data-test

#!/bin/bash
# data-test
# Test data conversion tool pull request file to ensure no new problems arise, and/or build is not slow or significantly slow down.

##########################################################
##################### DATA-TEST ##########################
### All files in this repositroy are public domain cc0 ###
### Any files created by the test are subject to the   ###
### license specified in that reposityor.              ###
##########################################################

### 1. SET CONFIGURATION AND READY TEST ELEMENTS ##########################################################################
###########################################################################################################################
# Set variables from config-variables.sh. 
# IMPORTANT - these should have been changed after making repo from template.
# IMPORTANT - you must agree to terms before use.
source config-variables.sh
check_if_user_agreed_exit_if_not # check if agreed to terms, exit if not
_file="source" # DO NOT CHANGE - sets file name for test pages
_start_timing_test="$(date +%s.%N)" # DO NOT CHANGE - compare to _max-run-time
_breakOutOfTest=0      # DO NOT CHANGE - safely exit build if max reached - currently off
_breakOutOfTestSize=0  # DO NOT CHANGE - safely exit build if max reached - currently off
_closeOutTest=0        # DO NOT CHANGE - indicates max time is reached - currently off
_maxTimeWasReached=0   # DO NOT CHANGE - alters test output if max reached - currently off
_maxSizeWasReached=0   # DO NOT CHANGE - alters test output if max reached - currently off

# The number of opttions that will run when testing.
# Array _optionsToRun is from congig-variables.sh.
_numberOfOptionsRan="${#_optionsToRun[@]}"

# Incrementers to show error stats.
_numberOfFilesTested=0
_numberOfFilesBuilt=0
_numberOfSourceFilesBuilt=0
_numberOfPullFilesBuilt=0
_countNumberOfErrors=0
_percentOfFilesWithError=0

# Determine which test is run.
if [ -z "$1" ]; then
	_currentTest=quick-test
else
	_currentTest=full-test
fi

# Make and store lists with two types of files used:
# 1. gen - general data files (using/not using command) with variety of elements.
# 2. unq - unique case for test files
ls -p1 data/"$_currentTest"/gen/ | grep -v '/$' > list-gen.txt
ls -p1 data/"$_currentTest"/unq/ | grep -v '/$' > list-unq.txt

# Make directories for results.
mkdir -p diff commands support tests
mkdir -p diff/unq diff/err diff/gen tests/unq tests/err tests/gen


### 2. CLONE SOURCE AND PULL GITHUB REPOSITORIES ##########################################################################
###########################################################################################################################
# Get source and pulled command file from github.
function _git_clone_repo() {
  _curGitHubUser="$1"
  git clone https://github.com/$_curGitHubUser/$_repoName.git commands/"$_curGitHubUser"-"$_COMMAND_TEST"/
}
# Make the dir to compare source and pull command.
if [ "$_cloneRepo" = "1" ]; then
  _git_clone_repo "$_sourceUserName" && _git_clone_repo "$_pullUserName"
fi


### 3. START TEST FUNCTION CONFIGURING FUNCTION FOR SOURCE OR PULL. #######################################################
###########################################################################################################################
# Define timing arrays.
declare -a _runTimes && declare -a _sourceRuns && declare -a _pullRuns

# Function to make html with option and no option call.
# Then make corresponding files in diff folder
# to test the differnences to ensure nothing new
# was broken as a result of pull request.
function run_test() {
  # set list to use
  _curList="$1" && _curData="$2" && _curDir="$_currentTest"/"$_curData"
  # set which coomand to use; source or pull
  if [ "$_useSource" = 1 ]; then
    _file="source"
    # copy to root of testing repo
    if [ $_toTestRoot = 1 ]; then
      cp commands/"$_sourceUserName"-"$_COMMAND_TEST"/"$_COMMAND_TEST" "$_COMMAND_TEST"
    fi
  else
    _file="pull"
    # copy to root of testing repo
    if [ $_toTestRoot = 1 ]; then
      cp commands/"$_pullUserName"-"$_COMMAND_TEST"/"$_COMMAND_TEST" "$_COMMAND_TEST"
    fi
  fi


### 4. SET A START AND END TIME FOR EACH BUILD WITH -CHANGE_COMMAND-, PUSHING START AND END TO TWO SEPARATE ARRAYS. 
###########################################################################################################################	
  # repetitive function to get time for each build
  calc_build() {
    # set starting point before making file
    if [ "$1" = 1 ]; then
      start_one_run="$(date +%s.%N)" # set start time mark
    elif [ "$1" = 2 ]; then
      # set end time mark
      end_one_run="$(date +%s.%N)"
      # get time difference
      _oneRun=$(echo "$end_one_run - $start_one_run" | bc)
      # append to correct array by either source or pull command
      # that is specified in config variab le "_useSource" at top
      if [ "$_file" = "source" ]; then
        (( _numberOfSourceFilesBuilt++ ))
        _sourceRuns+=("$_oneRun")
      else
        (( _numberOfPullFilesBuilt++ ))
        _pullRuns+=("$_oneRun")
      fi
    fi
  }


### 5. IN ` RUN_COMMAND() ` SUB-FUNCTION CALL -CHANGE_COMMAND- TO BUILD WITH -S_U- AND -P_U- AND CALCULATE BUILD TIME(S). 
#############################################################################################################################
  # repetitive function to make files
  run_command() {
    # loop through options specified to run command with
    for opts in "${_optionsToRun[@]}"; do
      if [ "$opts" == "" ]; then
        _nameOpt="-default"
      else
        _nameOpt="$opts"
      fi
      if [ "$_targetFileExtension" == ".html" ]; then
        echo "Command building with $1 $_COMMAND_TEST $opts :"
        # start start time, run build command, and calculate run time storing it in array
        calc_build "1" && _commandSyntax "$opts" data/"$_curDir"/"$line" tests/"$_curData"/"$line"-"$1""$_nameOpt".html && calc_build "2"
      else
        echo "Command building with $1 $_COMMAND_TEST $opts :"
        echo "<!DOCTYPE HTML><html><body><pre>" > tests/"$_curData"/"$line"-"$1""$_nameOpt".html # make html opening elements
        # start start time, run build command, and calculate run time storing it in array
        calc_build "1" && _commandSyntax "$opts" data/"$_curDir"/"$line" tests/"$_curData"/"$line"-"$1""$_nameOpt".html && calc_build "2"
        echo "</pre></body></html>" >> tests/"$_curData"/"$line"-"$1""$_nameOpt".html # make html closing elements					
      fi
      # increase the number of files built
      (( _numberOfFilesBuilt++ ))
    done


### 6. DETERMINE DIFFERENCE AFTER ALL FILES BUILT SECOND IN `RUNC_COMMAND() SUB-FUNCTION. ###################################
#############################################################################################################################		
    # loop through options that created files and check for differences in 
    # source and pull differences in created files
    for opts in "${_optionsToRun[@]}"; do
      if [ "$opts" = "" ]; then
        opts="-default"
      fi
      # compare the built files
      if [ "$1" = "source" ]; then
        _invertRepo="pull"
      else
        _invertRepo="source"
      fi
      # if file has been built
      if [ -e "tests/$_curData/$line-$_invertRepo$opts.html" ]; then
        # difference based on arguments setting variables.
        if ! diff -q tests/"$_curData"/"$line"-"$_invertRepo""$opts".html tests/"$_curData"/"$line"-"$1""$opts".html >/dev/null; then
          $_diffCommand tests/"$_curData"/"$line"-"$_invertRepo""$opts".html tests/"$_curData"/"$line"-"$1""$opts".html > diff/"$_curData"/"$line""$opts".txt
        fi
      fi
    done
    check_max_run_time
    if [ $_closeOutTest = 1 ]; then
      echo MAX TIME
      _maxTimeWasReached=1
      _breakOutOfTest=1
      return
    fi
    # check that max time is not reached
    check_max_test_size "check"
    if [ $_closeOutTest = 1 ]; then
      echo MAX SIZE
      _maxSizeWasReached=1
      _breakOutOfTestSize=1
      return
    fi		
  }


### 7. CALL ` RUN_COMMAND() ` SUB-FUNCTION. #################################################################################
#############################################################################################################################			
  # make files by command being used using repetitive functions above
  # file naming will be [file_name]-[source|pull]-[""|"${opt[x]}"].html
  while read -r line; do
    # end if max time has been reached
    if [ $_breakOutOfTest = 1 ] || [ $_breakOutOfTestSize = 1 ]; then
      break
    else		
      # increase count of files that were tested (files in data/[full|quick]-test folder(s)
      (( _numberOfFilesTested++ ))
      if [ "$_useSource" = 1 ]; then
        # run using source command
        _curUser="$_sourceUserName"
        run_command "source"
      else
        # run using pull command
        _curUser="$_pullUserName"
        run_command "pull"
      fi
    fi
    # loop through files listed in list-[gen|unq].txt file
  done < "$_curList"


### 8. DETERMINE IF THERE IS A DIFFERENCE, CHECKING EMPTY FILES AND COUNT NUMBER OF DIFFERENCES. ############################
#############################################################################################################################
  # move files with differences to errors
  while read -r line; do
    # for all options check files created by command
    for opts in "${_optionsToRun[@]}"; do
      # account for no option
      if [ "$opts" = "" ]; then
        opts="-default"
      fi
      # ensure differnces by seeing if file is NOT empty
      if [ -s diff/"$_curData"/"$line""$opts".txt ]; then
        mv diff/"$_curData"/"$line""$opts".txt diff/err/"$line""$opts".txt 2>/dev/null
        mv tests/"$_curData"/"$line"*"$opts".html tests/err/ 2>/dev/null
        # increase the number of errors
        (( _countNumberOfErrors++ ))
      fi
    done
          # loop through files listed in list-[gen|unq].txt file
  done < "$_curList"
}


### 9. CALL ` RUN_TEST() ` FUNCTION WITH SOURCE OR PULL, RECONFIGURE AND CALL ` RUN_TEST() ` AGAIN. #########################
#############################################################################################################################
# Run test with initial set source.
run_test "list-gen.txt" "gen" 
run_test "list-unq.txt" "unq"

# Use command in pull request.
# Redefine source command to use based on config from top (default is _useSource=1).
if [ "$_useSource" = 1 ]; then  _useSource=0; else  _useSource=1; fi

# Check if max size was reached and needs to be reset.
check_max_test_size "reset"
# Check if max time was reacehd and needs to be reset.
check_max_run_time

# Run test with invert of initial set source.
run_test "list-gen.txt" "gen" 
run_test "list-unq.txt" "unq"

# Check if either source or pull went over max time.
if [ $_maxTimeWasReached = 1 ]; then
  _generateTimeErrorMessage=1
else
  _generateTimeErrorMessage=0
  _numberOfFilesTested=$(( ($_numberOfFilesTested / 2) ))
fi
# Check if either source or pull went over max size.
if [ $_maxSizeWasReached = 1 ]; then
  _generateSizeErrorMessage=1
else
  _generateSizeErrorMessage=0	
fi


### 10. MAKE SUPPORT HTML ELEMENTS FOR DIFFERENCE OUTPUT AND DROPDOWN MENUS. ################################################
#############################################################################################################################
# Make support files for html links.
ls -1 tests/unq/*-source* > support/menu-unq-links.txt 2>/dev/null
ls -1 tests/err/*-source* > support/menu-err-links.txt 2>/dev/null
ls -1 tests/gen/*-source* > support/menu-gen-links.txt 2>/dev/null

# Replace the temporary filler text to fit test outputs.
sed -i "s/source/COMMAND_USED/g" support/menu-unq-links.txt 2>/dev/null
sed -i "s/source/COMMAND_USED/g" support/menu-err-links.txt 2>/dev/null
sed -i "s/source/COMMAND_USED/g" support/menu-gen-links.txt 2>/dev/null

# Create a text file with contents of all diff files. 
ls -p1 diff/err/* 2>/dev/null | grep -v '/$' > diff/errList.txt 2>/dev/null
touch diff/errs.txt

# Change script variable for default diff command use if not "diff"
if [ "$_diffCommand" = "diff" ]; then
  sed -i "s|var diffDefault=.|var diffDefault=1|" scripts.js
else
  sed -i "s|var diffDefault=.|var diffDefault=0|" scripts.js
fi


### 11. COMPARE TOTAL FIES BUILT TO FILES WITH DIFFERENCES, OUTPUTTING RESULTS TO SUPPORTING TEXT FILE. #####################
#############################################################################################################################
# Loop through diff file and output files' terminal output to html and count number built.
_builtFilesWithDifferences=0
while read -r line; do
  # header for file name
  echo "---h3-data-conversion-test_tag---${line##*/}---c-h3-data-conversion-test_tag---" >> diff/errs.txt	
  # file contents
  cat "$line" >> diff/errs.txt
  # add space and separation marker
  echo "---br_hr_br-data-conversion-test_tag---" >> diff/errs.txt
  # increment built files with differences
  (( _builtFilesWithDifferences++ ))
done < diff/errList.txt
# Account for source and pull
(( _builtFilesWithDifferences*="$_numberOfOptionsRan" ))

# Make statistic html section.
cp templates/diff-stats.html support/diff-stats.txt


### 12. CALCULATE STATS FOR BUILD TIMES, OUTPUTTING RESULTS TO SUPPORTING TEXT FILE. ########################################
#############################################################################################################################
# Calculate the percent of files with error/differences
_percentOfFilesWithError=$(( ( (_countNumberOfErrors/_numberOfOptionsRan) * 100) / _numberOfFilesTested ))
_percentOfBuiltFilesWithError=$(( (_builtFilesWithDifferences * 100) / _numberOfFilesBuilt ))
_percentOfFilesWithDifferences=$(( (_countNumberOfErrors /_numberOfOptionsRan) ))

# Edit copied template file for output differences. Comment at end correspond to comment in template file.
# Show time error message.
if [ $_generateTimeErrorMessage = 1 ]; then
  sed -i "s|_GENERATE-TIME-ERRORMESSAGE|block|" support/diff-stats.txt # time error message
else
  sed -i "s|_GENERATE-TIME-ERRORMESSAGE|none|" support/diff-stats.txt # time error message
fi
# Show size error message.
if [ $_generateSizeErrorMessage = 1 ]; then
  sed -i "s|_GENERATE-SIZE-ERRORMESSAGE|block|" support/diff-stats.txt # size error message
else
  sed -i "s|_GENERATE-SIZE-ERRORMESSAGE|none|" support/diff-stats.txt # size error message
fi


sed -i "s|_numberOfFilesTested|$_numberOfFilesTested|" support/diff-stats.txt # total files

sed -i "s|_percentOfFilesWithDifferences|$_percentOfFilesWithDifferences|" support/diff-stats.txt # percent with differneces
sed -i "s|_percentOfFilesWithError|$_percentOfFilesWithError|" support/diff-stats.txt # test with differeneces

sed -i "s|_numberOfOptionsRan|$_numberOfOptionsRan|" support/diff-stats.txt # options run
sed -i "s|_numberOfFilesTested|$_numberOfFilesTested|" support/diff-stats.txt # files tested
sed -i "s|_numberOfFilesBuilt|$_numberOfFilesBuilt|" support/diff-stats.txt # total built
sed -i "s|_builtFilesWithDifferences|$_builtFilesWithDifferences|" support/diff-stats.txt # total with difference
sed -i "s|_percentOfBuiltFilesWithError|$_percentOfBuiltFilesWithError|" support/diff-stats.txt # built difference percent

# Make run time html section.
cp templates/run-times.html support/run-times.txt

# Function to calculate basic run time stats.
calc_run_average() {
  cur_arr=("$@")           # array as parameter
  cur_sum=0                # start count at 0
  cur_len=${#cur_arr[@]}   # array length
  for num in "${cur_arr[@]}"; do
    # sum up all the time amounts to build file
    cur_sum=$(echo "scale=6; $cur_sum + $num" | bc)
  done
  # get average of summed build times
  cur_average=$(echo "scale=6; $cur_sum / $cur_len" | bc)
  # append array witb source and pull command's sums respectively
  _sumRuns+=("$cur_sum")
  # append array with source and pull command's averages respectively
  _averageRunTimes+=("$cur_average")
}

# Calculate individual run time stats. Declare arrays to be used.
declare -a _averageRunTimes && declare -a _sumRuns

# Calculate stats by source then pull repo command.
calc_run_average "${_sourceRuns[@]}" && calc_run_average "${_pullRuns[@]}"

# Calculate stats from function result. Source and pull sum and averages respectively.
_sourceSumRuns="${_sumRuns[0]}" && _pullSumRuns="${_sumRuns[1]}"
_sourceAvgRuns="${_averageRunTimes[0]}" && _pullAvgRuns="${_averageRunTimes[1]}"

# Calculate the differences from the time amounts of pull and source repo command.
_allFastOrSlow=$(echo "$_pullSumRuns - $_sourceSumRuns" | bc)
_avgFastOrSlow=$(echo "$_pullAvgRuns - $_sourceAvgRuns" | bc)

# Function to make repeating text.
function repetitive_diff() {
  # template text
  _thanSource="than source to build $1"
  # calculation displayed
  _theCurrentTimeDifference="$2"
  # used to run condition to check if calculation is less or more time
  _forCondition="$(echo "$_theCurrentTimeDifference >= 0" | bc -l)"
  if [ "$_forCondition" = 1 ]; then
    # pull command is slower
    _fastOrSlow="pull was <strong>$_theCurrentTimeDifference seconds</strong> slower $_thanSource."
  else
    # source command is slower
    _negateTime=$(echo "$_theCurrentTimeDifference * -1" | bc) # make positive number
    _fastOrSlow="pull was <strong>$_negateTime</strong> faster $_thanSource."
  fi	
}


# Edit copied template file with stats. Comment at end correspond to comment in template file.
sed -i "s|_numberOfSourceFilesBuilt|$_numberOfSourceFilesBuilt|" support/run-times.txt # total source file
sed -i "s|_sourceSumRuns|$_sourceSumRuns|" support/run-times.txt # total source time
sed -i "s|_numberOfPullFilesBuilt|$_numberOfPullFilesBuilt|" support/run-times.txt # total pull file
sed -i "s|_pullSumRuns|$_pullSumRuns|" support/run-times.txt # total pull time

# Sum of all builds compared.
repetitive_diff "$_numberOfFilesBuilt files" "$_allFastOrSlow" 

sed -i "s|_fastOrSlowOne|$_fastOrSlow|" support/run-times.txt # time difference
sed -i "s|_sourceAvgRuns|$_sourceAvgRuns|" support/run-times.txt # average source
sed -i "s|_pullAvgRuns|$_pullAvgRuns|" support/run-times.txt # average pull

# Average of all builds compared.
repetitive_diff "1 file on average" "$_avgFastOrSlow" 

sed -i "s|_fastOrSlowTwo|$_fastOrSlow|" support/run-times.txt # average of all builds compared


### 13. MAKE REMAINING SUPPORT HTML ELEMENTS AND CLEAN FILES AND DIRECTORIES. ###############################################
#############################################################################################################################
# Make drop items and menu text by commands used.
touch support/options-ran.txt
for opts in "${_optionsToRun[@]}"; do
  if [ "$opts" = "" ]; then
    echo "default" >> support/options-ran.txt
  else
    echo "$opts" >> support/options-ran.txt
  fi
done
echo "${_COMMAND_TEST^}" > support/COMMAND-USED.txt

# Change README.md to show test results.
cp README.md INSTRUCTIONS.md 2>/dev/null
if [ $_currentTest = "quick-test" ]; then
  cp templates/quick-README.md README.md 2>/dev/null
  # Make list of files tested.
  touch .temp-filesTested.txt
  # list of files to update
  list_files_tested=("list-gen.txt" "list-unq.txt")

  # loop through the files tested
  for files in "${list_files_tested[@]}"; do
    cat $files >> .temp-filesTested.txt
  done
  sed -i 's/^/   - /' .temp-filesTested.txt
else
  cp templates/full-README.md README.md 2>/dev/null
  # Make list of files tested.
  touch .temp-filesTested.txt
  echo "### File Extensions:" >> .temp-filesTested.txt
  # loop through the files tested
  for ext in "${_fileExtensions[@]}"; do
    echo "   - $ext" >> .temp-filesTested.txt
  done
  # add extra line
  echo >> .temp-filesTested.txt
  # headerfor repos
  echo "### GitHub Repositories Tested:" >> .temp-filesTested.txt

  while (( index < ${#_fullTestRepos[@]} -1 )); do
    _user=${_fullTestRepos[index]}
    _repo=${_fullTestRepos[index + 1]}
    echo "   - $_user/$_repo".git >> .temp-filesTested.txt
    index=$((index + 2))
  done
  echo >> .temp-filesTested.txt
fi

# Change link in README.md
sed -i "s/CHANGE_USER/$_pullUserName/g" README.md
sed -i "s/CHANGE_REPO/$_testResultRepo/g" README.md
sed -i "s/CHANGE_SOURCE_USER/$_sourceUserName/g" README.md
sed -i "s/CHANGE_COMMAND_TEST/$_COMMAND_TEST/g" README.md
awk -v _test_file_list="$(cat .temp-filesTested.txt)" '{gsub("CHANGE_FILE_LIST", _test_file_list)}1' README.md >> .outREADME.md
mv .outREADME.md README.md 2>/dev/null

# Change  user and repo in index.html
sed -i "s|var cur_test_results = \"https://github.com/CHANGE_USER/CHANGE_REPO/blob/main/\";|var cur_test_results = \"https://github.com/$_pullUserName/$_testResultRepo/blob/main/\";|" scripts.js

# remove option call signs
sed -i -E 's/([-]+)(.*)/\2/g' support/options-ran.txt

# Clean up folder.
rm list-gen.txt list-unq.txt .temp-filesTested.txt

# Remove command as no longer needed.
if [ ! -e .localCommandUsed ] || [ $_toTestRoot = 1 ]; then
  rm -rf commands "$_COMMAND_TEST"
fi

# Change comments in this file so markers reflect configuration.
./config/mark-data-test.sh "$_COMMAND_TEST" "$_sourceUserName" "$_pullUserName"