Skip to content

Commit

Permalink
Merge branch 'random_rt_fixes' of github.com:DusanJovic-NOAA/ufs-weat…
Browse files Browse the repository at this point in the history 
…her-model into c-libs
  • Loading branch information
@BrianCurtis-NOAA
BrianCurtis-NOAA committed May 30, 2024
2 parents 55a5c8e + 31d2e01 commit 83f6d0b
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 68 deletions.
47 changes: 16 additions & 31 deletions tests/rt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ usage() {
echo " -v verbose output"
echo " -w for weekly_test, skip comparing baseline results"
echo
set -x
}

[[ $# -eq 0 ]] && usage
Expand Down Expand Up @@ -155,11 +154,11 @@ update_rtconf() {

generate_log() {
echo "rt.sh: Generating Regression Testing Log..."
set -x
COMPILE_COUNTER=0
FAILED_COMPILES=()
TEST_COUNTER=0
FAILED_TESTS=()
SKIPPED_TESTS=()
FAILED_TEST_ID=()
FAILED_COMPILE_LOGS=()
FAILED_TEST_LOGS=()
Expand Down Expand Up @@ -255,14 +254,14 @@ EOF
COMPILE_RESULT="FAILED: UNABLE TO START COMPILE"
FAIL_LOG="N/A"
elif [[ -f fail_compile_${COMPILE_ID} ]]; then
COMPILE_RESULT="FAILED: UNABLE TO COMPILE"
COMPILE_RESULT="FAILED: UNABLE TO FINISH COMPILE"
FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
if grep -q "quota" "${LOG_DIR}/compile_${COMPILE_ID}.log"; then
COMPILE_RESULT="FAILED: DISK QUOTA ISSUE"
FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
elif grep -q "timeout" "${LOG_DIR}/compile_${COMPILE_ID}.log"; then
COMPILE_RESULT="FAILED: TEST TIMED OUT"
FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
elif grep -q "TIME LIMIT" "${RUNDIR_ROOT}/compile_${COMPILE_ID}/err"; then
COMPILE_RESULT="FAILED: COMPILE TIMED OUT"
FAIL_LOG="${RUNDIR_ROOT}/compile_${COMPILE_ID}/err"
fi
else
COMPILE_RESULT="PASS"
Expand Down Expand Up @@ -330,21 +329,25 @@ EOF
RT_TEST_TIME=""
RT_TEST_MEM=""
if [[ ${CREATE_BASELINE} == true && ${GEN_BASELINE} != "baseline" ]]; then
TEST_RESULT="SKIPPED (TEST DOES NOT GENERATE BASELINE)"
TEST_RESULT="SKIPPED: TEST DOES NOT GENERATE BASELINE"
SKIPPED_TESTS+=("TEST ${TEST_NAME}_${COMPILER}: ${TEST_RESULT}")
elif [[ ${COMPILE_RESULT} =~ FAILED ]]; then
TEST_RESULT="SKIPPED: ASSOCIATED COMPILE FAILED"
SKIPPED_TESTS+=("TEST ${TEST_NAME}_${COMPILER}: ${TEST_RESULT}")
elif [[ ! -f "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log" ]]; then
TEST_RESULT="FAILED: UNABLE TO START RUN"
TEST_RESULT="FAILED: UNABLE TO START TEST"
FAIL_LOG="N/A"
elif [[ -f fail_test_${TEST_NAME}_${COMPILER} ]]; then
if [[ -f "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log" ]]; then
if grep -q "FAIL" "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"; then
TEST_RESULT="FAILED: UNABLE TO RUN COMPARISON"
TEST_RESULT="FAILED: UNABLE TO COMPLETE COMPARISON"
FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
# We need to catch a "PASS" in rt_*.log even if a fail_test_* files exists
# I am not sure why this can happen.
elif grep -q "PASS" "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"; then
TEST_RESULT="PASS"
else
TEST_RESULT="FAILED: BASELINE COMPARISON"
TEST_RESULT="FAILED: UNSUCCESSFUL BASELINE COMPARISON"
FAIL_LOG="${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"
fi
else
Expand All @@ -354,9 +357,9 @@ EOF
if grep -q "quota" "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"; then
TEST_RESULT="FAILED: DISK QUOTA ISSUE"
FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
elif grep -q "timeout" "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"; then
elif grep -q "TIME LIMIT" "${RUNDIR_ROOT}/${TEST_NAME}_${COMPILER}/err"; then
TEST_RESULT="FAILED: TEST TIMED OUT"
FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
FAIL_LOG="${RUNDIR_ROOT}/${TEST_NAME}_${COMPILER}/err"
fi
else
TEST_RESULT="PASS"
Expand Down Expand Up @@ -399,7 +402,7 @@ Starting Date/Time: ${TEST_START_TIME}
Ending Date/Time: ${TEST_END_TIME}
Total Time: ${elapsed_time}
Compiles Completed: $((COMPILE_COUNTER-${#FAILED_COMPILES[@]}))/${COMPILE_COUNTER}
Tests Completed: $((TEST_COUNTER-${#FAILED_TESTS[@]}))/${TEST_COUNTER}
Tests Completed: $((TEST_COUNTER-${#FAILED_TESTS[@]}-${#SKIPPED_TESTS[@]}))/${TEST_COUNTER}
EOF
# PRINT FAILED COMPILES
if [[ "${#FAILED_COMPILES[@]}" -ne "0" ]]; then
Expand Down Expand Up @@ -674,14 +677,10 @@ done
#B&N not run together
[[ ${NEW_BASELINES_FILE} != '' && ${RUN_SINGLE_TEST} == true ]] && die "-b and -n options cannot be used at the same time"

[[ -o xtrace ]] && set_x='set -x' || set_x='set +x'

if [[ ${RTVERBOSE} == true ]]; then
set -x
fi

[[ -o xtrace ]] && set_x='set -x' || set_x='set +x'

if [[ -z "${ACCNR}" ]]; then
echo "Please use -a <account> to set group account to use on HPC"
exit 1
Expand All @@ -694,12 +693,10 @@ echo "Account: ${ACCNR}"
case ${MACHINE_ID} in
wcoss2|acorn)
echo "rt.sh: Setting up WCOSS2/Acorn"
set -x
if [[ "${ECFLOW:-false}" == true ]] ; then
module load ecflow/5.6.0.13
fi
module load intel/19.1.3.304 python/3.8.6
export colonifnco=":output" # hack

DISKNM="/lfs/h2/emc/nems/noscrub/emc.nems/RT"
QUEUE="dev"
Expand All @@ -714,7 +711,6 @@ case ${MACHINE_ID} in
;;
gaea)
echo "rt.sh: Setting up gaea..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module use /ncrc/proj/epic/rocoto/modulefiles
module load rocoto
Expand Down Expand Up @@ -747,7 +743,6 @@ case ${MACHINE_ID} in
;;
hera)
echo "rt.sh: Setting up hera..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module load rocoto
ROCOTO_SCHEDULER=slurm
Expand All @@ -770,7 +765,6 @@ case ${MACHINE_ID} in
;;
orion)
echo "rt.sh: Setting up orion..."
set -x
module load git/2.28.0
module load gcc/10.2.0
module load python/3.9.2
Expand Down Expand Up @@ -800,7 +794,6 @@ case ${MACHINE_ID} in
;;
hercules)
echo "rt.sh: Setting up hercules..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module load contrib rocoto
ROCOTO_SCHEDULER="slurm"
Expand Down Expand Up @@ -828,7 +821,6 @@ case ${MACHINE_ID} in
;;
jet)
echo "rt.sh: Setting up jet..."
set -x
CurJetOS=$(lsb_release -is)
echo "=======Running on ${CurJetOS}======="
if [[ ${CurJetOS} == "CentOS" ]]; then
Expand Down Expand Up @@ -861,7 +853,6 @@ case ${MACHINE_ID} in
;;
s4)
echo "rt.sh: Setting up s4..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module load rocoto/1.3.2
ROCOTO_SCHEDULER=slurm
Expand Down Expand Up @@ -892,7 +883,6 @@ case ${MACHINE_ID} in
;;
derecho)
echo "rt.sh: Setting up derecho..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module use /glade/work/epicufsrt/contrib/derecho/rocoto/modulefiles
module load rocoto
Expand Down Expand Up @@ -930,7 +920,6 @@ case ${MACHINE_ID} in
;;
stampede)
echo "rt.sh: Setting up stampede..."
set -x
export PYTHONPATH=
if [[ "${ECFLOW:-false}" == true ]] ; then
ECFLOW_START=
Expand All @@ -948,7 +937,6 @@ case ${MACHINE_ID} in
;;
expanse)
echo "rt.sh: Setting up expanse..."
set -x
export PYTHONPATH=

if [[ "${ECFLOW:-false}" == true ]] ; then
Expand All @@ -965,7 +953,6 @@ case ${MACHINE_ID} in
;;
noaacloud)
echo "rt.sh: Setting up noaacloud..."
set -x
export PATH="/contrib/EPIC/bin:${PATH}"
module use /apps/modules/modulefiles

Expand All @@ -987,7 +974,6 @@ case ${MACHINE_ID} in
die "Unknown machine ID, please edit detect_machine.sh file"
;;
esac
eval "${set_x}"

mkdir -p "${STMP}/${USER}"

Expand Down Expand Up @@ -1374,5 +1360,4 @@ fi

## Lets verify all tests were run and that they passed
generate_log
eval "${set_x}"
echo "******Regression Testing Script Completed******"
Loading

0 comments on commit 83f6d0b

Please sign in to comment.