diff --git a/spec/tests/check.fmf b/spec/tests/check.fmf index 2ea096b169..27334c0a6e 100644 --- a/spec/tests/check.fmf +++ b/spec/tests/check.fmf @@ -13,22 +13,25 @@ description: | the user can run into. Another useful checks would be kernel panic detection, core dump collection or collection of system logs. + By default, the check results affect the overall test outcome. - To change this behaviour, use the 'result' key, which accepts + To change this behaviour, use the ``result`` key, which accepts the following values: - respect (default) - The check result is respected and affects the overall test result. - xfail - The check result is expected to fail (pass becomes fail and vice-versa). - info - The check result is always treated as "INFO" and does not affect the - overall test result. + respect (default) + The check result is respected and affects the overall test result. + + xfail + The check result is expected to fail (pass becomes fail and vice-versa). + + info + The check result is always treated as "INFO" and does not affect the + overall test result. Note that running one check multiple times for the same test is - not yet supported." + not yet supported. - .. versionchanged:: 1.38.0 ``result`` key added + .. versionchanged:: 1.38.0 the ``result`` key added See :ref:`/plugins/test-checks` for the list of available checks. diff --git a/tests/execute/result/check_results.sh b/tests/execute/result/check.sh similarity index 91% rename from tests/execute/result/check_results.sh rename to tests/execute/result/check.sh index 6ea418a935..24b671fa14 100755 --- a/tests/execute/result/check_results.sh +++ b/tests/execute/result/check.sh @@ -18,8 +18,7 @@ run() rlJournalStart rlPhaseStartSetup rlRun "run=\$(mktemp -d)" 0 "Create run directory" - rlRun "pushd check_results" - rlRun "set -o pipefail" + rlRun "pushd check" rlPhaseEnd rlPhaseStartTest "Check Results Tests" @@ -32,7 +31,7 @@ rlJournalStart rlPhaseEnd rlPhaseStartTest "Verbose execute prints result" - rlRun -s "tmt run --id \${run} --scratch --until execute tests --filter tag:-cherry_pick provision --how local execute -v 2>&1 >/dev/null" "1" + rlRun -s "tmt run --id \${run} --scratch --until execute tests --filter provision --how local execute -v 2>&1 >/dev/null" "1" while read -r line; do rlAssertGrep "$line" "$rlRun_LOG" -F diff --git a/tests/execute/result/check_results/.fmf/version b/tests/execute/result/check/.fmf/version similarity index 100% rename from tests/execute/result/check_results/.fmf/version rename to tests/execute/result/check/.fmf/version diff --git a/tests/execute/result/check_results/test.fmf b/tests/execute/result/check/test.fmf similarity index 90% rename from tests/execute/result/check_results/test.fmf rename to tests/execute/result/check/test.fmf index 166f10d772..34bb61886d 100644 --- a/tests/execute/result/check_results/test.fmf +++ b/tests/execute/result/check/test.fmf @@ -1,12 +1,13 @@ summary: Tests for check results behaviour description: Verify that check results are correctly interpreted and affect test results +framework: shell +duration: 1m + /check-pass: summary: Test with passing checks description: Expected outcome: PASS (test passes, check passes) test: echo "Test passed" - framework: shell - duration: 1m check: - how: dmesg result: respect @@ -15,8 +16,6 @@ description: Verify that check results are correctly interpreted and affect test summary: Test with failing dmesg check (respect) description: Expected outcome: FAIL (test passes, but check fails and is respected) test: echo "Fail Test Check Pattern" | sudo tee /dev/kmsg - framework: shell - duration: 1m check: - how: dmesg failure-pattern: Fail Test Check Pattern @@ -26,8 +25,6 @@ description: Verify that check results are correctly interpreted and affect test summary: Test with failing dmesg check (info) description: Expected outcome: PASS (test passes, check fails, but should be just info) test: echo "Fail Test Check Pattern" | sudo tee /dev/kmsg - framework: shell - duration: 1m check: - how: dmesg failure-pattern: Fail Test Check Pattern @@ -37,8 +34,6 @@ description: Verify that check results are correctly interpreted and affect test summary: Test with passing dmesg check (xfail) description: Expected outcome: FAIL (test passes, check passes but xfail expects it to fail) test: echo "Test passed" - framework: shell - duration: 1m check: - how: dmesg result: xfail @@ -47,8 +42,6 @@ description: Verify that check results are correctly interpreted and affect test summary: Test with failing dmesg check (xfail) description: Expected outcome: PASS (test passes, check fails but xfail expects it to fail) test: echo "Fail Test Check Pattern" | sudo tee /dev/kmsg - framework: shell - duration: 1m check: - how: dmesg failure-pattern: Fail Test Check Pattern @@ -58,8 +51,6 @@ description: Verify that check results are correctly interpreted and affect test summary: Test with multiple checks with different result interpretations description: Expected outcome: FAIL (first dmesg check fails and is respected, second dmesg check passes but xfail expects it to fail, third failing dmesg check is just info) test: echo "Fail Test Check Pattern" | sudo tee /dev/kmsg - framework: shell - duration: 1m check: - how: dmesg failure-pattern: Fail Test Check Pattern @@ -75,8 +66,6 @@ description: Verify that check results are correctly interpreted and affect test summary: Test with failing dmesg check but overridden by test result description: Expected outcome: PASS (test passes, check fails but is overridden by 'result: pass') test: echo "Fail Test Check Pattern" | sudo tee /dev/kmsg - framework: shell - duration: 1m result: pass check: - how: dmesg diff --git a/tests/execute/result/main.fmf b/tests/execute/result/main.fmf index 0f7ca51221..a2fb8fb600 100644 --- a/tests/execute/result/main.fmf +++ b/tests/execute/result/main.fmf @@ -10,6 +10,6 @@ /special: summary: Test special characters generated to tmt-report-results.yaml test: ./special.sh -/check_results: +/check: summary: Test check result interpretations - test: ./check_results.sh + test: ./check.sh