feat(reana-dev): improve workflow testing in run-example command (#817)

Run-example now uses uses the test files in the specification file
to check the correctness of the results (by running the test command).

AUTHORS.md

@@ -7,6 +7,7 @@ organisation on GitHub, in alphabetical order:
 
 - [Adelina Lintuluoto](https://orcid.org/0000-0002-0726-1452)
 - [Agisilaos Kounelis](https://orcid.org/0000-0001-9312-3189)
+- [Alastair Lyall](https://orcid.org/0009-0000-4955-8935)
 - [Alizee Pace](https://www.linkedin.com/in/aliz%C3%A9e-pace-516b4314b/)
 - [Alp Tuna](https://orcid.org/0009-0001-1915-3993)
 - [Ana Trisovic](https://orcid.org/0000-0003-1991-0533)

reana/reana_dev/run.py

@@ -9,6 +9,7 @@
 """`reana-dev`'s run commands."""
 
 import os
+import re
 import subprocess
 import sys
 import time
@@ -18,9 +19,7 @@
 
 from reana.config import (
     COMPUTE_BACKEND_LIST_ALL,
-    EXAMPLE_LOG_MESSAGES,
     EXAMPLE_NON_STANDARD_REANA_YAML_FILENAME,
-    EXAMPLE_OUTPUT_FILENAMES,
     TIMECHECK,
     TIMEOUT,
     WORKFLOW_ENGINE_LIST_ALL,
@@ -59,32 +58,6 @@ def construct_workflow_name(example, workflow_engine, compute_backend):
     return output
 
 
-def get_expected_log_messages_for_example(example):
-    """Return expected log messages for given example.
-
-    :param example: name of the component
-    :return: Tuple with output log messages(s)
-    """
-    try:
-        output = EXAMPLE_LOG_MESSAGES[example]
-    except KeyError:
-        output = EXAMPLE_LOG_MESSAGES["*"]
-    return output
-
-
-def get_expected_output_filenames_for_example(example):
-    """Return expected output file names for given example.
-
-    :param example: name of the component
-    :return: Tuple with output file name(s)
-    """
-    try:
-        output = EXAMPLE_OUTPUT_FILENAMES[example]
-    except KeyError:
-        output = EXAMPLE_OUTPUT_FILENAMES["*"]
-    return output
-
-
 def select_workflow_engines(workflow_engines):
     """Return known workflow engine names that REANA supports.
 
@@ -511,28 +484,6 @@ def _get_test_matrix_summary() -> str:
 
     display_message(_get_test_matrix_summary(), component="reana")
 
-    def _verify_log_output(component: str, workflow_name: str) -> bool:
-        for log_message in get_expected_log_messages_for_example(component):
-            cmd = f"reana-client logs -w {workflow_name} | grep '{log_message}' | wc -l"
-            cmd_output = run_command(cmd, component, return_output=True)
-            click.secho(cmd_output)
-            line_count = int(cmd_output)
-
-            if line_count == 0:
-                return False
-        return True
-
-    def _return_missing_output_files(component: str, workflow_name: str) -> List[str]:
-        cmd = f"reana-client ls -w {workflow_name}"
-        listing = run_command(cmd, component, return_output=True)
-        click.secho(listing)
-        expected_files = file or get_expected_output_filenames_for_example(component)
-        missing_files = []
-        for expected_file in expected_files:
-            if expected_file not in listing:
-                missing_files.append(expected_file)
-        return missing_files
-
     run_statistics = {
         "queued": [],
         "pending": [],
@@ -604,18 +555,21 @@ def _return_missing_output_files(component: str, workflow_name: str) -> List[str
                     elif "failed" in status:
                         run_statistics["failed"].append(workflow_name)
                     elif "finished" in status or "stopped" in status:
-                        if not _verify_log_output(component, workflow_name):
-                            run_statistics["failed"].append(workflow_name)
-                            continue
-
-                        missing_files = _return_missing_output_files(
-                            component, workflow_name
-                        )
-
-                        if len(missing_files):
+                        try:
+                            tests = run_command(
+                                f"reana-client test -w {workflow_name}",
+                                component,
+                                return_output=True,
+                            )
+                            if re.findall(r"^\s*-> ERROR:.*", tests, re.MULTILINE):
+                                run_statistics["failed"].append(workflow_name)
+                            else:
+                                run_statistics["passed"].append(workflow_name)
+                        except SystemExit:
                             run_statistics["failed"].append(workflow_name)
-                        else:
-                            run_statistics["passed"].append(workflow_name)
+                            display_message(
+                                f"Workflow {workflow_name} has no tests.", component
+                            )
                     else:
                         run_statistics["failed"].append(workflow_name)
 

tests/test_cli.py

@@ -13,6 +13,9 @@
 import os
 import pytest
 import click
+from click.testing import CliRunner
+from unittest.mock import patch
+from reana.reana_dev.cli import reana_dev
 
 
 def test_shorten_component_name():
@@ -27,28 +30,97 @@ def test_shorten_component_name():
         assert name_short == shorten_component_name(name_long)
 
 
-def test_get_expected_output_filenames_for_example():
-    """Tests for get_expected_output_filenames_for_example()."""
-    from reana.reana_dev.run import get_expected_output_filenames_for_example
-
-    for example, output in (
-        ("", ("plot.png",)),
-        ("reana-demo-helloworld", ("greetings.txt",)),
-        ("reana-demo-root6-roofit", ("plot.png",)),
-        ("reana-demo-alice-lego-train-test-run", ("plot.pdf",)),
-    ):
-        assert output == get_expected_output_filenames_for_example(example)
-
-
-def test_get_expected_log_message_for_example():
-    """Tests for get_expected_log_messages_for_example()."""
-    from reana.reana_dev.run import get_expected_log_messages_for_example
-
-    for example, output in (
-        ("", ("job:",)),
-        ("reana-demo-helloworld", ("Parameters: inputfile=",)),
-    ):
-        assert output == get_expected_log_messages_for_example(example)
+def run_command_possibilities(command, component, return_output=False):
+    """Possible return values for run_command."""
+    if "reana-client test" in command and "cwl" in command:
+        return """
+        ==> Testing file "tests/cwl/log-messages.feature"...
+          -> ERROR: Scenario "-> SUCCESS: Writing SUCCESS in the scenario name should make no difference"
+          -> SUCCESS: Scenario "If one scenario fails, the whole test should fail"
+        """
+    elif "reana-client test" in command:
+        return """
+        ==> Testing file "tests/yadage/log-messages.feature"...
+          -> SUCCESS: Scenario "-> ERROR: Writing ERROR in the scenario name should make no difference"
+          -> SUCCESS: Scenario "If a different test fails, this one shouldn't"
+        """
+    elif "reana-client status" in command:
+        return "finished"
+    return ""
+
+
+@patch(
+    "reana.reana_dev.run.run_command",
+    side_effect=lambda command, component, return_output=False: (
+        """
+        ==> Testing file "tests/cwl/log-messages.feature"...
+          -> SUCCESS: Scenario "-> ERROR: Writing ERROR in the scenario name should make no difference"
+        """
+        if "reana-client test" in command
+        else "finished" if "reana-client status" in command else ""
+    ),
+)
+@patch(
+    "reana.reana_dev.run.get_example_reana_yaml_file_path",
+    return_value="reana-cwl.yaml",
+)
+def test_run_example_check_only_passes(
+    mock_run_command, mock_get_example_reana_yaml_file_path
+):
+    """Tests for run-example command with check-only flag, when all tests pass."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            reana_dev,
+            [
+                "run-example",
+                "-c",
+                "r-d-r-roofit",
+                "-w",
+                "cwl",
+                "--check-only",
+            ],
+        )
+        assert "1 passed" in result.output
+        assert "0 failed" in result.output
+        assert result.exit_code == 0
+
+
+@patch(
+    "reana.reana_dev.run.run_command",
+    side_effect=run_command_possibilities,
+)
+@patch(
+    "reana.reana_dev.run.get_example_reana_yaml_file_path",
+    return_value="reana-cwl.yaml",
+    side_effect=lambda component, workflow_engine, compute_backend: (
+        "reana-cwl.yaml" if workflow_engine == "cwl" else "reana-yadage.yaml"
+    ),
+)
+def test_run_example_check_only_one_fail_one_pass(
+    mock_run_command, mock_get_example_reana_yaml_file_path
+):
+    """Test for run-example command with check-only flag, and where one example fails and one passes."""
+    env = {"REANA_SERVER_URL": "localhost"}
+    runner = CliRunner(env=env)
+    with runner.isolation():
+        result = runner.invoke(
+            reana_dev,
+            [
+                "run-example",
+                "-c",
+                "r-d-r-roofit",
+                "-w",
+                "cwl",
+                "-w",
+                "yadage",
+                "--check-only",
+            ],
+        )
+        assert "2 submitted" in result.output
+        assert "1 passed" in result.output
+        assert "1 failed: root6-roofit-cwl-kubernetes" in result.output
 
 
 def test_is_component_python_package():