Override test metada from plan using adjust-tests (#2865)

New option for discover -h fmf to adjust discovered tests

Fix: #2430

Co-authored-by: Petr Šplíchal <psplicha@redhat.com>

docs/releases.rst

@@ -18,6 +18,12 @@ option or with the environment variable ``TMT_FEELING_SAFE`` set
 to ``True``. See the :ref:`/stories/features/feeling-safe` section
 for more details and motivation behind this change.
 
+The :ref:`/plugins/discover/fmf` discover plugin now supports
+a new ``adjust-tests`` key which allows modifying metadata of all
+discovered tests. This can be useful especially when fetching
+tests from remote repositories where the user does not have write
+access.
+
 
 tmt-1.37.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
 dependencies = [              # F39 / PyPI
     "click>=8.0.3,!=8.1.4",   # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558
     "docutils>=0.16",         # 0.16 is the current one available for RHEL9
-    "fmf>=1.3.0",
+    "fmf>=1.4.0",
     "jinja2>=2.11.3",         # 3.1.2 / 3.1.2
     "packaging>=20",          # 20 seems to be available with RHEL8
     "pint>=0.16.1",           # 0.16.1
@@ -87,7 +87,7 @@ docs = [
     "readthedocs-sphinx-ext",
     "docutils>=0.18.1",
     "Sphinx==7.3.7",
-    "fmf>=1.3.0",
+    "fmf>=1.4.0",
     ]
 
 [project.scripts]

tests/discover/adjust-tests.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "pushd data"
+        rlRun "run=\$(mktemp -d)"
+    rlPhaseEnd
+
+    rlPhaseStartTest
+        rlRun -s "tmt -c trigger=commit run -i $run discover plans --name /fmf/adjust-tests"
+        # If we ever change the path...
+        tests_yaml="$(find $run -name tests.yaml)"
+        rlAssertExits "$tests_yaml"
+        rlRun -s "yq '.[].require' < $tests_yaml"
+        rlAssertGrep "foo" "$rlRun_LOG"
+        rlRun -s "yq '.[].duration' < $tests_yaml"
+        # 'duration_to_seconds' takes care of injection the default '5m' as the base
+        rlAssertGrep "*2" "$rlRun_LOG"
+        # check added
+        rlRun -s "yq '.[].check' < $tests_yaml"
+        rlAssertGrep "avc" $rlRun_LOG
+        # recommend should not contain FAILURE
+        rlRun -s "yq '.[].recommend' < $tests_yaml"
+        rlAssertNotGrep "FAILURE" "$rlRun_LOG"
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "rm -rf $run"
+        rlRun "popd"
+    rlPhaseEnd
+rlJournalEnd

tests/discover/data/plans.fmf

@@ -203,3 +203,19 @@ execute:
                 url: https://github.com/teemtee/tmt
                 how: fmf
                 ref: "@tests/discover/data/dynamic-ref.fmf"
+    /adjust-tests:
+        discover:
+            how: fmf
+            test: /tests/discover1
+            adjust-tests:
+            - duration+: "*2"
+            - recommend:
+                - FAILURE
+              when: trigger is not defined
+              because: check if context is evaluated
+            - require+:
+                - foo
+              when: trigger == commit
+              because: check if context is evaluated
+            - check+:
+                - how: avc

tests/discover/main.fmf

@@ -72,3 +72,7 @@ tier: 3
 /exception:
     summary: Verify no color when throwing an exception if '--no-color' is specified
     test: ./exception.sh
+
+/adjust-tests:
+    summary: Change test metadata within discover phase
+    test: ./adjust-tests.sh

tests/unit/test_utils.py

@@ -341,6 +341,11 @@ def test_duration_to_seconds():
     assert duration_to_seconds('*2 *3 1m4') == 384
     # Round up
     assert duration_to_seconds('1s *3.3') == 4
+    # Value might be just the multiplication
+    #   without the default it thus equals zero
+    assert duration_to_seconds('*2') == 0
+    #   however the supplied "default" can be used: (1m * 2)
+    assert duration_to_seconds('*2', injected_default="1m") == 120
 
 
 @pytest.mark.parametrize("duration", [

tmt/base.py

@@ -2857,6 +2857,7 @@ def __init__(self,
                  path: Optional[Path] = None,
                  tree: Optional[fmf.Tree] = None,
                  fmf_context: Optional[FmfContext] = None,
+                 additional_rules: Optional[list[_RawAdjustRule]] = None,
                  logger: tmt.log.Logger) -> None:
         """ Initialize tmt tree from directory path or given fmf tree """
 
@@ -2866,6 +2867,7 @@ def __init__(self,
         self._path = path or Path.cwd()
         self._tree = tree
         self._custom_fmf_context = fmf_context or FmfContext()
+        self._additional_rules = additional_rules
 
     @classmethod
     def grow(
@@ -2980,7 +2982,8 @@ def tree(self) -> fmf.Tree:
             self._tree.adjust(
                 fmf.context.Context(**self._fmf_context),
                 case_sensitive=False,
-                decision_callback=create_adjust_callback(self._logger))
+                decision_callback=create_adjust_callback(self._logger),
+                additional_rules=self._additional_rules)
         return self._tree
 
     @tree.setter

tmt/schemas/discover/fmf.yaml

@@ -95,5 +95,8 @@ properties:
   where:
     $ref: "/schemas/common#/definitions/where"
 
+  adjust-tests:
+    $ref: "/schemas/core#/definitions/adjust"
+
 required:
   - how

tmt/steps/discover/fmf.py

@@ -18,6 +18,7 @@
 import tmt.steps.discover
 import tmt.utils
 import tmt.utils.git
+from tmt.base import _RawAdjustRule
 from tmt.steps.prepare.distgit import insert_to_prepare_step
 from tmt.utils import Command, Environment, EnvVarValue, Path, field
 
@@ -157,6 +158,15 @@ class DiscoverFmfStepData(tmt.steps.discover.DiscoverStepData):
         show_default=True,
         help="Copy only immediate directories of executed tests and their required files.")
 
+    # Edit discovered tests
+    adjust_tests: Optional[list[_RawAdjustRule]] = field(
+        default_factory=list,
+        normalize=tmt.utils.normalize_adjust,
+        help="""
+             Modify metadata of discovered tests from the plan itself. Use the
+             same format as for adjust rules.
+             """)
+
     # Upgrade plan path so the plan is not pruned
     upgrade_path: Optional[str] = None
 
@@ -260,6 +270,29 @@ class DiscoverFmf(tmt.steps.discover.DiscoverPlugin[DiscoverFmfStepData]):
     Note that internally the modified tests are appended to the list
     specified via ``test``, so those tests will also be selected even if
     not modified.
+
+    Use the ``adjust-tests`` key to modify the discovered tests'
+    metadata directly from the plan. For example, extend the test
+    duration for slow hardware or modify the list of required packages
+    when you do not have write access to the remote test repository.
+    The value should follow the ``adjust`` rules syntax.
+
+    The following example adds an ``avc`` check for each discovered
+    test, doubles its duration and replaces each occurrence of the word
+    ``python3.11`` in the list of required packages.
+
+    .. code-block:: yaml
+
+        discover:
+            how: fmf
+            adjust-tests:
+              - check+:
+                  - how: avc
+              - duration+: '*2'
+                because: Slow system under test
+                when: arch == i286
+              - require~:
+                  - '/python3.11/python3.12/'
     """
 
     _data_class = DiscoverFmfStepData
@@ -564,7 +597,8 @@ def do_the_discovery(self, path: Optional[Path] = None) -> None:
         tree = tmt.Tree(
             logger=self._logger,
             path=tree_path,
-            fmf_context=self.step.plan._fmf_context)
+            fmf_context=self.step.plan._fmf_context,
+            additional_rules=self.data.adjust_tests)
         self._tests = tree.tests(
             filters=filters,
             names=names,

tmt/steps/execute/internal.py

@@ -401,7 +401,8 @@ def _save_process(
                 timeout = None
 
             else:
-                timeout = tmt.utils.duration_to_seconds(test.duration)
+                timeout = tmt.utils.duration_to_seconds(
+                    test.duration, tmt.base.DEFAULT_TEST_DURATION_L1)
 
             try:
                 output = guest.execute(

tmt/utils/__init__.py

@@ -3344,8 +3344,13 @@ def shell_variables(
     return [f"{key}={shlex.quote(str(value))}" for key, value in data.items()]
 
 
-def duration_to_seconds(duration: str) -> int:
-    """ Convert extended sleep time format into seconds """
+def duration_to_seconds(duration: str, injected_default: Optional[str] = None) -> int:
+    """
+    Convert extended sleep time format into seconds
+
+    Optional 'injected_default' argument to evaluate 'duration' when
+    it contains only multiplication.
+    """
     units = {
         's': 1,
         'm': 60,
@@ -3389,6 +3394,9 @@ def duration_to_seconds(duration: str) -> int:
             multiply_by *= float(match['float'])
         else:
             total_time += int(match['digit']) * units.get(match['suffix'], 1)
+    # Inject value so we have something to multiply
+    if injected_default and total_time == 0:
+        total_time = duration_to_seconds(injected_default)
     # Multiply in the end and round up
     return ceil(total_time * multiply_by)
 
@@ -5405,6 +5413,18 @@ def normalize_shell_script(
     raise NormalizationError(key_address, value, 'a string')
 
 
+def normalize_adjust(
+        key_address: str,
+        raw_value: Any,
+        logger: tmt.log.Logger) -> Optional[list['tmt.base._RawAdjustRule']]:
+
+    if raw_value is None:
+        return []
+    if isinstance(raw_value, list):
+        return raw_value
+    return [raw_value]
+
+
 def normalize_string_dict(
         key_address: str,
         raw_value: Any,