diff --git a/docs/releases.rst b/docs/releases.rst index 777a4cdee4..182a85764b 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -18,6 +18,12 @@ option or with the environment variable ``TMT_FEELING_SAFE`` set to ``True``. See the :ref:`/stories/features/feeling-safe` section for more details and motivation behind this change. +The :ref:`/plugins/discover/fmf` discover plugin now supports +a new ``adjust-tests`` key which allows modifying metadata of all +discovered tests. This can be useful especially when fetching +tests from remote repositories where the user does not have write +access. + tmt-1.37.0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pyproject.toml b/pyproject.toml index a77a42a86d..10f2974c90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ dependencies = [ # F39 / PyPI "click>=8.0.3,!=8.1.4", # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 "docutils>=0.16", # 0.16 is the current one available for RHEL9 - "fmf>=1.3.0", + "fmf>=1.4.0", "jinja2>=2.11.3", # 3.1.2 / 3.1.2 "packaging>=20", # 20 seems to be available with RHEL8 "pint>=0.16.1", # 0.16.1 @@ -87,7 +87,7 @@ docs = [ "readthedocs-sphinx-ext", "docutils>=0.18.1", "Sphinx==7.3.7", - "fmf>=1.3.0", + "fmf>=1.4.0", ] [project.scripts] diff --git a/tests/discover/adjust-tests.sh b/tests/discover/adjust-tests.sh new file mode 100755 index 0000000000..c3756ae330 --- /dev/null +++ b/tests/discover/adjust-tests.sh @@ -0,0 +1,32 @@ +#!/bin/bash +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +rlJournalStart + rlPhaseStartSetup + rlRun "pushd data" + rlRun "run=\$(mktemp -d)" + rlPhaseEnd + + rlPhaseStartTest + rlRun -s "tmt -c trigger=commit run -i $run discover plans --name /fmf/adjust-tests" + # If we ever change the path... + tests_yaml="$(find $run -name tests.yaml)" + rlAssertExits "$tests_yaml" + rlRun -s "yq '.[].require' < $tests_yaml" + rlAssertGrep "foo" "$rlRun_LOG" + rlRun -s "yq '.[].duration' < $tests_yaml" + # 'duration_to_seconds' takes care of injection the default '5m' as the base + rlAssertGrep "*2" "$rlRun_LOG" + # check added + rlRun -s "yq '.[].check' < $tests_yaml" + rlAssertGrep "avc" $rlRun_LOG + # recommend should not contain FAILURE + rlRun -s "yq '.[].recommend' < $tests_yaml" + rlAssertNotGrep "FAILURE" "$rlRun_LOG" + rlPhaseEnd + + rlPhaseStartCleanup + rlRun "rm -rf $run" + rlRun "popd" + rlPhaseEnd +rlJournalEnd diff --git a/tests/discover/data/plans.fmf b/tests/discover/data/plans.fmf index 0e8245c7b1..9d315be9ff 100644 --- a/tests/discover/data/plans.fmf +++ b/tests/discover/data/plans.fmf @@ -203,3 +203,19 @@ execute: url: https://github.com/teemtee/tmt how: fmf ref: "@tests/discover/data/dynamic-ref.fmf" + /adjust-tests: + discover: + how: fmf + test: /tests/discover1 + adjust-tests: + - duration+: "*2" + - recommend: + - FAILURE + when: trigger is not defined + because: check if context is evaluated + - require+: + - foo + when: trigger == commit + because: check if context is evaluated + - check+: + - how: avc diff --git a/tests/discover/main.fmf b/tests/discover/main.fmf index 5a6632df19..85d2c7eac5 100644 --- a/tests/discover/main.fmf +++ b/tests/discover/main.fmf @@ -72,3 +72,7 @@ tier: 3 /exception: summary: Verify no color when throwing an exception if '--no-color' is specified test: ./exception.sh + +/adjust-tests: + summary: Change test metadata within discover phase + test: ./adjust-tests.sh diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ac02d0f466..f5b96ad3f6 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -341,6 +341,11 @@ def test_duration_to_seconds(): assert duration_to_seconds('*2 *3 1m4') == 384 # Round up assert duration_to_seconds('1s *3.3') == 4 + # Value might be just the multiplication + # without the default it thus equals zero + assert duration_to_seconds('*2') == 0 + # however the supplied "default" can be used: (1m * 2) + assert duration_to_seconds('*2', injected_default="1m") == 120 @pytest.mark.parametrize("duration", [ diff --git a/tmt/base.py b/tmt/base.py index 723091ac33..21999409e2 100644 --- a/tmt/base.py +++ b/tmt/base.py @@ -2857,6 +2857,7 @@ def __init__(self, path: Optional[Path] = None, tree: Optional[fmf.Tree] = None, fmf_context: Optional[FmfContext] = None, + additional_rules: Optional[list[_RawAdjustRule]] = None, logger: tmt.log.Logger) -> None: """ Initialize tmt tree from directory path or given fmf tree """ @@ -2866,6 +2867,7 @@ def __init__(self, self._path = path or Path.cwd() self._tree = tree self._custom_fmf_context = fmf_context or FmfContext() + self._additional_rules = additional_rules @classmethod def grow( @@ -2980,7 +2982,8 @@ def tree(self) -> fmf.Tree: self._tree.adjust( fmf.context.Context(**self._fmf_context), case_sensitive=False, - decision_callback=create_adjust_callback(self._logger)) + decision_callback=create_adjust_callback(self._logger), + additional_rules=self._additional_rules) return self._tree @tree.setter diff --git a/tmt/schemas/discover/fmf.yaml b/tmt/schemas/discover/fmf.yaml index 630aa62cdb..df79a1841c 100644 --- a/tmt/schemas/discover/fmf.yaml +++ b/tmt/schemas/discover/fmf.yaml @@ -95,5 +95,8 @@ properties: where: $ref: "/schemas/common#/definitions/where" + adjust-tests: + $ref: "/schemas/core#/definitions/adjust" + required: - how diff --git a/tmt/steps/discover/fmf.py b/tmt/steps/discover/fmf.py index 163ab04980..2c4f5d6e12 100644 --- a/tmt/steps/discover/fmf.py +++ b/tmt/steps/discover/fmf.py @@ -18,6 +18,7 @@ import tmt.steps.discover import tmt.utils import tmt.utils.git +from tmt.base import _RawAdjustRule from tmt.steps.prepare.distgit import insert_to_prepare_step from tmt.utils import Command, Environment, EnvVarValue, Path, field @@ -157,6 +158,15 @@ class DiscoverFmfStepData(tmt.steps.discover.DiscoverStepData): show_default=True, help="Copy only immediate directories of executed tests and their required files.") + # Edit discovered tests + adjust_tests: Optional[list[_RawAdjustRule]] = field( + default_factory=list, + normalize=tmt.utils.normalize_adjust, + help=""" + Modify metadata of discovered tests from the plan itself. Use the + same format as for adjust rules. + """) + # Upgrade plan path so the plan is not pruned upgrade_path: Optional[str] = None @@ -260,6 +270,29 @@ class DiscoverFmf(tmt.steps.discover.DiscoverPlugin[DiscoverFmfStepData]): Note that internally the modified tests are appended to the list specified via ``test``, so those tests will also be selected even if not modified. + + Use the ``adjust-tests`` key to modify the discovered tests' + metadata directly from the plan. For example, extend the test + duration for slow hardware or modify the list of required packages + when you do not have write access to the remote test repository. + The value should follow the ``adjust`` rules syntax. + + The following example adds an ``avc`` check for each discovered + test, doubles its duration and replaces each occurrence of the word + ``python3.11`` in the list of required packages. + + .. code-block:: yaml + + discover: + how: fmf + adjust-tests: + - check+: + - how: avc + - duration+: '*2' + because: Slow system under test + when: arch == i286 + - require~: + - '/python3.11/python3.12/' """ _data_class = DiscoverFmfStepData @@ -564,7 +597,8 @@ def do_the_discovery(self, path: Optional[Path] = None) -> None: tree = tmt.Tree( logger=self._logger, path=tree_path, - fmf_context=self.step.plan._fmf_context) + fmf_context=self.step.plan._fmf_context, + additional_rules=self.data.adjust_tests) self._tests = tree.tests( filters=filters, names=names, diff --git a/tmt/steps/execute/internal.py b/tmt/steps/execute/internal.py index 595d7b68e8..7f8332fe20 100644 --- a/tmt/steps/execute/internal.py +++ b/tmt/steps/execute/internal.py @@ -401,7 +401,8 @@ def _save_process( timeout = None else: - timeout = tmt.utils.duration_to_seconds(test.duration) + timeout = tmt.utils.duration_to_seconds( + test.duration, tmt.base.DEFAULT_TEST_DURATION_L1) try: output = guest.execute( diff --git a/tmt/utils/__init__.py b/tmt/utils/__init__.py index 583fe2eeb3..c0b1441f36 100644 --- a/tmt/utils/__init__.py +++ b/tmt/utils/__init__.py @@ -3344,8 +3344,13 @@ def shell_variables( return [f"{key}={shlex.quote(str(value))}" for key, value in data.items()] -def duration_to_seconds(duration: str) -> int: - """ Convert extended sleep time format into seconds """ +def duration_to_seconds(duration: str, injected_default: Optional[str] = None) -> int: + """ + Convert extended sleep time format into seconds + + Optional 'injected_default' argument to evaluate 'duration' when + it contains only multiplication. + """ units = { 's': 1, 'm': 60, @@ -3389,6 +3394,9 @@ def duration_to_seconds(duration: str) -> int: multiply_by *= float(match['float']) else: total_time += int(match['digit']) * units.get(match['suffix'], 1) + # Inject value so we have something to multiply + if injected_default and total_time == 0: + total_time = duration_to_seconds(injected_default) # Multiply in the end and round up return ceil(total_time * multiply_by) @@ -5405,6 +5413,18 @@ def normalize_shell_script( raise NormalizationError(key_address, value, 'a string') +def normalize_adjust( + key_address: str, + raw_value: Any, + logger: tmt.log.Logger) -> Optional[list['tmt.base._RawAdjustRule']]: + + if raw_value is None: + return [] + if isinstance(raw_value, list): + return raw_value + return [raw_value] + + def normalize_string_dict( key_address: str, raw_value: Any,