Introducing "testing" object (AcademySoftwareFoundation#1852)

Signed-off-by: Fabrice Macagno <fabrice.macagno@al.com.au>
Signed-off-by: Nathan Cheung <nathan.cheung.451@outlook.com>
Signed-off-by: Jean-Christophe Morin <38703886+JeanChristopheMorinPerso@users.noreply.github.com>
Co-authored-by: Fabrice Macagno <fabrice.macagno@al.com.au>
Co-authored-by: Nathan Cheung <nathan.cheung.451@outlook.com>

docs/source/package_commands.rst

@@ -707,6 +707,19 @@ Following is a list of the objects and functions available.
  if test.name == "unit":
  info("My unit test is about to run yay")
 
+.. py:attribute:: testing
+ :type: bool
+
+ This boolean variable is ``True`` if a test is occurring (typically done via the :ref:`rez-test` tool),
+ and ``False`` otherwise.
+
+ A package can use this variable to set environment variables that are only relevant during test execution.
+
+ .. code-block:: python
+
+ if testing:
+ env.FOO_TEST_DATA_PATH = "{root}/tests/data"
+
 .. py:attribute:: this
 
  The ``this`` object represents the current package. The following attributes are most commonly used

docs/source/package_definition.rst

@@ -290,6 +290,7 @@ is ``True``:
 * **context**: the :class:`~rez.resolved_context.ResolvedContext` instance this package belongs to;
 * **system**: see :attr:`system`;
 * **building**: see :attr:`building`;
+* **testing**: see :attr:`testing`;
 * **request**: see :attr:`request`;
 * **implicits**: see :attr:`implicits`.
 

src/rez/data/tests/builds/packages/testing_obj/1.0.0/build.py

@@ -0,0 +1,24 @@
+from build_util import build_directory_recurse
+import os.path
+
+
+def build(source_path, build_path, install_path, targets):
+
+ if "install" not in (targets or []):
+ install_path = None
+
+ build_directory_recurse(src_dir="testing_obj",
+ dest_dir=os.path.join("python", "testing_obj"),
+ source_path=source_path,
+ build_path=build_path,
+ install_path=install_path)
+
+
+if __name__ == '__main__':
+ import os, sys
+ build(
+ source_path=os.environ['REZ_BUILD_SOURCE_PATH'],
+ build_path=os.environ['REZ_BUILD_PATH'],
+ install_path=os.environ['REZ_BUILD_INSTALL_PATH'],
+ targets=sys.argv[1:]
+ )

src/rez/data/tests/builds/packages/testing_obj/1.0.0/package.py

@@ -0,0 +1,42 @@
+name = 'testing obj'
+version = '1.0.0'
+authors = ["Dan Flashes"]
+
+description = "testing the 'testing' attribute available during rez test"
+
+@late()
+def requires():
+ if in_context() and testing:
+ return ["floob"]
+ return ["hello"]
+
+private_build_requires = ["build_util", "python"]
+
+def commands():
+ env.PYTHONPATH.append('{root}/python')
+ if testing:
+ env.CAR_IDEA = "STURDY STEERING WHEEL"
+ else:
+ env.SKIP_LUNCH = "False"
+
+build_command = 'python {root}/build.py {install}'
+
+tests = {
+ "command_as_string_success": {
+ "command": "exit 0"
+ },
+ "command_as_string_fail": {
+ "command": "exit 1"
+ },
+ "check_car_ideas": {
+ "command": ["python", "-c", "import os; assert os.environ.get('CAR_IDEA') == 'STURDY STEERING WHEEL'"],
+ "requires": ["python"]
+ },
+ "move_meeting_to_noon": {
+ # We want this test to fail. SKIP_LUNCH should not be set.
+ # TODO: We should not test for failures here. Testing failures, str vs lsit commands, etc
+ # should we tested separately.
+ "command": ["python", "-c", "import os; assert os.environ.get('SKIP_LUNCH') is not None"],
+ "requires": ["python"]
+ }
+}

src/rez/data/tests/builds/packages/testing_obj/1.0.0/testing_obj/__init__.py

@@ -0,0 +1,2 @@
+def hello():
+ return "This shirt was $150 out the door and the pattern's not that complicated"

src/rez/package_test.py

@@ -603,6 +603,7 @@ def _get_context(self, requires, quiet=False):
  package_paths=self.package_paths,
  buf=(f if quiet else None),
  timestamp=self.timestamp,
+ testing=True,
  **self.context_kwargs
  )
 

src/rez/resolved_context.py

@@ -163,7 +163,7 @@ def __call__(self, state):
  return SolverCallbackReturn.keep_going, ''
 
  def __init__(self, package_requests, verbosity=0, timestamp=None,
- building=False, caching=None, package_paths=None,
+ building=False, testing=False, caching=None, package_paths=None,
  package_filter=None, package_orderers=None, max_fails=-1,
  add_implicit_packages=True, time_limit=-1, callback=None,
  package_load_callback=None, buf=None, suppress_passive=False,
@@ -176,6 +176,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
  timestamp (float): Ignore packages released after this epoch time. Packages
  released at exactly this time will not be ignored.
  building (bool): True if we're resolving for a build.
+ testing (bool): True if we're resolving for a test (rez-test).
  caching (bool): If True, cache(s) may be used to speed the resolve. If
  False, caches will not be used. If None, :data:`resolve_caching`
  is used.
@@ -214,6 +215,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
  self.requested_timestamp = timestamp
  self.timestamp = self.requested_timestamp or int(time.time())
  self.building = building
+ self.testing = testing
  self.implicit_packages = []
  self.caching = config.resolve_caching if caching is None else caching
  self.verbosity = verbosity
@@ -1553,6 +1555,7 @@ def _add(field):
  timestamp=self.timestamp,
  requested_timestamp=self.requested_timestamp,
  building=self.building,
+ testing=self.testing,
  caching=self.caching,
  implicit_packages=list(map(str, self.implicit_packages)),
  package_requests=list(map(str, self._package_requests)),
@@ -1627,6 +1630,7 @@ def _print_version(value):
 
  r.timestamp = d["timestamp"]
  r.building = d["building"]
+ r.testing = d["testing"]
  r.caching = d["caching"]
  r.implicit_packages = [PackageRequest(x) for x in d["implicit_packages"]]
  r._package_requests = [PackageRequest(x) for x in d["package_requests"]]
@@ -1959,6 +1963,7 @@ def _get_pre_resolve_bindings(self):
  self.pre_resolve_bindings = {
  "system": system,
  "building": self.building,
+ "testing": self.testing,
  "request": RequirementsBinding(self._package_requests),
  "implicits": RequirementsBinding(self.implicit_packages),
  "intersects": intersects

src/rez/resolver.py

@@ -37,8 +37,8 @@ class Resolver(object):
  """
  def __init__(self, context, package_requests, package_paths, package_filter=None,
  package_orderers=None, timestamp=0, callback=None, building=False,
- verbosity=False, buf=None, package_load_callback=None, caching=True,
- suppress_passive=False, print_stats=False):
+ testing=False, verbosity=False, buf=None, package_load_callback=None,
+ caching=True, suppress_passive=False, print_stats=False):
  """Create a Resolver.
 
  Args:
@@ -52,6 +52,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None
  prior to each package being loaded. It is passed a single
  `Package` object.
  building: True if we're resolving for a build.
+ testing: True if we're resolving for a rez (rez-test).
  caching: If True, cache(s) may be used to speed the resolve. If
  False, caches will not be used.
  print_stats (bool): If true, print advanced solver stats at the end.
@@ -64,6 +65,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None
  self.package_orderers = package_orderers
  self.package_load_callback = package_load_callback
  self.building = building
+ self.testing = testing
  self.verbosity = verbosity
  self.caching = caching
  self.buf = buf
@@ -384,6 +386,7 @@ def _memcache_key(self, timestamped=False):
  self.package_filter_hash,
  self.package_orderers_hash,
  self.building,
+ self.testing,
  config.prune_failed_graph]
 
  if timestamped and self.timestamp:

src/rez/serialise.py

@@ -184,6 +184,7 @@ def _load_file(filepath, format_, update_data_callback, original_filepath=None):
 # Default variables to avoid not-defined errors in early-bound attribs
 default_objects = {
  "building": False,
+ "testing": False,
  "build_variant_index": 0,
  "build_variant_requires": []
 }

src/rez/tests/test_context.py

@@ -71,6 +71,26 @@ def test_execute_command(self):
  stdout = stdout.strip()
  self.assertEqual(stdout, "Hello Rez World!")
 
+ def test_resolved_packages_testing_environ(self):
+ """Test resolving packages within a testing environment behaves correctly"""
+ packages_path = self.data_path("builds", "packages")
+
+ # Note how we use testing=True
+ r = ResolvedContext(["testing_obj"], testing=True, package_paths=[packages_path])
+ resolvedPackages = [x.qualified_package_name for x in r.resolved_packages]
+ self.assertEqual(resolvedPackages, ["floob", "testing_obj-1.0.0"])
+
+ def test_execute_command_testing_environ(self):
+ """Test that execute_command properly sets test specific environ dict"""
+ self.inject_python_repo()
+ packages_path = self.data_path("builds", "packages")
+ r = ResolvedContext(
+ ["testing_obj", "python"],
+ testing=True,
+ package_paths=[packages_path] + self.settings["packages_path"]
+ )
+ self.assertEqual(r.get_environ().get("CAR_IDEA"), "STURDY STEERING WHEEL")
+
  def test_execute_command_environ(self):
  """Test that execute_command properly sets environ dict."""
  self.inject_python_repo()

src/rez/tests/test_test.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Contributors to the Rez Project
+
+
+"""
+test rez package.py unit tests
+"""
+from rez.tests.util import TestBase, TempdirMixin
+from rez.resolved_context import ResolvedContext
+from rez.package_test import PackageTestRunner
+
+
+class TestTest(TestBase, TempdirMixin):
+ @classmethod
+ def setUpClass(cls):
+ TempdirMixin.setUpClass()
+
+ packages_path = cls.data_path("builds", "packages")
+ cls.settings = dict(
+ packages_path=[packages_path],
+ package_filter=None,
+ implicit_packages=[],
+ warn_untimestamped=False,
+ resolve_caching=False
+ )
+
+ @classmethod
+ def tearDownClass(cls):
+ TempdirMixin.tearDownClass()
+
+ def test_1(self):
+ """package.py unit tests are correctly run in a testing environment"""
+ self.inject_python_repo()
+ context = ResolvedContext(["testing_obj", "python"])
+ self._run_tests(context)
+
+ def test_2(self):
+ """package.py unit tests are correctly run in a testing environment when no verbosity is set"""
+ self.inject_python_repo()
+ context = ResolvedContext(["testing_obj", "python"])
+ # This will get us more code coverage :)
+ self._run_tests(context, verbose=0)
+
+ def _run_tests(self, r, verbose=2):
+ """Run unit tests in package.py"""
+ self.inject_python_repo()
+ runner = PackageTestRunner(
+ package_request="testing_obj",
+ package_paths=r.package_paths,
+ stop_on_fail=False,
+ verbose=verbose
+ )
+
+ test_names = runner.get_test_names()
+
+ for test_name in test_names:
+ runner.run_test(test_name)
+
+ self.assertEqual(runner.test_results.num_tests, 4)
+ self.assertEqual(
+ self._get_test_result(runner, "check_car_ideas")["status"],
+ "success",
+ "check_car_ideas did not succeed",
+ )
+ self.assertEqual(
+ self._get_test_result(runner, "move_meeting_to_noon")["status"],
+ "failed",
+ "move_meeting_to_noon did not fail",
+ )
+ self.assertEqual(
+ self._get_test_result(runner, "command_as_string_success")["status"],
+ "success",
+ "command_as_string_success did not succeed",
+ )
+ self.assertEqual(
+ self._get_test_result(runner, "command_as_string_fail")["status"],
+ "failed",
+ "command_as_string_fail did not fail",
+ )
+
+ def _get_test_result(self, runner, test_name):
+ return next(
+ (result for result in runner.test_results.test_results if result.get("test_name") == test_name),
+ None
+ )