Filter out log messages about non-existing pars- reffiles

.github/workflows/ci.yml

@@ -21,12 +21,18 @@ jobs:
  test:
  uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
  with:
+ setenv: |
+ CRDS_PATH: /tmp/data/crds_cache
+ CRDS_CLIENT_RETRY_COUNT: 3
+ CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
  envs: |
  - linux: py39-oldestdeps-cov-xdist
  - linux: py310-xdist
  - macos: py311-xdist
  - linux: py311-cov-xdist
  coverage: 'codecov'
+ - linux: py311-downstreamdeps-cov-xdist
+ coverage: 'codecov'
  - linux: py312-xdist
  test_downstream:
  uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1

.github/workflows/ci_cron.yml

@@ -23,6 +23,10 @@ jobs:
  uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
  if: (github.repository == 'spacetelescope/stpipe' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Weekly CI')))
  with:
+ setenv: |
+ CRDS_PATH: /tmp/data/crds_cache
+ CRDS_CLIENT_RETRY_COUNT: 3
+ CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
  envs: |
  - macos: py39-xdist
  - macos: py310-xdist

CHANGES.rst

@@ -6,6 +6,7 @@
 - Remove bundled ``configobj`` package in favor of the ``configobj`` package
  bundled into ``astropy``. [#122]
 - Fix ``datetime.utcnow()`` DeprecationWarning [#134]
+- Filter out log messages about non-existing pars- reffiles [#135]
 
 0.5.1 (2023-10-02)
 ==================

src/stpipe/crds_client.py

@@ -7,6 +7,7 @@
 directly in modules other than this crds_client so that dependency order and
 general integration can be managed here.
 """
+import logging
 import re
 
 import crds
@@ -49,6 +50,14 @@ def get_multiple_reference_paths(parameters, reference_file_types, observatory):
  raise TypeError("First argument must be a dict of parameters")
 
  log.set_log_time(True)
+
+ def parsfilter(record):
+ if "Error determining best reference for 'pars-" in record.getMessage():
+ return False
+ return True
+
+ logging.getLogger("CRDS").addFilter(parsfilter)
+
  return _get_refpaths(parameters, tuple(reference_file_types), observatory)
 
 

tests/test_crds_client.py

@@ -0,0 +1,30 @@
+import pytest
+from crds.core.exceptions import CrdsLookupError
+
+from stpipe import crds_client
+
+
+def test_missing_pars_log_filtering(capfd):
+ # CRDS needs stdatamodels to handle JWST lookups
+ pytest.importorskip("stdatamodels.jwst")
+
+ # A made-up pars- reffile will raise an exception in CRDS
+ with pytest.raises(CrdsLookupError):
+ crds_client.get_multiple_reference_paths(
+ parameters={
+ "meta.instrument.name": "NIRCAM",
+ "meta.telescope": "JWST",
+ },
+ reference_file_types=["pars-crunchyfrogstep"],
+ observatory="jwst",
+ )
+
+ # The following will always be true because of a bug in how pytest handles
+ # oddball logging setups, as used by crds. See issue
+ # https://github.com/pytest-dev/pytest/issues/5997
+ # So don't rely on this test passing (currently) to be actually testing what
+ # you think it is.
+ capture = capfd.readouterr()
+ assert (
+ "Error determining best reference for 'pars-crunchyfrogstep'" not in capture.err
+ )

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 envlist =
  check-{style,security,build}
- test{,-warnings,-cov,-xdist,-oldestdeps,-devdeps}
+ test{,-warnings,-cov,-xdist,-oldestdeps,-devdeps,-downstreamdeps}
  test-{jwst,romancal}-xdist
  build-{docs,dist}
 
@@ -37,6 +37,7 @@ description =
  jwst: of JWST pipeline
  romancal: of Romancal pipeline
  oldestdeps: with the oldest supported version of key dependencies
+ downstreamdeps: with the downstream packages that depend on stpipe
  warnings: treating warnings as errors
  cov: with coverage
  xdist: using parallel processing
@@ -49,12 +50,13 @@ deps =
  romancal: romancal[test] @ git+https://github.com/spacetelescope/romancal.git
  oldestdeps: minimum_dependencies
  devdeps: astropy>=0.0.dev0
+ downstreamdeps: stdatamodels
 pass_env =
  CRDS_*
  CI
 set_env =
  devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
- jwst: CRDS_SERVER_URL=https://jwst-crds.stsci.edu
+ !romancal: CRDS_SERVER_URL=https://jwst-crds.stsci.edu
  romancal: CRDS_SERVER_URL=https://roman-crds.stsci.edu
 package =
  !cov: wheel