Merge pull request #950 from akrherz/date422

Allow autoplot date parsing failures to be handled downstream as HTTP 422s

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
  autoupdate_schedule: quarterly
 repos:
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: "v0.5.7"
+ rev: "v0.6.0"
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this library are documented in this file.
 
 - Discontinue persisting text product into {mcd,mpd} table storage.
 - Discontinue storage of text product into sigmets, use `product_id` instead.
+- Raise `IncompleteWebRequest` exception when autoplot datetime parsing fails.
 
 ### New Features
 

src/pyiem/util.py

@@ -53,6 +53,18 @@ def format(self, record):
  )
 
 
+def _strptime(ins: str, fmt: str, rectify: bool = False) -> datetime:
+ """Wrapper around strptime."""
+ try:
+ return datetime.strptime(ins, fmt)
+ except ValueError as exp:
+ if rectify:
+ return handle_date_err(exp, ins, fmt)
+ raise IncompleteWebRequest(
+ f"String provided: `{ins}` does not match format: `{fmt}`"
+ ) from exp
+
+
 def web2ldm(url, ldm_product_name, md5_from_name=False, pqinsert="pqinsert"):
  """Download a URL and insert into LDM.
 
@@ -321,7 +333,8 @@ def handle_date_err(exp, value, fmt):
  res = lastday.strftime("%Y-%m-%d")
  if len(tokens) > 1:
  res += " " + tokens[1]
- return datetime.strptime(res, fmt)
+ # Careful here, we don't want a recursive loop
+ return _strptime(res, fmt, rectify=False)
 
 
 def get_autoplot_context(fdict, cfg, enforce_optional=False, **kwargs):
@@ -442,11 +455,11 @@ def _float(val):
  elif typ == "datetime":
  # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
  if default is not None:
- default = datetime.strptime(default, "%Y/%m/%d %H%M")
+ default = _strptime(default, "%Y/%m/%d %H%M")
  if minval is not None:
- minval = datetime.strptime(minval, "%Y/%m/%d %H%M")
+ minval = _strptime(minval, "%Y/%m/%d %H%M")
  if maxval is not None:
- maxval = datetime.strptime(maxval, "%Y/%m/%d %H%M")
+ maxval = _strptime(maxval, "%Y/%m/%d %H%M")
  if value is not None:
  # A common problem is for the space to be missing
  if value.find(" ") == -1:
@@ -455,50 +468,40 @@ def _float(val):
  else:
  value += " 0000"
  _dtfmt = "%Y-%m-%d %H%M"
- try:
- value = datetime.strptime(
- value[:15].replace("/", "-"), "%Y-%m-%d %H%M"
- )
- except ValueError as exp:
- if kwargs.get("rectify_dates", False):
- value = handle_date_err(exp, value, _dtfmt)
- else:
- # If we are not rectifying dates, we just raise the
- # exception
- raise
+ value = _strptime(
+ value[:15].replace("/", "-"),
+ "%Y-%m-%d %H%M",
+ rectify=kwargs.get("rectify_dates", False),
+ )
 
  elif typ == "sday":
  # supports legacy uris with yyyy-mm-dd, before migration to sday
  if default is not None:
- default = datetime.strptime(f"2000{default}", "%Y%m%d").date()
+ default = _strptime(f"2000{default}", "%Y%m%d").date()
  if minval is not None:
- minval = datetime.strptime(f"2000{minval}", "%Y%m%d").date()
+ minval = _strptime(f"2000{minval}", "%Y%m%d").date()
  if maxval is not None:
- maxval = datetime.strptime(f"2000{maxval}", "%Y%m%d").date()
+ maxval = _strptime(f"2000{maxval}", "%Y%m%d").date()
  if value is not None:
  if value.find("-") > -1:
- value = datetime.strptime(value, "%Y-%m-%d").date()
+ value = _strptime(value, "%Y-%m-%d").date()
  else:
- value = datetime.strptime(f"2000{value}", "%Y%m%d").date()
+ value = _strptime(f"2000{value}", "%Y%m%d").date()
 
  elif typ == "date":
  # tricky here, php has YYYY/mm/dd and CGI has YYYY-mm-dd
  if default is not None:
- default = datetime.strptime(default, "%Y/%m/%d").date()
+ default = _strptime(default, "%Y/%m/%d").date()
  if minval is not None:
- minval = datetime.strptime(minval, "%Y/%m/%d").date()
+ minval = _strptime(minval, "%Y/%m/%d").date()
  if maxval is not None:
- maxval = datetime.strptime(maxval, "%Y/%m/%d").date()
+ maxval = _strptime(maxval, "%Y/%m/%d").date()
  if value is not None:
- try:
- value = datetime.strptime(value, "%Y-%m-%d").date()
- except ValueError as exp:
- if kwargs.get("rectify_dates", False):
- value = handle_date_err(exp, value, "%Y-%m-%d").date()
- else:
- # If we are not rectifying dates, we just raise the
- # exception
- raise
+ value = _strptime(
+ value,
+ "%Y-%m-%d",
+ rectify=kwargs.get("rectify_dates", False),
+ ).date()
  elif typ == "vtec_ps":
  # VTEC phenomena and significance
  defaults = {}

src/pyiem/webutil.py

@@ -1,13 +1,13 @@
 """Utility functions for iemwebfarm applications."""
 
-import datetime
 import random
 import re
 import string
 import sys
 import traceback
 import warnings
 from collections import namedtuple
+from datetime import datetime
 from http import HTTPStatus
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
@@ -237,7 +237,7 @@ def compute_ts_from_string(form, key):
  fmt += ".%f"
  if len(tstr.split(":")) == 2:
  fmt = "%Y-%m-%d %H:%M"
- return datetime.datetime.strptime(tstr, fmt).replace(tzinfo=tz)
+ return datetime.strptime(tstr, fmt).replace(tzinfo=tz)
 
 
 def compute_ts(form, suffix):
@@ -260,7 +260,7 @@ def compute_ts(form, suffix):
  else:
  day = 28
 
- return datetime.datetime(
+ return datetime(
  int(yearval),
  month,
  day,
@@ -447,7 +447,7 @@ def _handle_exp(errormsg, routine=False, code=500):
  )
  return [msg.encode("ascii")]
 
- start_time = datetime.datetime.utcnow()
+ start_time = datetime.utcnow()
  status_code = 500
  try:
  # mixed convers this to a regular dict
@@ -489,7 +489,7 @@ def _handle_exp(errormsg, routine=False, code=500):
  )
  except Exception:
  res = _handle_exp(traceback.format_exc())
- end_time = datetime.datetime.utcnow()
+ end_time = datetime.utcnow()
  if kwargs.get("enable_telemetry", True):
  write_telemetry(
  TELEMETRY(

tests/grid/test_zs.py

@@ -3,9 +3,10 @@
 import numpy as np
 from affine import Affine
 from geopandas import GeoSeries
-from pyiem.grid import zs
 from shapely.geometry import Polygon
 
+from pyiem.grid import zs
+
 
 def test_polygon_to_left():
  """Test that we don't get an artifact in the first column."""

tests/models/test_models_shef.py

@@ -2,6 +2,7 @@
 # pylint: disable=redefined-outer-name
 
 import pytest
+
 from pyiem.models.shef import SHEFElement
 from pyiem.util import utc
 

tests/ncei/test_ds3505.py

@@ -2,6 +2,7 @@
 # pylint: disable=redefined-outer-name
 
 import numpy as np
+
 from pyiem.ncei import ds3505
 from pyiem.util import get_test_filepath, utc
 

tests/nws/products/test_cf6.py

@@ -3,6 +3,7 @@
 import datetime
 
 import pytest
+
 from pyiem.nws.products.cf6 import parser
 from pyiem.reference import TRACE_VALUE
 from pyiem.util import get_test_file, utc

tests/nws/products/test_cli.py

@@ -3,6 +3,7 @@
 import datetime
 
 import pytest
+
 from pyiem.nws.products import cli
 from pyiem.nws.products import parser as cliparser
 from pyiem.nws.products.cli import CLIException, get_number

tests/nws/products/test_cwa.py

@@ -2,11 +2,11 @@
 
 # third party
 import pytest
+from shapely.geometry import Polygon
 
 # this
 from pyiem.nws.products.cwa import parser
 from pyiem.util import get_test_file, utc
-from shapely.geometry import Polygon
 
 LOCS = {
  "AMG": {"lon": -82.51, "lat": 31.54},

tests/nws/products/test_dsm.py

@@ -4,6 +4,7 @@
 from zoneinfo import ZoneInfo
 
 import pytest
+
 from pyiem.nws.products.dsm import compute_time, parser, process
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_fd.py

@@ -1,6 +1,7 @@
 """Test FD."""
 
 import pytest
+
 from pyiem.nws.products.fd import parse_encoding, parser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_ffg.py

@@ -1,6 +1,7 @@
 """Testing FFG parsing."""
 
 import pytest
+
 from pyiem.nws.products.ffg import parser as ffgparser
 from pyiem.util import get_test_file
 

tests/nws/products/test_hml.py

@@ -3,6 +3,7 @@
 import datetime
 
 import pytest
+
 from pyiem.nws.products.hml import parser as hmlparser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_lsr.py

@@ -1,6 +1,7 @@
 """Test Local Storm Report parsing."""
 
 import pytest
+
 from pyiem.nws.products.lsr import parse_lsr, parser
 from pyiem.reference import TRACE_VALUE
 from pyiem.util import get_test_file

tests/nws/products/test_mcd.py

@@ -1,6 +1,7 @@
 """MCD/MPD tests."""
 
 import pytest
+
 from pyiem.exceptions import MCDException
 from pyiem.nws.products import parser
 from pyiem.util import get_test_file, utc

tests/nws/products/test_metarcollect.py

@@ -3,6 +3,7 @@
 from unittest import mock
 
 import pytest
+
 from pyiem.nws.products import metarcollect
 from pyiem.reference import TRACE_VALUE
 from pyiem.util import get_test_file, utc

tests/nws/products/test_mos.py

@@ -1,6 +1,7 @@
 """Test MOS Parsing."""
 
 import pytest
+
 from pyiem.nws.products.mos import parser as mosparser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_nldn.py

@@ -1,6 +1,7 @@
 """Test NLDN."""
 
 import pytest
+
 from pyiem.nws.products.nldn import parser
 from pyiem.util import get_test_filepath
 

tests/nws/products/test_pirep.py

@@ -1,6 +1,7 @@
 """PIREP."""
 
 import pytest
+
 from pyiem.nws.products.pirep import parser as pirepparser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_products_vtec.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 import pytest
+
 from pyiem.nws.nwsli import NWSLI
 from pyiem.nws.products.vtec import check_dup_ps
 from pyiem.nws.products.vtec import parser as _vtecparser

tests/nws/products/test_saw.py

@@ -1,6 +1,7 @@
 """Can we process the SAW"""
 
 import pytest
+
 from pyiem.nws.products import parser
 from pyiem.nws.products.saw import parser as sawparser
 from pyiem.util import get_test_file, utc

tests/nws/products/test_scp.py

@@ -1,6 +1,7 @@
 """Can we process the SCP"""
 
 import pytest
+
 from pyiem.nws.products.scp import parser
 from pyiem.util import get_test_file, utc
 

tests/nws/products/test_shef.py

@@ -6,6 +6,7 @@
 
 import mock
 import pytest
+
 from pyiem.exceptions import InvalidSHEFEncoding, InvalidSHEFValue
 from pyiem.nws.products.shef import (
  make_date,

tests/nws/products/test_spcpts.py

@@ -1,6 +1,7 @@
 """Unit Tests"""
 
 import pytest
+
 from pyiem.nws.products import parser
 from pyiem.nws.products._outlook_util import debug_draw
 from pyiem.nws.products.spcpts import (

tests/nws/products/test_sps.py

@@ -1,6 +1,7 @@
 """SPS Parsing"""
 
 import pytest
+
 from pyiem.nws.products import parser as spsparser
 from pyiem.nws.ugc import UGC
 from pyiem.reference import TWEET_CHARS

tests/nws/products/test_taf.py

@@ -2,6 +2,7 @@
 
 # Third Party
 import pytest
+
 from pyiem.nws.products import parser as tafparser
 from pyiem.nws.products.taf import parser as real_tafparser
 from pyiem.reference import TAF_VIS_OVER_6SM

tests/nws/test_bufkit.py

@@ -5,6 +5,7 @@
 
 # third party
 import pytest
+
 from pyiem.nws.bufkit import read_bufkit
 
 # Local

tests/nws/test_lsrclass.py

@@ -1,6 +1,7 @@
 """test lsr."""
 
 import pytest
+
 from pyiem.nws.lsr import _icestorm_remark as ir
 from pyiem.nws.products import lsr
 from pyiem.nws.products.lsr import _mylowercase

tests/nws/test_product.py

@@ -1,6 +1,7 @@
 """Testing!"""
 
 import pytest
+
 from pyiem.nws import product, ugc
 from pyiem.nws.product import (
  TextProduct,