Merge pull request #963 from akrherz/no_lsrs

Add new line to all text products not ending in new line

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
  autoupdate_schedule: quarterly
 repos:
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: "v0.6.7"
+ rev: "v0.6.8"
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this library are documented in this file.
 ### API Changes
 
 - Discontinue raw SPS product text within the database.
+- Ensure raw text products into `WMOProduct` end with a line feed.
 
 ### New Features
 

data/product_examples/LSR/LSRFWD.txt

@@ -0,0 +1,16 @@
+513 
+NWUS54 KFWD 242201
+LSRFWD
+
+PRELIMINARY LOCAL STORM REPORT
+NATIONAL WEATHER SERVICE FORT WORTH TX
+401 PM CST TUE JAN 24 2012
+
+..TIME.. ...EVENT... ...CITY LOCATION... ...LAT.LON...
+..DATE.. ....MAG.... ..COUNTY LOCATION..ST.. ...SOURCE....
+ ..REMARKS..
+
+0309 PM HAIL EUREKA 32.02N 96.30W
+01/24/2012 E0.75 INCH NAVARRO TX PUBLIC 
+
+$$

src/pyiem/nws/product.py

@@ -658,14 +658,15 @@ def __init__(
  @param parse_segments should the segments be parsed as well? True
  """
  super().__init__(text, utcnow=utcnow)
+ # NB: Don't use text as it could have been munged by this point
  self.afos = None
  if ugc_provider is None:
  ugc_provider = {}
  if nwsli_provider is None:
  nwsli_provider = {}
  self.ugc_provider = ugc_provider
  self.nwsli_provider = nwsli_provider
- self.unixtext = text.replace("\r", "")
+ self.unixtext = self.text.replace("\r", "")
  self.sections = self.unixtext.split("\n\n")
  # The "truth" timestamp
  self.valid = None

src/pyiem/nws/products/dsm.py

@@ -196,8 +196,6 @@ def __init__(
  # hold our parsing results
  self.data = []
  lines = self.text.replace("\r", "").split("\n")
- if len(lines) < 4:
- raise ValueError("Impossibly small DSM Text Product?")
  if len(lines[3]) < 10:
  meat = ("".join(lines[4:])).split("=")
  else:

src/pyiem/wmo.py

@@ -22,6 +22,9 @@ class WMOProduct:
  def __init__(self, text, utcnow=None):
  """Constructor."""
  self.warnings = []
+ # For better or worse, ensure the text string ends with a newline
+ if not text.endswith("\n"):
+ text = text + "\n"
  self.text = text
  self.source = None
  self.wmo = None

tests/nws/products/test_dsm.py

@@ -53,10 +53,7 @@ def test_jan1():
 
 def test_junk():
  """Test that we gracefully handle GIGO."""
- text = "000 \nCDUS27 KISU 020200\nBLAH BLAH="
- with pytest.raises(ValueError):
- parser(text)
- text += "\nBLAH BLAH BLAH=\n"
+ text = "000 \nCDUS27 KISU 020200\nBLAH BLAH=\nBLAH BLAH BLAH=\n"
  prod = parser(text)
  assert prod.warnings
 

tests/nws/products/test_lsr.py

@@ -7,6 +7,13 @@
 from pyiem.util import get_test_file
 
 
+def test_240926_nolsrs():
+ """Test that this processes a LSR!"""
+ prod = parser(get_test_file("LSR/LSRFWD.txt"))
+ assert not prod.warnings
+ assert len(prod.lsrs) == 1
+
+
 def test_240508_summary_link():
  """Test that the summary line is in UTC!"""
  prod = parser(get_test_file("LSR/LSRPIH_summary.txt"))