Fix ionosphereic date parsing and grouping (#432)

* move iono file parsing to separate function

* Update src/dolphin/workflows/_utils.py

src/dolphin/workflows/_utils.py

@@ -1,8 +1,15 @@
 from __future__ import annotations
 
 import contextlib
+import datetime
 import logging
+from collections import defaultdict
 from pathlib import Path
+from typing import Mapping, Sequence
+
+from opera_utils import group_by_date
+
+from dolphin._types import Filename
 
 from .config import DisplacementWorkflow
 
@@ -30,3 +37,38 @@ def _create_burst_cfg(
 def _remove_dir_if_empty(d: Path) -> None:
  with contextlib.suppress(OSError):
  d.rmdir()
+
+
+def parse_ionosphere_files(
+ ionosphere_files: Sequence[Filename],
+ iono_date_fmts: Sequence[str] = ["%j0.%y", "%Y%j0000"],
+) -> Mapping[tuple[datetime.datetime], list[Path]]:
+ """Parse ionosphere files and group them by date.
+
+ Parameters
+ ----------
+ ionosphere_files : Sequence[Union[str, Path]]
+ List of ionosphere file paths.
+ iono_date_fmts: Sequence[str]
+ Format of dates within ionosphere file names to search for.
+ Default is ["%j0.%y", "%Y%j0000"], which matches the old name
+ 'jplg2970.16i', and the new name format
+ 'JPL0OPSFIN_20232540000_01D_02H_GIM.INX' (respectively)
+
+
+ Returns
+ -------
+ grouped_iono_files : Mapping[Tuple[datetime], List[Path]]
+ Dictionary mapping dates to lists of files.
+
+ """
+ grouped_iono_files: Mapping[tuple[datetime.datetime], list[Path]] = defaultdict(
+ list
+ )
+ for fmt in iono_date_fmts:
+ group_iono = group_by_date(ionosphere_files, file_date_fmt=fmt)
+ if () in group_iono: # files which didn't match the date format
+ group_iono.pop(())
+ grouped_iono_files = {**grouped_iono_files, **group_iono}
+
+ return grouped_iono_files

src/dolphin/workflows/displacement.py

@@ -7,10 +7,8 @@
 import multiprocessing as mp
 from collections import defaultdict
 from concurrent.futures import ProcessPoolExecutor
-from datetime import datetime
-from os import PathLike
 from pathlib import Path
-from typing import Mapping, NamedTuple, Sequence
+from typing import NamedTuple
 
 from opera_utils import group_by_burst, group_by_date # , get_dates
 from tqdm.auto import tqdm
@@ -21,7 +19,7 @@
 from dolphin.workflows import CallFunc
 
 from . import stitching_bursts, unwrapping, wrapped_phase
-from ._utils import _create_burst_cfg, _remove_dir_if_empty
+from ._utils import _create_burst_cfg, _remove_dir_if_empty, parse_ionosphere_files
 from .config import DisplacementWorkflow # , TimeseriesOptions
 
 logger = logging.getLogger(__name__)
@@ -91,16 +89,9 @@ def run(
  else:
  grouped_amp_mean_files = defaultdict(list)
 
- grouped_iono_files: Mapping[tuple[datetime], Sequence[str | PathLike[str]]] = {}
- if len(cfg.correction_options.ionosphere_files) > 0:
- for fmt in cfg.correction_options._iono_date_fmt:
- group_iono = group_by_date(
- cfg.correction_options.ionosphere_files,
- file_date_fmt=fmt,
- )
- if len(next(iter(group_iono))) == 0:
- continue
- grouped_iono_files = {**grouped_iono_files, **group_iono}
+ grouped_iono_files = parse_ionosphere_files(
+ cfg.correction_options.ionosphere_files, cfg.correction_options._iono_date_fmt
+ )
 
  # ######################################
  # 1. Burst-wise Wrapped phase estimation

tests/test_workflows_utils.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from pathlib import Path
 
 import numpy as np
@@ -6,6 +7,7 @@
 
 from dolphin import stack
 from dolphin.io import _readers
+from dolphin.workflows._utils import parse_ionosphere_files
 from dolphin.workflows.single import setup_output_folder
 
 
@@ -82,3 +84,50 @@ def test_setup_output_folder_strided(tmpdir, tiled_file_list, strides):
  assert ds.RasterXSize == cols // strides["x"]
  assert ds.RasterYSize == rows // strides["y"]
  ds = None
+
+
+def test_parse_ionosphere_files():
+ ionosphere_files = [
+ Path("JPL0OPSFIN_20232780000_01D_02H_GIM.INX"),
+ Path("jplg1100.23i"),
+ Path("jplg1820.23i"),
+ Path("JPL0OPSFIN_20232660000_01D_02H_GIM.INX"),
+ Path("JPL0OPSFIN_20232300000_01D_02H_GIM.INX"),
+ Path("jplg2970.16i"),
+ Path("JPL0OPSFIN_20232420000_01D_02H_GIM.INX"),
+ Path("JPL0OPSFIN_20232540000_01D_02H_GIM.INX"),
+ ]
+
+ expected_output = {
+ (datetime(2023, 4, 20),): [Path("jplg1100.23i")],
+ (datetime(2023, 7, 1),): [Path("jplg1820.23i")],
+ (datetime(2016, 10, 23),): [Path("jplg2970.16i")],
+ (datetime(2023, 10, 5, 0, 0, 0),): [
+ Path("JPL0OPSFIN_20232780000_01D_02H_GIM.INX")
+ ],
+ (datetime(2023, 9, 23, 0, 0, 0),): [
+ Path("JPL0OPSFIN_20232660000_01D_02H_GIM.INX")
+ ],
+ (datetime(2023, 8, 18, 0, 0, 0),): [
+ Path("JPL0OPSFIN_20232300000_01D_02H_GIM.INX")
+ ],
+ (datetime(2023, 8, 30, 0, 0, 0),): [
+ Path("JPL0OPSFIN_20232420000_01D_02H_GIM.INX")
+ ],
+ (datetime(2023, 9, 11, 0, 0, 0),): [
+ Path("JPL0OPSFIN_20232540000_01D_02H_GIM.INX")
+ ],
+ }
+
+ grouped_iono_files = parse_ionosphere_files(ionosphere_files)
+ assert len(grouped_iono_files) == len(
+ expected_output
+ ), "Number of grouped dates does not match expected output."
+
+ for date_tuple, files in expected_output.items():
+ assert (
+ date_tuple in grouped_iono_files
+ ), f"Date {date_tuple} not found in grouped ionosphere files."
+ assert (
+ grouped_iono_files[date_tuple] == files
+ ), f"Files for date {date_tuple} do not match expected files."