diff --git a/src/dolphin/workflows/_utils.py b/src/dolphin/workflows/_utils.py index 4fa3eb57..a2ab9b33 100644 --- a/src/dolphin/workflows/_utils.py +++ b/src/dolphin/workflows/_utils.py @@ -1,8 +1,15 @@ from __future__ import annotations import contextlib +import datetime import logging +from collections import defaultdict from pathlib import Path +from typing import Mapping, Sequence + +from opera_utils import group_by_date + +from dolphin._types import Filename from .config import DisplacementWorkflow @@ -30,3 +37,38 @@ def _create_burst_cfg( def _remove_dir_if_empty(d: Path) -> None: with contextlib.suppress(OSError): d.rmdir() + + +def parse_ionosphere_files( + ionosphere_files: Sequence[Filename], + iono_date_fmts: Sequence[str] = ["%j0.%y", "%Y%j0000"], +) -> Mapping[tuple[datetime.datetime], list[Path]]: + """Parse ionosphere files and group them by date. + + Parameters + ---------- + ionosphere_files : Sequence[Union[str, Path]] + List of ionosphere file paths. + iono_date_fmts: Sequence[str] + Format of dates within ionosphere file names to search for. + Default is ["%j0.%y", "%Y%j0000"], which matches the old name + 'jplg2970.16i', and the new name format + 'JPL0OPSFIN_20232540000_01D_02H_GIM.INX' (respectively) + + + Returns + ------- + grouped_iono_files : Mapping[Tuple[datetime], List[Path]] + Dictionary mapping dates to lists of files. + + """ + grouped_iono_files: Mapping[tuple[datetime.datetime], list[Path]] = defaultdict( + list + ) + for fmt in iono_date_fmts: + group_iono = group_by_date(ionosphere_files, file_date_fmt=fmt) + if () in group_iono: # files which didn't match the date format + group_iono.pop(()) + grouped_iono_files = {**grouped_iono_files, **group_iono} + + return grouped_iono_files diff --git a/src/dolphin/workflows/displacement.py b/src/dolphin/workflows/displacement.py index baed680c..56c663f6 100755 --- a/src/dolphin/workflows/displacement.py +++ b/src/dolphin/workflows/displacement.py @@ -7,10 +7,8 @@ import multiprocessing as mp from collections import defaultdict from concurrent.futures import ProcessPoolExecutor -from datetime import datetime -from os import PathLike from pathlib import Path -from typing import Mapping, NamedTuple, Sequence +from typing import NamedTuple from opera_utils import group_by_burst, group_by_date # , get_dates from tqdm.auto import tqdm @@ -21,7 +19,7 @@ from dolphin.workflows import CallFunc from . import stitching_bursts, unwrapping, wrapped_phase -from ._utils import _create_burst_cfg, _remove_dir_if_empty +from ._utils import _create_burst_cfg, _remove_dir_if_empty, parse_ionosphere_files from .config import DisplacementWorkflow # , TimeseriesOptions logger = logging.getLogger(__name__) @@ -91,16 +89,9 @@ def run( else: grouped_amp_mean_files = defaultdict(list) - grouped_iono_files: Mapping[tuple[datetime], Sequence[str | PathLike[str]]] = {} - if len(cfg.correction_options.ionosphere_files) > 0: - for fmt in cfg.correction_options._iono_date_fmt: - group_iono = group_by_date( - cfg.correction_options.ionosphere_files, - file_date_fmt=fmt, - ) - if len(next(iter(group_iono))) == 0: - continue - grouped_iono_files = {**grouped_iono_files, **group_iono} + grouped_iono_files = parse_ionosphere_files( + cfg.correction_options.ionosphere_files, cfg.correction_options._iono_date_fmt + ) # ###################################### # 1. Burst-wise Wrapped phase estimation diff --git a/tests/test_workflows_utils.py b/tests/test_workflows_utils.py index 04b229d7..3fe132ff 100644 --- a/tests/test_workflows_utils.py +++ b/tests/test_workflows_utils.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path import numpy as np @@ -6,6 +7,7 @@ from dolphin import stack from dolphin.io import _readers +from dolphin.workflows._utils import parse_ionosphere_files from dolphin.workflows.single import setup_output_folder @@ -82,3 +84,50 @@ def test_setup_output_folder_strided(tmpdir, tiled_file_list, strides): assert ds.RasterXSize == cols // strides["x"] assert ds.RasterYSize == rows // strides["y"] ds = None + + +def test_parse_ionosphere_files(): + ionosphere_files = [ + Path("JPL0OPSFIN_20232780000_01D_02H_GIM.INX"), + Path("jplg1100.23i"), + Path("jplg1820.23i"), + Path("JPL0OPSFIN_20232660000_01D_02H_GIM.INX"), + Path("JPL0OPSFIN_20232300000_01D_02H_GIM.INX"), + Path("jplg2970.16i"), + Path("JPL0OPSFIN_20232420000_01D_02H_GIM.INX"), + Path("JPL0OPSFIN_20232540000_01D_02H_GIM.INX"), + ] + + expected_output = { + (datetime(2023, 4, 20),): [Path("jplg1100.23i")], + (datetime(2023, 7, 1),): [Path("jplg1820.23i")], + (datetime(2016, 10, 23),): [Path("jplg2970.16i")], + (datetime(2023, 10, 5, 0, 0, 0),): [ + Path("JPL0OPSFIN_20232780000_01D_02H_GIM.INX") + ], + (datetime(2023, 9, 23, 0, 0, 0),): [ + Path("JPL0OPSFIN_20232660000_01D_02H_GIM.INX") + ], + (datetime(2023, 8, 18, 0, 0, 0),): [ + Path("JPL0OPSFIN_20232300000_01D_02H_GIM.INX") + ], + (datetime(2023, 8, 30, 0, 0, 0),): [ + Path("JPL0OPSFIN_20232420000_01D_02H_GIM.INX") + ], + (datetime(2023, 9, 11, 0, 0, 0),): [ + Path("JPL0OPSFIN_20232540000_01D_02H_GIM.INX") + ], + } + + grouped_iono_files = parse_ionosphere_files(ionosphere_files) + assert len(grouped_iono_files) == len( + expected_output + ), "Number of grouped dates does not match expected output." + + for date_tuple, files in expected_output.items(): + assert ( + date_tuple in grouped_iono_files + ), f"Date {date_tuple} not found in grouped ionosphere files." + assert ( + grouped_iono_files[date_tuple] == files + ), f"Files for date {date_tuple} do not match expected files."