From 167c45de6f4b9306b9fb53db61a833ba69751dd0 Mon Sep 17 00:00:00 2001 From: "C.-W. HSU" <11950284+chiaweh2@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:41:29 -0600 Subject: [PATCH] fixing opendap test --- mom6/notebook/test_module.ipynb | 759 +++++++++++++++++++++++++++++++- tests/test_mom6_indexes.py | 2 +- tests/test_mom6_io.py | 396 +++++++++-------- 3 files changed, 955 insertions(+), 202 deletions(-) diff --git a/mom6/notebook/test_module.ipynb b/mom6/notebook/test_module.ipynb index c7c9c96..7d99102 100644 --- a/mom6/notebook/test_module.ipynb +++ b/mom6/notebook/test_module.ipynb @@ -323,16 +323,27 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "location = 'opendap'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ + "\n", + "\n", "histrun_raw_local = mom6_io.MOM6Historical(\n", " var='tob',\n", " data_relative_dir=historical_subdir,\n", " static_relative_dir=static_subdir,\n", " grid='raw',\n", - " source='local'\n", + " source=location\n", ")\n", "\n", "histrun_regrid_local = mom6_io.MOM6Historical(\n", @@ -340,7 +351,7 @@ " data_relative_dir=historical_subdir+'/regrid/',\n", " static_relative_dir=static_subdir,\n", " grid='regrid',\n", - " source='local'\n", + " source=location\n", ")\n", "\n", "try:\n", @@ -364,14 +375,738 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pytest" + ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "ds = histrun_regrid_local_nostaticdir.get_single()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 68MB\n",
+       "Dimensions:       (xh: 775, xq: 776, yh: 845, yq: 846, nv: 2)\n",
+       "Coordinates:\n",
+       "    time          datetime64[ns] 8B 1993-01-16T12:00:00\n",
+       "  * xh            (xh) float64 6kB -98.0 -97.92 -97.84 ... -36.24 -36.16 -36.08\n",
+       "  * xq            (xq) float64 6kB -98.04 -97.96 -97.88 ... -36.2 -36.12 -36.04\n",
+       "  * yh            (yh) float64 7kB 5.273 5.352 5.432 5.511 ... 51.9 51.91 51.93\n",
+       "  * yq            (yq) float64 7kB 5.233 5.312 5.392 5.472 ... 51.9 51.92 51.94\n",
+       "  * nv            (nv) float64 16B 1.0 2.0\n",
+       "Data variables: (12/30)\n",
+       "    Coriolis      (yq, xq) float32 3MB ...\n",
+       "    areacello     (yh, xh) float32 3MB ...\n",
+       "    areacello_bu  (yq, xq) float32 3MB ...\n",
+       "    areacello_cu  (yh, xq) float32 3MB ...\n",
+       "    areacello_cv  (yq, xh) float32 3MB ...\n",
+       "    deptho        (yh, xh) float32 3MB ...\n",
+       "    ...            ...\n",
+       "    wet_v         (yq, xh) float32 3MB ...\n",
+       "    average_DT    timedelta64[ns] 8B dask.array<chunksize=(), meta=np.ndarray>\n",
+       "    average_T1    datetime64[ns] 8B dask.array<chunksize=(), meta=np.ndarray>\n",
+       "    average_T2    datetime64[ns] 8B dask.array<chunksize=(), meta=np.ndarray>\n",
+       "    time_bnds     (nv) datetime64[ns] 16B dask.array<chunksize=(2,), meta=np.ndarray>\n",
+       "    tos           (yh, xh) float32 3MB dask.array<chunksize=(845, 775), meta=np.ndarray>\n",
+       "Attributes:\n",
+       "    _NCProperties:                   version=2,netcdf=4.9.0,hdf5=1.12.2\n",
+       "    NumFilesInSet:                   1\n",
+       "    title:                           NWA12_MOM6_v1.0\n",
+       "    grid_type:                       regular\n",
+       "    grid_tile:                       N/A\n",
+       "    history:                         Fri May 12 10:50:21 2023: ncks -4 -L 3 o...\n",
+       "    NCO:                             netCDF Operators version 5.0.1 (Homepage...\n",
+       "    DODS_EXTRA.Unlimited_Dimension:  time
" + ], + "text/plain": [ + " Size: 68MB\n", + "Dimensions: (xh: 775, xq: 776, yh: 845, yq: 846, nv: 2)\n", + "Coordinates:\n", + " time datetime64[ns] 8B 1993-01-16T12:00:00\n", + " * xh (xh) float64 6kB -98.0 -97.92 -97.84 ... -36.24 -36.16 -36.08\n", + " * xq (xq) float64 6kB -98.04 -97.96 -97.88 ... -36.2 -36.12 -36.04\n", + " * yh (yh) float64 7kB 5.273 5.352 5.432 5.511 ... 51.9 51.91 51.93\n", + " * yq (yq) float64 7kB 5.233 5.312 5.392 5.472 ... 51.9 51.92 51.94\n", + " * nv (nv) float64 16B 1.0 2.0\n", + "Data variables: (12/30)\n", + " Coriolis (yq, xq) float32 3MB ...\n", + " areacello (yh, xh) float32 3MB ...\n", + " areacello_bu (yq, xq) float32 3MB ...\n", + " areacello_cu (yh, xq) float32 3MB ...\n", + " areacello_cv (yq, xh) float32 3MB ...\n", + " deptho (yh, xh) float32 3MB ...\n", + " ... ...\n", + " wet_v (yq, xh) float32 3MB ...\n", + " average_DT timedelta64[ns] 8B dask.array\n", + " average_T1 datetime64[ns] 8B dask.array\n", + " average_T2 datetime64[ns] 8B dask.array\n", + " time_bnds (nv) datetime64[ns] 16B dask.array\n", + " tos (yh, xh) float32 3MB dask.array\n", + "Attributes:\n", + " _NCProperties: version=2,netcdf=4.9.0,hdf5=1.12.2\n", + " NumFilesInSet: 1\n", + " title: NWA12_MOM6_v1.0\n", + " grid_type: regular\n", + " grid_tile: N/A\n", + " history: Fri May 12 10:50:21 2023: ncks -4 -L 3 o...\n", + " NCO: netCDF Operators version 5.0.1 (Homepage...\n", + " DODS_EXTRA.Unlimited_Dimension: time" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds" + ] + }, + { + "cell_type": "code", + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -380,12 +1115,14 @@ " data_relative_dir=historical_subdir,\n", " static_relative_dir=None,\n", " grid='raw',\n", - " source='local'\n", + " source=location\n", ")\n", - "with pytest.raises(OSError):\n", - " ds = histrun_regrid_local_nostaticdir.get_single()\n", - "with pytest.raises(OSError):\n", - " ds = histrun_regrid_local_nostaticdir.get_single()\n" + "\n", + "if location == 'local':\n", + " with pytest.raises(OSError):\n", + " ds = histrun_regrid_local_nostaticdir.get_single()\n", + " with pytest.raises(OSError):\n", + " ds = histrun_regrid_local_nostaticdir.get_single()\n" ] }, { diff --git a/tests/test_mom6_indexes.py b/tests/test_mom6_indexes.py index 5f10bbb..a0f4b92 100644 --- a/tests/test_mom6_indexes.py +++ b/tests/test_mom6_indexes.py @@ -16,7 +16,7 @@ from mom6.mom6_module.mom6_io import MOM6Historical -def test_gulf_stream_index(location:str='opendap'): +def test_gulf_stream_index(location): """testing the gulf stream index calculation Parameters diff --git a/tests/test_mom6_io.py b/tests/test_mom6_io.py index b5d4765..63cb315 100644 --- a/tests/test_mom6_io.py +++ b/tests/test_mom6_io.py @@ -9,6 +9,8 @@ # TEST OPENDAP def test_OpenDapStore(): + """Test OpenDap Connection + """ opendap_raw = mom6_io.OpenDapStore('raw','historical') test_url = opendap_raw.get_catalog()[0] @@ -28,170 +30,187 @@ def test_OpenDapStore(): pytest.fail('OSError is raised OPeNDAP url not working') -# TEST FORECAST IO -def test_MOM6Forecast(): - forecast_subdir = 'hindcast' - static_subdir = 'static' - tercile_subdir = 'tercile_calculation' - - # create local raw instance (tercile foreced None) - fcast_raw_local = mom6_io.MOM6Forecast( - var='tob', - data_relative_dir=forecast_subdir, - static_relative_dir=static_subdir, - grid='raw', - source='local' - ) - - # create local regrid instance (tercile foreced None) - fcast_regrid_local = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=forecast_subdir+'/regrid/', - static_relative_dir=static_subdir, - grid='regrid', - source='local' - ) - - try: - ds = fcast_raw_local.get_single( - iyear=2006, - imonth=6) - if ds['init.year'] != 2006 or ds['init.month'] != 6 : - pytest.fail('Picked time not the same as output time') - ds = fcast_regrid_local.get_single( - iyear=2012, - imonth=9) - if ds['init.year'] != 2012 or ds['init.month'] != 9 : - pytest.fail('Picked time not the same as output time') - ds = fcast_raw_local.get_all() - ds = fcast_regrid_local.get_all() - except OSError : - pytest.fail('OSError is raised with correct function input') - - - # create local raw instance (static dir not provided expect error) - fcast_raw_local_nostaticdir = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=forecast_subdir, - static_relative_dir=None, - grid='raw', - source='local' - ) - with pytest.raises(OSError): - ds = fcast_raw_local_nostaticdir.get_single() - with pytest.raises(OSError): - ds = fcast_raw_local_nostaticdir.get_single() - - - # create local regrid instance (regrid dir location error expect error) - fcast_regrid_local_errorloc = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=forecast_subdir, - static_relative_dir=static_subdir, - grid='regrid', - source='local' - ) - - with pytest.raises(OSError): - ds = fcast_regrid_local_errorloc.get_single() - fcast_regrid_local_errorloc.get_all() - - # create local regrid instance (raw dir location error expect error) - fcast_regrid_local_errorgrid = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=forecast_subdir+'/regrid/', - static_relative_dir=static_subdir, - grid='raw', - source='local' - ) - with pytest.raises(OSError): - fcast_regrid_local_errorgrid.get_single() - fcast_regrid_local_errorgrid.get_all() - - # create local raw instance (error iyear and imonth input for method get_single expect error) - fcast_regrid_local_erroryear = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=forecast_subdir, - static_relative_dir=static_subdir, - grid='raw', - source='local' - ) - with pytest.raises(IndexError): - fcast_regrid_local_erroryear.get_single(iyear=2024,imonth=12) - fcast_regrid_local_erroryear.get_single(iyear=2024,imonth=8) - - # (first and last iyear and imonth input for method get_single expect NO error) - try: - ds = fcast_regrid_local_erroryear.get_single(iyear=1993,imonth=3) - if ds['init.year'] != 1993 or ds['init.month'] != 3 : - pytest.fail('Picked time not the same as output time') - ds = fcast_regrid_local_erroryear.get_single(iyear=2022,imonth=12) - if ds['init.year'] != 2022 or ds['init.month'] != 12 : - pytest.fail('Picked time not the same as output time') - except OSError : - pytest.fail('OSError is raised with correct function input') - - # create local raw instance (no data dir expect error) - fcast_raw_local_nodatadir = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=None, - static_relative_dir=static_subdir, - grid='raw', - source='local' - ) - - with pytest.raises(OSError): - fcast_raw_local_nodatadir.get_single() - - # create local raw/regrid instance will all argument provided correctly (expect no Error raised) - fcast_all3_raw_local = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=forecast_subdir, - static_relative_dir=static_subdir, - tercile_relative_dir=tercile_subdir, - grid='raw', - source='local' - ) - - fcast_all3_regrid_local = mom6_io.MOM6Forecast( - var='tos', - data_relative_dir=forecast_subdir+'/regrid/', - static_relative_dir=None, - tercile_relative_dir=tercile_subdir+'/regrid', - grid='regrid', - source='local' - ) - - try: - ds = fcast_all3_raw_local.get_single( - iyear=2006, - imonth=6) - if ds['init.year'] != 2006 or ds['init.month'] != 6 : - pytest.fail('Picked time not the same as output time') - ds = fcast_all3_regrid_local.get_single( - iyear=2012, - imonth=9) - if ds['init.year'] != 2012 or ds['init.month'] != 9 : - pytest.fail('Picked time not the same as output time') - except OSError : - pytest.fail('OSError is raised with correct function input') - - try: - ds = fcast_all3_raw_local.get_all() - ds = fcast_all3_regrid_local.get_all() - except OSError : - pytest.fail('OSError raised in get_all with correct function input') - - try: - ds = fcast_all3_raw_local.get_tercile() - ds = fcast_all3_regrid_local.get_tercile() - ds = fcast_all3_raw_local.get_tercile(average_type='region') - ds = fcast_all3_regrid_local.get_tercile(average_type='region') - except OSError : - pytest.fail('OSError raised in get_tercile with correct function input') - - -def test_MOM6Historical(): +# TEST FORECAST IO local +def test_MOM6Forecast(location): + """Test the forecast IO + + only available local + + Parameters + ---------- + location : str, optional + source of the data 'opendap' or 'local', by default 'opendap' + """ + if location == 'local': + forecast_subdir = 'hindcast' + static_subdir = 'static' + tercile_subdir = 'tercile_calculation' + + # create local raw instance (tercile foreced None) + fcast_raw_local = mom6_io.MOM6Forecast( + var='tob', + data_relative_dir=forecast_subdir, + static_relative_dir=static_subdir, + grid='raw', + source=location + ) + + # create local regrid instance (tercile foreced None) + fcast_regrid_local = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=forecast_subdir+'/regrid/', + static_relative_dir=static_subdir, + grid='regrid', + source=location + ) + + try: + ds = fcast_raw_local.get_single( + iyear=2006, + imonth=6) + if ds['init.year'] != 2006 or ds['init.month'] != 6 : + pytest.fail('Picked time not the same as output time') + ds = fcast_regrid_local.get_single( + iyear=2012, + imonth=9) + if ds['init.year'] != 2012 or ds['init.month'] != 9 : + pytest.fail('Picked time not the same as output time') + ds = fcast_raw_local.get_all() + ds = fcast_regrid_local.get_all() + except OSError : + pytest.fail('OSError is raised with correct function input') + + + # create local raw instance (static dir not provided expect error) + fcast_raw_local_nostaticdir = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=forecast_subdir, + static_relative_dir=None, + grid='raw', + source=location + ) + with pytest.raises(OSError): + ds = fcast_raw_local_nostaticdir.get_single() + with pytest.raises(OSError): + ds = fcast_raw_local_nostaticdir.get_single() + + + # create local regrid instance (regrid dir location error expect error) + fcast_regrid_local_errorloc = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=forecast_subdir, + static_relative_dir=static_subdir, + grid='regrid', + source=location + ) + + with pytest.raises(OSError): + ds = fcast_regrid_local_errorloc.get_single() + fcast_regrid_local_errorloc.get_all() + + # create local regrid instance (raw dir location error expect error) + fcast_regrid_local_errorgrid = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=forecast_subdir+'/regrid/', + static_relative_dir=static_subdir, + grid='raw', + source=location + ) + with pytest.raises(OSError): + fcast_regrid_local_errorgrid.get_single() + fcast_regrid_local_errorgrid.get_all() + + # create local raw instance (error iyear and imonth input for method get_single expect error) + fcast_regrid_local_erroryear = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=forecast_subdir, + static_relative_dir=static_subdir, + grid='raw', + source=location + ) + with pytest.raises(IndexError): + fcast_regrid_local_erroryear.get_single(iyear=2024,imonth=12) + fcast_regrid_local_erroryear.get_single(iyear=2024,imonth=8) + + # (first and last iyear and imonth input for method get_single expect NO error) + try: + ds = fcast_regrid_local_erroryear.get_single(iyear=1993,imonth=3) + if ds['init.year'] != 1993 or ds['init.month'] != 3 : + pytest.fail('Picked time not the same as output time') + ds = fcast_regrid_local_erroryear.get_single(iyear=2022,imonth=12) + if ds['init.year'] != 2022 or ds['init.month'] != 12 : + pytest.fail('Picked time not the same as output time') + except OSError : + pytest.fail('OSError is raised with correct function input') + + # create local raw instance (no data dir expect error) + fcast_raw_local_nodatadir = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=None, + static_relative_dir=static_subdir, + grid='raw', + source=location + ) + + with pytest.raises(OSError): + fcast_raw_local_nodatadir.get_single() + + # create local raw/regrid instance will all argument provided correctly (expect no Error raised) + fcast_all3_raw_local = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=forecast_subdir, + static_relative_dir=static_subdir, + tercile_relative_dir=tercile_subdir, + grid='raw', + source=location + ) + + fcast_all3_regrid_local = mom6_io.MOM6Forecast( + var='tos', + data_relative_dir=forecast_subdir+'/regrid/', + static_relative_dir=None, + tercile_relative_dir=tercile_subdir+'/regrid', + grid='regrid', + source=location + ) + + try: + ds = fcast_all3_raw_local.get_single( + iyear=2006, + imonth=6) + if ds['init.year'] != 2006 or ds['init.month'] != 6 : + pytest.fail('Picked time not the same as output time') + ds = fcast_all3_regrid_local.get_single( + iyear=2012, + imonth=9) + if ds['init.year'] != 2012 or ds['init.month'] != 9 : + pytest.fail('Picked time not the same as output time') + except OSError : + pytest.fail('OSError is raised with correct function input') + + try: + ds = fcast_all3_raw_local.get_all() + ds = fcast_all3_regrid_local.get_all() + except OSError : + pytest.fail('OSError raised in get_all with correct function input') + + try: + ds = fcast_all3_raw_local.get_tercile() + ds = fcast_all3_regrid_local.get_tercile() + ds = fcast_all3_raw_local.get_tercile(average_type='region') + # ds = fcast_all3_regrid_local.get_tercile(average_type='region') + except OSError : + pytest.fail('OSError raised in get_tercile with correct function input') + + +def test_MOM6Historical(location): + """Test the Historical IO + + Parameters + ---------- + location : str, optional + source of the data 'opendap' or 'local', by default 'opendap' + """ historical_subdir = 'hist_run' static_subdir = 'static' @@ -201,7 +220,7 @@ def test_MOM6Historical(): data_relative_dir=historical_subdir, static_relative_dir=static_subdir, grid='raw', - source='local' + source=location ) # create local regrid instance (tercile foreced None) @@ -210,7 +229,7 @@ def test_MOM6Historical(): data_relative_dir=historical_subdir+'/regrid/', static_relative_dir=static_subdir, grid='regrid', - source='local' + source=location ) try: @@ -236,12 +255,13 @@ def test_MOM6Historical(): data_relative_dir=historical_subdir, static_relative_dir=None, grid='raw', - source='local' + source=location ) - with pytest.raises(OSError): - ds = histrun_raw_local_nostaticdir.get_single() - with pytest.raises(OSError): - ds = histrun_raw_local_nostaticdir.get_single() + if location == 'local': + with pytest.raises(OSError): + ds = histrun_raw_local_nostaticdir.get_single() + with pytest.raises(OSError): + ds = histrun_raw_local_nostaticdir.get_single() # create local regrid instance (regrid dir location error expect error) @@ -250,12 +270,13 @@ def test_MOM6Historical(): data_relative_dir=historical_subdir, static_relative_dir=static_subdir, grid='regrid', - source='local' + source=location ) - with pytest.raises(OSError): - ds = histrun_regrid_local_errorloc.get_single() - histrun_regrid_local_errorloc.get_all() + if location == 'local': + with pytest.raises(OSError): + ds = histrun_regrid_local_errorloc.get_single() + histrun_regrid_local_errorloc.get_all() # create local regrid instance (raw dir location error expect error) histrun_regrid_local_errorgrid = mom6_io.MOM6Historical( @@ -263,11 +284,12 @@ def test_MOM6Historical(): data_relative_dir=historical_subdir+'/regrid/', static_relative_dir=static_subdir, grid='raw', - source='local' + source=location ) - with pytest.raises(OSError): - histrun_regrid_local_errorgrid.get_single() - histrun_regrid_local_errorgrid.get_all() + if location == 'local': + with pytest.raises(OSError): + histrun_regrid_local_errorgrid.get_single() + histrun_regrid_local_errorgrid.get_all() # create local raw instance (error iyear and imonth input for method get_single expect error) @@ -276,7 +298,7 @@ def test_MOM6Historical(): data_relative_dir=historical_subdir, static_relative_dir=static_subdir, grid='raw', - source='local' + source=location ) with pytest.raises(IndexError): histrun_regrid_local_erroryear.get_single(year=2024,month=12) @@ -299,14 +321,8 @@ def test_MOM6Historical(): data_relative_dir=None, static_relative_dir=static_subdir, grid='raw', - source='local' + source=location ) - - with pytest.raises(OSError): - histrun_raw_local_nodatadir.get_single() - - - - - - + if location == 'local': + with pytest.raises(OSError): + histrun_raw_local_nodatadir.get_single()