dsets = data_catalog_subset.to_dataset_dict(storage_options={'anon':True}) dsets

[yuxuefeifei-cf]

distributed.protocol.core - CRITICAL - Failed to deserialize
Traceback (most recent call last):
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\protocol\core.py", line 111, in loads
return msgpack.loads(
File "msgpack/_unpacker.pyx", line 194, in msgpack._cmsgpack.unpackb
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\protocol\core.py", line 103, in _decode_default
return merge_and_deserialize(
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\protocol\serialize.py", line 488, in merge_and_deserialize
return deserialize(header, merged_frames, deserializers=deserializers)
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\protocol\serialize.py", line 401, in deserialize
deserialize(
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\protocol\serialize.py", line 417, in deserialize
return loads(header, frames)
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\protocol\serialize.py", line 96, in pickle_loads
return pickle.loads(x, buffers=new)
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\protocol\pickle.py", line 66, in loads
return pickle.loads(x)
File "D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\pandas\core\indexes\base.py", line 255, in _new_Index
return cls.new(cls, **d)
TypeError: CFTimeIndex.new() got an unexpected keyword argument 'dtype'

[kmpaul]

@yuxuefeifei-cf: A little bit more context here would be appreciated. What were you trying to do? Where were you trying to do it? How did you set up your environment?

[kmpaul]

@andersy005: Perhaps you know what is going on here?

[andersy005]

@yuxuefeifei-cf, I believe this is a bug that originates from intake-esm. However, I don't know exactly which part of intake-esm causes this. Fortunately, the issue seems to be addressed when using the main branch of intake/intake-esm repo. Do you mind installing the development version of intake-esm in your environment and see if the issue goes away?

python -m pip install git+https://github.com/intake/intake-esm

[yuxuefeifei-cf]

http://localhost:8888/files/tutorial/intake-esm-01.ipynb?_xsrf=2%7C45848e59%7Cc6ac9d8660aa5907f60f39b92e80e4be%7C1645542065
Thanks for suggestion. I reinstall the development version of intake-esm in my environment, another error occured: Failed to load dataset with key='atm.CTRL.monthly', You can use cat['atm.CTRL.monthly'].df to inspect the assets/files for this key.

[andersy005]

Do you mind posting the entire error message you're running into? Or does your screenshot show the entire error message (it appears to be cropped)?

[yuxuefeifei-cf]

dsets = data_catalog_subset.to_dataset_dict(storage_options={'anon':True})
dsets

`NoCredentialsError                        Traceback (most recent call last)
D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\source.py in _open_dataset(self)
    225 
--> 226             datasets = dask.compute(*datasets)
    227             if len(datasets) == 1:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\dask\base.py in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    572 
--> 573     results = schedule(dsk, keys, **kwargs)
    574     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
   3009             try:
-> 3010                 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
   3011             finally:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\client.py in gather(self, futures, errors, direct, asynchronous)
   2161                 local_worker = None
-> 2162             return self.sync(
   2163                 self._gather,

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\utils.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    310         else:
--> 311             return sync(
    312                 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    377         typ, exc, tb = error
--> 378         raise exc.with_traceback(tb)
    379     else:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\utils.py in f()
    350             future = asyncio.ensure_future(future)
--> 351             result = yield future
    352         except Exception:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\tornado\gen.py in run(self)
    761                     try:
--> 762                         value = future.result()
    763                     except Exception:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\distributed\client.py in _gather(self, futures, errors, direct, local_worker)
   2024                         else:
-> 2025                             raise exception.with_traceback(traceback)
   2026                         raise exc

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\dask\utils.py in apply()
     38     if kwargs:
---> 39         return func(*args, **kwargs)
     40     else:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\source.py in _open_dataset()
     64     else:
---> 65         ds = xr.open_dataset(url, **xarray_open_kwargs)
     66         if preprocess is not None:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\xarray\backends\api.py in open_dataset()
    494     overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 495     backend_ds = backend.open_dataset(
    496         filename_or_obj,

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\xarray\backends\zarr.py in open_dataset()
    799         filename_or_obj = _normalize_path(filename_or_obj)
--> 800         store = ZarrStore.open_group(
    801             filename_or_obj,

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\xarray\backends\zarr.py in open_group()
    347             try:
--> 348                 zarr_group = zarr.open_consolidated(store, **open_kwargs)
    349             except KeyError:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\zarr\convenience.py in open_consolidated()
   1171     # normalize parameters
-> 1172     store = normalize_store_arg(store, storage_options=kwargs.get("storage_options", None))
   1173     if mode not in {'r', 'r+'}:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\zarr\creation.py in normalize_store_arg()
    153         if "://" in store or "::" in store:
--> 154             return FSStore(store, mode=mode, **(storage_options or {}))
    155         elif storage_options:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\zarr\storage.py in __init__()
   1068 
-> 1069         if self.fs.exists(self.path) and not self.fs.isdir(self.path):
   1070             raise FSPathExistNotDir(url)

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\fsspec\asyn.py in wrapper()
     90         self = obj or args[0]
---> 91         return sync(self.loop, func, *args, **kwargs)
     92 

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\fsspec\asyn.py in sync()
     70     elif isinstance(return_result, BaseException):
---> 71         raise return_result
     72     else:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\fsspec\asyn.py in _runner()
     24     try:
---> 25         result[0] = await coro
     26     except Exception as ex:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\s3fs\core.py in _exists()
    855             try:
--> 856                 await self._info(path, bucket, key, version_id=version_id)
    857                 return True

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\s3fs\core.py in _info()
   1064             try:
-> 1065                 out = await self._call_s3(
   1066                     "head_object",

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\s3fs\core.py in _call_s3()
    301         err = translate_boto_error(err)
--> 302         raise err
    303 

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\s3fs\core.py in _call_s3()
    281             try:
--> 282                 out = await method(**additional_kwargs)
    283                 return out

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\aiobotocore\client.py in _make_api_call()
    210         else:
--> 211             http, parsed_response = await self._make_request(
    212                 operation_model, request_dict, request_context)

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\aiobotocore\client.py in _make_request()
    230         try:
--> 231             return await self._endpoint.make_request(operation_model, request_dict)
    232         except Exception as e:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\aiobotocore\endpoint.py in _send_request()
     76         attempts = 1
---> 77         request = await self.create_request(request_dict, operation_model)
     78         context = request_dict['context']

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\aiobotocore\endpoint.py in create_request()
     69                 op_name=operation_model.name)
---> 70             await self._event_emitter.emit(event_name, request=request,
     71                                            operation_name=operation_model.name)

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\aiobotocore\hooks.py in _emit()
     26             if asyncio.iscoroutinefunction(handler):
---> 27                 response = await handler(**kwargs)
     28             else:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\aiobotocore\signers.py in handler()
     15         # Don't call this method directly.
---> 16         return await self.sign(operation_name, request)
     17 

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\aiobotocore\signers.py in sign()
     62 
---> 63             auth.add_auth(request)
     64 

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\botocore\auth.py in add_auth()
    377         if self.credentials is None:
--> 378             raise NoCredentialsError()
    379         datetime_now = datetime.datetime.utcnow()

NoCredentialsError: Unable to locate credentials

The above exception was the direct cause of the following exception:

ESMDataSourceError                        Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_9652/2228577492.py in <module>
----> 1 dsets = data_catalog_subset.to_dataset_dict(storage_options={'anon':True})
      2 dsets

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\pydantic\decorator.cp310-win_amd64.pyd in pydantic.decorator.validate_arguments.validate.wrapper_function()

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\pydantic\decorator.cp310-win_amd64.pyd in pydantic.decorator.ValidatedFunction.call()

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\pydantic\decorator.cp310-win_amd64.pyd in pydantic.decorator.ValidatedFunction.execute()

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\core.py in to_dataset_dict(self, xarray_open_kwargs, xarray_combine_by_coords_kwargs, preprocess, storage_options, progressbar, aggregate, skip_on_error, **kwargs)
    599                 except Exception as exc:
    600                     if not skip_on_error:
--> 601                         raise exc
    602         self.datasets = self._create_derived_variables(datasets, skip_on_error)
    603         return self.datasets

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\core.py in to_dataset_dict(self, xarray_open_kwargs, xarray_combine_by_coords_kwargs, preprocess, storage_options, progressbar, aggregate, skip_on_error, **kwargs)
    595             for task in gen:
    596                 try:
--> 597                     key, ds = task.result()
    598                     datasets[key] = ds
    599                 except Exception as exc:

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\concurrent\futures\_base.py in result(self, timeout)
    436                     raise CancelledError()
    437                 elif self._state == FINISHED:
--> 438                     return self.__get_result()
    439 
    440                 self._condition.wait(timeout)

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\concurrent\futures\_base.py in __get_result(self)
    388         if self._exception:
    389             try:
--> 390                 raise self._exception
    391             finally:
    392                 # Break a reference cycle with the exception in self._exception

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\concurrent\futures\thread.py in run(self)
     50 
     51         try:
---> 52             result = self.fn(*self.args, **self.kwargs)
     53         except BaseException as exc:
     54             self.future.set_exception(exc)

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\core.py in _load_source(key, source)
    723 
    724 def _load_source(key, source):
--> 725     return key, source.to_dask()

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\source.py in to_dask(self)
    257     def to_dask(self):
    258         """Return xarray object (which will have chunks)"""
--> 259         self._load_metadata()
    260         return self._ds
    261 

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake\source\base.py in _load_metadata(self)
    234         """load metadata only if needed"""
    235         if self._schema is None:
--> 236             self._schema = self._get_schema()
    237             self.dtype = self._schema.dtype
    238             self.shape = self._schema.shape

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\source.py in _get_schema(self)
    189 
    190         if self._ds is None:
--> 191             self._open_dataset()
    192             metadata = {'dims': {}, 'data_vars': {}, 'coords': ()}
    193             self._schema = Schema(

D:\Miniconda\envs\pythia-intake-esm-tutorial\lib\site-packages\intake_esm\source.py in _open_dataset(self)
    249 
    250         except Exception as exc:
--> 251             raise ESMDataSourceError(
    252                 f"""Failed to load dataset with key='{self.key}'
    253                  You can use `cat['{self.key}'].df` to inspect the assets/files for this key.

ESMDataSourceError: Failed to load dataset with key='atm.CTRL.monthly'
                 You can use `cat['atm.CTRL.monthly'].df` to inspect the assets/files for this key.`