I think it may have something to do with the set up of the Intake catalog itself, since there's not much going on in that call. The call to .to_dask() is where the error originates.
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[4], line 2
1 cat = catalog['NWM_Best_CONUS_Short_Range']
----> 2 ds_sr = cat.to_dask()
3 ds_sr
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/intake_xarray/base.py:8](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/intake_xarray/base.py#line=7), in IntakeXarraySourceAdapter.to_dask(self)
6 def to_dask(self):
7 if "chunks" not in self.reader.kwargs:
----> 8 return self.reader(chunks={}).read()
9 else:
10 return self.reader.read()
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/intake/readers/readers.py:121](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/intake/readers/readers.py#line=120), in BaseReader.read(self, *args, **kwargs)
119 kw.update(kwargs)
120 args = kw.pop("args", ()) or args
--> 121 return self._read(*args, **kw)
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/intake/readers/readers.py:1327](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/intake/readers/readers.py#line=1326), in XArrayDatasetReader._read(self, data, open_local, **kw)
1325 f = fsspec.open(data.url, **(data.storage_options or {})).open()
1326 return open_dataset(f, **kw)
-> 1327 return open_dataset(data.url, **kw)
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/api.py:687](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/api.py#line=686), in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
675 decoders = _resolve_decoders_kwargs(
676 decode_cf,
677 open_backend_dataset_parameters=backend.open_dataset_parameters,
(...) 683 decode_coords=decode_coords,
684 )
686 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 687 backend_ds = backend.open_dataset(
688 filename_or_obj,
689 drop_variables=drop_variables,
690 **decoders,
691 **kwargs,
692 )
693 ds = _dataset_from_backend_dataset(
694 backend_ds,
695 filename_or_obj,
(...) 705 **kwargs,
706 )
707 return ds
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/zarr.py:1584](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/zarr.py#line=1583), in ZarrBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask, cache_members)
1582 filename_or_obj = _normalize_path(filename_or_obj)
1583 if not store:
-> 1584 store = ZarrStore.open_group(
1585 filename_or_obj,
1586 group=group,
1587 mode=mode,
1588 synchronizer=synchronizer,
1589 consolidated=consolidated,
1590 consolidate_on_close=False,
1591 chunk_store=chunk_store,
1592 storage_options=storage_options,
1593 zarr_version=zarr_version,
1594 use_zarr_fill_value_as_mask=None,
1595 zarr_format=zarr_format,
1596 cache_members=cache_members,
1597 )
1599 store_entrypoint = StoreBackendEntrypoint()
1600 with close_on_error(store):
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/zarr.py:723](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/zarr.py#line=722), in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks, zarr_version, zarr_format, use_zarr_fill_value_as_mask, write_empty, cache_members)
698 @classmethod
699 def open_group(
700 cls,
(...) 716 cache_members: bool = True,
717 ):
718 (
719 zarr_group,
720 consolidate_on_close,
721 close_store_on_close,
722 use_zarr_fill_value_as_mask,
--> 723 ) = _get_open_params(
724 store=store,
725 mode=mode,
726 synchronizer=synchronizer,
727 group=group,
728 consolidated=consolidated,
729 consolidate_on_close=consolidate_on_close,
730 chunk_store=chunk_store,
731 storage_options=storage_options,
732 zarr_version=zarr_version,
733 use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask,
734 zarr_format=zarr_format,
735 )
737 return cls(
738 zarr_group,
739 mode,
(...) 747 cache_members,
748 )
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/zarr.py:1821](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/xarray/backends/zarr.py#line=1820), in _get_open_params(store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, zarr_version, use_zarr_fill_value_as_mask, zarr_format)
1817 if _zarr_v3():
1818 # we have determined that we don't want to use consolidated metadata
1819 # so we set that to False to avoid trying to read it
1820 open_kwargs["use_consolidated"] = False
-> 1821 zarr_group = zarr.open_group(store, **open_kwargs)
1823 close_store_on_close = zarr_group.store is not store
1825 # we use this to determine how to handle fill_value
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/_compat.py:43](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/_compat.py#line=42), in _deprecate_positional_args.<locals>._inner_deprecate_positional_args.<locals>.inner_f(*args, **kwargs)
41 extra_args = len(args) - len(all_args)
42 if extra_args <= 0:
---> 43 return f(*args, **kwargs)
45 # extra_args > 0
46 args_msg = [
47 f"{name}={arg}"
48 for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:], strict=False)
49 ]
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/api/synchronous.py:527](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/api/synchronous.py#line=526), in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version, zarr_format, meta_array, attributes, use_consolidated)
449 @_deprecate_positional_args
450 def open_group(
451 store: StoreLike | None = None,
(...) 463 use_consolidated: bool | str | None = None,
464 ) -> Group:
465 """Open a group using file-mode-like semantics.
466
467 Parameters
(...) 524 The new group.
525 """
526 return Group(
--> 527 sync(
528 async_api.open_group(
529 store=store,
530 mode=mode,
531 cache_attrs=cache_attrs,
532 synchronizer=synchronizer,
533 path=path,
534 chunk_store=chunk_store,
535 storage_options=storage_options,
536 zarr_version=zarr_version,
537 zarr_format=zarr_format,
538 meta_array=meta_array,
539 attributes=attributes,
540 use_consolidated=use_consolidated,
541 )
542 )
543 )
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/core/sync.py:163](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/core/sync.py#line=162), in sync(coro, loop, timeout)
160 return_result = next(iter(finished)).result()
162 if isinstance(return_result, BaseException):
--> 163 raise return_result
164 else:
165 return return_result
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/core/sync.py:119](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/core/sync.py#line=118), in _runner(coro)
114 """
115 Await a coroutine and return the result of running it. If awaiting the coroutine raises an
116 exception, the exception will be returned.
117 """
118 try:
--> 119 return await coro
120 except Exception as ex:
121 return ex
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/api/asynchronous.py:806](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/api/asynchronous.py#line=805), in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version, zarr_format, meta_array, attributes, use_consolidated)
803 if chunk_store is not None:
804 warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)
--> 806 store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)
808 if attributes is None:
809 attributes = {}
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/storage/_common.py:305](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/storage/_common.py#line=304), in make_store_path(store_like, path, mode, storage_options)
303 if _is_fsspec_uri(store_like):
304 used_storage_options = True
--> 305 store = FsspecStore.from_url(
306 store_like, storage_options=storage_options, read_only=_read_only
307 )
308 else:
309 store = await LocalStore.open(root=Path(store_like), read_only=_read_only)
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/storage/_fsspec.py:176](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/zarr/storage/_fsspec.py#line=175), in FsspecStore.from_url(cls, url, storage_options, read_only, allowed_exceptions)
173 opts = storage_options or {}
174 opts = {"asynchronous": True, **opts}
--> 176 fs, path = url_to_fs(url, **opts)
177 if not fs.async_impl:
178 try:
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/core.py:415](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/core.py#line=414), in url_to_fs(url, **kwargs)
413 inkwargs["fo"] = urls
414 urlpath, protocol, _ = chain[0]
--> 415 fs = filesystem(protocol, **inkwargs)
416 return fs, urlpath
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/registry.py:310](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/registry.py#line=309), in filesystem(protocol, **storage_options)
303 warnings.warn(
304 "The 'arrow_hdfs' protocol has been deprecated and will be "
305 "removed in the future. Specify it as 'hdfs'.",
306 DeprecationWarning,
307 )
309 cls = get_filesystem_class(protocol)
--> 310 return cls(**storage_options)
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/spec.py:81](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/spec.py#line=80), in _Cached.__call__(cls, *args, **kwargs)
79 return cls._cache[token]
80 else:
---> 81 obj = super().__call__(*args, **kwargs)
82 # Setting _fs_token here causes some static linters to complain.
83 obj._fs_token_ = token
File [/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/implementations/reference.py:770](https://nebari.esipfed.org/home/conda/global/5b9cbfd6-1742496550-169-pangeo-dev/lib/python3.12/site-packages/fsspec/implementations/reference.py#line=769), in ReferenceFileSystem.__init__(self, fo, target, ref_storage_args, target_protocol, target_options, remote_protocol, remote_options, fs, template_overrides, simple_templates, max_gap, max_block, cache_size, **kwargs)
768 self.fss[k] = AsyncFileSystemWrapper(f, asynchronous=self.asynchronous)
769 elif self.asynchronous ^ f.asynchronous:
--> 770 raise ValueError(
771 "Reference-FS's target filesystem must have same value"
772 "of asynchronous"
773 )
ValueError: Reference-FS's target filesystem must have same valueof asynchronous
@cheryldmorse Today I tried to run the above notebook, and encountered an error when running cell 4:
I think it may have something to do with the set up of the Intake catalog itself, since there's not much going on in that call. The call to
.to_dask()is where the error originates.