diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a21d5f16..93a54b15a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changed - Remove unused pystac.validation import ([#1583](https://github.com/stac-utils/pystac/pull/1583)) +- Updated storage extension to v2.0.0 ### Fixed diff --git a/docs/api/extensions.rst b/docs/api/extensions.rst index ce37d3215..19d17c84a 100644 --- a/docs/api/extensions.rst +++ b/docs/api/extensions.rst @@ -30,7 +30,8 @@ pystac.extensions sar.SarExtension sat.SatExtension scientific.ScientificExtension - storage.StorageExtension + storage.StorageSchemesExtension + storage.StorageRefsExtension table.TableExtension timestamps.TimestampsExtension version.VersionExtension diff --git a/pystac/extensions/ext.py b/pystac/extensions/ext.py index 84f60c39d..0b72e9feb 100644 --- a/pystac/extensions/ext.py +++ b/pystac/extensions/ext.py @@ -172,6 +172,10 @@ def render(self) -> dict[str, Render]: def sci(self) -> ScientificExtension[Collection]: return ScientificExtension.ext(self.stac_object) + @property + def storage(self) -> StorageExtension[Collection]: + return StorageExtension.ext(self.stac_object) + @property def table(self) -> TableExtension[Collection]: return TableExtension.ext(self.stac_object) @@ -432,6 +436,10 @@ class ItemAssetExt(_AssetExt[ItemAssetDefinition]): def mlm(self) -> MLMExtension[ItemAssetDefinition]: return MLMExtension.ext(self.stac_object) + @property + def storage(self) -> StorageExtension[ItemAssetDefinition]: + return StorageExtension.ext(self.stac_object) + @dataclass class LinkExt(_AssetsExt[Link]): @@ -444,3 +452,7 @@ class LinkExt(_AssetsExt[Link]): @property def file(self) -> FileExtension[Link]: return FileExtension.ext(self.stac_object) + + @property + def storage(self) -> StorageExtension[Link]: + return StorageExtension.ext(self.stac_object) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 4270a9dc3..f8064d23b 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -5,7 +5,6 @@ from __future__ import annotations -from collections.abc import Iterable from typing import ( Any, Generic, @@ -15,148 +14,307 @@ ) import pystac +from pystac.errors import RequiredPropertyMissing from pystac.extensions.base import ( ExtensionManagementMixin, PropertiesExtension, SummariesExtension, ) from pystac.extensions.hooks import ExtensionHooks -from pystac.utils import StringEnum - -#: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Asset` or -#: :class:`~pystac.ItemAssetDefinition` -T = TypeVar("T", pystac.Item, pystac.Asset, pystac.ItemAssetDefinition) +from pystac.utils import StringEnum, get_required, map_opt + +T = TypeVar( + "T", + pystac.Catalog, + pystac.Collection, + pystac.Item, + pystac.Asset, + pystac.Link, + pystac.ItemAssetDefinition, +) -SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v1.0.0/schema.json" +SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v2.0.0/schema.json" PREFIX: str = "storage:" # Field names -PLATFORM_PROP: str = PREFIX + "platform" -REGION_PROP: str = PREFIX + "region" -REQUESTER_PAYS_PROP: str = PREFIX + "requester_pays" -TIER_PROP: str = PREFIX + "tier" - +REFS_PROP: str = PREFIX + "refs" +SCHEMES_PROP: str = PREFIX + "schemes" -class CloudPlatform(StringEnum): - ALIBABA = "ALIBABA" - AWS = "AWS" - AZURE = "AZURE" - GCP = "GCP" - IBM = "IBM" - ORACLE = "ORACLE" - OTHER = "OTHER" +# Storage scheme object names +TYPE_PROP: str = "type" +PLATFORM_PROP: str = "platform" +REGION_PROP: str = "region" +REQUESTER_PAYS_PROP: str = "requester_pays" -class StorageExtension( - Generic[T], - PropertiesExtension, - ExtensionManagementMixin[pystac.Item | pystac.Collection], -): - """An abstract class that can be used to extend the properties of an - :class:`~pystac.Item` or :class:`~pystac.Asset` with properties from the - :stac-ext:`Storage Extension `. This class is generic over the type of - STAC Object to be extended (e.g. :class:`~pystac.Item`, - :class:`~pystac.Asset`). +class StorageSchemeType(StringEnum): + AWS_S3 = "aws-s3" + CUSTOM_S3 = "custom-s3" + AZURE = "ms-azure" - To create a concrete instance of :class:`StorageExtension`, use the - :meth:`StorageExtension.ext` method. For example: - .. code-block:: python +class StorageScheme: + """ + Helper class for storage scheme objects. - >>> item: pystac.Item = ... - >>> storage_ext = StorageExtension.ext(item) + Can set well-defined properties, or if needed, + any arbitrary property. """ - name: Literal["storage"] = "storage" + _known_fields = {"type", "platform", "region", "requester_pays"} + _properties: dict[str, Any] + + def __init__(self, properties: dict[str, Any]): + super().__setattr__("_properties", properties) + + def __setattr__(self, name: str, value: Any) -> None: + if hasattr(type(self), name): + object.__setattr__(self, name, value) + return + + if name in self._known_fields: + prop = getattr(type(self), name) + prop.fset(self, value) + return + + props = object.__getattribute__(self, "_properties") + props[name] = value + + def __getattr__(self, name: str) -> Any: + props = object.__getattribute__(self, "_properties") + + if name in props: + return props[name] + + raise AttributeError(name) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, StorageScheme): + return NotImplemented + return bool(self._properties == other._properties) + + def __repr__(self) -> str: + return f"" def apply( self, - platform: CloudPlatform | None = None, + type: str, + platform: str, region: str | None = None, requester_pays: bool | None = None, - tier: str | None = None, + **kwargs: Any, ) -> None: - """Applies Storage Extension properties to the extended :class:`~pystac.Item` or - :class:`~pystac.Asset`. - - Args: - platform (str, CloudPlatform) : The cloud provider where data is stored. - region (str) : The region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider). - requester_pays (bool) : Is the data requester pays or is it data - manager/cloud provider pays. - tier (str) : The title for the tier type (as defined by PaaS provider). - """ + self.type = type self.platform = platform self.region = region self.requester_pays = requester_pays - self.tier = tier + self._properties.update(kwargs) - @property - def platform(self) -> CloudPlatform | None: - """Get or sets the cloud provider where data is stored. + @classmethod + def create( + cls, + type: str, + platform: str, + region: str | None = None, + requester_pays: bool | None = None, + **kwargs: Any, + ) -> StorageScheme: + """Set the properties for a new StorageScheme object. + + Additional properties can be set through kwargs to fulfill + any additional variables in a templated uri. + + Args: + type (str): Type identifier for the platform. + platform (str): The cloud provider where data is stored as URI or URI + template to the API. + region (str | None): The region where the data is stored. + Defaults to None. + requester_pays (bool | None): requester pays or data manager/cloud + provider pays. Defaults to None. + kwargs (dict[str | Any]): Additional properties to set on scheme Returns: - str or None + StorageScheme: storage scheme """ - return self._get_property(PLATFORM_PROP, CloudPlatform) + c = cls({}) + c.apply( + type=type, + platform=platform, + region=region, + requester_pays=requester_pays, + **kwargs, + ) + return c + + @property + def type(self) -> str: + """ + Get or set the required type property + """ + return cast( + str, + get_required( + self._properties.get(TYPE_PROP), + self, + TYPE_PROP, + ), + ) + + @type.setter + def type(self, v: str) -> None: + self._properties[TYPE_PROP] = v + + @property + def platform(self) -> str: + """ + Get or set the required platform property + """ + return cast( + str, + get_required( + self._properties.get(PLATFORM_PROP), + self, + PLATFORM_PROP, + ), + ) @platform.setter - def platform(self, v: CloudPlatform | None) -> None: - self._set_property(PLATFORM_PROP, v) + def platform(self, v: str) -> None: + self._properties[PLATFORM_PROP] = v @property def region(self) -> str | None: - """Gets or sets the region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider).""" - return self._get_property(REGION_PROP, str) + """ + Get or set the optional region property + """ + return self._properties.get(REGION_PROP) @region.setter def region(self, v: str | None) -> None: - self._set_property(REGION_PROP, v) + if v is not None: + self._properties[REGION_PROP] = v + else: + self._properties.pop(REGION_PROP, None) @property def requester_pays(self) -> bool | None: - # This value "defaults to false", according to the extension spec. - return self._get_property(REQUESTER_PAYS_PROP, bool) + """ + Get or set the optional requester_pays property + """ + return self._properties.get(REQUESTER_PAYS_PROP) @requester_pays.setter def requester_pays(self, v: bool | None) -> None: - self._set_property(REQUESTER_PAYS_PROP, v) + if v is not None: + self._properties[REQUESTER_PAYS_PROP] = v + else: + self._properties.pop(REQUESTER_PAYS_PROP, None) - @property - def tier(self) -> str | None: - return self._get_property(TIER_PROP, str) + def to_dict(self) -> dict[str, Any]: + """ + Returns the dictionary encoding of this object + + Returns: + dict[str, Any + """ + return self._properties - @tier.setter - def tier(self, v: str | None) -> None: - self._set_property(TIER_PROP, v) + +class StorageExtension( + Generic[T], + PropertiesExtension, + ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], +): + """An class that can be used to extend the properties of an + :class:`~pystac.Catalog`, :class:`~pystac.Collection`, :class:`~pystac.Item`, + :class:`~pystac.Asset`, :class:`~pystac.Link`, or + :class:`~pystac.ItemAssetDefinition` with properties from the + :stac-ext:`Storage Extension `. + This class is generic over the type of STAC Object to be extended (e.g. + :class:`~pystac.Item`, :class:`~pystac.Collection`). + To create a concrete instance of :class:`StorageExtension`, use the + :meth:`StorageExtension.ext` method. For example: + + .. code-block:: python + + >>> item: pystac.Item = ... + >>> storage_ext = StorageExtension.ext(item) + """ + + name: Literal["storage"] = "storage" @classmethod def get_schema_uri(cls) -> str: return SCHEMA_URI + # For type checking purposes only, these methods are overridden in mixins + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + raise NotImplementedError() + + @property + def schemes(self) -> dict[str, StorageScheme]: + raise NotImplementedError() + + @schemes.setter + def schemes(self, v: dict[str, StorageScheme]) -> None: + raise NotImplementedError() + + def add_scheme(self, key: str, scheme: StorageScheme) -> None: + raise NotImplementedError() + + @property + def refs(self) -> list[str]: + raise NotImplementedError() + + @refs.setter + def refs(self, v: list[str]) -> None: + raise NotImplementedError() + + def add_ref(self, ref: str) -> None: + raise NotImplementedError() + @classmethod def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]: """Extends the given STAC Object with properties from the :stac-ext:`Storage Extension `. - This extension can be applied to instances of :class:`~pystac.Item` or - :class:`~pystac.Asset`. + This extension can be applied to instances of :class:`~pystac.Catalog`, + :class:`~pystac.Collection`, :class:`~pystac.Item`, :class:`~pystac.Asset`, + :class:`~pystac.Link`, or :class:`~pystac.ItemAssetDefinition`. Raises: - pystac.ExtensionTypeError : If an invalid object type is passed. """ if isinstance(obj, pystac.Item): cls.ensure_has_extension(obj, add_if_missing) return cast(StorageExtension[T], ItemStorageExtension(obj)) + + elif isinstance(obj, pystac.Collection): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], CollectionStorageExtension(obj)) + + elif isinstance(obj, pystac.Catalog): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], CatalogStorageExtension(obj)) + elif isinstance(obj, pystac.Asset): cls.ensure_owner_has_extension(obj, add_if_missing) return cast(StorageExtension[T], AssetStorageExtension(obj)) + + elif isinstance(obj, pystac.Link): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], LinkStorageExtension(obj)) + elif isinstance(obj, pystac.ItemAssetDefinition): cls.ensure_owner_has_extension(obj, add_if_missing) return cast(StorageExtension[T], ItemAssetsStorageExtension(obj)) + else: raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) @@ -169,21 +327,85 @@ def summaries( return SummariesStorageExtension(obj) -class ItemStorageExtension(StorageExtension[pystac.Item]): - """A concrete implementation of :class:`StorageExtension` on an - :class:`~pystac.Item` that extends the properties of the Item to include - properties defined in the :stac-ext:`Storage Extension `. +class _SchemesMixin: + """Mixin for objects that support Storage Schemes (Items, Collections, Catalogs).""" - This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Item` to extend it. - """ + properties: dict[str, Any] + _set_property: Any + + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + if refs is not None: + raise ValueError("'refs' cannot be applied with this STAC object type.") + if schemes is None: + raise RequiredPropertyMissing( + self, + SCHEMES_PROP, + "'schemes' property is required for this object type.", + ) + self.schemes = schemes + + @property + def schemes(self) -> dict[str, StorageScheme]: + schemes_dict: dict[str, Any] = get_required( + self.properties.get(SCHEMES_PROP), self, SCHEMES_PROP + ) + return {k: StorageScheme(v) for k, v in schemes_dict.items()} + + @schemes.setter + def schemes(self, v: dict[str, StorageScheme]) -> None: + v_trans = {k: c.to_dict() for k, c in v.items()} + self._set_property(SCHEMES_PROP, v_trans) + + def add_scheme(self, key: str, scheme: StorageScheme) -> None: + current = self.properties.get(SCHEMES_PROP, {}) + current[key] = scheme.to_dict() + self._set_property(SCHEMES_PROP, current) - item: pystac.Item - """The :class:`~pystac.Item` being extended.""" + +class _RefsMixin: + """Mixin for objects that support Storage Refs (Assets, Links).""" properties: dict[str, Any] - """The :class:`~pystac.Item` properties, including extension properties.""" + _set_property: Any + + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + if schemes is not None: + raise ValueError("'schemes' cannot be applied with this STAC object type.") + if refs is None: + raise RequiredPropertyMissing( + self, REFS_PROP, "'refs' property is required for this object type." + ) + self.refs = refs + @property + def refs(self) -> list[str]: + return get_required(self.properties.get(REFS_PROP), self, REFS_PROP) + + @refs.setter + def refs(self, v: list[str]) -> None: + self._set_property(REFS_PROP, v) + + def add_ref(self, ref: str) -> None: + try: + current = self.refs + if ref not in current: + current.append(ref) + self.refs = current + except RequiredPropertyMissing: + self.refs = [ref] + + +class ItemStorageExtension(_SchemesMixin, StorageExtension[pystac.Item]): def __init__(self, item: pystac.Item): self.item = item self.properties = item.properties @@ -192,99 +414,162 @@ def __repr__(self) -> str: return f"" -class AssetStorageExtension(StorageExtension[pystac.Asset]): +class CatalogStorageExtension(_SchemesMixin, StorageExtension[pystac.Catalog]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Catalog` that extends the properties of the Catalog to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Catalog` to extend it. + """ + + catalog: pystac.Catalog + """The :class:`~pystac.Catalog` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Catalog` properties, including extension properties.""" + + def __init__(self, catalog: pystac.Catalog): + self.catalog = catalog + self.properties = catalog.extra_fields + + def __repr__(self) -> str: + return f"" + + +class CollectionStorageExtension(_SchemesMixin, StorageExtension[pystac.Collection]): """A concrete implementation of :class:`StorageExtension` on an - :class:`~pystac.Asset` that extends the Asset fields to include properties defined - in the :stac-ext:`Storage Extension `. + :class:`~pystac.Collection` that extends the properties of the Collection to include + properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it. + :meth:`StorageExtension.ext` on an :class:`~pystac.Collection` to extend it. """ - asset_href: str - """The ``href`` value of the :class:`~pystac.Asset` being extended.""" + collection: pystac.Collection + """The :class:`~pystac.Collection` being extended.""" properties: dict[str, Any] - """The :class:`~pystac.Asset` fields, including extension properties.""" + """The :class:`~pystac.Collection` properties, including extension properties.""" + + def __init__(self, collection: pystac.Collection): + self.collection = collection + self.properties = collection.extra_fields + + def __repr__(self) -> str: + return f"" - additional_read_properties: Iterable[dict[str, Any]] | None = None - """If present, this will be a list containing 1 dictionary representing the - properties of the owning :class:`~pystac.Item`.""" + +class AssetStorageExtension(_RefsMixin, StorageExtension[pystac.Asset]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Asset` that extends the properties of the Asset to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it. + """ + + asset: pystac.Asset + """The :class:`~pystac.Asset` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Asset` properties, including extension properties.""" def __init__(self, asset: pystac.Asset): - self.asset_href = asset.href + self.asset = asset self.properties = asset.extra_fields - if asset.owner and isinstance(asset.owner, pystac.Item): - self.additional_read_properties = [asset.owner.properties] def __repr__(self) -> str: - return f"" + return f"" + + +class LinkStorageExtension(_RefsMixin, StorageExtension[pystac.Link]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Link` that extends the properties of the Link to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Link` to extend it. + """ + link: pystac.Link + """The :class:`~pystac.Link` being extended.""" -class ItemAssetsStorageExtension(StorageExtension[pystac.ItemAssetDefinition]): properties: dict[str, Any] - asset_defn: pystac.ItemAssetDefinition + """The :class:`~pystac.Link` properties, including extension properties.""" - def __init__(self, item_asset: pystac.ItemAssetDefinition): - self.asset_defn = item_asset - self.properties = item_asset.properties + def __init__(self, link: pystac.Link): + self.link = link + self.properties = link.extra_fields + def __repr__(self) -> str: + return f"" -class SummariesStorageExtension(SummariesExtension): - """A concrete implementation of :class:`~pystac.extensions.base.SummariesExtension` - that extends the ``summaries`` field of a :class:`~pystac.Collection` to include - properties defined in the :stac-ext:`Storage Extension `. + +class ItemAssetsStorageExtension( + _RefsMixin, StorageExtension[pystac.ItemAssetDefinition] +): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.ItemAssetDefinition` that extends the properties of the + ItemAssetDefinition to include properties defined in the + :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.ItemAssetDefinition` + to extend it. """ - @property - def platform(self) -> list[CloudPlatform] | None: - """Get or sets the summary of :attr:`StorageExtension.platform` values - for this Collection. - """ - return self.summaries.get_list(PLATFORM_PROP) + item_asset: pystac.ItemAssetDefinition + """The :class:`~pystac.ItemAssetDefinition` being extended.""" - @platform.setter - def platform(self, v: list[CloudPlatform] | None) -> None: - self._set_summary(PLATFORM_PROP, v) + properties: dict[str, Any] + """The :class:`~pystac.ItemAssetDefinition` properties, + including extension properties.""" - @property - def region(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.region` values - for this Collection. - """ - return self.summaries.get_list(REGION_PROP) + def __init__(self, item_asset: pystac.ItemAssetDefinition): + self.item_asset = item_asset + self.properties = item_asset.properties - @region.setter - def region(self, v: list[str] | None) -> None: - self._set_summary(REGION_PROP, v) + def __repr__(self) -> str: + return f"" - @property - def requester_pays(self) -> list[bool] | None: - """Get or sets the summary of :attr:`StorageExtension.requester_pays` values - for this Collection. - """ - return self.summaries.get_list(REQUESTER_PAYS_PROP) - @requester_pays.setter - def requester_pays(self, v: list[bool] | None) -> None: - self._set_summary(REQUESTER_PAYS_PROP, v) +class SummariesStorageExtension(SummariesExtension): + """A concrete implementation of :class:`~pystac.extensions.base.SummariesExtension` + that extends the ``summaries`` field of a :class:`~pystac.Collection` to include + properties defined in the :stac-ext:`Storage Extension `. + """ @property - def tier(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.tier` values + def schemes(self) -> list[dict[str, StorageScheme]] | None: + """Get or sets the summary of :attr:`StorageScheme.platform` values for this Collection. """ - return self.summaries.get_list(TIER_PROP) - - @tier.setter - def tier(self, v: list[str] | None) -> None: - self._set_summary(TIER_PROP, v) + return map_opt( + lambda schemes: [ + {k: StorageScheme(v) for k, v in x.items()} for x in schemes + ], + self.summaries.get_list(SCHEMES_PROP), + ) + + @schemes.setter + def schemes(self, v: list[dict[str, StorageScheme]] | None) -> None: + self._set_summary( + SCHEMES_PROP, + map_opt( + lambda schemes: [ + {k: c.to_dict() for k, c in x.items()} for x in schemes + ], + v, + ), + ) class StorageExtensionHooks(ExtensionHooks): schema_uri: str = SCHEMA_URI prev_extension_ids: set[str] = set() stac_object_types = { + pystac.STACObjectType.CATALOG, pystac.STACObjectType.COLLECTION, pystac.STACObjectType.ITEM, } diff --git a/tests/data-files/storage/collection-naip.json b/tests/data-files/storage/collection-naip.json index c24f38276..c3afac3da 100644 --- a/tests/data-files/storage/collection-naip.json +++ b/tests/data-files/storage/collection-naip.json @@ -16,29 +16,50 @@ } ], "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "summaries": { - "storage:platform": [ - "AZURE", - "GCP", - "AWS" - ], - "storage:region": [ - "westus2", - "us-central1", - "us-west-2", - "eastus" - ], - "storage:requester_pays": [ - true, - false - ], - "storage:tier": [ - "archive", - "COLDLINE", - "Standard", - "hot" + "storage:schemes": [ + { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + } + }, + { + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + } + }, + { + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + } + }, + { + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + } + }, + { + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } ] }, "extent": { diff --git a/tests/data-files/storage/item-naip.json b/tests/data-files/storage/item-naip.json index 22069111d..59ebefb14 100644 --- a/tests/data-files/storage/item-naip.json +++ b/tests/data-files/storage/item-naip.json @@ -1,7 +1,7 @@ { "stac_version": "1.1.0", "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "id": "m_3009743_sw_14_1_20160928_20161129", "collection": "NAIP_MOSAIC", @@ -43,55 +43,70 @@ "datetime": "2016-09-28T00:00:00+00:00", "mission": "NAIP", "platform": "UNKNOWN_PLATFORM", - "gsd": 1 + "gsd": 1, + "storage:schemes": { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + }, + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + }, + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + }, + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + }, + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } }, "assets": { "GEOTIFF_AZURE_RGBIR": { "href": "https://naip-nsl.blob.core.windows.net/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff", - "storage:platform": "AZURE", - "storage:region": "westus2", - "storage:tier": "archive" + "storage:refs": ["naip-azure-nsl"] }, "CO_GEOTIFF_GCP_RGB": { "href": "gs://naip-data/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "GCP", - "storage:region": "us-central1", - "storage:requester_pays": true, - "storage:tier": "COLDLINE" + "storage:refs": ["naip-gcs"] }, "CO_GEOTIFF_AWS_RGB": { "href": "s3://naip-visualization/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AWS", - "storage:region": "us-west-2", - "storage:requester_pays": true, - "storage:tier": "Standard" + "storage:refs": ["naip-aws"] }, "CO_GEOTIFF_AZURE_RGB": { "href": "https://naipeuwest.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "westeurope", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-eu"] }, "CO_GEOTIFF_AZURE_RGB_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] }, "THUMBNAIL_AZURE_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.200.jpg", "type": "image/jpeg", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] } }, "links": [ diff --git a/tests/extensions/cassettes/test_storage/test_refs_apply.yaml b/tests/extensions/cassettes/test_storage/test_refs_apply.yaml new file mode 100644 index 000000000..e5afbfe61 --- /dev/null +++ b/tests/extensions/cassettes/test_storage/test_refs_apply.yaml @@ -0,0 +1,332 @@ +interactions: +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n + \ }\n ]\n },\n {\n \"$comment\": \"This is the schema + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '4259' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-10a3"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - af64a991591f50737fadd2e89371cc12c8a2bb7d + X-GitHub-Request-Id: + - 591F:3B4DB0:4904A90:4C8204C:683CAF52 + X-Served-By: + - cache-iad-kcgs7200048-IAD + X-Timer: + - S1748816209.717952,VS0,VE1 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '706' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-2c2"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - 43d2b7092055b40e12b92793975aae512e94642f + X-GitHub-Request-Id: + - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 + X-Served-By: + - cache-iad-kiad7000035-IAD + X-Timer: + - S1748816209.780679,VS0,VE3 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '353' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-161"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - c0e4c57aae853bd12f2127008255fb0d13a187fd + X-GitHub-Request-Id: + - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 + X-Served-By: + - cache-iad-kiad7000157-IAD + X-Timer: + - S1748816209.846702,VS0,VE4 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '469' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-1d5"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - ea7c52a0de24dedf14c95d765216ad65b6bfa746 + X-GitHub-Request-Id: + - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 + X-Served-By: + - cache-iad-kiad7000135-IAD + X-Timer: + - S1748816209.915699,VS0,VE2 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml index e0f739b8a..ad4cfad1d 100644 --- a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml +++ b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml @@ -95,4 +95,334 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n + \ }\n ]\n },\n {\n \"$comment\": \"This is the schema + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '4259' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-10a3"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 855a50e2b2d2411c04eaca800019a8413b70b9a8 + X-GitHub-Request-Id: + - 591F:3B4DB0:4904A90:4C8204C:683CAF52 + X-Served-By: + - cache-iad-kiad7000152-IAD + X-Timer: + - S1748816208.373228,VS0,VE14 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '706' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-2c2"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 278480db61c132221d05f66f4d67dd50f9669b9a + X-GitHub-Request-Id: + - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 + X-Served-By: + - cache-iad-kiad7000064-IAD + X-Timer: + - S1748816208.451944,VS0,VE18 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '353' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-161"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 17cf88621af9473909a54f09b7280d23a8d03f5b + X-GitHub-Request-Id: + - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 + X-Served-By: + - cache-iad-kiad7000129-IAD + X-Timer: + - S1748816209.531423,VS0,VE21 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '469' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-1d5"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - ff0d43db7da3291d7786a451b605ce5d2e96e1c4 + X-GitHub-Request-Id: + - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 + X-Served-By: + - cache-iad-kiad7000060-IAD + X-Timer: + - S1748816209.609643,VS0,VE15 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK version: 1 diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 4c1ba2322..43dd54f7e 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -1,13 +1,19 @@ import json import random +from copy import deepcopy from string import ascii_letters import pytest import pystac -from pystac import ExtensionTypeError, Item +from pystac import ExtensionTypeError, Item, ItemAssetDefinition from pystac.collection import Collection -from pystac.extensions.storage import CloudPlatform, StorageExtension +from pystac.errors import RequiredPropertyMissing +from pystac.extensions.storage import ( + StorageExtension, + StorageScheme, + StorageSchemeType, +) from tests.utils import TestCases, assert_to_from_dict NAIP_EXAMPLE_URI = TestCases.get_path("data-files/storage/item-naip.json") @@ -24,6 +30,28 @@ def naip_collection() -> Collection: return Collection.from_file(NAIP_COLLECTION_URI) +@pytest.fixture +def sample_scheme() -> StorageScheme: + return StorageScheme.create( + type=StorageSchemeType.AWS_S3, + platform="https://{bucket}.s3.{region}.amazonaws.com", + region="us-west-2", + requester_pays=True, + ) + + +@pytest.fixture +def naip_asset(naip_item: Item) -> pystac.Asset: + # Grab a random asset with the platform property + return random.choice( + [ + _asset + for _asset in naip_item.assets.values() + if "storage:refs" in _asset.to_dict() + ] + ) + + def test_to_from_dict() -> None: with open(NAIP_EXAMPLE_URI) as f: item_dict = json.load(f) @@ -41,12 +69,12 @@ def test_add_to(sample_item: Item) -> None: StorageExtension.add_to(sample_item) StorageExtension.add_to(sample_item) - eo_uris = [ + uris = [ uri for uri in sample_item.stac_extensions if uri == StorageExtension.get_schema_uri() ] - assert len(eo_uris) == 1 + assert len(uris) == 1 @pytest.mark.vcr() @@ -54,13 +82,6 @@ def test_validate_storage(naip_item: Item) -> None: naip_item.validate() -def test_extend_invalid_object() -> None: - link = pystac.Link("child", "https://some-domain.com/some/path/to.json") - - with pytest.raises(pystac.ExtensionTypeError): - StorageExtension.ext(link) # type: ignore - - def test_extension_not_implemented(sample_item: Item) -> None: # Should raise exception if Item does not include extension URI with pytest.raises(pystac.ExtensionNotImplemented): @@ -77,6 +98,15 @@ def test_extension_not_implemented(sample_item: Item) -> None: _ = StorageExtension.ext(ownerless_asset) +def test_collection_ext_add_to(naip_collection: Collection) -> None: + naip_collection.stac_extensions = [] + assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions + + _ = StorageExtension.ext(naip_collection, add_if_missing=True) + + assert StorageExtension.get_schema_uri() in naip_collection.stac_extensions + + def test_item_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions @@ -85,6 +115,16 @@ def test_item_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() in sample_item.stac_extensions +def test_catalog_ext_add_to() -> None: + catalog = pystac.Catalog("stac", "a catalog") + + assert StorageExtension.get_schema_uri() not in catalog.stac_extensions + + _ = StorageExtension.ext(catalog, add_if_missing=True) + + assert StorageExtension.get_schema_uri() in catalog.stac_extensions + + def test_asset_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.assets["thumbnail"] @@ -94,6 +134,15 @@ def test_asset_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() in sample_item.stac_extensions +def test_link_ext_add_to(sample_item: Item) -> None: + assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + asset = sample_item.links[0] + + _ = StorageExtension.ext(asset, add_if_missing=True) + + assert StorageExtension.get_schema_uri() in sample_item.stac_extensions + + def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: asset_dict = sample_item.assets["thumbnail"].to_dict() asset = pystac.Asset.from_dict(asset_dict) @@ -104,79 +153,38 @@ def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: def test_should_raise_exception_when_passing_invalid_extension_object() -> None: with pytest.raises( - ExtensionTypeError, match=r"^StorageExtension does not apply to type 'object'$" + ExtensionTypeError, + match=r"^StorageExtension does not apply to type 'object'$", ): # calling it wrong purposely so ---------v StorageExtension.ext(object()) # type: ignore -def test_summaries_platform(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.platform == col_dict["summaries"]["storage:platform"] - # Set - new_platform_summary = [random.choice([v for v in CloudPlatform])] - assert storage_summaries.platform != new_platform_summary - storage_summaries.platform = new_platform_summary - assert storage_summaries.platform == new_platform_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:platform"] == new_platform_summary - - -def test_summaries_region(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.region == col_dict["summaries"]["storage:region"] - # Set - new_region_summary = [random.choice(ascii_letters)] - assert storage_summaries.region != new_region_summary - storage_summaries.region = new_region_summary - assert storage_summaries.region == new_region_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:region"] == new_region_summary - - -def test_summaries_requester_pays(naip_collection: Collection) -> None: +def test_summaries_schemes(naip_collection: Collection) -> None: col_dict = naip_collection.to_dict() storage_summaries = StorageExtension.summaries(naip_collection) - # Get assert ( - storage_summaries.requester_pays - == col_dict["summaries"]["storage:requester_pays"] + list( + map( + lambda x: {k: c.to_dict() for k, c in x.items()}, + storage_summaries.schemes or [], + ) + ) + == col_dict["summaries"]["storage:schemes"] ) - - # Set - new_requester_pays_summary = [True] - assert storage_summaries.requester_pays != new_requester_pays_summary - storage_summaries.requester_pays = new_requester_pays_summary - assert storage_summaries.requester_pays == new_requester_pays_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:requester_pays"] == new_requester_pays_summary - - -def test_summaries_tier(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.tier == col_dict["summaries"]["storage:tier"] - # Set - new_tier_summary = [random.choice(ascii_letters)] - assert storage_summaries.tier != new_tier_summary - storage_summaries.tier = new_tier_summary - assert storage_summaries.tier == new_tier_summary + new_schemes_summary = [ + {"key": StorageScheme.create("aws-s3", "https://a.platform.example.com")} + ] + assert storage_summaries.schemes != new_schemes_summary + storage_summaries.schemes = new_schemes_summary + assert storage_summaries.schemes == new_schemes_summary col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:tier"] == new_tier_summary + assert col_dict["summaries"]["storage:schemes"] == [ + {k: c.to_dict() for k, c in x.items()} for x in new_schemes_summary + ] def test_summaries_adds_uri(naip_collection: Collection) -> None: @@ -195,142 +203,145 @@ def test_summaries_adds_uri(naip_collection: Collection) -> None: assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions -def test_item_apply(naip_item: Item) -> None: - asset = random.choice(list(naip_item.assets.values())) - - storage_ext = StorageExtension.ext(asset) - - new_platform = random.choice( - [v for v in CloudPlatform if v != storage_ext.platform] - ) +def test_schemes_apply(naip_item: Item) -> None: + storage_ext = StorageExtension.ext(naip_item) + new_key = random.choice(ascii_letters) + new_type = random.choice(ascii_letters) + new_platform = random.choice(ascii_letters) new_region = random.choice(ascii_letters) - new_requestor_pays = random.choice( - [v for v in {True, False} if v != storage_ext.requester_pays] - ) - new_tier = random.choice(ascii_letters) + new_requestor_pays = random.choice([v for v in {True, False}]) storage_ext.apply( - platform=new_platform, - region=new_region, - requester_pays=new_requestor_pays, - tier=new_tier, + schemes={ + new_key: StorageScheme.create( + new_type, new_platform, new_region, new_requestor_pays + ), + } ) - assert storage_ext.platform == new_platform - assert storage_ext.region == new_region - assert storage_ext.requester_pays == new_requestor_pays - assert storage_ext.tier == new_tier + applied_schemes = storage_ext.schemes + assert list(applied_schemes) == [new_key] + assert applied_schemes[new_key].type == new_type + assert applied_schemes[new_key].platform == new_platform + assert applied_schemes[new_key].region == new_region + assert applied_schemes[new_key].requester_pays == new_requestor_pays @pytest.mark.vcr() -def test_asset_platform(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:platform" in _asset.to_dict() - ] - ) +def test_refs_apply(naip_asset: pystac.Asset) -> None: + test_refs = ["a_ref", "b_ref"] - storage_ext = StorageExtension.ext(asset) + storage_ext = StorageExtension.ext(naip_asset) + storage_ext.apply(refs=test_refs) # Get - assert storage_ext.platform == asset.extra_fields.get("storage:platform") + assert storage_ext.refs == test_refs # Set - new_platform = random.choice( - [val for val in CloudPlatform if val != storage_ext.platform] - ) - storage_ext.platform = new_platform - assert storage_ext.platform == new_platform + new_refs = [random.choice(ascii_letters)] + storage_ext.refs = new_refs + assert storage_ext.refs == new_refs - naip_item.validate() +def test_schemes_apply_raises(naip_item: Item) -> None: + storage_ext = StorageExtension.ext(naip_item) -@pytest.mark.vcr() -def test_asset_region(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:region" in _asset.to_dict() - ] - ) + with pytest.raises( + ValueError, + match="'refs' cannot be applied with this STAC object type.", + ): + storage_ext.apply( + schemes={ + "a_key": StorageScheme.create("a_type", "a_platform"), + }, + refs=["a_ref"], + ) + with pytest.raises( + RequiredPropertyMissing, + match="'schemes' property is required for this object type.", + ): + storage_ext.apply(refs=None) - storage_ext = StorageExtension.ext(asset) - # Get - assert storage_ext.region == asset.extra_fields.get("storage:region") +def test_refs_apply_raises(naip_asset: Item) -> None: + storage_ext = StorageExtension.ext(naip_asset) - # Set - new_region = random.choice( - [val for val in CloudPlatform if val != storage_ext.region] - ) - storage_ext.region = new_region - assert storage_ext.region == new_region + with pytest.raises( + ValueError, + match="'schemes' cannot be applied with this STAC object type.", + ): + storage_ext.apply( + schemes={ + "a_key": StorageScheme.create("a_type", "a_platform"), + }, + refs=["a_ref"], + ) - naip_item.validate() + with pytest.raises( + RequiredPropertyMissing, + match="'refs' property is required for this object type.", + ): + storage_ext.apply(schemes=None) - # Set to None - storage_ext.region = None - assert "storage:region" not in asset.extra_fields +def test_add_storage_scheme(naip_item: Item) -> None: + storage_ext = naip_item.ext.storage + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert "new_scheme" in storage_ext.schemes -@pytest.mark.vcr() -def test_asset_requester_pays(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:requester_pays" in _asset.to_dict() - ] - ) + storage_ext.properties.pop("storage:schemes") + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert len(storage_ext.schemes) == 1 + assert "new_scheme" in storage_ext.schemes - storage_ext = StorageExtension.ext(asset) - # Get - assert storage_ext.requester_pays == asset.extra_fields.get( - "storage:requester_pays" - ) +def test_add_refs(naip_item: Item) -> None: + scheme_name = random.choice(ascii_letters) + asset = naip_item.assets["GEOTIFF_AZURE_RGBIR"] + storage_ext = asset.ext.storage + assert isinstance(storage_ext, StorageExtension) - # Set - new_requester_pays = True if not storage_ext.requester_pays else False - storage_ext.requester_pays = new_requester_pays - assert storage_ext.requester_pays == new_requester_pays + storage_ext.add_ref(scheme_name) + assert scheme_name in storage_ext.refs - naip_item.validate() + storage_ext.properties.pop("storage:refs") + scheme_name_2 = random.choice(ascii_letters) + storage_ext.add_ref(scheme_name_2) + assert len(storage_ext.refs) == 1 + assert scheme_name_2 in storage_ext.refs - # Set to None - storage_ext.requester_pays = None - assert "storage:requester_pays" not in asset.extra_fields +def test_storage_scheme_create(sample_scheme: StorageScheme) -> None: + assert sample_scheme.type == StorageSchemeType.AWS_S3 + assert sample_scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" + assert sample_scheme.region == "us-west-2" + assert sample_scheme.requester_pays is True -@pytest.mark.vcr() -def test_asset_tier(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:tier" in _asset.to_dict() - ] - ) + sample_scheme.type = StorageSchemeType.AZURE + sample_scheme.platform = "https://{account}.blob.core.windows.net" + sample_scheme.region = "eastus" + sample_scheme.account = "account" + sample_scheme.requester_pays = False - storage_ext = StorageExtension.ext(asset) + assert sample_scheme.type == StorageSchemeType.AZURE + assert sample_scheme.platform == "https://{account}.blob.core.windows.net" + assert sample_scheme.region == "eastus" + assert sample_scheme.account == "account" + assert sample_scheme.requester_pays is False - # Get - assert storage_ext.tier == asset.extra_fields.get("storage:tier") - # Set - new_tier = random.choice(ascii_letters) - storage_ext.tier = new_tier - assert storage_ext.tier == new_tier +def test_storage_scheme_equality(sample_scheme: StorageScheme) -> None: + other = deepcopy(sample_scheme) + assert sample_scheme == other - naip_item.validate() + other.requester_pays = False + assert sample_scheme != other - # Set to None - storage_ext.tier = None - assert "storage:tier" not in asset.extra_fields + assert sample_scheme != object() + + +def test_item_asset_accessor() -> None: + item_asset = ItemAssetDefinition.create( + title="title", description="desc", media_type="media", roles=["a_role"] + ) + assert isinstance(item_asset.ext.storage, StorageExtension)