From 5ef3ee5a4266368400cba7f6863f14c4c2e0dcd2 Mon Sep 17 00:00:00 2001 From: Lukas Klingsbo Date: Fri, 14 Aug 2026 10:17:04 +0200 Subject: [PATCH] refactor(storage)!: move the Iceberg catalog into its own package Extracts IcebergRestCatalog, its exceptions and the table and namespace types out of storage_client and into a new iceberg package, mirroring the split between storage-js and iceberg-js. storage_client depends on it and re-exports the whole surface, so the public API of storage_client, supabase and supabase_flutter is unchanged and analyticsCatalog() still returns a catalog. Only imports of the private src paths break. --- .github/workflows/label-issues.yml | 1 + .github/workflows/release-pana.yml | 1 + .github/workflows/test.yml | 4 +- AGENTS.md | 1 + MIGRATION.md | 21 ++++++- packages/iceberg/CHANGELOG.md | 3 + packages/iceberg/LICENSE | 21 +++++++ packages/iceberg/README.md | 57 +++++++++++++++++ packages/iceberg/analysis_options.yaml | 11 ++++ packages/iceberg/example/main.dart | 62 +++++++++++++++++++ packages/iceberg/lib/iceberg.dart | 11 ++++ .../lib/src}/iceberg_error.dart | 0 .../lib/src}/iceberg_rest_catalog.dart | 8 +-- .../lib/src}/iceberg_types.dart | 0 .../lib/src}/table_requirement.dart | 0 .../lib/src}/table_update.dart | 2 +- packages/iceberg/pubspec.yaml | 30 +++++++++ .../test/iceberg_test.dart | 2 +- .../lib/src/storage_client.dart | 5 +- .../lib/supabase_storage.dart | 6 +- packages/supabase_storage/pubspec.yaml | 1 + pubspec.yaml | 1 + 22 files changed, 231 insertions(+), 17 deletions(-) create mode 100644 packages/iceberg/CHANGELOG.md create mode 100644 packages/iceberg/LICENSE create mode 100644 packages/iceberg/README.md create mode 100644 packages/iceberg/analysis_options.yaml create mode 100644 packages/iceberg/example/main.dart create mode 100644 packages/iceberg/lib/iceberg.dart rename packages/{supabase_storage/lib/src/iceberg => iceberg/lib/src}/iceberg_error.dart (100%) rename packages/{supabase_storage/lib/src/iceberg => iceberg/lib/src}/iceberg_rest_catalog.dart (98%) rename packages/{supabase_storage/lib/src/iceberg => iceberg/lib/src}/iceberg_types.dart (100%) rename packages/{supabase_storage/lib/src/iceberg => iceberg/lib/src}/table_requirement.dart (100%) rename packages/{supabase_storage/lib/src/iceberg => iceberg/lib/src}/table_update.dart (98%) create mode 100644 packages/iceberg/pubspec.yaml rename packages/{supabase_storage => iceberg}/test/iceberg_test.dart (99%) diff --git a/.github/workflows/label-issues.yml b/.github/workflows/label-issues.yml index c33d8cbcd..e4b1d43b6 100644 --- a/.github/workflows/label-issues.yml +++ b/.github/workflows/label-issues.yml @@ -26,6 +26,7 @@ jobs: "storage": "storage", "storage_client": "storage", "supabase_storage": "storage", + "iceberg": "storage", "realtime": "realtime", "realtime_client": "realtime", "supabase_realtime": "realtime", diff --git a/.github/workflows/release-pana.yml b/.github/workflows/release-pana.yml index af3f2d34d..9f238ac73 100644 --- a/.github/workflows/release-pana.yml +++ b/.github/workflows/release-pana.yml @@ -25,6 +25,7 @@ jobs: fail-fast: false matrix: package: + - iceberg - postgrest - supabase - supabase_auth diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d672ed45..680a9d588 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: echo "Affected packages:" echo "$CHANGED" - DART_PACKAGES="postgrest supabase supabase_auth supabase_common supabase_functions supabase_realtime supabase_storage supabase_typegen yet_another_json_isolate" + DART_PACKAGES="iceberg postgrest supabase supabase_auth supabase_common supabase_functions supabase_realtime supabase_storage supabase_typegen yet_another_json_isolate" entries=() for package in $DART_PACKAGES; do @@ -344,7 +344,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true - carryforward: 'postgrest,supabase,supabase_auth,supabase_common,supabase_functions,supabase_realtime,supabase_storage,supabase_typegen,yet_another_json_isolate,supabase_flutter' + carryforward: 'iceberg,postgrest,supabase,supabase_auth,supabase_common,supabase_functions,supabase_realtime,supabase_storage,supabase_typegen,yet_another_json_isolate,supabase_flutter' fail-on-error: false test-result: diff --git a/AGENTS.md b/AGENTS.md index cf70255d4..830d1b9c4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ The repository follows a layered dependency structure: - **postgrest**: Database query client with ORM-style API - **supabase_realtime**: WebSocket client for real-time subscriptions - **supabase_storage**: File storage client with retry logic +- **iceberg**: Apache Iceberg REST Catalog client used by storage analytics buckets - **supabase_functions**: Edge functions invocation client - **yet_another_json_isolate**: JSON parsing in separate isolate for performance diff --git a/MIGRATION.md b/MIGRATION.md index 3623f4cf6..55e5b0e74 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -5,7 +5,7 @@ versions of the Supabase Flutter SDK, together with the steps required to migrat All packages in this repository are released together for a major version, so a single section covers `supabase_flutter`, `supabase`, `supabase_auth`, `postgrest`, `supabase_realtime`, -`supabase_storage` and `supabase_functions`. Every symbol mentioned here is re-exported from +`supabase_storage`, `iceberg` and `supabase_functions`. Every symbol mentioned here is re-exported from `supabase_flutter`, so the snippets apply whether you depend on the individual package or on the Flutter one. @@ -973,6 +973,25 @@ try { Exhaustive switches over the sealed hierarchy still compile with the same set of cases, since the new base is sealed and every concrete subtype is unchanged. +### The Iceberg catalog moved to its own package + +`IcebergRestCatalog`, the exceptions above and the table and namespace types now live in +`iceberg`, mirroring the split between `storage-js` and `iceberg-js`. `supabase_storage` +depends on it and re-exports the whole surface, so importing +`package:supabase_storage/supabase_storage.dart` or `package:supabase_flutter/supabase_flutter.dart` +keeps working unchanged, and `storage.analyticsCatalog()` is still how you get a catalog for an +analytics bucket. + +Depend on `iceberg` directly to talk to an Iceberg REST Catalog without the rest of Storage: + +```dart +final catalog = IcebergRestCatalog( + baseUrl: 'https://example.com/iceberg', + headers: {'Authorization': 'Bearer $token'}, + warehouse: 'my-warehouse', +); +``` + ### Abbreviations in the public API are spelled out Public identifiers that still used abbreviations are spelled out, continuing the precedent set by diff --git a/packages/iceberg/CHANGELOG.md b/packages/iceberg/CHANGELOG.md new file mode 100644 index 000000000..f702d9732 --- /dev/null +++ b/packages/iceberg/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.1.0 + + - Initial release. Apache Iceberg REST Catalog client extracted from storage_client. diff --git a/packages/iceberg/LICENSE b/packages/iceberg/LICENSE new file mode 100644 index 000000000..ddeba6a00 --- /dev/null +++ b/packages/iceberg/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Supabase + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/iceberg/README.md b/packages/iceberg/README.md new file mode 100644 index 000000000..27d36cc49 --- /dev/null +++ b/packages/iceberg/README.md @@ -0,0 +1,57 @@ +
+

+ + Supabase Logo + + +

iceberg

+ +

+ Dart client library for the Apache Iceberg REST Catalog. +

+ +

+ Guides +

+

+ +
+ +[![pub package](https://img.shields.io/pub/v/iceberg.svg)](https://pub.dev/packages/iceberg) +[![pub test](https://github.com/supabase/supabase-flutter/workflows/Test/badge.svg)](https://github.com/supabase/supabase-flutter/actions?query=workflow%3ATest) + +
+ +## Usage + +Against a Supabase analytics bucket, get a catalog from `StorageClient` instead +of constructing one yourself: + +```dart +final catalog = supabase.storage.analyticsCatalog('my-analytics-bucket'); +await catalog.createNamespace(['my_namespace']); +``` + +Against any other Iceberg REST Catalog, point the client at its base URL: + +```dart +final catalog = IcebergRestCatalog( + baseUrl: 'https://example.com/iceberg', + warehouse: 'my-warehouse', + headers: {'Authorization': 'Bearer $token'}, +); +``` + +## Docs + +The docs can be found on the official Supabase website. + +- [Analytics buckets guide](https://supabase.com/docs/guides/storage/analytics/introduction) + +## License + +This repo is licensed under MIT. + +## Credits + +- https://github.com/supabase/iceberg-js - ported from supabase/iceberg-js diff --git a/packages/iceberg/analysis_options.yaml b/packages/iceberg/analysis_options.yaml new file mode 100644 index 000000000..7a7505a0b --- /dev/null +++ b/packages/iceberg/analysis_options.yaml @@ -0,0 +1,11 @@ +include: package:supabase_lints/analysis_options.yaml + +analyzer: + exclude: + - build/** + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** diff --git a/packages/iceberg/example/main.dart b/packages/iceberg/example/main.dart new file mode 100644 index 000000000..213c14f6e --- /dev/null +++ b/packages/iceberg/example/main.dart @@ -0,0 +1,62 @@ +// ignore_for_file: avoid_print + +import 'package:iceberg/iceberg.dart'; + +Future main() async { + const supabaseUrl = ''; + const supabaseKey = ''; + final catalog = IcebergRestCatalog( + baseUrl: '$supabaseUrl/storage/v1/iceberg', + headers: {'Authorization': 'Bearer $supabaseKey'}, + warehouse: 'my-analytics-bucket', + ); + + // Create a namespace to hold the tables + await catalog.createNamespaceIfNotExists(['analytics']); + + final namespaces = await catalog.listNamespaces(); + print('namespaces : ${namespaces.namespaces}'); + + // Create a table in that namespace + final metadata = await catalog.createTable( + ['analytics'], + const CreateTableRequest( + name: 'events', + schema: TableSchema( + fields: [ + TableField( + id: 1, + name: 'id', + type: PrimitiveType('long'), + required: true, + ), + TableField( + id: 2, + name: 'name', + type: PrimitiveType('string'), + required: false, + ), + ], + ), + ), + ); + print('table location : ${metadata.location}'); + + const identifier = TableIdentifier( + namespace: ['analytics'], + name: 'events', + ); + + // Load the table back + try { + final loaded = await catalog.loadTable(identifier); + print('metadata location : ${loaded.metadataLocation}'); + } on IcebergNotFoundException { + print('the table does not exist'); + } on IcebergNetworkException catch (error) { + print('no response was received : ${error.details}'); + } + + await catalog.dropTable(identifier); + await catalog.dropNamespace(['analytics']); +} diff --git a/packages/iceberg/lib/iceberg.dart b/packages/iceberg/lib/iceberg.dart new file mode 100644 index 000000000..72d0ba891 --- /dev/null +++ b/packages/iceberg/lib/iceberg.dart @@ -0,0 +1,11 @@ +/// Dart client library for the Apache Iceberg REST Catalog. +library; + +export 'package:supabase_common/supabase_common.dart' + show SupabaseApiException, SupabaseException; + +export 'src/iceberg_error.dart'; +export 'src/iceberg_rest_catalog.dart'; +export 'src/iceberg_types.dart'; +export 'src/table_requirement.dart'; +export 'src/table_update.dart'; diff --git a/packages/supabase_storage/lib/src/iceberg/iceberg_error.dart b/packages/iceberg/lib/src/iceberg_error.dart similarity index 100% rename from packages/supabase_storage/lib/src/iceberg/iceberg_error.dart rename to packages/iceberg/lib/src/iceberg_error.dart diff --git a/packages/supabase_storage/lib/src/iceberg/iceberg_rest_catalog.dart b/packages/iceberg/lib/src/iceberg_rest_catalog.dart similarity index 98% rename from packages/supabase_storage/lib/src/iceberg/iceberg_rest_catalog.dart rename to packages/iceberg/lib/src/iceberg_rest_catalog.dart index 95f276f29..11ac0a312 100644 --- a/packages/supabase_storage/lib/src/iceberg/iceberg_rest_catalog.dart +++ b/packages/iceberg/lib/src/iceberg_rest_catalog.dart @@ -2,11 +2,11 @@ import 'dart:convert'; import 'dart:math'; import 'package:http/http.dart' as http; +import 'package:iceberg/src/iceberg_error.dart'; +import 'package:iceberg/src/iceberg_types.dart'; +import 'package:iceberg/src/table_requirement.dart'; +import 'package:iceberg/src/table_update.dart'; import 'package:logging/logging.dart'; -import 'package:supabase_storage/src/iceberg/iceberg_error.dart'; -import 'package:supabase_storage/src/iceberg/iceberg_types.dart'; -import 'package:supabase_storage/src/iceberg/table_requirement.dart'; -import 'package:supabase_storage/src/iceberg/table_update.dart'; import 'package:supabase_common/supabase_common.dart'; class _IcebergResponse { diff --git a/packages/supabase_storage/lib/src/iceberg/iceberg_types.dart b/packages/iceberg/lib/src/iceberg_types.dart similarity index 100% rename from packages/supabase_storage/lib/src/iceberg/iceberg_types.dart rename to packages/iceberg/lib/src/iceberg_types.dart diff --git a/packages/supabase_storage/lib/src/iceberg/table_requirement.dart b/packages/iceberg/lib/src/table_requirement.dart similarity index 100% rename from packages/supabase_storage/lib/src/iceberg/table_requirement.dart rename to packages/iceberg/lib/src/table_requirement.dart diff --git a/packages/supabase_storage/lib/src/iceberg/table_update.dart b/packages/iceberg/lib/src/table_update.dart similarity index 98% rename from packages/supabase_storage/lib/src/iceberg/table_update.dart rename to packages/iceberg/lib/src/table_update.dart index c9480fc73..225478988 100644 --- a/packages/supabase_storage/lib/src/iceberg/table_update.dart +++ b/packages/iceberg/lib/src/table_update.dart @@ -1,4 +1,4 @@ -import 'package:supabase_storage/src/iceberg/iceberg_types.dart'; +import 'package:iceberg/src/iceberg_types.dart'; /// A single change applied to a table as part of a commit. The common data /// definition updates are modelled as dedicated subclasses. Updates that are diff --git a/packages/iceberg/pubspec.yaml b/packages/iceberg/pubspec.yaml new file mode 100644 index 000000000..e7310166e --- /dev/null +++ b/packages/iceberg/pubspec.yaml @@ -0,0 +1,30 @@ +name: iceberg +description: Dart client library for the Apache Iceberg REST Catalog, used by Supabase Storage analytics buckets. +version: 0.1.0 +homepage: 'https://supabase.com' +repository: 'https://github.com/supabase/supabase-flutter/tree/main/packages/iceberg' +issue_tracker: 'https://github.com/supabase/supabase-flutter/issues' +documentation: 'https://supabase.com/docs/guides/storage/analytics/introduction' +topics: + - supabase + - iceberg + - analytics + - storage + - backend + +environment: + sdk: '>=3.9.0 <4.0.0' + +resolution: workspace + +dependencies: + http: ^1.6.0 + logging: ^1.3.0 + supabase_common: 0.1.2 + +dev_dependencies: + test: ^1.25.0 + supabase_lints: ^0.1.1 + +false_secrets: + - /test/** diff --git a/packages/supabase_storage/test/iceberg_test.dart b/packages/iceberg/test/iceberg_test.dart similarity index 99% rename from packages/supabase_storage/test/iceberg_test.dart rename to packages/iceberg/test/iceberg_test.dart index f1c5099e5..7fcba9e65 100644 --- a/packages/supabase_storage/test/iceberg_test.dart +++ b/packages/iceberg/test/iceberg_test.dart @@ -2,7 +2,7 @@ import 'dart:convert'; import 'dart:io'; import 'package:http/http.dart'; -import 'package:supabase_storage/supabase_storage.dart'; +import 'package:iceberg/iceberg.dart'; import 'package:test/test.dart'; const String supabaseUrl = 'SUPABASE_TEST_URL'; diff --git a/packages/supabase_storage/lib/src/storage_client.dart b/packages/supabase_storage/lib/src/storage_client.dart index 24889cdd1..c135815ab 100644 --- a/packages/supabase_storage/lib/src/storage_client.dart +++ b/packages/supabase_storage/lib/src/storage_client.dart @@ -1,13 +1,12 @@ +import 'package:iceberg/iceberg.dart'; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; +import 'package:supabase_common/supabase_common.dart'; import 'package:supabase_storage/src/storage_constants.dart'; -import 'package:supabase_storage/src/iceberg/iceberg_rest_catalog.dart'; -import 'package:supabase_storage/src/iceberg/iceberg_types.dart'; import 'package:supabase_storage/src/storage_bucket_api.dart'; import 'package:supabase_storage/src/storage_file_api.dart'; import 'package:supabase_storage/src/vector_client.dart'; import 'package:supabase_storage/src/version.dart'; -import 'package:supabase_common/supabase_common.dart'; class SupabaseStorageClient extends StorageBucketApi { final int _defaultRetryAttempts; diff --git a/packages/supabase_storage/lib/supabase_storage.dart b/packages/supabase_storage/lib/supabase_storage.dart index ac70e6efe..d29bfc532 100644 --- a/packages/supabase_storage/lib/supabase_storage.dart +++ b/packages/supabase_storage/lib/supabase_storage.dart @@ -1,14 +1,10 @@ /// Dart client library for Supabase Storage. library; +export 'package:iceberg/iceberg.dart'; export 'package:supabase_common/supabase_common.dart' show SupabaseApiException, SupabaseException; -export 'src/iceberg/iceberg_error.dart'; -export 'src/iceberg/iceberg_rest_catalog.dart'; -export 'src/iceberg/iceberg_types.dart'; -export 'src/iceberg/table_requirement.dart'; -export 'src/iceberg/table_update.dart'; export 'src/storage_client.dart'; export 'src/storage_file_api.dart'; export 'src/types.dart'; diff --git a/packages/supabase_storage/pubspec.yaml b/packages/supabase_storage/pubspec.yaml index 7a1f61feb..12c2a448f 100644 --- a/packages/supabase_storage/pubspec.yaml +++ b/packages/supabase_storage/pubspec.yaml @@ -19,6 +19,7 @@ resolution: workspace dependencies: http: ^1.6.0 + iceberg: 0.1.0 mime: '>=2.0.0 <3.0.0' meta: ^1.16.0 logging: ^1.3.0 diff --git a/pubspec.yaml b/pubspec.yaml index 4d8d32902..1d4cf1354 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -7,6 +7,7 @@ environment: workspace: - packages/supabase_functions + - packages/iceberg - packages/supabase_auth - packages/postgrest - packages/supabase_realtime