diff --git a/lance_ray/compaction.py b/lance_ray/compaction.py index b5f624e1..b8ebbdb3 100644 --- a/lance_ray/compaction.py +++ b/lance_ray/compaction.py @@ -132,7 +132,9 @@ def compact_files( if describe_response.storage_options: merged_storage_options.update(describe_response.storage_options) - namespace_kwargs = get_namespace_kwargs(namespace_impl, namespace_properties, table_id) + namespace_kwargs = get_namespace_kwargs( + namespace_impl, namespace_properties, table_id + ) # Load dataset dataset = lance.LanceDataset( @@ -261,9 +263,7 @@ def compact_database( if not database: raise ValueError("'database' must be a non-empty list of path segments.") if not namespace_impl: - raise ValueError( - "'namespace_impl' is required when using compact_database." - ) + raise ValueError("'namespace_impl' is required when using compact_database.") from lance_namespace import ListTablesRequest @@ -314,8 +314,6 @@ def compact_database( results.append({"table_id": table_id, "metrics": metrics}) except Exception as e: logger.exception("Compaction failed for table %s: %s", table_id, e) - raise RuntimeError( - f"Compaction failed for table {table_id}: {e}" - ) from e + raise RuntimeError(f"Compaction failed for table {table_id}: {e}") from e return results diff --git a/lance_ray/datasink.py b/lance_ray/datasink.py index 814aa596..f2c525bb 100644 --- a/lance_ray/datasink.py +++ b/lance_ray/datasink.py @@ -145,9 +145,7 @@ def on_write_start(self, schema: Optional[pa.Schema] = None): if self.mode == "append": base_store_params_kwargs = {} if self.base_store_params: - base_store_params_kwargs = { - "base_store_params": self.base_store_params - } + base_store_params_kwargs = {"base_store_params": self.base_store_params} ds = lance.LanceDataset( self.uri, storage_options=self.storage_options, @@ -212,9 +210,7 @@ def on_write_complete( if op: base_store_params_kwargs = {} if self.base_store_params: - base_store_params_kwargs = { - "base_store_params": self.base_store_params - } + base_store_params_kwargs = {"base_store_params": self.base_store_params} lance.LanceDataset.commit( self.uri, op, diff --git a/lance_ray/datasource.py b/lance_ray/datasource.py index 323302e0..27bc2828 100644 --- a/lance_ray/datasource.py +++ b/lance_ray/datasource.py @@ -46,9 +46,7 @@ def __init__( _check_import(self, module="lance", package="pylance") self._dataset_options = dict(dataset_options or {}) - dataset_base_store_params = self._dataset_options.pop( - "base_store_params", None - ) + dataset_base_store_params = self._dataset_options.pop("base_store_params", None) if ( base_store_params is not None and dataset_base_store_params is not None diff --git a/lance_ray/index.py b/lance_ray/index.py index 287d805c..9da96cf9 100755 --- a/lance_ray/index.py +++ b/lance_ray/index.py @@ -371,7 +371,9 @@ def create_scalar_index( if describe_response.storage_options: merged_storage_options.update(describe_response.storage_options) - namespace_kwargs = get_namespace_kwargs(namespace_impl, namespace_properties, table_id) + namespace_kwargs = get_namespace_kwargs( + namespace_impl, namespace_properties, table_id + ) # Load dataset dataset = LanceDataset( @@ -441,7 +443,9 @@ def create_scalar_index( existing_indices = dataset.list_indices() existing_names = {idx["name"] for idx in existing_indices} index_exists = name in existing_names - except Exception: # pragma: no cover - list_indices() not available in older lance versions + except ( + Exception + ): # pragma: no cover - list_indices() not available in older lance versions pass if index_exists: raise ValueError( @@ -830,7 +834,9 @@ def create_index( dataset_obj = uri dataset_uri = dataset_obj.uri if not merged_storage_options: - merged_storage_options = getattr(dataset_obj, "_storage_options", None) or {} + merged_storage_options = ( + getattr(dataset_obj, "_storage_options", None) or {} + ) namespace_kwargs = {} try: @@ -850,7 +856,9 @@ def create_index( existing_indices = dataset_obj.list_indices() existing_names = {idx["name"] for idx in existing_indices} index_exists = name in existing_names - except Exception: # pragma: no cover - list_indices() not available in older lance versions + except ( + Exception + ): # pragma: no cover - list_indices() not available in older lance versions pass if index_exists: raise ValueError( @@ -1109,7 +1117,9 @@ def optimize_indices( uri, ) - namespace_kwargs = get_namespace_kwargs(namespace_impl, namespace_properties, table_id) + namespace_kwargs = get_namespace_kwargs( + namespace_impl, namespace_properties, table_id + ) dataset = LanceDataset( uri, diff --git a/lance_ray/io.py b/lance_ray/io.py index c1b33d7f..815e50e7 100644 --- a/lance_ray/io.py +++ b/lance_ray/io.py @@ -517,7 +517,9 @@ def add_columns( """ storage_options = storage_options or {} - namespace_kwargs = get_namespace_kwargs(namespace_impl, namespace_properties, table_id) + namespace_kwargs = get_namespace_kwargs( + namespace_impl, namespace_properties, table_id + ) lance_ds = LanceDataset( uri=uri, diff --git a/lance_ray/utils.py b/lance_ray/utils.py index 5698f78e..373d78c9 100644 --- a/lance_ray/utils.py +++ b/lance_ray/utils.py @@ -45,9 +45,7 @@ def materialize_initial_bases( bases = [] for base in initial_bases: if not isinstance(base, dict): - raise TypeError( - "initial_bases must be normalized before materialization" - ) + raise TypeError("initial_bases must be normalized before materialization") bases.append( DatasetBasePath( base["path"], @@ -176,7 +174,9 @@ def _create_storage_options_provider( if not hasattr(lance, "LanceNamespaceStorageOptionsProvider"): return None - return lance.LanceNamespaceStorageOptionsProvider(namespace=namespace, table_id=table_id) + return lance.LanceNamespaceStorageOptionsProvider( + namespace=namespace, table_id=table_id + ) def get_namespace_kwargs( @@ -232,7 +232,9 @@ def get_write_fragments_kwargs( return {} return {"namespace_client": namespace, "table_id": table_id} - provider = _create_storage_options_provider(namespace_impl, namespace_properties, table_id) + provider = _create_storage_options_provider( + namespace_impl, namespace_properties, table_id + ) if provider is None: return {} return {"storage_options_provider": provider} diff --git a/pyproject.toml b/pyproject.toml index 17fecc02..b6f3b93e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "ray[data]>=2.41.0", - "pylance>=6.0.0rc3", + "pylance>=7.0.0b7", "lance-namespace", "packaging", "pyarrow>=17.0.0", diff --git a/tests/test_blob.py b/tests/test_blob.py index b7a811e8..c4dcb629 100755 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -344,10 +344,7 @@ def test_stream_copy_basic_local(temp_dir): assert src.schema == dst.schema src_df = ( - ray.data.from_arrow(table) - .to_pandas() - .sort_values("id") - .reset_index(drop=True) + ray.data.from_arrow(table).to_pandas().sort_values("id").reset_index(drop=True) ) dst_df = ( lr.read_lance(str(dst_path)) @@ -367,7 +364,9 @@ def test_stream_copy_resume_local(temp_dir): # Legacy blob schema schema = pa.schema( [ - pa.field("blob", pa.large_binary(), metadata={"lance-encoding:blob": "true"}), + pa.field( + "blob", pa.large_binary(), metadata={"lance-encoding:blob": "true"} + ), pa.field("id", pa.int64()), pa.field("name", pa.string()), pa.field("val", pa.float64()), @@ -402,10 +401,7 @@ def test_stream_copy_resume_local(temp_dir): ) src_df = ( - ray.data.from_arrow(table) - .to_pandas() - .sort_values("id") - .reset_index(drop=True) + ray.data.from_arrow(table).to_pandas().sort_values("id").reset_index(drop=True) ) dst_df = ( lr.read_lance(str(dst_path)) diff --git a/tests/test_distributed_indexing.py b/tests/test_distributed_indexing.py index a900bba3..5c4c751f 100755 --- a/tests/test_distributed_indexing.py +++ b/tests/test_distributed_indexing.py @@ -465,9 +465,13 @@ def test_scalar_index_on_mixed_schema_list_indices(self, temp_dir): ) indices = updated_dataset.list_indices() - assert len(indices) >= 1, "list_indices should return at least the new scalar index" + assert len(indices) >= 1, ( + "list_indices should return at least the new scalar index" + ) names = [idx["name"] for idx in indices] - assert index_name in names, f"Expected index name {index_name!r} in list_indices: {names}" + assert index_name in names, ( + f"Expected index name {index_name!r} in list_indices: {names}" + ) label_index = next(idx for idx in indices if idx["name"] == index_name) assert label_index["type"] == "BTree", ( @@ -1018,7 +1022,9 @@ def test_optimize_indices_success_with_uri(self, multi_fragment_lance_dataset): assert result.count_rows() == lance.LanceDataset(dataset_uri).count_rows() indices = result.list_indices() - assert len(indices) >= 1, "list_indices should include at least the existing index" + assert len(indices) >= 1, ( + "list_indices should include at least the existing index" + ) names = [idx["name"] for idx in indices] assert "text_idx" in names, f"Expected 'text_idx' in list_indices: {names}" @@ -1029,9 +1035,10 @@ def test_optimize_indices_runtime_error_when_api_missing(self, temp_dir): lr.write_lance(ray.data.from_pandas(df), str(path)) ds = lance.LanceDataset(str(path)) - if getattr(ds, "optimize_indices", None) is not None or getattr( - ds, "optimize", None - ) is not None: + if ( + getattr(ds, "optimize_indices", None) is not None + or getattr(ds, "optimize", None) is not None + ): pytest.skip( "This lance version exposes optimize_indices/optimize; " "cannot test RuntimeError path." diff --git a/uv.lock b/uv.lock index 23220eb5..470fd684 100644 --- a/uv.lock +++ b/uv.lock @@ -387,7 +387,7 @@ requires-dist = [ { name = "more-itertools", marker = "python_full_version < '3.12'", specifier = ">=2.6.0" }, { name = "packaging" }, { name = "pyarrow", specifier = ">=17.0.0" }, - { name = "pylance", specifier = ">=6.0.0rc3" }, + { name = "pylance", specifier = ">=7.0.0b7" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.0.0" }, { name = "pytest-cov", specifier = ">=5.0.0" }, @@ -1024,7 +1024,7 @@ wheels = [ [[package]] name = "pylance" -version = "6.0.0rc3" +version = "7.0.0b7" source = { registry = "https://pypi.fury.io/lance-format" } dependencies = [ { name = "lance-namespace" }, @@ -1033,12 +1033,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_2gaNbS/pylance-6.0.0rc3-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d1e7a5b10204166973ad273cb4b761703b5dd5cb02c3da4f7a05c45d5f64892" }, - { url = "https://pypi.fury.io/lance-format/-/ver_eZwKh/pylance-6.0.0rc3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd473456c5ae4e22a287024b4cb6668a47be5d6c79ae796c76709e225e65b663" }, - { url = "https://pypi.fury.io/lance-format/-/ver_BuSJt/pylance-6.0.0rc3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c777b27667349fda0d400a13e5be08ffb8a2d59275f4f42437730567e75e2ce9" }, - { url = "https://pypi.fury.io/lance-format/-/ver_25fNsj/pylance-6.0.0rc3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02d1fb8eb65cc64f71a4257f8704365ff2fc246d6aee22b8c0316d4d4c4add83" }, - { url = "https://pypi.fury.io/lance-format/-/ver_44z8I/pylance-6.0.0rc3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:7c72068bfce55762356686a1a9c0dcab64397d742bc62a4414b2074684643d17" }, - { url = "https://pypi.fury.io/lance-format/-/ver_1Hyw9U/pylance-6.0.0rc3-cp39-abi3-win_amd64.whl", hash = "sha256:34601f31e7d4d5c5f0b87c0c71d7de43e8e8295f1e6d54ad39c11eb0a4ff5db6" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1zsVV4/pylance-7.0.0b7-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:66f2ba7969c0fad259f5d13842e89d664956c1415eed644d6956af333e848a62" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1TtYis/pylance-7.0.0b7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:029ac2359cccbac62345392c78f5aae6596eda4a3398fbf8bef5aad708591d14" }, + { url = "https://pypi.fury.io/lance-format/-/ver_uZkce/pylance-7.0.0b7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a53841757573d7c63238d79df94236eb6e46d9508d2690d98ce983e5d04f8c7a" }, + { url = "https://pypi.fury.io/lance-format/-/ver_3iSeD/pylance-7.0.0b7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:34c67a0102d47f870d4b795087622c16087aa1928c1925b1acdeb61206f4663a" }, + { url = "https://pypi.fury.io/lance-format/-/ver_BpURn/pylance-7.0.0b7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ac70f0248239b50ae718e38e69c1418899a1a7575e7b911beac91eeb60b174ec" }, + { url = "https://pypi.fury.io/lance-format/-/ver_jVZS2/pylance-7.0.0b7-cp39-abi3-win_amd64.whl", hash = "sha256:0f1caea57ea998f6e59dcb2684a08e4b06f993ecc77c8d0f032a4a43a82ed085" }, ] [[package]]