Extract diskann-storage crate from diskann-providers

Cargo.toml

@@ -12,6 +12,8 @@ members = [
     "diskann-platform",
     # Algorithm
     "diskann",
+    # Storage
+    "diskann-storage",
     # Providers
     "diskann-providers",
     "diskann-disk",
@@ -56,6 +58,8 @@ diskann-quantization = { path = "diskann-quantization", default-features = false
 diskann-platform = { path = "diskann-platform", version = "0.50.0" }
 # Algorithm
 diskann = { path = "diskann", version = "0.50.0" }
+# Storage
+diskann-storage = { path = "diskann-storage", default-features = false, version = "0.50.0" }
 # Providers
 diskann-providers = { path = "diskann-providers", default-features = false, version = "0.50.0" }
 diskann-disk = { path = "diskann-disk", version = "0.50.0" }

diskann-benchmark/Cargo.toml

@@ -15,6 +15,7 @@ bytemuck.workspace = true
 roaring.workspace = true
 clap = { workspace = true, features = ["derive"] }
 diskann-providers = { workspace = true }
+diskann-storage = { workspace = true }
 diskann.workspace = true
 diskann-utils.workspace = true
 half = { workspace = true, features = ["rand_distr", "num-traits"] }

diskann-benchmark/src/backend/disk_index/benchmarks.rs

@@ -13,7 +13,7 @@ use diskann_benchmark_runner::{
     utils::datatype::{DataType, Type},
     Benchmark, Checkpoint,
 };
-use diskann_providers::storage::FileStorageProvider;
+use diskann_storage::FileStorageProvider;
 use half::f16;
 
 use crate::{

diskann-benchmark/src/backend/disk_index/build.rs

@@ -18,8 +18,8 @@ use diskann_disk::{
     },
     storage::DiskIndexWriter,
 };
-use diskann_providers::storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_providers::{model::IndexConfiguration, utils::load_metadata_from_file};
+use diskann_storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_vector::distance::Metric;
 use opentelemetry::global;
 use opentelemetry::trace::Tracer;

diskann-benchmark/src/backend/disk_index/search.rs

@@ -19,12 +19,10 @@ use diskann_disk::{
     storage::disk_index_reader::DiskIndexReader,
     utils::{instrumentation::PerfLogger, statistics, AlignedFileReaderFactory, QueryStatistics},
 };
-use diskann_providers::storage::StorageReadProvider;
-use diskann_providers::{
-    storage::{
-        get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file, FileStorageProvider,
-    },
-    utils::{create_thread_pool, ParallelIteratorInPool},
+use diskann_providers::utils::{create_thread_pool, ParallelIteratorInPool};
+use diskann_storage::{
+    get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file, FileStorageProvider,
+    StorageReadProvider,
 };
 use diskann_tools::utils::{search_index_utils, KRecallAtN};
 use diskann_utils::views::Matrix;

diskann-benchmark/src/inputs/disk.rs

@@ -11,7 +11,7 @@ use diskann_benchmark_runner::{
 };
 #[cfg(feature = "disk-index")]
 use diskann_disk::QuantizationType;
-use diskann_providers::storage::{get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file};
+use diskann_storage::{get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file};
 use serde::{Deserialize, Serialize};
 
 use crate::{

diskann-benchmark/src/inputs/save_and_load.rs

@@ -5,7 +5,7 @@
 use std::{io::Read, mem::size_of, num::NonZeroUsize};
 
 use diskann::{ANNError, ANNResult};
-use diskann_providers::storage::StorageReadProvider;
+use diskann_storage::StorageReadProvider;
 
 pub fn get_graph_num_frozen_points(
     storage_provider: &impl StorageReadProvider,

diskann-benchmark/src/main.rs

@@ -131,7 +131,7 @@ mod tests {
     use super::*;
 
     use diskann_benchmark_runner::{app::Commands, output::Memory};
-    use diskann_providers::storage::FileStorageProvider;
+    use diskann_storage::FileStorageProvider;
     use diskann_tools::utils::{compute_ground_truth_from_datafiles, GraphDataF32Vector};
     use diskann_vector::distance::Metric;
 

diskann-benchmark/src/utils/datafiles.rs

@@ -9,7 +9,7 @@ use anyhow::Context;
 use bit_set::BitSet;
 use diskann::utils::IntoUsize;
 use diskann_benchmark_runner::utils::datatype::DataType;
-use diskann_providers::storage::StorageReadProvider;
+use diskann_storage::StorageReadProvider;
 use diskann_utils::views::Matrix;
 use serde::{Deserialize, Serialize};
 
@@ -23,8 +23,7 @@ where
     T: Copy + bytemuck::Pod,
 {
     let data = diskann_utils::io::read_bin::<T>(
-        &mut diskann_providers::storage::FileStorageProvider
-            .open_reader(&path.0.to_string_lossy())?,
+        &mut diskann_storage::FileStorageProvider.open_reader(&path.0.to_string_lossy())?,
     )?;
     Ok(data)
 }

diskann-disk/Cargo.toml

@@ -21,6 +21,7 @@ targets = [
 diskann = { workspace = true }
 diskann-utils = { workspace = true }
 diskann-providers = { workspace = true }
+diskann-storage = { workspace = true }
 diskann-vector = { workspace = true }
 diskann-linalg = { workspace = true }
 diskann-quantization = { workspace = true, features = ["rayon"] }
@@ -58,6 +59,7 @@ diskann-providers = { workspace = true, default-features = false, features = [
     "testing",
     "virtual_storage",
 ] }
+diskann-storage = { workspace = true, features = ["virtual_storage"] }
 diskann-utils = { workspace = true, features = ["testing"] }
 criterion.workspace = true
 iai-callgrind.workspace = true

diskann-disk/src/build/builder/build.rs

@@ -15,7 +15,6 @@ use diskann::{
     utils::{async_tools, vecid_from_usize, TryIntoVectorId, VectorRepr, ONE},
     ANNError, ANNErrorKind, ANNResult,
 };
-use diskann_providers::storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_providers::{
     model::{
         graph::{
@@ -30,6 +29,7 @@ use diskann_providers::{
         MAX_MEDOID_SAMPLE_SIZE,
     },
 };
+use diskann_storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_utils::io::{read_bin, write_bin};
 use diskann_utils::views::MatrixView;
 use tokio::task::JoinSet;
@@ -913,7 +913,7 @@ impl StartPoint {
 mod start_point_tests {
     use std::io::Write;
 
-    use diskann_providers::storage::VirtualStorageProvider;
+    use diskann_storage::VirtualStorageProvider;
     use diskann_utils::io::Metadata;
 
     use super::*;

diskann-disk/src/build/builder/core.rs

@@ -5,7 +5,6 @@
 use std::mem::{self, size_of};
 
 use diskann::ANNResult;
-use diskann_providers::storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_providers::{
     model::{
         graph::traits::GraphDataType, IndexConfiguration, GRAPH_SLACK_FACTOR,
@@ -17,6 +16,7 @@ use diskann_providers::{
         READ_WRITE_BLOCK_SIZE,
     },
 };
+use diskann_storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_utils::io::read_bin;
 use rand::{seq::SliceRandom, Rng};
 use tracing::info;
@@ -633,15 +633,16 @@ pub(crate) mod disk_index_builder_tests {
         utils::{IntoUsize, VectorRepr, ONE},
         ANNResult,
     };
-    use diskann_providers::storage::VirtualStorageProvider;
     use diskann_providers::{
         common::AlignedBoxWithSlice,
-        storage::{get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file},
         test_utils::graph_data_type_utils::{
             GraphDataF32VectorU32Data, GraphDataF32VectorUnitData,
         },
         utils::Timer,
     };
+    use diskann_storage::{
+        get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file, VirtualStorageProvider,
+    };
     use diskann_utils::test_data_root;
     use diskann_vector::{
         distance::Metric::{self, L2},

diskann-disk/src/build/builder/inmem_builder.rs

@@ -14,7 +14,6 @@ use diskann::{
     utils::VectorRepr,
     ANNError, ANNResult,
 };
-use diskann_providers::storage::{DynWriteProvider, StorageReadProvider, WriteProviderWrapper};
 use diskann_providers::{
     index::diskann_async,
     model::{
@@ -31,6 +30,7 @@ use diskann_providers::{
         index_storage::load_index, load_fp_index, AsyncIndexMetadata, DiskGraphOnly, SaveWith,
     },
 };
+use diskann_storage::{DynWriteProvider, StorageReadProvider, WriteProviderWrapper};
 use diskann_utils::future::{AsyncFriendly, SendFuture};
 
 use super::quantizer::BuildQuantizer;

diskann-disk/src/build/builder/quantizer.rs

@@ -4,7 +4,6 @@
  */
 //! Disk index quantizer implementation.
 use diskann::{ANNError, ANNResult};
-use diskann_providers::storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_providers::{
     index::diskann_async::train_pq,
     model::{
@@ -18,6 +17,7 @@ use diskann_providers::{
     utils::{BridgeErr, PQPathNames},
 };
 use diskann_quantization::scalar::train::ScalarQuantizationParameters;
+use diskann_storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_utils::views::MatrixView;
 use tracing::info;
 

diskann-disk/src/build/chunking/checkpoint/checkpoint_record_manager_with_file.rs

@@ -9,7 +9,8 @@ use std::{
 };
 
 use diskann::{ANNError, ANNResult};
-use diskann_providers::{storage::FileStorageProvider, utils::file_exists};
+use diskann_providers::utils::file_exists;
+use diskann_storage::FileStorageProvider;
 
 use super::{CheckpointManager, CheckpointRecord, Progress, WorkStage};
 

diskann-disk/src/search/provider/disk_provider.rs

@@ -36,14 +36,15 @@ use diskann::{
     },
     ANNError, ANNResult,
 };
-use diskann_providers::storage::StorageReadProvider;
 use diskann_providers::{
     model::{
         compute_pq_distance, compute_pq_distance_for_pq_coordinates, graph::traits::GraphDataType,
         pq::quantizer_preprocess, PQData, PQScratch,
     },
-    storage::{get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file, LoadWith},
+    storage::LoadWith,
 };
+use diskann_storage::StorageReadProvider;
+use diskann_storage::{get_compressed_pq_file, get_disk_index_file, get_pq_pivot_file};
 use diskann_vector::{distance::Metric, DistanceFunction, PreprocessedDistanceFunction};
 use futures_util::future;
 use tokio::runtime::Runtime;
@@ -1060,16 +1061,14 @@ mod disk_provider_tests {
         utils::IntoUsize,
         ANNErrorKind,
     };
-    use diskann_providers::storage::{
-        DynWriteProvider, StorageReadProvider, VirtualStorageProvider,
-    };
     use diskann_providers::{
         common::AlignedBoxWithSlice,
         test_utils::graph_data_type_utils::{
             GraphDataF32VectorU32Data, GraphDataF32VectorUnitData,
         },
         utils::{create_thread_pool, load_aligned_bin, PQPathNames, ParallelIteratorInPool},
     };
+    use diskann_storage::{DynWriteProvider, StorageReadProvider, VirtualStorageProvider};
     use diskann_utils::{io::read_bin, test_data_root};
     use diskann_vector::distance::Metric;
     use rayon::prelude::{IndexedParallelIterator, IntoParallelRefIterator};

diskann-disk/src/search/provider/disk_vertex_provider.rs

@@ -264,15 +264,14 @@ mod disk_vertex_provider_tests {
     use std::sync::Arc;
 
     use diskann::{graph::config, utils::ONE};
-    use diskann_providers::storage::{
-        StorageReadProvider, StorageWriteProvider, VirtualStorageProvider,
-    };
     use diskann_providers::{
         model::{graph::traits::GraphDataType, IndexConfiguration},
-        storage::get_disk_index_file,
         test_utils::graph_data_type_utils::GraphDataF32VectorU32Data,
         utils::load_metadata_from_file,
     };
+    use diskann_storage::{
+        get_disk_index_file, StorageReadProvider, StorageWriteProvider, VirtualStorageProvider,
+    };
     use diskann_utils::test_data_root;
     use vfs::OverlayFS;
 

diskann-disk/src/storage/cached_reader.rs

@@ -5,7 +5,7 @@
 use std::io::{Read, Seek};
 
 use diskann::{ANNError, ANNResult};
-use diskann_providers::storage::StorageReadProvider;
+use diskann_storage::StorageReadProvider;
 use tracing::info;
 
 /// Sequential cached reads with a generic storage provider with read access.
@@ -114,7 +114,7 @@ where
 #[cfg(test)]
 mod cached_reader_test {
 
-    use diskann_providers::storage::{StorageWriteProvider, VirtualStorageProvider};
+    use diskann_storage::{StorageWriteProvider, VirtualStorageProvider};
     use vfs::MemoryFS;
 
     use super::*;

diskann-disk/src/storage/cached_writer.rs

@@ -4,7 +4,7 @@
  */
 use std::io::{Seek, SeekFrom, Write};
 
-use diskann_providers::storage::StorageWriteProvider;
+use diskann_storage::StorageWriteProvider;
 use tracing::info;
 
 /// Sequential cached writes with a generic storage provider with write access.
@@ -114,7 +114,7 @@ where
 
 #[cfg(test)]
 mod cached_writer_test {
-    use diskann_providers::storage::VirtualStorageProvider;
+    use diskann_storage::VirtualStorageProvider;
     use vfs::OverlayFS;
 
     use super::*;

diskann-disk/src/storage/disk_index_reader.rs

@@ -5,8 +5,8 @@
 use std::{marker::PhantomData, sync::Arc};
 
 use diskann::ANNResult;
-use diskann_providers::storage::StorageReadProvider;
 use diskann_providers::{model::pq::PQData, storage::PQStorage, utils::load_metadata_from_file};
+use diskann_storage::StorageReadProvider;
 use tracing::info;
 
 /// This struct is used by the DiskIndexSearcher to read the index data from storage. Noted that the index data here is different from index graph,
@@ -69,7 +69,7 @@ impl<VectorType> DiskIndexReader<VectorType> {
 #[cfg(test)]
 mod disk_index_storage_test {
     use diskann::ANNErrorKind;
-    use diskann_providers::storage::VirtualStorageProvider;
+    use diskann_storage::VirtualStorageProvider;
     use diskann_utils::test_data_root;
     use vfs::OverlayFS;