π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Require explicit VecDB index configuration
laurenspriemd0bf5920a0a4
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rsindex c7df5bf8b3..4aff1d6b51 100644--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs@@ -191,23 +191,33 @@ impl VecDb {pub fn new(file_path: String,dimensions: u32,- storage: Option<VecDbStorage>,- metric: Option<VecDbMetric>,+ storage: VecDbStorage,+ metric: VecDbMetric,) -> Result<Self, RustVecDbError> {let path = Path::new(&file_path);let dims = dimensions as usize;- let inner = vecdb::VecDb::open_with(+ let inner = vecdb::VecDb::open(path,dims,- storage.map(to_engine_storage),- metric.map(to_engine_metric),+ to_engine_storage(storage),+ to_engine_metric(metric),)?;Ok(Self { inner })}- pub fn open_read_only(file_path: String, dimensions: u32) -> Result<Self, RustVecDbError> {+ pub fn open_read_only(+ file_path: String,+ dimensions: u32,+ storage: VecDbStorage,+ metric: VecDbMetric,+ ) -> Result<Self, RustVecDbError> {Ok(Self {- inner: vecdb::VecDb::open_read_only(Path::new(&file_path), dimensions as usize)?,+ inner: vecdb::VecDb::open_read_only(+ Path::new(&file_path),+ dimensions as usize,+ to_engine_storage(storage),+ to_engine_metric(metric),+ )?,})}@@ -436,15 +446,66 @@ mod tests {}#[test]- fn metric_defaults_to_cosine_and_explicit_inner_product_is_immutable() {+ fn read_only_verifies_explicit_configuration() {+ for (storage, metric, wrong_storage, wrong_metric) in [+ (+ VecDbStorage::F32,+ VecDbMetric::InnerProduct,+ VecDbStorage::I8,+ VecDbMetric::Cosine,+ ),+ (+ VecDbStorage::I8,+ VecDbMetric::Cosine,+ VecDbStorage::F32,+ VecDbMetric::InnerProduct,+ ),+ ] {+ let dir = TestDir::create();+ let db = VecDb::new(dir.db_path(), 32, storage, metric).unwrap();+ db.add_vector(key("kept"), vec![1.0; 32]).unwrap();+ db.flush().unwrap();+ let verify = || {+ assert!(matches!(+ VecDb::open_read_only(dir.db_path(), 32, wrong_storage, metric),+ Err(RustVecDbError::StorageMismatch { .. })+ ));+ assert!(matches!(+ VecDb::open_read_only(dir.db_path(), 32, storage, wrong_metric),+ Err(RustVecDbError::MetricMismatch { .. })+ ));+ assert!(matches!(+ VecDb::open_read_only(dir.db_path(), 64, storage, metric),+ Err(RustVecDbError::DimensionMismatch { .. })+ ));+ let reader = VecDb::open_read_only(dir.db_path(), 32, storage, metric).unwrap();+ assert_eq!(reader.get_index_stats().unwrap().live_count, 1);+ assert!(matches!(+ reader.add_vector(key("no"), vec![1.0; 32]),+ Err(RustVecDbError::ReadOnly { .. })+ ));+ };+ verify();+ drop(db);+ verify();+ }+ }++ #[test]+ fn explicitly_selected_metrics_are_immutable() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), 32, None, None).unwrap();+ let db = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();assert!(matches!(db.get_index_stats().unwrap().metric,VecDbMetric::Cosine));assert!(matches!(- VecDb::new(dir.db_path(), 32, None, Some(VecDbMetric::InnerProduct)),+ VecDb::new(+ dir.db_path(),+ 32,+ VecDbStorage::I8,+ VecDbMetric::InnerProduct+ ),Err(RustVecDbError::MetricMismatch { .. })));drop(db);@@ -452,8 +513,8 @@ mod tests {let db = VecDb::new(dir.db_path(),DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::InnerProduct),+ VecDbStorage::F32,+ VecDbMetric::InnerProduct,).unwrap();let vector: Vec<f32> = basis(0).iter().map(|value| value * 3.0).collect();@@ -467,10 +528,16 @@ mod tests {db.flush().unwrap();drop(db);assert!(matches!(- VecDb::new(dir.db_path(), DIMS, None, Some(VecDbMetric::Cosine)),+ VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine),Err(RustVecDbError::MetricMismatch { message }) if message.contains("cosine") && message.contains("inner product")));- let db = VecDb::new(dir.db_path(), DIMS, None, None).unwrap();+ let db = VecDb::new(+ dir.db_path(),+ DIMS,+ VecDbStorage::F32,+ VecDbMetric::InnerProduct,+ )+ .unwrap();assert!(matches!(db.get_index_stats().unwrap().metric,VecDbMetric::InnerProduct@@ -484,7 +551,7 @@ mod tests {#[test]fn add_search_stats_round_trip() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();db.add_vector(key("a"), basis(0)).unwrap();db.bulk_add_vectors(vec![key("b"), key("c")], vec![basis(1), basis(2)]).unwrap();@@ -517,7 +584,9 @@ mod tests {assert!(stats.approximate_memory_bytes > 0);db.flush().unwrap();assert_eq!(db.get_index_stats().unwrap().records_since_snapshot, 0);- let read_only = VecDb::open_read_only(dir.db_path(), DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine)+ .unwrap();assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);assert!(matches!(read_only.add_vector(key("x"), basis(3)),@@ -539,7 +608,7 @@ mod tests {#[test]fn bulk_add_rejects_length_mismatch() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();let error = db.bulk_add_vectors(vec![key("a")], vec![basis(0), basis(1)]).unwrap_err();@@ -550,7 +619,7 @@ mod tests {#[test]fn similarity_threshold_conversion_edges() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();db.add_vector(key("a"), basis(0)).unwrap();db.add_vector(key("b"), basis(1)).unwrap();let close = db@@ -574,7 +643,7 @@ mod tests {#[test]fn bulk_search_within_similarity_honors_per_query_thresholds() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();db.bulk_add_vectors(vec![key("a"), key("b"), key("c")],vec![basis(0), basis(1), basis(2)],@@ -640,7 +709,7 @@ mod tests {#[test]fn bulk_filtered_search_stays_query_aligned() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();db.bulk_add_vectors(vec![key("a"), key("b"), key("c")],vec![basis(0), basis(1), basis(2)],@@ -694,7 +763,7 @@ mod tests {#[test]fn bulk_search_keys_excludes_self_and_honors_the_filters() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();db.bulk_add_vectors(vec![key("a"), key("b"), key("c")],vec![basis(0), basis(1), basis(2)],@@ -732,7 +801,7 @@ mod tests {#[test]fn attrs_round_trip_through_the_api() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();let attrs = vec![VecDbAttr {name: key("model"),@@ -824,7 +893,7 @@ mod tests {check_open_cost(dir.db_path()),VecDbOpenCost::Absent));- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();db.add_vector(key("a"), basis(0)).unwrap();assert!(matches!(check_open_cost(dir.db_path()),@@ -846,7 +915,7 @@ mod tests {#[test]fn delete_vec_db_files_purges_with_and_without_live_instances() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();db.add_vector(key("a"), basis(0)).unwrap();db.flush().unwrap();delete_vec_db_files(dir.db_path()).unwrap();@@ -857,7 +926,8 @@ mod tests {db.get_index_stats(),Err(RustVecDbError::Closed { .. })));- let reopened = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();+ let reopened =+ VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();assert_eq!(reopened.get_index_stats().unwrap().live_count, 0);reopened.add_vector(key("b"), basis(1)).unwrap();drop(reopened);@@ -877,7 +947,7 @@ mod tests {#[test]fn storage_is_chosen_at_creation_reported_in_stats_and_verified_on_reopen() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8), None).unwrap();+ let db = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();assert!(matches!(db.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -904,10 +974,10 @@ mod tests {assert_eq!(stats.dims, 32);assert!(matches!(stats.storage, VecDbStorage::I8));assert!(matches!(- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), None),+ VecDb::new(dir.db_path(), 32, VecDbStorage::F32, VecDbMetric::Cosine),Err(RustVecDbError::StorageMismatch { .. })));- let joined = VecDb::new(dir.db_path(), 32, None, None).unwrap();+ let joined = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();assert!(matches!(joined.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -915,7 +985,9 @@ mod tests {drop(joined);db.flush().unwrap();drop(db);- let read_only = VecDb::open_read_only(dir.db_path(), 32).unwrap();+ let read_only =+ VecDb::open_read_only(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine)+ .unwrap();assert!(matches!(read_only.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -923,10 +995,11 @@ mod tests {assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);drop(read_only);assert!(matches!(- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), None),+ VecDb::new(dir.db_path(), 32, VecDbStorage::F32, VecDbMetric::Cosine),Err(RustVecDbError::StorageMismatch { message }) if message.contains("f32") && message.contains("i8")));- let reopened = VecDb::new(dir.db_path(), 32, None, None).unwrap();+ let reopened =+ VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();assert!(matches!(reopened.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -936,28 +1009,37 @@ mod tests {}#[test]- fn storage_defaults_to_i8_and_rejects_dims_off_the_lane_grid() {+ fn explicit_i8_storage_rejects_dims_off_the_lane_grid() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), 32, None, None).unwrap();+ let db = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();assert!(matches!(db.get_index_stats().unwrap().storage,VecDbStorage::I8));- let explicit = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8), None).unwrap();+ let explicit =+ VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();assert!(matches!(explicit.get_index_stats().unwrap().storage,VecDbStorage::I8));assert!(matches!(- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), None),+ VecDb::new(dir.db_path(), 32, VecDbStorage::F32, VecDbMetric::Cosine),Err(RustVecDbError::StorageMismatch { .. })));let other = TestDir::create();assert!(matches!(- VecDb::new(other.db_path(), DIMS, None, None),+ VecDb::new(other.db_path(), DIMS, VecDbStorage::I8, VecDbMetric::Cosine),Err(RustVecDbError::InvalidDimensions { message }) if message.contains("i8") && message.contains("32")));assert!(!PathBuf::from(other.db_path()).exists());- assert!(VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::F32), None).is_ok());+ assert!(+ VecDb::new(+ other.db_path(),+ DIMS,+ VecDbStorage::F32,+ VecDbMetric::Cosine+ )+ .is_ok()+ );}}diff --git a/rust/crates/ml/examples/vecdb_bench.rs b/rust/crates/ml/examples/vecdb_bench.rsindex d911e6221b..78c5915e3d 100644--- a/rust/crates/ml/examples/vecdb_bench.rs+++ b/rust/crates/ml/examples/vecdb_bench.rs@@ -7,7 +7,9 @@ use std::collections::HashSet;use std::path::{Path, PathBuf};use std::time::{Duration, Instant};-use ente_ml::vecdb::{AttrValue, Attribute, Match, SearchParams, StorageKind, VecDb};+use ente_ml::vecdb::{+ AttrValue, Attribute, DistanceMetric, Match, SearchParams, StorageKind, VecDb,+};const SEED: u64 = 0xE47E_0000_0000_0001;const LATENT_DIMS: usize = 24;@@ -180,7 +182,15 @@ fn run_scale(scale: usize, dims: usize, attrs: bool, storage: StorageKind, temp_);let dir = temp_root.join(format!("scale-{scale}-{dims}"));std::fs::create_dir_all(&dir).expect("create bench dir");- drop(VecDb::open_with_storage(&dir.join("bench.vecdb"), dims, storage).expect("open vecdb"));+ drop(+ VecDb::open(+ &dir.join("bench.vecdb"),+ dims,+ storage,+ DistanceMetric::Cosine,+ )+ .expect("open vecdb"),+ );eprintln!("[scale {scale}] generating data");let data = generate_data(scale, dims, clusters);let vecdb = run_vecdb(&data, dims, attrs, storage, &dir, scale);@@ -317,7 +327,7 @@ fn run_vecdb(scale: usize,) -> VecdbReport {let path = dir.join("bench.vecdb");- let mut db = VecDb::open_with_storage(&path, dims, storage).expect("open vecdb");+ let mut db = VecDb::open(&path, dims, storage, DistanceMetric::Cosine).expect("open vecdb");let ingest = ingest_phase(&mut db, data, attrs, scale);let searches = search_phase(&db, data, storage, scale);let stats = db.stats().expect("vecdb stats");@@ -326,7 +336,7 @@ fn run_vecdb(.map(|meta| meta.len()).unwrap_or(0);drop(db);- let (reopens, mut db) = reopen_phase(&path, dims, &snapshot_file, scale);+ let (reopens, mut db) = reopen_phase(&path, dims, storage, &snapshot_file, scale);let compaction = compaction_phase(&mut db, data, scale);db.delete().expect("vecdb delete");VecdbReport {@@ -529,18 +539,21 @@ fn search_phase(db: &VecDb, data: &BenchData, storage: StorageKind, scale: usizefn reopen_phase(path: &Path,dims: usize,+ storage: StorageKind,snapshot_file: &Path,scale: usize,) -> (ReopenTimings, VecDb) {eprintln!("[scale {scale}] vecdb cold open with snapshot");let started = Instant::now();- let reopened = VecDb::open(path, dims).expect("vecdb reopen with snapshot");+ let reopened = VecDb::open(path, dims, storage, DistanceMetric::Cosine)+ .expect("vecdb reopen with snapshot");let open_with_snapshot = started.elapsed();drop(reopened);std::fs::remove_file(snapshot_file).expect("remove snapshot");eprintln!("[scale {scale}] vecdb cold open without snapshot (full rebuild)");let started = Instant::now();- let db = VecDb::open(path, dims).expect("vecdb reopen without snapshot");+ let db = VecDb::open(path, dims, storage, DistanceMetric::Cosine)+ .expect("vecdb reopen without snapshot");let open_full_rebuild = started.elapsed();(ReopenTimings {diff --git a/rust/crates/ml/src/vecdb/README.md b/rust/crates/ml/src/vecdb/README.mdindex 165f7f80dd..3e260b20aa 100644--- a/rust/crates/ml/src/vecdb/README.md+++ b/rust/crates/ml/src/vecdb/README.md@@ -1,24 +1,26 @@# VecDB index configuration-New indexes default to i8 storage and cosine distance. Storage and distance metric-are persisted at creation and remain unchanged through reopen, compaction, and reset.--`VecDb::open(path, dims)` detects both settings on existing indexes, including older-inner-product indexes. `open_with_storage` and `open_with_metric` select or verify-one setting; `open_with` accepts optional storage and metric together. An explicit-setting that differs from the saved value returns `StorageMismatch` or-`MetricMismatch`. Read-only opens use the saved settings.+Every open requires the path, dimensions, storage, and distance metric. There are+no defaults. These settings are persisted at creation and remain unchanged through+reopen, compaction, and reset.```rustuse ente_ml::vecdb::{DistanceMetric, StorageKind, VecDb};-let cosine = VecDb::open(path, 512)?;-let inner_product = VecDb::open_with_metric(other_path, 512, DistanceMetric::InnerProduct)?;-let f32_cosine = VecDb::open_with(third_path, 512, Some(StorageKind::F32), Some(DistanceMetric::Cosine))?;+let index = VecDb::open(path, 512, StorageKind::I8, DistanceMetric::Cosine)?;+let reader = VecDb::open_read_only(path, 512, StorageKind::I8, DistanceMetric::Cosine)?;```-The Flutter constructor exposes optional `storage` and `metric` parameters, and-index stats report both settings.+Both open methods validate the supplied configuration against the saved header or+an already-open instance. A mismatch returns `DimensionMismatch`, `StorageMismatch`,+or `MetricMismatch`. Older indexes remain readable by specifying their original+storage and `DistanceMetric::InnerProduct`.++A read-only open of a missing or incomplete index returns an empty view with the+supplied configuration without creating or modifying the file.++The Flutter constructor and read-only open also require `storage` and `metric`.+Index stats report both settings.Cosine distance is `1 - dot(a, b) / (norm(a) * norm(b))`, clamped to `[0, 2]`.A comparison involving a zero vector has distance `1`, including two zero vectors.diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rsindex fca93796c4..64d30bd3a4 100644--- a/rust/crates/ml/src/vecdb/log.rs+++ b/rust/crates/ml/src/vecdb/log.rs@@ -127,10 +127,9 @@ impl Log {mut file: File,path: &Path,expected_dims: usize,- requested: Option<StorageKind>,- requested_metric: Option<DistanceMetric>,+ requested: StorageKind,+ requested_metric: DistanceMetric,) -> Result<Self, VecDbError> {- let fallback = requested.unwrap_or(StorageKind::I8);let file_len = file.metadata().map_err(|source| VecDbError::io(path, source))?@@ -140,22 +139,38 @@ impl Log {"reinitializing {} whose {file_len}-byte header was never completed",path.display());- validate_dims(expected_dims, fallback)?;- return Self::initialize(- file,- path,- expected_dims,- fallback,- requested_metric.unwrap_or_default(),- );+ validate_dims(expected_dims, requested)?;+ return Self::initialize(file, path, expected_dims, requested, requested_metric);}file.seek(SeekFrom::Start(0)).map_err(|source| VecDbError::io(path, source))?;let mut header = [0u8; HEADER_LEN];file.read_exact(&mut header).map_err(|source| VecDbError::io(path, source))?;- let (generation, storage, metric) =- decode_header(&header, expected_dims, requested, requested_metric)?;+ let Header {+ generation,+ dims,+ storage,+ metric,+ } = decode_header(&header)?;+ if requested != storage {+ return Err(VecDbError::StorageMismatch {+ expected: requested,+ actual: storage,+ });+ }+ if requested_metric != metric {+ return Err(VecDbError::MetricMismatch {+ expected: requested_metric,+ actual: metric,+ });+ }+ if dims != expected_dims {+ return Err(VecDbError::DimensionMismatch {+ expected: expected_dims,+ actual: dims,+ });+ }validate_dims(expected_dims, storage)?;Ok(Self {file,@@ -779,16 +794,18 @@ fn encode_header(}pub(crate) fn header_generation(bytes: &[u8; HEADER_LEN]) -> Result<[u8; 16], VecDbError> {- let dims = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;- decode_header(bytes, dims, None, None).map(|(generation, _, _)| generation)+ decode_header(bytes).map(|header| header.generation)+}++#[derive(Debug, PartialEq, Eq)]+struct Header {+ generation: [u8; 16],+ dims: usize,+ storage: StorageKind,+ metric: DistanceMetric,}-fn decode_header(- bytes: &[u8; HEADER_LEN],- expected_dims: usize,- requested: Option<StorageKind>,- requested_metric: Option<DistanceMetric>,-) -> Result<([u8; 16], StorageKind, DistanceMetric), VecDbError> {+fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {if bytes[0..4] != MAGIC {return Err(VecDbError::Corrupt(format!("bad log magic {:02x?}",@@ -811,35 +828,18 @@ fn decode_header(let storage = StorageKind::from_header_tag(bytes[STORAGE_TAG_OFFSET]).ok_or_else(|| {VecDbError::Corrupt(format!("unknown scalar tag {}", bytes[STORAGE_TAG_OFFSET]))})?;- if let Some(requested) = requested- && requested != storage- {- return Err(VecDbError::StorageMismatch {- expected: requested,- actual: storage,- });- }let metric = DistanceMetric::from_header_tag(bytes[METRIC_TAG_OFFSET]).ok_or_else(|| {VecDbError::Corrupt(format!("unknown metric tag {}", bytes[METRIC_TAG_OFFSET]))})?;- if let Some(requested) = requested_metric- && requested != metric- {- return Err(VecDbError::MetricMismatch {- expected: requested,- actual: metric,- });- }let dims = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;- if dims != expected_dims {- return Err(VecDbError::DimensionMismatch {- expected: expected_dims,- actual: dims,- });- }let mut generation = [0u8; 16];generation.copy_from_slice(&bytes[12..28]);- Ok((generation, storage, metric))+ Ok(Header {+ generation,+ dims,+ storage,+ metric,+ })}fn validate_entry(@@ -1056,9 +1056,9 @@ mod tests {.collect()}- fn reopen(path: &Path, dims: usize) -> Log {+ fn reopen(path: &Path, dims: usize, storage: StorageKind) -> Log {let file = open_writer_file(path, false).unwrap();- Log::open(file, path, dims, None, None).unwrap()+ Log::open(file, path, dims, storage, InnerProduct).unwrap()}fn append_bounds(log: &mut Log, entries: &[LogEntry<'_>]) -> (u64, u64) {@@ -1149,7 +1149,7 @@ mod tests {mutate(&mut bytes);std::fs::write(&path, bytes).unwrap();let file = File::options().read(true).write(true).open(&path).unwrap();- Log::open(file, &path, expected_dims, None, None)+ Log::open(file, &path, expected_dims, StorageKind::F32, InnerProduct)}#[test]@@ -1161,7 +1161,7 @@ mod tests {assert_eq!(created.current_end_offset(), HEADER_LEN as u64);assert_eq!(created.path(), path.as_path());drop(created.into_file());- let reopened = reopen(&path, 512);+ let reopened = reopen(&path, 512, StorageKind::F32);assert_eq!(reopened.generation(), generation);assert_eq!(reopened.current_end_offset(), HEADER_LEN as u64);}@@ -1187,7 +1187,7 @@ mod tests {Err(VecDbError::Locked(_))));drop(file);- let reopened = reopen(&alias, 8);+ let reopened = reopen(&alias, 8, StorageKind::F32);assert!(matches!(open_writer_file(&path, false),Err(VecDbError::Locked(_))@@ -1287,7 +1287,7 @@ mod tests {let path = dir.path().join("log");std::fs::write(&path, vec![1u8; junk_len]).unwrap();let file = File::options().read(true).write(true).open(&path).unwrap();- let mut log = Log::open(file, &path, 8, Some(StorageKind::F32), None).unwrap();+ let mut log = Log::open(file, &path, 8, StorageKind::F32, InnerProduct).unwrap();assert_eq!(log.current_end_offset(), HEADER_LEN as u64);assert_eq!(std::fs::metadata(&path).unwrap().len(), HEADER_LEN as u64);let (records, _) = scan_all(&mut log);@@ -1304,7 +1304,7 @@ mod tests {assert_eq!(start, HEADER_LEN as u64);let generation = log.generation();drop(log);- let reopened = reopen(&path, 8);+ let reopened = reopen(&path, 8, StorageKind::F32);assert_eq!(reopened.generation(), generation);assert_eq!(reopened.current_end_offset(), end);}@@ -1334,7 +1334,7 @@ mod tests {);let file = File::options().read(true).write(true).open(&path).unwrap();assert!(matches!(- Log::open(file, &path, 12, None, None),+ Log::open(file, &path, 12, StorageKind::F32, InnerProduct),Err(VecDbError::DimensionMismatch {expected: 12,actual: 8@@ -1417,7 +1417,7 @@ mod tests {bytes.extend_from_slice(&golden_attr_add);bytes.extend_from_slice(&golden_tombstone);std::fs::write(&path, &bytes).unwrap();- let mut log = reopen(&path, 8);+ let mut log = reopen(&path, 8, StorageKind::F32);assert_eq!(log.generation(), generation);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(@@ -1586,7 +1586,7 @@ mod tests {for cut in intact_end..boundaries[2].1 {let torn_path = dir.path().join(format!("torn-{cut}"));std::fs::write(&torn_path, &full_bytes[..cut as usize]).unwrap();- let mut torn_log = reopen(&torn_path, 8);+ let mut torn_log = reopen(&torn_path, 8, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut torn_log);assert_eq!(records.len(), 2);assert_eq!(records[0].1, boundaries[0].0);@@ -1622,7 +1622,7 @@ mod tests {let file = File::options().read(true).write(true).open(&path).unwrap();file.set_len(cut).unwrap();drop(file);- let mut log = reopen(&path, 8);+ let mut log = reopen(&path, 8, StorageKind::F32);assert_eq!(log.current_end_offset(), cut);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);@@ -1683,7 +1683,7 @@ mod tests {assert_eq!(log.current_end_offset(), grown_end);let generation = log.generation();drop(log);- assert_eq!(generation, reopen(&path, 8).generation());+ assert_eq!(generation, reopen(&path, 8, StorageKind::F32).generation());}#[test]@@ -1709,7 +1709,7 @@ mod tests {log.discard_unacked_tail().unwrap();assert_eq!(std::fs::metadata(&path).unwrap().len(), acked_end);drop(log);- let mut reopened = reopen(&path, 8);+ let mut reopened = reopen(&path, 8, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut reopened);assert_eq!(records.len(), 1);assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "acked"));@@ -1751,7 +1751,7 @@ mod tests {assert!(end < acked_end + stale.len() as u64);assert_eq!(std::fs::metadata(&path).unwrap().len(), end);drop(log);- let mut reopened = reopen(&path, 8);+ let mut reopened = reopen(&path, 8, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut reopened);assert_eq!(records.len(), 2);assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "acked"));@@ -1806,8 +1806,8 @@ mod tests {let staged_bytes = std::fs::read(&staged_path).unwrap();assert_eq!(staged_bytes.len(), synced_bytes.len());assert_eq!(staged_bytes[HEADER_LEN..], synced_bytes[HEADER_LEN..]);- let (synced_records, synced_end) = scan_all(&mut reopen(&synced_path, 8));- let (staged_records, staged_end) = scan_all(&mut reopen(&staged_path, 8));+ let (synced_records, synced_end) = scan_all(&mut reopen(&synced_path, 8, StorageKind::F32));+ let (staged_records, staged_end) = scan_all(&mut reopen(&staged_path, 8, StorageKind::F32));assert_eq!(staged_records.len(), 4);assert_eq!(staged_records, synced_records);assert_eq!(staged_end, synced_end);@@ -1829,7 +1829,7 @@ mod tests {);drop(log);let unwritable = File::open(&path).unwrap();- let mut log = Log::open(unwritable, &path, 8, None, None).unwrap();+ let mut log = Log::open(unwritable, &path, 8, StorageKind::F32, InnerProduct).unwrap();assert!(matches!(log.append(&[LogEntry::Add {key: "lost",@@ -1841,7 +1841,7 @@ mod tests {assert_eq!(log.current_end_offset(), acked_end);drop(log);assert_eq!(std::fs::metadata(&path).unwrap().len(), acked_end);- let mut reopened = reopen(&path, 8);+ let mut reopened = reopen(&path, 8, StorageKind::F32);let (records, _) = scan_all(&mut reopened);assert_eq!(records.len(), 1);assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "kept"));@@ -1878,7 +1878,7 @@ mod tests {assert!(matches!(&records[1].0, LogRecord::Add { key, .. } if key == "third"));assert_eq!(recoverable_end, end);drop(log);- let mut reopened = reopen(&path, 8);+ let mut reopened = reopen(&path, 8, StorageKind::F32);let (records, _) = scan_all(&mut reopened);assert_eq!(records.len(), 2);assert!(@@ -1921,7 +1921,7 @@ mod tests {let mut corrupted = bytes.clone();corrupted[*position] ^= 0x01;std::fs::write(&corrupt_path, &corrupted).unwrap();- let mut corrupt_log = reopen(&corrupt_path, 8);+ let mut corrupt_log = reopen(&corrupt_path, 8, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut corrupt_log);assert_eq!(records.len(), 1);assert_eq!(records[0].1, boundaries[0].0);@@ -1959,7 +1959,7 @@ mod tests {let mut record_bytes = good.clone();record_bytes.extend_from_slice(bad);write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(records[0].1, HEADER_LEN as u64);@@ -1998,7 +1998,7 @@ mod tests {assert_eq!(offsets, expected_offsets);assert_eq!(recoverable_end, previous_end);drop(log);- let reopened = reopen(&path, 8);+ let reopened = reopen(&path, 8, StorageKind::F32);assert_eq!(reopened.current_end_offset(), previous_end);}@@ -2078,7 +2078,7 @@ mod tests {);let temp_generation = temp_log.generation();drop(temp_log);- let reopened = reopen(&temp_path, 8);+ let reopened = reopen(&temp_path, 8, StorageKind::F32);assert_eq!(reopened.generation(), temp_generation);}@@ -2203,7 +2203,7 @@ mod tests {.unwrap();assert_eq!(&scan_attrs_of_single_add(&mut log), attrs);drop(log);- let mut reopened = reopen(&path, 8);+ let mut reopened = reopen(&path, 8, StorageKind::F32);assert_eq!(&scan_attrs_of_single_add(&mut reopened), attrs);}}@@ -2406,7 +2406,7 @@ mod tests {let mut record_bytes = good.clone();record_bytes.extend_from_slice(bad);write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1, "case {index}");assert_eq!(records[0].1, HEADER_LEN as u64, "case {index}");@@ -2450,7 +2450,7 @@ mod tests {let mut record_bytes = good.clone();record_bytes.extend_from_slice(&torn[..cut]);write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1, "cut {cut}");assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);@@ -2473,7 +2473,7 @@ mod tests {record_bytes.extend_from_slice(&vec![0xAA; garbage_len]);record_bytes.extend_from_slice(survivor);write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(recoverable_end, HEADER_LEN as u64 + first.len() as u64);@@ -2524,7 +2524,7 @@ mod tests {record_bytes.push(0xAA);record_bytes.extend_from_slice(tail);write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1, "case {index}");assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);@@ -2549,7 +2549,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("log");write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);@@ -2580,7 +2580,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("log");write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);@@ -2606,7 +2606,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("log");write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);@@ -2653,7 +2653,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("log");write_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::F32);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);@@ -2671,7 +2671,7 @@ mod tests {corrupted_bytes[crc_last] ^= 0xFF;let corrupted_path = dir.path().join("log-corrupted");write_log_file(&corrupted_path, dims as u32, &corrupted_bytes);- let mut corrupted_log = reopen(&corrupted_path, dims);+ let mut corrupted_log = reopen(&corrupted_path, dims, StorageKind::F32);let (corrupted_records, corrupted_end) = scan_all(&mut corrupted_log);assert_eq!(corrupted_records.len(), 1);assert!(@@ -2716,7 +2716,7 @@ mod tests {bytes.extend_from_slice(&first);bytes.extend_from_slice(&after_first);std::fs::write(&path, &bytes).unwrap();- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::I8);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "first"));@@ -2790,7 +2790,7 @@ mod tests {bytes.extend_from_slice(&golden_add);bytes.extend_from_slice(&tombstone);std::fs::write(&path, &bytes).unwrap();- let mut log = reopen(&path, 32);+ let mut log = reopen(&path, 32, StorageKind::I8);assert_eq!(log.storage(), StorageKind::I8);assert_eq!(log.generation(), generation);let (records, recoverable_end) = scan_all(&mut log);@@ -2821,8 +2821,13 @@ mod tests {let header = encode_header(8, &[7u8; 16], StorageKind::F32, InnerProduct);assert_eq!(header[STORAGE_TAG_OFFSET], 0);assert_eq!(- decode_header(&header, 8, None, None).unwrap(),- ([7u8; 16], StorageKind::F32, InnerProduct)+ decode_header(&header).unwrap(),+ Header {+ generation: [7u8; 16],+ dims: 8,+ storage: StorageKind::F32,+ metric: InnerProduct+ });}@@ -2855,27 +2860,30 @@ mod tests {.unwrap().into_file(),);- let open = |path: &Path, requested: Option<StorageKind>| {+ let open = |path: &Path, requested: StorageKind| {let file = File::options().read(true).write(true).open(path).unwrap();- Log::open(file, path, 32, requested, None)+ Log::open(file, path, 32, requested, InnerProduct)};- let detected = open(&i8_path, None).unwrap();+ let detected = open(&i8_path, StorageKind::I8).unwrap();assert_eq!(detected.storage(), StorageKind::I8);assert_eq!(detected.generation(), generation);assert_eq!(- open(&i8_path, Some(StorageKind::I8)).unwrap().storage(),+ open(&i8_path, StorageKind::I8).unwrap().storage(),StorageKind::I8);assert!(matches!(- open(&i8_path, Some(StorageKind::F32)),+ open(&i8_path, StorageKind::F32),Err(VecDbError::StorageMismatch {expected: StorageKind::F32,actual: StorageKind::I8})));- assert_eq!(open(&f32_path, None).unwrap().storage(), StorageKind::F32);+ assert_eq!(+ open(&f32_path, StorageKind::F32).unwrap().storage(),+ StorageKind::F32+ );assert!(matches!(- open(&f32_path, Some(StorageKind::I8)),+ open(&f32_path, StorageKind::I8),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32@@ -2891,15 +2899,20 @@ mod tests {let path = dir.path().join("log");std::fs::write(&path, [1u8; 5]).unwrap();let file = File::options().read(true).write(true).open(&path).unwrap();- let log = Log::open(file, &path, 64, Some(StorageKind::I8), None).unwrap();+ let log = Log::open(file, &path, 64, StorageKind::I8, InnerProduct).unwrap();assert_eq!(log.storage(), StorageKind::I8);drop(log);- assert_eq!(reopen(&path, 64).storage(), StorageKind::I8);+ assert_eq!(+ reopen(&path, 64, StorageKind::I8).storage(),+ StorageKind::I8+ );let other = dir.path().join("other");std::fs::write(&other, [1u8; 5]).unwrap();let file = File::options().read(true).write(true).open(&other).unwrap();assert_eq!(- Log::open(file, &other, 64, None, None).unwrap().storage(),+ Log::open(file, &other, 64, StorageKind::I8, InnerProduct)+ .unwrap()+ .storage(),StorageKind::I8);}@@ -2929,7 +2942,7 @@ mod tests {let path = dir.path().join("c");let file = File::options().read(true).write(true).open(&path).unwrap();assert!(matches!(- Log::open(file, &path, 8, Some(StorageKind::I8), None),+ Log::open(file, &path, 8, StorageKind::I8, InnerProduct),Err(VecDbError::DimensionMismatch {expected: 8,actual: 32@@ -2945,7 +2958,7 @@ mod tests {.open(&forged_path).unwrap();assert!(matches!(- Log::open(file, &forged_path, 8, None, None),+ Log::open(file, &forged_path, 8, StorageKind::I8, InnerProduct),Err(VecDbError::InvalidDimensions {dims: 8,storage: StorageKind::I8@@ -3006,7 +3019,7 @@ mod tests {assert_eq!(decoded_values, values);}drop(log);- let (reopened, _) = scan_all(&mut reopen(&path, dims));+ let (reopened, _) = scan_all(&mut reopen(&path, dims, StorageKind::I8));assert_eq!(reopened.len(), records.len());for ((record, offset), (again, again_offset)) in records.iter().zip(&reopened) {assert_eq!(offset, again_offset);@@ -3111,7 +3124,7 @@ mod tests {for cut in intact_end..boundaries[2].1 {let torn_path = dir.path().join(format!("torn-{cut}"));std::fs::write(&torn_path, &full_bytes[..cut as usize]).unwrap();- let mut torn_log = reopen(&torn_path, dims);+ let mut torn_log = reopen(&torn_path, dims, StorageKind::I8);let (records, recoverable_end) = scan_all(&mut torn_log);assert_eq!(records.len(), 2);assert_eq!(recoverable_end, intact_end);@@ -3121,7 +3134,7 @@ mod tests {mixed.extend_from_slice(&f32_sized);let mixed_path = dir.path().join("mixed");std::fs::write(&mixed_path, &mixed).unwrap();- let mut mixed_log = reopen(&mixed_path, dims);+ let mut mixed_log = reopen(&mixed_path, dims, StorageKind::I8);let (records, recoverable_end) = scan_all(&mut mixed_log);assert_eq!(records.len(), 2);assert_eq!(recoverable_end, intact_end);@@ -3150,7 +3163,7 @@ mod tests {record_bytes.extend_from_slice(&vec![0xAA; garbage_len]);record_bytes.extend_from_slice(survivor);write_i8_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::I8);assert_eq!(log.storage(), StorageKind::I8);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);@@ -3195,7 +3208,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("log");write_i8_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::I8);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);@@ -3213,7 +3226,7 @@ mod tests {corrupted_bytes[crc_last] ^= 0xFF;let corrupted_path = dir.path().join("log-corrupted");write_i8_log_file(&corrupted_path, dims as u32, &corrupted_bytes);- let mut corrupted_log = reopen(&corrupted_path, dims);+ let mut corrupted_log = reopen(&corrupted_path, dims, StorageKind::I8);let (corrupted_records, corrupted_end) = scan_all(&mut corrupted_log);assert_eq!(corrupted_records.len(), 1);assert!(@@ -3237,7 +3250,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("log");write_i8_log_file(&path, dims as u32, &record_bytes);- let mut log = reopen(&path, dims);+ let mut log = reopen(&path, dims, StorageKind::I8);let (records, recoverable_end) = scan_all(&mut log);assert_eq!(records.len(), 1);assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);diff --git a/rust/crates/ml/src/vecdb/mod.rs b/rust/crates/ml/src/vecdb/mod.rsindex 40111e7926..af104180de 100644--- a/rust/crates/ml/src/vecdb/mod.rs+++ b/rust/crates/ml/src/vecdb/mod.rs@@ -13,10 +13,9 @@ mod store;pub use store::{OpenCost, Stats, VecDb};-#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum DistanceMetric {InnerProduct,- #[default]Cosine,}diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rsindex 7b8b419131..080df6d2de 100644--- a/rust/crates/ml/src/vecdb/store.rs+++ b/rust/crates/ml/src/vecdb/store.rs@@ -307,8 +307,8 @@ fn registry_key_for(path: &Path) -> Result<PathBuf, VecDbError> {fn join_live(shared: Arc<Shared>,dims: usize,- storage: Option<StorageKind>,- metric: Option<DistanceMetric>,+ storage: StorageKind,+ metric: DistanceMetric,read_only: bool,) -> Result<VecDb, VecDbError> {if shared.dims != dims {@@ -317,19 +317,15 @@ fn join_live(actual: shared.dims,});}- if let Some(requested) = storage- && requested != shared.storage- {+ if storage != shared.storage {return Err(VecDbError::StorageMismatch {- expected: requested,+ expected: storage,actual: shared.storage,});}- if let Some(requested) = metric- && requested != shared.metric- {+ if metric != shared.metric {return Err(VecDbError::MetricMismatch {- expected: requested,+ expected: metric,actual: shared.metric,});}@@ -383,31 +379,11 @@ fn warn_handoff_cap(path: &Path) {}impl VecDb {- pub fn open(path: &Path, dims: usize) -> Result<Self, VecDbError> {- Self::open_with(path, dims, None, None)- }-- pub fn open_with_storage(+ pub fn open(path: &Path,dims: usize,storage: StorageKind,- ) -> Result<Self, VecDbError> {- Self::open_with(path, dims, Some(storage), None)- }-- pub fn open_with_metric(- path: &Path,- dims: usize,metric: DistanceMetric,- ) -> Result<Self, VecDbError> {- Self::open_with(path, dims, None, Some(metric))- }-- pub fn open_with(- path: &Path,- dims: usize,- storage: Option<StorageKind>,- metric: Option<DistanceMetric>,) -> Result<Self, VecDbError> {let key = registry_key_for(path)?;let slot = path_slot(&key);@@ -475,19 +451,24 @@ impl VecDb {open_cost_from_files(&key)}- pub fn open_read_only(path: &Path, dims: usize) -> Result<Self, VecDbError> {+ pub fn open_read_only(+ path: &Path,+ dims: usize,+ storage: StorageKind,+ metric: DistanceMetric,+ ) -> Result<Self, VecDbError> {let resolved = registry_key_for(path).unwrap_or_else(|_| path.to_path_buf());if let Some(slot) = existing_path_slot(&resolved) {let observed = lock_slot(&slot).live.as_ref().and_then(Weak::upgrade);match observed {Some(shared) if !shared.is_closed() => {- return join_live(shared, dims, None, None, true);+ return join_live(shared, dims, storage, metric, true);}Some(closing) => {drop(closing);match wait_for_teardown_handoff(&slot, &resolved) {SlotHandoff::Join(shared) => {- return join_live(shared, dims, None, None, true);+ return join_live(shared, dims, storage, metric, true);}SlotHandoff::Released(released) | SlotHandoff::Expired(released) => {drop(released);@@ -498,7 +479,7 @@ impl VecDb {}}Ok(Self {- shared: Arc::new(build_read_only(&resolved, dims)?),+ shared: Arc::new(build_read_only(&resolved, dims, storage, metric)?),read_only: true,})}@@ -1232,8 +1213,8 @@ fn restore_writer_mode(file,&shared.path,shared.dims,- Some(shared.storage),- Some(shared.metric),+ shared.storage,+ shared.metric,)?;let mut pending = mutations_since_snapshot;if log.checkpoint() != expected_checkpoint {@@ -1308,19 +1289,14 @@ fn build_writer(path: &Path,registry_key: PathBuf,dims: usize,- storage: Option<StorageKind>,- metric: Option<DistanceMetric>,+ storage: StorageKind,+ metric: DistanceMetric,) -> Result<BuiltWriter, VecDbError> {let lock = WriterLock::acquire(path)?;let mut log = if std::fs::metadata(path).is_ok() {open_existing_writer_log(path, dims, storage, metric)?} else {- match Log::create(- path,- dims,- storage.unwrap_or(StorageKind::I8),- metric.unwrap_or_default(),- ) {+ match Log::create(path, dims, storage, metric) {Ok(created) => created,Err(VecDbError::Io { source, .. }) if source.kind() == ErrorKind::AlreadyExists => {open_existing_writer_log(path, dims, storage, metric)?@@ -1469,11 +1445,16 @@ fn install_graph(shared: &Shared, graph: Graph) {}}-fn build_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {+fn build_read_only(+ path: &Path,+ dims: usize,+ storage: StorageKind,+ metric: DistanceMetric,+) -> Result<Shared, VecDbError> {let file = match File::open(path) {Ok(file) => file,Err(error) if error.kind() == ErrorKind::NotFound => {- return empty_read_only(path, dims);+ return empty_read_only(path, dims, storage, metric);}Err(source) => return Err(VecDbError::io(path, source)),};@@ -1482,9 +1463,9 @@ fn build_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {.map_err(|source| VecDbError::io(path, source))?.len();if file_len < HEADER_LEN as u64 {- return empty_read_only(path, dims);+ return empty_read_only(path, dims, storage, metric);}- let mut log = Log::open(file, path, dims, None, None)?;+ let mut log = Log::open(file, path, dims, storage, metric)?;let replayed = replay(&mut log, dims, ReplayMode::ReadOnly)?;drop(log);let ReplayedState {@@ -1506,10 +1487,15 @@ fn build_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {))}-fn empty_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {+fn empty_read_only(+ path: &Path,+ dims: usize,+ storage: StorageKind,+ metric: DistanceMetric,+) -> Result<Shared, VecDbError> {Ok(read_only_shared(path,- VectorArena::with_metric(dims, StorageKind::I8, DistanceMetric::default())?,+ VectorArena::with_metric(dims, storage, metric)?,Graph::new(),AttrTable::default(),0,@@ -1701,8 +1687,8 @@ fn ensure_finite(key: &str, vector: &[f32]) -> Result<(), VecDbError> {fn open_existing_writer_log(path: &Path,dims: usize,- storage: Option<StorageKind>,- metric: Option<DistanceMetric>,+ storage: StorageKind,+ metric: DistanceMetric,) -> Result<Log, VecDbError> {let file = open_writer_file(path, false)?;Log::open(file, path, dims, storage, metric)@@ -1782,75 +1768,119 @@ mod tests {const TOMBSTONE_RECORD_LEN: u64 = 3 + 5 + 4;#[test]- fn cosine_defaults_and_metric_mismatches_cover_live_and_disk_opens() {+ fn explicit_configuration_is_verified_on_live_and_disk_opens() {let dir = TempDir::new().unwrap();- let default_path = dir.path().join("default");- let default = VecDb::open(&default_path, 32).unwrap();- assert_eq!(default.stats().unwrap().metric, DistanceMetric::Cosine);- assert_eq!(default.stats().unwrap().storage, StorageKind::I8);for storage in [StorageKind::F32, StorageKind::I8] {for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {let path = dir.path().join(format!("{storage}-{metric}"));- let other = match metric {+ let other_storage = match storage {+ StorageKind::F32 => StorageKind::I8,+ StorageKind::I8 => StorageKind::F32,+ };+ let other_metric = match metric {DistanceMetric::Cosine => DistanceMetric::InnerProduct,DistanceMetric::InnerProduct => DistanceMetric::Cosine,};- let db = VecDb::open_with(&path, 32, Some(storage), Some(metric)).unwrap();- assert_eq!(db.stats().unwrap().metric, metric);- assert_eq!(- VecDb::open(&path, 32).unwrap().stats().unwrap().metric,- metric- );- assert_eq!(- VecDb::open_with_metric(&path, 32, metric)- .unwrap()- .stats()- .unwrap()- .metric,- metric- );- assert_eq!(- VecDb::open_read_only(&path, 32)- .unwrap()- .stats()- .unwrap()- .metric,- metric- );- assert!(matches!(- VecDb::open_with_metric(&path, 32, other),- Err(VecDbError::MetricMismatch { expected, actual })- if expected == other && actual == metric- ));+ let db = VecDb::open(&path, 32, storage, metric).unwrap();+ db.add("kept", &seeded_unit_vector(1, 32)).unwrap();+ db.flush().unwrap();+ let verify = |live: Option<&VecDb>| {+ let before = fs::read(&path).unwrap();+ let snapshot_before = fs::read(snapshot_path(&path)).unwrap();+ for open in [VecDb::open, VecDb::open_read_only] {+ assert!(matches!(open(&path, 32, other_storage, metric),+ Err(VecDbError::StorageMismatch { expected, actual })+ if expected == other_storage && actual == storage));+ assert!(matches!(open(&path, 32, storage, other_metric),+ Err(VecDbError::MetricMismatch { expected, actual })+ if expected == other_metric && actual == metric));+ assert!(matches!(+ open(&path, 64, storage, metric),+ Err(VecDbError::DimensionMismatch {+ expected: 64,+ actual: 32+ })+ ));+ assert_eq!(fs::read(&path).unwrap(), before);+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_before);+ let reopened = open(&path, 32, storage, metric).unwrap();+ assert_eq!(reopened.stats().unwrap().storage, storage);+ assert_eq!(reopened.stats().unwrap().metric, metric);+ assert!(reopened.contains("kept"));+ if let Some(live) = live {+ assert!(Arc::ptr_eq(&live.shared, &reopened.shared));+ }+ }+ };+ verify(Some(&db));drop(db);- let before = std::fs::read(&path).unwrap();- assert!(matches!(- VecDb::open_with_metric(&path, 32, other),- Err(VecDbError::MetricMismatch { .. })- ));- assert_eq!(std::fs::read(&path).unwrap(), before);- assert_eq!(- VecDb::open_read_only(&path, 32)- .unwrap()- .stats()- .unwrap()- .metric,- metric- );- assert_eq!(- VecDb::open(&path, 32).unwrap().stats().unwrap().metric,- metric- );+ verify(None);}}}+ #[test]+ fn missing_and_incomplete_indexes_use_the_explicit_configuration() {+ let dir = TempDir::new().unwrap();+ for storage in [StorageKind::F32, StorageKind::I8] {+ for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {+ for incomplete in [false, true] {+ let path = dir.path().join(format!("{storage}-{metric}-{incomplete}"));+ let dims = storage.lane_width();+ if incomplete {+ fs::write(&path, [1, 2, 3]).unwrap();+ }+ let reader = VecDb::open_read_only(&path, dims, storage, metric).unwrap();+ let stats = reader.stats().unwrap();+ assert_eq!(stats.dims, dims);+ assert_eq!(stats.storage, storage);+ assert_eq!(stats.metric, metric);+ assert!(reader.is_empty());+ assert!(matches!(+ reader.add("no", &vec![1.0; dims]),+ Err(VecDbError::ReadOnly)+ ));+ if incomplete {+ assert_eq!(fs::read(&path).unwrap(), [1, 2, 3]);+ } else {+ assert!(!path.exists());+ }+ drop(reader);+ let writer = VecDb::open(&path, dims, storage, metric).unwrap();+ assert_eq!(writer.stats().unwrap().storage, storage);+ assert_eq!(writer.stats().unwrap().metric, metric);+ writer.add("kept", &vec![1.0; dims]).unwrap();+ drop(writer);+ let reopened = VecDb::open_read_only(&path, dims, storage, metric).unwrap();+ assert!(reopened.contains("kept"));+ assert_eq!(reopened.stats().unwrap().metric, metric);+ }+ }+ }+ for open in [VecDb::open, VecDb::open_read_only] {+ let path = dir.path().join("invalid");+ assert!(matches!(+ open(&path, 8, StorageKind::I8, DistanceMetric::Cosine),+ Err(VecDbError::InvalidDimensions {+ dims: 8,+ storage: StorageKind::I8+ })+ ));+ assert!(!path.exists());+ }+ }+#[test]fn cosine_scores_unnormalized_vectors_in_every_search_path() {let dir = TempDir::new().unwrap();for storage in [StorageKind::F32, StorageKind::I8] {- let db = VecDb::open_with_storage(&dir.path().join(storage.to_string()), 32, storage)- .unwrap();+ let db = VecDb::open(+ &dir.path().join(storage.to_string()),+ 32,+ storage,+ DistanceMetric::Cosine,+ )+ .unwrap();let vector = |x, y| {let mut values = vec![0.0; 32];values[0] = x;@@ -1930,7 +1960,7 @@ mod tests {for storage in [StorageKind::F32, StorageKind::I8] {for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {let path = dir.path().join(format!("{storage}-{metric}"));- let db = VecDb::open_with(&path, 32, Some(storage), Some(metric)).unwrap();+ let db = VecDb::open(&path, 32, storage, metric).unwrap();let mut vector = vec![0.0; 32];vector[0] = 4.0;db.add("removed", &vector).unwrap();@@ -1971,15 +2001,15 @@ mod tests {if !snapshot {remove_snapshot(&path).unwrap();}- assert_score(&VecDb::open_read_only(&path, 32).unwrap());- assert_score(&VecDb::open(&path, 32).unwrap());+ assert_score(&VecDb::open_read_only(&path, 32, storage, metric).unwrap());+ assert_score(&VecDb::open(&path, 32, storage, metric).unwrap());}- let db = VecDb::open(&path, 32).unwrap();+ let db = VecDb::open(&path, 32, storage, metric).unwrap();db.reset().unwrap();db.add("kept", &vector).unwrap();assert_score(&db);drop(db);- assert_score(&VecDb::open(&path, 32).unwrap());+ assert_score(&VecDb::open(&path, 32, storage, metric).unwrap());}}}@@ -1987,7 +2017,13 @@ mod tests {#[test]fn cosine_f32_handles_extreme_finite_magnitudes() {let dir = TempDir::new().unwrap();- let db = VecDb::open_with_storage(&dir.path().join("db"), 8, StorageKind::F32).unwrap();+ let db = VecDb::open(+ &dir.path().join("db"),+ 8,+ StorageKind::F32,+ DistanceMetric::Cosine,+ )+ .unwrap();for magnitude in [f32::MAX, f32::MIN_POSITIVE, f32::from_bits(1)] {let vector = vec![magnitude; 8];db.add("same", &vector).unwrap();@@ -2057,7 +2093,7 @@ mod tests {}fn open_writer(path: &Path) -> VecDb {- VecDb::open_with_storage(path, DIMS, StorageKind::F32).unwrap()+ VecDb::open(path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap()}fn generation_of(path: &Path) -> [u8; 16] {@@ -2195,7 +2231,8 @@ mod tests {db.flush().unwrap();assert_eq!(db.stats().unwrap().records_since_snapshot, 0);drop(db);- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();let reopened = open_writer(&path);let mut index = 0;for query in &queries {@@ -2350,11 +2387,11 @@ mod tests {let log_bytes = fs::read(path).unwrap();let snapshot_bytes = snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap());assert!(matches!(- VecDb::open(path, DIMS),+ VecDb::open(path, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Corrupt(_))));assert!(matches!(- VecDb::open_read_only(path, DIMS),+ VecDb::open_read_only(path, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Corrupt(_))));assert_eq!(fs::read(path).unwrap(), log_bytes);@@ -2391,10 +2428,22 @@ mod tests {bulk_add(&db, &bulk_entries(0, 2, 500)).unwrap();db.flush().unwrap();drop(db);- let mut captured_for_read_only =- Log::open(File::open(&path).unwrap(), &path, DIMS, None, None).unwrap();- let mut captured_for_writer =- Log::open(File::open(&path).unwrap(), &path, DIMS, None, None).unwrap();+ let mut captured_for_read_only = Log::open(+ File::open(&path).unwrap(),+ &path,+ DIMS,+ StorageKind::F32,+ DistanceMetric::Cosine,+ )+ .unwrap();+ let mut captured_for_writer = Log::open(+ File::open(&path).unwrap(),+ &path,+ DIMS,+ StorageKind::F32,+ DistanceMetric::Cosine,+ )+ .unwrap();let stale_end = captured_for_read_only.current_end_offset();let racer = open_writer(&path);bulk_add(&racer, &bulk_entries(2, 2, 500)).unwrap();@@ -2429,8 +2478,14 @@ mod tests {bulk_add(&db, &bulk_entries(0, 2, 700)).unwrap();db.flush().unwrap();drop(db);- let mut captured_for_read_only =- Log::open(File::open(&path).unwrap(), &path, DIMS, None, None).unwrap();+ let mut captured_for_read_only = Log::open(+ File::open(&path).unwrap(),+ &path,+ DIMS,+ StorageKind::F32,+ DistanceMetric::Cosine,+ )+ .unwrap();let stale_end = captured_for_read_only.current_end_offset();let racer = open_writer(&path);bulk_add(&racer, &bulk_entries(2, 2, 700)).unwrap();@@ -2583,7 +2638,8 @@ mod tests {let last_start = bytes.len() - ADD_RECORD_LEN as usize;bytes[last_start + 10] ^= 0x01;fs::write(&path, &bytes).unwrap();- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert_eq!(read_only.len(), 3);assert!(!read_only.contains("key-3"));drop(read_only);@@ -2612,11 +2668,11 @@ mod tests {fs::write(&path, &bytes).unwrap();let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();assert!(matches!(- VecDb::open(&path, DIMS),+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Corrupt(message)) if message.contains("version")));assert!(matches!(- VecDb::open_read_only(&path, DIMS),+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Corrupt(message)) if message.contains("version")));assert_eq!(fs::read(&path).unwrap(), bytes);@@ -2704,7 +2760,7 @@ mod tests {let path = dir.path().join("db");let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 3, 120)).unwrap();- let joined = VecDb::open(&path, DIMS).unwrap();+ let joined = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&db.shared, &joined.shared));assert_eq!(joined.len(), 3);joined.add("joined", &seeded_unit_vector(9, DIMS)).unwrap();@@ -2725,7 +2781,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let first = open_writer(&path);- let second = VecDb::open(&path, DIMS).unwrap();+ let second = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&first.shared, &second.shared));let via_first = seeded_unit_vector(11, DIMS);let via_second = seeded_unit_vector(22, DIMS);@@ -2740,7 +2796,7 @@ mod tests {assert!(first.contains("via-second"));assert_eq!(first.len(), 2);assert!(matches!(- VecDb::open(&path, DIMS + 8),+ VecDb::open(&path, DIMS + 8, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::DimensionMismatch {expected,actual@@ -2758,12 +2814,13 @@ mod tests {fs::hard_link(&path, &alias).unwrap();let bytes = fs::read(&path).unwrap();assert!(matches!(- VecDb::open(&alias, DIMS),+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Locked(_))));assert_eq!(fs::read(&path).unwrap(), bytes);let second = open_writer(&path);- let reader = VecDb::open_read_only(&path, DIMS).unwrap();+ let reader =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();first.add("first", &basis_vector(0)).unwrap();second.add("second", &basis_vector(1)).unwrap();assert!(reader.contains("first"));@@ -2789,12 +2846,16 @@ mod tests {fs::hard_link(&path, &alias).unwrap();for name in [&path, &alias] {assert!(matches!(- VecDb::open(name, DIMS),+ VecDb::open(name, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Locked(_))));}assert_eq!(fs::read(&path).unwrap(), partial);- assert!(VecDb::open_read_only(&alias, I8_DIMS).unwrap().is_empty());+ assert!(+ VecDb::open_read_only(&alias, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine)+ .unwrap()+ .is_empty()+ );drop(holder);let writer = open_writer(&alias);assert!(writer.is_empty());@@ -2813,9 +2874,11 @@ mod tests {.as_str(){"blocked" => {- let result = VecDb::open(&path, DIMS);+ let result = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine);assert!(matches!(result, Err(VecDbError::Locked(_))));- let reader = VecDb::open_read_only(&path, DIMS).unwrap();+ let reader =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ .unwrap();assert!(reader.contains("first"));}"write" => {@@ -2860,7 +2923,8 @@ mod tests {assert_eq!(first.len(), 1);drop(first);run_hard_link_writer_process(&alias, "write");- let reader = VecDb::open_read_only(&alias, DIMS).unwrap();+ let reader =+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert_eq!(reader.len(), 2);assert!(reader.contains("first"));assert!(reader.contains("next"));@@ -2890,10 +2954,12 @@ mod tests {}assert_eq!(fs::metadata(&alias).unwrap().nlink(), 1);assert!(matches!(- VecDb::open(&alias, DIMS),+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Locked(_))));- let reader = VecDb::open_read_only(&alias, DIMS).unwrap();+ let reader =+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ .unwrap();assert!(reader.contains("first"));drop(reader);run_hard_link_writer_process(&alias, "blocked");@@ -2918,14 +2984,15 @@ mod tests {let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 10, 800)).unwrap();fs::hard_link(&path, &alias).unwrap();- let observer = VecDb::open_read_only(&alias, DIMS).unwrap();+ let observer =+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert_eq!(observer.len(), 10);compact(&db.shared, &mut db.shared.writer_half()).unwrap();assert_eq!(db.len(), 10);fs::remove_file(&alias).unwrap();fs::hard_link(&path, &alias).unwrap();assert!(matches!(- VecDb::open(&alias, DIMS),+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Locked(_))));db.reset().unwrap();@@ -2933,7 +3000,7 @@ mod tests {fs::remove_file(&alias).unwrap();fs::hard_link(&path, &alias).unwrap();assert!(matches!(- VecDb::open(&alias, DIMS),+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Locked(_))));assert_eq!(db.len(), 1);@@ -2948,7 +3015,7 @@ mod tests {let plain = subdir.join("db");let dotted = subdir.join("..").join("indexes").join("db");let first = open_writer(&plain);- let second = VecDb::open(&dotted, DIMS).unwrap();+ let second = VecDb::open(&dotted, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&first.shared, &second.shared));let vector = seeded_unit_vector(5, DIMS);first.add("shared", &vector).unwrap();@@ -2961,7 +3028,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let first = open_writer(&path);- let second = VecDb::open(&path, DIMS).unwrap();+ let second = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();first.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();drop(first);assert!(second.contains("kept"));@@ -2975,7 +3042,7 @@ mod tests {assert_eq!(reopened.len(), 2);assert!(reopened.contains("kept"));assert!(reopened.contains("still-open"));- let again = VecDb::open(&path, DIMS).unwrap();+ let again = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&reopened.shared, &again.shared));}@@ -2995,7 +3062,8 @@ mod tests {thread::spawn(move || {barrier.wait();for _ in 0..CYCLES {- let db = VecDb::open(&path, DIMS).unwrap();+ let db = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ .unwrap();assert!(db.contains("seed"));drop(db);}@@ -3139,7 +3207,9 @@ mod tests {}let opener = {let path = path.clone();- thread::spawn(move || VecDb::open_with_storage(&path, DIMS, StorageKind::F32))+ thread::spawn(move || {+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ })};thread::sleep(Duration::from_millis(5));drop(frozen);@@ -3210,7 +3280,7 @@ mod tests {fs::create_dir(&path).unwrap();let key = registry_key_for(&path).unwrap();assert!(matches!(- VecDb::open(&path, DIMS),+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::Io { .. })));assert!(!registry().contains_key(&key));@@ -3262,7 +3332,8 @@ mod tests {let path = dir.path().join("db");let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 3, 130)).unwrap();- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&db.shared, &read_only.shared));assert_eq!(read_only.len(), 3);assert!(read_only.contains("key-1"));@@ -3285,7 +3356,8 @@ mod tests {));assert!(matches!(read_only.flush(), Err(VecDbError::ReadOnly)));assert!(matches!(read_only.reset(), Err(VecDbError::ReadOnly)));- let another = VecDb::open_read_only(&path, DIMS).unwrap();+ let another =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(matches!(another.delete(), Err(VecDbError::ReadOnly)));assert!(db.contains("later"));assert!(path.exists());@@ -3296,7 +3368,8 @@ mod tests {const DIMS: usize = I8_DIMS;let dir = TempDir::new().unwrap();let path = dir.path().join("absent");- let db = VecDb::open_read_only(&path, DIMS).unwrap();+ let db =+ VecDb::open_read_only(&path, DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();assert_eq!(db.len(), 0);assert!(db.is_empty());assert!(!db.contains("anything"));@@ -3331,7 +3404,8 @@ mod tests {let db = open_writer(&path);db.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();drop(db);- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();let vector = seeded_unit_vector(4, DIMS);assert!(matches!(read_only.add("x", &vector),@@ -3367,7 +3441,8 @@ mod tests {let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 100, 140)).unwrap();drop(db);- let snapshot_view = VecDb::open_read_only(&path, DIMS).unwrap();+ let snapshot_view =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert_eq!(snapshot_view.len(), 100);let db = open_writer(&path);assert!(!Arc::ptr_eq(&snapshot_view.shared, &db.shared));@@ -3384,11 +3459,13 @@ mod tests {assert!(snapshot_view.contains("key-5"));assert!(!snapshot_view.contains("fresh"));assert_own_nearest(&snapshot_view, "key-5", &seeded_unit_vector(145, DIMS));- let live_view = VecDb::open_read_only(&path, DIMS).unwrap();+ let live_view =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&live_view.shared, &db.shared));assert!(live_view.contains("fresh"));drop(db);- let fresh_view = VecDb::open_read_only(&path, DIMS).unwrap();+ let fresh_view =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert_eq!(fresh_view.len(), 69);assert!(!fresh_view.contains("key-5"));assert!(fresh_view.contains("fresh"));@@ -3885,7 +3962,8 @@ mod tests {let path = dir.path().join("db");let partial = [0x45u8, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00];fs::write(&path, partial).unwrap();- let db = VecDb::open_read_only(&path, I8_DIMS).unwrap();+ let db =+ VecDb::open_read_only(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();assert!(db.is_empty());assert_eq!(db.stats().unwrap().log_bytes, 0);assert_eq!(fs::read(&path).unwrap(), partial);@@ -3914,7 +3992,9 @@ mod tests {assert!(!temp.exists());assert_eq!(generation_of(&path), new_generation);assert_ne!(generation_of(&path), old_generation);- let log = open_existing_writer_log(&path, DIMS, None, None).unwrap();+ let log =+ open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::InnerProduct)+ .unwrap();assert_eq!(log.generation(), new_generation);drop(log);assert!(matches!(@@ -3984,7 +4064,9 @@ mod tests {let WriterState { lock, log, .. } = take_writer(&db);let checkpoint = log.checkpoint();drop(log);- let mut foreign = open_existing_writer_log(&path, DIMS, None, None).unwrap();+ let mut foreign =+ open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ .unwrap();foreign.append(&[LogEntry::Add {key: "foreign",@@ -4272,7 +4354,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let first = open_writer(&path);- let second = VecDb::open(&path, DIMS).unwrap();+ let second = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();bulk_add(&first, &bulk_entries(0, 10, 220)).unwrap();assert_eq!(second.len(), 10);second.reset().unwrap();@@ -4448,8 +4530,9 @@ mod tests {let first = open_writer(&path);bulk_add(&first, &bulk_entries(0, 5, 400)).unwrap();first.flush().unwrap();- let survivor = VecDb::open(&path, DIMS).unwrap();- let observer = VecDb::open_read_only(&path, DIMS).unwrap();+ let survivor = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();+ let observer =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();first.delete().unwrap();assert!(!path.exists());assert!(!snapshot_exists(&path));@@ -4506,7 +4589,8 @@ mod tests {let barrier = Arc::new(Barrier::new(READERS + 2));let mut readers = Vec::new();for reader_index in 0..READERS {- let handle = VecDb::open(&path, DIMS).unwrap();+ let handle =+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();let stop = Arc::clone(&stop);let barrier = Arc::clone(&barrier);let expected = Arc::clone(&expected);@@ -4529,7 +4613,8 @@ mod tests {}));}let flusher = {- let handle = VecDb::open(&path, DIMS).unwrap();+ let handle =+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();let stop = Arc::clone(&stop);let barrier = Arc::clone(&barrier);thread::spawn(move || {@@ -4931,7 +5016,8 @@ mod tests {assert_eq!(db.len(), 1);assert!(!db.contains("x"));assert_eq!(db.stats().unwrap().log_bytes, log_bytes);- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(matches!(read_only.bulk_add(&keys, &vectors),Err(VecDbError::LengthMismatch { .. })@@ -4946,8 +5032,9 @@ mod tests {bulk_add(&first, &bulk_entries(0, 5, 950)).unwrap();first.flush().unwrap();fs::write(temp_sibling(&path), [7u8; 10]).unwrap();- let survivor = VecDb::open(&path, DIMS).unwrap();- let observer = VecDb::open_read_only(&path, DIMS).unwrap();+ let survivor = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();+ let observer =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();VecDb::purge(&path).unwrap();assert!(!path.exists());assert!(!snapshot_exists(&path));@@ -5051,7 +5138,8 @@ mod tests {let alias = dir.path().join("alias");let via_path = open_writer(&path);std::os::unix::fs::symlink(&path, &alias).unwrap();- let via_alias = VecDb::open(&alias, DIMS).unwrap();+ let via_alias =+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();via_path.add("through-real", &seeded_unit_vector(5, DIMS)).unwrap();@@ -5063,7 +5151,8 @@ mod tests {assert_eq!(via_path.len(), 2);assert!(lock_path(&path).exists());assert!(!lock_path(&alias).exists());- let read_alias = VecDb::open_read_only(&alias, DIMS).unwrap();+ let read_alias =+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();via_path.add("late", &seeded_unit_vector(7, DIMS)).unwrap();assert!(read_alias.contains("late"));}@@ -5208,7 +5297,9 @@ mod tests {assert!(!snapshot_exists(&path));let opener = {let path = path.clone();- thread::spawn(move || VecDb::open(&path, DIMS).unwrap())+ thread::spawn(move || {+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap()+ })};let key = registry_key_for(&path).unwrap();let deadline = Instant::now() + Duration::from_secs(60);@@ -5217,7 +5308,8 @@ mod tests {let registered =existing_path_slot(&key).is_some_and(|slot| lock_slot(&slot).holds_live());if registered {- break VecDb::open_read_only(&path, DIMS).unwrap();+ break VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ .unwrap();}thread::yield_now();};@@ -5236,7 +5328,7 @@ mod tests {expected_stored);assert!(!opener.is_finished());- let writer = VecDb::open(&path, DIMS).unwrap();+ let writer = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&writer.shared, &joined.shared));let adder = {let vector = replacement.clone();@@ -5391,7 +5483,8 @@ mod tests {assert_eq!(db.get_attrs("removed"), None);assert_eq!(db.get_attrs("recycled").unwrap(), sample_attrs(22));};- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();verify(&read_only);drop(read_only);let reopened = open_writer(&path);@@ -5531,7 +5624,8 @@ mod tests {vec![Some(sample_attrs(7)), None, None, None]);assert_eq!(db.get_attrs("x").unwrap(), sample_attrs(7));- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(matches!(read_only.bulk_add_with_attrs(&keys, &vectors, &[None]),Err(VecDbError::LengthMismatch { .. })@@ -5587,7 +5681,7 @@ mod tests {}fn open_i8(path: &Path) -> VecDb {- VecDb::open_with_storage(path, I8_DIMS, StorageKind::I8).unwrap()+ VecDb::open(path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap()}fn stored_vectors(db: &VecDb) -> Vec<(String, StoredVector)> {@@ -5618,7 +5712,14 @@ mod tests {}fn logged_add_records(path: &Path, dims: usize) -> Vec<(String, StoredVector)> {- let mut log = Log::open(File::open(path).unwrap(), path, dims, None, None).unwrap();+ let mut log = Log::open(+ File::open(path).unwrap(),+ path,+ dims,+ StorageKind::I8,+ DistanceMetric::Cosine,+ )+ .unwrap();let mut scanner = log.scan().unwrap();let mut records = Vec::new();while let Some((record, _)) = scanner.next_record().unwrap() {@@ -5634,10 +5735,10 @@ mod tests {let dir = TempDir::new().unwrap();let narrow = dir.path().join("narrow");let wide = dir.path().join("wide");- let live = VecDb::open_with_storage(&narrow, 8, StorageKind::F32).unwrap();+ let live = VecDb::open(&narrow, 8, StorageKind::F32, DistanceMetric::Cosine).unwrap();live.add("n", &seeded_unit_vector(1, 8)).unwrap();assert!(matches!(- VecDb::open_with_storage(&narrow, 8, StorageKind::I8),+ VecDb::open(&narrow, 8, StorageKind::I8, DistanceMetric::Cosine),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32@@ -5645,22 +5746,22 @@ mod tests {));drop(live);assert!(matches!(- VecDb::open_with_storage(&narrow, 8, StorageKind::I8),+ VecDb::open(&narrow, 8, StorageKind::I8, DistanceMetric::Cosine),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));- drop(VecDb::open_with_storage(&wide, I8_DIMS, StorageKind::F32).unwrap());+ drop(VecDb::open(&wide, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap());assert!(matches!(- VecDb::open_with_storage(&wide, 8, StorageKind::I8),+ VecDb::open(&wide, 8, StorageKind::I8, DistanceMetric::Cosine),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));assert!(matches!(- VecDb::open(&wide, 8),+ VecDb::open(&wide, 8, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::DimensionMismatch {expected: 8,actual: I8_DIMS@@ -5668,14 +5769,24 @@ mod tests {));drop(open_i8(&dir.path().join("i8")));assert!(matches!(- VecDb::open_read_only(&dir.path().join("i8"), 8),+ VecDb::open_read_only(+ &dir.path().join("i8"),+ 8,+ StorageKind::I8,+ DistanceMetric::Cosine+ ),Err(VecDbError::DimensionMismatch {expected: 8,actual: I8_DIMS})));assert!(matches!(- VecDb::open_with_storage(&dir.path().join("fresh"), 8, StorageKind::I8),+ VecDb::open(+ &dir.path().join("fresh"),+ 8,+ StorageKind::I8,+ DistanceMetric::Cosine+ ),Err(VecDbError::InvalidDimensions {dims: 8,storage: StorageKind::I8@@ -5685,54 +5796,60 @@ mod tests {}#[test]- fn storage_kind_is_verified_on_every_open_path_and_auto_detected_otherwise() {+ fn storage_kind_is_verified_on_every_open_path() {let dir = TempDir::new().unwrap();let f32_path = dir.path().join("f32");let i8_path = dir.path().join("i8");- let f32_db = VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::F32).unwrap();+ let f32_db =+ VecDb::open(&f32_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();f32_db.add("f", &seeded_unit_vector(1, I8_DIMS)).unwrap();assert_eq!(f32_db.stats().unwrap().storage, StorageKind::F32);assert!(matches!(- VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::I8),+ VecDb::open(&f32_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));- let verified = VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::F32).unwrap();+ let verified =+ VecDb::open(&f32_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&verified.shared, &f32_db.shared));drop(verified);drop(f32_db);assert!(matches!(- VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::I8),+ VecDb::open(&f32_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));assert_eq!(- VecDb::open(&f32_path, I8_DIMS)+ VecDb::open(&f32_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap().stats().unwrap().storage,StorageKind::F32);- let i8_db = VecDb::open(&i8_path, I8_DIMS).unwrap();+ let i8_db =+ VecDb::open(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();let vector = seeded_unit_vector(2, I8_DIMS);i8_db.add("i", &vector).unwrap();assert_eq!(i8_db.stats().unwrap().storage, StorageKind::I8);- let joined = VecDb::open(&i8_path, I8_DIMS).unwrap();+ let joined =+ VecDb::open(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();assert!(Arc::ptr_eq(&joined.shared, &i8_db.shared));assert_eq!(joined.stats().unwrap().storage, StorageKind::I8);assert!(matches!(- VecDb::open_with_storage(&i8_path, I8_DIMS, StorageKind::F32),+ VecDb::open(&i8_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::StorageMismatch {expected: StorageKind::F32,actual: StorageKind::I8})));- let read_alias = VecDb::open_read_only(&i8_path, I8_DIMS).unwrap();+ let read_alias =+ VecDb::open_read_only(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine)+ .unwrap();assert!(Arc::ptr_eq(&read_alias.shared, &i8_db.shared));assert_eq!(read_alias.stats().unwrap().storage, StorageKind::I8);i8_db.flush().unwrap();@@ -5740,36 +5857,57 @@ mod tests {drop(joined);drop(i8_db);assert_eq!(VecDb::open_cost(&i8_path), OpenCost::Ready);- let standalone = VecDb::open_read_only(&i8_path, I8_DIMS).unwrap();+ let standalone =+ VecDb::open_read_only(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine)+ .unwrap();assert_eq!(standalone.stats().unwrap().storage, StorageKind::I8);assert!(standalone.contains("i"));assert_own_nearest_i8(&standalone, "i", &vector);drop(standalone);assert!(matches!(- VecDb::open_with_storage(&i8_path, I8_DIMS, StorageKind::F32),+ VecDb::open(&i8_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine),Err(VecDbError::StorageMismatch {expected: StorageKind::F32,actual: StorageKind::I8})));- let detected = VecDb::open(&i8_path, I8_DIMS).unwrap();+ let detected =+ VecDb::open(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();assert_eq!(detected.stats().unwrap().storage, StorageKind::I8);assert!(detected.contains("i"));assert_own_nearest_i8(&detected, "i", &vector);assert!(matches!(- VecDb::open_with_storage(&dir.path().join("narrow"), 16, StorageKind::I8),+ VecDb::open(+ &dir.path().join("narrow"),+ 16,+ StorageKind::I8,+ DistanceMetric::Cosine+ ),Err(VecDbError::InvalidDimensions {dims: 16,storage: StorageKind::I8})));- assert!(VecDb::open_with_storage(&dir.path().join("narrow"), 16, StorageKind::F32).is_ok());+ assert!(+ VecDb::open(+ &dir.path().join("narrow"),+ 16,+ StorageKind::F32,+ DistanceMetric::Cosine+ )+ .is_ok()+ );assert_eq!(- VecDb::open_read_only(&dir.path().join("absent"), I8_DIMS)- .unwrap()- .stats()- .unwrap()- .storage,+ VecDb::open_read_only(+ &dir.path().join("absent"),+ I8_DIMS,+ StorageKind::I8,+ DistanceMetric::Cosine+ )+ .unwrap()+ .stats()+ .unwrap()+ .storage,StorageKind::I8);}@@ -5900,14 +6038,16 @@ mod tests {stored_after);};- let read_only = VecDb::open_read_only(&path, I8_DIMS).unwrap();+ let read_only =+ VecDb::open_read_only(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();verify(&read_only);drop(read_only);- let with_snapshot = VecDb::open(&path, I8_DIMS).unwrap();+ let with_snapshot =+ VecDb::open(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();verify(&with_snapshot);drop(with_snapshot);fs::remove_file(snapshot_path(&path)).unwrap();- let rebuilt = VecDb::open(&path, I8_DIMS).unwrap();+ let rebuilt = VecDb::open(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();verify(&rebuilt);let from_copy = open_i8(©_path);verify(&from_copy);@@ -5940,12 +6080,24 @@ mod tests {db.flush().unwrap();}{- let db = VecDb::open(&split_path, I8_DIMS).unwrap();+ let db = VecDb::open(+ &split_path,+ I8_DIMS,+ StorageKind::I8,+ DistanceMetric::Cosine,+ )+ .unwrap();assert_eq!(db.stats().unwrap().storage, StorageKind::I8);db.add("tail-new", &tail_new).unwrap();db.add("key-3", &older_upsert).unwrap();}- let split = VecDb::open(&split_path, I8_DIMS).unwrap();+ let split = VecDb::open(+ &split_path,+ I8_DIMS,+ StorageKind::I8,+ DistanceMetric::Cosine,+ )+ .unwrap();let whole = open_i8(&whole_path);bulk_add(&whole, &i8_entries(0, 40, 500)).unwrap();whole.add("tail-new", &tail_new).unwrap();@@ -5969,10 +6121,20 @@ mod tests {fn i8_stats_report_the_storage_and_a_quarter_of_the_vector_memory() {let dir = TempDir::new().unwrap();let dims = 512;- let f32_db =- VecDb::open_with_storage(&dir.path().join("f32"), dims, StorageKind::F32).unwrap();- let i8_db =- VecDb::open_with_storage(&dir.path().join("i8"), dims, StorageKind::I8).unwrap();+ let f32_db = VecDb::open(+ &dir.path().join("f32"),+ dims,+ StorageKind::F32,+ DistanceMetric::Cosine,+ )+ .unwrap();+ let i8_db = VecDb::open(+ &dir.path().join("i8"),+ dims,+ StorageKind::I8,+ DistanceMetric::Cosine,+ )+ .unwrap();let entries: Vec<(String, Vec<f32>)> = (0..200u64).map(|index| (format!("key-{index}"), seeded_unit_vector(index, dims))).collect();