π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: drop the inner product metric and keep cosine
laurenspriem84c6da6000f7
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rsindex b927005996..a75b35e5f5 100644--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs@@ -18,7 +18,6 @@ pub enum RustVecDbError {UnboundedSearch { message: String },LengthMismatch { message: String },StorageMismatch { message: String },- MetricMismatch { message: String },}impl From<vecdb::VecDbError> for RustVecDbError {@@ -38,7 +37,6 @@ impl From<vecdb::VecDbError> for RustVecDbError {vecdb::VecDbError::UnboundedSearch => Self::UnboundedSearch { message },vecdb::VecDbError::LengthMismatch { .. } => Self::LengthMismatch { message },vecdb::VecDbError::StorageMismatch { .. } => Self::StorageMismatch { message },- vecdb::VecDbError::MetricMismatch { .. } => Self::MetricMismatch { message },}}}@@ -63,26 +61,6 @@ fn to_api_storage(storage: vecdb::StorageKind) -> VecDbStorage {}}-#[derive(Clone, Copy, Debug)]-pub enum VecDbMetric {- InnerProduct,- Cosine,-}--fn to_engine_metric(metric: VecDbMetric) -> vecdb::DistanceMetric {- match metric {- VecDbMetric::InnerProduct => vecdb::DistanceMetric::InnerProduct,- VecDbMetric::Cosine => vecdb::DistanceMetric::Cosine,- }-}--fn to_api_metric(metric: vecdb::DistanceMetric) -> VecDbMetric {- match metric {- vecdb::DistanceMetric::InnerProduct => VecDbMetric::InnerProduct,- vecdb::DistanceMetric::Cosine => VecDbMetric::Cosine,- }-}-#[derive(Clone, Debug)]pub enum VecDbAttrValue {Str(String),@@ -146,7 +124,6 @@ pub struct VecDbStats {pub dead_count: u32,pub dims: u32,pub storage: VecDbStorage,- pub metric: VecDbMetric,pub log_bytes: u64,pub records_since_snapshot: u32,pub approximate_memory_bytes: u64,@@ -192,16 +169,10 @@ impl VecDb {file_path: String,dimensions: u32,storage: Option<VecDbStorage>,- metric: Option<VecDbMetric>,) -> Result<Self, RustVecDbError> {let path = Path::new(&file_path);let dims = dimensions as usize;- let inner = vecdb::VecDb::open(- path,- dims,- storage.map(to_engine_storage),- metric.map(to_engine_metric),- )?;+ let inner = vecdb::VecDb::open(path, dims, storage.map(to_engine_storage))?;Ok(Self { inner })}@@ -209,14 +180,12 @@ impl VecDb {file_path: String,dimensions: u32,storage: Option<VecDbStorage>,- metric: Option<VecDbMetric>,) -> Result<Self, RustVecDbError> {Ok(Self {inner: vecdb::VecDb::open_read_only(Path::new(&file_path),dimensions as usize,storage.map(to_engine_storage),- metric.map(to_engine_metric),)?,})}@@ -391,7 +360,6 @@ impl VecDb {dead_count: stats.dead_count as u32,dims: stats.dims as u32,storage: to_api_storage(stats.storage),- metric: to_api_metric(stats.metric),log_bytes: stats.log_bytes,records_since_snapshot: stats.records_since_snapshot as u32,approximate_memory_bytes: stats.approximate_memory_bytes as u64,@@ -451,56 +419,26 @@ mod tests {name.to_string()}- #[test]- fn omitted_metric_defaults_to_cosine() {- let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), 32, None, None).unwrap();- assert!(matches!(- db.get_index_stats().unwrap().metric,- VecDbMetric::Cosine- ));- drop(db);- let reader = VecDb::open_read_only(dir.db_path(), 32, None, None).unwrap();- assert!(matches!(- reader.get_index_stats().unwrap().metric,- VecDbMetric::Cosine- ));- }-#[test]fn omitted_storage_defaults_to_i8() {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), 32, None, Some(VecDbMetric::InnerProduct)).unwrap();+ let db = VecDb::new(dir.db_path(), 32, None).unwrap();assert!(matches!(db.get_index_stats().unwrap().storage,VecDbStorage::I8));- assert!(matches!(- db.get_index_stats().unwrap().metric,- VecDbMetric::InnerProduct- ));drop(db);- let reader =- VecDb::open_read_only(dir.db_path(), 32, None, Some(VecDbMetric::InnerProduct))- .unwrap();+ let reader = VecDb::open_read_only(dir.db_path(), 32, None).unwrap();assert!(matches!(reader.get_index_stats().unwrap().storage,VecDbStorage::I8));drop(reader);let dir = TestDir::create();- drop(- VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap(),- );+ drop(VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)).unwrap());for open in [VecDb::new, VecDb::open_read_only] {assert!(matches!(- open(dir.db_path(), 32, None, Some(VecDbMetric::Cosine)),+ open(dir.db_path(), 32, None),Err(RustVecDbError::StorageMismatch { .. })));}@@ -508,39 +446,24 @@ mod tests {#[test]fn read_only_verifies_explicit_configuration() {- for (storage, metric, wrong_storage, wrong_metric) in [- (- VecDbStorage::F32,- VecDbMetric::InnerProduct,- VecDbStorage::I8,- VecDbMetric::Cosine,- ),- (- VecDbStorage::I8,- VecDbMetric::Cosine,- VecDbStorage::F32,- VecDbMetric::InnerProduct,- ),+ for (storage, wrong_storage) in [+ (VecDbStorage::F32, VecDbStorage::I8),+ (VecDbStorage::I8, VecDbStorage::F32),] {let dir = TestDir::create();- let db = VecDb::new(dir.db_path(), 32, Some(storage), Some(metric)).unwrap();+ let db = VecDb::new(dir.db_path(), 32, Some(storage)).unwrap();db.add_vector(key("kept"), unit_axis(32)).unwrap();db.flush().unwrap();let verify = || {assert!(matches!(- VecDb::open_read_only(dir.db_path(), 32, Some(wrong_storage), Some(metric)),+ VecDb::open_read_only(dir.db_path(), 32, Some(wrong_storage)),Err(RustVecDbError::StorageMismatch { .. })));assert!(matches!(- VecDb::open_read_only(dir.db_path(), 32, Some(storage), Some(wrong_metric)),- Err(RustVecDbError::MetricMismatch { .. })- ));- assert!(matches!(- VecDb::open_read_only(dir.db_path(), 64, Some(storage), Some(metric)),+ VecDb::open_read_only(dir.db_path(), 64, Some(storage)),Err(RustVecDbError::DimensionMismatch { .. })));- let reader =- VecDb::open_read_only(dir.db_path(), 32, Some(storage), Some(metric)).unwrap();+ let reader = VecDb::open_read_only(dir.db_path(), 32, Some(storage)).unwrap();assert_eq!(reader.get_index_stats().unwrap().live_count, 1);assert!(matches!(reader.add_vector(key("no"), unit_axis(32)),@@ -553,79 +476,10 @@ mod tests {}}- #[test]- fn explicitly_selected_metrics_are_immutable() {- let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::Cosine),- )- .unwrap();- assert!(matches!(- db.get_index_stats().unwrap().metric,- VecDbMetric::Cosine- ));- assert!(matches!(- VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::InnerProduct)- ),- Err(RustVecDbError::MetricMismatch { .. })- ));- drop(db);- let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::InnerProduct),- )- .unwrap();- db.add_vector(key("a"), basis(0)).unwrap();- let vector: Vec<f32> = basis(0).iter().map(|value| value * 3.0).collect();- assert_eq!(- db.search(vector.clone(), Some(1), None, true, None)- .unwrap()[0]- .distance,- -2.0- );- db.flush().unwrap();- drop(db);- assert!(matches!(- VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), Some(VecDbMetric::Cosine)),- Err(RustVecDbError::MetricMismatch { message }) if message.contains("cosine") && message.contains("inner product")- ));- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::InnerProduct),- )- .unwrap();- assert!(matches!(- db.get_index_stats().unwrap().metric,- VecDbMetric::InnerProduct- ));- assert_eq!(- db.search(vector, Some(1), None, false, None).unwrap()[0].distance,- -2.0- );- }-#[test]fn add_search_stats_round_trip() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();db.add_vector(key("a"), basis(0)).unwrap();db.bulk_add_vectors(vec![key("b"), key("c")], vec![basis(1), basis(2)]).unwrap();@@ -658,13 +512,8 @@ mod tests {assert!(stats.approximate_memory_bytes > 0);db.flush().unwrap();assert_eq!(db.get_index_stats().unwrap().records_since_snapshot, 0);- let read_only = VecDb::open_read_only(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let read_only =+ VecDb::open_read_only(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);assert!(matches!(read_only.add_vector(key("x"), basis(3)),@@ -686,13 +535,7 @@ mod tests {#[test]fn bulk_add_rejects_length_mismatch() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();let error = db.bulk_add_vectors(vec![key("a")], vec![basis(0), basis(1)]).unwrap_err();@@ -703,13 +546,7 @@ mod tests {#[test]fn similarity_threshold_conversion_edges() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();db.add_vector(key("a"), basis(0)).unwrap();db.add_vector(key("b"), basis(1)).unwrap();let close = db@@ -733,13 +570,7 @@ mod tests {#[test]fn bulk_search_within_similarity_honors_per_query_thresholds() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();db.bulk_add_vectors(vec![key("a"), key("b"), key("c")],vec![basis(0), basis(1), basis(2)],@@ -805,13 +636,7 @@ mod tests {#[test]fn bulk_filtered_search_stays_query_aligned() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();db.bulk_add_vectors(vec![key("a"), key("b"), key("c")],vec![basis(0), basis(1), basis(2)],@@ -865,13 +690,7 @@ mod tests {#[test]fn bulk_search_keys_excludes_self_and_honors_the_filters() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();db.bulk_add_vectors(vec![key("a"), key("b"), key("c")],vec![basis(0), basis(1), basis(2)],@@ -909,13 +728,7 @@ mod tests {#[test]fn attrs_round_trip_through_the_api() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();let attrs = vec![VecDbAttr {name: key("model"),@@ -1007,13 +820,7 @@ mod tests {check_open_cost(dir.db_path()),VecDbOpenCost::Absent));- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();db.add_vector(key("a"), basis(0)).unwrap();assert!(matches!(check_open_cost(dir.db_path()),@@ -1035,13 +842,7 @@ mod tests {#[test]fn delete_vec_db_files_purges_with_and_without_live_instances() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();db.add_vector(key("a"), basis(0)).unwrap();db.flush().unwrap();delete_vec_db_files(dir.db_path()).unwrap();@@ -1052,13 +853,7 @@ mod tests {db.get_index_stats(),Err(RustVecDbError::Closed { .. })));- let reopened = VecDb::new(- dir.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let reopened = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();assert_eq!(reopened.get_index_stats().unwrap().live_count, 0);reopened.add_vector(key("b"), basis(1)).unwrap();drop(reopened);@@ -1078,13 +873,7 @@ mod tests {#[test]fn storage_is_chosen_at_creation_reported_in_stats_and_verified_on_reopen() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();assert!(matches!(db.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -1111,21 +900,10 @@ mod tests {assert_eq!(stats.dims, 32);assert!(matches!(stats.storage, VecDbStorage::I8));assert!(matches!(- VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine)- ),+ VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),Err(RustVecDbError::StorageMismatch { .. })));- let joined = VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let joined = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();assert!(matches!(joined.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -1133,13 +911,7 @@ mod tests {drop(joined);db.flush().unwrap();drop(db);- let read_only = VecDb::open_read_only(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();assert!(matches!(read_only.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -1147,16 +919,10 @@ mod tests {assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);drop(read_only);assert!(matches!(- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), Some(VecDbMetric::Cosine)),+ VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),Err(RustVecDbError::StorageMismatch { message }) if message.contains("f32") && message.contains("i8")));- let reopened = VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let reopened = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();assert!(matches!(reopened.get_index_stats().unwrap().storage,VecDbStorage::I8@@ -1168,51 +934,26 @@ mod tests {#[test]fn explicit_i8_storage_rejects_dims_off_the_lane_grid() {let dir = TestDir::create();- let db = VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();assert!(matches!(db.get_index_stats().unwrap().storage,VecDbStorage::I8));- let explicit = VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::I8),- Some(VecDbMetric::Cosine),- )- .unwrap();+ let explicit = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();assert!(matches!(explicit.get_index_stats().unwrap().storage,VecDbStorage::I8));assert!(matches!(- VecDb::new(- dir.db_path(),- 32,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine)- ),+ VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),Err(RustVecDbError::StorageMismatch { .. })));let other = TestDir::create();assert!(matches!(- VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::I8), Some(VecDbMetric::Cosine)),+ VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::I8)),Err(RustVecDbError::InvalidDimensions { message }) if message.contains("i8") && message.contains("32")));assert!(!PathBuf::from(other.db_path()).exists());- assert!(- VecDb::new(- other.db_path(),- DIMS,- Some(VecDbStorage::F32),- Some(VecDbMetric::Cosine)- )- .is_ok()- );+ assert!(VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::F32)).is_ok());}}diff --git a/rust/crates/vecdb/README.md b/rust/crates/vecdb/README.mdindex cc477df47b..28533afbf4 100644--- a/rust/crates/vecdb/README.md+++ b/rust/crates/vecdb/README.md@@ -1,34 +1,30 @@# VecDB index configuration-Every open requires the path and dimensions. Storage and distance metric are-optional and default to i8 and cosine. These settings are persisted at creation and-remain unchanged through reopen, compaction, and reset.+Every open requires the path and dimensions. Storage is optional and defaults to+i8. It is persisted at creation and remains unchanged through reopen, compaction,+and reset.```rust-use ente_vecdb::{DistanceMetric, StorageKind, VecDb};+use ente_vecdb::{StorageKind, VecDb};-let index = VecDb::open(path, 512, None, None)?;-let reader = VecDb::open_read_only(path, 512, None, None)?;-let f32_index = VecDb::open(other_path, 512, Some(StorageKind::F32), None)?;-let dot_index = VecDb::open(third_path, 512, None, Some(DistanceMetric::InnerProduct))?;+let index = VecDb::open(path, 512, None)?;+let reader = VecDb::open_read_only(path, 512, None)?;+let f32_index = VecDb::open(other_path, 512, Some(StorageKind::F32))?;```Both open methods validate the supplied configuration against the saved header or-an already-open instance. A mismatch returns `DimensionMismatch`, `StorageMismatch`,-or `MetricMismatch`. Omitting storage or metric requires i8 or cosine on existing-indexes too; it does not detect the saved settings. Older indexes remain readable by-specifying their original storage and `DistanceMetric::InnerProduct`.+an already-open instance. A mismatch returns `DimensionMismatch` or+`StorageMismatch`. Omitting storage requires i8 on existing indexes too; it does+not detect the saved setting. Older indexes remain readable by specifying their+original storage.A read-only open of a missing or incomplete index returns an empty view with thesupplied configuration without creating or modifying the file.-The Flutter constructor and read-only open accept optional `storage` and `metric`,-defaulting to i8 and cosine. Index stats report both settings.+The Flutter constructor and read-only open accept an optional `storage`,+defaulting to i8. Index stats report the setting.-Cosine distance is `1 - dot(a, b) / (norm(a) * norm(b))`, clamped to `[0, 2]`.-A comparison involving a zero vector has distance `1`, including two zero vectors.-Stored vectors retain their magnitudes; i8 cosine uses the quantized vectors.-Inner-product distance remains `1 - dot(a, b)`, which only ranks like cosine for-unit vectors, so an inner-product index rejects an added vector whose squared norm-is further than `1e-2` from `1` with `InvalidVector`. Cosine indexes take any-magnitude, and queries are never checked.+Cosine is the only distance: `1 - dot(a, b) / (norm(a) * norm(b))`, clamped to+`[0, 2]`. A comparison involving a zero vector has distance `1`, including two+zero vectors. Stored vectors retain their magnitudes; i8 cosine uses the+quantized vectors.diff --git a/rust/crates/vecdb/examples/vecdb_bench.rs b/rust/crates/vecdb/examples/vecdb_bench.rsindex 429206c4b5..87380e22b9 100644--- a/rust/crates/vecdb/examples/vecdb_bench.rs+++ b/rust/crates/vecdb/examples/vecdb_bench.rs@@ -7,7 +7,7 @@ use std::collections::HashSet;use std::path::{Path, PathBuf};use std::time::{Duration, Instant};-use ente_vecdb::{AttrValue, Attribute, DistanceMetric, Match, SearchParams, StorageKind, VecDb};+use ente_vecdb::{AttrValue, Attribute, Match, SearchParams, StorageKind, VecDb};const SEED: u64 = 0xE47E_0000_0000_0001;const LATENT_DIMS: usize = 24;@@ -26,7 +26,6 @@ struct Config {dims: usize,attrs: bool,storage: StorageKind,- metric: DistanceMetric,}struct BenchData {@@ -67,10 +66,9 @@ struct VecdbReport {fn main() {let config = parse_args();println!(- "vecdb bench dims={} storage={} metric={} queries={} seed={:#018x}{}",+ "vecdb bench dims={} storage={} queries={} seed={:#018x}{}",config.dims,storage_name(config.storage),- metric_name(config.metric),QUERY_COUNT,SEED,if config.attrs { " attrs=on" } else { "" }@@ -91,7 +89,6 @@ fn main() {config.dims,config.attrs,config.storage,- config.metric,temp_root.path(),);}@@ -102,7 +99,6 @@ fn parse_args() -> Config {let mut dims = DEFAULT_DIMS;let mut attrs = false;let mut storage = StorageKind::I8;- let mut metric = DistanceMetric::Cosine;let mut args = std::env::args().skip(1);while let Some(arg) = args.next() {match arg.as_str() {@@ -120,13 +116,6 @@ fn parse_args() -> Config {_ => usage_exit(),}}- "--metric" => {- metric = match required_value(args.next()).as_str() {- "cosine" => DistanceMetric::Cosine,- "ip" => DistanceMetric::InnerProduct,- _ => usage_exit(),- }- }_ => usage_exit(),}}@@ -146,7 +135,6 @@ fn parse_args() -> Config {dims,attrs,storage,- metric,}}@@ -171,18 +159,11 @@ fn parse_scales(list: &str) -> Vec<usize> {fn usage_exit() -> ! {eprintln!("usage: cargo run -p ente-vecdb --example vecdb_bench --release -- \- [--scales 10000,100000] [--dims 512] [--attrs] [--storage f32|i8] [--metric cosine|ip]"+ [--scales 10000,100000] [--dims 512] [--attrs] [--storage f32|i8]");std::process::exit(2);}-fn metric_name(metric: DistanceMetric) -> &'static str {- match metric {- DistanceMetric::Cosine => "cosine",- DistanceMetric::InnerProduct => "ip",- }-}-fn storage_name(storage: StorageKind) -> &'static str {match storage {StorageKind::F32 => "f32",@@ -190,30 +171,19 @@ fn storage_name(storage: StorageKind) -> &'static str {}}-fn run_scale(- scale: usize,- dims: usize,- attrs: bool,- storage: StorageKind,- metric: DistanceMetric,- temp_root: &Path,-) {+fn run_scale(scale: usize, dims: usize, attrs: bool, storage: StorageKind, temp_root: &Path) {let clusters = (scale / 150).max(1) as u64;println!();println!(- "=== scale {scale} dims {dims} storage {} metric {} clusters {clusters} ===",- storage_name(storage),- metric_name(metric)+ "=== scale {scale} dims {dims} storage {} clusters {clusters} ===",+ storage_name(storage));let dir = temp_root.join(format!("scale-{scale}-{dims}"));std::fs::create_dir_all(&dir).expect("create bench dir");- drop(- VecDb::open(&dir.join("bench.vecdb"), dims, Some(storage), Some(metric))- .expect("open vecdb"),- );+ drop(VecDb::open(&dir.join("bench.vecdb"), dims, Some(storage)).expect("open vecdb"));eprintln!("[scale {scale}] generating data");let data = generate_data(scale, dims, clusters);- let vecdb = run_vecdb(&data, dims, attrs, storage, metric, &dir, scale);+ let vecdb = run_vecdb(&data, dims, attrs, storage, &dir, scale);print_vecdb_report(&vecdb);let _ = std::fs::remove_dir_all(&dir);}@@ -343,12 +313,11 @@ fn run_vecdb(dims: usize,attrs: bool,storage: StorageKind,- metric: DistanceMetric,dir: &Path,scale: usize,) -> VecdbReport {let path = dir.join("bench.vecdb");- let mut db = VecDb::open(&path, dims, Some(storage), Some(metric)).expect("open vecdb");+ let mut db = VecDb::open(&path, dims, Some(storage)).expect("open vecdb");let ingest = ingest_phase(&mut db, data, attrs, scale);let searches = search_phase(&db, data, storage, scale);let stats = db.stats().expect("vecdb stats");@@ -357,7 +326,7 @@ fn run_vecdb(.map(|meta| meta.len()).unwrap_or(0);drop(db);- let (reopens, mut db) = reopen_phase(&path, dims, storage, metric, &snapshot_file, scale);+ let (reopens, mut db) = reopen_phase(&path, dims, storage, &snapshot_file, scale);let compaction = compaction_phase(&mut db, data, scale);db.delete().expect("vecdb delete");VecdbReport {@@ -561,21 +530,18 @@ fn reopen_phase(path: &Path,dims: usize,storage: StorageKind,- metric: DistanceMetric,snapshot_file: &Path,scale: usize,) -> (ReopenTimings, VecDb) {eprintln!("[scale {scale}] vecdb cold open with snapshot");let started = Instant::now();- let reopened =- VecDb::open(path, dims, Some(storage), Some(metric)).expect("vecdb reopen with snapshot");+ let reopened = VecDb::open(path, dims, Some(storage)).expect("vecdb reopen with snapshot");let open_with_snapshot = started.elapsed();drop(reopened);std::fs::remove_file(snapshot_file).expect("remove snapshot");eprintln!("[scale {scale}] vecdb cold open without snapshot (full rebuild)");let started = Instant::now();- let db = VecDb::open(path, dims, Some(storage), Some(metric))- .expect("vecdb reopen without snapshot");+ let db = VecDb::open(path, dims, Some(storage)).expect("vecdb reopen without snapshot");let open_full_rebuild = started.elapsed();(ReopenTimings {diff --git a/rust/crates/vecdb/src/arena.rs b/rust/crates/vecdb/src/arena.rsindex 77278483d9..3277b66908 100644--- a/rust/crates/vecdb/src/arena.rs+++ b/rust/crates/vecdb/src/arena.rs@@ -5,7 +5,7 @@ use super::kernel::{pack_lanes, pack_lanes_i8, pack_lanes_i8_into, pack_lanes_into, quantize_for, unpack_lanes,unpack_lanes_i8, validate_dims,};-use super::{DistanceMetric, StorageKind, VecDbError};+use super::{StorageKind, VecDbError};pub(crate) const VECTORS_PER_CHUNK: usize = 4096;pub(crate) const MAX_KEY_BYTES: usize = 256;@@ -114,7 +114,6 @@ pub(crate) struct VectorArena {dims: usize,lanes_per_vector: usize,storage: Storage,- metric: DistanceMetric,norms: Vec<f64>,keys_to_slots: HashMap<Box<str>, u32>,slots_to_keys: Vec<Box<str>>,@@ -129,16 +128,7 @@ impl VectorArena {Self::with_storage(dims, StorageKind::F32)}- #[cfg(test)]pub(crate) fn with_storage(dims: usize, storage: StorageKind) -> Result<Self, VecDbError> {- Self::with_metric(dims, storage, DistanceMetric::InnerProduct)- }-- pub(crate) fn with_metric(- dims: usize,- storage: StorageKind,- metric: DistanceMetric,- ) -> Result<Self, VecDbError> {validate_dims(dims, storage)?;let storage = match storage {StorageKind::F32 => Storage::F32 { chunks: Vec::new() },@@ -151,7 +141,6 @@ impl VectorArena {dims,lanes_per_vector: dims / storage.kind().lane_width(),storage,- metric,norms: Vec::new(),keys_to_slots: HashMap::new(),slots_to_keys: Vec::new(),@@ -161,10 +150,6 @@ impl VectorArena {})}- pub(crate) fn metric(&self) -> DistanceMetric {- self.metric- }-pub(crate) fn dims(&self) -> usize {self.dims}@@ -250,9 +235,7 @@ impl VectorArena {if slot as usize / 64 == self.alive.len() {self.alive.push(0);}- if self.metric == DistanceMetric::Cosine {- self.norms.push(0.0);- }+ self.norms.push(0.0);self.slots_to_keys.push(Box::from(key));self.keys_to_slots.insert(Box::from(key), slot);self.mark_alive(slot);@@ -317,9 +300,7 @@ impl VectorArena {}fn move_vector(&mut self, src: u32, dst: u32) {- if self.metric == DistanceMetric::Cosine {- self.norms[dst as usize] = self.norms[src as usize];- }+ self.norms[dst as usize] = self.norms[src as usize];let lanes = self.lanes_per_vector;match &mut self.storage {Storage::F32 { chunks } => move_lanes(chunks, lanes, src, dst),@@ -392,20 +373,12 @@ impl VectorArena {Ok(match quantize_for(self.storage_kind(), values) {Some(StoredVector::I8 { scale, values }) => {let lanes = pack_lanes_i8(&values);- let norm = if self.metric == DistanceMetric::Cosine {- i8_lanes_norm(scale, &lanes)- } else {- 0.0- };+ let norm = i8_lanes_norm(scale, &lanes);PackedQuery::I8 { scale, lanes, norm }}Some(StoredVector::F32(_)) | None => {let lanes = pack_lanes(values);- let norm = if self.metric == DistanceMetric::Cosine {- f32_lanes_norm(&lanes)- } else {- 0.0- };+ let norm = f32_lanes_norm(&lanes);PackedQuery::F32(lanes, norm)}})@@ -418,9 +391,6 @@ impl VectorArena {pub(crate) fn distance_to_query(&self, query: Query<'_>, slot: u32) -> f32 {match (query, self.stored_query(slot)) {(Query::F32(a, norm_a), Query::F32(b, norm_b)) => {- if self.metric == DistanceMetric::InnerProduct {- return F32Kernel::distance(a, b);- }let denominator = norm_a * norm_b;let dot = if denominator >= f64::from(f32::MIN_POSITIVE)&& denominator <= f64::from(f32::MAX)@@ -437,21 +407,16 @@ impl VectorArena {}(Query::I8 {- scale: scale_a,lanes: a,norm: norm_a,+ ..},Query::I8 {- scale: scale_b,lanes: b,norm: norm_b,+ ..},- ) => {- if self.metric == DistanceMetric::InnerProduct {- return I8Kernel::distance(a, scale_a, b, scale_b);- }- cosine_distance(f64::from(I8Kernel::dot(a, b)), norm_a * norm_b)- }+ ) => cosine_distance(f64::from(I8Kernel::dot(a, b)), norm_a * norm_b),_ => unreachable!("queries are packed by the arena that searches them"),}}@@ -481,9 +446,7 @@ impl VectorArena {unreachable!("payload kind is validated before a slot is written")}}- if self.metric == DistanceMetric::Cosine {- self.norms[slot as usize] = self.stored_norm(slot);- }+ self.norms[slot as usize] = self.stored_norm(slot);}fn mark_alive(&mut self, slot: u32) {@@ -576,8 +539,7 @@ mod tests {fn cosine_norms_match_their_scalar_references_and_agree_between_query_and_slot() {for dims in [512usize, 768] {for storage in [StorageKind::F32, StorageKind::I8] {- let mut arena =- VectorArena::with_metric(dims, storage, DistanceMetric::Cosine).unwrap();+ let mut arena = VectorArena::with_storage(dims, storage).unwrap();let vectors: Vec<Vec<f32>> = (0..16u64).map(|seed| seeded_vector(0x5E1F_0000 + seed, dims)).collect();@@ -1175,7 +1137,7 @@ mod tests {}#[test]- fn distances_use_the_kernel_metric() {+ fn distances_are_cosine() {let mut arena = VectorArena::new(16).unwrap();arena.upsert("x", &basis_vector(16, 0)).unwrap();arena.upsert("y", &basis_vector(16, 9)).unwrap();@@ -1254,21 +1216,24 @@ mod tests {}#[test]- fn i8_distances_follow_the_scaled_dot_formula() {+ fn i8_distances_follow_the_quantized_cosine_formula() {let dims = 64;let mut arena = VectorArena::with_storage(dims, StorageKind::I8).unwrap();let a = seeded_unit_vector(11, dims);let b = seeded_unit_vector(12, dims);arena.upsert("a", &a).unwrap();arena.upsert("b", &b).unwrap();- let (scale_a, values_a) = stored_i8(&arena, 0);- let (scale_b, values_b) = stored_i8(&arena, 1);- let dot: i32 = values_a- .iter()- .zip(&values_b)- .map(|(x, y)| i32::from(*x) * i32::from(*y))- .sum();- let expected = 1.0 - dot as f32 * scale_a * scale_b;+ let (_, values_a) = stored_i8(&arena, 0);+ let (_, values_b) = stored_i8(&arena, 1);+ let dot = |x: &[i8], y: &[i8]| -> f64 {+ x.iter()+ .zip(y)+ .map(|(x, y)| f64::from(*x) * f64::from(*y))+ .sum()+ };+ let norm = |values: &[i8]| dot(values, values).sqrt();+ let expected =+ (1.0 - dot(&values_a, &values_b) / (norm(&values_a) * norm(&values_b))) as f32;assert_eq!(arena.distance_between_slots(0, 1).to_bits(),expected.to_bits()@@ -1288,7 +1253,7 @@ mod tests {);let f32_distance = 1.0 - a.iter().zip(&b).map(|(x, y)| x * y).sum::<f32>();assert!((expected - f32_distance).abs() < 0.01);- assert!(arena.distance_between_slots(0, 0).abs() < 0.02);+ assert!(arena.distance_between_slots(0, 0).abs() < 1.0e-6);}#[test]@@ -1390,13 +1355,15 @@ mod tests {f32_arena.upsert(&format!("key-{index}"), &values).unwrap();i8_arena.upsert(&format!("key-{index}"), &values).unwrap();}+ let norm_bytes = f32_arena.norms.capacity() * size_of::<f64>();+ assert_eq!(i8_arena.norms.capacity() * size_of::<f64>(), norm_bytes);assert_eq!(f32_arena.vector_memory_bytes(),- VECTORS_PER_CHUNK * dims * size_of::<f32>()+ VECTORS_PER_CHUNK * dims * size_of::<f32>() + norm_bytes);assert_eq!(i8_arena.vector_memory_bytes(),- VECTORS_PER_CHUNK * dims + 100 * size_of::<f32>()+ VECTORS_PER_CHUNK * dims + 100 * size_of::<f32>() + norm_bytes);}}diff --git a/rust/crates/vecdb/src/graph.rs b/rust/crates/vecdb/src/graph.rsindex de57bccf48..9218739fad 100644--- a/rust/crates/vecdb/src/graph.rs+++ b/rust/crates/vecdb/src/graph.rs@@ -1012,9 +1012,9 @@ mod tests {use std::f32::consts::FRAC_1_SQRT_2;use std::sync::LazyLock;+ use super::super::StorageKind;use super::super::arena::UpsertOutcome;use super::super::test_support::{assert_identical_graphs, stale_downward_edge_exists};- use super::super::{DistanceMetric, StorageKind};use super::*;const FIXTURE_DIMS: usize = 16;@@ -1160,23 +1160,16 @@ mod tests {const JITTER_SEED: u64 = 0x71E7_0000;fn build_duplicate_groups(groups: usize, per_group: usize) -> (VectorArena, Graph) {- build_duplicate_groups_with(- 16,- StorageKind::F32,- DistanceMetric::InnerProduct,- groups,- per_group,- )+ build_duplicate_groups_with(16, StorageKind::F32, groups, per_group)}fn build_duplicate_groups_with(dims: usize,storage: StorageKind,- metric: DistanceMetric,groups: usize,per_group: usize,) -> (VectorArena, Graph) {- build_groups(dims, storage, metric, groups, per_group, |group, _| {+ build_groups(dims, storage, groups, per_group, |group, _| {group_center(group, dims)})}@@ -1184,12 +1177,11 @@ mod tests {fn build_groups(dims: usize,storage: StorageKind,- metric: DistanceMetric,groups: usize,per_group: usize,member: impl Fn(usize, usize) -> Vec<f32>,) -> (VectorArena, Graph) {- let mut arena = VectorArena::with_metric(dims, storage, metric).unwrap();+ let mut arena = VectorArena::with_storage(dims, storage).unwrap();for index in 0..groups * per_group {let vector = member(index / per_group, index % per_group);arena.upsert(&format!("key-{index}"), &vector).unwrap();@@ -1202,10 +1194,6 @@ mod tests {seeded_unit_vector(DUPLICATE_GROUP_SEED + group as u64, dims)}- fn scaled(values: &[f32], factor: f32) -> Vec<f32> {- values.iter().map(|value| value * factor).collect()- }-fn peaky_unit_vector(group: usize, dims: usize) -> Vec<f32> {let mut state = PEAKY_GROUP_SEED + group as u64;let mut values: Vec<f32> = (0..dims)@@ -1644,39 +1632,9 @@ mod tests {#[test]fn duplicate_groups_near_the_neighbor_cap_stay_whole_at_production_dims() {for storage in [StorageKind::F32, StorageKind::I8] {- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {- for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 100)] {- let (arena, graph) =- build_duplicate_groups_with(512, storage, metric, groups, per_group);- let query = arena.pack_query(&group_center(0, 512)).unwrap();- let found = search(- &graph,- &arena,- &query,- ¶ms(Some(limit), None, false),- None,- );- assert_eq!(found.len(), limit);- assert_sorted(&found);- assert_leading_group(&arena, &found, per_group, 0, per_group);- }- }- }- }-- #[test]- fn duplicate_groups_of_peaky_vectors_stay_whole_in_int8() {- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {- for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 10), (4, 80, 80)] {- let (arena, graph) = build_groups(- 512,- StorageKind::I8,- metric,- groups,- per_group,- |group, _| peaky_unit_vector(group, 512),- );- let query = arena.pack_query(&peaky_unit_vector(0, 512)).unwrap();+ for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 100)] {+ let (arena, graph) = build_duplicate_groups_with(512, storage, groups, per_group);+ let query = arena.pack_query(&group_center(0, 512)).unwrap();let found = search(&graph,&arena,@@ -1686,37 +1644,29 @@ mod tests {);assert_eq!(found.len(), limit);assert_sorted(&found);- assert_leading_group(&arena, &found, per_group.min(limit), 0, per_group);+ assert_leading_group(&arena, &found, per_group, 0, per_group);}}}#[test]- fn duplicate_groups_stored_below_unit_norm_stay_whole() {- for storage in [StorageKind::F32, StorageKind::I8] {- for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 100)] {- let (arena, graph) = build_groups(- 512,- storage,- DistanceMetric::InnerProduct,- groups,- per_group,- |group, _| scaled(&group_center(group, 512), 0.5),- );- let query = arena- .pack_query(&scaled(&group_center(0, 512), 0.5))- .unwrap();- let found = search(- &graph,- &arena,- &query,- ¶ms(Some(limit), None, false),- None,- );- assert_eq!(found.len(), limit);- assert_sorted(&found);- assert_leading_group(&arena, &found, per_group, 0, per_group);- }+ fn duplicate_groups_of_peaky_vectors_stay_whole_in_int8() {+ for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 10), (4, 80, 80)] {+ let (arena, graph) =+ build_groups(512, StorageKind::I8, groups, per_group, |group, _| {+ peaky_unit_vector(group, 512)+ });+ let query = arena.pack_query(&peaky_unit_vector(0, 512)).unwrap();+ let found = search(+ &graph,+ &arena,+ &query,+ ¶ms(Some(limit), None, false),+ None,+ );+ assert_eq!(found.len(), limit);+ assert_sorted(&found);+ assert_leading_group(&arena, &found, per_group.min(limit), 0, per_group);}}@@ -1724,35 +1674,33 @@ mod tests {fn a_duplicate_burst_at_the_neighbor_cap_survives_background_and_churn() {const BURST: usize = LEVEL_ZERO_NEIGHBOR_CAP;const BACKGROUND: usize = 2000;- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {- let duplicate = group_center(0, 512);- let mut arena = VectorArena::with_metric(512, StorageKind::I8, metric).unwrap();- let mut graph = Graph::new();- for index in 0..BACKGROUND {- apply_upsert(- &mut arena,- &mut graph,- &format!("background-{index}"),- &seeded_unit_vector(BACKGROUND_SEED + index as u64, 512),- );- }- for index in 0..BURST {- apply_upsert(- &mut arena,- &mut graph,- &format!("duplicate-{index}"),- &duplicate,- );- }- let query = arena.pack_query(&duplicate).unwrap();- assert_whole_burst(&graph, &arena, &query, BURST);- for index in (0..BURST).step_by(2) {- let key = format!("duplicate-{index}");- assert!(arena.remove(&key).is_some());- apply_upsert(&mut arena, &mut graph, &key, &duplicate);- }- assert_whole_burst(&graph, &arena, &query, BURST);+ let duplicate = group_center(0, 512);+ let mut arena = VectorArena::with_storage(512, StorageKind::I8).unwrap();+ let mut graph = Graph::new();+ for index in 0..BACKGROUND {+ apply_upsert(+ &mut arena,+ &mut graph,+ &format!("background-{index}"),+ &seeded_unit_vector(BACKGROUND_SEED + index as u64, 512),+ );+ }+ for index in 0..BURST {+ apply_upsert(+ &mut arena,+ &mut graph,+ &format!("duplicate-{index}"),+ &duplicate,+ );+ }+ let query = arena.pack_query(&duplicate).unwrap();+ assert_whole_burst(&graph, &arena, &query, BURST);+ for index in (0..BURST).step_by(2) {+ let key = format!("duplicate-{index}");+ assert!(arena.remove(&key).is_some());+ apply_upsert(&mut arena, &mut graph, &key, &duplicate);}+ assert_whole_burst(&graph, &arena, &query, BURST);}fn assert_whole_burst(graph: &Graph, arena: &VectorArena, query: &PackedQuery, burst: usize) {@@ -1786,20 +1734,13 @@ mod tests {fn jittered_groups_near_the_neighbor_cap_stay_whole() {for storage in [StorageKind::F32, StorageKind::I8] {for spread in [1e-3, 1e-5, 1e-6] {- let (arena, graph) = build_groups(- 512,- storage,- DistanceMetric::Cosine,- 4,- 80,- |group, member| {- jittered(- &group_center(group, 512),- JITTER_SEED + (group * 80 + member) as u64,- spread,- )- },- );+ let (arena, graph) = build_groups(512, storage, 4, 80, |group, member| {+ jittered(+ &group_center(group, 512),+ JITTER_SEED + (group * 80 + member) as u64,+ spread,+ )+ });let query = arena.pack_query(&group_center(0, 512)).unwrap();let found = search(&graph,diff --git a/rust/crates/vecdb/src/kernel.rs b/rust/crates/vecdb/src/kernel.rsindex b59b6b1994..b83ed06f6d 100644--- a/rust/crates/vecdb/src/kernel.rs+++ b/rust/crates/vecdb/src/kernel.rs@@ -143,10 +143,6 @@ pub(crate) trait VectorKernel {type Lane;fn dot(a: &[Self::Lane], b: &[Self::Lane]) -> f32;-- fn distance(a: &[Self::Lane], b: &[Self::Lane]) -> f32 {- 1.0 - Self::dot(a, b)- }}pub(crate) struct F32Kernel;@@ -181,10 +177,6 @@ impl I8Kernel {}total}-- pub(crate) fn distance(a: &[LaneI8], scale_a: f32, b: &[LaneI8], scale_b: f32) -> f32 {- 1.0 - Self::dot(a, b) as f32 * scale_a * scale_b- }}pub(crate) fn pack_lanes_i8(values: &[i8]) -> Vec<LaneI8> {@@ -290,11 +282,11 @@ mod tests {}#[test]- fn distance_of_basis_vectors_is_exact() {+ fn dot_of_basis_vectors_is_exact() {let e0 = pack_lanes(&basis_vector(128, 0));let e1 = pack_lanes(&basis_vector(128, 100));- assert_eq!(F32Kernel::distance(&e0, &e0), 0.0);- assert_eq!(F32Kernel::distance(&e0, &e1), 1.0);+ assert_eq!(F32Kernel::dot(&e0, &e0), 1.0);+ assert_eq!(F32Kernel::dot(&e0, &e1), 0.0);}#[test]@@ -388,26 +380,7 @@ mod tests {}#[test]- fn i8_distance_pins_the_evaluation_order() {- let a = pack_lanes_i8(&seeded_i8_values(5, 64));- let b = pack_lanes_i8(&seeded_i8_values(6, 64));- let scale_a = 0.012_345_6f32;- let scale_b = 0.045_678_9f32;- let dot = I8Kernel::dot(&a, &b);- let scaled_once = dot as f32 * scale_a;- let expected = 1.0 - scaled_once * scale_b;- assert_eq!(- I8Kernel::distance(&a, scale_a, &b, scale_b).to_bits(),- expected.to_bits()- );- assert_eq!(- I8Kernel::distance(&b, scale_b, &a, scale_a).to_bits(),- (1.0 - (dot as f32 * scale_b) * scale_a).to_bits()- );- }-- #[test]- fn i8_distance_of_basis_vectors_is_orthogonal_exactly_and_self_within_rounding() {+ fn i8_dot_of_basis_vectors_is_orthogonal_exactly_and_self_within_rounding() {let (scale_0, values_0) = quantized_parts(&basis_vector(32, 0));let (scale_5, values_5) = quantized_parts(&basis_vector(32, 5));assert_eq!(scale_0, 1.0 / 127.0);@@ -415,8 +388,9 @@ mod tests {assert!(values_0[1..].iter().all(|&value| value == 0));let e0 = pack_lanes_i8(&values_0);let e5 = pack_lanes_i8(&values_5);- assert_eq!(I8Kernel::distance(&e0, scale_0, &e5, scale_5), 1.0);- assert!(I8Kernel::distance(&e0, scale_0, &e0, scale_0).abs() < 1.0e-6);+ assert_eq!(I8Kernel::dot(&e0, &e5), 0);+ assert!((I8Kernel::dot(&e0, &e0) as f32 * scale_0 * scale_0 - 1.0).abs() < 1.0e-6);+ assert_eq!(scale_5, scale_0);}#[test]@@ -538,18 +512,19 @@ mod tests {}#[test]- fn i8_distance_tracks_f32_distance_on_unit_vectors_within_the_bound() {+ fn i8_dot_tracks_f32_dot_on_unit_vectors_within_the_bound() {let mut max_delta = 0.0f32;for (dims, pairs, seed) in [(512usize, 300u64, 0x1000u64), (64, 300, 0x2000)] {for pair in 0..pairs {let a = seeded_unit_values(seed + pair * 2, dims);let b = seeded_unit_values(seed + pair * 2 + 1, dims);- let f32_distance = F32Kernel::distance(&pack_lanes(&a), &pack_lanes(&b));+ let f32_dot = F32Kernel::dot(&pack_lanes(&a), &pack_lanes(&b));let (scale_a, qa) = quantized_parts(&a);let (scale_b, qb) = quantized_parts(&b);- let i8_distance =- I8Kernel::distance(&pack_lanes_i8(&qa), scale_a, &pack_lanes_i8(&qb), scale_b);- let delta = (i8_distance - f32_distance).abs();+ let i8_dot = I8Kernel::dot(&pack_lanes_i8(&qa), &pack_lanes_i8(&qb)) as f32+ * scale_a+ * scale_b;+ let delta = (i8_dot - f32_dot).abs();assert!(delta <= 0.01, "dims {dims} pair {pair}: delta {delta}");max_delta = max_delta.max(delta);}diff --git a/rust/crates/vecdb/src/lib.rs b/rust/crates/vecdb/src/lib.rsindex af104180de..1a573a908f 100644--- a/rust/crates/vecdb/src/lib.rs+++ b/rust/crates/vecdb/src/lib.rs@@ -13,38 +13,6 @@ mod store;pub use store::{OpenCost, Stats, VecDb};-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum DistanceMetric {- InnerProduct,- Cosine,-}--impl DistanceMetric {- pub(crate) fn header_tag(self) -> u8 {- match self {- Self::InnerProduct => 0,- Self::Cosine => 1,- }- }-- pub(crate) fn from_header_tag(tag: u8) -> Option<Self> {- match tag {- 0 => Some(Self::InnerProduct),- 1 => Some(Self::Cosine),- _ => None,- }- }-}--impl std::fmt::Display for DistanceMetric {- fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- formatter.write_str(match self {- Self::InnerProduct => "inner product",- Self::Cosine => "cosine",- })- }-}-#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum StorageKind {F32,@@ -156,11 +124,6 @@ pub enum VecDbError {expected: StorageKind,actual: StorageKind,},- #[error("distance metric mismatch: expected {expected}, found {actual}")]- MetricMismatch {- expected: DistanceMetric,- actual: DistanceMetric,- },#[error("search requires a limit or a max distance")]UnboundedSearch,#[error("length mismatch: {keys} keys, {vectors} vectors")]diff --git a/rust/crates/vecdb/src/log.rs b/rust/crates/vecdb/src/log.rsindex 64d30bd3a4..307900d2d0 100644--- a/rust/crates/vecdb/src/log.rs+++ b/rust/crates/vecdb/src/log.rs@@ -9,13 +9,14 @@ use super::crc::{Crc32, crc32};use super::kernel::{StoredVector, VectorPayload, is_within_quantized_range, splitmix64, validate_dims,};-use super::{AttrValue, Attribute, DistanceMetric, StorageKind, VecDbError};+use super::{AttrValue, Attribute, StorageKind, VecDbError};pub(crate) const HEADER_LEN: usize = 32;const MAGIC: [u8; 4] = *b"EVDB";const FORMAT_VERSION: u16 = 1;const STORAGE_TAG_OFFSET: usize = 6;const METRIC_TAG_OFFSET: usize = 7;+const COSINE_METRIC_TAG: u8 = 1;const RECORD_TYPE_ADD: u8 = 1;const RECORD_TYPE_TOMBSTONE: u8 = 2;const RECORD_PREFIX_LEN: usize = 3;@@ -63,7 +64,6 @@ pub(crate) struct Log {path: PathBuf,dims: usize,storage: StorageKind,- metric: DistanceMetric,generation: [u8; 16],end_offset: u64,encode_buffer: Vec<u8>,@@ -116,11 +116,10 @@ impl Log {path: &Path,dims: usize,storage: StorageKind,- metric: DistanceMetric,) -> Result<Self, VecDbError> {validate_dims(dims, storage)?;let file = open_writer_file(path, true)?;- Self::initialize(file, path, dims, storage, metric)+ Self::initialize(file, path, dims, storage)}pub(crate) fn open(@@ -128,7 +127,6 @@ impl Log {path: &Path,expected_dims: usize,requested: StorageKind,- requested_metric: DistanceMetric,) -> Result<Self, VecDbError> {let file_len = file.metadata()@@ -140,7 +138,7 @@ impl Log {path.display());validate_dims(expected_dims, requested)?;- return Self::initialize(file, path, expected_dims, requested, requested_metric);+ return Self::initialize(file, path, expected_dims, requested);}file.seek(SeekFrom::Start(0)).map_err(|source| VecDbError::io(path, source))?;@@ -151,7 +149,6 @@ impl Log {generation,dims,storage,- metric,} = decode_header(&header)?;if requested != storage {return Err(VecDbError::StorageMismatch {@@ -159,12 +156,6 @@ impl Log {actual: storage,});}- if requested_metric != metric {- return Err(VecDbError::MetricMismatch {- expected: requested_metric,- actual: metric,- });- }if dims != expected_dims {return Err(VecDbError::DimensionMismatch {expected: expected_dims,@@ -177,7 +168,6 @@ impl Log {path: path.to_path_buf(),dims: expected_dims,storage,- metric,generation,end_offset: file_len,encode_buffer: Vec::new(),@@ -190,12 +180,11 @@ impl Log {path: &Path,dims: usize,storage: StorageKind,- metric: DistanceMetric,) -> Result<Self, VecDbError> {let generation = fresh_generation();file.seek(SeekFrom::Start(0)).map_err(|source| VecDbError::io(path, source))?;- file.write_all(&encode_header(dims as u32, &generation, storage, metric))+ file.write_all(&encode_header(dims as u32, &generation, storage)).map_err(|source| VecDbError::io(path, source))?;file.sync_all().map_err(|source| VecDbError::io(path, source))?;@@ -205,7 +194,6 @@ impl Log {path: path.to_path_buf(),dims,storage,- metric,generation,end_offset: HEADER_LEN as u64,encode_buffer: Vec::new(),@@ -217,11 +205,10 @@ impl Log {path: &Path,dims: usize,storage: StorageKind,- metric: DistanceMetric,) -> Result<(Self, PathBuf), VecDbError> {remove_stale_temp_sibling(path)?;let temp_path = temp_sibling_path(path);- let log = Self::create(&temp_path, dims, storage, metric)?;+ let log = Self::create(&temp_path, dims, storage)?;Ok((log, temp_path))}@@ -410,10 +397,6 @@ impl Log {self.generation}- pub(crate) fn metric(&self) -> DistanceMetric {- self.metric- }-pub(crate) fn storage(&self) -> StorageKind {self.storage}@@ -775,17 +758,12 @@ fn plausible_attrs_len(bytes: &[u8]) -> Option<usize> {Some(1 + attr_bytes)}-fn encode_header(- dims: u32,- generation: &[u8; 16],- storage: StorageKind,- metric: DistanceMetric,-) -> [u8; HEADER_LEN] {+fn encode_header(dims: u32, generation: &[u8; 16], storage: StorageKind) -> [u8; HEADER_LEN] {let mut bytes = [0u8; HEADER_LEN];bytes[0..4].copy_from_slice(&MAGIC);bytes[4..6].copy_from_slice(&FORMAT_VERSION.to_le_bytes());bytes[STORAGE_TAG_OFFSET] = storage.header_tag();- bytes[METRIC_TAG_OFFSET] = metric.header_tag();+ bytes[METRIC_TAG_OFFSET] = COSINE_METRIC_TAG;bytes[8..12].copy_from_slice(&dims.to_le_bytes());bytes[12..28].copy_from_slice(generation);let crc = crc32(&bytes[0..28]);@@ -802,7 +780,6 @@ struct Header {generation: [u8; 16],dims: usize,storage: StorageKind,- metric: DistanceMetric,}fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {@@ -828,9 +805,12 @@ fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {let storage = StorageKind::from_header_tag(bytes[STORAGE_TAG_OFFSET]).ok_or_else(|| {VecDbError::Corrupt(format!("unknown scalar tag {}", bytes[STORAGE_TAG_OFFSET]))})?;- let metric = DistanceMetric::from_header_tag(bytes[METRIC_TAG_OFFSET]).ok_or_else(|| {- VecDbError::Corrupt(format!("unknown metric tag {}", bytes[METRIC_TAG_OFFSET]))- })?;+ if bytes[METRIC_TAG_OFFSET] != COSINE_METRIC_TAG {+ return Err(VecDbError::Corrupt(format!(+ "unknown metric tag {}",+ bytes[METRIC_TAG_OFFSET]+ )));+ }let dims = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;let mut generation = [0u8; 16];generation.copy_from_slice(&bytes[12..28]);@@ -838,7 +818,6 @@ fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {generation,dims,storage,- metric,})}@@ -1042,7 +1021,6 @@ pub(crate) fn sync_parent_dir(_path: &Path) -> Result<(), VecDbError> {#[cfg(test)]mod tests {- use super::DistanceMetric::InnerProduct;use super::*;use tempfile::TempDir;@@ -1058,7 +1036,7 @@ mod tests {fn reopen(path: &Path, dims: usize, storage: StorageKind) -> Log {let file = open_writer_file(path, false).unwrap();- Log::open(file, path, dims, storage, InnerProduct).unwrap()+ Log::open(file, path, dims, storage).unwrap()}fn append_bounds(log: &mut Log, entries: &[LogEntry<'_>]) -> (u64, u64) {@@ -1129,7 +1107,7 @@ mod tests {}fn write_log_file(path: &Path, dims: u32, record_bytes: &[u8]) {- let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::F32, InnerProduct).to_vec();+ let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::F32).to_vec();bytes.extend_from_slice(record_bytes);std::fs::write(path, bytes).unwrap();}@@ -1145,18 +1123,18 @@ mod tests {) -> Result<Log, VecDbError> {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut bytes = encode_header(8, &[7u8; 16], StorageKind::F32, InnerProduct);+ let mut bytes = encode_header(8, &[7u8; 16], StorageKind::F32);mutate(&mut bytes);std::fs::write(&path, bytes).unwrap();let file = File::options().read(true).write(true).open(&path).unwrap();- Log::open(file, &path, expected_dims, StorageKind::F32, InnerProduct)+ Log::open(file, &path, expected_dims, StorageKind::F32)}#[test]fn header_round_trips_through_create_and_reopen() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let created = Log::create(&path, 512, StorageKind::F32, InnerProduct).unwrap();+ let created = Log::create(&path, 512, StorageKind::F32).unwrap();let generation = created.generation();assert_eq!(created.current_end_offset(), HEADER_LEN as u64);assert_eq!(created.path(), path.as_path());@@ -1172,7 +1150,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("log");let alias = dir.path().join("alias");- let created = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let created = Log::create(&path, 8, StorageKind::F32).unwrap();std::fs::hard_link(&path, &alias).unwrap();let reader = File::open(&alias).unwrap();assert_eq!(reader.metadata().unwrap().len(), HEADER_LEN as u64);@@ -1250,16 +1228,18 @@ mod tests {}#[test]- fn rejects_unknown_metric_tag() {- let error = open_with_header(- |bytes| {- bytes[7] = 3;- refresh_crc(bytes);- },- 8,- )- .unwrap_err();- assert!(matches!(&error, VecDbError::Corrupt(message) if message.contains("metric")));+ fn rejects_every_metric_tag_but_cosine() {+ for tag in [0u8, 2, 3, 0xff] {+ let error = open_with_header(+ |bytes| {+ bytes[METRIC_TAG_OFFSET] = tag;+ refresh_crc(bytes);+ },+ 8,+ )+ .unwrap_err();+ assert!(matches!(&error, VecDbError::Corrupt(message) if message.contains("metric")));+ }}#[test]@@ -1287,7 +1267,7 @@ mod tests {let path = dir.path().join("log");std::fs::write(&path, vec![1u8; junk_len]).unwrap();let file = File::options().read(true).write(true).open(&path).unwrap();- let mut log = Log::open(file, &path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::open(file, &path, 8, StorageKind::F32).unwrap();assert_eq!(log.current_end_offset(), HEADER_LEN as u64);assert_eq!(std::fs::metadata(&path).unwrap().len(), HEADER_LEN as u64);let (records, _) = scan_all(&mut log);@@ -1314,11 +1294,11 @@ mod tests {fn create_rejects_invalid_dimensions() {let dir = TempDir::new().unwrap();assert!(matches!(- Log::create(&dir.path().join("a"), 0, StorageKind::F32, InnerProduct),+ Log::create(&dir.path().join("a"), 0, StorageKind::F32),Err(VecDbError::InvalidDimensions { dims: 0, .. })));assert!(matches!(- Log::create(&dir.path().join("b"), 12, StorageKind::F32, InnerProduct),+ Log::create(&dir.path().join("b"), 12, StorageKind::F32),Err(VecDbError::InvalidDimensions { dims: 12, .. })));}@@ -1327,14 +1307,10 @@ mod tests {fn open_rejects_invalid_dimensions() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- drop(- Log::create(&path, 8, StorageKind::F32, InnerProduct)- .unwrap()- .into_file(),- );+ drop(Log::create(&path, 8, StorageKind::F32).unwrap().into_file());let file = File::options().read(true).write(true).open(&path).unwrap();assert!(matches!(- Log::open(file, &path, 12, StorageKind::F32, InnerProduct),+ Log::open(file, &path, 12, StorageKind::F32),Err(VecDbError::DimensionMismatch {expected: 12,actual: 8@@ -1346,9 +1322,9 @@ mod tests {fn golden_bytes_pin_format_v1() {let generation: [u8; 16] = std::array::from_fn(|index| index as u8);let golden_header: [u8; 32] = [- 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x01,+ 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x01,0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,- 0x56, 0x32, 0x6d, 0x51,+ 0xc8, 0xb1, 0xb7, 0xce,];let golden_add: [u8; 42] = [0x01, 0x02, 0x00, 0x6b, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00,@@ -1384,7 +1360,7 @@ mod tests {},];assert_eq!(- encode_header(8, &generation, StorageKind::F32, InnerProduct),+ encode_header(8, &generation, StorageKind::F32),golden_header);let mut encoded = Vec::new();@@ -1455,7 +1431,7 @@ mod tests {for dims in [8usize, 512] {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, dims, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, dims, StorageKind::F32).unwrap();let vector = seeded_vector(1, dims);let entries = [LogEntry::Add {@@ -1495,7 +1471,7 @@ mod tests {];let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(5, 8);for key in keys {log.append(&[LogEntry::Add {@@ -1530,7 +1506,7 @@ mod tests {fn decodes_non_finite_components_without_panicking() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = [f32::NAN,f32::INFINITY,@@ -1566,7 +1542,7 @@ mod tests {fn scanner_stops_at_last_complete_record_for_every_torn_length() {let dir = TempDir::new().unwrap();let build_path = dir.path().join("log");- let mut log = Log::create(&build_path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&build_path, 8, StorageKind::F32).unwrap();let vectors: Vec<Vec<f32>> = (0..3).map(|seed| seeded_vector(seed, 8)).collect();let mut boundaries = Vec::new();for (index, vector) in vectors.iter().enumerate() {@@ -1599,7 +1575,7 @@ mod tests {fn truncate_to_repairs_torn_tail_for_clean_appends() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(1, 8);let (_, first_end) = append_bounds(&mut log,@@ -1650,7 +1626,7 @@ mod tests {fn extend_end_offset_grows_to_target_clamps_to_file_and_never_shrinks() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(1, 8);let (_, captured_end) = append_bounds(&mut log,@@ -1690,7 +1666,7 @@ mod tests {fn discard_unacked_tail_drops_phantom_frames_before_reopen() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(1, 8);let (_, acked_end) = append_bounds(&mut log,@@ -1720,7 +1696,7 @@ mod tests {fn pending_rollback_truncates_stale_records_before_the_next_append() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(6, 8);let (_, acked_end) = append_bounds(&mut log,@@ -1766,8 +1742,8 @@ mod tests {let dir = TempDir::new().unwrap();let synced_path = dir.path().join("synced");let staged_path = dir.path().join("staged");- let mut synced = Log::create(&synced_path, 8, StorageKind::F32, InnerProduct).unwrap();- let mut staged = Log::create(&staged_path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut synced = Log::create(&synced_path, 8, StorageKind::F32).unwrap();+ let mut staged = Log::create(&staged_path, 8, StorageKind::F32).unwrap();let vectors: Vec<Vec<f32>> = (0..3).map(|seed| seeded_vector(seed, 8)).collect();let attrs = [Attribute {name: "model".to_string(),@@ -1817,7 +1793,7 @@ mod tests {fn append_failure_rolls_back_without_disturbing_acked_records() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(2, 8);let (_, acked_end) = append_bounds(&mut log,@@ -1829,7 +1805,7 @@ mod tests {);drop(log);let unwritable = File::open(&path).unwrap();- let mut log = Log::open(unwritable, &path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::open(unwritable, &path, 8, StorageKind::F32).unwrap();assert!(matches!(log.append(&[LogEntry::Add {key: "lost",@@ -1851,7 +1827,7 @@ mod tests {fn append_after_phantom_frames_overwrites_at_the_acked_offset() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(3, 8);let (_, acked_end) = append_bounds(&mut log,@@ -1892,7 +1868,7 @@ mod tests {fn scanner_stops_at_corrupted_middle_record() {let dir = TempDir::new().unwrap();let build_path = dir.path().join("log");- let mut log = Log::create(&build_path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&build_path, 8, StorageKind::F32).unwrap();let vectors: Vec<Vec<f32>> = (0..3).map(|seed| seeded_vector(seed, 8)).collect();let mut boundaries = Vec::new();for (index, vector) in vectors.iter().enumerate() {@@ -1971,7 +1947,7 @@ mod tests {fn append_offsets_match_scanner_offsets() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();log.append(&[]).unwrap();assert_eq!(log.current_end_offset(), HEADER_LEN as u64);let vector = seeded_vector(7, 8);@@ -2006,7 +1982,7 @@ mod tests {fn append_validates_keys_and_vector_dims() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(1, 8);let wrong_dims = seeded_vector(1, 16);assert!(matches!(@@ -2048,15 +2024,8 @@ mod tests {#[test]fn generation_is_distinct_across_creates() {let dir = TempDir::new().unwrap();- let first =- Log::create(&dir.path().join("first"), 8, StorageKind::F32, InnerProduct).unwrap();- let second = Log::create(- &dir.path().join("second"),- 8,- StorageKind::F32,- InnerProduct,- )- .unwrap();+ let first = Log::create(&dir.path().join("first"), 8, StorageKind::F32).unwrap();+ let second = Log::create(&dir.path().join("second"), 8, StorageKind::F32).unwrap();assert_ne!(first.generation(), second.generation());}@@ -2064,11 +2033,10 @@ mod tests {fn temp_sibling_gets_fresh_generation_and_replaces_stale_tmp() {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let log = Log::create(&path, 8, StorageKind::F32).unwrap();let stale_path = dir.path().join("log.tmp");std::fs::write(&stale_path, b"stale leftover").unwrap();- let (temp_log, temp_path) =- Log::create_temp_sibling(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let (temp_log, temp_path) = Log::create_temp_sibling(&path, 8, StorageKind::F32).unwrap();assert_eq!(temp_path, stale_path);assert_ne!(temp_log.generation(), log.generation());assert_eq!(temp_log.current_end_offset(), HEADER_LEN as u64);@@ -2101,9 +2069,8 @@ mod tests {attrs: &[],}).collect();- let mut bulk_log = Log::create(&bulk_path, dims, StorageKind::F32, InnerProduct).unwrap();- let mut sequential_log =- Log::create(&sequential_path, dims, StorageKind::F32, InnerProduct).unwrap();+ let mut bulk_log = Log::create(&bulk_path, dims, StorageKind::F32).unwrap();+ let mut sequential_log = Log::create(&sequential_path, dims, StorageKind::F32).unwrap();let (_, bulk_end) = append_bounds(&mut bulk_log, &entries);let mut sequential_end = 0;for entry in &entries {@@ -2143,9 +2110,8 @@ mod tests {attrs: &[],},];- let mut bulk_log = Log::create(&bulk_path, 8, StorageKind::F32, InnerProduct).unwrap();- let mut sequential_log =- Log::create(&sequential_path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut bulk_log = Log::create(&bulk_path, 8, StorageKind::F32).unwrap();+ let mut sequential_log = Log::create(&sequential_path, 8, StorageKind::F32).unwrap();let (bulk_start, bulk_end) = append_bounds(&mut bulk_log, &entries);let mut sequential_end = 0;for entry in &entries {@@ -2194,7 +2160,7 @@ mod tests {let vector = seeded_vector(8, 8);for (index, attrs) in singles.iter().chain([&full_set]).enumerate() {let path = dir.path().join(format!("log-{index}"));- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();log.append(&[LogEntry::Add {key: "k",vector: VectorPayload::F32(&vector),@@ -2229,7 +2195,7 @@ mod tests {let vector = seeded_vector(9, 8);for (index, attrs) in cases.iter().enumerate() {let path = dir.path().join(format!("log-{index}"));- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();log.append(&[LogEntry::Add {key: "k",vector: VectorPayload::F32(&vector),@@ -2267,7 +2233,7 @@ mod tests {];let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();let vector = seeded_vector(10, 8);for attrs in &invalid {assert!(matches!(@@ -2307,8 +2273,7 @@ mod tests {.map(|(index, value)| attr(&format!("f{index}"), AttrValue::F64(*value))).collect();let dir = TempDir::new().unwrap();- let mut log =- Log::create(&dir.path().join("log"), 8, StorageKind::F32, InnerProduct).unwrap();+ let mut log = Log::create(&dir.path().join("log"), 8, StorageKind::F32).unwrap();let vector = seeded_vector(11, 8);log.append(&[LogEntry::Add {key: "k",@@ -2711,8 +2676,7 @@ mod tests {];for (name, after_first, intact_record_beyond) in cases {let path = dir.path().join(name);- let mut bytes =- encode_header(dims as u32, &[9u8; 16], StorageKind::I8, InnerProduct).to_vec();+ let mut bytes = encode_header(dims as u32, &[9u8; 16], StorageKind::I8).to_vec();bytes.extend_from_slice(&first);bytes.extend_from_slice(&after_first);std::fs::write(&path, &bytes).unwrap();@@ -2729,7 +2693,7 @@ mod tests {}fn write_i8_log_file(path: &Path, dims: u32, record_bytes: &[u8]) {- let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::I8, InnerProduct).to_vec();+ let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::I8).to_vec();bytes.extend_from_slice(record_bytes);std::fs::write(path, bytes).unwrap();}@@ -2757,9 +2721,9 @@ mod tests {fn i8_golden_bytes_pin_the_header_and_add_record() {let generation: [u8; 16] = std::array::from_fn(|index| index as u8);let golden_header: [u8; 32] = [- 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x01,+ 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x01, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, 0x01,0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,- 0x2f, 0xd0, 0x62, 0xc1,+ 0xb1, 0x53, 0xb8, 0x5e,];let golden_add: [u8; 46] = [0x01, 0x02, 0x00, 0x6b, 0x31, 0x0a, 0xd7, 0x23, 0x3c, 0x81, 0x7f, 0x00, 0xff, 0xa4,@@ -2770,7 +2734,7 @@ mod tests {let scale = 0.01f32;assert_eq!(scale.to_bits(), 0x3c23_d70a);let values = golden_i8_values();- let header = encode_header(32, &generation, StorageKind::I8, InnerProduct);+ let header = encode_header(32, &generation, StorageKind::I8);assert_eq!(header[STORAGE_TAG_OFFSET], 1);assert_eq!(header[..28], golden_header[..28]);let encoded = encoded_i8_add("k1", scale, &values);@@ -2818,15 +2782,15 @@ mod tests {#[test]fn f32_headers_leave_the_storage_tag_zero() {- let header = encode_header(8, &[7u8; 16], StorageKind::F32, InnerProduct);+ let header = encode_header(8, &[7u8; 16], StorageKind::F32);assert_eq!(header[STORAGE_TAG_OFFSET], 0);+ assert_eq!(header[METRIC_TAG_OFFSET], COSINE_METRIC_TAG);assert_eq!(decode_header(&header).unwrap(),Header {generation: [7u8; 16],dims: 8,storage: StorageKind::F32,- metric: InnerProduct});}@@ -2851,18 +2815,18 @@ mod tests {let dir = TempDir::new().unwrap();let i8_path = dir.path().join("i8");let f32_path = dir.path().join("f32");- let created = Log::create(&i8_path, 32, StorageKind::I8, InnerProduct).unwrap();+ let created = Log::create(&i8_path, 32, StorageKind::I8).unwrap();assert_eq!(created.storage(), StorageKind::I8);let generation = created.generation();drop(created.into_file());drop(- Log::create(&f32_path, 32, StorageKind::F32, InnerProduct)+ Log::create(&f32_path, 32, StorageKind::F32).unwrap().into_file(),);let open = |path: &Path, requested: StorageKind| {let file = File::options().read(true).write(true).open(path).unwrap();- Log::open(file, path, 32, requested, InnerProduct)+ Log::open(file, path, 32, requested)};let detected = open(&i8_path, StorageKind::I8).unwrap();assert_eq!(detected.storage(), StorageKind::I8);@@ -2899,7 +2863,7 @@ mod tests {let path = dir.path().join("log");std::fs::write(&path, [1u8; 5]).unwrap();let file = File::options().read(true).write(true).open(&path).unwrap();- let log = Log::open(file, &path, 64, StorageKind::I8, InnerProduct).unwrap();+ let log = Log::open(file, &path, 64, StorageKind::I8).unwrap();assert_eq!(log.storage(), StorageKind::I8);drop(log);assert_eq!(@@ -2910,7 +2874,7 @@ mod tests {std::fs::write(&other, [1u8; 5]).unwrap();let file = File::options().read(true).write(true).open(&other).unwrap();assert_eq!(- Log::open(file, &other, 64, StorageKind::I8, InnerProduct)+ Log::open(file, &other, 64, StorageKind::I8).unwrap().storage(),StorageKind::I8@@ -2922,33 +2886,27 @@ mod tests {let dir = TempDir::new().unwrap();for dims in [8usize, 16, 24, 40] {assert!(matches!(- Log::create(&dir.path().join(format!("a{dims}")), dims, StorageKind::I8, InnerProduct),+ Log::create(&dir.path().join(format!("a{dims}")), dims, StorageKind::I8),Err(VecDbError::InvalidDimensions {dims: rejected,storage: StorageKind::I8}) if rejected == dims));assert!(- Log::create(- &dir.path().join(format!("b{dims}")),- dims,- StorageKind::F32,- InnerProduct- )- .is_ok()+ Log::create(&dir.path().join(format!("b{dims}")), dims, StorageKind::F32).is_ok());}- assert!(Log::create(&dir.path().join("c"), 32, StorageKind::I8, InnerProduct).is_ok());+ assert!(Log::create(&dir.path().join("c"), 32, StorageKind::I8).is_ok());let path = dir.path().join("c");let file = File::options().read(true).write(true).open(&path).unwrap();assert!(matches!(- Log::open(file, &path, 8, StorageKind::I8, InnerProduct),+ Log::open(file, &path, 8, StorageKind::I8),Err(VecDbError::DimensionMismatch {expected: 8,actual: 32})));- let mut forged = encode_header(8, &[3u8; 16], StorageKind::I8, InnerProduct);+ let mut forged = encode_header(8, &[3u8; 16], StorageKind::I8);refresh_crc(&mut forged);let forged_path = dir.path().join("forged");std::fs::write(&forged_path, forged).unwrap();@@ -2958,7 +2916,7 @@ mod tests {.open(&forged_path).unwrap();assert!(matches!(- Log::open(file, &forged_path, 8, StorageKind::I8, InnerProduct),+ Log::open(file, &forged_path, 8, StorageKind::I8),Err(VecDbError::InvalidDimensions {dims: 8,storage: StorageKind::I8@@ -2971,7 +2929,7 @@ mod tests {for dims in [32usize, 512] {let dir = TempDir::new().unwrap();let path = dir.path().join("log");- let mut log = Log::create(&path, dims, StorageKind::I8, InnerProduct).unwrap();+ let mut log = Log::create(&path, dims, StorageKind::I8).unwrap();let scales = [f32::from_bits(0x3a1f_8f3c),0.0,@@ -3033,8 +2991,8 @@ mod tests {let dir = TempDir::new().unwrap();let i8_path = dir.path().join("i8");let f32_path = dir.path().join("f32");- let mut i8_log = Log::create(&i8_path, 32, StorageKind::I8, InnerProduct).unwrap();- let mut f32_log = Log::create(&f32_path, 32, StorageKind::F32, InnerProduct).unwrap();+ let mut i8_log = Log::create(&i8_path, 32, StorageKind::I8).unwrap();+ let mut f32_log = Log::create(&f32_path, 32, StorageKind::F32).unwrap();let values = seeded_vector(4, 32);let quantized = StoredVector::quantize(&values);assert!(matches!(@@ -3101,7 +3059,7 @@ mod tests {let dims = 32usize;let dir = TempDir::new().unwrap();let build_path = dir.path().join("log");- let mut log = Log::create(&build_path, dims, StorageKind::I8, InnerProduct).unwrap();+ let mut log = Log::create(&build_path, dims, StorageKind::I8).unwrap();let mut boundaries = Vec::new();for index in 0..3u64 {let key = format!("key-{index}");diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rsindex 5144498a55..75dfb61af4 100644--- a/rust/crates/vecdb/src/store.rs+++ b/rust/crates/vecdb/src/store.rs@@ -20,9 +20,7 @@ use super::log::{use super::snapshot::{LoadedSnapshot, load_snapshot, remove_snapshot, snapshot_path, write_snapshot,};-use super::{- AttrValue, Attribute, DistanceMetric, KeyMatches, Match, SearchParams, StorageKind, VecDbError,-};+use super::{AttrValue, Attribute, KeyMatches, Match, SearchParams, StorageKind, VecDbError};const SNAPSHOT_HARD_CAP: usize = 5000;const SNAPSHOT_QUIET_THRESHOLD: usize = 1000;@@ -35,7 +33,6 @@ const HANDOFF_CLOSE_WAIT_ROUNDS: u32 = 2500;const HANDOFF_WAIT_PARK: Duration = Duration::from_millis(2);const KEY_TABLE_COPIES: usize = 2;const KEY_ENTRY_OVERHEAD_BYTES: usize = 48;-const INNER_PRODUCT_SQUARED_NORM_TOLERANCE: f64 = 1e-2;static REGISTRY: LazyLock<Mutex<HashMap<PathBuf, Arc<Mutex<PathSlot>>>>> =LazyLock::new(|| Mutex::new(HashMap::new()));@@ -67,7 +64,6 @@ pub struct Stats {pub dead_count: usize,pub dims: usize,pub storage: StorageKind,- pub metric: DistanceMetric,pub log_bytes: u64,pub records_since_snapshot: usize,pub approximate_memory_bytes: usize,@@ -82,7 +78,6 @@ struct Shared {path: PathBuf,dims: usize,storage: StorageKind,- metric: DistanceMetric,registry_key: Option<PathBuf>,closed: AtomicBool,writer: Mutex<WriterHalf>,@@ -309,7 +304,6 @@ fn join_live(shared: Arc<Shared>,dims: usize,storage: StorageKind,- metric: DistanceMetric,read_only: bool,) -> Result<VecDb, VecDbError> {if shared.dims != dims {@@ -324,12 +318,6 @@ fn join_live(actual: shared.storage,});}- if metric != shared.metric {- return Err(VecDbError::MetricMismatch {- expected: metric,- actual: shared.metric,- });- }Ok(VecDb { shared, read_only })}@@ -384,10 +372,8 @@ impl VecDb {path: &Path,dims: usize,storage: Option<StorageKind>,- metric: Option<DistanceMetric>,) -> Result<Self, VecDbError> {let storage = storage.unwrap_or(StorageKind::I8);- let metric = metric.unwrap_or(DistanceMetric::Cosine);let key = registry_key_for(path)?;let slot = path_slot(&key);let mut guard = lock_slot(&slot);@@ -396,20 +382,20 @@ impl VecDb {drop(guard);match observed {Some(shared) if !shared.is_closed() => {- return join_live(shared, dims, storage, metric, false);+ return join_live(shared, dims, storage, false);}observed => drop(observed),}match wait_for_teardown_handoff(&slot, &key) {SlotHandoff::Join(shared) => {- return join_live(shared, dims, storage, metric, false);+ return join_live(shared, dims, storage, false);}SlotHandoff::Released(reacquired) | SlotHandoff::Expired(reacquired) => {guard = reacquired;}}}- let built = match build_writer(&key, key.clone(), dims, storage, metric) {+ let built = match build_writer(&key, key.clone(), dims, storage) {Ok(built) => built,Err(error) => {drop(guard);@@ -458,22 +444,20 @@ impl VecDb {path: &Path,dims: usize,storage: Option<StorageKind>,- metric: Option<DistanceMetric>,) -> Result<Self, VecDbError> {let storage = storage.unwrap_or(StorageKind::I8);- let metric = metric.unwrap_or(DistanceMetric::Cosine);let resolved = registry_key_for(path).unwrap_or_else(|_| path.to_path_buf());if let Some(slot) = existing_path_slot(&resolved) {let observed = lock_slot(&slot).live.as_ref().and_then(Weak::upgrade);match observed {Some(shared) if !shared.is_closed() => {- return join_live(shared, dims, storage, metric, true);+ return join_live(shared, dims, storage, true);}Some(closing) => {drop(closing);match wait_for_teardown_handoff(&slot, &resolved) {SlotHandoff::Join(shared) => {- return join_live(shared, dims, storage, metric, true);+ return join_live(shared, dims, storage, true);}SlotHandoff::Released(released) | SlotHandoff::Expired(released) => {drop(released);@@ -484,7 +468,7 @@ impl VecDb {}}Ok(Self {- shared: Arc::new(build_read_only(&resolved, dims, storage, metric)?),+ shared: Arc::new(build_read_only(&resolved, dims, storage)?),read_only: true,})}@@ -502,7 +486,7 @@ impl VecDb {let now = Instant::now();let mut half = self.writable_half()?;let state = active_state(&mut half)?;- ensure_storable(self.shared.metric, key, vector)?;+ ensure_finite(key, vector)?;let quantized = quantize_for(self.shared.storage, vector);let payload = quantized.as_ref()@@ -559,7 +543,7 @@ impl VecDb {return Ok(());}for (key, vector) in keys.iter().zip(vectors) {- ensure_storable(self.shared.metric, key, vector)?;+ ensure_finite(key, vector)?;}let quantized: Option<Vec<StoredVector>> = match self.shared.storage {StorageKind::F32 => None,@@ -792,7 +776,7 @@ impl VecDb {let dims = self.shared.dims;let storage = self.shared.storage;let recreated = remove_data_files(&self.shared.path)- .and_then(|()| Log::create(&self.shared.path, dims, storage, self.shared.metric));+ .and_then(|()| Log::create(&self.shared.path, dims, storage));let log = match recreated {Ok(log) => log,Err(error) => {@@ -806,7 +790,7 @@ impl VecDb {return Err(error);}};- let arena = VectorArena::with_metric(dims, storage, self.shared.metric)?;+ let arena = VectorArena::with_storage(dims, storage)?;{let mut st = self.shared.state_write();st.arena = arena;@@ -892,7 +876,6 @@ impl VecDb {dead_count: st.total_records.saturating_sub(live_count as u64) as usize,dims: st.arena.dims(),storage: self.shared.storage,- metric: self.shared.metric,log_bytes,records_since_snapshot,approximate_memory_bytes: approximate_memory_bytes(&st),@@ -1019,7 +1002,7 @@ fn close_live_instance(shared: &Arc<Shared>) -> Result<(), VecDbError> {shared.closed.store(true, Ordering::Release);drop(log);let removed = remove_data_files(&shared.path);- if let Ok(empty) = VectorArena::with_metric(shared.dims, shared.storage, shared.metric) {+ if let Ok(empty) = VectorArena::with_storage(shared.dims, shared.storage) {st.arena = empty;st.graph = Some(Graph::new());st.attrs.reset();@@ -1135,7 +1118,7 @@ fn write_snapshot_now(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecDfn compact(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecDbError> {active_state(half)?;let (mut temp_log, temp_path) =- Log::create_temp_sibling(&shared.path, shared.dims, shared.storage, shared.metric)?;+ Log::create_temp_sibling(&shared.path, shared.dims, shared.storage)?;let staged = {let st = shared.state_read();stage_live_entries(&st, &mut temp_log)@@ -1214,13 +1197,7 @@ fn restore_writer_mode("log header was truncated while the writer was closed".to_string(),));}- let mut log = Log::open(- file,- &shared.path,- shared.dims,- shared.storage,- shared.metric,- )?;+ let mut log = Log::open(file, &shared.path, shared.dims, shared.storage)?;let mut pending = mutations_since_snapshot;if log.checkpoint() != expected_checkpoint {let replayed = replay(&mut log, shared.dims, ReplayMode::Writer)?;@@ -1271,7 +1248,7 @@ fn recover_after_failed_reset(if restore_writer_mode(shared, half, lock, None, old_pending, old_checkpoint).is_ok() {return;}- if let Ok(empty) = VectorArena::with_metric(shared.dims, shared.storage, shared.metric) {+ if let Ok(empty) = VectorArena::with_storage(shared.dims, shared.storage) {let mut st = shared.state_write();st.arena = empty;st.graph = Some(Graph::new());@@ -1295,16 +1272,15 @@ fn build_writer(registry_key: PathBuf,dims: usize,storage: StorageKind,- metric: DistanceMetric,) -> Result<BuiltWriter, VecDbError> {let lock = WriterLock::acquire(path)?;let mut log = if std::fs::metadata(path).is_ok() {- open_existing_writer_log(path, dims, storage, metric)?+ open_existing_writer_log(path, dims, storage)?} else {- match Log::create(path, dims, storage, metric) {+ match Log::create(path, dims, storage) {Ok(created) => created,Err(VecDbError::Io { source, .. }) if source.kind() == ErrorKind::AlreadyExists => {- open_existing_writer_log(path, dims, storage, metric)?+ open_existing_writer_log(path, dims, storage)?}Err(error) => return Err(error),}@@ -1397,7 +1373,6 @@ fn writer_shared(path: path.to_path_buf(),dims,storage,- metric: state.arena.metric(),registry_key: Some(registry_key),closed: AtomicBool::new(false),writer: Mutex::new(WriterHalf {@@ -1450,16 +1425,11 @@ fn install_graph(shared: &Shared, graph: Graph) {}}-fn build_read_only(- path: &Path,- dims: usize,- storage: StorageKind,- metric: DistanceMetric,-) -> Result<Shared, VecDbError> {+fn build_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Shared, VecDbError> {let file = match File::open(path) {Ok(file) => file,Err(error) if error.kind() == ErrorKind::NotFound => {- return empty_read_only(path, dims, storage, metric);+ return empty_read_only(path, dims, storage);}Err(source) => return Err(VecDbError::io(path, source)),};@@ -1468,9 +1438,9 @@ fn build_read_only(.map_err(|source| VecDbError::io(path, source))?.len();if file_len < HEADER_LEN as u64 {- return empty_read_only(path, dims, storage, metric);+ return empty_read_only(path, dims, storage);}- let mut log = Log::open(file, path, dims, storage, metric)?;+ let mut log = Log::open(file, path, dims, storage)?;let replayed = replay(&mut log, dims, ReplayMode::ReadOnly)?;drop(log);let ReplayedState {@@ -1492,15 +1462,10 @@ fn build_read_only())}-fn empty_read_only(- path: &Path,- dims: usize,- storage: StorageKind,- metric: DistanceMetric,-) -> Result<Shared, VecDbError> {+fn empty_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Shared, VecDbError> {Ok(read_only_shared(path,- VectorArena::with_metric(dims, storage, metric)?,+ VectorArena::with_storage(dims, storage)?,Graph::new(),AttrTable::default(),0,@@ -1520,7 +1485,6 @@ fn read_only_shared(path: path.to_path_buf(),dims: arena.dims(),storage: arena.storage_kind(),- metric: arena.metric(),registry_key: None,closed: AtomicBool::new(false),writer: Mutex::new(WriterHalf {@@ -1584,7 +1548,7 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,let snapshot_covered = snapshot.as_ref().map(|loaded| loaded.covered_log_offset);let covered = snapshot_covered.unwrap_or(u64::MAX);let path = log.path().to_path_buf();- let mut arena = VectorArena::with_metric(dims, log.storage(), log.metric())?;+ let mut arena = VectorArena::with_storage(dims, log.storage())?;let mut attrs = AttrTable::default();let mut graph: Option<Graph> = None;let mut tail_records = 0u64;@@ -1680,14 +1644,6 @@ fn apply_outcome(graph: &mut Graph, arena: &VectorArena, outcome: UpsertOutcome)}}-fn ensure_storable(metric: DistanceMetric, key: &str, vector: &[f32]) -> Result<(), VecDbError> {- ensure_finite(key, vector)?;- if metric == DistanceMetric::InnerProduct {- ensure_unit_norm(key, vector)?;- }- Ok(())-}-fn ensure_finite(key: &str, vector: &[f32]) -> Result<(), VecDbError> {if let Some(index) = vector.iter().position(|value| !value.is_finite()) {return Err(VecDbError::InvalidVector(format!(@@ -1697,24 +1653,13 @@ fn ensure_finite(key: &str, vector: &[f32]) -> Result<(), VecDbError> {Ok(())}-fn ensure_unit_norm(key: &str, vector: &[f32]) -> Result<(), VecDbError> {- let squared_norm: f64 = vector.iter().map(|&value| f64::from(value).powi(2)).sum();- if (squared_norm - 1.0).abs() > INNER_PRODUCT_SQUARED_NORM_TOLERANCE {- return Err(VecDbError::InvalidVector(format!(- "the vector for key {key:?} has squared norm {squared_norm}, and an inner product index needs unit vectors"- )));- }- Ok(())-}-fn open_existing_writer_log(path: &Path,dims: usize,storage: StorageKind,- metric: DistanceMetric,) -> Result<Log, VecDbError> {let file = open_writer_file(path, false)?;- Log::open(file, path, dims, storage, metric)+ Log::open(file, path, dims, storage)}fn promote_compacted_log(temp_path: &Path, path: &Path) -> Result<(), VecDbError> {@@ -1791,96 +1736,48 @@ mod tests {const TOMBSTONE_RECORD_LEN: u64 = 3 + 5 + 4;#[test]- fn omitted_metric_requires_cosine_for_creation_and_every_reopen() {+ fn omitted_storage_requires_i8_for_creation_and_every_reopen() {let dir = TempDir::new().unwrap();- let path = dir.path().join("cosine");- let reader = VecDb::open_read_only(&path, 32, None, None).unwrap();- assert_eq!(reader.stats().unwrap().metric, DistanceMetric::Cosine);+ let path = dir.path().join("index");+ let reader = VecDb::open_read_only(&path, 32, None).unwrap();assert_eq!(reader.stats().unwrap().storage, StorageKind::I8);assert!(!path.exists());drop(reader);- let db = VecDb::open(&path, 32, None, None).unwrap();+ let db = VecDb::open(&path, 32, None).unwrap();db.add("kept", &seeded_unit_vector(1, 32)).unwrap();db.flush().unwrap();let verify = || {for open in [VecDb::open, VecDb::open_read_only] {- let reopened = open(&path, 32, None, None).unwrap();- assert_eq!(reopened.stats().unwrap().metric, DistanceMetric::Cosine);+ let reopened = open(&path, 32, None).unwrap();+ assert_eq!(reopened.stats().unwrap().storage, StorageKind::I8);assert!(reopened.contains("kept"));- let explicit = open(&path, 32, None, Some(DistanceMetric::Cosine)).unwrap();- assert!(explicit.contains("kept"));}};verify();drop(db);verify();- let dot_path = dir.path().join("inner-product");- let db = VecDb::open(&dot_path, 32, None, Some(DistanceMetric::InnerProduct)).unwrap();+ let f32_path = dir.path().join("f32");+ let db = VecDb::open(&f32_path, 32, Some(StorageKind::F32)).unwrap();let reject = || {- let before = fs::read(&dot_path).unwrap();+ let before = fs::read(&f32_path).unwrap();for open in [VecDb::open, VecDb::open_read_only] {assert!(matches!(- open(&dot_path, 32, None, None),- Err(VecDbError::MetricMismatch {- expected: DistanceMetric::Cosine,- actual: DistanceMetric::InnerProduct+ open(&f32_path, 32, None),+ Err(VecDbError::StorageMismatch {+ expected: StorageKind::I8,+ actual: StorageKind::F32})));}- assert_eq!(fs::read(&dot_path).unwrap(), before);+ assert_eq!(fs::read(&f32_path).unwrap(), before);};reject();drop(db);reject();- }-- #[test]- fn omitted_storage_requires_i8_for_creation_and_every_reopen() {- let dir = TempDir::new().unwrap();- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {- let path = dir.path().join(metric.to_string());- let reader = VecDb::open_read_only(&path, 32, None, Some(metric)).unwrap();- assert_eq!(reader.stats().unwrap().storage, StorageKind::I8);- assert_eq!(reader.stats().unwrap().metric, metric);- assert!(!path.exists());- drop(reader);- let db = VecDb::open(&path, 32, None, Some(metric)).unwrap();- db.add("kept", &seeded_unit_vector(1, 32)).unwrap();- db.flush().unwrap();- let verify = || {- for open in [VecDb::open, VecDb::open_read_only] {- let reopened = open(&path, 32, None, Some(metric)).unwrap();- assert_eq!(reopened.stats().unwrap().storage, StorageKind::I8);- assert_eq!(reopened.stats().unwrap().metric, metric);- assert!(reopened.contains("kept"));- }- };- verify();- drop(db);- verify();- let f32_path = dir.path().join(format!("f32-{metric}"));- let db = VecDb::open(&f32_path, 32, Some(StorageKind::F32), Some(metric)).unwrap();- let reject = || {- let before = fs::read(&f32_path).unwrap();- for open in [VecDb::open, VecDb::open_read_only] {- assert!(matches!(- open(&f32_path, 32, None, Some(metric)),- Err(VecDbError::StorageMismatch {- expected: StorageKind::I8,- actual: StorageKind::F32- })- ));- }- assert_eq!(fs::read(&f32_path).unwrap(), before);- };- reject();- drop(db);- reject();- }let path = dir.path().join("invalid");for open in [VecDb::open, VecDb::open_read_only] {assert!(matches!(- open(&path, 8, None, Some(DistanceMetric::Cosine)),+ open(&path, 8, None),Err(VecDbError::InvalidDimensions {dims: 8,storage: StorageKind::I8@@ -1894,51 +1791,41 @@ mod tests {fn explicit_configuration_is_verified_on_live_and_disk_opens() {let dir = TempDir::new().unwrap();for storage in [StorageKind::F32, StorageKind::I8] {- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {- let path = dir.path().join(format!("{storage}-{metric}"));- let other_storage = match storage {- StorageKind::F32 => StorageKind::I8,- StorageKind::I8 => StorageKind::F32,- };- let other_metric = match metric {- DistanceMetric::Cosine => DistanceMetric::InnerProduct,- DistanceMetric::InnerProduct => DistanceMetric::Cosine,- };- let db = VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap();- db.add("kept", &seeded_unit_vector(1, 32)).unwrap();- db.flush().unwrap();- let verify = |live: Option<&VecDb>| {- let before = fs::read(&path).unwrap();- let snapshot_before = fs::read(snapshot_path(&path)).unwrap();- for open in [VecDb::open, VecDb::open_read_only] {- assert!(matches!(open(&path, 32, Some(other_storage), Some(metric)),- Err(VecDbError::StorageMismatch { expected, actual })- if expected == other_storage && actual == storage));- assert!(matches!(open(&path, 32, Some(storage), Some(other_metric)),- Err(VecDbError::MetricMismatch { expected, actual })- if expected == other_metric && actual == metric));- assert!(matches!(- open(&path, 64, Some(storage), Some(metric)),- Err(VecDbError::DimensionMismatch {- expected: 64,- actual: 32- })- ));- assert_eq!(fs::read(&path).unwrap(), before);- assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_before);- let reopened = open(&path, 32, Some(storage), Some(metric)).unwrap();- assert_eq!(reopened.stats().unwrap().storage, storage);- assert_eq!(reopened.stats().unwrap().metric, metric);- assert!(reopened.contains("kept"));- if let Some(live) = live {- assert!(Arc::ptr_eq(&live.shared, &reopened.shared));- }+ let path = dir.path().join(storage.to_string());+ let other_storage = match storage {+ StorageKind::F32 => StorageKind::I8,+ StorageKind::I8 => StorageKind::F32,+ };+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();+ db.add("kept", &seeded_unit_vector(1, 32)).unwrap();+ db.flush().unwrap();+ let verify = |live: Option<&VecDb>| {+ let before = fs::read(&path).unwrap();+ let snapshot_before = fs::read(snapshot_path(&path)).unwrap();+ for open in [VecDb::open, VecDb::open_read_only] {+ assert!(matches!(open(&path, 32, Some(other_storage)),+ Err(VecDbError::StorageMismatch { expected, actual })+ if expected == other_storage && actual == storage));+ assert!(matches!(+ open(&path, 64, Some(storage)),+ Err(VecDbError::DimensionMismatch {+ expected: 64,+ actual: 32+ })+ ));+ assert_eq!(fs::read(&path).unwrap(), before);+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_before);+ let reopened = open(&path, 32, Some(storage)).unwrap();+ assert_eq!(reopened.stats().unwrap().storage, storage);+ assert!(reopened.contains("kept"));+ if let Some(live) = live {+ assert!(Arc::ptr_eq(&live.shared, &reopened.shared));}- };- verify(Some(&db));- drop(db);- verify(None);- }+ }+ };+ verify(Some(&db));+ drop(db);+ verify(None);}}@@ -1946,51 +1833,40 @@ mod tests {fn missing_and_incomplete_indexes_use_the_explicit_configuration() {let dir = TempDir::new().unwrap();for storage in [StorageKind::F32, StorageKind::I8] {- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {- for incomplete in [false, true] {- let path = dir.path().join(format!("{storage}-{metric}-{incomplete}"));- let dims = storage.lane_width();- if incomplete {- fs::write(&path, [1, 2, 3]).unwrap();- }- let reader =- VecDb::open_read_only(&path, dims, Some(storage), Some(metric)).unwrap();- let stats = reader.stats().unwrap();- assert_eq!(stats.dims, dims);- assert_eq!(stats.storage, storage);- assert_eq!(stats.metric, metric);- assert!(reader.is_empty());- assert!(matches!(- reader.add("no", &seeded_unit_vector(1, dims)),- Err(VecDbError::ReadOnly)- ));- if incomplete {- assert_eq!(fs::read(&path).unwrap(), [1, 2, 3]);- } else {- assert!(!path.exists());- }- drop(reader);- let writer = VecDb::open(&path, dims, Some(storage), Some(metric)).unwrap();- assert_eq!(writer.stats().unwrap().storage, storage);- assert_eq!(writer.stats().unwrap().metric, metric);- writer.add("kept", &seeded_unit_vector(1, dims)).unwrap();- drop(writer);- let reopened =- VecDb::open_read_only(&path, dims, Some(storage), Some(metric)).unwrap();- assert!(reopened.contains("kept"));- assert_eq!(reopened.stats().unwrap().metric, metric);+ for incomplete in [false, true] {+ let path = dir.path().join(format!("{storage}-{incomplete}"));+ let dims = storage.lane_width();+ if incomplete {+ fs::write(&path, [1, 2, 3]).unwrap();+ }+ let reader = VecDb::open_read_only(&path, dims, Some(storage)).unwrap();+ let stats = reader.stats().unwrap();+ assert_eq!(stats.dims, dims);+ assert_eq!(stats.storage, storage);+ assert!(reader.is_empty());+ assert!(matches!(+ reader.add("no", &seeded_unit_vector(1, dims)),+ Err(VecDbError::ReadOnly)+ ));+ if incomplete {+ assert_eq!(fs::read(&path).unwrap(), [1, 2, 3]);+ } else {+ assert!(!path.exists());}+ drop(reader);+ let writer = VecDb::open(&path, dims, Some(storage)).unwrap();+ assert_eq!(writer.stats().unwrap().storage, storage);+ writer.add("kept", &seeded_unit_vector(1, dims)).unwrap();+ drop(writer);+ let reopened = VecDb::open_read_only(&path, dims, Some(storage)).unwrap();+ assert!(reopened.contains("kept"));+ assert_eq!(reopened.stats().unwrap().storage, storage);}}for open in [VecDb::open, VecDb::open_read_only] {let path = dir.path().join("invalid");assert!(matches!(- open(- &path,- 8,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ open(&path, 8, Some(StorageKind::I8)),Err(VecDbError::InvalidDimensions {dims: 8,storage: StorageKind::I8@@ -2004,13 +1880,7 @@ mod tests {fn cosine_scores_unnormalized_vectors_in_every_search_path() {let dir = TempDir::new().unwrap();for storage in [StorageKind::F32, StorageKind::I8] {- let db = VecDb::open(- &dir.path().join(storage.to_string()),- 32,- Some(storage),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let db = VecDb::open(&dir.path().join(storage.to_string()), 32, Some(storage)).unwrap();let vector = |x, y| {let mut values = vec![0.0; 32];values[0] = x;@@ -2085,80 +1955,64 @@ mod tests {}#[test]- fn metrics_survive_replacement_compaction_snapshot_replay_and_reset() {+ fn cosine_scores_survive_replacement_compaction_snapshot_replay_and_reset() {let dir = TempDir::new().unwrap();for storage in [StorageKind::F32, StorageKind::I8] {- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {- let path = dir.path().join(format!("{storage}-{metric}"));- let db = VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap();- let mut vector = vec![0.0; 32];- vector[0] = 1.0;- db.add("removed", &vector).unwrap();- db.add("kept", &vector).unwrap();- db.remove("removed").unwrap();- vector[0] = 0.0;- vector[1] = 1.0;- db.add("kept", &vector).unwrap();- let mut recycled = vec![0.0; 32];- recycled[2] = 1.0;- db.add("recycled", &recycled).unwrap();- db.remove("recycled").unwrap();- compact(&db.shared, &mut db.shared.writer_half()).unwrap();- db.flush().unwrap();- let query = scaled(&vector, 2.0);- let expected = if metric == DistanceMetric::Cosine {- 0.0- } else {- -1.0- };- let assert_score = |db: &VecDb| {- assert_eq!(db.stats().unwrap().metric, metric);- for exact in [false, true] {- let found = db- .search(- &query,- &SearchParams {- limit: Some(1),- exact,- ..Default::default()- },- )- .unwrap();- assert_eq!(found[0].key, "kept");- assert!((found[0].distance - expected).abs() < 1.0e-5);- }- };- assert_score(&db);- drop(db);- for snapshot in [true, false] {- if !snapshot {- remove_snapshot(&path).unwrap();- }- assert_score(- &VecDb::open_read_only(&path, 32, Some(storage), Some(metric)).unwrap(),- );- assert_score(&VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap());+ let path = dir.path().join(storage.to_string());+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();+ let mut vector = vec![0.0; 32];+ vector[0] = 1.0;+ db.add("removed", &vector).unwrap();+ db.add("kept", &vector).unwrap();+ db.remove("removed").unwrap();+ vector[0] = 0.0;+ vector[1] = 1.0;+ db.add("kept", &vector).unwrap();+ let mut recycled = vec![0.0; 32];+ recycled[2] = 1.0;+ db.add("recycled", &recycled).unwrap();+ db.remove("recycled").unwrap();+ compact(&db.shared, &mut db.shared.writer_half()).unwrap();+ db.flush().unwrap();+ let query = scaled(&vector, 2.0);+ let assert_score = |db: &VecDb| {+ for exact in [false, true] {+ let found = db+ .search(+ &query,+ &SearchParams {+ limit: Some(1),+ exact,+ ..Default::default()+ },+ )+ .unwrap();+ assert_eq!(found[0].key, "kept");+ assert!(found[0].distance.abs() < 1.0e-5);}- let db = VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap();- db.reset().unwrap();- db.add("kept", &vector).unwrap();- assert_score(&db);- drop(db);- assert_score(&VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap());+ };+ assert_score(&db);+ drop(db);+ for snapshot in [true, false] {+ if !snapshot {+ remove_snapshot(&path).unwrap();+ }+ assert_score(&VecDb::open_read_only(&path, 32, Some(storage)).unwrap());+ assert_score(&VecDb::open(&path, 32, Some(storage)).unwrap());}+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();+ db.reset().unwrap();+ db.add("kept", &vector).unwrap();+ assert_score(&db);+ drop(db);+ assert_score(&VecDb::open(&path, 32, Some(storage)).unwrap());}}#[test]fn cosine_f32_handles_extreme_finite_magnitudes() {let dir = TempDir::new().unwrap();- let db = VecDb::open(- &dir.path().join("db"),- 8,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let db = VecDb::open(&dir.path().join("db"), 8, Some(StorageKind::F32)).unwrap();for magnitude in [f32::MAX, f32::MIN_POSITIVE, f32::from_bits(1)] {let vector = vec![magnitude; 8];db.add("same", &vector).unwrap();@@ -2232,13 +2086,7 @@ mod tests {}fn open_writer(path: &Path) -> VecDb {- VecDb::open(- path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap()+ VecDb::open(path, DIMS, Some(StorageKind::F32)).unwrap()}fn generation_of(path: &Path) -> [u8; 16] {@@ -2376,13 +2224,7 @@ mod tests {db.flush().unwrap();assert_eq!(db.stats().unwrap().records_since_snapshot, 0);drop(db);- let read_only = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();let reopened = open_writer(&path);let mut index = 0;for query in &queries {@@ -2537,21 +2379,11 @@ mod tests {let log_bytes = fs::read(path).unwrap();let snapshot_bytes = snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap());assert!(matches!(- VecDb::open(- path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(path, DIMS, Some(StorageKind::F32)),Err(VecDbError::Corrupt(_))));assert!(matches!(- VecDb::open_read_only(- path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open_read_only(path, DIMS, Some(StorageKind::F32)),Err(VecDbError::Corrupt(_))));assert_eq!(fs::read(path).unwrap(), log_bytes);@@ -2588,22 +2420,10 @@ mod tests {bulk_add(&db, &bulk_entries(0, 2, 500)).unwrap();db.flush().unwrap();drop(db);- let mut captured_for_read_only = Log::open(- File::open(&path).unwrap(),- &path,- DIMS,- StorageKind::F32,- DistanceMetric::Cosine,- )- .unwrap();- let mut captured_for_writer = Log::open(- File::open(&path).unwrap(),- &path,- DIMS,- StorageKind::F32,- DistanceMetric::Cosine,- )- .unwrap();+ let mut captured_for_read_only =+ Log::open(File::open(&path).unwrap(), &path, DIMS, StorageKind::F32).unwrap();+ let mut captured_for_writer =+ Log::open(File::open(&path).unwrap(), &path, DIMS, StorageKind::F32).unwrap();let stale_end = captured_for_read_only.current_end_offset();let racer = open_writer(&path);bulk_add(&racer, &bulk_entries(2, 2, 500)).unwrap();@@ -2638,14 +2458,8 @@ mod tests {bulk_add(&db, &bulk_entries(0, 2, 700)).unwrap();db.flush().unwrap();drop(db);- let mut captured_for_read_only = Log::open(- File::open(&path).unwrap(),- &path,- DIMS,- StorageKind::F32,- DistanceMetric::Cosine,- )- .unwrap();+ let mut captured_for_read_only =+ Log::open(File::open(&path).unwrap(), &path, DIMS, StorageKind::F32).unwrap();let stale_end = captured_for_read_only.current_end_offset();let racer = open_writer(&path);bulk_add(&racer, &bulk_entries(2, 2, 700)).unwrap();@@ -2798,13 +2612,7 @@ mod tests {let last_start = bytes.len() - ADD_RECORD_LEN as usize;bytes[last_start + 10] ^= 0x01;fs::write(&path, &bytes).unwrap();- let read_only = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert_eq!(read_only.len(), 3);assert!(!read_only.contains("key-3"));drop(read_only);@@ -2833,11 +2641,11 @@ mod tests {fs::write(&path, &bytes).unwrap();let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();assert!(matches!(- VecDb::open(&path, DIMS, Some(StorageKind::F32), Some(DistanceMetric::Cosine)),+ VecDb::open(&path, DIMS, Some(StorageKind::F32)),Err(VecDbError::Corrupt(message)) if message.contains("version")));assert!(matches!(- VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32), Some(DistanceMetric::Cosine)),+ VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)),Err(VecDbError::Corrupt(message)) if message.contains("version")));assert_eq!(fs::read(&path).unwrap(), bytes);@@ -2925,13 +2733,7 @@ mod tests {let path = dir.path().join("db");let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 3, 120)).unwrap();- let joined = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let joined = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&db.shared, &joined.shared));assert_eq!(joined.len(), 3);joined.add("joined", &seeded_unit_vector(9, DIMS)).unwrap();@@ -2952,13 +2754,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let first = open_writer(&path);- let second = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let second = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&first.shared, &second.shared));let via_first = seeded_unit_vector(11, DIMS);let via_second = seeded_unit_vector(22, DIMS);@@ -2973,7 +2769,7 @@ mod tests {assert!(first.contains("via-second"));assert_eq!(first.len(), 2);assert!(matches!(- VecDb::open(&path, DIMS + 8, Some(StorageKind::F32), Some(DistanceMetric::Cosine)),+ VecDb::open(&path, DIMS + 8, Some(StorageKind::F32)),Err(VecDbError::DimensionMismatch {expected,actual@@ -2991,23 +2787,12 @@ mod tests {fs::hard_link(&path, &alias).unwrap();let bytes = fs::read(&path).unwrap();assert!(matches!(- VecDb::open(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),Err(VecDbError::Locked(_))));assert_eq!(fs::read(&path).unwrap(), bytes);let second = open_writer(&path);- let reader = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let reader = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();first.add("first", &basis_vector(0)).unwrap();second.add("second", &basis_vector(1)).unwrap();assert!(reader.contains("first"));@@ -3033,25 +2818,15 @@ mod tests {fs::hard_link(&path, &alias).unwrap();for name in [&path, &alias] {assert!(matches!(- VecDb::open(- name,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(name, DIMS, Some(StorageKind::F32)),Err(VecDbError::Locked(_))));}assert_eq!(fs::read(&path).unwrap(), partial);assert!(- VecDb::open_read_only(- &alias,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- )- .unwrap()- .is_empty()+ VecDb::open_read_only(&alias, I8_DIMS, Some(StorageKind::I8))+ .unwrap()+ .is_empty());drop(holder);let writer = open_writer(&alias);@@ -3071,20 +2846,9 @@ mod tests {.as_str(){"blocked" => {- let result = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- );+ let result = VecDb::open(&path, DIMS, Some(StorageKind::F32));assert!(matches!(result, Err(VecDbError::Locked(_))));- let reader = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let reader = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(reader.contains("first"));}"write" => {@@ -3129,13 +2893,7 @@ mod tests {assert_eq!(first.len(), 1);drop(first);run_hard_link_writer_process(&alias, "write");- let reader = VecDb::open_read_only(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let reader = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();assert_eq!(reader.len(), 2);assert!(reader.contains("first"));assert!(reader.contains("next"));@@ -3165,21 +2923,10 @@ mod tests {}assert_eq!(fs::metadata(&alias).unwrap().nlink(), 1);assert!(matches!(- VecDb::open(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),Err(VecDbError::Locked(_))));- let reader = VecDb::open_read_only(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let reader = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();assert!(reader.contains("first"));drop(reader);run_hard_link_writer_process(&alias, "blocked");@@ -3204,25 +2951,14 @@ mod tests {let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 10, 800)).unwrap();fs::hard_link(&path, &alias).unwrap();- let observer = VecDb::open_read_only(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let observer = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();assert_eq!(observer.len(), 10);compact(&db.shared, &mut db.shared.writer_half()).unwrap();assert_eq!(db.len(), 10);fs::remove_file(&alias).unwrap();fs::hard_link(&path, &alias).unwrap();assert!(matches!(- VecDb::open(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),Err(VecDbError::Locked(_))));db.reset().unwrap();@@ -3230,12 +2966,7 @@ mod tests {fs::remove_file(&alias).unwrap();fs::hard_link(&path, &alias).unwrap();assert!(matches!(- VecDb::open(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),Err(VecDbError::Locked(_))));assert_eq!(db.len(), 1);@@ -3250,13 +2981,7 @@ mod tests {let plain = subdir.join("db");let dotted = subdir.join("..").join("indexes").join("db");let first = open_writer(&plain);- let second = VecDb::open(- &dotted,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let second = VecDb::open(&dotted, DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&first.shared, &second.shared));let vector = seeded_unit_vector(5, DIMS);first.add("shared", &vector).unwrap();@@ -3269,13 +2994,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let first = open_writer(&path);- let second = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let second = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();first.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();drop(first);assert!(second.contains("kept"));@@ -3289,13 +3008,7 @@ mod tests {assert_eq!(reopened.len(), 2);assert!(reopened.contains("kept"));assert!(reopened.contains("still-open"));- let again = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let again = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&reopened.shared, &again.shared));}@@ -3315,13 +3028,7 @@ mod tests {thread::spawn(move || {barrier.wait();for _ in 0..CYCLES {- let db = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let db = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(db.contains("seed"));drop(db);}@@ -3465,14 +3172,7 @@ mod tests {}let opener = {let path = path.clone();- thread::spawn(move || {- VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- })+ thread::spawn(move || VecDb::open(&path, DIMS, Some(StorageKind::F32)))};thread::sleep(Duration::from_millis(5));drop(frozen);@@ -3543,12 +3243,7 @@ mod tests {fs::create_dir(&path).unwrap();let key = registry_key_for(&path).unwrap();assert!(matches!(- VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&path, DIMS, Some(StorageKind::F32)),Err(VecDbError::Io { .. })));assert!(!registry().contains_key(&key));@@ -3600,13 +3295,7 @@ mod tests {let path = dir.path().join("db");let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 3, 130)).unwrap();- let read_only = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&db.shared, &read_only.shared));assert_eq!(read_only.len(), 3);assert!(read_only.contains("key-1"));@@ -3629,13 +3318,7 @@ mod tests {));assert!(matches!(read_only.flush(), Err(VecDbError::ReadOnly)));assert!(matches!(read_only.reset(), Err(VecDbError::ReadOnly)));- let another = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let another = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(matches!(another.delete(), Err(VecDbError::ReadOnly)));assert!(db.contains("later"));assert!(path.exists());@@ -3646,13 +3329,7 @@ mod tests {const DIMS: usize = I8_DIMS;let dir = TempDir::new().unwrap();let path = dir.path().join("absent");- let db = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let db = VecDb::open_read_only(&path, DIMS, Some(StorageKind::I8)).unwrap();assert_eq!(db.len(), 0);assert!(db.is_empty());assert!(!db.contains("anything"));@@ -3669,7 +3346,6 @@ mod tests {dead_count: 0,dims: DIMS,storage: StorageKind::I8,- metric: DistanceMetric::Cosine,log_bytes: 0,records_since_snapshot: 0,approximate_memory_bytes: 0,@@ -3687,13 +3363,7 @@ mod tests {let db = open_writer(&path);db.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();drop(db);- let read_only = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();let vector = seeded_unit_vector(4, DIMS);assert!(matches!(read_only.add("x", &vector),@@ -3729,13 +3399,7 @@ mod tests {let db = open_writer(&path);bulk_add(&db, &bulk_entries(0, 100, 140)).unwrap();drop(db);- let snapshot_view = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let snapshot_view = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert_eq!(snapshot_view.len(), 100);let db = open_writer(&path);assert!(!Arc::ptr_eq(&snapshot_view.shared, &db.shared));@@ -3752,23 +3416,11 @@ mod tests {assert!(snapshot_view.contains("key-5"));assert!(!snapshot_view.contains("fresh"));assert_own_nearest(&snapshot_view, "key-5", &seeded_unit_vector(145, DIMS));- let live_view = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let live_view = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&live_view.shared, &db.shared));assert!(live_view.contains("fresh"));drop(db);- let fresh_view = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let fresh_view = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert_eq!(fresh_view.len(), 69);assert!(!fresh_view.contains("key-5"));assert!(fresh_view.contains("fresh"));@@ -4265,13 +3917,7 @@ mod tests {let path = dir.path().join("db");let partial = [0x45u8, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00];fs::write(&path, partial).unwrap();- let db = VecDb::open_read_only(- &path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let db = VecDb::open_read_only(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();assert!(db.is_empty());assert_eq!(db.stats().unwrap().log_bytes, 0);assert_eq!(fs::read(&path).unwrap(), partial);@@ -4284,14 +3930,14 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");drop(- Log::create(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ Log::create(&path, DIMS, StorageKind::F32).unwrap().into_file(),);let old_generation = generation_of(&path);let temp = temp_sibling(&path);drop(- Log::create(&temp, DIMS, StorageKind::F32, DistanceMetric::InnerProduct)+ Log::create(&temp, DIMS, StorageKind::F32).unwrap().into_file(),);@@ -4300,9 +3946,7 @@ mod tests {assert!(!temp.exists());assert_eq!(generation_of(&path), new_generation);assert_ne!(generation_of(&path), old_generation);- let log =- open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::InnerProduct)- .unwrap();+ let log = open_existing_writer_log(&path, DIMS, StorageKind::F32).unwrap();assert_eq!(log.generation(), new_generation);drop(log);assert!(matches!(@@ -4372,9 +4016,7 @@ mod tests {let WriterState { lock, log, .. } = take_writer(&db);let checkpoint = log.checkpoint();drop(log);- let mut foreign =- open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)- .unwrap();+ let mut foreign = open_existing_writer_log(&path, DIMS, StorageKind::F32).unwrap();foreign.append(&[LogEntry::Add {key: "foreign",@@ -4409,8 +4051,7 @@ mod tests {let old_len = log.current_end_offset();drop(log);fs::remove_file(&path).unwrap();- let mut replacement =- Log::create(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();+ let mut replacement = Log::create(&path, DIMS, StorageKind::F32).unwrap();replacement.append(&[LogEntry::Add {key: "other",@@ -4642,7 +4283,6 @@ mod tests {dead_count: 0,dims: DIMS,storage: StorageKind::F32,- metric: DistanceMetric::Cosine,log_bytes: 32,records_since_snapshot: 0,approximate_memory_bytes: 0,@@ -4662,13 +4302,7 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let first = open_writer(&path);- let second = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let second = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();bulk_add(&first, &bulk_entries(0, 10, 220)).unwrap();assert_eq!(second.len(), 10);second.reset().unwrap();@@ -4740,7 +4374,7 @@ mod tests {drop(log);remove_data_files(&path).unwrap();drop(- Log::create(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)+ Log::create(&path, DIMS, StorageKind::F32).unwrap().into_file(),);@@ -4844,20 +4478,8 @@ mod tests {let first = open_writer(&path);bulk_add(&first, &bulk_entries(0, 5, 400)).unwrap();first.flush().unwrap();- let survivor = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();- let observer = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let survivor = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();+ let observer = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();first.delete().unwrap();assert!(!path.exists());assert!(!snapshot_exists(&path));@@ -4914,13 +4536,7 @@ mod tests {let barrier = Arc::new(Barrier::new(READERS + 2));let mut readers = Vec::new();for reader_index in 0..READERS {- let handle = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let handle = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();let stop = Arc::clone(&stop);let barrier = Arc::clone(&barrier);let expected = Arc::clone(&expected);@@ -4943,13 +4559,7 @@ mod tests {}));}let flusher = {- let handle = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let handle = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();let stop = Arc::clone(&stop);let barrier = Arc::clone(&barrier);thread::spawn(move || {@@ -5061,66 +4671,11 @@ mod tests {assert!(!reopened.contains("bad"));}- #[test]- fn inner_product_indexes_reject_vectors_off_the_unit_sphere() {- let dir = TempDir::new().unwrap();- let path = dir.path().join("db");- let db = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::InnerProduct),- )- .unwrap();- let unit = seeded_unit_vector(1, DIMS);- db.add("unit", &unit).unwrap();- for delta in [-1.0e-4f32, 1.0e-4] {- let nudged = scaled(&unit, (1.0 + delta).sqrt());- db.add(&format!("nudged{delta}"), &nudged).unwrap();- }- let live = db.len();- let log_bytes = db.stats().unwrap().log_bytes;- for factor in [0.5, 0.99] {- let off_sphere = scaled(&unit, factor);- assert!(matches!(- db.add("off-sphere", &off_sphere),- Err(VecDbError::InvalidVector(_))- ));- assert!(!db.contains("off-sphere"));- let mut batch = bulk_entries(10, 3, 200);- batch[1].1 = off_sphere;- assert!(matches!(- bulk_add(&db, &batch),- Err(VecDbError::InvalidVector(_))- ));- assert!(!db.contains("key-10"));- assert!(!db.contains("key-12"));- }- assert_eq!(db.len(), live);- assert_eq!(db.stats().unwrap().log_bytes, log_bytes);- drop(db);- let reopened = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::InnerProduct),- )- .unwrap();- assert_eq!(reopened.len(), live);- assert!(reopened.contains("unit"));- }-#[test]fn cosine_indexes_keep_taking_vectors_off_the_unit_sphere() {let dir = TempDir::new().unwrap();let path = dir.path().join("db");- let db = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let db = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();let half = scaled(&seeded_unit_vector(1, DIMS), 0.5);db.add("half", &half).unwrap();let batch: Vec<(String, Vec<f32>)> = bulk_entries(10, 3, 200)@@ -5422,13 +4977,7 @@ mod tests {assert_eq!(db.len(), 1);assert!(!db.contains("x"));assert_eq!(db.stats().unwrap().log_bytes, log_bytes);- let read_only = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(matches!(read_only.bulk_add(&keys, &vectors),Err(VecDbError::LengthMismatch { .. })@@ -5443,20 +4992,8 @@ mod tests {bulk_add(&first, &bulk_entries(0, 5, 950)).unwrap();first.flush().unwrap();fs::write(temp_sibling(&path), [7u8; 10]).unwrap();- let survivor = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();- let observer = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let survivor = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();+ let observer = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();VecDb::purge(&path).unwrap();assert!(!path.exists());assert!(!snapshot_exists(&path));@@ -5560,13 +5097,7 @@ mod tests {let alias = dir.path().join("alias");let via_path = open_writer(&path);std::os::unix::fs::symlink(&path, &alias).unwrap();- let via_alias = VecDb::open(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let via_alias = VecDb::open(&alias, DIMS, Some(StorageKind::F32)).unwrap();via_path.add("through-real", &seeded_unit_vector(5, DIMS)).unwrap();@@ -5578,13 +5109,7 @@ mod tests {assert_eq!(via_path.len(), 2);assert!(lock_path(&path).exists());assert!(!lock_path(&alias).exists());- let read_alias = VecDb::open_read_only(- &alias,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_alias = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();via_path.add("late", &seeded_unit_vector(7, DIMS)).unwrap();assert!(read_alias.contains("late"));}@@ -5729,15 +5254,7 @@ mod tests {assert!(!snapshot_exists(&path));let opener = {let path = path.clone();- thread::spawn(move || {- VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap()- })+ thread::spawn(move || VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap())};let key = registry_key_for(&path).unwrap();let deadline = Instant::now() + Duration::from_secs(60);@@ -5746,13 +5263,7 @@ mod tests {let registered =existing_path_slot(&key).is_some_and(|slot| lock_slot(&slot).holds_live());if registered {- break VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ break VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();}thread::yield_now();};@@ -5771,13 +5282,7 @@ mod tests {expected_stored);assert!(!opener.is_finished());- let writer = VecDb::open(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let writer = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&writer.shared, &joined.shared));let adder = {let vector = replacement.clone();@@ -5932,13 +5437,7 @@ mod tests {assert_eq!(db.get_attrs("removed"), None);assert_eq!(db.get_attrs("recycled").unwrap(), sample_attrs(22));};- let read_only = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();verify(&read_only);drop(read_only);let reopened = open_writer(&path);@@ -6078,13 +5577,7 @@ mod tests {vec![Some(sample_attrs(7)), None, None, None]);assert_eq!(db.get_attrs("x").unwrap(), sample_attrs(7));- let read_only = VecDb::open_read_only(- &path,- DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();assert!(matches!(read_only.bulk_add_with_attrs(&keys, &vectors, &[None]),Err(VecDbError::LengthMismatch { .. })@@ -6140,13 +5633,7 @@ mod tests {}fn open_i8(path: &Path) -> VecDb {- VecDb::open(- path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap()+ VecDb::open(path, I8_DIMS, Some(StorageKind::I8)).unwrap()}fn stored_vectors(db: &VecDb) -> Vec<(String, StoredVector)> {@@ -6177,14 +5664,7 @@ mod tests {}fn logged_add_records(path: &Path, dims: usize) -> Vec<(String, StoredVector)> {- let mut log = Log::open(- File::open(path).unwrap(),- path,- dims,- StorageKind::I8,- DistanceMetric::Cosine,- )- .unwrap();+ let mut log = Log::open(File::open(path).unwrap(), path, dims, StorageKind::I8).unwrap();let mut scanner = log.scan().unwrap();let mut records = Vec::new();while let Some((record, _)) = scanner.next_record().unwrap() {@@ -6200,21 +5680,10 @@ mod tests {let dir = TempDir::new().unwrap();let narrow = dir.path().join("narrow");let wide = dir.path().join("wide");- let live = VecDb::open(- &narrow,- 8,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let live = VecDb::open(&narrow, 8, Some(StorageKind::F32)).unwrap();live.add("n", &seeded_unit_vector(1, 8)).unwrap();assert!(matches!(- VecDb::open(- &narrow,- 8,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&narrow, 8, Some(StorageKind::I8)),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32@@ -6222,45 +5691,22 @@ mod tests {));drop(live);assert!(matches!(- VecDb::open(- &narrow,- 8,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&narrow, 8, Some(StorageKind::I8)),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));- drop(- VecDb::open(- &wide,- I8_DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap(),- );+ drop(VecDb::open(&wide, I8_DIMS, Some(StorageKind::F32)).unwrap());assert!(matches!(- VecDb::open(- &wide,- 8,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&wide, 8, Some(StorageKind::I8)),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));assert!(matches!(- VecDb::open(- &wide,- 8,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&wide, 8, Some(StorageKind::F32)),Err(VecDbError::DimensionMismatch {expected: 8,actual: I8_DIMS@@ -6268,24 +5714,14 @@ mod tests {));drop(open_i8(&dir.path().join("i8")));assert!(matches!(- VecDb::open_read_only(- &dir.path().join("i8"),- 8,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open_read_only(&dir.path().join("i8"), 8, Some(StorageKind::I8)),Err(VecDbError::DimensionMismatch {expected: 8,actual: I8_DIMS})));assert!(matches!(- VecDb::open(- &dir.path().join("fresh"),- 8,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&dir.path().join("fresh"), 8, Some(StorageKind::I8)),Err(VecDbError::InvalidDimensions {dims: 8,storage: StorageKind::I8@@ -6299,100 +5735,50 @@ mod tests {let dir = TempDir::new().unwrap();let f32_path = dir.path().join("f32");let i8_path = dir.path().join("i8");- let f32_db = VecDb::open(- &f32_path,- I8_DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let f32_db = VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::F32)).unwrap();f32_db.add("f", &seeded_unit_vector(1, I8_DIMS)).unwrap();assert_eq!(f32_db.stats().unwrap().storage, StorageKind::F32);assert!(matches!(- VecDb::open(- &f32_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::I8)),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));- let verified = VecDb::open(- &f32_path,- I8_DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let verified = VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::F32)).unwrap();assert!(Arc::ptr_eq(&verified.shared, &f32_db.shared));drop(verified);drop(f32_db);assert!(matches!(- VecDb::open(- &f32_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::I8)),Err(VecDbError::StorageMismatch {expected: StorageKind::I8,actual: StorageKind::F32})));assert_eq!(- VecDb::open(- &f32_path,- I8_DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- )- .unwrap()- .stats()- .unwrap()- .storage,+ VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::F32))+ .unwrap()+ .stats()+ .unwrap()+ .storage,StorageKind::F32);- let i8_db = VecDb::open(- &i8_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let i8_db = VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();let vector = seeded_unit_vector(2, I8_DIMS);i8_db.add("i", &vector).unwrap();assert_eq!(i8_db.stats().unwrap().storage, StorageKind::I8);- let joined = VecDb::open(- &i8_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let joined = VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();assert!(Arc::ptr_eq(&joined.shared, &i8_db.shared));assert_eq!(joined.stats().unwrap().storage, StorageKind::I8);assert!(matches!(- VecDb::open(- &i8_path,- I8_DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::F32)),Err(VecDbError::StorageMismatch {expected: StorageKind::F32,actual: StorageKind::I8})));- let read_alias = VecDb::open_read_only(- &i8_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_alias = VecDb::open_read_only(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();assert!(Arc::ptr_eq(&read_alias.shared, &i8_db.shared));assert_eq!(read_alias.stats().unwrap().storage, StorageKind::I8);i8_db.flush().unwrap();@@ -6400,71 +5786,36 @@ mod tests {drop(joined);drop(i8_db);assert_eq!(VecDb::open_cost(&i8_path), OpenCost::Ready);- let standalone = VecDb::open_read_only(- &i8_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let standalone = VecDb::open_read_only(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();assert_eq!(standalone.stats().unwrap().storage, StorageKind::I8);assert!(standalone.contains("i"));assert_own_nearest_i8(&standalone, "i", &vector);drop(standalone);assert!(matches!(- VecDb::open(- &i8_path,- I8_DIMS,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::F32)),Err(VecDbError::StorageMismatch {expected: StorageKind::F32,actual: StorageKind::I8})));- let detected = VecDb::open(- &i8_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let detected = VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();assert_eq!(detected.stats().unwrap().storage, StorageKind::I8);assert!(detected.contains("i"));assert_own_nearest_i8(&detected, "i", &vector);assert!(matches!(- VecDb::open(- &dir.path().join("narrow"),- 16,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- ),+ VecDb::open(&dir.path().join("narrow"), 16, Some(StorageKind::I8)),Err(VecDbError::InvalidDimensions {dims: 16,storage: StorageKind::I8})));- assert!(- VecDb::open(- &dir.path().join("narrow"),- 16,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine)- )- .is_ok()- );+ assert!(VecDb::open(&dir.path().join("narrow"), 16, Some(StorageKind::F32)).is_ok());assert_eq!(- VecDb::open_read_only(- &dir.path().join("absent"),- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine)- )- .unwrap()- .stats()- .unwrap()- .storage,+ VecDb::open_read_only(&dir.path().join("absent"), I8_DIMS, Some(StorageKind::I8))+ .unwrap()+ .stats()+ .unwrap()+ .storage,StorageKind::I8);}@@ -6595,32 +5946,14 @@ mod tests {stored_after);};- let read_only = VecDb::open_read_only(- &path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let read_only = VecDb::open_read_only(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();verify(&read_only);drop(read_only);- let with_snapshot = VecDb::open(- &path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let with_snapshot = VecDb::open(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();verify(&with_snapshot);drop(with_snapshot);fs::remove_file(snapshot_path(&path)).unwrap();- let rebuilt = VecDb::open(- &path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let rebuilt = VecDb::open(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();verify(&rebuilt);let from_copy = open_i8(©_path);verify(&from_copy);@@ -6653,24 +5986,12 @@ mod tests {db.flush().unwrap();}{- let db = VecDb::open(- &split_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let db = VecDb::open(&split_path, I8_DIMS, Some(StorageKind::I8)).unwrap();assert_eq!(db.stats().unwrap().storage, StorageKind::I8);db.add("tail-new", &tail_new).unwrap();db.add("key-3", &older_upsert).unwrap();}- let split = VecDb::open(- &split_path,- I8_DIMS,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let split = VecDb::open(&split_path, I8_DIMS, Some(StorageKind::I8)).unwrap();let whole = open_i8(&whole_path);bulk_add(&whole, &i8_entries(0, 40, 500)).unwrap();whole.add("tail-new", &tail_new).unwrap();@@ -6694,20 +6015,8 @@ mod tests {fn i8_stats_report_the_storage_and_a_quarter_of_the_vector_memory() {let dir = TempDir::new().unwrap();let dims = 512;- let f32_db = VecDb::open(- &dir.path().join("f32"),- dims,- Some(StorageKind::F32),- Some(DistanceMetric::Cosine),- )- .unwrap();- let i8_db = VecDb::open(- &dir.path().join("i8"),- dims,- Some(StorageKind::I8),- Some(DistanceMetric::Cosine),- )- .unwrap();+ let f32_db = VecDb::open(&dir.path().join("f32"), dims, Some(StorageKind::F32)).unwrap();+ let i8_db = VecDb::open(&dir.path().join("i8"), dims, Some(StorageKind::I8)).unwrap();let entries: Vec<(String, Vec<f32>)> = (0..200u64).map(|index| (format!("key-{index}"), seeded_unit_vector(index, dims))).collect();