Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

vecdb: drop the inner product metric and keep cosine

laurenspriem84c6da6000f7
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rs
index b927005996..a75b35e5f5 100644
--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs
+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs
@@ -18,7 +18,6 @@ pub enum RustVecDbError {
UnboundedSearch { message: String },
LengthMismatch { message: String },
StorageMismatch { message: String },
- MetricMismatch { message: String },
}
impl From<vecdb::VecDbError> for RustVecDbError {
@@ -38,7 +37,6 @@ impl From<vecdb::VecDbError> for RustVecDbError {
vecdb::VecDbError::UnboundedSearch => Self::UnboundedSearch { message },
vecdb::VecDbError::LengthMismatch { .. } => Self::LengthMismatch { message },
vecdb::VecDbError::StorageMismatch { .. } => Self::StorageMismatch { message },
- vecdb::VecDbError::MetricMismatch { .. } => Self::MetricMismatch { message },
}
}
}
@@ -63,26 +61,6 @@ fn to_api_storage(storage: vecdb::StorageKind) -> VecDbStorage {
}
}
-#[derive(Clone, Copy, Debug)]
-pub enum VecDbMetric {
- InnerProduct,
- Cosine,
-}
-
-fn to_engine_metric(metric: VecDbMetric) -> vecdb::DistanceMetric {
- match metric {
- VecDbMetric::InnerProduct => vecdb::DistanceMetric::InnerProduct,
- VecDbMetric::Cosine => vecdb::DistanceMetric::Cosine,
- }
-}
-
-fn to_api_metric(metric: vecdb::DistanceMetric) -> VecDbMetric {
- match metric {
- vecdb::DistanceMetric::InnerProduct => VecDbMetric::InnerProduct,
- vecdb::DistanceMetric::Cosine => VecDbMetric::Cosine,
- }
-}
-
#[derive(Clone, Debug)]
pub enum VecDbAttrValue {
Str(String),
@@ -146,7 +124,6 @@ pub struct VecDbStats {
pub dead_count: u32,
pub dims: u32,
pub storage: VecDbStorage,
- pub metric: VecDbMetric,
pub log_bytes: u64,
pub records_since_snapshot: u32,
pub approximate_memory_bytes: u64,
@@ -192,16 +169,10 @@ impl VecDb {
file_path: String,
dimensions: u32,
storage: Option<VecDbStorage>,
- metric: Option<VecDbMetric>,
) -> Result<Self, RustVecDbError> {
let path = Path::new(&file_path);
let dims = dimensions as usize;
- let inner = vecdb::VecDb::open(
- path,
- dims,
- storage.map(to_engine_storage),
- metric.map(to_engine_metric),
- )?;
+ let inner = vecdb::VecDb::open(path, dims, storage.map(to_engine_storage))?;
Ok(Self { inner })
}
@@ -209,14 +180,12 @@ impl VecDb {
file_path: String,
dimensions: u32,
storage: Option<VecDbStorage>,
- metric: Option<VecDbMetric>,
) -> Result<Self, RustVecDbError> {
Ok(Self {
inner: vecdb::VecDb::open_read_only(
Path::new(&file_path),
dimensions as usize,
storage.map(to_engine_storage),
- metric.map(to_engine_metric),
)?,
})
}
@@ -391,7 +360,6 @@ impl VecDb {
dead_count: stats.dead_count as u32,
dims: stats.dims as u32,
storage: to_api_storage(stats.storage),
- metric: to_api_metric(stats.metric),
log_bytes: stats.log_bytes,
records_since_snapshot: stats.records_since_snapshot as u32,
approximate_memory_bytes: stats.approximate_memory_bytes as u64,
@@ -451,56 +419,26 @@ mod tests {
name.to_string()
}
- #[test]
- fn omitted_metric_defaults_to_cosine() {
- let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), 32, None, None).unwrap();
- assert!(matches!(
- db.get_index_stats().unwrap().metric,
- VecDbMetric::Cosine
- ));
- drop(db);
- let reader = VecDb::open_read_only(dir.db_path(), 32, None, None).unwrap();
- assert!(matches!(
- reader.get_index_stats().unwrap().metric,
- VecDbMetric::Cosine
- ));
- }
-
#[test]
fn omitted_storage_defaults_to_i8() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), 32, None, Some(VecDbMetric::InnerProduct)).unwrap();
+ let db = VecDb::new(dir.db_path(), 32, None).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
- assert!(matches!(
- db.get_index_stats().unwrap().metric,
- VecDbMetric::InnerProduct
- ));
drop(db);
- let reader =
- VecDb::open_read_only(dir.db_path(), 32, None, Some(VecDbMetric::InnerProduct))
- .unwrap();
+ let reader = VecDb::open_read_only(dir.db_path(), 32, None).unwrap();
assert!(matches!(
reader.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
drop(reader);
let dir = TestDir::create();
- drop(
- VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap(),
- );
+ drop(VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)).unwrap());
for open in [VecDb::new, VecDb::open_read_only] {
assert!(matches!(
- open(dir.db_path(), 32, None, Some(VecDbMetric::Cosine)),
+ open(dir.db_path(), 32, None),
Err(RustVecDbError::StorageMismatch { .. })
));
}
@@ -508,39 +446,24 @@ mod tests {
#[test]
fn read_only_verifies_explicit_configuration() {
- for (storage, metric, wrong_storage, wrong_metric) in [
- (
- VecDbStorage::F32,
- VecDbMetric::InnerProduct,
- VecDbStorage::I8,
- VecDbMetric::Cosine,
- ),
- (
- VecDbStorage::I8,
- VecDbMetric::Cosine,
- VecDbStorage::F32,
- VecDbMetric::InnerProduct,
- ),
+ for (storage, wrong_storage) in [
+ (VecDbStorage::F32, VecDbStorage::I8),
+ (VecDbStorage::I8, VecDbStorage::F32),
] {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), 32, Some(storage), Some(metric)).unwrap();
+ let db = VecDb::new(dir.db_path(), 32, Some(storage)).unwrap();
db.add_vector(key("kept"), unit_axis(32)).unwrap();
db.flush().unwrap();
let verify = || {
assert!(matches!(
- VecDb::open_read_only(dir.db_path(), 32, Some(wrong_storage), Some(metric)),
+ VecDb::open_read_only(dir.db_path(), 32, Some(wrong_storage)),
Err(RustVecDbError::StorageMismatch { .. })
));
assert!(matches!(
- VecDb::open_read_only(dir.db_path(), 32, Some(storage), Some(wrong_metric)),
- Err(RustVecDbError::MetricMismatch { .. })
- ));
- assert!(matches!(
- VecDb::open_read_only(dir.db_path(), 64, Some(storage), Some(metric)),
+ VecDb::open_read_only(dir.db_path(), 64, Some(storage)),
Err(RustVecDbError::DimensionMismatch { .. })
));
- let reader =
- VecDb::open_read_only(dir.db_path(), 32, Some(storage), Some(metric)).unwrap();
+ let reader = VecDb::open_read_only(dir.db_path(), 32, Some(storage)).unwrap();
assert_eq!(reader.get_index_stats().unwrap().live_count, 1);
assert!(matches!(
reader.add_vector(key("no"), unit_axis(32)),
@@ -553,79 +476,10 @@ mod tests {
}
}
- #[test]
- fn explicitly_selected_metrics_are_immutable() {
- let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
- assert!(matches!(
- db.get_index_stats().unwrap().metric,
- VecDbMetric::Cosine
- ));
- assert!(matches!(
- VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::InnerProduct)
- ),
- Err(RustVecDbError::MetricMismatch { .. })
- ));
- drop(db);
- let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::InnerProduct),
- )
- .unwrap();
- db.add_vector(key("a"), basis(0)).unwrap();
- let vector: Vec<f32> = basis(0).iter().map(|value| value * 3.0).collect();
- assert_eq!(
- db.search(vector.clone(), Some(1), None, true, None)
- .unwrap()[0]
- .distance,
- -2.0
- );
- db.flush().unwrap();
- drop(db);
- assert!(matches!(
- VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), Some(VecDbMetric::Cosine)),
- Err(RustVecDbError::MetricMismatch { message }) if message.contains("cosine") && message.contains("inner product")
- ));
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::InnerProduct),
- )
- .unwrap();
- assert!(matches!(
- db.get_index_stats().unwrap().metric,
- VecDbMetric::InnerProduct
- ));
- assert_eq!(
- db.search(vector, Some(1), None, false, None).unwrap()[0].distance,
- -2.0
- );
- }
-
#[test]
fn add_search_stats_round_trip() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.bulk_add_vectors(vec![key("b"), key("c")], vec![basis(1), basis(2)])
.unwrap();
@@ -658,13 +512,8 @@ mod tests {
assert!(stats.approximate_memory_bytes > 0);
db.flush().unwrap();
assert_eq!(db.get_index_stats().unwrap().records_since_snapshot, 0);
- let read_only = VecDb::open_read_only(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let read_only =
+ VecDb::open_read_only(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);
assert!(matches!(
read_only.add_vector(key("x"), basis(3)),
@@ -686,13 +535,7 @@ mod tests {
#[test]
fn bulk_add_rejects_length_mismatch() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
let error = db
.bulk_add_vectors(vec![key("a")], vec![basis(0), basis(1)])
.unwrap_err();
@@ -703,13 +546,7 @@ mod tests {
#[test]
fn similarity_threshold_conversion_edges() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.add_vector(key("b"), basis(1)).unwrap();
let close = db
@@ -733,13 +570,7 @@ mod tests {
#[test]
fn bulk_search_within_similarity_honors_per_query_thresholds() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
@@ -805,13 +636,7 @@ mod tests {
#[test]
fn bulk_filtered_search_stays_query_aligned() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
@@ -865,13 +690,7 @@ mod tests {
#[test]
fn bulk_search_keys_excludes_self_and_honors_the_filters() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
@@ -909,13 +728,7 @@ mod tests {
#[test]
fn attrs_round_trip_through_the_api() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
let attrs = vec![
VecDbAttr {
name: key("model"),
@@ -1007,13 +820,7 @@ mod tests {
check_open_cost(dir.db_path()),
VecDbOpenCost::Absent
));
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
assert!(matches!(
check_open_cost(dir.db_path()),
@@ -1035,13 +842,7 @@ mod tests {
#[test]
fn delete_vec_db_files_purges_with_and_without_live_instances() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.flush().unwrap();
delete_vec_db_files(dir.db_path()).unwrap();
@@ -1052,13 +853,7 @@ mod tests {
db.get_index_stats(),
Err(RustVecDbError::Closed { .. })
));
- let reopened = VecDb::new(
- dir.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let reopened = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
assert_eq!(reopened.get_index_stats().unwrap().live_count, 0);
reopened.add_vector(key("b"), basis(1)).unwrap();
drop(reopened);
@@ -1078,13 +873,7 @@ mod tests {
#[test]
fn storage_is_chosen_at_creation_reported_in_stats_and_verified_on_reopen() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -1111,21 +900,10 @@ mod tests {
assert_eq!(stats.dims, 32);
assert!(matches!(stats.storage, VecDbStorage::I8));
assert!(matches!(
- VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine)
- ),
+ VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),
Err(RustVecDbError::StorageMismatch { .. })
));
- let joined = VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let joined = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
joined.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -1133,13 +911,7 @@ mod tests {
drop(joined);
db.flush().unwrap();
drop(db);
- let read_only = VecDb::open_read_only(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
read_only.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -1147,16 +919,10 @@ mod tests {
assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);
drop(read_only);
assert!(matches!(
- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), Some(VecDbMetric::Cosine)),
+ VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),
Err(RustVecDbError::StorageMismatch { message }) if message.contains("f32") && message.contains("i8")
));
- let reopened = VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let reopened = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
reopened.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -1168,51 +934,26 @@ mod tests {
#[test]
fn explicit_i8_storage_rejects_dims_off_the_lane_grid() {
let dir = TestDir::create();
- let db = VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
- let explicit = VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::I8),
- Some(VecDbMetric::Cosine),
- )
- .unwrap();
+ let explicit = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
explicit.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
assert!(matches!(
- VecDb::new(
- dir.db_path(),
- 32,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine)
- ),
+ VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),
Err(RustVecDbError::StorageMismatch { .. })
));
let other = TestDir::create();
assert!(matches!(
- VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::I8), Some(VecDbMetric::Cosine)),
+ VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::I8)),
Err(RustVecDbError::InvalidDimensions { message }) if message.contains("i8") && message.contains("32")
));
assert!(!PathBuf::from(other.db_path()).exists());
- assert!(
- VecDb::new(
- other.db_path(),
- DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::Cosine)
- )
- .is_ok()
- );
+ assert!(VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::F32)).is_ok());
}
}
diff --git a/rust/crates/vecdb/README.md b/rust/crates/vecdb/README.md
index cc477df47b..28533afbf4 100644
--- a/rust/crates/vecdb/README.md
+++ b/rust/crates/vecdb/README.md
@@ -1,34 +1,30 @@
# VecDB index configuration
-Every open requires the path and dimensions. Storage and distance metric are
-optional and default to i8 and cosine. These settings are persisted at creation and
-remain unchanged through reopen, compaction, and reset.
+Every open requires the path and dimensions. Storage is optional and defaults to
+i8. It is persisted at creation and remains unchanged through reopen, compaction,
+and reset.
```rust
-use ente_vecdb::{DistanceMetric, StorageKind, VecDb};
+use ente_vecdb::{StorageKind, VecDb};
-let index = VecDb::open(path, 512, None, None)?;
-let reader = VecDb::open_read_only(path, 512, None, None)?;
-let f32_index = VecDb::open(other_path, 512, Some(StorageKind::F32), None)?;
-let dot_index = VecDb::open(third_path, 512, None, Some(DistanceMetric::InnerProduct))?;
+let index = VecDb::open(path, 512, None)?;
+let reader = VecDb::open_read_only(path, 512, None)?;
+let f32_index = VecDb::open(other_path, 512, Some(StorageKind::F32))?;
```
Both open methods validate the supplied configuration against the saved header or
-an already-open instance. A mismatch returns `DimensionMismatch`, `StorageMismatch`,
-or `MetricMismatch`. Omitting storage or metric requires i8 or cosine on existing
-indexes too; it does not detect the saved settings. Older indexes remain readable by
-specifying their original storage and `DistanceMetric::InnerProduct`.
+an already-open instance. A mismatch returns `DimensionMismatch` or
+`StorageMismatch`. Omitting storage requires i8 on existing indexes too; it does
+not detect the saved setting. Older indexes remain readable by specifying their
+original storage.
A read-only open of a missing or incomplete index returns an empty view with the
supplied configuration without creating or modifying the file.
-The Flutter constructor and read-only open accept optional `storage` and `metric`,
-defaulting to i8 and cosine. Index stats report both settings.
+The Flutter constructor and read-only open accept an optional `storage`,
+defaulting to i8. Index stats report the setting.
-Cosine distance is `1 - dot(a, b) / (norm(a) * norm(b))`, clamped to `[0, 2]`.
-A comparison involving a zero vector has distance `1`, including two zero vectors.
-Stored vectors retain their magnitudes; i8 cosine uses the quantized vectors.
-Inner-product distance remains `1 - dot(a, b)`, which only ranks like cosine for
-unit vectors, so an inner-product index rejects an added vector whose squared norm
-is further than `1e-2` from `1` with `InvalidVector`. Cosine indexes take any
-magnitude, and queries are never checked.
+Cosine is the only distance: `1 - dot(a, b) / (norm(a) * norm(b))`, clamped to
+`[0, 2]`. A comparison involving a zero vector has distance `1`, including two
+zero vectors. Stored vectors retain their magnitudes; i8 cosine uses the
+quantized vectors.
diff --git a/rust/crates/vecdb/examples/vecdb_bench.rs b/rust/crates/vecdb/examples/vecdb_bench.rs
index 429206c4b5..87380e22b9 100644
--- a/rust/crates/vecdb/examples/vecdb_bench.rs
+++ b/rust/crates/vecdb/examples/vecdb_bench.rs
@@ -7,7 +7,7 @@ use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
-use ente_vecdb::{AttrValue, Attribute, DistanceMetric, Match, SearchParams, StorageKind, VecDb};
+use ente_vecdb::{AttrValue, Attribute, Match, SearchParams, StorageKind, VecDb};
const SEED: u64 = 0xE47E_0000_0000_0001;
const LATENT_DIMS: usize = 24;
@@ -26,7 +26,6 @@ struct Config {
dims: usize,
attrs: bool,
storage: StorageKind,
- metric: DistanceMetric,
}
struct BenchData {
@@ -67,10 +66,9 @@ struct VecdbReport {
fn main() {
let config = parse_args();
println!(
- "vecdb bench dims={} storage={} metric={} queries={} seed={:#018x}{}",
+ "vecdb bench dims={} storage={} queries={} seed={:#018x}{}",
config.dims,
storage_name(config.storage),
- metric_name(config.metric),
QUERY_COUNT,
SEED,
if config.attrs { " attrs=on" } else { "" }
@@ -91,7 +89,6 @@ fn main() {
config.dims,
config.attrs,
config.storage,
- config.metric,
temp_root.path(),
);
}
@@ -102,7 +99,6 @@ fn parse_args() -> Config {
let mut dims = DEFAULT_DIMS;
let mut attrs = false;
let mut storage = StorageKind::I8;
- let mut metric = DistanceMetric::Cosine;
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
@@ -120,13 +116,6 @@ fn parse_args() -> Config {
_ => usage_exit(),
}
}
- "--metric" => {
- metric = match required_value(args.next()).as_str() {
- "cosine" => DistanceMetric::Cosine,
- "ip" => DistanceMetric::InnerProduct,
- _ => usage_exit(),
- }
- }
_ => usage_exit(),
}
}
@@ -146,7 +135,6 @@ fn parse_args() -> Config {
dims,
attrs,
storage,
- metric,
}
}
@@ -171,18 +159,11 @@ fn parse_scales(list: &str) -> Vec<usize> {
fn usage_exit() -> ! {
eprintln!(
"usage: cargo run -p ente-vecdb --example vecdb_bench --release -- \
- [--scales 10000,100000] [--dims 512] [--attrs] [--storage f32|i8] [--metric cosine|ip]"
+ [--scales 10000,100000] [--dims 512] [--attrs] [--storage f32|i8]"
);
std::process::exit(2);
}
-fn metric_name(metric: DistanceMetric) -> &'static str {
- match metric {
- DistanceMetric::Cosine => "cosine",
- DistanceMetric::InnerProduct => "ip",
- }
-}
-
fn storage_name(storage: StorageKind) -> &'static str {
match storage {
StorageKind::F32 => "f32",
@@ -190,30 +171,19 @@ fn storage_name(storage: StorageKind) -> &'static str {
}
}
-fn run_scale(
- scale: usize,
- dims: usize,
- attrs: bool,
- storage: StorageKind,
- metric: DistanceMetric,
- temp_root: &Path,
-) {
+fn run_scale(scale: usize, dims: usize, attrs: bool, storage: StorageKind, temp_root: &Path) {
let clusters = (scale / 150).max(1) as u64;
println!();
println!(
- "=== scale {scale} dims {dims} storage {} metric {} clusters {clusters} ===",
- storage_name(storage),
- metric_name(metric)
+ "=== scale {scale} dims {dims} storage {} clusters {clusters} ===",
+ storage_name(storage)
);
let dir = temp_root.join(format!("scale-{scale}-{dims}"));
std::fs::create_dir_all(&dir).expect("create bench dir");
- drop(
- VecDb::open(&dir.join("bench.vecdb"), dims, Some(storage), Some(metric))
- .expect("open vecdb"),
- );
+ drop(VecDb::open(&dir.join("bench.vecdb"), dims, Some(storage)).expect("open vecdb"));
eprintln!("[scale {scale}] generating data");
let data = generate_data(scale, dims, clusters);
- let vecdb = run_vecdb(&data, dims, attrs, storage, metric, &dir, scale);
+ let vecdb = run_vecdb(&data, dims, attrs, storage, &dir, scale);
print_vecdb_report(&vecdb);
let _ = std::fs::remove_dir_all(&dir);
}
@@ -343,12 +313,11 @@ fn run_vecdb(
dims: usize,
attrs: bool,
storage: StorageKind,
- metric: DistanceMetric,
dir: &Path,
scale: usize,
) -> VecdbReport {
let path = dir.join("bench.vecdb");
- let mut db = VecDb::open(&path, dims, Some(storage), Some(metric)).expect("open vecdb");
+ let mut db = VecDb::open(&path, dims, Some(storage)).expect("open vecdb");
let ingest = ingest_phase(&mut db, data, attrs, scale);
let searches = search_phase(&db, data, storage, scale);
let stats = db.stats().expect("vecdb stats");
@@ -357,7 +326,7 @@ fn run_vecdb(
.map(|meta| meta.len())
.unwrap_or(0);
drop(db);
- let (reopens, mut db) = reopen_phase(&path, dims, storage, metric, &snapshot_file, scale);
+ let (reopens, mut db) = reopen_phase(&path, dims, storage, &snapshot_file, scale);
let compaction = compaction_phase(&mut db, data, scale);
db.delete().expect("vecdb delete");
VecdbReport {
@@ -561,21 +530,18 @@ fn reopen_phase(
path: &Path,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
snapshot_file: &Path,
scale: usize,
) -> (ReopenTimings, VecDb) {
eprintln!("[scale {scale}] vecdb cold open with snapshot");
let started = Instant::now();
- let reopened =
- VecDb::open(path, dims, Some(storage), Some(metric)).expect("vecdb reopen with snapshot");
+ let reopened = VecDb::open(path, dims, Some(storage)).expect("vecdb reopen with snapshot");
let open_with_snapshot = started.elapsed();
drop(reopened);
std::fs::remove_file(snapshot_file).expect("remove snapshot");
eprintln!("[scale {scale}] vecdb cold open without snapshot (full rebuild)");
let started = Instant::now();
- let db = VecDb::open(path, dims, Some(storage), Some(metric))
- .expect("vecdb reopen without snapshot");
+ let db = VecDb::open(path, dims, Some(storage)).expect("vecdb reopen without snapshot");
let open_full_rebuild = started.elapsed();
(
ReopenTimings {
diff --git a/rust/crates/vecdb/src/arena.rs b/rust/crates/vecdb/src/arena.rs
index 77278483d9..3277b66908 100644
--- a/rust/crates/vecdb/src/arena.rs
+++ b/rust/crates/vecdb/src/arena.rs
@@ -5,7 +5,7 @@ use super::kernel::{
pack_lanes, pack_lanes_i8, pack_lanes_i8_into, pack_lanes_into, quantize_for, unpack_lanes,
unpack_lanes_i8, validate_dims,
};
-use super::{DistanceMetric, StorageKind, VecDbError};
+use super::{StorageKind, VecDbError};
pub(crate) const VECTORS_PER_CHUNK: usize = 4096;
pub(crate) const MAX_KEY_BYTES: usize = 256;
@@ -114,7 +114,6 @@ pub(crate) struct VectorArena {
dims: usize,
lanes_per_vector: usize,
storage: Storage,
- metric: DistanceMetric,
norms: Vec<f64>,
keys_to_slots: HashMap<Box<str>, u32>,
slots_to_keys: Vec<Box<str>>,
@@ -129,16 +128,7 @@ impl VectorArena {
Self::with_storage(dims, StorageKind::F32)
}
- #[cfg(test)]
pub(crate) fn with_storage(dims: usize, storage: StorageKind) -> Result<Self, VecDbError> {
- Self::with_metric(dims, storage, DistanceMetric::InnerProduct)
- }
-
- pub(crate) fn with_metric(
- dims: usize,
- storage: StorageKind,
- metric: DistanceMetric,
- ) -> Result<Self, VecDbError> {
validate_dims(dims, storage)?;
let storage = match storage {
StorageKind::F32 => Storage::F32 { chunks: Vec::new() },
@@ -151,7 +141,6 @@ impl VectorArena {
dims,
lanes_per_vector: dims / storage.kind().lane_width(),
storage,
- metric,
norms: Vec::new(),
keys_to_slots: HashMap::new(),
slots_to_keys: Vec::new(),
@@ -161,10 +150,6 @@ impl VectorArena {
})
}
- pub(crate) fn metric(&self) -> DistanceMetric {
- self.metric
- }
-
pub(crate) fn dims(&self) -> usize {
self.dims
}
@@ -250,9 +235,7 @@ impl VectorArena {
if slot as usize / 64 == self.alive.len() {
self.alive.push(0);
}
- if self.metric == DistanceMetric::Cosine {
- self.norms.push(0.0);
- }
+ self.norms.push(0.0);
self.slots_to_keys.push(Box::from(key));
self.keys_to_slots.insert(Box::from(key), slot);
self.mark_alive(slot);
@@ -317,9 +300,7 @@ impl VectorArena {
}
fn move_vector(&mut self, src: u32, dst: u32) {
- if self.metric == DistanceMetric::Cosine {
- self.norms[dst as usize] = self.norms[src as usize];
- }
+ self.norms[dst as usize] = self.norms[src as usize];
let lanes = self.lanes_per_vector;
match &mut self.storage {
Storage::F32 { chunks } => move_lanes(chunks, lanes, src, dst),
@@ -392,20 +373,12 @@ impl VectorArena {
Ok(match quantize_for(self.storage_kind(), values) {
Some(StoredVector::I8 { scale, values }) => {
let lanes = pack_lanes_i8(&values);
- let norm = if self.metric == DistanceMetric::Cosine {
- i8_lanes_norm(scale, &lanes)
- } else {
- 0.0
- };
+ let norm = i8_lanes_norm(scale, &lanes);
PackedQuery::I8 { scale, lanes, norm }
}
Some(StoredVector::F32(_)) | None => {
let lanes = pack_lanes(values);
- let norm = if self.metric == DistanceMetric::Cosine {
- f32_lanes_norm(&lanes)
- } else {
- 0.0
- };
+ let norm = f32_lanes_norm(&lanes);
PackedQuery::F32(lanes, norm)
}
})
@@ -418,9 +391,6 @@ impl VectorArena {
pub(crate) fn distance_to_query(&self, query: Query<'_>, slot: u32) -> f32 {
match (query, self.stored_query(slot)) {
(Query::F32(a, norm_a), Query::F32(b, norm_b)) => {
- if self.metric == DistanceMetric::InnerProduct {
- return F32Kernel::distance(a, b);
- }
let denominator = norm_a * norm_b;
let dot = if denominator >= f64::from(f32::MIN_POSITIVE)
&& denominator <= f64::from(f32::MAX)
@@ -437,21 +407,16 @@ impl VectorArena {
}
(
Query::I8 {
- scale: scale_a,
lanes: a,
norm: norm_a,
+ ..
},
Query::I8 {
- scale: scale_b,
lanes: b,
norm: norm_b,
+ ..
},
- ) => {
- if self.metric == DistanceMetric::InnerProduct {
- return I8Kernel::distance(a, scale_a, b, scale_b);
- }
- cosine_distance(f64::from(I8Kernel::dot(a, b)), norm_a * norm_b)
- }
+ ) => cosine_distance(f64::from(I8Kernel::dot(a, b)), norm_a * norm_b),
_ => unreachable!("queries are packed by the arena that searches them"),
}
}
@@ -481,9 +446,7 @@ impl VectorArena {
unreachable!("payload kind is validated before a slot is written")
}
}
- if self.metric == DistanceMetric::Cosine {
- self.norms[slot as usize] = self.stored_norm(slot);
- }
+ self.norms[slot as usize] = self.stored_norm(slot);
}
fn mark_alive(&mut self, slot: u32) {
@@ -576,8 +539,7 @@ mod tests {
fn cosine_norms_match_their_scalar_references_and_agree_between_query_and_slot() {
for dims in [512usize, 768] {
for storage in [StorageKind::F32, StorageKind::I8] {
- let mut arena =
- VectorArena::with_metric(dims, storage, DistanceMetric::Cosine).unwrap();
+ let mut arena = VectorArena::with_storage(dims, storage).unwrap();
let vectors: Vec<Vec<f32>> = (0..16u64)
.map(|seed| seeded_vector(0x5E1F_0000 + seed, dims))
.collect();
@@ -1175,7 +1137,7 @@ mod tests {
}
#[test]
- fn distances_use_the_kernel_metric() {
+ fn distances_are_cosine() {
let mut arena = VectorArena::new(16).unwrap();
arena.upsert("x", &basis_vector(16, 0)).unwrap();
arena.upsert("y", &basis_vector(16, 9)).unwrap();
@@ -1254,21 +1216,24 @@ mod tests {
}
#[test]
- fn i8_distances_follow_the_scaled_dot_formula() {
+ fn i8_distances_follow_the_quantized_cosine_formula() {
let dims = 64;
let mut arena = VectorArena::with_storage(dims, StorageKind::I8).unwrap();
let a = seeded_unit_vector(11, dims);
let b = seeded_unit_vector(12, dims);
arena.upsert("a", &a).unwrap();
arena.upsert("b", &b).unwrap();
- let (scale_a, values_a) = stored_i8(&arena, 0);
- let (scale_b, values_b) = stored_i8(&arena, 1);
- let dot: i32 = values_a
- .iter()
- .zip(&values_b)
- .map(|(x, y)| i32::from(*x) * i32::from(*y))
- .sum();
- let expected = 1.0 - dot as f32 * scale_a * scale_b;
+ let (_, values_a) = stored_i8(&arena, 0);
+ let (_, values_b) = stored_i8(&arena, 1);
+ let dot = |x: &[i8], y: &[i8]| -> f64 {
+ x.iter()
+ .zip(y)
+ .map(|(x, y)| f64::from(*x) * f64::from(*y))
+ .sum()
+ };
+ let norm = |values: &[i8]| dot(values, values).sqrt();
+ let expected =
+ (1.0 - dot(&values_a, &values_b) / (norm(&values_a) * norm(&values_b))) as f32;
assert_eq!(
arena.distance_between_slots(0, 1).to_bits(),
expected.to_bits()
@@ -1288,7 +1253,7 @@ mod tests {
);
let f32_distance = 1.0 - a.iter().zip(&b).map(|(x, y)| x * y).sum::<f32>();
assert!((expected - f32_distance).abs() < 0.01);
- assert!(arena.distance_between_slots(0, 0).abs() < 0.02);
+ assert!(arena.distance_between_slots(0, 0).abs() < 1.0e-6);
}
#[test]
@@ -1390,13 +1355,15 @@ mod tests {
f32_arena.upsert(&format!("key-{index}"), &values).unwrap();
i8_arena.upsert(&format!("key-{index}"), &values).unwrap();
}
+ let norm_bytes = f32_arena.norms.capacity() * size_of::<f64>();
+ assert_eq!(i8_arena.norms.capacity() * size_of::<f64>(), norm_bytes);
assert_eq!(
f32_arena.vector_memory_bytes(),
- VECTORS_PER_CHUNK * dims * size_of::<f32>()
+ VECTORS_PER_CHUNK * dims * size_of::<f32>() + norm_bytes
);
assert_eq!(
i8_arena.vector_memory_bytes(),
- VECTORS_PER_CHUNK * dims + 100 * size_of::<f32>()
+ VECTORS_PER_CHUNK * dims + 100 * size_of::<f32>() + norm_bytes
);
}
}
diff --git a/rust/crates/vecdb/src/graph.rs b/rust/crates/vecdb/src/graph.rs
index de57bccf48..9218739fad 100644
--- a/rust/crates/vecdb/src/graph.rs
+++ b/rust/crates/vecdb/src/graph.rs
@@ -1012,9 +1012,9 @@ mod tests {
use std::f32::consts::FRAC_1_SQRT_2;
use std::sync::LazyLock;
+ use super::super::StorageKind;
use super::super::arena::UpsertOutcome;
use super::super::test_support::{assert_identical_graphs, stale_downward_edge_exists};
- use super::super::{DistanceMetric, StorageKind};
use super::*;
const FIXTURE_DIMS: usize = 16;
@@ -1160,23 +1160,16 @@ mod tests {
const JITTER_SEED: u64 = 0x71E7_0000;
fn build_duplicate_groups(groups: usize, per_group: usize) -> (VectorArena, Graph) {
- build_duplicate_groups_with(
- 16,
- StorageKind::F32,
- DistanceMetric::InnerProduct,
- groups,
- per_group,
- )
+ build_duplicate_groups_with(16, StorageKind::F32, groups, per_group)
}
fn build_duplicate_groups_with(
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
groups: usize,
per_group: usize,
) -> (VectorArena, Graph) {
- build_groups(dims, storage, metric, groups, per_group, |group, _| {
+ build_groups(dims, storage, groups, per_group, |group, _| {
group_center(group, dims)
})
}
@@ -1184,12 +1177,11 @@ mod tests {
fn build_groups(
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
groups: usize,
per_group: usize,
member: impl Fn(usize, usize) -> Vec<f32>,
) -> (VectorArena, Graph) {
- let mut arena = VectorArena::with_metric(dims, storage, metric).unwrap();
+ let mut arena = VectorArena::with_storage(dims, storage).unwrap();
for index in 0..groups * per_group {
let vector = member(index / per_group, index % per_group);
arena.upsert(&format!("key-{index}"), &vector).unwrap();
@@ -1202,10 +1194,6 @@ mod tests {
seeded_unit_vector(DUPLICATE_GROUP_SEED + group as u64, dims)
}
- fn scaled(values: &[f32], factor: f32) -> Vec<f32> {
- values.iter().map(|value| value * factor).collect()
- }
-
fn peaky_unit_vector(group: usize, dims: usize) -> Vec<f32> {
let mut state = PEAKY_GROUP_SEED + group as u64;
let mut values: Vec<f32> = (0..dims)
@@ -1644,39 +1632,9 @@ mod tests {
#[test]
fn duplicate_groups_near_the_neighbor_cap_stay_whole_at_production_dims() {
for storage in [StorageKind::F32, StorageKind::I8] {
- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
- for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 100)] {
- let (arena, graph) =
- build_duplicate_groups_with(512, storage, metric, groups, per_group);
- let query = arena.pack_query(&group_center(0, 512)).unwrap();
- let found = search(
- &graph,
- &arena,
- &query,
- &params(Some(limit), None, false),
- None,
- );
- assert_eq!(found.len(), limit);
- assert_sorted(&found);
- assert_leading_group(&arena, &found, per_group, 0, per_group);
- }
- }
- }
- }
-
- #[test]
- fn duplicate_groups_of_peaky_vectors_stay_whole_in_int8() {
- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
- for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 10), (4, 80, 80)] {
- let (arena, graph) = build_groups(
- 512,
- StorageKind::I8,
- metric,
- groups,
- per_group,
- |group, _| peaky_unit_vector(group, 512),
- );
- let query = arena.pack_query(&peaky_unit_vector(0, 512)).unwrap();
+ for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 100)] {
+ let (arena, graph) = build_duplicate_groups_with(512, storage, groups, per_group);
+ let query = arena.pack_query(&group_center(0, 512)).unwrap();
let found = search(
&graph,
&arena,
@@ -1686,37 +1644,29 @@ mod tests {
);
assert_eq!(found.len(), limit);
assert_sorted(&found);
- assert_leading_group(&arena, &found, per_group.min(limit), 0, per_group);
+ assert_leading_group(&arena, &found, per_group, 0, per_group);
}
}
}
#[test]
- fn duplicate_groups_stored_below_unit_norm_stay_whole() {
- for storage in [StorageKind::F32, StorageKind::I8] {
- for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 100)] {
- let (arena, graph) = build_groups(
- 512,
- storage,
- DistanceMetric::InnerProduct,
- groups,
- per_group,
- |group, _| scaled(&group_center(group, 512), 0.5),
- );
- let query = arena
- .pack_query(&scaled(&group_center(0, 512), 0.5))
- .unwrap();
- let found = search(
- &graph,
- &arena,
- &query,
- &params(Some(limit), None, false),
- None,
- );
- assert_eq!(found.len(), limit);
- assert_sorted(&found);
- assert_leading_group(&arena, &found, per_group, 0, per_group);
- }
+ fn duplicate_groups_of_peaky_vectors_stay_whole_in_int8() {
+ for (groups, per_group, limit) in [(4, 40, 60), (8, 40, 60), (4, 80, 10), (4, 80, 80)] {
+ let (arena, graph) =
+ build_groups(512, StorageKind::I8, groups, per_group, |group, _| {
+ peaky_unit_vector(group, 512)
+ });
+ let query = arena.pack_query(&peaky_unit_vector(0, 512)).unwrap();
+ let found = search(
+ &graph,
+ &arena,
+ &query,
+ &params(Some(limit), None, false),
+ None,
+ );
+ assert_eq!(found.len(), limit);
+ assert_sorted(&found);
+ assert_leading_group(&arena, &found, per_group.min(limit), 0, per_group);
}
}
@@ -1724,35 +1674,33 @@ mod tests {
fn a_duplicate_burst_at_the_neighbor_cap_survives_background_and_churn() {
const BURST: usize = LEVEL_ZERO_NEIGHBOR_CAP;
const BACKGROUND: usize = 2000;
- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
- let duplicate = group_center(0, 512);
- let mut arena = VectorArena::with_metric(512, StorageKind::I8, metric).unwrap();
- let mut graph = Graph::new();
- for index in 0..BACKGROUND {
- apply_upsert(
- &mut arena,
- &mut graph,
- &format!("background-{index}"),
- &seeded_unit_vector(BACKGROUND_SEED + index as u64, 512),
- );
- }
- for index in 0..BURST {
- apply_upsert(
- &mut arena,
- &mut graph,
- &format!("duplicate-{index}"),
- &duplicate,
- );
- }
- let query = arena.pack_query(&duplicate).unwrap();
- assert_whole_burst(&graph, &arena, &query, BURST);
- for index in (0..BURST).step_by(2) {
- let key = format!("duplicate-{index}");
- assert!(arena.remove(&key).is_some());
- apply_upsert(&mut arena, &mut graph, &key, &duplicate);
- }
- assert_whole_burst(&graph, &arena, &query, BURST);
+ let duplicate = group_center(0, 512);
+ let mut arena = VectorArena::with_storage(512, StorageKind::I8).unwrap();
+ let mut graph = Graph::new();
+ for index in 0..BACKGROUND {
+ apply_upsert(
+ &mut arena,
+ &mut graph,
+ &format!("background-{index}"),
+ &seeded_unit_vector(BACKGROUND_SEED + index as u64, 512),
+ );
+ }
+ for index in 0..BURST {
+ apply_upsert(
+ &mut arena,
+ &mut graph,
+ &format!("duplicate-{index}"),
+ &duplicate,
+ );
+ }
+ let query = arena.pack_query(&duplicate).unwrap();
+ assert_whole_burst(&graph, &arena, &query, BURST);
+ for index in (0..BURST).step_by(2) {
+ let key = format!("duplicate-{index}");
+ assert!(arena.remove(&key).is_some());
+ apply_upsert(&mut arena, &mut graph, &key, &duplicate);
}
+ assert_whole_burst(&graph, &arena, &query, BURST);
}
fn assert_whole_burst(graph: &Graph, arena: &VectorArena, query: &PackedQuery, burst: usize) {
@@ -1786,20 +1734,13 @@ mod tests {
fn jittered_groups_near_the_neighbor_cap_stay_whole() {
for storage in [StorageKind::F32, StorageKind::I8] {
for spread in [1e-3, 1e-5, 1e-6] {
- let (arena, graph) = build_groups(
- 512,
- storage,
- DistanceMetric::Cosine,
- 4,
- 80,
- |group, member| {
- jittered(
- &group_center(group, 512),
- JITTER_SEED + (group * 80 + member) as u64,
- spread,
- )
- },
- );
+ let (arena, graph) = build_groups(512, storage, 4, 80, |group, member| {
+ jittered(
+ &group_center(group, 512),
+ JITTER_SEED + (group * 80 + member) as u64,
+ spread,
+ )
+ });
let query = arena.pack_query(&group_center(0, 512)).unwrap();
let found = search(
&graph,
diff --git a/rust/crates/vecdb/src/kernel.rs b/rust/crates/vecdb/src/kernel.rs
index b59b6b1994..b83ed06f6d 100644
--- a/rust/crates/vecdb/src/kernel.rs
+++ b/rust/crates/vecdb/src/kernel.rs
@@ -143,10 +143,6 @@ pub(crate) trait VectorKernel {
type Lane;
fn dot(a: &[Self::Lane], b: &[Self::Lane]) -> f32;
-
- fn distance(a: &[Self::Lane], b: &[Self::Lane]) -> f32 {
- 1.0 - Self::dot(a, b)
- }
}
pub(crate) struct F32Kernel;
@@ -181,10 +177,6 @@ impl I8Kernel {
}
total
}
-
- pub(crate) fn distance(a: &[LaneI8], scale_a: f32, b: &[LaneI8], scale_b: f32) -> f32 {
- 1.0 - Self::dot(a, b) as f32 * scale_a * scale_b
- }
}
pub(crate) fn pack_lanes_i8(values: &[i8]) -> Vec<LaneI8> {
@@ -290,11 +282,11 @@ mod tests {
}
#[test]
- fn distance_of_basis_vectors_is_exact() {
+ fn dot_of_basis_vectors_is_exact() {
let e0 = pack_lanes(&basis_vector(128, 0));
let e1 = pack_lanes(&basis_vector(128, 100));
- assert_eq!(F32Kernel::distance(&e0, &e0), 0.0);
- assert_eq!(F32Kernel::distance(&e0, &e1), 1.0);
+ assert_eq!(F32Kernel::dot(&e0, &e0), 1.0);
+ assert_eq!(F32Kernel::dot(&e0, &e1), 0.0);
}
#[test]
@@ -388,26 +380,7 @@ mod tests {
}
#[test]
- fn i8_distance_pins_the_evaluation_order() {
- let a = pack_lanes_i8(&seeded_i8_values(5, 64));
- let b = pack_lanes_i8(&seeded_i8_values(6, 64));
- let scale_a = 0.012_345_6f32;
- let scale_b = 0.045_678_9f32;
- let dot = I8Kernel::dot(&a, &b);
- let scaled_once = dot as f32 * scale_a;
- let expected = 1.0 - scaled_once * scale_b;
- assert_eq!(
- I8Kernel::distance(&a, scale_a, &b, scale_b).to_bits(),
- expected.to_bits()
- );
- assert_eq!(
- I8Kernel::distance(&b, scale_b, &a, scale_a).to_bits(),
- (1.0 - (dot as f32 * scale_b) * scale_a).to_bits()
- );
- }
-
- #[test]
- fn i8_distance_of_basis_vectors_is_orthogonal_exactly_and_self_within_rounding() {
+ fn i8_dot_of_basis_vectors_is_orthogonal_exactly_and_self_within_rounding() {
let (scale_0, values_0) = quantized_parts(&basis_vector(32, 0));
let (scale_5, values_5) = quantized_parts(&basis_vector(32, 5));
assert_eq!(scale_0, 1.0 / 127.0);
@@ -415,8 +388,9 @@ mod tests {
assert!(values_0[1..].iter().all(|&value| value == 0));
let e0 = pack_lanes_i8(&values_0);
let e5 = pack_lanes_i8(&values_5);
- assert_eq!(I8Kernel::distance(&e0, scale_0, &e5, scale_5), 1.0);
- assert!(I8Kernel::distance(&e0, scale_0, &e0, scale_0).abs() < 1.0e-6);
+ assert_eq!(I8Kernel::dot(&e0, &e5), 0);
+ assert!((I8Kernel::dot(&e0, &e0) as f32 * scale_0 * scale_0 - 1.0).abs() < 1.0e-6);
+ assert_eq!(scale_5, scale_0);
}
#[test]
@@ -538,18 +512,19 @@ mod tests {
}
#[test]
- fn i8_distance_tracks_f32_distance_on_unit_vectors_within_the_bound() {
+ fn i8_dot_tracks_f32_dot_on_unit_vectors_within_the_bound() {
let mut max_delta = 0.0f32;
for (dims, pairs, seed) in [(512usize, 300u64, 0x1000u64), (64, 300, 0x2000)] {
for pair in 0..pairs {
let a = seeded_unit_values(seed + pair * 2, dims);
let b = seeded_unit_values(seed + pair * 2 + 1, dims);
- let f32_distance = F32Kernel::distance(&pack_lanes(&a), &pack_lanes(&b));
+ let f32_dot = F32Kernel::dot(&pack_lanes(&a), &pack_lanes(&b));
let (scale_a, qa) = quantized_parts(&a);
let (scale_b, qb) = quantized_parts(&b);
- let i8_distance =
- I8Kernel::distance(&pack_lanes_i8(&qa), scale_a, &pack_lanes_i8(&qb), scale_b);
- let delta = (i8_distance - f32_distance).abs();
+ let i8_dot = I8Kernel::dot(&pack_lanes_i8(&qa), &pack_lanes_i8(&qb)) as f32
+ * scale_a
+ * scale_b;
+ let delta = (i8_dot - f32_dot).abs();
assert!(delta <= 0.01, "dims {dims} pair {pair}: delta {delta}");
max_delta = max_delta.max(delta);
}
diff --git a/rust/crates/vecdb/src/lib.rs b/rust/crates/vecdb/src/lib.rs
index af104180de..1a573a908f 100644
--- a/rust/crates/vecdb/src/lib.rs
+++ b/rust/crates/vecdb/src/lib.rs
@@ -13,38 +13,6 @@ mod store;
pub use store::{OpenCost, Stats, VecDb};
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DistanceMetric {
- InnerProduct,
- Cosine,
-}
-
-impl DistanceMetric {
- pub(crate) fn header_tag(self) -> u8 {
- match self {
- Self::InnerProduct => 0,
- Self::Cosine => 1,
- }
- }
-
- pub(crate) fn from_header_tag(tag: u8) -> Option<Self> {
- match tag {
- 0 => Some(Self::InnerProduct),
- 1 => Some(Self::Cosine),
- _ => None,
- }
- }
-}
-
-impl std::fmt::Display for DistanceMetric {
- fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- formatter.write_str(match self {
- Self::InnerProduct => "inner product",
- Self::Cosine => "cosine",
- })
- }
-}
-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StorageKind {
F32,
@@ -156,11 +124,6 @@ pub enum VecDbError {
expected: StorageKind,
actual: StorageKind,
},
- #[error("distance metric mismatch: expected {expected}, found {actual}")]
- MetricMismatch {
- expected: DistanceMetric,
- actual: DistanceMetric,
- },
#[error("search requires a limit or a max distance")]
UnboundedSearch,
#[error("length mismatch: {keys} keys, {vectors} vectors")]
diff --git a/rust/crates/vecdb/src/log.rs b/rust/crates/vecdb/src/log.rs
index 64d30bd3a4..307900d2d0 100644
--- a/rust/crates/vecdb/src/log.rs
+++ b/rust/crates/vecdb/src/log.rs
@@ -9,13 +9,14 @@ use super::crc::{Crc32, crc32};
use super::kernel::{
StoredVector, VectorPayload, is_within_quantized_range, splitmix64, validate_dims,
};
-use super::{AttrValue, Attribute, DistanceMetric, StorageKind, VecDbError};
+use super::{AttrValue, Attribute, StorageKind, VecDbError};
pub(crate) const HEADER_LEN: usize = 32;
const MAGIC: [u8; 4] = *b"EVDB";
const FORMAT_VERSION: u16 = 1;
const STORAGE_TAG_OFFSET: usize = 6;
const METRIC_TAG_OFFSET: usize = 7;
+const COSINE_METRIC_TAG: u8 = 1;
const RECORD_TYPE_ADD: u8 = 1;
const RECORD_TYPE_TOMBSTONE: u8 = 2;
const RECORD_PREFIX_LEN: usize = 3;
@@ -63,7 +64,6 @@ pub(crate) struct Log {
path: PathBuf,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
generation: [u8; 16],
end_offset: u64,
encode_buffer: Vec<u8>,
@@ -116,11 +116,10 @@ impl Log {
path: &Path,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
) -> Result<Self, VecDbError> {
validate_dims(dims, storage)?;
let file = open_writer_file(path, true)?;
- Self::initialize(file, path, dims, storage, metric)
+ Self::initialize(file, path, dims, storage)
}
pub(crate) fn open(
@@ -128,7 +127,6 @@ impl Log {
path: &Path,
expected_dims: usize,
requested: StorageKind,
- requested_metric: DistanceMetric,
) -> Result<Self, VecDbError> {
let file_len = file
.metadata()
@@ -140,7 +138,7 @@ impl Log {
path.display()
);
validate_dims(expected_dims, requested)?;
- return Self::initialize(file, path, expected_dims, requested, requested_metric);
+ return Self::initialize(file, path, expected_dims, requested);
}
file.seek(SeekFrom::Start(0))
.map_err(|source| VecDbError::io(path, source))?;
@@ -151,7 +149,6 @@ impl Log {
generation,
dims,
storage,
- metric,
} = decode_header(&header)?;
if requested != storage {
return Err(VecDbError::StorageMismatch {
@@ -159,12 +156,6 @@ impl Log {
actual: storage,
});
}
- if requested_metric != metric {
- return Err(VecDbError::MetricMismatch {
- expected: requested_metric,
- actual: metric,
- });
- }
if dims != expected_dims {
return Err(VecDbError::DimensionMismatch {
expected: expected_dims,
@@ -177,7 +168,6 @@ impl Log {
path: path.to_path_buf(),
dims: expected_dims,
storage,
- metric,
generation,
end_offset: file_len,
encode_buffer: Vec::new(),
@@ -190,12 +180,11 @@ impl Log {
path: &Path,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
) -> Result<Self, VecDbError> {
let generation = fresh_generation();
file.seek(SeekFrom::Start(0))
.map_err(|source| VecDbError::io(path, source))?;
- file.write_all(&encode_header(dims as u32, &generation, storage, metric))
+ file.write_all(&encode_header(dims as u32, &generation, storage))
.map_err(|source| VecDbError::io(path, source))?;
file.sync_all()
.map_err(|source| VecDbError::io(path, source))?;
@@ -205,7 +194,6 @@ impl Log {
path: path.to_path_buf(),
dims,
storage,
- metric,
generation,
end_offset: HEADER_LEN as u64,
encode_buffer: Vec::new(),
@@ -217,11 +205,10 @@ impl Log {
path: &Path,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
) -> Result<(Self, PathBuf), VecDbError> {
remove_stale_temp_sibling(path)?;
let temp_path = temp_sibling_path(path);
- let log = Self::create(&temp_path, dims, storage, metric)?;
+ let log = Self::create(&temp_path, dims, storage)?;
Ok((log, temp_path))
}
@@ -410,10 +397,6 @@ impl Log {
self.generation
}
- pub(crate) fn metric(&self) -> DistanceMetric {
- self.metric
- }
-
pub(crate) fn storage(&self) -> StorageKind {
self.storage
}
@@ -775,17 +758,12 @@ fn plausible_attrs_len(bytes: &[u8]) -> Option<usize> {
Some(1 + attr_bytes)
}
-fn encode_header(
- dims: u32,
- generation: &[u8; 16],
- storage: StorageKind,
- metric: DistanceMetric,
-) -> [u8; HEADER_LEN] {
+fn encode_header(dims: u32, generation: &[u8; 16], storage: StorageKind) -> [u8; HEADER_LEN] {
let mut bytes = [0u8; HEADER_LEN];
bytes[0..4].copy_from_slice(&MAGIC);
bytes[4..6].copy_from_slice(&FORMAT_VERSION.to_le_bytes());
bytes[STORAGE_TAG_OFFSET] = storage.header_tag();
- bytes[METRIC_TAG_OFFSET] = metric.header_tag();
+ bytes[METRIC_TAG_OFFSET] = COSINE_METRIC_TAG;
bytes[8..12].copy_from_slice(&dims.to_le_bytes());
bytes[12..28].copy_from_slice(generation);
let crc = crc32(&bytes[0..28]);
@@ -802,7 +780,6 @@ struct Header {
generation: [u8; 16],
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
}
fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {
@@ -828,9 +805,12 @@ fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {
let storage = StorageKind::from_header_tag(bytes[STORAGE_TAG_OFFSET]).ok_or_else(|| {
VecDbError::Corrupt(format!("unknown scalar tag {}", bytes[STORAGE_TAG_OFFSET]))
})?;
- let metric = DistanceMetric::from_header_tag(bytes[METRIC_TAG_OFFSET]).ok_or_else(|| {
- VecDbError::Corrupt(format!("unknown metric tag {}", bytes[METRIC_TAG_OFFSET]))
- })?;
+ if bytes[METRIC_TAG_OFFSET] != COSINE_METRIC_TAG {
+ return Err(VecDbError::Corrupt(format!(
+ "unknown metric tag {}",
+ bytes[METRIC_TAG_OFFSET]
+ )));
+ }
let dims = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
let mut generation = [0u8; 16];
generation.copy_from_slice(&bytes[12..28]);
@@ -838,7 +818,6 @@ fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {
generation,
dims,
storage,
- metric,
})
}
@@ -1042,7 +1021,6 @@ pub(crate) fn sync_parent_dir(_path: &Path) -> Result<(), VecDbError> {
#[cfg(test)]
mod tests {
- use super::DistanceMetric::InnerProduct;
use super::*;
use tempfile::TempDir;
@@ -1058,7 +1036,7 @@ mod tests {
fn reopen(path: &Path, dims: usize, storage: StorageKind) -> Log {
let file = open_writer_file(path, false).unwrap();
- Log::open(file, path, dims, storage, InnerProduct).unwrap()
+ Log::open(file, path, dims, storage).unwrap()
}
fn append_bounds(log: &mut Log, entries: &[LogEntry<'_>]) -> (u64, u64) {
@@ -1129,7 +1107,7 @@ mod tests {
}
fn write_log_file(path: &Path, dims: u32, record_bytes: &[u8]) {
- let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::F32, InnerProduct).to_vec();
+ let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::F32).to_vec();
bytes.extend_from_slice(record_bytes);
std::fs::write(path, bytes).unwrap();
}
@@ -1145,18 +1123,18 @@ mod tests {
) -> Result<Log, VecDbError> {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut bytes = encode_header(8, &[7u8; 16], StorageKind::F32, InnerProduct);
+ let mut bytes = encode_header(8, &[7u8; 16], StorageKind::F32);
mutate(&mut bytes);
std::fs::write(&path, bytes).unwrap();
let file = File::options().read(true).write(true).open(&path).unwrap();
- Log::open(file, &path, expected_dims, StorageKind::F32, InnerProduct)
+ Log::open(file, &path, expected_dims, StorageKind::F32)
}
#[test]
fn header_round_trips_through_create_and_reopen() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let created = Log::create(&path, 512, StorageKind::F32, InnerProduct).unwrap();
+ let created = Log::create(&path, 512, StorageKind::F32).unwrap();
let generation = created.generation();
assert_eq!(created.current_end_offset(), HEADER_LEN as u64);
assert_eq!(created.path(), path.as_path());
@@ -1172,7 +1150,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
let alias = dir.path().join("alias");
- let created = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let created = Log::create(&path, 8, StorageKind::F32).unwrap();
std::fs::hard_link(&path, &alias).unwrap();
let reader = File::open(&alias).unwrap();
assert_eq!(reader.metadata().unwrap().len(), HEADER_LEN as u64);
@@ -1250,16 +1228,18 @@ mod tests {
}
#[test]
- fn rejects_unknown_metric_tag() {
- let error = open_with_header(
- |bytes| {
- bytes[7] = 3;
- refresh_crc(bytes);
- },
- 8,
- )
- .unwrap_err();
- assert!(matches!(&error, VecDbError::Corrupt(message) if message.contains("metric")));
+ fn rejects_every_metric_tag_but_cosine() {
+ for tag in [0u8, 2, 3, 0xff] {
+ let error = open_with_header(
+ |bytes| {
+ bytes[METRIC_TAG_OFFSET] = tag;
+ refresh_crc(bytes);
+ },
+ 8,
+ )
+ .unwrap_err();
+ assert!(matches!(&error, VecDbError::Corrupt(message) if message.contains("metric")));
+ }
}
#[test]
@@ -1287,7 +1267,7 @@ mod tests {
let path = dir.path().join("log");
std::fs::write(&path, vec![1u8; junk_len]).unwrap();
let file = File::options().read(true).write(true).open(&path).unwrap();
- let mut log = Log::open(file, &path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::open(file, &path, 8, StorageKind::F32).unwrap();
assert_eq!(log.current_end_offset(), HEADER_LEN as u64);
assert_eq!(std::fs::metadata(&path).unwrap().len(), HEADER_LEN as u64);
let (records, _) = scan_all(&mut log);
@@ -1314,11 +1294,11 @@ mod tests {
fn create_rejects_invalid_dimensions() {
let dir = TempDir::new().unwrap();
assert!(matches!(
- Log::create(&dir.path().join("a"), 0, StorageKind::F32, InnerProduct),
+ Log::create(&dir.path().join("a"), 0, StorageKind::F32),
Err(VecDbError::InvalidDimensions { dims: 0, .. })
));
assert!(matches!(
- Log::create(&dir.path().join("b"), 12, StorageKind::F32, InnerProduct),
+ Log::create(&dir.path().join("b"), 12, StorageKind::F32),
Err(VecDbError::InvalidDimensions { dims: 12, .. })
));
}
@@ -1327,14 +1307,10 @@ mod tests {
fn open_rejects_invalid_dimensions() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- drop(
- Log::create(&path, 8, StorageKind::F32, InnerProduct)
- .unwrap()
- .into_file(),
- );
+ drop(Log::create(&path, 8, StorageKind::F32).unwrap().into_file());
let file = File::options().read(true).write(true).open(&path).unwrap();
assert!(matches!(
- Log::open(file, &path, 12, StorageKind::F32, InnerProduct),
+ Log::open(file, &path, 12, StorageKind::F32),
Err(VecDbError::DimensionMismatch {
expected: 12,
actual: 8
@@ -1346,9 +1322,9 @@ mod tests {
fn golden_bytes_pin_format_v1() {
let generation: [u8; 16] = std::array::from_fn(|index| index as u8);
let golden_header: [u8; 32] = [
- 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, 0x01,
0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
- 0x56, 0x32, 0x6d, 0x51,
+ 0xc8, 0xb1, 0xb7, 0xce,
];
let golden_add: [u8; 42] = [
0x01, 0x02, 0x00, 0x6b, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00,
@@ -1384,7 +1360,7 @@ mod tests {
},
];
assert_eq!(
- encode_header(8, &generation, StorageKind::F32, InnerProduct),
+ encode_header(8, &generation, StorageKind::F32),
golden_header
);
let mut encoded = Vec::new();
@@ -1455,7 +1431,7 @@ mod tests {
for dims in [8usize, 512] {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, dims, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, dims, StorageKind::F32).unwrap();
let vector = seeded_vector(1, dims);
let entries = [
LogEntry::Add {
@@ -1495,7 +1471,7 @@ mod tests {
];
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(5, 8);
for key in keys {
log.append(&[LogEntry::Add {
@@ -1530,7 +1506,7 @@ mod tests {
fn decodes_non_finite_components_without_panicking() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = [
f32::NAN,
f32::INFINITY,
@@ -1566,7 +1542,7 @@ mod tests {
fn scanner_stops_at_last_complete_record_for_every_torn_length() {
let dir = TempDir::new().unwrap();
let build_path = dir.path().join("log");
- let mut log = Log::create(&build_path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&build_path, 8, StorageKind::F32).unwrap();
let vectors: Vec<Vec<f32>> = (0..3).map(|seed| seeded_vector(seed, 8)).collect();
let mut boundaries = Vec::new();
for (index, vector) in vectors.iter().enumerate() {
@@ -1599,7 +1575,7 @@ mod tests {
fn truncate_to_repairs_torn_tail_for_clean_appends() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(1, 8);
let (_, first_end) = append_bounds(
&mut log,
@@ -1650,7 +1626,7 @@ mod tests {
fn extend_end_offset_grows_to_target_clamps_to_file_and_never_shrinks() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(1, 8);
let (_, captured_end) = append_bounds(
&mut log,
@@ -1690,7 +1666,7 @@ mod tests {
fn discard_unacked_tail_drops_phantom_frames_before_reopen() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(1, 8);
let (_, acked_end) = append_bounds(
&mut log,
@@ -1720,7 +1696,7 @@ mod tests {
fn pending_rollback_truncates_stale_records_before_the_next_append() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(6, 8);
let (_, acked_end) = append_bounds(
&mut log,
@@ -1766,8 +1742,8 @@ mod tests {
let dir = TempDir::new().unwrap();
let synced_path = dir.path().join("synced");
let staged_path = dir.path().join("staged");
- let mut synced = Log::create(&synced_path, 8, StorageKind::F32, InnerProduct).unwrap();
- let mut staged = Log::create(&staged_path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut synced = Log::create(&synced_path, 8, StorageKind::F32).unwrap();
+ let mut staged = Log::create(&staged_path, 8, StorageKind::F32).unwrap();
let vectors: Vec<Vec<f32>> = (0..3).map(|seed| seeded_vector(seed, 8)).collect();
let attrs = [Attribute {
name: "model".to_string(),
@@ -1817,7 +1793,7 @@ mod tests {
fn append_failure_rolls_back_without_disturbing_acked_records() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(2, 8);
let (_, acked_end) = append_bounds(
&mut log,
@@ -1829,7 +1805,7 @@ mod tests {
);
drop(log);
let unwritable = File::open(&path).unwrap();
- let mut log = Log::open(unwritable, &path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::open(unwritable, &path, 8, StorageKind::F32).unwrap();
assert!(matches!(
log.append(&[LogEntry::Add {
key: "lost",
@@ -1851,7 +1827,7 @@ mod tests {
fn append_after_phantom_frames_overwrites_at_the_acked_offset() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(3, 8);
let (_, acked_end) = append_bounds(
&mut log,
@@ -1892,7 +1868,7 @@ mod tests {
fn scanner_stops_at_corrupted_middle_record() {
let dir = TempDir::new().unwrap();
let build_path = dir.path().join("log");
- let mut log = Log::create(&build_path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&build_path, 8, StorageKind::F32).unwrap();
let vectors: Vec<Vec<f32>> = (0..3).map(|seed| seeded_vector(seed, 8)).collect();
let mut boundaries = Vec::new();
for (index, vector) in vectors.iter().enumerate() {
@@ -1971,7 +1947,7 @@ mod tests {
fn append_offsets_match_scanner_offsets() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
log.append(&[]).unwrap();
assert_eq!(log.current_end_offset(), HEADER_LEN as u64);
let vector = seeded_vector(7, 8);
@@ -2006,7 +1982,7 @@ mod tests {
fn append_validates_keys_and_vector_dims() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(1, 8);
let wrong_dims = seeded_vector(1, 16);
assert!(matches!(
@@ -2048,15 +2024,8 @@ mod tests {
#[test]
fn generation_is_distinct_across_creates() {
let dir = TempDir::new().unwrap();
- let first =
- Log::create(&dir.path().join("first"), 8, StorageKind::F32, InnerProduct).unwrap();
- let second = Log::create(
- &dir.path().join("second"),
- 8,
- StorageKind::F32,
- InnerProduct,
- )
- .unwrap();
+ let first = Log::create(&dir.path().join("first"), 8, StorageKind::F32).unwrap();
+ let second = Log::create(&dir.path().join("second"), 8, StorageKind::F32).unwrap();
assert_ne!(first.generation(), second.generation());
}
@@ -2064,11 +2033,10 @@ mod tests {
fn temp_sibling_gets_fresh_generation_and_replaces_stale_tmp() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let log = Log::create(&path, 8, StorageKind::F32).unwrap();
let stale_path = dir.path().join("log.tmp");
std::fs::write(&stale_path, b"stale leftover").unwrap();
- let (temp_log, temp_path) =
- Log::create_temp_sibling(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let (temp_log, temp_path) = Log::create_temp_sibling(&path, 8, StorageKind::F32).unwrap();
assert_eq!(temp_path, stale_path);
assert_ne!(temp_log.generation(), log.generation());
assert_eq!(temp_log.current_end_offset(), HEADER_LEN as u64);
@@ -2101,9 +2069,8 @@ mod tests {
attrs: &[],
})
.collect();
- let mut bulk_log = Log::create(&bulk_path, dims, StorageKind::F32, InnerProduct).unwrap();
- let mut sequential_log =
- Log::create(&sequential_path, dims, StorageKind::F32, InnerProduct).unwrap();
+ let mut bulk_log = Log::create(&bulk_path, dims, StorageKind::F32).unwrap();
+ let mut sequential_log = Log::create(&sequential_path, dims, StorageKind::F32).unwrap();
let (_, bulk_end) = append_bounds(&mut bulk_log, &entries);
let mut sequential_end = 0;
for entry in &entries {
@@ -2143,9 +2110,8 @@ mod tests {
attrs: &[],
},
];
- let mut bulk_log = Log::create(&bulk_path, 8, StorageKind::F32, InnerProduct).unwrap();
- let mut sequential_log =
- Log::create(&sequential_path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut bulk_log = Log::create(&bulk_path, 8, StorageKind::F32).unwrap();
+ let mut sequential_log = Log::create(&sequential_path, 8, StorageKind::F32).unwrap();
let (bulk_start, bulk_end) = append_bounds(&mut bulk_log, &entries);
let mut sequential_end = 0;
for entry in &entries {
@@ -2194,7 +2160,7 @@ mod tests {
let vector = seeded_vector(8, 8);
for (index, attrs) in singles.iter().chain([&full_set]).enumerate() {
let path = dir.path().join(format!("log-{index}"));
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
log.append(&[LogEntry::Add {
key: "k",
vector: VectorPayload::F32(&vector),
@@ -2229,7 +2195,7 @@ mod tests {
let vector = seeded_vector(9, 8);
for (index, attrs) in cases.iter().enumerate() {
let path = dir.path().join(format!("log-{index}"));
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
log.append(&[LogEntry::Add {
key: "k",
vector: VectorPayload::F32(&vector),
@@ -2267,7 +2233,7 @@ mod tests {
];
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&path, 8, StorageKind::F32).unwrap();
let vector = seeded_vector(10, 8);
for attrs in &invalid {
assert!(matches!(
@@ -2307,8 +2273,7 @@ mod tests {
.map(|(index, value)| attr(&format!("f{index}"), AttrValue::F64(*value)))
.collect();
let dir = TempDir::new().unwrap();
- let mut log =
- Log::create(&dir.path().join("log"), 8, StorageKind::F32, InnerProduct).unwrap();
+ let mut log = Log::create(&dir.path().join("log"), 8, StorageKind::F32).unwrap();
let vector = seeded_vector(11, 8);
log.append(&[LogEntry::Add {
key: "k",
@@ -2711,8 +2676,7 @@ mod tests {
];
for (name, after_first, intact_record_beyond) in cases {
let path = dir.path().join(name);
- let mut bytes =
- encode_header(dims as u32, &[9u8; 16], StorageKind::I8, InnerProduct).to_vec();
+ let mut bytes = encode_header(dims as u32, &[9u8; 16], StorageKind::I8).to_vec();
bytes.extend_from_slice(&first);
bytes.extend_from_slice(&after_first);
std::fs::write(&path, &bytes).unwrap();
@@ -2729,7 +2693,7 @@ mod tests {
}
fn write_i8_log_file(path: &Path, dims: u32, record_bytes: &[u8]) {
- let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::I8, InnerProduct).to_vec();
+ let mut bytes = encode_header(dims, &[9u8; 16], StorageKind::I8).to_vec();
bytes.extend_from_slice(record_bytes);
std::fs::write(path, bytes).unwrap();
}
@@ -2757,9 +2721,9 @@ mod tests {
fn i8_golden_bytes_pin_the_header_and_add_record() {
let generation: [u8; 16] = std::array::from_fn(|index| index as u8);
let golden_header: [u8; 32] = [
- 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x45, 0x56, 0x44, 0x42, 0x01, 0x00, 0x01, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, 0x01,
0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
- 0x2f, 0xd0, 0x62, 0xc1,
+ 0xb1, 0x53, 0xb8, 0x5e,
];
let golden_add: [u8; 46] = [
0x01, 0x02, 0x00, 0x6b, 0x31, 0x0a, 0xd7, 0x23, 0x3c, 0x81, 0x7f, 0x00, 0xff, 0xa4,
@@ -2770,7 +2734,7 @@ mod tests {
let scale = 0.01f32;
assert_eq!(scale.to_bits(), 0x3c23_d70a);
let values = golden_i8_values();
- let header = encode_header(32, &generation, StorageKind::I8, InnerProduct);
+ let header = encode_header(32, &generation, StorageKind::I8);
assert_eq!(header[STORAGE_TAG_OFFSET], 1);
assert_eq!(header[..28], golden_header[..28]);
let encoded = encoded_i8_add("k1", scale, &values);
@@ -2818,15 +2782,15 @@ mod tests {
#[test]
fn f32_headers_leave_the_storage_tag_zero() {
- let header = encode_header(8, &[7u8; 16], StorageKind::F32, InnerProduct);
+ let header = encode_header(8, &[7u8; 16], StorageKind::F32);
assert_eq!(header[STORAGE_TAG_OFFSET], 0);
+ assert_eq!(header[METRIC_TAG_OFFSET], COSINE_METRIC_TAG);
assert_eq!(
decode_header(&header).unwrap(),
Header {
generation: [7u8; 16],
dims: 8,
storage: StorageKind::F32,
- metric: InnerProduct
}
);
}
@@ -2851,18 +2815,18 @@ mod tests {
let dir = TempDir::new().unwrap();
let i8_path = dir.path().join("i8");
let f32_path = dir.path().join("f32");
- let created = Log::create(&i8_path, 32, StorageKind::I8, InnerProduct).unwrap();
+ let created = Log::create(&i8_path, 32, StorageKind::I8).unwrap();
assert_eq!(created.storage(), StorageKind::I8);
let generation = created.generation();
drop(created.into_file());
drop(
- Log::create(&f32_path, 32, StorageKind::F32, InnerProduct)
+ Log::create(&f32_path, 32, StorageKind::F32)
.unwrap()
.into_file(),
);
let open = |path: &Path, requested: StorageKind| {
let file = File::options().read(true).write(true).open(path).unwrap();
- Log::open(file, path, 32, requested, InnerProduct)
+ Log::open(file, path, 32, requested)
};
let detected = open(&i8_path, StorageKind::I8).unwrap();
assert_eq!(detected.storage(), StorageKind::I8);
@@ -2899,7 +2863,7 @@ mod tests {
let path = dir.path().join("log");
std::fs::write(&path, [1u8; 5]).unwrap();
let file = File::options().read(true).write(true).open(&path).unwrap();
- let log = Log::open(file, &path, 64, StorageKind::I8, InnerProduct).unwrap();
+ let log = Log::open(file, &path, 64, StorageKind::I8).unwrap();
assert_eq!(log.storage(), StorageKind::I8);
drop(log);
assert_eq!(
@@ -2910,7 +2874,7 @@ mod tests {
std::fs::write(&other, [1u8; 5]).unwrap();
let file = File::options().read(true).write(true).open(&other).unwrap();
assert_eq!(
- Log::open(file, &other, 64, StorageKind::I8, InnerProduct)
+ Log::open(file, &other, 64, StorageKind::I8)
.unwrap()
.storage(),
StorageKind::I8
@@ -2922,33 +2886,27 @@ mod tests {
let dir = TempDir::new().unwrap();
for dims in [8usize, 16, 24, 40] {
assert!(matches!(
- Log::create(&dir.path().join(format!("a{dims}")), dims, StorageKind::I8, InnerProduct),
+ Log::create(&dir.path().join(format!("a{dims}")), dims, StorageKind::I8),
Err(VecDbError::InvalidDimensions {
dims: rejected,
storage: StorageKind::I8
}) if rejected == dims
));
assert!(
- Log::create(
- &dir.path().join(format!("b{dims}")),
- dims,
- StorageKind::F32,
- InnerProduct
- )
- .is_ok()
+ Log::create(&dir.path().join(format!("b{dims}")), dims, StorageKind::F32).is_ok()
);
}
- assert!(Log::create(&dir.path().join("c"), 32, StorageKind::I8, InnerProduct).is_ok());
+ assert!(Log::create(&dir.path().join("c"), 32, StorageKind::I8).is_ok());
let path = dir.path().join("c");
let file = File::options().read(true).write(true).open(&path).unwrap();
assert!(matches!(
- Log::open(file, &path, 8, StorageKind::I8, InnerProduct),
+ Log::open(file, &path, 8, StorageKind::I8),
Err(VecDbError::DimensionMismatch {
expected: 8,
actual: 32
})
));
- let mut forged = encode_header(8, &[3u8; 16], StorageKind::I8, InnerProduct);
+ let mut forged = encode_header(8, &[3u8; 16], StorageKind::I8);
refresh_crc(&mut forged);
let forged_path = dir.path().join("forged");
std::fs::write(&forged_path, forged).unwrap();
@@ -2958,7 +2916,7 @@ mod tests {
.open(&forged_path)
.unwrap();
assert!(matches!(
- Log::open(file, &forged_path, 8, StorageKind::I8, InnerProduct),
+ Log::open(file, &forged_path, 8, StorageKind::I8),
Err(VecDbError::InvalidDimensions {
dims: 8,
storage: StorageKind::I8
@@ -2971,7 +2929,7 @@ mod tests {
for dims in [32usize, 512] {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
- let mut log = Log::create(&path, dims, StorageKind::I8, InnerProduct).unwrap();
+ let mut log = Log::create(&path, dims, StorageKind::I8).unwrap();
let scales = [
f32::from_bits(0x3a1f_8f3c),
0.0,
@@ -3033,8 +2991,8 @@ mod tests {
let dir = TempDir::new().unwrap();
let i8_path = dir.path().join("i8");
let f32_path = dir.path().join("f32");
- let mut i8_log = Log::create(&i8_path, 32, StorageKind::I8, InnerProduct).unwrap();
- let mut f32_log = Log::create(&f32_path, 32, StorageKind::F32, InnerProduct).unwrap();
+ let mut i8_log = Log::create(&i8_path, 32, StorageKind::I8).unwrap();
+ let mut f32_log = Log::create(&f32_path, 32, StorageKind::F32).unwrap();
let values = seeded_vector(4, 32);
let quantized = StoredVector::quantize(&values);
assert!(matches!(
@@ -3101,7 +3059,7 @@ mod tests {
let dims = 32usize;
let dir = TempDir::new().unwrap();
let build_path = dir.path().join("log");
- let mut log = Log::create(&build_path, dims, StorageKind::I8, InnerProduct).unwrap();
+ let mut log = Log::create(&build_path, dims, StorageKind::I8).unwrap();
let mut boundaries = Vec::new();
for index in 0..3u64 {
let key = format!("key-{index}");
diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rs
index 5144498a55..75dfb61af4 100644
--- a/rust/crates/vecdb/src/store.rs
+++ b/rust/crates/vecdb/src/store.rs
@@ -20,9 +20,7 @@ use super::log::{
use super::snapshot::{
LoadedSnapshot, load_snapshot, remove_snapshot, snapshot_path, write_snapshot,
};
-use super::{
- AttrValue, Attribute, DistanceMetric, KeyMatches, Match, SearchParams, StorageKind, VecDbError,
-};
+use super::{AttrValue, Attribute, KeyMatches, Match, SearchParams, StorageKind, VecDbError};
const SNAPSHOT_HARD_CAP: usize = 5000;
const SNAPSHOT_QUIET_THRESHOLD: usize = 1000;
@@ -35,7 +33,6 @@ const HANDOFF_CLOSE_WAIT_ROUNDS: u32 = 2500;
const HANDOFF_WAIT_PARK: Duration = Duration::from_millis(2);
const KEY_TABLE_COPIES: usize = 2;
const KEY_ENTRY_OVERHEAD_BYTES: usize = 48;
-const INNER_PRODUCT_SQUARED_NORM_TOLERANCE: f64 = 1e-2;
static REGISTRY: LazyLock<Mutex<HashMap<PathBuf, Arc<Mutex<PathSlot>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
@@ -67,7 +64,6 @@ pub struct Stats {
pub dead_count: usize,
pub dims: usize,
pub storage: StorageKind,
- pub metric: DistanceMetric,
pub log_bytes: u64,
pub records_since_snapshot: usize,
pub approximate_memory_bytes: usize,
@@ -82,7 +78,6 @@ struct Shared {
path: PathBuf,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
registry_key: Option<PathBuf>,
closed: AtomicBool,
writer: Mutex<WriterHalf>,
@@ -309,7 +304,6 @@ fn join_live(
shared: Arc<Shared>,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
read_only: bool,
) -> Result<VecDb, VecDbError> {
if shared.dims != dims {
@@ -324,12 +318,6 @@ fn join_live(
actual: shared.storage,
});
}
- if metric != shared.metric {
- return Err(VecDbError::MetricMismatch {
- expected: metric,
- actual: shared.metric,
- });
- }
Ok(VecDb { shared, read_only })
}
@@ -384,10 +372,8 @@ impl VecDb {
path: &Path,
dims: usize,
storage: Option<StorageKind>,
- metric: Option<DistanceMetric>,
) -> Result<Self, VecDbError> {
let storage = storage.unwrap_or(StorageKind::I8);
- let metric = metric.unwrap_or(DistanceMetric::Cosine);
let key = registry_key_for(path)?;
let slot = path_slot(&key);
let mut guard = lock_slot(&slot);
@@ -396,20 +382,20 @@ impl VecDb {
drop(guard);
match observed {
Some(shared) if !shared.is_closed() => {
- return join_live(shared, dims, storage, metric, false);
+ return join_live(shared, dims, storage, false);
}
observed => drop(observed),
}
match wait_for_teardown_handoff(&slot, &key) {
SlotHandoff::Join(shared) => {
- return join_live(shared, dims, storage, metric, false);
+ return join_live(shared, dims, storage, false);
}
SlotHandoff::Released(reacquired) | SlotHandoff::Expired(reacquired) => {
guard = reacquired;
}
}
}
- let built = match build_writer(&key, key.clone(), dims, storage, metric) {
+ let built = match build_writer(&key, key.clone(), dims, storage) {
Ok(built) => built,
Err(error) => {
drop(guard);
@@ -458,22 +444,20 @@ impl VecDb {
path: &Path,
dims: usize,
storage: Option<StorageKind>,
- metric: Option<DistanceMetric>,
) -> Result<Self, VecDbError> {
let storage = storage.unwrap_or(StorageKind::I8);
- let metric = metric.unwrap_or(DistanceMetric::Cosine);
let resolved = registry_key_for(path).unwrap_or_else(|_| path.to_path_buf());
if let Some(slot) = existing_path_slot(&resolved) {
let observed = lock_slot(&slot).live.as_ref().and_then(Weak::upgrade);
match observed {
Some(shared) if !shared.is_closed() => {
- return join_live(shared, dims, storage, metric, true);
+ return join_live(shared, dims, storage, true);
}
Some(closing) => {
drop(closing);
match wait_for_teardown_handoff(&slot, &resolved) {
SlotHandoff::Join(shared) => {
- return join_live(shared, dims, storage, metric, true);
+ return join_live(shared, dims, storage, true);
}
SlotHandoff::Released(released) | SlotHandoff::Expired(released) => {
drop(released);
@@ -484,7 +468,7 @@ impl VecDb {
}
}
Ok(Self {
- shared: Arc::new(build_read_only(&resolved, dims, storage, metric)?),
+ shared: Arc::new(build_read_only(&resolved, dims, storage)?),
read_only: true,
})
}
@@ -502,7 +486,7 @@ impl VecDb {
let now = Instant::now();
let mut half = self.writable_half()?;
let state = active_state(&mut half)?;
- ensure_storable(self.shared.metric, key, vector)?;
+ ensure_finite(key, vector)?;
let quantized = quantize_for(self.shared.storage, vector);
let payload = quantized
.as_ref()
@@ -559,7 +543,7 @@ impl VecDb {
return Ok(());
}
for (key, vector) in keys.iter().zip(vectors) {
- ensure_storable(self.shared.metric, key, vector)?;
+ ensure_finite(key, vector)?;
}
let quantized: Option<Vec<StoredVector>> = match self.shared.storage {
StorageKind::F32 => None,
@@ -792,7 +776,7 @@ impl VecDb {
let dims = self.shared.dims;
let storage = self.shared.storage;
let recreated = remove_data_files(&self.shared.path)
- .and_then(|()| Log::create(&self.shared.path, dims, storage, self.shared.metric));
+ .and_then(|()| Log::create(&self.shared.path, dims, storage));
let log = match recreated {
Ok(log) => log,
Err(error) => {
@@ -806,7 +790,7 @@ impl VecDb {
return Err(error);
}
};
- let arena = VectorArena::with_metric(dims, storage, self.shared.metric)?;
+ let arena = VectorArena::with_storage(dims, storage)?;
{
let mut st = self.shared.state_write();
st.arena = arena;
@@ -892,7 +876,6 @@ impl VecDb {
dead_count: st.total_records.saturating_sub(live_count as u64) as usize,
dims: st.arena.dims(),
storage: self.shared.storage,
- metric: self.shared.metric,
log_bytes,
records_since_snapshot,
approximate_memory_bytes: approximate_memory_bytes(&st),
@@ -1019,7 +1002,7 @@ fn close_live_instance(shared: &Arc<Shared>) -> Result<(), VecDbError> {
shared.closed.store(true, Ordering::Release);
drop(log);
let removed = remove_data_files(&shared.path);
- if let Ok(empty) = VectorArena::with_metric(shared.dims, shared.storage, shared.metric) {
+ if let Ok(empty) = VectorArena::with_storage(shared.dims, shared.storage) {
st.arena = empty;
st.graph = Some(Graph::new());
st.attrs.reset();
@@ -1135,7 +1118,7 @@ fn write_snapshot_now(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecD
fn compact(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecDbError> {
active_state(half)?;
let (mut temp_log, temp_path) =
- Log::create_temp_sibling(&shared.path, shared.dims, shared.storage, shared.metric)?;
+ Log::create_temp_sibling(&shared.path, shared.dims, shared.storage)?;
let staged = {
let st = shared.state_read();
stage_live_entries(&st, &mut temp_log)
@@ -1214,13 +1197,7 @@ fn restore_writer_mode(
"log header was truncated while the writer was closed".to_string(),
));
}
- let mut log = Log::open(
- file,
- &shared.path,
- shared.dims,
- shared.storage,
- shared.metric,
- )?;
+ let mut log = Log::open(file, &shared.path, shared.dims, shared.storage)?;
let mut pending = mutations_since_snapshot;
if log.checkpoint() != expected_checkpoint {
let replayed = replay(&mut log, shared.dims, ReplayMode::Writer)?;
@@ -1271,7 +1248,7 @@ fn recover_after_failed_reset(
if restore_writer_mode(shared, half, lock, None, old_pending, old_checkpoint).is_ok() {
return;
}
- if let Ok(empty) = VectorArena::with_metric(shared.dims, shared.storage, shared.metric) {
+ if let Ok(empty) = VectorArena::with_storage(shared.dims, shared.storage) {
let mut st = shared.state_write();
st.arena = empty;
st.graph = Some(Graph::new());
@@ -1295,16 +1272,15 @@ fn build_writer(
registry_key: PathBuf,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
) -> Result<BuiltWriter, VecDbError> {
let lock = WriterLock::acquire(path)?;
let mut log = if std::fs::metadata(path).is_ok() {
- open_existing_writer_log(path, dims, storage, metric)?
+ open_existing_writer_log(path, dims, storage)?
} else {
- match Log::create(path, dims, storage, metric) {
+ match Log::create(path, dims, storage) {
Ok(created) => created,
Err(VecDbError::Io { source, .. }) if source.kind() == ErrorKind::AlreadyExists => {
- open_existing_writer_log(path, dims, storage, metric)?
+ open_existing_writer_log(path, dims, storage)?
}
Err(error) => return Err(error),
}
@@ -1397,7 +1373,6 @@ fn writer_shared(
path: path.to_path_buf(),
dims,
storage,
- metric: state.arena.metric(),
registry_key: Some(registry_key),
closed: AtomicBool::new(false),
writer: Mutex::new(WriterHalf {
@@ -1450,16 +1425,11 @@ fn install_graph(shared: &Shared, graph: Graph) {
}
}
-fn build_read_only(
- path: &Path,
- dims: usize,
- storage: StorageKind,
- metric: DistanceMetric,
-) -> Result<Shared, VecDbError> {
+fn build_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Shared, VecDbError> {
let file = match File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == ErrorKind::NotFound => {
- return empty_read_only(path, dims, storage, metric);
+ return empty_read_only(path, dims, storage);
}
Err(source) => return Err(VecDbError::io(path, source)),
};
@@ -1468,9 +1438,9 @@ fn build_read_only(
.map_err(|source| VecDbError::io(path, source))?
.len();
if file_len < HEADER_LEN as u64 {
- return empty_read_only(path, dims, storage, metric);
+ return empty_read_only(path, dims, storage);
}
- let mut log = Log::open(file, path, dims, storage, metric)?;
+ let mut log = Log::open(file, path, dims, storage)?;
let replayed = replay(&mut log, dims, ReplayMode::ReadOnly)?;
drop(log);
let ReplayedState {
@@ -1492,15 +1462,10 @@ fn build_read_only(
))
}
-fn empty_read_only(
- path: &Path,
- dims: usize,
- storage: StorageKind,
- metric: DistanceMetric,
-) -> Result<Shared, VecDbError> {
+fn empty_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Shared, VecDbError> {
Ok(read_only_shared(
path,
- VectorArena::with_metric(dims, storage, metric)?,
+ VectorArena::with_storage(dims, storage)?,
Graph::new(),
AttrTable::default(),
0,
@@ -1520,7 +1485,6 @@ fn read_only_shared(
path: path.to_path_buf(),
dims: arena.dims(),
storage: arena.storage_kind(),
- metric: arena.metric(),
registry_key: None,
closed: AtomicBool::new(false),
writer: Mutex::new(WriterHalf {
@@ -1584,7 +1548,7 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,
let snapshot_covered = snapshot.as_ref().map(|loaded| loaded.covered_log_offset);
let covered = snapshot_covered.unwrap_or(u64::MAX);
let path = log.path().to_path_buf();
- let mut arena = VectorArena::with_metric(dims, log.storage(), log.metric())?;
+ let mut arena = VectorArena::with_storage(dims, log.storage())?;
let mut attrs = AttrTable::default();
let mut graph: Option<Graph> = None;
let mut tail_records = 0u64;
@@ -1680,14 +1644,6 @@ fn apply_outcome(graph: &mut Graph, arena: &VectorArena, outcome: UpsertOutcome)
}
}
-fn ensure_storable(metric: DistanceMetric, key: &str, vector: &[f32]) -> Result<(), VecDbError> {
- ensure_finite(key, vector)?;
- if metric == DistanceMetric::InnerProduct {
- ensure_unit_norm(key, vector)?;
- }
- Ok(())
-}
-
fn ensure_finite(key: &str, vector: &[f32]) -> Result<(), VecDbError> {
if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
return Err(VecDbError::InvalidVector(format!(
@@ -1697,24 +1653,13 @@ fn ensure_finite(key: &str, vector: &[f32]) -> Result<(), VecDbError> {
Ok(())
}
-fn ensure_unit_norm(key: &str, vector: &[f32]) -> Result<(), VecDbError> {
- let squared_norm: f64 = vector.iter().map(|&value| f64::from(value).powi(2)).sum();
- if (squared_norm - 1.0).abs() > INNER_PRODUCT_SQUARED_NORM_TOLERANCE {
- return Err(VecDbError::InvalidVector(format!(
- "the vector for key {key:?} has squared norm {squared_norm}, and an inner product index needs unit vectors"
- )));
- }
- Ok(())
-}
-
fn open_existing_writer_log(
path: &Path,
dims: usize,
storage: StorageKind,
- metric: DistanceMetric,
) -> Result<Log, VecDbError> {
let file = open_writer_file(path, false)?;
- Log::open(file, path, dims, storage, metric)
+ Log::open(file, path, dims, storage)
}
fn promote_compacted_log(temp_path: &Path, path: &Path) -> Result<(), VecDbError> {
@@ -1791,96 +1736,48 @@ mod tests {
const TOMBSTONE_RECORD_LEN: u64 = 3 + 5 + 4;
#[test]
- fn omitted_metric_requires_cosine_for_creation_and_every_reopen() {
+ fn omitted_storage_requires_i8_for_creation_and_every_reopen() {
let dir = TempDir::new().unwrap();
- let path = dir.path().join("cosine");
- let reader = VecDb::open_read_only(&path, 32, None, None).unwrap();
- assert_eq!(reader.stats().unwrap().metric, DistanceMetric::Cosine);
+ let path = dir.path().join("index");
+ let reader = VecDb::open_read_only(&path, 32, None).unwrap();
assert_eq!(reader.stats().unwrap().storage, StorageKind::I8);
assert!(!path.exists());
drop(reader);
- let db = VecDb::open(&path, 32, None, None).unwrap();
+ let db = VecDb::open(&path, 32, None).unwrap();
db.add("kept", &seeded_unit_vector(1, 32)).unwrap();
db.flush().unwrap();
let verify = || {
for open in [VecDb::open, VecDb::open_read_only] {
- let reopened = open(&path, 32, None, None).unwrap();
- assert_eq!(reopened.stats().unwrap().metric, DistanceMetric::Cosine);
+ let reopened = open(&path, 32, None).unwrap();
+ assert_eq!(reopened.stats().unwrap().storage, StorageKind::I8);
assert!(reopened.contains("kept"));
- let explicit = open(&path, 32, None, Some(DistanceMetric::Cosine)).unwrap();
- assert!(explicit.contains("kept"));
}
};
verify();
drop(db);
verify();
- let dot_path = dir.path().join("inner-product");
- let db = VecDb::open(&dot_path, 32, None, Some(DistanceMetric::InnerProduct)).unwrap();
+ let f32_path = dir.path().join("f32");
+ let db = VecDb::open(&f32_path, 32, Some(StorageKind::F32)).unwrap();
let reject = || {
- let before = fs::read(&dot_path).unwrap();
+ let before = fs::read(&f32_path).unwrap();
for open in [VecDb::open, VecDb::open_read_only] {
assert!(matches!(
- open(&dot_path, 32, None, None),
- Err(VecDbError::MetricMismatch {
- expected: DistanceMetric::Cosine,
- actual: DistanceMetric::InnerProduct
+ open(&f32_path, 32, None),
+ Err(VecDbError::StorageMismatch {
+ expected: StorageKind::I8,
+ actual: StorageKind::F32
})
));
}
- assert_eq!(fs::read(&dot_path).unwrap(), before);
+ assert_eq!(fs::read(&f32_path).unwrap(), before);
};
reject();
drop(db);
reject();
- }
-
- #[test]
- fn omitted_storage_requires_i8_for_creation_and_every_reopen() {
- let dir = TempDir::new().unwrap();
- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
- let path = dir.path().join(metric.to_string());
- let reader = VecDb::open_read_only(&path, 32, None, Some(metric)).unwrap();
- assert_eq!(reader.stats().unwrap().storage, StorageKind::I8);
- assert_eq!(reader.stats().unwrap().metric, metric);
- assert!(!path.exists());
- drop(reader);
- let db = VecDb::open(&path, 32, None, Some(metric)).unwrap();
- db.add("kept", &seeded_unit_vector(1, 32)).unwrap();
- db.flush().unwrap();
- let verify = || {
- for open in [VecDb::open, VecDb::open_read_only] {
- let reopened = open(&path, 32, None, Some(metric)).unwrap();
- assert_eq!(reopened.stats().unwrap().storage, StorageKind::I8);
- assert_eq!(reopened.stats().unwrap().metric, metric);
- assert!(reopened.contains("kept"));
- }
- };
- verify();
- drop(db);
- verify();
- let f32_path = dir.path().join(format!("f32-{metric}"));
- let db = VecDb::open(&f32_path, 32, Some(StorageKind::F32), Some(metric)).unwrap();
- let reject = || {
- let before = fs::read(&f32_path).unwrap();
- for open in [VecDb::open, VecDb::open_read_only] {
- assert!(matches!(
- open(&f32_path, 32, None, Some(metric)),
- Err(VecDbError::StorageMismatch {
- expected: StorageKind::I8,
- actual: StorageKind::F32
- })
- ));
- }
- assert_eq!(fs::read(&f32_path).unwrap(), before);
- };
- reject();
- drop(db);
- reject();
- }
let path = dir.path().join("invalid");
for open in [VecDb::open, VecDb::open_read_only] {
assert!(matches!(
- open(&path, 8, None, Some(DistanceMetric::Cosine)),
+ open(&path, 8, None),
Err(VecDbError::InvalidDimensions {
dims: 8,
storage: StorageKind::I8
@@ -1894,51 +1791,41 @@ mod tests {
fn explicit_configuration_is_verified_on_live_and_disk_opens() {
let dir = TempDir::new().unwrap();
for storage in [StorageKind::F32, StorageKind::I8] {
- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
- let path = dir.path().join(format!("{storage}-{metric}"));
- let other_storage = match storage {
- StorageKind::F32 => StorageKind::I8,
- StorageKind::I8 => StorageKind::F32,
- };
- let other_metric = match metric {
- DistanceMetric::Cosine => DistanceMetric::InnerProduct,
- DistanceMetric::InnerProduct => DistanceMetric::Cosine,
- };
- let db = VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap();
- db.add("kept", &seeded_unit_vector(1, 32)).unwrap();
- db.flush().unwrap();
- let verify = |live: Option<&VecDb>| {
- let before = fs::read(&path).unwrap();
- let snapshot_before = fs::read(snapshot_path(&path)).unwrap();
- for open in [VecDb::open, VecDb::open_read_only] {
- assert!(matches!(open(&path, 32, Some(other_storage), Some(metric)),
- Err(VecDbError::StorageMismatch { expected, actual })
- if expected == other_storage && actual == storage));
- assert!(matches!(open(&path, 32, Some(storage), Some(other_metric)),
- Err(VecDbError::MetricMismatch { expected, actual })
- if expected == other_metric && actual == metric));
- assert!(matches!(
- open(&path, 64, Some(storage), Some(metric)),
- Err(VecDbError::DimensionMismatch {
- expected: 64,
- actual: 32
- })
- ));
- assert_eq!(fs::read(&path).unwrap(), before);
- assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_before);
- let reopened = open(&path, 32, Some(storage), Some(metric)).unwrap();
- assert_eq!(reopened.stats().unwrap().storage, storage);
- assert_eq!(reopened.stats().unwrap().metric, metric);
- assert!(reopened.contains("kept"));
- if let Some(live) = live {
- assert!(Arc::ptr_eq(&live.shared, &reopened.shared));
- }
+ let path = dir.path().join(storage.to_string());
+ let other_storage = match storage {
+ StorageKind::F32 => StorageKind::I8,
+ StorageKind::I8 => StorageKind::F32,
+ };
+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();
+ db.add("kept", &seeded_unit_vector(1, 32)).unwrap();
+ db.flush().unwrap();
+ let verify = |live: Option<&VecDb>| {
+ let before = fs::read(&path).unwrap();
+ let snapshot_before = fs::read(snapshot_path(&path)).unwrap();
+ for open in [VecDb::open, VecDb::open_read_only] {
+ assert!(matches!(open(&path, 32, Some(other_storage)),
+ Err(VecDbError::StorageMismatch { expected, actual })
+ if expected == other_storage && actual == storage));
+ assert!(matches!(
+ open(&path, 64, Some(storage)),
+ Err(VecDbError::DimensionMismatch {
+ expected: 64,
+ actual: 32
+ })
+ ));
+ assert_eq!(fs::read(&path).unwrap(), before);
+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_before);
+ let reopened = open(&path, 32, Some(storage)).unwrap();
+ assert_eq!(reopened.stats().unwrap().storage, storage);
+ assert!(reopened.contains("kept"));
+ if let Some(live) = live {
+ assert!(Arc::ptr_eq(&live.shared, &reopened.shared));
}
- };
- verify(Some(&db));
- drop(db);
- verify(None);
- }
+ }
+ };
+ verify(Some(&db));
+ drop(db);
+ verify(None);
}
}
@@ -1946,51 +1833,40 @@ mod tests {
fn missing_and_incomplete_indexes_use_the_explicit_configuration() {
let dir = TempDir::new().unwrap();
for storage in [StorageKind::F32, StorageKind::I8] {
- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
- for incomplete in [false, true] {
- let path = dir.path().join(format!("{storage}-{metric}-{incomplete}"));
- let dims = storage.lane_width();
- if incomplete {
- fs::write(&path, [1, 2, 3]).unwrap();
- }
- let reader =
- VecDb::open_read_only(&path, dims, Some(storage), Some(metric)).unwrap();
- let stats = reader.stats().unwrap();
- assert_eq!(stats.dims, dims);
- assert_eq!(stats.storage, storage);
- assert_eq!(stats.metric, metric);
- assert!(reader.is_empty());
- assert!(matches!(
- reader.add("no", &seeded_unit_vector(1, dims)),
- Err(VecDbError::ReadOnly)
- ));
- if incomplete {
- assert_eq!(fs::read(&path).unwrap(), [1, 2, 3]);
- } else {
- assert!(!path.exists());
- }
- drop(reader);
- let writer = VecDb::open(&path, dims, Some(storage), Some(metric)).unwrap();
- assert_eq!(writer.stats().unwrap().storage, storage);
- assert_eq!(writer.stats().unwrap().metric, metric);
- writer.add("kept", &seeded_unit_vector(1, dims)).unwrap();
- drop(writer);
- let reopened =
- VecDb::open_read_only(&path, dims, Some(storage), Some(metric)).unwrap();
- assert!(reopened.contains("kept"));
- assert_eq!(reopened.stats().unwrap().metric, metric);
+ for incomplete in [false, true] {
+ let path = dir.path().join(format!("{storage}-{incomplete}"));
+ let dims = storage.lane_width();
+ if incomplete {
+ fs::write(&path, [1, 2, 3]).unwrap();
+ }
+ let reader = VecDb::open_read_only(&path, dims, Some(storage)).unwrap();
+ let stats = reader.stats().unwrap();
+ assert_eq!(stats.dims, dims);
+ assert_eq!(stats.storage, storage);
+ assert!(reader.is_empty());
+ assert!(matches!(
+ reader.add("no", &seeded_unit_vector(1, dims)),
+ Err(VecDbError::ReadOnly)
+ ));
+ if incomplete {
+ assert_eq!(fs::read(&path).unwrap(), [1, 2, 3]);
+ } else {
+ assert!(!path.exists());
}
+ drop(reader);
+ let writer = VecDb::open(&path, dims, Some(storage)).unwrap();
+ assert_eq!(writer.stats().unwrap().storage, storage);
+ writer.add("kept", &seeded_unit_vector(1, dims)).unwrap();
+ drop(writer);
+ let reopened = VecDb::open_read_only(&path, dims, Some(storage)).unwrap();
+ assert!(reopened.contains("kept"));
+ assert_eq!(reopened.stats().unwrap().storage, storage);
}
}
for open in [VecDb::open, VecDb::open_read_only] {
let path = dir.path().join("invalid");
assert!(matches!(
- open(
- &path,
- 8,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ open(&path, 8, Some(StorageKind::I8)),
Err(VecDbError::InvalidDimensions {
dims: 8,
storage: StorageKind::I8
@@ -2004,13 +1880,7 @@ mod tests {
fn cosine_scores_unnormalized_vectors_in_every_search_path() {
let dir = TempDir::new().unwrap();
for storage in [StorageKind::F32, StorageKind::I8] {
- let db = VecDb::open(
- &dir.path().join(storage.to_string()),
- 32,
- Some(storage),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::open(&dir.path().join(storage.to_string()), 32, Some(storage)).unwrap();
let vector = |x, y| {
let mut values = vec![0.0; 32];
values[0] = x;
@@ -2085,80 +1955,64 @@ mod tests {
}
#[test]
- fn metrics_survive_replacement_compaction_snapshot_replay_and_reset() {
+ fn cosine_scores_survive_replacement_compaction_snapshot_replay_and_reset() {
let dir = TempDir::new().unwrap();
for storage in [StorageKind::F32, StorageKind::I8] {
- for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
- let path = dir.path().join(format!("{storage}-{metric}"));
- let db = VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap();
- let mut vector = vec![0.0; 32];
- vector[0] = 1.0;
- db.add("removed", &vector).unwrap();
- db.add("kept", &vector).unwrap();
- db.remove("removed").unwrap();
- vector[0] = 0.0;
- vector[1] = 1.0;
- db.add("kept", &vector).unwrap();
- let mut recycled = vec![0.0; 32];
- recycled[2] = 1.0;
- db.add("recycled", &recycled).unwrap();
- db.remove("recycled").unwrap();
- compact(&db.shared, &mut db.shared.writer_half()).unwrap();
- db.flush().unwrap();
- let query = scaled(&vector, 2.0);
- let expected = if metric == DistanceMetric::Cosine {
- 0.0
- } else {
- -1.0
- };
- let assert_score = |db: &VecDb| {
- assert_eq!(db.stats().unwrap().metric, metric);
- for exact in [false, true] {
- let found = db
- .search(
- &query,
- &SearchParams {
- limit: Some(1),
- exact,
- ..Default::default()
- },
- )
- .unwrap();
- assert_eq!(found[0].key, "kept");
- assert!((found[0].distance - expected).abs() < 1.0e-5);
- }
- };
- assert_score(&db);
- drop(db);
- for snapshot in [true, false] {
- if !snapshot {
- remove_snapshot(&path).unwrap();
- }
- assert_score(
- &VecDb::open_read_only(&path, 32, Some(storage), Some(metric)).unwrap(),
- );
- assert_score(&VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap());
+ let path = dir.path().join(storage.to_string());
+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();
+ let mut vector = vec![0.0; 32];
+ vector[0] = 1.0;
+ db.add("removed", &vector).unwrap();
+ db.add("kept", &vector).unwrap();
+ db.remove("removed").unwrap();
+ vector[0] = 0.0;
+ vector[1] = 1.0;
+ db.add("kept", &vector).unwrap();
+ let mut recycled = vec![0.0; 32];
+ recycled[2] = 1.0;
+ db.add("recycled", &recycled).unwrap();
+ db.remove("recycled").unwrap();
+ compact(&db.shared, &mut db.shared.writer_half()).unwrap();
+ db.flush().unwrap();
+ let query = scaled(&vector, 2.0);
+ let assert_score = |db: &VecDb| {
+ for exact in [false, true] {
+ let found = db
+ .search(
+ &query,
+ &SearchParams {
+ limit: Some(1),
+ exact,
+ ..Default::default()
+ },
+ )
+ .unwrap();
+ assert_eq!(found[0].key, "kept");
+ assert!(found[0].distance.abs() < 1.0e-5);
}
- let db = VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap();
- db.reset().unwrap();
- db.add("kept", &vector).unwrap();
- assert_score(&db);
- drop(db);
- assert_score(&VecDb::open(&path, 32, Some(storage), Some(metric)).unwrap());
+ };
+ assert_score(&db);
+ drop(db);
+ for snapshot in [true, false] {
+ if !snapshot {
+ remove_snapshot(&path).unwrap();
+ }
+ assert_score(&VecDb::open_read_only(&path, 32, Some(storage)).unwrap());
+ assert_score(&VecDb::open(&path, 32, Some(storage)).unwrap());
}
+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();
+ db.reset().unwrap();
+ db.add("kept", &vector).unwrap();
+ assert_score(&db);
+ drop(db);
+ assert_score(&VecDb::open(&path, 32, Some(storage)).unwrap());
}
}
#[test]
fn cosine_f32_handles_extreme_finite_magnitudes() {
let dir = TempDir::new().unwrap();
- let db = VecDb::open(
- &dir.path().join("db"),
- 8,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::open(&dir.path().join("db"), 8, Some(StorageKind::F32)).unwrap();
for magnitude in [f32::MAX, f32::MIN_POSITIVE, f32::from_bits(1)] {
let vector = vec![magnitude; 8];
db.add("same", &vector).unwrap();
@@ -2232,13 +2086,7 @@ mod tests {
}
fn open_writer(path: &Path) -> VecDb {
- VecDb::open(
- path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap()
+ VecDb::open(path, DIMS, Some(StorageKind::F32)).unwrap()
}
fn generation_of(path: &Path) -> [u8; 16] {
@@ -2376,13 +2224,7 @@ mod tests {
db.flush().unwrap();
assert_eq!(db.stats().unwrap().records_since_snapshot, 0);
drop(db);
- let read_only = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
let reopened = open_writer(&path);
let mut index = 0;
for query in &queries {
@@ -2537,21 +2379,11 @@ mod tests {
let log_bytes = fs::read(path).unwrap();
let snapshot_bytes = snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap());
assert!(matches!(
- VecDb::open(
- path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(path, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Corrupt(_))
));
assert!(matches!(
- VecDb::open_read_only(
- path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open_read_only(path, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Corrupt(_))
));
assert_eq!(fs::read(path).unwrap(), log_bytes);
@@ -2588,22 +2420,10 @@ mod tests {
bulk_add(&db, &bulk_entries(0, 2, 500)).unwrap();
db.flush().unwrap();
drop(db);
- let mut captured_for_read_only = Log::open(
- File::open(&path).unwrap(),
- &path,
- DIMS,
- StorageKind::F32,
- DistanceMetric::Cosine,
- )
- .unwrap();
- let mut captured_for_writer = Log::open(
- File::open(&path).unwrap(),
- &path,
- DIMS,
- StorageKind::F32,
- DistanceMetric::Cosine,
- )
- .unwrap();
+ let mut captured_for_read_only =
+ Log::open(File::open(&path).unwrap(), &path, DIMS, StorageKind::F32).unwrap();
+ let mut captured_for_writer =
+ Log::open(File::open(&path).unwrap(), &path, DIMS, StorageKind::F32).unwrap();
let stale_end = captured_for_read_only.current_end_offset();
let racer = open_writer(&path);
bulk_add(&racer, &bulk_entries(2, 2, 500)).unwrap();
@@ -2638,14 +2458,8 @@ mod tests {
bulk_add(&db, &bulk_entries(0, 2, 700)).unwrap();
db.flush().unwrap();
drop(db);
- let mut captured_for_read_only = Log::open(
- File::open(&path).unwrap(),
- &path,
- DIMS,
- StorageKind::F32,
- DistanceMetric::Cosine,
- )
- .unwrap();
+ let mut captured_for_read_only =
+ Log::open(File::open(&path).unwrap(), &path, DIMS, StorageKind::F32).unwrap();
let stale_end = captured_for_read_only.current_end_offset();
let racer = open_writer(&path);
bulk_add(&racer, &bulk_entries(2, 2, 700)).unwrap();
@@ -2798,13 +2612,7 @@ mod tests {
let last_start = bytes.len() - ADD_RECORD_LEN as usize;
bytes[last_start + 10] ^= 0x01;
fs::write(&path, &bytes).unwrap();
- let read_only = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert_eq!(read_only.len(), 3);
assert!(!read_only.contains("key-3"));
drop(read_only);
@@ -2833,11 +2641,11 @@ mod tests {
fs::write(&path, &bytes).unwrap();
let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();
assert!(matches!(
- VecDb::open(&path, DIMS, Some(StorageKind::F32), Some(DistanceMetric::Cosine)),
+ VecDb::open(&path, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Corrupt(message)) if message.contains("version")
));
assert!(matches!(
- VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32), Some(DistanceMetric::Cosine)),
+ VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Corrupt(message)) if message.contains("version")
));
assert_eq!(fs::read(&path).unwrap(), bytes);
@@ -2925,13 +2733,7 @@ mod tests {
let path = dir.path().join("db");
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 3, 120)).unwrap();
- let joined = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let joined = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&db.shared, &joined.shared));
assert_eq!(joined.len(), 3);
joined.add("joined", &seeded_unit_vector(9, DIMS)).unwrap();
@@ -2952,13 +2754,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let first = open_writer(&path);
- let second = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let second = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&first.shared, &second.shared));
let via_first = seeded_unit_vector(11, DIMS);
let via_second = seeded_unit_vector(22, DIMS);
@@ -2973,7 +2769,7 @@ mod tests {
assert!(first.contains("via-second"));
assert_eq!(first.len(), 2);
assert!(matches!(
- VecDb::open(&path, DIMS + 8, Some(StorageKind::F32), Some(DistanceMetric::Cosine)),
+ VecDb::open(&path, DIMS + 8, Some(StorageKind::F32)),
Err(VecDbError::DimensionMismatch {
expected,
actual
@@ -2991,23 +2787,12 @@ mod tests {
fs::hard_link(&path, &alias).unwrap();
let bytes = fs::read(&path).unwrap();
assert!(matches!(
- VecDb::open(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Locked(_))
));
assert_eq!(fs::read(&path).unwrap(), bytes);
let second = open_writer(&path);
- let reader = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let reader = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
first.add("first", &basis_vector(0)).unwrap();
second.add("second", &basis_vector(1)).unwrap();
assert!(reader.contains("first"));
@@ -3033,25 +2818,15 @@ mod tests {
fs::hard_link(&path, &alias).unwrap();
for name in [&path, &alias] {
assert!(matches!(
- VecDb::open(
- name,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(name, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Locked(_))
));
}
assert_eq!(fs::read(&path).unwrap(), partial);
assert!(
- VecDb::open_read_only(
- &alias,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- )
- .unwrap()
- .is_empty()
+ VecDb::open_read_only(&alias, I8_DIMS, Some(StorageKind::I8))
+ .unwrap()
+ .is_empty()
);
drop(holder);
let writer = open_writer(&alias);
@@ -3071,20 +2846,9 @@ mod tests {
.as_str()
{
"blocked" => {
- let result = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- );
+ let result = VecDb::open(&path, DIMS, Some(StorageKind::F32));
assert!(matches!(result, Err(VecDbError::Locked(_))));
- let reader = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let reader = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(reader.contains("first"));
}
"write" => {
@@ -3129,13 +2893,7 @@ mod tests {
assert_eq!(first.len(), 1);
drop(first);
run_hard_link_writer_process(&alias, "write");
- let reader = VecDb::open_read_only(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let reader = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();
assert_eq!(reader.len(), 2);
assert!(reader.contains("first"));
assert!(reader.contains("next"));
@@ -3165,21 +2923,10 @@ mod tests {
}
assert_eq!(fs::metadata(&alias).unwrap().nlink(), 1);
assert!(matches!(
- VecDb::open(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Locked(_))
));
- let reader = VecDb::open_read_only(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let reader = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();
assert!(reader.contains("first"));
drop(reader);
run_hard_link_writer_process(&alias, "blocked");
@@ -3204,25 +2951,14 @@ mod tests {
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 10, 800)).unwrap();
fs::hard_link(&path, &alias).unwrap();
- let observer = VecDb::open_read_only(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let observer = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();
assert_eq!(observer.len(), 10);
compact(&db.shared, &mut db.shared.writer_half()).unwrap();
assert_eq!(db.len(), 10);
fs::remove_file(&alias).unwrap();
fs::hard_link(&path, &alias).unwrap();
assert!(matches!(
- VecDb::open(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Locked(_))
));
db.reset().unwrap();
@@ -3230,12 +2966,7 @@ mod tests {
fs::remove_file(&alias).unwrap();
fs::hard_link(&path, &alias).unwrap();
assert!(matches!(
- VecDb::open(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&alias, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Locked(_))
));
assert_eq!(db.len(), 1);
@@ -3250,13 +2981,7 @@ mod tests {
let plain = subdir.join("db");
let dotted = subdir.join("..").join("indexes").join("db");
let first = open_writer(&plain);
- let second = VecDb::open(
- &dotted,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let second = VecDb::open(&dotted, DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&first.shared, &second.shared));
let vector = seeded_unit_vector(5, DIMS);
first.add("shared", &vector).unwrap();
@@ -3269,13 +2994,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let first = open_writer(&path);
- let second = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let second = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
first.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();
drop(first);
assert!(second.contains("kept"));
@@ -3289,13 +3008,7 @@ mod tests {
assert_eq!(reopened.len(), 2);
assert!(reopened.contains("kept"));
assert!(reopened.contains("still-open"));
- let again = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let again = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&reopened.shared, &again.shared));
}
@@ -3315,13 +3028,7 @@ mod tests {
thread::spawn(move || {
barrier.wait();
for _ in 0..CYCLES {
- let db = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(db.contains("seed"));
drop(db);
}
@@ -3465,14 +3172,7 @@ mod tests {
}
let opener = {
let path = path.clone();
- thread::spawn(move || {
- VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- })
+ thread::spawn(move || VecDb::open(&path, DIMS, Some(StorageKind::F32)))
};
thread::sleep(Duration::from_millis(5));
drop(frozen);
@@ -3543,12 +3243,7 @@ mod tests {
fs::create_dir(&path).unwrap();
let key = registry_key_for(&path).unwrap();
assert!(matches!(
- VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&path, DIMS, Some(StorageKind::F32)),
Err(VecDbError::Io { .. })
));
assert!(!registry().contains_key(&key));
@@ -3600,13 +3295,7 @@ mod tests {
let path = dir.path().join("db");
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 3, 130)).unwrap();
- let read_only = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&db.shared, &read_only.shared));
assert_eq!(read_only.len(), 3);
assert!(read_only.contains("key-1"));
@@ -3629,13 +3318,7 @@ mod tests {
));
assert!(matches!(read_only.flush(), Err(VecDbError::ReadOnly)));
assert!(matches!(read_only.reset(), Err(VecDbError::ReadOnly)));
- let another = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let another = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(matches!(another.delete(), Err(VecDbError::ReadOnly)));
assert!(db.contains("later"));
assert!(path.exists());
@@ -3646,13 +3329,7 @@ mod tests {
const DIMS: usize = I8_DIMS;
let dir = TempDir::new().unwrap();
let path = dir.path().join("absent");
- let db = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::open_read_only(&path, DIMS, Some(StorageKind::I8)).unwrap();
assert_eq!(db.len(), 0);
assert!(db.is_empty());
assert!(!db.contains("anything"));
@@ -3669,7 +3346,6 @@ mod tests {
dead_count: 0,
dims: DIMS,
storage: StorageKind::I8,
- metric: DistanceMetric::Cosine,
log_bytes: 0,
records_since_snapshot: 0,
approximate_memory_bytes: 0,
@@ -3687,13 +3363,7 @@ mod tests {
let db = open_writer(&path);
db.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();
drop(db);
- let read_only = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
let vector = seeded_unit_vector(4, DIMS);
assert!(matches!(
read_only.add("x", &vector),
@@ -3729,13 +3399,7 @@ mod tests {
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 100, 140)).unwrap();
drop(db);
- let snapshot_view = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let snapshot_view = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert_eq!(snapshot_view.len(), 100);
let db = open_writer(&path);
assert!(!Arc::ptr_eq(&snapshot_view.shared, &db.shared));
@@ -3752,23 +3416,11 @@ mod tests {
assert!(snapshot_view.contains("key-5"));
assert!(!snapshot_view.contains("fresh"));
assert_own_nearest(&snapshot_view, "key-5", &seeded_unit_vector(145, DIMS));
- let live_view = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let live_view = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&live_view.shared, &db.shared));
assert!(live_view.contains("fresh"));
drop(db);
- let fresh_view = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let fresh_view = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert_eq!(fresh_view.len(), 69);
assert!(!fresh_view.contains("key-5"));
assert!(fresh_view.contains("fresh"));
@@ -4265,13 +3917,7 @@ mod tests {
let path = dir.path().join("db");
let partial = [0x45u8, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00];
fs::write(&path, partial).unwrap();
- let db = VecDb::open_read_only(
- &path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::open_read_only(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();
assert!(db.is_empty());
assert_eq!(db.stats().unwrap().log_bytes, 0);
assert_eq!(fs::read(&path).unwrap(), partial);
@@ -4284,14 +3930,14 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
drop(
- Log::create(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ Log::create(&path, DIMS, StorageKind::F32)
.unwrap()
.into_file(),
);
let old_generation = generation_of(&path);
let temp = temp_sibling(&path);
drop(
- Log::create(&temp, DIMS, StorageKind::F32, DistanceMetric::InnerProduct)
+ Log::create(&temp, DIMS, StorageKind::F32)
.unwrap()
.into_file(),
);
@@ -4300,9 +3946,7 @@ mod tests {
assert!(!temp.exists());
assert_eq!(generation_of(&path), new_generation);
assert_ne!(generation_of(&path), old_generation);
- let log =
- open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::InnerProduct)
- .unwrap();
+ let log = open_existing_writer_log(&path, DIMS, StorageKind::F32).unwrap();
assert_eq!(log.generation(), new_generation);
drop(log);
assert!(matches!(
@@ -4372,9 +4016,7 @@ mod tests {
let WriterState { lock, log, .. } = take_writer(&db);
let checkpoint = log.checkpoint();
drop(log);
- let mut foreign =
- open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
- .unwrap();
+ let mut foreign = open_existing_writer_log(&path, DIMS, StorageKind::F32).unwrap();
foreign
.append(&[LogEntry::Add {
key: "foreign",
@@ -4409,8 +4051,7 @@ mod tests {
let old_len = log.current_end_offset();
drop(log);
fs::remove_file(&path).unwrap();
- let mut replacement =
- Log::create(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
+ let mut replacement = Log::create(&path, DIMS, StorageKind::F32).unwrap();
replacement
.append(&[LogEntry::Add {
key: "other",
@@ -4642,7 +4283,6 @@ mod tests {
dead_count: 0,
dims: DIMS,
storage: StorageKind::F32,
- metric: DistanceMetric::Cosine,
log_bytes: 32,
records_since_snapshot: 0,
approximate_memory_bytes: 0,
@@ -4662,13 +4302,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let first = open_writer(&path);
- let second = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let second = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
bulk_add(&first, &bulk_entries(0, 10, 220)).unwrap();
assert_eq!(second.len(), 10);
second.reset().unwrap();
@@ -4740,7 +4374,7 @@ mod tests {
drop(log);
remove_data_files(&path).unwrap();
drop(
- Log::create(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ Log::create(&path, DIMS, StorageKind::F32)
.unwrap()
.into_file(),
);
@@ -4844,20 +4478,8 @@ mod tests {
let first = open_writer(&path);
bulk_add(&first, &bulk_entries(0, 5, 400)).unwrap();
first.flush().unwrap();
- let survivor = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
- let observer = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let survivor = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
+ let observer = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
first.delete().unwrap();
assert!(!path.exists());
assert!(!snapshot_exists(&path));
@@ -4914,13 +4536,7 @@ mod tests {
let barrier = Arc::new(Barrier::new(READERS + 2));
let mut readers = Vec::new();
for reader_index in 0..READERS {
- let handle = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let handle = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
let stop = Arc::clone(&stop);
let barrier = Arc::clone(&barrier);
let expected = Arc::clone(&expected);
@@ -4943,13 +4559,7 @@ mod tests {
}));
}
let flusher = {
- let handle = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let handle = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
let stop = Arc::clone(&stop);
let barrier = Arc::clone(&barrier);
thread::spawn(move || {
@@ -5061,66 +4671,11 @@ mod tests {
assert!(!reopened.contains("bad"));
}
- #[test]
- fn inner_product_indexes_reject_vectors_off_the_unit_sphere() {
- let dir = TempDir::new().unwrap();
- let path = dir.path().join("db");
- let db = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::InnerProduct),
- )
- .unwrap();
- let unit = seeded_unit_vector(1, DIMS);
- db.add("unit", &unit).unwrap();
- for delta in [-1.0e-4f32, 1.0e-4] {
- let nudged = scaled(&unit, (1.0 + delta).sqrt());
- db.add(&format!("nudged{delta}"), &nudged).unwrap();
- }
- let live = db.len();
- let log_bytes = db.stats().unwrap().log_bytes;
- for factor in [0.5, 0.99] {
- let off_sphere = scaled(&unit, factor);
- assert!(matches!(
- db.add("off-sphere", &off_sphere),
- Err(VecDbError::InvalidVector(_))
- ));
- assert!(!db.contains("off-sphere"));
- let mut batch = bulk_entries(10, 3, 200);
- batch[1].1 = off_sphere;
- assert!(matches!(
- bulk_add(&db, &batch),
- Err(VecDbError::InvalidVector(_))
- ));
- assert!(!db.contains("key-10"));
- assert!(!db.contains("key-12"));
- }
- assert_eq!(db.len(), live);
- assert_eq!(db.stats().unwrap().log_bytes, log_bytes);
- drop(db);
- let reopened = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::InnerProduct),
- )
- .unwrap();
- assert_eq!(reopened.len(), live);
- assert!(reopened.contains("unit"));
- }
-
#[test]
fn cosine_indexes_keep_taking_vectors_off_the_unit_sphere() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
- let db = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
let half = scaled(&seeded_unit_vector(1, DIMS), 0.5);
db.add("half", &half).unwrap();
let batch: Vec<(String, Vec<f32>)> = bulk_entries(10, 3, 200)
@@ -5422,13 +4977,7 @@ mod tests {
assert_eq!(db.len(), 1);
assert!(!db.contains("x"));
assert_eq!(db.stats().unwrap().log_bytes, log_bytes);
- let read_only = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(matches!(
read_only.bulk_add(&keys, &vectors),
Err(VecDbError::LengthMismatch { .. })
@@ -5443,20 +4992,8 @@ mod tests {
bulk_add(&first, &bulk_entries(0, 5, 950)).unwrap();
first.flush().unwrap();
fs::write(temp_sibling(&path), [7u8; 10]).unwrap();
- let survivor = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
- let observer = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let survivor = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
+ let observer = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
VecDb::purge(&path).unwrap();
assert!(!path.exists());
assert!(!snapshot_exists(&path));
@@ -5560,13 +5097,7 @@ mod tests {
let alias = dir.path().join("alias");
let via_path = open_writer(&path);
std::os::unix::fs::symlink(&path, &alias).unwrap();
- let via_alias = VecDb::open(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let via_alias = VecDb::open(&alias, DIMS, Some(StorageKind::F32)).unwrap();
via_path
.add("through-real", &seeded_unit_vector(5, DIMS))
.unwrap();
@@ -5578,13 +5109,7 @@ mod tests {
assert_eq!(via_path.len(), 2);
assert!(lock_path(&path).exists());
assert!(!lock_path(&alias).exists());
- let read_alias = VecDb::open_read_only(
- &alias,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_alias = VecDb::open_read_only(&alias, DIMS, Some(StorageKind::F32)).unwrap();
via_path.add("late", &seeded_unit_vector(7, DIMS)).unwrap();
assert!(read_alias.contains("late"));
}
@@ -5729,15 +5254,7 @@ mod tests {
assert!(!snapshot_exists(&path));
let opener = {
let path = path.clone();
- thread::spawn(move || {
- VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap()
- })
+ thread::spawn(move || VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap())
};
let key = registry_key_for(&path).unwrap();
let deadline = Instant::now() + Duration::from_secs(60);
@@ -5746,13 +5263,7 @@ mod tests {
let registered =
existing_path_slot(&key).is_some_and(|slot| lock_slot(&slot).holds_live());
if registered {
- break VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ break VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
}
thread::yield_now();
};
@@ -5771,13 +5282,7 @@ mod tests {
expected_stored
);
assert!(!opener.is_finished());
- let writer = VecDb::open(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let writer = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&writer.shared, &joined.shared));
let adder = {
let vector = replacement.clone();
@@ -5932,13 +5437,7 @@ mod tests {
assert_eq!(db.get_attrs("removed"), None);
assert_eq!(db.get_attrs("recycled").unwrap(), sample_attrs(22));
};
- let read_only = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
verify(&read_only);
drop(read_only);
let reopened = open_writer(&path);
@@ -6078,13 +5577,7 @@ mod tests {
vec![Some(sample_attrs(7)), None, None, None]
);
assert_eq!(db.get_attrs("x").unwrap(), sample_attrs(7));
- let read_only = VecDb::open_read_only(
- &path,
- DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, DIMS, Some(StorageKind::F32)).unwrap();
assert!(matches!(
read_only.bulk_add_with_attrs(&keys, &vectors, &[None]),
Err(VecDbError::LengthMismatch { .. })
@@ -6140,13 +5633,7 @@ mod tests {
}
fn open_i8(path: &Path) -> VecDb {
- VecDb::open(
- path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap()
+ VecDb::open(path, I8_DIMS, Some(StorageKind::I8)).unwrap()
}
fn stored_vectors(db: &VecDb) -> Vec<(String, StoredVector)> {
@@ -6177,14 +5664,7 @@ mod tests {
}
fn logged_add_records(path: &Path, dims: usize) -> Vec<(String, StoredVector)> {
- let mut log = Log::open(
- File::open(path).unwrap(),
- path,
- dims,
- StorageKind::I8,
- DistanceMetric::Cosine,
- )
- .unwrap();
+ let mut log = Log::open(File::open(path).unwrap(), path, dims, StorageKind::I8).unwrap();
let mut scanner = log.scan().unwrap();
let mut records = Vec::new();
while let Some((record, _)) = scanner.next_record().unwrap() {
@@ -6200,21 +5680,10 @@ mod tests {
let dir = TempDir::new().unwrap();
let narrow = dir.path().join("narrow");
let wide = dir.path().join("wide");
- let live = VecDb::open(
- &narrow,
- 8,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let live = VecDb::open(&narrow, 8, Some(StorageKind::F32)).unwrap();
live.add("n", &seeded_unit_vector(1, 8)).unwrap();
assert!(matches!(
- VecDb::open(
- &narrow,
- 8,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&narrow, 8, Some(StorageKind::I8)),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
@@ -6222,45 +5691,22 @@ mod tests {
));
drop(live);
assert!(matches!(
- VecDb::open(
- &narrow,
- 8,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&narrow, 8, Some(StorageKind::I8)),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
- drop(
- VecDb::open(
- &wide,
- I8_DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap(),
- );
+ drop(VecDb::open(&wide, I8_DIMS, Some(StorageKind::F32)).unwrap());
assert!(matches!(
- VecDb::open(
- &wide,
- 8,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&wide, 8, Some(StorageKind::I8)),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
assert!(matches!(
- VecDb::open(
- &wide,
- 8,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&wide, 8, Some(StorageKind::F32)),
Err(VecDbError::DimensionMismatch {
expected: 8,
actual: I8_DIMS
@@ -6268,24 +5714,14 @@ mod tests {
));
drop(open_i8(&dir.path().join("i8")));
assert!(matches!(
- VecDb::open_read_only(
- &dir.path().join("i8"),
- 8,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open_read_only(&dir.path().join("i8"), 8, Some(StorageKind::I8)),
Err(VecDbError::DimensionMismatch {
expected: 8,
actual: I8_DIMS
})
));
assert!(matches!(
- VecDb::open(
- &dir.path().join("fresh"),
- 8,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&dir.path().join("fresh"), 8, Some(StorageKind::I8)),
Err(VecDbError::InvalidDimensions {
dims: 8,
storage: StorageKind::I8
@@ -6299,100 +5735,50 @@ mod tests {
let dir = TempDir::new().unwrap();
let f32_path = dir.path().join("f32");
let i8_path = dir.path().join("i8");
- let f32_db = VecDb::open(
- &f32_path,
- I8_DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let f32_db = VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::F32)).unwrap();
f32_db.add("f", &seeded_unit_vector(1, I8_DIMS)).unwrap();
assert_eq!(f32_db.stats().unwrap().storage, StorageKind::F32);
assert!(matches!(
- VecDb::open(
- &f32_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::I8)),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
- let verified = VecDb::open(
- &f32_path,
- I8_DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let verified = VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::F32)).unwrap();
assert!(Arc::ptr_eq(&verified.shared, &f32_db.shared));
drop(verified);
drop(f32_db);
assert!(matches!(
- VecDb::open(
- &f32_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::I8)),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
assert_eq!(
- VecDb::open(
- &f32_path,
- I8_DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- )
- .unwrap()
- .stats()
- .unwrap()
- .storage,
+ VecDb::open(&f32_path, I8_DIMS, Some(StorageKind::F32))
+ .unwrap()
+ .stats()
+ .unwrap()
+ .storage,
StorageKind::F32
);
- let i8_db = VecDb::open(
- &i8_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let i8_db = VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();
let vector = seeded_unit_vector(2, I8_DIMS);
i8_db.add("i", &vector).unwrap();
assert_eq!(i8_db.stats().unwrap().storage, StorageKind::I8);
- let joined = VecDb::open(
- &i8_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let joined = VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();
assert!(Arc::ptr_eq(&joined.shared, &i8_db.shared));
assert_eq!(joined.stats().unwrap().storage, StorageKind::I8);
assert!(matches!(
- VecDb::open(
- &i8_path,
- I8_DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::F32)),
Err(VecDbError::StorageMismatch {
expected: StorageKind::F32,
actual: StorageKind::I8
})
));
- let read_alias = VecDb::open_read_only(
- &i8_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_alias = VecDb::open_read_only(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();
assert!(Arc::ptr_eq(&read_alias.shared, &i8_db.shared));
assert_eq!(read_alias.stats().unwrap().storage, StorageKind::I8);
i8_db.flush().unwrap();
@@ -6400,71 +5786,36 @@ mod tests {
drop(joined);
drop(i8_db);
assert_eq!(VecDb::open_cost(&i8_path), OpenCost::Ready);
- let standalone = VecDb::open_read_only(
- &i8_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let standalone = VecDb::open_read_only(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();
assert_eq!(standalone.stats().unwrap().storage, StorageKind::I8);
assert!(standalone.contains("i"));
assert_own_nearest_i8(&standalone, "i", &vector);
drop(standalone);
assert!(matches!(
- VecDb::open(
- &i8_path,
- I8_DIMS,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::F32)),
Err(VecDbError::StorageMismatch {
expected: StorageKind::F32,
actual: StorageKind::I8
})
));
- let detected = VecDb::open(
- &i8_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let detected = VecDb::open(&i8_path, I8_DIMS, Some(StorageKind::I8)).unwrap();
assert_eq!(detected.stats().unwrap().storage, StorageKind::I8);
assert!(detected.contains("i"));
assert_own_nearest_i8(&detected, "i", &vector);
assert!(matches!(
- VecDb::open(
- &dir.path().join("narrow"),
- 16,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- ),
+ VecDb::open(&dir.path().join("narrow"), 16, Some(StorageKind::I8)),
Err(VecDbError::InvalidDimensions {
dims: 16,
storage: StorageKind::I8
})
));
- assert!(
- VecDb::open(
- &dir.path().join("narrow"),
- 16,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine)
- )
- .is_ok()
- );
+ assert!(VecDb::open(&dir.path().join("narrow"), 16, Some(StorageKind::F32)).is_ok());
assert_eq!(
- VecDb::open_read_only(
- &dir.path().join("absent"),
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine)
- )
- .unwrap()
- .stats()
- .unwrap()
- .storage,
+ VecDb::open_read_only(&dir.path().join("absent"), I8_DIMS, Some(StorageKind::I8))
+ .unwrap()
+ .stats()
+ .unwrap()
+ .storage,
StorageKind::I8
);
}
@@ -6595,32 +5946,14 @@ mod tests {
stored_after
);
};
- let read_only = VecDb::open_read_only(
- &path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let read_only = VecDb::open_read_only(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();
verify(&read_only);
drop(read_only);
- let with_snapshot = VecDb::open(
- &path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let with_snapshot = VecDb::open(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();
verify(&with_snapshot);
drop(with_snapshot);
fs::remove_file(snapshot_path(&path)).unwrap();
- let rebuilt = VecDb::open(
- &path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let rebuilt = VecDb::open(&path, I8_DIMS, Some(StorageKind::I8)).unwrap();
verify(&rebuilt);
let from_copy = open_i8(&copy_path);
verify(&from_copy);
@@ -6653,24 +5986,12 @@ mod tests {
db.flush().unwrap();
}
{
- let db = VecDb::open(
- &split_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let db = VecDb::open(&split_path, I8_DIMS, Some(StorageKind::I8)).unwrap();
assert_eq!(db.stats().unwrap().storage, StorageKind::I8);
db.add("tail-new", &tail_new).unwrap();
db.add("key-3", &older_upsert).unwrap();
}
- let split = VecDb::open(
- &split_path,
- I8_DIMS,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let split = VecDb::open(&split_path, I8_DIMS, Some(StorageKind::I8)).unwrap();
let whole = open_i8(&whole_path);
bulk_add(&whole, &i8_entries(0, 40, 500)).unwrap();
whole.add("tail-new", &tail_new).unwrap();
@@ -6694,20 +6015,8 @@ mod tests {
fn i8_stats_report_the_storage_and_a_quarter_of_the_vector_memory() {
let dir = TempDir::new().unwrap();
let dims = 512;
- let f32_db = VecDb::open(
- &dir.path().join("f32"),
- dims,
- Some(StorageKind::F32),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
- let i8_db = VecDb::open(
- &dir.path().join("i8"),
- dims,
- Some(StorageKind::I8),
- Some(DistanceMetric::Cosine),
- )
- .unwrap();
+ let f32_db = VecDb::open(&dir.path().join("f32"), dims, Some(StorageKind::F32)).unwrap();
+ let i8_db = VecDb::open(&dir.path().join("i8"), dims, Some(StorageKind::I8)).unwrap();
let entries: Vec<(String, Vec<f32>)> = (0..200u64)
.map(|index| (format!("key-{index}"), seeded_unit_vector(index, dims)))
.collect();