Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Require explicit VecDB index configuration

laurenspriemd0bf5920a0a4
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rs
index c7df5bf8b3..4aff1d6b51 100644
--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs
+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs
@@ -191,23 +191,33 @@ impl VecDb {
pub fn new(
file_path: String,
dimensions: u32,
- storage: Option<VecDbStorage>,
- metric: Option<VecDbMetric>,
+ storage: VecDbStorage,
+ metric: VecDbMetric,
) -> Result<Self, RustVecDbError> {
let path = Path::new(&file_path);
let dims = dimensions as usize;
- let inner = vecdb::VecDb::open_with(
+ let inner = vecdb::VecDb::open(
path,
dims,
- storage.map(to_engine_storage),
- metric.map(to_engine_metric),
+ to_engine_storage(storage),
+ to_engine_metric(metric),
)?;
Ok(Self { inner })
}
- pub fn open_read_only(file_path: String, dimensions: u32) -> Result<Self, RustVecDbError> {
+ pub fn open_read_only(
+ file_path: String,
+ dimensions: u32,
+ storage: VecDbStorage,
+ metric: VecDbMetric,
+ ) -> Result<Self, RustVecDbError> {
Ok(Self {
- inner: vecdb::VecDb::open_read_only(Path::new(&file_path), dimensions as usize)?,
+ inner: vecdb::VecDb::open_read_only(
+ Path::new(&file_path),
+ dimensions as usize,
+ to_engine_storage(storage),
+ to_engine_metric(metric),
+ )?,
})
}
@@ -436,15 +446,66 @@ mod tests {
}
#[test]
- fn metric_defaults_to_cosine_and_explicit_inner_product_is_immutable() {
+ fn read_only_verifies_explicit_configuration() {
+ for (storage, metric, wrong_storage, wrong_metric) in [
+ (
+ VecDbStorage::F32,
+ VecDbMetric::InnerProduct,
+ VecDbStorage::I8,
+ VecDbMetric::Cosine,
+ ),
+ (
+ VecDbStorage::I8,
+ VecDbMetric::Cosine,
+ VecDbStorage::F32,
+ VecDbMetric::InnerProduct,
+ ),
+ ] {
+ let dir = TestDir::create();
+ let db = VecDb::new(dir.db_path(), 32, storage, metric).unwrap();
+ db.add_vector(key("kept"), vec![1.0; 32]).unwrap();
+ db.flush().unwrap();
+ let verify = || {
+ assert!(matches!(
+ VecDb::open_read_only(dir.db_path(), 32, wrong_storage, metric),
+ Err(RustVecDbError::StorageMismatch { .. })
+ ));
+ assert!(matches!(
+ VecDb::open_read_only(dir.db_path(), 32, storage, wrong_metric),
+ Err(RustVecDbError::MetricMismatch { .. })
+ ));
+ assert!(matches!(
+ VecDb::open_read_only(dir.db_path(), 64, storage, metric),
+ Err(RustVecDbError::DimensionMismatch { .. })
+ ));
+ let reader = VecDb::open_read_only(dir.db_path(), 32, storage, metric).unwrap();
+ assert_eq!(reader.get_index_stats().unwrap().live_count, 1);
+ assert!(matches!(
+ reader.add_vector(key("no"), vec![1.0; 32]),
+ Err(RustVecDbError::ReadOnly { .. })
+ ));
+ };
+ verify();
+ drop(db);
+ verify();
+ }
+ }
+
+ #[test]
+ fn explicitly_selected_metrics_are_immutable() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), 32, None, None).unwrap();
+ let db = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().metric,
VecDbMetric::Cosine
));
assert!(matches!(
- VecDb::new(dir.db_path(), 32, None, Some(VecDbMetric::InnerProduct)),
+ VecDb::new(
+ dir.db_path(),
+ 32,
+ VecDbStorage::I8,
+ VecDbMetric::InnerProduct
+ ),
Err(RustVecDbError::MetricMismatch { .. })
));
drop(db);
@@ -452,8 +513,8 @@ mod tests {
let db = VecDb::new(
dir.db_path(),
DIMS,
- Some(VecDbStorage::F32),
- Some(VecDbMetric::InnerProduct),
+ VecDbStorage::F32,
+ VecDbMetric::InnerProduct,
)
.unwrap();
let vector: Vec<f32> = basis(0).iter().map(|value| value * 3.0).collect();
@@ -467,10 +528,16 @@ mod tests {
db.flush().unwrap();
drop(db);
assert!(matches!(
- VecDb::new(dir.db_path(), DIMS, None, Some(VecDbMetric::Cosine)),
+ VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine),
Err(RustVecDbError::MetricMismatch { message }) if message.contains("cosine") && message.contains("inner product")
));
- let db = VecDb::new(dir.db_path(), DIMS, None, None).unwrap();
+ let db = VecDb::new(
+ dir.db_path(),
+ DIMS,
+ VecDbStorage::F32,
+ VecDbMetric::InnerProduct,
+ )
+ .unwrap();
assert!(matches!(
db.get_index_stats().unwrap().metric,
VecDbMetric::InnerProduct
@@ -484,7 +551,7 @@ mod tests {
#[test]
fn add_search_stats_round_trip() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.bulk_add_vectors(vec![key("b"), key("c")], vec![basis(1), basis(2)])
.unwrap();
@@ -517,7 +584,9 @@ mod tests {
assert!(stats.approximate_memory_bytes > 0);
db.flush().unwrap();
assert_eq!(db.get_index_stats().unwrap().records_since_snapshot, 0);
- let read_only = VecDb::open_read_only(dir.db_path(), DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine)
+ .unwrap();
assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);
assert!(matches!(
read_only.add_vector(key("x"), basis(3)),
@@ -539,7 +608,7 @@ mod tests {
#[test]
fn bulk_add_rejects_length_mismatch() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
let error = db
.bulk_add_vectors(vec![key("a")], vec![basis(0), basis(1)])
.unwrap_err();
@@ -550,7 +619,7 @@ mod tests {
#[test]
fn similarity_threshold_conversion_edges() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.add_vector(key("b"), basis(1)).unwrap();
let close = db
@@ -574,7 +643,7 @@ mod tests {
#[test]
fn bulk_search_within_similarity_honors_per_query_thresholds() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
@@ -640,7 +709,7 @@ mod tests {
#[test]
fn bulk_filtered_search_stays_query_aligned() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
@@ -694,7 +763,7 @@ mod tests {
#[test]
fn bulk_search_keys_excludes_self_and_honors_the_filters() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
@@ -732,7 +801,7 @@ mod tests {
#[test]
fn attrs_round_trip_through_the_api() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
let attrs = vec![
VecDbAttr {
name: key("model"),
@@ -824,7 +893,7 @@ mod tests {
check_open_cost(dir.db_path()),
VecDbOpenCost::Absent
));
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
assert!(matches!(
check_open_cost(dir.db_path()),
@@ -846,7 +915,7 @@ mod tests {
#[test]
fn delete_vec_db_files_purges_with_and_without_live_instances() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let db = VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.flush().unwrap();
delete_vec_db_files(dir.db_path()).unwrap();
@@ -857,7 +926,8 @@ mod tests {
db.get_index_stats(),
Err(RustVecDbError::Closed { .. })
));
- let reopened = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32), None).unwrap();
+ let reopened =
+ VecDb::new(dir.db_path(), DIMS, VecDbStorage::F32, VecDbMetric::Cosine).unwrap();
assert_eq!(reopened.get_index_stats().unwrap().live_count, 0);
reopened.add_vector(key("b"), basis(1)).unwrap();
drop(reopened);
@@ -877,7 +947,7 @@ mod tests {
#[test]
fn storage_is_chosen_at_creation_reported_in_stats_and_verified_on_reopen() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8), None).unwrap();
+ let db = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -904,10 +974,10 @@ mod tests {
assert_eq!(stats.dims, 32);
assert!(matches!(stats.storage, VecDbStorage::I8));
assert!(matches!(
- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), None),
+ VecDb::new(dir.db_path(), 32, VecDbStorage::F32, VecDbMetric::Cosine),
Err(RustVecDbError::StorageMismatch { .. })
));
- let joined = VecDb::new(dir.db_path(), 32, None, None).unwrap();
+ let joined = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();
assert!(matches!(
joined.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -915,7 +985,9 @@ mod tests {
drop(joined);
db.flush().unwrap();
drop(db);
- let read_only = VecDb::open_read_only(dir.db_path(), 32).unwrap();
+ let read_only =
+ VecDb::open_read_only(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine)
+ .unwrap();
assert!(matches!(
read_only.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -923,10 +995,11 @@ mod tests {
assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);
drop(read_only);
assert!(matches!(
- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), None),
+ VecDb::new(dir.db_path(), 32, VecDbStorage::F32, VecDbMetric::Cosine),
Err(RustVecDbError::StorageMismatch { message }) if message.contains("f32") && message.contains("i8")
));
- let reopened = VecDb::new(dir.db_path(), 32, None, None).unwrap();
+ let reopened =
+ VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();
assert!(matches!(
reopened.get_index_stats().unwrap().storage,
VecDbStorage::I8
@@ -936,28 +1009,37 @@ mod tests {
}
#[test]
- fn storage_defaults_to_i8_and_rejects_dims_off_the_lane_grid() {
+ fn explicit_i8_storage_rejects_dims_off_the_lane_grid() {
let dir = TestDir::create();
- let db = VecDb::new(dir.db_path(), 32, None, None).unwrap();
+ let db = VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
- let explicit = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8), None).unwrap();
+ let explicit =
+ VecDb::new(dir.db_path(), 32, VecDbStorage::I8, VecDbMetric::Cosine).unwrap();
assert!(matches!(
explicit.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
assert!(matches!(
- VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32), None),
+ VecDb::new(dir.db_path(), 32, VecDbStorage::F32, VecDbMetric::Cosine),
Err(RustVecDbError::StorageMismatch { .. })
));
let other = TestDir::create();
assert!(matches!(
- VecDb::new(other.db_path(), DIMS, None, None),
+ VecDb::new(other.db_path(), DIMS, VecDbStorage::I8, VecDbMetric::Cosine),
Err(RustVecDbError::InvalidDimensions { message }) if message.contains("i8") && message.contains("32")
));
assert!(!PathBuf::from(other.db_path()).exists());
- assert!(VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::F32), None).is_ok());
+ assert!(
+ VecDb::new(
+ other.db_path(),
+ DIMS,
+ VecDbStorage::F32,
+ VecDbMetric::Cosine
+ )
+ .is_ok()
+ );
}
}
diff --git a/rust/crates/ml/examples/vecdb_bench.rs b/rust/crates/ml/examples/vecdb_bench.rs
index d911e6221b..78c5915e3d 100644
--- a/rust/crates/ml/examples/vecdb_bench.rs
+++ b/rust/crates/ml/examples/vecdb_bench.rs
@@ -7,7 +7,9 @@ use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
-use ente_ml::vecdb::{AttrValue, Attribute, Match, SearchParams, StorageKind, VecDb};
+use ente_ml::vecdb::{
+ AttrValue, Attribute, DistanceMetric, Match, SearchParams, StorageKind, VecDb,
+};
const SEED: u64 = 0xE47E_0000_0000_0001;
const LATENT_DIMS: usize = 24;
@@ -180,7 +182,15 @@ fn run_scale(scale: usize, dims: usize, attrs: bool, storage: StorageKind, temp_
);
let dir = temp_root.join(format!("scale-{scale}-{dims}"));
std::fs::create_dir_all(&dir).expect("create bench dir");
- drop(VecDb::open_with_storage(&dir.join("bench.vecdb"), dims, storage).expect("open vecdb"));
+ drop(
+ VecDb::open(
+ &dir.join("bench.vecdb"),
+ dims,
+ storage,
+ DistanceMetric::Cosine,
+ )
+ .expect("open vecdb"),
+ );
eprintln!("[scale {scale}] generating data");
let data = generate_data(scale, dims, clusters);
let vecdb = run_vecdb(&data, dims, attrs, storage, &dir, scale);
@@ -317,7 +327,7 @@ fn run_vecdb(
scale: usize,
) -> VecdbReport {
let path = dir.join("bench.vecdb");
- let mut db = VecDb::open_with_storage(&path, dims, storage).expect("open vecdb");
+ let mut db = VecDb::open(&path, dims, storage, DistanceMetric::Cosine).expect("open vecdb");
let ingest = ingest_phase(&mut db, data, attrs, scale);
let searches = search_phase(&db, data, storage, scale);
let stats = db.stats().expect("vecdb stats");
@@ -326,7 +336,7 @@ fn run_vecdb(
.map(|meta| meta.len())
.unwrap_or(0);
drop(db);
- let (reopens, mut db) = reopen_phase(&path, dims, &snapshot_file, scale);
+ let (reopens, mut db) = reopen_phase(&path, dims, storage, &snapshot_file, scale);
let compaction = compaction_phase(&mut db, data, scale);
db.delete().expect("vecdb delete");
VecdbReport {
@@ -529,18 +539,21 @@ fn search_phase(db: &VecDb, data: &BenchData, storage: StorageKind, scale: usize
fn reopen_phase(
path: &Path,
dims: usize,
+ storage: StorageKind,
snapshot_file: &Path,
scale: usize,
) -> (ReopenTimings, VecDb) {
eprintln!("[scale {scale}] vecdb cold open with snapshot");
let started = Instant::now();
- let reopened = VecDb::open(path, dims).expect("vecdb reopen with snapshot");
+ let reopened = VecDb::open(path, dims, storage, DistanceMetric::Cosine)
+ .expect("vecdb reopen with snapshot");
let open_with_snapshot = started.elapsed();
drop(reopened);
std::fs::remove_file(snapshot_file).expect("remove snapshot");
eprintln!("[scale {scale}] vecdb cold open without snapshot (full rebuild)");
let started = Instant::now();
- let db = VecDb::open(path, dims).expect("vecdb reopen without snapshot");
+ let db = VecDb::open(path, dims, storage, DistanceMetric::Cosine)
+ .expect("vecdb reopen without snapshot");
let open_full_rebuild = started.elapsed();
(
ReopenTimings {
diff --git a/rust/crates/ml/src/vecdb/README.md b/rust/crates/ml/src/vecdb/README.md
index 165f7f80dd..3e260b20aa 100644
--- a/rust/crates/ml/src/vecdb/README.md
+++ b/rust/crates/ml/src/vecdb/README.md
@@ -1,24 +1,26 @@
# VecDB index configuration
-New indexes default to i8 storage and cosine distance. Storage and distance metric
-are persisted at creation and remain unchanged through reopen, compaction, and reset.
-
-`VecDb::open(path, dims)` detects both settings on existing indexes, including older
-inner-product indexes. `open_with_storage` and `open_with_metric` select or verify
-one setting; `open_with` accepts optional storage and metric together. An explicit
-setting that differs from the saved value returns `StorageMismatch` or
-`MetricMismatch`. Read-only opens use the saved settings.
+Every open requires the path, dimensions, storage, and distance metric. There are
+no defaults. These settings are persisted at creation and remain unchanged through
+reopen, compaction, and reset.
```rust
use ente_ml::vecdb::{DistanceMetric, StorageKind, VecDb};
-let cosine = VecDb::open(path, 512)?;
-let inner_product = VecDb::open_with_metric(other_path, 512, DistanceMetric::InnerProduct)?;
-let f32_cosine = VecDb::open_with(third_path, 512, Some(StorageKind::F32), Some(DistanceMetric::Cosine))?;
+let index = VecDb::open(path, 512, StorageKind::I8, DistanceMetric::Cosine)?;
+let reader = VecDb::open_read_only(path, 512, StorageKind::I8, DistanceMetric::Cosine)?;
```
-The Flutter constructor exposes optional `storage` and `metric` parameters, and
-index stats report both settings.
+Both open methods validate the supplied configuration against the saved header or
+an already-open instance. A mismatch returns `DimensionMismatch`, `StorageMismatch`,
+or `MetricMismatch`. Older indexes remain readable by specifying their original
+storage and `DistanceMetric::InnerProduct`.
+
+A read-only open of a missing or incomplete index returns an empty view with the
+supplied configuration without creating or modifying the file.
+
+The Flutter constructor and read-only open also require `storage` and `metric`.
+Index stats report both settings.
Cosine distance is `1 - dot(a, b) / (norm(a) * norm(b))`, clamped to `[0, 2]`.
A comparison involving a zero vector has distance `1`, including two zero vectors.
diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rs
index fca93796c4..64d30bd3a4 100644
--- a/rust/crates/ml/src/vecdb/log.rs
+++ b/rust/crates/ml/src/vecdb/log.rs
@@ -127,10 +127,9 @@ impl Log {
mut file: File,
path: &Path,
expected_dims: usize,
- requested: Option<StorageKind>,
- requested_metric: Option<DistanceMetric>,
+ requested: StorageKind,
+ requested_metric: DistanceMetric,
) -> Result<Self, VecDbError> {
- let fallback = requested.unwrap_or(StorageKind::I8);
let file_len = file
.metadata()
.map_err(|source| VecDbError::io(path, source))?
@@ -140,22 +139,38 @@ impl Log {
"reinitializing {} whose {file_len}-byte header was never completed",
path.display()
);
- validate_dims(expected_dims, fallback)?;
- return Self::initialize(
- file,
- path,
- expected_dims,
- fallback,
- requested_metric.unwrap_or_default(),
- );
+ validate_dims(expected_dims, requested)?;
+ return Self::initialize(file, path, expected_dims, requested, requested_metric);
}
file.seek(SeekFrom::Start(0))
.map_err(|source| VecDbError::io(path, source))?;
let mut header = [0u8; HEADER_LEN];
file.read_exact(&mut header)
.map_err(|source| VecDbError::io(path, source))?;
- let (generation, storage, metric) =
- decode_header(&header, expected_dims, requested, requested_metric)?;
+ let Header {
+ generation,
+ dims,
+ storage,
+ metric,
+ } = decode_header(&header)?;
+ if requested != storage {
+ return Err(VecDbError::StorageMismatch {
+ expected: requested,
+ actual: storage,
+ });
+ }
+ if requested_metric != metric {
+ return Err(VecDbError::MetricMismatch {
+ expected: requested_metric,
+ actual: metric,
+ });
+ }
+ if dims != expected_dims {
+ return Err(VecDbError::DimensionMismatch {
+ expected: expected_dims,
+ actual: dims,
+ });
+ }
validate_dims(expected_dims, storage)?;
Ok(Self {
file,
@@ -779,16 +794,18 @@ fn encode_header(
}
pub(crate) fn header_generation(bytes: &[u8; HEADER_LEN]) -> Result<[u8; 16], VecDbError> {
- let dims = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
- decode_header(bytes, dims, None, None).map(|(generation, _, _)| generation)
+ decode_header(bytes).map(|header| header.generation)
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct Header {
+ generation: [u8; 16],
+ dims: usize,
+ storage: StorageKind,
+ metric: DistanceMetric,
}
-fn decode_header(
- bytes: &[u8; HEADER_LEN],
- expected_dims: usize,
- requested: Option<StorageKind>,
- requested_metric: Option<DistanceMetric>,
-) -> Result<([u8; 16], StorageKind, DistanceMetric), VecDbError> {
+fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {
if bytes[0..4] != MAGIC {
return Err(VecDbError::Corrupt(format!(
"bad log magic {:02x?}",
@@ -811,35 +828,18 @@ fn decode_header(
let storage = StorageKind::from_header_tag(bytes[STORAGE_TAG_OFFSET]).ok_or_else(|| {
VecDbError::Corrupt(format!("unknown scalar tag {}", bytes[STORAGE_TAG_OFFSET]))
})?;
- if let Some(requested) = requested
- && requested != storage
- {
- return Err(VecDbError::StorageMismatch {
- expected: requested,
- actual: storage,
- });
- }
let metric = DistanceMetric::from_header_tag(bytes[METRIC_TAG_OFFSET]).ok_or_else(|| {
VecDbError::Corrupt(format!("unknown metric tag {}", bytes[METRIC_TAG_OFFSET]))
})?;
- if let Some(requested) = requested_metric
- && requested != metric
- {
- return Err(VecDbError::MetricMismatch {
- expected: requested,
- actual: metric,
- });
- }
let dims = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
- if dims != expected_dims {
- return Err(VecDbError::DimensionMismatch {
- expected: expected_dims,
- actual: dims,
- });
- }
let mut generation = [0u8; 16];
generation.copy_from_slice(&bytes[12..28]);
- Ok((generation, storage, metric))
+ Ok(Header {
+ generation,
+ dims,
+ storage,
+ metric,
+ })
}
fn validate_entry(
@@ -1056,9 +1056,9 @@ mod tests {
.collect()
}
- fn reopen(path: &Path, dims: usize) -> Log {
+ fn reopen(path: &Path, dims: usize, storage: StorageKind) -> Log {
let file = open_writer_file(path, false).unwrap();
- Log::open(file, path, dims, None, None).unwrap()
+ Log::open(file, path, dims, storage, InnerProduct).unwrap()
}
fn append_bounds(log: &mut Log, entries: &[LogEntry<'_>]) -> (u64, u64) {
@@ -1149,7 +1149,7 @@ mod tests {
mutate(&mut bytes);
std::fs::write(&path, bytes).unwrap();
let file = File::options().read(true).write(true).open(&path).unwrap();
- Log::open(file, &path, expected_dims, None, None)
+ Log::open(file, &path, expected_dims, StorageKind::F32, InnerProduct)
}
#[test]
@@ -1161,7 +1161,7 @@ mod tests {
assert_eq!(created.current_end_offset(), HEADER_LEN as u64);
assert_eq!(created.path(), path.as_path());
drop(created.into_file());
- let reopened = reopen(&path, 512);
+ let reopened = reopen(&path, 512, StorageKind::F32);
assert_eq!(reopened.generation(), generation);
assert_eq!(reopened.current_end_offset(), HEADER_LEN as u64);
}
@@ -1187,7 +1187,7 @@ mod tests {
Err(VecDbError::Locked(_))
));
drop(file);
- let reopened = reopen(&alias, 8);
+ let reopened = reopen(&alias, 8, StorageKind::F32);
assert!(matches!(
open_writer_file(&path, false),
Err(VecDbError::Locked(_))
@@ -1287,7 +1287,7 @@ mod tests {
let path = dir.path().join("log");
std::fs::write(&path, vec![1u8; junk_len]).unwrap();
let file = File::options().read(true).write(true).open(&path).unwrap();
- let mut log = Log::open(file, &path, 8, Some(StorageKind::F32), None).unwrap();
+ let mut log = Log::open(file, &path, 8, StorageKind::F32, InnerProduct).unwrap();
assert_eq!(log.current_end_offset(), HEADER_LEN as u64);
assert_eq!(std::fs::metadata(&path).unwrap().len(), HEADER_LEN as u64);
let (records, _) = scan_all(&mut log);
@@ -1304,7 +1304,7 @@ mod tests {
assert_eq!(start, HEADER_LEN as u64);
let generation = log.generation();
drop(log);
- let reopened = reopen(&path, 8);
+ let reopened = reopen(&path, 8, StorageKind::F32);
assert_eq!(reopened.generation(), generation);
assert_eq!(reopened.current_end_offset(), end);
}
@@ -1334,7 +1334,7 @@ mod tests {
);
let file = File::options().read(true).write(true).open(&path).unwrap();
assert!(matches!(
- Log::open(file, &path, 12, None, None),
+ Log::open(file, &path, 12, StorageKind::F32, InnerProduct),
Err(VecDbError::DimensionMismatch {
expected: 12,
actual: 8
@@ -1417,7 +1417,7 @@ mod tests {
bytes.extend_from_slice(&golden_attr_add);
bytes.extend_from_slice(&golden_tombstone);
std::fs::write(&path, &bytes).unwrap();
- let mut log = reopen(&path, 8);
+ let mut log = reopen(&path, 8, StorageKind::F32);
assert_eq!(log.generation(), generation);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(
@@ -1586,7 +1586,7 @@ mod tests {
for cut in intact_end..boundaries[2].1 {
let torn_path = dir.path().join(format!("torn-{cut}"));
std::fs::write(&torn_path, &full_bytes[..cut as usize]).unwrap();
- let mut torn_log = reopen(&torn_path, 8);
+ let mut torn_log = reopen(&torn_path, 8, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut torn_log);
assert_eq!(records.len(), 2);
assert_eq!(records[0].1, boundaries[0].0);
@@ -1622,7 +1622,7 @@ mod tests {
let file = File::options().read(true).write(true).open(&path).unwrap();
file.set_len(cut).unwrap();
drop(file);
- let mut log = reopen(&path, 8);
+ let mut log = reopen(&path, 8, StorageKind::F32);
assert_eq!(log.current_end_offset(), cut);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
@@ -1683,7 +1683,7 @@ mod tests {
assert_eq!(log.current_end_offset(), grown_end);
let generation = log.generation();
drop(log);
- assert_eq!(generation, reopen(&path, 8).generation());
+ assert_eq!(generation, reopen(&path, 8, StorageKind::F32).generation());
}
#[test]
@@ -1709,7 +1709,7 @@ mod tests {
log.discard_unacked_tail().unwrap();
assert_eq!(std::fs::metadata(&path).unwrap().len(), acked_end);
drop(log);
- let mut reopened = reopen(&path, 8);
+ let mut reopened = reopen(&path, 8, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut reopened);
assert_eq!(records.len(), 1);
assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "acked"));
@@ -1751,7 +1751,7 @@ mod tests {
assert!(end < acked_end + stale.len() as u64);
assert_eq!(std::fs::metadata(&path).unwrap().len(), end);
drop(log);
- let mut reopened = reopen(&path, 8);
+ let mut reopened = reopen(&path, 8, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut reopened);
assert_eq!(records.len(), 2);
assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "acked"));
@@ -1806,8 +1806,8 @@ mod tests {
let staged_bytes = std::fs::read(&staged_path).unwrap();
assert_eq!(staged_bytes.len(), synced_bytes.len());
assert_eq!(staged_bytes[HEADER_LEN..], synced_bytes[HEADER_LEN..]);
- let (synced_records, synced_end) = scan_all(&mut reopen(&synced_path, 8));
- let (staged_records, staged_end) = scan_all(&mut reopen(&staged_path, 8));
+ let (synced_records, synced_end) = scan_all(&mut reopen(&synced_path, 8, StorageKind::F32));
+ let (staged_records, staged_end) = scan_all(&mut reopen(&staged_path, 8, StorageKind::F32));
assert_eq!(staged_records.len(), 4);
assert_eq!(staged_records, synced_records);
assert_eq!(staged_end, synced_end);
@@ -1829,7 +1829,7 @@ mod tests {
);
drop(log);
let unwritable = File::open(&path).unwrap();
- let mut log = Log::open(unwritable, &path, 8, None, None).unwrap();
+ let mut log = Log::open(unwritable, &path, 8, StorageKind::F32, InnerProduct).unwrap();
assert!(matches!(
log.append(&[LogEntry::Add {
key: "lost",
@@ -1841,7 +1841,7 @@ mod tests {
assert_eq!(log.current_end_offset(), acked_end);
drop(log);
assert_eq!(std::fs::metadata(&path).unwrap().len(), acked_end);
- let mut reopened = reopen(&path, 8);
+ let mut reopened = reopen(&path, 8, StorageKind::F32);
let (records, _) = scan_all(&mut reopened);
assert_eq!(records.len(), 1);
assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "kept"));
@@ -1878,7 +1878,7 @@ mod tests {
assert!(matches!(&records[1].0, LogRecord::Add { key, .. } if key == "third"));
assert_eq!(recoverable_end, end);
drop(log);
- let mut reopened = reopen(&path, 8);
+ let mut reopened = reopen(&path, 8, StorageKind::F32);
let (records, _) = scan_all(&mut reopened);
assert_eq!(records.len(), 2);
assert!(
@@ -1921,7 +1921,7 @@ mod tests {
let mut corrupted = bytes.clone();
corrupted[*position] ^= 0x01;
std::fs::write(&corrupt_path, &corrupted).unwrap();
- let mut corrupt_log = reopen(&corrupt_path, 8);
+ let mut corrupt_log = reopen(&corrupt_path, 8, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut corrupt_log);
assert_eq!(records.len(), 1);
assert_eq!(records[0].1, boundaries[0].0);
@@ -1959,7 +1959,7 @@ mod tests {
let mut record_bytes = good.clone();
record_bytes.extend_from_slice(bad);
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(records[0].1, HEADER_LEN as u64);
@@ -1998,7 +1998,7 @@ mod tests {
assert_eq!(offsets, expected_offsets);
assert_eq!(recoverable_end, previous_end);
drop(log);
- let reopened = reopen(&path, 8);
+ let reopened = reopen(&path, 8, StorageKind::F32);
assert_eq!(reopened.current_end_offset(), previous_end);
}
@@ -2078,7 +2078,7 @@ mod tests {
);
let temp_generation = temp_log.generation();
drop(temp_log);
- let reopened = reopen(&temp_path, 8);
+ let reopened = reopen(&temp_path, 8, StorageKind::F32);
assert_eq!(reopened.generation(), temp_generation);
}
@@ -2203,7 +2203,7 @@ mod tests {
.unwrap();
assert_eq!(&scan_attrs_of_single_add(&mut log), attrs);
drop(log);
- let mut reopened = reopen(&path, 8);
+ let mut reopened = reopen(&path, 8, StorageKind::F32);
assert_eq!(&scan_attrs_of_single_add(&mut reopened), attrs);
}
}
@@ -2406,7 +2406,7 @@ mod tests {
let mut record_bytes = good.clone();
record_bytes.extend_from_slice(bad);
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1, "case {index}");
assert_eq!(records[0].1, HEADER_LEN as u64, "case {index}");
@@ -2450,7 +2450,7 @@ mod tests {
let mut record_bytes = good.clone();
record_bytes.extend_from_slice(&torn[..cut]);
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1, "cut {cut}");
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
@@ -2473,7 +2473,7 @@ mod tests {
record_bytes.extend_from_slice(&vec![0xAA; garbage_len]);
record_bytes.extend_from_slice(survivor);
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(recoverable_end, HEADER_LEN as u64 + first.len() as u64);
@@ -2524,7 +2524,7 @@ mod tests {
record_bytes.push(0xAA);
record_bytes.extend_from_slice(tail);
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1, "case {index}");
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
@@ -2549,7 +2549,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
@@ -2580,7 +2580,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
@@ -2606,7 +2606,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
@@ -2653,7 +2653,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
write_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::F32);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
@@ -2671,7 +2671,7 @@ mod tests {
corrupted_bytes[crc_last] ^= 0xFF;
let corrupted_path = dir.path().join("log-corrupted");
write_log_file(&corrupted_path, dims as u32, &corrupted_bytes);
- let mut corrupted_log = reopen(&corrupted_path, dims);
+ let mut corrupted_log = reopen(&corrupted_path, dims, StorageKind::F32);
let (corrupted_records, corrupted_end) = scan_all(&mut corrupted_log);
assert_eq!(corrupted_records.len(), 1);
assert!(
@@ -2716,7 +2716,7 @@ mod tests {
bytes.extend_from_slice(&first);
bytes.extend_from_slice(&after_first);
std::fs::write(&path, &bytes).unwrap();
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::I8);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "first"));
@@ -2790,7 +2790,7 @@ mod tests {
bytes.extend_from_slice(&golden_add);
bytes.extend_from_slice(&tombstone);
std::fs::write(&path, &bytes).unwrap();
- let mut log = reopen(&path, 32);
+ let mut log = reopen(&path, 32, StorageKind::I8);
assert_eq!(log.storage(), StorageKind::I8);
assert_eq!(log.generation(), generation);
let (records, recoverable_end) = scan_all(&mut log);
@@ -2821,8 +2821,13 @@ mod tests {
let header = encode_header(8, &[7u8; 16], StorageKind::F32, InnerProduct);
assert_eq!(header[STORAGE_TAG_OFFSET], 0);
assert_eq!(
- decode_header(&header, 8, None, None).unwrap(),
- ([7u8; 16], StorageKind::F32, InnerProduct)
+ decode_header(&header).unwrap(),
+ Header {
+ generation: [7u8; 16],
+ dims: 8,
+ storage: StorageKind::F32,
+ metric: InnerProduct
+ }
);
}
@@ -2855,27 +2860,30 @@ mod tests {
.unwrap()
.into_file(),
);
- let open = |path: &Path, requested: Option<StorageKind>| {
+ let open = |path: &Path, requested: StorageKind| {
let file = File::options().read(true).write(true).open(path).unwrap();
- Log::open(file, path, 32, requested, None)
+ Log::open(file, path, 32, requested, InnerProduct)
};
- let detected = open(&i8_path, None).unwrap();
+ let detected = open(&i8_path, StorageKind::I8).unwrap();
assert_eq!(detected.storage(), StorageKind::I8);
assert_eq!(detected.generation(), generation);
assert_eq!(
- open(&i8_path, Some(StorageKind::I8)).unwrap().storage(),
+ open(&i8_path, StorageKind::I8).unwrap().storage(),
StorageKind::I8
);
assert!(matches!(
- open(&i8_path, Some(StorageKind::F32)),
+ open(&i8_path, StorageKind::F32),
Err(VecDbError::StorageMismatch {
expected: StorageKind::F32,
actual: StorageKind::I8
})
));
- assert_eq!(open(&f32_path, None).unwrap().storage(), StorageKind::F32);
+ assert_eq!(
+ open(&f32_path, StorageKind::F32).unwrap().storage(),
+ StorageKind::F32
+ );
assert!(matches!(
- open(&f32_path, Some(StorageKind::I8)),
+ open(&f32_path, StorageKind::I8),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
@@ -2891,15 +2899,20 @@ mod tests {
let path = dir.path().join("log");
std::fs::write(&path, [1u8; 5]).unwrap();
let file = File::options().read(true).write(true).open(&path).unwrap();
- let log = Log::open(file, &path, 64, Some(StorageKind::I8), None).unwrap();
+ let log = Log::open(file, &path, 64, StorageKind::I8, InnerProduct).unwrap();
assert_eq!(log.storage(), StorageKind::I8);
drop(log);
- assert_eq!(reopen(&path, 64).storage(), StorageKind::I8);
+ assert_eq!(
+ reopen(&path, 64, StorageKind::I8).storage(),
+ StorageKind::I8
+ );
let other = dir.path().join("other");
std::fs::write(&other, [1u8; 5]).unwrap();
let file = File::options().read(true).write(true).open(&other).unwrap();
assert_eq!(
- Log::open(file, &other, 64, None, None).unwrap().storage(),
+ Log::open(file, &other, 64, StorageKind::I8, InnerProduct)
+ .unwrap()
+ .storage(),
StorageKind::I8
);
}
@@ -2929,7 +2942,7 @@ mod tests {
let path = dir.path().join("c");
let file = File::options().read(true).write(true).open(&path).unwrap();
assert!(matches!(
- Log::open(file, &path, 8, Some(StorageKind::I8), None),
+ Log::open(file, &path, 8, StorageKind::I8, InnerProduct),
Err(VecDbError::DimensionMismatch {
expected: 8,
actual: 32
@@ -2945,7 +2958,7 @@ mod tests {
.open(&forged_path)
.unwrap();
assert!(matches!(
- Log::open(file, &forged_path, 8, None, None),
+ Log::open(file, &forged_path, 8, StorageKind::I8, InnerProduct),
Err(VecDbError::InvalidDimensions {
dims: 8,
storage: StorageKind::I8
@@ -3006,7 +3019,7 @@ mod tests {
assert_eq!(decoded_values, values);
}
drop(log);
- let (reopened, _) = scan_all(&mut reopen(&path, dims));
+ let (reopened, _) = scan_all(&mut reopen(&path, dims, StorageKind::I8));
assert_eq!(reopened.len(), records.len());
for ((record, offset), (again, again_offset)) in records.iter().zip(&reopened) {
assert_eq!(offset, again_offset);
@@ -3111,7 +3124,7 @@ mod tests {
for cut in intact_end..boundaries[2].1 {
let torn_path = dir.path().join(format!("torn-{cut}"));
std::fs::write(&torn_path, &full_bytes[..cut as usize]).unwrap();
- let mut torn_log = reopen(&torn_path, dims);
+ let mut torn_log = reopen(&torn_path, dims, StorageKind::I8);
let (records, recoverable_end) = scan_all(&mut torn_log);
assert_eq!(records.len(), 2);
assert_eq!(recoverable_end, intact_end);
@@ -3121,7 +3134,7 @@ mod tests {
mixed.extend_from_slice(&f32_sized);
let mixed_path = dir.path().join("mixed");
std::fs::write(&mixed_path, &mixed).unwrap();
- let mut mixed_log = reopen(&mixed_path, dims);
+ let mut mixed_log = reopen(&mixed_path, dims, StorageKind::I8);
let (records, recoverable_end) = scan_all(&mut mixed_log);
assert_eq!(records.len(), 2);
assert_eq!(recoverable_end, intact_end);
@@ -3150,7 +3163,7 @@ mod tests {
record_bytes.extend_from_slice(&vec![0xAA; garbage_len]);
record_bytes.extend_from_slice(survivor);
write_i8_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::I8);
assert_eq!(log.storage(), StorageKind::I8);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
@@ -3195,7 +3208,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
write_i8_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::I8);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
@@ -3213,7 +3226,7 @@ mod tests {
corrupted_bytes[crc_last] ^= 0xFF;
let corrupted_path = dir.path().join("log-corrupted");
write_i8_log_file(&corrupted_path, dims as u32, &corrupted_bytes);
- let mut corrupted_log = reopen(&corrupted_path, dims);
+ let mut corrupted_log = reopen(&corrupted_path, dims, StorageKind::I8);
let (corrupted_records, corrupted_end) = scan_all(&mut corrupted_log);
assert_eq!(corrupted_records.len(), 1);
assert!(
@@ -3237,7 +3250,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("log");
write_i8_log_file(&path, dims as u32, &record_bytes);
- let mut log = reopen(&path, dims);
+ let mut log = reopen(&path, dims, StorageKind::I8);
let (records, recoverable_end) = scan_all(&mut log);
assert_eq!(records.len(), 1);
assert_eq!(recoverable_end, HEADER_LEN as u64 + good.len() as u64);
diff --git a/rust/crates/ml/src/vecdb/mod.rs b/rust/crates/ml/src/vecdb/mod.rs
index 40111e7926..af104180de 100644
--- a/rust/crates/ml/src/vecdb/mod.rs
+++ b/rust/crates/ml/src/vecdb/mod.rs
@@ -13,10 +13,9 @@ mod store;
pub use store::{OpenCost, Stats, VecDb};
-#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DistanceMetric {
InnerProduct,
- #[default]
Cosine,
}
diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rs
index 7b8b419131..080df6d2de 100644
--- a/rust/crates/ml/src/vecdb/store.rs
+++ b/rust/crates/ml/src/vecdb/store.rs
@@ -307,8 +307,8 @@ fn registry_key_for(path: &Path) -> Result<PathBuf, VecDbError> {
fn join_live(
shared: Arc<Shared>,
dims: usize,
- storage: Option<StorageKind>,
- metric: Option<DistanceMetric>,
+ storage: StorageKind,
+ metric: DistanceMetric,
read_only: bool,
) -> Result<VecDb, VecDbError> {
if shared.dims != dims {
@@ -317,19 +317,15 @@ fn join_live(
actual: shared.dims,
});
}
- if let Some(requested) = storage
- && requested != shared.storage
- {
+ if storage != shared.storage {
return Err(VecDbError::StorageMismatch {
- expected: requested,
+ expected: storage,
actual: shared.storage,
});
}
- if let Some(requested) = metric
- && requested != shared.metric
- {
+ if metric != shared.metric {
return Err(VecDbError::MetricMismatch {
- expected: requested,
+ expected: metric,
actual: shared.metric,
});
}
@@ -383,31 +379,11 @@ fn warn_handoff_cap(path: &Path) {
}
impl VecDb {
- pub fn open(path: &Path, dims: usize) -> Result<Self, VecDbError> {
- Self::open_with(path, dims, None, None)
- }
-
- pub fn open_with_storage(
+ pub fn open(
path: &Path,
dims: usize,
storage: StorageKind,
- ) -> Result<Self, VecDbError> {
- Self::open_with(path, dims, Some(storage), None)
- }
-
- pub fn open_with_metric(
- path: &Path,
- dims: usize,
metric: DistanceMetric,
- ) -> Result<Self, VecDbError> {
- Self::open_with(path, dims, None, Some(metric))
- }
-
- pub fn open_with(
- path: &Path,
- dims: usize,
- storage: Option<StorageKind>,
- metric: Option<DistanceMetric>,
) -> Result<Self, VecDbError> {
let key = registry_key_for(path)?;
let slot = path_slot(&key);
@@ -475,19 +451,24 @@ impl VecDb {
open_cost_from_files(&key)
}
- pub fn open_read_only(path: &Path, dims: usize) -> Result<Self, VecDbError> {
+ pub fn open_read_only(
+ path: &Path,
+ dims: usize,
+ storage: StorageKind,
+ metric: DistanceMetric,
+ ) -> Result<Self, VecDbError> {
let resolved = registry_key_for(path).unwrap_or_else(|_| path.to_path_buf());
if let Some(slot) = existing_path_slot(&resolved) {
let observed = lock_slot(&slot).live.as_ref().and_then(Weak::upgrade);
match observed {
Some(shared) if !shared.is_closed() => {
- return join_live(shared, dims, None, None, true);
+ return join_live(shared, dims, storage, metric, true);
}
Some(closing) => {
drop(closing);
match wait_for_teardown_handoff(&slot, &resolved) {
SlotHandoff::Join(shared) => {
- return join_live(shared, dims, None, None, true);
+ return join_live(shared, dims, storage, metric, true);
}
SlotHandoff::Released(released) | SlotHandoff::Expired(released) => {
drop(released);
@@ -498,7 +479,7 @@ impl VecDb {
}
}
Ok(Self {
- shared: Arc::new(build_read_only(&resolved, dims)?),
+ shared: Arc::new(build_read_only(&resolved, dims, storage, metric)?),
read_only: true,
})
}
@@ -1232,8 +1213,8 @@ fn restore_writer_mode(
file,
&shared.path,
shared.dims,
- Some(shared.storage),
- Some(shared.metric),
+ shared.storage,
+ shared.metric,
)?;
let mut pending = mutations_since_snapshot;
if log.checkpoint() != expected_checkpoint {
@@ -1308,19 +1289,14 @@ fn build_writer(
path: &Path,
registry_key: PathBuf,
dims: usize,
- storage: Option<StorageKind>,
- metric: Option<DistanceMetric>,
+ storage: StorageKind,
+ metric: DistanceMetric,
) -> Result<BuiltWriter, VecDbError> {
let lock = WriterLock::acquire(path)?;
let mut log = if std::fs::metadata(path).is_ok() {
open_existing_writer_log(path, dims, storage, metric)?
} else {
- match Log::create(
- path,
- dims,
- storage.unwrap_or(StorageKind::I8),
- metric.unwrap_or_default(),
- ) {
+ match Log::create(path, dims, storage, metric) {
Ok(created) => created,
Err(VecDbError::Io { source, .. }) if source.kind() == ErrorKind::AlreadyExists => {
open_existing_writer_log(path, dims, storage, metric)?
@@ -1469,11 +1445,16 @@ fn install_graph(shared: &Shared, graph: Graph) {
}
}
-fn build_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {
+fn build_read_only(
+ path: &Path,
+ dims: usize,
+ storage: StorageKind,
+ metric: DistanceMetric,
+) -> Result<Shared, VecDbError> {
let file = match File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == ErrorKind::NotFound => {
- return empty_read_only(path, dims);
+ return empty_read_only(path, dims, storage, metric);
}
Err(source) => return Err(VecDbError::io(path, source)),
};
@@ -1482,9 +1463,9 @@ fn build_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {
.map_err(|source| VecDbError::io(path, source))?
.len();
if file_len < HEADER_LEN as u64 {
- return empty_read_only(path, dims);
+ return empty_read_only(path, dims, storage, metric);
}
- let mut log = Log::open(file, path, dims, None, None)?;
+ let mut log = Log::open(file, path, dims, storage, metric)?;
let replayed = replay(&mut log, dims, ReplayMode::ReadOnly)?;
drop(log);
let ReplayedState {
@@ -1506,10 +1487,15 @@ fn build_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {
))
}
-fn empty_read_only(path: &Path, dims: usize) -> Result<Shared, VecDbError> {
+fn empty_read_only(
+ path: &Path,
+ dims: usize,
+ storage: StorageKind,
+ metric: DistanceMetric,
+) -> Result<Shared, VecDbError> {
Ok(read_only_shared(
path,
- VectorArena::with_metric(dims, StorageKind::I8, DistanceMetric::default())?,
+ VectorArena::with_metric(dims, storage, metric)?,
Graph::new(),
AttrTable::default(),
0,
@@ -1701,8 +1687,8 @@ fn ensure_finite(key: &str, vector: &[f32]) -> Result<(), VecDbError> {
fn open_existing_writer_log(
path: &Path,
dims: usize,
- storage: Option<StorageKind>,
- metric: Option<DistanceMetric>,
+ storage: StorageKind,
+ metric: DistanceMetric,
) -> Result<Log, VecDbError> {
let file = open_writer_file(path, false)?;
Log::open(file, path, dims, storage, metric)
@@ -1782,75 +1768,119 @@ mod tests {
const TOMBSTONE_RECORD_LEN: u64 = 3 + 5 + 4;
#[test]
- fn cosine_defaults_and_metric_mismatches_cover_live_and_disk_opens() {
+ fn explicit_configuration_is_verified_on_live_and_disk_opens() {
let dir = TempDir::new().unwrap();
- let default_path = dir.path().join("default");
- let default = VecDb::open(&default_path, 32).unwrap();
- assert_eq!(default.stats().unwrap().metric, DistanceMetric::Cosine);
- assert_eq!(default.stats().unwrap().storage, StorageKind::I8);
for storage in [StorageKind::F32, StorageKind::I8] {
for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
let path = dir.path().join(format!("{storage}-{metric}"));
- let other = match metric {
+ let other_storage = match storage {
+ StorageKind::F32 => StorageKind::I8,
+ StorageKind::I8 => StorageKind::F32,
+ };
+ let other_metric = match metric {
DistanceMetric::Cosine => DistanceMetric::InnerProduct,
DistanceMetric::InnerProduct => DistanceMetric::Cosine,
};
- let db = VecDb::open_with(&path, 32, Some(storage), Some(metric)).unwrap();
- assert_eq!(db.stats().unwrap().metric, metric);
- assert_eq!(
- VecDb::open(&path, 32).unwrap().stats().unwrap().metric,
- metric
- );
- assert_eq!(
- VecDb::open_with_metric(&path, 32, metric)
- .unwrap()
- .stats()
- .unwrap()
- .metric,
- metric
- );
- assert_eq!(
- VecDb::open_read_only(&path, 32)
- .unwrap()
- .stats()
- .unwrap()
- .metric,
- metric
- );
- assert!(matches!(
- VecDb::open_with_metric(&path, 32, other),
- Err(VecDbError::MetricMismatch { expected, actual })
- if expected == other && actual == metric
- ));
+ let db = VecDb::open(&path, 32, storage, metric).unwrap();
+ db.add("kept", &seeded_unit_vector(1, 32)).unwrap();
+ db.flush().unwrap();
+ let verify = |live: Option<&VecDb>| {
+ let before = fs::read(&path).unwrap();
+ let snapshot_before = fs::read(snapshot_path(&path)).unwrap();
+ for open in [VecDb::open, VecDb::open_read_only] {
+ assert!(matches!(open(&path, 32, other_storage, metric),
+ Err(VecDbError::StorageMismatch { expected, actual })
+ if expected == other_storage && actual == storage));
+ assert!(matches!(open(&path, 32, storage, other_metric),
+ Err(VecDbError::MetricMismatch { expected, actual })
+ if expected == other_metric && actual == metric));
+ assert!(matches!(
+ open(&path, 64, storage, metric),
+ Err(VecDbError::DimensionMismatch {
+ expected: 64,
+ actual: 32
+ })
+ ));
+ assert_eq!(fs::read(&path).unwrap(), before);
+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot_before);
+ let reopened = open(&path, 32, storage, metric).unwrap();
+ assert_eq!(reopened.stats().unwrap().storage, storage);
+ assert_eq!(reopened.stats().unwrap().metric, metric);
+ assert!(reopened.contains("kept"));
+ if let Some(live) = live {
+ assert!(Arc::ptr_eq(&live.shared, &reopened.shared));
+ }
+ }
+ };
+ verify(Some(&db));
drop(db);
- let before = std::fs::read(&path).unwrap();
- assert!(matches!(
- VecDb::open_with_metric(&path, 32, other),
- Err(VecDbError::MetricMismatch { .. })
- ));
- assert_eq!(std::fs::read(&path).unwrap(), before);
- assert_eq!(
- VecDb::open_read_only(&path, 32)
- .unwrap()
- .stats()
- .unwrap()
- .metric,
- metric
- );
- assert_eq!(
- VecDb::open(&path, 32).unwrap().stats().unwrap().metric,
- metric
- );
+ verify(None);
}
}
}
+ #[test]
+ fn missing_and_incomplete_indexes_use_the_explicit_configuration() {
+ let dir = TempDir::new().unwrap();
+ for storage in [StorageKind::F32, StorageKind::I8] {
+ for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
+ for incomplete in [false, true] {
+ let path = dir.path().join(format!("{storage}-{metric}-{incomplete}"));
+ let dims = storage.lane_width();
+ if incomplete {
+ fs::write(&path, [1, 2, 3]).unwrap();
+ }
+ let reader = VecDb::open_read_only(&path, dims, storage, metric).unwrap();
+ let stats = reader.stats().unwrap();
+ assert_eq!(stats.dims, dims);
+ assert_eq!(stats.storage, storage);
+ assert_eq!(stats.metric, metric);
+ assert!(reader.is_empty());
+ assert!(matches!(
+ reader.add("no", &vec![1.0; dims]),
+ Err(VecDbError::ReadOnly)
+ ));
+ if incomplete {
+ assert_eq!(fs::read(&path).unwrap(), [1, 2, 3]);
+ } else {
+ assert!(!path.exists());
+ }
+ drop(reader);
+ let writer = VecDb::open(&path, dims, storage, metric).unwrap();
+ assert_eq!(writer.stats().unwrap().storage, storage);
+ assert_eq!(writer.stats().unwrap().metric, metric);
+ writer.add("kept", &vec![1.0; dims]).unwrap();
+ drop(writer);
+ let reopened = VecDb::open_read_only(&path, dims, storage, metric).unwrap();
+ assert!(reopened.contains("kept"));
+ assert_eq!(reopened.stats().unwrap().metric, metric);
+ }
+ }
+ }
+ for open in [VecDb::open, VecDb::open_read_only] {
+ let path = dir.path().join("invalid");
+ assert!(matches!(
+ open(&path, 8, StorageKind::I8, DistanceMetric::Cosine),
+ Err(VecDbError::InvalidDimensions {
+ dims: 8,
+ storage: StorageKind::I8
+ })
+ ));
+ assert!(!path.exists());
+ }
+ }
+
#[test]
fn cosine_scores_unnormalized_vectors_in_every_search_path() {
let dir = TempDir::new().unwrap();
for storage in [StorageKind::F32, StorageKind::I8] {
- let db = VecDb::open_with_storage(&dir.path().join(storage.to_string()), 32, storage)
- .unwrap();
+ let db = VecDb::open(
+ &dir.path().join(storage.to_string()),
+ 32,
+ storage,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
let vector = |x, y| {
let mut values = vec![0.0; 32];
values[0] = x;
@@ -1930,7 +1960,7 @@ mod tests {
for storage in [StorageKind::F32, StorageKind::I8] {
for metric in [DistanceMetric::Cosine, DistanceMetric::InnerProduct] {
let path = dir.path().join(format!("{storage}-{metric}"));
- let db = VecDb::open_with(&path, 32, Some(storage), Some(metric)).unwrap();
+ let db = VecDb::open(&path, 32, storage, metric).unwrap();
let mut vector = vec![0.0; 32];
vector[0] = 4.0;
db.add("removed", &vector).unwrap();
@@ -1971,15 +2001,15 @@ mod tests {
if !snapshot {
remove_snapshot(&path).unwrap();
}
- assert_score(&VecDb::open_read_only(&path, 32).unwrap());
- assert_score(&VecDb::open(&path, 32).unwrap());
+ assert_score(&VecDb::open_read_only(&path, 32, storage, metric).unwrap());
+ assert_score(&VecDb::open(&path, 32, storage, metric).unwrap());
}
- let db = VecDb::open(&path, 32).unwrap();
+ let db = VecDb::open(&path, 32, storage, metric).unwrap();
db.reset().unwrap();
db.add("kept", &vector).unwrap();
assert_score(&db);
drop(db);
- assert_score(&VecDb::open(&path, 32).unwrap());
+ assert_score(&VecDb::open(&path, 32, storage, metric).unwrap());
}
}
}
@@ -1987,7 +2017,13 @@ mod tests {
#[test]
fn cosine_f32_handles_extreme_finite_magnitudes() {
let dir = TempDir::new().unwrap();
- let db = VecDb::open_with_storage(&dir.path().join("db"), 8, StorageKind::F32).unwrap();
+ let db = VecDb::open(
+ &dir.path().join("db"),
+ 8,
+ StorageKind::F32,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
for magnitude in [f32::MAX, f32::MIN_POSITIVE, f32::from_bits(1)] {
let vector = vec![magnitude; 8];
db.add("same", &vector).unwrap();
@@ -2057,7 +2093,7 @@ mod tests {
}
fn open_writer(path: &Path) -> VecDb {
- VecDb::open_with_storage(path, DIMS, StorageKind::F32).unwrap()
+ VecDb::open(path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap()
}
fn generation_of(path: &Path) -> [u8; 16] {
@@ -2195,7 +2231,8 @@ mod tests {
db.flush().unwrap();
assert_eq!(db.stats().unwrap().records_since_snapshot, 0);
drop(db);
- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
let reopened = open_writer(&path);
let mut index = 0;
for query in &queries {
@@ -2350,11 +2387,11 @@ mod tests {
let log_bytes = fs::read(path).unwrap();
let snapshot_bytes = snapshot_exists(path).then(|| fs::read(snapshot_path(path)).unwrap());
assert!(matches!(
- VecDb::open(path, DIMS),
+ VecDb::open(path, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Corrupt(_))
));
assert!(matches!(
- VecDb::open_read_only(path, DIMS),
+ VecDb::open_read_only(path, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Corrupt(_))
));
assert_eq!(fs::read(path).unwrap(), log_bytes);
@@ -2391,10 +2428,22 @@ mod tests {
bulk_add(&db, &bulk_entries(0, 2, 500)).unwrap();
db.flush().unwrap();
drop(db);
- let mut captured_for_read_only =
- Log::open(File::open(&path).unwrap(), &path, DIMS, None, None).unwrap();
- let mut captured_for_writer =
- Log::open(File::open(&path).unwrap(), &path, DIMS, None, None).unwrap();
+ let mut captured_for_read_only = Log::open(
+ File::open(&path).unwrap(),
+ &path,
+ DIMS,
+ StorageKind::F32,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
+ let mut captured_for_writer = Log::open(
+ File::open(&path).unwrap(),
+ &path,
+ DIMS,
+ StorageKind::F32,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
let stale_end = captured_for_read_only.current_end_offset();
let racer = open_writer(&path);
bulk_add(&racer, &bulk_entries(2, 2, 500)).unwrap();
@@ -2429,8 +2478,14 @@ mod tests {
bulk_add(&db, &bulk_entries(0, 2, 700)).unwrap();
db.flush().unwrap();
drop(db);
- let mut captured_for_read_only =
- Log::open(File::open(&path).unwrap(), &path, DIMS, None, None).unwrap();
+ let mut captured_for_read_only = Log::open(
+ File::open(&path).unwrap(),
+ &path,
+ DIMS,
+ StorageKind::F32,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
let stale_end = captured_for_read_only.current_end_offset();
let racer = open_writer(&path);
bulk_add(&racer, &bulk_entries(2, 2, 700)).unwrap();
@@ -2583,7 +2638,8 @@ mod tests {
let last_start = bytes.len() - ADD_RECORD_LEN as usize;
bytes[last_start + 10] ^= 0x01;
fs::write(&path, &bytes).unwrap();
- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert_eq!(read_only.len(), 3);
assert!(!read_only.contains("key-3"));
drop(read_only);
@@ -2612,11 +2668,11 @@ mod tests {
fs::write(&path, &bytes).unwrap();
let snapshot_bytes = fs::read(snapshot_path(&path)).unwrap();
assert!(matches!(
- VecDb::open(&path, DIMS),
+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Corrupt(message)) if message.contains("version")
));
assert!(matches!(
- VecDb::open_read_only(&path, DIMS),
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Corrupt(message)) if message.contains("version")
));
assert_eq!(fs::read(&path).unwrap(), bytes);
@@ -2704,7 +2760,7 @@ mod tests {
let path = dir.path().join("db");
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 3, 120)).unwrap();
- let joined = VecDb::open(&path, DIMS).unwrap();
+ let joined = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&db.shared, &joined.shared));
assert_eq!(joined.len(), 3);
joined.add("joined", &seeded_unit_vector(9, DIMS)).unwrap();
@@ -2725,7 +2781,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let first = open_writer(&path);
- let second = VecDb::open(&path, DIMS).unwrap();
+ let second = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&first.shared, &second.shared));
let via_first = seeded_unit_vector(11, DIMS);
let via_second = seeded_unit_vector(22, DIMS);
@@ -2740,7 +2796,7 @@ mod tests {
assert!(first.contains("via-second"));
assert_eq!(first.len(), 2);
assert!(matches!(
- VecDb::open(&path, DIMS + 8),
+ VecDb::open(&path, DIMS + 8, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::DimensionMismatch {
expected,
actual
@@ -2758,12 +2814,13 @@ mod tests {
fs::hard_link(&path, &alias).unwrap();
let bytes = fs::read(&path).unwrap();
assert!(matches!(
- VecDb::open(&alias, DIMS),
+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Locked(_))
));
assert_eq!(fs::read(&path).unwrap(), bytes);
let second = open_writer(&path);
- let reader = VecDb::open_read_only(&path, DIMS).unwrap();
+ let reader =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
first.add("first", &basis_vector(0)).unwrap();
second.add("second", &basis_vector(1)).unwrap();
assert!(reader.contains("first"));
@@ -2789,12 +2846,16 @@ mod tests {
fs::hard_link(&path, &alias).unwrap();
for name in [&path, &alias] {
assert!(matches!(
- VecDb::open(name, DIMS),
+ VecDb::open(name, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Locked(_))
));
}
assert_eq!(fs::read(&path).unwrap(), partial);
- assert!(VecDb::open_read_only(&alias, I8_DIMS).unwrap().is_empty());
+ assert!(
+ VecDb::open_read_only(&alias, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine)
+ .unwrap()
+ .is_empty()
+ );
drop(holder);
let writer = open_writer(&alias);
assert!(writer.is_empty());
@@ -2813,9 +2874,11 @@ mod tests {
.as_str()
{
"blocked" => {
- let result = VecDb::open(&path, DIMS);
+ let result = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine);
assert!(matches!(result, Err(VecDbError::Locked(_))));
- let reader = VecDb::open_read_only(&path, DIMS).unwrap();
+ let reader =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ .unwrap();
assert!(reader.contains("first"));
}
"write" => {
@@ -2860,7 +2923,8 @@ mod tests {
assert_eq!(first.len(), 1);
drop(first);
run_hard_link_writer_process(&alias, "write");
- let reader = VecDb::open_read_only(&alias, DIMS).unwrap();
+ let reader =
+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert_eq!(reader.len(), 2);
assert!(reader.contains("first"));
assert!(reader.contains("next"));
@@ -2890,10 +2954,12 @@ mod tests {
}
assert_eq!(fs::metadata(&alias).unwrap().nlink(), 1);
assert!(matches!(
- VecDb::open(&alias, DIMS),
+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Locked(_))
));
- let reader = VecDb::open_read_only(&alias, DIMS).unwrap();
+ let reader =
+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ .unwrap();
assert!(reader.contains("first"));
drop(reader);
run_hard_link_writer_process(&alias, "blocked");
@@ -2918,14 +2984,15 @@ mod tests {
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 10, 800)).unwrap();
fs::hard_link(&path, &alias).unwrap();
- let observer = VecDb::open_read_only(&alias, DIMS).unwrap();
+ let observer =
+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert_eq!(observer.len(), 10);
compact(&db.shared, &mut db.shared.writer_half()).unwrap();
assert_eq!(db.len(), 10);
fs::remove_file(&alias).unwrap();
fs::hard_link(&path, &alias).unwrap();
assert!(matches!(
- VecDb::open(&alias, DIMS),
+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Locked(_))
));
db.reset().unwrap();
@@ -2933,7 +3000,7 @@ mod tests {
fs::remove_file(&alias).unwrap();
fs::hard_link(&path, &alias).unwrap();
assert!(matches!(
- VecDb::open(&alias, DIMS),
+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Locked(_))
));
assert_eq!(db.len(), 1);
@@ -2948,7 +3015,7 @@ mod tests {
let plain = subdir.join("db");
let dotted = subdir.join("..").join("indexes").join("db");
let first = open_writer(&plain);
- let second = VecDb::open(&dotted, DIMS).unwrap();
+ let second = VecDb::open(&dotted, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&first.shared, &second.shared));
let vector = seeded_unit_vector(5, DIMS);
first.add("shared", &vector).unwrap();
@@ -2961,7 +3028,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let first = open_writer(&path);
- let second = VecDb::open(&path, DIMS).unwrap();
+ let second = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
first.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();
drop(first);
assert!(second.contains("kept"));
@@ -2975,7 +3042,7 @@ mod tests {
assert_eq!(reopened.len(), 2);
assert!(reopened.contains("kept"));
assert!(reopened.contains("still-open"));
- let again = VecDb::open(&path, DIMS).unwrap();
+ let again = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&reopened.shared, &again.shared));
}
@@ -2995,7 +3062,8 @@ mod tests {
thread::spawn(move || {
barrier.wait();
for _ in 0..CYCLES {
- let db = VecDb::open(&path, DIMS).unwrap();
+ let db = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ .unwrap();
assert!(db.contains("seed"));
drop(db);
}
@@ -3139,7 +3207,9 @@ mod tests {
}
let opener = {
let path = path.clone();
- thread::spawn(move || VecDb::open_with_storage(&path, DIMS, StorageKind::F32))
+ thread::spawn(move || {
+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ })
};
thread::sleep(Duration::from_millis(5));
drop(frozen);
@@ -3210,7 +3280,7 @@ mod tests {
fs::create_dir(&path).unwrap();
let key = registry_key_for(&path).unwrap();
assert!(matches!(
- VecDb::open(&path, DIMS),
+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::Io { .. })
));
assert!(!registry().contains_key(&key));
@@ -3262,7 +3332,8 @@ mod tests {
let path = dir.path().join("db");
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 3, 130)).unwrap();
- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&db.shared, &read_only.shared));
assert_eq!(read_only.len(), 3);
assert!(read_only.contains("key-1"));
@@ -3285,7 +3356,8 @@ mod tests {
));
assert!(matches!(read_only.flush(), Err(VecDbError::ReadOnly)));
assert!(matches!(read_only.reset(), Err(VecDbError::ReadOnly)));
- let another = VecDb::open_read_only(&path, DIMS).unwrap();
+ let another =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(matches!(another.delete(), Err(VecDbError::ReadOnly)));
assert!(db.contains("later"));
assert!(path.exists());
@@ -3296,7 +3368,8 @@ mod tests {
const DIMS: usize = I8_DIMS;
let dir = TempDir::new().unwrap();
let path = dir.path().join("absent");
- let db = VecDb::open_read_only(&path, DIMS).unwrap();
+ let db =
+ VecDb::open_read_only(&path, DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
assert_eq!(db.len(), 0);
assert!(db.is_empty());
assert!(!db.contains("anything"));
@@ -3331,7 +3404,8 @@ mod tests {
let db = open_writer(&path);
db.add("kept", &seeded_unit_vector(3, DIMS)).unwrap();
drop(db);
- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
let vector = seeded_unit_vector(4, DIMS);
assert!(matches!(
read_only.add("x", &vector),
@@ -3367,7 +3441,8 @@ mod tests {
let db = open_writer(&path);
bulk_add(&db, &bulk_entries(0, 100, 140)).unwrap();
drop(db);
- let snapshot_view = VecDb::open_read_only(&path, DIMS).unwrap();
+ let snapshot_view =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert_eq!(snapshot_view.len(), 100);
let db = open_writer(&path);
assert!(!Arc::ptr_eq(&snapshot_view.shared, &db.shared));
@@ -3384,11 +3459,13 @@ mod tests {
assert!(snapshot_view.contains("key-5"));
assert!(!snapshot_view.contains("fresh"));
assert_own_nearest(&snapshot_view, "key-5", &seeded_unit_vector(145, DIMS));
- let live_view = VecDb::open_read_only(&path, DIMS).unwrap();
+ let live_view =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&live_view.shared, &db.shared));
assert!(live_view.contains("fresh"));
drop(db);
- let fresh_view = VecDb::open_read_only(&path, DIMS).unwrap();
+ let fresh_view =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert_eq!(fresh_view.len(), 69);
assert!(!fresh_view.contains("key-5"));
assert!(fresh_view.contains("fresh"));
@@ -3885,7 +3962,8 @@ mod tests {
let path = dir.path().join("db");
let partial = [0x45u8, 0x56, 0x44, 0x42, 0x01, 0x00, 0x00];
fs::write(&path, partial).unwrap();
- let db = VecDb::open_read_only(&path, I8_DIMS).unwrap();
+ let db =
+ VecDb::open_read_only(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
assert!(db.is_empty());
assert_eq!(db.stats().unwrap().log_bytes, 0);
assert_eq!(fs::read(&path).unwrap(), partial);
@@ -3914,7 +3992,9 @@ mod tests {
assert!(!temp.exists());
assert_eq!(generation_of(&path), new_generation);
assert_ne!(generation_of(&path), old_generation);
- let log = open_existing_writer_log(&path, DIMS, None, None).unwrap();
+ let log =
+ open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::InnerProduct)
+ .unwrap();
assert_eq!(log.generation(), new_generation);
drop(log);
assert!(matches!(
@@ -3984,7 +4064,9 @@ mod tests {
let WriterState { lock, log, .. } = take_writer(&db);
let checkpoint = log.checkpoint();
drop(log);
- let mut foreign = open_existing_writer_log(&path, DIMS, None, None).unwrap();
+ let mut foreign =
+ open_existing_writer_log(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ .unwrap();
foreign
.append(&[LogEntry::Add {
key: "foreign",
@@ -4272,7 +4354,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let first = open_writer(&path);
- let second = VecDb::open(&path, DIMS).unwrap();
+ let second = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
bulk_add(&first, &bulk_entries(0, 10, 220)).unwrap();
assert_eq!(second.len(), 10);
second.reset().unwrap();
@@ -4448,8 +4530,9 @@ mod tests {
let first = open_writer(&path);
bulk_add(&first, &bulk_entries(0, 5, 400)).unwrap();
first.flush().unwrap();
- let survivor = VecDb::open(&path, DIMS).unwrap();
- let observer = VecDb::open_read_only(&path, DIMS).unwrap();
+ let survivor = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
+ let observer =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
first.delete().unwrap();
assert!(!path.exists());
assert!(!snapshot_exists(&path));
@@ -4506,7 +4589,8 @@ mod tests {
let barrier = Arc::new(Barrier::new(READERS + 2));
let mut readers = Vec::new();
for reader_index in 0..READERS {
- let handle = VecDb::open(&path, DIMS).unwrap();
+ let handle =
+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
let stop = Arc::clone(&stop);
let barrier = Arc::clone(&barrier);
let expected = Arc::clone(&expected);
@@ -4529,7 +4613,8 @@ mod tests {
}));
}
let flusher = {
- let handle = VecDb::open(&path, DIMS).unwrap();
+ let handle =
+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
let stop = Arc::clone(&stop);
let barrier = Arc::clone(&barrier);
thread::spawn(move || {
@@ -4931,7 +5016,8 @@ mod tests {
assert_eq!(db.len(), 1);
assert!(!db.contains("x"));
assert_eq!(db.stats().unwrap().log_bytes, log_bytes);
- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(matches!(
read_only.bulk_add(&keys, &vectors),
Err(VecDbError::LengthMismatch { .. })
@@ -4946,8 +5032,9 @@ mod tests {
bulk_add(&first, &bulk_entries(0, 5, 950)).unwrap();
first.flush().unwrap();
fs::write(temp_sibling(&path), [7u8; 10]).unwrap();
- let survivor = VecDb::open(&path, DIMS).unwrap();
- let observer = VecDb::open_read_only(&path, DIMS).unwrap();
+ let survivor = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
+ let observer =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
VecDb::purge(&path).unwrap();
assert!(!path.exists());
assert!(!snapshot_exists(&path));
@@ -5051,7 +5138,8 @@ mod tests {
let alias = dir.path().join("alias");
let via_path = open_writer(&path);
std::os::unix::fs::symlink(&path, &alias).unwrap();
- let via_alias = VecDb::open(&alias, DIMS).unwrap();
+ let via_alias =
+ VecDb::open(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
via_path
.add("through-real", &seeded_unit_vector(5, DIMS))
.unwrap();
@@ -5063,7 +5151,8 @@ mod tests {
assert_eq!(via_path.len(), 2);
assert!(lock_path(&path).exists());
assert!(!lock_path(&alias).exists());
- let read_alias = VecDb::open_read_only(&alias, DIMS).unwrap();
+ let read_alias =
+ VecDb::open_read_only(&alias, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
via_path.add("late", &seeded_unit_vector(7, DIMS)).unwrap();
assert!(read_alias.contains("late"));
}
@@ -5208,7 +5297,9 @@ mod tests {
assert!(!snapshot_exists(&path));
let opener = {
let path = path.clone();
- thread::spawn(move || VecDb::open(&path, DIMS).unwrap())
+ thread::spawn(move || {
+ VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap()
+ })
};
let key = registry_key_for(&path).unwrap();
let deadline = Instant::now() + Duration::from_secs(60);
@@ -5217,7 +5308,8 @@ mod tests {
let registered =
existing_path_slot(&key).is_some_and(|slot| lock_slot(&slot).holds_live());
if registered {
- break VecDb::open_read_only(&path, DIMS).unwrap();
+ break VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine)
+ .unwrap();
}
thread::yield_now();
};
@@ -5236,7 +5328,7 @@ mod tests {
expected_stored
);
assert!(!opener.is_finished());
- let writer = VecDb::open(&path, DIMS).unwrap();
+ let writer = VecDb::open(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&writer.shared, &joined.shared));
let adder = {
let vector = replacement.clone();
@@ -5391,7 +5483,8 @@ mod tests {
assert_eq!(db.get_attrs("removed"), None);
assert_eq!(db.get_attrs("recycled").unwrap(), sample_attrs(22));
};
- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
verify(&read_only);
drop(read_only);
let reopened = open_writer(&path);
@@ -5531,7 +5624,8 @@ mod tests {
vec![Some(sample_attrs(7)), None, None, None]
);
assert_eq!(db.get_attrs("x").unwrap(), sample_attrs(7));
- let read_only = VecDb::open_read_only(&path, DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(matches!(
read_only.bulk_add_with_attrs(&keys, &vectors, &[None]),
Err(VecDbError::LengthMismatch { .. })
@@ -5587,7 +5681,7 @@ mod tests {
}
fn open_i8(path: &Path) -> VecDb {
- VecDb::open_with_storage(path, I8_DIMS, StorageKind::I8).unwrap()
+ VecDb::open(path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap()
}
fn stored_vectors(db: &VecDb) -> Vec<(String, StoredVector)> {
@@ -5618,7 +5712,14 @@ mod tests {
}
fn logged_add_records(path: &Path, dims: usize) -> Vec<(String, StoredVector)> {
- let mut log = Log::open(File::open(path).unwrap(), path, dims, None, None).unwrap();
+ let mut log = Log::open(
+ File::open(path).unwrap(),
+ path,
+ dims,
+ StorageKind::I8,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
let mut scanner = log.scan().unwrap();
let mut records = Vec::new();
while let Some((record, _)) = scanner.next_record().unwrap() {
@@ -5634,10 +5735,10 @@ mod tests {
let dir = TempDir::new().unwrap();
let narrow = dir.path().join("narrow");
let wide = dir.path().join("wide");
- let live = VecDb::open_with_storage(&narrow, 8, StorageKind::F32).unwrap();
+ let live = VecDb::open(&narrow, 8, StorageKind::F32, DistanceMetric::Cosine).unwrap();
live.add("n", &seeded_unit_vector(1, 8)).unwrap();
assert!(matches!(
- VecDb::open_with_storage(&narrow, 8, StorageKind::I8),
+ VecDb::open(&narrow, 8, StorageKind::I8, DistanceMetric::Cosine),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
@@ -5645,22 +5746,22 @@ mod tests {
));
drop(live);
assert!(matches!(
- VecDb::open_with_storage(&narrow, 8, StorageKind::I8),
+ VecDb::open(&narrow, 8, StorageKind::I8, DistanceMetric::Cosine),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
- drop(VecDb::open_with_storage(&wide, I8_DIMS, StorageKind::F32).unwrap());
+ drop(VecDb::open(&wide, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap());
assert!(matches!(
- VecDb::open_with_storage(&wide, 8, StorageKind::I8),
+ VecDb::open(&wide, 8, StorageKind::I8, DistanceMetric::Cosine),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
assert!(matches!(
- VecDb::open(&wide, 8),
+ VecDb::open(&wide, 8, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::DimensionMismatch {
expected: 8,
actual: I8_DIMS
@@ -5668,14 +5769,24 @@ mod tests {
));
drop(open_i8(&dir.path().join("i8")));
assert!(matches!(
- VecDb::open_read_only(&dir.path().join("i8"), 8),
+ VecDb::open_read_only(
+ &dir.path().join("i8"),
+ 8,
+ StorageKind::I8,
+ DistanceMetric::Cosine
+ ),
Err(VecDbError::DimensionMismatch {
expected: 8,
actual: I8_DIMS
})
));
assert!(matches!(
- VecDb::open_with_storage(&dir.path().join("fresh"), 8, StorageKind::I8),
+ VecDb::open(
+ &dir.path().join("fresh"),
+ 8,
+ StorageKind::I8,
+ DistanceMetric::Cosine
+ ),
Err(VecDbError::InvalidDimensions {
dims: 8,
storage: StorageKind::I8
@@ -5685,54 +5796,60 @@ mod tests {
}
#[test]
- fn storage_kind_is_verified_on_every_open_path_and_auto_detected_otherwise() {
+ fn storage_kind_is_verified_on_every_open_path() {
let dir = TempDir::new().unwrap();
let f32_path = dir.path().join("f32");
let i8_path = dir.path().join("i8");
- let f32_db = VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::F32).unwrap();
+ let f32_db =
+ VecDb::open(&f32_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
f32_db.add("f", &seeded_unit_vector(1, I8_DIMS)).unwrap();
assert_eq!(f32_db.stats().unwrap().storage, StorageKind::F32);
assert!(matches!(
- VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::I8),
+ VecDb::open(&f32_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
- let verified = VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::F32).unwrap();
+ let verified =
+ VecDb::open(&f32_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&verified.shared, &f32_db.shared));
drop(verified);
drop(f32_db);
assert!(matches!(
- VecDb::open_with_storage(&f32_path, I8_DIMS, StorageKind::I8),
+ VecDb::open(&f32_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine),
Err(VecDbError::StorageMismatch {
expected: StorageKind::I8,
actual: StorageKind::F32
})
));
assert_eq!(
- VecDb::open(&f32_path, I8_DIMS)
+ VecDb::open(&f32_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine)
.unwrap()
.stats()
.unwrap()
.storage,
StorageKind::F32
);
- let i8_db = VecDb::open(&i8_path, I8_DIMS).unwrap();
+ let i8_db =
+ VecDb::open(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
let vector = seeded_unit_vector(2, I8_DIMS);
i8_db.add("i", &vector).unwrap();
assert_eq!(i8_db.stats().unwrap().storage, StorageKind::I8);
- let joined = VecDb::open(&i8_path, I8_DIMS).unwrap();
+ let joined =
+ VecDb::open(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
assert!(Arc::ptr_eq(&joined.shared, &i8_db.shared));
assert_eq!(joined.stats().unwrap().storage, StorageKind::I8);
assert!(matches!(
- VecDb::open_with_storage(&i8_path, I8_DIMS, StorageKind::F32),
+ VecDb::open(&i8_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::StorageMismatch {
expected: StorageKind::F32,
actual: StorageKind::I8
})
));
- let read_alias = VecDb::open_read_only(&i8_path, I8_DIMS).unwrap();
+ let read_alias =
+ VecDb::open_read_only(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine)
+ .unwrap();
assert!(Arc::ptr_eq(&read_alias.shared, &i8_db.shared));
assert_eq!(read_alias.stats().unwrap().storage, StorageKind::I8);
i8_db.flush().unwrap();
@@ -5740,36 +5857,57 @@ mod tests {
drop(joined);
drop(i8_db);
assert_eq!(VecDb::open_cost(&i8_path), OpenCost::Ready);
- let standalone = VecDb::open_read_only(&i8_path, I8_DIMS).unwrap();
+ let standalone =
+ VecDb::open_read_only(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine)
+ .unwrap();
assert_eq!(standalone.stats().unwrap().storage, StorageKind::I8);
assert!(standalone.contains("i"));
assert_own_nearest_i8(&standalone, "i", &vector);
drop(standalone);
assert!(matches!(
- VecDb::open_with_storage(&i8_path, I8_DIMS, StorageKind::F32),
+ VecDb::open(&i8_path, I8_DIMS, StorageKind::F32, DistanceMetric::Cosine),
Err(VecDbError::StorageMismatch {
expected: StorageKind::F32,
actual: StorageKind::I8
})
));
- let detected = VecDb::open(&i8_path, I8_DIMS).unwrap();
+ let detected =
+ VecDb::open(&i8_path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
assert_eq!(detected.stats().unwrap().storage, StorageKind::I8);
assert!(detected.contains("i"));
assert_own_nearest_i8(&detected, "i", &vector);
assert!(matches!(
- VecDb::open_with_storage(&dir.path().join("narrow"), 16, StorageKind::I8),
+ VecDb::open(
+ &dir.path().join("narrow"),
+ 16,
+ StorageKind::I8,
+ DistanceMetric::Cosine
+ ),
Err(VecDbError::InvalidDimensions {
dims: 16,
storage: StorageKind::I8
})
));
- assert!(VecDb::open_with_storage(&dir.path().join("narrow"), 16, StorageKind::F32).is_ok());
+ assert!(
+ VecDb::open(
+ &dir.path().join("narrow"),
+ 16,
+ StorageKind::F32,
+ DistanceMetric::Cosine
+ )
+ .is_ok()
+ );
assert_eq!(
- VecDb::open_read_only(&dir.path().join("absent"), I8_DIMS)
- .unwrap()
- .stats()
- .unwrap()
- .storage,
+ VecDb::open_read_only(
+ &dir.path().join("absent"),
+ I8_DIMS,
+ StorageKind::I8,
+ DistanceMetric::Cosine
+ )
+ .unwrap()
+ .stats()
+ .unwrap()
+ .storage,
StorageKind::I8
);
}
@@ -5900,14 +6038,16 @@ mod tests {
stored_after
);
};
- let read_only = VecDb::open_read_only(&path, I8_DIMS).unwrap();
+ let read_only =
+ VecDb::open_read_only(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
verify(&read_only);
drop(read_only);
- let with_snapshot = VecDb::open(&path, I8_DIMS).unwrap();
+ let with_snapshot =
+ VecDb::open(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
verify(&with_snapshot);
drop(with_snapshot);
fs::remove_file(snapshot_path(&path)).unwrap();
- let rebuilt = VecDb::open(&path, I8_DIMS).unwrap();
+ let rebuilt = VecDb::open(&path, I8_DIMS, StorageKind::I8, DistanceMetric::Cosine).unwrap();
verify(&rebuilt);
let from_copy = open_i8(&copy_path);
verify(&from_copy);
@@ -5940,12 +6080,24 @@ mod tests {
db.flush().unwrap();
}
{
- let db = VecDb::open(&split_path, I8_DIMS).unwrap();
+ let db = VecDb::open(
+ &split_path,
+ I8_DIMS,
+ StorageKind::I8,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
assert_eq!(db.stats().unwrap().storage, StorageKind::I8);
db.add("tail-new", &tail_new).unwrap();
db.add("key-3", &older_upsert).unwrap();
}
- let split = VecDb::open(&split_path, I8_DIMS).unwrap();
+ let split = VecDb::open(
+ &split_path,
+ I8_DIMS,
+ StorageKind::I8,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
let whole = open_i8(&whole_path);
bulk_add(&whole, &i8_entries(0, 40, 500)).unwrap();
whole.add("tail-new", &tail_new).unwrap();
@@ -5969,10 +6121,20 @@ mod tests {
fn i8_stats_report_the_storage_and_a_quarter_of_the_vector_memory() {
let dir = TempDir::new().unwrap();
let dims = 512;
- let f32_db =
- VecDb::open_with_storage(&dir.path().join("f32"), dims, StorageKind::F32).unwrap();
- let i8_db =
- VecDb::open_with_storage(&dir.path().join("i8"), dims, StorageKind::I8).unwrap();
+ let f32_db = VecDb::open(
+ &dir.path().join("f32"),
+ dims,
+ StorageKind::F32,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
+ let i8_db = VecDb::open(
+ &dir.path().join("i8"),
+ dims,
+ StorageKind::I8,
+ DistanceMetric::Cosine,
+ )
+ .unwrap();
let entries: Vec<(String, Vec<f32>)> = (0..200u64)
.map(|index| (format!("key-{index}"), seeded_unit_vector(index, dims)))
.collect();