π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
use std::path::Path;
use ente_vecdb as vecdb;
use flutter_rust_bridge::frb;
#[derive(Clone, Debug)]
pub enum RustVecDbError {
Io { message: String },
Corrupt { message: String },
Locked { message: String },
ReadOnly { message: String },
Closed { message: String },
InvalidKey { message: String },
InvalidVector { message: String },
InvalidAttributes { message: String },
DimensionMismatch { message: String },
InvalidDimensions { message: String },
UnboundedSearch { message: String },
LengthMismatch { message: String },
StorageMismatch { message: String },
}
impl From<vecdb::VecDbError> for RustVecDbError {
fn from(value: vecdb::VecDbError) -> Self {
let message = value.to_string();
match value {
vecdb::VecDbError::Io { .. } => Self::Io { message },
vecdb::VecDbError::Corrupt(_) => Self::Corrupt { message },
vecdb::VecDbError::Locked(_) => Self::Locked { message },
vecdb::VecDbError::ReadOnly => Self::ReadOnly { message },
vecdb::VecDbError::Closed => Self::Closed { message },
vecdb::VecDbError::InvalidKey(_) => Self::InvalidKey { message },
vecdb::VecDbError::InvalidVector(_) => Self::InvalidVector { message },
vecdb::VecDbError::InvalidAttributes(_) => Self::InvalidAttributes { message },
vecdb::VecDbError::DimensionMismatch { .. } => Self::DimensionMismatch { message },
vecdb::VecDbError::InvalidDimensions { .. } => Self::InvalidDimensions { message },
vecdb::VecDbError::UnboundedSearch => Self::UnboundedSearch { message },
vecdb::VecDbError::LengthMismatch { .. } => Self::LengthMismatch { message },
vecdb::VecDbError::StorageMismatch { .. } => Self::StorageMismatch { message },
}
}
}
#[derive(Clone, Copy, Debug)]
pub enum VecDbStorage {
F32,
I8,
}
fn to_engine_storage(storage: VecDbStorage) -> vecdb::StorageKind {
match storage {
VecDbStorage::F32 => vecdb::StorageKind::F32,
VecDbStorage::I8 => vecdb::StorageKind::I8,
}
}
pub(crate) fn to_api_storage(storage: vecdb::StorageKind) -> VecDbStorage {
match storage {
vecdb::StorageKind::F32 => VecDbStorage::F32,
vecdb::StorageKind::I8 => VecDbStorage::I8,
}
}
#[derive(Clone, Debug)]
pub enum VecDbAttrValue {
Str(String),
Bool(bool),
I64(i64),
F64(f64),
}
#[derive(Clone, Debug)]
pub struct VecDbAttr {
pub name: String,
pub value: VecDbAttrValue,
}
fn to_engine_attr(attr: VecDbAttr) -> vecdb::Attribute {
vecdb::Attribute {
name: attr.name,
value: match attr.value {
VecDbAttrValue::Str(value) => vecdb::AttrValue::Str(value),
VecDbAttrValue::Bool(value) => vecdb::AttrValue::Bool(value),
VecDbAttrValue::I64(value) => vecdb::AttrValue::I64(value),
VecDbAttrValue::F64(value) => vecdb::AttrValue::F64(value),
},
}
}
fn to_engine_attrs(attrs: Vec<VecDbAttr>) -> Vec<vecdb::Attribute> {
attrs.into_iter().map(to_engine_attr).collect()
}
fn to_api_attrs(attrs: Vec<vecdb::Attribute>) -> Vec<VecDbAttr> {
attrs
.into_iter()
.map(|attr| VecDbAttr {
name: attr.name,
value: match attr.value {
vecdb::AttrValue::Str(value) => VecDbAttrValue::Str(value),
vecdb::AttrValue::Bool(value) => VecDbAttrValue::Bool(value),
vecdb::AttrValue::I64(value) => VecDbAttrValue::I64(value),
vecdb::AttrValue::F64(value) => VecDbAttrValue::F64(value),
},
})
.collect()
}
#[derive(Clone, Debug)]
pub struct VecDbMatch {
pub key: String,
pub distance: f32,
}
#[derive(Clone, Debug)]
pub struct VecDbKeyMatches {
pub key: String,
pub matches: Vec<VecDbMatch>,
}
#[derive(Clone, Debug)]
pub struct VecDbStats {
pub live_count: u32,
pub dead_count: u32,
pub dims: u32,
pub storage: VecDbStorage,
pub log_bytes: u64,
pub records_since_snapshot: u32,
pub approximate_memory_bytes: u64,
}
fn to_api_match(entry: vecdb::Match) -> VecDbMatch {
VecDbMatch {
key: entry.key,
distance: entry.distance,
}
}
pub(crate) fn to_api_matches(matches: Vec<vecdb::Match>) -> Vec<VecDbMatch> {
matches.into_iter().map(to_api_match).collect()
}
pub(crate) fn to_api_key_matches(entry: vecdb::KeyMatches) -> VecDbKeyMatches {
VecDbKeyMatches {
key: entry.key,
matches: to_api_matches(entry.matches),
}
}
pub(crate) fn to_api_stats(stats: vecdb::Stats) -> VecDbStats {
VecDbStats {
live_count: stats.live_count as u32,
dead_count: stats.dead_count as u32,
dims: stats.dims as u32,
storage: to_api_storage(stats.storage),
log_bytes: stats.log_bytes,
records_since_snapshot: stats.records_since_snapshot as u32,
approximate_memory_bytes: stats.approximate_memory_bytes as u64,
}
}
pub fn delete_vec_db_files(file_path: String) -> Result<(), RustVecDbError> {
Ok(vecdb::VecDb::purge(Path::new(&file_path))?)
}
#[derive(Clone, Copy, Debug)]
pub enum VecDbOpenCost {
Ready,
NeedsRebuild,
Absent,
}
pub fn check_open_cost(file_path: String) -> VecDbOpenCost {
match vecdb::VecDb::open_cost(Path::new(&file_path)) {
vecdb::OpenCost::Ready => VecDbOpenCost::Ready,
vecdb::OpenCost::NeedsRebuild => VecDbOpenCost::NeedsRebuild,
vecdb::OpenCost::Absent => VecDbOpenCost::Absent,
}
}
#[frb(opaque)]
pub struct VecDb {
inner: vecdb::VecDb,
}
impl VecDb {
pub fn new(
file_path: String,
dimensions: u32,
storage: Option<VecDbStorage>,
) -> Result<Self, RustVecDbError> {
let path = Path::new(&file_path);
let dims = dimensions as usize;
let inner = vecdb::VecDb::open(path, dims, storage.map(to_engine_storage))?;
Ok(Self { inner })
}
pub fn open_read_only(
file_path: String,
dimensions: u32,
storage: Option<VecDbStorage>,
) -> Result<Self, RustVecDbError> {
Ok(Self {
inner: vecdb::VecDb::open_read_only(
Path::new(&file_path),
dimensions as usize,
storage.map(to_engine_storage),
)?,
})
}
pub fn search(
&self,
query: Vec<f32>,
limit: Option<u32>,
max_distance: Option<f32>,
exact: bool,
allowed_keys: Option<Vec<String>>,
) -> Result<Vec<VecDbMatch>, RustVecDbError> {
let params = vecdb::SearchParams {
limit: limit.map(|value| value as usize),
max_distance,
exact,
allowed_keys,
};
Ok(to_api_matches(self.inner.search(&query, ¶ms)?))
}
pub fn add_vector(&self, key: String, vector: Vec<f32>) -> Result<(), RustVecDbError> {
Ok(self.inner.add(&key, &vector)?)
}
pub fn bulk_add_vectors(
&self,
keys: Vec<String>,
vectors: Vec<Vec<f32>>,
) -> Result<(), RustVecDbError> {
Ok(self.inner.bulk_add(&keys, &vectors)?)
}
pub fn add_vector_with_attrs(
&self,
key: String,
vector: Vec<f32>,
attrs: Vec<VecDbAttr>,
) -> Result<(), RustVecDbError> {
Ok(self
.inner
.add_with_attrs(&key, &vector, &to_engine_attrs(attrs))?)
}
pub fn bulk_add_vectors_with_attrs(
&self,
keys: Vec<String>,
vectors: Vec<Vec<f32>>,
attrs: Vec<Option<Vec<VecDbAttr>>>,
) -> Result<(), RustVecDbError> {
let attrs: Vec<Option<Vec<vecdb::Attribute>>> = attrs
.into_iter()
.map(|entry| entry.map(to_engine_attrs))
.collect();
Ok(self.inner.bulk_add_with_attrs(&keys, &vectors, &attrs)?)
}
pub fn get_attrs(&self, key: String) -> Result<Option<Vec<VecDbAttr>>, RustVecDbError> {
Ok(self.inner.get_attrs(&key)?.map(to_api_attrs))
}
pub fn bulk_get_attrs(
&self,
keys: Vec<String>,
) -> Result<Vec<Option<Vec<VecDbAttr>>>, RustVecDbError> {
Ok(self
.inner
.bulk_get_attrs(&keys)?
.into_iter()
.map(|entry| entry.map(to_api_attrs))
.collect())
}
pub fn approx_search_vectors_within_similarity(
&self,
query: Vec<f32>,
minimum_similarity: f32,
) -> Result<Vec<VecDbMatch>, RustVecDbError> {
self.search(query, None, Some(1.0 - minimum_similarity), false, None)
}
pub fn bulk_search_within_similarity(
&self,
queries: Vec<Vec<f32>>,
minimum_similarities: Vec<f32>,
limit: Option<u32>,
exact: bool,
) -> Result<Vec<Vec<VecDbMatch>>, RustVecDbError> {
if queries.len() != minimum_similarities.len() {
return Err(RustVecDbError::LengthMismatch {
message: format!(
"queries length {} does not match similarities length {}",
queries.len(),
minimum_similarities.len()
),
});
}
let params: Vec<vecdb::SearchParams> = minimum_similarities
.iter()
.map(|similarity| vecdb::SearchParams {
limit: limit.map(|value| value as usize),
max_distance: Some(1.0 - similarity),
exact,
allowed_keys: None,
})
.collect();
let results = self.inner.bulk_search_varied(&queries, ¶ms)?;
Ok(results.into_iter().map(to_api_matches).collect())
}
pub fn bulk_approx_filtered_search_vectors_within_distance(
&self,
queries: Vec<Vec<f32>>,
allowed_keys: Vec<String>,
count: u32,
max_distance: f32,
) -> Result<Vec<Vec<VecDbMatch>>, RustVecDbError> {
let params = vecdb::SearchParams {
limit: Some(count as usize),
max_distance: Some(max_distance),
exact: false,
allowed_keys: Some(allowed_keys),
};
let results = self.inner.bulk_search(&queries, ¶ms)?;
Ok(results.into_iter().map(to_api_matches).collect())
}
pub fn bulk_search_keys(
&self,
potential_keys: Vec<String>,
count: u32,
max_distance: Option<f32>,
exact: bool,
restrict_to_input: bool,
) -> Result<Vec<VecDbKeyMatches>, RustVecDbError> {
let results = self.inner.bulk_search_stored(
&potential_keys,
count as usize,
max_distance,
exact,
restrict_to_input,
)?;
Ok(results.into_iter().map(to_api_key_matches).collect())
}
pub fn bulk_get_vectors(
&self,
keys: Vec<String>,
) -> Result<Vec<Option<Vec<f32>>>, RustVecDbError> {
Ok(keys
.iter()
.map(|key| self.inner.get(key))
.collect::<Result<_, _>>()?)
}
pub fn bulk_remove_vectors(&self, keys: Vec<String>) -> Result<u32, RustVecDbError> {
Ok(self.inner.bulk_remove(&keys)? as u32)
}
pub fn flush(&self) -> Result<(), RustVecDbError> {
Ok(self.inner.flush()?)
}
pub fn reset_index(&self) -> Result<(), RustVecDbError> {
Ok(self.inner.reset()?)
}
pub fn delete_index(self) -> Result<(), RustVecDbError> {
Ok(self.inner.delete()?)
}
pub fn get_index_stats(&self) -> Result<VecDbStats, RustVecDbError> {
Ok(to_api_stats(self.inner.stats()?))
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::atomic::{AtomicU32, Ordering};
use super::*;
const DIMS: u32 = 8;
static TEST_DIR_COUNTER: AtomicU32 = AtomicU32::new(0);
struct TestDir(PathBuf);
impl TestDir {
fn create() -> Self {
let sequence = TEST_DIR_COUNTER.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"ente-photos-frb-vecdb-{}-{sequence}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&path);
std::fs::create_dir_all(&path).unwrap();
Self(path)
}
fn db_path(&self) -> String {
self.0.join("db").to_str().unwrap().to_string()
}
}
impl Drop for TestDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn basis(axis: usize) -> Vec<f32> {
let mut vector = vec![0.0; DIMS as usize];
vector[axis] = 1.0;
vector
}
fn unit_axis(dims: usize) -> Vec<f32> {
let mut vector = vec![0.0; dims];
vector[0] = 1.0;
vector
}
fn key(name: &str) -> String {
name.to_string()
}
#[test]
fn omitted_storage_defaults_to_i8() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), 32, None).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
drop(db);
let reader = VecDb::open_read_only(dir.db_path(), 32, None).unwrap();
assert!(matches!(
reader.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
drop(reader);
let dir = TestDir::create();
drop(VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)).unwrap());
for open in [VecDb::new, VecDb::open_read_only] {
assert!(matches!(
open(dir.db_path(), 32, None),
Err(RustVecDbError::StorageMismatch { .. })
));
}
}
#[test]
fn read_only_verifies_explicit_configuration() {
for (storage, wrong_storage) in [
(VecDbStorage::F32, VecDbStorage::I8),
(VecDbStorage::I8, VecDbStorage::F32),
] {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), 32, Some(storage)).unwrap();
db.add_vector(key("kept"), unit_axis(32)).unwrap();
db.flush().unwrap();
let verify = || {
assert!(matches!(
VecDb::open_read_only(dir.db_path(), 32, Some(wrong_storage)),
Err(RustVecDbError::StorageMismatch { .. })
));
assert!(matches!(
VecDb::open_read_only(dir.db_path(), 64, Some(storage)),
Err(RustVecDbError::DimensionMismatch { .. })
));
let reader = VecDb::open_read_only(dir.db_path(), 32, Some(storage)).unwrap();
assert_eq!(reader.get_index_stats().unwrap().live_count, 1);
assert!(matches!(
reader.add_vector(key("no"), unit_axis(32)),
Err(RustVecDbError::ReadOnly { .. })
));
};
verify();
drop(db);
verify();
}
}
#[test]
fn add_search_stats_round_trip() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.bulk_add_vectors(vec![key("b"), key("c")], vec![basis(1), basis(2)])
.unwrap();
assert_eq!(
db.bulk_get_vectors(vec![key("c"), key("missing")]).unwrap(),
vec![Some(basis(2)), None]
);
let matches = db.search(basis(0), Some(2), None, true, None).unwrap();
assert_eq!(matches[0].key, "a");
assert!(matches[0].distance.abs() < 1.0e-6);
let within = db.search(basis(0), None, Some(0.5), true, None).unwrap();
assert_eq!(within.len(), 1);
assert_eq!(within[0].key, "a");
let filtered = db
.search(
basis(0),
Some(3),
Some(2.0),
false,
Some(vec![key("b"), key("c")]),
)
.unwrap();
assert!(filtered.iter().all(|found| found.key != "a"));
assert_eq!(filtered.len(), 2);
let stats = db.get_index_stats().unwrap();
assert_eq!(stats.live_count, 3);
assert_eq!(stats.dead_count, 0);
assert_eq!(stats.dims, DIMS);
assert!(stats.log_bytes > 32);
assert!(stats.approximate_memory_bytes > 0);
db.flush().unwrap();
assert_eq!(db.get_index_stats().unwrap().records_since_snapshot, 0);
let read_only =
VecDb::open_read_only(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);
assert!(matches!(
read_only.add_vector(key("x"), basis(3)),
Err(RustVecDbError::ReadOnly { .. })
));
assert_eq!(
db.bulk_remove_vectors(vec![key("a"), key("b"), key("nope")])
.unwrap(),
2
);
assert_eq!(db.get_index_stats().unwrap().live_count, 1);
db.reset_index().unwrap();
assert_eq!(db.get_index_stats().unwrap().live_count, 0);
db.add_vector(key("again"), basis(3)).unwrap();
db.delete_index().unwrap();
assert!(!PathBuf::from(dir.db_path()).exists());
}
#[test]
fn bulk_add_rejects_length_mismatch() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
let error = db
.bulk_add_vectors(vec![key("a")], vec![basis(0), basis(1)])
.unwrap_err();
assert!(matches!(error, RustVecDbError::LengthMismatch { .. }));
assert_eq!(db.get_index_stats().unwrap().live_count, 0);
}
#[test]
fn similarity_threshold_conversion_edges() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.add_vector(key("b"), basis(1)).unwrap();
let close = db
.approx_search_vectors_within_similarity(basis(0), 0.5)
.unwrap();
assert_eq!(close.len(), 1);
assert_eq!(close[0].key, "a");
let all = db
.approx_search_vectors_within_similarity(basis(0), -1.0)
.unwrap();
assert_eq!(all.len(), 2);
for degenerate in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, 2.0] {
assert!(
db.approx_search_vectors_within_similarity(basis(0), degenerate)
.unwrap()
.is_empty()
);
}
}
#[test]
fn bulk_search_within_similarity_honors_per_query_thresholds() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
)
.unwrap();
let queries = vec![basis(0), basis(0), basis(1)];
let results = db
.bulk_search_within_similarity(queries.clone(), vec![0.5, -1.0, 0.5], None, true)
.unwrap();
assert_eq!(results.len(), 3);
assert_eq!(results[0].len(), 1);
assert_eq!(results[0][0].key, "a");
assert_eq!(results[1].len(), 3);
assert_eq!(results[2].len(), 1);
assert_eq!(results[2][0].key, "b");
let limited = db
.bulk_search_within_similarity(queries.clone(), vec![-1.0, -1.0, -1.0], Some(2), false)
.unwrap();
assert!(limited.iter().all(|matches| matches.len() == 2));
let mixed = vec![0.5, -1.0, 0.9];
for exact in [true, false] {
let bulk = db
.bulk_search_within_similarity(queries.clone(), mixed.clone(), Some(2), exact)
.unwrap();
for ((query, similarity), matches) in queries.iter().zip(&mixed).zip(&bulk) {
let direct = db
.search(query.clone(), Some(2), Some(1.0 - similarity), exact, None)
.unwrap();
assert_eq!(matches.len(), direct.len());
for (found, expected) in matches.iter().zip(&direct) {
assert_eq!(found.key, expected.key);
assert_eq!(found.distance, expected.distance);
}
}
}
assert!(matches!(
db.bulk_search_within_similarity(queries.clone(), vec![0.5], None, false),
Err(RustVecDbError::LengthMismatch { .. })
));
let degenerate = db
.bulk_search_within_similarity(
queries.clone(),
vec![f32::NAN, 1.5, -1.0],
Some(3),
true,
)
.unwrap();
assert!(degenerate[0].is_empty());
assert!(degenerate[1].is_empty());
assert_eq!(degenerate[2].len(), 3);
let all_degenerate = db
.bulk_search_within_similarity(
queries,
vec![f32::NAN, f32::INFINITY, 3.0],
Some(3),
false,
)
.unwrap();
assert_eq!(all_degenerate.len(), 3);
assert!(all_degenerate.iter().all(Vec::is_empty));
}
#[test]
fn bulk_filtered_search_stays_query_aligned() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
)
.unwrap();
let queries = vec![basis(0), basis(1), basis(5)];
let aligned = db
.bulk_approx_filtered_search_vectors_within_distance(
queries.clone(),
vec![key("a"), key("b"), key("absent")],
2,
0.5,
)
.unwrap();
assert_eq!(aligned.len(), 3);
assert_eq!(aligned[0].len(), 1);
assert_eq!(aligned[0][0].key, "a");
assert_eq!(aligned[1].len(), 1);
assert_eq!(aligned[1][0].key, "b");
assert!(aligned[2].is_empty());
let empty_allowed = db
.bulk_approx_filtered_search_vectors_within_distance(
queries.clone(),
Vec::new(),
2,
0.5,
)
.unwrap();
assert_eq!(empty_allowed.len(), 3);
assert!(empty_allowed.iter().all(Vec::is_empty));
for degenerate_distance in [f32::NAN, f32::INFINITY, -0.5] {
let degenerate = db
.bulk_approx_filtered_search_vectors_within_distance(
queries.clone(),
vec![key("a")],
2,
degenerate_distance,
)
.unwrap();
assert_eq!(degenerate.len(), 3);
assert!(degenerate.iter().all(Vec::is_empty));
}
let unfiltered = db.search(basis(0), Some(3), None, false, None).unwrap();
assert_eq!(unfiltered.len(), 3);
let filtered_empty = db
.search(basis(0), Some(3), None, false, Some(Vec::new()))
.unwrap();
assert!(filtered_empty.is_empty());
}
#[test]
fn bulk_search_keys_excludes_self_and_honors_the_filters() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.bulk_add_vectors(
vec![key("a"), key("b"), key("c")],
vec![basis(0), basis(1), basis(2)],
)
.unwrap();
let results = db
.bulk_search_keys(
vec![key("missing"), key("b"), key("a")],
2,
None,
true,
false,
)
.unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0].key, "b");
assert_eq!(results[0].matches.len(), 2);
assert!(results[0].matches.iter().all(|found| found.key != "b"));
assert_eq!(results[1].key, "a");
assert!(results[1].matches.iter().all(|found| found.key != "a"));
let restricted = db
.bulk_search_keys(vec![key("a"), key("b")], 5, None, true, true)
.unwrap();
for entry in &restricted {
assert_eq!(entry.matches.len(), 1);
}
assert_eq!(restricted[0].matches[0].key, "b");
assert_eq!(restricted[1].matches[0].key, "a");
let cut = db
.bulk_search_keys(vec![key("a")], 2, Some(0.5), false, false)
.unwrap();
assert!(cut[0].matches.is_empty());
}
#[test]
fn attrs_round_trip_through_the_api() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
let attrs = vec![
VecDbAttr {
name: key("model"),
value: VecDbAttrValue::Str("clip".to_string()),
},
VecDbAttr {
name: key("version"),
value: VecDbAttrValue::I64(3),
},
VecDbAttr {
name: key("indexed"),
value: VecDbAttrValue::Bool(true),
},
VecDbAttr {
name: key("score"),
value: VecDbAttrValue::F64(0.75),
},
];
db.add_vector_with_attrs(key("a"), basis(0), attrs).unwrap();
let fetched = db.get_attrs(key("a")).unwrap().unwrap();
assert_eq!(fetched.len(), 4);
assert!(matches!(
&fetched[0],
VecDbAttr { name, value: VecDbAttrValue::Str(value) }
if name == "model" && value == "clip"
));
assert!(matches!(
&fetched[1],
VecDbAttr { name, value: VecDbAttrValue::I64(3) } if name == "version"
));
assert!(matches!(
&fetched[2],
VecDbAttr { name, value: VecDbAttrValue::Bool(true) } if name == "indexed"
));
assert!(matches!(
&fetched[3],
VecDbAttr { name, value: VecDbAttrValue::F64(value) } if name == "score" && *value == 0.75
));
db.bulk_add_vectors_with_attrs(
vec![key("b"), key("c")],
vec![basis(1), basis(2)],
vec![
Some(vec![VecDbAttr {
name: key("only"),
value: VecDbAttrValue::Bool(false),
}]),
None,
],
)
.unwrap();
let bulk = db
.bulk_get_attrs(vec![key("b"), key("c"), key("missing")])
.unwrap();
assert_eq!(bulk.len(), 3);
assert!(matches!(
bulk[0].as_deref(),
Some([VecDbAttr { name, value: VecDbAttrValue::Bool(false) }]) if name == "only"
));
assert!(bulk[1].is_none());
assert!(bulk[2].is_none());
db.add_vector(key("a"), basis(0)).unwrap();
assert!(db.get_attrs(key("a")).unwrap().is_none());
assert!(matches!(
db.add_vector_with_attrs(
key("a"),
basis(0),
vec![
VecDbAttr {
name: key("dup"),
value: VecDbAttrValue::Bool(true),
},
VecDbAttr {
name: key("dup"),
value: VecDbAttrValue::Bool(false),
},
],
),
Err(RustVecDbError::InvalidAttributes { .. })
));
assert!(matches!(
db.bulk_add_vectors_with_attrs(vec![key("x")], vec![basis(3)], vec![None, None]),
Err(RustVecDbError::LengthMismatch { .. })
));
assert_eq!(db.get_index_stats().unwrap().live_count, 3);
}
#[test]
fn check_open_cost_mirrors_the_engine_decision() {
let dir = TestDir::create();
assert!(matches!(
check_open_cost(dir.db_path()),
VecDbOpenCost::Absent
));
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
assert!(matches!(
check_open_cost(dir.db_path()),
VecDbOpenCost::Ready
));
db.flush().unwrap();
drop(db);
assert!(matches!(
check_open_cost(dir.db_path()),
VecDbOpenCost::Ready
));
std::fs::remove_file(format!("{}.graph", dir.db_path())).unwrap();
assert!(matches!(
check_open_cost(dir.db_path()),
VecDbOpenCost::NeedsRebuild
));
}
#[test]
fn delete_vec_db_files_purges_with_and_without_live_instances() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
db.add_vector(key("a"), basis(0)).unwrap();
db.flush().unwrap();
delete_vec_db_files(dir.db_path()).unwrap();
assert!(!PathBuf::from(dir.db_path()).exists());
assert!(!PathBuf::from(format!("{}.graph", dir.db_path())).exists());
assert!(PathBuf::from(format!("{}.lock", dir.db_path())).exists());
assert!(matches!(
db.get_index_stats(),
Err(RustVecDbError::Closed { .. })
));
let reopened = VecDb::new(dir.db_path(), DIMS, Some(VecDbStorage::F32)).unwrap();
assert_eq!(reopened.get_index_stats().unwrap().live_count, 0);
reopened.add_vector(key("b"), basis(1)).unwrap();
drop(reopened);
delete_vec_db_files(dir.db_path()).unwrap();
assert!(!PathBuf::from(dir.db_path()).exists());
delete_vec_db_files(dir.db_path()).unwrap();
}
fn i8_vector(seed: u32) -> Vec<f32> {
let raw: Vec<f32> = (0..32)
.map(|index| ((index * 7 + seed * 13) % 23) as f32 - 11.0)
.collect();
let norm = raw.iter().map(|value| value * value).sum::<f32>().sqrt();
raw.into_iter().map(|value| value / norm).collect()
}
#[test]
fn storage_is_chosen_at_creation_reported_in_stats_and_verified_on_reopen() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
let vector = i8_vector(1);
db.add_vector(key("a"), vector.clone()).unwrap();
db.bulk_add_vectors(vec![key("b"), key("c")], vec![i8_vector(2), i8_vector(3)])
.unwrap();
let max_abs = vector
.iter()
.fold(0.0f32, |acc, value| acc.max(value.abs()));
let fetched = db.bulk_get_vectors(vec![key("a"), key("missing")]).unwrap();
assert!(fetched[1].is_none());
for (restored, original) in fetched[0].as_ref().unwrap().iter().zip(&vector) {
assert!((restored - original).abs() <= max_abs / 254.0 + 1.0e-6);
}
let found = db
.search(vector.clone(), Some(1), None, true, None)
.unwrap();
assert_eq!(found[0].key, "a");
assert!(found[0].distance.abs() < 0.02);
let stats = db.get_index_stats().unwrap();
assert_eq!(stats.live_count, 3);
assert_eq!(stats.dims, 32);
assert!(matches!(stats.storage, VecDbStorage::I8));
assert!(matches!(
VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),
Err(RustVecDbError::StorageMismatch { .. })
));
let joined = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
joined.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
drop(joined);
db.flush().unwrap();
drop(db);
let read_only = VecDb::open_read_only(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
read_only.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
assert_eq!(read_only.get_index_stats().unwrap().live_count, 3);
drop(read_only);
assert!(matches!(
VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),
Err(RustVecDbError::StorageMismatch { message }) if message.contains("f32") && message.contains("i8")
));
let reopened = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
reopened.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
let again = reopened.search(vector, Some(1), None, false, None).unwrap();
assert_eq!(again[0].key, "a");
}
#[test]
fn explicit_i8_storage_rejects_dims_off_the_lane_grid() {
let dir = TestDir::create();
let db = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
db.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
let explicit = VecDb::new(dir.db_path(), 32, Some(VecDbStorage::I8)).unwrap();
assert!(matches!(
explicit.get_index_stats().unwrap().storage,
VecDbStorage::I8
));
assert!(matches!(
VecDb::new(dir.db_path(), 32, Some(VecDbStorage::F32)),
Err(RustVecDbError::StorageMismatch { .. })
));
let other = TestDir::create();
assert!(matches!(
VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::I8)),
Err(RustVecDbError::InvalidDimensions { message }) if message.contains("i8") && message.contains("32")
));
assert!(!PathBuf::from(other.db_path()).exists());
assert!(VecDb::new(other.db_path(), DIMS, Some(VecDbStorage::F32)).is_ok());
}
}