π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
fix(vecdb): prefer live nodes when filling neighbor lists
laurenspriembeccbf0208e8
diff --git a/rust/crates/vecdb/src/graph.rs b/rust/crates/vecdb/src/graph.rsindex 82f9dcd8ee..73292abf1a 100644--- a/rust/crates/vecdb/src/graph.rs+++ b/rust/crates/vecdb/src/graph.rs@@ -80,13 +80,17 @@ fn select_neighbors(count += 1;}}- for candidate in candidates {- if count == cap {- break;- }- if !selected[..count].contains(&candidate.slot) {- selected[count] = candidate.slot;- count += 1;+ for alive in [true, false] {+ for candidate in candidates {+ if count == cap {+ return count;+ }+ if arena.is_alive(candidate.slot) == alive+ && !selected[..count].contains(&candidate.slot)+ {+ selected[count] = candidate.slot;+ count += 1;+ }}}count@@ -1708,6 +1712,64 @@ mod tests {}}+ #[test]+ fn overflowing_backlinks_preserve_a_diverse_retired_bridge() {+ for storage in [StorageKind::F32, StorageKind::I8] {+ let mut arena = VectorArena::with_storage(32, storage).unwrap();+ let owner = axis_vector(32, 0);+ arena.upsert("owner", &owner).unwrap();+ let mut nearby = owner;+ nearby[1] = 0.1;+ let nearby = normalized(nearby);+ for index in 1..32 {+ arena.upsert(&format!("near-{index}"), &nearby).unwrap();+ }+ let mut bridge = vec![0.0; 32];+ bridge[1] = -1.0;+ arena.upsert("bridge", &bridge).unwrap();+ let mut survivor = bridge;+ survivor[2] = 0.1;+ let survivor = normalized(survivor);+ arena.upsert("survivor", &survivor).unwrap();+ let parts = (0..34)+ .map(|slot| GraphNodeParts {+ slot,+ level: 0,+ neighbors: vec![match slot {+ 0 => (1..=32).collect(),+ 32 => vec![33],+ _ => vec![0],+ }],+ })+ .collect();+ let mut graph = Graph::from_parts(Some(0), parts, 34, 34).unwrap();+ arena.remove("bridge").unwrap();+ let query = arena.pack_query(&survivor).unwrap();+ for round in 0..80 {+ let UpsertOutcome::NewSlot(slot) = arena.upsert("incoming", &nearby).unwrap()+ else {+ panic!("incoming was not retired");+ };+ graph.reserve_slots(arena.slot_count());+ graph.attach_empty_node(slot, 0);+ graph.write_level_list(slot, 0, &[0]);+ graph.link_back(0, 0, slot, LEVEL_ZERO_NEIGHBOR_CAP, &arena);+ assert!(graph.neighbors_of(0, 0).contains(&32), "round {round}");+ for (limit, distance) in [(Some(1), None), (None, Some(0.05))] {+ let found = search(+ &graph,+ &arena,+ &query,+ ¶ms(limit, distance, false),+ None,+ );+ assert_eq!(keys(&found), ["survivor"]);+ }+ arena.remove("incoming").unwrap();+ }+ }+ }+#[test]fn jittered_groups_near_the_neighbor_cap_stay_whole() {for storage in [StorageKind::F32, StorageKind::I8] {diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rsindex f11aa7be35..6aa3012352 100644--- a/rust/crates/vecdb/src/store.rs+++ b/rust/crates/vecdb/src/store.rs@@ -1943,6 +1943,79 @@ mod tests {assert_ne!(state.arena.stored_vector(1), previous);}+ #[test]+ fn repeated_equivalent_vectors_remain_searchable_across_replay() {+ for storage in [StorageKind::F32, StorageKind::I8] {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let replay_path = dir.path().join("replay");+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();+ let keys: Vec<_> = (0..3000).map(|index| format!("key-{index}")).collect();+ let vectors: Vec<_> = (1..=3000)+ .map(|index| seeded_unit_vector(index, 32))+ .collect();+ db.bulk_add(&keys, &vectors).unwrap();+ for round in 0..48 {+ let scale = if round % 2 == 0 { 2.0 } else { 1.0 };+ for (key, vector) in keys.iter().zip(&vectors).take(3) {+ let scaled: Vec<_> = vector.iter().map(|value| value * scale).collect();+ db.add(key, &scaled).unwrap();+ }+ if round == 23 {+ db.flush().unwrap();+ }+ }+ let verify = |db: &VecDb| {+ for key in keys.iter().take(3) {+ let query = db.get(key).unwrap().unwrap();+ let nearest = db.search(&query, &limit_params(1)).unwrap();+ assert_eq!(nearest[0].key, *key, "storage={storage:?}");+ let nearby = db+ .search(+ &query,+ &SearchParams {+ max_distance: Some(0.2),+ ..SearchParams::default()+ },+ )+ .unwrap();+ assert!(+ nearby.iter().any(|hit| hit.key == *key),+ "storage={storage:?}, key={key}",+ );+ }+ let state = db.shared.state_read();+ assert_eq!(state.arena.dead_count(), 144);+ assert_eq!(state.arena.live_count(), 3000);+ assert_eq!(state.graph.as_ref().unwrap().insert_ordinal(), 3144);+ };+ verify(&db);+ fs::copy(&path, &replay_path).unwrap();+ fs::copy(snapshot_path(&path), snapshot_path(&replay_path)).unwrap();+ let read_only = VecDb::open_read_only(&replay_path, 32, Some(storage)).unwrap();+ verify(&read_only);+ test_support::assert_identical_graphs(+ db.shared.state_read().graph.as_ref().unwrap(),+ read_only.shared.state_read().graph.as_ref().unwrap(),+ );+ drop(read_only);+ let replayed = VecDb::open(&replay_path, 32, Some(storage)).unwrap();+ verify(&replayed);+ test_support::assert_identical_graphs(+ db.shared.state_read().graph.as_ref().unwrap(),+ replayed.shared.state_read().graph.as_ref().unwrap(),+ );+ db.flush().unwrap();+ drop(db);+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();+ verify(&reopened);+ test_support::assert_identical_graphs(+ reopened.shared.state_read().graph.as_ref().unwrap(),+ replayed.shared.state_read().graph.as_ref().unwrap(),+ );+ }+ }+#[test]fn f32_replacement_preserves_distinct_stored_bits() {let dir = TempDir::new().unwrap();