Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

fix(vecdb): prefer live nodes when filling neighbor lists

laurenspriembeccbf0208e8
diff --git a/rust/crates/vecdb/src/graph.rs b/rust/crates/vecdb/src/graph.rs
index 82f9dcd8ee..73292abf1a 100644
--- a/rust/crates/vecdb/src/graph.rs
+++ b/rust/crates/vecdb/src/graph.rs
@@ -80,13 +80,17 @@ fn select_neighbors(
count += 1;
}
}
- for candidate in candidates {
- if count == cap {
- break;
- }
- if !selected[..count].contains(&candidate.slot) {
- selected[count] = candidate.slot;
- count += 1;
+ for alive in [true, false] {
+ for candidate in candidates {
+ if count == cap {
+ return count;
+ }
+ if arena.is_alive(candidate.slot) == alive
+ && !selected[..count].contains(&candidate.slot)
+ {
+ selected[count] = candidate.slot;
+ count += 1;
+ }
}
}
count
@@ -1708,6 +1712,64 @@ mod tests {
}
}
+ #[test]
+ fn overflowing_backlinks_preserve_a_diverse_retired_bridge() {
+ for storage in [StorageKind::F32, StorageKind::I8] {
+ let mut arena = VectorArena::with_storage(32, storage).unwrap();
+ let owner = axis_vector(32, 0);
+ arena.upsert("owner", &owner).unwrap();
+ let mut nearby = owner;
+ nearby[1] = 0.1;
+ let nearby = normalized(nearby);
+ for index in 1..32 {
+ arena.upsert(&format!("near-{index}"), &nearby).unwrap();
+ }
+ let mut bridge = vec![0.0; 32];
+ bridge[1] = -1.0;
+ arena.upsert("bridge", &bridge).unwrap();
+ let mut survivor = bridge;
+ survivor[2] = 0.1;
+ let survivor = normalized(survivor);
+ arena.upsert("survivor", &survivor).unwrap();
+ let parts = (0..34)
+ .map(|slot| GraphNodeParts {
+ slot,
+ level: 0,
+ neighbors: vec![match slot {
+ 0 => (1..=32).collect(),
+ 32 => vec![33],
+ _ => vec![0],
+ }],
+ })
+ .collect();
+ let mut graph = Graph::from_parts(Some(0), parts, 34, 34).unwrap();
+ arena.remove("bridge").unwrap();
+ let query = arena.pack_query(&survivor).unwrap();
+ for round in 0..80 {
+ let UpsertOutcome::NewSlot(slot) = arena.upsert("incoming", &nearby).unwrap()
+ else {
+ panic!("incoming was not retired");
+ };
+ graph.reserve_slots(arena.slot_count());
+ graph.attach_empty_node(slot, 0);
+ graph.write_level_list(slot, 0, &[0]);
+ graph.link_back(0, 0, slot, LEVEL_ZERO_NEIGHBOR_CAP, &arena);
+ assert!(graph.neighbors_of(0, 0).contains(&32), "round {round}");
+ for (limit, distance) in [(Some(1), None), (None, Some(0.05))] {
+ let found = search(
+ &graph,
+ &arena,
+ &query,
+ &params(limit, distance, false),
+ None,
+ );
+ assert_eq!(keys(&found), ["survivor"]);
+ }
+ arena.remove("incoming").unwrap();
+ }
+ }
+ }
+
#[test]
fn jittered_groups_near_the_neighbor_cap_stay_whole() {
for storage in [StorageKind::F32, StorageKind::I8] {
diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rs
index f11aa7be35..6aa3012352 100644
--- a/rust/crates/vecdb/src/store.rs
+++ b/rust/crates/vecdb/src/store.rs
@@ -1943,6 +1943,79 @@ mod tests {
assert_ne!(state.arena.stored_vector(1), previous);
}
+ #[test]
+ fn repeated_equivalent_vectors_remain_searchable_across_replay() {
+ for storage in [StorageKind::F32, StorageKind::I8] {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let replay_path = dir.path().join("replay");
+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();
+ let keys: Vec<_> = (0..3000).map(|index| format!("key-{index}")).collect();
+ let vectors: Vec<_> = (1..=3000)
+ .map(|index| seeded_unit_vector(index, 32))
+ .collect();
+ db.bulk_add(&keys, &vectors).unwrap();
+ for round in 0..48 {
+ let scale = if round % 2 == 0 { 2.0 } else { 1.0 };
+ for (key, vector) in keys.iter().zip(&vectors).take(3) {
+ let scaled: Vec<_> = vector.iter().map(|value| value * scale).collect();
+ db.add(key, &scaled).unwrap();
+ }
+ if round == 23 {
+ db.flush().unwrap();
+ }
+ }
+ let verify = |db: &VecDb| {
+ for key in keys.iter().take(3) {
+ let query = db.get(key).unwrap().unwrap();
+ let nearest = db.search(&query, &limit_params(1)).unwrap();
+ assert_eq!(nearest[0].key, *key, "storage={storage:?}");
+ let nearby = db
+ .search(
+ &query,
+ &SearchParams {
+ max_distance: Some(0.2),
+ ..SearchParams::default()
+ },
+ )
+ .unwrap();
+ assert!(
+ nearby.iter().any(|hit| hit.key == *key),
+ "storage={storage:?}, key={key}",
+ );
+ }
+ let state = db.shared.state_read();
+ assert_eq!(state.arena.dead_count(), 144);
+ assert_eq!(state.arena.live_count(), 3000);
+ assert_eq!(state.graph.as_ref().unwrap().insert_ordinal(), 3144);
+ };
+ verify(&db);
+ fs::copy(&path, &replay_path).unwrap();
+ fs::copy(snapshot_path(&path), snapshot_path(&replay_path)).unwrap();
+ let read_only = VecDb::open_read_only(&replay_path, 32, Some(storage)).unwrap();
+ verify(&read_only);
+ test_support::assert_identical_graphs(
+ db.shared.state_read().graph.as_ref().unwrap(),
+ read_only.shared.state_read().graph.as_ref().unwrap(),
+ );
+ drop(read_only);
+ let replayed = VecDb::open(&replay_path, 32, Some(storage)).unwrap();
+ verify(&replayed);
+ test_support::assert_identical_graphs(
+ db.shared.state_read().graph.as_ref().unwrap(),
+ replayed.shared.state_read().graph.as_ref().unwrap(),
+ );
+ db.flush().unwrap();
+ drop(db);
+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();
+ verify(&reopened);
+ test_support::assert_identical_graphs(
+ reopened.shared.state_read().graph.as_ref().unwrap(),
+ replayed.shared.state_read().graph.as_ref().unwrap(),
+ );
+ }
+ }
+
#[test]
fn f32_replacement_preserves_distinct_stored_bits() {
let dir = TempDir::new().unwrap();