π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
fix(vecdb): retain retired routing nodes until rebuild
laurenspriemffe4b3a2773a
diff --git a/rust/crates/vecdb/src/arena.rs b/rust/crates/vecdb/src/arena.rsindex 3277b66908..12451e0159 100644--- a/rust/crates/vecdb/src/arena.rs+++ b/rust/crates/vecdb/src/arena.rs@@ -13,8 +13,7 @@ pub(crate) const MAX_KEY_BYTES: usize = 256;#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub(crate) enum UpsertOutcome {NewSlot(u32),- RecycledSlot(u32),- ReplacedInPlace(u32),+ Unchanged(u32),}pub(crate) fn validate_key(key: &str) -> Result<(), VecDbError> {@@ -118,7 +117,6 @@ pub(crate) struct VectorArena {keys_to_slots: HashMap<Box<str>, u32>,slots_to_keys: Vec<Box<str>>,alive: Vec<u64>,- free_slots: Vec<u32>,live_count: usize,}@@ -145,7 +143,6 @@ impl VectorArena {keys_to_slots: HashMap::new(),slots_to_keys: Vec::new(),alive: Vec::new(),- free_slots: Vec::new(),live_count: 0,})}@@ -163,7 +160,7 @@ impl VectorArena {}pub(crate) fn dead_count(&self) -> usize {- self.free_slots.len()+ self.slot_count() - self.live_count}pub(crate) fn slot_count(&self) -> usize {@@ -214,16 +211,10 @@ impl VectorArena {});}if let Some(&slot) = self.keys_to_slots.get(key) {- self.write_vector(slot, payload);- return Ok(UpsertOutcome::ReplacedInPlace(slot));- }- if let Some(slot) = self.free_slots.pop() {- self.slots_to_keys[slot as usize] = Box::from(key);- self.keys_to_slots.insert(Box::from(key), slot);- self.mark_alive(slot);- self.live_count += 1;- self.write_vector(slot, payload);- return Ok(UpsertOutcome::RecycledSlot(slot));+ if self.matches_payload(slot, payload) {+ return Ok(UpsertOutcome::Unchanged(slot));+ }+ self.retire_slot(slot);}let slot = self.slots_to_keys.len() as u32;if slot as usize == self.storage.chunk_count() * VECTORS_PER_CHUNK {@@ -246,10 +237,14 @@ impl VectorArena {pub(crate) fn remove(&mut self, key: &str) -> Option<u32> {let slot = self.keys_to_slots.remove(key)?;+ self.retire_slot(slot);+ Some(slot)+ }++ fn retire_slot(&mut self, slot: u32) {self.mark_dead(slot);self.live_count -= 1;- self.free_slots.push(slot);- Some(slot)+ self.slots_to_keys[slot as usize] = Box::from("");}pub(crate) fn compact_in_place(&mut self) {@@ -296,7 +291,6 @@ impl VectorArena {{*word = (1u64 << (live % 64)) - 1;}- self.free_slots = Vec::new();}fn move_vector(&mut self, src: u32, dst: u32) {@@ -363,6 +357,33 @@ impl VectorArena {}}+ pub(crate) fn matches_payload(&self, slot: u32, payload: VectorPayload<'_>) -> bool {+ if payload.dims() != self.dims {+ return false;+ }+ match (self.stored_query(slot), payload) {+ (Query::F32(lanes, _), VectorPayload::F32(values)) => lanes+ .iter()+ .flat_map(|lane| lane.to_array())+ .zip(values)+ .all(|(stored, value)| stored.to_bits() == value.to_bits()),+ (+ Query::I8 { scale, lanes, .. },+ VectorPayload::I8 {+ scale: other_scale,+ values,+ },+ ) => {+ scale.to_bits() == other_scale.to_bits()+ && lanes+ .iter()+ .flat_map(|lane| lane.to_array())+ .eq(values.iter().copied())+ }+ _ => false,+ }+ }+pub(crate) fn pack_query(&self, values: &[f32]) -> Result<PackedQuery, VecDbError> {if values.len() != self.dims {return Err(VecDbError::DimensionMismatch {@@ -763,7 +784,7 @@ mod tests {}#[test]- fn upsert_replaces_in_place() {+ fn upsert_appends_a_version_and_preserves_the_retired_vector() {let mut arena = VectorArena::new(8).unwrap();let first = seeded_vector(1, 8);let second = seeded_vector(2, 8);@@ -773,29 +794,32 @@ mod tests {);assert_eq!(arena.upsert("key", &second).unwrap(),- UpsertOutcome::ReplacedInPlace(0)+ UpsertOutcome::NewSlot(1));- assert_eq!(arena.vector_values(0), second);+ assert_eq!(arena.vector_values(0), first);+ assert_eq!(arena.vector_values(1), second);+ assert_eq!(arena.slot_of_key("key"), Some(1));+ assert!(!arena.is_alive(0));assert_eq!(arena.live_count(), 1);- assert_eq!(arena.slot_count(), 1);+ assert_eq!(arena.slot_count(), 2);}#[test]- fn free_list_reuses_most_recently_freed_slot_first() {+ fn new_keys_never_reuse_retired_slots() {let mut arena = arena_with_keys(8, &["a", "b", "c"]);assert_eq!(arena.remove("a"), Some(0));assert_eq!(arena.remove("c"), Some(2));assert_eq!(arena.upsert("d", &seeded_vector(4, 8)).unwrap(),- UpsertOutcome::RecycledSlot(2)+ UpsertOutcome::NewSlot(3));assert_eq!(arena.upsert("e", &seeded_vector(5, 8)).unwrap(),- UpsertOutcome::RecycledSlot(0)+ UpsertOutcome::NewSlot(4));assert_eq!(arena.upsert("f", &seeded_vector(6, 8)).unwrap(),- UpsertOutcome::NewSlot(3)+ UpsertOutcome::NewSlot(5));}@@ -832,8 +856,8 @@ mod tests {assert_eq!(arena.dead_count(), 1);arena.upsert("d", &seeded_vector(9, 8)).unwrap();assert_eq!(arena.live_count(), 3);- assert_eq!(arena.dead_count(), 0);- assert_eq!(arena.live_slots().collect::<Vec<_>>(), vec![0, 1, 2]);+ assert_eq!(arena.dead_count(), 1);+ assert_eq!(arena.live_slots().collect::<Vec<_>>(), vec![0, 2, 3]);}#[test]@@ -846,26 +870,27 @@ mod tests {}#[test]- fn readding_a_removed_key_recycles_like_any_new_key() {+ fn readding_a_removed_key_appends_like_any_new_key() {let mut arena = arena_with_keys(8, &["a", "b"]);assert_eq!(arena.remove("a"), Some(0));let vector = seeded_vector(9, 8);assert_eq!(arena.upsert("a", &vector).unwrap(),- UpsertOutcome::RecycledSlot(0)+ UpsertOutcome::NewSlot(2));- assert_eq!(arena.key_of_slot(0), Some("a"));- assert_eq!(arena.vector_values(0), vector);+ assert_eq!(arena.key_of_slot(0), None);+ assert_eq!(arena.key_of_slot(2), Some("a"));+ assert_eq!(arena.vector_values(2), vector);assert_eq!(arena.live_count(), 2);- assert_eq!(arena.dead_count(), 0);+ assert_eq!(arena.dead_count(), 1);assert_eq!(arena.remove("b"), Some(1));assert_eq!(arena.upsert("c", &seeded_vector(10, 8)).unwrap(),- UpsertOutcome::RecycledSlot(1)+ UpsertOutcome::NewSlot(3));assert_eq!(arena.upsert("b", &seeded_vector(11, 8)).unwrap(),- UpsertOutcome::NewSlot(2)+ UpsertOutcome::NewSlot(4));}@@ -928,11 +953,11 @@ mod tests {assert_eq!(arena.remove("key-9"), Some(9));assert_eq!(arena.upsert("key-5", &seeded_vector(50, 8)).unwrap(),- UpsertOutcome::ReplacedInPlace(5)+ UpsertOutcome::NewSlot(10));assert_eq!(arena.upsert("extra", &seeded_vector(60, 8)).unwrap(),- UpsertOutcome::RecycledSlot(9)+ UpsertOutcome::NewSlot(11));assert_eq!(arena.remove("key-2"), Some(2));let expected: Vec<(String, Vec<f32>)> = arena@@ -945,12 +970,11 @@ mod tests {}).collect();assert_eq!(expected.len(), 7);- assert_eq!(arena.dead_count(), 3);+ assert_eq!(arena.dead_count(), 5);arena.compact_in_place();assert_eq!(arena.live_count(), 7);assert_eq!(arena.slot_count(), 7);assert_eq!(arena.dead_count(), 0);- assert!(arena.free_slots.is_empty());for (index, (key, vector)) in expected.iter().enumerate() {let slot = index as u32;assert!(arena.is_alive(slot));@@ -967,12 +991,12 @@ mod tests {arena.upsert("fresh", &seeded_vector(70, 8)).unwrap(),UpsertOutcome::NewSlot(7));- let recycled = arena.slot_of_key("key-5").unwrap();- assert_eq!(arena.remove("key-5"), Some(recycled));+ let retired = arena.slot_of_key("key-5").unwrap();+ assert_eq!(arena.remove("key-5"), Some(retired));assert_eq!(arena.dead_count(), 1);assert_eq!(arena.upsert("reused", &seeded_vector(80, 8)).unwrap(),- UpsertOutcome::RecycledSlot(recycled)+ UpsertOutcome::NewSlot(8));}@@ -1004,7 +1028,6 @@ mod tests {assert_eq!(arena.slot_count(), live);assert_eq!(arena.storage.chunk_count(), 1);assert_eq!(arena.alive.len(), live.div_ceil(64));- assert!(arena.free_slots.is_empty());for (index, (key, vector)) in expected.iter().enumerate() {assert_eq!(arena.slot_of_key(key), Some(index as u32));assert_eq!(&arena.vector_values(index as u32), vector);@@ -1199,7 +1222,7 @@ mod tests {assert_eq!(arena.vector_values(0), dequantize(scale, &values));assert_eq!(arena.upsert_payload("exact", payload).unwrap(),- UpsertOutcome::ReplacedInPlace(0)+ UpsertOutcome::Unchanged(0));assert_eq!(stored_i8(&arena, 0), (scale, values.clone()));assert_eq!(arena.remove("exact"), Some(0));@@ -1210,9 +1233,10 @@ mod tests {};assert_eq!(arena.upsert_payload("recycled", other).unwrap(),- UpsertOutcome::RecycledSlot(0)+ UpsertOutcome::NewSlot(1));- assert_eq!(stored_i8(&arena, 0), (0.5, vec![3i8; dims]));+ assert_eq!(stored_i8(&arena, 0), (scale, values));+ assert_eq!(stored_i8(&arena, 1), (0.5, vec![3i8; dims]));}#[test]diff --git a/rust/crates/vecdb/src/graph.rs b/rust/crates/vecdb/src/graph.rsindex dec260fb24..6b3a246157 100644--- a/rust/crates/vecdb/src/graph.rs+++ b/rust/crates/vecdb/src/graph.rs@@ -146,7 +146,6 @@ struct UpperLevel {dense_of_slot: Vec<u32>,neighbors: Vec<u32>,lengths: Vec<u16>,- free: Vec<u32>,}impl UpperLevel {@@ -155,7 +154,6 @@ impl UpperLevel {dense_of_slot: Vec::new(),neighbors: Vec::new(),lengths: Vec::new(),- free: Vec::new(),}}@@ -163,7 +161,6 @@ impl UpperLevel {self.dense_of_slot.capacity() * size_of::<u32>()+ self.neighbors.capacity() * size_of::<u32>()+ self.lengths.capacity() * size_of::<u16>()- + self.free.capacity() * size_of::<u32>()}}@@ -331,11 +328,6 @@ impl Graph {}}- pub(crate) fn reinsert(&mut self, slot: u32, arena: &VectorArena) {- self.detach(slot);- self.insert(slot, arena);- }-pub(crate) fn entry_point(&self) -> Option<u32> {self.entry_point}@@ -455,79 +447,14 @@ impl Graph {if upper.dense_of_slot.len() < span {grow_amortized(&mut upper.dense_of_slot, span, ABSENT_DENSE);}- let dense = match upper.free.pop() {- Some(dense) => dense,- None => {- let dense = upper.lengths.len() as u32;- upper.lengths.push(0);- upper- .neighbors- .resize(upper.lengths.len() * UPPER_LEVEL_NEIGHBOR_CAP, 0);- dense- }- };- upper.lengths[dense as usize] = 0;+ let dense = upper.lengths.len() as u32;+ upper.lengths.push(0);+ upper+ .neighbors+ .resize(upper.lengths.len() * UPPER_LEVEL_NEIGHBOR_CAP, 0);upper.dense_of_slot[slot as usize] = dense;}- fn release_node(&mut self, slot: u32, level: usize) {- self.zero_lengths[slot as usize] = 0;- for layer in 1..=level {- let upper = &mut self.upper[layer - 1];- let dense = upper.dense_of_slot[slot as usize];- if dense == ABSENT_DENSE {- continue;- }- upper.dense_of_slot[slot as usize] = ABSENT_DENSE;- upper.lengths[dense as usize] = 0;- upper.free.push(dense);- }- }-- fn detach(&mut self, slot: u32) {- let Some(level) = self.level_of(slot).map(usize::from) else {- return;- };- self.node_levels[slot as usize] = ABSENT_LEVEL;- for layer in 0..=level {- let mut buffer = [0u32; LEVEL_ZERO_NEIGHBOR_CAP];- let count = {- let list = self.level_neighbors(slot, layer);- buffer[..list.len()].copy_from_slice(list);- list.len()- };- for &neighbor in &buffer[..count] {- self.unlink(neighbor, layer, slot);- }- }- self.release_node(slot, level);- if self.entry_point == Some(slot) {- self.entry_point = self.highest_slot();- }- }-- fn unlink(&mut self, slot: u32, level: usize, other: u32) {- if !self.holds_level(slot, level) {- return;- }- let mut buffer = [0u32; LEVEL_ZERO_NEIGHBOR_CAP];- let count = {- let list = self.level_neighbors(slot, level);- if !list.contains(&other) {- return;- }- let mut count = 0;- for &value in list {- if value != other {- buffer[count] = value;- count += 1;- }- }- count- };- self.write_level_list(slot, level, &buffer[..count]);- }-fn link_back(&mut self, slot: u32, level: usize, other: u32, cap: usize, arena: &VectorArena) {if !self.holds_level(slot, level) {return;@@ -563,19 +490,6 @@ impl Graph {self.write_level_list(slot, level, &kept[..kept_count]);}- fn highest_slot(&self) -> Option<u32> {- let mut best: Option<(u16, u32)> = None;- for (index, &level) in self.node_levels.iter().enumerate() {- if level == ABSENT_LEVEL {- continue;- }- if best.is_none_or(|(best_level, _)| level > best_level) {- best = Some((level, index as u32));- }- }- best.map(|(_, slot)| slot)- }-fn next_level(&mut self) -> usize {let mut state = LEVEL_SEED ^ self.insert_ordinal;self.insert_ordinal = self.insert_ordinal.wrapping_add(1);@@ -1290,11 +1204,8 @@ mod tests {}fn apply_upsert(arena: &mut VectorArena, graph: &mut Graph, key: &str, vector: &[f32]) {- match arena.upsert(key, vector).unwrap() {- UpsertOutcome::NewSlot(slot) => graph.insert(slot, arena),- UpsertOutcome::RecycledSlot(slot) | UpsertOutcome::ReplacedInPlace(slot) => {- graph.reinsert(slot, arena);- }+ if let UpsertOutcome::NewSlot(slot) = arena.upsert(key, vector).unwrap() {+ graph.insert(slot, arena);}}@@ -1911,13 +1822,14 @@ mod tests {}#[test]- fn reinsert_moves_key_to_its_new_position() {+ fn replacement_moves_key_to_its_new_position() {let (mut arena, mut graph) = build_fixture(300, 16, 0x4000_0000);let old_vector = arena.vector_values(7);let new_vector = seeded_unit_vector(0x4222_0000, 16);let outcome = arena.upsert("key-7", &new_vector).unwrap();- assert_eq!(outcome, UpsertOutcome::ReplacedInPlace(7));- graph.reinsert(7, &arena);+ assert_eq!(outcome, UpsertOutcome::NewSlot(300));+ graph.insert(300, &arena);+ assert_eq!(arena.vector_values(7), old_vector);let new_query = arena.pack_query(&new_vector).unwrap();for exact in [false, true] {let top = search(@@ -1931,7 +1843,7 @@ mod tests {assert!(top[0].distance < 1e-3);}let old_query = arena.pack_query(&old_vector).unwrap();- let moved_distance = arena.distance_to_query(old_query.as_query(), 7);+ let moved_distance = arena.distance_to_query(old_query.as_query(), 300);assert!(moved_distance > 0.3);let near_old = search(&graph,@@ -1954,14 +1866,14 @@ mod tests {}#[test]- fn reinsert_of_entry_point_keeps_graph_searchable() {+ fn replacement_of_entry_point_keeps_graph_searchable() {let (mut arena, mut graph) = build_fixture(300, 16, 0x5000_0000);let entry_slot = graph.entry_point().unwrap();let entry_key = arena.key_of_slot(entry_slot).unwrap().to_string();let new_vector = seeded_unit_vector(0x5222_0000, 16);let outcome = arena.upsert(&entry_key, &new_vector).unwrap();- assert_eq!(outcome, UpsertOutcome::ReplacedInPlace(entry_slot));- graph.reinsert(entry_slot, &arena);+ assert_eq!(outcome, UpsertOutcome::NewSlot(300));+ graph.insert(300, &arena);assert!(graph.entry_point().is_some());let query = arena.pack_query(&new_vector).unwrap();let top = search(&graph, &arena, &query, ¶ms(Some(1), None, false), None);@@ -1977,13 +1889,13 @@ mod tests {}#[test]- fn reinsert_recycled_slot_serves_the_new_key() {+ fn insert_after_deletion_serves_the_new_key() {let (mut arena, mut graph) = build_fixture(300, 16, 0x6000_0000);assert_eq!(arena.remove("key-3"), Some(3));let new_vector = seeded_unit_vector(0x6222_0000, 16);let outcome = arena.upsert("fresh", &new_vector).unwrap();- assert_eq!(outcome, UpsertOutcome::RecycledSlot(3));- graph.reinsert(3, &arena);+ assert_eq!(outcome, UpsertOutcome::NewSlot(300));+ graph.insert(300, &arena);let query = arena.pack_query(&new_vector).unwrap();for exact in [false, true] {let top = search(&graph, &arena, &query, ¶ms(Some(1), None, exact), None);@@ -2002,13 +1914,13 @@ mod tests {}#[test]- fn reinsert_of_the_only_node_resets_entry_point() {+ fn replacement_of_the_only_node_preserves_the_routing_node() {let mut arena = VectorArena::new(8).unwrap();arena.upsert("solo", &axis_vector(8, 0)).unwrap();let mut graph = Graph::rebuild(&arena);assert_eq!(graph.entry_point(), Some(0));arena.upsert("solo", &axis_vector(8, 3)).unwrap();- graph.reinsert(0, &arena);+ graph.insert(1, &arena);assert_eq!(graph.entry_point(), Some(0));let query = arena.pack_query(&axis_vector(8, 3)).unwrap();let found = search(&graph, &arena, &query, ¶ms(Some(5), None, false), None);@@ -2503,19 +2415,28 @@ mod tests {}#[test]- fn repeated_reinserts_shrink_levels_yet_keep_every_node_reachable() {+ fn repeated_replacements_preserve_levels_and_keep_live_nodes_reachable() {let (mut arena, mut graph) = build_fixture(600, 16, 0xB000_0000);- let mut stale_seen = false;+ let original_levels: Vec<_> = graph+ .slots()+ .map(|slot| (slot, graph.level_of(slot)))+ .collect();+ let key = arena+ .key_of_slot(graph.entry_point().unwrap())+ .unwrap()+ .to_string();for round in 0..40u64 {- let victim = graph.entry_point().unwrap();- let key = arena.key_of_slot(victim).unwrap().to_string();let vector = seeded_unit_vector(0xB100_0000 + round, 16);+ let slot = arena.slot_count() as u32;assert_eq!(arena.upsert(&key, &vector).unwrap(),- UpsertOutcome::ReplacedInPlace(victim)+ UpsertOutcome::NewSlot(slot));- graph.reinsert(victim, &arena);- stale_seen |= stale_downward_edge_exists(&graph);+ graph.insert(slot, &arena);+ assert!(!stale_downward_edge_exists(&graph));+ for &(slot, level) in &original_levels {+ assert_eq!(graph.level_of(slot), level);+ }assert_graph_invariants(&graph);let reloaded = Graph::from_parts(graph.entry_point(),@@ -2537,7 +2458,6 @@ mod tests {);assert_eq!(everything.len(), 600);}- assert!(stale_seen);let rebuilt = Graph::from_parts(graph.entry_point(),graph_parts(&graph),diff --git a/rust/crates/vecdb/src/kernel.rs b/rust/crates/vecdb/src/kernel.rsindex b83ed06f6d..1ab00e88f1 100644--- a/rust/crates/vecdb/src/kernel.rs+++ b/rust/crates/vecdb/src/kernel.rs@@ -53,6 +53,29 @@ pub(crate) enum VectorPayload<'a> {}impl VectorPayload<'_> {+ pub(crate) fn bitwise_eq(self, other: Self) -> bool {+ match (self, other) {+ (Self::F32(first), Self::F32(second)) => {+ first.len() == second.len()+ && first+ .iter()+ .zip(second)+ .all(|(a, b)| a.to_bits() == b.to_bits())+ }+ (+ Self::I8 {+ scale: first_scale,+ values: first,+ },+ Self::I8 {+ scale: second_scale,+ values: second,+ },+ ) => first_scale.to_bits() == second_scale.to_bits() && first == second,+ _ => false,+ }+ }+pub(crate) fn storage(&self) -> StorageKind {match self {Self::F32(_) => StorageKind::F32,diff --git a/rust/crates/vecdb/src/log.rs b/rust/crates/vecdb/src/log.rsindex 307900d2d0..f389b35fed 100644--- a/rust/crates/vecdb/src/log.rs+++ b/rust/crates/vecdb/src/log.rs@@ -821,7 +821,7 @@ fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {})}-fn validate_entry(+pub(crate) fn validate_entry(entry: &LogEntry<'_>,dims: usize,storage: StorageKind,@@ -902,6 +902,20 @@ fn encoded_attr_len(attr: &Attribute) -> usize {}}+pub(crate) fn add_record_len(+ key: &str,+ dims: usize,+ storage: StorageKind,+ attrs: &[Attribute],+) -> u64 {+ (RECORD_PREFIX_LEN+ + key.len()+ + vector_payload_len(storage, dims)+ + 1+ + attrs.iter().map(encoded_attr_len).sum::<usize>()+ + RECORD_CRC_LEN) as u64+}+fn encode_record_into(buffer: &mut Vec<u8>, entry: &LogEntry<'_>) {let start = buffer.len();match entry {@@ -1024,6 +1038,48 @@ mod tests {use super::*;use tempfile::TempDir;+ #[test]+ fn add_record_size_matches_encoded_bytes_for_both_storage_formats() {+ let vector = seeded_vector(9, 32);+ let attrs = vec![+ Attribute {+ name: "name".into(),+ value: AttrValue::Str("ε½±ε".into()),+ },+ Attribute {+ name: "flag".into(),+ value: AttrValue::Bool(true),+ },+ Attribute {+ name: "count".into(),+ value: AttrValue::I64(-23),+ },+ Attribute {+ name: "score".into(),+ value: AttrValue::F64(0.125),+ },+ ];+ for stored in [+ StoredVector::F32(vector.clone()),+ StoredVector::quantize(&vector),+ ] {+ for attrs in [&[][..], attrs.as_slice()] {+ let payload = stored.as_payload();+ let entry = LogEntry::Add {+ key: "photo-ε½±ε",+ vector: payload,+ attrs,+ };+ let mut encoded = Vec::new();+ encode_record_into(&mut encoded, &entry);+ assert_eq!(+ add_record_len("photo-ε½±ε", 32, payload.storage(), attrs),+ encoded.len() as u64+ );+ }+ }+ }+fn seeded_vector(seed: u64, dims: usize) -> Vec<f32> {let mut state = seed;(0..dims)diff --git a/rust/crates/vecdb/src/snapshot.rs b/rust/crates/vecdb/src/snapshot.rsindex b31f9ed2f8..8e5a095794 100644--- a/rust/crates/vecdb/src/snapshot.rs+++ b/rust/crates/vecdb/src/snapshot.rs@@ -359,30 +359,29 @@ mod tests {for index in [3, 41, 87, 150, 199] {arena.remove(&format!("key-{index}")).unwrap();}- let mut stale_seen = false;for round in 0..40u64 {let victim = graph.entry_point().filter(|&slot| arena.is_alive(slot)).unwrap_or_else(|| arena.live_slots().next().unwrap());let key = arena.key_of_slot(victim).unwrap().to_string();+ let slot = arena.slot_count() as u32;assert_eq!(arena.upsert(&key, &seeded_unit_vector(0x00A2_0000 + round, dims)).unwrap(),- UpsertOutcome::ReplacedInPlace(victim)+ UpsertOutcome::NewSlot(slot));- graph.reinsert(victim, &arena);- stale_seen |= stale_downward_edge_exists(&graph);+ graph.insert(slot, &arena);+ assert!(!stale_downward_edge_exists(&graph));}- let UpsertOutcome::RecycledSlot(recycled) = arena- .upsert("recycled", &seeded_unit_vector(0x00A3_0000, dims))+ let UpsertOutcome::NewSlot(slot) = arena+ .upsert("fresh", &seeded_unit_vector(0x00A3_0000, dims)).unwrap()else {- panic!("expected a recycled slot");+ panic!("expected a new slot");};- graph.reinsert(recycled, &arena);- assert!(stale_seen);+ graph.insert(slot, &arena);(arena, graph)}diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rsindex 32423db79d..f11aa7be35 100644--- a/rust/crates/vecdb/src/store.rs+++ b/rust/crates/vecdb/src/store.rs@@ -14,8 +14,9 @@ use super::graph::{Graph, search as graph_search, search_stored};use super::kernel::{StoredVector, VectorPayload, quantize_for};use super::lock::WriterLock;use super::log::{- HEADER_LEN, Log, LogCheckpoint, LogEntry, LogRecord, header_generation, open_writer_file,- remove_if_present, remove_stale_temp_sibling, sync_parent_dir,+ HEADER_LEN, Log, LogCheckpoint, LogEntry, LogRecord, add_record_len, header_generation,+ open_writer_file, remove_if_present, remove_stale_temp_sibling, sync_parent_dir,+ validate_entry,};use super::snapshot::{LoadedSnapshot, load_snapshot, remove_snapshot, snapshot_path, write_snapshot,@@ -26,6 +27,7 @@ const SNAPSHOT_HARD_CAP: usize = 5000;const SNAPSHOT_QUIET_THRESHOLD: usize = 1000;const SNAPSHOT_QUIET_GAP: Duration = Duration::from_secs(10);const COMPACTION_MIN_DEAD_RECORDS: u64 = 64;+const COMPACTION_MIN_RETIRED_VECTORS: usize = 64;const COMPACTION_DEAD_RATIO: u64 = 10;const COMPACTION_BATCH_SIZE: usize = 1000;const HANDOFF_WAIT_ROUNDS: u32 = 750;@@ -89,12 +91,31 @@ struct SearchState {graph: Option<Graph>,attrs: AttrTable,total_records: u64,+ live_log_bytes: u64,}impl SearchState {fn search_graph(&self) -> &Graph {self.graph.as_ref().unwrap_or(&PENDING_SEARCH_GRAPH)}++ fn entry_matches(&self, key: &str, vector: VectorPayload<'_>, attrs: &[Attribute]) -> bool {+ self.arena.slot_of_key(key).is_some_and(|slot| {+ self.arena.matches_payload(slot, vector)+ && attrs_match(self.attrs.get(slot).unwrap_or(&[]), attrs)+ })+ }+}++fn attrs_match(first: &[Attribute], second: &[Attribute]) -> bool {+ first.len() == second.len()+ && first.iter().zip(second).all(|(a, b)| {+ a.name == b.name+ && match (&a.value, &b.value) {+ (AttrValue::F64(a), AttrValue::F64(b)) => a.to_bits() == b.to_bits(),+ _ => a.value == b.value,+ }+ })}#[derive(Default)]@@ -167,9 +188,7 @@ impl AttrTable {fn slot_of_outcome(outcome: UpsertOutcome) -> u32 {match outcome {- UpsertOutcome::NewSlot(slot)- | UpsertOutcome::RecycledSlot(slot)- | UpsertOutcome::ReplacedInPlace(slot) => slot,+ UpsertOutcome::NewSlot(slot) | UpsertOutcome::Unchanged(slot) => slot,}}@@ -491,11 +510,16 @@ impl VecDb {let payload = quantized.as_ref().map_or(VectorPayload::F32(vector), StoredVector::as_payload);- state.log.append(&[LogEntry::Add {+ let entry = LogEntry::Add {key,vector: payload,attrs,- }])?;+ };+ validate_entry(&entry, self.shared.dims, self.shared.storage)?;+ if self.shared.state_read().entry_matches(key, payload, attrs) {+ return Ok(());+ }+ state.log.append(&[entry])?;apply_add(&self.shared, key, payload, attrs)?;apply_policy(&self.shared, &mut half, 1, now)}@@ -558,20 +582,51 @@ impl VecDb {Some(stored) => stored[index].as_payload(),None => VectorPayload::F32(&vectors[index]),};- let records: Vec<LogEntry<'_>> = keys+ let scheduled = {+ let st = self.shared.state_read();+ let mut previous = HashMap::<&str, usize>::new();+ let mut scheduled = Vec::new();+ for (index, key) in keys.iter().enumerate() {+ let entry = LogEntry::Add {+ key,+ vector: payload_of(index),+ attrs: attrs_of(index),+ };+ validate_entry(&entry, self.shared.dims, self.shared.storage)?;+ let unchanged = match previous.insert(key, index) {+ Some(prior) => {+ payload_of(prior).bitwise_eq(payload_of(index))+ && attrs_match(attrs_of(prior), attrs_of(index))+ }+ None => st.entry_matches(key, payload_of(index), attrs_of(index)),+ };+ if !unchanged {+ scheduled.push(index);+ }+ }+ scheduled+ };+ if scheduled.is_empty() {+ return Ok(());+ }+ let records: Vec<LogEntry<'_>> = scheduled.iter()- .enumerate()- .map(|(index, key)| LogEntry::Add {- key,+ .map(|&index| LogEntry::Add {+ key: &keys[index],vector: payload_of(index),attrs: attrs_of(index),}).collect();state.log.append(&records)?;- for (index, key) in keys.iter().enumerate() {- apply_add(&self.shared, key, payload_of(index), attrs_of(index))?;+ for &index in &scheduled {+ apply_add(+ &self.shared,+ &keys[index],+ payload_of(index),+ attrs_of(index),+ )?;}- apply_policy(&self.shared, &mut half, keys.len(), now)+ apply_policy(&self.shared, &mut half, scheduled.len(), now)}pub fn remove(&self, key: &str) -> Result<bool, VecDbError> {@@ -805,6 +860,7 @@ impl VecDb {st.graph = Some(Graph::new());st.attrs.reset();st.total_records = 0;+ st.live_log_bytes = 0;}half.compaction_retry_at_dead = 0;half.mode = WriterMode::Active(WriterState {@@ -1022,6 +1078,7 @@ fn close_live_instance(shared: &Arc<Shared>) -> Result<(), VecDbError> {st.graph = Some(Graph::new());st.attrs.reset();st.total_records = 0;+ st.live_log_bytes = 0;}drop(lock);if let Some(key) = &shared.registry_key@@ -1053,9 +1110,9 @@ fn apply_add(graph,attrs: attr_table,total_records,+ live_log_bytes,} = &mut *st;- let outcome = arena.upsert_payload(key, vector)?;- attr_table.set(slot_of_outcome(outcome), attrs);+ let outcome = upsert_entry(arena, attr_table, live_log_bytes, key, vector, attrs)?;if let Some(graph) = graph {apply_outcome(graph, arena, outcome);}@@ -1065,36 +1122,97 @@ fn apply_add(fn apply_remove(shared: &Shared, key: &str) {let mut st = shared.state_write();- if let Some(slot) = st.arena.remove(key) {- st.attrs.clear(slot);- }+ let SearchState {+ arena,+ attrs,+ live_log_bytes,+ ..+ } = &mut *st;+ remove_entry(arena, attrs, live_log_bytes, key);st.total_records += 1;}+fn upsert_entry(+ arena: &mut VectorArena,+ attrs: &mut AttrTable,+ live_log_bytes: &mut u64,+ key: &str,+ vector: VectorPayload<'_>,+ new_attrs: &[Attribute],+) -> Result<UpsertOutcome, VecDbError> {+ let previous = arena.slot_of_key(key);+ let outcome = arena.upsert_payload(key, vector)?;+ if let Some(slot) = previous {+ *live_log_bytes -= add_record_len(+ key,+ arena.dims(),+ arena.storage_kind(),+ attrs.get(slot).unwrap_or(&[]),+ );+ attrs.clear(slot);+ }+ attrs.set(slot_of_outcome(outcome), new_attrs);+ *live_log_bytes += add_record_len(key, arena.dims(), arena.storage_kind(), new_attrs);+ Ok(outcome)+}++fn remove_entry(+ arena: &mut VectorArena,+ attrs: &mut AttrTable,+ live_log_bytes: &mut u64,+ key: &str,+) {+ if let Some(slot) = arena.remove(key) {+ *live_log_bytes -= add_record_len(+ key,+ arena.dims(),+ arena.storage_kind(),+ attrs.get(slot).unwrap_or(&[]),+ );+ attrs.clear(slot);+ }+}+fn apply_policy(shared: &Shared,half: &mut WriterHalf,count: usize,now: Instant,) -> Result<(), VecDbError> {- let (live, total) = {+ let (dead, retired, slots, live_log_bytes) = {let st = shared.state_read();- (st.arena.live_count() as u64, st.total_records)+ (+ st.total_records+ .saturating_sub(st.arena.live_count() as u64),+ st.arena.dead_count(),+ st.arena.slot_count(),+ st.live_log_bytes,+ )};- let (quiet, mutations) = {+ let (quiet, mutations, log_bytes) = {let state = active_state(half)?;state.mutations_since_snapshot += count;let quiet = state.last_write.is_some_and(|previous| now.duration_since(previous) >= SNAPSHOT_QUIET_GAP);state.last_write = Some(now);- (quiet, state.mutations_since_snapshot)+ (+ quiet,+ state.mutations_since_snapshot,+ state+ .log+ .current_end_offset()+ .saturating_sub(HEADER_LEN as u64),+ )};- let dead = total.saturating_sub(live);- if dead >= COMPACTION_MIN_DEAD_RECORDS- && dead.saturating_mul(COMPACTION_DEAD_RATIO) >= total- && dead >= half.compaction_retry_at_dead- {+ let retired_budget_reached = retired >= COMPACTION_MIN_RETIRED_VECTORS+ && (retired as u64).saturating_mul(COMPACTION_DEAD_RATIO) >= slots as u64;+ let log_budget_reached = dead >= COMPACTION_MIN_DEAD_RECORDS+ && log_bytes+ .saturating_sub(live_log_bytes)+ .saturating_mul(COMPACTION_DEAD_RATIO)+ >= log_bytes;+ if (retired_budget_reached || log_budget_reached) && dead >= half.compaction_retry_at_dead {match compact(shared, half) {Ok(()) => half.compaction_retry_at_dead = 0,Err(error) => {@@ -1164,25 +1282,31 @@ fn compact(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecDbError> {let _ = std::fs::remove_file(&temp_path);(mutations_since_snapshot, old_checkpoint)} else {- let live_records = {+ let (live_records, rebuild) = {let mut st = shared.state_write();let SearchState {arena,graph,attrs,total_records,+ ..} = &mut *st;- attrs.repack(arena.live_slots());- arena.compact_in_place();- *graph = None;+ let rebuild = arena.dead_count() > 0;+ if rebuild {+ attrs.repack(arena.live_slots());+ arena.compact_in_place();+ *graph = None;+ }*total_records = arena.live_count() as u64;- *total_records as usize- };- let graph = {- let st = shared.state_read();- Graph::rebuild(&st.arena)+ (*total_records as usize, rebuild)};- shared.state_write().graph = Some(graph);+ if rebuild {+ let graph = {+ let st = shared.state_read();+ Graph::rebuild(&st.arena)+ };+ shared.state_write().graph = Some(graph);+ }(live_records, compacted_checkpoint)};let restored =@@ -1224,6 +1348,7 @@ fn restore_writer_mode(graph: Some(graph),attrs: replayed.attrs,total_records: replayed.total_records,+ live_log_bytes: replayed.live_log_bytes,};half.compaction_retry_at_dead = 0;}@@ -1269,6 +1394,7 @@ fn recover_after_failed_reset(st.graph = Some(Graph::new());st.attrs.reset();st.total_records = 0;+ st.live_log_bytes = 0;half.compaction_retry_at_dead = 0;}}@@ -1312,6 +1438,7 @@ fn build_writer(graph,attrs,total_records,+ live_log_bytes,tail_records,..} = replayed;@@ -1348,6 +1475,7 @@ fn build_writer(graph: Some(graph),attrs,total_records,+ live_log_bytes,},)))}@@ -1368,6 +1496,7 @@ fn build_writer(graph: None,attrs,total_records,+ live_log_bytes,},),snapshot_stale: total_records > 0 || snapshot_present,@@ -1463,6 +1592,7 @@ fn build_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Shagraph,attrs,total_records,+ live_log_bytes,recoverable_end,..} = replayed;@@ -1473,6 +1603,7 @@ fn build_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Shagraph,attrs,total_records,+ live_log_bytes,recoverable_end,))}@@ -1485,6 +1616,7 @@ fn empty_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<ShaAttrTable::default(),0,0,+ 0,))}@@ -1494,6 +1626,7 @@ fn read_only_shared(graph: Graph,attrs: AttrTable,total_records: u64,+ live_log_bytes: u64,log_bytes: u64,) -> Shared {Shared {@@ -1511,6 +1644,7 @@ fn read_only_shared(graph: Some(graph),attrs,total_records,+ live_log_bytes,}),}}@@ -1523,9 +1657,9 @@ fn approximate_memory_bytes(state: &SearchState) -> usize {.filter_map(|slot| state.arena.key_of_slot(slot)).map(|key| KEY_TABLE_COPIES * (key.len() + KEY_ENTRY_OVERHEAD_BYTES)).sum();- let free_list_bytes = state.arena.dead_count() * size_of::<u32>();let graph_bytes = state.graph.as_ref().map_or(0, Graph::memory_bytes);- vector_bytes + key_bytes + free_list_bytes + graph_bytes + state.attrs.memory_bytes()+ let retired_key_slots = state.arena.dead_count() * size_of::<Box<str>>();+ vector_bytes + key_bytes + retired_key_slots + graph_bytes + state.attrs.memory_bytes()}struct ReplayedState {@@ -1533,6 +1667,7 @@ struct ReplayedState {graph: Option<Graph>,attrs: AttrTable,total_records: u64,+ live_log_bytes: u64,recoverable_end: u64,tail_records: u64,}@@ -1568,6 +1703,7 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,let mut graph: Option<Graph> = None;let mut tail_records = 0u64;let mut total_records = 0u64;+ let mut live_log_bytes = 0u64;let recoverable_end;{let mut scanner = log.scan()?;@@ -1583,16 +1719,20 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,vector,attrs: record_attrs,} => {- let outcome = arena.upsert_payload(&key, vector.as_payload())?;- attrs.set(slot_of_outcome(outcome), &record_attrs);+ let outcome = upsert_entry(+ &mut arena,+ &mut attrs,+ &mut live_log_bytes,+ &key,+ vector.as_payload(),+ &record_attrs,+ )?;if let Some(graph) = &mut graph {apply_outcome(graph, &arena, outcome);}}LogRecord::Tombstone { key } => {- if let Some(slot) = arena.remove(&key) {- attrs.clear(slot);- }+ remove_entry(&mut arena, &mut attrs, &mut live_log_bytes, &key);}}}@@ -1623,6 +1763,7 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,graph,attrs,total_records,+ live_log_bytes,recoverable_end,tail_records,})@@ -1651,11 +1792,8 @@ fn attach_snapshot_graph(}fn apply_outcome(graph: &mut Graph, arena: &VectorArena, outcome: UpsertOutcome) {- match outcome {- UpsertOutcome::NewSlot(slot) => graph.insert(slot, arena),- UpsertOutcome::RecycledSlot(slot) | UpsertOutcome::ReplacedInPlace(slot) => {- graph.reinsert(slot, arena);- }+ if let UpsertOutcome::NewSlot(slot) = outcome {+ graph.insert(slot, arena);}}@@ -1750,6 +1888,316 @@ mod tests {const ADD_RECORD_LEN: u64 = 3 + 5 + (DIMS as u64) * 4 + 1 + 4;const TOMBSTONE_RECORD_LEN: u64 = 3 + 5 + 4;+ #[test]+ fn unchanged_writes_preserve_the_log_graph_and_snapshot() {+ for storage in [StorageKind::F32, StorageKind::I8] {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();+ let vector = seeded_unit_vector(1, 32);+ let attrs = sample_attrs(7);+ db.add_with_attrs("key", &vector, &attrs).unwrap();+ db.flush().unwrap();+ let before = db.stats().unwrap();+ let snapshot = fs::read(snapshot_path(&path)).unwrap();+ db.add_with_attrs("key", &vector, &attrs).unwrap();+ db.bulk_add_with_attrs(+ &["key".into(), "key".into()],+ &[vector.clone(), vector.clone()],+ &[Some(attrs.clone()), Some(attrs.clone())],+ )+ .unwrap();+ db.flush().unwrap();+ assert_eq!(db.stats().unwrap(), before);+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot);+ let invalid = vec![attr("", AttrValue::Bool(true))];+ assert!(matches!(+ db.add_with_attrs("key", &vector, &invalid),+ Err(VecDbError::InvalidAttributes(_))+ ));+ assert_eq!(db.stats().unwrap(), before);+ }+ }++ #[test]+ fn unchanged_i8_payloads_skip_writes_but_scale_changes_append_versions() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = VecDb::open(&path, 32, Some(StorageKind::I8)).unwrap();+ let mut vector = vec![0.0; 32];+ vector[0] = 1.0;+ vector[1] = 0.1;+ db.add("key", &vector).unwrap();+ let before = db.stats().unwrap();+ vector[1] += 0.00001;+ db.add("key", &vector).unwrap();+ assert_eq!(db.stats().unwrap(), before);+ let previous = db.shared.state_read().arena.stored_vector(0);+ for value in &mut vector {+ *value *= 2.0;+ }+ db.add("key", &vector).unwrap();+ let state = db.shared.state_read();+ assert_eq!(state.arena.slot_of_key("key"), Some(1));+ assert_eq!(state.arena.stored_vector(0), previous);+ assert_ne!(state.arena.stored_vector(1), previous);+ }++ #[test]+ fn f32_replacement_preserves_distinct_stored_bits() {+ let dir = TempDir::new().unwrap();+ let db = open_writer(&dir.path().join("db"));+ let mut vector = basis_vector(0);+ db.add("key", &vector).unwrap();+ vector[1] = -0.0;+ db.add("key", &vector).unwrap();+ let state = db.shared.state_read();+ assert_eq!(state.arena.slot_of_key("key"), Some(1));+ assert_eq!(state.arena.vector_values(0)[1].to_bits(), 0.0f32.to_bits());+ assert_eq!(+ state.arena.vector_values(1)[1].to_bits(),+ (-0.0f32).to_bits()+ );+ }++ #[test]+ fn batch_noops_follow_prior_entries_and_replay_the_same_versions() {+ for storage in [StorageKind::F32, StorageKind::I8] {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();+ let first = seeded_unit_vector(1, 32);+ let second = seeded_unit_vector(2, 32);+ let attrs = sample_attrs(7);+ db.add("key", &first).unwrap();+ db.flush().unwrap();+ db.bulk_add_with_attrs(+ &vec!["key".to_string(); 5],+ &[first.clone(), first.clone(), second, first.clone(), first],+ &[None, Some(attrs.clone()), None, None, None],+ )+ .unwrap();+ let verify = |db: &VecDb| {+ let state = db.shared.state_read();+ assert_eq!(state.total_records, 4);+ assert_eq!(state.arena.slot_of_key("key"), Some(2));+ assert_eq!(state.arena.dead_count(), 2);+ assert_eq!(state.graph.as_ref().unwrap().insert_ordinal(), 3);+ assert_eq!(+ state.live_log_bytes,+ add_record_len("key", 32, storage, &[])+ );+ assert!(state.attrs.get(0).is_none());+ assert!(state.attrs.get(1).is_none());+ assert!(state.attrs.get(2).is_none());+ };+ verify(&db);+ drop(db);+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();+ verify(&reopened);+ drop(reopened);+ let read_only = VecDb::open_read_only(&path, 32, Some(storage)).unwrap();+ verify(&read_only);+ }+ }++ #[test]+ fn attribute_only_compaction_preserves_graph_slots_and_levels() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ let entries = bulk_entries(0, 100, 733);+ bulk_add(&db, &entries).unwrap();+ db.flush().unwrap();+ let generation = generation_of(&path);+ let loaded = load_snapshot(&path, generation).unwrap();+ let graph = Graph::from_parts(loaded.entry_point, loaded.parts, 100, loaded.insert_ordinal)+ .unwrap();+ for (index, (key, vector)) in entries.iter().take(64).enumerate() {+ db.add_with_attrs(key, vector, &sample_attrs(index as i64))+ .unwrap();+ }+ assert_ne!(generation_of(&path), generation);+ let state = db.shared.state_read();+ assert_eq!(state.arena.dead_count(), 0);+ assert_eq!(state.total_records, 100);+ assert_eq!(state.graph.as_ref().unwrap().insert_ordinal(), 100);+ test_support::assert_identical_graphs(&graph, state.graph.as_ref().unwrap());+ for (index, (key, _)) in entries.iter().enumerate() {+ assert_eq!(state.arena.slot_of_key(key), Some(index as u32));+ }+ drop(state);+ let compacted_bytes = db.stats().unwrap().log_bytes - HEADER_LEN as u64;+ assert_eq!(db.shared.state_read().live_log_bytes, compacted_bytes);+ drop(db);+ let reopened = open_writer(&path);+ assert_eq!(reopened.shared.state_read().live_log_bytes, compacted_bytes);+ assert_eq!(+ reopened.get_attrs("key-63").unwrap(),+ Some(sample_attrs(63))+ );+ }++ #[test]+ fn maintenance_accounts_for_log_bytes_and_retired_slots_separately() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ let (keys, vectors): (Vec<_>, Vec<_>) = bulk_entries(0, 1000, 888).into_iter().unzip();+ let large_attrs = vec![attr("padding", AttrValue::Str("x".repeat(1024)))];+ let attrs: Vec<_> = (0..1000)+ .map(|index| (index >= 200).then(|| large_attrs.clone()))+ .collect();+ db.bulk_add_with_attrs(&keys, &vectors, &attrs).unwrap();+ let generation = generation_of(&path);+ let small_attrs = vec![Some(vec![attr("flag", AttrValue::Bool(true))]); 112];+ db.bulk_add_with_attrs(&keys[..112], &vectors[..112], &small_attrs)+ .unwrap();+ assert_eq!(generation_of(&path), generation);+ assert_eq!(db.stats().unwrap().dead_count, 112);+ assert_eq!(db.shared.state_read().arena.dead_count(), 0);+ db.bulk_remove(&keys[..99]).unwrap();+ assert_eq!(generation_of(&path), generation);+ assert_eq!(db.shared.state_read().arena.dead_count(), 99);+ db.remove(&keys[99]).unwrap();+ assert_ne!(generation_of(&path), generation);+ assert_eq!(db.shared.state_read().arena.dead_count(), 0);+ assert_eq!(db.stats().unwrap().dead_count, 0);+ assert_eq!(db.len().unwrap(), 900);+ }++ fn churn_vectors(clusters: usize) -> (Vec<String>, Vec<Vec<f32>>) {+ let mut seed = 2;+ let keys = (0..3000).map(|index| format!("key-{index}")).collect();+ let vectors = (0..3000)+ .map(|index| {+ let mut vector: Vec<_> = (0..32)+ .map(|_| {+ ((splitmix64(&mut seed) >> 40) as f32 / (1u64 << 24) as f32 * 2.0 - 1.0)+ * 0.01+ })+ .collect();+ vector[index % clusters] += 1.0;+ let norm = vector.iter().map(|v| v * v).sum::<f32>().sqrt();+ for value in &mut vector {+ *value /= norm;+ }+ vector+ })+ .collect();+ (keys, vectors)+ }++ fn assert_cluster_recall(db: &VecDb, vectors: &[Vec<f32>], survivors: &[usize]) {+ let approx = limit_params(10);+ let exact = SearchParams {+ exact: true,+ ..approx.clone()+ };+ let range = SearchParams {+ max_distance: Some(0.01),+ ..SearchParams::default()+ };+ let exact_range = SearchParams {+ exact: true,+ ..range.clone()+ };+ for &index in survivors {+ assert_eq!(+ db.search(&vectors[index], &approx).unwrap(),+ db.search(&vectors[index], &exact).unwrap()+ );+ assert_eq!(+ db.search(&vectors[index], &range).unwrap(),+ db.search(&vectors[index], &exact_range).unwrap()+ );+ }+ }++ #[test]+ fn concentrated_changes_preserve_recall_before_compaction_and_across_reopen() {+ for storage in [StorageKind::F32, StorageKind::I8] {+ for delete_first in [false, true] {+ let clusters = if delete_first { 20 } else { 10 };+ let victims = if delete_first { 140 } else { 270 };+ let (keys, vectors) = churn_vectors(clusters);+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();+ db.bulk_add(&keys, &vectors).unwrap();+ db.flush().unwrap();+ let survivors: Vec<_> = (victims * clusters..3000).step_by(clusters).collect();+ assert_cluster_recall(&db, &vectors, &survivors);+ let changed_keys: Vec<_> = (0..victims)+ .map(|index| keys[index * clusters].clone())+ .collect();+ let changed_vectors: Vec<_> = (0..victims)+ .map(|index| vectors[index * clusters + 1].clone())+ .collect();+ let unchanged_vectors: Vec<_> = (0..victims)+ .map(|index| vectors[index * clusters].clone())+ .collect();+ let before = db.stats().unwrap();+ db.bulk_add(&changed_keys, &unchanged_vectors).unwrap();+ assert_eq!(db.stats().unwrap(), before);+ let all = SearchParams {+ max_distance: Some(2.5),+ ..SearchParams::default()+ };+ assert_eq!(db.search(&vectors[survivors[0]], &all).unwrap().len(), 3000);+ if delete_first {+ db.bulk_remove(&changed_keys).unwrap();+ assert_cluster_recall(&db, &vectors, &survivors);+ }+ db.bulk_add(&changed_keys, &changed_vectors).unwrap();+ assert_eq!(db.shared.state_read().arena.dead_count(), victims);+ assert_cluster_recall(&db, &vectors, &survivors);+ let stored_keys: Vec<_> =+ survivors.iter().map(|&index| keys[index].clone()).collect();+ let stored_before = db+ .bulk_search_stored(&stored_keys, 10, None, false, false)+ .unwrap();+ assert_eq!(+ stored_before,+ db.bulk_search_stored(&stored_keys, 10, None, true, false)+ .unwrap()+ );+ drop(db);+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();+ assert_cluster_recall(&reopened, &vectors, &survivors);+ assert_eq!(+ reopened+ .bulk_search_stored(&stored_keys, 10, None, false, false)+ .unwrap(),+ stored_before+ );+ reopened.flush().unwrap();+ drop(reopened);+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();+ assert_cluster_recall(&reopened, &vectors, &survivors);+ let exact = SearchParams {+ exact: true,+ ..limit_params(10)+ };+ let queries: Vec<_> = survivors+ .iter()+ .map(|&index| vectors[index].clone())+ .collect();+ let truth = reopened.bulk_search(&queries, &exact).unwrap();+ compact(&reopened.shared, &mut reopened.shared.writer_half()).unwrap();+ assert_eq!(reopened.shared.state_read().arena.dead_count(), 0);+ assert_eq!(reopened.bulk_search(&queries, &exact).unwrap(), truth);+ let fresh_path = dir.path().join("fresh");+ fs::copy(&path, &fresh_path).unwrap();+ let fresh = VecDb::open(&fresh_path, 32, Some(storage)).unwrap();+ assert_eq!(+ reopened.bulk_search(&queries, &limit_params(10)).unwrap(),+ fresh.bulk_search(&queries, &limit_params(10)).unwrap()+ );+ }+ }+ }+#[test]fn omitted_storage_requires_i8_for_creation_and_every_reopen() {let dir = TempDir::new().unwrap();@@ -3781,15 +4229,15 @@ mod tests {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let db = open_writer(&path);- bulk_add(&db, &bulk_entries(0, 1100, 3000)).unwrap();+ bulk_add(&db, &bulk_entries(0, 1200, 3000)).unwrap();let generation_before = generation_of(&path);- let removals: Vec<String> = (0..58).map(|index| format!("key-{index}")).collect();- assert_eq!(db.bulk_remove(&removals).unwrap(), 58);+ let removals: Vec<String> = (0..120).map(|index| format!("key-{index}")).collect();+ assert_eq!(db.bulk_remove(&removals).unwrap(), 120);assert_ne!(generation_of(&path), generation_before);let stats = db.stats().unwrap();- assert_eq!(stats.live_count, 1042);+ assert_eq!(stats.live_count, 1080);assert_eq!(stats.dead_count, 0);- for index in (58..1100u64).step_by(97) {+ for index in (120..1200u64).step_by(97) {assert_own_nearest(&db,&format!("key-{index}"),@@ -3798,9 +4246,9 @@ mod tests {}drop(db);let reopened = open_writer(&path);- assert_eq!(reopened.len().unwrap(), 1042);+ assert_eq!(reopened.len().unwrap(), 1080);assert!(!reopened.contains("key-0").unwrap());- assert!(reopened.contains("key-58").unwrap());+ assert!(reopened.contains("key-120").unwrap());}#[test]@@ -3872,6 +4320,7 @@ mod tests {graph,attrs,total_records,+ ..} = &mut *st;attrs.repack(arena.live_slots());arena.compact_in_place();@@ -5432,7 +5881,7 @@ mod tests {}#[test]- fn attrs_do_not_leak_across_remove_and_slot_recycle() {+ fn attrs_do_not_leak_across_retired_slots_and_new_versions() {let dir = TempDir::new().unwrap();let path = dir.path().join("db");let db = open_writer(&path);@@ -5442,11 +5891,11 @@ mod tests {assert!(db.remove("a").unwrap());assert_eq!(db.get_attrs("a").unwrap(), None);db.add("c", &vector).unwrap();- assert_eq!(db.shared.state_read().arena.slot_of_key("c"), Some(0));+ assert_eq!(db.shared.state_read().arena.slot_of_key("c"), Some(2));assert_eq!(db.get_attrs("c").unwrap(), None);assert!(db.remove("b").unwrap());db.add_with_attrs("d", &vector, &sample_attrs(12)).unwrap();- assert_eq!(db.shared.state_read().arena.slot_of_key("d"), Some(1));+ assert_eq!(db.shared.state_read().arena.slot_of_key("d"), Some(3));assert_eq!(db.get_attrs("d").unwrap().unwrap(), sample_attrs(12));db.add("a", &vector).unwrap();assert_eq!(db.get_attrs("a").unwrap(), None);@@ -5561,9 +6010,9 @@ mod tests {};verify(&db);fs::remove_dir(temp_sibling(&path)).unwrap();- let rewrite = |range: std::ops::Range<usize>| {+ let rewrite = |range: std::ops::Range<usize>, seed| {let (keys, vectors): (Vec<String>, Vec<Vec<f32>>) =- bulk_entries(range.start, range.len(), 910)+ bulk_entries(range.start, range.len(), seed).iter().cloned().unzip();@@ -5572,11 +6021,11 @@ mod tests {.collect();db.bulk_add_with_attrs(&keys, &vectors, &attrs).unwrap();};- rewrite(64..120);+ rewrite(64..120, 920);assert_eq!(db.stats().unwrap().dead_count, 184);assert_eq!(generation_of(&path), generation_before);verify(&db);- rewrite(64..72);+ rewrite(64..72, 930);assert_eq!(db.stats().unwrap().dead_count, 0);assert_ne!(generation_of(&path), generation_before);verify(&db);