Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

fix(vecdb): retain retired routing nodes until rebuild

laurenspriemffe4b3a2773a
diff --git a/rust/crates/vecdb/src/arena.rs b/rust/crates/vecdb/src/arena.rs
index 3277b66908..12451e0159 100644
--- a/rust/crates/vecdb/src/arena.rs
+++ b/rust/crates/vecdb/src/arena.rs
@@ -13,8 +13,7 @@ pub(crate) const MAX_KEY_BYTES: usize = 256;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum UpsertOutcome {
NewSlot(u32),
- RecycledSlot(u32),
- ReplacedInPlace(u32),
+ Unchanged(u32),
}
pub(crate) fn validate_key(key: &str) -> Result<(), VecDbError> {
@@ -118,7 +117,6 @@ pub(crate) struct VectorArena {
keys_to_slots: HashMap<Box<str>, u32>,
slots_to_keys: Vec<Box<str>>,
alive: Vec<u64>,
- free_slots: Vec<u32>,
live_count: usize,
}
@@ -145,7 +143,6 @@ impl VectorArena {
keys_to_slots: HashMap::new(),
slots_to_keys: Vec::new(),
alive: Vec::new(),
- free_slots: Vec::new(),
live_count: 0,
})
}
@@ -163,7 +160,7 @@ impl VectorArena {
}
pub(crate) fn dead_count(&self) -> usize {
- self.free_slots.len()
+ self.slot_count() - self.live_count
}
pub(crate) fn slot_count(&self) -> usize {
@@ -214,16 +211,10 @@ impl VectorArena {
});
}
if let Some(&slot) = self.keys_to_slots.get(key) {
- self.write_vector(slot, payload);
- return Ok(UpsertOutcome::ReplacedInPlace(slot));
- }
- if let Some(slot) = self.free_slots.pop() {
- self.slots_to_keys[slot as usize] = Box::from(key);
- self.keys_to_slots.insert(Box::from(key), slot);
- self.mark_alive(slot);
- self.live_count += 1;
- self.write_vector(slot, payload);
- return Ok(UpsertOutcome::RecycledSlot(slot));
+ if self.matches_payload(slot, payload) {
+ return Ok(UpsertOutcome::Unchanged(slot));
+ }
+ self.retire_slot(slot);
}
let slot = self.slots_to_keys.len() as u32;
if slot as usize == self.storage.chunk_count() * VECTORS_PER_CHUNK {
@@ -246,10 +237,14 @@ impl VectorArena {
pub(crate) fn remove(&mut self, key: &str) -> Option<u32> {
let slot = self.keys_to_slots.remove(key)?;
+ self.retire_slot(slot);
+ Some(slot)
+ }
+
+ fn retire_slot(&mut self, slot: u32) {
self.mark_dead(slot);
self.live_count -= 1;
- self.free_slots.push(slot);
- Some(slot)
+ self.slots_to_keys[slot as usize] = Box::from("");
}
pub(crate) fn compact_in_place(&mut self) {
@@ -296,7 +291,6 @@ impl VectorArena {
{
*word = (1u64 << (live % 64)) - 1;
}
- self.free_slots = Vec::new();
}
fn move_vector(&mut self, src: u32, dst: u32) {
@@ -363,6 +357,33 @@ impl VectorArena {
}
}
+ pub(crate) fn matches_payload(&self, slot: u32, payload: VectorPayload<'_>) -> bool {
+ if payload.dims() != self.dims {
+ return false;
+ }
+ match (self.stored_query(slot), payload) {
+ (Query::F32(lanes, _), VectorPayload::F32(values)) => lanes
+ .iter()
+ .flat_map(|lane| lane.to_array())
+ .zip(values)
+ .all(|(stored, value)| stored.to_bits() == value.to_bits()),
+ (
+ Query::I8 { scale, lanes, .. },
+ VectorPayload::I8 {
+ scale: other_scale,
+ values,
+ },
+ ) => {
+ scale.to_bits() == other_scale.to_bits()
+ && lanes
+ .iter()
+ .flat_map(|lane| lane.to_array())
+ .eq(values.iter().copied())
+ }
+ _ => false,
+ }
+ }
+
pub(crate) fn pack_query(&self, values: &[f32]) -> Result<PackedQuery, VecDbError> {
if values.len() != self.dims {
return Err(VecDbError::DimensionMismatch {
@@ -763,7 +784,7 @@ mod tests {
}
#[test]
- fn upsert_replaces_in_place() {
+ fn upsert_appends_a_version_and_preserves_the_retired_vector() {
let mut arena = VectorArena::new(8).unwrap();
let first = seeded_vector(1, 8);
let second = seeded_vector(2, 8);
@@ -773,29 +794,32 @@ mod tests {
);
assert_eq!(
arena.upsert("key", &second).unwrap(),
- UpsertOutcome::ReplacedInPlace(0)
+ UpsertOutcome::NewSlot(1)
);
- assert_eq!(arena.vector_values(0), second);
+ assert_eq!(arena.vector_values(0), first);
+ assert_eq!(arena.vector_values(1), second);
+ assert_eq!(arena.slot_of_key("key"), Some(1));
+ assert!(!arena.is_alive(0));
assert_eq!(arena.live_count(), 1);
- assert_eq!(arena.slot_count(), 1);
+ assert_eq!(arena.slot_count(), 2);
}
#[test]
- fn free_list_reuses_most_recently_freed_slot_first() {
+ fn new_keys_never_reuse_retired_slots() {
let mut arena = arena_with_keys(8, &["a", "b", "c"]);
assert_eq!(arena.remove("a"), Some(0));
assert_eq!(arena.remove("c"), Some(2));
assert_eq!(
arena.upsert("d", &seeded_vector(4, 8)).unwrap(),
- UpsertOutcome::RecycledSlot(2)
+ UpsertOutcome::NewSlot(3)
);
assert_eq!(
arena.upsert("e", &seeded_vector(5, 8)).unwrap(),
- UpsertOutcome::RecycledSlot(0)
+ UpsertOutcome::NewSlot(4)
);
assert_eq!(
arena.upsert("f", &seeded_vector(6, 8)).unwrap(),
- UpsertOutcome::NewSlot(3)
+ UpsertOutcome::NewSlot(5)
);
}
@@ -832,8 +856,8 @@ mod tests {
assert_eq!(arena.dead_count(), 1);
arena.upsert("d", &seeded_vector(9, 8)).unwrap();
assert_eq!(arena.live_count(), 3);
- assert_eq!(arena.dead_count(), 0);
- assert_eq!(arena.live_slots().collect::<Vec<_>>(), vec![0, 1, 2]);
+ assert_eq!(arena.dead_count(), 1);
+ assert_eq!(arena.live_slots().collect::<Vec<_>>(), vec![0, 2, 3]);
}
#[test]
@@ -846,26 +870,27 @@ mod tests {
}
#[test]
- fn readding_a_removed_key_recycles_like_any_new_key() {
+ fn readding_a_removed_key_appends_like_any_new_key() {
let mut arena = arena_with_keys(8, &["a", "b"]);
assert_eq!(arena.remove("a"), Some(0));
let vector = seeded_vector(9, 8);
assert_eq!(
arena.upsert("a", &vector).unwrap(),
- UpsertOutcome::RecycledSlot(0)
+ UpsertOutcome::NewSlot(2)
);
- assert_eq!(arena.key_of_slot(0), Some("a"));
- assert_eq!(arena.vector_values(0), vector);
+ assert_eq!(arena.key_of_slot(0), None);
+ assert_eq!(arena.key_of_slot(2), Some("a"));
+ assert_eq!(arena.vector_values(2), vector);
assert_eq!(arena.live_count(), 2);
- assert_eq!(arena.dead_count(), 0);
+ assert_eq!(arena.dead_count(), 1);
assert_eq!(arena.remove("b"), Some(1));
assert_eq!(
arena.upsert("c", &seeded_vector(10, 8)).unwrap(),
- UpsertOutcome::RecycledSlot(1)
+ UpsertOutcome::NewSlot(3)
);
assert_eq!(
arena.upsert("b", &seeded_vector(11, 8)).unwrap(),
- UpsertOutcome::NewSlot(2)
+ UpsertOutcome::NewSlot(4)
);
}
@@ -928,11 +953,11 @@ mod tests {
assert_eq!(arena.remove("key-9"), Some(9));
assert_eq!(
arena.upsert("key-5", &seeded_vector(50, 8)).unwrap(),
- UpsertOutcome::ReplacedInPlace(5)
+ UpsertOutcome::NewSlot(10)
);
assert_eq!(
arena.upsert("extra", &seeded_vector(60, 8)).unwrap(),
- UpsertOutcome::RecycledSlot(9)
+ UpsertOutcome::NewSlot(11)
);
assert_eq!(arena.remove("key-2"), Some(2));
let expected: Vec<(String, Vec<f32>)> = arena
@@ -945,12 +970,11 @@ mod tests {
})
.collect();
assert_eq!(expected.len(), 7);
- assert_eq!(arena.dead_count(), 3);
+ assert_eq!(arena.dead_count(), 5);
arena.compact_in_place();
assert_eq!(arena.live_count(), 7);
assert_eq!(arena.slot_count(), 7);
assert_eq!(arena.dead_count(), 0);
- assert!(arena.free_slots.is_empty());
for (index, (key, vector)) in expected.iter().enumerate() {
let slot = index as u32;
assert!(arena.is_alive(slot));
@@ -967,12 +991,12 @@ mod tests {
arena.upsert("fresh", &seeded_vector(70, 8)).unwrap(),
UpsertOutcome::NewSlot(7)
);
- let recycled = arena.slot_of_key("key-5").unwrap();
- assert_eq!(arena.remove("key-5"), Some(recycled));
+ let retired = arena.slot_of_key("key-5").unwrap();
+ assert_eq!(arena.remove("key-5"), Some(retired));
assert_eq!(arena.dead_count(), 1);
assert_eq!(
arena.upsert("reused", &seeded_vector(80, 8)).unwrap(),
- UpsertOutcome::RecycledSlot(recycled)
+ UpsertOutcome::NewSlot(8)
);
}
@@ -1004,7 +1028,6 @@ mod tests {
assert_eq!(arena.slot_count(), live);
assert_eq!(arena.storage.chunk_count(), 1);
assert_eq!(arena.alive.len(), live.div_ceil(64));
- assert!(arena.free_slots.is_empty());
for (index, (key, vector)) in expected.iter().enumerate() {
assert_eq!(arena.slot_of_key(key), Some(index as u32));
assert_eq!(&arena.vector_values(index as u32), vector);
@@ -1199,7 +1222,7 @@ mod tests {
assert_eq!(arena.vector_values(0), dequantize(scale, &values));
assert_eq!(
arena.upsert_payload("exact", payload).unwrap(),
- UpsertOutcome::ReplacedInPlace(0)
+ UpsertOutcome::Unchanged(0)
);
assert_eq!(stored_i8(&arena, 0), (scale, values.clone()));
assert_eq!(arena.remove("exact"), Some(0));
@@ -1210,9 +1233,10 @@ mod tests {
};
assert_eq!(
arena.upsert_payload("recycled", other).unwrap(),
- UpsertOutcome::RecycledSlot(0)
+ UpsertOutcome::NewSlot(1)
);
- assert_eq!(stored_i8(&arena, 0), (0.5, vec![3i8; dims]));
+ assert_eq!(stored_i8(&arena, 0), (scale, values));
+ assert_eq!(stored_i8(&arena, 1), (0.5, vec![3i8; dims]));
}
#[test]
diff --git a/rust/crates/vecdb/src/graph.rs b/rust/crates/vecdb/src/graph.rs
index dec260fb24..6b3a246157 100644
--- a/rust/crates/vecdb/src/graph.rs
+++ b/rust/crates/vecdb/src/graph.rs
@@ -146,7 +146,6 @@ struct UpperLevel {
dense_of_slot: Vec<u32>,
neighbors: Vec<u32>,
lengths: Vec<u16>,
- free: Vec<u32>,
}
impl UpperLevel {
@@ -155,7 +154,6 @@ impl UpperLevel {
dense_of_slot: Vec::new(),
neighbors: Vec::new(),
lengths: Vec::new(),
- free: Vec::new(),
}
}
@@ -163,7 +161,6 @@ impl UpperLevel {
self.dense_of_slot.capacity() * size_of::<u32>()
+ self.neighbors.capacity() * size_of::<u32>()
+ self.lengths.capacity() * size_of::<u16>()
- + self.free.capacity() * size_of::<u32>()
}
}
@@ -331,11 +328,6 @@ impl Graph {
}
}
- pub(crate) fn reinsert(&mut self, slot: u32, arena: &VectorArena) {
- self.detach(slot);
- self.insert(slot, arena);
- }
-
pub(crate) fn entry_point(&self) -> Option<u32> {
self.entry_point
}
@@ -455,79 +447,14 @@ impl Graph {
if upper.dense_of_slot.len() < span {
grow_amortized(&mut upper.dense_of_slot, span, ABSENT_DENSE);
}
- let dense = match upper.free.pop() {
- Some(dense) => dense,
- None => {
- let dense = upper.lengths.len() as u32;
- upper.lengths.push(0);
- upper
- .neighbors
- .resize(upper.lengths.len() * UPPER_LEVEL_NEIGHBOR_CAP, 0);
- dense
- }
- };
- upper.lengths[dense as usize] = 0;
+ let dense = upper.lengths.len() as u32;
+ upper.lengths.push(0);
+ upper
+ .neighbors
+ .resize(upper.lengths.len() * UPPER_LEVEL_NEIGHBOR_CAP, 0);
upper.dense_of_slot[slot as usize] = dense;
}
- fn release_node(&mut self, slot: u32, level: usize) {
- self.zero_lengths[slot as usize] = 0;
- for layer in 1..=level {
- let upper = &mut self.upper[layer - 1];
- let dense = upper.dense_of_slot[slot as usize];
- if dense == ABSENT_DENSE {
- continue;
- }
- upper.dense_of_slot[slot as usize] = ABSENT_DENSE;
- upper.lengths[dense as usize] = 0;
- upper.free.push(dense);
- }
- }
-
- fn detach(&mut self, slot: u32) {
- let Some(level) = self.level_of(slot).map(usize::from) else {
- return;
- };
- self.node_levels[slot as usize] = ABSENT_LEVEL;
- for layer in 0..=level {
- let mut buffer = [0u32; LEVEL_ZERO_NEIGHBOR_CAP];
- let count = {
- let list = self.level_neighbors(slot, layer);
- buffer[..list.len()].copy_from_slice(list);
- list.len()
- };
- for &neighbor in &buffer[..count] {
- self.unlink(neighbor, layer, slot);
- }
- }
- self.release_node(slot, level);
- if self.entry_point == Some(slot) {
- self.entry_point = self.highest_slot();
- }
- }
-
- fn unlink(&mut self, slot: u32, level: usize, other: u32) {
- if !self.holds_level(slot, level) {
- return;
- }
- let mut buffer = [0u32; LEVEL_ZERO_NEIGHBOR_CAP];
- let count = {
- let list = self.level_neighbors(slot, level);
- if !list.contains(&other) {
- return;
- }
- let mut count = 0;
- for &value in list {
- if value != other {
- buffer[count] = value;
- count += 1;
- }
- }
- count
- };
- self.write_level_list(slot, level, &buffer[..count]);
- }
-
fn link_back(&mut self, slot: u32, level: usize, other: u32, cap: usize, arena: &VectorArena) {
if !self.holds_level(slot, level) {
return;
@@ -563,19 +490,6 @@ impl Graph {
self.write_level_list(slot, level, &kept[..kept_count]);
}
- fn highest_slot(&self) -> Option<u32> {
- let mut best: Option<(u16, u32)> = None;
- for (index, &level) in self.node_levels.iter().enumerate() {
- if level == ABSENT_LEVEL {
- continue;
- }
- if best.is_none_or(|(best_level, _)| level > best_level) {
- best = Some((level, index as u32));
- }
- }
- best.map(|(_, slot)| slot)
- }
-
fn next_level(&mut self) -> usize {
let mut state = LEVEL_SEED ^ self.insert_ordinal;
self.insert_ordinal = self.insert_ordinal.wrapping_add(1);
@@ -1290,11 +1204,8 @@ mod tests {
}
fn apply_upsert(arena: &mut VectorArena, graph: &mut Graph, key: &str, vector: &[f32]) {
- match arena.upsert(key, vector).unwrap() {
- UpsertOutcome::NewSlot(slot) => graph.insert(slot, arena),
- UpsertOutcome::RecycledSlot(slot) | UpsertOutcome::ReplacedInPlace(slot) => {
- graph.reinsert(slot, arena);
- }
+ if let UpsertOutcome::NewSlot(slot) = arena.upsert(key, vector).unwrap() {
+ graph.insert(slot, arena);
}
}
@@ -1911,13 +1822,14 @@ mod tests {
}
#[test]
- fn reinsert_moves_key_to_its_new_position() {
+ fn replacement_moves_key_to_its_new_position() {
let (mut arena, mut graph) = build_fixture(300, 16, 0x4000_0000);
let old_vector = arena.vector_values(7);
let new_vector = seeded_unit_vector(0x4222_0000, 16);
let outcome = arena.upsert("key-7", &new_vector).unwrap();
- assert_eq!(outcome, UpsertOutcome::ReplacedInPlace(7));
- graph.reinsert(7, &arena);
+ assert_eq!(outcome, UpsertOutcome::NewSlot(300));
+ graph.insert(300, &arena);
+ assert_eq!(arena.vector_values(7), old_vector);
let new_query = arena.pack_query(&new_vector).unwrap();
for exact in [false, true] {
let top = search(
@@ -1931,7 +1843,7 @@ mod tests {
assert!(top[0].distance < 1e-3);
}
let old_query = arena.pack_query(&old_vector).unwrap();
- let moved_distance = arena.distance_to_query(old_query.as_query(), 7);
+ let moved_distance = arena.distance_to_query(old_query.as_query(), 300);
assert!(moved_distance > 0.3);
let near_old = search(
&graph,
@@ -1954,14 +1866,14 @@ mod tests {
}
#[test]
- fn reinsert_of_entry_point_keeps_graph_searchable() {
+ fn replacement_of_entry_point_keeps_graph_searchable() {
let (mut arena, mut graph) = build_fixture(300, 16, 0x5000_0000);
let entry_slot = graph.entry_point().unwrap();
let entry_key = arena.key_of_slot(entry_slot).unwrap().to_string();
let new_vector = seeded_unit_vector(0x5222_0000, 16);
let outcome = arena.upsert(&entry_key, &new_vector).unwrap();
- assert_eq!(outcome, UpsertOutcome::ReplacedInPlace(entry_slot));
- graph.reinsert(entry_slot, &arena);
+ assert_eq!(outcome, UpsertOutcome::NewSlot(300));
+ graph.insert(300, &arena);
assert!(graph.entry_point().is_some());
let query = arena.pack_query(&new_vector).unwrap();
let top = search(&graph, &arena, &query, &params(Some(1), None, false), None);
@@ -1977,13 +1889,13 @@ mod tests {
}
#[test]
- fn reinsert_recycled_slot_serves_the_new_key() {
+ fn insert_after_deletion_serves_the_new_key() {
let (mut arena, mut graph) = build_fixture(300, 16, 0x6000_0000);
assert_eq!(arena.remove("key-3"), Some(3));
let new_vector = seeded_unit_vector(0x6222_0000, 16);
let outcome = arena.upsert("fresh", &new_vector).unwrap();
- assert_eq!(outcome, UpsertOutcome::RecycledSlot(3));
- graph.reinsert(3, &arena);
+ assert_eq!(outcome, UpsertOutcome::NewSlot(300));
+ graph.insert(300, &arena);
let query = arena.pack_query(&new_vector).unwrap();
for exact in [false, true] {
let top = search(&graph, &arena, &query, &params(Some(1), None, exact), None);
@@ -2002,13 +1914,13 @@ mod tests {
}
#[test]
- fn reinsert_of_the_only_node_resets_entry_point() {
+ fn replacement_of_the_only_node_preserves_the_routing_node() {
let mut arena = VectorArena::new(8).unwrap();
arena.upsert("solo", &axis_vector(8, 0)).unwrap();
let mut graph = Graph::rebuild(&arena);
assert_eq!(graph.entry_point(), Some(0));
arena.upsert("solo", &axis_vector(8, 3)).unwrap();
- graph.reinsert(0, &arena);
+ graph.insert(1, &arena);
assert_eq!(graph.entry_point(), Some(0));
let query = arena.pack_query(&axis_vector(8, 3)).unwrap();
let found = search(&graph, &arena, &query, &params(Some(5), None, false), None);
@@ -2503,19 +2415,28 @@ mod tests {
}
#[test]
- fn repeated_reinserts_shrink_levels_yet_keep_every_node_reachable() {
+ fn repeated_replacements_preserve_levels_and_keep_live_nodes_reachable() {
let (mut arena, mut graph) = build_fixture(600, 16, 0xB000_0000);
- let mut stale_seen = false;
+ let original_levels: Vec<_> = graph
+ .slots()
+ .map(|slot| (slot, graph.level_of(slot)))
+ .collect();
+ let key = arena
+ .key_of_slot(graph.entry_point().unwrap())
+ .unwrap()
+ .to_string();
for round in 0..40u64 {
- let victim = graph.entry_point().unwrap();
- let key = arena.key_of_slot(victim).unwrap().to_string();
let vector = seeded_unit_vector(0xB100_0000 + round, 16);
+ let slot = arena.slot_count() as u32;
assert_eq!(
arena.upsert(&key, &vector).unwrap(),
- UpsertOutcome::ReplacedInPlace(victim)
+ UpsertOutcome::NewSlot(slot)
);
- graph.reinsert(victim, &arena);
- stale_seen |= stale_downward_edge_exists(&graph);
+ graph.insert(slot, &arena);
+ assert!(!stale_downward_edge_exists(&graph));
+ for &(slot, level) in &original_levels {
+ assert_eq!(graph.level_of(slot), level);
+ }
assert_graph_invariants(&graph);
let reloaded = Graph::from_parts(
graph.entry_point(),
@@ -2537,7 +2458,6 @@ mod tests {
);
assert_eq!(everything.len(), 600);
}
- assert!(stale_seen);
let rebuilt = Graph::from_parts(
graph.entry_point(),
graph_parts(&graph),
diff --git a/rust/crates/vecdb/src/kernel.rs b/rust/crates/vecdb/src/kernel.rs
index b83ed06f6d..1ab00e88f1 100644
--- a/rust/crates/vecdb/src/kernel.rs
+++ b/rust/crates/vecdb/src/kernel.rs
@@ -53,6 +53,29 @@ pub(crate) enum VectorPayload<'a> {
}
impl VectorPayload<'_> {
+ pub(crate) fn bitwise_eq(self, other: Self) -> bool {
+ match (self, other) {
+ (Self::F32(first), Self::F32(second)) => {
+ first.len() == second.len()
+ && first
+ .iter()
+ .zip(second)
+ .all(|(a, b)| a.to_bits() == b.to_bits())
+ }
+ (
+ Self::I8 {
+ scale: first_scale,
+ values: first,
+ },
+ Self::I8 {
+ scale: second_scale,
+ values: second,
+ },
+ ) => first_scale.to_bits() == second_scale.to_bits() && first == second,
+ _ => false,
+ }
+ }
+
pub(crate) fn storage(&self) -> StorageKind {
match self {
Self::F32(_) => StorageKind::F32,
diff --git a/rust/crates/vecdb/src/log.rs b/rust/crates/vecdb/src/log.rs
index 307900d2d0..f389b35fed 100644
--- a/rust/crates/vecdb/src/log.rs
+++ b/rust/crates/vecdb/src/log.rs
@@ -821,7 +821,7 @@ fn decode_header(bytes: &[u8; HEADER_LEN]) -> Result<Header, VecDbError> {
})
}
-fn validate_entry(
+pub(crate) fn validate_entry(
entry: &LogEntry<'_>,
dims: usize,
storage: StorageKind,
@@ -902,6 +902,20 @@ fn encoded_attr_len(attr: &Attribute) -> usize {
}
}
+pub(crate) fn add_record_len(
+ key: &str,
+ dims: usize,
+ storage: StorageKind,
+ attrs: &[Attribute],
+) -> u64 {
+ (RECORD_PREFIX_LEN
+ + key.len()
+ + vector_payload_len(storage, dims)
+ + 1
+ + attrs.iter().map(encoded_attr_len).sum::<usize>()
+ + RECORD_CRC_LEN) as u64
+}
+
fn encode_record_into(buffer: &mut Vec<u8>, entry: &LogEntry<'_>) {
let start = buffer.len();
match entry {
@@ -1024,6 +1038,48 @@ mod tests {
use super::*;
use tempfile::TempDir;
+ #[test]
+ fn add_record_size_matches_encoded_bytes_for_both_storage_formats() {
+ let vector = seeded_vector(9, 32);
+ let attrs = vec![
+ Attribute {
+ name: "name".into(),
+ value: AttrValue::Str("影像".into()),
+ },
+ Attribute {
+ name: "flag".into(),
+ value: AttrValue::Bool(true),
+ },
+ Attribute {
+ name: "count".into(),
+ value: AttrValue::I64(-23),
+ },
+ Attribute {
+ name: "score".into(),
+ value: AttrValue::F64(0.125),
+ },
+ ];
+ for stored in [
+ StoredVector::F32(vector.clone()),
+ StoredVector::quantize(&vector),
+ ] {
+ for attrs in [&[][..], attrs.as_slice()] {
+ let payload = stored.as_payload();
+ let entry = LogEntry::Add {
+ key: "photo-影像",
+ vector: payload,
+ attrs,
+ };
+ let mut encoded = Vec::new();
+ encode_record_into(&mut encoded, &entry);
+ assert_eq!(
+ add_record_len("photo-影像", 32, payload.storage(), attrs),
+ encoded.len() as u64
+ );
+ }
+ }
+ }
+
fn seeded_vector(seed: u64, dims: usize) -> Vec<f32> {
let mut state = seed;
(0..dims)
diff --git a/rust/crates/vecdb/src/snapshot.rs b/rust/crates/vecdb/src/snapshot.rs
index b31f9ed2f8..8e5a095794 100644
--- a/rust/crates/vecdb/src/snapshot.rs
+++ b/rust/crates/vecdb/src/snapshot.rs
@@ -359,30 +359,29 @@ mod tests {
for index in [3, 41, 87, 150, 199] {
arena.remove(&format!("key-{index}")).unwrap();
}
- let mut stale_seen = false;
for round in 0..40u64 {
let victim = graph
.entry_point()
.filter(|&slot| arena.is_alive(slot))
.unwrap_or_else(|| arena.live_slots().next().unwrap());
let key = arena.key_of_slot(victim).unwrap().to_string();
+ let slot = arena.slot_count() as u32;
assert_eq!(
arena
.upsert(&key, &seeded_unit_vector(0x00A2_0000 + round, dims))
.unwrap(),
- UpsertOutcome::ReplacedInPlace(victim)
+ UpsertOutcome::NewSlot(slot)
);
- graph.reinsert(victim, &arena);
- stale_seen |= stale_downward_edge_exists(&graph);
+ graph.insert(slot, &arena);
+ assert!(!stale_downward_edge_exists(&graph));
}
- let UpsertOutcome::RecycledSlot(recycled) = arena
- .upsert("recycled", &seeded_unit_vector(0x00A3_0000, dims))
+ let UpsertOutcome::NewSlot(slot) = arena
+ .upsert("fresh", &seeded_unit_vector(0x00A3_0000, dims))
.unwrap()
else {
- panic!("expected a recycled slot");
+ panic!("expected a new slot");
};
- graph.reinsert(recycled, &arena);
- assert!(stale_seen);
+ graph.insert(slot, &arena);
(arena, graph)
}
diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rs
index 32423db79d..f11aa7be35 100644
--- a/rust/crates/vecdb/src/store.rs
+++ b/rust/crates/vecdb/src/store.rs
@@ -14,8 +14,9 @@ use super::graph::{Graph, search as graph_search, search_stored};
use super::kernel::{StoredVector, VectorPayload, quantize_for};
use super::lock::WriterLock;
use super::log::{
- HEADER_LEN, Log, LogCheckpoint, LogEntry, LogRecord, header_generation, open_writer_file,
- remove_if_present, remove_stale_temp_sibling, sync_parent_dir,
+ HEADER_LEN, Log, LogCheckpoint, LogEntry, LogRecord, add_record_len, header_generation,
+ open_writer_file, remove_if_present, remove_stale_temp_sibling, sync_parent_dir,
+ validate_entry,
};
use super::snapshot::{
LoadedSnapshot, load_snapshot, remove_snapshot, snapshot_path, write_snapshot,
@@ -26,6 +27,7 @@ const SNAPSHOT_HARD_CAP: usize = 5000;
const SNAPSHOT_QUIET_THRESHOLD: usize = 1000;
const SNAPSHOT_QUIET_GAP: Duration = Duration::from_secs(10);
const COMPACTION_MIN_DEAD_RECORDS: u64 = 64;
+const COMPACTION_MIN_RETIRED_VECTORS: usize = 64;
const COMPACTION_DEAD_RATIO: u64 = 10;
const COMPACTION_BATCH_SIZE: usize = 1000;
const HANDOFF_WAIT_ROUNDS: u32 = 750;
@@ -89,12 +91,31 @@ struct SearchState {
graph: Option<Graph>,
attrs: AttrTable,
total_records: u64,
+ live_log_bytes: u64,
}
impl SearchState {
fn search_graph(&self) -> &Graph {
self.graph.as_ref().unwrap_or(&PENDING_SEARCH_GRAPH)
}
+
+ fn entry_matches(&self, key: &str, vector: VectorPayload<'_>, attrs: &[Attribute]) -> bool {
+ self.arena.slot_of_key(key).is_some_and(|slot| {
+ self.arena.matches_payload(slot, vector)
+ && attrs_match(self.attrs.get(slot).unwrap_or(&[]), attrs)
+ })
+ }
+}
+
+fn attrs_match(first: &[Attribute], second: &[Attribute]) -> bool {
+ first.len() == second.len()
+ && first.iter().zip(second).all(|(a, b)| {
+ a.name == b.name
+ && match (&a.value, &b.value) {
+ (AttrValue::F64(a), AttrValue::F64(b)) => a.to_bits() == b.to_bits(),
+ _ => a.value == b.value,
+ }
+ })
}
#[derive(Default)]
@@ -167,9 +188,7 @@ impl AttrTable {
fn slot_of_outcome(outcome: UpsertOutcome) -> u32 {
match outcome {
- UpsertOutcome::NewSlot(slot)
- | UpsertOutcome::RecycledSlot(slot)
- | UpsertOutcome::ReplacedInPlace(slot) => slot,
+ UpsertOutcome::NewSlot(slot) | UpsertOutcome::Unchanged(slot) => slot,
}
}
@@ -491,11 +510,16 @@ impl VecDb {
let payload = quantized
.as_ref()
.map_or(VectorPayload::F32(vector), StoredVector::as_payload);
- state.log.append(&[LogEntry::Add {
+ let entry = LogEntry::Add {
key,
vector: payload,
attrs,
- }])?;
+ };
+ validate_entry(&entry, self.shared.dims, self.shared.storage)?;
+ if self.shared.state_read().entry_matches(key, payload, attrs) {
+ return Ok(());
+ }
+ state.log.append(&[entry])?;
apply_add(&self.shared, key, payload, attrs)?;
apply_policy(&self.shared, &mut half, 1, now)
}
@@ -558,20 +582,51 @@ impl VecDb {
Some(stored) => stored[index].as_payload(),
None => VectorPayload::F32(&vectors[index]),
};
- let records: Vec<LogEntry<'_>> = keys
+ let scheduled = {
+ let st = self.shared.state_read();
+ let mut previous = HashMap::<&str, usize>::new();
+ let mut scheduled = Vec::new();
+ for (index, key) in keys.iter().enumerate() {
+ let entry = LogEntry::Add {
+ key,
+ vector: payload_of(index),
+ attrs: attrs_of(index),
+ };
+ validate_entry(&entry, self.shared.dims, self.shared.storage)?;
+ let unchanged = match previous.insert(key, index) {
+ Some(prior) => {
+ payload_of(prior).bitwise_eq(payload_of(index))
+ && attrs_match(attrs_of(prior), attrs_of(index))
+ }
+ None => st.entry_matches(key, payload_of(index), attrs_of(index)),
+ };
+ if !unchanged {
+ scheduled.push(index);
+ }
+ }
+ scheduled
+ };
+ if scheduled.is_empty() {
+ return Ok(());
+ }
+ let records: Vec<LogEntry<'_>> = scheduled
.iter()
- .enumerate()
- .map(|(index, key)| LogEntry::Add {
- key,
+ .map(|&index| LogEntry::Add {
+ key: &keys[index],
vector: payload_of(index),
attrs: attrs_of(index),
})
.collect();
state.log.append(&records)?;
- for (index, key) in keys.iter().enumerate() {
- apply_add(&self.shared, key, payload_of(index), attrs_of(index))?;
+ for &index in &scheduled {
+ apply_add(
+ &self.shared,
+ &keys[index],
+ payload_of(index),
+ attrs_of(index),
+ )?;
}
- apply_policy(&self.shared, &mut half, keys.len(), now)
+ apply_policy(&self.shared, &mut half, scheduled.len(), now)
}
pub fn remove(&self, key: &str) -> Result<bool, VecDbError> {
@@ -805,6 +860,7 @@ impl VecDb {
st.graph = Some(Graph::new());
st.attrs.reset();
st.total_records = 0;
+ st.live_log_bytes = 0;
}
half.compaction_retry_at_dead = 0;
half.mode = WriterMode::Active(WriterState {
@@ -1022,6 +1078,7 @@ fn close_live_instance(shared: &Arc<Shared>) -> Result<(), VecDbError> {
st.graph = Some(Graph::new());
st.attrs.reset();
st.total_records = 0;
+ st.live_log_bytes = 0;
}
drop(lock);
if let Some(key) = &shared.registry_key
@@ -1053,9 +1110,9 @@ fn apply_add(
graph,
attrs: attr_table,
total_records,
+ live_log_bytes,
} = &mut *st;
- let outcome = arena.upsert_payload(key, vector)?;
- attr_table.set(slot_of_outcome(outcome), attrs);
+ let outcome = upsert_entry(arena, attr_table, live_log_bytes, key, vector, attrs)?;
if let Some(graph) = graph {
apply_outcome(graph, arena, outcome);
}
@@ -1065,36 +1122,97 @@ fn apply_add(
fn apply_remove(shared: &Shared, key: &str) {
let mut st = shared.state_write();
- if let Some(slot) = st.arena.remove(key) {
- st.attrs.clear(slot);
- }
+ let SearchState {
+ arena,
+ attrs,
+ live_log_bytes,
+ ..
+ } = &mut *st;
+ remove_entry(arena, attrs, live_log_bytes, key);
st.total_records += 1;
}
+fn upsert_entry(
+ arena: &mut VectorArena,
+ attrs: &mut AttrTable,
+ live_log_bytes: &mut u64,
+ key: &str,
+ vector: VectorPayload<'_>,
+ new_attrs: &[Attribute],
+) -> Result<UpsertOutcome, VecDbError> {
+ let previous = arena.slot_of_key(key);
+ let outcome = arena.upsert_payload(key, vector)?;
+ if let Some(slot) = previous {
+ *live_log_bytes -= add_record_len(
+ key,
+ arena.dims(),
+ arena.storage_kind(),
+ attrs.get(slot).unwrap_or(&[]),
+ );
+ attrs.clear(slot);
+ }
+ attrs.set(slot_of_outcome(outcome), new_attrs);
+ *live_log_bytes += add_record_len(key, arena.dims(), arena.storage_kind(), new_attrs);
+ Ok(outcome)
+}
+
+fn remove_entry(
+ arena: &mut VectorArena,
+ attrs: &mut AttrTable,
+ live_log_bytes: &mut u64,
+ key: &str,
+) {
+ if let Some(slot) = arena.remove(key) {
+ *live_log_bytes -= add_record_len(
+ key,
+ arena.dims(),
+ arena.storage_kind(),
+ attrs.get(slot).unwrap_or(&[]),
+ );
+ attrs.clear(slot);
+ }
+}
+
fn apply_policy(
shared: &Shared,
half: &mut WriterHalf,
count: usize,
now: Instant,
) -> Result<(), VecDbError> {
- let (live, total) = {
+ let (dead, retired, slots, live_log_bytes) = {
let st = shared.state_read();
- (st.arena.live_count() as u64, st.total_records)
+ (
+ st.total_records
+ .saturating_sub(st.arena.live_count() as u64),
+ st.arena.dead_count(),
+ st.arena.slot_count(),
+ st.live_log_bytes,
+ )
};
- let (quiet, mutations) = {
+ let (quiet, mutations, log_bytes) = {
let state = active_state(half)?;
state.mutations_since_snapshot += count;
let quiet = state
.last_write
.is_some_and(|previous| now.duration_since(previous) >= SNAPSHOT_QUIET_GAP);
state.last_write = Some(now);
- (quiet, state.mutations_since_snapshot)
+ (
+ quiet,
+ state.mutations_since_snapshot,
+ state
+ .log
+ .current_end_offset()
+ .saturating_sub(HEADER_LEN as u64),
+ )
};
- let dead = total.saturating_sub(live);
- if dead >= COMPACTION_MIN_DEAD_RECORDS
- && dead.saturating_mul(COMPACTION_DEAD_RATIO) >= total
- && dead >= half.compaction_retry_at_dead
- {
+ let retired_budget_reached = retired >= COMPACTION_MIN_RETIRED_VECTORS
+ && (retired as u64).saturating_mul(COMPACTION_DEAD_RATIO) >= slots as u64;
+ let log_budget_reached = dead >= COMPACTION_MIN_DEAD_RECORDS
+ && log_bytes
+ .saturating_sub(live_log_bytes)
+ .saturating_mul(COMPACTION_DEAD_RATIO)
+ >= log_bytes;
+ if (retired_budget_reached || log_budget_reached) && dead >= half.compaction_retry_at_dead {
match compact(shared, half) {
Ok(()) => half.compaction_retry_at_dead = 0,
Err(error) => {
@@ -1164,25 +1282,31 @@ fn compact(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecDbError> {
let _ = std::fs::remove_file(&temp_path);
(mutations_since_snapshot, old_checkpoint)
} else {
- let live_records = {
+ let (live_records, rebuild) = {
let mut st = shared.state_write();
let SearchState {
arena,
graph,
attrs,
total_records,
+ ..
} = &mut *st;
- attrs.repack(arena.live_slots());
- arena.compact_in_place();
- *graph = None;
+ let rebuild = arena.dead_count() > 0;
+ if rebuild {
+ attrs.repack(arena.live_slots());
+ arena.compact_in_place();
+ *graph = None;
+ }
*total_records = arena.live_count() as u64;
- *total_records as usize
- };
- let graph = {
- let st = shared.state_read();
- Graph::rebuild(&st.arena)
+ (*total_records as usize, rebuild)
};
- shared.state_write().graph = Some(graph);
+ if rebuild {
+ let graph = {
+ let st = shared.state_read();
+ Graph::rebuild(&st.arena)
+ };
+ shared.state_write().graph = Some(graph);
+ }
(live_records, compacted_checkpoint)
};
let restored =
@@ -1224,6 +1348,7 @@ fn restore_writer_mode(
graph: Some(graph),
attrs: replayed.attrs,
total_records: replayed.total_records,
+ live_log_bytes: replayed.live_log_bytes,
};
half.compaction_retry_at_dead = 0;
}
@@ -1269,6 +1394,7 @@ fn recover_after_failed_reset(
st.graph = Some(Graph::new());
st.attrs.reset();
st.total_records = 0;
+ st.live_log_bytes = 0;
half.compaction_retry_at_dead = 0;
}
}
@@ -1312,6 +1438,7 @@ fn build_writer(
graph,
attrs,
total_records,
+ live_log_bytes,
tail_records,
..
} = replayed;
@@ -1348,6 +1475,7 @@ fn build_writer(
graph: Some(graph),
attrs,
total_records,
+ live_log_bytes,
},
)))
}
@@ -1368,6 +1496,7 @@ fn build_writer(
graph: None,
attrs,
total_records,
+ live_log_bytes,
},
),
snapshot_stale: total_records > 0 || snapshot_present,
@@ -1463,6 +1592,7 @@ fn build_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Sha
graph,
attrs,
total_records,
+ live_log_bytes,
recoverable_end,
..
} = replayed;
@@ -1473,6 +1603,7 @@ fn build_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Sha
graph,
attrs,
total_records,
+ live_log_bytes,
recoverable_end,
))
}
@@ -1485,6 +1616,7 @@ fn empty_read_only(path: &Path, dims: usize, storage: StorageKind) -> Result<Sha
AttrTable::default(),
0,
0,
+ 0,
))
}
@@ -1494,6 +1626,7 @@ fn read_only_shared(
graph: Graph,
attrs: AttrTable,
total_records: u64,
+ live_log_bytes: u64,
log_bytes: u64,
) -> Shared {
Shared {
@@ -1511,6 +1644,7 @@ fn read_only_shared(
graph: Some(graph),
attrs,
total_records,
+ live_log_bytes,
}),
}
}
@@ -1523,9 +1657,9 @@ fn approximate_memory_bytes(state: &SearchState) -> usize {
.filter_map(|slot| state.arena.key_of_slot(slot))
.map(|key| KEY_TABLE_COPIES * (key.len() + KEY_ENTRY_OVERHEAD_BYTES))
.sum();
- let free_list_bytes = state.arena.dead_count() * size_of::<u32>();
let graph_bytes = state.graph.as_ref().map_or(0, Graph::memory_bytes);
- vector_bytes + key_bytes + free_list_bytes + graph_bytes + state.attrs.memory_bytes()
+ let retired_key_slots = state.arena.dead_count() * size_of::<Box<str>>();
+ vector_bytes + key_bytes + retired_key_slots + graph_bytes + state.attrs.memory_bytes()
}
struct ReplayedState {
@@ -1533,6 +1667,7 @@ struct ReplayedState {
graph: Option<Graph>,
attrs: AttrTable,
total_records: u64,
+ live_log_bytes: u64,
recoverable_end: u64,
tail_records: u64,
}
@@ -1568,6 +1703,7 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,
let mut graph: Option<Graph> = None;
let mut tail_records = 0u64;
let mut total_records = 0u64;
+ let mut live_log_bytes = 0u64;
let recoverable_end;
{
let mut scanner = log.scan()?;
@@ -1583,16 +1719,20 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,
vector,
attrs: record_attrs,
} => {
- let outcome = arena.upsert_payload(&key, vector.as_payload())?;
- attrs.set(slot_of_outcome(outcome), &record_attrs);
+ let outcome = upsert_entry(
+ &mut arena,
+ &mut attrs,
+ &mut live_log_bytes,
+ &key,
+ vector.as_payload(),
+ &record_attrs,
+ )?;
if let Some(graph) = &mut graph {
apply_outcome(graph, &arena, outcome);
}
}
LogRecord::Tombstone { key } => {
- if let Some(slot) = arena.remove(&key) {
- attrs.clear(slot);
- }
+ remove_entry(&mut arena, &mut attrs, &mut live_log_bytes, &key);
}
}
}
@@ -1623,6 +1763,7 @@ fn replay(log: &mut Log, dims: usize, mode: ReplayMode) -> Result<ReplayedState,
graph,
attrs,
total_records,
+ live_log_bytes,
recoverable_end,
tail_records,
})
@@ -1651,11 +1792,8 @@ fn attach_snapshot_graph(
}
fn apply_outcome(graph: &mut Graph, arena: &VectorArena, outcome: UpsertOutcome) {
- match outcome {
- UpsertOutcome::NewSlot(slot) => graph.insert(slot, arena),
- UpsertOutcome::RecycledSlot(slot) | UpsertOutcome::ReplacedInPlace(slot) => {
- graph.reinsert(slot, arena);
- }
+ if let UpsertOutcome::NewSlot(slot) = outcome {
+ graph.insert(slot, arena);
}
}
@@ -1750,6 +1888,316 @@ mod tests {
const ADD_RECORD_LEN: u64 = 3 + 5 + (DIMS as u64) * 4 + 1 + 4;
const TOMBSTONE_RECORD_LEN: u64 = 3 + 5 + 4;
+ #[test]
+ fn unchanged_writes_preserve_the_log_graph_and_snapshot() {
+ for storage in [StorageKind::F32, StorageKind::I8] {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();
+ let vector = seeded_unit_vector(1, 32);
+ let attrs = sample_attrs(7);
+ db.add_with_attrs("key", &vector, &attrs).unwrap();
+ db.flush().unwrap();
+ let before = db.stats().unwrap();
+ let snapshot = fs::read(snapshot_path(&path)).unwrap();
+ db.add_with_attrs("key", &vector, &attrs).unwrap();
+ db.bulk_add_with_attrs(
+ &["key".into(), "key".into()],
+ &[vector.clone(), vector.clone()],
+ &[Some(attrs.clone()), Some(attrs.clone())],
+ )
+ .unwrap();
+ db.flush().unwrap();
+ assert_eq!(db.stats().unwrap(), before);
+ assert_eq!(fs::read(snapshot_path(&path)).unwrap(), snapshot);
+ let invalid = vec![attr("", AttrValue::Bool(true))];
+ assert!(matches!(
+ db.add_with_attrs("key", &vector, &invalid),
+ Err(VecDbError::InvalidAttributes(_))
+ ));
+ assert_eq!(db.stats().unwrap(), before);
+ }
+ }
+
+ #[test]
+ fn unchanged_i8_payloads_skip_writes_but_scale_changes_append_versions() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = VecDb::open(&path, 32, Some(StorageKind::I8)).unwrap();
+ let mut vector = vec![0.0; 32];
+ vector[0] = 1.0;
+ vector[1] = 0.1;
+ db.add("key", &vector).unwrap();
+ let before = db.stats().unwrap();
+ vector[1] += 0.00001;
+ db.add("key", &vector).unwrap();
+ assert_eq!(db.stats().unwrap(), before);
+ let previous = db.shared.state_read().arena.stored_vector(0);
+ for value in &mut vector {
+ *value *= 2.0;
+ }
+ db.add("key", &vector).unwrap();
+ let state = db.shared.state_read();
+ assert_eq!(state.arena.slot_of_key("key"), Some(1));
+ assert_eq!(state.arena.stored_vector(0), previous);
+ assert_ne!(state.arena.stored_vector(1), previous);
+ }
+
+ #[test]
+ fn f32_replacement_preserves_distinct_stored_bits() {
+ let dir = TempDir::new().unwrap();
+ let db = open_writer(&dir.path().join("db"));
+ let mut vector = basis_vector(0);
+ db.add("key", &vector).unwrap();
+ vector[1] = -0.0;
+ db.add("key", &vector).unwrap();
+ let state = db.shared.state_read();
+ assert_eq!(state.arena.slot_of_key("key"), Some(1));
+ assert_eq!(state.arena.vector_values(0)[1].to_bits(), 0.0f32.to_bits());
+ assert_eq!(
+ state.arena.vector_values(1)[1].to_bits(),
+ (-0.0f32).to_bits()
+ );
+ }
+
+ #[test]
+ fn batch_noops_follow_prior_entries_and_replay_the_same_versions() {
+ for storage in [StorageKind::F32, StorageKind::I8] {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();
+ let first = seeded_unit_vector(1, 32);
+ let second = seeded_unit_vector(2, 32);
+ let attrs = sample_attrs(7);
+ db.add("key", &first).unwrap();
+ db.flush().unwrap();
+ db.bulk_add_with_attrs(
+ &vec!["key".to_string(); 5],
+ &[first.clone(), first.clone(), second, first.clone(), first],
+ &[None, Some(attrs.clone()), None, None, None],
+ )
+ .unwrap();
+ let verify = |db: &VecDb| {
+ let state = db.shared.state_read();
+ assert_eq!(state.total_records, 4);
+ assert_eq!(state.arena.slot_of_key("key"), Some(2));
+ assert_eq!(state.arena.dead_count(), 2);
+ assert_eq!(state.graph.as_ref().unwrap().insert_ordinal(), 3);
+ assert_eq!(
+ state.live_log_bytes,
+ add_record_len("key", 32, storage, &[])
+ );
+ assert!(state.attrs.get(0).is_none());
+ assert!(state.attrs.get(1).is_none());
+ assert!(state.attrs.get(2).is_none());
+ };
+ verify(&db);
+ drop(db);
+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();
+ verify(&reopened);
+ drop(reopened);
+ let read_only = VecDb::open_read_only(&path, 32, Some(storage)).unwrap();
+ verify(&read_only);
+ }
+ }
+
+ #[test]
+ fn attribute_only_compaction_preserves_graph_slots_and_levels() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ let entries = bulk_entries(0, 100, 733);
+ bulk_add(&db, &entries).unwrap();
+ db.flush().unwrap();
+ let generation = generation_of(&path);
+ let loaded = load_snapshot(&path, generation).unwrap();
+ let graph = Graph::from_parts(loaded.entry_point, loaded.parts, 100, loaded.insert_ordinal)
+ .unwrap();
+ for (index, (key, vector)) in entries.iter().take(64).enumerate() {
+ db.add_with_attrs(key, vector, &sample_attrs(index as i64))
+ .unwrap();
+ }
+ assert_ne!(generation_of(&path), generation);
+ let state = db.shared.state_read();
+ assert_eq!(state.arena.dead_count(), 0);
+ assert_eq!(state.total_records, 100);
+ assert_eq!(state.graph.as_ref().unwrap().insert_ordinal(), 100);
+ test_support::assert_identical_graphs(&graph, state.graph.as_ref().unwrap());
+ for (index, (key, _)) in entries.iter().enumerate() {
+ assert_eq!(state.arena.slot_of_key(key), Some(index as u32));
+ }
+ drop(state);
+ let compacted_bytes = db.stats().unwrap().log_bytes - HEADER_LEN as u64;
+ assert_eq!(db.shared.state_read().live_log_bytes, compacted_bytes);
+ drop(db);
+ let reopened = open_writer(&path);
+ assert_eq!(reopened.shared.state_read().live_log_bytes, compacted_bytes);
+ assert_eq!(
+ reopened.get_attrs("key-63").unwrap(),
+ Some(sample_attrs(63))
+ );
+ }
+
+ #[test]
+ fn maintenance_accounts_for_log_bytes_and_retired_slots_separately() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ let (keys, vectors): (Vec<_>, Vec<_>) = bulk_entries(0, 1000, 888).into_iter().unzip();
+ let large_attrs = vec![attr("padding", AttrValue::Str("x".repeat(1024)))];
+ let attrs: Vec<_> = (0..1000)
+ .map(|index| (index >= 200).then(|| large_attrs.clone()))
+ .collect();
+ db.bulk_add_with_attrs(&keys, &vectors, &attrs).unwrap();
+ let generation = generation_of(&path);
+ let small_attrs = vec![Some(vec![attr("flag", AttrValue::Bool(true))]); 112];
+ db.bulk_add_with_attrs(&keys[..112], &vectors[..112], &small_attrs)
+ .unwrap();
+ assert_eq!(generation_of(&path), generation);
+ assert_eq!(db.stats().unwrap().dead_count, 112);
+ assert_eq!(db.shared.state_read().arena.dead_count(), 0);
+ db.bulk_remove(&keys[..99]).unwrap();
+ assert_eq!(generation_of(&path), generation);
+ assert_eq!(db.shared.state_read().arena.dead_count(), 99);
+ db.remove(&keys[99]).unwrap();
+ assert_ne!(generation_of(&path), generation);
+ assert_eq!(db.shared.state_read().arena.dead_count(), 0);
+ assert_eq!(db.stats().unwrap().dead_count, 0);
+ assert_eq!(db.len().unwrap(), 900);
+ }
+
+ fn churn_vectors(clusters: usize) -> (Vec<String>, Vec<Vec<f32>>) {
+ let mut seed = 2;
+ let keys = (0..3000).map(|index| format!("key-{index}")).collect();
+ let vectors = (0..3000)
+ .map(|index| {
+ let mut vector: Vec<_> = (0..32)
+ .map(|_| {
+ ((splitmix64(&mut seed) >> 40) as f32 / (1u64 << 24) as f32 * 2.0 - 1.0)
+ * 0.01
+ })
+ .collect();
+ vector[index % clusters] += 1.0;
+ let norm = vector.iter().map(|v| v * v).sum::<f32>().sqrt();
+ for value in &mut vector {
+ *value /= norm;
+ }
+ vector
+ })
+ .collect();
+ (keys, vectors)
+ }
+
+ fn assert_cluster_recall(db: &VecDb, vectors: &[Vec<f32>], survivors: &[usize]) {
+ let approx = limit_params(10);
+ let exact = SearchParams {
+ exact: true,
+ ..approx.clone()
+ };
+ let range = SearchParams {
+ max_distance: Some(0.01),
+ ..SearchParams::default()
+ };
+ let exact_range = SearchParams {
+ exact: true,
+ ..range.clone()
+ };
+ for &index in survivors {
+ assert_eq!(
+ db.search(&vectors[index], &approx).unwrap(),
+ db.search(&vectors[index], &exact).unwrap()
+ );
+ assert_eq!(
+ db.search(&vectors[index], &range).unwrap(),
+ db.search(&vectors[index], &exact_range).unwrap()
+ );
+ }
+ }
+
+ #[test]
+ fn concentrated_changes_preserve_recall_before_compaction_and_across_reopen() {
+ for storage in [StorageKind::F32, StorageKind::I8] {
+ for delete_first in [false, true] {
+ let clusters = if delete_first { 20 } else { 10 };
+ let victims = if delete_first { 140 } else { 270 };
+ let (keys, vectors) = churn_vectors(clusters);
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = VecDb::open(&path, 32, Some(storage)).unwrap();
+ db.bulk_add(&keys, &vectors).unwrap();
+ db.flush().unwrap();
+ let survivors: Vec<_> = (victims * clusters..3000).step_by(clusters).collect();
+ assert_cluster_recall(&db, &vectors, &survivors);
+ let changed_keys: Vec<_> = (0..victims)
+ .map(|index| keys[index * clusters].clone())
+ .collect();
+ let changed_vectors: Vec<_> = (0..victims)
+ .map(|index| vectors[index * clusters + 1].clone())
+ .collect();
+ let unchanged_vectors: Vec<_> = (0..victims)
+ .map(|index| vectors[index * clusters].clone())
+ .collect();
+ let before = db.stats().unwrap();
+ db.bulk_add(&changed_keys, &unchanged_vectors).unwrap();
+ assert_eq!(db.stats().unwrap(), before);
+ let all = SearchParams {
+ max_distance: Some(2.5),
+ ..SearchParams::default()
+ };
+ assert_eq!(db.search(&vectors[survivors[0]], &all).unwrap().len(), 3000);
+ if delete_first {
+ db.bulk_remove(&changed_keys).unwrap();
+ assert_cluster_recall(&db, &vectors, &survivors);
+ }
+ db.bulk_add(&changed_keys, &changed_vectors).unwrap();
+ assert_eq!(db.shared.state_read().arena.dead_count(), victims);
+ assert_cluster_recall(&db, &vectors, &survivors);
+ let stored_keys: Vec<_> =
+ survivors.iter().map(|&index| keys[index].clone()).collect();
+ let stored_before = db
+ .bulk_search_stored(&stored_keys, 10, None, false, false)
+ .unwrap();
+ assert_eq!(
+ stored_before,
+ db.bulk_search_stored(&stored_keys, 10, None, true, false)
+ .unwrap()
+ );
+ drop(db);
+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();
+ assert_cluster_recall(&reopened, &vectors, &survivors);
+ assert_eq!(
+ reopened
+ .bulk_search_stored(&stored_keys, 10, None, false, false)
+ .unwrap(),
+ stored_before
+ );
+ reopened.flush().unwrap();
+ drop(reopened);
+ let reopened = VecDb::open(&path, 32, Some(storage)).unwrap();
+ assert_cluster_recall(&reopened, &vectors, &survivors);
+ let exact = SearchParams {
+ exact: true,
+ ..limit_params(10)
+ };
+ let queries: Vec<_> = survivors
+ .iter()
+ .map(|&index| vectors[index].clone())
+ .collect();
+ let truth = reopened.bulk_search(&queries, &exact).unwrap();
+ compact(&reopened.shared, &mut reopened.shared.writer_half()).unwrap();
+ assert_eq!(reopened.shared.state_read().arena.dead_count(), 0);
+ assert_eq!(reopened.bulk_search(&queries, &exact).unwrap(), truth);
+ let fresh_path = dir.path().join("fresh");
+ fs::copy(&path, &fresh_path).unwrap();
+ let fresh = VecDb::open(&fresh_path, 32, Some(storage)).unwrap();
+ assert_eq!(
+ reopened.bulk_search(&queries, &limit_params(10)).unwrap(),
+ fresh.bulk_search(&queries, &limit_params(10)).unwrap()
+ );
+ }
+ }
+ }
+
#[test]
fn omitted_storage_requires_i8_for_creation_and_every_reopen() {
let dir = TempDir::new().unwrap();
@@ -3781,15 +4229,15 @@ mod tests {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let db = open_writer(&path);
- bulk_add(&db, &bulk_entries(0, 1100, 3000)).unwrap();
+ bulk_add(&db, &bulk_entries(0, 1200, 3000)).unwrap();
let generation_before = generation_of(&path);
- let removals: Vec<String> = (0..58).map(|index| format!("key-{index}")).collect();
- assert_eq!(db.bulk_remove(&removals).unwrap(), 58);
+ let removals: Vec<String> = (0..120).map(|index| format!("key-{index}")).collect();
+ assert_eq!(db.bulk_remove(&removals).unwrap(), 120);
assert_ne!(generation_of(&path), generation_before);
let stats = db.stats().unwrap();
- assert_eq!(stats.live_count, 1042);
+ assert_eq!(stats.live_count, 1080);
assert_eq!(stats.dead_count, 0);
- for index in (58..1100u64).step_by(97) {
+ for index in (120..1200u64).step_by(97) {
assert_own_nearest(
&db,
&format!("key-{index}"),
@@ -3798,9 +4246,9 @@ mod tests {
}
drop(db);
let reopened = open_writer(&path);
- assert_eq!(reopened.len().unwrap(), 1042);
+ assert_eq!(reopened.len().unwrap(), 1080);
assert!(!reopened.contains("key-0").unwrap());
- assert!(reopened.contains("key-58").unwrap());
+ assert!(reopened.contains("key-120").unwrap());
}
#[test]
@@ -3872,6 +4320,7 @@ mod tests {
graph,
attrs,
total_records,
+ ..
} = &mut *st;
attrs.repack(arena.live_slots());
arena.compact_in_place();
@@ -5432,7 +5881,7 @@ mod tests {
}
#[test]
- fn attrs_do_not_leak_across_remove_and_slot_recycle() {
+ fn attrs_do_not_leak_across_retired_slots_and_new_versions() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("db");
let db = open_writer(&path);
@@ -5442,11 +5891,11 @@ mod tests {
assert!(db.remove("a").unwrap());
assert_eq!(db.get_attrs("a").unwrap(), None);
db.add("c", &vector).unwrap();
- assert_eq!(db.shared.state_read().arena.slot_of_key("c"), Some(0));
+ assert_eq!(db.shared.state_read().arena.slot_of_key("c"), Some(2));
assert_eq!(db.get_attrs("c").unwrap(), None);
assert!(db.remove("b").unwrap());
db.add_with_attrs("d", &vector, &sample_attrs(12)).unwrap();
- assert_eq!(db.shared.state_read().arena.slot_of_key("d"), Some(1));
+ assert_eq!(db.shared.state_read().arena.slot_of_key("d"), Some(3));
assert_eq!(db.get_attrs("d").unwrap().unwrap(), sample_attrs(12));
db.add("a", &vector).unwrap();
assert_eq!(db.get_attrs("a").unwrap(), None);
@@ -5561,9 +6010,9 @@ mod tests {
};
verify(&db);
fs::remove_dir(temp_sibling(&path)).unwrap();
- let rewrite = |range: std::ops::Range<usize>| {
+ let rewrite = |range: std::ops::Range<usize>, seed| {
let (keys, vectors): (Vec<String>, Vec<Vec<f32>>) =
- bulk_entries(range.start, range.len(), 910)
+ bulk_entries(range.start, range.len(), seed)
.iter()
.cloned()
.unzip();
@@ -5572,11 +6021,11 @@ mod tests {
.collect();
db.bulk_add_with_attrs(&keys, &vectors, &attrs).unwrap();
};
- rewrite(64..120);
+ rewrite(64..120, 920);
assert_eq!(db.stats().unwrap().dead_count, 184);
assert_eq!(generation_of(&path), generation_before);
verify(&db);
- rewrite(64..72);
+ rewrite(64..72, 930);
assert_eq!(db.stats().unwrap().dead_count, 0);
assert_ne!(generation_of(&path), generation_before);
verify(&db);