π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: compact the arena in place and rebuild the graph without doubling resident memory
laurensprieme83f4d88da20
diff --git a/rust/crates/ml/src/vecdb/arena.rs b/rust/crates/ml/src/vecdb/arena.rsindex e3d1a49936..32d8c910be 100644--- a/rust/crates/ml/src/vecdb/arena.rs+++ b/rust/crates/ml/src/vecdb/arena.rs@@ -124,6 +124,54 @@ impl VectorArena {Some(slot)}+ pub(crate) fn compact_in_place(&mut self) {+ let mut dense: u32 = 0;+ for slot in 0..self.slots_to_keys.len() as u32 {+ if !self.is_alive(slot) {+ continue;+ }+ if dense != slot {+ self.move_vector(slot, dense);+ let key = std::mem::take(&mut self.slots_to_keys[slot as usize]);+ if let Some(mapped) = self.keys_to_slots.get_mut(&key) {+ *mapped = dense;+ }+ self.slots_to_keys[dense as usize] = key;+ }+ dense += 1;+ }+ let live = dense as usize;+ debug_assert_eq!(live, self.live_count);+ self.slots_to_keys.truncate(live);+ self.slots_to_keys.shrink_to_fit();+ self.chunks.truncate(live.div_ceil(VECTORS_PER_CHUNK));+ self.chunks.shrink_to_fit();+ self.alive.clear();+ self.alive.resize(live.div_ceil(64), u64::MAX);+ self.alive.shrink_to_fit();+ if !live.is_multiple_of(64)+ && let Some(word) = self.alive.last_mut()+ {+ *word = (1u64 << (live % 64)) - 1;+ }+ self.free_slots = Vec::new();+ }++ fn move_vector(&mut self, src: u32, dst: u32) {+ let lanes = self.lanes_per_vector;+ let src_chunk = src as usize / VECTORS_PER_CHUNK;+ let dst_chunk = dst as usize / VECTORS_PER_CHUNK;+ let src_start = (src as usize % VECTORS_PER_CHUNK) * lanes;+ let dst_start = (dst as usize % VECTORS_PER_CHUNK) * lanes;+ if src_chunk == dst_chunk {+ self.chunks[src_chunk].copy_within(src_start..src_start + lanes, dst_start);+ } else {+ let (front, back) = self.chunks.split_at_mut(src_chunk);+ front[dst_chunk][dst_start..dst_start + lanes]+ .copy_from_slice(&back[0][src_start..src_start + lanes]);+ }+ }+pub(crate) fn slot_of_key(&self, key: &str) -> Option<u32> {self.keys_to_slots.get(key).copied()}@@ -439,6 +487,157 @@ mod tests {}}+ #[test]+ fn compact_in_place_repacks_live_entries_densely_in_order() {+ let mut arena = VectorArena::new(8).unwrap();+ for index in 0..10u64 {+ arena+ .upsert(&format!("key-{index}"), &seeded_vector(index, 8))+ .unwrap();+ }+ assert_eq!(arena.remove("key-0"), Some(0));+ assert_eq!(arena.remove("key-4"), Some(4));+ assert_eq!(arena.remove("key-9"), Some(9));+ assert_eq!(+ arena.upsert("key-5", &seeded_vector(50, 8)).unwrap(),+ UpsertOutcome::ReplacedInPlace(5)+ );+ assert_eq!(+ arena.upsert("extra", &seeded_vector(60, 8)).unwrap(),+ UpsertOutcome::RecycledSlot(9)+ );+ assert_eq!(arena.remove("key-2"), Some(2));+ let expected: Vec<(String, Vec<f32>)> = arena+ .live_slots()+ .map(|slot| {+ (+ arena.key_of_slot(slot).unwrap().to_string(),+ arena.vector_values(slot),+ )+ })+ .collect();+ assert_eq!(expected.len(), 7);+ assert_eq!(arena.dead_count(), 3);+ arena.compact_in_place();+ assert_eq!(arena.live_count(), 7);+ assert_eq!(arena.slot_count(), 7);+ assert_eq!(arena.dead_count(), 0);+ assert!(arena.free_slots.is_empty());+ for (index, (key, vector)) in expected.iter().enumerate() {+ let slot = index as u32;+ assert!(arena.is_alive(slot));+ assert_eq!(arena.key_of_slot(slot), Some(key.as_str()));+ assert_eq!(arena.slot_of_key(key), Some(slot));+ assert_eq!(&arena.vector_values(slot), vector);+ }+ assert!(!arena.is_alive(7));+ assert_eq!(+ arena.live_slots().collect::<Vec<_>>(),+ (0..7).collect::<Vec<_>>()+ );+ assert_eq!(+ arena.upsert("fresh", &seeded_vector(70, 8)).unwrap(),+ UpsertOutcome::NewSlot(7)+ );+ let recycled = arena.slot_of_key("key-5").unwrap();+ assert_eq!(arena.remove("key-5"), Some(recycled));+ assert_eq!(arena.dead_count(), 1);+ assert_eq!(+ arena.upsert("reused", &seeded_vector(80, 8)).unwrap(),+ UpsertOutcome::RecycledSlot(recycled)+ );+ }++ #[test]+ fn compact_in_place_moves_vectors_across_chunks_and_frees_the_tail() {+ let mut arena = VectorArena::new(8).unwrap();+ let count = VECTORS_PER_CHUNK + 300;+ for index in 0..count as u64 {+ arena+ .upsert(&format!("key-{index}"), &seeded_vector(index, 8))+ .unwrap();+ }+ assert_eq!(arena.chunks.len(), 2);+ for index in (0..1000u64).step_by(2) {+ assert!(arena.remove(&format!("key-{index}")).is_some());+ }+ let expected: Vec<(String, Vec<f32>)> = arena+ .live_slots()+ .map(|slot| {+ (+ arena.key_of_slot(slot).unwrap().to_string(),+ arena.vector_values(slot),+ )+ })+ .collect();+ arena.compact_in_place();+ let live = count - 500;+ assert_eq!(arena.live_count(), live);+ assert_eq!(arena.slot_count(), live);+ assert_eq!(arena.chunks.len(), 1);+ assert_eq!(arena.alive.len(), live.div_ceil(64));+ assert!(arena.free_slots.is_empty());+ for (index, (key, vector)) in expected.iter().enumerate() {+ assert_eq!(arena.slot_of_key(key), Some(index as u32));+ assert_eq!(&arena.vector_values(index as u32), vector);+ }+ for slot in arena.live_slots() {+ assert_eq!(arena.vector_lanes(slot).as_ptr() as usize % 32, 0);+ }+ }++ #[test]+ fn compact_in_place_with_no_dead_slots_preserves_everything() {+ let mut arena = arena_with_keys(8, &["a", "b", "c"]);+ arena.compact_in_place();+ assert_eq!(arena.live_count(), 3);+ assert_eq!(arena.slot_count(), 3);+ assert_eq!(arena.dead_count(), 0);+ for (index, key) in ["a", "b", "c"].iter().enumerate() {+ assert_eq!(arena.key_of_slot(index as u32), Some(*key));+ assert_eq!(+ arena.vector_values(index as u32),+ seeded_vector(index as u64, 8)+ );+ }+ assert_eq!(+ arena.upsert("d", &seeded_vector(9, 8)).unwrap(),+ UpsertOutcome::NewSlot(3)+ );+ }++ #[test]+ fn compact_in_place_empties_a_fully_dead_arena() {+ let mut arena = arena_with_keys(8, &["a", "b", "c"]);+ for key in ["a", "b", "c"] {+ assert!(arena.remove(key).is_some());+ }+ arena.compact_in_place();+ assert!(arena.is_empty());+ assert_eq!(arena.live_count(), 0);+ assert_eq!(arena.slot_count(), 0);+ assert_eq!(arena.dead_count(), 0);+ assert!(arena.chunks.is_empty());+ assert!(arena.alive.is_empty());+ assert_eq!(+ arena.upsert("d", &seeded_vector(4, 8)).unwrap(),+ UpsertOutcome::NewSlot(0)+ );+ assert_eq!(arena.vector_values(0), seeded_vector(4, 8));+ }++ #[test]+ fn compact_in_place_on_an_empty_arena_is_a_noop() {+ let mut arena = VectorArena::new(8).unwrap();+ arena.compact_in_place();+ assert!(arena.is_empty());+ assert_eq!(arena.slot_count(), 0);+ assert_eq!(+ arena.upsert("a", &seeded_vector(0, 8)).unwrap(),+ UpsertOutcome::NewSlot(0)+ );+ }+#[test]fn grows_across_chunks_with_correct_addressing() {let mut arena = VectorArena::new(8).unwrap();diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rsindex 24430877b7..3a95258e46 100644--- a/rust/crates/ml/src/vecdb/store.rs+++ b/rust/crates/ml/src/vecdb/store.rs@@ -114,6 +114,23 @@ impl AttrTable {self.0.as_ref()?.get(slot as usize)?.as_deref()}+ fn repack(&mut self, live_slots: impl Iterator<Item = u32>) {+ let Some(table) = &mut self.0 else {+ return;+ };+ let mut dense = 0usize;+ for slot in live_slots {+ let moved = table.get_mut(slot as usize).and_then(Option::take);+ match table.get_mut(dense) {+ Some(entry) => *entry = moved,+ None => debug_assert!(moved.is_none()),+ }+ dense += 1;+ }+ table.truncate(dense);+ table.shrink_to_fit();+ }+fn reset(&mut self) {self.0 = None;}@@ -941,11 +958,9 @@ fn compact(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecDbError> {active_state(half)?;let dims = shared.dims;let (mut temp_log, temp_path) = Log::create_temp_sibling(&shared.path, dims)?;- let mut compacted = VectorArena::new(dims)?;- let mut compacted_attrs = AttrTable::default();let staged = {let st = shared.state_read();- stage_live_entries(&st, &mut compacted, &mut compacted_attrs, &mut temp_log)+ stage_live_entries(&st, &mut temp_log)};if let Err(error) = staged {drop(temp_log);@@ -971,14 +986,26 @@ fn compact(shared: &Shared, half: &mut WriterHalf) -> Result<(), VecDbError> {let _ = std::fs::remove_file(&temp_path);mutations_since_snapshot} else {- let graph = Graph::rebuild(&compacted);- let total_records = compacted.live_count() as u64;- let mut st = shared.state_write();- st.arena = compacted;- st.graph = Some(graph);- st.attrs = compacted_attrs;- st.total_records = total_records;- total_records as usize+ let live_records = {+ let mut st = shared.state_write();+ let SearchState {+ arena,+ graph,+ attrs,+ total_records,+ } = &mut *st;+ attrs.repack(arena.live_slots());+ arena.compact_in_place();+ *graph = None;+ *total_records = arena.live_count() as u64;+ *total_records as usize+ };+ let graph = {+ let st = shared.state_read();+ Graph::rebuild(&st.arena)+ };+ shared.state_write().graph = Some(graph);+ live_records};let restored = restore_writer_mode(shared, half, lock, last_write, pending);promoted?;@@ -1456,12 +1483,7 @@ fn remove_data_files(path: &Path) -> Result<(), VecDbError> {sync_parent_dir(path)}-fn stage_live_entries(- source: &SearchState,- target: &mut VectorArena,- target_attrs: &mut AttrTable,- log: &mut Log,-) -> Result<(), VecDbError> {+fn stage_live_entries(source: &SearchState, log: &mut Log) -> Result<(), VecDbError> {let mut batch: Vec<(String, Vec<f32>, Vec<Attribute>)> =Vec::with_capacity(COMPACTION_BATCH_SIZE);for slot in source.arena.live_slots() {@@ -1478,16 +1500,14 @@ fn stage_live_entries(.unwrap_or_default(),));if batch.len() == COMPACTION_BATCH_SIZE {- flush_stage_batch(&mut batch, target, target_attrs, log)?;+ flush_stage_batch(&mut batch, log)?;}}- flush_stage_batch(&mut batch, target, target_attrs, log)+ flush_stage_batch(&mut batch, log)}fn flush_stage_batch(batch: &mut Vec<(String, Vec<f32>, Vec<Attribute>)>,- target: &mut VectorArena,- target_attrs: &mut AttrTable,log: &mut Log,) -> Result<(), VecDbError> {if batch.is_empty() {@@ -1498,10 +1518,7 @@ fn flush_stage_batch(.map(|(key, vector, attrs)| LogEntry::Add { key, vector, attrs }).collect();log.append_unsynced_for_staging(&records)?;- for (key, vector, attrs) in batch.drain(..) {- let outcome = target.upsert(&key, &vector)?;- target_attrs.set(slot_of_outcome(outcome), &attrs);- }+ batch.clear();Ok(())}@@ -2780,6 +2797,112 @@ mod tests {assert!(reopened.contains("key-58"));}+ #[test]+ fn compaction_installs_the_graph_a_fresh_rebuild_of_the_compacted_log_builds() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let copy_path = dir.path().join("copy");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 300, 500)).unwrap();+ let removals: Vec<String> = (0..40).map(|index| format!("key-{}", index * 7)).collect();+ assert_eq!(db.bulk_remove(&removals).unwrap(), 40);+ assert_eq!(db.stats().unwrap().dead_count, 0);+ fs::copy(&path, ©_path).unwrap();+ let rebuilt = open_writer(©_path);+ let st = db.shared.state_read();+ let rebuilt_st = rebuilt.shared.state_read();+ let installed = st.graph.as_ref().unwrap();+ let from_scratch = rebuilt_st.graph.as_ref().unwrap();+ assert_eq!(st.arena.live_count(), 260);+ assert_eq!(st.arena.slot_count(), 260);+ for slot in st.arena.live_slots() {+ assert_eq!(+ st.arena.key_of_slot(slot),+ rebuilt_st.arena.key_of_slot(slot)+ );+ }+ assert!(+ installed+ .slots()+ .any(|slot| installed.level_of(slot).unwrap() > 0)+ );+ test_support::assert_identical_graphs(installed, from_scratch);+ }++ #[test]+ fn a_repacked_state_with_no_graph_serves_exact_results_for_approx_searches() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ let entries = bulk_entries(0, 200, 640);+ let (keys, vectors): (Vec<String>, Vec<Vec<f32>>) = entries.iter().cloned().unzip();+ let attrs: Vec<Option<Vec<Attribute>>> = (0..200)+ .map(|index| (index % 3 == 0).then(|| sample_attrs(index)))+ .collect();+ db.bulk_add_with_attrs(&keys, &vectors, &attrs).unwrap();+ let removals: Vec<String> = (0..20).map(|index| format!("key-{}", index * 9)).collect();+ assert_eq!(db.bulk_remove(&removals).unwrap(), 20);+ let query = seeded_unit_vector(777, DIMS);+ let approx = limit_params(10);+ let exact = SearchParams {+ limit: Some(10),+ exact: true,+ ..SearchParams::default()+ };+ let expected = db.search(&query, &exact).unwrap();+ assert_eq!(expected.len(), 10);+ let stored_keys = vec![+ "key-1".to_string(),+ "key-50".to_string(),+ "key-100".to_string(),+ ];+ let expected_stored = db+ .bulk_search_stored(&stored_keys, 5, None, true, false)+ .unwrap();+ {+ let mut st = db.shared.state_write();+ let SearchState {+ arena,+ graph,+ attrs,+ total_records,+ } = &mut *st;+ attrs.repack(arena.live_slots());+ arena.compact_in_place();+ *graph = None;+ *total_records = arena.live_count() as u64;+ }+ {+ let st = db.shared.state_read();+ assert!(st.graph.is_none());+ assert_eq!(st.arena.slot_count(), 180);+ }+ assert_eq!(db.len(), 180);+ assert_eq!(db.search(&query, &approx).unwrap(), expected);+ assert_eq!(db.search(&query, &exact).unwrap(), expected);+ assert_eq!(+ db.bulk_search(std::slice::from_ref(&query), &approx)+ .unwrap(),+ vec![expected.clone()]+ );+ assert_eq!(+ db.bulk_search_stored(&stored_keys, 5, None, false, false)+ .unwrap(),+ expected_stored+ );+ for index in 0..200i64 {+ let key = format!("key-{index}");+ if index % 9 == 0 && index <= 171 {+ assert!(!db.contains(&key));+ assert_eq!(db.get_attrs(&key), None);+ } else if index % 3 == 0 {+ assert_eq!(db.get_attrs(&key).unwrap(), sample_attrs(index));+ } else {+ assert_eq!(db.get_attrs(&key), None);+ }+ }+ }+#[test]fn stale_snapshot_left_by_interrupted_compaction_is_discarded_on_open() {let dir = TempDir::new().unwrap();@@ -3351,11 +3474,18 @@ mod tests {let mut half = shared.writer_half();compact(&shared, &mut half)});+ let deadline = Instant::now() + Duration::from_secs(60);+ while generation_of(&path) == generation_before {+ assert!(Instant::now() < deadline);+ thread::yield_now();+ }let packed = held.arena.pack_query(&query).unwrap();for _ in 0..25 {let during = graph_search(held.search_graph(), &held.arena, &packed, &exact, None);assert_eq!(during, before);}+ assert!(held.graph.is_some());+ assert!(!compactor.is_finished());drop(held);compactor.join().unwrap().unwrap();assert_ne!(generation_of(&path), generation_before);