π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: reclaim key-map capacity during compaction and give close teardown its own handoff budget
laurenspriem56c97800f343
diff --git a/rust/crates/ml/src/vecdb/arena.rs b/rust/crates/ml/src/vecdb/arena.rsindex 32d8c910be..9641aaac2b 100644--- a/rust/crates/ml/src/vecdb/arena.rs+++ b/rust/crates/ml/src/vecdb/arena.rs@@ -144,6 +144,7 @@ impl VectorArena {debug_assert_eq!(live, self.live_count);self.slots_to_keys.truncate(live);self.slots_to_keys.shrink_to_fit();+ self.keys_to_slots.shrink_to_fit();self.chunks.truncate(live.div_ceil(VECTORS_PER_CHUNK));self.chunks.shrink_to_fit();self.alive.clear();@@ -626,6 +627,28 @@ mod tests {assert_eq!(arena.vector_values(0), seeded_vector(4, 8));}+ #[test]+ fn compact_in_place_reclaims_key_map_capacity() {+ let mut arena = VectorArena::new(8).unwrap();+ for index in 0..5000u64 {+ arena+ .upsert(&format!("key-{index}"), &seeded_vector(index, 8))+ .unwrap();+ }+ for index in 0..4900u64 {+ assert!(arena.remove(&format!("key-{index}")).is_some());+ }+ let before = arena.keys_to_slots.capacity();+ assert!(before > arena.live_count() * 4);+ arena.compact_in_place();+ let after = arena.keys_to_slots.capacity();+ assert_eq!(arena.live_count(), 100);+ assert!(after < before);+ assert!(after >= arena.live_count());+ assert!(after <= arena.live_count() * 4);+ assert_eq!(arena.slot_of_key("key-4999"), Some(99));+ }+#[test]fn compact_in_place_on_an_empty_arena_is_a_noop() {let mut arena = VectorArena::new(8).unwrap();diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rsindex 5fc1edbb13..ac72bda306 100644--- a/rust/crates/ml/src/vecdb/store.rs+++ b/rust/crates/ml/src/vecdb/store.rs@@ -28,6 +28,7 @@ const COMPACTION_MIN_DEAD_RECORDS: u64 = 64;const COMPACTION_DEAD_RATIO: u64 = 10;const COMPACTION_BATCH_SIZE: usize = 1000;const HANDOFF_WAIT_ROUNDS: u32 = 750;+const HANDOFF_CLOSE_WAIT_ROUNDS: u32 = 2500;const HANDOFF_WAIT_PARK: Duration = Duration::from_millis(2);const KEY_TABLE_COPIES: usize = 2;const KEY_ENTRY_OVERHEAD_BYTES: usize = 48;@@ -315,7 +316,8 @@ enum SlotHandoff<'a> {}fn wait_for_teardown_handoff<'a>(slot: &'a Mutex<PathSlot>, path: &Path) -> SlotHandoff<'a> {- let mut rounds = HANDOFF_WAIT_ROUNDS;+ let mut dead_rounds = HANDOFF_WAIT_ROUNDS;+ let mut close_rounds = HANDOFF_CLOSE_WAIT_ROUNDS;loop {let guard = lock_slot(slot);let observed = match &guard.live {@@ -327,20 +329,21 @@ fn wait_for_teardown_handoff<'a>(slot: &'a Mutex<PathSlot>, path: &Path) -> SlotSome(closing) => {drop(guard);drop(closing);- if rounds == 0 {+ if close_rounds == 0 {warn_handoff_cap(path);return SlotHandoff::Expired(lock_slot(slot));}+ close_rounds -= 1;}None => {- if rounds == 0 {+ if dead_rounds == 0 {warn_handoff_cap(path);return SlotHandoff::Expired(guard);}drop(guard);+ dead_rounds -= 1;}}- rounds -= 1;std::thread::sleep(HANDOFF_WAIT_PARK);}}@@ -2507,6 +2510,7 @@ mod tests {SlotHandoff::Released(_) => panic!("expected an expired wait, saw a release"),}assert!(waited >= HANDOFF_WAIT_PARK * HANDOFF_WAIT_ROUNDS);+ assert!(waited < HANDOFF_WAIT_PARK * HANDOFF_WAIT_ROUNDS * 2);}fn closed_instance(path: &Path) -> Arc<Shared> {@@ -2540,7 +2544,7 @@ mod tests {SlotHandoff::Join(_) => panic!("expected an expired wait, saw a join"),SlotHandoff::Released(_) => panic!("expected an expired wait, saw a release"),}- assert!(waited >= HANDOFF_WAIT_PARK * HANDOFF_WAIT_ROUNDS);+ assert!(waited >= HANDOFF_WAIT_PARK * HANDOFF_CLOSE_WAIT_ROUNDS);}#[test]