Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

vecdb: reclaim key-map capacity during compaction and give close teardown its own handoff budget

laurenspriem56c97800f343
diff --git a/rust/crates/ml/src/vecdb/arena.rs b/rust/crates/ml/src/vecdb/arena.rs
index 32d8c910be..9641aaac2b 100644
--- a/rust/crates/ml/src/vecdb/arena.rs
+++ b/rust/crates/ml/src/vecdb/arena.rs
@@ -144,6 +144,7 @@ impl VectorArena {
debug_assert_eq!(live, self.live_count);
self.slots_to_keys.truncate(live);
self.slots_to_keys.shrink_to_fit();
+ self.keys_to_slots.shrink_to_fit();
self.chunks.truncate(live.div_ceil(VECTORS_PER_CHUNK));
self.chunks.shrink_to_fit();
self.alive.clear();
@@ -626,6 +627,28 @@ mod tests {
assert_eq!(arena.vector_values(0), seeded_vector(4, 8));
}
+ #[test]
+ fn compact_in_place_reclaims_key_map_capacity() {
+ let mut arena = VectorArena::new(8).unwrap();
+ for index in 0..5000u64 {
+ arena
+ .upsert(&format!("key-{index}"), &seeded_vector(index, 8))
+ .unwrap();
+ }
+ for index in 0..4900u64 {
+ assert!(arena.remove(&format!("key-{index}")).is_some());
+ }
+ let before = arena.keys_to_slots.capacity();
+ assert!(before > arena.live_count() * 4);
+ arena.compact_in_place();
+ let after = arena.keys_to_slots.capacity();
+ assert_eq!(arena.live_count(), 100);
+ assert!(after < before);
+ assert!(after >= arena.live_count());
+ assert!(after <= arena.live_count() * 4);
+ assert_eq!(arena.slot_of_key("key-4999"), Some(99));
+ }
+
#[test]
fn compact_in_place_on_an_empty_arena_is_a_noop() {
let mut arena = VectorArena::new(8).unwrap();
diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rs
index 5fc1edbb13..ac72bda306 100644
--- a/rust/crates/ml/src/vecdb/store.rs
+++ b/rust/crates/ml/src/vecdb/store.rs
@@ -28,6 +28,7 @@ const COMPACTION_MIN_DEAD_RECORDS: u64 = 64;
const COMPACTION_DEAD_RATIO: u64 = 10;
const COMPACTION_BATCH_SIZE: usize = 1000;
const HANDOFF_WAIT_ROUNDS: u32 = 750;
+const HANDOFF_CLOSE_WAIT_ROUNDS: u32 = 2500;
const HANDOFF_WAIT_PARK: Duration = Duration::from_millis(2);
const KEY_TABLE_COPIES: usize = 2;
const KEY_ENTRY_OVERHEAD_BYTES: usize = 48;
@@ -315,7 +316,8 @@ enum SlotHandoff<'a> {
}
fn wait_for_teardown_handoff<'a>(slot: &'a Mutex<PathSlot>, path: &Path) -> SlotHandoff<'a> {
- let mut rounds = HANDOFF_WAIT_ROUNDS;
+ let mut dead_rounds = HANDOFF_WAIT_ROUNDS;
+ let mut close_rounds = HANDOFF_CLOSE_WAIT_ROUNDS;
loop {
let guard = lock_slot(slot);
let observed = match &guard.live {
@@ -327,20 +329,21 @@ fn wait_for_teardown_handoff<'a>(slot: &'a Mutex<PathSlot>, path: &Path) -> Slot
Some(closing) => {
drop(guard);
drop(closing);
- if rounds == 0 {
+ if close_rounds == 0 {
warn_handoff_cap(path);
return SlotHandoff::Expired(lock_slot(slot));
}
+ close_rounds -= 1;
}
None => {
- if rounds == 0 {
+ if dead_rounds == 0 {
warn_handoff_cap(path);
return SlotHandoff::Expired(guard);
}
drop(guard);
+ dead_rounds -= 1;
}
}
- rounds -= 1;
std::thread::sleep(HANDOFF_WAIT_PARK);
}
}
@@ -2507,6 +2510,7 @@ mod tests {
SlotHandoff::Released(_) => panic!("expected an expired wait, saw a release"),
}
assert!(waited >= HANDOFF_WAIT_PARK * HANDOFF_WAIT_ROUNDS);
+ assert!(waited < HANDOFF_WAIT_PARK * HANDOFF_WAIT_ROUNDS * 2);
}
fn closed_instance(path: &Path) -> Arc<Shared> {
@@ -2540,7 +2544,7 @@ mod tests {
SlotHandoff::Join(_) => panic!("expected an expired wait, saw a join"),
SlotHandoff::Released(_) => panic!("expected an expired wait, saw a release"),
}
- assert!(waited >= HANDOFF_WAIT_PARK * HANDOFF_WAIT_ROUNDS);
+ assert!(waited >= HANDOFF_WAIT_PARK * HANDOFF_CLOSE_WAIT_ROUNDS);
}
#[test]