π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: step the ef search floor once per doubling of the index
laurenspriemf4e5e7405e47
diff --git a/rust/crates/vecdb/src/graph.rs b/rust/crates/vecdb/src/graph.rsindex 55bf6f02c4..dec260fb24 100644--- a/rust/crates/vecdb/src/graph.rs+++ b/rust/crates/vecdb/src/graph.rs@@ -12,11 +12,12 @@ const EF_CONSTRUCTION: usize = 96;const SELECTION_WINDOW_FACTOR: usize = 2;const REVERSE_PRUNE_SLACK: usize = 4;const EF_SEARCH_UPPER: usize = 1;-const EF_SEARCH_FLOOR_SMALL: usize = 72;-const EF_SEARCH_FLOOR_LARGE: usize = 56;-const EF_SEARCH_FLOOR_HUGE: usize = 128;-const EF_SEARCH_FLOOR_CROSSOVER: usize = 20_000;-const EF_SEARCH_FLOOR_CROSSOVER_HUGE: usize = 250_000;+const EF_SEARCH_FLOOR_MIN: usize = 72;+const EF_SEARCH_FLOOR_MAX: usize = 128;+const EF_SEARCH_FLOOR_STEP: usize = 16;+const EF_SEARCH_FLOOR_CROSSOVER: usize = 65_536;+const EF_SEARCH_FLOOR_AT_CROSSOVER: usize = 80;+const _: () = assert!(EF_SEARCH_FLOOR_CROSSOVER.is_power_of_two());const EF_SEARCH_LIMIT_FACTOR: usize = 4;const SMALL_FILTER_FLOOR: usize = 1024;const SMALL_FILTER_LIMIT_FACTOR: usize = 4;@@ -880,12 +881,10 @@ fn result_bound(arena: &VectorArena, allowed: Option<&HashSet<u32>>) -> usize {fn ef_search_floor(live: usize) -> usize {if live < EF_SEARCH_FLOOR_CROSSOVER {- EF_SEARCH_FLOOR_SMALL- } else if live < EF_SEARCH_FLOOR_CROSSOVER_HUGE {- EF_SEARCH_FLOOR_LARGE- } else {- EF_SEARCH_FLOOR_HUGE+ return EF_SEARCH_FLOOR_MIN;}+ let doublings = (live.ilog2() - EF_SEARCH_FLOOR_CROSSOVER.ilog2()) as usize;+ (EF_SEARCH_FLOOR_AT_CROSSOVER + doublings * EF_SEARCH_FLOOR_STEP).min(EF_SEARCH_FLOOR_MAX)}fn admissible(arena: &VectorArena, allowed: Option<&HashSet<u32>>, slot: u32) -> bool {@@ -2760,29 +2759,39 @@ mod tests {}#[test]- fn ef_search_floor_steps_at_both_crossovers() {+ fn ef_search_floor_steps_once_per_doubling_above_the_crossover() {assert_eq!(ef_search_floor(0), 72);- assert_eq!(ef_search_floor(19_999), 72);- assert_eq!(ef_search_floor(20_000), 56);- assert_eq!(ef_search_floor(249_999), 56);- assert_eq!(ef_search_floor(250_000), 128);- assert_eq!(ef_search_floor(4_000_000), 128);+ assert_eq!(ef_search_floor(1), 72);+ assert_eq!(ef_search_floor(65_535), 72);+ assert_eq!(ef_search_floor(65_536), 80);+ assert_eq!(ef_search_floor(131_071), 80);+ assert_eq!(ef_search_floor(131_072), 96);+ assert_eq!(ef_search_floor(262_143), 96);+ assert_eq!(ef_search_floor(262_144), 112);+ assert_eq!(ef_search_floor(524_287), 112);+ assert_eq!(ef_search_floor(524_288), 128);+ assert_eq!(ef_search_floor(1_000_000), 128);+ assert_eq!(ef_search_floor(2_000_000), 128);assert_eq!(ef_search_floor(usize::MAX), 128);}#[test]- fn ef_search_floor_drops_once_and_then_only_grows() {- assert!(- ef_search_floor(EF_SEARCH_FLOOR_CROSSOVER - 1)- > ef_search_floor(EF_SEARCH_FLOOR_CROSSOVER)- );- let mut previous = ef_search_floor(EF_SEARCH_FLOOR_CROSSOVER);+ fn ef_search_floor_never_drops_and_stays_within_the_cap() {+ let mut previous = ef_search_floor(0);for live in [+ 0,+ 1,+ EF_SEARCH_FLOOR_CROSSOVER - 1,EF_SEARCH_FLOOR_CROSSOVER,100_000,- EF_SEARCH_FLOOR_CROSSOVER_HUGE - 1,- EF_SEARCH_FLOOR_CROSSOVER_HUGE,+ 2 * EF_SEARCH_FLOOR_CROSSOVER - 1,+ 2 * EF_SEARCH_FLOOR_CROSSOVER,+ 4 * EF_SEARCH_FLOOR_CROSSOVER - 1,+ 4 * EF_SEARCH_FLOOR_CROSSOVER,+ 8 * EF_SEARCH_FLOOR_CROSSOVER - 1,+ 8 * EF_SEARCH_FLOOR_CROSSOVER,1_000_000,+ 4_000_000,usize::MAX,] {let floor = ef_search_floor(live);@@ -2790,6 +2799,10 @@ mod tests {floor >= previous,"floor at {live} was {floor}, below {previous}");+ assert!(+ floor <= EF_SEARCH_FLOOR_MAX,+ "floor at {live} was {floor}, above the cap"+ );previous = floor;}}