π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
vecdb: retry a failed append rollback before the next write; search bulk similarity batches under per-query cutoffs
laurenspriem1d2b1e16c4b7
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rsindex b22c93c23b..f1c81698ff 100644--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs@@ -117,15 +117,6 @@ fn to_api_matches(matches: Vec<vecdb::Match>) -> Vec<VecDbMatch> {matches.into_iter().map(to_api_match).collect()}-fn distance_cut(matches: &mut Vec<VecDbMatch>, max_distance: f32) {- let keep = matches.partition_point(|entry| entry.distance <= max_distance);- matches.truncate(keep);-}--fn valid_distance(distance: f32) -> bool {- distance.is_finite() && distance >= 0.0-}-pub fn delete_vec_db_files(file_path: String) -> Result<(), RustVecDbError> {Ok(vecdb::VecDb::purge(Path::new(&file_path))?)}@@ -252,34 +243,17 @@ impl VecDb {),});}- let cuts: Vec<f32> = minimum_similarities+ let params: Vec<vecdb::SearchParams> = minimum_similarities.iter()- .map(|similarity| 1.0 - similarity)- .collect();- let widest = cuts- .iter()- .copied()- .filter(|cut| valid_distance(*cut))- .fold(f32::NEG_INFINITY, f32::max);- let params = vecdb::SearchParams {- limit: limit.map(|value| value as usize),- max_distance: Some(widest),- exact,- allowed_keys: None,- };- let results = self.inner.bulk_search(&queries, ¶ms)?;- Ok(results- .into_iter()- .zip(cuts)- .map(|(matches, cut)| {- if !valid_distance(cut) {- return Vec::new();- }- let mut matches = to_api_matches(matches);- distance_cut(&mut matches, cut);- matches+ .map(|similarity| vecdb::SearchParams {+ limit: limit.map(|value| value as usize),+ max_distance: Some(1.0 - similarity),+ exact,+ allowed_keys: None,})- .collect())+ .collect();+ let results = self.inner.bulk_search_varied(&queries, ¶ms)?;+ Ok(results.into_iter().map(to_api_matches).collect())}pub fn bulk_approx_filtered_search_vectors_within_distance(diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rsindex f25dd26ade..383dcc28d2 100644--- a/rust/crates/ml/src/vecdb/log.rs+++ b/rust/crates/ml/src/vecdb/log.rs@@ -63,6 +63,7 @@ pub(crate) struct Log {generation: [u8; 16],end_offset: u64,encode_buffer: Vec<u8>,+ rollback_pending: bool,}impl Log {@@ -107,6 +108,7 @@ impl Log {generation,end_offset: file_len,encode_buffer: Vec::new(),+ rollback_pending: false,})}@@ -126,6 +128,7 @@ impl Log {generation,end_offset: HEADER_LEN as u64,encode_buffer: Vec::new(),+ rollback_pending: false,})}@@ -172,13 +175,23 @@ impl Log {for entry in entries {validate_entry(entry, self.dims)?;}+ if self.rollback_pending {+ self.discard_unacked_tail()?;+ self.rollback_pending = false;+ }match self.write_records(entries, sync) {Ok(written) => {self.end_offset += written;Ok(())}Err(error) => {- self.discard_unacked_tail();+ if let Err(rollback_error) = self.discard_unacked_tail() {+ log::warn!(+ "failed to discard the unacked tail of {} after a write failure, deferring the rollback: {rollback_error}",+ self.path.display()+ );+ self.rollback_pending = true;+ }Err(error)}}@@ -205,9 +218,14 @@ impl Log {Ok(written)}- fn discard_unacked_tail(&mut self) {- let _ = self.file.set_len(self.end_offset);- let _ = self.file.sync_all();+ fn discard_unacked_tail(&mut self) -> Result<(), VecDbError> {+ self.file+ .set_len(self.end_offset)+ .map_err(|source| VecDbError::io(&self.path, source))?;+ self.file+ .sync_all()+ .map_err(|source| VecDbError::io(&self.path, source))?;+ Ok(())}fn drain_encode_buffer(&mut self) -> Result<u64, VecDbError> {@@ -1398,7 +1416,7 @@ mod tests {std::fs::metadata(&path).unwrap().len(),acked_end + phantom.len() as u64);- log.discard_unacked_tail();+ log.discard_unacked_tail().unwrap();assert_eq!(std::fs::metadata(&path).unwrap().len(), acked_end);drop(log);let mut reopened = reopen(&path, 8);@@ -1408,6 +1426,51 @@ mod tests {assert_eq!(recoverable_end, acked_end);}+ #[test]+ fn pending_rollback_truncates_stale_records_before_the_next_append() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("log");+ let mut log = Log::create(&path, 8).unwrap();+ let vector = seeded_vector(6, 8);+ let (_, acked_end) = append_bounds(+ &mut log,+ &[LogEntry::Add {+ key: "acked",+ vector: &vector,+ attrs: &[],+ }],+ );+ let stale = encoded_add("stale-with-a-much-longer-key", &vector);+ append_raw_bytes(&path, &stale);+ assert_eq!(+ std::fs::metadata(&path).unwrap().len(),+ acked_end + stale.len() as u64+ );+ log.rollback_pending = true;+ let fresh = seeded_vector(7, 8);+ let (start, end) = append_bounds(+ &mut log,+ &[LogEntry::Add {+ key: "fresh",+ vector: &fresh,+ attrs: &[],+ }],+ );+ assert!(!log.rollback_pending);+ assert_eq!(start, acked_end);+ assert!(end < acked_end + stale.len() as u64);+ assert_eq!(std::fs::metadata(&path).unwrap().len(), end);+ drop(log);+ let mut reopened = reopen(&path, 8);+ let (records, recoverable_end) = scan_all(&mut reopened);+ assert_eq!(records.len(), 2);+ assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "acked"));+ assert!(matches!(&records[1].0, LogRecord::Add { key, vector, .. }+ if key == "fresh" && vector == &fresh));+ assert_eq!(records[1].1, acked_end);+ assert_eq!(recoverable_end, end);+ }+#[test]fn unsynced_staging_appends_produce_bytes_identical_to_synced_appends() {let dir = TempDir::new().unwrap();diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rsindex 18335341ee..60c0c15691 100644--- a/rust/crates/ml/src/vecdb/store.rs+++ b/rust/crates/ml/src/vecdb/store.rs@@ -516,6 +516,34 @@ impl VecDb {.collect()}+ pub fn bulk_search_varied(+ &self,+ queries: &[Vec<f32>],+ params: &[SearchParams],+ ) -> Result<Vec<Vec<Match>>, VecDbError> {+ if queries.len() != params.len() {+ return Err(VecDbError::LengthMismatch {+ keys: queries.len(),+ vectors: params.len(),+ });+ }+ self.shared.ensure_open()?;+ let st = self.shared.state_read();+ self.shared.ensure_open()?;+ queries+ .iter()+ .zip(params)+ .map(|(query, query_params)| {+ if degenerate_distance_cut(query_params)? {+ return Ok(Vec::new());+ }+ let allowed_slots =+ resolve_allowed_slots(&st, query_params.allowed_keys.as_deref());+ search_in_state(&st, query, query_params, allowed_slots.as_ref())+ })+ .collect()+ }+pub fn bulk_search_stored(&self,keys: &[String],@@ -3388,6 +3416,89 @@ mod tests {assert!(db.bulk_search(&[], &limit_params(3)).unwrap().is_empty());}+ #[test]+ fn bulk_search_varied_matches_per_query_search() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let db = open_writer(&path);+ bulk_add(&db, &bulk_entries(0, 200, 640)).unwrap();+ let queries: Vec<Vec<f32>> = (0..5)+ .map(|index| seeded_unit_vector(9600 + index, DIMS))+ .collect();+ let params = vec![+ SearchParams {+ limit: Some(8),+ max_distance: Some(2.0),+ exact: true,+ ..SearchParams::default()+ },+ SearchParams {+ limit: Some(4),+ max_distance: Some(0.9),+ exact: true,+ ..SearchParams::default()+ },+ SearchParams {+ limit: None,+ max_distance: Some(1.05),+ exact: true,+ ..SearchParams::default()+ },+ SearchParams {+ limit: Some(6),+ exact: true,+ allowed_keys: Some(vec![+ "key-3".to_string(),+ "key-77".to_string(),+ "key-150".to_string(),+ "absent".to_string(),+ ]),+ ..SearchParams::default()+ },+ SearchParams {+ limit: Some(3),+ max_distance: Some(-0.25),+ exact: true,+ ..SearchParams::default()+ },+ ];+ let varied = db.bulk_search_varied(&queries, ¶ms).unwrap();+ assert_eq!(varied.len(), queries.len());+ for ((query, shape), matches) in queries.iter().zip(¶ms).zip(&varied) {+ assert_eq!(matches, &db.search(query, shape).unwrap());+ }+ assert_eq!(varied[0].len(), 8);+ assert!(varied[4].is_empty());+ assert!(varied[..4].iter().all(|matches| !matches.is_empty()));+ let approx: Vec<SearchParams> = params+ .iter()+ .cloned()+ .map(|shape| SearchParams {+ exact: false,+ ..shape+ })+ .collect();+ let approx_varied = db.bulk_search_varied(&queries, &approx).unwrap();+ for ((query, shape), matches) in queries.iter().zip(&approx).zip(&approx_varied) {+ assert_eq!(matches, &db.search(query, shape).unwrap());+ }+ assert!(approx_varied[4].is_empty());+ let mut unbounded = params.clone();+ unbounded[2] = SearchParams::default();+ assert!(matches!(+ db.bulk_search_varied(&queries, &unbounded),+ Err(VecDbError::UnboundedSearch)+ ));+ assert!(matches!(+ db.bulk_search_varied(&queries, ¶ms[..3]),+ Err(VecDbError::LengthMismatch {+ keys: 5,+ vectors: 3+ })+ ));+ assert!(db.bulk_search_varied(&[], &[]).unwrap().is_empty());+ }+#[test]fn bulk_search_stored_preserves_order_skips_absent_and_excludes_self() {let dir = TempDir::new().unwrap();