Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

vecdb: retry a failed append rollback before the next write; search bulk similarity batches under per-query cutoffs

laurenspriem1d2b1e16c4b7
diff --git a/rust/bindings/frb/photos/src/api/vecdb_api.rs b/rust/bindings/frb/photos/src/api/vecdb_api.rs
index b22c93c23b..f1c81698ff 100644
--- a/rust/bindings/frb/photos/src/api/vecdb_api.rs
+++ b/rust/bindings/frb/photos/src/api/vecdb_api.rs
@@ -117,15 +117,6 @@ fn to_api_matches(matches: Vec<vecdb::Match>) -> Vec<VecDbMatch> {
matches.into_iter().map(to_api_match).collect()
}
-fn distance_cut(matches: &mut Vec<VecDbMatch>, max_distance: f32) {
- let keep = matches.partition_point(|entry| entry.distance <= max_distance);
- matches.truncate(keep);
-}
-
-fn valid_distance(distance: f32) -> bool {
- distance.is_finite() && distance >= 0.0
-}
-
pub fn delete_vec_db_files(file_path: String) -> Result<(), RustVecDbError> {
Ok(vecdb::VecDb::purge(Path::new(&file_path))?)
}
@@ -252,34 +243,17 @@ impl VecDb {
),
});
}
- let cuts: Vec<f32> = minimum_similarities
+ let params: Vec<vecdb::SearchParams> = minimum_similarities
.iter()
- .map(|similarity| 1.0 - similarity)
- .collect();
- let widest = cuts
- .iter()
- .copied()
- .filter(|cut| valid_distance(*cut))
- .fold(f32::NEG_INFINITY, f32::max);
- let params = vecdb::SearchParams {
- limit: limit.map(|value| value as usize),
- max_distance: Some(widest),
- exact,
- allowed_keys: None,
- };
- let results = self.inner.bulk_search(&queries, &params)?;
- Ok(results
- .into_iter()
- .zip(cuts)
- .map(|(matches, cut)| {
- if !valid_distance(cut) {
- return Vec::new();
- }
- let mut matches = to_api_matches(matches);
- distance_cut(&mut matches, cut);
- matches
+ .map(|similarity| vecdb::SearchParams {
+ limit: limit.map(|value| value as usize),
+ max_distance: Some(1.0 - similarity),
+ exact,
+ allowed_keys: None,
})
- .collect())
+ .collect();
+ let results = self.inner.bulk_search_varied(&queries, &params)?;
+ Ok(results.into_iter().map(to_api_matches).collect())
}
pub fn bulk_approx_filtered_search_vectors_within_distance(
diff --git a/rust/crates/ml/src/vecdb/log.rs b/rust/crates/ml/src/vecdb/log.rs
index f25dd26ade..383dcc28d2 100644
--- a/rust/crates/ml/src/vecdb/log.rs
+++ b/rust/crates/ml/src/vecdb/log.rs
@@ -63,6 +63,7 @@ pub(crate) struct Log {
generation: [u8; 16],
end_offset: u64,
encode_buffer: Vec<u8>,
+ rollback_pending: bool,
}
impl Log {
@@ -107,6 +108,7 @@ impl Log {
generation,
end_offset: file_len,
encode_buffer: Vec::new(),
+ rollback_pending: false,
})
}
@@ -126,6 +128,7 @@ impl Log {
generation,
end_offset: HEADER_LEN as u64,
encode_buffer: Vec::new(),
+ rollback_pending: false,
})
}
@@ -172,13 +175,23 @@ impl Log {
for entry in entries {
validate_entry(entry, self.dims)?;
}
+ if self.rollback_pending {
+ self.discard_unacked_tail()?;
+ self.rollback_pending = false;
+ }
match self.write_records(entries, sync) {
Ok(written) => {
self.end_offset += written;
Ok(())
}
Err(error) => {
- self.discard_unacked_tail();
+ if let Err(rollback_error) = self.discard_unacked_tail() {
+ log::warn!(
+ "failed to discard the unacked tail of {} after a write failure, deferring the rollback: {rollback_error}",
+ self.path.display()
+ );
+ self.rollback_pending = true;
+ }
Err(error)
}
}
@@ -205,9 +218,14 @@ impl Log {
Ok(written)
}
- fn discard_unacked_tail(&mut self) {
- let _ = self.file.set_len(self.end_offset);
- let _ = self.file.sync_all();
+ fn discard_unacked_tail(&mut self) -> Result<(), VecDbError> {
+ self.file
+ .set_len(self.end_offset)
+ .map_err(|source| VecDbError::io(&self.path, source))?;
+ self.file
+ .sync_all()
+ .map_err(|source| VecDbError::io(&self.path, source))?;
+ Ok(())
}
fn drain_encode_buffer(&mut self) -> Result<u64, VecDbError> {
@@ -1398,7 +1416,7 @@ mod tests {
std::fs::metadata(&path).unwrap().len(),
acked_end + phantom.len() as u64
);
- log.discard_unacked_tail();
+ log.discard_unacked_tail().unwrap();
assert_eq!(std::fs::metadata(&path).unwrap().len(), acked_end);
drop(log);
let mut reopened = reopen(&path, 8);
@@ -1408,6 +1426,51 @@ mod tests {
assert_eq!(recoverable_end, acked_end);
}
+ #[test]
+ fn pending_rollback_truncates_stale_records_before_the_next_append() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("log");
+ let mut log = Log::create(&path, 8).unwrap();
+ let vector = seeded_vector(6, 8);
+ let (_, acked_end) = append_bounds(
+ &mut log,
+ &[LogEntry::Add {
+ key: "acked",
+ vector: &vector,
+ attrs: &[],
+ }],
+ );
+ let stale = encoded_add("stale-with-a-much-longer-key", &vector);
+ append_raw_bytes(&path, &stale);
+ assert_eq!(
+ std::fs::metadata(&path).unwrap().len(),
+ acked_end + stale.len() as u64
+ );
+ log.rollback_pending = true;
+ let fresh = seeded_vector(7, 8);
+ let (start, end) = append_bounds(
+ &mut log,
+ &[LogEntry::Add {
+ key: "fresh",
+ vector: &fresh,
+ attrs: &[],
+ }],
+ );
+ assert!(!log.rollback_pending);
+ assert_eq!(start, acked_end);
+ assert!(end < acked_end + stale.len() as u64);
+ assert_eq!(std::fs::metadata(&path).unwrap().len(), end);
+ drop(log);
+ let mut reopened = reopen(&path, 8);
+ let (records, recoverable_end) = scan_all(&mut reopened);
+ assert_eq!(records.len(), 2);
+ assert!(matches!(&records[0].0, LogRecord::Add { key, .. } if key == "acked"));
+ assert!(matches!(&records[1].0, LogRecord::Add { key, vector, .. }
+ if key == "fresh" && vector == &fresh));
+ assert_eq!(records[1].1, acked_end);
+ assert_eq!(recoverable_end, end);
+ }
+
#[test]
fn unsynced_staging_appends_produce_bytes_identical_to_synced_appends() {
let dir = TempDir::new().unwrap();
diff --git a/rust/crates/ml/src/vecdb/store.rs b/rust/crates/ml/src/vecdb/store.rs
index 18335341ee..60c0c15691 100644
--- a/rust/crates/ml/src/vecdb/store.rs
+++ b/rust/crates/ml/src/vecdb/store.rs
@@ -516,6 +516,34 @@ impl VecDb {
.collect()
}
+ pub fn bulk_search_varied(
+ &self,
+ queries: &[Vec<f32>],
+ params: &[SearchParams],
+ ) -> Result<Vec<Vec<Match>>, VecDbError> {
+ if queries.len() != params.len() {
+ return Err(VecDbError::LengthMismatch {
+ keys: queries.len(),
+ vectors: params.len(),
+ });
+ }
+ self.shared.ensure_open()?;
+ let st = self.shared.state_read();
+ self.shared.ensure_open()?;
+ queries
+ .iter()
+ .zip(params)
+ .map(|(query, query_params)| {
+ if degenerate_distance_cut(query_params)? {
+ return Ok(Vec::new());
+ }
+ let allowed_slots =
+ resolve_allowed_slots(&st, query_params.allowed_keys.as_deref());
+ search_in_state(&st, query, query_params, allowed_slots.as_ref())
+ })
+ .collect()
+ }
+
pub fn bulk_search_stored(
&self,
keys: &[String],
@@ -3388,6 +3416,89 @@ mod tests {
assert!(db.bulk_search(&[], &limit_params(3)).unwrap().is_empty());
}
+ #[test]
+ fn bulk_search_varied_matches_per_query_search() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let db = open_writer(&path);
+ bulk_add(&db, &bulk_entries(0, 200, 640)).unwrap();
+ let queries: Vec<Vec<f32>> = (0..5)
+ .map(|index| seeded_unit_vector(9600 + index, DIMS))
+ .collect();
+ let params = vec![
+ SearchParams {
+ limit: Some(8),
+ max_distance: Some(2.0),
+ exact: true,
+ ..SearchParams::default()
+ },
+ SearchParams {
+ limit: Some(4),
+ max_distance: Some(0.9),
+ exact: true,
+ ..SearchParams::default()
+ },
+ SearchParams {
+ limit: None,
+ max_distance: Some(1.05),
+ exact: true,
+ ..SearchParams::default()
+ },
+ SearchParams {
+ limit: Some(6),
+ exact: true,
+ allowed_keys: Some(vec![
+ "key-3".to_string(),
+ "key-77".to_string(),
+ "key-150".to_string(),
+ "absent".to_string(),
+ ]),
+ ..SearchParams::default()
+ },
+ SearchParams {
+ limit: Some(3),
+ max_distance: Some(-0.25),
+ exact: true,
+ ..SearchParams::default()
+ },
+ ];
+ let varied = db.bulk_search_varied(&queries, &params).unwrap();
+ assert_eq!(varied.len(), queries.len());
+ for ((query, shape), matches) in queries.iter().zip(&params).zip(&varied) {
+ assert_eq!(matches, &db.search(query, shape).unwrap());
+ }
+ assert_eq!(varied[0].len(), 8);
+ assert!(varied[4].is_empty());
+ assert!(varied[..4].iter().all(|matches| !matches.is_empty()));
+ let approx: Vec<SearchParams> = params
+ .iter()
+ .cloned()
+ .map(|shape| SearchParams {
+ exact: false,
+ ..shape
+ })
+ .collect();
+ let approx_varied = db.bulk_search_varied(&queries, &approx).unwrap();
+ for ((query, shape), matches) in queries.iter().zip(&approx).zip(&approx_varied) {
+ assert_eq!(matches, &db.search(query, shape).unwrap());
+ }
+ assert!(approx_varied[4].is_empty());
+ let mut unbounded = params.clone();
+ unbounded[2] = SearchParams::default();
+ assert!(matches!(
+ db.bulk_search_varied(&queries, &unbounded),
+ Err(VecDbError::UnboundedSearch)
+ ));
+ assert!(matches!(
+ db.bulk_search_varied(&queries, &params[..3]),
+ Err(VecDbError::LengthMismatch {
+ keys: 5,
+ vectors: 3
+ })
+ ));
+ assert!(db.bulk_search_varied(&[], &[]).unwrap().is_empty());
+ }
+
#[test]
fn bulk_search_stored_preserves_order_skips_absent_and_excludes_self() {
let dir = TempDir::new().unwrap();