π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Purge the ML store index when an index write fails
laurenspriemd867e7fe3cba
diff --git a/rust/crates/photos/src/ml_store/fill.rs b/rust/crates/photos/src/ml_store/fill.rsindex 0d1ed37343..820ac99332 100644--- a/rust/crates/photos/src/ml_store/fill.rs+++ b/rust/crates/photos/src/ml_store/fill.rs@@ -169,7 +169,7 @@ impl MlStore {}),(true, _) | (false, FillState::Stale) => {state::mark_stale(&self.db, index)?;- self.with_index(index, VecDb::reset)?;+ self.index_write(index, VecDb::reset)?;state::mark_filling(&self.db, index)?;Ok(Start::Fresh)}diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rsindex e4c4346ac8..3df7acc8db 100644--- a/rust/crates/photos/src/ml_store/mod.rs+++ b/rust/crates/photos/src/ml_store/mod.rs@@ -254,6 +254,19 @@ mod tests {}}+ #[cfg(unix)]+ pub(super) fn degrade_clip_index_through_a_failed_compaction(+ directory: &TempDir,+ store: &MlStore,+ ) {+ store.put_clip(&clips(0..70)).unwrap();+ let log = index_path(directory, Index::Clip);+ fs::remove_file(&log).unwrap();+ fs::create_dir(&log).unwrap();+ store.delete_clip(&(0..65).collect::<Vec<_>>()).unwrap();+ fs::remove_dir(&log).unwrap();+ }+pub(super) fn lose_index_files(store: &MlStore, path: &Path) {store.release().unwrap();VecDb::purge(path).unwrap();diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rsindex f520d8fe97..facea47261 100644--- a/rust/crates/photos/src/ml_store/writes.rs+++ b/rust/crates/photos/src/ml_store/writes.rs@@ -125,7 +125,7 @@ impl MlStore {}}- fn index_write<T>(+ pub(super) fn index_write<T>(&self,index: Index,operation: impl Fn(&VecDb) -> IndexResult<T>,@@ -133,6 +133,9 @@ impl MlStore {let outcome = self.with_index(index, operation);if let Err(Error::Index(_)) = &outcome {state::mark_stale(&self.db, index)?;+ if let Err(error) = self.indexes[index.position()].purge() {+ log::warn!("could not purge the stale {} index: {error}", index.name());+ }}outcome.map(drop)}@@ -184,6 +187,8 @@ mod tests {DB_FILE, centroid, clips, empty, hot_position, index_path, live_count, meta, nearest,one_hot, open, open_with_unusable_clip_index,};+ #[cfg(unix)]+ use crate::ml_store::tests::{degrade_clip_index_through_a_failed_compaction, open_in};use crate::ml_store::{Error, FillOutcome, FillState, Index};#[test]@@ -554,4 +559,75 @@ mod tests {store.put_clip(&clips([2])).unwrap();assert_eq!(store.db().count_clip_rows().unwrap(), 2);}++ #[cfg(unix)]+ #[test]+ fn failed_index_writes_leave_no_vectors_of_rows_deleted_while_stale() {+ let (directory, store) = open();+ let other = open_in(&directory);+ store.fill_clip_index(false).unwrap();+ assert_eq!(other.fill_state(Index::Clip).unwrap(), FillState::Filled);+ degrade_clip_index_through_a_failed_compaction(&directory, &store);+ assert!(other.contains(Index::Clip, "66").unwrap());++ assert!(matches!(+ store.delete_clip(&[66]),+ Err(Error::Index(VecDbError::ReadOnly))+ ));+ store.delete_clip(&[67]).unwrap();+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);+ assert_eq!(store.db().count_clip_rows().unwrap(), 3);+ let query = one_hot(CLIP_EMBEDDING_DIMENSIONS, 66);+ let params = SearchParams {+ max_distance: Some(0.5),+ exact: true,+ ..SearchParams::default()+ };+ for reader in [&store, &other] {+ assert!(!reader.contains(Index::Clip, "66").unwrap());+ assert!(!reader.contains(Index::Clip, "67").unwrap());+ assert!(+ reader+ .search(Index::Clip, &query, ¶ms)+ .unwrap()+ .is_empty()+ );+ assert_eq!(live_count(reader, Index::Clip), 0);+ }++ let refill = store.fill_clip_index(false).unwrap();+ assert_eq!(+ (refill.outcome, refill.indexed),+ (FillOutcome::Completed, 3)+ );+ assert_eq!(other.fill_state(Index::Clip).unwrap(), FillState::Filled);+ assert_eq!(live_count(&other, Index::Clip), 3);+ assert!(other.contains(Index::Clip, "65").unwrap());+ assert!(!other.contains(Index::Clip, "66").unwrap());+ }++ #[cfg(unix)]+ #[test]+ fn failed_fill_resets_leave_no_vectors_of_rows_deleted_while_stale() {+ let (directory, store) = open();+ store.fill_clip_index(false).unwrap();+ degrade_clip_index_through_a_failed_compaction(&directory, &store);++ assert!(matches!(+ store.fill_clip_index(true),+ Err(Error::Index(VecDbError::ReadOnly))+ ));+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);+ store.delete_clip(&[66]).unwrap();+ assert!(!store.contains(Index::Clip, "66").unwrap());+ assert_eq!(live_count(&store, Index::Clip), 0);++ let refill = store.fill_clip_index(false).unwrap();+ assert_eq!(+ (refill.outcome, refill.indexed),+ (FillOutcome::Completed, 4)+ );+ assert!(store.contains(Index::Clip, "65").unwrap());+ assert!(!store.contains(Index::Clip, "66").unwrap());+ }}diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rsindex d7b7442a78..ea60818f87 100644--- a/rust/crates/vecdb/src/store.rs+++ b/rust/crates/vecdb/src/store.rs@@ -1072,7 +1072,11 @@ fn open_cost_from_files(path: &Path) -> OpenCost {fn close_live_instance(shared: &Arc<Shared>) -> Result<(), VecDbError> {let mut half = shared.writer_half();shared.ensure_open()?;- let WriterState { lock, log, .. } = take_writer_state(&mut half)?;+ let (lock, log) = match take_writer_state(&mut half) {+ Ok(WriterState { lock, log, .. }) => (lock, Some(log)),+ Err(VecDbError::ReadOnly) => (WriterLock::acquire(&shared.path)?, None),+ Err(error) => return Err(error),+ };let mut st = shared.state_write();shared.closed.store(true, Ordering::Release);drop(log);@@ -5616,6 +5620,42 @@ mod tests {assert_key_reads_open(&fresh, "anew", 1);}+ #[cfg(unix)]+ #[test]+ fn purge_closes_a_live_instance_degraded_to_read_only() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let first = open_writer(&path);+ let survivor = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();+ bulk_add(&first, &bulk_entries(0, 70, 955)).unwrap();+ fs::remove_file(&path).unwrap();+ fs::create_dir(&path).unwrap();+ let removals: Vec<String> = (0..65).map(|index| format!("key-{index}")).collect();+ assert_eq!(first.bulk_remove(&removals).unwrap(), 65);+ fs::remove_dir(&path).unwrap();+ assert!(is_degraded(&first));+ assert!(matches!(+ survivor.remove("key-66"),+ Err(VecDbError::ReadOnly)+ ));+ assert_key_reads_open(&survivor, "key-66", 5);++ let held = WriterLock::acquire(&path).unwrap();+ assert!(matches!(VecDb::purge(&path), Err(VecDbError::Locked(_))));+ assert_key_reads_open(&first, "key-66", 5);+ assert_key_reads_open(&survivor, "key-66", 5);+ drop(held);++ VecDb::purge(&path).unwrap();+ assert_key_reads_closed(&first);+ assert_key_reads_closed(&survivor);+ assert!(lock_path(&path).exists());+ let fresh = open_writer(&path);+ assert!(fresh.is_empty().unwrap());+ fresh.add("anew", &seeded_unit_vector(2, DIMS)).unwrap();+ assert_key_reads_open(&fresh, "anew", 1);+ }+#[test]fn purge_without_a_live_instance_removes_files_and_tolerates_missing_ones() {let dir = TempDir::new().unwrap();