Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Purge the ML store index when an index write fails

laurenspriemd867e7fe3cba
diff --git a/rust/crates/photos/src/ml_store/fill.rs b/rust/crates/photos/src/ml_store/fill.rs
index 0d1ed37343..820ac99332 100644
--- a/rust/crates/photos/src/ml_store/fill.rs
+++ b/rust/crates/photos/src/ml_store/fill.rs
@@ -169,7 +169,7 @@ impl MlStore {
}),
(true, _) | (false, FillState::Stale) => {
state::mark_stale(&self.db, index)?;
- self.with_index(index, VecDb::reset)?;
+ self.index_write(index, VecDb::reset)?;
state::mark_filling(&self.db, index)?;
Ok(Start::Fresh)
}
diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rs
index e4c4346ac8..3df7acc8db 100644
--- a/rust/crates/photos/src/ml_store/mod.rs
+++ b/rust/crates/photos/src/ml_store/mod.rs
@@ -254,6 +254,19 @@ mod tests {
}
}
+ #[cfg(unix)]
+ pub(super) fn degrade_clip_index_through_a_failed_compaction(
+ directory: &TempDir,
+ store: &MlStore,
+ ) {
+ store.put_clip(&clips(0..70)).unwrap();
+ let log = index_path(directory, Index::Clip);
+ fs::remove_file(&log).unwrap();
+ fs::create_dir(&log).unwrap();
+ store.delete_clip(&(0..65).collect::<Vec<_>>()).unwrap();
+ fs::remove_dir(&log).unwrap();
+ }
+
pub(super) fn lose_index_files(store: &MlStore, path: &Path) {
store.release().unwrap();
VecDb::purge(path).unwrap();
diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rs
index f520d8fe97..facea47261 100644
--- a/rust/crates/photos/src/ml_store/writes.rs
+++ b/rust/crates/photos/src/ml_store/writes.rs
@@ -125,7 +125,7 @@ impl MlStore {
}
}
- fn index_write<T>(
+ pub(super) fn index_write<T>(
&self,
index: Index,
operation: impl Fn(&VecDb) -> IndexResult<T>,
@@ -133,6 +133,9 @@ impl MlStore {
let outcome = self.with_index(index, operation);
if let Err(Error::Index(_)) = &outcome {
state::mark_stale(&self.db, index)?;
+ if let Err(error) = self.indexes[index.position()].purge() {
+ log::warn!("could not purge the stale {} index: {error}", index.name());
+ }
}
outcome.map(drop)
}
@@ -184,6 +187,8 @@ mod tests {
DB_FILE, centroid, clips, empty, hot_position, index_path, live_count, meta, nearest,
one_hot, open, open_with_unusable_clip_index,
};
+ #[cfg(unix)]
+ use crate::ml_store::tests::{degrade_clip_index_through_a_failed_compaction, open_in};
use crate::ml_store::{Error, FillOutcome, FillState, Index};
#[test]
@@ -554,4 +559,75 @@ mod tests {
store.put_clip(&clips([2])).unwrap();
assert_eq!(store.db().count_clip_rows().unwrap(), 2);
}
+
+ #[cfg(unix)]
+ #[test]
+ fn failed_index_writes_leave_no_vectors_of_rows_deleted_while_stale() {
+ let (directory, store) = open();
+ let other = open_in(&directory);
+ store.fill_clip_index(false).unwrap();
+ assert_eq!(other.fill_state(Index::Clip).unwrap(), FillState::Filled);
+ degrade_clip_index_through_a_failed_compaction(&directory, &store);
+ assert!(other.contains(Index::Clip, "66").unwrap());
+
+ assert!(matches!(
+ store.delete_clip(&[66]),
+ Err(Error::Index(VecDbError::ReadOnly))
+ ));
+ store.delete_clip(&[67]).unwrap();
+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);
+ assert_eq!(store.db().count_clip_rows().unwrap(), 3);
+ let query = one_hot(CLIP_EMBEDDING_DIMENSIONS, 66);
+ let params = SearchParams {
+ max_distance: Some(0.5),
+ exact: true,
+ ..SearchParams::default()
+ };
+ for reader in [&store, &other] {
+ assert!(!reader.contains(Index::Clip, "66").unwrap());
+ assert!(!reader.contains(Index::Clip, "67").unwrap());
+ assert!(
+ reader
+ .search(Index::Clip, &query, &params)
+ .unwrap()
+ .is_empty()
+ );
+ assert_eq!(live_count(reader, Index::Clip), 0);
+ }
+
+ let refill = store.fill_clip_index(false).unwrap();
+ assert_eq!(
+ (refill.outcome, refill.indexed),
+ (FillOutcome::Completed, 3)
+ );
+ assert_eq!(other.fill_state(Index::Clip).unwrap(), FillState::Filled);
+ assert_eq!(live_count(&other, Index::Clip), 3);
+ assert!(other.contains(Index::Clip, "65").unwrap());
+ assert!(!other.contains(Index::Clip, "66").unwrap());
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn failed_fill_resets_leave_no_vectors_of_rows_deleted_while_stale() {
+ let (directory, store) = open();
+ store.fill_clip_index(false).unwrap();
+ degrade_clip_index_through_a_failed_compaction(&directory, &store);
+
+ assert!(matches!(
+ store.fill_clip_index(true),
+ Err(Error::Index(VecDbError::ReadOnly))
+ ));
+ assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Stale);
+ store.delete_clip(&[66]).unwrap();
+ assert!(!store.contains(Index::Clip, "66").unwrap());
+ assert_eq!(live_count(&store, Index::Clip), 0);
+
+ let refill = store.fill_clip_index(false).unwrap();
+ assert_eq!(
+ (refill.outcome, refill.indexed),
+ (FillOutcome::Completed, 4)
+ );
+ assert!(store.contains(Index::Clip, "65").unwrap());
+ assert!(!store.contains(Index::Clip, "66").unwrap());
+ }
}
diff --git a/rust/crates/vecdb/src/store.rs b/rust/crates/vecdb/src/store.rs
index d7b7442a78..ea60818f87 100644
--- a/rust/crates/vecdb/src/store.rs
+++ b/rust/crates/vecdb/src/store.rs
@@ -1072,7 +1072,11 @@ fn open_cost_from_files(path: &Path) -> OpenCost {
fn close_live_instance(shared: &Arc<Shared>) -> Result<(), VecDbError> {
let mut half = shared.writer_half();
shared.ensure_open()?;
- let WriterState { lock, log, .. } = take_writer_state(&mut half)?;
+ let (lock, log) = match take_writer_state(&mut half) {
+ Ok(WriterState { lock, log, .. }) => (lock, Some(log)),
+ Err(VecDbError::ReadOnly) => (WriterLock::acquire(&shared.path)?, None),
+ Err(error) => return Err(error),
+ };
let mut st = shared.state_write();
shared.closed.store(true, Ordering::Release);
drop(log);
@@ -5616,6 +5620,42 @@ mod tests {
assert_key_reads_open(&fresh, "anew", 1);
}
+ #[cfg(unix)]
+ #[test]
+ fn purge_closes_a_live_instance_degraded_to_read_only() {
+ let dir = TempDir::new().unwrap();
+ let path = dir.path().join("db");
+ let first = open_writer(&path);
+ let survivor = VecDb::open(&path, DIMS, Some(StorageKind::F32)).unwrap();
+ bulk_add(&first, &bulk_entries(0, 70, 955)).unwrap();
+ fs::remove_file(&path).unwrap();
+ fs::create_dir(&path).unwrap();
+ let removals: Vec<String> = (0..65).map(|index| format!("key-{index}")).collect();
+ assert_eq!(first.bulk_remove(&removals).unwrap(), 65);
+ fs::remove_dir(&path).unwrap();
+ assert!(is_degraded(&first));
+ assert!(matches!(
+ survivor.remove("key-66"),
+ Err(VecDbError::ReadOnly)
+ ));
+ assert_key_reads_open(&survivor, "key-66", 5);
+
+ let held = WriterLock::acquire(&path).unwrap();
+ assert!(matches!(VecDb::purge(&path), Err(VecDbError::Locked(_))));
+ assert_key_reads_open(&first, "key-66", 5);
+ assert_key_reads_open(&survivor, "key-66", 5);
+ drop(held);
+
+ VecDb::purge(&path).unwrap();
+ assert_key_reads_closed(&first);
+ assert_key_reads_closed(&survivor);
+ assert!(lock_path(&path).exists());
+ let fresh = open_writer(&path);
+ assert!(fresh.is_empty().unwrap());
+ fresh.add("anew", &seeded_unit_vector(2, DIMS)).unwrap();
+ assert_key_reads_open(&fresh, "anew", 1);
+ }
+
#[test]
fn purge_without_a_live_instance_removes_files_and_tolerates_missing_ones() {
let dir = TempDir::new().unwrap();