Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Clear ML database tables atomically

laurenspriem3f965a85b693
diff --git a/mobile/apps/photos/changes/ml-store-atomic-clear.md b/mobile/apps/photos/changes/ml-store-atomic-clear.md
new file mode 100644
index 0000000000..a616e6e923
--- /dev/null
+++ b/mobile/apps/photos/changes/ml-store-atomic-clear.md
@@ -0,0 +1 @@
+- Fixed partial ML database clearing when a database error occurs with the Rust ML backend.
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
index fffd11f5b9..d410b67345 100644
--- a/rust/crates/photos/src/ml_db/mod.rs
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -37,6 +37,24 @@ pub struct MlDb {
db: Database,
}
+const CLEAR_NON_PET_TABLES: [&str; 8] = [
+ schema::DELETE_FACES,
+ schema::DELETE_FACE_CLUSTERS,
+ schema::DELETE_CLUSTER_PERSON,
+ schema::DELETE_CLUSTER_SUMMARY,
+ schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
+ schema::DELETE_NOT_PERSON_FEEDBACK,
+ schema::DELETE_CLIP_EMBEDDINGS,
+ schema::DELETE_FILE_DATA,
+];
+
+const CLEAR_PET_TABLES: [&str; 4] = [
+ schema::DELETE_PET_FACES,
+ schema::DELETE_PET_BODIES,
+ schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
+ schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
+];
+
impl MlDb {
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
Ok(Self {
@@ -56,25 +74,20 @@ impl MlDb {
}
pub fn clear_non_pet_tables(&self) -> Result<()> {
- self.execute_statements([
- schema::DELETE_FACES,
- schema::DELETE_FACE_CLUSTERS,
- schema::DELETE_CLUSTER_PERSON,
- schema::DELETE_CLUSTER_SUMMARY,
- schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,
- schema::DELETE_NOT_PERSON_FEEDBACK,
- schema::DELETE_CLIP_EMBEDDINGS,
- schema::DELETE_FILE_DATA,
- ])
+ self.execute_statements(CLEAR_NON_PET_TABLES)
}
pub fn clear_pet_tables(&self) -> Result<()> {
- self.execute_statements([
- schema::DELETE_PET_FACES,
- schema::DELETE_PET_BODIES,
- schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,
- schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,
- ])
+ self.execute_statements(CLEAR_PET_TABLES)
+ }
+
+ pub(crate) fn clear_all_tables(&self) -> Result<()> {
+ self.execute_statements(
+ CLEAR_NON_PET_TABLES
+ .into_iter()
+ .chain(CLEAR_PET_TABLES)
+ .chain([schema::DELETE_ML_STORE_META]),
+ )
}
}
diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rs
index ba0ff35435..c372803286 100644
--- a/rust/crates/photos/src/ml_store/writes.rs
+++ b/rust/crates/photos/src/ml_store/writes.rs
@@ -105,9 +105,7 @@ impl MlStore {
pub fn clear_all(&self) -> Result<()> {
let _mutations = self.lock_mutations();
- self.db.clear_non_pet_tables()?;
- self.db.clear_pet_tables()?;
- self.db.clear_meta()?;
+ self.db.clear_all_tables()?;
for slot in &self.indexes {
slot.purge()?;
}
@@ -195,6 +193,7 @@ mod tests {
use ente_vecdb::{OpenCost, SearchParams, VecDb, VecDbError};
+ use crate::db::Connection;
use crate::ml_db::tests::deny_cluster_summary_inserts_after;
use crate::ml_db::vector_encoding::encode_evector;
use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, ClusterSummary};
@@ -204,7 +203,7 @@ mod tests {
};
#[cfg(unix)]
use crate::ml_store::tests::{degrade_clip_index_through_a_failed_compaction, open_in};
- use crate::ml_store::{Error, FillOutcome, FillState, Index};
+ use crate::ml_store::{Error, FillOutcome, FillState, Index, MlStore};
#[test]
fn put_clip_round_trips_through_sql_and_index() {
@@ -560,6 +559,119 @@ mod tests {
assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Filled);
}
+ #[test]
+ fn clear_all_rolls_back_when_pet_clear_fails() {
+ assert_clear_all_rolls_back("pet_face_vector_id_map");
+ }
+
+ #[test]
+ fn clear_all_rolls_back_when_metadata_clear_fails() {
+ assert_clear_all_rolls_back("ml_store_meta");
+ }
+
+ fn assert_clear_all_rolls_back(table: &str) {
+ let (directory, store) = open();
+ let path = directory.path().join(DB_FILE);
+ store.put_clip(&clips([1])).unwrap();
+ store
+ .cluster_summary_update(&HashMap::from([centroid("c1", 1)]))
+ .unwrap();
+ store.fill_clip_index(false).unwrap();
+ store.fill_cluster_centroid_index(false).unwrap();
+ store
+ .db()
+ .put_repeated_text_embedding_cache("dog", &[0.5])
+ .unwrap();
+ store
+ .db()
+ .put_face_id_cached_for_person_or_cluster("p1", "1_0")
+ .unwrap();
+ let connection = Connection::open(&path).unwrap();
+ connection
+ .execute_batch(
+ "INSERT INTO pet_faces (file_id, pet_face_id, detection, species, score)
+ VALUES (1, 'pet_face', '{}', 1, 0.9);
+ INSERT INTO pet_bodies (file_id, pet_body_id, detection, species, score)
+ VALUES (1, 'pet_body', '{}', 1, 0.9);
+ INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('pet_face');
+ INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('pet_body');",
+ )
+ .unwrap();
+ let pet_rows = || {
+ connection
+ .query_row(
+ "SELECT (SELECT COUNT(*) FROM pet_faces)
+ + (SELECT COUNT(*) FROM pet_bodies)
+ + (SELECT COUNT(*) FROM pet_face_vector_id_map)
+ + (SELECT COUNT(*) FROM pet_body_vector_id_map)",
+ (),
+ |row| Ok(row.get::<_, i64>(0)?),
+ )
+ .unwrap()
+ };
+ connection
+ .execute_batch(&format!(
+ "CREATE TRIGGER fail_clear BEFORE DELETE ON {table}
+ BEGIN SELECT RAISE(ABORT, 'injected clear failure'); END;"
+ ))
+ .unwrap();
+
+ let error = store.clear_all().unwrap_err();
+ assert!(matches!(error, Error::Database(_)));
+ assert!(error.to_string().contains("injected clear failure"));
+ assert_eq!(store.db().count_clip_rows().unwrap(), 1);
+ assert_eq!(store.db().count_cluster_summaries().unwrap(), 1);
+ assert_eq!(pet_rows(), 4);
+ for index in Index::ALL {
+ assert_eq!(store.fill_state(index).unwrap(), FillState::Filled);
+ assert_eq!(live_count(&store, index), 1);
+ }
+ drop(store);
+
+ let reopened = MlStore::open(&path).unwrap();
+ assert_eq!(reopened.db().count_clip_rows().unwrap(), 1);
+ assert_eq!(reopened.db().count_cluster_summaries().unwrap(), 1);
+ assert_eq!(
+ reopened.fill_clip_index(false).unwrap().outcome,
+ FillOutcome::AlreadyFilled
+ );
+ assert_eq!(
+ reopened.fill_cluster_centroid_index(false).unwrap().outcome,
+ FillOutcome::AlreadyFilled
+ );
+ for (index, key) in [(Index::Clip, "1"), (Index::ClusterCentroid, "c1")] {
+ assert_eq!(nearest(&reopened, index, &one_hot(index.dims(), 1)), key);
+ }
+
+ connection.execute_batch("DROP TRIGGER fail_clear").unwrap();
+ reopened.clear_all().unwrap();
+ assert_eq!(reopened.db().count_clip_rows().unwrap(), 0);
+ assert_eq!(reopened.db().count_cluster_summaries().unwrap(), 0);
+ assert_eq!(pet_rows(), 0);
+ assert_eq!(
+ reopened
+ .db()
+ .get_repeated_text_embedding_cache("dog")
+ .unwrap(),
+ Some(vec![0.5])
+ );
+ assert_eq!(
+ reopened
+ .db()
+ .get_face_id_used_for_person_or_cluster("p1")
+ .unwrap()
+ .as_deref(),
+ Some("1_0")
+ );
+ drop(reopened);
+
+ let cleared = MlStore::open(&path).unwrap();
+ for index in Index::ALL {
+ assert_eq!(cleared.fill_state(index).unwrap(), FillState::Stale);
+ assert_eq!(live_count(&cleared, index), 0);
+ }
+ }
+
#[test]
fn index_write_failure_marks_the_index_stale() {
let (_directory, store) = open_with_unusable_clip_index();