π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[rust] Clear ML database tables atomically
laurenspriem3f965a85b693
diff --git a/mobile/apps/photos/changes/ml-store-atomic-clear.md b/mobile/apps/photos/changes/ml-store-atomic-clear.mdnew file mode 100644index 0000000000..a616e6e923--- /dev/null+++ b/mobile/apps/photos/changes/ml-store-atomic-clear.md@@ -0,0 +1 @@+- Fixed partial ML database clearing when a database error occurs with the Rust ML backend.diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rsindex fffd11f5b9..d410b67345 100644--- a/rust/crates/photos/src/ml_db/mod.rs+++ b/rust/crates/photos/src/ml_db/mod.rs@@ -37,6 +37,24 @@ pub struct MlDb {db: Database,}+const CLEAR_NON_PET_TABLES: [&str; 8] = [+ schema::DELETE_FACES,+ schema::DELETE_FACE_CLUSTERS,+ schema::DELETE_CLUSTER_PERSON,+ schema::DELETE_CLUSTER_SUMMARY,+ schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,+ schema::DELETE_NOT_PERSON_FEEDBACK,+ schema::DELETE_CLIP_EMBEDDINGS,+ schema::DELETE_FILE_DATA,+];++const CLEAR_PET_TABLES: [&str; 4] = [+ schema::DELETE_PET_FACES,+ schema::DELETE_PET_BODIES,+ schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,+ schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,+];+impl MlDb {pub fn open(path: impl AsRef<Path>) -> Result<Self> {Ok(Self {@@ -56,25 +74,20 @@ impl MlDb {}pub fn clear_non_pet_tables(&self) -> Result<()> {- self.execute_statements([- schema::DELETE_FACES,- schema::DELETE_FACE_CLUSTERS,- schema::DELETE_CLUSTER_PERSON,- schema::DELETE_CLUSTER_SUMMARY,- schema::DELETE_CLUSTER_CENTROID_VECTOR_ID_MAPPING,- schema::DELETE_NOT_PERSON_FEEDBACK,- schema::DELETE_CLIP_EMBEDDINGS,- schema::DELETE_FILE_DATA,- ])+ self.execute_statements(CLEAR_NON_PET_TABLES)}pub fn clear_pet_tables(&self) -> Result<()> {- self.execute_statements([- schema::DELETE_PET_FACES,- schema::DELETE_PET_BODIES,- schema::DELETE_PET_FACE_VECTOR_ID_MAPPING,- schema::DELETE_PET_BODY_VECTOR_ID_MAPPING,- ])+ self.execute_statements(CLEAR_PET_TABLES)+ }++ pub(crate) fn clear_all_tables(&self) -> Result<()> {+ self.execute_statements(+ CLEAR_NON_PET_TABLES+ .into_iter()+ .chain(CLEAR_PET_TABLES)+ .chain([schema::DELETE_ML_STORE_META]),+ )}}diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rsindex ba0ff35435..c372803286 100644--- a/rust/crates/photos/src/ml_store/writes.rs+++ b/rust/crates/photos/src/ml_store/writes.rs@@ -105,9 +105,7 @@ impl MlStore {pub fn clear_all(&self) -> Result<()> {let _mutations = self.lock_mutations();- self.db.clear_non_pet_tables()?;- self.db.clear_pet_tables()?;- self.db.clear_meta()?;+ self.db.clear_all_tables()?;for slot in &self.indexes {slot.purge()?;}@@ -195,6 +193,7 @@ mod tests {use ente_vecdb::{OpenCost, SearchParams, VecDb, VecDbError};+ use crate::db::Connection;use crate::ml_db::tests::deny_cluster_summary_inserts_after;use crate::ml_db::vector_encoding::encode_evector;use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, ClusterSummary};@@ -204,7 +203,7 @@ mod tests {};#[cfg(unix)]use crate::ml_store::tests::{degrade_clip_index_through_a_failed_compaction, open_in};- use crate::ml_store::{Error, FillOutcome, FillState, Index};+ use crate::ml_store::{Error, FillOutcome, FillState, Index, MlStore};#[test]fn put_clip_round_trips_through_sql_and_index() {@@ -560,6 +559,119 @@ mod tests {assert_eq!(store.fill_state(Index::Clip).unwrap(), FillState::Filled);}+ #[test]+ fn clear_all_rolls_back_when_pet_clear_fails() {+ assert_clear_all_rolls_back("pet_face_vector_id_map");+ }++ #[test]+ fn clear_all_rolls_back_when_metadata_clear_fails() {+ assert_clear_all_rolls_back("ml_store_meta");+ }++ fn assert_clear_all_rolls_back(table: &str) {+ let (directory, store) = open();+ let path = directory.path().join(DB_FILE);+ store.put_clip(&clips([1])).unwrap();+ store+ .cluster_summary_update(&HashMap::from([centroid("c1", 1)]))+ .unwrap();+ store.fill_clip_index(false).unwrap();+ store.fill_cluster_centroid_index(false).unwrap();+ store+ .db()+ .put_repeated_text_embedding_cache("dog", &[0.5])+ .unwrap();+ store+ .db()+ .put_face_id_cached_for_person_or_cluster("p1", "1_0")+ .unwrap();+ let connection = Connection::open(&path).unwrap();+ connection+ .execute_batch(+ "INSERT INTO pet_faces (file_id, pet_face_id, detection, species, score)+ VALUES (1, 'pet_face', '{}', 1, 0.9);+ INSERT INTO pet_bodies (file_id, pet_body_id, detection, species, score)+ VALUES (1, 'pet_body', '{}', 1, 0.9);+ INSERT INTO pet_face_vector_id_map (pet_face_id) VALUES ('pet_face');+ INSERT INTO pet_body_vector_id_map (pet_body_id) VALUES ('pet_body');",+ )+ .unwrap();+ let pet_rows = || {+ connection+ .query_row(+ "SELECT (SELECT COUNT(*) FROM pet_faces)+ + (SELECT COUNT(*) FROM pet_bodies)+ + (SELECT COUNT(*) FROM pet_face_vector_id_map)+ + (SELECT COUNT(*) FROM pet_body_vector_id_map)",+ (),+ |row| Ok(row.get::<_, i64>(0)?),+ )+ .unwrap()+ };+ connection+ .execute_batch(&format!(+ "CREATE TRIGGER fail_clear BEFORE DELETE ON {table}+ BEGIN SELECT RAISE(ABORT, 'injected clear failure'); END;"+ ))+ .unwrap();++ let error = store.clear_all().unwrap_err();+ assert!(matches!(error, Error::Database(_)));+ assert!(error.to_string().contains("injected clear failure"));+ assert_eq!(store.db().count_clip_rows().unwrap(), 1);+ assert_eq!(store.db().count_cluster_summaries().unwrap(), 1);+ assert_eq!(pet_rows(), 4);+ for index in Index::ALL {+ assert_eq!(store.fill_state(index).unwrap(), FillState::Filled);+ assert_eq!(live_count(&store, index), 1);+ }+ drop(store);++ let reopened = MlStore::open(&path).unwrap();+ assert_eq!(reopened.db().count_clip_rows().unwrap(), 1);+ assert_eq!(reopened.db().count_cluster_summaries().unwrap(), 1);+ assert_eq!(+ reopened.fill_clip_index(false).unwrap().outcome,+ FillOutcome::AlreadyFilled+ );+ assert_eq!(+ reopened.fill_cluster_centroid_index(false).unwrap().outcome,+ FillOutcome::AlreadyFilled+ );+ for (index, key) in [(Index::Clip, "1"), (Index::ClusterCentroid, "c1")] {+ assert_eq!(nearest(&reopened, index, &one_hot(index.dims(), 1)), key);+ }++ connection.execute_batch("DROP TRIGGER fail_clear").unwrap();+ reopened.clear_all().unwrap();+ assert_eq!(reopened.db().count_clip_rows().unwrap(), 0);+ assert_eq!(reopened.db().count_cluster_summaries().unwrap(), 0);+ assert_eq!(pet_rows(), 0);+ assert_eq!(+ reopened+ .db()+ .get_repeated_text_embedding_cache("dog")+ .unwrap(),+ Some(vec![0.5])+ );+ assert_eq!(+ reopened+ .db()+ .get_face_id_used_for_person_or_cluster("p1")+ .unwrap()+ .as_deref(),+ Some("1_0")+ );+ drop(reopened);++ let cleared = MlStore::open(&path).unwrap();+ for index in Index::ALL {+ assert_eq!(cleared.fill_state(index).unwrap(), FillState::Stale);+ assert_eq!(live_count(&cleared, index), 0);+ }+ }+#[test]fn index_write_failure_marks_the_index_stale() {let (_directory, store) = open_with_unusable_clip_index();