π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Recluster selected faces in the Rust ML store
laurenspriem140fd4761724
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dartindex 33985d1983..a70e0cf9c7 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart@@ -155,4 +155,24 @@ class DartFaceClustering implements FaceClustering {bucket++;}}++ @override+ Future<Map<String, String>> reclusterFaces(+ List<String> faceIDs, {+ required Map<int, int> fileIDToCreationTime,+ required double distanceThreshold,+ }) async {+ final embeddings = await _mlDataDB.getFaceEmbeddingMapForFaces(faceIDs);+ if (embeddings.isEmpty) return {};+ final clusterResult = await FaceClusteringService.instance+ .predictWithinClusterComputer(+ embeddings,+ fileIDToCreationTime: fileIDToCreationTime,+ distanceThreshold: distanceThreshold,+ );+ if (clusterResult.isEmpty) return {};+ await _mlDataDB.forceUpdateClusterIds(clusterResult.newFaceIdToCluster);+ await _mlDataDB.clusterSummaryUpdate(clusterResult.newClusterSummaries);+ return clusterResult.newFaceIdToCluster;+ }}diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dartindex 39cc2d8a7e..caae979f3a 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart@@ -16,4 +16,10 @@ abstract class FaceClustering {required Map<String, List<String>> rejectedClusterIDsByFaceID,required MlRunControl control,});++ Future<Map<String, String>> reclusterFaces(+ List<String> faceIDs, {+ required Map<int, int> fileIDToCreationTime,+ required double distanceThreshold,+ });}diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dartindex 99d626a59f..e7f71d299b 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart@@ -4,6 +4,7 @@ import "package:photos/db/ml/rust_db.dart";import "package:photos/events/people_changed_event.dart";import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";+import "package:photos/services/machine_learning/ml_result.dart";import "package:photos/services/machine_learning/ml_run_control.dart";import "package:photos/src/rust/api/ml_db_api.dart" as rust;@@ -70,4 +71,20 @@ class StoreFaceClustering implements FaceClustering {run.dispose();}}++ @override+ Future<Map<String, String>> reclusterFaces(+ List<String> faceIDs, {+ required Map<int, int> fileIDToCreationTime,+ required double distanceThreshold,+ }) async {+ final fileIDs = faceIDs.map(getFileIdFromFaceId<int>).toSet();+ return (await _mlDataDB.store).reclusterFaces(+ faceIds: faceIDs,+ fileCreationTimes: {+ for (final fileID in fileIDs) fileID: ?fileIDToCreationTime[fileID],+ },+ distanceThreshold: distanceThreshold,+ );+ }}diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dartindex 872c7646c2..1aa05670b7 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart@@ -18,6 +18,7 @@ import "package:photos/models/base/id.dart";import "package:photos/models/file/file.dart";import "package:photos/models/ml/face/person.dart";import "package:photos/service_locator.dart";+import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";import "package:photos/services/machine_learning/face_ml/person/person_service.dart";@@ -209,30 +210,19 @@ class ClusterFeedbackService<T> {return;}- final embeddings = await mlDataDB.getFaceEmbeddingMapForFaces(faceIDs);- if (embeddings.isEmpty) {- _logger.severe('No embeddings found for faces $faceIDs');- Bus.instance.fire(PeopleChangedEvent());- return;- }-final fileIDToCreationTime = await FilesDB.instance.getFileIDToCreationTime();- final clusterResult = await FaceClusteringService.instance- .predictWithinClusterComputer(- embeddings,- fileIDToCreationTime: fileIDToCreationTime,- distanceThreshold: 0.20,- );- if (clusterResult.isEmpty) {- _logger.severe('No clusters found or something went wrong');+ final newFaceIdToClusterID = await FaceClustering.instance.reclusterFaces(+ faceIDs,+ fileIDToCreationTime: fileIDToCreationTime,+ distanceThreshold: 0.20,+ );+ if (newFaceIdToClusterID.isEmpty) {+ _logger.severe('No embeddings found for faces $faceIDs');+ Bus.instance.fire(PeopleChangedEvent());return;}- final newFaceIdToClusterID = clusterResult.newFaceIdToCluster;-- await mlDataDB.forceUpdateClusterIds(newFaceIdToClusterID);- await mlDataDB.clusterSummaryUpdate(clusterResult.newClusterSummaries);// Record rejection so removed faces are not suggested again.final notClusterIdToPersonId = <String, String>{};@@ -305,33 +295,21 @@ class ClusterFeedbackService<T> {final fileID = getFileIdFromFaceId<int>(faceID);return files.any((file) => file.uploadedFileID == fileID);});- final embeddings = await mlDataDB.getFaceEmbeddingMapForFaces(faceIDs);+ final fileIDToCreationTime = await FilesDB.instance+ .getFileIDToCreationTime();- if (faceIDs.isEmpty || embeddings.isEmpty) {+ final newFaceIdToClusterID = await FaceClustering.instance.reclusterFaces(+ faceIDs,+ fileIDToCreationTime: fileIDToCreationTime,+ distanceThreshold: 0.20,+ );+ if (newFaceIdToClusterID.isEmpty) {_logger.severe('No faces or embeddings found for cluster $clusterID that match the given files',);return;}- final fileIDToCreationTime = await FilesDB.instance- .getFileIDToCreationTime();-- final clusterResult = await FaceClusteringService.instance- .predictWithinClusterComputer(- embeddings,- fileIDToCreationTime: fileIDToCreationTime,- distanceThreshold: 0.20,- );- if (clusterResult.isEmpty) {- _logger.severe('No clusters found or something went wrong');- return;- }- final newFaceIdToClusterID = clusterResult.newFaceIdToCluster;-- await mlDataDB.forceUpdateClusterIds(newFaceIdToClusterID);- await mlDataDB.clusterSummaryUpdate(clusterResult.newClusterSummaries);-Bus.instance.fire(PeopleChangedEvent(relevantFiles: files,@@ -1432,6 +1410,32 @@ class ClusterFeedbackService<T> {return susClusters;}+ Future<Map<String, List<String>>> splitCluster(String clusterID) async {+ final faceIDs = await mlDataDB.getFaceIDsForCluster(clusterID);+ final fileIDToCreationTime = await FilesDB.instance+ .getFileIDToCreationTime();+ final newFaceIdToClusterID = await FaceClustering.instance.reclusterFaces(+ faceIDs.toList(),+ fileIDToCreationTime: fileIDToCreationTime,+ distanceThreshold: 0.22,+ );+ if (newFaceIdToClusterID.isEmpty) {+ _logger.warning('No embeddings found for cluster $clusterID');+ return {};+ }+ await mlDataDB.deleteClusterSummary(clusterID);++ final newClusterIDToFaceIDs = <String, List<String>>{};+ for (final MapEntry(key: faceID, value: newClusterID)+ in newFaceIdToClusterID.entries) {+ newClusterIDToFaceIDs.putIfAbsent(newClusterID, () => []).add(faceID);+ }+ _logger.info(+ 'Split cluster $clusterID into ${newClusterIDToFaceIDs.length} clusters',+ );+ return newClusterIDToFaceIDs;+ }+Future<ClusteringResult> breakUpCluster(String clusterID, {bool useDbscan = false,diff --git a/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dart b/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dartindex 3ae0db27a9..d40236f88d 100644--- a/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dart+++ b/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dart@@ -9,8 +9,6 @@ import 'package:logging/logging.dart';import 'package:photos/core/configuration.dart';import 'package:photos/core/event_bus.dart';import "package:photos/db/files_db.dart";-import 'package:photos/db/ml/base.dart';-import "package:photos/db/ml/db.dart";import "package:photos/events/people_changed_event.dart";import 'package:photos/events/subscription_purchased_event.dart';import "package:photos/models/file/file.dart";@@ -112,7 +110,6 @@ class _AppBarWidgetState extends State<ClusterAppBar> {final GlobalKey shareButtonKey = GlobalKey();bool isQuickLink = false;late GalleryType galleryType;- late final IMLDataDB mlDataDB = MLDataDB.instance;@overridevoid initState() {@@ -328,18 +325,8 @@ class _AppBarWidgetState extends State<ClusterAppBar> {firstButtonLabel: context.strings.confirm,firstButtonOnTap: () async {try {- final breakupResult = await ClusterFeedbackService.instance- .breakUpCluster(widget.clusterID);- final Map<String, List<String>> newClusterIDToFaceIDs =- breakupResult.newClusterIdToFaceIds;- final Map<String, String> newFaceIdToClusterID =- breakupResult.newFaceIdToCluster;-- await mlDataDB.deleteClusterSummary(widget.clusterID);- await MLDataDB.instance.clusterSummaryUpdate(- breakupResult.newClusterSummaries,- );- await mlDataDB.updateFaceIdToClusterId(newFaceIdToClusterID);+ final newClusterIDToFaceIDs = await ClusterFeedbackService.instance+ .splitCluster(widget.clusterID);biggestClusterID = '';int biggestClusterSize = 0;diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rsindex 48298f96bb..f4e2715419 100644--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs@@ -303,6 +303,17 @@ impl MlStore {.map(Into::into))}+ pub fn recluster_faces(+ &self,+ face_ids: Vec<String>,+ file_creation_times: HashMap<i64, i64>,+ distance_threshold: f64,+ ) -> Result<HashMap<String, String>, MlDbError> {+ Ok(self+ .inner+ .recluster_faces(&face_ids, &file_creation_times, distance_threshold)?)+ }+pub fn clear_all(&self) -> Result<(), MlDbError> {Ok(self.inner.clear_all()?)}diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rsindex 3f34372493..fffd11f5b9 100644--- a/rust/crates/photos/src/ml_db/mod.rs+++ b/rust/crates/photos/src/ml_db/mod.rs@@ -13,6 +13,7 @@ pub use queries::clip::{ClipRow, EmbeddingVector,};pub use queries::clusters::{ClusterCentroidRow, ClusterSummary};+pub(crate) use queries::faces::try_file_id_from_face_id;pub use queries::faces::{FACE_ML_VERSION, FaceDbInfoForClustering, FaceForClustering, FaceRow, FaceWithoutEmbedding,};diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rsindex 6e4a9c7f33..4342db992d 100644--- a/rust/crates/photos/src/ml_db/queries/faces.rs+++ b/rust/crates/photos/src/ml_db/queries/faces.rs@@ -604,7 +604,7 @@ pub(super) fn file_id_from_face_id(face_id: &str) -> Result<i64> {.ok_or_else(|| Error::Invalid(format!("Error parsing faceId: {face_id}")))}-fn try_file_id_from_face_id(face_id: &str) -> Option<i64> {+pub(crate) fn try_file_id_from_face_id(face_id: &str) -> Option<i64> {face_id.split_once('_').and_then(|(file_id, _)| file_id.parse().ok())diff --git a/rust/crates/photos/src/ml_store/clustering.rs b/rust/crates/photos/src/ml_store/clustering.rsindex a3556eab70..a0cf138103 100644--- a/rust/crates/photos/src/ml_store/clustering.rs+++ b/rust/crates/photos/src/ml_store/clustering.rs@@ -4,9 +4,11 @@ use std::ops::Range;use super::fill::decode_centroid;use super::{CLUSTER_CENTROID_DIMENSIONS, Error, MlStore, Result};-use crate::face_clustering::{ClusterMean, Face, IncrementalFace, cluster_incrementally};-use crate::ml_db::ClusterSummary;-use crate::ml_db::vector_encoding::encode_evector;+use crate::face_clustering::{+ ClusterMean, Face, IncrementalFace, cluster_incrementally, recluster,+};+use crate::ml_db::vector_encoding::{decode_evector, encode_evector};+use crate::ml_db::{ClusterSummary, try_file_id_from_face_id};const BUCKET_SIZE: usize = 10_000;const BUCKET_STEP: usize = 7_500;@@ -74,12 +76,7 @@ impl MlStore {&self,mut request: FaceClusteringRequest,) -> Result<FaceClusteringRun> {- if !request.distance_threshold.is_finite() {- return Err(Error::InvalidArgument(format!(- "face clustering distance threshold {} is not finite",- request.distance_threshold- )));- }+ check_distance_threshold(request.distance_threshold)?;let mut faces = Vec::new();let mut missing_files = HashSet::new();let mut invalid_faces = 0;@@ -150,12 +147,8 @@ impl MlStore {run.distance_threshold,&run.cluster_means,);- let summary: HashMap<String, ClusterSummary> = clustering- .means- .iter()- .map(|(cluster_id, mean)| (cluster_id.clone(), cluster_summary(mean)))- .collect();- let written = self.assign_face_clusters(&clustering.assignments, &summary);+ let written =+ self.assign_face_clusters(&clustering.assignments, &summaries(&clustering.means));if let Ok(()) | Err(Error::Index(_)) = written {run.record(bucket.clone(), &clustering.assignments);run.cluster_means.extend(clustering.means);@@ -168,6 +161,43 @@ impl MlStore {})}+ pub fn recluster_faces(+ &self,+ face_ids: &[String],+ file_creation_times: &HashMap<i64, i64>,+ distance_threshold: f64,+ ) -> Result<HashMap<String, String>> {+ check_distance_threshold(distance_threshold)?;+ let faces = self+ .db+ .get_face_embedding_rows_for_faces(face_ids)?+ .into_iter()+ .filter_map(|(face_id, embedding)| {+ let embedding: Vec<f32> = decode_evector(&embedding)+ .ok()?+ .into_iter()+ .map(|value| value as f32)+ .collect();+ let creation_time = try_file_id_from_face_id(&face_id)+ .and_then(|file_id| file_creation_times.get(&file_id).copied());+ is_clusterable(&embedding).then_some(Face {+ face_id,+ embedding,+ creation_time,+ })+ })+ .collect();+ let clustering = recluster(faces, distance_threshold);+ match self.assign_face_clusters(&clustering.assignments, &summaries(&clustering.means)) {+ Ok(()) => {}+ Err(Error::Index(error)) => {+ log::warn!("cluster centroid index write failed while reclustering faces: {error}");+ }+ Err(error) => return Err(error),+ }+ Ok(clustering.assignments)+ }+fn cluster_means(&self) -> Result<HashMap<String, ClusterMean>> {Ok(self.db@@ -182,17 +212,33 @@ impl MlStore {}}+fn check_distance_threshold(distance_threshold: f64) -> Result<()> {+ if distance_threshold.is_finite() {+ Ok(())+ } else {+ Err(Error::InvalidArgument(format!(+ "face clustering distance threshold {distance_threshold} is not finite"+ )))+ }+}+fn is_clusterable(embedding: &[f32]) -> bool {embedding.len() == CLUSTER_CENTROID_DIMENSIONS&& embedding.iter().all(|value| value.is_finite())}-fn cluster_summary(mean: &ClusterMean) -> ClusterSummary {- let values: Vec<f64> = mean.mean.iter().copied().map(f64::from).collect();- ClusterSummary {- avg: encode_evector(&values),- count: mean.count as i64,- }+fn summaries(means: &HashMap<String, ClusterMean>) -> HashMap<String, ClusterSummary> {+ means+ .iter()+ .map(|(cluster_id, mean)| {+ let values: Vec<f64> = mean.mean.iter().copied().map(f64::from).collect();+ let summary = ClusterSummary {+ avg: encode_evector(&values),+ count: mean.count as i64,+ };+ (cluster_id.clone(), summary)+ })+ .collect()}#[cfg(test)]@@ -502,6 +548,98 @@ mod tests {assert_eq!(groups(&store), [face_ids(1..=7)]);}+ fn times(file_ids: impl IntoIterator<Item = i64>) -> HashMap<i64, i64> {+ file_ids+ .into_iter()+ .map(|file_id| (file_id, file_id))+ .collect()+ }++ fn face_id_list(file_ids: impl IntoIterator<Item = i64>) -> Vec<String> {+ file_ids+ .into_iter()+ .map(|file_id| format!("{file_id}_0"))+ .collect()+ }++ #[test]+ fn reclustering_moves_the_given_faces_into_new_clusters() {+ let (_directory, store) = open();+ store.fill_cluster_centroid_index(false).unwrap();+ seed(+ &store,+ &[(1, 0.0), (2, 2.0), (3, 0.0), (4, 90.0), (5, 92.0), (6, 0.0)],+ );+ assign(+ &store,+ &[(1, "p"), (2, "p"), (3, "p"), (4, "p"), (5, "p"), (6, "p")],+ );++ let assignments = store+ .recluster_faces(&face_id_list(1..=5), ×(1..=4), THRESHOLD)+ .unwrap();++ assert_eq!(assignments.len(), 5);+ assert!(assignments.values().all(|cluster_id| cluster_id != "p"));+ assert_eq!(+ groups(&store),+ [face_ids([1, 2, 3]), face_ids([4, 5]), face_ids([6])]+ );+ for (file_id, members) in [(1, 3), (4, 2)] {+ let cluster_id = cluster_of(&store, file_id).unwrap();+ assert_eq!(assignments[&format!("{file_id}_0")], cluster_id);+ assert_eq!(summary_of(&store, &cluster_id).1, members);+ assert!(store.contains(Index::ClusterCentroid, &cluster_id).unwrap());+ }+ }++ #[test]+ fn reclustering_leaves_faces_without_a_valid_embedding_in_place() {+ let (_directory, store) = open();+ seed(&store, &[(1, 0.0), (2, 0.0)]);+ let mut invalid = face(3, 0.0);+ invalid.embedding[5] = f64::NAN;+ let mut short = face(4, 0.0);+ short.embedding.truncate(3);+ store.db().bulk_insert_faces(&[invalid, short]).unwrap();+ assign(&store, &[(1, "p"), (2, "p"), (3, "p"), (4, "p")]);++ let assignments = store+ .recluster_faces(&face_id_list(1..=5), ×(1..=5), THRESHOLD)+ .unwrap();++ let mut moved: Vec<&String> = assignments.keys().collect();+ moved.sort();+ assert_eq!(moved, face_ids([1, 2]).iter().collect::<Vec<_>>());+ assert_eq!(cluster_of(&store, 3).as_deref(), Some("p"));+ assert_eq!(cluster_of(&store, 4).as_deref(), Some("p"));+ assert!(+ store+ .recluster_faces(&face_id_list([9]), ×([9]), THRESHOLD)+ .unwrap()+ .is_empty()+ );+ }++ #[test]+ fn reclustering_keeps_its_assignments_when_the_centroid_write_fails() {+ let (directory, store) = open();+ fs::create_dir(index_path(&directory, Index::ClusterCentroid)).unwrap();+ store+ .db()+ .set_meta("cluster_centroid.fill", "filled")+ .unwrap();+ seed(&store, &[(1, 0.0), (2, 0.0)]);++ let assignments = store+ .recluster_faces(&face_id_list([1, 2]), ×([1, 2]), THRESHOLD)+ .unwrap();++ assert_eq!(assignments.len(), 2);+ assert_eq!(groups(&store), [face_ids([1, 2])]);+ assert_eq!(meta(&store, "cluster_centroid.fill"), None);+ }+#[test]fn non_finite_thresholds_are_rejected() {let (_directory, store) = open();@@ -511,5 +649,9 @@ mod tests {store.start_face_clustering(invalid),Err(Error::InvalidArgument(_))));+ assert!(matches!(+ store.recluster_faces(&[], &HashMap::new(), f64::INFINITY),+ Err(Error::InvalidArgument(_))+ ));}}diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rsindex 2ba4f1a6e0..ba0ff35435 100644--- a/rust/crates/photos/src/ml_store/writes.rs+++ b/rust/crates/photos/src/ml_store/writes.rs@@ -78,6 +78,9 @@ impl MlStore {face_id_to_cluster_id: &HashMap<String, String>,summary: &HashMap<String, ClusterSummary>,) -> Result<()> {+ if face_id_to_cluster_id.is_empty() {+ return Ok(());+ }let _mutations = self.lock_mutations();self.upsert(Index::ClusterCentroid, ¢roid_batch(summary), || {self.db