Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Recluster selected faces in the Rust ML store

laurenspriem140fd4761724
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart
index 33985d1983..a70e0cf9c7 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart
@@ -155,4 +155,24 @@ class DartFaceClustering implements FaceClustering {
bucket++;
}
}
+
+ @override
+ Future<Map<String, String>> reclusterFaces(
+ List<String> faceIDs, {
+ required Map<int, int> fileIDToCreationTime,
+ required double distanceThreshold,
+ }) async {
+ final embeddings = await _mlDataDB.getFaceEmbeddingMapForFaces(faceIDs);
+ if (embeddings.isEmpty) return {};
+ final clusterResult = await FaceClusteringService.instance
+ .predictWithinClusterComputer(
+ embeddings,
+ fileIDToCreationTime: fileIDToCreationTime,
+ distanceThreshold: distanceThreshold,
+ );
+ if (clusterResult.isEmpty) return {};
+ await _mlDataDB.forceUpdateClusterIds(clusterResult.newFaceIdToCluster);
+ await _mlDataDB.clusterSummaryUpdate(clusterResult.newClusterSummaries);
+ return clusterResult.newFaceIdToCluster;
+ }
}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart
index 39cc2d8a7e..caae979f3a 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart
@@ -16,4 +16,10 @@ abstract class FaceClustering {
required Map<String, List<String>> rejectedClusterIDsByFaceID,
required MlRunControl control,
});
+
+ Future<Map<String, String>> reclusterFaces(
+ List<String> faceIDs, {
+ required Map<int, int> fileIDToCreationTime,
+ required double distanceThreshold,
+ });
}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart
index 99d626a59f..e7f71d299b 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart
@@ -4,6 +4,7 @@ import "package:photos/db/ml/rust_db.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
+import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/services/machine_learning/ml_run_control.dart";
import "package:photos/src/rust/api/ml_db_api.dart" as rust;
@@ -70,4 +71,20 @@ class StoreFaceClustering implements FaceClustering {
run.dispose();
}
}
+
+ @override
+ Future<Map<String, String>> reclusterFaces(
+ List<String> faceIDs, {
+ required Map<int, int> fileIDToCreationTime,
+ required double distanceThreshold,
+ }) async {
+ final fileIDs = faceIDs.map(getFileIdFromFaceId<int>).toSet();
+ return (await _mlDataDB.store).reclusterFaces(
+ faceIds: faceIDs,
+ fileCreationTimes: {
+ for (final fileID in fileIDs) fileID: ?fileIDToCreationTime[fileID],
+ },
+ distanceThreshold: distanceThreshold,
+ );
+ }
}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
index 872c7646c2..1aa05670b7 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
@@ -18,6 +18,7 @@ import "package:photos/models/base/id.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/service_locator.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
@@ -209,30 +210,19 @@ class ClusterFeedbackService<T> {
return;
}
- final embeddings = await mlDataDB.getFaceEmbeddingMapForFaces(faceIDs);
- if (embeddings.isEmpty) {
- _logger.severe('No embeddings found for faces $faceIDs');
- Bus.instance.fire(PeopleChangedEvent());
- return;
- }
-
final fileIDToCreationTime = await FilesDB.instance
.getFileIDToCreationTime();
- final clusterResult = await FaceClusteringService.instance
- .predictWithinClusterComputer(
- embeddings,
- fileIDToCreationTime: fileIDToCreationTime,
- distanceThreshold: 0.20,
- );
- if (clusterResult.isEmpty) {
- _logger.severe('No clusters found or something went wrong');
+ final newFaceIdToClusterID = await FaceClustering.instance.reclusterFaces(
+ faceIDs,
+ fileIDToCreationTime: fileIDToCreationTime,
+ distanceThreshold: 0.20,
+ );
+ if (newFaceIdToClusterID.isEmpty) {
+ _logger.severe('No embeddings found for faces $faceIDs');
+ Bus.instance.fire(PeopleChangedEvent());
return;
}
- final newFaceIdToClusterID = clusterResult.newFaceIdToCluster;
-
- await mlDataDB.forceUpdateClusterIds(newFaceIdToClusterID);
- await mlDataDB.clusterSummaryUpdate(clusterResult.newClusterSummaries);
// Record rejection so removed faces are not suggested again.
final notClusterIdToPersonId = <String, String>{};
@@ -305,33 +295,21 @@ class ClusterFeedbackService<T> {
final fileID = getFileIdFromFaceId<int>(faceID);
return files.any((file) => file.uploadedFileID == fileID);
});
- final embeddings = await mlDataDB.getFaceEmbeddingMapForFaces(faceIDs);
+ final fileIDToCreationTime = await FilesDB.instance
+ .getFileIDToCreationTime();
- if (faceIDs.isEmpty || embeddings.isEmpty) {
+ final newFaceIdToClusterID = await FaceClustering.instance.reclusterFaces(
+ faceIDs,
+ fileIDToCreationTime: fileIDToCreationTime,
+ distanceThreshold: 0.20,
+ );
+ if (newFaceIdToClusterID.isEmpty) {
_logger.severe(
'No faces or embeddings found for cluster $clusterID that match the given files',
);
return;
}
- final fileIDToCreationTime = await FilesDB.instance
- .getFileIDToCreationTime();
-
- final clusterResult = await FaceClusteringService.instance
- .predictWithinClusterComputer(
- embeddings,
- fileIDToCreationTime: fileIDToCreationTime,
- distanceThreshold: 0.20,
- );
- if (clusterResult.isEmpty) {
- _logger.severe('No clusters found or something went wrong');
- return;
- }
- final newFaceIdToClusterID = clusterResult.newFaceIdToCluster;
-
- await mlDataDB.forceUpdateClusterIds(newFaceIdToClusterID);
- await mlDataDB.clusterSummaryUpdate(clusterResult.newClusterSummaries);
-
Bus.instance.fire(
PeopleChangedEvent(
relevantFiles: files,
@@ -1432,6 +1410,32 @@ class ClusterFeedbackService<T> {
return susClusters;
}
+ Future<Map<String, List<String>>> splitCluster(String clusterID) async {
+ final faceIDs = await mlDataDB.getFaceIDsForCluster(clusterID);
+ final fileIDToCreationTime = await FilesDB.instance
+ .getFileIDToCreationTime();
+ final newFaceIdToClusterID = await FaceClustering.instance.reclusterFaces(
+ faceIDs.toList(),
+ fileIDToCreationTime: fileIDToCreationTime,
+ distanceThreshold: 0.22,
+ );
+ if (newFaceIdToClusterID.isEmpty) {
+ _logger.warning('No embeddings found for cluster $clusterID');
+ return {};
+ }
+ await mlDataDB.deleteClusterSummary(clusterID);
+
+ final newClusterIDToFaceIDs = <String, List<String>>{};
+ for (final MapEntry(key: faceID, value: newClusterID)
+ in newFaceIdToClusterID.entries) {
+ newClusterIDToFaceIDs.putIfAbsent(newClusterID, () => []).add(faceID);
+ }
+ _logger.info(
+ 'Split cluster $clusterID into ${newClusterIDToFaceIDs.length} clusters',
+ );
+ return newClusterIDToFaceIDs;
+ }
+
Future<ClusteringResult> breakUpCluster(
String clusterID, {
bool useDbscan = false,
diff --git a/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dart b/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dart
index 3ae0db27a9..d40236f88d 100644
--- a/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dart
+++ b/mobile/apps/photos/lib/ui/viewer/people/cluster_app_bar.dart
@@ -9,8 +9,6 @@ import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/core/event_bus.dart';
import "package:photos/db/files_db.dart";
-import 'package:photos/db/ml/base.dart';
-import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import 'package:photos/events/subscription_purchased_event.dart';
import "package:photos/models/file/file.dart";
@@ -112,7 +110,6 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
final GlobalKey shareButtonKey = GlobalKey();
bool isQuickLink = false;
late GalleryType galleryType;
- late final IMLDataDB mlDataDB = MLDataDB.instance;
@override
void initState() {
@@ -328,18 +325,8 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
firstButtonLabel: context.strings.confirm,
firstButtonOnTap: () async {
try {
- final breakupResult = await ClusterFeedbackService.instance
- .breakUpCluster(widget.clusterID);
- final Map<String, List<String>> newClusterIDToFaceIDs =
- breakupResult.newClusterIdToFaceIds;
- final Map<String, String> newFaceIdToClusterID =
- breakupResult.newFaceIdToCluster;
-
- await mlDataDB.deleteClusterSummary(widget.clusterID);
- await MLDataDB.instance.clusterSummaryUpdate(
- breakupResult.newClusterSummaries,
- );
- await mlDataDB.updateFaceIdToClusterId(newFaceIdToClusterID);
+ final newClusterIDToFaceIDs = await ClusterFeedbackService.instance
+ .splitCluster(widget.clusterID);
biggestClusterID = '';
int biggestClusterSize = 0;
diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rs
index 48298f96bb..f4e2715419 100644
--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs
@@ -303,6 +303,17 @@ impl MlStore {
.map(Into::into))
}
+ pub fn recluster_faces(
+ &self,
+ face_ids: Vec<String>,
+ file_creation_times: HashMap<i64, i64>,
+ distance_threshold: f64,
+ ) -> Result<HashMap<String, String>, MlDbError> {
+ Ok(self
+ .inner
+ .recluster_faces(&face_ids, &file_creation_times, distance_threshold)?)
+ }
+
pub fn clear_all(&self) -> Result<(), MlDbError> {
Ok(self.inner.clear_all()?)
}
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
index 3f34372493..fffd11f5b9 100644
--- a/rust/crates/photos/src/ml_db/mod.rs
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -13,6 +13,7 @@ pub use queries::clip::{
ClipRow, EmbeddingVector,
};
pub use queries::clusters::{ClusterCentroidRow, ClusterSummary};
+pub(crate) use queries::faces::try_file_id_from_face_id;
pub use queries::faces::{
FACE_ML_VERSION, FaceDbInfoForClustering, FaceForClustering, FaceRow, FaceWithoutEmbedding,
};
diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rs
index 6e4a9c7f33..4342db992d 100644
--- a/rust/crates/photos/src/ml_db/queries/faces.rs
+++ b/rust/crates/photos/src/ml_db/queries/faces.rs
@@ -604,7 +604,7 @@ pub(super) fn file_id_from_face_id(face_id: &str) -> Result<i64> {
.ok_or_else(|| Error::Invalid(format!("Error parsing faceId: {face_id}")))
}
-fn try_file_id_from_face_id(face_id: &str) -> Option<i64> {
+pub(crate) fn try_file_id_from_face_id(face_id: &str) -> Option<i64> {
face_id
.split_once('_')
.and_then(|(file_id, _)| file_id.parse().ok())
diff --git a/rust/crates/photos/src/ml_store/clustering.rs b/rust/crates/photos/src/ml_store/clustering.rs
index a3556eab70..a0cf138103 100644
--- a/rust/crates/photos/src/ml_store/clustering.rs
+++ b/rust/crates/photos/src/ml_store/clustering.rs
@@ -4,9 +4,11 @@ use std::ops::Range;
use super::fill::decode_centroid;
use super::{CLUSTER_CENTROID_DIMENSIONS, Error, MlStore, Result};
-use crate::face_clustering::{ClusterMean, Face, IncrementalFace, cluster_incrementally};
-use crate::ml_db::ClusterSummary;
-use crate::ml_db::vector_encoding::encode_evector;
+use crate::face_clustering::{
+ ClusterMean, Face, IncrementalFace, cluster_incrementally, recluster,
+};
+use crate::ml_db::vector_encoding::{decode_evector, encode_evector};
+use crate::ml_db::{ClusterSummary, try_file_id_from_face_id};
const BUCKET_SIZE: usize = 10_000;
const BUCKET_STEP: usize = 7_500;
@@ -74,12 +76,7 @@ impl MlStore {
&self,
mut request: FaceClusteringRequest,
) -> Result<FaceClusteringRun> {
- if !request.distance_threshold.is_finite() {
- return Err(Error::InvalidArgument(format!(
- "face clustering distance threshold {} is not finite",
- request.distance_threshold
- )));
- }
+ check_distance_threshold(request.distance_threshold)?;
let mut faces = Vec::new();
let mut missing_files = HashSet::new();
let mut invalid_faces = 0;
@@ -150,12 +147,8 @@ impl MlStore {
run.distance_threshold,
&run.cluster_means,
);
- let summary: HashMap<String, ClusterSummary> = clustering
- .means
- .iter()
- .map(|(cluster_id, mean)| (cluster_id.clone(), cluster_summary(mean)))
- .collect();
- let written = self.assign_face_clusters(&clustering.assignments, &summary);
+ let written =
+ self.assign_face_clusters(&clustering.assignments, &summaries(&clustering.means));
if let Ok(()) | Err(Error::Index(_)) = written {
run.record(bucket.clone(), &clustering.assignments);
run.cluster_means.extend(clustering.means);
@@ -168,6 +161,43 @@ impl MlStore {
})
}
+ pub fn recluster_faces(
+ &self,
+ face_ids: &[String],
+ file_creation_times: &HashMap<i64, i64>,
+ distance_threshold: f64,
+ ) -> Result<HashMap<String, String>> {
+ check_distance_threshold(distance_threshold)?;
+ let faces = self
+ .db
+ .get_face_embedding_rows_for_faces(face_ids)?
+ .into_iter()
+ .filter_map(|(face_id, embedding)| {
+ let embedding: Vec<f32> = decode_evector(&embedding)
+ .ok()?
+ .into_iter()
+ .map(|value| value as f32)
+ .collect();
+ let creation_time = try_file_id_from_face_id(&face_id)
+ .and_then(|file_id| file_creation_times.get(&file_id).copied());
+ is_clusterable(&embedding).then_some(Face {
+ face_id,
+ embedding,
+ creation_time,
+ })
+ })
+ .collect();
+ let clustering = recluster(faces, distance_threshold);
+ match self.assign_face_clusters(&clustering.assignments, &summaries(&clustering.means)) {
+ Ok(()) => {}
+ Err(Error::Index(error)) => {
+ log::warn!("cluster centroid index write failed while reclustering faces: {error}");
+ }
+ Err(error) => return Err(error),
+ }
+ Ok(clustering.assignments)
+ }
+
fn cluster_means(&self) -> Result<HashMap<String, ClusterMean>> {
Ok(self
.db
@@ -182,17 +212,33 @@ impl MlStore {
}
}
+fn check_distance_threshold(distance_threshold: f64) -> Result<()> {
+ if distance_threshold.is_finite() {
+ Ok(())
+ } else {
+ Err(Error::InvalidArgument(format!(
+ "face clustering distance threshold {distance_threshold} is not finite"
+ )))
+ }
+}
+
fn is_clusterable(embedding: &[f32]) -> bool {
embedding.len() == CLUSTER_CENTROID_DIMENSIONS
&& embedding.iter().all(|value| value.is_finite())
}
-fn cluster_summary(mean: &ClusterMean) -> ClusterSummary {
- let values: Vec<f64> = mean.mean.iter().copied().map(f64::from).collect();
- ClusterSummary {
- avg: encode_evector(&values),
- count: mean.count as i64,
- }
+fn summaries(means: &HashMap<String, ClusterMean>) -> HashMap<String, ClusterSummary> {
+ means
+ .iter()
+ .map(|(cluster_id, mean)| {
+ let values: Vec<f64> = mean.mean.iter().copied().map(f64::from).collect();
+ let summary = ClusterSummary {
+ avg: encode_evector(&values),
+ count: mean.count as i64,
+ };
+ (cluster_id.clone(), summary)
+ })
+ .collect()
}
#[cfg(test)]
@@ -502,6 +548,98 @@ mod tests {
assert_eq!(groups(&store), [face_ids(1..=7)]);
}
+ fn times(file_ids: impl IntoIterator<Item = i64>) -> HashMap<i64, i64> {
+ file_ids
+ .into_iter()
+ .map(|file_id| (file_id, file_id))
+ .collect()
+ }
+
+ fn face_id_list(file_ids: impl IntoIterator<Item = i64>) -> Vec<String> {
+ file_ids
+ .into_iter()
+ .map(|file_id| format!("{file_id}_0"))
+ .collect()
+ }
+
+ #[test]
+ fn reclustering_moves_the_given_faces_into_new_clusters() {
+ let (_directory, store) = open();
+ store.fill_cluster_centroid_index(false).unwrap();
+ seed(
+ &store,
+ &[(1, 0.0), (2, 2.0), (3, 0.0), (4, 90.0), (5, 92.0), (6, 0.0)],
+ );
+ assign(
+ &store,
+ &[(1, "p"), (2, "p"), (3, "p"), (4, "p"), (5, "p"), (6, "p")],
+ );
+
+ let assignments = store
+ .recluster_faces(&face_id_list(1..=5), &times(1..=4), THRESHOLD)
+ .unwrap();
+
+ assert_eq!(assignments.len(), 5);
+ assert!(assignments.values().all(|cluster_id| cluster_id != "p"));
+ assert_eq!(
+ groups(&store),
+ [face_ids([1, 2, 3]), face_ids([4, 5]), face_ids([6])]
+ );
+ for (file_id, members) in [(1, 3), (4, 2)] {
+ let cluster_id = cluster_of(&store, file_id).unwrap();
+ assert_eq!(assignments[&format!("{file_id}_0")], cluster_id);
+ assert_eq!(summary_of(&store, &cluster_id).1, members);
+ assert!(store.contains(Index::ClusterCentroid, &cluster_id).unwrap());
+ }
+ }
+
+ #[test]
+ fn reclustering_leaves_faces_without_a_valid_embedding_in_place() {
+ let (_directory, store) = open();
+ seed(&store, &[(1, 0.0), (2, 0.0)]);
+ let mut invalid = face(3, 0.0);
+ invalid.embedding[5] = f64::NAN;
+ let mut short = face(4, 0.0);
+ short.embedding.truncate(3);
+ store.db().bulk_insert_faces(&[invalid, short]).unwrap();
+ assign(&store, &[(1, "p"), (2, "p"), (3, "p"), (4, "p")]);
+
+ let assignments = store
+ .recluster_faces(&face_id_list(1..=5), &times(1..=5), THRESHOLD)
+ .unwrap();
+
+ let mut moved: Vec<&String> = assignments.keys().collect();
+ moved.sort();
+ assert_eq!(moved, face_ids([1, 2]).iter().collect::<Vec<_>>());
+ assert_eq!(cluster_of(&store, 3).as_deref(), Some("p"));
+ assert_eq!(cluster_of(&store, 4).as_deref(), Some("p"));
+ assert!(
+ store
+ .recluster_faces(&face_id_list([9]), &times([9]), THRESHOLD)
+ .unwrap()
+ .is_empty()
+ );
+ }
+
+ #[test]
+ fn reclustering_keeps_its_assignments_when_the_centroid_write_fails() {
+ let (directory, store) = open();
+ fs::create_dir(index_path(&directory, Index::ClusterCentroid)).unwrap();
+ store
+ .db()
+ .set_meta("cluster_centroid.fill", "filled")
+ .unwrap();
+ seed(&store, &[(1, 0.0), (2, 0.0)]);
+
+ let assignments = store
+ .recluster_faces(&face_id_list([1, 2]), &times([1, 2]), THRESHOLD)
+ .unwrap();
+
+ assert_eq!(assignments.len(), 2);
+ assert_eq!(groups(&store), [face_ids([1, 2])]);
+ assert_eq!(meta(&store, "cluster_centroid.fill"), None);
+ }
+
#[test]
fn non_finite_thresholds_are_rejected() {
let (_directory, store) = open();
@@ -511,5 +649,9 @@ mod tests {
store.start_face_clustering(invalid),
Err(Error::InvalidArgument(_))
));
+ assert!(matches!(
+ store.recluster_faces(&[], &HashMap::new(), f64::INFINITY),
+ Err(Error::InvalidArgument(_))
+ ));
}
}
diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rs
index 2ba4f1a6e0..ba0ff35435 100644
--- a/rust/crates/photos/src/ml_store/writes.rs
+++ b/rust/crates/photos/src/ml_store/writes.rs
@@ -78,6 +78,9 @@ impl MlStore {
face_id_to_cluster_id: &HashMap<String, String>,
summary: &HashMap<String, ClusterSummary>,
) -> Result<()> {
+ if face_id_to_cluster_id.is_empty() {
+ return Ok(());
+ }
let _mutations = self.lock_mutations();
self.upsert(Index::ClusterCentroid, &centroid_batch(summary), || {
self.db