Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Run merge sheet similarity in the Rust ML store

laurenspriem20edf66df82c
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dart
index ebe0071e65..6d7e0fae6c 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dart
@@ -1,4 +1,4 @@
-import "dart:math" show Random;
+import "dart:math" show Random, max;
import "package:ente_pure_utils/ente_pure_utils.dart";
import "package:flutter/foundation.dart";
@@ -223,6 +223,48 @@ class DartFaceDistances implements FaceDistances {
}
}
+ @override
+ Future<Map<String, double>> maxSimilaritiesToPersons(
+ String clusterID,
+ Set<String> personIDs,
+ ) async {
+ final allClusterSummary = await _mlDataDB.getAllClusterSummary();
+ final clusterEmbeddingData = allClusterSummary[clusterID]?.$1;
+ if (clusterEmbeddingData == null) {
+ return {};
+ }
+ final Vector clusterEmbedding = Vector.fromList(
+ EVector.fromBuffer(clusterEmbeddingData).values,
+ dtype: DType.float32,
+ );
+
+ final clusterToPerson = await _mlDataDB.getClusterIDToPersonID();
+ clusterToPerson.removeWhere((_, personID) => !personIDs.contains(personID));
+ allClusterSummary.removeWhere(
+ (key, value) => !clusterToPerson.containsKey(key),
+ );
+ final Map<String, Vector> allClusterEmbeddings = allClusterSummary.map(
+ (key, value) => MapEntry(
+ key,
+ Vector.fromList(
+ EVector.fromBuffer(value.$1).values,
+ dtype: DType.float32,
+ ),
+ ),
+ );
+
+ final Map<String, double> personToMaxSimilarity = {};
+ for (final entry in allClusterEmbeddings.entries) {
+ final personID = clusterToPerson[entry.key]!;
+ final similarity = clusterEmbedding.dot(entry.value);
+ personToMaxSimilarity[personID] = max(
+ personToMaxSimilarity[personID] ?? double.negativeInfinity,
+ similarity,
+ );
+ }
+ return personToMaxSimilarity;
+ }
+
Future<Vector?> _getPersonAvg(String personID) async {
final w = (kDebugMode ? EnteWatch('_getPersonAvg') : null)?..start();
final personClusters = await _mlDataDB.getPersonClusterIDs(personID);
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dart
index f6e3f04354..9c2b5814a9 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dart
@@ -26,4 +26,9 @@ abstract class FaceDistances {
Stream<List<(String, double)>?> distancesToPersons(
List<(String, List<String>)> personFaces,
);
+
+ Future<Map<String, double>> maxSimilaritiesToPersons(
+ String clusterID,
+ Set<String> personIDs,
+ );
}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dart
index 3664767ab5..71b8f159c4 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dart
@@ -75,6 +75,17 @@ class StoreFaceDistances implements FaceDistances {
}
}
+ @override
+ Future<Map<String, double>> maxSimilaritiesToPersons(
+ String clusterID,
+ Set<String> personIDs,
+ ) async {
+ return (await _mlDataDB.store).maxSimilaritiesToPersons(
+ clusterId: clusterID,
+ personIds: personIDs.toList(growable: false),
+ );
+ }
+
Future<List<List<(String, double)>>> _suggestByMedian(
List<rust.MedianQuery> queries, {
required int sampleSize,
diff --git a/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dart b/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dart
index 0d3078d8c4..dd2b5a4b0f 100644
--- a/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dart
+++ b/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dart
@@ -1,5 +1,4 @@
import "dart:async";
-import "dart:math" show max;
import "package:collection/collection.dart";
import "package:dotted_border/dotted_border.dart";
@@ -10,16 +9,14 @@ import "package:ente_ui/components/loading_widget.dart";
import "package:flutter/material.dart";
import "package:hugeicons/hugeicons.dart";
import "package:logging/logging.dart";
-import "package:ml_linalg/linalg.dart" as ml;
import "package:photos/core/event_bus.dart";
-import "package:photos/db/ml/db.dart";
import "package:photos/events/people_sort_order_change_event.dart";
-import "package:photos/generated/protos/ente/common/vector.pb.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/models/search/generic_search_result.dart";
import "package:photos/models/search/search_constants.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
+import "package:photos/services/machine_learning/face_ml/feedback/face_distances.dart";
import "package:photos/services/search_service.dart";
import "package:photos/settings/local_settings.dart";
import "package:photos/theme/ente_theme.dart";
@@ -626,16 +623,6 @@ class _MergeClustersToPersonPageState extends State<MergeClustersToPersonPage> {
if (!_canUseSimilaritySort || seedClusterId == null || persons.isEmpty) {
return {};
}
- final allClusterSummary = await MLDataDB.instance.getAllClusterSummary();
- final currentClusterEmbeddingData = allClusterSummary[seedClusterId]?.$1;
- if (currentClusterEmbeddingData == null) {
- return {};
- }
- final ml.Vector currentClusterEmbedding = ml.Vector.fromList(
- EVector.fromBuffer(currentClusterEmbeddingData).values,
- dtype: ml.DType.float32,
- );
-
final personIds = persons
.map((result) => result.params[kPersonParamID] as String?)
.where((id) => id != null && id.isNotEmpty)
@@ -644,31 +631,10 @@ class _MergeClustersToPersonPageState extends State<MergeClustersToPersonPage> {
if (personIds.isEmpty) {
return {};
}
-
- final clusterToPerson = await MLDataDB.instance.getClusterIDToPersonID();
- clusterToPerson.removeWhere((_, personId) => !personIds.contains(personId));
- allClusterSummary.removeWhere(
- (key, value) => !clusterToPerson.containsKey(key),
+ return FaceDistances.instance.maxSimilaritiesToPersons(
+ seedClusterId,
+ personIds,
);
- final Map<String, ml.Vector> allClusterEmbeddings = allClusterSummary.map(
- (key, value) => MapEntry(
- key,
- ml.Vector.fromList(
- EVector.fromBuffer(value.$1).values,
- dtype: ml.DType.float32,
- ),
- ),
- );
-
- for (final entry in allClusterEmbeddings.entries) {
- final personId = clusterToPerson[entry.key]!;
- final similarity = currentClusterEmbedding.dot(entry.value);
- _personToMaxSimilarity[personId] = max(
- _personToMaxSimilarity[personId] ?? double.negativeInfinity,
- similarity,
- );
- }
- return _personToMaxSimilarity;
}
}
diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rs
index ce26e94753..f110352cf8 100644
--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs
@@ -425,6 +425,16 @@ impl MlStore {
Ok(self.inner.distances_to_person_averages(&requests)?)
}
+ pub fn max_similarities_to_persons(
+ &self,
+ cluster_id: String,
+ person_ids: Vec<String>,
+ ) -> Result<HashMap<String, f64>, MlDbError> {
+ Ok(self
+ .inner
+ .max_similarities_to_persons(&cluster_id, &person_ids.into_iter().collect())?)
+ }
+
pub fn clear_all(&self) -> Result<(), MlDbError> {
Ok(self.inner.clear_all()?)
}
diff --git a/rust/crates/photos/src/face_clustering/mod.rs b/rust/crates/photos/src/face_clustering/mod.rs
index b4d132993c..1ef63e5b1d 100644
--- a/rust/crates/photos/src/face_clustering/mod.rs
+++ b/rust/crates/photos/src/face_clustering/mod.rs
@@ -126,7 +126,7 @@ pub(crate) fn distance(a: &[f32], b: &[f32]) -> f64 {
1.0 - dot(a, b)
}
-fn dot(a: &[f32], b: &[f32]) -> f64 {
+pub(crate) fn dot(a: &[f32], b: &[f32]) -> f64 {
debug_assert_eq!(a.len(), b.len());
let (a_chunks, a_rest) = a.as_chunks::<SIMD_LANES>();
let (b_chunks, b_rest) = b.as_chunks::<SIMD_LANES>();
diff --git a/rust/crates/photos/src/ml_store/cluster_averages.rs b/rust/crates/photos/src/ml_store/cluster_averages.rs
index 21ae1fc425..2f5133be0e 100644
--- a/rust/crates/photos/src/ml_store/cluster_averages.rs
+++ b/rust/crates/photos/src/ml_store/cluster_averages.rs
@@ -147,7 +147,10 @@ impl MlStore {
.collect())
}
- fn cluster_centroids(&self, cluster_ids: &[String]) -> Result<HashMap<String, Vec<f32>>> {
+ pub(super) fn cluster_centroids(
+ &self,
+ cluster_ids: &[String],
+ ) -> Result<HashMap<String, Vec<f32>>> {
Ok(self
.db
.get_cluster_to_cluster_summary(cluster_ids)?
diff --git a/rust/crates/photos/src/ml_store/face_distances.rs b/rust/crates/photos/src/ml_store/face_distances.rs
index 0c66501b8a..abc00e8016 100644
--- a/rust/crates/photos/src/ml_store/face_distances.rs
+++ b/rust/crates/photos/src/ml_store/face_distances.rs
@@ -7,7 +7,7 @@ use ente_core::crypto::random_bytes;
use super::clustering::clusterable_embedding;
use super::fill::decode_centroid;
use super::{MlStore, Result};
-use crate::face_clustering::{distance, weighted_sum};
+use crate::face_clustering::{distance, dot, weighted_sum};
const SAMPLE_OFFSET_BYTES: usize = size_of::<u64>();
@@ -113,6 +113,35 @@ impl MlStore {
Ok(distances)
}
+ pub fn max_similarities_to_persons(
+ &self,
+ cluster_id: &str,
+ person_ids: &HashSet<String>,
+ ) -> Result<HashMap<String, f64>> {
+ let Some(average) = self
+ .cluster_centroids(&[cluster_id.to_owned()])?
+ .remove(cluster_id)
+ else {
+ return Ok(HashMap::new());
+ };
+ let mut cluster_to_person = self.db.get_cluster_id_to_person_id()?;
+ cluster_to_person.retain(|_, person_id| person_ids.contains(person_id));
+ let cluster_ids: Vec<String> = cluster_to_person.keys().cloned().collect();
+ let centroids = self.cluster_centroids(&cluster_ids)?;
+ let mut similarities: HashMap<String, f64> = HashMap::new();
+ for (cluster_id, person_id) in cluster_to_person {
+ let Some(centroid) = centroids.get(&cluster_id) else {
+ continue;
+ };
+ let similarity = dot(&average, centroid);
+ similarities
+ .entry(person_id)
+ .and_modify(|best| *best = best.max(similarity))
+ .or_insert(similarity);
+ }
+ Ok(similarities)
+ }
+
fn sample_embeddings(
&self,
face_ids: Vec<&String>,
@@ -322,6 +351,25 @@ mod tests {
.collect()
}
+ fn similarities(
+ store: &MlStore,
+ cluster_id: &str,
+ person_ids: &[&str],
+ ) -> HashMap<String, f64> {
+ let person_ids = person_ids.iter().map(ToString::to_string).collect();
+ store
+ .max_similarities_to_persons(cluster_id, &person_ids)
+ .unwrap()
+ }
+
+ fn assert_similarities(actual: &HashMap<String, f64>, expected: &[(&str, f64)]) {
+ assert_eq!(actual.len(), expected.len());
+ for (person_id, expected) in expected {
+ let actual = actual[*person_id];
+ assert!((actual - expected).abs() < 1e-5, "{actual} != {expected}");
+ }
+ }
+
#[test]
fn median_distance_is_the_upper_middle_of_all_pairwise_distances() {
let person = vectors(&[0.0]);
@@ -523,4 +571,65 @@ mod tests {
assert_eq!(result[3], Some(Vec::new()));
assert!(result[4].is_none());
}
+
+ #[test]
+ fn similarity_is_the_closest_cluster_of_each_requested_person() {
+ let (_directory, store) = open();
+ put_summaries(
+ &store,
+ &[
+ ("seed", 0.0, 1),
+ ("far", 60.0, 1),
+ ("near", 30.0, 1),
+ ("right_angle", 90.0, 1),
+ ("unrequested", 10.0, 1),
+ ],
+ );
+ assign_to_person(&store, "closest_second", &["far", "near"]);
+ assign_to_person(&store, "other", &["right_angle"]);
+ assign_to_person(&store, "not_requested", &["unrequested"]);
+
+ let result = similarities(&store, "seed", &["closest_second", "other", "no_clusters"]);
+
+ let expected = [
+ ("closest_second", 30.0_f64.to_radians().cos()),
+ ("other", 0.0),
+ ];
+ assert_similarities(&result, &expected);
+ }
+
+ #[test]
+ fn similarity_includes_the_cluster_itself_when_a_requested_person_owns_it() {
+ let (_directory, store) = open();
+ put_summaries(&store, &[("seed", 0.0, 1), ("other", 90.0, 1)]);
+ assign_to_person(&store, "person", &["seed", "other"]);
+
+ let result = similarities(&store, "seed", &["person"]);
+
+ assert_similarities(&result, &[("person", 1.0)]);
+ }
+
+ #[test]
+ fn similarities_are_empty_without_a_usable_average_for_the_cluster() {
+ let (_directory, store) = open();
+ put_summaries(&store, &[("a", 0.0, 1)]);
+ put_unusable_summary(&store, "unusable");
+ assign_to_person(&store, "person", &["a"]);
+
+ assert!(similarities(&store, "missing", &["person"]).is_empty());
+ assert!(similarities(&store, "unusable", &["person"]).is_empty());
+ }
+
+ #[test]
+ fn similarity_skips_person_clusters_without_a_usable_average() {
+ let (_directory, store) = open();
+ put_summaries(&store, &[("seed", 0.0, 1), ("usable", 60.0, 1)]);
+ put_unusable_summary(&store, "unusable");
+ assign_to_person(&store, "person", &["no_summary", "unusable", "usable"]);
+ assign_to_person(&store, "unusable_only", &["no_summary", "unusable"]);
+
+ let result = similarities(&store, "seed", &["person", "unusable_only"]);
+
+ assert_similarities(&result, &[("person", 0.5)]);
+ }
}