π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Run merge sheet similarity in the Rust ML store
laurenspriem20edf66df82c
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dartindex ebe0071e65..6d7e0fae6c 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_face_distances.dart@@ -1,4 +1,4 @@-import "dart:math" show Random;+import "dart:math" show Random, max;import "package:ente_pure_utils/ente_pure_utils.dart";import "package:flutter/foundation.dart";@@ -223,6 +223,48 @@ class DartFaceDistances implements FaceDistances {}}+ @override+ Future<Map<String, double>> maxSimilaritiesToPersons(+ String clusterID,+ Set<String> personIDs,+ ) async {+ final allClusterSummary = await _mlDataDB.getAllClusterSummary();+ final clusterEmbeddingData = allClusterSummary[clusterID]?.$1;+ if (clusterEmbeddingData == null) {+ return {};+ }+ final Vector clusterEmbedding = Vector.fromList(+ EVector.fromBuffer(clusterEmbeddingData).values,+ dtype: DType.float32,+ );++ final clusterToPerson = await _mlDataDB.getClusterIDToPersonID();+ clusterToPerson.removeWhere((_, personID) => !personIDs.contains(personID));+ allClusterSummary.removeWhere(+ (key, value) => !clusterToPerson.containsKey(key),+ );+ final Map<String, Vector> allClusterEmbeddings = allClusterSummary.map(+ (key, value) => MapEntry(+ key,+ Vector.fromList(+ EVector.fromBuffer(value.$1).values,+ dtype: DType.float32,+ ),+ ),+ );++ final Map<String, double> personToMaxSimilarity = {};+ for (final entry in allClusterEmbeddings.entries) {+ final personID = clusterToPerson[entry.key]!;+ final similarity = clusterEmbedding.dot(entry.value);+ personToMaxSimilarity[personID] = max(+ personToMaxSimilarity[personID] ?? double.negativeInfinity,+ similarity,+ );+ }+ return personToMaxSimilarity;+ }+Future<Vector?> _getPersonAvg(String personID) async {final w = (kDebugMode ? EnteWatch('_getPersonAvg') : null)?..start();final personClusters = await _mlDataDB.getPersonClusterIDs(personID);diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dartindex f6e3f04354..9c2b5814a9 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/face_distances.dart@@ -26,4 +26,9 @@ abstract class FaceDistances {Stream<List<(String, double)>?> distancesToPersons(List<(String, List<String>)> personFaces,);++ Future<Map<String, double>> maxSimilaritiesToPersons(+ String clusterID,+ Set<String> personIDs,+ );}diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dartindex 3664767ab5..71b8f159c4 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_face_distances.dart@@ -75,6 +75,17 @@ class StoreFaceDistances implements FaceDistances {}}+ @override+ Future<Map<String, double>> maxSimilaritiesToPersons(+ String clusterID,+ Set<String> personIDs,+ ) async {+ return (await _mlDataDB.store).maxSimilaritiesToPersons(+ clusterId: clusterID,+ personIds: personIDs.toList(growable: false),+ );+ }+Future<List<List<(String, double)>>> _suggestByMedian(List<rust.MedianQuery> queries, {required int sampleSize,diff --git a/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dart b/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dartindex 0d3078d8c4..dd2b5a4b0f 100644--- a/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dart+++ b/mobile/apps/photos/lib/ui/viewer/people/merge_clusters_to_person_sheet.dart@@ -1,5 +1,4 @@import "dart:async";-import "dart:math" show max;import "package:collection/collection.dart";import "package:dotted_border/dotted_border.dart";@@ -10,16 +9,14 @@ import "package:ente_ui/components/loading_widget.dart";import "package:flutter/material.dart";import "package:hugeicons/hugeicons.dart";import "package:logging/logging.dart";-import "package:ml_linalg/linalg.dart" as ml;import "package:photos/core/event_bus.dart";-import "package:photos/db/ml/db.dart";import "package:photos/events/people_sort_order_change_event.dart";-import "package:photos/generated/protos/ente/common/vector.pb.dart";import "package:photos/models/ml/face/person.dart";import "package:photos/models/search/generic_search_result.dart";import "package:photos/models/search/search_constants.dart";import "package:photos/service_locator.dart";import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";+import "package:photos/services/machine_learning/face_ml/feedback/face_distances.dart";import "package:photos/services/search_service.dart";import "package:photos/settings/local_settings.dart";import "package:photos/theme/ente_theme.dart";@@ -626,16 +623,6 @@ class _MergeClustersToPersonPageState extends State<MergeClustersToPersonPage> {if (!_canUseSimilaritySort || seedClusterId == null || persons.isEmpty) {return {};}- final allClusterSummary = await MLDataDB.instance.getAllClusterSummary();- final currentClusterEmbeddingData = allClusterSummary[seedClusterId]?.$1;- if (currentClusterEmbeddingData == null) {- return {};- }- final ml.Vector currentClusterEmbedding = ml.Vector.fromList(- EVector.fromBuffer(currentClusterEmbeddingData).values,- dtype: ml.DType.float32,- );-final personIds = persons.map((result) => result.params[kPersonParamID] as String?).where((id) => id != null && id.isNotEmpty)@@ -644,31 +631,10 @@ class _MergeClustersToPersonPageState extends State<MergeClustersToPersonPage> {if (personIds.isEmpty) {return {};}-- final clusterToPerson = await MLDataDB.instance.getClusterIDToPersonID();- clusterToPerson.removeWhere((_, personId) => !personIds.contains(personId));- allClusterSummary.removeWhere(- (key, value) => !clusterToPerson.containsKey(key),+ return FaceDistances.instance.maxSimilaritiesToPersons(+ seedClusterId,+ personIds,);- final Map<String, ml.Vector> allClusterEmbeddings = allClusterSummary.map(- (key, value) => MapEntry(- key,- ml.Vector.fromList(- EVector.fromBuffer(value.$1).values,- dtype: ml.DType.float32,- ),- ),- );-- for (final entry in allClusterEmbeddings.entries) {- final personId = clusterToPerson[entry.key]!;- final similarity = currentClusterEmbedding.dot(entry.value);- _personToMaxSimilarity[personId] = max(- _personToMaxSimilarity[personId] ?? double.negativeInfinity,- similarity,- );- }- return _personToMaxSimilarity;}}diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rsindex ce26e94753..f110352cf8 100644--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs@@ -425,6 +425,16 @@ impl MlStore {Ok(self.inner.distances_to_person_averages(&requests)?)}+ pub fn max_similarities_to_persons(+ &self,+ cluster_id: String,+ person_ids: Vec<String>,+ ) -> Result<HashMap<String, f64>, MlDbError> {+ Ok(self+ .inner+ .max_similarities_to_persons(&cluster_id, &person_ids.into_iter().collect())?)+ }+pub fn clear_all(&self) -> Result<(), MlDbError> {Ok(self.inner.clear_all()?)}diff --git a/rust/crates/photos/src/face_clustering/mod.rs b/rust/crates/photos/src/face_clustering/mod.rsindex b4d132993c..1ef63e5b1d 100644--- a/rust/crates/photos/src/face_clustering/mod.rs+++ b/rust/crates/photos/src/face_clustering/mod.rs@@ -126,7 +126,7 @@ pub(crate) fn distance(a: &[f32], b: &[f32]) -> f64 {1.0 - dot(a, b)}-fn dot(a: &[f32], b: &[f32]) -> f64 {+pub(crate) fn dot(a: &[f32], b: &[f32]) -> f64 {debug_assert_eq!(a.len(), b.len());let (a_chunks, a_rest) = a.as_chunks::<SIMD_LANES>();let (b_chunks, b_rest) = b.as_chunks::<SIMD_LANES>();diff --git a/rust/crates/photos/src/ml_store/cluster_averages.rs b/rust/crates/photos/src/ml_store/cluster_averages.rsindex 21ae1fc425..2f5133be0e 100644--- a/rust/crates/photos/src/ml_store/cluster_averages.rs+++ b/rust/crates/photos/src/ml_store/cluster_averages.rs@@ -147,7 +147,10 @@ impl MlStore {.collect())}- fn cluster_centroids(&self, cluster_ids: &[String]) -> Result<HashMap<String, Vec<f32>>> {+ pub(super) fn cluster_centroids(+ &self,+ cluster_ids: &[String],+ ) -> Result<HashMap<String, Vec<f32>>> {Ok(self.db.get_cluster_to_cluster_summary(cluster_ids)?diff --git a/rust/crates/photos/src/ml_store/face_distances.rs b/rust/crates/photos/src/ml_store/face_distances.rsindex 0c66501b8a..abc00e8016 100644--- a/rust/crates/photos/src/ml_store/face_distances.rs+++ b/rust/crates/photos/src/ml_store/face_distances.rs@@ -7,7 +7,7 @@ use ente_core::crypto::random_bytes;use super::clustering::clusterable_embedding;use super::fill::decode_centroid;use super::{MlStore, Result};-use crate::face_clustering::{distance, weighted_sum};+use crate::face_clustering::{distance, dot, weighted_sum};const SAMPLE_OFFSET_BYTES: usize = size_of::<u64>();@@ -113,6 +113,35 @@ impl MlStore {Ok(distances)}+ pub fn max_similarities_to_persons(+ &self,+ cluster_id: &str,+ person_ids: &HashSet<String>,+ ) -> Result<HashMap<String, f64>> {+ let Some(average) = self+ .cluster_centroids(&[cluster_id.to_owned()])?+ .remove(cluster_id)+ else {+ return Ok(HashMap::new());+ };+ let mut cluster_to_person = self.db.get_cluster_id_to_person_id()?;+ cluster_to_person.retain(|_, person_id| person_ids.contains(person_id));+ let cluster_ids: Vec<String> = cluster_to_person.keys().cloned().collect();+ let centroids = self.cluster_centroids(&cluster_ids)?;+ let mut similarities: HashMap<String, f64> = HashMap::new();+ for (cluster_id, person_id) in cluster_to_person {+ let Some(centroid) = centroids.get(&cluster_id) else {+ continue;+ };+ let similarity = dot(&average, centroid);+ similarities+ .entry(person_id)+ .and_modify(|best| *best = best.max(similarity))+ .or_insert(similarity);+ }+ Ok(similarities)+ }+fn sample_embeddings(&self,face_ids: Vec<&String>,@@ -322,6 +351,25 @@ mod tests {.collect()}+ fn similarities(+ store: &MlStore,+ cluster_id: &str,+ person_ids: &[&str],+ ) -> HashMap<String, f64> {+ let person_ids = person_ids.iter().map(ToString::to_string).collect();+ store+ .max_similarities_to_persons(cluster_id, &person_ids)+ .unwrap()+ }++ fn assert_similarities(actual: &HashMap<String, f64>, expected: &[(&str, f64)]) {+ assert_eq!(actual.len(), expected.len());+ for (person_id, expected) in expected {+ let actual = actual[*person_id];+ assert!((actual - expected).abs() < 1e-5, "{actual} != {expected}");+ }+ }+#[test]fn median_distance_is_the_upper_middle_of_all_pairwise_distances() {let person = vectors(&[0.0]);@@ -523,4 +571,65 @@ mod tests {assert_eq!(result[3], Some(Vec::new()));assert!(result[4].is_none());}++ #[test]+ fn similarity_is_the_closest_cluster_of_each_requested_person() {+ let (_directory, store) = open();+ put_summaries(+ &store,+ &[+ ("seed", 0.0, 1),+ ("far", 60.0, 1),+ ("near", 30.0, 1),+ ("right_angle", 90.0, 1),+ ("unrequested", 10.0, 1),+ ],+ );+ assign_to_person(&store, "closest_second", &["far", "near"]);+ assign_to_person(&store, "other", &["right_angle"]);+ assign_to_person(&store, "not_requested", &["unrequested"]);++ let result = similarities(&store, "seed", &["closest_second", "other", "no_clusters"]);++ let expected = [+ ("closest_second", 30.0_f64.to_radians().cos()),+ ("other", 0.0),+ ];+ assert_similarities(&result, &expected);+ }++ #[test]+ fn similarity_includes_the_cluster_itself_when_a_requested_person_owns_it() {+ let (_directory, store) = open();+ put_summaries(&store, &[("seed", 0.0, 1), ("other", 90.0, 1)]);+ assign_to_person(&store, "person", &["seed", "other"]);++ let result = similarities(&store, "seed", &["person"]);++ assert_similarities(&result, &[("person", 1.0)]);+ }++ #[test]+ fn similarities_are_empty_without_a_usable_average_for_the_cluster() {+ let (_directory, store) = open();+ put_summaries(&store, &[("a", 0.0, 1)]);+ put_unusable_summary(&store, "unusable");+ assign_to_person(&store, "person", &["a"]);++ assert!(similarities(&store, "missing", &["person"]).is_empty());+ assert!(similarities(&store, "unusable", &["person"]).is_empty());+ }++ #[test]+ fn similarity_skips_person_clusters_without_a_usable_average() {+ let (_directory, store) = open();+ put_summaries(&store, &[("seed", 0.0, 1), ("usable", 60.0, 1)]);+ put_unusable_summary(&store, "unusable");+ assign_to_person(&store, "person", &["no_summary", "unusable", "usable"]);+ assign_to_person(&store, "unusable_only", &["no_summary", "unusable"]);++ let result = similarities(&store, "seed", &["person", "unusable_only"]);++ assert_similarities(&result, &[("person", 0.5)]);+ }}