π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Bound memory used by People suggestions (#14307)
laurens-pilota38c4e31b474
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dartindex 4046483cd1..5bfcd7a2c2 100644--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart@@ -1461,9 +1461,6 @@ class ClusterFeedbackService<T> {w?.log('${p.data.name} has ${personClusters.length} existing clusters, getting all database data done',);- final allClusterIdToFaceIDs = await mlDataDB.getAllClusterIdToFaceIDs();- w?.log('getAllClusterIdToFaceIDs done');-final smallestPersonClusterSize = personClusters.map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0).reduce((value, element) => min(value, element));@@ -1495,10 +1492,14 @@ class ClusterFeedbackService<T> {w?.log('Calculate suggestions using mean for ${clusterAvgBigClusters.clusterIDs.length} clusters of min size $minimumSize',);+ final candidateFaceIDs = await mlDataDB.getClusterToFaceIDs(+ suggestionsMeanBigClusters.map((suggestion) => suggestion.$1).toSet(),+ );for (final suggestion in suggestionsMeanBigClusters) {- final suggestionSet = allClusterIdToFaceIDs[suggestion.$1]!- .map((faceID) => getFileIdFromFaceId<int>(faceID))- .toSet();+ final suggestionSet =+ (candidateFaceIDs[suggestion.$1] ?? const <String>[])+ .map((faceID) => getFileIdFromFaceId<int>(faceID))+ .toSet();final overlap = personFileIDs.intersection(suggestionSet);if (overlap.isNotEmpty &&((overlap.length / suggestionSet.length) > 0.1)) {diff --git a/rust/crates/photos/src/face_clustering/mod.rs b/rust/crates/photos/src/face_clustering/mod.rsindex 7130ec4600..db172cac60 100644--- a/rust/crates/photos/src/face_clustering/mod.rs+++ b/rust/crates/photos/src/face_clustering/mod.rs@@ -104,14 +104,19 @@ pub(crate) fn new_cluster_id() -> String {format!("{CLUSTER_ID_PREFIX}{suffix}")}-#[cfg(any(test, not(target_arch = "wasm32")))]+#[cfg(test)]pub(crate) fn normalized_mean(embeddings: &[Vec<f32>]) -> Option<Vec<f32>> {let (first, rest) = embeddings.split_first()?;let mut sum = first.clone();for embedding in rest {add_scaled(&mut sum, embedding, 1.0);}- Some(normalized(scaled(sum, 1.0 / embeddings.len() as f64)))+ Some(normalized_mean_from_sum(sum, embeddings.len()))+}++#[cfg(any(test, not(target_arch = "wasm32")))]+pub(crate) fn normalized_mean_from_sum(sum: Vec<f32>, count: usize) -> Vec<f32> {+ normalized(scaled(sum, 1.0 / count as f64))}pub(crate) fn weighted_sum(terms: &[(&[f32], f64)]) -> Option<Vec<f32>> {diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rsindex 4342db992d..a07e342e0e 100644--- a/rust/crates/photos/src/ml_db/queries/faces.rs+++ b/rust/crates/photos/src/ml_db/queries/faces.rs@@ -184,6 +184,35 @@ impl MlDb {Ok(result)}+ pub(crate) fn for_each_face_embedding_for_clusters(+ &self,+ cluster_ids: &[String],+ mut visit: impl FnMut(String, Vec<u8>),+ ) -> Result<()> {+ let cluster_ids = unique_in_order(cluster_ids);+ self.db+ .read(|connection| {+ for chunk in cluster_ids.chunks(MAX_SQL_BIND_PARAMS_PER_QUERY) {+ let sql = format!(+ r#"+ SELECT fc.cluster_id, fe.embedding+ FROM face_clusters fc+ INNER JOIN faces fe ON fc.face_id = fe.face_id+ WHERE fc.cluster_id IN ({})+ "#,+ bind_placeholders(chunk.len())+ );+ let mut statement = connection.prepare_cached(&sql)?;+ for row in statement.query_map(params_from_iter(chunk), pair)? {+ let (cluster_id, embedding) = row?;+ visit(cluster_id, embedding);+ }+ }+ Ok(())+ })+ .map_err(Into::into)+ }+pub fn get_cover_face_for_person(&self,recent_file_id: i64,diff --git a/rust/crates/photos/src/ml_store/cluster_averages.rs b/rust/crates/photos/src/ml_store/cluster_averages.rsindex 2f5133be0e..715065a2f7 100644--- a/rust/crates/photos/src/ml_store/cluster_averages.rs+++ b/rust/crates/photos/src/ml_store/cluster_averages.rs@@ -4,11 +4,12 @@ use std::collections::{HashMap, HashSet};use super::clustering::{clusterable_embedding, is_clusterable, summaries};use super::fill::decode_centroid;use super::{Error, Index, Match, MlStore, Result, SearchParams};-use crate::face_clustering::{ClusterMean, distance, normalized_mean};+use crate::face_clustering::{ClusterMean, distance, normalized_mean_from_sum};const MAX_CLUSTERS_PER_REFRESH: usize = 500;const MAX_EMBEDDINGS_PER_REFRESH: i64 = 10_000;const MAX_MEAN_SUGGESTIONS: usize = 20;+const SUMMARY_READ_BATCH_SIZE: usize = 500;#[derive(Clone, Debug, PartialEq)]pub struct MeanSuggestion {@@ -17,6 +18,41 @@ pub struct MeanSuggestion {pub person_cluster_id: String,}+#[derive(Default)]+struct ClusterAverage {+ sum: Vec<f32>,+ valid_count: usize,+ count: usize,+}++impl ClusterAverage {+ fn add(&mut self, embedding: &[u8]) {+ self.count += 1;+ let Some(embedding) = clusterable_embedding(embedding) else {+ return;+ };+ if self.valid_count == 0 {+ self.sum = embedding;+ } else {+ for (sum, value) in self.sum.iter_mut().zip(embedding) {+ *sum += value;+ }+ }+ self.valid_count += 1;+ }++ fn into_mean(self) -> Option<ClusterMean> {+ if self.valid_count == 0 {+ return None;+ }+ let mean = normalized_mean_from_sum(self.sum, self.valid_count);+ is_clusterable(&mean).then_some(ClusterMean {+ mean,+ count: self.count,+ })+ }+}+impl MlStore {pub fn refresh_cluster_averages(&self,@@ -24,17 +60,26 @@ impl MlStore {ignored_cluster_ids: &HashSet<String>,min_cluster_size: i64,) -> Result<HashSet<String>> {- let summaries_by_id = self.db.get_all_cluster_summary(Some(min_cluster_size))?;let mut averaged = HashSet::new();let mut stale = Vec::new();- for (cluster_id, &face_count) in cluster_face_counts {- let current = summaries_by_id.get(cluster_id).is_some_and(|summary| {- summary.count == face_count && decode_centroid(cluster_id, &summary.avg).is_some()- });- if current {- averaged.insert(cluster_id.clone());- } else if face_count >= min_cluster_size && !ignored_cluster_ids.contains(cluster_id) {- stale.push((cluster_id.clone(), face_count));+ let cluster_ids: Vec<String> = cluster_face_counts+ .iter()+ .filter(|&(_, &face_count)| face_count >= min_cluster_size)+ .map(|(cluster_id, _)| cluster_id.clone())+ .collect();+ for chunk in cluster_ids.chunks(SUMMARY_READ_BATCH_SIZE) {+ let summaries = self.db.get_cluster_to_cluster_summary(chunk)?;+ for cluster_id in chunk {+ let face_count = cluster_face_counts[cluster_id];+ let current = summaries.get(cluster_id).is_some_and(|summary| {+ summary.count == face_count+ && decode_centroid(cluster_id, &summary.avg).is_some()+ });+ if current {+ averaged.insert(cluster_id.clone());+ } else if !ignored_cluster_ids.contains(cluster_id) {+ stale.push((cluster_id.clone(), face_count));+ }}}let means = self.average_clusters(&largest_within_budget(stale))?;@@ -91,20 +136,18 @@ impl MlStore {max_distance: f64,ignored_by_person_cluster: &HashMap<String, HashSet<String>>,) -> Result<Vec<MeanSuggestion>> {- let cluster_ids: Vec<String> = person_cluster_ids- .iter()- .chain(candidate_cluster_ids)- .cloned()- .collect();- let centroids = self.cluster_centroids(&cluster_ids)?;+ let centroids = self.cluster_centroids(person_cluster_ids)?;let person_centroids: Vec<(&String, &Vec<f32>)> = person_cluster_ids.iter().filter_map(|cluster_id| Some((cluster_id, centroids.get(cluster_id)?))).collect();- let mut suggestions: Vec<MeanSuggestion> = candidate_cluster_ids- .iter()- .filter_map(|candidate_id| {- let candidate = centroids.get(candidate_id)?;+ let mut suggestions: Vec<MeanSuggestion> = Vec::new();+ for chunk in candidate_cluster_ids.chunks(SUMMARY_READ_BATCH_SIZE) {+ let candidates = self.cluster_centroids(chunk)?;+ for candidate_id in chunk {+ let Some(candidate) = candidates.get(candidate_id) else {+ continue;+ };let mut nearest: Option<(&String, f64)> = None;for &(person_cluster_id, person_centroid) in &person_centroids {if ignored_by_person_cluster@@ -118,32 +161,36 @@ impl MlStore {nearest = Some((person_cluster_id, distance));}}- nearest.map(|(person_cluster_id, distance)| MeanSuggestion {- cluster_id: candidate_id.clone(),- distance,- person_cluster_id: person_cluster_id.clone(),- })- })- .collect();- suggestions.sort_by(|a, b| a.distance.total_cmp(&b.distance));- suggestions.truncate(MAX_MEAN_SUGGESTIONS);+ if let Some((person_cluster_id, distance)) = nearest {+ let position = suggestions.partition_point(|suggestion| {+ suggestion.distance.total_cmp(&distance).is_le()+ });+ if position < MAX_MEAN_SUGGESTIONS {+ suggestions.insert(+ position,+ MeanSuggestion {+ cluster_id: candidate_id.clone(),+ distance,+ person_cluster_id: person_cluster_id.clone(),+ },+ );+ suggestions.truncate(MAX_MEAN_SUGGESTIONS);+ }+ }+ }+ }Ok(suggestions)}fn average_clusters(&self, cluster_ids: &[String]) -> Result<HashMap<String, ClusterMean>> {- Ok(self- .db- .get_face_embeddings_for_clusters(cluster_ids, None)?+ let mut averages: HashMap<String, ClusterAverage> = HashMap::new();+ self.db+ .for_each_face_embedding_for_clusters(cluster_ids, |cluster_id, embedding| {+ averages.entry(cluster_id).or_default().add(&embedding);+ })?;+ Ok(averages.into_iter()- .filter_map(|(cluster_id, embeddings)| {- let valid: Vec<Vec<f32>> = embeddings- .iter()- .filter_map(|embedding| clusterable_embedding(embedding))- .collect();- let mean = normalized_mean(&valid).filter(|mean| is_clusterable(mean))?;- let count = embeddings.len();- Some((cluster_id, ClusterMean { mean, count }))- })+ .filter_map(|(cluster_id, average)| Some((cluster_id, average.into_mean()?))).collect())}@@ -293,6 +340,28 @@ mod tests {assert_eq!(refreshed.keys().collect::<Vec<_>>(), ["large"]);}+ #[test]+ fn refreshing_excludes_current_small_clusters_and_recomputes_eligible_stale_counts() {+ let (_directory, store) = open();+ seed(&store, &[(1, 0.0), (2, 0.0), (3, 90.0)]);+ assign(&store, &[(1, "small"), (2, "stale"), (3, "stale")]);+ store+ .cluster_summary_update(&HashMap::from([+ ("small".to_owned(), summary(30.0, 1)),+ ("stale".to_owned(), summary(0.0, 1)),+ ]))+ .unwrap();++ assert_eq!(refresh(&store, &[], 2), id_set(&["stale"]));++ let (mean, count) = summary_of(&store, "small");+ assert_eq!(count, 1);+ assert_direction(&mean, 30.0);+ let (mean, count) = summary_of(&store, "stale");+ assert_eq!(count, 2);+ assert_direction(&mean, 45.0);+ }+#[test]fn refreshing_averages_only_valid_embeddings() {let (_directory, store) = open();@@ -493,4 +562,91 @@ mod tests {let expected: Vec<String> = (1..=20).map(|degrees| format!("c{degrees}")).collect();assert_eq!(found, expected);}++ #[test]+ fn warm_refresh_spans_summary_batches_and_excludes_invalid_summaries() {+ let (_directory, store) = open();+ let clusters: Vec<String> = (0..1100).map(|i| format!("c{i}")).collect();+ let summaries: Vec<(&str, f64)> = clusters.iter().map(|id| (id.as_str(), 0.0)).collect();+ put_summaries(&store, &summaries);+ put_summaries(&store, &[("outside", 0.0)]);+ store+ .db()+ .upsert_cluster_summary_rows(&HashMap::from([(+ clusters[500].clone(),+ ClusterSummary {+ avg: vec![255],+ count: 1,+ },+ )]))+ .unwrap();+ let counts = clusters.iter().map(|id| (id.clone(), 1)).collect();++ let averaged = store+ .refresh_cluster_averages(&counts, &HashSet::new(), 1)+ .unwrap();++ let expected = clusters.into_iter().filter(|id| id != "c500").collect();+ assert_eq!(averaged, expected);+ }++ #[test]+ fn oversized_refresh_matches_collected_mean_and_counts_invalid_rows() {+ let (_directory, store) = open();+ let mut faces: Vec<_> = (1..=10003).map(|id| face(id, (id % 17) as f64)).collect();+ faces[0].embedding.truncate(3);+ faces[1].embedding[0] = f64::NAN;+ store.db().bulk_insert_faces(&faces).unwrap();+ let assignments: Vec<_> = (1..=10003).map(|id| (id, "large")).collect();+ assign(&store, &assignments);+ let stored = store+ .db()+ .get_face_embeddings_for_clusters(&ids(&["large"]), None)+ .unwrap();+ let valid: Vec<_> = stored["large"]+ .iter()+ .filter_map(|embedding| crate::ml_store::clustering::clusterable_embedding(embedding))+ .collect();+ let expected = crate::face_clustering::normalized_mean(&valid).unwrap();++ assert_eq!(refresh(&store, &[], 1), id_set(&["large"]));++ let (mean, count) = summary_of(&store, "large");+ assert_eq!(count, 10003);+ assert_eq!(+ mean,+ expected.into_iter().map(f64::from).collect::<Vec<_>>()+ );+ assert_eq!(refresh(&store, &[], 1), id_set(&["large"]));+ }++ #[test]+ fn mean_suggestions_preserve_ties_and_rejections_across_batches() {+ let (_directory, store) = open();+ let candidates: Vec<String> = (0..1100).map(|i| format!("c{i}")).collect();+ let summaries: Vec<(&str, f64)> = candidates.iter().map(|id| (id.as_str(), 30.0)).collect();+ put_summaries(&store, &summaries);+ put_summaries(&store, &[("p1", 0.0), ("p2", 0.0), ("c1099", 1.0)]);+ let ignored = HashMap::from([+ ("p1".to_owned(), id_set(&["c1"])),+ ("p2".to_owned(), id_set(&["c1"])),+ ]);++ let found = store+ .suggest_clusters_by_mean(&ids(&["p2", "p1"]), &candidates, 0.5, &ignored)+ .unwrap();++ let expected: Vec<_> = ["c1099".to_owned()]+ .into_iter()+ .chain((0..20).filter(|&i| i != 1).map(|i| format!("c{i}")))+ .collect();+ assert_eq!(+ found+ .iter()+ .map(|s| s.cluster_id.clone())+ .collect::<Vec<_>>(),+ expected+ );+ assert!(found.iter().all(|s| s.person_cluster_id == "p2"));+ }}