Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Refresh cluster averages and search suggestions by cluster ID in the Rust ML store

laurenspriem6f4f6d6f4d3a
diff --git a/mobile/apps/photos/lib/db/ml/cluster_centroid_vector_db.dart b/mobile/apps/photos/lib/db/ml/cluster_centroid_vector_db.dart
index f55058205e..f46415fa11 100644
--- a/mobile/apps/photos/lib/db/ml/cluster_centroid_vector_db.dart
+++ b/mobile/apps/photos/lib/db/ml/cluster_centroid_vector_db.dart
@@ -20,12 +20,6 @@ abstract class ClusterCentroidVectorDB {
Future<bool> isReady();
Future<void> warmup();
- Future<List<List<(String, double)>>> bulkSearchNearestAllowed(
- List<List<double>> queries,
- Set<String> allowedClusterIDs, {
- required double maxDistance,
- int count = 10,
- });
Future<Map<String, ClusterCentroidMatches>> bulkSearchNearestForClusters(
Set<String> queryClusterIDs, {
required Set<String> candidateClusterIDs,
diff --git a/mobile/apps/photos/lib/db/ml/store_cluster_centroid_vector_db.dart b/mobile/apps/photos/lib/db/ml/store_cluster_centroid_vector_db.dart
index ad0b2eb4c7..f2e1f144a6 100644
--- a/mobile/apps/photos/lib/db/ml/store_cluster_centroid_vector_db.dart
+++ b/mobile/apps/photos/lib/db/ml/store_cluster_centroid_vector_db.dart
@@ -1,5 +1,3 @@
-import "dart:typed_data" show Float32List;
-
import "package:logging/logging.dart";
import "package:photos/db/ml/cluster_centroid_vector_db.dart";
import "package:photos/db/ml/rust_db.dart";
@@ -51,36 +49,29 @@ class StoreClusterCentroidVectorDB implements ClusterCentroidVectorDB {
}
}
- @override
- Future<List<List<(String, double)>>> bulkSearchNearestAllowed(
- List<List<double>> queries,
+ Future<List<List<(String, double)>>> bulkSearchNearestAllowedForClusters(
+ List<String> queryClusterIDs,
Set<String> allowedClusterIDs, {
required double maxDistance,
- int count = 10,
+ required int count,
}) async {
await _ensureReady();
- if (count <= 0 || queries.isEmpty || allowedClusterIDs.isEmpty) {
- return List.generate(queries.length, (_) => const <(String, double)>[]);
+ if (count <= 0 || queryClusterIDs.isEmpty || allowedClusterIDs.isEmpty) {
+ return List.generate(
+ queryClusterIDs.length,
+ (_) => const <(String, double)>[],
+ );
}
- final matchesPerQuery = await (await _mlDataDB.store).bulkSearch(
- index: rust.Index.clusterCentroid,
- queries: [
- for (final query in queries)
- query is Float32List ? query : Float32List.fromList(query),
- ],
- limit: count,
- maxDistance: maxDistance,
- exact: false,
- allowedKeys: allowedClusterIDs.toList(),
- );
+ final matchesPerQuery = await (await _mlDataDB.store)
+ .searchClusterCentroids(
+ clusterIds: queryClusterIDs,
+ allowedClusterIds: allowedClusterIDs.toList(),
+ limit: count,
+ maxDistance: maxDistance,
+ );
return [
- for (var i = 0; i < queries.length; i++)
- i < matchesPerQuery.length
- ? [
- for (final match in matchesPerQuery[i])
- (match.key, match.distance),
- ]
- : const <(String, double)>[],
+ for (final matches in matchesPerQuery)
+ [for (final match in matches) (match.key, match.distance)],
];
}
diff --git a/mobile/apps/photos/lib/db/ml/usearch_cluster_centroid_vector_db.dart b/mobile/apps/photos/lib/db/ml/usearch_cluster_centroid_vector_db.dart
index 93b3cc2874..3d105e5383 100644
--- a/mobile/apps/photos/lib/db/ml/usearch_cluster_centroid_vector_db.dart
+++ b/mobile/apps/photos/lib/db/ml/usearch_cluster_centroid_vector_db.dart
@@ -258,7 +258,6 @@ class UsearchClusterCentroidVectorDB implements ClusterCentroidVectorDB {
return output;
}
- @override
Future<List<List<(String, double)>>> bulkSearchNearestAllowed(
List<List<double>> queries,
Set<String> allowedClusterIDs, {
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_averages.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_averages.dart
new file mode 100644
index 0000000000..26046cdd89
--- /dev/null
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_averages.dart
@@ -0,0 +1,17 @@
+abstract class ClusterAverages {
+ Set<String> get clusterIDs;
+
+ Future<List<List<(String, double)>>> searchNearest(
+ List<String> queryClusterIDs,
+ Set<String> allowedClusterIDs, {
+ required double maxDistance,
+ required int count,
+ });
+
+ Future<List<(String, double, String)>> suggestByMean(
+ Set<String> personClusters,
+ Set<String> ignoredClusters,
+ double maxClusterDistance, {
+ Map<String, Set<String>>? personClusterToIgnoredClusters,
+ });
+}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
index 1aa05670b7..f7bc2818d5 100644
--- a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
@@ -1,8 +1,6 @@
import "dart:async";
-import 'dart:developer' as dev show log;
import "dart:math" show Random, min;
-import "package:computer/computer.dart";
import "package:ente_pure_utils/ente_pure_utils.dart";
import "package:flutter/foundation.dart";
import "package:logging/logging.dart";
@@ -21,6 +19,9 @@ import "package:photos/service_locator.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
+import "package:photos/services/machine_learning/face_ml/feedback/cluster_averages.dart";
+import "package:photos/services/machine_learning/face_ml/feedback/dart_cluster_averages.dart";
+import "package:photos/services/machine_learning/face_ml/feedback/store_cluster_averages.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/services/search_service.dart";
@@ -45,7 +46,6 @@ class ClusterSuggestion {
class ClusterFeedbackService<T> {
final Logger _logger = Logger("ClusterFeedbackService");
- final _computer = Computer.shared();
static const int _allPeopleSuggestionNamedSampleLimit = 20;
static const int _allPeopleSuggestionVectorResultCount = 120;
@@ -436,7 +436,7 @@ class ClusterFeedbackService<T> {
ignoredClustersForAverages,
minClusterSize: 1,
);
- if (clusterAvg.isEmpty) {
+ if (clusterAvg.clusterIDs.isEmpty) {
return [];
}
@@ -445,7 +445,7 @@ class ClusterFeedbackService<T> {
for (final entry in personIdToClusters.entries) {
final validClusters = <String>{};
for (final clusterID in entry.value) {
- if (!clusterAvg.containsKey(clusterID)) {
+ if (!clusterAvg.clusterIDs.contains(clusterID)) {
continue;
}
validClusters.add(clusterID);
@@ -464,7 +464,7 @@ class ClusterFeedbackService<T> {
resultByQueryCluster = await _clusterCentroidVectorDB
.bulkSearchNearestForClusters(
allQueryClusterIDs,
- candidateClusterIDs: clusterAvg.keys.toSet(),
+ candidateClusterIDs: clusterAvg.clusterIDs,
count: _allPeopleSuggestionVectorResultCount,
maxDistance: 0.65,
);
@@ -553,7 +553,7 @@ class ClusterFeedbackService<T> {
personClusters.contains(candidateClusterID)) {
continue;
}
- if (!clusterAvg.containsKey(candidateClusterID)) {
+ if (!clusterAvg.clusterIDs.contains(candidateClusterID)) {
continue;
}
hasUsableCandidate = true;
@@ -580,7 +580,7 @@ class ClusterFeedbackService<T> {
final personClusters = personIDToQueryClusters[personID]!;
final ignoredClusters = personIDToIgnoredClusters[personID]!;
final allowedClusterIDs = <String>{};
- for (final clusterID in clusterAvg.keys) {
+ for (final clusterID in clusterAvg.clusterIDs) {
if (ignoredClusters.contains(clusterID) ||
personClusters.contains(clusterID)) {
continue;
@@ -599,29 +599,21 @@ class ClusterFeedbackService<T> {
continue;
}
- final queryClusterIDs = <String>[];
- final queryVectors = <List<double>>[];
- for (final queryClusterID in entry.value) {
- final queryVector = clusterAvg[queryClusterID];
- if (queryVector == null) {
- continue;
- }
- queryClusterIDs.add(queryClusterID);
- queryVectors.add(queryVector.toList(growable: false));
- }
+ final queryClusterIDs = entry.value
+ .where(clusterAvg.clusterIDs.contains)
+ .toList(growable: false);
if (queryClusterIDs.isEmpty) {
continue;
}
List<List<(String, double)>> filteredMatchesByQuery;
try {
- filteredMatchesByQuery = await _clusterCentroidVectorDB
- .bulkSearchNearestAllowed(
- queryVectors,
- allowedClusterIDs,
- maxDistance: 0.65,
- count: _allPeopleSuggestionVectorResultCount,
- );
+ filteredMatchesByQuery = await clusterAvg.searchNearest(
+ queryClusterIDs,
+ allowedClusterIDs,
+ maxDistance: 0.65,
+ count: _allPeopleSuggestionVectorResultCount,
+ );
} catch (e, s) {
_logger.warning(
"Filtered centroid fallback search failed for person $personID",
@@ -675,7 +667,7 @@ class ClusterFeedbackService<T> {
personClusters.contains(candidateClusterID)) {
continue;
}
- if (!clusterAvg.containsKey(candidateClusterID)) {
+ if (!clusterAvg.clusterIDs.contains(candidateClusterID)) {
continue;
}
final previous = bestDistanceByCluster[candidateClusterID];
@@ -1103,8 +1095,7 @@ class ClusterFeedbackService<T> {
personClusterToIgnoredClusters[personCluster] = ignoredClusters;
}
- final foundSuggestions = await calcSuggestionsMeanInComputer(
- clusterAvg,
+ final foundSuggestions = await clusterAvg.suggestByMean(
allPersonClusters,
allOtherPersonClustersToIgnore,
0.55,
@@ -1560,7 +1551,7 @@ class ClusterFeedbackService<T> {
.map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0)
.reduce((value, element) => min(value, element));
final checkSizes = [20, kMinimumClusterSizeSearchResult, 10, 5, 1];
- Map<String, Vector> clusterAvgBigClusters = <String, Vector>{};
+ ClusterAverages? clusterAvgBigClusters;
final List<(String, double)> suggestionsMean = [];
bool canUseVectorDb = await _canUseClusterCentroidVectorDbForSuggestions();
for (final minimumSize in checkSizes.toSet()) {
@@ -1572,7 +1563,7 @@ class ClusterFeedbackService<T> {
minClusterSize: minimumSize,
);
w?.log(
- 'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize',
+ 'Calculate avg for ${clusterAvgBigClusters.clusterIDs.length} clusters of min size $minimumSize',
);
final suggestionsMeanBigClusters =
await _calcSuggestionsMeanWithFallback(
@@ -1585,7 +1576,7 @@ class ClusterFeedbackService<T> {
onVectorDbFailed: () => canUseVectorDb = false,
);
w?.log(
- 'Calculate suggestions using mean for ${clusterAvgBigClusters.length} clusters of min size $minimumSize',
+ 'Calculate suggestions using mean for ${clusterAvgBigClusters.clusterIDs.length} clusters of min size $minimumSize',
);
for (final suggestion in suggestionsMeanBigClusters) {
final suggestionSet = allClusterIdToFaceIDs[suggestion.$1]!
@@ -1612,15 +1603,17 @@ class ClusterFeedbackService<T> {
w?.reset();
final clusterAvg = clusterAvgBigClusters;
- final moreSuggestionsMean = await _calcSuggestionsMeanWithFallback(
- clusterAvg,
- personClusters,
- ignoredClusters,
- maxMeanDistance,
- canUseVectorDb: canUseVectorDb,
- perClusterResultCount: 12,
- onVectorDbFailed: () => canUseVectorDb = false,
- );
+ final moreSuggestionsMean = clusterAvg == null
+ ? <(String, double, String)>[]
+ : await _calcSuggestionsMeanWithFallback(
+ clusterAvg,
+ personClusters,
+ ignoredClusters,
+ maxMeanDistance,
+ canUseVectorDb: canUseVectorDb,
+ perClusterResultCount: 12,
+ onVectorDbFailed: () => canUseVectorDb = false,
+ );
if (moreSuggestionsMean.isEmpty) {
_logger.info(
"No suggestions found using mean, even with higher threshold",
@@ -1741,20 +1734,15 @@ class ClusterFeedbackService<T> {
extraIgnoredClusters ?? {},
);
final startTime = DateTime.now();
- final Map<String, Vector> clusterAvg = await _getUpdateClusterAvg(
+ final clusterAvg = await _getUpdateClusterAvg(
allClusterIdsToCountMap,
ignoredClusters,
minClusterSize: kMinimumClusterSizeSearchResult,
);
final avgCalcTime = DateTime.now();
- final List<(String, double, String)> foundSuggestions =
- await calcSuggestionsMeanInComputer(
- clusterAvg,
- {clusterID},
- ignoredClusters,
- threshold,
- );
+ final List<(String, double, String)> foundSuggestions = await clusterAvg
+ .suggestByMean({clusterID}, ignoredClusters, threshold);
final suggestionCalcTime = DateTime.now();
_logger.info(
"Calculated average vectors in ${avgCalcTime.difference(startTime).inMilliseconds}ms and suggestions in ${suggestionCalcTime.difference(avgCalcTime).inMilliseconds}ms",
@@ -1762,144 +1750,29 @@ class ClusterFeedbackService<T> {
return foundSuggestions;
}
- Future<Map<String, Vector>> _getUpdateClusterAvg(
+ Future<ClusterAverages> _getUpdateClusterAvg(
Map<String, int> allClusterIdsToCountMap,
Set<String> ignoredClusters, {
int minClusterSize = 1,
- int maxClusterInCurrentRun = 500,
- int maxEmbeddingToRead = 10000,
- }) async {
- final w = (kDebugMode ? EnteWatch('_getUpdateClusterAvg') : null)?..start();
- final startTime = DateTime.now();
- final shouldPersistClusterSummaryUpdates =
- _prepareClusterCentroidVectorDbFuture == null;
- _logger.info(
- 'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters, minClusterSize $minClusterSize, maxClusterInCurrentRun $maxClusterInCurrentRun',
- );
- if (!shouldPersistClusterSummaryUpdates) {
- _logger.info(
- "Skipping cluster summary cache writes while centroid vector DB migration is in progress",
- );
- }
-
- final Map<String, (Uint8List, int)> clusterToSummary = await mlDataDB
- .getAllClusterSummary(minClusterSize);
- final Map<String, (Uint8List, int)> updatesForClusterSummary = {};
-
- w?.log('getUpdateClusterAvg database call for getAllClusterSummary');
-
- final serializationEmbeddings =
- await _computer.compute(
- checkAndSerializeCurrentClusterMeans,
- param: {
- 'allClusterIdsToCountMap': allClusterIdsToCountMap,
- 'minClusterSize': minClusterSize,
- 'ignoredClusters': ignoredClusters,
- 'clusterToSummary': clusterToSummary,
- },
- )
- as (Map<String, Vector>, Set<String>, int, int, int);
- final clusterAvg = serializationEmbeddings.$1;
- final allClusterIds = serializationEmbeddings.$2;
- final ignoredClustersCnt = serializationEmbeddings.$3;
- final alreadyUpdatedClustersCnt = serializationEmbeddings.$4;
- final smallerClustersCnt = serializationEmbeddings.$5;
-
- for (final avg in clusterAvg.values) {
- assert((avg.norm() - 1.0).abs() < 1e-5);
- }
-
- w?.log('serialization of embeddings');
- _logger.info(
- 'Ignored $ignoredClustersCnt clusters, already updated $alreadyUpdatedClustersCnt clusters, $smallerClustersCnt clusters are smaller than $minClusterSize',
- );
-
- if (allClusterIds.isEmpty) {
- _logger.info(
- 'No clusters to update, getUpdateClusterAvg done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
- );
- return clusterAvg;
- }
-
- final sortedClusterIDs = allClusterIds.toList();
- sortedClusterIDs.sort(
- (a, b) =>
- allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!),
- );
- int indexedInCurrentRun = 0;
- w?.reset();
-
- int currentPendingRead = 0;
- final List<String> clusterIdsToRead = [];
- for (final clusterID in sortedClusterIDs) {
- if (maxClusterInCurrentRun-- <= 0) {
- break;
- }
- if (currentPendingRead == 0) {
- currentPendingRead = allClusterIdsToCountMap[clusterID] ?? 0;
- clusterIdsToRead.add(clusterID);
- } else {
- if ((currentPendingRead + allClusterIdsToCountMap[clusterID]!) <
- maxEmbeddingToRead) {
- clusterIdsToRead.add(clusterID);
- currentPendingRead += allClusterIdsToCountMap[clusterID]!;
- } else {
- break;
- }
- }
- }
-
- final Map<String, Iterable<Uint8List>> clusterEmbeddings = await mlDataDB
- .getFaceEmbeddingsForClusters(clusterIdsToRead);
-
- w?.logAndReset(
- 'read $currentPendingRead embeddings for ${clusterEmbeddings.length} clusters',
- );
-
- for (final clusterID in clusterEmbeddings.keys) {
- final Iterable<Uint8List> embeddings = clusterEmbeddings[clusterID]!;
- final Iterable<Vector> vectors = embeddings.map(
- (e) =>
- Vector.fromList(EVector.fromBuffer(e).values, dtype: DType.float32),
+ }) {
+ if (MLDataDB.isRustBackend) {
+ return StoreClusterAverages.refresh(
+ allClusterIdsToCountMap,
+ ignoredClusters,
+ minClusterSize: minClusterSize,
);
- final avg = vectors.reduce((a, b) => a + b) / vectors.length;
- final avgNormalized = avg / avg.norm();
- if (shouldPersistClusterSummaryUpdates) {
- final avgEmbeddingBuffer = EVector(
- values: avgNormalized,
- ).writeToBuffer();
- updatesForClusterSummary[clusterID] = (
- avgEmbeddingBuffer,
- embeddings.length,
- );
- }
- indexedInCurrentRun++;
- if (shouldPersistClusterSummaryUpdates &&
- updatesForClusterSummary.length > 100) {
- await mlDataDB.clusterSummaryUpdate(updatesForClusterSummary);
- updatesForClusterSummary.clear();
- if (kDebugMode) {
- _logger.info(
- 'getUpdateClusterAvg $indexedInCurrentRun clusters in current one',
- );
- }
- }
- clusterAvg[clusterID] = avgNormalized;
}
- if (shouldPersistClusterSummaryUpdates &&
- updatesForClusterSummary.isNotEmpty) {
- await mlDataDB.clusterSummaryUpdate(updatesForClusterSummary);
- }
- w?.logAndReset('done computing avg ');
- _logger.info(
- 'end getUpdateClusterAvg for ${clusterAvg.length} clusters, done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
+ return DartClusterAverages.refresh(
+ allClusterIdsToCountMap,
+ ignoredClusters,
+ minClusterSize: minClusterSize,
+ shouldPersistClusterSummaryUpdates:
+ _prepareClusterCentroidVectorDbFuture == null,
);
-
- return clusterAvg;
}
Future<List<(String, double, String)>> _calcSuggestionsMeanWithFallback(
- Map<String, Vector> clusterAvg,
+ ClusterAverages clusterAvg,
Set<String> personClusters,
Set<String> ignoredClusters,
double maxClusterDistance, {
@@ -1925,8 +1798,7 @@ class ClusterFeedbackService<T> {
onVectorDbFailed();
}
}
- return calcSuggestionsMeanInComputer(
- clusterAvg,
+ return clusterAvg.suggestByMean(
personClusters,
ignoredClusters,
maxClusterDistance,
@@ -1934,13 +1806,13 @@ class ClusterFeedbackService<T> {
}
Future<List<(String, double, String)>> _calcSuggestionsMeanWithVectorDb(
- Map<String, Vector> clusterAvg,
+ ClusterAverages clusterAvg,
Set<String> personClusters,
Set<String> ignoredClusters,
double maxClusterDistance, {
int perClusterResultCount = 8,
}) async {
- if (clusterAvg.isEmpty) {
+ if (clusterAvg.clusterIDs.isEmpty) {
return [];
}
if (perClusterResultCount <= 0) {
@@ -1948,41 +1820,26 @@ class ClusterFeedbackService<T> {
}
final validPersonClusters = personClusters
- .where((clusterID) => clusterAvg.containsKey(clusterID))
+ .where(clusterAvg.clusterIDs.contains)
.toSet();
if (validPersonClusters.isEmpty) {
return [];
}
- final otherClusters = clusterAvg.keys
- .toSet()
+ final otherClusters = clusterAvg.clusterIDs
.difference(validPersonClusters)
.difference(ignoredClusters);
if (otherClusters.isEmpty) {
return [];
}
- final orderedPersonClusterIDs = <String>[];
- final queryVectors = <List<double>>[];
- for (final personClusterID in validPersonClusters) {
- final queryVector = clusterAvg[personClusterID];
- if (queryVector == null) {
- continue;
- }
- orderedPersonClusterIDs.add(personClusterID);
- queryVectors.add(queryVector.toList(growable: false));
- }
- if (orderedPersonClusterIDs.isEmpty) {
- return [];
- }
-
- final matchesPerQuery = await _clusterCentroidVectorDB
- .bulkSearchNearestAllowed(
- queryVectors,
- otherClusters,
- maxDistance: maxClusterDistance,
- count: perClusterResultCount,
- );
+ final orderedPersonClusterIDs = validPersonClusters.toList(growable: false);
+ final matchesPerQuery = await clusterAvg.searchNearest(
+ orderedPersonClusterIDs,
+ otherClusters,
+ maxDistance: maxClusterDistance,
+ count: perClusterResultCount,
+ );
final bestSuggestionPerCluster = <String, (double, String)>{};
final alignedQueryLength = min(
@@ -2017,25 +1874,6 @@ class ClusterFeedbackService<T> {
return suggestions.sublist(0, min(suggestions.length, 20));
}
- Future<List<(String, double, String)>> calcSuggestionsMeanInComputer(
- Map<String, Vector> clusterAvg,
- Set<String> personClusters,
- Set<String> ignoredClusters,
- double maxClusterDistance, {
- Map<String, Set<String>>? personClusterToIgnoredClusters,
- }) async {
- return await _computer.compute(
- _calcSuggestionsMean,
- param: {
- 'clusterAvg': clusterAvg,
- 'personClusters': personClusters,
- 'ignoredClusters': ignoredClusters,
- 'maxClusterDistance': maxClusterDistance,
- 'personClusterToIgnoredClusters': personClusterToIgnoredClusters,
- },
- );
- }
-
List<S> _randomSampleWithoutReplacement<S>(
Iterable<S> embeddings,
int sampleSize,
@@ -2320,133 +2158,3 @@ class ClusterFeedbackService<T> {
return;
}
}
-
-// Returns (suggestion cluster ID, distance, person cluster ID).
-List<(String, double, String)> _calcSuggestionsMean(Map<String, dynamic> args) {
- final Map<String, Vector> clusterAvg = args['clusterAvg'];
- final Set<String> personClusters = args['personClusters'];
- final Set<String> ignoredClusters = args['ignoredClusters'];
- final double maxClusterDistance = args['maxClusterDistance'];
- final Map<String, Set<String>>? personClusterToIgnoredClusters =
- args['personClusterToIgnoredClusters'];
- final bool extraIgnoreCheck = personClusterToIgnoredClusters != null;
-
- final Map<String, List<(String, double)>> suggestions = {};
- const suggestionMax = 2000;
- int suggestionCount = 0;
- int comparisons = 0;
- final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start();
-
- Set<String> otherClusters = clusterAvg.keys.toSet().difference(
- personClusters,
- );
- otherClusters = otherClusters.difference(ignoredClusters);
-
- for (final otherClusterID in otherClusters) {
- final Vector? otherAvg = clusterAvg[otherClusterID];
- if (otherAvg == null) {
- dev.log('[WARNING] no avg for othercluster $otherClusterID');
- continue;
- }
- String? nearestPersonCluster;
- double? minDistance;
- for (final personCluster in personClusters) {
- if (clusterAvg[personCluster] == null) {
- dev.log('[WARNING] no avg for personcluster $personCluster');
- continue;
- }
- if (extraIgnoreCheck &&
- personClusterToIgnoredClusters[personCluster] != null &&
- personClusterToIgnoredClusters[personCluster]!.contains(
- otherClusterID,
- )) {
- continue;
- }
- final Vector avg = clusterAvg[personCluster]!;
- final distance = 1 - avg.dot(otherAvg);
- comparisons++;
- if (distance < maxClusterDistance) {
- if (minDistance == null || distance < minDistance) {
- minDistance = distance;
- nearestPersonCluster = personCluster;
- }
- }
- }
- if (nearestPersonCluster != null && minDistance != null) {
- suggestions.putIfAbsent(nearestPersonCluster, () => []).add((
- otherClusterID,
- minDistance,
- ));
- suggestionCount++;
- }
- if (suggestionCount >= suggestionMax) {
- break;
- }
- }
- w?.log(
- 'calculation inside calcSuggestionsMean for ${personClusters.length} person clusters and ${otherClusters.length} other clusters (so ${personClusters.length * otherClusters.length} combinations, $comparisons comparisons made resulted in $suggestionCount suggestions)',
- );
-
- if (suggestions.isNotEmpty) {
- final List<(String, double, String)> suggestClusterIds = [];
- for (final String personClusterID in suggestions.keys) {
- final suggestionss = suggestions[personClusterID]!;
- suggestClusterIds.addAll(
- suggestionss.map(
- (suggestion) => (suggestion.$1, suggestion.$2, personClusterID),
- ),
- );
- }
- suggestClusterIds.sort((a, b) => a.$2.compareTo(b.$2));
-
- dev.log(
- "Already found ${suggestClusterIds.length} good suggestions using mean",
- );
- return suggestClusterIds.sublist(0, min(suggestClusterIds.length, 20));
- } else {
- dev.log("No suggestions found using mean");
- return <(String, double, String)>[];
- }
-}
-
-Future<(Map<String, Vector>, Set<String>, int, int, int)>
-checkAndSerializeCurrentClusterMeans(Map args) async {
- final Map<String, int> allClusterIdsToCountMap =
- args['allClusterIdsToCountMap'];
- final int minClusterSize = args['minClusterSize'] ?? 1;
- final Set<String> ignoredClusters = args['ignoredClusters'] ?? {};
- final Map<String, (Uint8List, int)> clusterToSummary =
- args['clusterToSummary'];
-
- final Map<String, Vector> clusterAvg = {};
-
- final allClusterIds = allClusterIdsToCountMap.keys.toSet();
- int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0;
- int smallerClustersCnt = 0;
- for (final id in allClusterIdsToCountMap.keys) {
- if (ignoredClusters.contains(id)) {
- allClusterIds.remove(id);
- ignoredClustersCnt++;
- }
- if (clusterToSummary[id]?.$2 == allClusterIdsToCountMap[id]) {
- allClusterIds.remove(id);
- clusterAvg[id] = Vector.fromList(
- EVector.fromBuffer(clusterToSummary[id]!.$1).values,
- dtype: DType.float32,
- );
- alreadyUpdatedClustersCnt++;
- }
- if (allClusterIdsToCountMap[id]! < minClusterSize) {
- allClusterIds.remove(id);
- smallerClustersCnt++;
- }
- }
-
- return (
- clusterAvg,
- allClusterIds,
- ignoredClustersCnt,
- alreadyUpdatedClustersCnt,
- smallerClustersCnt,
- );
-}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_cluster_averages.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_cluster_averages.dart
new file mode 100644
index 0000000000..66a7ad64c7
--- /dev/null
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/dart_cluster_averages.dart
@@ -0,0 +1,327 @@
+import 'dart:developer' as dev show log;
+import "dart:math" show min;
+
+import "package:computer/computer.dart";
+import "package:ente_pure_utils/ente_pure_utils.dart";
+import "package:flutter/foundation.dart";
+import "package:logging/logging.dart";
+import "package:ml_linalg/linalg.dart";
+import "package:photos/db/ml/dart_db.dart";
+import "package:photos/db/ml/usearch_cluster_centroid_vector_db.dart";
+import "package:photos/generated/protos/ente/common/vector.pb.dart";
+import "package:photos/services/machine_learning/face_ml/feedback/cluster_averages.dart";
+
+class DartClusterAverages implements ClusterAverages {
+ static final Logger _logger = Logger("DartClusterAverages");
+ static final _computer = Computer.shared();
+
+ final Map<String, Vector> _clusterAvg;
+
+ DartClusterAverages._(this._clusterAvg);
+
+ @override
+ late final Set<String> clusterIDs = _clusterAvg.keys.toSet();
+
+ static Future<DartClusterAverages> refresh(
+ Map<String, int> allClusterIdsToCountMap,
+ Set<String> ignoredClusters, {
+ required bool shouldPersistClusterSummaryUpdates,
+ int minClusterSize = 1,
+ int maxClusterInCurrentRun = 500,
+ int maxEmbeddingToRead = 10000,
+ }) async {
+ final mlDataDB = DartMLDataDB.instance;
+ final w = (kDebugMode ? EnteWatch('_getUpdateClusterAvg') : null)?..start();
+ final startTime = DateTime.now();
+ _logger.info(
+ 'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters, minClusterSize $minClusterSize, maxClusterInCurrentRun $maxClusterInCurrentRun',
+ );
+ if (!shouldPersistClusterSummaryUpdates) {
+ _logger.info(
+ "Skipping cluster summary cache writes while centroid vector DB migration is in progress",
+ );
+ }
+
+ final Map<String, (Uint8List, int)> clusterToSummary = await mlDataDB
+ .getAllClusterSummary(minClusterSize);
+ final Map<String, (Uint8List, int)> updatesForClusterSummary = {};
+
+ w?.log('getUpdateClusterAvg database call for getAllClusterSummary');
+
+ final serializationEmbeddings =
+ await _computer.compute(
+ checkAndSerializeCurrentClusterMeans,
+ param: {
+ 'allClusterIdsToCountMap': allClusterIdsToCountMap,
+ 'minClusterSize': minClusterSize,
+ 'ignoredClusters': ignoredClusters,
+ 'clusterToSummary': clusterToSummary,
+ },
+ )
+ as (Map<String, Vector>, Set<String>, int, int, int);
+ final clusterAvg = serializationEmbeddings.$1;
+ final allClusterIds = serializationEmbeddings.$2;
+ final ignoredClustersCnt = serializationEmbeddings.$3;
+ final alreadyUpdatedClustersCnt = serializationEmbeddings.$4;
+ final smallerClustersCnt = serializationEmbeddings.$5;
+
+ for (final avg in clusterAvg.values) {
+ assert((avg.norm() - 1.0).abs() < 1e-5);
+ }
+
+ w?.log('serialization of embeddings');
+ _logger.info(
+ 'Ignored $ignoredClustersCnt clusters, already updated $alreadyUpdatedClustersCnt clusters, $smallerClustersCnt clusters are smaller than $minClusterSize',
+ );
+
+ if (allClusterIds.isEmpty) {
+ _logger.info(
+ 'No clusters to update, getUpdateClusterAvg done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
+ );
+ return DartClusterAverages._(clusterAvg);
+ }
+
+ final sortedClusterIDs = allClusterIds.toList();
+ sortedClusterIDs.sort(
+ (a, b) =>
+ allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!),
+ );
+ int indexedInCurrentRun = 0;
+ w?.reset();
+
+ int currentPendingRead = 0;
+ final List<String> clusterIdsToRead = [];
+ for (final clusterID in sortedClusterIDs) {
+ if (maxClusterInCurrentRun-- <= 0) {
+ break;
+ }
+ if (currentPendingRead == 0) {
+ currentPendingRead = allClusterIdsToCountMap[clusterID] ?? 0;
+ clusterIdsToRead.add(clusterID);
+ } else {
+ if ((currentPendingRead + allClusterIdsToCountMap[clusterID]!) <
+ maxEmbeddingToRead) {
+ clusterIdsToRead.add(clusterID);
+ currentPendingRead += allClusterIdsToCountMap[clusterID]!;
+ } else {
+ break;
+ }
+ }
+ }
+
+ final Map<String, Iterable<Uint8List>> clusterEmbeddings = await mlDataDB
+ .getFaceEmbeddingsForClusters(clusterIdsToRead);
+
+ w?.logAndReset(
+ 'read $currentPendingRead embeddings for ${clusterEmbeddings.length} clusters',
+ );
+
+ for (final clusterID in clusterEmbeddings.keys) {
+ final Iterable<Uint8List> embeddings = clusterEmbeddings[clusterID]!;
+ final Iterable<Vector> vectors = embeddings.map(
+ (e) =>
+ Vector.fromList(EVector.fromBuffer(e).values, dtype: DType.float32),
+ );
+ final avg = vectors.reduce((a, b) => a + b) / vectors.length;
+ final avgNormalized = avg / avg.norm();
+ if (shouldPersistClusterSummaryUpdates) {
+ final avgEmbeddingBuffer = EVector(
+ values: avgNormalized,
+ ).writeToBuffer();
+ updatesForClusterSummary[clusterID] = (
+ avgEmbeddingBuffer,
+ embeddings.length,
+ );
+ }
+ indexedInCurrentRun++;
+ if (shouldPersistClusterSummaryUpdates &&
+ updatesForClusterSummary.length > 100) {
+ await mlDataDB.clusterSummaryUpdate(updatesForClusterSummary);
+ updatesForClusterSummary.clear();
+ if (kDebugMode) {
+ _logger.info(
+ 'getUpdateClusterAvg $indexedInCurrentRun clusters in current one',
+ );
+ }
+ }
+ clusterAvg[clusterID] = avgNormalized;
+ }
+ if (shouldPersistClusterSummaryUpdates &&
+ updatesForClusterSummary.isNotEmpty) {
+ await mlDataDB.clusterSummaryUpdate(updatesForClusterSummary);
+ }
+ w?.logAndReset('done computing avg ');
+ _logger.info(
+ 'end getUpdateClusterAvg for ${clusterAvg.length} clusters, done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
+ );
+
+ return DartClusterAverages._(clusterAvg);
+ }
+
+ @override
+ Future<List<List<(String, double)>>> searchNearest(
+ List<String> queryClusterIDs,
+ Set<String> allowedClusterIDs, {
+ required double maxDistance,
+ required int count,
+ }) {
+ return UsearchClusterCentroidVectorDB.instance.bulkSearchNearestAllowed(
+ [
+ for (final clusterID in queryClusterIDs)
+ _clusterAvg[clusterID]!.toList(growable: false),
+ ],
+ allowedClusterIDs,
+ maxDistance: maxDistance,
+ count: count,
+ );
+ }
+
+ @override
+ Future<List<(String, double, String)>> suggestByMean(
+ Set<String> personClusters,
+ Set<String> ignoredClusters,
+ double maxClusterDistance, {
+ Map<String, Set<String>>? personClusterToIgnoredClusters,
+ }) async {
+ return await _computer.compute(
+ _calcSuggestionsMean,
+ param: {
+ 'clusterAvg': _clusterAvg,
+ 'personClusters': personClusters,
+ 'ignoredClusters': ignoredClusters,
+ 'maxClusterDistance': maxClusterDistance,
+ 'personClusterToIgnoredClusters': personClusterToIgnoredClusters,
+ },
+ );
+ }
+}
+
+// Returns (suggestion cluster ID, distance, person cluster ID).
+List<(String, double, String)> _calcSuggestionsMean(Map<String, dynamic> args) {
+ final Map<String, Vector> clusterAvg = args['clusterAvg'];
+ final Set<String> personClusters = args['personClusters'];
+ final Set<String> ignoredClusters = args['ignoredClusters'];
+ final double maxClusterDistance = args['maxClusterDistance'];
+ final Map<String, Set<String>>? personClusterToIgnoredClusters =
+ args['personClusterToIgnoredClusters'];
+ final bool extraIgnoreCheck = personClusterToIgnoredClusters != null;
+
+ final Map<String, List<(String, double)>> suggestions = {};
+ const suggestionMax = 2000;
+ int suggestionCount = 0;
+ int comparisons = 0;
+ final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start();
+
+ Set<String> otherClusters = clusterAvg.keys.toSet().difference(
+ personClusters,
+ );
+ otherClusters = otherClusters.difference(ignoredClusters);
+
+ for (final otherClusterID in otherClusters) {
+ final Vector? otherAvg = clusterAvg[otherClusterID];
+ if (otherAvg == null) {
+ dev.log('[WARNING] no avg for othercluster $otherClusterID');
+ continue;
+ }
+ String? nearestPersonCluster;
+ double? minDistance;
+ for (final personCluster in personClusters) {
+ if (clusterAvg[personCluster] == null) {
+ dev.log('[WARNING] no avg for personcluster $personCluster');
+ continue;
+ }
+ if (extraIgnoreCheck &&
+ personClusterToIgnoredClusters[personCluster] != null &&
+ personClusterToIgnoredClusters[personCluster]!.contains(
+ otherClusterID,
+ )) {
+ continue;
+ }
+ final Vector avg = clusterAvg[personCluster]!;
+ final distance = 1 - avg.dot(otherAvg);
+ comparisons++;
+ if (distance < maxClusterDistance) {
+ if (minDistance == null || distance < minDistance) {
+ minDistance = distance;
+ nearestPersonCluster = personCluster;
+ }
+ }
+ }
+ if (nearestPersonCluster != null && minDistance != null) {
+ suggestions.putIfAbsent(nearestPersonCluster, () => []).add((
+ otherClusterID,
+ minDistance,
+ ));
+ suggestionCount++;
+ }
+ if (suggestionCount >= suggestionMax) {
+ break;
+ }
+ }
+ w?.log(
+ 'calculation inside calcSuggestionsMean for ${personClusters.length} person clusters and ${otherClusters.length} other clusters (so ${personClusters.length * otherClusters.length} combinations, $comparisons comparisons made resulted in $suggestionCount suggestions)',
+ );
+
+ if (suggestions.isNotEmpty) {
+ final List<(String, double, String)> suggestClusterIds = [];
+ for (final String personClusterID in suggestions.keys) {
+ final suggestionss = suggestions[personClusterID]!;
+ suggestClusterIds.addAll(
+ suggestionss.map(
+ (suggestion) => (suggestion.$1, suggestion.$2, personClusterID),
+ ),
+ );
+ }
+ suggestClusterIds.sort((a, b) => a.$2.compareTo(b.$2));
+
+ dev.log(
+ "Already found ${suggestClusterIds.length} good suggestions using mean",
+ );
+ return suggestClusterIds.sublist(0, min(suggestClusterIds.length, 20));
+ } else {
+ dev.log("No suggestions found using mean");
+ return <(String, double, String)>[];
+ }
+}
+
+Future<(Map<String, Vector>, Set<String>, int, int, int)>
+checkAndSerializeCurrentClusterMeans(Map args) async {
+ final Map<String, int> allClusterIdsToCountMap =
+ args['allClusterIdsToCountMap'];
+ final int minClusterSize = args['minClusterSize'] ?? 1;
+ final Set<String> ignoredClusters = args['ignoredClusters'] ?? {};
+ final Map<String, (Uint8List, int)> clusterToSummary =
+ args['clusterToSummary'];
+
+ final Map<String, Vector> clusterAvg = {};
+
+ final allClusterIds = allClusterIdsToCountMap.keys.toSet();
+ int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0;
+ int smallerClustersCnt = 0;
+ for (final id in allClusterIdsToCountMap.keys) {
+ if (ignoredClusters.contains(id)) {
+ allClusterIds.remove(id);
+ ignoredClustersCnt++;
+ }
+ if (clusterToSummary[id]?.$2 == allClusterIdsToCountMap[id]) {
+ allClusterIds.remove(id);
+ clusterAvg[id] = Vector.fromList(
+ EVector.fromBuffer(clusterToSummary[id]!.$1).values,
+ dtype: DType.float32,
+ );
+ alreadyUpdatedClustersCnt++;
+ }
+ if (allClusterIdsToCountMap[id]! < minClusterSize) {
+ allClusterIds.remove(id);
+ smallerClustersCnt++;
+ }
+ }
+
+ return (
+ clusterAvg,
+ allClusterIds,
+ ignoredClustersCnt,
+ alreadyUpdatedClustersCnt,
+ smallerClustersCnt,
+ );
+}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_cluster_averages.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_cluster_averages.dart
new file mode 100644
index 0000000000..527d2218bd
--- /dev/null
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/feedback/store_cluster_averages.dart
@@ -0,0 +1,74 @@
+import "package:logging/logging.dart";
+import "package:photos/db/ml/rust_db.dart";
+import "package:photos/db/ml/store_cluster_centroid_vector_db.dart";
+import "package:photos/services/machine_learning/face_ml/feedback/cluster_averages.dart";
+
+class StoreClusterAverages implements ClusterAverages {
+ static final Logger _logger = Logger("StoreClusterAverages");
+
+ final RustMLDataDB _mlDataDB;
+
+ @override
+ final Set<String> clusterIDs;
+
+ StoreClusterAverages._(this._mlDataDB, this.clusterIDs);
+
+ static Future<StoreClusterAverages> refresh(
+ Map<String, int> allClusterIdsToCountMap,
+ Set<String> ignoredClusters, {
+ required int minClusterSize,
+ }) async {
+ final mlDataDB = RustMLDataDB.instance;
+ final stopwatch = Stopwatch()..start();
+ final clusterIDs = await (await mlDataDB.store).refreshClusterAverages(
+ clusterFaceCounts: allClusterIdsToCountMap,
+ ignoredClusterIds: ignoredClusters.toList(),
+ minClusterSize: minClusterSize,
+ );
+ _logger.info(
+ 'Refreshed cluster averages in ${stopwatch.elapsedMilliseconds} ms, ${clusterIDs.length} clusters have an average (minClusterSize $minClusterSize)',
+ );
+ return StoreClusterAverages._(mlDataDB, clusterIDs);
+ }
+
+ @override
+ Future<List<List<(String, double)>>> searchNearest(
+ List<String> queryClusterIDs,
+ Set<String> allowedClusterIDs, {
+ required double maxDistance,
+ required int count,
+ }) {
+ return StoreClusterCentroidVectorDB.instance
+ .bulkSearchNearestAllowedForClusters(
+ queryClusterIDs,
+ allowedClusterIDs,
+ maxDistance: maxDistance,
+ count: count,
+ );
+ }
+
+ @override
+ Future<List<(String, double, String)>> suggestByMean(
+ Set<String> personClusters,
+ Set<String> ignoredClusters,
+ double maxClusterDistance, {
+ Map<String, Set<String>>? personClusterToIgnoredClusters,
+ }) async {
+ final suggestions = await (await _mlDataDB.store).suggestClustersByMean(
+ personClusterIds: personClusters.where(clusterIDs.contains).toList(),
+ candidateClusterIds: clusterIDs
+ .difference(personClusters)
+ .difference(ignoredClusters)
+ .toList(),
+ maxDistance: maxClusterDistance,
+ ignoredByPersonCluster: {
+ for (final entry in (personClusterToIgnoredClusters ?? {}).entries)
+ entry.key: entry.value.toList(),
+ },
+ );
+ return [
+ for (final suggestion in suggestions)
+ (suggestion.clusterId, suggestion.distance, suggestion.personClusterId),
+ ];
+ }
+}
diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rs
index f4e2715419..c9f2374c66 100644
--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs
@@ -7,7 +7,7 @@ pub use ente_photos::ml_db::{
FaceWithoutEmbedding, FdStatus, PreviewInfo,
};
use ente_photos::ml_store;
-pub use ente_photos::ml_store::{FillOutcome, FillState, Index};
+pub use ente_photos::ml_store::{FillOutcome, FillState, Index, MeanSuggestion};
use flutter_rust_bridge::frb;
use super::vecdb_api::{
@@ -104,6 +104,13 @@ pub struct _ClusterSummary {
pub count: i64,
}
+#[frb(mirror(MeanSuggestion))]
+pub struct _MeanSuggestion {
+ pub cluster_id: String,
+ pub distance: f64,
+ pub person_cluster_id: String,
+}
+
#[frb(mirror(FdStatus))]
pub struct _FdStatus {
pub file_id: i64,
@@ -314,6 +321,61 @@ impl MlStore {
.recluster_faces(&face_ids, &file_creation_times, distance_threshold)?)
}
+ pub fn refresh_cluster_averages(
+ &self,
+ cluster_face_counts: HashMap<String, i64>,
+ ignored_cluster_ids: Vec<String>,
+ min_cluster_size: i64,
+ ) -> Result<HashSet<String>, MlDbError> {
+ let ignored_cluster_ids = ignored_cluster_ids.into_iter().collect();
+ Ok(self.inner.refresh_cluster_averages(
+ &cluster_face_counts,
+ &ignored_cluster_ids,
+ min_cluster_size,
+ )?)
+ }
+
+ pub fn search_cluster_centroids(
+ &self,
+ cluster_ids: Vec<String>,
+ allowed_cluster_ids: Vec<String>,
+ limit: u32,
+ max_distance: f32,
+ ) -> Result<Vec<Vec<VecDbMatch>>, MlDbError> {
+ Ok(self
+ .inner
+ .search_cluster_centroids(
+ &cluster_ids,
+ allowed_cluster_ids,
+ limit as usize,
+ max_distance,
+ )?
+ .into_iter()
+ .map(to_api_matches)
+ .collect())
+ }
+
+ pub fn suggest_clusters_by_mean(
+ &self,
+ person_cluster_ids: Vec<String>,
+ candidate_cluster_ids: Vec<String>,
+ max_distance: f64,
+ ignored_by_person_cluster: HashMap<String, Vec<String>>,
+ ) -> Result<Vec<MeanSuggestion>, MlDbError> {
+ let ignored_by_person_cluster = ignored_by_person_cluster
+ .into_iter()
+ .map(|(person_cluster_id, cluster_ids)| {
+ (person_cluster_id, cluster_ids.into_iter().collect())
+ })
+ .collect();
+ Ok(self.inner.suggest_clusters_by_mean(
+ &person_cluster_ids,
+ &candidate_cluster_ids,
+ max_distance,
+ &ignored_by_person_cluster,
+ )?)
+ }
+
pub fn clear_all(&self) -> Result<(), MlDbError> {
Ok(self.inner.clear_all()?)
}
diff --git a/rust/crates/photos/src/face_clustering/mod.rs b/rust/crates/photos/src/face_clustering/mod.rs
index 53b5108d7d..33c4ab6657 100644
--- a/rust/crates/photos/src/face_clustering/mod.rs
+++ b/rust/crates/photos/src/face_clustering/mod.rs
@@ -104,7 +104,16 @@ pub(crate) fn new_cluster_id() -> String {
format!("{CLUSTER_ID_PREFIX}{suffix}")
}
-fn distance(a: &[f32], b: &[f32]) -> f64 {
+pub(crate) fn normalized_mean(embeddings: &[Vec<f32>]) -> Option<Vec<f32>> {
+ let (first, rest) = embeddings.split_first()?;
+ let mut sum = first.clone();
+ for embedding in rest {
+ add_scaled(&mut sum, embedding, 1.0);
+ }
+ Some(normalized(scaled(sum, 1.0 / embeddings.len() as f64)))
+}
+
+pub(crate) fn distance(a: &[f32], b: &[f32]) -> f64 {
1.0 - dot(a, b)
}
@@ -180,7 +189,7 @@ mod tests {
use super::{
CLUSTER_ID_LENGTH, CLUSTER_ID_PREFIX, Clustering, Face, IncrementalFace, dot,
- new_cluster_id, newest_first, recluster,
+ new_cluster_id, newest_first, normalized_mean, recluster,
};
pub(super) fn direction(degrees: f32) -> Vec<f32> {
@@ -299,6 +308,17 @@ mod tests {
assert_eq!(dot(&a, &[1.0; 8]), 1.0 + f64::from(tiny));
}
+ #[test]
+ fn normalized_mean_averages_then_scales_to_unit_length() {
+ let mean = normalized_mean(&[vec![3.0, 0.0], vec![0.0, 4.0], vec![3.0, 4.0]]).unwrap();
+ assert_close(&mean, &[0.6, 0.8]);
+ }
+
+ #[test]
+ fn normalized_mean_of_no_embeddings_is_none() {
+ assert_eq!(normalized_mean(&[]), None);
+ }
+
#[test]
fn unknown_creation_times_sort_first_then_newest() {
let mut times = vec![Some(1), None, Some(3), Some(2)];
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rs
index 6c4c0bc929..cadc28b051 100644
--- a/rust/crates/photos/src/ml_db/queries/clusters.rs
+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs
@@ -313,11 +313,11 @@ impl MlDb {
&self,
cluster_ids: &[String],
) -> Result<HashMap<String, ClusterSummary>> {
- let sql = format!(
+ self.read_chunked_in(
"SELECT * FROM cluster_summary WHERE cluster_id IN ({})",
- bind_placeholders(cluster_ids.len())
- );
- self.read_all(&sql, params_from_iter(cluster_ids), read_cluster_summary)
+ cluster_ids,
+ read_cluster_summary,
+ )
}
pub fn count_cluster_summaries(&self) -> Result<i64> {
diff --git a/rust/crates/photos/src/ml_store/cluster_averages.rs b/rust/crates/photos/src/ml_store/cluster_averages.rs
new file mode 100644
index 0000000000..21ae1fc425
--- /dev/null
+++ b/rust/crates/photos/src/ml_store/cluster_averages.rs
@@ -0,0 +1,493 @@
+use std::cmp::Reverse;
+use std::collections::{HashMap, HashSet};
+
+use super::clustering::{clusterable_embedding, is_clusterable, summaries};
+use super::fill::decode_centroid;
+use super::{Error, Index, Match, MlStore, Result, SearchParams};
+use crate::face_clustering::{ClusterMean, distance, normalized_mean};
+
+const MAX_CLUSTERS_PER_REFRESH: usize = 500;
+const MAX_EMBEDDINGS_PER_REFRESH: i64 = 10_000;
+const MAX_MEAN_SUGGESTIONS: usize = 20;
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct MeanSuggestion {
+ pub cluster_id: String,
+ pub distance: f64,
+ pub person_cluster_id: String,
+}
+
+impl MlStore {
+ pub fn refresh_cluster_averages(
+ &self,
+ cluster_face_counts: &HashMap<String, i64>,
+ ignored_cluster_ids: &HashSet<String>,
+ min_cluster_size: i64,
+ ) -> Result<HashSet<String>> {
+ let summaries_by_id = self.db.get_all_cluster_summary(Some(min_cluster_size))?;
+ let mut averaged = HashSet::new();
+ let mut stale = Vec::new();
+ for (cluster_id, &face_count) in cluster_face_counts {
+ let current = summaries_by_id.get(cluster_id).is_some_and(|summary| {
+ summary.count == face_count && decode_centroid(cluster_id, &summary.avg).is_some()
+ });
+ if current {
+ averaged.insert(cluster_id.clone());
+ } else if face_count >= min_cluster_size && !ignored_cluster_ids.contains(cluster_id) {
+ stale.push((cluster_id.clone(), face_count));
+ }
+ }
+ let means = self.average_clusters(&largest_within_budget(stale))?;
+ match self.cluster_summary_update(&summaries(&means)) {
+ Ok(()) => {}
+ Err(Error::Index(error)) => {
+ log::warn!(
+ "cluster centroid index write failed while refreshing cluster averages: {error}"
+ );
+ }
+ Err(error) => return Err(error),
+ }
+ averaged.extend(means.into_keys());
+ Ok(averaged)
+ }
+
+ pub fn search_cluster_centroids(
+ &self,
+ cluster_ids: &[String],
+ allowed_cluster_ids: Vec<String>,
+ limit: usize,
+ max_distance: f32,
+ ) -> Result<Vec<Vec<Match>>> {
+ let centroids = self.cluster_centroids(cluster_ids)?;
+ let queries: Vec<Vec<f32>> = cluster_ids
+ .iter()
+ .filter_map(|cluster_id| centroids.get(cluster_id).cloned())
+ .collect();
+ let params = SearchParams {
+ limit: Some(limit),
+ max_distance: Some(max_distance),
+ exact: false,
+ allowed_keys: Some(allowed_cluster_ids),
+ };
+ let mut matches = self
+ .bulk_search(Index::ClusterCentroid, &queries, &params)?
+ .into_iter();
+ Ok(cluster_ids
+ .iter()
+ .map(|cluster_id| {
+ if centroids.contains_key(cluster_id) {
+ matches.next().unwrap_or_default()
+ } else {
+ Vec::new()
+ }
+ })
+ .collect())
+ }
+
+ pub fn suggest_clusters_by_mean(
+ &self,
+ person_cluster_ids: &[String],
+ candidate_cluster_ids: &[String],
+ max_distance: f64,
+ ignored_by_person_cluster: &HashMap<String, HashSet<String>>,
+ ) -> Result<Vec<MeanSuggestion>> {
+ let cluster_ids: Vec<String> = person_cluster_ids
+ .iter()
+ .chain(candidate_cluster_ids)
+ .cloned()
+ .collect();
+ let centroids = self.cluster_centroids(&cluster_ids)?;
+ let person_centroids: Vec<(&String, &Vec<f32>)> = person_cluster_ids
+ .iter()
+ .filter_map(|cluster_id| Some((cluster_id, centroids.get(cluster_id)?)))
+ .collect();
+ let mut suggestions: Vec<MeanSuggestion> = candidate_cluster_ids
+ .iter()
+ .filter_map(|candidate_id| {
+ let candidate = centroids.get(candidate_id)?;
+ let mut nearest: Option<(&String, f64)> = None;
+ for &(person_cluster_id, person_centroid) in &person_centroids {
+ if ignored_by_person_cluster
+ .get(person_cluster_id)
+ .is_some_and(|ignored| ignored.contains(candidate_id))
+ {
+ continue;
+ }
+ let distance = distance(person_centroid, candidate);
+ if distance < max_distance && nearest.is_none_or(|(_, best)| distance < best) {
+ nearest = Some((person_cluster_id, distance));
+ }
+ }
+ nearest.map(|(person_cluster_id, distance)| MeanSuggestion {
+ cluster_id: candidate_id.clone(),
+ distance,
+ person_cluster_id: person_cluster_id.clone(),
+ })
+ })
+ .collect();
+ suggestions.sort_by(|a, b| a.distance.total_cmp(&b.distance));
+ suggestions.truncate(MAX_MEAN_SUGGESTIONS);
+ Ok(suggestions)
+ }
+
+ fn average_clusters(&self, cluster_ids: &[String]) -> Result<HashMap<String, ClusterMean>> {
+ Ok(self
+ .db
+ .get_face_embeddings_for_clusters(cluster_ids, None)?
+ .into_iter()
+ .filter_map(|(cluster_id, embeddings)| {
+ let valid: Vec<Vec<f32>> = embeddings
+ .iter()
+ .filter_map(|embedding| clusterable_embedding(embedding))
+ .collect();
+ let mean = normalized_mean(&valid).filter(|mean| is_clusterable(mean))?;
+ let count = embeddings.len();
+ Some((cluster_id, ClusterMean { mean, count }))
+ })
+ .collect())
+ }
+
+ fn cluster_centroids(&self, cluster_ids: &[String]) -> Result<HashMap<String, Vec<f32>>> {
+ Ok(self
+ .db
+ .get_cluster_to_cluster_summary(cluster_ids)?
+ .into_iter()
+ .filter_map(|(cluster_id, summary)| {
+ let centroid = decode_centroid(&cluster_id, &summary.avg)?;
+ Some((cluster_id, centroid))
+ })
+ .collect())
+ }
+}
+
+fn largest_within_budget(mut stale: Vec<(String, i64)>) -> Vec<String> {
+ stale.sort_by_key(|&(_, face_count)| Reverse(face_count));
+ let mut selected = Vec::new();
+ let mut embeddings = 0;
+ for (cluster_id, face_count) in stale.into_iter().take(MAX_CLUSTERS_PER_REFRESH) {
+ if !selected.is_empty() && embeddings + face_count >= MAX_EMBEDDINGS_PER_REFRESH {
+ break;
+ }
+ embeddings += face_count;
+ selected.push(cluster_id);
+ }
+ selected
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::{HashMap, HashSet};
+ use std::fs;
+
+ use super::{MeanSuggestion, largest_within_budget};
+ use crate::ml_db::ClusterSummary;
+ use crate::ml_db::vector_encoding::encode_evector;
+ use crate::ml_store::tests::{
+ assert_direction, assign, embedding, face, index_path, meta, open, seed, summary_of,
+ };
+ use crate::ml_store::{CLUSTER_CENTROID_DIMENSIONS, Index, MlStore};
+
+ fn summary(degrees: f64, count: i64) -> ClusterSummary {
+ ClusterSummary {
+ avg: encode_evector(&embedding(degrees)),
+ count,
+ }
+ }
+
+ fn put_summaries(store: &MlStore, summaries: &[(&str, f64)]) {
+ let summaries: HashMap<String, ClusterSummary> = summaries
+ .iter()
+ .map(|&(cluster_id, degrees)| (cluster_id.to_string(), summary(degrees, 1)))
+ .collect();
+ store.cluster_summary_update(&summaries).unwrap();
+ }
+
+ fn ids(cluster_ids: &[&str]) -> Vec<String> {
+ cluster_ids.iter().map(ToString::to_string).collect()
+ }
+
+ fn id_set(cluster_ids: &[&str]) -> HashSet<String> {
+ cluster_ids.iter().map(ToString::to_string).collect()
+ }
+
+ fn refresh(store: &MlStore, ignored: &[&str], min_cluster_size: i64) -> HashSet<String> {
+ let face_counts = store.db().cluster_id_to_face_count().unwrap();
+ store
+ .refresh_cluster_averages(&face_counts, &id_set(ignored), min_cluster_size)
+ .unwrap()
+ }
+
+ #[test]
+ fn refreshing_recomputes_stale_averages_and_keeps_current_ones() {
+ let (_directory, store) = open();
+ store.fill_cluster_centroid_index(false).unwrap();
+ seed(&store, &[(1, 0.0), (2, 90.0), (3, 0.0)]);
+ assign(&store, &[(1, "stale"), (2, "stale"), (3, "current")]);
+ store
+ .cluster_summary_update(&HashMap::from([
+ ("stale".to_string(), summary(0.0, 1)),
+ ("current".to_string(), summary(45.0, 1)),
+ ]))
+ .unwrap();
+
+ assert_eq!(refresh(&store, &[], 1), id_set(&["stale", "current"]));
+
+ let (mean, count) = summary_of(&store, "stale");
+ assert_eq!(count, 2);
+ assert_direction(&mean, 45.0);
+ let (mean, count) = summary_of(&store, "current");
+ assert_eq!(count, 1);
+ assert_direction(&mean, 45.0);
+ let indexed = store
+ .get_vector(Index::ClusterCentroid, "stale")
+ .unwrap()
+ .unwrap();
+ assert!(indexed[0] > 0.7 && indexed[1] > 0.7);
+ }
+
+ #[test]
+ fn refreshing_skips_ignored_and_small_stale_clusters_but_returns_their_current_averages() {
+ let (_directory, store) = open();
+ seed(
+ &store,
+ &[
+ (1, 0.0),
+ (2, 0.0),
+ (3, 0.0),
+ (4, 0.0),
+ (5, 0.0),
+ (6, 0.0),
+ (7, 0.0),
+ ],
+ );
+ assign(
+ &store,
+ &[
+ (1, "ignored_current"),
+ (2, "ignored_current"),
+ (3, "ignored_stale"),
+ (4, "ignored_stale"),
+ (5, "small"),
+ (6, "large"),
+ (7, "large"),
+ ],
+ );
+ store
+ .cluster_summary_update(&HashMap::from([(
+ "ignored_current".to_string(),
+ summary(0.0, 2),
+ )]))
+ .unwrap();
+
+ assert_eq!(
+ refresh(&store, &["ignored_current", "ignored_stale"], 2),
+ id_set(&["ignored_current", "large"])
+ );
+ let refreshed = store
+ .db()
+ .get_cluster_to_cluster_summary(&ids(&["ignored_stale", "small", "large"]))
+ .unwrap();
+ assert_eq!(refreshed.keys().collect::<Vec<_>>(), ["large"]);
+ }
+
+ #[test]
+ fn refreshing_averages_only_valid_embeddings() {
+ let (_directory, store) = open();
+ store.fill_cluster_centroid_index(false).unwrap();
+ seed(&store, &[(1, 0.0), (2, 90.0), (5, 30.0)]);
+ let mut invalid = face(3, 0.0);
+ invalid.embedding[5] = f64::NAN;
+ let mut short = face(4, 0.0);
+ short.embedding.truncate(3);
+ store.db().bulk_insert_faces(&[invalid, short]).unwrap();
+ assign(
+ &store,
+ &[
+ (1, "mixed"),
+ (2, "mixed"),
+ (3, "mixed"),
+ (4, "invalid"),
+ (5, "corrupt"),
+ ],
+ );
+ store
+ .cluster_summary_update(&HashMap::from([(
+ "corrupt".to_string(),
+ ClusterSummary {
+ avg: encode_evector(&[f64::NAN; CLUSTER_CENTROID_DIMENSIONS]),
+ count: 1,
+ },
+ )]))
+ .unwrap();
+
+ assert_eq!(refresh(&store, &[], 1), id_set(&["mixed", "corrupt"]));
+
+ let (mean, count) = summary_of(&store, "mixed");
+ assert_eq!(count, 3);
+ assert_direction(&mean, 45.0);
+ let (mean, _) = summary_of(&store, "corrupt");
+ assert_direction(&mean, 30.0);
+ assert!(
+ store
+ .db()
+ .get_cluster_to_cluster_summary(&ids(&["invalid"]))
+ .unwrap()
+ .is_empty()
+ );
+ assert!(!store.contains(Index::ClusterCentroid, "invalid").unwrap());
+ assert_eq!(refresh(&store, &[], 1), id_set(&["mixed", "corrupt"]));
+ }
+
+ #[test]
+ fn refreshing_considers_only_the_callers_clusters() {
+ let (_directory, store) = open();
+ seed(&store, &[(1, 0.0), (2, 90.0), (3, 0.0)]);
+ assign(&store, &[(1, "known")]);
+ let face_counts = store.db().cluster_id_to_face_count().unwrap();
+ assign(&store, &[(2, "new_stale"), (3, "new_current")]);
+ store
+ .cluster_summary_update(&HashMap::from([(
+ "new_current".to_string(),
+ summary(0.0, 1),
+ )]))
+ .unwrap();
+
+ let averaged = store
+ .refresh_cluster_averages(&face_counts, &HashSet::new(), 1)
+ .unwrap();
+
+ assert_eq!(averaged, id_set(&["known"]));
+ assert!(
+ store
+ .db()
+ .get_cluster_to_cluster_summary(&ids(&["new_stale"]))
+ .unwrap()
+ .is_empty()
+ );
+ }
+
+ #[test]
+ fn largest_stale_clusters_are_refreshed_first_within_the_embedding_budget() {
+ let stale = |clusters: &[(&str, i64)]| -> Vec<(String, i64)> {
+ clusters
+ .iter()
+ .map(|&(cluster_id, count)| (cluster_id.to_string(), count))
+ .collect()
+ };
+ assert_eq!(
+ largest_within_budget(stale(&[("a", 20), ("b", 9_000), ("c", 1), ("d", 990)])),
+ ids(&["b", "d"])
+ );
+ assert_eq!(
+ largest_within_budget(stale(&[("a", 5), ("b", 12_000), ("c", 1)])),
+ ids(&["b"])
+ );
+ let singletons: Vec<(String, i64)> = (0..600).map(|i| (format!("c{i}"), 1)).collect();
+ assert_eq!(largest_within_budget(singletons).len(), 500);
+ }
+
+ #[test]
+ fn refreshing_keeps_the_averages_when_the_centroid_write_fails() {
+ let (directory, store) = open();
+ fs::create_dir(index_path(&directory, Index::ClusterCentroid)).unwrap();
+ store
+ .db()
+ .set_meta("cluster_centroid.fill", "filled")
+ .unwrap();
+ seed(&store, &[(1, 0.0), (2, 90.0)]);
+ assign(&store, &[(1, "c"), (2, "c")]);
+
+ assert_eq!(refresh(&store, &[], 1), id_set(&["c"]));
+
+ let (mean, count) = summary_of(&store, "c");
+ assert_eq!(count, 2);
+ assert_direction(&mean, 45.0);
+ assert_eq!(meta(&store, "cluster_centroid.fill"), None);
+ }
+
+ #[test]
+ fn centroid_search_queries_with_the_stored_averages_in_query_order() {
+ let (_directory, store) = open();
+ store.fill_cluster_centroid_index(false).unwrap();
+ put_summaries(
+ &store,
+ &[
+ ("p1", 0.0),
+ ("p2", 90.0),
+ ("c1", 10.0),
+ ("c2", 80.0),
+ ("c3", 5.0),
+ ],
+ );
+ let results = store
+ .search_cluster_centroids(
+ &ids(&["p2", "missing", "p1"]),
+ ids(&["c1", "c2", "c3"]),
+ 2,
+ 0.5,
+ )
+ .unwrap();
+
+ let keys: Vec<Vec<&str>> = results
+ .iter()
+ .map(|matches| matches.iter().map(|m| m.key.as_str()).collect())
+ .collect();
+ assert_eq!(keys, [vec!["c2"], vec![], vec!["c3", "c1"]]);
+ }
+
+ #[test]
+ fn mean_suggestions_pick_the_nearest_allowed_person_cluster_within_the_threshold() {
+ let (_directory, store) = open();
+ put_summaries(
+ &store,
+ &[
+ ("p1", 0.0),
+ ("p2", 90.0),
+ ("c1", 10.0),
+ ("c2", 70.0),
+ ("c3", 180.0),
+ ("c4", 40.0),
+ ],
+ );
+
+ let suggestions = store
+ .suggest_clusters_by_mean(
+ &ids(&["p1", "p2", "missing"]),
+ &ids(&["c4", "c3", "missing", "c2", "c1"]),
+ 0.5,
+ &HashMap::from([("p1".to_string(), id_set(&["c4"]))]),
+ )
+ .unwrap();
+
+ let found: Vec<(&str, &str)> = suggestions
+ .iter()
+ .map(|s| (s.cluster_id.as_str(), s.person_cluster_id.as_str()))
+ .collect();
+ assert_eq!(found, [("c1", "p1"), ("c2", "p2"), ("c4", "p2")]);
+ let MeanSuggestion { distance, .. } = &suggestions[2];
+ assert!((distance - (1.0 - 50.0_f64.to_radians().cos())).abs() < 1e-6);
+ }
+
+ #[test]
+ fn mean_suggestions_keep_the_nearest_twenty() {
+ let (_directory, store) = open();
+ let candidates: Vec<(String, f64)> = (1..=25)
+ .map(|degrees| (format!("c{degrees}"), f64::from(degrees)))
+ .collect();
+ let mut summaries: Vec<(&str, f64)> = candidates
+ .iter()
+ .map(|(cluster_id, degrees)| (cluster_id.as_str(), *degrees))
+ .collect();
+ summaries.push(("p", 0.0));
+ put_summaries(&store, &summaries);
+ let candidate_ids: Vec<String> = candidates.into_iter().rev().map(|(id, _)| id).collect();
+
+ let suggestions = store
+ .suggest_clusters_by_mean(&ids(&["p"]), &candidate_ids, 0.5, &HashMap::new())
+ .unwrap();
+
+ let found: Vec<&str> = suggestions.iter().map(|s| s.cluster_id.as_str()).collect();
+ let expected: Vec<String> = (1..=20).map(|degrees| format!("c{degrees}")).collect();
+ assert_eq!(found, expected);
+ }
+}
diff --git a/rust/crates/photos/src/ml_store/clustering.rs b/rust/crates/photos/src/ml_store/clustering.rs
index ec9def42d4..923cd1c646 100644
--- a/rust/crates/photos/src/ml_store/clustering.rs
+++ b/rust/crates/photos/src/ml_store/clustering.rs
@@ -171,11 +171,7 @@ impl MlStore {
let mut faces = Vec::new();
let mut unclusterable_face_ids = Vec::new();
for (face_id, embedding) in self.db.get_face_embedding_rows_for_faces(face_ids)? {
- let embedding = decode_evector(&embedding)
- .ok()
- .map(|values| values.into_iter().map(|value| value as f32).collect())
- .filter(|embedding: &Vec<f32>| is_clusterable(embedding));
- match embedding {
+ match clusterable_embedding(&embedding) {
Some(embedding) => faces.push(Face {
creation_time: try_file_id_from_face_id(&face_id)
.and_then(|file_id| file_creation_times.get(&file_id).copied()),
@@ -225,12 +221,19 @@ fn check_distance_threshold(distance_threshold: f64) -> Result<()> {
}
}
-fn is_clusterable(embedding: &[f32]) -> bool {
+pub(super) fn is_clusterable(embedding: &[f32]) -> bool {
embedding.len() == CLUSTER_CENTROID_DIMENSIONS
&& embedding.iter().all(|value| value.is_finite())
}
-fn summaries(means: &HashMap<String, ClusterMean>) -> HashMap<String, ClusterSummary> {
+pub(super) fn clusterable_embedding(embedding: &[u8]) -> Option<Vec<f32>> {
+ decode_evector(embedding)
+ .ok()
+ .map(|values| values.into_iter().map(|value| value as f32).collect())
+ .filter(|embedding: &Vec<f32>| is_clusterable(embedding))
+}
+
+pub(super) fn summaries(means: &HashMap<String, ClusterMean>) -> HashMap<String, ClusterSummary> {
means
.iter()
.map(|(cluster_id, mean)| {
@@ -250,55 +253,15 @@ mod tests {
use std::fs;
use super::{ClusteredBucket, FaceClusteringPlan, FaceClusteringRequest, FaceClusteringRun};
- use crate::ml_db::vector_encoding::{decode_evector, encode_evector};
- use crate::ml_db::{ClusterSummary, FaceRow};
- use crate::ml_store::tests::{index_path, meta, open};
- use crate::ml_store::{CLUSTER_CENTROID_DIMENSIONS, Error, Index, MlStore};
+ use crate::ml_db::ClusterSummary;
+ use crate::ml_db::vector_encoding::encode_evector;
+ use crate::ml_store::tests::{
+ assert_direction, assign, embedding, face, index_path, meta, open, seed, summary_of,
+ };
+ use crate::ml_store::{Error, Index, MlStore};
const THRESHOLD: f64 = 0.24;
- fn embedding(degrees: f64) -> Vec<f64> {
- let mut values = vec![0.0; CLUSTER_CENTROID_DIMENSIONS];
- let radians = degrees.to_radians();
- values[0] = radians.cos();
- values[1] = radians.sin();
- values
- }
-
- fn face(file_id: i64, degrees: f64) -> FaceRow {
- FaceRow {
- file_id,
- face_id: format!("{file_id}_0"),
- detection_json: "{}".to_string(),
- embedding: embedding(degrees),
- score: 0.9,
- blur: 300.0,
- is_sideways: false,
- image_height: 100,
- image_width: 100,
- ml_version: 1,
- }
- }
-
- fn seed(store: &MlStore, faces: &[(i64, f64)]) {
- let rows: Vec<FaceRow> = faces
- .iter()
- .map(|&(file_id, degrees)| face(file_id, degrees))
- .collect();
- store.db().bulk_insert_faces(&rows).unwrap();
- }
-
- fn assign(store: &MlStore, assignments: &[(i64, &str)]) {
- let assignments: HashMap<String, String> = assignments
- .iter()
- .map(|&(file_id, cluster_id)| (format!("{file_id}_0"), cluster_id.to_string()))
- .collect();
- store
- .db()
- .update_face_id_to_cluster_id(&assignments)
- .unwrap();
- }
-
fn request(file_ids: impl IntoIterator<Item = i64>) -> FaceClusteringRequest {
FaceClusteringRequest {
file_creation_times: file_ids
@@ -349,23 +312,6 @@ mod tests {
.unwrap()
}
- fn summary_of(store: &MlStore, cluster_id: &str) -> (Vec<f64>, i64) {
- let summary = store
- .db()
- .get_cluster_to_cluster_summary(&[cluster_id.to_string()])
- .unwrap()
- .remove(cluster_id)
- .unwrap();
- (decode_evector(&summary.avg).unwrap(), summary.count)
- }
-
- fn assert_direction(actual: &[f64], degrees: f64) {
- let expected = embedding(degrees);
- for (actual, expected) in actual.iter().zip(&expected) {
- assert!((actual - expected).abs() < 1e-5, "{actual} != {expected}");
- }
- }
-
#[test]
fn new_faces_are_clustered_and_written_with_summaries_and_centroids() {
let (_directory, store) = open();
diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rs
index 4d02dd89c1..8089f663b8 100644
--- a/rust/crates/photos/src/ml_store/mod.rs
+++ b/rust/crates/photos/src/ml_store/mod.rs
@@ -1,3 +1,4 @@
+mod cluster_averages;
mod clustering;
mod fill;
mod index;
@@ -12,6 +13,7 @@ use ente_vecdb::{VecDb, VecDbError};
use crate::ml_db::MlDb;
+pub use cluster_averages::MeanSuggestion;
pub use clustering::{
ClusteredBucket, FaceClusteringPlan, FaceClusteringRequest, FaceClusteringRun,
};
@@ -161,15 +163,16 @@ impl MlStore {
#[cfg(test)]
mod tests {
use std::cell::{Cell, RefCell};
+ use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use ente_vecdb::{OpenCost, SearchParams, VecDb};
use tempfile::TempDir;
- use super::{FillOutcome, FillReport, FillState, Index, MlStore};
- use crate::ml_db::vector_encoding::encode_evector;
- use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, ClipEmbedding, ClusterSummary};
+ use super::{CLUSTER_CENTROID_DIMENSIONS, FillOutcome, FillReport, FillState, Index, MlStore};
+ use crate::ml_db::vector_encoding::{decode_evector, encode_evector};
+ use crate::ml_db::{CLIP_EMBEDDING_DIMENSIONS, ClipEmbedding, ClusterSummary, FaceRow};
pub(super) const DB_FILE: &str = "ente.ml.db";
@@ -228,6 +231,65 @@ mod tests {
(cluster_id.to_string(), summary)
}
+ pub(super) fn embedding(degrees: f64) -> Vec<f64> {
+ let mut values = vec![0.0; CLUSTER_CENTROID_DIMENSIONS];
+ let radians = degrees.to_radians();
+ values[0] = radians.cos();
+ values[1] = radians.sin();
+ values
+ }
+
+ pub(super) fn face(file_id: i64, degrees: f64) -> FaceRow {
+ FaceRow {
+ file_id,
+ face_id: format!("{file_id}_0"),
+ detection_json: "{}".to_string(),
+ embedding: embedding(degrees),
+ score: 0.9,
+ blur: 300.0,
+ is_sideways: false,
+ image_height: 100,
+ image_width: 100,
+ ml_version: 1,
+ }
+ }
+
+ pub(super) fn seed(store: &MlStore, faces: &[(i64, f64)]) {
+ let rows: Vec<FaceRow> = faces
+ .iter()
+ .map(|&(file_id, degrees)| face(file_id, degrees))
+ .collect();
+ store.db().bulk_insert_faces(&rows).unwrap();
+ }
+
+ pub(super) fn assign(store: &MlStore, assignments: &[(i64, &str)]) {
+ let assignments: HashMap<String, String> = assignments
+ .iter()
+ .map(|&(file_id, cluster_id)| (format!("{file_id}_0"), cluster_id.to_string()))
+ .collect();
+ store
+ .db()
+ .update_face_id_to_cluster_id(&assignments)
+ .unwrap();
+ }
+
+ pub(super) fn summary_of(store: &MlStore, cluster_id: &str) -> (Vec<f64>, i64) {
+ let summary = store
+ .db()
+ .get_cluster_to_cluster_summary(&[cluster_id.to_string()])
+ .unwrap()
+ .remove(cluster_id)
+ .unwrap();
+ (decode_evector(&summary.avg).unwrap(), summary.count)
+ }
+
+ pub(super) fn assert_direction(actual: &[f64], degrees: f64) {
+ let expected = embedding(degrees);
+ for (actual, expected) in actual.iter().zip(&expected) {
+ assert!((actual - expected).abs() < 1e-5, "{actual} != {expected}");
+ }
+ }
+
pub(super) fn nearest(store: &MlStore, index: Index, query: &[f32]) -> String {
let params = SearchParams {
limit: Some(1),