Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Run bulk face clustering in the Rust ML store

laurenspriem866c3901fbc6
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart
new file mode 100644
index 0000000000..33985d1983
--- /dev/null
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart
@@ -0,0 +1,158 @@
+import "dart:math" show min;
+import "dart:typed_data" show Uint8List;
+
+import "package:logging/logging.dart";
+import "package:photos/core/event_bus.dart";
+import "package:photos/db/ml/base.dart";
+import "package:photos/db/ml/dart_db.dart";
+import "package:photos/events/people_changed_event.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart";
+import "package:photos/services/machine_learning/ml_run_control.dart";
+
+class DartFaceClustering implements FaceClustering {
+ static final Logger _logger = Logger("DartFaceClustering");
+
+ final IMLDataDB<int> _mlDataDB;
+
+ DartFaceClustering._(this._mlDataDB);
+
+ static final instance = DartFaceClustering._(DartMLDataDB.instance);
+ static final localGalleryInstance = DartFaceClustering._(
+ DartMLDataDB.localGalleryInstance,
+ );
+
+ @override
+ Future<void> clusterAllFaces({
+ required Map<int, int> fileIDToCreationTime,
+ required Map<String, List<String>> rejectedClusterIDsByFaceID,
+ required MlRunControl control,
+ }) async {
+ final mlDataDB = _mlDataDB;
+ final int totalFaces = await mlDataDB.getTotalFaceCount();
+ final startEmbeddingFetch = DateTime.now();
+ final result = await mlDataDB.getFaceInfoForClustering(
+ maxFaces: totalFaces,
+ );
+ final Set<int> missingFileIDs = {};
+ final allFaceInfoForClustering = <FaceDbInfoForClustering>[];
+ for (final faceInfo in result) {
+ if (!fileIDToCreationTime.containsKey(faceInfo.fileID)) {
+ missingFileIDs.add(faceInfo.fileID);
+ } else {
+ if (rejectedClusterIDsByFaceID.containsKey(faceInfo.faceID)) {
+ faceInfo.rejectedClusterIds =
+ rejectedClusterIDsByFaceID[faceInfo.faceID];
+ }
+ allFaceInfoForClustering.add(faceInfo);
+ }
+ }
+ allFaceInfoForClustering.sort((b, a) {
+ return fileIDToCreationTime[a.fileID]!.compareTo(
+ fileIDToCreationTime[b.fileID]!,
+ );
+ });
+ _logger.info(
+ 'Getting and sorting embeddings took ${DateTime.now().difference(startEmbeddingFetch).inMilliseconds} ms for ${allFaceInfoForClustering.length} embeddings'
+ 'and ${missingFileIDs.length} missing fileIDs',
+ );
+
+ if (missingFileIDs.isNotEmpty) {
+ await mlDataDB.deleteUnclusteredFaceIndexForFiles(
+ missingFileIDs.toList(),
+ );
+ _logger.info(
+ 'Pruned unclustered face data for ${missingFileIDs.length} missing files',
+ );
+ }
+
+ if (allFaceInfoForClustering.every((face) => face.clusterId != null)) {
+ _logger.info('All faces clustered');
+ return;
+ }
+
+ final Map<String, (Uint8List, int)> oldClusterSummaries = await mlDataDB
+ .getAllClusterSummary();
+
+ const int bucketSize = 10000;
+ const int offsetIncrement = 7500;
+ int offset = 0;
+ int bucket = 1;
+
+ while (true) {
+ if (control.stopRequested) {
+ _logger.info(
+ "Clustering stopped before bucket $bucket (${control.stopReason?.name})",
+ );
+ break;
+ }
+ if (offset > allFaceInfoForClustering.length - 1) {
+ _logger.warning(
+ 'faceIdToEmbeddingBucket is empty, this should ideally not happen as it should have stopped earlier. offset: $offset, totalFaces: $totalFaces',
+ );
+ break;
+ }
+ if (offset > totalFaces) {
+ _logger.warning(
+ 'offset > totalFaces, this should ideally not happen. offset: $offset, totalFaces: $totalFaces',
+ );
+ break;
+ }
+
+ final bucketStartTime = DateTime.now();
+ final faceInfoForClustering = allFaceInfoForClustering.sublist(
+ offset,
+ min(offset + bucketSize, allFaceInfoForClustering.length),
+ );
+
+ if (faceInfoForClustering.every((face) => face.clusterId != null)) {
+ _logger.info('Everything in bucket $bucket is already clustered');
+ if (offset + bucketSize >= totalFaces) {
+ _logger.info('All faces clustered');
+ break;
+ } else {
+ _logger.info('Skipping to next bucket');
+ offset += offsetIncrement;
+ bucket++;
+ continue;
+ }
+ }
+
+ final clusteringResult = await FaceClusteringService.instance
+ .predictLinearIsolate(
+ faceInfoForClustering.toSet(),
+ fileIDToCreationTime: fileIDToCreationTime,
+ offset: offset,
+ oldClusterSummaries: oldClusterSummaries,
+ );
+ if (clusteringResult == null) {
+ _logger.warning("faceIdToCluster is null");
+ return;
+ }
+
+ await mlDataDB.updateFaceIdToClusterId(
+ clusteringResult.newFaceIdToCluster,
+ );
+ await mlDataDB.clusterSummaryUpdate(clusteringResult.newClusterSummaries);
+ Bus.instance.fire(PeopleChangedEvent());
+ for (final faceInfo in faceInfoForClustering) {
+ faceInfo.clusterId ??=
+ clusteringResult.newFaceIdToCluster[faceInfo.faceID];
+ }
+ for (final clusterUpdate
+ in clusteringResult.newClusterSummaries.entries) {
+ oldClusterSummaries[clusterUpdate.key] = clusterUpdate.value;
+ }
+ _logger.info(
+ 'Done with clustering ${offset + faceInfoForClustering.length} embeddings (${(100 * (offset + faceInfoForClustering.length) / totalFaces).toStringAsFixed(0)}%) in bucket $bucket, offset: $offset, in ${DateTime.now().difference(bucketStartTime).inSeconds} seconds',
+ );
+ if (offset + bucketSize >= totalFaces) {
+ _logger.info('All faces clustered');
+ break;
+ }
+ offset += offsetIncrement;
+ bucket++;
+ }
+ }
+}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart
new file mode 100644
index 0000000000..39cc2d8a7e
--- /dev/null
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/face_clustering.dart
@@ -0,0 +1,19 @@
+import "package:photos/db/ml/db.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/dart_face_clustering.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart";
+import "package:photos/services/machine_learning/ml_run_control.dart";
+
+abstract class FaceClustering {
+ static FaceClustering get instance => MLDataDB.isRustBackend
+ ? StoreFaceClustering.instance
+ : DartFaceClustering.instance;
+ static FaceClustering get localGalleryInstance => MLDataDB.isRustBackend
+ ? StoreFaceClustering.localGalleryInstance
+ : DartFaceClustering.localGalleryInstance;
+
+ Future<void> clusterAllFaces({
+ required Map<int, int> fileIDToCreationTime,
+ required Map<String, List<String>> rejectedClusterIDsByFaceID,
+ required MlRunControl control,
+ });
+}
diff --git a/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart
new file mode 100644
index 0000000000..99d626a59f
--- /dev/null
+++ b/mobile/apps/photos/lib/services/machine_learning/face_ml/face_clustering/store_face_clustering.dart
@@ -0,0 +1,73 @@
+import "package:logging/logging.dart";
+import "package:photos/core/event_bus.dart";
+import "package:photos/db/ml/rust_db.dart";
+import "package:photos/events/people_changed_event.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
+import "package:photos/services/machine_learning/ml_run_control.dart";
+import "package:photos/src/rust/api/ml_db_api.dart" as rust;
+
+class StoreFaceClustering implements FaceClustering {
+ static final Logger _logger = Logger("StoreFaceClustering");
+
+ final RustMLDataDB _mlDataDB;
+
+ StoreFaceClustering._(this._mlDataDB);
+
+ static final instance = StoreFaceClustering._(RustMLDataDB.instance);
+ static final localGalleryInstance = StoreFaceClustering._(
+ RustMLDataDB.localGalleryInstance,
+ );
+
+ @override
+ Future<void> clusterAllFaces({
+ required Map<int, int> fileIDToCreationTime,
+ required Map<String, List<String>> rejectedClusterIDsByFaceID,
+ required MlRunControl control,
+ }) async {
+ final store = await _mlDataDB.store;
+ final stopwatch = Stopwatch()..start();
+ final run = await store.startFaceClustering(
+ fileCreationTimes: fileIDToCreationTime,
+ rejectedClusterIds: rejectedClusterIDsByFaceID,
+ distanceThreshold: FaceClusteringService.defaultDistanceThreshold,
+ );
+ try {
+ final plan = run.plan();
+ _logger.info(
+ "Prepared ${plan.faces} faces for clustering in ${stopwatch.elapsedMilliseconds} ms: "
+ "${plan.unclusteredFaces} unclustered, ${plan.invalidFaces} skipped for invalid embeddings, "
+ "unclustered faces of ${plan.missingFiles} missing files pruned",
+ );
+ while (true) {
+ if (control.stopRequested) {
+ _logger.info("Clustering stopped (${control.stopReason?.name})");
+ return;
+ }
+ stopwatch.reset();
+ final rust.ClusteredBucket? bucket;
+ try {
+ bucket = await store.clusterNextBucket(run: run);
+ } on rust.MlDbError_Index catch (e, s) {
+ _logger.severe(
+ "ClusterCentroidVectorDB write failed while clustering. The index is marked stale and refills on the next fill check.",
+ e,
+ s,
+ );
+ Bus.instance.fire(PeopleChangedEvent());
+ continue;
+ }
+ if (bucket == null) {
+ _logger.info("All faces clustered");
+ return;
+ }
+ Bus.instance.fire(PeopleChangedEvent());
+ _logger.info(
+ "Clustered ${bucket.clusteredFaces} new faces, ${bucket.processedFaces} of ${plan.faces} done, in ${stopwatch.elapsedMilliseconds} ms",
+ );
+ }
+ } finally {
+ run.dispose();
+ }
+ }
+}
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
index 6e086fdae2..aa95f5e5de 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
@@ -1,7 +1,5 @@
import "dart:async";
import "dart:io" show File, Platform;
-import "dart:math" show min;
-import "dart:typed_data" show Uint8List;
import "package:flutter/foundation.dart" show kDebugMode;
import "package:flutter/widgets.dart" show AppLifecycleState, WidgetsBinding;
@@ -13,7 +11,6 @@ import "package:photos/db/ml/db.dart";
import "package:photos/db/offline_files_db.dart";
import "package:photos/events/app_mode_changed_event.dart";
import "package:photos/events/compute_control_event.dart";
-import "package:photos/events/people_changed_event.dart";
import "package:photos/main.dart";
import "package:photos/models/file/file.dart";
import "package:photos/models/file/file_type.dart";
@@ -23,8 +20,8 @@ import "package:photos/models/ml/ml_versions.dart";
import "package:photos/module/download/file.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/filedata/model/file_data.dart";
+import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
-import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/machine_learning/ml_exceptions.dart";
import "package:photos/services/machine_learning/ml_indexing_isolate.dart";
@@ -80,8 +77,9 @@ class MLService {
: _kForceClusteringFaceCount;
}
- IMLDataDB<int> get _mlDataDB =>
- isLocalGalleryMode ? MLDataDB.localGalleryInstance : MLDataDB.instance;
+ FaceClustering get _faceClustering => isLocalGalleryMode
+ ? FaceClustering.localGalleryInstance
+ : FaceClustering.instance;
IMLDataDB<int> _dbForMode(MLMode mode) {
return mode == MLMode.localGallery
@@ -720,19 +718,12 @@ class MLService {
}
}
- Future<MlLockAttempt> clusterAllImages({
- bool clusterInBuckets = true,
- bool force = false,
- }) async {
+ Future<MlLockAttempt> clusterAllImages({bool force = false}) async {
final control = MlRunControl();
final attempt = await _runExclusiveWithControl(
MlOperation.clustering,
control,
- () => _clusterAllImages(
- clusterInBuckets: clusterInBuckets,
- force: force,
- control: control,
- ),
+ () => _clusterAllImages(force: force, control: control),
);
if (attempt != MlLockAttempt.ran) {
_logger.info(
@@ -744,7 +735,6 @@ class MLService {
Future<void> _clusterAllImages({
required MlRunControl control,
- bool clusterInBuckets = true,
bool force = false,
}) async {
assert(MlProcessLock.instance.isBusy, "ml funnel must be held");
@@ -753,7 +743,7 @@ class MLService {
return;
}
if (!_canRunMLFunction(function: "Clustering") && !force) return;
- final mlDataDB = _mlDataDB;
+ final faceClustering = _faceClustering;
_logger.info("`clusterAllImages()` called");
_clusteringIsHappening = true;
final clusterAllImagesTime = DateTime.now();
@@ -776,169 +766,14 @@ class MLService {
}
try {
- final int totalFaces = await mlDataDB.getTotalFaceCount();
final fileIDToCreationTime = isLocalGalleryMode
? await _getLocalGalleryFileIdToCreationTime()
: await FilesDB.instance.getFileIDToCreationTime();
- final startEmbeddingFetch = DateTime.now();
- final result = await mlDataDB.getFaceInfoForClustering(
- maxFaces: totalFaces,
- );
- final Set<int> missingFileIDs = {};
- final allFaceInfoForClustering = <FaceDbInfoForClustering>[];
- for (final faceInfo in result) {
- if (!fileIDToCreationTime.containsKey(faceInfo.fileID)) {
- missingFileIDs.add(faceInfo.fileID);
- } else {
- if (faceIdNotToCluster.containsKey(faceInfo.faceID)) {
- faceInfo.rejectedClusterIds = faceIdNotToCluster[faceInfo.faceID];
- }
- allFaceInfoForClustering.add(faceInfo);
- }
- }
- allFaceInfoForClustering.sort((b, a) {
- return fileIDToCreationTime[a.fileID]!.compareTo(
- fileIDToCreationTime[b.fileID]!,
- );
- });
- _logger.info(
- 'Getting and sorting embeddings took ${DateTime.now().difference(startEmbeddingFetch).inMilliseconds} ms for ${allFaceInfoForClustering.length} embeddings'
- 'and ${missingFileIDs.length} missing fileIDs',
+ await faceClustering.clusterAllFaces(
+ fileIDToCreationTime: fileIDToCreationTime,
+ rejectedClusterIDsByFaceID: faceIdNotToCluster,
+ control: control,
);
-
- if (missingFileIDs.isNotEmpty) {
- await mlDataDB.deleteUnclusteredFaceIndexForFiles(
- missingFileIDs.toList(),
- );
- _logger.info(
- 'Pruned unclustered face data for ${missingFileIDs.length} missing files',
- );
- }
-
- if (allFaceInfoForClustering.every((face) => face.clusterId != null)) {
- _logger.info('All faces clustered');
- _logger.info(
- 'clusterAllImages() finished, in '
- '${DateTime.now().difference(clusterAllImagesTime).inSeconds} seconds',
- );
- return;
- }
-
- final Map<String, (Uint8List, int)> oldClusterSummaries = await mlDataDB
- .getAllClusterSummary();
-
- if (clusterInBuckets) {
- const int bucketSize = 10000;
- const int offsetIncrement = 7500;
- int offset = 0;
- int bucket = 1;
-
- while (true) {
- if (control.stopRequested) {
- _logRunStopped(control, "before clustering bucket $bucket");
- break;
- }
- if (offset > allFaceInfoForClustering.length - 1) {
- _logger.warning(
- 'faceIdToEmbeddingBucket is empty, this should ideally not happen as it should have stopped earlier. offset: $offset, totalFaces: $totalFaces',
- );
- break;
- }
- if (offset > totalFaces) {
- _logger.warning(
- 'offset > totalFaces, this should ideally not happen. offset: $offset, totalFaces: $totalFaces',
- );
- break;
- }
-
- final bucketStartTime = DateTime.now();
- final faceInfoForClustering = allFaceInfoForClustering.sublist(
- offset,
- min(offset + bucketSize, allFaceInfoForClustering.length),
- );
-
- if (faceInfoForClustering.every((face) => face.clusterId != null)) {
- _logger.info('Everything in bucket $bucket is already clustered');
- if (offset + bucketSize >= totalFaces) {
- _logger.info('All faces clustered');
- break;
- } else {
- _logger.info('Skipping to next bucket');
- offset += offsetIncrement;
- bucket++;
- continue;
- }
- }
-
- final clusteringResult = await FaceClusteringService.instance
- .predictLinearIsolate(
- faceInfoForClustering.toSet(),
- fileIDToCreationTime: fileIDToCreationTime,
- offset: offset,
- oldClusterSummaries: oldClusterSummaries,
- );
- if (clusteringResult == null) {
- _logger.warning("faceIdToCluster is null");
- return;
- }
-
- await mlDataDB.updateFaceIdToClusterId(
- clusteringResult.newFaceIdToCluster,
- );
- await mlDataDB.clusterSummaryUpdate(
- clusteringResult.newClusterSummaries,
- );
- Bus.instance.fire(PeopleChangedEvent());
- for (final faceInfo in faceInfoForClustering) {
- faceInfo.clusterId ??=
- clusteringResult.newFaceIdToCluster[faceInfo.faceID];
- }
- for (final clusterUpdate
- in clusteringResult.newClusterSummaries.entries) {
- oldClusterSummaries[clusterUpdate.key] = clusterUpdate.value;
- }
- _logger.info(
- 'Done with clustering ${offset + faceInfoForClustering.length} embeddings (${(100 * (offset + faceInfoForClustering.length) / totalFaces).toStringAsFixed(0)}%) in bucket $bucket, offset: $offset, in ${DateTime.now().difference(bucketStartTime).inSeconds} seconds',
- );
- if (offset + bucketSize >= totalFaces) {
- _logger.info('All faces clustered');
- break;
- }
- offset += offsetIncrement;
- bucket++;
- }
- } else {
- final clusterStartTime = DateTime.now();
- final clusteringResult = await FaceClusteringService.instance
- .predictLinearIsolate(
- allFaceInfoForClustering.toSet(),
- fileIDToCreationTime: fileIDToCreationTime,
- oldClusterSummaries: oldClusterSummaries,
- );
- if (clusteringResult == null) {
- _logger.warning("faceIdToCluster is null");
- return;
- }
- final clusterDoneTime = DateTime.now();
- _logger.info(
- 'done with clustering ${allFaceInfoForClustering.length} in ${clusterDoneTime.difference(clusterStartTime).inSeconds} seconds ',
- );
-
- _logger.info(
- 'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB',
- );
- await mlDataDB.updateFaceIdToClusterId(
- clusteringResult.newFaceIdToCluster,
- );
- await mlDataDB.clusterSummaryUpdate(
- clusteringResult.newClusterSummaries,
- );
- Bus.instance.fire(PeopleChangedEvent());
- _logger.info(
- 'Done updating FaceIDs with clusterIDs in the DB, in '
- '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds',
- );
- }
_logger.info(
'clusterAllImages() finished, in '
'${DateTime.now().difference(clusterAllImagesTime).inSeconds} seconds',
diff --git a/rust/bindings/frb/photos/src/api/ml_db_api.rs b/rust/bindings/frb/photos/src/api/ml_db_api.rs
index 6be3943bec..48298f96bb 100644
--- a/rust/bindings/frb/photos/src/api/ml_db_api.rs
+++ b/rust/bindings/frb/photos/src/api/ml_db_api.rs
@@ -161,6 +161,50 @@ impl From<ml_store::FillReport> for FillReport {
}
}
+pub struct FaceClusteringPlan {
+ pub faces: u32,
+ pub unclustered_faces: u32,
+ pub missing_files: u32,
+ pub invalid_faces: u32,
+}
+
+impl From<ml_store::FaceClusteringPlan> for FaceClusteringPlan {
+ fn from(plan: ml_store::FaceClusteringPlan) -> Self {
+ Self {
+ faces: plan.faces as u32,
+ unclustered_faces: plan.unclustered_faces as u32,
+ missing_files: plan.missing_files as u32,
+ invalid_faces: plan.invalid_faces as u32,
+ }
+ }
+}
+
+pub struct ClusteredBucket {
+ pub clustered_faces: u32,
+ pub processed_faces: u32,
+}
+
+impl From<ml_store::ClusteredBucket> for ClusteredBucket {
+ fn from(bucket: ml_store::ClusteredBucket) -> Self {
+ Self {
+ clustered_faces: bucket.clustered_faces as u32,
+ processed_faces: bucket.processed_faces as u32,
+ }
+ }
+}
+
+#[frb(opaque)]
+pub struct FaceClusteringRun {
+ inner: ml_store::FaceClusteringRun,
+}
+
+impl FaceClusteringRun {
+ #[frb(sync)]
+ pub fn plan(&self) -> FaceClusteringPlan {
+ self.inner.plan().into()
+ }
+}
+
fn search_params(
limit: Option<u32>,
max_distance: Option<f32>,
@@ -230,6 +274,35 @@ impl MlStore {
Ok(self.inner.drop_clusters_and_person_table(faces)?)
}
+ pub fn start_face_clustering(
+ &self,
+ file_creation_times: HashMap<i64, i64>,
+ rejected_cluster_ids: HashMap<String, Vec<String>>,
+ distance_threshold: f64,
+ ) -> Result<FaceClusteringRun, MlDbError> {
+ let request = ml_store::FaceClusteringRequest {
+ file_creation_times,
+ rejected_cluster_ids: rejected_cluster_ids
+ .into_iter()
+ .map(|(face_id, cluster_ids)| (face_id, cluster_ids.into_iter().collect()))
+ .collect(),
+ distance_threshold,
+ };
+ Ok(FaceClusteringRun {
+ inner: self.inner.start_face_clustering(request)?,
+ })
+ }
+
+ pub fn cluster_next_bucket(
+ &self,
+ run: &mut FaceClusteringRun,
+ ) -> Result<Option<ClusteredBucket>, MlDbError> {
+ Ok(self
+ .inner
+ .cluster_next_bucket(&mut run.inner)?
+ .map(Into::into))
+ }
+
pub fn clear_all(&self) -> Result<(), MlDbError> {
Ok(self.inner.clear_all()?)
}
diff --git a/rust/crates/photos/src/ml_db/mod.rs b/rust/crates/photos/src/ml_db/mod.rs
index 9825fc0a44..3f34372493 100644
--- a/rust/crates/photos/src/ml_db/mod.rs
+++ b/rust/crates/photos/src/ml_db/mod.rs
@@ -13,7 +13,9 @@ pub use queries::clip::{
ClipRow, EmbeddingVector,
};
pub use queries::clusters::{ClusterCentroidRow, ClusterSummary};
-pub use queries::faces::{FACE_ML_VERSION, FaceDbInfoForClustering, FaceRow, FaceWithoutEmbedding};
+pub use queries::faces::{
+ FACE_ML_VERSION, FaceDbInfoForClustering, FaceForClustering, FaceRow, FaceWithoutEmbedding,
+};
pub use queries::filedata::{FdStatus, PreviewInfo};
pub use queries::persons::PersonToClusterIdToFaceIds;
pub use queries::pets::PET_ML_VERSION;
diff --git a/rust/crates/photos/src/ml_db/queries/clusters.rs b/rust/crates/photos/src/ml_db/queries/clusters.rs
index 6f43a9b4b7..6c4c0bc929 100644
--- a/rust/crates/photos/src/ml_db/queries/clusters.rs
+++ b/rust/crates/photos/src/ml_db/queries/clusters.rs
@@ -264,6 +264,24 @@ impl MlDb {
)
}
+ pub fn update_face_clusters_and_summaries(
+ &self,
+ face_id_to_cluster_id: &HashMap<String, String>,
+ summary: &HashMap<String, ClusterSummary>,
+ ) -> Result<()> {
+ self.write_transaction(|transaction| {
+ let mut assign = transaction.prepare_cached(UPSERT_FACE_CLUSTER)?;
+ for assignment in face_id_to_cluster_id {
+ assign.execute(assignment)?;
+ }
+ let mut summarize = transaction.prepare_cached(UPSERT_CLUSTER_SUMMARY)?;
+ for (cluster_id, summary) in summary {
+ summarize.execute((cluster_id, &summary.avg, summary.count))?;
+ }
+ Ok(())
+ })
+ }
+
pub fn delete_cluster_summary_row(&self, cluster_id: &str) -> Result<()> {
self.execute(
"DELETE FROM cluster_summary WHERE cluster_id = ?",
@@ -724,6 +742,37 @@ pub(in crate::ml_db) mod tests {
assert_eq!(db.count_cluster_summaries().unwrap(), 0);
}
+ #[test]
+ fn face_clusters_and_summaries_are_written_in_one_transaction() {
+ let (directory, db) = open();
+ db.update_face_clusters_and_summaries(
+ &map([("1_0", "c1"), ("2_0", "c1")]),
+ &summaries([("c1", 2)]),
+ )
+ .unwrap();
+ deny_cluster_summary_inserts_after(&directory.path().join("ente.ml.db"), 1);
+
+ assert!(
+ db.update_face_clusters_and_summaries(
+ &map([("2_0", "c2"), ("3_0", "c2")]),
+ &summaries([("c2", 2)]),
+ )
+ .is_err()
+ );
+ assert_eq!(
+ db.get_face_ids_to_cluster_ids(&strings(["1_0", "2_0", "3_0"]))
+ .unwrap(),
+ HashMap::from([
+ ("1_0".to_string(), Some("c1".to_string())),
+ ("2_0".to_string(), Some("c1".to_string())),
+ ])
+ );
+ assert_eq!(
+ db.get_all_cluster_summary(None).unwrap(),
+ summaries([("c1", 2)])
+ );
+ }
+
#[test]
fn cluster_centroid_vector_ids() {
let (_directory, db) = seeded();
diff --git a/rust/crates/photos/src/ml_db/queries/faces.rs b/rust/crates/photos/src/ml_db/queries/faces.rs
index 6ba868350e..6e4a9c7f33 100644
--- a/rust/crates/photos/src/ml_db/queries/faces.rs
+++ b/rust/crates/photos/src/ml_db/queries/faces.rs
@@ -52,6 +52,15 @@ pub struct FaceDbInfoForClustering {
pub is_sideways: bool,
}
+#[derive(Clone, Debug, PartialEq)]
+pub struct FaceForClustering {
+ pub face_id: String,
+ pub file_id: i64,
+ pub cluster_id: Option<String>,
+ pub embedding: Option<Vec<f32>>,
+ pub bad: bool,
+}
+
const FACE_EMBEDDING_MAX_ROWS: usize = 20000;
const UPSERT_FACE: &str = r#"
@@ -329,6 +338,38 @@ impl MlDb {
Ok(result)
}
+ pub fn get_faces_for_clustering(&self) -> Result<Vec<FaceForClustering>> {
+ self.read_all(
+ r#"
+ SELECT faces.face_id, faces.file_id, face_clusters.cluster_id, faces.embedding,
+ faces.score, faces.blur, faces.is_sideways
+ FROM faces
+ LEFT JOIN face_clusters
+ ON face_clusters.face_id = faces.face_id
+ WHERE faces.score > ?
+ AND faces.blur > ?
+ ORDER BY faces.face_id DESC
+ "#,
+ (MINIMUM_QUALITY_FACE_SCORE, LAPLACIAN_HARD_THRESHOLD),
+ |row| {
+ let embedding: Vec<u8> = row.get(3)?;
+ Ok(FaceForClustering {
+ face_id: row.get(0)?,
+ file_id: row.get(1)?,
+ cluster_id: row.get(2)?,
+ embedding: decode_evector(&embedding)
+ .ok()
+ .map(|values| values.into_iter().map(|value| value as f32).collect()),
+ bad: is_bad_face_for_clustering(
+ row.get(4)?,
+ row.get(5)?,
+ row.get::<_, i64>(6)? == 1,
+ ),
+ })
+ },
+ )
+ }
+
pub fn get_face_embedding_rows_for_faces(
&self,
face_ids: &[String],
@@ -811,6 +852,44 @@ pub(in crate::ml_db) mod tests {
);
}
+ #[test]
+ fn faces_for_clustering_join_clusters_decode_embeddings_and_flag_bad_faces() {
+ let (_directory, db) = seeded();
+ db.execute(
+ "UPDATE faces SET embedding = x'0a' WHERE face_id = '3_0'",
+ (),
+ )
+ .unwrap();
+ let faces = db.get_faces_for_clustering().unwrap();
+ let rows: Vec<(&str, i64, Option<&str>, bool)> = faces
+ .iter()
+ .map(|face| {
+ (
+ face.face_id.as_str(),
+ face.file_id,
+ face.cluster_id.as_deref(),
+ face.bad,
+ )
+ })
+ .collect();
+ assert_eq!(
+ rows,
+ [
+ ("9_0", 9, Some("c9"), true),
+ ("8_0", 8, Some("c8"), true),
+ ("7_0", 7, Some("c7"), true),
+ ("6_1", 6, None, false),
+ ("3_0", 3, Some("c3"), false),
+ ("2_0", 2, Some("c1"), false),
+ ("1_1", 1, Some("c2"), false),
+ ("1_0", 1, Some("c1"), false),
+ ("10_0", 10, None, false),
+ ]
+ );
+ assert_eq!(faces[0].embedding, Some(vec![9.0, 0.0, 0.5]));
+ assert_eq!(faces[4].embedding, None);
+ }
+
#[test]
fn face_embedding_rows_keep_twenty_thousand_highest_face_ids_in_order() {
let (_directory, db) = open();
diff --git a/rust/crates/photos/src/ml_db/queries/helpers.rs b/rust/crates/photos/src/ml_db/queries/helpers.rs
index 38d712409a..db45904cd9 100644
--- a/rust/crates/photos/src/ml_db/queries/helpers.rs
+++ b/rust/crates/photos/src/ml_db/queries/helpers.rs
@@ -1,7 +1,7 @@
use std::collections::HashMap;
use std::hash::Hash;
-use crate::db::{self, FromSql, Params, Row, ToSql, params_from_iter};
+use crate::db::{self, FromSql, Params, Row, ToSql, Transaction, params_from_iter};
use crate::ml_db::{MlDb, Result};
@@ -128,6 +128,13 @@ impl MlDb {
.map_err(Into::into)
}
+ pub(in crate::ml_db) fn write_transaction<T>(
+ &self,
+ write: impl FnOnce(&Transaction<'_>) -> db::Result<T>,
+ ) -> Result<T> {
+ self.db.write_transaction(write).map_err(Into::into)
+ }
+
pub(in crate::ml_db) fn write_batch_atomic<P: Params>(
&self,
sql: &str,
diff --git a/rust/crates/photos/src/ml_store/clustering.rs b/rust/crates/photos/src/ml_store/clustering.rs
new file mode 100644
index 0000000000..a3556eab70
--- /dev/null
+++ b/rust/crates/photos/src/ml_store/clustering.rs
@@ -0,0 +1,515 @@
+use std::cmp::Reverse;
+use std::collections::{HashMap, HashSet};
+use std::ops::Range;
+
+use super::fill::decode_centroid;
+use super::{CLUSTER_CENTROID_DIMENSIONS, Error, MlStore, Result};
+use crate::face_clustering::{ClusterMean, Face, IncrementalFace, cluster_incrementally};
+use crate::ml_db::ClusterSummary;
+use crate::ml_db::vector_encoding::encode_evector;
+
+const BUCKET_SIZE: usize = 10_000;
+const BUCKET_STEP: usize = 7_500;
+
+pub struct FaceClusteringRequest {
+ pub file_creation_times: HashMap<i64, i64>,
+ pub rejected_cluster_ids: HashMap<String, HashSet<String>>,
+ pub distance_threshold: f64,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct FaceClusteringPlan {
+ pub faces: usize,
+ pub unclustered_faces: usize,
+ pub missing_files: usize,
+ pub invalid_faces: usize,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct ClusteredBucket {
+ pub clustered_faces: usize,
+ pub processed_faces: usize,
+}
+
+pub struct FaceClusteringRun {
+ faces: Vec<IncrementalFace>,
+ cluster_means: HashMap<String, ClusterMean>,
+ distance_threshold: f64,
+ plan: FaceClusteringPlan,
+ next_offset: Option<usize>,
+ bucket_size: usize,
+ bucket_step: usize,
+}
+
+impl FaceClusteringRun {
+ pub fn plan(&self) -> FaceClusteringPlan {
+ self.plan
+ }
+
+ fn next_bucket(&mut self) -> Option<Range<usize>> {
+ while let Some(offset) = self.next_offset {
+ let end = (offset + self.bucket_size).min(self.faces.len());
+ self.next_offset = (end < self.faces.len()).then_some(offset + self.bucket_step);
+ if self.faces[offset..end]
+ .iter()
+ .any(|face| face.cluster_id.is_none())
+ {
+ return Some(offset..end);
+ }
+ }
+ None
+ }
+
+ fn record(&mut self, bucket: Range<usize>, assignments: &HashMap<String, String>) {
+ for face in &mut self.faces[bucket] {
+ if face.cluster_id.is_none() {
+ face.cluster_id = assignments.get(&face.face.face_id).cloned();
+ }
+ }
+ }
+}
+
+impl MlStore {
+ pub fn start_face_clustering(
+ &self,
+ mut request: FaceClusteringRequest,
+ ) -> Result<FaceClusteringRun> {
+ if !request.distance_threshold.is_finite() {
+ return Err(Error::InvalidArgument(format!(
+ "face clustering distance threshold {} is not finite",
+ request.distance_threshold
+ )));
+ }
+ let mut faces = Vec::new();
+ let mut missing_files = HashSet::new();
+ let mut invalid_faces = 0;
+ for face in self.db.get_faces_for_clustering()? {
+ let Some(&creation_time) = request.file_creation_times.get(&face.file_id) else {
+ missing_files.insert(face.file_id);
+ continue;
+ };
+ let Some(embedding) = face.embedding.filter(|embedding| is_clusterable(embedding))
+ else {
+ invalid_faces += 1;
+ continue;
+ };
+ let rejected_cluster_ids = request
+ .rejected_cluster_ids
+ .remove(&face.face_id)
+ .unwrap_or_default();
+ faces.push(IncrementalFace {
+ face: Face {
+ face_id: face.face_id,
+ embedding,
+ creation_time: Some(creation_time),
+ },
+ cluster_id: face.cluster_id,
+ bad: face.bad,
+ rejected_cluster_ids,
+ });
+ }
+ let missing_files: Vec<i64> = missing_files.into_iter().collect();
+ self.db
+ .delete_unclustered_face_index_for_files(&missing_files)?;
+ faces.sort_by_key(|face| Reverse(face.face.creation_time));
+
+ let unclustered_faces = faces
+ .iter()
+ .filter(|face| face.cluster_id.is_none())
+ .count();
+ let cluster_means = if unclustered_faces == 0 {
+ HashMap::new()
+ } else {
+ self.cluster_means()?
+ };
+ Ok(FaceClusteringRun {
+ plan: FaceClusteringPlan {
+ faces: faces.len(),
+ unclustered_faces,
+ missing_files: missing_files.len(),
+ invalid_faces,
+ },
+ next_offset: (unclustered_faces > 0).then_some(0),
+ faces,
+ cluster_means,
+ distance_threshold: request.distance_threshold,
+ bucket_size: BUCKET_SIZE,
+ bucket_step: BUCKET_STEP,
+ })
+ }
+
+ pub fn cluster_next_bucket(
+ &self,
+ run: &mut FaceClusteringRun,
+ ) -> Result<Option<ClusteredBucket>> {
+ let Some(bucket) = run.next_bucket() else {
+ return Ok(None);
+ };
+ let clustering = cluster_incrementally(
+ run.faces[bucket.clone()].to_vec(),
+ run.distance_threshold,
+ &run.cluster_means,
+ );
+ let summary: HashMap<String, ClusterSummary> = clustering
+ .means
+ .iter()
+ .map(|(cluster_id, mean)| (cluster_id.clone(), cluster_summary(mean)))
+ .collect();
+ let written = self.assign_face_clusters(&clustering.assignments, &summary);
+ if let Ok(()) | Err(Error::Index(_)) = written {
+ run.record(bucket.clone(), &clustering.assignments);
+ run.cluster_means.extend(clustering.means);
+ }
+ written.map(|()| {
+ Some(ClusteredBucket {
+ clustered_faces: clustering.assignments.len(),
+ processed_faces: bucket.end,
+ })
+ })
+ }
+
+ fn cluster_means(&self) -> Result<HashMap<String, ClusterMean>> {
+ Ok(self
+ .db
+ .get_all_cluster_summary(None)?
+ .into_iter()
+ .filter_map(|(cluster_id, summary)| {
+ let mean = decode_centroid(&cluster_id, &summary.avg)?;
+ let count = usize::try_from(summary.count).ok()?;
+ Some((cluster_id, ClusterMean { mean, count }))
+ })
+ .collect())
+ }
+}
+
+fn is_clusterable(embedding: &[f32]) -> bool {
+ embedding.len() == CLUSTER_CENTROID_DIMENSIONS
+ && embedding.iter().all(|value| value.is_finite())
+}
+
+fn cluster_summary(mean: &ClusterMean) -> ClusterSummary {
+ let values: Vec<f64> = mean.mean.iter().copied().map(f64::from).collect();
+ ClusterSummary {
+ avg: encode_evector(&values),
+ count: mean.count as i64,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::{HashMap, HashSet};
+ use std::fs;
+
+ use super::{ClusteredBucket, FaceClusteringPlan, FaceClusteringRequest, FaceClusteringRun};
+ use crate::ml_db::vector_encoding::{decode_evector, encode_evector};
+ use crate::ml_db::{ClusterSummary, FaceRow};
+ use crate::ml_store::tests::{index_path, meta, open};
+ use crate::ml_store::{CLUSTER_CENTROID_DIMENSIONS, Error, Index, MlStore};
+
+ const THRESHOLD: f64 = 0.24;
+
+ fn embedding(degrees: f64) -> Vec<f64> {
+ let mut values = vec![0.0; CLUSTER_CENTROID_DIMENSIONS];
+ let radians = degrees.to_radians();
+ values[0] = radians.cos();
+ values[1] = radians.sin();
+ values
+ }
+
+ fn face(file_id: i64, degrees: f64) -> FaceRow {
+ FaceRow {
+ file_id,
+ face_id: format!("{file_id}_0"),
+ detection_json: "{}".to_string(),
+ embedding: embedding(degrees),
+ score: 0.9,
+ blur: 300.0,
+ is_sideways: false,
+ image_height: 100,
+ image_width: 100,
+ ml_version: 1,
+ }
+ }
+
+ fn seed(store: &MlStore, faces: &[(i64, f64)]) {
+ let rows: Vec<FaceRow> = faces
+ .iter()
+ .map(|&(file_id, degrees)| face(file_id, degrees))
+ .collect();
+ store.db().bulk_insert_faces(&rows).unwrap();
+ }
+
+ fn assign(store: &MlStore, assignments: &[(i64, &str)]) {
+ let assignments: HashMap<String, String> = assignments
+ .iter()
+ .map(|&(file_id, cluster_id)| (format!("{file_id}_0"), cluster_id.to_string()))
+ .collect();
+ store
+ .db()
+ .update_face_id_to_cluster_id(&assignments)
+ .unwrap();
+ }
+
+ fn request(file_ids: impl IntoIterator<Item = i64>) -> FaceClusteringRequest {
+ FaceClusteringRequest {
+ file_creation_times: file_ids
+ .into_iter()
+ .map(|file_id| (file_id, file_id))
+ .collect(),
+ rejected_cluster_ids: HashMap::new(),
+ distance_threshold: THRESHOLD,
+ }
+ }
+
+ fn run_to_end(store: &MlStore, run: &mut FaceClusteringRun) -> Vec<ClusteredBucket> {
+ let mut buckets = Vec::new();
+ while let Some(bucket) = store.cluster_next_bucket(run).unwrap() {
+ buckets.push(bucket);
+ }
+ buckets
+ }
+
+ fn groups(store: &MlStore) -> Vec<Vec<String>> {
+ let mut groups: Vec<Vec<String>> = store
+ .db()
+ .get_all_cluster_id_to_face_ids()
+ .unwrap()
+ .into_values()
+ .map(|mut faces| {
+ faces.sort();
+ faces
+ })
+ .collect();
+ groups.sort();
+ groups
+ }
+
+ fn face_ids(file_ids: impl IntoIterator<Item = i64>) -> Vec<String> {
+ let mut face_ids: Vec<String> = file_ids
+ .into_iter()
+ .map(|file_id| format!("{file_id}_0"))
+ .collect();
+ face_ids.sort();
+ face_ids
+ }
+
+ fn cluster_of(store: &MlStore, file_id: i64) -> Option<String> {
+ store
+ .db()
+ .get_cluster_id_for_face_id(&format!("{file_id}_0"))
+ .unwrap()
+ }
+
+ fn summary_of(store: &MlStore, cluster_id: &str) -> (Vec<f64>, i64) {
+ let summary = store
+ .db()
+ .get_cluster_to_cluster_summary(&[cluster_id.to_string()])
+ .unwrap()
+ .remove(cluster_id)
+ .unwrap();
+ (decode_evector(&summary.avg).unwrap(), summary.count)
+ }
+
+ fn assert_direction(actual: &[f64], degrees: f64) {
+ let expected = embedding(degrees);
+ for (actual, expected) in actual.iter().zip(&expected) {
+ assert!((actual - expected).abs() < 1e-5, "{actual} != {expected}");
+ }
+ }
+
+ #[test]
+ fn new_faces_are_clustered_and_written_with_summaries_and_centroids() {
+ let (_directory, store) = open();
+ store.fill_cluster_centroid_index(false).unwrap();
+ seed(
+ &store,
+ &[(1, 0.0), (2, 2.0), (3, 0.0), (4, 90.0), (5, 92.0)],
+ );
+
+ let mut run = store.start_face_clustering(request(1..=5)).unwrap();
+ assert_eq!(
+ run.plan(),
+ FaceClusteringPlan {
+ faces: 5,
+ unclustered_faces: 5,
+ missing_files: 0,
+ invalid_faces: 0,
+ }
+ );
+ assert_eq!(
+ run_to_end(&store, &mut run),
+ [ClusteredBucket {
+ clustered_faces: 5,
+ processed_faces: 5,
+ }]
+ );
+
+ assert_eq!(groups(&store), [face_ids([1, 2, 3]), face_ids([4, 5])]);
+ let cluster_id = cluster_of(&store, 1).unwrap();
+ let (mean, count) = summary_of(&store, &cluster_id);
+ assert_eq!(count, 3);
+ assert_direction(&mean, 2.0 / 3.0);
+ for file_id in [1, 4] {
+ let cluster_id = cluster_of(&store, file_id).unwrap();
+ assert!(store.contains(Index::ClusterCentroid, &cluster_id).unwrap());
+ }
+ }
+
+ #[test]
+ fn new_faces_join_existing_clusters_and_fold_into_their_summaries() {
+ let (_directory, store) = open();
+ seed(&store, &[(1, 0.0), (2, 0.0), (3, 0.0)]);
+ assign(&store, &[(1, "p")]);
+ let previous = ClusterSummary {
+ avg: encode_evector(&embedding(0.0)),
+ count: 3,
+ };
+ store
+ .db()
+ .upsert_cluster_summary_rows(&HashMap::from([("p".to_string(), previous)]))
+ .unwrap();
+ let mut rejecting = request(1..=3);
+ rejecting
+ .rejected_cluster_ids
+ .insert("2_0".to_string(), HashSet::from(["p".to_string()]));
+
+ let mut run = store.start_face_clustering(rejecting).unwrap();
+ assert_eq!(run.plan().unclustered_faces, 2);
+ run_to_end(&store, &mut run);
+
+ assert_eq!(cluster_of(&store, 3).as_deref(), Some("p"));
+ assert_ne!(cluster_of(&store, 2).as_deref(), Some("p"));
+ let (mean, count) = summary_of(&store, "p");
+ assert_eq!(count, 4);
+ assert_direction(&mean, 0.0);
+ }
+
+ #[test]
+ fn faces_of_missing_files_are_pruned_unless_clustered_and_invalid_embeddings_are_skipped() {
+ let (_directory, store) = open();
+ seed(&store, &[(1, 0.0), (2, 0.0), (4, 0.0)]);
+ let mut invalid = face(3, 0.0);
+ invalid.embedding[5] = f64::NAN;
+ let mut short = face(5, 0.0);
+ short.embedding.truncate(3);
+ store.db().bulk_insert_faces(&[invalid, short]).unwrap();
+ assign(&store, &[(1, "p")]);
+
+ let mut run = store.start_face_clustering(request([3, 4, 5])).unwrap();
+ assert_eq!(
+ run.plan(),
+ FaceClusteringPlan {
+ faces: 1,
+ unclustered_faces: 1,
+ missing_files: 2,
+ invalid_faces: 2,
+ }
+ );
+ run_to_end(&store, &mut run);
+
+ assert!(
+ store
+ .db()
+ .get_faces_for_given_file_id(2)
+ .unwrap()
+ .is_empty()
+ );
+ assert_eq!(cluster_of(&store, 1).as_deref(), Some("p"));
+ assert_eq!(cluster_of(&store, 3), None);
+ assert_eq!(cluster_of(&store, 5), None);
+ assert!(cluster_of(&store, 4).is_some());
+ }
+
+ #[test]
+ fn clustered_libraries_have_no_buckets() {
+ let (_directory, store) = open();
+ seed(&store, &[(1, 0.0)]);
+ assign(&store, &[(1, "p")]);
+ let mut run = store.start_face_clustering(request([1])).unwrap();
+ assert_eq!(run.plan().unclustered_faces, 0);
+ assert!(run_to_end(&store, &mut run).is_empty());
+
+ let mut empty = store.start_face_clustering(request([])).unwrap();
+ assert!(run_to_end(&store, &mut empty).is_empty());
+ }
+
+ #[test]
+ fn overlapping_buckets_continue_from_faces_clustered_in_earlier_buckets() {
+ let (_directory, store) = open();
+ seed(
+ &store,
+ &(1..=7).map(|file_id| (file_id, 0.0)).collect::<Vec<_>>(),
+ );
+ let mut run = store.start_face_clustering(request(1..=7)).unwrap();
+ run.bucket_size = 4;
+ run.bucket_step = 3;
+
+ let processed: Vec<usize> = run_to_end(&store, &mut run)
+ .iter()
+ .map(|bucket| bucket.processed_faces)
+ .collect();
+ assert_eq!(processed, [4, 7]);
+ assert_eq!(groups(&store), [face_ids(1..=7)]);
+ let (_, count) = summary_of(&store, &cluster_of(&store, 1).unwrap());
+ assert_eq!(count, 7);
+ }
+
+ #[test]
+ fn fully_clustered_buckets_are_skipped() {
+ let (_directory, store) = open();
+ seed(
+ &store,
+ &(1..=7).map(|file_id| (file_id, 0.0)).collect::<Vec<_>>(),
+ );
+ assign(&store, &[(7, "p"), (6, "p"), (5, "p"), (4, "p")]);
+ let mut run = store.start_face_clustering(request(1..=7)).unwrap();
+ run.bucket_size = 4;
+ run.bucket_step = 3;
+
+ let buckets = run_to_end(&store, &mut run);
+ assert_eq!(
+ buckets,
+ [ClusteredBucket {
+ clustered_faces: 3,
+ processed_faces: 7,
+ }]
+ );
+ assert_eq!(groups(&store), [face_ids(1..=7)]);
+ }
+
+ #[test]
+ fn buckets_committed_despite_a_failed_centroid_write_are_kept() {
+ let (directory, store) = open();
+ fs::create_dir(index_path(&directory, Index::ClusterCentroid)).unwrap();
+ store
+ .db()
+ .set_meta("cluster_centroid.fill", "filled")
+ .unwrap();
+ seed(
+ &store,
+ &(1..=7).map(|file_id| (file_id, 0.0)).collect::<Vec<_>>(),
+ );
+ let mut run = store.start_face_clustering(request(1..=7)).unwrap();
+ run.bucket_size = 4;
+ run.bucket_step = 3;
+
+ assert!(matches!(
+ store.cluster_next_bucket(&mut run),
+ Err(Error::Index(_))
+ ));
+ assert_eq!(meta(&store, "cluster_centroid.fill"), None);
+ run_to_end(&store, &mut run);
+ assert_eq!(groups(&store), [face_ids(1..=7)]);
+ }
+
+ #[test]
+ fn non_finite_thresholds_are_rejected() {
+ let (_directory, store) = open();
+ let mut invalid = request([]);
+ invalid.distance_threshold = f64::NAN;
+ assert!(matches!(
+ store.start_face_clustering(invalid),
+ Err(Error::InvalidArgument(_))
+ ));
+ }
+}
diff --git a/rust/crates/photos/src/ml_store/mod.rs b/rust/crates/photos/src/ml_store/mod.rs
index 3df7acc8db..4d02dd89c1 100644
--- a/rust/crates/photos/src/ml_store/mod.rs
+++ b/rust/crates/photos/src/ml_store/mod.rs
@@ -1,3 +1,4 @@
+mod clustering;
mod fill;
mod index;
mod lock;
@@ -11,6 +12,9 @@ use ente_vecdb::{VecDb, VecDbError};
use crate::ml_db::MlDb;
+pub use clustering::{
+ ClusteredBucket, FaceClusteringPlan, FaceClusteringRequest, FaceClusteringRun,
+};
pub use ente_vecdb::{KeyMatches, Match, SearchParams, Stats};
pub use fill::{FillOutcome, FillReport};
pub use index::{CLUSTER_CENTROID_DIMENSIONS, Index};
diff --git a/rust/crates/photos/src/ml_store/writes.rs b/rust/crates/photos/src/ml_store/writes.rs
index facea47261..2ba4f1a6e0 100644
--- a/rust/crates/photos/src/ml_store/writes.rs
+++ b/rust/crates/photos/src/ml_store/writes.rs
@@ -73,6 +73,18 @@ impl MlStore {
})
}
+ pub(super) fn assign_face_clusters(
+ &self,
+ face_id_to_cluster_id: &HashMap<String, String>,
+ summary: &HashMap<String, ClusterSummary>,
+ ) -> Result<()> {
+ let _mutations = self.lock_mutations();
+ self.upsert(Index::ClusterCentroid, &centroid_batch(summary), || {
+ self.db
+ .update_face_clusters_and_summaries(face_id_to_cluster_id, summary)
+ })
+ }
+
pub fn delete_cluster_summary(&self, cluster_id: &str) -> Result<()> {
let _mutations = self.lock_mutations();
self.db.delete_cluster_summary_row(cluster_id)?;