Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile][photos] Guard vector migrations with ML process lock

laurenspriem6e864f8fd42e
diff --git a/mobile/apps/photos/lib/db/ml/db.dart b/mobile/apps/photos/lib/db/ml/db.dart
index 4ddb3730f2..e6393e0b29 100644
--- a/mobile/apps/photos/lib/db/ml/db.dart
+++ b/mobile/apps/photos/lib/db/ml/db.dart
@@ -18,6 +18,7 @@ import "package:photos/db/ml/pet_vector_db.dart";
import 'package:photos/db/ml/schema.dart';
import "package:photos/events/embedding_updated_event.dart";
import "package:photos/generated/protos/ente/common/vector.pb.dart";
+import "package:photos/main.dart" show isProcessBg;
import "package:photos/models/ml/clip.dart";
import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/face/face_with_embedding.dart";
@@ -27,6 +28,7 @@ import "package:photos/service_locator.dart";
import "package:photos/services/machine_learning/compute_controller.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
+import "package:photos/services/machine_learning/ml_process_lock.dart";
import "package:photos/services/machine_learning/ml_result.dart";
import "package:photos/utils/ml_util.dart";
import 'package:sqlite_async/sqlite_async.dart';
@@ -1853,6 +1855,18 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
Future<void> checkMigrateFillClusterCentroidVectorDB({
bool force = false,
+ }) async {
+ if (!force && await _clusterCentroidVectorDB.checkIfMigrationDone()) {
+ return;
+ }
+ await _runVectorMigrationExclusive(
+ MlOperation.clusterCentroidVectorMigration,
+ () => _checkMigrateFillClusterCentroidVectorDB(force: force),
+ );
+ }
+
+ Future<void> _checkMigrateFillClusterCentroidVectorDB({
+ required bool force,
}) async {
await _clusterCentroidVectorMigrationLock.synchronized(() async {
final migrationDone = await _clusterCentroidVectorDB
@@ -2151,6 +2165,16 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
}
Future<void> checkMigrateFillClipVectorDB({bool force = false}) async {
+ if (!force && await _clipVectorDB.checkIfMigrationDone()) {
+ return;
+ }
+ await _runVectorMigrationExclusive(
+ MlOperation.clipVectorMigration,
+ () => _checkMigrateFillClipVectorDB(force: force),
+ );
+ }
+
+ Future<void> _checkMigrateFillClipVectorDB({required bool force}) async {
await _clipVectorMigrationLock.synchronized(() async {
final migrationDone = await _clipVectorDB.checkIfMigrationDone();
if (migrationDone && !force) {
@@ -2295,6 +2319,24 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
});
}
+ Future<void> _runVectorMigrationExclusive(
+ MlOperation operation,
+ Future<void> Function() migration,
+ ) async {
+ final attempt = await MlProcessLock.instance.tryRunExclusive(
+ operation,
+ migration,
+ background: isProcessBg,
+ waitForAvailability: true,
+ );
+ if (attempt != MlLockAttempt.ran) {
+ throw StateError(
+ "${operation.name} could not acquire the ML process lock "
+ "(${attempt.name})",
+ );
+ }
+ }
+
Future<void> _withClipVectorWriteRecovery({
required String operation,
required Future<void> Function() writeOperation,
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart b/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart
index 90006d42b9..125c19ca65 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart
@@ -4,7 +4,14 @@ import "package:ente_photos_platform/ente_photos_platform.dart";
import "package:logging/logging.dart";
import "package:synchronized/synchronized.dart";
-enum MlOperation { fullRun, indexing, clustering, startupRemoteHydration }
+enum MlOperation {
+ fullRun,
+ indexing,
+ clustering,
+ startupRemoteHydration,
+ clipVectorMigration,
+ clusterCentroidVectorMigration,
+}
enum MlLockAttempt {
ran,
@@ -19,13 +26,14 @@ enum MlRunDisposition { completed, denied, stopped, failed }
///
/// All top-level ML operations must run through [tryRunExclusive]: the funnel
/// serializes operations within this engine, and the native lock excludes the
-/// other engine. Single attempt, no waiting β€” callers defer to their normal
-/// triggers on denial.
+/// other engine. Calls make a single attempt unless [waitForAvailability] is
+/// set for maintenance work that must complete before its caller continues.
class MlProcessLock {
MlProcessLock._();
static final instance = MlProcessLock._();
static const _lockName = "ml";
+ static const _availabilityPollInterval = Duration(milliseconds: 500);
final _logger = Logger("MlProcessLock");
final Lock _funnel = Lock();
MlOperation? _activeOperation;
@@ -40,12 +48,15 @@ class MlProcessLock {
/// is released only after [body] completes, so [body] must drain any
/// protected ML work it started before returning. Errors from [body]
/// propagate after release. [background] labels this engine's origin.
+ /// [waitForAvailability] waits for both the local funnel and the other
+ /// engine instead of returning a busy denial.
Future<MlLockAttempt> tryRunExclusive(
MlOperation operation,
Future<void> Function() body, {
required bool background,
+ bool waitForAvailability = false,
}) async {
- if (_funnel.locked) {
+ if (!waitForAvailability && _funnel.locked) {
_logger.info(
"Funnel busy with ${_activeOperation?.name}, denying ${operation.name}",
);
@@ -55,25 +66,29 @@ class MlProcessLock {
_activeOperation = operation;
try {
final origin = background ? "bg" : "fg";
- final bool acquired;
- try {
- acquired = await ProcessLockClient.instance.tryAcquire(
- name: _lockName,
- origin: origin,
- operation: operation.name,
- );
- } catch (e, s) {
- // Fail closed: no protected ML without the lock.
- _logger.severe(
- "Process lock channel failure for ${operation.name}",
- e,
- s,
- );
- return MlLockAttempt.deniedChannelError;
- }
- if (!acquired) {
- unawaited(_logOwner(operation));
- return MlLockAttempt.deniedByOtherEngine;
+ while (true) {
+ final bool acquired;
+ try {
+ acquired = await ProcessLockClient.instance.tryAcquire(
+ name: _lockName,
+ origin: origin,
+ operation: operation.name,
+ );
+ } catch (e, s) {
+ // Fail closed: no protected ML without the lock.
+ _logger.severe(
+ "Process lock channel failure for ${operation.name}",
+ e,
+ s,
+ );
+ return MlLockAttempt.deniedChannelError;
+ }
+ if (acquired) break;
+ if (!waitForAvailability) {
+ unawaited(_logOwner(operation));
+ return MlLockAttempt.deniedByOtherEngine;
+ }
+ await Future<void>.delayed(_availabilityPollInterval);
}
_logger.info("Acquired ml lock ($origin/${operation.name})");
final heldSince = DateTime.now();