Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile][photos] Enforce ML foreground/background mutual exclusion

laurenspriem127a1cc8f9cd
diff --git a/mobile/apps/photos/lib/main.dart b/mobile/apps/photos/lib/main.dart
index dfafa9a424..fd534533be 100644
--- a/mobile/apps/photos/lib/main.dart
+++ b/mobile/apps/photos/lib/main.dart
@@ -46,12 +46,14 @@ import 'package:photos/services/collections_service.dart';
import 'package:photos/services/favorites_service.dart';
import 'package:photos/services/home_widget_service.dart';
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
+import "package:photos/services/machine_learning/ml_run_control.dart";
import 'package:photos/services/machine_learning/ml_service.dart';
import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart';
import 'package:photos/services/memory_lane/memory_lane_service.dart';
import 'package:photos/services/memory_share_service.dart';
import "package:photos/services/notification_service.dart";
import "package:photos/services/photos_contacts_service.dart";
+import "package:photos/services/process_activity.dart";
import 'package:photos/services/push_service.dart';
import 'package:photos/services/search_service.dart';
import 'package:photos/services/social_notification_coordinator.dart';
@@ -70,14 +72,16 @@ import 'package:shared_preferences/shared_preferences.dart';
final _logger = Logger("main");
-const kLastBGTaskHeartBeatTime = "bg_task_hb_time";
-const kLastFGTaskHeartBeatTime = "fg_task_hb_time";
const kHeartBeatFrequency = Duration(seconds: 1);
const kFGSyncFrequency = Duration(minutes: 5);
const kFGHomeWidgetSyncFrequency = Duration(minutes: 15);
const kBGTaskTimeout = Duration(seconds: 28);
const kBGPushTimeout = Duration(seconds: 28);
-const kFGTaskDeathTimeoutInMicroseconds = 5000000;
+// ML self-stops before the platform hard-kills the BG engine (kBGTaskTimeout
+// on iOS, the ~10-minute WorkManager system stop on Android), leaving margin
+// to drain in-flight work and release the process lock cleanly.
+const kBGTaskMLSelfStopIOS = Duration(seconds: 26);
+const kBGTaskMLSelfStopAndroid = Duration(minutes: 9);
bool isProcessBg = true;
bool _stopHearBeat = false;
bool _isSyncInitialized = false;
@@ -242,27 +246,41 @@ Future<void> runBackgroundTask(
TimeLogger tlog, {
String mode = 'normal',
}) async {
- // Check if foreground is recently active to avoid conflicts
- final isRunningInFG = await _isRunningInForeground();
+ // Created at task start so a stop that fires before ML begins stays
+ // latched for the whole task.
+ final mlRunControl = MlRunControl();
+ final mlSelfStopTimer = Timer(
+ Platform.isIOS ? kBGTaskMLSelfStopIOS : kBGTaskMLSelfStopAndroid,
+ () => mlRunControl.requestStop(MlStopReason.backgroundDeadline),
+ );
- // If FG was active in last 30 seconds, skip BG work
- if (isRunningInFG) {
- _logger.info(
- "[BG TASK] Foreground recently active, skipping background work",
- );
- return;
- }
+ try {
+ // Check if foreground is recently active to avoid conflicts
+ final isRunningInFG = await isForegroundEngineActive();
- _logger.info(
- "[BG TASK] No recent foreground activity, proceeding with background work",
- );
+ // If FG was active in the last 5 seconds, skip BG work
+ if (isRunningInFG) {
+ _logger.info(
+ "[BG TASK] Foreground recently active, skipping background work",
+ );
+ return;
+ }
- // Mark BG as active
+ _logger.info(
+ "[BG TASK] No recent foreground activity, proceeding with background work",
+ );
- await _runMinimally(taskId, tlog);
+ await _runMinimally(taskId, tlog, mlRunControl);
+ } finally {
+ mlSelfStopTimer.cancel();
+ }
}
-Future<void> _runMinimally(String taskId, TimeLogger tlog) async {
+Future<void> _runMinimally(
+ String taskId,
+ TimeLogger tlog,
+ MlRunControl mlRunControl,
+) async {
try {
final PackageInfo packageInfo = await PackageInfo.fromPlatform();
final SharedPreferences prefs = await SharedPreferences.getInstance();
@@ -355,7 +373,11 @@ Future<void> _runMinimally(String taskId, TimeLogger tlog) async {
await MLService.instance.init();
PersonService.init(entityService, MLDataDB.instance, prefs);
mlRunStarted = true;
- await MLService.instance.runAllML(force: false);
+ final disposition = await MLService.instance.runAllML(
+ force: false,
+ control: mlRunControl,
+ );
+ _logger.info("[BG TASK] ML run disposition: ${disposition.name}");
} finally {
if (!mlRunStarted) {
controller.releaseCompute(ml: true);
@@ -685,19 +707,8 @@ Future<void> _scheduleFGSync(String caller) async {
});
}
-Future<bool> _isRunningInForeground() async {
- final prefs = await SharedPreferences.getInstance();
- await prefs.reload();
- final currentTime = DateTime.now().microsecondsSinceEpoch;
- final lastFGHeartBeatTime = DateTime.fromMicrosecondsSinceEpoch(
- prefs.getInt(kLastFGTaskHeartBeatTime) ?? 0,
- );
- return lastFGHeartBeatTime.microsecondsSinceEpoch >
- (currentTime - kFGTaskDeathTimeoutInMicroseconds);
-}
-
Future<void> _handleBackgroundPush(Object message) async {
- final bool isRunningInFG = await _isRunningInForeground(); // hb
+ final bool isRunningInFG = await isForegroundEngineActive(); // hb
final bool isInForeground = AppLifecycleService.instance.isForeground;
if (isRunningInFG) {
_logger.info(
@@ -733,7 +744,7 @@ Future<void> _handleBackgroundPush(Object message) async {
}
Future<void> _logFGHeartBeatInfo(SharedPreferences prefs) async {
- final bool isRunningInFG = await _isRunningInForeground();
+ final bool isRunningInFG = await isForegroundEngineActive();
await prefs.reload();
final lastFGTaskHeartBeatTime = prefs.getInt(kLastFGTaskHeartBeatTime) ?? 0;
final String lastRun = lastFGTaskHeartBeatTime == 0
diff --git a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart b/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
index ec7b934751..5444c335e6 100644
--- a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
+++ b/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
@@ -20,7 +20,7 @@ import "package:photos/events/file_uploaded_event.dart";
import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/local_photos_updated_event.dart';
import "package:photos/gateways/files/file_upload_gateway.dart";
-import "package:photos/main.dart" show isProcessBg, kLastBGTaskHeartBeatTime;
+import "package:photos/main.dart" show isProcessBg;
import "package:photos/models/backup/backup_item.dart";
import 'package:photos/models/file/file.dart';
import 'package:photos/models/file/file_type.dart';
@@ -38,6 +38,8 @@ import "package:photos/service_locator.dart";
import "package:photos/services/account/user_service.dart";
import 'package:photos/services/collections_service.dart';
import 'package:photos/services/file_magic_service.dart';
+import "package:photos/services/process_activity.dart"
+ show kLastBGTaskHeartBeatTime;
import 'package:photos/services/sync/local_sync_service.dart';
import 'package:photos/services/sync/sync_service.dart';
import "package:photos/utils/device_storage_error.dart";
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
index 95cc9fe364..72d70488aa 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
@@ -28,8 +28,11 @@ import "package:photos/services/machine_learning/face_ml/face_detection/detectio
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/machine_learning/ml_indexing_isolate.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
+import "package:photos/services/machine_learning/ml_process_lock.dart";
import "package:photos/services/machine_learning/ml_result.dart";
+import "package:photos/services/machine_learning/ml_run_control.dart";
import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";
+import "package:photos/services/process_activity.dart";
import "package:photos/services/search_service.dart";
import "package:photos/services/video_preview_service.dart";
import "package:photos/utils/isolate/isolate_operations.dart";
@@ -61,11 +64,17 @@ class MLService {
bool _clusteringIsHappening = false;
bool _mlControllerStatus = false;
bool _isIndexingOrClusteringRunning = false;
- bool _isRunningML = false;
- bool _shouldPauseIndexingAndClustering = false;
+ MlRunControl? _activeRunControl;
+ Timer? _bgYieldPollTimer;
+ Timer? _deniedRunRetryTimer;
Timer? _predownloadLocalModelsTimer;
static const _kPredownloadLocalModelsDelay = Duration(seconds: 10);
+ static const _kBgYieldPollInterval = Duration(seconds: 3);
+ static const _kDeniedRunRetryDelay = Duration(seconds: 15);
+
+ bool get _isRunningML =>
+ MlProcessLock.instance.activeOperation == MlOperation.fullRun;
bool get isRunningML =>
_isRunningML || memoriesCacheService.isUpdatingMemories;
@@ -122,16 +131,9 @@ class MLService {
_mlControllerStatus = event.shouldRun;
if (_mlControllerStatus) {
- if (_shouldPauseIndexingAndClustering) {
- _cancelPauseIndexingAndClustering();
- _logger.info(
- "MLController allowed running ML, faces indexing undoing previous pause",
- );
- } else {
- _logger.info(
- "MLController allowed running ML, faces indexing starting",
- );
- }
+ _logger.info(
+ "MLController allowed running ML, faces indexing starting",
+ );
// Background start is driven manually from _runMinimally to avoid
// duplicate runAllML invocations in the same cycle.
if (!isProcessBg) {
@@ -139,9 +141,9 @@ class MLService {
}
} else {
_logger.info(
- "MLController stopped running ML, faces indexing will be paused (unless it's fetching embeddings)",
+ "MLController stopped running ML, stopping any active ML run",
);
- pauseIndexingAndClustering();
+ stopActiveRun(MlStopReason.controller);
}
});
_syncMlControllerStatusForBg();
@@ -247,18 +249,30 @@ class MLService {
);
return existing;
}
- final future = _runOwnedRemoteHydrationSafely(
- reason: reason,
- skipHydrationIfCandidateFileCountAtMost:
- skipHydrationIfCandidateFileCountAtMost,
+ final attempt = await MlProcessLock.instance.tryRunExclusive(
+ MlOperation.startupRemoteHydration,
+ () async {
+ final future = _runOwnedRemoteHydrationSafely(
+ reason: reason,
+ skipHydrationIfCandidateFileCountAtMost:
+ skipHydrationIfCandidateFileCountAtMost,
+ );
+ _ownedRemoteHydrationFuture = future;
+ try {
+ await future;
+ } finally {
+ if (identical(_ownedRemoteHydrationFuture, future)) {
+ _ownedRemoteHydrationFuture = null;
+ }
+ }
+ },
+ background: isProcessBg,
);
- _ownedRemoteHydrationFuture = future;
- try {
- await future;
- } finally {
- if (identical(_ownedRemoteHydrationFuture, future)) {
- _ownedRemoteHydrationFuture = null;
- }
+ if (attempt != MlLockAttempt.ran) {
+ _logger.info(
+ "Skipping owned remote ML hydration ($reason): ml process lock "
+ "denied (${attempt.name})",
+ );
}
}
@@ -266,6 +280,7 @@ class MLService {
required String reason,
int? skipHydrationIfCandidateFileCountAtMost,
}) async {
+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");
try {
await _hydrateRemoteEmbeddingsForOwnedFilesInternal(
reason: reason,
@@ -307,25 +322,6 @@ class MLService {
);
}
- Future<void> _waitForOwnedRemoteHydrationIfRunning() async {
- final existing = _ownedRemoteHydrationFuture;
- if (existing == null) {
- return;
- }
- _logger.info(
- "Waiting for owned remote ML hydration to finish before indexing",
- );
- try {
- await existing;
- } catch (e, s) {
- _logger.warning(
- "Owned remote ML hydration failed while indexing was waiting, continuing",
- e,
- s,
- );
- }
- }
-
void _schedulePredownloadLocalModels() {
if (isProcessBg || _predownloadLocalModelsTimer?.isActive == true) {
return;
@@ -355,30 +351,98 @@ class MLService {
await personFeedbackService.syncPersonFeedback();
}
- Future<void> runAllML({bool force = false}) async {
+ Future<MlRunDisposition> runAllML({
+ bool force = false,
+ MlRunControl? control,
+ }) async {
+ final runControl = control ?? MlRunControl();
+ // A latched stop always wins, force included.
+ if (runControl.stopRequested) {
+ _logger.info(
+ "runAllML skipped, stop already requested "
+ "(${runControl.stopReason?.name})",
+ );
+ return MlRunDisposition.stopped;
+ }
if (_isRunningML) {
_logger.info("runAllML called while already running, skipping");
- return;
+ return MlRunDisposition.denied;
}
+ if (!hasGrantedMLConsent) {
+ _logger.info("runAllML called without ML consent, skipping");
+ return MlRunDisposition.denied;
+ }
+ final MLMode mode = isLocalGalleryMode
+ ? MLMode.localGallery
+ : MLMode.enteGallery;
+ final mlDataDB = _dbForMode(mode);
+ if (force) {
+ _mlControllerStatus = true;
+ }
+ if (!_canRunMLFunction(function: "AllML") && !force) {
+ return MlRunDisposition.denied;
+ }
+ if (!force && !computeController.requestCompute(ml: true)) {
+ return MlRunDisposition.denied;
+ }
+ var disposition = MlRunDisposition.completed;
try {
- if (!hasGrantedMLConsent) {
- _logger.info("runAllML called without ML consent, skipping");
- return;
- }
- final MLMode mode = isLocalGalleryMode
- ? MLMode.localGallery
- : MLMode.enteGallery;
- final mlDataDB = _dbForMode(mode);
- if (force) {
- _mlControllerStatus = true;
+ final attempt = await MlProcessLock.instance.tryRunExclusive(
+ MlOperation.fullRun,
+ () async {
+ disposition = await _runAllMLProtected(
+ mode: mode,
+ mlDataDB: mlDataDB,
+ force: force,
+ control: runControl,
+ );
+ },
+ background: isProcessBg,
+ );
+ if (attempt != MlLockAttempt.ran) {
+ _logger.info("runAllML denied the ml process lock (${attempt.name})");
+ disposition = MlRunDisposition.denied;
+ _scheduleDeniedRunRetry();
}
- if (!_canRunMLFunction(function: "AllML") && !force) return;
- if (!force && !computeController.requestCompute(ml: true)) return;
- _isRunningML = true;
+ } catch (e, s) {
+ _logger.severe("runAllML failed", e, s);
+ disposition = MlRunDisposition.failed;
+ } finally {
+ computeController.releaseCompute(ml: true);
+ }
+ return disposition;
+ }
+
+ // FG-only: one coalesced retry after a lock denial, because the natural
+ // ComputeControlEvent re-trigger only fires on an interaction or health
+ // transition β€” without this, an untouched device would stall until the
+ // next user interaction. Preconditions are re-validated when it fires.
+ void _scheduleDeniedRunRetry() {
+ if (isProcessBg) return;
+ if (_deniedRunRetryTimer != null) return;
+ _deniedRunRetryTimer = Timer(_kDeniedRunRetryDelay, () {
+ _deniedRunRetryTimer = null;
+ if (!hasGrantedMLConsent) return;
+ triggerML();
+ });
+ }
+
+ Future<MlRunDisposition> _runAllMLProtected({
+ required MLMode mode,
+ required MLDataDB mlDataDB,
+ required bool force,
+ required MlRunControl control,
+ }) async {
+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");
+ _installRunControl(control);
+ try {
await sync();
+ if (control.stopRequested) {
+ return _stoppedDisposition(control, "after sync");
+ }
if (_hasModeChanged(mode)) {
_logger.info("App mode changed during ML run, stopping");
- return;
+ return MlRunDisposition.stopped;
}
final int unclusteredFacesCount = await mlDataDB
@@ -387,81 +451,142 @@ class MLService {
_logger.info(
"There are $unclusteredFacesCount unclustered faces, doing clustering first",
);
- await clusterAllImages();
+ await _clusterAllImages(control: control);
+ }
+ if (control.stopRequested) {
+ return _stoppedDisposition(control, "after initial clustering");
}
if (_mlControllerStatus == true) {
if (_hasModeChanged(mode)) {
_logger.info("App mode changed during ML run, stopping");
- return;
+ return MlRunDisposition.stopped;
}
// Refresh discover/memories caches before indexing using the same
// path in foreground and background runs.
magicCacheService.updateCache(forced: force).ignore();
memoriesCacheService.updateCache(forced: force).ignore();
}
+ if (control.stopRequested) {
+ return _stoppedDisposition(control, "before indexing");
+ }
if (canFetch()) {
- await fetchAndIndexAllImages(mode: mode);
+ await _fetchAndIndexAllImages(mode: mode, control: control);
}
if (_hasModeChanged(mode)) {
_logger.info("App mode changed during ML run, stopping");
- return;
+ return MlRunDisposition.stopped;
+ }
+ if (control.stopRequested) {
+ return _stoppedDisposition(control, "before final clustering");
}
if ((await mlDataDB.getUnclusteredFaceCount()) > 0) {
- await clusterAllImages();
+ await _clusterAllImages(control: control);
+ }
+ if (control.stopRequested) {
+ return _stoppedDisposition(control, "before post-run cache scheduling");
}
if (_mlControllerStatus == true) {
if (_hasModeChanged(mode)) {
_logger.info("App mode changed during ML run, stopping");
- return;
+ return MlRunDisposition.stopped;
}
// Persist refreshed caches after ML so foreground can pick them up
// on the next resume, even when the work ran headlessly in background.
magicCacheService.updateCache().ignore();
memoriesCacheService.updateCache(forced: force).ignore();
}
- } catch (e, s) {
- _logger.severe("runAllML failed", e, s);
- rethrow;
+ return MlRunDisposition.completed;
} finally {
_logger.info("ML finished running");
- _isRunningML = false;
- computeController.releaseCompute(ml: true);
+ _clearRunControl(control);
if (!isProcessBg) {
VideoPreviewService.instance.queueFiles();
}
}
}
- void triggerML() {
- if (_mlControllerStatus &&
- !_isIndexingOrClusteringRunning &&
- !_isRunningML) {
- unawaited(runAllML());
- }
+ MlRunDisposition _stoppedDisposition(MlRunControl control, String where) {
+ _logger.info("ML run stopped $where (${control.stopReason?.name})");
+ return MlRunDisposition.stopped;
}
- void pauseIndexingAndClustering() {
- if (_isIndexingOrClusteringRunning) {
- _shouldPauseIndexingAndClustering = true;
+ void _installRunControl(MlRunControl control) {
+ _activeRunControl = control;
+ control.attachOnStop(() {
MLIndexingIsolate.instance.shouldPauseIndexingAndClustering = true;
+ });
+ // BG always yields to FG: while a BG run holds the lock, watch for the
+ // foreground engine's heartbeat and wind down as soon as it appears.
+ if (isProcessBg) {
+ _bgYieldPollTimer = Timer.periodic(_kBgYieldPollInterval, (_) async {
+ if (control.stopRequested) return;
+ if (await isForegroundEngineActive()) {
+ control.requestStop(MlStopReason.foregroundActive);
+ }
+ });
}
}
- void _cancelPauseIndexingAndClustering() {
- _shouldPauseIndexingAndClustering = false;
+ // The isolate stop flag is cleared here, from the outer run cleanup, and
+ // nowhere else: a durable stop must survive stage-level finally blocks.
+ void _clearRunControl(MlRunControl control) {
+ _bgYieldPollTimer?.cancel();
+ _bgYieldPollTimer = null;
+ control.detachOnStop();
+ if (identical(_activeRunControl, control)) {
+ _activeRunControl = null;
+ }
MLIndexingIsolate.instance.shouldPauseIndexingAndClustering = false;
}
+ void triggerML() {
+ if (_mlControllerStatus && !MlProcessLock.instance.isBusy) {
+ unawaited(runAllML());
+ }
+ }
+
+ /// Requests a durable stop of the active ML run, if any. The run drains
+ /// already-started work and does not begin further stages; a new run may
+ /// start later through the normal triggers.
+ void stopActiveRun(MlStopReason reason) {
+ _activeRunControl?.requestStop(reason);
+ }
+
/// Analyzes all the images in the user library with the latest ml version and stores the results in the database.
///
/// This function first fetches from remote and checks if the image has already been analyzed
/// with the lastest faceMlVersion and stored on remote or local database. If so, it skips the image.
Future<void> fetchAndIndexAllImages({required MLMode mode}) async {
- if (!_canRunMLFunction(function: "Indexing")) return;
- if (mode == MLMode.enteGallery && !isLocalGalleryMode) {
- await _waitForOwnedRemoteHydrationIfRunning();
+ final control = MlRunControl();
+ final attempt = await MlProcessLock.instance.tryRunExclusive(
+ MlOperation.indexing,
+ () async {
+ _installRunControl(control);
+ try {
+ await _fetchAndIndexAllImages(mode: mode, control: control);
+ } finally {
+ _clearRunControl(control);
+ }
+ },
+ background: isProcessBg,
+ );
+ if (attempt != MlLockAttempt.ran) {
+ _logger.info(
+ "fetchAndIndexAllImages denied the ml process lock (${attempt.name})",
+ );
}
+ }
+
+ Future<void> _fetchAndIndexAllImages({
+ required MLMode mode,
+ required MlRunControl control,
+ }) async {
+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");
if (!_canRunMLFunction(function: "Indexing")) return;
+ if (control.stopRequested) {
+ _stoppedDisposition(control, "before indexing started");
+ return;
+ }
bool rustRuntimePrepared = false;
try {
@@ -478,14 +603,17 @@ class MLService {
int fileAnalyzedCount = 0;
final Stopwatch stopwatch = Stopwatch()..start();
- stream:
+ bool stopRun = false;
await for (final chunk in instructionStream) {
- if ((isLocalGalleryMode ? MLMode.localGallery : MLMode.enteGallery) !=
- mode) {
+ if (_hasModeChanged(mode)) {
_logger.info(
"App mode changed during indexing, stopping current ML run",
);
- break stream;
+ break;
+ }
+ if (control.stopRequested) {
+ _stoppedDisposition(control, "between indexing chunks");
+ break;
}
if (!localSettings.isMLLocalIndexingEnabled) {
if (rustRuntimePrepared) {
@@ -498,7 +626,7 @@ class MLService {
_logger.info(
'stopping indexing because user is not connected to wifi and in online mode',
);
- break stream;
+ break;
} else {
await MLModelDownloadService.instance.ensureModelsDownloaded(
onlyIndexingModels: true,
@@ -510,25 +638,31 @@ class MLService {
}
final futures = <Future<bool>>[];
for (final instruction in chunk) {
- if ((isLocalGalleryMode ? MLMode.localGallery : MLMode.enteGallery) !=
- mode) {
+ if (_hasModeChanged(mode)) {
_logger.info(
"App mode changed during indexing, stopping current ML run",
);
- break stream;
+ stopRun = true;
+ break;
}
- if (_shouldPauseIndexingAndClustering) {
- _logger.info("indexAllImages() was paused, stopping");
- break stream;
+ if (control.stopRequested) {
+ _stoppedDisposition(control, "between indexing instructions");
+ stopRun = true;
+ break;
}
- futures.add(processImage(instruction));
+ futures.add(_processImage(instruction));
}
+ // Drain: work that was already started must complete (and commit)
+ // before this run can exit and release the process lock.
final awaitedFutures = await Future.wait(futures);
final sumFutures = awaitedFutures.fold<int>(
0,
(previousValue, element) => previousValue + (element ? 1 : 0),
);
fileAnalyzedCount += sumFutures;
+ if (stopRun) {
+ break;
+ }
}
if (fileAnalyzedCount > 0) {
magicCacheService.queueUpdate('fileIndexed');
@@ -547,7 +681,6 @@ class MLService {
await MLIndexingIsolate.instance.releaseRustRuntime();
MLModelDownloadService.instance.invalidateModelDownloadCache();
_isIndexingOrClusteringRunning = false;
- _cancelPauseIndexingAndClustering();
}
}
@@ -555,6 +688,40 @@ class MLService {
bool clusterInBuckets = true,
bool force = false,
}) async {
+ final control = MlRunControl();
+ final attempt = await MlProcessLock.instance.tryRunExclusive(
+ MlOperation.clustering,
+ () async {
+ _installRunControl(control);
+ try {
+ await _clusterAllImages(
+ clusterInBuckets: clusterInBuckets,
+ force: force,
+ control: control,
+ );
+ } finally {
+ _clearRunControl(control);
+ }
+ },
+ background: isProcessBg,
+ );
+ if (attempt != MlLockAttempt.ran) {
+ _logger.info(
+ "clusterAllImages denied the ml process lock (${attempt.name})",
+ );
+ }
+ }
+
+ Future<void> _clusterAllImages({
+ required MlRunControl control,
+ bool clusterInBuckets = true,
+ bool force = false,
+ }) async {
+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");
+ if (control.stopRequested) {
+ _stoppedDisposition(control, "before clustering started");
+ return;
+ }
if (!_canRunMLFunction(function: "Clustering") && !force) return;
if (_clusteringIsHappening) {
_logger.info("clusterAllImages() is already running, returning");
@@ -628,10 +795,8 @@ class MLService {
int bucket = 1;
while (true) {
- if (_shouldPauseIndexingAndClustering) {
- _logger.info(
- "MLController does not allow running ML, stopping before clustering bucket $bucket",
- );
+ if (control.stopRequested) {
+ _stoppedDisposition(control, "before clustering bucket $bucket");
break;
}
if (offset > allFaceInfoForClustering.length - 1) {
@@ -746,11 +911,10 @@ class MLService {
} finally {
_clusteringIsHappening = false;
_isIndexingOrClusteringRunning = false;
- _cancelPauseIndexingAndClustering();
}
}
- Future<bool> processImage(FileMLInstruction instruction) async {
+ Future<bool> _processImage(FileMLInstruction instruction) async {
bool actuallyRanML = false;
final mlDataDB = _dbForMode(instruction.mode);
@@ -768,9 +932,9 @@ class MLService {
instruction,
filePath,
);
- // Check if there's no result simply because MLController paused indexing
+ // Check if there's no result simply because the run was stopped
if (result == null) {
- if (!_shouldPauseIndexingAndClustering &&
+ if (!(_activeRunControl?.stopRequested ?? false) &&
!MLIndexingIsolate.instance.shouldPauseIndexingAndClustering) {
_logger.severe(
"Failed to analyze image with fileID: ${instruction.fileKey}",
@@ -918,7 +1082,7 @@ class MLService {
final int? size = instruction.file.fileSize;
final fileType = instruction.file.fileType;
if (e is RustCorruptModelException) {
- pauseIndexingAndClustering();
+ stopActiveRun(MlStopReason.corruptModel);
_logger.severe(
"Stopping ML indexing for fileID ${instruction.fileKey} "
"(format $format, type $fileType, size $size) because Rust ML "
@@ -1040,14 +1204,6 @@ class MLService {
_logStatus();
return false;
}
- if (_shouldPauseIndexingAndClustering) {
- // This should ideally not be triggered, because one of the above should be triggered instead.
- _logger.warning(
- "Cannot run $function because indexing and clustering is being paused",
- );
- _logStatus();
- return false;
- }
return true;
}
@@ -1081,7 +1237,8 @@ class MLService {
Local indexing: ${localSettings.isMLLocalIndexingEnabled}
canRunMLController: $_mlControllerStatus
isIndexingOrClusteringRunning: $_isIndexingOrClusteringRunning
- shouldPauseIndexingAndClustering: $_shouldPauseIndexingAndClustering
+ activeOperation: ${MlProcessLock.instance.activeOperation?.name}
+ stopRequested: ${_activeRunControl?.stopRequested} (reason: ${_activeRunControl?.stopReason?.name})
debugIndexingDisabled: $debugIndexingDisabled
''';
_logger.info(status);
diff --git a/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart b/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart
index 98bd2af827..cb0b9826ff 100644
--- a/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart
+++ b/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart
@@ -1,5 +1,6 @@
import "dart:async";
+import "package:ente_photos_platform/ente_photos_platform.dart";
import "package:ente_pure_utils/ente_pure_utils.dart";
import "package:flutter/material.dart";
import "package:hugeicons/hugeicons.dart";
@@ -15,6 +16,8 @@ import "package:photos/services/machine_learning/face_ml/face_clustering/face_cl
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/machine_learning/ml_indexing_isolate.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
+import "package:photos/services/machine_learning/ml_process_lock.dart";
+import "package:photos/services/machine_learning/ml_run_control.dart";
import "package:photos/services/machine_learning/ml_service.dart";
import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";
import "package:photos/theme/ente_theme.dart";
@@ -248,6 +251,16 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {
trailingIconIsMuted: true,
onTap: () async => _onTriggerClustering(context),
),
+ MenuItemWidgetNew(
+ title: "ML process lock state",
+ leadingIconWidget: _buildIconWidget(
+ context,
+ HugeIcons.strokeRoundedLock,
+ ),
+ trailingIcon: Icons.chevron_right_outlined,
+ trailingIconIsMuted: true,
+ onTap: () async => _onShowProcessLockState(context),
+ ),
MenuItemWidgetNew(
title: "Update discover",
leadingIconWidget: _buildIconWidget(
@@ -637,7 +650,7 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {
await setMLConsent(mlConsent);
logger.info('ML consent turned ${mlConsent ? 'on' : 'off'}');
if (!mlConsent) {
- MLService.instance.pauseIndexingAndClustering();
+ MLService.instance.stopActiveRun(MlStopReason.manual);
unawaited(MLIndexingIsolate.instance.cleanupLocalIndexingModels());
} else {
await MLService.instance.init();
@@ -677,7 +690,7 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {
if (localIndexing) {
unawaited(MLService.instance.runAllML(force: true));
} else {
- MLService.instance.pauseIndexingAndClustering();
+ MLService.instance.stopActiveRun(MlStopReason.manual);
unawaited(MLIndexingIsolate.instance.cleanupLocalIndexingModels());
}
if (mounted) {
@@ -690,7 +703,7 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {
MLService.instance.debugIndexingDisabled =
!MLService.instance.debugIndexingDisabled;
if (MLService.instance.debugIndexingDisabled) {
- MLService.instance.pauseIndexingAndClustering();
+ MLService.instance.stopActiveRun(MlStopReason.manual);
} else {
unawaited(MLService.instance.runAllML());
}
@@ -789,6 +802,23 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {
}
}
+ Future<void> _onShowProcessLockState(BuildContext context) async {
+ try {
+ final owner = await ProcessLockClient.instance.state(name: "ml");
+ final funnelOp = MlProcessLock.instance.activeOperation?.name;
+ final native = owner == null
+ ? "free"
+ : "held by ${owner.origin}/${owner.operation} "
+ "for ${owner.heldFor.inSeconds}s";
+ if (!context.mounted) return;
+ showShortToast(context, "Lock: $native Β· funnel: ${funnelOp ?? 'idle'}");
+ } catch (e, s) {
+ logger.warning('fetching ml process lock state failed', e, s);
+ if (!context.mounted) return;
+ await showGenericErrorDialog(context: context, error: e);
+ }
+ }
+
Future<void> _onTriggerClustering(BuildContext context) async {
try {
await PersonService.instance.fetchRemoteClusterFeedback();
diff --git a/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dart b/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dart
index 1cbe38f016..ae0122a63b 100644
--- a/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dart
+++ b/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dart
@@ -11,6 +11,7 @@ import "package:photos/service_locator.dart";
import "package:photos/services/machine_learning/ml_indexing_isolate.dart";
import "package:photos/services/machine_learning/ml_model_assets.dart";
import "package:photos/services/machine_learning/ml_model_download_service.dart";
+import "package:photos/services/machine_learning/ml_run_control.dart";
import "package:photos/services/machine_learning/ml_service.dart";
import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";
import "package:photos/services/remote_assets_service.dart";
@@ -185,7 +186,7 @@ class _MachineLearningSettingsPageState
memoriesCacheService.queueUpdateCache();
Bus.instance.fire(NotificationEvent());
if (!mlConsent) {
- MLService.instance.pauseIndexingAndClustering();
+ MLService.instance.stopActiveRun(MlStopReason.manual);
unawaited(MLIndexingIsolate.instance.cleanupLocalIndexingModels());
if (oldMlEnabled && !newMlEnabled) {
await memoriesCacheService.purgeMlOnlyMemoriesFromCache();
@@ -313,7 +314,7 @@ class _MachineLearningSettingsPageState
if (localIndexing) {
unawaited(MLService.instance.runAllML(force: true));
} else {
- MLService.instance.pauseIndexingAndClustering();
+ MLService.instance.stopActiveRun(MlStopReason.manual);
unawaited(
MLIndexingIsolate.instance.cleanupLocalIndexingModels(),
);
diff --git a/mobile/apps/photos/lib/utils/bg_task_utils.dart b/mobile/apps/photos/lib/utils/bg_task_utils.dart
index 506e932199..b5145bb880 100644
--- a/mobile/apps/photos/lib/utils/bg_task_utils.dart
+++ b/mobile/apps/photos/lib/utils/bg_task_utils.dart
@@ -7,6 +7,7 @@ import "package:permission_handler/permission_handler.dart";
import "package:photos/db/upload_locks_db.dart";
import "package:photos/main.dart";
import "package:photos/module/upload/service/file_uploader.dart";
+import "package:photos/services/process_activity.dart";
import "package:shared_preferences/shared_preferences.dart";
import "package:workmanager/workmanager.dart" as workmanager;