π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mobile][photos] Enforce ML foreground/background mutual exclusion
laurenspriem127a1cc8f9cd
diff --git a/mobile/apps/photos/lib/main.dart b/mobile/apps/photos/lib/main.dartindex dfafa9a424..fd534533be 100644--- a/mobile/apps/photos/lib/main.dart+++ b/mobile/apps/photos/lib/main.dart@@ -46,12 +46,14 @@ import 'package:photos/services/collections_service.dart';import 'package:photos/services/favorites_service.dart';import 'package:photos/services/home_widget_service.dart';import "package:photos/services/machine_learning/face_ml/person/person_service.dart";+import "package:photos/services/machine_learning/ml_run_control.dart";import 'package:photos/services/machine_learning/ml_service.dart';import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart';import 'package:photos/services/memory_lane/memory_lane_service.dart';import 'package:photos/services/memory_share_service.dart';import "package:photos/services/notification_service.dart";import "package:photos/services/photos_contacts_service.dart";+import "package:photos/services/process_activity.dart";import 'package:photos/services/push_service.dart';import 'package:photos/services/search_service.dart';import 'package:photos/services/social_notification_coordinator.dart';@@ -70,14 +72,16 @@ import 'package:shared_preferences/shared_preferences.dart';final _logger = Logger("main");-const kLastBGTaskHeartBeatTime = "bg_task_hb_time";-const kLastFGTaskHeartBeatTime = "fg_task_hb_time";const kHeartBeatFrequency = Duration(seconds: 1);const kFGSyncFrequency = Duration(minutes: 5);const kFGHomeWidgetSyncFrequency = Duration(minutes: 15);const kBGTaskTimeout = Duration(seconds: 28);const kBGPushTimeout = Duration(seconds: 28);-const kFGTaskDeathTimeoutInMicroseconds = 5000000;+// ML self-stops before the platform hard-kills the BG engine (kBGTaskTimeout+// on iOS, the ~10-minute WorkManager system stop on Android), leaving margin+// to drain in-flight work and release the process lock cleanly.+const kBGTaskMLSelfStopIOS = Duration(seconds: 26);+const kBGTaskMLSelfStopAndroid = Duration(minutes: 9);bool isProcessBg = true;bool _stopHearBeat = false;bool _isSyncInitialized = false;@@ -242,27 +246,41 @@ Future<void> runBackgroundTask(TimeLogger tlog, {String mode = 'normal',}) async {- // Check if foreground is recently active to avoid conflicts- final isRunningInFG = await _isRunningInForeground();+ // Created at task start so a stop that fires before ML begins stays+ // latched for the whole task.+ final mlRunControl = MlRunControl();+ final mlSelfStopTimer = Timer(+ Platform.isIOS ? kBGTaskMLSelfStopIOS : kBGTaskMLSelfStopAndroid,+ () => mlRunControl.requestStop(MlStopReason.backgroundDeadline),+ );- // If FG was active in last 30 seconds, skip BG work- if (isRunningInFG) {- _logger.info(- "[BG TASK] Foreground recently active, skipping background work",- );- return;- }+ try {+ // Check if foreground is recently active to avoid conflicts+ final isRunningInFG = await isForegroundEngineActive();- _logger.info(- "[BG TASK] No recent foreground activity, proceeding with background work",- );+ // If FG was active in the last 5 seconds, skip BG work+ if (isRunningInFG) {+ _logger.info(+ "[BG TASK] Foreground recently active, skipping background work",+ );+ return;+ }- // Mark BG as active+ _logger.info(+ "[BG TASK] No recent foreground activity, proceeding with background work",+ );- await _runMinimally(taskId, tlog);+ await _runMinimally(taskId, tlog, mlRunControl);+ } finally {+ mlSelfStopTimer.cancel();+ }}-Future<void> _runMinimally(String taskId, TimeLogger tlog) async {+Future<void> _runMinimally(+ String taskId,+ TimeLogger tlog,+ MlRunControl mlRunControl,+) async {try {final PackageInfo packageInfo = await PackageInfo.fromPlatform();final SharedPreferences prefs = await SharedPreferences.getInstance();@@ -355,7 +373,11 @@ Future<void> _runMinimally(String taskId, TimeLogger tlog) async {await MLService.instance.init();PersonService.init(entityService, MLDataDB.instance, prefs);mlRunStarted = true;- await MLService.instance.runAllML(force: false);+ final disposition = await MLService.instance.runAllML(+ force: false,+ control: mlRunControl,+ );+ _logger.info("[BG TASK] ML run disposition: ${disposition.name}");} finally {if (!mlRunStarted) {controller.releaseCompute(ml: true);@@ -685,19 +707,8 @@ Future<void> _scheduleFGSync(String caller) async {});}-Future<bool> _isRunningInForeground() async {- final prefs = await SharedPreferences.getInstance();- await prefs.reload();- final currentTime = DateTime.now().microsecondsSinceEpoch;- final lastFGHeartBeatTime = DateTime.fromMicrosecondsSinceEpoch(- prefs.getInt(kLastFGTaskHeartBeatTime) ?? 0,- );- return lastFGHeartBeatTime.microsecondsSinceEpoch >- (currentTime - kFGTaskDeathTimeoutInMicroseconds);-}-Future<void> _handleBackgroundPush(Object message) async {- final bool isRunningInFG = await _isRunningInForeground(); // hb+ final bool isRunningInFG = await isForegroundEngineActive(); // hbfinal bool isInForeground = AppLifecycleService.instance.isForeground;if (isRunningInFG) {_logger.info(@@ -733,7 +744,7 @@ Future<void> _handleBackgroundPush(Object message) async {}Future<void> _logFGHeartBeatInfo(SharedPreferences prefs) async {- final bool isRunningInFG = await _isRunningInForeground();+ final bool isRunningInFG = await isForegroundEngineActive();await prefs.reload();final lastFGTaskHeartBeatTime = prefs.getInt(kLastFGTaskHeartBeatTime) ?? 0;final String lastRun = lastFGTaskHeartBeatTime == 0diff --git a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart b/mobile/apps/photos/lib/module/upload/service/file_uploader.dartindex ec7b934751..5444c335e6 100644--- a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart+++ b/mobile/apps/photos/lib/module/upload/service/file_uploader.dart@@ -20,7 +20,7 @@ import "package:photos/events/file_uploaded_event.dart";import 'package:photos/events/files_updated_event.dart';import 'package:photos/events/local_photos_updated_event.dart';import "package:photos/gateways/files/file_upload_gateway.dart";-import "package:photos/main.dart" show isProcessBg, kLastBGTaskHeartBeatTime;+import "package:photos/main.dart" show isProcessBg;import "package:photos/models/backup/backup_item.dart";import 'package:photos/models/file/file.dart';import 'package:photos/models/file/file_type.dart';@@ -38,6 +38,8 @@ import "package:photos/service_locator.dart";import "package:photos/services/account/user_service.dart";import 'package:photos/services/collections_service.dart';import 'package:photos/services/file_magic_service.dart';+import "package:photos/services/process_activity.dart"+ show kLastBGTaskHeartBeatTime;import 'package:photos/services/sync/local_sync_service.dart';import 'package:photos/services/sync/sync_service.dart';import "package:photos/utils/device_storage_error.dart";diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dartindex 95cc9fe364..72d70488aa 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart@@ -28,8 +28,11 @@ import "package:photos/services/machine_learning/face_ml/face_detection/detectioimport "package:photos/services/machine_learning/face_ml/person/person_service.dart";import "package:photos/services/machine_learning/ml_indexing_isolate.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";+import "package:photos/services/machine_learning/ml_process_lock.dart";import "package:photos/services/machine_learning/ml_result.dart";+import "package:photos/services/machine_learning/ml_run_control.dart";import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";+import "package:photos/services/process_activity.dart";import "package:photos/services/search_service.dart";import "package:photos/services/video_preview_service.dart";import "package:photos/utils/isolate/isolate_operations.dart";@@ -61,11 +64,17 @@ class MLService {bool _clusteringIsHappening = false;bool _mlControllerStatus = false;bool _isIndexingOrClusteringRunning = false;- bool _isRunningML = false;- bool _shouldPauseIndexingAndClustering = false;+ MlRunControl? _activeRunControl;+ Timer? _bgYieldPollTimer;+ Timer? _deniedRunRetryTimer;Timer? _predownloadLocalModelsTimer;static const _kPredownloadLocalModelsDelay = Duration(seconds: 10);+ static const _kBgYieldPollInterval = Duration(seconds: 3);+ static const _kDeniedRunRetryDelay = Duration(seconds: 15);++ bool get _isRunningML =>+ MlProcessLock.instance.activeOperation == MlOperation.fullRun;bool get isRunningML =>_isRunningML || memoriesCacheService.isUpdatingMemories;@@ -122,16 +131,9 @@ class MLService {_mlControllerStatus = event.shouldRun;if (_mlControllerStatus) {- if (_shouldPauseIndexingAndClustering) {- _cancelPauseIndexingAndClustering();- _logger.info(- "MLController allowed running ML, faces indexing undoing previous pause",- );- } else {- _logger.info(- "MLController allowed running ML, faces indexing starting",- );- }+ _logger.info(+ "MLController allowed running ML, faces indexing starting",+ );// Background start is driven manually from _runMinimally to avoid// duplicate runAllML invocations in the same cycle.if (!isProcessBg) {@@ -139,9 +141,9 @@ class MLService {}} else {_logger.info(- "MLController stopped running ML, faces indexing will be paused (unless it's fetching embeddings)",+ "MLController stopped running ML, stopping any active ML run",);- pauseIndexingAndClustering();+ stopActiveRun(MlStopReason.controller);}});_syncMlControllerStatusForBg();@@ -247,18 +249,30 @@ class MLService {);return existing;}- final future = _runOwnedRemoteHydrationSafely(- reason: reason,- skipHydrationIfCandidateFileCountAtMost:- skipHydrationIfCandidateFileCountAtMost,+ final attempt = await MlProcessLock.instance.tryRunExclusive(+ MlOperation.startupRemoteHydration,+ () async {+ final future = _runOwnedRemoteHydrationSafely(+ reason: reason,+ skipHydrationIfCandidateFileCountAtMost:+ skipHydrationIfCandidateFileCountAtMost,+ );+ _ownedRemoteHydrationFuture = future;+ try {+ await future;+ } finally {+ if (identical(_ownedRemoteHydrationFuture, future)) {+ _ownedRemoteHydrationFuture = null;+ }+ }+ },+ background: isProcessBg,);- _ownedRemoteHydrationFuture = future;- try {- await future;- } finally {- if (identical(_ownedRemoteHydrationFuture, future)) {- _ownedRemoteHydrationFuture = null;- }+ if (attempt != MlLockAttempt.ran) {+ _logger.info(+ "Skipping owned remote ML hydration ($reason): ml process lock "+ "denied (${attempt.name})",+ );}}@@ -266,6 +280,7 @@ class MLService {required String reason,int? skipHydrationIfCandidateFileCountAtMost,}) async {+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");try {await _hydrateRemoteEmbeddingsForOwnedFilesInternal(reason: reason,@@ -307,25 +322,6 @@ class MLService {);}- Future<void> _waitForOwnedRemoteHydrationIfRunning() async {- final existing = _ownedRemoteHydrationFuture;- if (existing == null) {- return;- }- _logger.info(- "Waiting for owned remote ML hydration to finish before indexing",- );- try {- await existing;- } catch (e, s) {- _logger.warning(- "Owned remote ML hydration failed while indexing was waiting, continuing",- e,- s,- );- }- }-void _schedulePredownloadLocalModels() {if (isProcessBg || _predownloadLocalModelsTimer?.isActive == true) {return;@@ -355,30 +351,98 @@ class MLService {await personFeedbackService.syncPersonFeedback();}- Future<void> runAllML({bool force = false}) async {+ Future<MlRunDisposition> runAllML({+ bool force = false,+ MlRunControl? control,+ }) async {+ final runControl = control ?? MlRunControl();+ // A latched stop always wins, force included.+ if (runControl.stopRequested) {+ _logger.info(+ "runAllML skipped, stop already requested "+ "(${runControl.stopReason?.name})",+ );+ return MlRunDisposition.stopped;+ }if (_isRunningML) {_logger.info("runAllML called while already running, skipping");- return;+ return MlRunDisposition.denied;}+ if (!hasGrantedMLConsent) {+ _logger.info("runAllML called without ML consent, skipping");+ return MlRunDisposition.denied;+ }+ final MLMode mode = isLocalGalleryMode+ ? MLMode.localGallery+ : MLMode.enteGallery;+ final mlDataDB = _dbForMode(mode);+ if (force) {+ _mlControllerStatus = true;+ }+ if (!_canRunMLFunction(function: "AllML") && !force) {+ return MlRunDisposition.denied;+ }+ if (!force && !computeController.requestCompute(ml: true)) {+ return MlRunDisposition.denied;+ }+ var disposition = MlRunDisposition.completed;try {- if (!hasGrantedMLConsent) {- _logger.info("runAllML called without ML consent, skipping");- return;- }- final MLMode mode = isLocalGalleryMode- ? MLMode.localGallery- : MLMode.enteGallery;- final mlDataDB = _dbForMode(mode);- if (force) {- _mlControllerStatus = true;+ final attempt = await MlProcessLock.instance.tryRunExclusive(+ MlOperation.fullRun,+ () async {+ disposition = await _runAllMLProtected(+ mode: mode,+ mlDataDB: mlDataDB,+ force: force,+ control: runControl,+ );+ },+ background: isProcessBg,+ );+ if (attempt != MlLockAttempt.ran) {+ _logger.info("runAllML denied the ml process lock (${attempt.name})");+ disposition = MlRunDisposition.denied;+ _scheduleDeniedRunRetry();}- if (!_canRunMLFunction(function: "AllML") && !force) return;- if (!force && !computeController.requestCompute(ml: true)) return;- _isRunningML = true;+ } catch (e, s) {+ _logger.severe("runAllML failed", e, s);+ disposition = MlRunDisposition.failed;+ } finally {+ computeController.releaseCompute(ml: true);+ }+ return disposition;+ }++ // FG-only: one coalesced retry after a lock denial, because the natural+ // ComputeControlEvent re-trigger only fires on an interaction or health+ // transition β without this, an untouched device would stall until the+ // next user interaction. Preconditions are re-validated when it fires.+ void _scheduleDeniedRunRetry() {+ if (isProcessBg) return;+ if (_deniedRunRetryTimer != null) return;+ _deniedRunRetryTimer = Timer(_kDeniedRunRetryDelay, () {+ _deniedRunRetryTimer = null;+ if (!hasGrantedMLConsent) return;+ triggerML();+ });+ }++ Future<MlRunDisposition> _runAllMLProtected({+ required MLMode mode,+ required MLDataDB mlDataDB,+ required bool force,+ required MlRunControl control,+ }) async {+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");+ _installRunControl(control);+ try {await sync();+ if (control.stopRequested) {+ return _stoppedDisposition(control, "after sync");+ }if (_hasModeChanged(mode)) {_logger.info("App mode changed during ML run, stopping");- return;+ return MlRunDisposition.stopped;}final int unclusteredFacesCount = await mlDataDB@@ -387,81 +451,142 @@ class MLService {_logger.info("There are $unclusteredFacesCount unclustered faces, doing clustering first",);- await clusterAllImages();+ await _clusterAllImages(control: control);+ }+ if (control.stopRequested) {+ return _stoppedDisposition(control, "after initial clustering");}if (_mlControllerStatus == true) {if (_hasModeChanged(mode)) {_logger.info("App mode changed during ML run, stopping");- return;+ return MlRunDisposition.stopped;}// Refresh discover/memories caches before indexing using the same// path in foreground and background runs.magicCacheService.updateCache(forced: force).ignore();memoriesCacheService.updateCache(forced: force).ignore();}+ if (control.stopRequested) {+ return _stoppedDisposition(control, "before indexing");+ }if (canFetch()) {- await fetchAndIndexAllImages(mode: mode);+ await _fetchAndIndexAllImages(mode: mode, control: control);}if (_hasModeChanged(mode)) {_logger.info("App mode changed during ML run, stopping");- return;+ return MlRunDisposition.stopped;+ }+ if (control.stopRequested) {+ return _stoppedDisposition(control, "before final clustering");}if ((await mlDataDB.getUnclusteredFaceCount()) > 0) {- await clusterAllImages();+ await _clusterAllImages(control: control);+ }+ if (control.stopRequested) {+ return _stoppedDisposition(control, "before post-run cache scheduling");}if (_mlControllerStatus == true) {if (_hasModeChanged(mode)) {_logger.info("App mode changed during ML run, stopping");- return;+ return MlRunDisposition.stopped;}// Persist refreshed caches after ML so foreground can pick them up// on the next resume, even when the work ran headlessly in background.magicCacheService.updateCache().ignore();memoriesCacheService.updateCache(forced: force).ignore();}- } catch (e, s) {- _logger.severe("runAllML failed", e, s);- rethrow;+ return MlRunDisposition.completed;} finally {_logger.info("ML finished running");- _isRunningML = false;- computeController.releaseCompute(ml: true);+ _clearRunControl(control);if (!isProcessBg) {VideoPreviewService.instance.queueFiles();}}}- void triggerML() {- if (_mlControllerStatus &&- !_isIndexingOrClusteringRunning &&- !_isRunningML) {- unawaited(runAllML());- }+ MlRunDisposition _stoppedDisposition(MlRunControl control, String where) {+ _logger.info("ML run stopped $where (${control.stopReason?.name})");+ return MlRunDisposition.stopped;}- void pauseIndexingAndClustering() {- if (_isIndexingOrClusteringRunning) {- _shouldPauseIndexingAndClustering = true;+ void _installRunControl(MlRunControl control) {+ _activeRunControl = control;+ control.attachOnStop(() {MLIndexingIsolate.instance.shouldPauseIndexingAndClustering = true;+ });+ // BG always yields to FG: while a BG run holds the lock, watch for the+ // foreground engine's heartbeat and wind down as soon as it appears.+ if (isProcessBg) {+ _bgYieldPollTimer = Timer.periodic(_kBgYieldPollInterval, (_) async {+ if (control.stopRequested) return;+ if (await isForegroundEngineActive()) {+ control.requestStop(MlStopReason.foregroundActive);+ }+ });}}- void _cancelPauseIndexingAndClustering() {- _shouldPauseIndexingAndClustering = false;+ // The isolate stop flag is cleared here, from the outer run cleanup, and+ // nowhere else: a durable stop must survive stage-level finally blocks.+ void _clearRunControl(MlRunControl control) {+ _bgYieldPollTimer?.cancel();+ _bgYieldPollTimer = null;+ control.detachOnStop();+ if (identical(_activeRunControl, control)) {+ _activeRunControl = null;+ }MLIndexingIsolate.instance.shouldPauseIndexingAndClustering = false;}+ void triggerML() {+ if (_mlControllerStatus && !MlProcessLock.instance.isBusy) {+ unawaited(runAllML());+ }+ }++ /// Requests a durable stop of the active ML run, if any. The run drains+ /// already-started work and does not begin further stages; a new run may+ /// start later through the normal triggers.+ void stopActiveRun(MlStopReason reason) {+ _activeRunControl?.requestStop(reason);+ }+/// Analyzes all the images in the user library with the latest ml version and stores the results in the database.////// This function first fetches from remote and checks if the image has already been analyzed/// with the lastest faceMlVersion and stored on remote or local database. If so, it skips the image.Future<void> fetchAndIndexAllImages({required MLMode mode}) async {- if (!_canRunMLFunction(function: "Indexing")) return;- if (mode == MLMode.enteGallery && !isLocalGalleryMode) {- await _waitForOwnedRemoteHydrationIfRunning();+ final control = MlRunControl();+ final attempt = await MlProcessLock.instance.tryRunExclusive(+ MlOperation.indexing,+ () async {+ _installRunControl(control);+ try {+ await _fetchAndIndexAllImages(mode: mode, control: control);+ } finally {+ _clearRunControl(control);+ }+ },+ background: isProcessBg,+ );+ if (attempt != MlLockAttempt.ran) {+ _logger.info(+ "fetchAndIndexAllImages denied the ml process lock (${attempt.name})",+ );}+ }++ Future<void> _fetchAndIndexAllImages({+ required MLMode mode,+ required MlRunControl control,+ }) async {+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");if (!_canRunMLFunction(function: "Indexing")) return;+ if (control.stopRequested) {+ _stoppedDisposition(control, "before indexing started");+ return;+ }bool rustRuntimePrepared = false;try {@@ -478,14 +603,17 @@ class MLService {int fileAnalyzedCount = 0;final Stopwatch stopwatch = Stopwatch()..start();- stream:+ bool stopRun = false;await for (final chunk in instructionStream) {- if ((isLocalGalleryMode ? MLMode.localGallery : MLMode.enteGallery) !=- mode) {+ if (_hasModeChanged(mode)) {_logger.info("App mode changed during indexing, stopping current ML run",);- break stream;+ break;+ }+ if (control.stopRequested) {+ _stoppedDisposition(control, "between indexing chunks");+ break;}if (!localSettings.isMLLocalIndexingEnabled) {if (rustRuntimePrepared) {@@ -498,7 +626,7 @@ class MLService {_logger.info('stopping indexing because user is not connected to wifi and in online mode',);- break stream;+ break;} else {await MLModelDownloadService.instance.ensureModelsDownloaded(onlyIndexingModels: true,@@ -510,25 +638,31 @@ class MLService {}final futures = <Future<bool>>[];for (final instruction in chunk) {- if ((isLocalGalleryMode ? MLMode.localGallery : MLMode.enteGallery) !=- mode) {+ if (_hasModeChanged(mode)) {_logger.info("App mode changed during indexing, stopping current ML run",);- break stream;+ stopRun = true;+ break;}- if (_shouldPauseIndexingAndClustering) {- _logger.info("indexAllImages() was paused, stopping");- break stream;+ if (control.stopRequested) {+ _stoppedDisposition(control, "between indexing instructions");+ stopRun = true;+ break;}- futures.add(processImage(instruction));+ futures.add(_processImage(instruction));}+ // Drain: work that was already started must complete (and commit)+ // before this run can exit and release the process lock.final awaitedFutures = await Future.wait(futures);final sumFutures = awaitedFutures.fold<int>(0,(previousValue, element) => previousValue + (element ? 1 : 0),);fileAnalyzedCount += sumFutures;+ if (stopRun) {+ break;+ }}if (fileAnalyzedCount > 0) {magicCacheService.queueUpdate('fileIndexed');@@ -547,7 +681,6 @@ class MLService {await MLIndexingIsolate.instance.releaseRustRuntime();MLModelDownloadService.instance.invalidateModelDownloadCache();_isIndexingOrClusteringRunning = false;- _cancelPauseIndexingAndClustering();}}@@ -555,6 +688,40 @@ class MLService {bool clusterInBuckets = true,bool force = false,}) async {+ final control = MlRunControl();+ final attempt = await MlProcessLock.instance.tryRunExclusive(+ MlOperation.clustering,+ () async {+ _installRunControl(control);+ try {+ await _clusterAllImages(+ clusterInBuckets: clusterInBuckets,+ force: force,+ control: control,+ );+ } finally {+ _clearRunControl(control);+ }+ },+ background: isProcessBg,+ );+ if (attempt != MlLockAttempt.ran) {+ _logger.info(+ "clusterAllImages denied the ml process lock (${attempt.name})",+ );+ }+ }++ Future<void> _clusterAllImages({+ required MlRunControl control,+ bool clusterInBuckets = true,+ bool force = false,+ }) async {+ assert(MlProcessLock.instance.isBusy, "ml funnel must be held");+ if (control.stopRequested) {+ _stoppedDisposition(control, "before clustering started");+ return;+ }if (!_canRunMLFunction(function: "Clustering") && !force) return;if (_clusteringIsHappening) {_logger.info("clusterAllImages() is already running, returning");@@ -628,10 +795,8 @@ class MLService {int bucket = 1;while (true) {- if (_shouldPauseIndexingAndClustering) {- _logger.info(- "MLController does not allow running ML, stopping before clustering bucket $bucket",- );+ if (control.stopRequested) {+ _stoppedDisposition(control, "before clustering bucket $bucket");break;}if (offset > allFaceInfoForClustering.length - 1) {@@ -746,11 +911,10 @@ class MLService {} finally {_clusteringIsHappening = false;_isIndexingOrClusteringRunning = false;- _cancelPauseIndexingAndClustering();}}- Future<bool> processImage(FileMLInstruction instruction) async {+ Future<bool> _processImage(FileMLInstruction instruction) async {bool actuallyRanML = false;final mlDataDB = _dbForMode(instruction.mode);@@ -768,9 +932,9 @@ class MLService {instruction,filePath,);- // Check if there's no result simply because MLController paused indexing+ // Check if there's no result simply because the run was stoppedif (result == null) {- if (!_shouldPauseIndexingAndClustering &&+ if (!(_activeRunControl?.stopRequested ?? false) &&!MLIndexingIsolate.instance.shouldPauseIndexingAndClustering) {_logger.severe("Failed to analyze image with fileID: ${instruction.fileKey}",@@ -918,7 +1082,7 @@ class MLService {final int? size = instruction.file.fileSize;final fileType = instruction.file.fileType;if (e is RustCorruptModelException) {- pauseIndexingAndClustering();+ stopActiveRun(MlStopReason.corruptModel);_logger.severe("Stopping ML indexing for fileID ${instruction.fileKey} ""(format $format, type $fileType, size $size) because Rust ML "@@ -1040,14 +1204,6 @@ class MLService {_logStatus();return false;}- if (_shouldPauseIndexingAndClustering) {- // This should ideally not be triggered, because one of the above should be triggered instead.- _logger.warning(- "Cannot run $function because indexing and clustering is being paused",- );- _logStatus();- return false;- }return true;}@@ -1081,7 +1237,8 @@ class MLService {Local indexing: ${localSettings.isMLLocalIndexingEnabled}canRunMLController: $_mlControllerStatusisIndexingOrClusteringRunning: $_isIndexingOrClusteringRunning- shouldPauseIndexingAndClustering: $_shouldPauseIndexingAndClustering+ activeOperation: ${MlProcessLock.instance.activeOperation?.name}+ stopRequested: ${_activeRunControl?.stopRequested} (reason: ${_activeRunControl?.stopReason?.name})debugIndexingDisabled: $debugIndexingDisabled''';_logger.info(status);diff --git a/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart b/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dartindex 98bd2af827..cb0b9826ff 100644--- a/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart+++ b/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart@@ -1,5 +1,6 @@import "dart:async";+import "package:ente_photos_platform/ente_photos_platform.dart";import "package:ente_pure_utils/ente_pure_utils.dart";import "package:flutter/material.dart";import "package:hugeicons/hugeicons.dart";@@ -15,6 +16,8 @@ import "package:photos/services/machine_learning/face_ml/face_clustering/face_climport "package:photos/services/machine_learning/face_ml/person/person_service.dart";import "package:photos/services/machine_learning/ml_indexing_isolate.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";+import "package:photos/services/machine_learning/ml_process_lock.dart";+import "package:photos/services/machine_learning/ml_run_control.dart";import "package:photos/services/machine_learning/ml_service.dart";import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";import "package:photos/theme/ente_theme.dart";@@ -248,6 +251,16 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {trailingIconIsMuted: true,onTap: () async => _onTriggerClustering(context),),+ MenuItemWidgetNew(+ title: "ML process lock state",+ leadingIconWidget: _buildIconWidget(+ context,+ HugeIcons.strokeRoundedLock,+ ),+ trailingIcon: Icons.chevron_right_outlined,+ trailingIconIsMuted: true,+ onTap: () async => _onShowProcessLockState(context),+ ),MenuItemWidgetNew(title: "Update discover",leadingIconWidget: _buildIconWidget(@@ -637,7 +650,7 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {await setMLConsent(mlConsent);logger.info('ML consent turned ${mlConsent ? 'on' : 'off'}');if (!mlConsent) {- MLService.instance.pauseIndexingAndClustering();+ MLService.instance.stopActiveRun(MlStopReason.manual);unawaited(MLIndexingIsolate.instance.cleanupLocalIndexingModels());} else {await MLService.instance.init();@@ -677,7 +690,7 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {if (localIndexing) {unawaited(MLService.instance.runAllML(force: true));} else {- MLService.instance.pauseIndexingAndClustering();+ MLService.instance.stopActiveRun(MlStopReason.manual);unawaited(MLIndexingIsolate.instance.cleanupLocalIndexingModels());}if (mounted) {@@ -690,7 +703,7 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {MLService.instance.debugIndexingDisabled =!MLService.instance.debugIndexingDisabled;if (MLService.instance.debugIndexingDisabled) {- MLService.instance.pauseIndexingAndClustering();+ MLService.instance.stopActiveRun(MlStopReason.manual);} else {unawaited(MLService.instance.runAllML());}@@ -789,6 +802,23 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {}}+ Future<void> _onShowProcessLockState(BuildContext context) async {+ try {+ final owner = await ProcessLockClient.instance.state(name: "ml");+ final funnelOp = MlProcessLock.instance.activeOperation?.name;+ final native = owner == null+ ? "free"+ : "held by ${owner.origin}/${owner.operation} "+ "for ${owner.heldFor.inSeconds}s";+ if (!context.mounted) return;+ showShortToast(context, "Lock: $native Β· funnel: ${funnelOp ?? 'idle'}");+ } catch (e, s) {+ logger.warning('fetching ml process lock state failed', e, s);+ if (!context.mounted) return;+ await showGenericErrorDialog(context: context, error: e);+ }+ }+Future<void> _onTriggerClustering(BuildContext context) async {try {await PersonService.instance.fetchRemoteClusterFeedback();diff --git a/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dart b/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dartindex 1cbe38f016..ae0122a63b 100644--- a/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dart+++ b/mobile/apps/photos/lib/ui/settings/ml/machine_learning_settings_page.dart@@ -11,6 +11,7 @@ import "package:photos/service_locator.dart";import "package:photos/services/machine_learning/ml_indexing_isolate.dart";import "package:photos/services/machine_learning/ml_model_assets.dart";import "package:photos/services/machine_learning/ml_model_download_service.dart";+import "package:photos/services/machine_learning/ml_run_control.dart";import "package:photos/services/machine_learning/ml_service.dart";import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";import "package:photos/services/remote_assets_service.dart";@@ -185,7 +186,7 @@ class _MachineLearningSettingsPageStatememoriesCacheService.queueUpdateCache();Bus.instance.fire(NotificationEvent());if (!mlConsent) {- MLService.instance.pauseIndexingAndClustering();+ MLService.instance.stopActiveRun(MlStopReason.manual);unawaited(MLIndexingIsolate.instance.cleanupLocalIndexingModels());if (oldMlEnabled && !newMlEnabled) {await memoriesCacheService.purgeMlOnlyMemoriesFromCache();@@ -313,7 +314,7 @@ class _MachineLearningSettingsPageStateif (localIndexing) {unawaited(MLService.instance.runAllML(force: true));} else {- MLService.instance.pauseIndexingAndClustering();+ MLService.instance.stopActiveRun(MlStopReason.manual);unawaited(MLIndexingIsolate.instance.cleanupLocalIndexingModels(),);diff --git a/mobile/apps/photos/lib/utils/bg_task_utils.dart b/mobile/apps/photos/lib/utils/bg_task_utils.dartindex 506e932199..b5145bb880 100644--- a/mobile/apps/photos/lib/utils/bg_task_utils.dart+++ b/mobile/apps/photos/lib/utils/bg_task_utils.dart@@ -7,6 +7,7 @@ import "package:permission_handler/permission_handler.dart";import "package:photos/db/upload_locks_db.dart";import "package:photos/main.dart";import "package:photos/module/upload/service/file_uploader.dart";+import "package:photos/services/process_activity.dart";import "package:shared_preferences/shared_preferences.dart";import "package:workmanager/workmanager.dart" as workmanager;