Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mobile][photos] Round out ML lock ergonomics

laurenspriem6ff719a5fd2c
diff --git a/mobile/apps/photos/lib/db/ml/db.dart b/mobile/apps/photos/lib/db/ml/db.dart
index a1771ad215..034bdda334 100644
--- a/mobile/apps/photos/lib/db/ml/db.dart
+++ b/mobile/apps/photos/lib/db/ml/db.dart
@@ -50,6 +50,9 @@ import "package:synchronized/synchronized.dart";
class MLDataDB with SqlDbBase implements IMLDataDB<int> {
static final Logger _logger = Logger("MLDataDB");
static const int _maxSqlBindParamsPerQuery = 10000;
+ // Migrations are opportunistic and retried by their callers; logout's
+ // clearData intentionally waits unbounded because it must complete.
+ static const _kMigrationLockWaitDeadline = Duration(minutes: 2);
static Logger get logger => _logger;
@@ -1865,6 +1868,7 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
await _runMlOperationExclusive(
MlOperation.clusterCentroidVectorMigration,
() => _checkMigrateFillClusterCentroidVectorDB(force: force),
+ waitDeadline: _kMigrationLockWaitDeadline,
);
}
@@ -2174,6 +2178,7 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
await _runMlOperationExclusive(
MlOperation.clipVectorMigration,
() => _checkMigrateFillClipVectorDB(force: force),
+ waitDeadline: _kMigrationLockWaitDeadline,
);
}
@@ -2324,13 +2329,15 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
Future<void> _runMlOperationExclusive(
MlOperation operation,
- Future<void> Function() body,
- ) async {
+ Future<void> Function() body, {
+ Duration? waitDeadline,
+ }) async {
final attempt = await MlProcessLock.instance.tryRunExclusive(
operation,
body,
background: isProcessBg,
waitForAvailability: true,
+ waitDeadline: waitDeadline,
);
if (attempt != MlLockAttempt.ran) {
throw StateError(
diff --git a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart b/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
index 5444c335e6..a6152190f2 100644
--- a/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
+++ b/mobile/apps/photos/lib/module/upload/service/file_uploader.dart
@@ -39,7 +39,7 @@ import "package:photos/services/account/user_service.dart";
import 'package:photos/services/collections_service.dart';
import 'package:photos/services/file_magic_service.dart';
import "package:photos/services/process_activity.dart"
- show kLastBGTaskHeartBeatTime;
+ show isBackgroundEngineActive;
import 'package:photos/services/sync/local_sync_service.dart';
import 'package:photos/services/sync/sync_service.dart';
import "package:photos/utils/device_storage_error.dart";
@@ -72,7 +72,6 @@ class FileUploader {
);
final _existingUploadResolver = ExistingUploadResolver.forApp();
final kSafeBufferForLockExpiry = const Duration(hours: 4).inMicroseconds;
- final kBGTaskDeathTimeout = const Duration(seconds: 5).inMicroseconds;
Map<String, BackupItem> get allBackups => _queue.backupItems;
/// Returns true if any file uploads are currently in progress
@@ -106,11 +105,7 @@ class FileUploader {
currentTime - kSafeBufferForLockExpiry,
);
if (!isBackground) {
- await _prefs.reload();
- final lastBGTaskHeartBeatTime =
- _prefs.getInt(kLastBGTaskHeartBeatTime) ?? 0;
- final isBGTaskDead =
- lastBGTaskHeartBeatTime < (currentTime - kBGTaskDeathTimeout);
+ final isBGTaskDead = !await isBackgroundEngineActive();
if (isBGTaskDead) {
await _uploadLocks.releaseLocksAcquiredByOwnerBefore(
ProcessType.background.toString(),
@@ -118,9 +113,7 @@ class FileUploader {
);
_logger.info("BG task was found dead, cleared all locks");
} else {
- _logger.info(
- "BG task is alive, not clearing locks ${DateTime.fromMicrosecondsSinceEpoch(lastBGTaskHeartBeatTime)}",
- );
+ _logger.info("BG task is alive, not clearing locks");
}
if (!_hasStartedBackgroundUploadPolling) {
_hasStartedBackgroundUploadPolling = true;
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart b/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart
index 4bd8c9ea5b..8bf7bbefd8 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_process_lock.dart
@@ -19,6 +19,7 @@ enum MlLockAttempt {
deniedFunnelBusy,
deniedByOtherEngine,
deniedChannelError,
+ deniedTimeout,
}
enum MlRunDisposition { completed, denied, stopped, failed }
@@ -50,12 +51,14 @@ class MlProcessLock {
/// protected ML work it started before returning. Errors from [body]
/// propagate after release. [background] labels this engine's origin.
/// [waitForAvailability] waits for both the local funnel and the other
- /// engine instead of returning a busy denial.
+ /// engine instead of returning a busy denial, bounded by [waitDeadline]
+ /// per wait when given.
Future<MlLockAttempt> tryRunExclusive(
MlOperation operation,
Future<void> Function() body, {
required bool background,
bool waitForAvailability = false,
+ Duration? waitDeadline,
}) async {
if (!waitForAvailability && _funnel.locked) {
_logger.info(
@@ -63,7 +66,33 @@ class MlProcessLock {
);
return MlLockAttempt.deniedFunnelBusy;
}
- return _funnel.synchronized(() async {
+ final nativeDeadline = waitDeadline == null
+ ? null
+ : DateTime.now().add(waitDeadline);
+ try {
+ return await _runSynchronized(
+ operation,
+ body,
+ background: background,
+ waitForAvailability: waitForAvailability,
+ nativeDeadline: nativeDeadline,
+ funnelTimeout: waitForAvailability ? waitDeadline : null,
+ );
+ } on TimeoutException {
+ _logger.warning("Timed out waiting for the funnel (${operation.name})");
+ return MlLockAttempt.deniedTimeout;
+ }
+ }
+
+ Future<MlLockAttempt> _runSynchronized(
+ MlOperation operation,
+ Future<void> Function() body, {
+ required bool background,
+ required bool waitForAvailability,
+ required DateTime? nativeDeadline,
+ required Duration? funnelTimeout,
+ }) {
+ return _funnel.synchronized(timeout: funnelTimeout, () async {
_activeOperation = operation;
try {
final origin = background ? "bg" : "fg";
@@ -89,6 +118,14 @@ class MlProcessLock {
unawaited(_logOwner(operation));
return MlLockAttempt.deniedByOtherEngine;
}
+ if (nativeDeadline != null &&
+ DateTime.now().isAfter(nativeDeadline)) {
+ _logger.warning(
+ "Timed out waiting for the ml lock (${operation.name})",
+ );
+ unawaited(_logOwner(operation));
+ return MlLockAttempt.deniedTimeout;
+ }
await Future<void>.delayed(_availabilityPollInterval);
}
_logger.info("Acquired ml lock ($origin/${operation.name})");
@@ -118,6 +155,20 @@ class MlProcessLock {
});
}
+ Future<String> describeState() async {
+ String native;
+ try {
+ final owner = await ProcessLockClient.instance.state(name: _lockName);
+ native = owner == null
+ ? "free"
+ : "held by ${owner.origin}/${owner.operation} "
+ "for ${owner.heldFor.inSeconds}s";
+ } catch (e) {
+ native = "unavailable ($e)";
+ }
+ return "Lock: $native Β· funnel: ${_activeOperation?.name ?? 'idle'}";
+ }
+
Future<void> _logOwner(MlOperation denied) async {
try {
final owner = await ProcessLockClient.instance.state(name: _lockName);
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_run_control.dart b/mobile/apps/photos/lib/services/machine_learning/ml_run_control.dart
index a8b17a1b83..6ae526ee8e 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_run_control.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_run_control.dart
@@ -5,6 +5,7 @@ enum MlStopReason {
backgroundDeadline,
controller,
logout,
+ modeChanged,
manual,
corruptModel,
}
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
index ef1904b580..f743fcd646 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
@@ -11,6 +11,7 @@ import "package:photos/db/files_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/db/ml/db_pet_model_mappers.dart";
import "package:photos/db/offline_files_db.dart";
+import "package:photos/events/app_mode_changed_event.dart";
import "package:photos/events/compute_control_event.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/main.dart";
@@ -96,11 +97,6 @@ class MLService {
: MLDataDB.instance;
}
- bool _hasModeChanged(MLMode mode) {
- return (isLocalGalleryMode ? MLMode.localGallery : MLMode.enteGallery) !=
- mode;
- }
-
/// Only call this function once at app startup, after that you can directly call [runAllML]
Future<void> init() async {
if (_isInitialized) {
@@ -146,6 +142,11 @@ class MLService {
stopActiveRun(MlStopReason.controller);
}
});
+ // Every setAppMode call site fires this event, so a latched stop fully
+ // replaces polled mode checks inside the pipeline.
+ Bus.instance.on<AppModeChangedEvent>().listen((_) {
+ stopActiveRun(MlStopReason.modeChanged);
+ });
_syncMlControllerStatusForBg();
_isInitialized = true;
@@ -450,10 +451,6 @@ class MLService {
if (control.stopRequested) {
return _stoppedDisposition(control, "after sync");
}
- if (_hasModeChanged(mode)) {
- _logger.info("App mode changed during ML run, stopping");
- return MlRunDisposition.stopped;
- }
final int unclusteredFacesCount = await mlDataDB
.getUnclusteredFaceCount();
@@ -467,10 +464,6 @@ class MLService {
return _stoppedDisposition(control, "after initial clustering");
}
if (_mlControllerStatus == true) {
- if (_hasModeChanged(mode)) {
- _logger.info("App mode changed during ML run, stopping");
- return MlRunDisposition.stopped;
- }
// Cache refreshes only read ML stores, so they may safely outlive
// this run and the process lock.
magicCacheService.updateCache(forced: force).ignore();
@@ -482,10 +475,6 @@ class MLService {
if (canFetch()) {
await _fetchAndIndexAllImages(mode: mode, control: control);
}
- if (_hasModeChanged(mode)) {
- _logger.info("App mode changed during ML run, stopping");
- return MlRunDisposition.stopped;
- }
if (control.stopRequested) {
return _stoppedDisposition(control, "before final clustering");
}
@@ -496,10 +485,6 @@ class MLService {
return _stoppedDisposition(control, "before post-run cache scheduling");
}
if (_mlControllerStatus == true) {
- if (_hasModeChanged(mode)) {
- _logger.info("App mode changed during ML run, stopping");
- return MlRunDisposition.stopped;
- }
// Persist refreshed caches after ML so foreground can pick them up
// on the next resume, even when the work ran headlessly in background.
magicCacheService.updateCache().ignore();
@@ -580,7 +565,7 @@ class MLService {
///
/// This function first fetches from remote and checks if the image has already been analyzed
/// with the lastest faceMlVersion and stored on remote or local database. If so, it skips the image.
- Future<void> fetchAndIndexAllImages({required MLMode mode}) async {
+ Future<MlLockAttempt> fetchAndIndexAllImages({required MLMode mode}) async {
final control = MlRunControl();
final attempt = await _runExclusiveWithControl(
MlOperation.indexing,
@@ -592,6 +577,7 @@ class MLService {
"fetchAndIndexAllImages denied the ml process lock (${attempt.name})",
);
}
+ return attempt;
}
Future<void> _fetchAndIndexAllImages({
@@ -622,12 +608,6 @@ class MLService {
bool stopRun = false;
await for (final chunk in instructionStream) {
- if (_hasModeChanged(mode)) {
- _logger.info(
- "App mode changed during indexing, stopping current ML run",
- );
- break;
- }
if (control.stopRequested) {
_stoppedDisposition(control, "between indexing chunks");
break;
@@ -655,13 +635,6 @@ class MLService {
}
final futures = <Future<bool>>[];
for (final instruction in chunk) {
- if (_hasModeChanged(mode)) {
- _logger.info(
- "App mode changed during indexing, stopping current ML run",
- );
- stopRun = true;
- break;
- }
if (control.stopRequested) {
_stoppedDisposition(control, "between indexing instructions");
stopRun = true;
@@ -701,7 +674,7 @@ class MLService {
}
}
- Future<void> clusterAllImages({
+ Future<MlLockAttempt> clusterAllImages({
bool clusterInBuckets = true,
bool force = false,
}) async {
@@ -720,6 +693,7 @@ class MLService {
"clusterAllImages denied the ml process lock (${attempt.name})",
);
}
+ return attempt;
}
Future<void> _clusterAllImages({
diff --git a/mobile/apps/photos/lib/services/process_activity.dart b/mobile/apps/photos/lib/services/process_activity.dart
index 2fb28bb62b..a52c36a41e 100644
--- a/mobile/apps/photos/lib/services/process_activity.dart
+++ b/mobile/apps/photos/lib/services/process_activity.dart
@@ -2,17 +2,22 @@ import "package:shared_preferences/shared_preferences.dart";
const kLastBGTaskHeartBeatTime = "bg_task_hb_time";
const kLastFGTaskHeartBeatTime = "fg_task_hb_time";
-const kFGTaskDeathTimeoutInMicroseconds = 5000000;
+const kEngineDeathTimeoutInMicroseconds = 5000000;
/// Whether the foreground engine has written its heartbeat within the last
/// five seconds. Reloads prefs so writes from the other engine are seen.
///
/// Heartbeats influence priority and yield latency only β€” never correctness.
-Future<bool> isForegroundEngineActive() async {
+Future<bool> isForegroundEngineActive() =>
+ _isEngineActive(kLastFGTaskHeartBeatTime);
+
+Future<bool> isBackgroundEngineActive() =>
+ _isEngineActive(kLastBGTaskHeartBeatTime);
+
+Future<bool> _isEngineActive(String heartBeatKey) async {
final prefs = await SharedPreferences.getInstance();
await prefs.reload();
final currentTime = DateTime.now().microsecondsSinceEpoch;
- final lastFGHeartBeatTime = prefs.getInt(kLastFGTaskHeartBeatTime) ?? 0;
- return lastFGHeartBeatTime >
- (currentTime - kFGTaskDeathTimeoutInMicroseconds);
+ final lastHeartBeatTime = prefs.getInt(heartBeatKey) ?? 0;
+ return lastHeartBeatTime > (currentTime - kEngineDeathTimeoutInMicroseconds);
}
diff --git a/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart b/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart
index 339ad05f69..b1b3f7d9cd 100644
--- a/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart
+++ b/mobile/apps/photos/lib/ui/settings/debug/ml_debug_settings_page.dart
@@ -1,6 +1,5 @@
import "dart:async";
-import "package:ente_photos_platform/ente_photos_platform.dart";
import "package:ente_pure_utils/ente_pure_utils.dart";
import "package:flutter/material.dart";
import "package:hugeicons/hugeicons.dart";
@@ -804,14 +803,9 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {
Future<void> _onShowProcessLockState(BuildContext context) async {
try {
- final owner = await ProcessLockClient.instance.state(name: "ml");
- final funnelOp = MlProcessLock.instance.activeOperation?.name;
- final native = owner == null
- ? "free"
- : "held by ${owner.origin}/${owner.operation} "
- "for ${owner.heldFor.inSeconds}s";
+ final description = await MlProcessLock.instance.describeState();
if (!context.mounted) return;
- showShortToast(context, "Lock: $native Β· funnel: ${funnelOp ?? 'idle'}");
+ showShortToast(context, description);
} catch (e, s) {
logger.warning('fetching ml process lock state failed', e, s);
if (!context.mounted) return;
@@ -823,9 +817,13 @@ class _MLDebugSettingsPageState extends State<MLDebugSettingsPage> {
try {
await PersonService.instance.sync();
MLService.instance.debugIndexingDisabled = false;
- await MLService.instance.clusterAllImages();
- Bus.instance.fire(PeopleChangedEvent());
+ final attempt = await MLService.instance.clusterAllImages();
if (!context.mounted) return;
+ if (attempt != MlLockAttempt.ran) {
+ showShortToast(context, "Denied (${attempt.name})");
+ return;
+ }
+ Bus.instance.fire(PeopleChangedEvent());
showShortToast(context, "Done");
} catch (e, s) {
logger.warning('clustering failed ', e, s);