π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][photos] Add Android background processing for ML
laurenspriemca4c87ce7da6
diff --git a/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.kt b/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.ktindex d74d1af180..641988ebd1 100644--- a/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.kt+++ b/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.kt@@ -6,6 +6,7 @@ import android.app.NotificationManagerimport android.content.Contextimport android.util.Logimport androidx.core.app.NotificationCompat+import androidx.core.content.editimport dev.fluttercommunity.workmanager.TaskDebugInfoimport dev.fluttercommunity.workmanager.TaskResultimport dev.fluttercommunity.workmanager.WorkmanagerDebug@@ -16,7 +17,7 @@ import kotlin.random.Randomclass EnteApplication : Application() {override fun onCreate() {super.onCreate()- WorkmanagerDebug.setCurrent(InternalUserWorkmanagerDebugHandler())+ WorkmanagerDebug.setCurrent(EnteWorkmanagerDebugHandler())ForegroundHeartbeat.install(this)}@@ -30,13 +31,18 @@ class EnteApplication : Application() {}}-private class InternalUserWorkmanagerDebugHandler : WorkmanagerDebug() {+private class EnteWorkmanagerDebugHandler : WorkmanagerDebug() {override fun onTaskStatusUpdate(context: Context,taskInfo: TaskDebugInfo,status: TaskStatus,result: TaskResult?,) {+ if (status == TaskStatus.STARTED || status == TaskStatus.RETRYING) {+ context+ .getSharedPreferences(EnteApplication.FLUTTER_SHARED_PREFERENCES, Context.MODE_PRIVATE)+ .edit { putLong("flutter.bg_task_start_${taskInfo.taskName}", taskInfo.startTime) }+ }if (!shouldEnableWorkmanagerDebugNotifications(context)) {return}diff --git a/mobile/apps/photos/lib/db/ml/base.dart b/mobile/apps/photos/lib/db/ml/base.dartindex 0439b0287d..35219442b8 100644--- a/mobile/apps/photos/lib/db/ml/base.dart+++ b/mobile/apps/photos/lib/db/ml/base.dart@@ -136,6 +136,7 @@ abstract class IMLDataDB<T> {Future<Map<int, int>> petIndexedFileIds({int minimumMlVersion});Future<int> getPetIndexedFileCount({int minimumMlVersion});Future<void> deletePetDataForFiles(List<int> fileIDs);+ Future<void> markPetFileIndexed(int fileID, int mlVersion);Future<Set<int>> getFullyIndexedFileIds({required bool includePets});diff --git a/mobile/apps/photos/lib/db/ml/db.dart b/mobile/apps/photos/lib/db/ml/db.dartindex 5aa5405076..99e73ce4ff 100644--- a/mobile/apps/photos/lib/db/ml/db.dart+++ b/mobile/apps/photos/lib/db/ml/db.dart@@ -206,6 +206,15 @@ class DartMLDataDB}}+ @override+ Future<void> markPetFileIndexed(int fileID, int mlVersion) async {+ final db = await asyncDB;+ await db.execute(+ 'UPDATE $petFacesTable SET $mlVersionColumn = ? WHERE $fileIDColumn = ?',+ [mlVersion, fileID],+ );+ }+@overrideFuture<void> bulkInsertPetBodies(List<DBPetBody> petBodies) async {final db = await asyncDB;diff --git a/mobile/apps/photos/lib/main.dart b/mobile/apps/photos/lib/main.dartindex 09231b8c62..1adfdb2f90 100644--- a/mobile/apps/photos/lib/main.dart+++ b/mobile/apps/photos/lib/main.dart@@ -89,6 +89,7 @@ const kBGProcessingTaskMLSelfStopIOS = Duration(minutes: 3, seconds: 45);// opens; it drains its latched stop within seconds once the process is woken.const kBGProcessingTaskMLLockWaitIOS = Duration(seconds: 60);const kBGTaskMLSelfStopAndroid = Duration(minutes: 9);+const kBGProcessingTaskMLSelfStopAndroid = Duration(minutes: 8);bool isProcessBg = true;bool _stopHearBeat = false;bool _isSyncInitialized = false;@@ -244,11 +245,18 @@ Future<void> runBackgroundTask(// Created at task start so a stop that fires before ML begins stays// latched for the whole task.final mlRunControl = MlRunControl();- final mlSelfStopTimer = Timer(- mlSelfStop ??- (Platform.isIOS ? kBGTaskMLSelfStopIOS : kBGTaskMLSelfStopAndroid),- () => mlRunControl.requestStop(MlStopReason.backgroundDeadline),- );+ final mlBudget =+ mlSelfStop ??+ (Platform.isIOS ? kBGTaskMLSelfStopIOS : kBGTaskMLSelfStopAndroid);+ if (mlBudget <= Duration.zero) {+ mlRunControl.requestStop(MlStopReason.backgroundDeadline);+ }+ final mlSelfStopTimer = mlBudget > Duration.zero+ ? Timer(+ mlBudget,+ () => mlRunControl.requestStop(MlStopReason.backgroundDeadline),+ )+ : null;final mlForegroundWatchTimer = Timer.periodic(const Duration(milliseconds: 500),(_) async {@@ -274,7 +282,7 @@ Future<void> runBackgroundTask(await _runMinimally(taskId, tlog, mlRunControl, mlLockWait);} finally {- mlSelfStopTimer.cancel();+ mlSelfStopTimer?.cancel();mlForegroundWatchTimer.cancel();}}@@ -377,9 +385,7 @@ Future<void> _runMinimally(await MLService.instance.init();final disposition = await MLService.instance.runAllML(force: false,- allowImageIndexing:- !Platform.isIOS ||- taskId == BgTaskUtils.iOSBackgroundProcessingTask,+ allowImageIndexing: BgTaskUtils.allowsImageIndexing(taskId),control: mlRunControl,lockWait: mlLockWait,);diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dartindex 05feb84134..e91388b9f6 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart@@ -1023,15 +1023,11 @@ class MLService {faceScore: pf.detection.score,imageHeight: result.decodedImageSize.height,imageWidth: result.decodedImageSize.width,- mlVersion: petMlVersion,+ mlVersion: -1,);}).toList();await mlDataDB.bulkInsertPetFaces(dbPetFaces);await mlDataDB.storePetFaceEmbeddings(dbPetFaces, result.petFaces!);- } else if (instruction.shouldRunPets) {- // No pet faces detected; insert empty marker so the file is- // considered pet-indexed (mirrors Face.empty for human faces).- await mlDataDB.bulkInsertPetFaces([DBPetFace.empty(result.fileId)]);}if (result.petBodies != null && result.petBodies!.isNotEmpty) {@@ -1061,6 +1057,11 @@ class MLService {await mlDataDB.bulkInsertPetBodies(dbPetBodies);await mlDataDB.storePetBodyEmbeddings(dbPetBodies, result.petBodies!);}+ if (result.petFaces?.isNotEmpty ?? false) {+ await mlDataDB.markPetFileIndexed(result.fileId, petMlVersion);+ } else if (instruction.shouldRunPets) {+ await mlDataDB.bulkInsertPetFaces([DBPetFace.empty(result.fileId)]);+ }}_logger.info("ML result for fileID ${result.fileId} stored remote+local");indexedOrSkipped = true;diff --git a/mobile/apps/photos/lib/utils/bg_task_utils.dart b/mobile/apps/photos/lib/utils/bg_task_utils.dartindex 15e3b30531..02442613dd 100644--- a/mobile/apps/photos/lib/utils/bg_task_utils.dart+++ b/mobile/apps/photos/lib/utils/bg_task_utils.dart@@ -23,10 +23,15 @@ void callbackDispatcher() {// listener surfaces as an unhandled exception even on success.String? failure = "Task didn't run";bool timedOut = false;- bool ownsPipeline = !Platform.isIOS;+ bool ownsPipeline = false;final isIOSProcessingTask =Platform.isIOS && taskName == BgTaskUtils.iOSBackgroundProcessingTask;final prefs = await SharedPreferences.getInstance();+ final startupElapsed = BgTaskUtils.taskStartupElapsedFor(+ taskName,+ prefs,+ taskStopwatch,+ );await runWithLogs(() async {@@ -35,33 +40,32 @@ void callbackDispatcher() {if (isIOSProcessingTask) {await BgTaskUtils.scheduleIOSBackgroundProcessingTask();}- if (Platform.isIOS) {- ownsPipeline = await BgTaskUtils.acquireIOSBackgroundPipeline(- taskName,- taskStopwatch,- );- if (!ownsPipeline) {- BgTaskUtils.$.info(- 'Skipping $taskName, background pipeline busy',- );- failure = null;- return;- }+ ownsPipeline = await BgTaskUtils.acquireBackgroundPipeline(+ taskName,+ taskStopwatch,+ startupElapsed: startupElapsed,+ );+ if (!ownsPipeline) {+ BgTaskUtils.$.info('Skipping $taskName, background pipeline busy');+ failure = null;+ return;}if (isIOSProcessingTask) {await BgTaskUtils.markProcessingTaskStart(prefs);}final Duration taskBudget = BgTaskUtils.taskTimeoutFor(taskName);- final Duration remainingBudget = Platform.isIOS- ? taskBudget - taskStopwatch.elapsed+ final accountForStartup =+ Platform.isIOS ||+ taskName == BgTaskUtils.androidBackgroundProcessingTask;+ final elapsed = startupElapsed + taskStopwatch.elapsed;+ final Duration remainingBudget = accountForStartup+ ? taskBudget - elapsed: taskBudget;final mlSelfStop = BgTaskUtils.mlSelfStopFor(taskName);await runBackgroundTask(taskName,tlog,- mlSelfStop: Platform.isIOS- ? mlSelfStop - taskStopwatch.elapsed- : mlSelfStop,+ mlSelfStop: accountForStartup ? mlSelfStop - elapsed : mlSelfStop,mlLockWait: BgTaskUtils.mlLockWaitFor(taskName),).timeout(remainingBudget.isNegative ? Duration.zero : remainingBudget,@@ -115,36 +119,74 @@ class BgTaskUtils {static const iOSBackgroundProcessingTask ="io.ente.frame.iOSBackgroundProcessing";static const androidPeriodicTask = "io.ente.photos.androidPeriodicTask";+ static const androidBackgroundProcessingTask =+ "io.ente.photos.androidBackgroundProcessing";++ static bool allowsImageIndexing(String taskName) =>+ taskName == iOSBackgroundProcessingTask ||+ taskName == androidBackgroundProcessingTask;- static Future<bool> acquireIOSBackgroundPipeline(+ static Duration taskStartupElapsedFor(String taskName,+ SharedPreferences prefs,Stopwatch taskStopwatch,- ) async {- final waitDeadline =- taskStopwatch.elapsed +- (taskName == iOSBackgroundProcessingTask- ? const Duration(seconds: 30)- : Duration.zero);- final taskBudget = taskTimeoutFor(taskName);+ ) {+ if (!Platform.isAndroid || taskName != androidBackgroundProcessingTask) {+ return Duration.zero;+ }+ final startedAt = prefs.getInt("bg_task_start_$taskName");+ if (startedAt == null) {+ $.warning("Native start time unavailable for $taskName");+ return Duration.zero;+ }+ final elapsed =+ DateTime.now().difference(+ DateTime.fromMillisecondsSinceEpoch(startedAt),+ ) -+ taskStopwatch.elapsed;+ return elapsed.isNegative ? Duration.zero : elapsed;+ }++ static Future<bool> acquireBackgroundPipeline(+ String taskName,+ Stopwatch taskStopwatch, {+ Duration startupElapsed = Duration.zero,+ }) async {+ final waits = switch (taskName) {+ iOSBackgroundProcessingTask => const [Duration(seconds: 30)],+ androidBackgroundProcessingTask => const [+ Duration(seconds: 30),+ Duration(minutes: 2),+ ],+ _ => const [Duration.zero],+ };+ var waitDeadline = taskStopwatch.elapsed;+ final taskBudget = taskTimeoutFor(taskName) - startupElapsed;const pollInterval = Duration(milliseconds: 500);- do {- if (taskStopwatch.elapsed >= taskBudget) return false;- if (await ProcessLockClient.instance.tryAcquire(- name: "background_process",- origin: "bg",- operation: taskName,- )) {- $.info(- 'Acquired background pipeline for $taskName until engine detach',+ for (final wait in waits) {+ waitDeadline += wait;+ do {+ if (taskStopwatch.elapsed >= taskBudget) return false;+ if (await ProcessLockClient.instance.tryAcquire(+ name: "background_process",+ origin: "bg",+ operation: taskName,+ )) {+ $.info(+ 'Acquired background pipeline for $taskName until engine detach',+ );+ return true;+ }+ final remainingWait = waitDeadline - taskStopwatch.elapsed;+ if (remainingWait <= Duration.zero) break;+ await Future<void>.delayed(+ remainingWait < pollInterval ? remainingWait : pollInterval,);- return true;+ } while (taskStopwatch.elapsed < waitDeadline);+ if (taskName == androidBackgroundProcessingTask && wait == waits.first) {+ $.info("Background pipeline still busy after 30s, waiting another 2m");}- final remainingWait = waitDeadline - taskStopwatch.elapsed;- if (remainingWait <= Duration.zero) return false;- await Future<void>.delayed(- remainingWait < pollInterval ? remainingWait : pollInterval,- );- } while (taskStopwatch.elapsed < waitDeadline);+ }return false;}@@ -156,7 +198,11 @@ class BgTaskUtils {}static Duration mlSelfStopFor(String taskName) {- if (!Platform.isIOS) return kBGTaskMLSelfStopAndroid;+ if (!Platform.isIOS) {+ return taskName == androidBackgroundProcessingTask+ ? kBGProcessingTaskMLSelfStopAndroid+ : kBGTaskMLSelfStopAndroid;+ }return taskName == iOSBackgroundProcessingTask? kBGProcessingTaskMLSelfStopIOS: kBGTaskMLSelfStopIOS;@@ -259,6 +305,7 @@ class BgTaskUtils {$.info("WorkManager configured");if (Platform.isAndroid) {+ await scheduleAndroidBackgroundProcessingTask();final isScheduled = await workmanager.Workmanager().isScheduledByUniqueName(backgroundTaskIdentifier);if (!isScheduled) {@@ -274,6 +321,23 @@ class BgTaskUtils {static bool _processingTaskArmedThisProcess = false;+ static Future<void> scheduleAndroidBackgroundProcessingTask() async {+ await workmanager.Workmanager().registerPeriodicTask(+ androidBackgroundProcessingTask,+ androidBackgroundProcessingTask,+ frequency: const Duration(hours: 2),+ flexInterval: const Duration(hours: 2),+ initialDelay: Duration.zero,+ constraints: workmanager.Constraints(+ networkType: workmanager.NetworkType.connected,+ requiresCharging: true,+ requiresDeviceIdle: true,+ ),+ existingWorkPolicy: workmanager.ExistingPeriodicWorkPolicy.update,+ );+ $.info("Scheduled Android background processing task");+ }+// Foreground arm to restart the chain after a first install or force-quit;// once per process, so resumes don't keep pushing the pending request out.static Future<void> ensureIOSProcessingTaskScheduled() async {