Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos] Add Android background processing for ML

laurenspriemca4c87ce7da6
diff --git a/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.kt b/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.kt
index d74d1af180..641988ebd1 100644
--- a/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.kt
+++ b/mobile/apps/photos/android/app/src/main/kotlin/io/ente/photos/EnteApplication.kt
@@ -6,6 +6,7 @@ import android.app.NotificationManager
import android.content.Context
import android.util.Log
import androidx.core.app.NotificationCompat
+import androidx.core.content.edit
import dev.fluttercommunity.workmanager.TaskDebugInfo
import dev.fluttercommunity.workmanager.TaskResult
import dev.fluttercommunity.workmanager.WorkmanagerDebug
@@ -16,7 +17,7 @@ import kotlin.random.Random
class EnteApplication : Application() {
override fun onCreate() {
super.onCreate()
- WorkmanagerDebug.setCurrent(InternalUserWorkmanagerDebugHandler())
+ WorkmanagerDebug.setCurrent(EnteWorkmanagerDebugHandler())
ForegroundHeartbeat.install(this)
}
@@ -30,13 +31,18 @@ class EnteApplication : Application() {
}
}
-private class InternalUserWorkmanagerDebugHandler : WorkmanagerDebug() {
+private class EnteWorkmanagerDebugHandler : WorkmanagerDebug() {
override fun onTaskStatusUpdate(
context: Context,
taskInfo: TaskDebugInfo,
status: TaskStatus,
result: TaskResult?,
) {
+ if (status == TaskStatus.STARTED || status == TaskStatus.RETRYING) {
+ context
+ .getSharedPreferences(EnteApplication.FLUTTER_SHARED_PREFERENCES, Context.MODE_PRIVATE)
+ .edit { putLong("flutter.bg_task_start_${taskInfo.taskName}", taskInfo.startTime) }
+ }
if (!shouldEnableWorkmanagerDebugNotifications(context)) {
return
}
diff --git a/mobile/apps/photos/lib/db/ml/base.dart b/mobile/apps/photos/lib/db/ml/base.dart
index 0439b0287d..35219442b8 100644
--- a/mobile/apps/photos/lib/db/ml/base.dart
+++ b/mobile/apps/photos/lib/db/ml/base.dart
@@ -136,6 +136,7 @@ abstract class IMLDataDB<T> {
Future<Map<int, int>> petIndexedFileIds({int minimumMlVersion});
Future<int> getPetIndexedFileCount({int minimumMlVersion});
Future<void> deletePetDataForFiles(List<int> fileIDs);
+ Future<void> markPetFileIndexed(int fileID, int mlVersion);
Future<Set<int>> getFullyIndexedFileIds({required bool includePets});
diff --git a/mobile/apps/photos/lib/db/ml/db.dart b/mobile/apps/photos/lib/db/ml/db.dart
index 5aa5405076..99e73ce4ff 100644
--- a/mobile/apps/photos/lib/db/ml/db.dart
+++ b/mobile/apps/photos/lib/db/ml/db.dart
@@ -206,6 +206,15 @@ class DartMLDataDB
}
}
+ @override
+ Future<void> markPetFileIndexed(int fileID, int mlVersion) async {
+ final db = await asyncDB;
+ await db.execute(
+ 'UPDATE $petFacesTable SET $mlVersionColumn = ? WHERE $fileIDColumn = ?',
+ [mlVersion, fileID],
+ );
+ }
+
@override
Future<void> bulkInsertPetBodies(List<DBPetBody> petBodies) async {
final db = await asyncDB;
diff --git a/mobile/apps/photos/lib/main.dart b/mobile/apps/photos/lib/main.dart
index 09231b8c62..1adfdb2f90 100644
--- a/mobile/apps/photos/lib/main.dart
+++ b/mobile/apps/photos/lib/main.dart
@@ -89,6 +89,7 @@ const kBGProcessingTaskMLSelfStopIOS = Duration(minutes: 3, seconds: 45);
// opens; it drains its latched stop within seconds once the process is woken.
const kBGProcessingTaskMLLockWaitIOS = Duration(seconds: 60);
const kBGTaskMLSelfStopAndroid = Duration(minutes: 9);
+const kBGProcessingTaskMLSelfStopAndroid = Duration(minutes: 8);
bool isProcessBg = true;
bool _stopHearBeat = false;
bool _isSyncInitialized = false;
@@ -244,11 +245,18 @@ Future<void> runBackgroundTask(
// Created at task start so a stop that fires before ML begins stays
// latched for the whole task.
final mlRunControl = MlRunControl();
- final mlSelfStopTimer = Timer(
- mlSelfStop ??
- (Platform.isIOS ? kBGTaskMLSelfStopIOS : kBGTaskMLSelfStopAndroid),
- () => mlRunControl.requestStop(MlStopReason.backgroundDeadline),
- );
+ final mlBudget =
+ mlSelfStop ??
+ (Platform.isIOS ? kBGTaskMLSelfStopIOS : kBGTaskMLSelfStopAndroid);
+ if (mlBudget <= Duration.zero) {
+ mlRunControl.requestStop(MlStopReason.backgroundDeadline);
+ }
+ final mlSelfStopTimer = mlBudget > Duration.zero
+ ? Timer(
+ mlBudget,
+ () => mlRunControl.requestStop(MlStopReason.backgroundDeadline),
+ )
+ : null;
final mlForegroundWatchTimer = Timer.periodic(
const Duration(milliseconds: 500),
(_) async {
@@ -274,7 +282,7 @@ Future<void> runBackgroundTask(
await _runMinimally(taskId, tlog, mlRunControl, mlLockWait);
} finally {
- mlSelfStopTimer.cancel();
+ mlSelfStopTimer?.cancel();
mlForegroundWatchTimer.cancel();
}
}
@@ -377,9 +385,7 @@ Future<void> _runMinimally(
await MLService.instance.init();
final disposition = await MLService.instance.runAllML(
force: false,
- allowImageIndexing:
- !Platform.isIOS ||
- taskId == BgTaskUtils.iOSBackgroundProcessingTask,
+ allowImageIndexing: BgTaskUtils.allowsImageIndexing(taskId),
control: mlRunControl,
lockWait: mlLockWait,
);
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
index 05feb84134..e91388b9f6 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
@@ -1023,15 +1023,11 @@ class MLService {
faceScore: pf.detection.score,
imageHeight: result.decodedImageSize.height,
imageWidth: result.decodedImageSize.width,
- mlVersion: petMlVersion,
+ mlVersion: -1,
);
}).toList();
await mlDataDB.bulkInsertPetFaces(dbPetFaces);
await mlDataDB.storePetFaceEmbeddings(dbPetFaces, result.petFaces!);
- } else if (instruction.shouldRunPets) {
- // No pet faces detected; insert empty marker so the file is
- // considered pet-indexed (mirrors Face.empty for human faces).
- await mlDataDB.bulkInsertPetFaces([DBPetFace.empty(result.fileId)]);
}
if (result.petBodies != null && result.petBodies!.isNotEmpty) {
@@ -1061,6 +1057,11 @@ class MLService {
await mlDataDB.bulkInsertPetBodies(dbPetBodies);
await mlDataDB.storePetBodyEmbeddings(dbPetBodies, result.petBodies!);
}
+ if (result.petFaces?.isNotEmpty ?? false) {
+ await mlDataDB.markPetFileIndexed(result.fileId, petMlVersion);
+ } else if (instruction.shouldRunPets) {
+ await mlDataDB.bulkInsertPetFaces([DBPetFace.empty(result.fileId)]);
+ }
}
_logger.info("ML result for fileID ${result.fileId} stored remote+local");
indexedOrSkipped = true;
diff --git a/mobile/apps/photos/lib/utils/bg_task_utils.dart b/mobile/apps/photos/lib/utils/bg_task_utils.dart
index 15e3b30531..02442613dd 100644
--- a/mobile/apps/photos/lib/utils/bg_task_utils.dart
+++ b/mobile/apps/photos/lib/utils/bg_task_utils.dart
@@ -23,10 +23,15 @@ void callbackDispatcher() {
// listener surfaces as an unhandled exception even on success.
String? failure = "Task didn't run";
bool timedOut = false;
- bool ownsPipeline = !Platform.isIOS;
+ bool ownsPipeline = false;
final isIOSProcessingTask =
Platform.isIOS && taskName == BgTaskUtils.iOSBackgroundProcessingTask;
final prefs = await SharedPreferences.getInstance();
+ final startupElapsed = BgTaskUtils.taskStartupElapsedFor(
+ taskName,
+ prefs,
+ taskStopwatch,
+ );
await runWithLogs(
() async {
@@ -35,33 +40,32 @@ void callbackDispatcher() {
if (isIOSProcessingTask) {
await BgTaskUtils.scheduleIOSBackgroundProcessingTask();
}
- if (Platform.isIOS) {
- ownsPipeline = await BgTaskUtils.acquireIOSBackgroundPipeline(
- taskName,
- taskStopwatch,
- );
- if (!ownsPipeline) {
- BgTaskUtils.$.info(
- 'Skipping $taskName, background pipeline busy',
- );
- failure = null;
- return;
- }
+ ownsPipeline = await BgTaskUtils.acquireBackgroundPipeline(
+ taskName,
+ taskStopwatch,
+ startupElapsed: startupElapsed,
+ );
+ if (!ownsPipeline) {
+ BgTaskUtils.$.info('Skipping $taskName, background pipeline busy');
+ failure = null;
+ return;
}
if (isIOSProcessingTask) {
await BgTaskUtils.markProcessingTaskStart(prefs);
}
final Duration taskBudget = BgTaskUtils.taskTimeoutFor(taskName);
- final Duration remainingBudget = Platform.isIOS
- ? taskBudget - taskStopwatch.elapsed
+ final accountForStartup =
+ Platform.isIOS ||
+ taskName == BgTaskUtils.androidBackgroundProcessingTask;
+ final elapsed = startupElapsed + taskStopwatch.elapsed;
+ final Duration remainingBudget = accountForStartup
+ ? taskBudget - elapsed
: taskBudget;
final mlSelfStop = BgTaskUtils.mlSelfStopFor(taskName);
await runBackgroundTask(
taskName,
tlog,
- mlSelfStop: Platform.isIOS
- ? mlSelfStop - taskStopwatch.elapsed
- : mlSelfStop,
+ mlSelfStop: accountForStartup ? mlSelfStop - elapsed : mlSelfStop,
mlLockWait: BgTaskUtils.mlLockWaitFor(taskName),
).timeout(
remainingBudget.isNegative ? Duration.zero : remainingBudget,
@@ -115,36 +119,74 @@ class BgTaskUtils {
static const iOSBackgroundProcessingTask =
"io.ente.frame.iOSBackgroundProcessing";
static const androidPeriodicTask = "io.ente.photos.androidPeriodicTask";
+ static const androidBackgroundProcessingTask =
+ "io.ente.photos.androidBackgroundProcessing";
+
+ static bool allowsImageIndexing(String taskName) =>
+ taskName == iOSBackgroundProcessingTask ||
+ taskName == androidBackgroundProcessingTask;
- static Future<bool> acquireIOSBackgroundPipeline(
+ static Duration taskStartupElapsedFor(
String taskName,
+ SharedPreferences prefs,
Stopwatch taskStopwatch,
- ) async {
- final waitDeadline =
- taskStopwatch.elapsed +
- (taskName == iOSBackgroundProcessingTask
- ? const Duration(seconds: 30)
- : Duration.zero);
- final taskBudget = taskTimeoutFor(taskName);
+ ) {
+ if (!Platform.isAndroid || taskName != androidBackgroundProcessingTask) {
+ return Duration.zero;
+ }
+ final startedAt = prefs.getInt("bg_task_start_$taskName");
+ if (startedAt == null) {
+ $.warning("Native start time unavailable for $taskName");
+ return Duration.zero;
+ }
+ final elapsed =
+ DateTime.now().difference(
+ DateTime.fromMillisecondsSinceEpoch(startedAt),
+ ) -
+ taskStopwatch.elapsed;
+ return elapsed.isNegative ? Duration.zero : elapsed;
+ }
+
+ static Future<bool> acquireBackgroundPipeline(
+ String taskName,
+ Stopwatch taskStopwatch, {
+ Duration startupElapsed = Duration.zero,
+ }) async {
+ final waits = switch (taskName) {
+ iOSBackgroundProcessingTask => const [Duration(seconds: 30)],
+ androidBackgroundProcessingTask => const [
+ Duration(seconds: 30),
+ Duration(minutes: 2),
+ ],
+ _ => const [Duration.zero],
+ };
+ var waitDeadline = taskStopwatch.elapsed;
+ final taskBudget = taskTimeoutFor(taskName) - startupElapsed;
const pollInterval = Duration(milliseconds: 500);
- do {
- if (taskStopwatch.elapsed >= taskBudget) return false;
- if (await ProcessLockClient.instance.tryAcquire(
- name: "background_process",
- origin: "bg",
- operation: taskName,
- )) {
- $.info(
- 'Acquired background pipeline for $taskName until engine detach',
+ for (final wait in waits) {
+ waitDeadline += wait;
+ do {
+ if (taskStopwatch.elapsed >= taskBudget) return false;
+ if (await ProcessLockClient.instance.tryAcquire(
+ name: "background_process",
+ origin: "bg",
+ operation: taskName,
+ )) {
+ $.info(
+ 'Acquired background pipeline for $taskName until engine detach',
+ );
+ return true;
+ }
+ final remainingWait = waitDeadline - taskStopwatch.elapsed;
+ if (remainingWait <= Duration.zero) break;
+ await Future<void>.delayed(
+ remainingWait < pollInterval ? remainingWait : pollInterval,
);
- return true;
+ } while (taskStopwatch.elapsed < waitDeadline);
+ if (taskName == androidBackgroundProcessingTask && wait == waits.first) {
+ $.info("Background pipeline still busy after 30s, waiting another 2m");
}
- final remainingWait = waitDeadline - taskStopwatch.elapsed;
- if (remainingWait <= Duration.zero) return false;
- await Future<void>.delayed(
- remainingWait < pollInterval ? remainingWait : pollInterval,
- );
- } while (taskStopwatch.elapsed < waitDeadline);
+ }
return false;
}
@@ -156,7 +198,11 @@ class BgTaskUtils {
}
static Duration mlSelfStopFor(String taskName) {
- if (!Platform.isIOS) return kBGTaskMLSelfStopAndroid;
+ if (!Platform.isIOS) {
+ return taskName == androidBackgroundProcessingTask
+ ? kBGProcessingTaskMLSelfStopAndroid
+ : kBGTaskMLSelfStopAndroid;
+ }
return taskName == iOSBackgroundProcessingTask
? kBGProcessingTaskMLSelfStopIOS
: kBGTaskMLSelfStopIOS;
@@ -259,6 +305,7 @@ class BgTaskUtils {
$.info("WorkManager configured");
if (Platform.isAndroid) {
+ await scheduleAndroidBackgroundProcessingTask();
final isScheduled = await workmanager.Workmanager()
.isScheduledByUniqueName(backgroundTaskIdentifier);
if (!isScheduled) {
@@ -274,6 +321,23 @@ class BgTaskUtils {
static bool _processingTaskArmedThisProcess = false;
+ static Future<void> scheduleAndroidBackgroundProcessingTask() async {
+ await workmanager.Workmanager().registerPeriodicTask(
+ androidBackgroundProcessingTask,
+ androidBackgroundProcessingTask,
+ frequency: const Duration(hours: 2),
+ flexInterval: const Duration(hours: 2),
+ initialDelay: Duration.zero,
+ constraints: workmanager.Constraints(
+ networkType: workmanager.NetworkType.connected,
+ requiresCharging: true,
+ requiresDeviceIdle: true,
+ ),
+ existingWorkPolicy: workmanager.ExistingPeriodicWorkPolicy.update,
+ );
+ $.info("Scheduled Android background processing task");
+ }
+
// Foreground arm to restart the chain after a first install or force-quit;
// once per process, so resumes don't keep pushing the pending request out.
static Future<void> ensureIOSProcessingTaskScheduled() async {