High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(mobile): retry the background backup when connectivity is restored (#30909)
Santo Shakil3e2e8e0fe8a9
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.ktindex 60387f434a..522183c66c 100644--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.kt+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.kt@@ -24,6 +24,7 @@ import io.flutter.embedding.engine.FlutterEngineCacheimport io.flutter.embedding.engine.dart.DartExecutorimport io.flutter.embedding.engine.loader.FlutterLoaderimport java.util.concurrent.TimeUnit+import java.util.concurrent.atomic.AtomicReferenceprivate const val TAG = "BackgroundWorker"@@ -59,56 +60,68 @@ class BackgroundWorker(context: Context, params: WorkerParameters) :companion object {private const val NOTIFICATION_CHANNEL_ID = "immich::background_worker::notif"private const val NOTIFICATION_ID = 100+ private val activeWorker = AtomicReference<BackgroundWorker?>(null)}override fun startWork(): ListenableFuture<Result> {- if (BackgroundWorkerPreferences(ctx).isLocked() && BackgroundEngineLock.connectEngines > 0) {- Log.i(TAG, "Foreground engine active, skipping background worker")+ if (!activeWorker.compareAndSet(null, this)) {+ Log.i(TAG, "Another background worker is running, skipping")return Futures.immediateFuture(Result.success())}- Log.i(TAG, "Starting background upload worker")-- if (!loader.initialized()) {- loader.startInitialization(ctx)+ if (BackgroundWorkerPreferences(ctx).isLocked() && BackgroundEngineLock.connectEngines > 0) {+ Log.i(TAG, "Foreground engine active, skipping background worker")+ activeWorker.compareAndSet(this, null)+ return Futures.immediateFuture(Result.success())}- val notificationChannel = NotificationChannel(- NOTIFICATION_CHANNEL_ID,- ctx.getString(R.string.background_worker_notification_channel_name),- NotificationManager.IMPORTANCE_LOW- )- notificationManager.createNotificationChannel(notificationChannel)- val notificationConfig = BackgroundWorkerPreferences(ctx).getNotificationConfig()- showNotification(notificationConfig.first, notificationConfig.second)+ try {+ Log.i(TAG, "Starting background upload worker")- loader.ensureInitializationCompleteAsync(ctx, null, Handler(Looper.getMainLooper())) {- if (isStopped || isComplete) {- return@ensureInitializationCompleteAsync+ if (!loader.initialized()) {+ loader.startInitialization(ctx)}- engine = FlutterEngine(ctx)- FlutterEngineCache.getInstance().put(BackgroundWorkerApiImpl.ENGINE_CACHE_KEY, engine!!)-- // Register custom plugins- MainActivity.registerPlugins(ctx, engine!!)- flutterApi =- BackgroundWorkerFlutterApi(binaryMessenger = engine!!.dartExecutor.binaryMessenger)- BackgroundWorkerBgHostApi.setUp(- binaryMessenger = engine!!.dartExecutor.binaryMessenger,- api = this+ val notificationChannel = NotificationChannel(+ NOTIFICATION_CHANNEL_ID,+ ctx.getString(R.string.background_worker_notification_channel_name),+ NotificationManager.IMPORTANCE_LOW)+ notificationManager.createNotificationChannel(notificationChannel)+ val notificationConfig = BackgroundWorkerPreferences(ctx).getNotificationConfig()+ showNotification(notificationConfig.first, notificationConfig.second)++ loader.ensureInitializationCompleteAsync(ctx, null, Handler(Looper.getMainLooper())) {+ if (isStopped || isComplete) {+ return@ensureInitializationCompleteAsync+ }++ engine = FlutterEngine(ctx)+ FlutterEngineCache.getInstance().put(BackgroundWorkerApiImpl.ENGINE_CACHE_KEY, engine!!)++ // Register custom plugins+ MainActivity.registerPlugins(ctx, engine!!)+ flutterApi =+ BackgroundWorkerFlutterApi(binaryMessenger = engine!!.dartExecutor.binaryMessenger)+ BackgroundWorkerBgHostApi.setUp(+ binaryMessenger = engine!!.dartExecutor.binaryMessenger,+ api = this+ )- engine!!.dartExecutor.executeDartEntrypoint(- DartExecutor.DartEntrypoint(- loader.findAppBundlePath(),- "package:immich_mobile/domain/services/background_worker.service.dart",- "backgroundSyncNativeEntrypoint"+ engine!!.dartExecutor.executeDartEntrypoint(+ DartExecutor.DartEntrypoint(+ loader.findAppBundlePath(),+ "package:immich_mobile/domain/services/background_worker.service.dart",+ "backgroundSyncNativeEntrypoint"+ ))- )- }+ }- return completionHandler+ return completionHandler+ } catch (error: Throwable) {+ activeWorker.compareAndSet(this, null)+ throw error+ }}/**@@ -184,13 +197,21 @@ class BackgroundWorker(context: Context, params: WorkerParameters) :close()}- private fun handleHostResult(result: kotlin.Result<Unit>) {+ private fun handleHostResult(result: kotlin.Result<Boolean>) {if (isComplete) {return}result.fold(- onSuccess = { _ -> complete(Result.success()) },+ onSuccess = { needsRetry ->+ // The connected worker cannot enqueue itself while it is running, so it retries in place+ if (needsRetry && !tags.contains(BackgroundWorkerApiImpl.RETRY_TAG)) {+ BackgroundWorkerApiImpl.enqueueBackgroundWorkerWhenConnected(ctx)+ complete(Result.success())+ } else {+ complete(if (needsRetry) Result.retry() else Result.success())+ }+ },onFailure = { _ -> onStopped() })}@@ -214,6 +235,7 @@ class BackgroundWorker(context: Context, params: WorkerParameters) :flutterApi = nullnotificationManager.cancel(NOTIFICATION_ID)FlutterEngineCache.getInstance().remove(BackgroundWorkerApiImpl.ENGINE_CACHE_KEY)+ activeWorker.compareAndSet(this, null)waitForForegroundPromotion()completionHandler.set(success)}diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.ktindex bc0766bee5..b971da45c7 100644--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.kt+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.kt@@ -7,6 +7,7 @@ import androidx.work.BackoffPolicyimport androidx.work.Constraintsimport androidx.work.ExistingPeriodicWorkPolicyimport androidx.work.ExistingWorkPolicy+import androidx.work.NetworkTypeimport androidx.work.OneTimeWorkRequestBuilderimport androidx.work.PeriodicWorkRequestBuilderimport androidx.work.WorkManager@@ -37,6 +38,7 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {WorkManager.getInstance(ctx).apply {cancelUniqueWork(OBSERVER_WORKER_NAME)cancelUniqueWork(BACKGROUND_WORKER_NAME)+ cancelUniqueWork(RETRY_WORKER_NAME)cancelUniqueWork(PERIODIC_WORKER_NAME)}Log.i(TAG, "Cancelled background upload tasks")@@ -44,9 +46,11 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {companion object {private const val BACKGROUND_WORKER_NAME = "immich/BackgroundWorkerV1"+ private const val RETRY_WORKER_NAME = "immich/BackgroundWorkerRetryV1"private const val OBSERVER_WORKER_NAME = "immich/MediaObserverV1"private const val PERIODIC_WORKER_NAME = "immich/PeriodicBackgroundWorkerV1"const val ENGINE_CACHE_KEY = "immich::background_worker::engine"+ const val RETRY_TAG = "immich/retryWhenConnected"fun enqueueMediaObserver(ctx: Context) {@@ -106,6 +110,24 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {Log.i(TAG, "Enqueued background worker with name: $BACKGROUND_WORKER_NAME")}+ fun enqueueBackgroundWorkerWhenConnected(ctx: Context) {+ val settings = BackgroundWorkerPreferences(ctx).getSettings()+ val constraints = Constraints.Builder().apply {+ setRequiredNetworkType(NetworkType.CONNECTED)+ setRequiresBatteryNotLow(true)+ setRequiresCharging(settings.requiresCharging)+ }.build()+ val work = OneTimeWorkRequestBuilder<BackgroundWorker>()+ .setConstraints(constraints)+ .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 1, TimeUnit.MINUTES)+ .addTag(RETRY_TAG)+ .build()+ WorkManager.getInstance(ctx)+ .enqueueUniqueWork(RETRY_WORKER_NAME, ExistingWorkPolicy.KEEP, work)++ Log.i(TAG, "Enqueued background worker to run once connected")+ }+fun isBackgroundWorkerRunning(): Boolean {// Easier to check if the engine is cached as we always cache the engine when starting the worker// and remove it when the worker is finished@@ -114,6 +136,7 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {fun cancelBackgroundWorker(ctx: Context) {WorkManager.getInstance(ctx).cancelUniqueWork(BACKGROUND_WORKER_NAME)+ WorkManager.getInstance(ctx).cancelUniqueWork(RETRY_WORKER_NAME)FlutterEngineCache.getInstance().remove(ENGINE_CACHE_KEY)Log.i(TAG, "Cancelled background upload task")diff --git a/mobile/lib/domain/services/background_worker.service.dart b/mobile/lib/domain/services/background_worker.service.dartindex 6e04c4646c..6286d37482 100644--- a/mobile/lib/domain/services/background_worker.service.dart+++ b/mobile/lib/domain/services/background_worker.service.dart@@ -141,7 +141,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {}@override- Future<void> onAndroidUpload(int? maxMinutes) async {+ Future<bool> onAndroidUpload(int? maxMinutes) async {final hashTimeout = Duration(minutes: _isBackupEnabled ? 3 : 6);final backupTimeout = maxMinutes != null ? Duration(minutes: maxMinutes - 1) : null;await _optimizeDB();@@ -199,7 +199,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {}}- Future<void> _backgroundLoop({+ Future<bool> _backgroundLoop({required Duration hashTimeout,required Duration? backupTimeout,required String debugLabel,@@ -211,7 +211,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {try {if (!await _syncAssets(hashTimeout: hashTimeout)) {_logger.warning("Remote sync did not complete successfully, skipping backup");- return;+ return true;}final backupFuture = _handleBackup();@@ -225,12 +225,13 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {});}try {- await backupFuture;+ return await backupFuture;} finally {cancelTimer?.cancel();}} catch (error, stack) {_logger.severe("Failed to complete $debugLabel", error, stack);+ return true;} finally {sw.stop();_logger.info("$debugLabel completed in ${sw.elapsed.inSeconds}s");@@ -290,36 +291,41 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {}}- Future<void> _handleBackup() async {- await runZonedGuarded(- () async {- if (_isCleanedUp) {- return;- }-- if (!_isBackupEnabled) {- _logger.info("Backup is disabled. Skipping backup routine");- return;- }-- final currentUser = _ref?.read(currentUserProvider);- if (currentUser == null) {- _logger.warning("No current user found. Skipping backup from background");- return;- }-- if (Platform.isIOS) {- return _ref?.read(backupProvider.notifier).startBackupWithURLSession(currentUser.id);- }-- return _ref- ?.read(foregroundUploadServiceProvider)- .uploadCandidates(currentUser.id, _cancellationToken, useSequentialUpload: true);- },- (error, stack) {- dPrint(() => "Error in backup zone $error, $stack");- },- );+ Future<bool> _handleBackup() async {+ final needsRetry = await runZonedGuarded(() async {+ if (_isCleanedUp) {+ return false;+ }++ if (!_isBackupEnabled) {+ _logger.info("Backup is disabled. Skipping backup routine");+ return false;+ }++ final currentUser = _ref?.read(currentUserProvider);+ if (currentUser == null) {+ _logger.warning("No current user found. Skipping backup from background");+ return false;+ }++ if (Platform.isIOS) {+ await _ref?.read(backupProvider.notifier).startBackupWithURLSession(currentUser.id);+ return false;+ }++ final uploadService = _ref?.read(foregroundUploadServiceProvider);+ var failed = 0;+ await uploadService?.uploadCandidates(+ currentUser.id,+ _cancellationToken,+ callbacks: UploadCallbacks(onError: (_, _) => failed++),+ useSequentialUpload: true,+ );+ _logger.info("Background backup finished, $failed uploads failed");+ // Retry when at least one upload failed+ return failed > 0;+ }, (error, stack) => dPrint(() => "Error in backup zone $error, $stack"));+ return needsRetry ?? true;}Future<bool> _syncAssets({Duration? hashTimeout}) async {diff --git a/mobile/pigeon/background_worker_api.dart b/mobile/pigeon/background_worker_api.dartindex 06395fae7b..dd62dd069d 100644--- a/mobile/pigeon/background_worker_api.dart+++ b/mobile/pigeon/background_worker_api.dart@@ -46,8 +46,9 @@ abstract class BackgroundWorkerFlutterApi {void onIosUpload(bool isRefresh, int? maxSeconds);// Android Only: Called when the Android background upload is triggered+ // Returns whether the native side needs to retry the run@async- void onAndroidUpload(int? maxMinutes);+ bool onAndroidUpload(int? maxMinutes);@asyncvoid cancel();