Skip to content

High performance self-hosted photo and video management solution.

fix(mobile): retry the background backup when connectivity is restored (#30909)

Santo Shakil3e2e8e0fe8a9
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.kt
index 60387f434a..522183c66c 100644
--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.kt
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorker.kt
@@ -24,6 +24,7 @@ import io.flutter.embedding.engine.FlutterEngineCache
import io.flutter.embedding.engine.dart.DartExecutor
import io.flutter.embedding.engine.loader.FlutterLoader
import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
private const val TAG = "BackgroundWorker"
@@ -59,56 +60,68 @@ class BackgroundWorker(context: Context, params: WorkerParameters) :
companion object {
private const val NOTIFICATION_CHANNEL_ID = "immich::background_worker::notif"
private const val NOTIFICATION_ID = 100
+ private val activeWorker = AtomicReference<BackgroundWorker?>(null)
}
override fun startWork(): ListenableFuture<Result> {
- if (BackgroundWorkerPreferences(ctx).isLocked() && BackgroundEngineLock.connectEngines > 0) {
- Log.i(TAG, "Foreground engine active, skipping background worker")
+ if (!activeWorker.compareAndSet(null, this)) {
+ Log.i(TAG, "Another background worker is running, skipping")
return Futures.immediateFuture(Result.success())
}
- Log.i(TAG, "Starting background upload worker")
-
- if (!loader.initialized()) {
- loader.startInitialization(ctx)
+ if (BackgroundWorkerPreferences(ctx).isLocked() && BackgroundEngineLock.connectEngines > 0) {
+ Log.i(TAG, "Foreground engine active, skipping background worker")
+ activeWorker.compareAndSet(this, null)
+ return Futures.immediateFuture(Result.success())
}
- val notificationChannel = NotificationChannel(
- NOTIFICATION_CHANNEL_ID,
- ctx.getString(R.string.background_worker_notification_channel_name),
- NotificationManager.IMPORTANCE_LOW
- )
- notificationManager.createNotificationChannel(notificationChannel)
- val notificationConfig = BackgroundWorkerPreferences(ctx).getNotificationConfig()
- showNotification(notificationConfig.first, notificationConfig.second)
+ try {
+ Log.i(TAG, "Starting background upload worker")
- loader.ensureInitializationCompleteAsync(ctx, null, Handler(Looper.getMainLooper())) {
- if (isStopped || isComplete) {
- return@ensureInitializationCompleteAsync
+ if (!loader.initialized()) {
+ loader.startInitialization(ctx)
}
- engine = FlutterEngine(ctx)
- FlutterEngineCache.getInstance().put(BackgroundWorkerApiImpl.ENGINE_CACHE_KEY, engine!!)
-
- // Register custom plugins
- MainActivity.registerPlugins(ctx, engine!!)
- flutterApi =
- BackgroundWorkerFlutterApi(binaryMessenger = engine!!.dartExecutor.binaryMessenger)
- BackgroundWorkerBgHostApi.setUp(
- binaryMessenger = engine!!.dartExecutor.binaryMessenger,
- api = this
+ val notificationChannel = NotificationChannel(
+ NOTIFICATION_CHANNEL_ID,
+ ctx.getString(R.string.background_worker_notification_channel_name),
+ NotificationManager.IMPORTANCE_LOW
)
+ notificationManager.createNotificationChannel(notificationChannel)
+ val notificationConfig = BackgroundWorkerPreferences(ctx).getNotificationConfig()
+ showNotification(notificationConfig.first, notificationConfig.second)
+
+ loader.ensureInitializationCompleteAsync(ctx, null, Handler(Looper.getMainLooper())) {
+ if (isStopped || isComplete) {
+ return@ensureInitializationCompleteAsync
+ }
+
+ engine = FlutterEngine(ctx)
+ FlutterEngineCache.getInstance().put(BackgroundWorkerApiImpl.ENGINE_CACHE_KEY, engine!!)
+
+ // Register custom plugins
+ MainActivity.registerPlugins(ctx, engine!!)
+ flutterApi =
+ BackgroundWorkerFlutterApi(binaryMessenger = engine!!.dartExecutor.binaryMessenger)
+ BackgroundWorkerBgHostApi.setUp(
+ binaryMessenger = engine!!.dartExecutor.binaryMessenger,
+ api = this
+ )
- engine!!.dartExecutor.executeDartEntrypoint(
- DartExecutor.DartEntrypoint(
- loader.findAppBundlePath(),
- "package:immich_mobile/domain/services/background_worker.service.dart",
- "backgroundSyncNativeEntrypoint"
+ engine!!.dartExecutor.executeDartEntrypoint(
+ DartExecutor.DartEntrypoint(
+ loader.findAppBundlePath(),
+ "package:immich_mobile/domain/services/background_worker.service.dart",
+ "backgroundSyncNativeEntrypoint"
+ )
)
- )
- }
+ }
- return completionHandler
+ return completionHandler
+ } catch (error: Throwable) {
+ activeWorker.compareAndSet(this, null)
+ throw error
+ }
}
/**
@@ -184,13 +197,21 @@ class BackgroundWorker(context: Context, params: WorkerParameters) :
close()
}
- private fun handleHostResult(result: kotlin.Result<Unit>) {
+ private fun handleHostResult(result: kotlin.Result<Boolean>) {
if (isComplete) {
return
}
result.fold(
- onSuccess = { _ -> complete(Result.success()) },
+ onSuccess = { needsRetry ->
+ // The connected worker cannot enqueue itself while it is running, so it retries in place
+ if (needsRetry && !tags.contains(BackgroundWorkerApiImpl.RETRY_TAG)) {
+ BackgroundWorkerApiImpl.enqueueBackgroundWorkerWhenConnected(ctx)
+ complete(Result.success())
+ } else {
+ complete(if (needsRetry) Result.retry() else Result.success())
+ }
+ },
onFailure = { _ -> onStopped() }
)
}
@@ -214,6 +235,7 @@ class BackgroundWorker(context: Context, params: WorkerParameters) :
flutterApi = null
notificationManager.cancel(NOTIFICATION_ID)
FlutterEngineCache.getInstance().remove(BackgroundWorkerApiImpl.ENGINE_CACHE_KEY)
+ activeWorker.compareAndSet(this, null)
waitForForegroundPromotion()
completionHandler.set(success)
}
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.kt
index bc0766bee5..b971da45c7 100644
--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.kt
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/background/BackgroundWorkerApiImpl.kt
@@ -7,6 +7,7 @@ import androidx.work.BackoffPolicy
import androidx.work.Constraints
import androidx.work.ExistingPeriodicWorkPolicy
import androidx.work.ExistingWorkPolicy
+import androidx.work.NetworkType
import androidx.work.OneTimeWorkRequestBuilder
import androidx.work.PeriodicWorkRequestBuilder
import androidx.work.WorkManager
@@ -37,6 +38,7 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {
WorkManager.getInstance(ctx).apply {
cancelUniqueWork(OBSERVER_WORKER_NAME)
cancelUniqueWork(BACKGROUND_WORKER_NAME)
+ cancelUniqueWork(RETRY_WORKER_NAME)
cancelUniqueWork(PERIODIC_WORKER_NAME)
}
Log.i(TAG, "Cancelled background upload tasks")
@@ -44,9 +46,11 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {
companion object {
private const val BACKGROUND_WORKER_NAME = "immich/BackgroundWorkerV1"
+ private const val RETRY_WORKER_NAME = "immich/BackgroundWorkerRetryV1"
private const val OBSERVER_WORKER_NAME = "immich/MediaObserverV1"
private const val PERIODIC_WORKER_NAME = "immich/PeriodicBackgroundWorkerV1"
const val ENGINE_CACHE_KEY = "immich::background_worker::engine"
+ const val RETRY_TAG = "immich/retryWhenConnected"
fun enqueueMediaObserver(ctx: Context) {
@@ -106,6 +110,24 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {
Log.i(TAG, "Enqueued background worker with name: $BACKGROUND_WORKER_NAME")
}
+ fun enqueueBackgroundWorkerWhenConnected(ctx: Context) {
+ val settings = BackgroundWorkerPreferences(ctx).getSettings()
+ val constraints = Constraints.Builder().apply {
+ setRequiredNetworkType(NetworkType.CONNECTED)
+ setRequiresBatteryNotLow(true)
+ setRequiresCharging(settings.requiresCharging)
+ }.build()
+ val work = OneTimeWorkRequestBuilder<BackgroundWorker>()
+ .setConstraints(constraints)
+ .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 1, TimeUnit.MINUTES)
+ .addTag(RETRY_TAG)
+ .build()
+ WorkManager.getInstance(ctx)
+ .enqueueUniqueWork(RETRY_WORKER_NAME, ExistingWorkPolicy.KEEP, work)
+
+ Log.i(TAG, "Enqueued background worker to run once connected")
+ }
+
fun isBackgroundWorkerRunning(): Boolean {
// Easier to check if the engine is cached as we always cache the engine when starting the worker
// and remove it when the worker is finished
@@ -114,6 +136,7 @@ class BackgroundWorkerApiImpl(context: Context) : BackgroundWorkerFgHostApi {
fun cancelBackgroundWorker(ctx: Context) {
WorkManager.getInstance(ctx).cancelUniqueWork(BACKGROUND_WORKER_NAME)
+ WorkManager.getInstance(ctx).cancelUniqueWork(RETRY_WORKER_NAME)
FlutterEngineCache.getInstance().remove(ENGINE_CACHE_KEY)
Log.i(TAG, "Cancelled background upload task")
diff --git a/mobile/lib/domain/services/background_worker.service.dart b/mobile/lib/domain/services/background_worker.service.dart
index 6e04c4646c..6286d37482 100644
--- a/mobile/lib/domain/services/background_worker.service.dart
+++ b/mobile/lib/domain/services/background_worker.service.dart
@@ -141,7 +141,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
}
@override
- Future<void> onAndroidUpload(int? maxMinutes) async {
+ Future<bool> onAndroidUpload(int? maxMinutes) async {
final hashTimeout = Duration(minutes: _isBackupEnabled ? 3 : 6);
final backupTimeout = maxMinutes != null ? Duration(minutes: maxMinutes - 1) : null;
await _optimizeDB();
@@ -199,7 +199,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
}
}
- Future<void> _backgroundLoop({
+ Future<bool> _backgroundLoop({
required Duration hashTimeout,
required Duration? backupTimeout,
required String debugLabel,
@@ -211,7 +211,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
try {
if (!await _syncAssets(hashTimeout: hashTimeout)) {
_logger.warning("Remote sync did not complete successfully, skipping backup");
- return;
+ return true;
}
final backupFuture = _handleBackup();
@@ -225,12 +225,13 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
});
}
try {
- await backupFuture;
+ return await backupFuture;
} finally {
cancelTimer?.cancel();
}
} catch (error, stack) {
_logger.severe("Failed to complete $debugLabel", error, stack);
+ return true;
} finally {
sw.stop();
_logger.info("$debugLabel completed in ${sw.elapsed.inSeconds}s");
@@ -290,36 +291,41 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
}
}
- Future<void> _handleBackup() async {
- await runZonedGuarded(
- () async {
- if (_isCleanedUp) {
- return;
- }
-
- if (!_isBackupEnabled) {
- _logger.info("Backup is disabled. Skipping backup routine");
- return;
- }
-
- final currentUser = _ref?.read(currentUserProvider);
- if (currentUser == null) {
- _logger.warning("No current user found. Skipping backup from background");
- return;
- }
-
- if (Platform.isIOS) {
- return _ref?.read(backupProvider.notifier).startBackupWithURLSession(currentUser.id);
- }
-
- return _ref
- ?.read(foregroundUploadServiceProvider)
- .uploadCandidates(currentUser.id, _cancellationToken, useSequentialUpload: true);
- },
- (error, stack) {
- dPrint(() => "Error in backup zone $error, $stack");
- },
- );
+ Future<bool> _handleBackup() async {
+ final needsRetry = await runZonedGuarded(() async {
+ if (_isCleanedUp) {
+ return false;
+ }
+
+ if (!_isBackupEnabled) {
+ _logger.info("Backup is disabled. Skipping backup routine");
+ return false;
+ }
+
+ final currentUser = _ref?.read(currentUserProvider);
+ if (currentUser == null) {
+ _logger.warning("No current user found. Skipping backup from background");
+ return false;
+ }
+
+ if (Platform.isIOS) {
+ await _ref?.read(backupProvider.notifier).startBackupWithURLSession(currentUser.id);
+ return false;
+ }
+
+ final uploadService = _ref?.read(foregroundUploadServiceProvider);
+ var failed = 0;
+ await uploadService?.uploadCandidates(
+ currentUser.id,
+ _cancellationToken,
+ callbacks: UploadCallbacks(onError: (_, _) => failed++),
+ useSequentialUpload: true,
+ );
+ _logger.info("Background backup finished, $failed uploads failed");
+ // Retry when at least one upload failed
+ return failed > 0;
+ }, (error, stack) => dPrint(() => "Error in backup zone $error, $stack"));
+ return needsRetry ?? true;
}
Future<bool> _syncAssets({Duration? hashTimeout}) async {
diff --git a/mobile/pigeon/background_worker_api.dart b/mobile/pigeon/background_worker_api.dart
index 06395fae7b..dd62dd069d 100644
--- a/mobile/pigeon/background_worker_api.dart
+++ b/mobile/pigeon/background_worker_api.dart
@@ -46,8 +46,9 @@ abstract class BackgroundWorkerFlutterApi {
void onIosUpload(bool isRefresh, int? maxSeconds);
// Android Only: Called when the Android background upload is triggered
+ // Returns whether the native side needs to retry the run
@async
- void onAndroidUpload(int? maxMinutes);
+ bool onAndroidUpload(int? maxMinutes);
@async
void cancel();