Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos] Coalesce overlapping album refreshes (#14308)

ashil-pilotb225aa8a8314
diff --git a/mobile/apps/photos/changes/coalesce-album-refreshes.md b/mobile/apps/photos/changes/coalesce-album-refreshes.md
new file mode 100644
index 0000000000..377cae4d50
--- /dev/null
+++ b/mobile/apps/photos/changes/coalesce-album-refreshes.md
@@ -0,0 +1 @@
+- Improved backup performance for large photo libraries.
diff --git a/mobile/apps/photos/lib/services/sync/album_refresh_coordinator.dart b/mobile/apps/photos/lib/services/sync/album_refresh_coordinator.dart
new file mode 100644
index 0000000000..a7feed5437
--- /dev/null
+++ b/mobile/apps/photos/lib/services/sync/album_refresh_coordinator.dart
@@ -0,0 +1,76 @@
+import "dart:async";
+
+import "package:logging/logging.dart";
+
+class AlbumRefreshCoordinator {
+ AlbumRefreshCoordinator(this._refresh);
+
+ final Future<bool> Function(
+ bool isFirstSync,
+ void Function(String stage) setStage,
+ )
+ _refresh;
+ final _logger = Logger("AlbumRefreshCoordinator");
+ Future<bool>? _running;
+ Completer<bool>? _pending;
+ int _nextPassID = 0;
+
+ Future<bool> request({bool isFirstSync = false}) {
+ // First import must complete without joining ordinary refresh events.
+ if (isFirstSync) return _runPass(isFirstSync: true);
+ if (_running != null) {
+ return (_pending ??= Completer<bool>()).future;
+ }
+ return _start(Completer<bool>());
+ }
+
+ Future<bool> _start(Completer<bool> completion) {
+ _running = completion.future;
+ unawaited(_completePass(completion));
+ return completion.future;
+ }
+
+ Future<void> _completePass(Completer<bool> completion) async {
+ try {
+ completion.complete(await _runPass());
+ } catch (error, stack) {
+ completion.completeError(error, stack);
+ } finally {
+ final pending = _pending;
+ _pending = null;
+ _running = null;
+ if (pending != null) unawaited(_start(pending));
+ }
+ }
+
+ Future<bool> _runPass({bool isFirstSync = false}) async {
+ final passID = ++_nextPassID;
+ final stopwatch = Stopwatch()..start();
+ var stage = "starting";
+ _logger.info("Album refresh $passID started, firstSync=$isFirstSync");
+ final warning = Timer(const Duration(minutes: 2), () {
+ _logger.warning(
+ "Album refresh $passID still running after ${stopwatch.elapsedMilliseconds}ms, "
+ "stage=$stage, pending=${!isFirstSync && _pending != null}",
+ );
+ });
+ try {
+ final hasUpdated = await _refresh(isFirstSync, (value) => stage = value);
+ _logger.info(
+ "Album refresh $passID completed in ${stopwatch.elapsedMilliseconds}ms, "
+ "updated=$hasUpdated",
+ );
+ return hasUpdated;
+ } catch (error, stack) {
+ _logger.severe(
+ "Album refresh $passID failed after ${stopwatch.elapsedMilliseconds}ms, "
+ "stage=$stage",
+ error,
+ stack,
+ );
+ rethrow;
+ } finally {
+ warning.cancel();
+ }
+ }
+}
diff --git a/mobile/apps/photos/lib/services/sync/local_sync_service.dart b/mobile/apps/photos/lib/services/sync/local_sync_service.dart
index a63ee6b910..374b4b4e33 100644
--- a/mobile/apps/photos/lib/services/sync/local_sync_service.dart
+++ b/mobile/apps/photos/lib/services/sync/local_sync_service.dart
@@ -22,6 +22,7 @@ import "package:photos/models/ignored_file.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/app_lifecycle_service.dart";
import "package:photos/services/ignored_files_service.dart";
+import "package:photos/services/sync/album_refresh_coordinator.dart";
import "package:photos/services/sync/import/diff.dart";
import "package:photos/services/sync/import/local_assets.dart";
import "package:photos/services/sync/import/model.dart";
@@ -39,6 +40,7 @@ class LocalSyncService {
StreamSubscription<PermissionGrantedEvent>? _permissionGrantedSubscription;
bool _isChangeCallbackRegistered = false;
final Lock _lock = Lock();
+ late final _albumRefresh = AlbumRefreshCoordinator(_refreshDeviceFolders);
static const kDbUpdationTimeKey = "db_updation_time";
static const kHasCompletedFirstImportKey = "has_completed_firstImport";
@@ -165,11 +167,17 @@ class LocalSyncService {
_existingSync = null;
}
- Future<bool> _refreshDeviceFolderCountAndCover({
- bool isFirstSync = false,
- }) async {
+ Future<bool> _refreshDeviceFolderCountAndCover({bool isFirstSync = false}) =>
+ _albumRefresh.request(isFirstSync: isFirstSync);
+
+ Future<bool> _refreshDeviceFolders(
+ bool isFirstSync,
+ void Function(String) setStage,
+ ) async {
+ setStage("reading albums and covers");
final List<Tuple2<AssetPathEntity, String>> result =
await getDeviceFolderWithCountAndCoverID();
+ setStage("reading counts and updating database");
final bool hasUpdated = await _db.updateDeviceCoverWithCount(
result,
shouldBackup: backupPreferenceService.hasSelectedAllFoldersForBackup,
@@ -504,7 +512,9 @@ class LocalSyncService {
if (permissionService.hasGrantedLimitedPermissions()) {
unawaited(syncAll());
} else {
- unawaited(sync().then((value) => _refreshDeviceFolderCountAndCover()));
+ unawaited(
+ sync().then((value) => _refreshDeviceFolderCountAndCover().ignore()),
+ );
}
}
}
diff --git a/mobile/apps/photos/test/services/sync/album_refresh_coordinator_test.dart b/mobile/apps/photos/test/services/sync/album_refresh_coordinator_test.dart
new file mode 100644
index 0000000000..d1a5a80495
--- /dev/null
+++ b/mobile/apps/photos/test/services/sync/album_refresh_coordinator_test.dart
@@ -0,0 +1,190 @@
+import "dart:async";
+
+import "package:flutter_test/flutter_test.dart";
+import "package:logging/logging.dart";
+import "package:photos/services/sync/album_refresh_coordinator.dart";
+
+void main() {
+ final records = <LogRecord>[];
+ late StreamSubscription<LogRecord> subscription;
+ late Level previousLevel;
+
+ setUp(() {
+ records.clear();
+ previousLevel = Logger.root.level;
+ Logger.root.level = Level.ALL;
+ subscription = Logger.root.onRecord
+ .where((record) => record.loggerName == "AlbumRefreshCoordinator")
+ .listen(records.add);
+ });
+
+ tearDown(() async {
+ await subscription.cancel();
+ Logger.root.level = previousLevel;
+ });
+
+ test(
+ "coalesces bursts and preserves changes during follow-up passes",
+ () async {
+ final gates = <Completer<bool>>[];
+ final observedCounts = <int>[];
+ var photoCount = 100;
+ var active = 0;
+ var maxActive = 0;
+ final coordinator = AlbumRefreshCoordinator((_, setStage) async {
+ final gate = Completer<bool>();
+ gates.add(gate);
+ observedCounts.add(photoCount);
+ active++;
+ if (active > maxActive) maxActive = active;
+ try {
+ return await gate.future;
+ } finally {
+ active--;
+ }
+ });
+
+ final first = coordinator.request();
+ await _flush();
+ photoCount = 101;
+ final burst = List.generate(200, (_) => coordinator.request());
+ expect(gates, hasLength(1));
+ gates[0].complete(true);
+ await _flush();
+ expect(gates, hasLength(2));
+ expect(await first, isTrue);
+
+ photoCount = 102;
+ final laterScan = coordinator.request();
+ gates[1].complete(false);
+ await _flush();
+ expect(gates, hasLength(3));
+ expect(await Future.wait(burst), everyElement(isFalse));
+ gates[2].complete(true);
+ await Future.wait([first, ...burst, laterScan]);
+
+ expect(observedCounts, [100, 101, 102]);
+ expect(maxActive, 1);
+ expect(records.where((r) => r.level == Level.INFO), hasLength(6));
+ expect(records.where((r) => r.level >= Level.WARNING), isEmpty);
+ },
+ );
+
+ test(
+ "runs requested follow-up after failure and releases the guard",
+ () async {
+ final gates = <Completer<bool>>[];
+ final failure = StateError("album lookup failed");
+ final coordinator = AlbumRefreshCoordinator((_, setStage) {
+ final gate = Completer<bool>();
+ gates.add(gate);
+ return gate.future;
+ });
+
+ final first = coordinator.request();
+ final firstError = expectLater(first, throwsA(same(failure)));
+ await _flush();
+ final joined = coordinator.request();
+ gates[0].completeError(failure);
+ await _flush();
+ expect(gates, hasLength(2));
+ gates[1].complete(false);
+ await firstError;
+ expect(await joined, isFalse);
+
+ final retry = coordinator.request();
+ await _flush();
+ expect(gates, hasLength(3));
+ gates[2].complete(true);
+ expect(await retry, isTrue);
+ expect(records.where((r) => r.level == Level.SEVERE), hasLength(1));
+ },
+ );
+
+ test(
+ "keeps first-import completion separate from ordinary refreshes",
+ () async {
+ final gates = <Completer<bool>>[];
+ final firstSyncFlags = <bool>[];
+ final coordinator = AlbumRefreshCoordinator((isFirstSync, setStage) {
+ firstSyncFlags.add(isFirstSync);
+ final gate = Completer<bool>();
+ gates.add(gate);
+ return gate.future;
+ });
+
+ final firstImport = coordinator.request(isFirstSync: true);
+ final ordinary = coordinator.request();
+ await _flush();
+ final joined = coordinator.request();
+ expect(firstSyncFlags, [true, false]);
+
+ gates[0].complete(true);
+ expect(await firstImport, isTrue);
+ expect(gates[1].isCompleted, isFalse);
+ gates[1].complete(false);
+ await _flush();
+ expect(firstSyncFlags, [true, false, false]);
+ gates[2].complete(false);
+ await Future.wait([ordinary, joined]);
+ },
+ );
+
+ testWidgets(
+ "warns once without retrying and cancels completed-pass warnings",
+ (tester) async {
+ final gates = <Completer<bool>>[];
+ final coordinator = AlbumRefreshCoordinator((_, setStage) {
+ setStage("reading counts and updating database");
+ final gate = Completer<bool>();
+ gates.add(gate);
+ return gate.future;
+ });
+
+ var completed = false;
+ final first = coordinator.request();
+ unawaited(first.then((_) => completed = true));
+ await tester.pump();
+ final pending = coordinator.request();
+ await tester.pump(const Duration(minutes: 2));
+ final warnings = records.where((r) => r.level == Level.WARNING).toList();
+ expect(warnings, hasLength(1));
+ expect(
+ warnings.single.message,
+ contains("Album refresh 1 still running"),
+ );
+ expect(
+ warnings.single.message,
+ contains("reading counts and updating database"),
+ );
+ expect(warnings.single.message, contains("pending=true"));
+ expect(completed, isFalse);
+ expect(gates, hasLength(1));
+ await tester.pump(const Duration(minutes: 5));
+ expect(records.where((r) => r.level == Level.WARNING), hasLength(1));
+
+ gates[0].complete(false);
+ await tester.pump();
+ gates[1].complete(false);
+ await tester.pump();
+ await Future.wait([first, pending]);
+ final retry = coordinator.request();
+ final retryError = expectLater(retry, throwsStateError);
+ await tester.pump();
+ gates[2].completeError(StateError("database update failed"));
+ await tester.pump();
+ await retryError;
+ final afterFailure = coordinator.request();
+ await tester.pump();
+ gates[3].complete(false);
+ await tester.pump();
+ await afterFailure;
+ await tester.pump(const Duration(minutes: 3));
+ expect(records.where((r) => r.level == Level.WARNING), hasLength(1));
+ expect(records.where((r) => r.level == Level.INFO), hasLength(7));
+ expect(records.where((r) => r.level == Level.SEVERE), hasLength(1));
+ },
+ );
+}
+
+Future<void> _flush() => Future<void>.delayed(Duration.zero);