π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][photos] Coalesce overlapping album refreshes (#14308)
ashil-pilotb225aa8a8314
diff --git a/mobile/apps/photos/changes/coalesce-album-refreshes.md b/mobile/apps/photos/changes/coalesce-album-refreshes.mdnew file mode 100644index 0000000000..377cae4d50--- /dev/null+++ b/mobile/apps/photos/changes/coalesce-album-refreshes.md@@ -0,0 +1 @@+- Improved backup performance for large photo libraries.diff --git a/mobile/apps/photos/lib/services/sync/album_refresh_coordinator.dart b/mobile/apps/photos/lib/services/sync/album_refresh_coordinator.dartnew file mode 100644index 0000000000..a7feed5437--- /dev/null+++ b/mobile/apps/photos/lib/services/sync/album_refresh_coordinator.dart@@ -0,0 +1,76 @@+import "dart:async";++import "package:logging/logging.dart";++class AlbumRefreshCoordinator {+ AlbumRefreshCoordinator(this._refresh);++ final Future<bool> Function(+ bool isFirstSync,+ void Function(String stage) setStage,+ )+ _refresh;+ final _logger = Logger("AlbumRefreshCoordinator");+ Future<bool>? _running;+ Completer<bool>? _pending;+ int _nextPassID = 0;++ Future<bool> request({bool isFirstSync = false}) {+ // First import must complete without joining ordinary refresh events.+ if (isFirstSync) return _runPass(isFirstSync: true);+ if (_running != null) {+ return (_pending ??= Completer<bool>()).future;+ }+ return _start(Completer<bool>());+ }++ Future<bool> _start(Completer<bool> completion) {+ _running = completion.future;+ unawaited(_completePass(completion));+ return completion.future;+ }++ Future<void> _completePass(Completer<bool> completion) async {+ try {+ completion.complete(await _runPass());+ } catch (error, stack) {+ completion.completeError(error, stack);+ } finally {+ final pending = _pending;+ _pending = null;+ _running = null;+ if (pending != null) unawaited(_start(pending));+ }+ }++ Future<bool> _runPass({bool isFirstSync = false}) async {+ final passID = ++_nextPassID;+ final stopwatch = Stopwatch()..start();+ var stage = "starting";+ _logger.info("Album refresh $passID started, firstSync=$isFirstSync");+ final warning = Timer(const Duration(minutes: 2), () {+ _logger.warning(+ "Album refresh $passID still running after ${stopwatch.elapsedMilliseconds}ms, "+ "stage=$stage, pending=${!isFirstSync && _pending != null}",+ );+ });+ try {+ final hasUpdated = await _refresh(isFirstSync, (value) => stage = value);+ _logger.info(+ "Album refresh $passID completed in ${stopwatch.elapsedMilliseconds}ms, "+ "updated=$hasUpdated",+ );+ return hasUpdated;+ } catch (error, stack) {+ _logger.severe(+ "Album refresh $passID failed after ${stopwatch.elapsedMilliseconds}ms, "+ "stage=$stage",+ error,+ stack,+ );+ rethrow;+ } finally {+ warning.cancel();+ }+ }+}diff --git a/mobile/apps/photos/lib/services/sync/local_sync_service.dart b/mobile/apps/photos/lib/services/sync/local_sync_service.dartindex a63ee6b910..374b4b4e33 100644--- a/mobile/apps/photos/lib/services/sync/local_sync_service.dart+++ b/mobile/apps/photos/lib/services/sync/local_sync_service.dart@@ -22,6 +22,7 @@ import "package:photos/models/ignored_file.dart";import "package:photos/service_locator.dart";import "package:photos/services/app_lifecycle_service.dart";import "package:photos/services/ignored_files_service.dart";+import "package:photos/services/sync/album_refresh_coordinator.dart";import "package:photos/services/sync/import/diff.dart";import "package:photos/services/sync/import/local_assets.dart";import "package:photos/services/sync/import/model.dart";@@ -39,6 +40,7 @@ class LocalSyncService {StreamSubscription<PermissionGrantedEvent>? _permissionGrantedSubscription;bool _isChangeCallbackRegistered = false;final Lock _lock = Lock();+ late final _albumRefresh = AlbumRefreshCoordinator(_refreshDeviceFolders);static const kDbUpdationTimeKey = "db_updation_time";static const kHasCompletedFirstImportKey = "has_completed_firstImport";@@ -165,11 +167,17 @@ class LocalSyncService {_existingSync = null;}- Future<bool> _refreshDeviceFolderCountAndCover({- bool isFirstSync = false,- }) async {+ Future<bool> _refreshDeviceFolderCountAndCover({bool isFirstSync = false}) =>+ _albumRefresh.request(isFirstSync: isFirstSync);++ Future<bool> _refreshDeviceFolders(+ bool isFirstSync,+ void Function(String) setStage,+ ) async {+ setStage("reading albums and covers");final List<Tuple2<AssetPathEntity, String>> result =await getDeviceFolderWithCountAndCoverID();+ setStage("reading counts and updating database");final bool hasUpdated = await _db.updateDeviceCoverWithCount(result,shouldBackup: backupPreferenceService.hasSelectedAllFoldersForBackup,@@ -504,7 +512,9 @@ class LocalSyncService {if (permissionService.hasGrantedLimitedPermissions()) {unawaited(syncAll());} else {- unawaited(sync().then((value) => _refreshDeviceFolderCountAndCover()));+ unawaited(+ sync().then((value) => _refreshDeviceFolderCountAndCover().ignore()),+ );}}}diff --git a/mobile/apps/photos/test/services/sync/album_refresh_coordinator_test.dart b/mobile/apps/photos/test/services/sync/album_refresh_coordinator_test.dartnew file mode 100644index 0000000000..d1a5a80495--- /dev/null+++ b/mobile/apps/photos/test/services/sync/album_refresh_coordinator_test.dart@@ -0,0 +1,190 @@+import "dart:async";++import "package:flutter_test/flutter_test.dart";+import "package:logging/logging.dart";+import "package:photos/services/sync/album_refresh_coordinator.dart";++void main() {+ final records = <LogRecord>[];+ late StreamSubscription<LogRecord> subscription;+ late Level previousLevel;++ setUp(() {+ records.clear();+ previousLevel = Logger.root.level;+ Logger.root.level = Level.ALL;+ subscription = Logger.root.onRecord+ .where((record) => record.loggerName == "AlbumRefreshCoordinator")+ .listen(records.add);+ });++ tearDown(() async {+ await subscription.cancel();+ Logger.root.level = previousLevel;+ });++ test(+ "coalesces bursts and preserves changes during follow-up passes",+ () async {+ final gates = <Completer<bool>>[];+ final observedCounts = <int>[];+ var photoCount = 100;+ var active = 0;+ var maxActive = 0;+ final coordinator = AlbumRefreshCoordinator((_, setStage) async {+ final gate = Completer<bool>();+ gates.add(gate);+ observedCounts.add(photoCount);+ active++;+ if (active > maxActive) maxActive = active;+ try {+ return await gate.future;+ } finally {+ active--;+ }+ });++ final first = coordinator.request();+ await _flush();+ photoCount = 101;+ final burst = List.generate(200, (_) => coordinator.request());+ expect(gates, hasLength(1));+ gates[0].complete(true);+ await _flush();+ expect(gates, hasLength(2));+ expect(await first, isTrue);++ photoCount = 102;+ final laterScan = coordinator.request();+ gates[1].complete(false);+ await _flush();+ expect(gates, hasLength(3));+ expect(await Future.wait(burst), everyElement(isFalse));+ gates[2].complete(true);+ await Future.wait([first, ...burst, laterScan]);++ expect(observedCounts, [100, 101, 102]);+ expect(maxActive, 1);+ expect(records.where((r) => r.level == Level.INFO), hasLength(6));+ expect(records.where((r) => r.level >= Level.WARNING), isEmpty);+ },+ );++ test(+ "runs requested follow-up after failure and releases the guard",+ () async {+ final gates = <Completer<bool>>[];+ final failure = StateError("album lookup failed");+ final coordinator = AlbumRefreshCoordinator((_, setStage) {+ final gate = Completer<bool>();+ gates.add(gate);+ return gate.future;+ });++ final first = coordinator.request();+ final firstError = expectLater(first, throwsA(same(failure)));+ await _flush();+ final joined = coordinator.request();+ gates[0].completeError(failure);+ await _flush();+ expect(gates, hasLength(2));+ gates[1].complete(false);+ await firstError;+ expect(await joined, isFalse);++ final retry = coordinator.request();+ await _flush();+ expect(gates, hasLength(3));+ gates[2].complete(true);+ expect(await retry, isTrue);+ expect(records.where((r) => r.level == Level.SEVERE), hasLength(1));+ },+ );++ test(+ "keeps first-import completion separate from ordinary refreshes",+ () async {+ final gates = <Completer<bool>>[];+ final firstSyncFlags = <bool>[];+ final coordinator = AlbumRefreshCoordinator((isFirstSync, setStage) {+ firstSyncFlags.add(isFirstSync);+ final gate = Completer<bool>();+ gates.add(gate);+ return gate.future;+ });++ final firstImport = coordinator.request(isFirstSync: true);+ final ordinary = coordinator.request();+ await _flush();+ final joined = coordinator.request();+ expect(firstSyncFlags, [true, false]);++ gates[0].complete(true);+ expect(await firstImport, isTrue);+ expect(gates[1].isCompleted, isFalse);+ gates[1].complete(false);+ await _flush();+ expect(firstSyncFlags, [true, false, false]);+ gates[2].complete(false);+ await Future.wait([ordinary, joined]);+ },+ );++ testWidgets(+ "warns once without retrying and cancels completed-pass warnings",+ (tester) async {+ final gates = <Completer<bool>>[];+ final coordinator = AlbumRefreshCoordinator((_, setStage) {+ setStage("reading counts and updating database");+ final gate = Completer<bool>();+ gates.add(gate);+ return gate.future;+ });++ var completed = false;+ final first = coordinator.request();+ unawaited(first.then((_) => completed = true));+ await tester.pump();+ final pending = coordinator.request();+ await tester.pump(const Duration(minutes: 2));+ final warnings = records.where((r) => r.level == Level.WARNING).toList();+ expect(warnings, hasLength(1));+ expect(+ warnings.single.message,+ contains("Album refresh 1 still running"),+ );+ expect(+ warnings.single.message,+ contains("reading counts and updating database"),+ );+ expect(warnings.single.message, contains("pending=true"));+ expect(completed, isFalse);+ expect(gates, hasLength(1));+ await tester.pump(const Duration(minutes: 5));+ expect(records.where((r) => r.level == Level.WARNING), hasLength(1));++ gates[0].complete(false);+ await tester.pump();+ gates[1].complete(false);+ await tester.pump();+ await Future.wait([first, pending]);+ final retry = coordinator.request();+ final retryError = expectLater(retry, throwsStateError);+ await tester.pump();+ gates[2].completeError(StateError("database update failed"));+ await tester.pump();+ await retryError;+ final afterFailure = coordinator.request();+ await tester.pump();+ gates[3].complete(false);+ await tester.pump();+ await afterFailure;+ await tester.pump(const Duration(minutes: 3));+ expect(records.where((r) => r.level == Level.WARNING), hasLength(1));+ expect(records.where((r) => r.level == Level.INFO), hasLength(7));+ expect(records.where((r) => r.level == Level.SEVERE), hasLength(1));+ },+ );+}++Future<void> _flush() => Future<void>.delayed(Duration.zero);