Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos] Reuse backup preparation checklist across folders (#14357)

ashil-pilotdd9f6e723e83
diff --git a/mobile/apps/photos/changes/shared-backup-checklist.md b/mobile/apps/photos/changes/shared-backup-checklist.md
new file mode 100644
index 0000000000..b180700167
--- /dev/null
+++ b/mobile/apps/photos/changes/shared-backup-checklist.md
@@ -0,0 +1 @@
+- Improved backup preparation speed for libraries with many folders.
diff --git a/mobile/apps/photos/integration_test/shared_checklist_validation_test.dart b/mobile/apps/photos/integration_test/shared_checklist_validation_test.dart
new file mode 100644
index 0000000000..1c35dc4081
--- /dev/null
+++ b/mobile/apps/photos/integration_test/shared_checklist_validation_test.dart
@@ -0,0 +1,19 @@
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/integration_test.dart';
+
+import '../test/services/sync/backup_preparation_test.dart' as regression;
+import '../tool/shared_checklist_validation/measure.dart';
+
+void main() {
+ IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+ late regression.PreparationFixture fixture;
+ regression.registerPreparationTests(
+ registerTest: (name, body) {
+ testWidgets(name, (tester) async => tester.runAsync(body));
+ },
+ onFixtureCreated: (value) => fixture = value,
+ );
+ testWidgets('measure integrated backup preparation', (tester) async {
+ await tester.runAsync(() => measurePreparation(fixture));
+ }, timeout: const Timeout(Duration(minutes: 20)));
+}
diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dart
index 9846a01dc1..b96f23729b 100644
--- a/mobile/apps/photos/lib/db/files_db.dart
+++ b/mobile/apps/photos/lib/db/files_db.dart
@@ -1094,6 +1094,7 @@ class FilesDB with SqlDbBase {
DELETE FROM $filesTable
WHERE $columnLocalID IN ($localIdPlaceholder)
AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)
+ RETURNING $columnLocalID
''', alreadyUploadedLocalIDs);
final removedCount = deleteResult.length;
@@ -1131,18 +1132,32 @@ class FilesDB with SqlDbBase {
return result;
}
- Future<void> setCollectionIDForUnMappedLocalFiles(
+ Future<Set<String>> setCollectionIDForUnMappedLocalFiles(
int collectionID,
- Set<String> localIDs,
- ) async {
+ Set<String> localIDs, {
+ int? trackClaimsForOwner,
+ }) async {
final db = await instance.sqliteAsyncDB;
final inParam = localIDs.map((id) => "'$id'").join(',');
- await db.execute('''
+ final returning = trackClaimsForOwner == null
+ ? ''
+ : 'RETURNING $columnLocalID, $columnOwnerID';
+ final rows = await db.execute('''
UPDATE $filesTable
SET $columnCollectionID = $collectionID
WHERE $columnLocalID IN ($inParam) AND ($columnCollectionID IS NULL OR
- $columnCollectionID = -1);
+ $columnCollectionID = -1)
+ $returning;
''');
+ if (trackClaimsForOwner == null || collectionID == -1) {
+ return const {};
+ }
+ return {
+ for (final row in rows)
+ if (row[columnOwnerID] == null ||
+ row[columnOwnerID] == trackClaimsForOwner)
+ row[columnLocalID] as String,
+ };
}
Future<void> markFilesForReUpload(
diff --git a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart
index 7d754e9b15..d0349a47c3 100644
--- a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart
+++ b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart
@@ -45,10 +45,10 @@ import 'package:shared_preferences/shared_preferences.dart';
class RemoteSyncService {
final _logger = Logger("RemoteSyncService");
- final _db = FilesDB.instance;
+ final FilesDB _db;
final FileUploader _uploader = FileUploader.instance;
- final Configuration _config = Configuration.instance;
- final CollectionsService _collectionsService = CollectionsService.instance;
+ final Configuration _config;
+ final CollectionsService _collectionsService;
final DiffFetcher _diffFetcher = DiffFetcher();
final PhotoManagerPlugin _photoManagerPlugin = PhotoManagerPlugin();
final LocalFileUpdateService _localFileUpdateService =
@@ -78,7 +78,19 @@ class RemoteSyncService {
static final RemoteSyncService instance =
RemoteSyncService._privateConstructor();
- RemoteSyncService._privateConstructor();
+ RemoteSyncService._privateConstructor()
+ : _db = FilesDB.instance,
+ _config = Configuration.instance,
+ _collectionsService = CollectionsService.instance;
+
+ @visibleForTesting
+ RemoteSyncService.forTesting({
+ required FilesDB filesDB,
+ required Configuration configuration,
+ required CollectionsService collectionsService,
+ }) : _db = filesDB,
+ _config = configuration,
+ _collectionsService = collectionsService;
void init(SharedPreferences preferences) {
_prefs = preferences;
@@ -399,19 +411,36 @@ class RemoteSyncService {
_logger.info("Found ${newerLocalIDs.length} newer files");
}
+ var remainingIfMissingFolders = deviceCollections
+ .where((folder) => folder.uploadStrategy == UploadStrategy.ifMissing)
+ .length;
+ Set<String>? claimedLocalIDs;
bool moreFilesMarkedForBackup = false;
for (final deviceCollection in deviceCollections) {
final Set<String> localIDsToSync =
pathIdToLocalIDs[deviceCollection.id]?.toSet() ?? {};
if (deviceCollection.uploadStrategy == UploadStrategy.ifMissing) {
- final Set<String> alreadyClaimedLocalIDs = await _db
- .getLocalIDsMarkedForOrAlreadyUploaded(ownerID);
+ remainingIfMissingFolders--;
+ final alreadyClaimedLocalIDs =
+ claimedLocalIDs ??
+ await _db.getLocalIDsMarkedForOrAlreadyUploaded(ownerID);
+ claimedLocalIDs = remainingIfMissingFolders > 0
+ ? alreadyClaimedLocalIDs
+ : null;
localIDsToSync.removeAll(alreadyClaimedLocalIDs);
if (alreadyClaimedLocalIDs.isNotEmpty && !_hasCleanupStaleEntry) {
try {
- await _db.removeQueuedLocalFiles(alreadyClaimedLocalIDs, ownerID);
+ final removed = await _db.removeQueuedLocalFiles(
+ alreadyClaimedLocalIDs,
+ ownerID,
+ );
+ if (removed > 0) {
+ // Cleanup can remove the last mapped row for a local ID.
+ claimedLocalIDs = null;
+ }
_hasCleanupStaleEntry = true;
} catch (e, s) {
+ claimedLocalIDs = null;
_logger.severe("removeQueuedLocalFiles failed", e, s);
}
}
@@ -437,10 +466,13 @@ class RemoteSyncService {
}
moreFilesMarkedForBackup = true;
- await _db.setCollectionIDForUnMappedLocalFiles(
- collectionID,
- localIDsToSync,
- );
+ final newlyClaimedLocalIDs = await _db
+ .setCollectionIDForUnMappedLocalFiles(
+ collectionID,
+ localIDsToSync,
+ trackClaimsForOwner: claimedLocalIDs == null ? null : ownerID,
+ );
+ claimedLocalIDs?.addAll(newlyClaimedLocalIDs);
// Re-enabling a backup folder must not duplicate its existing rows.
final Set<String> existingMapping = await _db
@@ -464,11 +496,23 @@ class RemoteSyncService {
final filesWithCollectionID = await _db.getLocalFiles(
localIDsToSync.toList(),
);
+ if (deviceCollection.uploadStrategy == UploadStrategy.ifMissing) {
+ final claimedSinceRead = {
+ for (final file in filesWithCollectionID)
+ if (file.collectionID != null &&
+ file.collectionID != -1 &&
+ (file.ownerID == null || file.ownerID == ownerID))
+ file.localID!,
+ };
+ localIDsToSync.removeAll(claimedSinceRead);
+ claimedLocalIDs?.addAll(claimedSinceRead);
+ }
final List<EnteFile> newFilesToInsert = [];
final Set<String> fileFoundForLocalIDs = {};
for (var existingFile in filesWithCollectionID) {
final String localID = existingFile.localID!;
- if (!fileFoundForLocalIDs.contains(localID)) {
+ if (localIDsToSync.contains(localID) &&
+ !fileFoundForLocalIDs.contains(localID)) {
existingFile.generatedID = null;
existingFile.collectionID = collectionID;
existingFile.uploadedFileID = null;
@@ -478,6 +522,7 @@ class RemoteSyncService {
}
}
await _db.insertMultiple(newFilesToInsert);
+ claimedLocalIDs?.addAll(fileFoundForLocalIDs);
if (fileFoundForLocalIDs.length != localIDsToSync.length) {
_logger.warning(
"mismatch in num of filesToSync ${localIDsToSync.length} to "
diff --git a/mobile/apps/photos/test/services/sync/backup_preparation_test.dart b/mobile/apps/photos/test/services/sync/backup_preparation_test.dart
new file mode 100644
index 0000000000..3516ac8eaa
--- /dev/null
+++ b/mobile/apps/photos/test/services/sync/backup_preparation_test.dart
@@ -0,0 +1,530 @@
+import 'dart:io';
+
+import 'package:dio/dio.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:package_info_plus/package_info_plus.dart';
+import 'package:path_provider_platform_interface/path_provider_platform_interface.dart';
+import 'package:photos/core/configuration.dart';
+import 'package:photos/db/common/conflict_algo.dart';
+import 'package:photos/db/files_db.dart';
+import 'package:photos/models/collection/collection.dart';
+import 'package:photos/models/file/file.dart';
+import 'package:photos/module/upload/service/file_uploader.dart';
+import 'package:photos/service_locator.dart';
+import 'package:photos/services/collections_service.dart';
+import 'package:photos/services/sync/remote_sync_service.dart';
+import 'package:shared_preferences/shared_preferences.dart';
+import 'package:sqlite_async/sqlite_async.dart';
+
+void main() => registerPreparationTests();
+
+void registerPreparationTests({
+ void Function(String, Future<void> Function())? registerTest,
+ void Function(PreparationFixture)? onFixtureCreated,
+}) {
+ final runTest =
+ registerTest ??
+ (String name, Future<void> Function() body) {
+ test(name, body);
+ };
+ TestWidgetsFlutterBinding.ensureInitialized();
+ late PreparationFixture fixture;
+ setUpAll(() async {
+ fixture = await PreparationFixture.create();
+ onFixtureCreated?.call(fixture);
+ });
+ setUp(() async => fixture.reset());
+ tearDownAll(() async => fixture.close());
+
+ runTest(
+ 'shares claims across overlapping folders and starts fresh next pass',
+ () async {
+ await fixture.file('local');
+ await fixture.folder('first', 1000, ['local']);
+ await fixture.folder('second', 1001, ['local']);
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(fixture.db.mappingCalls, 1);
+ expect(await fixture.pendingIDs(), ['local']);
+
+ await fixture.file('new');
+ await fixture.raw.executeBatch(
+ 'INSERT INTO device_files(id,path_id) VALUES (?,?)',
+ [
+ ['new', 'first'],
+ ['new', 'second'],
+ ],
+ );
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(fixture.db.mappingCalls, 1);
+ expect(await fixture.pendingIDs(), ['local', 'new']);
+ },
+ );
+
+ runTest(
+ 'single-folder preparation does not request mapping return rows',
+ () async {
+ await fixture.file('local');
+ await fixture.folder('only', 1000, ['local']);
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(fixture.db.trackedMappings, 0);
+ expect(await fixture.pendingIDs(), ['local']);
+ },
+ );
+
+ runTest(
+ 'foreign mappings do not prematurely claim an owned backup',
+ () async {
+ await fixture.file(
+ 'foreign_mapped',
+ owner: 99,
+ uploaded: 456,
+ collection: 9000,
+ );
+ await fixture.file('foreign_unmapped', owner: 99, uploaded: 457);
+ await fixture.file('current', owner: 42, uploaded: 458);
+ await fixture.file('local');
+ await fixture.file('second_only');
+ final shared = ['foreign_mapped', 'foreign_unmapped', 'current', 'local'];
+ await fixture.folder('second', 1001, [...shared, 'second_only']);
+ await fixture.folder('first', 1000, shared);
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(fixture.db.claimedRows, 0);
+ expect(await fixture.pendingIDs(), [
+ 'foreign_mapped',
+ 'foreign_unmapped',
+ 'local',
+ 'second_only',
+ ]);
+ final row = await fixture.raw.get('''SELECT collection_id FROM files
+ WHERE local_id = 'foreign_unmapped' AND owner_id IS NULL''');
+ expect(row['collection_id'], 1001);
+ expect(
+ (await fixture.raw.get(
+ 'SELECT COUNT(*) AS n FROM files WHERE owner_id=99',
+ ))['n'],
+ 2,
+ );
+ },
+ );
+
+ runTest(
+ 'Always mappings and copies update claims without changing only-new filtering',
+ () async {
+ await backupPreferenceService.setOnlyNewSinceEpoch(100);
+ await fixture.file('old', created: 99);
+ await fixture.file('new', created: 100);
+ await fixture.file(
+ 'uploaded',
+ owner: 42,
+ uploaded: 456,
+ collection: 9000,
+ );
+ await fixture.file('always_new');
+ await fixture.folder('first', 1000, ['old', 'new']);
+ await fixture.folder('always', 1001, [
+ 'uploaded',
+ 'new',
+ 'always_new',
+ ], strategy: 1);
+ await fixture.folder('last', 1002, [
+ 'old',
+ 'new',
+ 'uploaded',
+ 'always_new',
+ ]);
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(fixture.db.mappingCalls, 2);
+ expect(await fixture.pendingIDs(), [
+ 'always_new',
+ 'new',
+ 'new',
+ 'uploaded',
+ ]);
+ expect(
+ (await fixture.raw.get(
+ "SELECT collection_id FROM files WHERE local_id='old'",
+ ))['collection_id'],
+ isNull,
+ );
+ },
+ );
+
+ for (final failsAfterDelete in [false, true]) {
+ runTest(
+ 'refreshes claims after ${failsAfterDelete ? 'partially failed' : 'successful'} stale cleanup',
+ () async {
+ await fixture.file('edge', owner: 42, uploaded: 456);
+ await fixture.file('edge', collection: 9000);
+ await fixture.file(
+ 'backed_up',
+ owner: 42,
+ uploaded: 457,
+ collection: 1001,
+ );
+ await fixture.folder('second', 1001, ['edge', 'backed_up']);
+ await fixture.folder('first', 1000, ['edge']);
+ fixture.db.failCleanupOnce = failsAfterDelete;
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 2);
+ expect(await fixture.pendingIDs(), isEmpty);
+ final row = await fixture.raw.get(
+ "SELECT collection_id FROM files WHERE local_id='edge'",
+ );
+ expect(row['collection_id'], 1001);
+ },
+ );
+ }
+
+ runTest(
+ 'missing file rows are not claimed before a later folder can import them',
+ () async {
+ await fixture.file('second_only');
+ await fixture.folder('second', 1001, ['late', 'second_only']);
+ await fixture.folder('first', 1000, ['late']);
+ fixture.db.afterLocalRead = (_) async {
+ fixture.db.afterLocalRead = null;
+ await fixture.file('late');
+ };
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(await fixture.pendingIDs(), ['late', 'second_only']);
+ expect(
+ (await fixture.raw.get(
+ "SELECT collection_id FROM files WHERE local_id='late'",
+ ))['collection_id'],
+ 1001,
+ );
+ },
+ );
+
+ runTest(
+ 'an intervening owned mapping is not cloned from a stale checklist',
+ () async {
+ await fixture.file('raced');
+ // Keep this row mapped so it reaches the clone filter before being queued.
+ await fixture.file(
+ 'unaffected',
+ owner: 99,
+ uploaded: 457,
+ collection: 9001,
+ );
+ await fixture.file(
+ 'backed_up',
+ owner: 42,
+ uploaded: 458,
+ collection: 9002,
+ );
+ await fixture.folder('third', 1002, ['raced', 'unaffected', 'backed_up']);
+ await fixture.folder('empty', 1000, []);
+ await fixture.folder('second', 1001, ['raced', 'unaffected']);
+ fixture.db.beforeMapping = (_, _) async {
+ fixture.db.beforeMapping = null;
+ await fixture.raw.execute(
+ "UPDATE files SET collection_id=9000 WHERE local_id='raced'",
+ );
+ };
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(fixture.db.mappingCalls, 1);
+ expect(await fixture.pendingIDs(), ['raced', 'unaffected']);
+ expect(
+ (await fixture.raw.get(
+ "SELECT collection_id FROM files WHERE local_id='raced'",
+ ))['collection_id'],
+ 9000,
+ );
+ expect(
+ (await fixture.raw.get(
+ "SELECT collection_id FROM files WHERE local_id='unaffected' AND owner_id IS NULL",
+ ))['collection_id'],
+ 1001,
+ );
+ },
+ );
+
+ runTest(
+ 'missing collections do not claim files and disabled folders stay unmapped',
+ () async {
+ await fixture.file('local');
+ await fixture.file('disabled');
+ await fixture.file('second_only');
+ await fixture.folder('second', 1001, ['local', 'second_only']);
+ await fixture.folder('missing', 1000, ['local']);
+ await fixture.folder('disabled', 1002, ['disabled'], selected: false);
+ fixture.collections.missing.add(1000);
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(await fixture.pendingIDs(), ['local', 'second_only']);
+ expect(
+ (await fixture.raw.get(
+ "SELECT collection_id FROM files WHERE local_id='disabled'",
+ ))['collection_id'],
+ isNull,
+ );
+ },
+ );
+
+ runTest(
+ 'retained empty records do not repeat the account-wide read',
+ () async {
+ await fixture.file(
+ 'uploaded',
+ owner: 42,
+ uploaded: 456,
+ collection: 9000,
+ );
+ for (var i = 0; i < 100; i++) {
+ await fixture.folder(
+ 'folder-$i',
+ 1000 + i,
+ i == 99 ? ['uploaded'] : [],
+ );
+ }
+ await fixture.prepare();
+ expect(fixture.db.claimedReads, 1);
+ expect(await fixture.pendingIDs(), isEmpty);
+ },
+ );
+}
+
+class PreparationFixture {
+ PreparationFixture._(
+ this.directory,
+ this.previousPaths,
+ this.previousPrint,
+ this.raw,
+ );
+ final Directory directory;
+ final PathProviderPlatform previousPaths;
+ final DebugPrintCallback previousPrint;
+ final SqliteDatabase raw;
+ final db = PreparationDatabase();
+ final collections = _Collections();
+ late RemoteSyncService service;
+
+ static Future<PreparationFixture> create() async {
+ final directory = await Directory.systemTemp.createTemp(
+ 'backup_preparation_',
+ );
+ final previousPaths = PathProviderPlatform.instance;
+ final previousPrint = debugPrint;
+ PathProviderPlatform.instance = _Paths(directory.path);
+ debugPrint = (String? message, {int? wrapWidth}) {};
+ // FileUploader's normal constructor requires an initialized network client.
+ FileUploader.instance = _Uploader();
+ SharedPreferences.setMockInitialValues({
+ 'has_selected_all_folders_for_backup': true,
+ });
+ final prefs = await SharedPreferences.getInstance();
+ ServiceLocator.instance.init(
+ prefs,
+ Dio(),
+ Dio(),
+ Dio(),
+ PackageInfo(
+ appName: 'test',
+ packageName: 'test',
+ version: '1',
+ buildNumber: '1',
+ ),
+ );
+ final raw = await FilesDB.instance.sqliteAsyncDB;
+ return PreparationFixture._(directory, previousPaths, previousPrint, raw);
+ }
+
+ Future<void> reset() async {
+ for (final table in ['files', 'device_files', 'device_collections']) {
+ await raw.execute('DELETE FROM $table');
+ }
+ await backupPreferenceService.clearOnlyNewSinceEpoch();
+ db.beforeMapping = null;
+ db.afterLocalRead = null;
+ db.failCleanupOnce = false;
+ collections.missing.clear();
+ service = RemoteSyncService.forTesting(
+ filesDB: db,
+ configuration: _Configuration(),
+ collectionsService: collections,
+ );
+ }
+
+ Future<Map<String, Object?>> prepare() async {
+ db.claimedReads = 0;
+ db.claimedRows = 0;
+ db.mappingCalls = 0;
+ db.trackedMappings = 0;
+ db.claimedQueryMs = 0;
+ final watch = Stopwatch()..start();
+ await service.syncDeviceCollectionFilesForUpload();
+ return {
+ 'elapsedMs': watch.elapsedMicroseconds / 1000,
+ 'claimedReads': db.claimedReads,
+ 'claimedRows': db.claimedRows,
+ 'claimedQueryMs': db.claimedQueryMs,
+ 'mappingCalls': db.mappingCalls,
+ 'trackedMappings': db.trackedMappings,
+ };
+ }
+
+ Future<void> file(
+ String id, {
+ int? owner,
+ int? uploaded,
+ int? collection,
+ int created = 100,
+ }) => raw.execute(
+ '''INSERT INTO files
+ (local_id,owner_id,uploaded_file_id,collection_id,title,file_type,creation_time,modification_time)
+ VALUES (?,?,?,?,?,0,?,100)''',
+ [id, owner, uploaded, collection, '$id.jpg', created],
+ );
+
+ Future<void> folder(
+ String id,
+ int collection,
+ List<String> ids, {
+ int strategy = 0,
+ bool selected = true,
+ }) async {
+ await raw.execute(
+ '''INSERT INTO device_collections
+ (id,name,count,collection_id,upload_strategy,should_backup) VALUES (?,?,?,?,?,?)''',
+ [id, id, ids.length, collection, strategy, selected ? 1 : 0],
+ );
+ if (ids.isNotEmpty) {
+ await raw.executeBatch(
+ 'INSERT INTO device_files (id,path_id) VALUES (?,?)',
+ [
+ for (final localID in ids) [localID, id],
+ ],
+ );
+ }
+ }
+
+ Future<List<String>> pendingIDs() async {
+ final rows = await raw.getAll(
+ '''SELECT local_id FROM files WHERE collection_id IS NOT NULL AND collection_id !=-1
+ AND (uploaded_file_id IS NULL OR uploaded_file_id=-1) AND (owner_id IS NULL OR owner_id=42) ORDER BY local_id''',
+ );
+ return rows.map((r) => r['local_id'] as String).toList();
+ }
+
+ Future<void> close() async {
+ await raw.close();
+ PathProviderPlatform.instance = previousPaths;
+ debugPrint = previousPrint;
+ await directory.delete(recursive: true);
+ }
+}
+
+class PreparationDatabase extends Fake implements FilesDB {
+ final FilesDB real = FilesDB.instance;
+ double claimedQueryMs = 0;
+ int claimedReads = 0;
+ int claimedRows = 0;
+ int mappingCalls = 0;
+ int trackedMappings = 0;
+ bool failCleanupOnce = false;
+ Future<void> Function(int, Set<String>)? beforeMapping;
+ Future<void> Function(List<String>)? afterLocalRead;
+ @override
+ Future<SqliteDatabase> get sqliteAsyncDB => real.sqliteAsyncDB;
+ @override
+ Future<Set<String>> getLocalIDsMarkedForOrAlreadyUploaded(int ownerID) async {
+ claimedReads++;
+ final watch = Stopwatch()..start();
+ final result = await real.getLocalIDsMarkedForOrAlreadyUploaded(ownerID);
+ claimedQueryMs += watch.elapsedMicroseconds / 1000;
+ claimedRows += result.length;
+ return result;
+ }
+
+ @override
+ Future<int> removeQueuedLocalFiles(Set<String> ids, int ownerID) async {
+ final result = await real.removeQueuedLocalFiles(ids, ownerID);
+ if (failCleanupOnce) {
+ failCleanupOnce = false;
+ throw StateError('cleanup failed after deletion');
+ }
+ return result;
+ }
+
+ @override
+ Future<Set<String>> setCollectionIDForUnMappedLocalFiles(
+ int id,
+ Set<String> ids, {
+ int? trackClaimsForOwner,
+ }) async {
+ mappingCalls++;
+ if (trackClaimsForOwner != null) trackedMappings++;
+ await beforeMapping?.call(id, ids);
+ return real.setCollectionIDForUnMappedLocalFiles(
+ id,
+ ids,
+ trackClaimsForOwner: trackClaimsForOwner,
+ );
+ }
+
+ @override
+ Future<Set<String>> getLocalFileIDsForCollection(int id) =>
+ real.getLocalFileIDsForCollection(id);
+ @override
+ Future<Set<String>> getAllLocalIDsNewerThan(int time) =>
+ real.getAllLocalIDsNewerThan(time);
+ @override
+ Future<List<EnteFile>> getLocalFiles(
+ List<String> ids, {
+ bool dedupeByLocalID = false,
+ }) async {
+ final result = await real.getLocalFiles(
+ ids,
+ dedupeByLocalID: dedupeByLocalID,
+ );
+ await afterLocalRead?.call(ids);
+ return result;
+ }
+
+ @override
+ Future<void> insertMultiple(
+ List<EnteFile> files, {
+ SqliteAsyncConflictAlgorithm conflictAlgorithm =
+ SqliteAsyncConflictAlgorithm.replace,
+ }) => real.insertMultiple(files, conflictAlgorithm: conflictAlgorithm);
+}
+
+class _Configuration extends Fake implements Configuration {
+ @override
+ int? getUserID() => 42;
+}
+
+class _Uploader extends Fake implements FileUploader {}
+
+class _Collections extends Fake implements CollectionsService {
+ final missing = <int>{};
+ @override
+ Collection? getCollectionByID(int id) =>
+ missing.contains(id) ? null : _Collection(id);
+ @override
+ Future<Collection> fetchCollectionByID(int id) async => _Collection(id);
+}
+
+class _Collection extends Fake implements Collection {
+ _Collection(this.id);
+ @override
+ final int id;
+ @override
+ bool get isDeleted => false;
+}
+
+class _Paths extends PathProviderPlatform {
+ _Paths(this.path);
+ final String path;
+ @override
+ Future<String?> getApplicationDocumentsPath() async => path;
+}
diff --git a/mobile/apps/photos/tool/shared_checklist_validation/measure.dart b/mobile/apps/photos/tool/shared_checklist_validation/measure.dart
new file mode 100644
index 0000000000..2bac8404ba
--- /dev/null
+++ b/mobile/apps/photos/tool/shared_checklist_validation/measure.dart
@@ -0,0 +1,210 @@
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+import 'dart:math';
+
+import 'package:flutter_test/flutter_test.dart';
+
+import '../../test/services/sync/backup_preparation_test.dart';
+
+Future<void> measurePreparation(PreparationFixture fixture) async {
+ const cases = [
+ (
+ name: 'empty',
+ folders: 0,
+ active: 0,
+ assets: 0,
+ uploaded: 0,
+ queued: 0,
+ overlap: false,
+ ),
+ (
+ name: 'one_mixed',
+ folders: 1,
+ active: 1,
+ assets: 10000,
+ uploaded: 4000,
+ queued: 3000,
+ overlap: false,
+ ),
+ (
+ name: 'two_mixed',
+ folders: 2,
+ active: 2,
+ assets: 10000,
+ uploaded: 4000,
+ queued: 3000,
+ overlap: false,
+ ),
+ (
+ name: 'typical_new',
+ folders: 50,
+ active: 50,
+ assets: 10000,
+ uploaded: 1000,
+ queued: 0,
+ overlap: false,
+ ),
+ (
+ name: 'two_large',
+ folders: 2,
+ active: 2,
+ assets: 100000,
+ uploaded: 40000,
+ queued: 30000,
+ overlap: false,
+ ),
+ (
+ name: 'many_backed',
+ folders: 500,
+ active: 500,
+ assets: 10000,
+ uploaded: 10000,
+ queued: 0,
+ overlap: false,
+ ),
+ (
+ name: 'both_large',
+ folders: 100,
+ active: 100,
+ assets: 100000,
+ uploaded: 10000,
+ queued: 0,
+ overlap: false,
+ ),
+ (
+ name: 'customer_scale',
+ folders: 1903,
+ active: 1903,
+ assets: 37302,
+ uploaded: 15000,
+ queued: 22302,
+ overlap: false,
+ ),
+ (
+ name: 'retained',
+ folders: 1903,
+ active: 10,
+ assets: 37302,
+ uploaded: 15000,
+ queued: 22302,
+ overlap: false,
+ ),
+ (
+ name: 'overlap',
+ folders: 5,
+ active: 5,
+ assets: 10000,
+ uploaded: 1000,
+ queued: 0,
+ overlap: true,
+ ),
+ ];
+ var record = 0;
+ for (final c in cases) {
+ await fixture.reset();
+ await fixture.raw.writeTransaction((tx) async {
+ await tx.executeBatch(
+ '''INSERT INTO device_collections
+ (id,name,count,collection_id,upload_strategy,should_backup)
+ VALUES (?,?,?,?,0,1)''',
+ [
+ for (var i = 0; i < c.folders; i++)
+ [
+ 'folder-$i',
+ 'Folder $i',
+ i < c.active
+ ? (c.assets ~/ c.active + (i < c.assets % c.active ? 1 : 0)) *
+ (c.overlap ? 2 : 1)
+ : 0,
+ 1000 + i,
+ ],
+ ],
+ );
+ await tx.executeBatch(
+ 'INSERT INTO device_files(id,path_id) VALUES (?,?)',
+ [
+ for (var i = 0; i < c.assets; i++) ...[
+ ['${1000000000 + i}', 'folder-${i % c.active}'],
+ if (c.overlap)
+ ['${1000000000 + i}', 'folder-${(i + 1) % c.active}'],
+ ],
+ ],
+ );
+ await tx.executeBatch(
+ '''INSERT INTO files
+ (local_id,uploaded_file_id,owner_id,collection_id,title,file_type,creation_time,modification_time)
+ VALUES (?,?,?,?,?,0,100,100)''',
+ [
+ for (var i = 0; i < c.assets; i++)
+ [
+ '${1000000000 + i}',
+ i < c.uploaded ? 2000000000 + i : null,
+ i < c.uploaded ? 42 : null,
+ i < c.uploaded + c.queued ? 1000 + i % c.active : null,
+ 'fixture_$i.jpg',
+ ],
+ ],
+ );
+ });
+ for (final pass in [
+ 'first',
+ 'subsequent',
+ if (c.name == 'many_backed') 'after_discovery',
+ ]) {
+ if (pass == 'after_discovery') {
+ await fixture.file('new_photo');
+ await fixture.raw.execute(
+ 'INSERT INTO device_files(id,path_id) VALUES (?,?)',
+ ['new_photo', 'folder-0'],
+ );
+ await fixture.raw.execute(
+ "UPDATE device_collections SET count=count+1 WHERE id='folder-0'",
+ );
+ }
+ final baselineRss = ProcessInfo.currentRss;
+ var peakRss = baselineRss;
+ final timer = Timer.periodic(const Duration(milliseconds: 10), (_) {
+ peakRss = max(peakRss, ProcessInfo.currentRss);
+ });
+ late Map<String, Object?> result;
+ try {
+ result = await fixture.prepare();
+ } finally {
+ timer.cancel();
+ peakRss = max(peakRss, ProcessInfo.currentRss);
+ }
+ expect(fixture.db.claimedReads, c.folders == 0 ? 0 : 1);
+ final pending = await fixture.pendingIDs();
+ expect(
+ pending.length,
+ c.assets - c.uploaded + (pass == 'after_discovery' ? 1 : 0),
+ );
+ expect(pending.toSet().length, pending.length);
+ final shape = await fixture.raw.get(
+ 'SELECT COUNT(DISTINCT local_id) AS n FROM files',
+ );
+ expect(shape['n'], c.assets + (pass == 'after_discovery' ? 1 : 0));
+ final encoded = jsonEncode({
+ 'case': c.name,
+ 'folders': c.folders,
+ 'populated': c.active,
+ 'assets': c.assets,
+ 'pass': pass,
+ ...result,
+ 'baselineRssBytes': baselineRss,
+ 'peakSampledRssBytes': peakRss,
+ 'pending': pending.length,
+ 'correctness': 'passed',
+ });
+ final parts = (encoded.length / 650).ceil();
+ for (var part = 0; part < parts; part++) {
+ // ignore: avoid_print
+ print(
+ 'SHARED_CHECKLIST_PART $record $part $parts ${encoded.substring(part * 650, min((part + 1) * 650, encoded.length))}',
+ );
+ }
+ record++;
+ }
+ }
+}