π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Speed up offline metadata processing
Neeraj Guptae4073a933c66
diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dartindex 546fb1ee5c..5a00e29802 100644--- a/mobile/apps/photos/lib/db/files_db.dart+++ b/mobile/apps/photos/lib/db/files_db.dart@@ -960,20 +960,31 @@ class FilesDB with SqlDbBase {Future<List<EnteFile>> getUnUploadedLocalFilesPendingOfflineProcessing(int processingVersion, {- int? limit,+ required int limit,+ ({int creationTime, int generatedID})? cursor,}) async {final db = await instance.sqliteAsyncDB;final args = <Object?>[processingVersion];var query =- 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID IS '+ 'SELECT * FROM $filesTable WHERE $columnGeneratedID IN ('+ 'SELECT MAX($columnGeneratedID) FROM $filesTable WHERE '+ '($columnUploadedFileID IS ''NULL OR $columnUploadedFileID IS -1) AND $columnLocalID IS NOT NULL ''AND $columnLocalID IS NOT -1 AND ($columnMetadataVersion IS NULL OR '- '$columnMetadataVersion < ?) GROUP BY $columnLocalID ORDER BY '- '$columnCreationTime DESC';- if (limit != null) {- query += ' LIMIT ?';- args.add(limit);- }+ '$columnMetadataVersion < ?) GROUP BY $columnLocalID)';+ if (cursor != null) {+ query +=+ ' AND ($columnCreationTime < ? OR ($columnCreationTime = ? AND '+ '$columnGeneratedID < ?))';+ args.addAll([+ cursor.creationTime,+ cursor.creationTime,+ cursor.generatedID,+ ]);+ }+ query +=+ ' ORDER BY $columnCreationTime DESC, $columnGeneratedID DESC LIMIT ?';+ args.add(limit);final results = await db.getAll(query, args);return convertToFiles(results);}diff --git a/mobile/apps/photos/lib/module/metadata/exif.dart b/mobile/apps/photos/lib/module/metadata/exif.dartindex ae3148bbe0..1588ec5771 100644--- a/mobile/apps/photos/lib/module/metadata/exif.dart+++ b/mobile/apps/photos/lib/module/metadata/exif.dart@@ -284,15 +284,14 @@ Location? locationFromExif(Map<String, IfdTag> exif) {}}-Future<Map<String, IfdTag>> _readExifArgs(Map<String, dynamic> args) {+Future<Map<String, IfdTag>> _readExifArgs(Map<String, dynamic> args) async {final file = args["file"] as File;- return FileRASource.loadFile(file).then((src) async {- try {- return _normalizeExifResult(await readExifFromSource(src));- } finally {- await src.close();- }- });+ final src = ReadAheadRASource(await FileRASource.loadFile(file));+ try {+ return _normalizeExifResult(await readExifFromSource(src));+ } finally {+ await src.close();+ }}Future<Map<String, IfdTag>> readExifAsync(File file) {diff --git a/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dart b/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dartindex 5fe78bdd38..f817dbcd33 100644--- a/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dart+++ b/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dart@@ -7,6 +7,7 @@ import "package:photos/core/event_bus.dart";import "package:photos/db/files_db.dart";import "package:photos/events/files_updated_event.dart";import "package:photos/events/local_photos_updated_event.dart";+import "package:photos/main.dart" show isProcessBg;import "package:photos/models/file/file.dart";import "package:photos/models/location/location.dart";import "package:photos/module/download/file.dart";@@ -14,6 +15,7 @@ import "package:photos/module/metadata/exif.dart";import "package:photos/module/metadata/local_file.dart";import 'package:photos/module/metadata/location.dart';import "package:photos/service_locator.dart";+import "package:photos/services/process_activity.dart";class OfflineImportMetadataService {static const kProcessingVersion = 1;@@ -28,10 +30,7 @@ class OfflineImportMetadataService {static final instance = OfflineImportMetadataService._privateConstructor();- Future<void> processPendingFiles({- int batchSize = kDefaultBatchSize,- int maxBatches = 4,- }) async {+ Future<void> processPendingFiles({int batchSize = kDefaultBatchSize}) async {if (!Platform.isAndroid || !isLocalGalleryMode) {return;}@@ -42,19 +41,24 @@ class OfflineImportMetadataService {_running = Completer<void>();try {- for (var batch = 0; batch < maxBatches; batch++) {- if (!isLocalGalleryMode) {- _logger.info("Offline mode disabled, stopping metadata processing");+ ({int creationTime, int generatedID})? cursor;+ while (isLocalGalleryMode) {+ if (isProcessBg && await isForegroundEngineActive()) {+ _logger.info("Foreground active, stopping background processing");return;}-final files = await _db.getUnUploadedLocalFilesPendingOfflineProcessing(kProcessingVersion,limit: batchSize,+ cursor: cursor,);if (files.isEmpty) {return;}+ cursor = (+ creationTime: files.last.creationTime!,+ generatedID: files.last.generatedID!,+ );final updatedFiles = <EnteFile>[];for (final file in files) {@@ -87,6 +91,7 @@ class OfflineImportMetadataService {return;}}+ _logger.info("Offline mode disabled, stopping metadata processing");} finally {_running?.complete();_running = null;diff --git a/mobile/apps/photos/lib/services/sync/sync_service.dart b/mobile/apps/photos/lib/services/sync/sync_service.dartindex 9b28082f1f..f3751c6c18 100644--- a/mobile/apps/photos/lib/services/sync/sync_service.dart+++ b/mobile/apps/photos/lib/services/sync/sync_service.dart@@ -235,14 +235,17 @@ class SyncService {if (isLocalGalleryMode) {await _localSyncService.syncAll();if (Platform.isAndroid) {- unawaited(- OfflineImportMetadataService.instance.processPendingFiles(- batchSize: isProcessBg- ? 10- : OfflineImportMetadataService.kDefaultBatchSize,- maxBatches: isProcessBg ? 1 : 4,- ),- );+ final processing = OfflineImportMetadataService.instance+ .processPendingFiles(+ batchSize: isProcessBg+ ? 10+ : OfflineImportMetadataService.kDefaultBatchSize,+ );+ if (isProcessBg) {+ await processing;+ } else {+ unawaited(processing);+ }}_logger.info("[SYNC] Local gallery mode${Platform.isAndroid ? '' : ' (non-Android, no metadata processing)'}, skipping remote sync",diff --git a/mobile/apps/photos/pubspec.yaml b/mobile/apps/photos/pubspec.yamlindex d021a9259c..1edddfcdd9 100644--- a/mobile/apps/photos/pubspec.yaml+++ b/mobile/apps/photos/pubspec.yaml@@ -78,7 +78,7 @@ dependencies:path: ../../packages/uiequatable: 2.0.7event_bus: 2.0.1- exif_reader: 4.1.0+ exif_reader: 4.2.0extended_image: 9.1.0fade_indexed_stack: 0.2.2fast_base58: 0.2.1@@ -176,7 +176,7 @@ dependencies:pointycastle: 3.9.1pro_image_editor: 11.23.0protobuf: 4.2.0- random_access_source: 4.2.1+ random_access_source: 4.3.1receive_sharing_intent: # pub.dev is behindgit:url: https://github.com/KasemJaffer/receive_sharing_intent.gitdiff --git a/mobile/apps/photos/test/db/files_db_test.dart b/mobile/apps/photos/test/db/files_db_test.dartnew file mode 100644index 0000000000..2e14ff8f99--- /dev/null+++ b/mobile/apps/photos/test/db/files_db_test.dart@@ -0,0 +1,104 @@+import "dart:io";++import "package:flutter_test/flutter_test.dart";+import "package:path_provider_platform_interface/path_provider_platform_interface.dart";+import "package:photos/db/files_db.dart";+import "package:photos/models/file/file.dart";+import "package:photos/models/file/file_type.dart";++void main() {+ late Directory tempDir;+ late PathProviderPlatform previousPathProvider;++ setUpAll(() async {+ previousPathProvider = PathProviderPlatform.instance;+ tempDir = await Directory.systemTemp.createTemp("files_db_test_");+ PathProviderPlatform.instance = _FakePathProvider(tempDir.path);+ });++ tearDownAll(() async {+ await FilesDB.instance.clearTable();+ PathProviderPlatform.instance = previousPathProvider;+ await tempDir.delete(recursive: true);+ });++ test("pagination reaches older rows when failures remain pending", () async {+ final files = List.generate(11, (index) {+ final id = index + 1;+ return EnteFile()+ ..generatedID = id+ ..localID = "local-$id"+ ..title = "$id.jpg"+ ..fileType = FileType.image+ ..creationTime = id <= 2 ? 11 : 12 - id+ ..modificationTime = id <= 2 ? 11 : 12 - id+ ..metadataVersion = 0;+ });+ files.add(+ EnteFile()+ ..generatedID = 31+ ..localID = "local-3"+ ..collectionID = 31+ ..title = "3-copy.jpg"+ ..fileType = FileType.image+ ..creationTime = 9+ ..modificationTime = 9+ ..metadataVersion = 0,+ );+ await FilesDB.instance.insertMultiple(files);++ final visited = <String>[];+ final visitedIDs = <int>[];+ ({int creationTime, int generatedID})? cursor;+ while (true) {+ final page = await FilesDB.instance+ .getUnUploadedLocalFilesPendingOfflineProcessing(+ 1,+ limit: 1,+ cursor: cursor,+ );+ if (page.isEmpty) break;++ cursor = (+ creationTime: page.last.creationTime!,+ generatedID: page.last.generatedID!,+ );+ for (final file in page) {+ visited.add(file.localID!);+ visitedIDs.add(file.generatedID!);+ if (int.parse(file.localID!.substring(6)).isEven) {+ await FilesDB.instance.updateOfflineImportMetadataForLocalID(+ file.localID!,+ processingVersion: 1,+ );+ }+ }+ }++ expect(visited, [+ "local-2",+ "local-1",+ ...List.generate(9, (index) => "local-${index + 3}"),+ ]);+ expect(visitedIDs[2], 31);+ final pending = await FilesDB.instance+ .getUnUploadedLocalFilesPendingOfflineProcessing(1, limit: 20);+ expect(pending.map((file) => file.localID), [+ "local-1",+ "local-3",+ "local-5",+ "local-7",+ "local-9",+ "local-11",+ ]);+ });+}++class _FakePathProvider extends PathProviderPlatform {+ _FakePathProvider(this.documentsPath);++ final String documentsPath;++ @override+ Future<String?> getApplicationDocumentsPath() async => documentsPath;+}diff --git a/mobile/pubspec.lock b/mobile/pubspec.lockindex f4abca9b6f..89f909c900 100644--- a/mobile/pubspec.lock+++ b/mobile/pubspec.lock@@ -621,10 +621,10 @@ packages:dependency: transitivedescription:name: exif_reader- sha256: "8491a1733f31b5b0c49d05e3fd2690538f1906b52f156d9d1fe7fad9d76bcced"+ sha256: bdb0f14e6550eea85059513354e7feda2267ba3a0e108356aa283a71038ac535url: "https://pub.dev"source: hosted- version: "4.1.0"+ version: "4.2.0"expandable:dependency: transitivedescription:@@ -2225,10 +2225,10 @@ packages:dependency: transitivedescription:name: random_access_source- sha256: "2cb3578e3027d6f7e44a66ec9bf72f5e0b73e9c0e62f4768977a0ed962d08be7"+ sha256: "0d33a16014f72da17adfb44fdafe8976b45d51a22a192d2fe832d1c1da2bee34"url: "https://pub.dev"source: hosted- version: "4.2.1"+ version: "4.3.1"receive_sharing_intent:dependency: transitivedescription: