Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Speed up offline metadata processing

Neeraj Guptae4073a933c66
diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dart
index 546fb1ee5c..5a00e29802 100644
--- a/mobile/apps/photos/lib/db/files_db.dart
+++ b/mobile/apps/photos/lib/db/files_db.dart
@@ -960,20 +960,31 @@ class FilesDB with SqlDbBase {
Future<List<EnteFile>> getUnUploadedLocalFilesPendingOfflineProcessing(
int processingVersion, {
- int? limit,
+ required int limit,
+ ({int creationTime, int generatedID})? cursor,
}) async {
final db = await instance.sqliteAsyncDB;
final args = <Object?>[processingVersion];
var query =
- 'SELECT * FROM $filesTable WHERE ($columnUploadedFileID IS '
+ 'SELECT * FROM $filesTable WHERE $columnGeneratedID IN ('
+ 'SELECT MAX($columnGeneratedID) FROM $filesTable WHERE '
+ '($columnUploadedFileID IS '
'NULL OR $columnUploadedFileID IS -1) AND $columnLocalID IS NOT NULL '
'AND $columnLocalID IS NOT -1 AND ($columnMetadataVersion IS NULL OR '
- '$columnMetadataVersion < ?) GROUP BY $columnLocalID ORDER BY '
- '$columnCreationTime DESC';
- if (limit != null) {
- query += ' LIMIT ?';
- args.add(limit);
- }
+ '$columnMetadataVersion < ?) GROUP BY $columnLocalID)';
+ if (cursor != null) {
+ query +=
+ ' AND ($columnCreationTime < ? OR ($columnCreationTime = ? AND '
+ '$columnGeneratedID < ?))';
+ args.addAll([
+ cursor.creationTime,
+ cursor.creationTime,
+ cursor.generatedID,
+ ]);
+ }
+ query +=
+ ' ORDER BY $columnCreationTime DESC, $columnGeneratedID DESC LIMIT ?';
+ args.add(limit);
final results = await db.getAll(query, args);
return convertToFiles(results);
}
diff --git a/mobile/apps/photos/lib/module/metadata/exif.dart b/mobile/apps/photos/lib/module/metadata/exif.dart
index ae3148bbe0..1588ec5771 100644
--- a/mobile/apps/photos/lib/module/metadata/exif.dart
+++ b/mobile/apps/photos/lib/module/metadata/exif.dart
@@ -284,15 +284,14 @@ Location? locationFromExif(Map<String, IfdTag> exif) {
}
}
-Future<Map<String, IfdTag>> _readExifArgs(Map<String, dynamic> args) {
+Future<Map<String, IfdTag>> _readExifArgs(Map<String, dynamic> args) async {
final file = args["file"] as File;
- return FileRASource.loadFile(file).then((src) async {
- try {
- return _normalizeExifResult(await readExifFromSource(src));
- } finally {
- await src.close();
- }
- });
+ final src = ReadAheadRASource(await FileRASource.loadFile(file));
+ try {
+ return _normalizeExifResult(await readExifFromSource(src));
+ } finally {
+ await src.close();
+ }
}
Future<Map<String, IfdTag>> readExifAsync(File file) {
diff --git a/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dart b/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dart
index 5fe78bdd38..f817dbcd33 100644
--- a/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dart
+++ b/mobile/apps/photos/lib/services/sync/offline_import_metadata_service.dart
@@ -7,6 +7,7 @@ import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
import "package:photos/events/files_updated_event.dart";
import "package:photos/events/local_photos_updated_event.dart";
+import "package:photos/main.dart" show isProcessBg;
import "package:photos/models/file/file.dart";
import "package:photos/models/location/location.dart";
import "package:photos/module/download/file.dart";
@@ -14,6 +15,7 @@ import "package:photos/module/metadata/exif.dart";
import "package:photos/module/metadata/local_file.dart";
import 'package:photos/module/metadata/location.dart';
import "package:photos/service_locator.dart";
+import "package:photos/services/process_activity.dart";
class OfflineImportMetadataService {
static const kProcessingVersion = 1;
@@ -28,10 +30,7 @@ class OfflineImportMetadataService {
static final instance = OfflineImportMetadataService._privateConstructor();
- Future<void> processPendingFiles({
- int batchSize = kDefaultBatchSize,
- int maxBatches = 4,
- }) async {
+ Future<void> processPendingFiles({int batchSize = kDefaultBatchSize}) async {
if (!Platform.isAndroid || !isLocalGalleryMode) {
return;
}
@@ -42,19 +41,24 @@ class OfflineImportMetadataService {
_running = Completer<void>();
try {
- for (var batch = 0; batch < maxBatches; batch++) {
- if (!isLocalGalleryMode) {
- _logger.info("Offline mode disabled, stopping metadata processing");
+ ({int creationTime, int generatedID})? cursor;
+ while (isLocalGalleryMode) {
+ if (isProcessBg && await isForegroundEngineActive()) {
+ _logger.info("Foreground active, stopping background processing");
return;
}
-
final files = await _db.getUnUploadedLocalFilesPendingOfflineProcessing(
kProcessingVersion,
limit: batchSize,
+ cursor: cursor,
);
if (files.isEmpty) {
return;
}
+ cursor = (
+ creationTime: files.last.creationTime!,
+ generatedID: files.last.generatedID!,
+ );
final updatedFiles = <EnteFile>[];
for (final file in files) {
@@ -87,6 +91,7 @@ class OfflineImportMetadataService {
return;
}
}
+ _logger.info("Offline mode disabled, stopping metadata processing");
} finally {
_running?.complete();
_running = null;
diff --git a/mobile/apps/photos/lib/services/sync/sync_service.dart b/mobile/apps/photos/lib/services/sync/sync_service.dart
index 9b28082f1f..f3751c6c18 100644
--- a/mobile/apps/photos/lib/services/sync/sync_service.dart
+++ b/mobile/apps/photos/lib/services/sync/sync_service.dart
@@ -235,14 +235,17 @@ class SyncService {
if (isLocalGalleryMode) {
await _localSyncService.syncAll();
if (Platform.isAndroid) {
- unawaited(
- OfflineImportMetadataService.instance.processPendingFiles(
- batchSize: isProcessBg
- ? 10
- : OfflineImportMetadataService.kDefaultBatchSize,
- maxBatches: isProcessBg ? 1 : 4,
- ),
- );
+ final processing = OfflineImportMetadataService.instance
+ .processPendingFiles(
+ batchSize: isProcessBg
+ ? 10
+ : OfflineImportMetadataService.kDefaultBatchSize,
+ );
+ if (isProcessBg) {
+ await processing;
+ } else {
+ unawaited(processing);
+ }
}
_logger.info(
"[SYNC] Local gallery mode${Platform.isAndroid ? '' : ' (non-Android, no metadata processing)'}, skipping remote sync",
diff --git a/mobile/apps/photos/pubspec.yaml b/mobile/apps/photos/pubspec.yaml
index d021a9259c..1edddfcdd9 100644
--- a/mobile/apps/photos/pubspec.yaml
+++ b/mobile/apps/photos/pubspec.yaml
@@ -78,7 +78,7 @@ dependencies:
path: ../../packages/ui
equatable: 2.0.7
event_bus: 2.0.1
- exif_reader: 4.1.0
+ exif_reader: 4.2.0
extended_image: 9.1.0
fade_indexed_stack: 0.2.2
fast_base58: 0.2.1
@@ -176,7 +176,7 @@ dependencies:
pointycastle: 3.9.1
pro_image_editor: 11.23.0
protobuf: 4.2.0
- random_access_source: 4.2.1
+ random_access_source: 4.3.1
receive_sharing_intent: # pub.dev is behind
git:
url: https://github.com/KasemJaffer/receive_sharing_intent.git
diff --git a/mobile/apps/photos/test/db/files_db_test.dart b/mobile/apps/photos/test/db/files_db_test.dart
new file mode 100644
index 0000000000..2e14ff8f99
--- /dev/null
+++ b/mobile/apps/photos/test/db/files_db_test.dart
@@ -0,0 +1,104 @@
+import "dart:io";
+
+import "package:flutter_test/flutter_test.dart";
+import "package:path_provider_platform_interface/path_provider_platform_interface.dart";
+import "package:photos/db/files_db.dart";
+import "package:photos/models/file/file.dart";
+import "package:photos/models/file/file_type.dart";
+
+void main() {
+ late Directory tempDir;
+ late PathProviderPlatform previousPathProvider;
+
+ setUpAll(() async {
+ previousPathProvider = PathProviderPlatform.instance;
+ tempDir = await Directory.systemTemp.createTemp("files_db_test_");
+ PathProviderPlatform.instance = _FakePathProvider(tempDir.path);
+ });
+
+ tearDownAll(() async {
+ await FilesDB.instance.clearTable();
+ PathProviderPlatform.instance = previousPathProvider;
+ await tempDir.delete(recursive: true);
+ });
+
+ test("pagination reaches older rows when failures remain pending", () async {
+ final files = List.generate(11, (index) {
+ final id = index + 1;
+ return EnteFile()
+ ..generatedID = id
+ ..localID = "local-$id"
+ ..title = "$id.jpg"
+ ..fileType = FileType.image
+ ..creationTime = id <= 2 ? 11 : 12 - id
+ ..modificationTime = id <= 2 ? 11 : 12 - id
+ ..metadataVersion = 0;
+ });
+ files.add(
+ EnteFile()
+ ..generatedID = 31
+ ..localID = "local-3"
+ ..collectionID = 31
+ ..title = "3-copy.jpg"
+ ..fileType = FileType.image
+ ..creationTime = 9
+ ..modificationTime = 9
+ ..metadataVersion = 0,
+ );
+ await FilesDB.instance.insertMultiple(files);
+
+ final visited = <String>[];
+ final visitedIDs = <int>[];
+ ({int creationTime, int generatedID})? cursor;
+ while (true) {
+ final page = await FilesDB.instance
+ .getUnUploadedLocalFilesPendingOfflineProcessing(
+ 1,
+ limit: 1,
+ cursor: cursor,
+ );
+ if (page.isEmpty) break;
+
+ cursor = (
+ creationTime: page.last.creationTime!,
+ generatedID: page.last.generatedID!,
+ );
+ for (final file in page) {
+ visited.add(file.localID!);
+ visitedIDs.add(file.generatedID!);
+ if (int.parse(file.localID!.substring(6)).isEven) {
+ await FilesDB.instance.updateOfflineImportMetadataForLocalID(
+ file.localID!,
+ processingVersion: 1,
+ );
+ }
+ }
+ }
+
+ expect(visited, [
+ "local-2",
+ "local-1",
+ ...List.generate(9, (index) => "local-${index + 3}"),
+ ]);
+ expect(visitedIDs[2], 31);
+ final pending = await FilesDB.instance
+ .getUnUploadedLocalFilesPendingOfflineProcessing(1, limit: 20);
+ expect(pending.map((file) => file.localID), [
+ "local-1",
+ "local-3",
+ "local-5",
+ "local-7",
+ "local-9",
+ "local-11",
+ ]);
+ });
+}
+
+class _FakePathProvider extends PathProviderPlatform {
+ _FakePathProvider(this.documentsPath);
+
+ final String documentsPath;
+
+ @override
+ Future<String?> getApplicationDocumentsPath() async => documentsPath;
+}
diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock
index f4abca9b6f..89f909c900 100644
--- a/mobile/pubspec.lock
+++ b/mobile/pubspec.lock
@@ -621,10 +621,10 @@ packages:
dependency: transitive
description:
name: exif_reader
- sha256: "8491a1733f31b5b0c49d05e3fd2690538f1906b52f156d9d1fe7fad9d76bcced"
+ sha256: bdb0f14e6550eea85059513354e7feda2267ba3a0e108356aa283a71038ac535
url: "https://pub.dev"
source: hosted
- version: "4.1.0"
+ version: "4.2.0"
expandable:
dependency: transitive
description:
@@ -2225,10 +2225,10 @@ packages:
dependency: transitive
description:
name: random_access_source
- sha256: "2cb3578e3027d6f7e44a66ec9bf72f5e0b73e9c0e62f4768977a0ed962d08be7"
+ sha256: "0d33a16014f72da17adfb44fdafe8976b45d51a22a192d2fe832d1c1da2bee34"
url: "https://pub.dev"
source: hosted
- version: "4.2.1"
+ version: "4.3.1"
receive_sharing_intent:
dependency: transitive
description: