π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[photos] Keep FilesDB materialization branch production-only
ashil-pilot7390b9dd39bc
diff --git a/mobile/apps/photos/benchmark/run_files_db_materialization_benchmark.sh b/mobile/apps/photos/benchmark/run_files_db_materialization_benchmark.shdeleted file mode 100755index 366809d480..0000000000--- a/mobile/apps/photos/benchmark/run_files_db_materialization_benchmark.sh+++ /dev/null@@ -1,96 +0,0 @@-#!/usr/bin/env bash--set -euo pipefail--if [[ $# -ne 1 || -z "$1" || "$1" == "/" ]]; then- printf 'Usage: %s <persistent-result-directory>\n' "$0" >&2- exit 64-fi--result_dir="$1"-mkdir -p "$result_dir"-result_dir="$(cd "$result_dir" && pwd -P)"-if [[ "$result_dir" == "/" ]]; then- printf 'Refusing to use the filesystem root as the result directory\n' >&2- exit 64-fi-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"-photos_dir="$(cd "$script_dir/.." && pwd -P)"-database_path="$result_dir/files-db-100000.sqlite"-raw_log="$result_dir/raw-runs.log"-measurements="$result_dir/measurements.jsonl"-report="$result_dir/benchmark-report.md"-benchmark_test="test/benchmark/files_db_materialization_benchmark_test.dart"--for output in "$database_path" "$raw_log" "$measurements" "$report"; do- if [[ -e "$output" ]]; then- printf 'Refusing to overwrite existing benchmark output: %s\n' "$output" >&2- exit 1- fi-done--cleanup_fixture() {- if [[ "${FILESDB_BENCHMARK_KEEP_FIXTURE:-0}" == "1" ]]; then- return- fi- for generated_database_file in \- "$database_path" \- "$database_path-shm" \- "$database_path-wal"; do- if [[ -e "$generated_database_file" ]]; then- find "$generated_database_file" -delete- fi- done-}--trap cleanup_fixture EXIT-trap 'exit 130' INT-trap 'exit 143' TERM--cd "$photos_dir"-touch "$raw_log" "$measurements"--run_worker() {- local variant="$1"- local run_kind="$2"- local page_size="$3"- local family="$4"- FILESDB_BENCHMARK_MODE=run \- FILESDB_BENCHMARK_DB="$database_path" \- FILESDB_BENCHMARK_VARIANT="$variant" \- FILESDB_BENCHMARK_RUN_KIND="$run_kind" \- FILESDB_BENCHMARK_PAGE_SIZE="$page_size" \- FILESDB_BENCHMARK_FAMILY="$family" \- flutter test --no-pub "$benchmark_test" 2>&1 \- | tee -a "$raw_log" \- | tr '\r' '\n' \- | sed -n 's/^.*FILESDB_BENCHMARK_JSON //p' >> "$measurements"-}--FILESDB_BENCHMARK_MODE=build \-FILESDB_BENCHMARK_DB="$database_path" \- flutter test --no-pub "$benchmark_test" 2>&1 \- | tee -a "$raw_log" \- | tr '\r' '\n' \- | sed -n 's/^.*FILESDB_BENCHMARK_JSON //p' >> "$measurements"--for family in \- search_all_files \- gallery_pending_or_uploaded \- gallery_local_and_uploaded; do- run_worker legacy warmup 2000 "$family"- for run in 1 2 3 4 5; do- run_worker legacy measured 2000 "$family"- done- for page_size in 500 1000 2000 5000; do- run_worker paged warmup "$page_size" "$family"- for run in 1 2 3 4 5; do- run_worker paged measured "$page_size" "$family"- done- done-done--dart benchmark/summarize_files_db_materialization_benchmark.dart \- "$measurements" > "$report"--printf 'Benchmark report: %s\n' "$report"diff --git a/mobile/apps/photos/benchmark/summarize_files_db_materialization_benchmark.dart b/mobile/apps/photos/benchmark/summarize_files_db_materialization_benchmark.dartdeleted file mode 100644index b93b2d494b..0000000000--- a/mobile/apps/photos/benchmark/summarize_files_db_materialization_benchmark.dart+++ /dev/null@@ -1,196 +0,0 @@-import 'dart:convert';-import 'dart:io';--void main(List<String> arguments) {- if (arguments.length != 1) {- stderr.writeln('Usage: dart run <script> <measurements.jsonl>');- exitCode = 64;- return;- }-- final allRecords = File(arguments.single)- .readAsLinesSync()- .where((line) => line.trim().isNotEmpty)- .map((line) => jsonDecode(line) as Map<String, dynamic>)- .toList();- final records = allRecords- .where((record) => record['event'] == 'measurement')- .where((record) => record['runKind'] == 'measured')- .toList();- final grouped = <String, List<Map<String, dynamic>>>{};- for (final record in records) {- final variant = record['variant'] as String;- final family = record['benchmarkFamily'] as String;- final label = variant == 'legacy'- ? 'legacy-$family'- : '$family-paged-${record['pageSize'] as int}';- grouped.putIfAbsent(label, () => []).add(record);- }-- const expectedLabels = [- 'legacy-search_all_files',- 'search_all_files-paged-500',- 'search_all_files-paged-1000',- 'search_all_files-paged-2000',- 'search_all_files-paged-5000',- 'legacy-gallery_pending_or_uploaded',- 'gallery_pending_or_uploaded-paged-500',- 'gallery_pending_or_uploaded-paged-1000',- 'gallery_pending_or_uploaded-paged-2000',- 'gallery_pending_or_uploaded-paged-5000',- 'legacy-gallery_local_and_uploaded',- 'gallery_local_and_uploaded-paged-500',- 'gallery_local_and_uploaded-paged-1000',- 'gallery_local_and_uploaded-paged-2000',- 'gallery_local_and_uploaded-paged-5000',- ];- for (final label in expectedLabels) {- final runs = grouped[label] ?? const [];- if (runs.length < 5) {- stderr.writeln(- '$label has ${runs.length} measured runs; expected at least 5',- );- exitCode = 1;- return;- }- }-- stdout.writeln('# FilesDB 100,000-row materialization benchmark');- stdout.writeln();- stdout.writeln('- Fixture rows: 100,000');- stdout.writeln('- Fixture seed: 20260825');- stdout.writeln(- '- One fresh-process warm-up and five fresh-process measured runs per query family/page size',- );- stdout.writeln(- '- RSS uses `dart:io` `ProcessInfo.currentRss`/`maxRss`, sampled '- 'immediately before and after materialization in each fresh process; it '- 'is process-wide, informational, and not an acceptance gate',- );- stdout.writeln();- stdout.writeln(- '| Variant | Runs | Median wall ms | Median transaction ms | Median query ms | Median conversion ms | Median filter ms | Max raw page | End current RSS MiB values | End max RSS MiB values |',- );- stdout.writeln('|---|---:|---:|---:|---:|---:|---:|---:|---|---|');-- for (final label in expectedLabels) {- final runs = grouped[label]!;- final wall = _median(_values(runs, 'wallMicros')) / 1000;- final transaction = _median(_values(runs, 'transactionMicros')) / 1000;- final query = _median(_values(runs, 'queryMicros')) / 1000;- final conversion = _median(_values(runs, 'conversionMicros')) / 1000;- final filter = _median(_values(runs, 'filterMicros')) / 1000;- final maxRawPage = runs- .map((run) => run['maxRawPage'] as int)- .reduce((left, right) => left > right ? left : right);- final currentRss = runs- .map((run) => _mib(run['endCurrentRss'] as int))- .map((value) => value.toStringAsFixed(1))- .join(', ');- final maxRss = runs- .map((run) => _mib(run['endMaxRss'] as int))- .map((value) => value.toStringAsFixed(1))- .join(', ');-- stdout.writeln(- '| $label | ${runs.length} | ${wall.toStringAsFixed(1)} | '- '${transaction.toStringAsFixed(1)} | ${query.toStringAsFixed(1)} | '- '${conversion.toStringAsFixed(1)} | ${filter.toStringAsFixed(1)} | '- '$maxRawPage | $currentRss | $maxRss |',- );- }-- stdout.writeln();- for (final family in [- 'search_all_files',- 'gallery_pending_or_uploaded',- 'gallery_local_and_uploaded',- ]) {- final legacyMax = grouped['legacy-$family']!- .map((run) => run['maxRawPage'] as int)- .reduce((left, right) => left > right ? left : right);- final productionMax = grouped['$family-paged-2000']!- .map((run) => run['maxRawPage'] as int)- .reduce((left, right) => left > right ? left : right);- final reduction = (1 - (productionMax / legacyMax)) * 100;- stdout.writeln(- 'The selected 2,000-row `$family` load bounds each raw payload at '- '$productionMax rows versus $legacyMax for the legacy one-shot load '- '(${reduction.toStringAsFixed(1)}% fewer rows per maximum payload).',- );- }-- stdout.writeln();- stdout.writeln('## Gallery paging investigation');- stdout.writeln();- for (final family in [- 'gallery_pending_or_uploaded',- 'gallery_local_and_uploaded',- ]) {- final legacyRuns = grouped['legacy-$family']!;- final selectedRuns = grouped['$family-paged-2000']!;- final legacyWall = _median(_values(legacyRuns, 'wallMicros')) / 1000;- final selectedWall = _median(_values(selectedRuns, 'wallMicros')) / 1000;- final legacyTransaction =- _median(_values(legacyRuns, 'transactionMicros')) / 1000;- final selectedTransaction =- _median(_values(selectedRuns, 'transactionMicros')) / 1000;- stdout.writeln(- '- `$family`: selected 2,000-row paging median wall time '- '${selectedWall.toStringAsFixed(1)} ms versus '- '${legacyWall.toStringAsFixed(1)} ms one-shot '- '(${_percentDelta(selectedWall, legacyWall)}); median transaction time '- '${selectedTransaction.toStringAsFixed(1)} ms versus '- '${legacyTransaction.toStringAsFixed(1)} ms '- '(${_percentDelta(selectedTransaction, legacyTransaction)}).',- );- }- stdout.writeln(- '- An unindexed diagnostic run exposed `USE TEMP B-TREE FOR LAST 2 TERMS '- 'OF ORDER BY` and selected-page medians of 1,479.8 ms (pending) and '- '1,408.6 ms (local). The selected composite '- '`(creation_time, modification_time, _id)` index removes that repeated '- 'sort from the plans below.',- );- stdout.writeln(- '- On this fixed fixture, creating the candidate index took 0.10 seconds '- 'and increased the database from 98,906,112 to 101,859,328 bytes '- '(2,953,216 bytes, 3.0%). This startup migration and storage cost was '- 'accepted to avoid the measured Gallery regression while retaining the '- '98% payload reduction.',- );-- stdout.writeln();- stdout.writeln('## Query plans');- stdout.writeln();- final plansByFamily = <String, Set<String>>{};- for (final record in allRecords.where(- (record) => record['event'] == 'queryPlan',- )) {- final family = record['family'] as String;- final details = (record['details'] as List<dynamic>).cast<String>();- plansByFamily.putIfAbsent(family, () => {}).addAll(details);- }- for (final family in plansByFamily.keys.toList()..sort()) {- stdout.writeln('- `$family`: ${plansByFamily[family]!.join('; ')}');- }-}--List<int> _values(List<Map<String, dynamic>> records, String key) =>- records.map((record) => record[key] as int).toList();--double _median(List<int> values) {- values.sort();- final middle = values.length ~/ 2;- if (values.length.isOdd) {- return values[middle].toDouble();- }- return (values[middle - 1] + values[middle]) / 2;-}--double _mib(int bytes) => bytes / (1024 * 1024);--String _percentDelta(double selected, double legacy) {- final delta = ((selected / legacy) - 1) * 100;- return '${delta >= 0 ? '+' : ''}${delta.toStringAsFixed(1)}%';-}diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dartindex b3d176ccd8..bcf174aaa2 100644--- a/mobile/apps/photos/lib/db/files_db.dart+++ b/mobile/apps/photos/lib/db/files_db.dart@@ -130,56 +130,17 @@ class FilesDB with SqlDbBase {static final String _materializedFileProjection = _columnNames.join(", ");static int _nextMaterializationOperationID = 0;- final Future<SqliteDatabase>? _databaseOverride;- final int materializationPageSize;- final FilesDBMaterializationHooks? _materializationHooks;- final FilesDBPageConverter? _searchPageConverterOverride;- final FilesDBPageConverter? _galleryPageConverterOverride;- final FilesDBFilter? _filterOverride;-- FilesDB._privateConstructor()- : _databaseOverride = null,- materializationPageSize = defaultMaterializationPageSize,- _materializationHooks = null,- _searchPageConverterOverride = null,- _galleryPageConverterOverride = null,- _filterOverride = null;-- @visibleForTesting- FilesDB.forTesting({- required SqliteDatabase database,- this.materializationPageSize = defaultMaterializationPageSize,- FilesDBMaterializationHooks? materializationHooks,- FilesDBPageConverter? searchPageConverter,- FilesDBPageConverter? galleryPageConverter,- FilesDBFilter? filter,- }) : assert(materializationPageSize > 0),- _databaseOverride = Future.value(database),- _materializationHooks = materializationHooks,- _searchPageConverterOverride = searchPageConverter,- _galleryPageConverterOverride = galleryPageConverter,- _filterOverride = filter;+ FilesDB._privateConstructor();static final FilesDB instance = FilesDB._privateConstructor();static Future<SqliteDatabase>? _sqliteAsyncDBFuture;Future<SqliteDatabase> get sqliteAsyncDB async {- if (_databaseOverride != null) {- return _databaseOverride;- }_sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase();return _sqliteAsyncDBFuture!;}- @visibleForTesting- static List<String> get materializedFileColumns =>- List.unmodifiable(_columnNames);-- @visibleForTesting- static Future<void> migrateTestDatabase(SqliteDatabase database) =>- instance.migrate(database, _migrationScripts);-Future<SqliteDatabase> _initSqliteAsyncDatabase() async {final Directory documentsDirectory =await getApplicationDocumentsDirectory();@@ -1969,10 +1930,6 @@ class FilesDB with SqlDbBase {}Future<List<EnteFile>> _convertSearchPage(List<Map<String, dynamic>> rows) {- final override = _searchPageConverterOverride;- if (override != null) {- return override(rows);- }return Computer.shared().compute(_convertFilesDBRowsForIsolate,param: {"result": rows},@@ -1983,10 +1940,6 @@ class FilesDB with SqlDbBase {Future<List<EnteFile>> _convertGalleryPage(List<Map<String, dynamic>> rows,) async {- final override = _galleryPageConverterOverride;- if (override != null) {- return override(rows);- }return convertToFiles(rows);}@@ -1995,14 +1948,14 @@ class FilesDB with SqlDbBase {required String whereClause,required List<Object?> whereArguments,required _MaterializedFileOrder order,- required FilesDBPageConverter converter,+ required _FilesDBPageConverter converter,DBFilterOptions? filterOptions,int? limit,}) async {final operation = _FilesDBMaterializationOperation(id: ++_nextMaterializationOperationID,family: family,- pageSize: materializationPageSize,+ pageSize: defaultMaterializationPageSize,);final files = <EnteFile>[];final boundedLimit = limit != null && limit >= 0 ? limit : null;@@ -2018,13 +1971,9 @@ class FilesDB with SqlDbBase {while (remaining == null || remaining > 0) {final requestedRows = remaining == null- ? materializationPageSize- : min(materializationPageSize, remaining);+ ? defaultMaterializationPageSize+ : min(defaultMaterializationPageSize, remaining);final pageNumber = operation.pageCount + 1;- await _materializationHooks?.beforePageQuery?.call(- pageNumber,- transaction,- );final queryWatch = Stopwatch()..start();late final List<Map<String, dynamic>> rows;@@ -2050,7 +1999,7 @@ class FilesDB with SqlDbBase {operation.rawRows += rows.length;operation.maxRawPage = max(operation.maxRawPage, rows.length);if (rows.length > requestedRows ||- rows.length > materializationPageSize) {+ rows.length > defaultMaterializationPageSize) {throw StateError('FilesDB materialization page exceeded its configured bound',);@@ -2065,21 +2014,11 @@ class FilesDB with SqlDbBase {);}- await _materializationHooks?.afterPageQuery?.call(- pageNumber,- rows.length,- transaction,- );-if (rows.isEmpty) {break;}cursor = order.cursorFrom(rows.last);- await _materializationHooks?.beforePageConversion?.call(- pageNumber,- rows.length,- );final conversionWatch = Stopwatch()..start();late final List<EnteFile> convertedPage;try {@@ -2088,10 +2027,6 @@ class FilesDB with SqlDbBase {conversionWatch.stop();operation.recordConversion(conversionWatch.elapsed);}- await _materializationHooks?.afterPageConversion?.call(- pageNumber,- rows.length,- );files.addAll(convertedPage);if (remaining != null) {@@ -2109,8 +2044,7 @@ class FilesDB with SqlDbBase {operation.filterWatch.start();late final List<EnteFile> filteredFiles;try {- final filter = _filterOverride ?? applyDBFilters;- filteredFiles = await filter(files, filterOptions);+ filteredFiles = await applyDBFilters(files, filterOptions);} finally {operation.filterWatch.stop();}@@ -2155,15 +2089,6 @@ class FilesDB with SqlDbBase {} else {_logger.warning(message);}- try {- _materializationHooks?.onComplete?.call(metrics);- } catch (hookError, hookStackTrace) {- _logger.warning(- 'FilesDBMaterialization metrics hook failed',- hookError,- hookStackTrace,- );- }}Future<bool> hasAnyFile() async {@@ -2446,42 +2371,10 @@ class FilesDB with SqlDbBase {}}-typedef FilesDBPageConverter =+typedef _FilesDBPageConverter =Future<List<EnteFile>> Function(List<Map<String, dynamic>> rows);-typedef FilesDBFilter =- Future<List<EnteFile>> Function(- List<EnteFile> files,- DBFilterOptions? options,- );--@visibleForTesting-class FilesDBMaterializationHooks {- final Future<void> Function(int pageNumber, SqliteReadContext transaction)?- beforePageQuery;- final Future<void> Function(- int pageNumber,- int rowCount,- SqliteReadContext transaction,- )?- afterPageQuery;- final Future<void> Function(int pageNumber, int rowCount)?- beforePageConversion;- final Future<void> Function(int pageNumber, int rowCount)?- afterPageConversion;- final void Function(FilesDBMaterializationMetrics metrics)? onComplete;-- const FilesDBMaterializationHooks({- this.beforePageQuery,- this.afterPageQuery,- this.beforePageConversion,- this.afterPageConversion,- this.onComplete,- });-}--@visibleForTesting-class FilesDBMaterializationMetrics {+class _FilesDBMaterializationMetrics {final int operationID;final String family;final int pageSize;@@ -2503,7 +2396,7 @@ class FilesDBMaterializationMetrics {final int? startMaxRss;final int? endMaxRss;- const FilesDBMaterializationMetrics({+ const _FilesDBMaterializationMetrics({required this.operationID,required this.family,required this.pageSize,@@ -2548,29 +2441,6 @@ class FilesDBMaterializationMetrics {'startMaxRss=${startMaxRss ?? 'unavailable'}','endMaxRss=${endMaxRss ?? 'unavailable'}',].join(' ');-- Map<String, Object?> toJson() => {- 'operationID': operationID,- 'family': family,- 'status': success ? 'success' : 'error',- 'errorType': errorType,- 'pageSize': pageSize,- 'pageCount': pageCount,- 'rawRows': rawRows,- 'finalRows': finalRows,- 'maxRawPage': maxRawPage,- 'queryMicros': queryDuration.inMicroseconds,- 'maxQueryMicros': maxQueryDuration.inMicroseconds,- 'conversionMicros': conversionDuration.inMicroseconds,- 'maxConversionMicros': maxConversionDuration.inMicroseconds,- 'filterMicros': filterDuration.inMicroseconds,- 'transactionMicros': transactionDuration.inMicroseconds,- 'totalMicros': totalDuration.inMicroseconds,- 'startCurrentRss': startCurrentRss,- 'endCurrentRss': endCurrentRss,- 'startMaxRss': startMaxRss,- 'endMaxRss': endMaxRss,- };}enum _MaterializedFileOrder {@@ -2662,7 +2532,7 @@ class _FilesDBMaterializationOperation {endMaxRss = _readMaxRss();}- FilesDBMaterializationMetrics get metrics => FilesDBMaterializationMetrics(+ _FilesDBMaterializationMetrics get metrics => _FilesDBMaterializationMetrics(operationID: id,family: family,pageSize: pageSize,diff --git a/mobile/apps/photos/test/benchmark/files_db_materialization_benchmark_test.dart b/mobile/apps/photos/test/benchmark/files_db_materialization_benchmark_test.dartdeleted file mode 100644index 27a432807d..0000000000--- a/mobile/apps/photos/test/benchmark/files_db_materialization_benchmark_test.dart+++ /dev/null@@ -1,428 +0,0 @@-import 'dart:convert';-import 'dart:io';-import 'dart:math';--import 'package:computer/computer.dart';-import 'package:flutter_test/flutter_test.dart';-import 'package:photos/db/files_db.dart';-import 'package:photos/models/file/file.dart';-import 'package:photos/services/filter/db_filters.dart';-import 'package:sqlite_async/sqlite_async.dart';--const _fixtureRows = 100000;-const _fixtureSeed = 20260825;-const _marker = 'FILESDB_BENCHMARK_JSON ';-const _searchFamily = 'search_all_files';-const _pendingFamily = 'gallery_pending_or_uploaded';-const _localFamily = 'gallery_local_and_uploaded';--void main() {- final environment = Platform.environment;- final mode = environment['FILESDB_BENCHMARK_MODE'];- final databasePath = environment['FILESDB_BENCHMARK_DB'];-- if (mode == null || databasePath == null) {- test(- 'FilesDB materialization benchmark worker',- () {},- skip: 'non-CI benchmark; run through the benchmark script',- );- return;- }-- test(- 'FilesDB materialization benchmark worker',- () async {- if (mode == 'build') {- await _buildFixture(databasePath);- return;- }- if (mode != 'run') {- fail('Unknown FILESDB_BENCHMARK_MODE: $mode');- }-- final variant = environment['FILESDB_BENCHMARK_VARIANT'] ?? 'paged';- final runKind = environment['FILESDB_BENCHMARK_RUN_KIND'] ?? 'measured';- final family = environment['FILESDB_BENCHMARK_FAMILY'] ?? _searchFamily;- final pageSize = int.parse(- environment['FILESDB_BENCHMARK_PAGE_SIZE'] ??- FilesDB.defaultMaterializationPageSize.toString(),- );- await _runBenchmark(- databasePath: databasePath,- variant: variant,- runKind: runKind,- family: family,- pageSize: pageSize,- );- },- timeout: const Timeout(Duration(minutes: 15)),- );-}--Future<void> _buildFixture(String databasePath) async {- final file = File(databasePath);- await file.parent.create(recursive: true);- if (await file.exists()) {- await file.delete();- }- final database = SqliteDatabase(path: databasePath);- try {- await FilesDB.migrateTestDatabase(database);- final columns = FilesDB.materializedFileColumns;- final placeholders = List.filled(columns.length, '?').join(', ');- final insertStatement =- 'INSERT INTO ${FilesDB.filesTable} (${columns.join(', ')}) '- 'VALUES ($placeholders)';- final random = Random(_fixtureSeed);- const batchSize = 1000;-- for (var first = 1; first <= _fixtureRows; first += batchSize) {- final last = min(_fixtureRows + 1, first + batchSize);- final parameterSets = <List<Object?>>[];- for (var id = first; id < last; id++) {- final creationTime = 2000000000000000 - (id ~/ 4);- final modificationTime = creationTime + random.nextInt(7);- final values = _fixtureValues(- id: id,- creationTime: creationTime,- modificationTime: modificationTime,- );- parameterSets.add(- columns.map<Object?>((column) => values[column]).toList(),- );- }- await database.executeBatch(insertStatement, parameterSets);- }-- final countRows = await database.getAll(- 'SELECT COUNT(*) AS count FROM ${FilesDB.filesTable}',- );- expect(countRows.single['count'], _fixtureRows);- await database.execute('PRAGMA wal_checkpoint(TRUNCATE)');- stdout.writeln(- '$_marker${jsonEncode({'event': 'fixture', 'rows': _fixtureRows, 'seed': _fixtureSeed, 'databaseBytes': await file.length()})}',- );- } finally {- await database.close();- }-}--Future<void> _runBenchmark({- required String databasePath,- required String variant,- required String runKind,- required String family,- required int pageSize,-}) async {- final database = SqliteDatabase(path: databasePath);- await database.initialize();- await Computer.shared().turnOn(workersCount: 1);- try {- if (runKind == 'warmup') {- await _writeQueryPlan(database, family);- }- final startCurrentRss = ProcessInfo.currentRss;- final startMaxRss = ProcessInfo.maxRss;- final wallWatch = Stopwatch()..start();-- late final Map<String, Object?> measurement;- if (variant == 'legacy') {- measurement = await _runLegacy(database, family);- } else if (variant == 'paged') {- measurement = await _runPaged(database, pageSize, family);- } else {- fail('Unknown FILESDB_BENCHMARK_VARIANT: $variant');- }-- wallWatch.stop();- measurement.addAll({- 'event': 'measurement',- 'variant': variant,- 'benchmarkFamily': family,- 'runKind': runKind,- 'seed': _fixtureSeed,- 'fixtureRows': _fixtureRows,- 'pageSize': variant == 'legacy' ? null : pageSize,- 'wallMicros': wallWatch.elapsedMicroseconds,- 'startCurrentRss': startCurrentRss,- 'endCurrentRss': ProcessInfo.currentRss,- 'startMaxRss': startMaxRss,- 'endMaxRss': ProcessInfo.maxRss,- });- expect(measurement['rawRows'], _fixtureRows);- expect(measurement['finalRows'], _fixtureRows);- if (variant == 'legacy') {- expect(measurement['maxRawPage'], _fixtureRows);- } else {- expect(measurement['maxRawPage'], lessThanOrEqualTo(pageSize));- expect(measurement['maxRawPage'], lessThanOrEqualTo(5000));- }- stdout.writeln('$_marker${jsonEncode(measurement)}');- } finally {- await Computer.shared().turnOff();- await database.close();- }-}--Future<Map<String, Object?>> _runPaged(- SqliteDatabase database,- int pageSize,- String family,-) async {- late FilesDBMaterializationMetrics metrics;- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: pageSize,- materializationHooks: FilesDBMaterializationHooks(- onComplete: (value) => metrics = value,- ),- );- final options = DBFilterOptions(dedupeUploadID: false);- final int finalRows;- switch (family) {- case _searchFamily:- finalRows = (await filesDB.getAllFilesFromDB(- const {},- dedupeByUploadId: false,- )).length;- case _pendingFamily:- finalRows = (await filesDB.getAllPendingOrUploadedFiles(- 0,- 3000000000000000,- 1,- filterOptions: options,- )).files.length;- case _localFamily:- finalRows = (await filesDB.getAllLocalAndUploadedFiles(- 0,- 3000000000000000,- 1,- filterOptions: options,- )).files.length;- default:- fail('Unknown FILESDB_BENCHMARK_FAMILY: $family');- }- expect(finalRows, _fixtureRows);- return metrics.toJson();-}--Future<Map<String, Object?>> _runLegacy(- SqliteDatabase database,- String family,-) async {- late final String query;- late final List<Object?> arguments;- late final bool convertInWorker;- switch (family) {- case _searchFamily:- query =- 'SELECT * FROM ${FilesDB.filesTable} '- 'ORDER BY ${FilesDB.columnCreationTime} DESC';- arguments = const [];- convertInWorker = true;- case _pendingFamily:- query =- 'SELECT * FROM ${FilesDB.filesTable} '- 'WHERE ${FilesDB.columnCreationTime} >= ? '- 'AND ${FilesDB.columnCreationTime} <= ? '- 'AND (${FilesDB.columnCollectionID} IS NOT NULL '- 'AND ${FilesDB.columnCollectionID} IS NOT -1) '- 'AND ${FilesDB.columnMMdVisibility} = ? '- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnModificationTime} DESC';- arguments = const [0, 3000000000000000, 0];- convertInWorker = false;- case _localFamily:- query =- 'SELECT * FROM ${FilesDB.filesTable} '- 'WHERE ${FilesDB.columnCreationTime} >= ? '- 'AND ${FilesDB.columnCreationTime} <= ? '- 'AND (${FilesDB.columnMMdVisibility} IS NULL '- 'OR ${FilesDB.columnMMdVisibility} = ?) '- 'AND (${FilesDB.columnLocalID} IS NOT NULL OR '- '(${FilesDB.columnCollectionID} IS NOT NULL '- 'AND ${FilesDB.columnCollectionID} IS NOT -1)) '- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnModificationTime} DESC';- arguments = const [0, 3000000000000000, 0];- convertInWorker = false;- default:- fail('Unknown FILESDB_BENCHMARK_FAMILY: $family');- }-- final transactionWatch = Stopwatch()..start();- final queryWatch = Stopwatch()..start();- late final List<Map<String, dynamic>> rows;- try {- rows = await database.readTransaction(- (transaction) => transaction.getAll(query, arguments),- );- } finally {- queryWatch.stop();- transactionWatch.stop();- }-- final conversionWatch = Stopwatch()..start();- late final List<EnteFile> files;- try {- files = convertInWorker- ? await Computer.shared().compute(- _convertLegacyRows,- param: {'result': rows},- taskName: 'FilesDB.legacyBenchmark',- )- : FilesDB.instance.convertToFiles(rows);- } finally {- conversionWatch.stop();- }-- final filterWatch = Stopwatch()..start();- late final List<EnteFile> filteredFiles;- try {- filteredFiles = await applyDBFilters(- files,- DBFilterOptions(dedupeUploadID: false),- );- } finally {- filterWatch.stop();- }-- return {- 'pageCount': 1,- 'rawRows': rows.length,- 'finalRows': filteredFiles.length,- 'maxRawPage': rows.length,- 'queryMicros': queryWatch.elapsedMicroseconds,- 'maxQueryMicros': queryWatch.elapsedMicroseconds,- 'conversionMicros': conversionWatch.elapsedMicroseconds,- 'maxConversionMicros': conversionWatch.elapsedMicroseconds,- 'filterMicros': filterWatch.elapsedMicroseconds,- 'transactionMicros': transactionWatch.elapsedMicroseconds,- };-}--Future<void> _writeQueryPlan(SqliteDatabase database, String family) async {- final projection = FilesDB.materializedFileColumns.join(', ');- late final String query;- late final List<Object?> arguments;- switch (family) {- case _searchFamily:- query =- 'EXPLAIN QUERY PLAN SELECT $projection '- 'FROM ${FilesDB.filesTable} '- 'WHERE (${FilesDB.columnCreationTime}, ${FilesDB.columnGeneratedID}) '- '< (?, ?) '- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnGeneratedID} DESC LIMIT ?';- arguments = [2000000000000000, _fixtureRows, 2000];- case _pendingFamily:- query =- 'EXPLAIN QUERY PLAN SELECT $projection '- 'FROM ${FilesDB.filesTable} '- 'WHERE ${FilesDB.columnCreationTime} >= ? '- 'AND ${FilesDB.columnCreationTime} <= ? '- 'AND (${FilesDB.columnCollectionID} IS NOT NULL '- 'AND ${FilesDB.columnCollectionID} IS NOT -1) '- 'AND ${FilesDB.columnMMdVisibility} = ? '- 'AND (${FilesDB.columnCreationTime}, '- '${FilesDB.columnModificationTime}, ${FilesDB.columnGeneratedID}) '- '< (?, ?, ?) '- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnModificationTime} DESC, '- '${FilesDB.columnGeneratedID} DESC LIMIT ?';- arguments = [- 0,- 3000000000000000,- 0,- 2000000000000000,- 2000000000000000,- _fixtureRows,- 2000,- ];- case _localFamily:- query =- 'EXPLAIN QUERY PLAN SELECT $projection '- 'FROM ${FilesDB.filesTable} '- 'WHERE ${FilesDB.columnCreationTime} >= ? '- 'AND ${FilesDB.columnCreationTime} <= ? '- 'AND (${FilesDB.columnMMdVisibility} IS NULL '- 'OR ${FilesDB.columnMMdVisibility} = ?) '- 'AND (${FilesDB.columnLocalID} IS NOT NULL OR '- '(${FilesDB.columnCollectionID} IS NOT NULL '- 'AND ${FilesDB.columnCollectionID} IS NOT -1)) '- 'AND (${FilesDB.columnCreationTime}, '- '${FilesDB.columnModificationTime}, ${FilesDB.columnGeneratedID}) '- '< (?, ?, ?) '- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnModificationTime} DESC, '- '${FilesDB.columnGeneratedID} DESC LIMIT ?';- arguments = [- 0,- 3000000000000000,- 0,- 2000000000000000,- 2000000000000000,- _fixtureRows,- 2000,- ];- default:- fail('Unknown FILESDB_BENCHMARK_FAMILY: $family');- }- final rows = await database.getAll(query, arguments);- stdout.writeln(- '$_marker${jsonEncode({'event': 'queryPlan', 'family': family, 'details': rows.map((row) => row['detail']).toList()})}',- );-}--Map<String, Object?> _fixtureValues({- required int id,- required int creationTime,- required int modificationTime,-}) {- const cryptoValue =- '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef';- const headerValue =- 'header-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'- '0123456789abcdef0123456789abcdef';- const exifValue =- '{"Make":"Deterministic","Model":"Benchmark","payload":"'- '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'- '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}';- return {- FilesDB.columnGeneratedID: id,- FilesDB.columnLocalID: id % 10 == 0 ? 'local-$id' : null,- FilesDB.columnUploadedFileID: id,- FilesDB.columnOwnerID: id.isEven ? 1 : 2,- FilesDB.columnCollectionID: 100 + (id % 997),- FilesDB.columnTitle: 'deterministic-$id.jpg',- FilesDB.columnDeviceFolder: id % 3 == 0 ? 'camera' : null,- FilesDB.columnLatitude: id % 17 == 0 ? 12.25 : null,- FilesDB.columnLongitude: id % 17 == 0 ? 77.75 : null,- FilesDB.columnFileType: id % 8 == 0 ? 1 : 0,- FilesDB.columnModificationTime: modificationTime,- FilesDB.columnEncryptedKey: cryptoValue,- FilesDB.columnKeyDecryptionNonce: cryptoValue,- FilesDB.columnFileDecryptionHeader: headerValue,- FilesDB.columnThumbnailDecryptionHeader: headerValue,- FilesDB.columnMetadataDecryptionHeader: headerValue,- FilesDB.columnCreationTime: creationTime,- FilesDB.columnUpdationTime: creationTime + 1,- FilesDB.columnFileSubType: 0,- FilesDB.columnDuration: id % 8 == 0 ? 12000 : 0,- FilesDB.columnExif: exifValue,- FilesDB.columnHash: 'hash-${id % 99991}',- FilesDB.columnMetadataVersion: 3,- FilesDB.columnMMdEncodedJson: '{"visibility":0}',- FilesDB.columnMMdVersion: 1,- FilesDB.columnMMdVisibility: 0,- FilesDB.columnPubMMdEncodedJson:- '{"caption":"benchmark","w":4032,"h":3024}',- FilesDB.columnPubMMdVersion: 1,- FilesDB.columnFileSize: 1000000 + id,- FilesDB.columnAddedTime: creationTime + 2,- };-}--List<EnteFile> _convertLegacyRows(Map<dynamic, dynamic> arguments) =>- FilesDB.instance.convertToFilesForIsolate(arguments);diff --git a/mobile/apps/photos/test/db/files_db_materialization_test.dart b/mobile/apps/photos/test/db/files_db_materialization_test.dartdeleted file mode 100644index c87edcb59c..0000000000--- a/mobile/apps/photos/test/db/files_db_materialization_test.dart+++ /dev/null@@ -1,1123 +0,0 @@-import 'dart:async';-import 'dart:io';--import 'package:computer/computer.dart';-import 'package:flutter_secure_storage/flutter_secure_storage.dart';-import 'package:flutter_test/flutter_test.dart';-import 'package:path/path.dart' as path;-import 'package:path_provider_platform_interface/path_provider_platform_interface.dart';-import 'package:photos/core/configuration.dart';-import 'package:photos/db/files_db.dart';-import 'package:photos/models/file/file.dart';-import 'package:photos/models/file/file_type.dart';-import 'package:photos/models/metadata/common_keys.dart';-import 'package:photos/services/filter/db_filters.dart';-import 'package:shared_preferences/shared_preferences.dart';-import 'package:sqlite_async/sqlite_async.dart';--const _ownerID = 0;-const _sharedOwnerID = 99;-const _ignoredCollectionID = 900;-const _useGeneratedValue = Object();--void main() {- late Directory temporaryDirectory;- late SqliteDatabase database;- late PathProviderPlatform previousPathProvider;- var startedComputer = false;-- setUpAll(() async {- temporaryDirectory = await Directory.systemTemp.createTemp(- 'files-db-materialization-test-',- );- previousPathProvider = PathProviderPlatform.instance;- PathProviderPlatform.instance = _FakePathProvider(temporaryDirectory.path);- FlutterSecureStorage.setMockInitialValues({});- SharedPreferences.setMockInitialValues({Configuration.userIDKey: _ownerID});- await Configuration.instance.init(await SharedPreferences.getInstance());- database = SqliteDatabase(- path: path.join(temporaryDirectory.path, 'files.sqlite'),- );- await FilesDB.migrateTestDatabase(database);- if (!Computer.shared().isRunning) {- await Computer.shared().turnOn(workersCount: 1);- startedComputer = true;- }- });-- setUp(() async {- await database.execute('DELETE FROM ${FilesDB.filesTable}');- await _insertRows(database, _fixtureRows());- });-- tearDownAll(() async {- if (startedComputer) {- await Computer.shared().turnOff();- }- await database.close();- PathProviderPlatform.instance = previousPathProvider;- await temporaryDirectory.delete(recursive: true);- });-- test(- 'explicit projection preserves the complete EnteFile contract',- () async {- expect(FilesDB.materializedFileColumns, <String>[- FilesDB.columnGeneratedID,- FilesDB.columnLocalID,- FilesDB.columnUploadedFileID,- FilesDB.columnOwnerID,- FilesDB.columnCollectionID,- FilesDB.columnTitle,- FilesDB.columnDeviceFolder,- FilesDB.columnLatitude,- FilesDB.columnLongitude,- FilesDB.columnFileType,- FilesDB.columnModificationTime,- FilesDB.columnEncryptedKey,- FilesDB.columnKeyDecryptionNonce,- FilesDB.columnFileDecryptionHeader,- FilesDB.columnThumbnailDecryptionHeader,- FilesDB.columnMetadataDecryptionHeader,- FilesDB.columnCreationTime,- FilesDB.columnUpdationTime,- FilesDB.columnFileSubType,- FilesDB.columnDuration,- FilesDB.columnExif,- FilesDB.columnHash,- FilesDB.columnMetadataVersion,- FilesDB.columnMMdEncodedJson,- FilesDB.columnMMdVersion,- FilesDB.columnMMdVisibility,- FilesDB.columnPubMMdEncodedJson,- FilesDB.columnPubMMdVersion,- FilesDB.columnFileSize,- FilesDB.columnAddedTime,- ]);-- final result =- await FilesDB.forTesting(- database: database,- materializationPageSize: 3,- ).getAllLocalAndUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- final file = result.files.singleWhere((file) => file.generatedID == 16);- final largeValue = _largeValue('sentinel');-- expect(file.localID, 'sentinel-local');- expect(file.uploadedFileID, 116);- expect(file.ownerID, _ownerID);- expect(file.collectionID, 26);- expect(file.title, 'sentinel-title');- expect(file.deviceFolder, 'sentinel-folder');- expect(file.location?.latitude, 12.25);- expect(file.location?.longitude, 77.75);- expect(file.fileType, FileType.video);- expect(file.creationTime, 210);- expect(file.modificationTime, 29);- expect(file.updationTime, 211);- expect(file.addedTime, 212);- expect(file.encryptedKey, largeValue);- expect(file.keyDecryptionNonce, 'sentinel-nonce');- expect(file.fileDecryptionHeader, largeValue);- expect(file.thumbnailDecryptionHeader, largeValue);- expect(file.metadataDecryptionHeader, largeValue);- expect(file.fileSubType, 7);- expect(file.duration, 1234);- expect(file.exif, largeValue);- expect(file.hash, 'sentinel-hash');- expect(file.metadataVersion, 3);- expect(file.fileSize, 987654321);- expect(file.mMdEncodedJson, '{"visibility":0}');- expect(file.mMdVersion, 4);- expect(- file.pubMmdEncodedJson,- '{"caption":"sentinel","w":8000,"h":4000}',- );- expect(file.pubMmdVersion, 5);- },- );-- test(- 'Search matches one-shot total order and defined legacy order',- () async {- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 3,- );- final paged = await filesDB.getAllFilesFromDB(- const {},- dedupeByUploadId: false,- );- final projection = FilesDB.materializedFileColumns.join(', ');- final totalOrderRows = await database.getAll(- 'SELECT $projection FROM ${FilesDB.filesTable} '- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnGeneratedID} DESC',- );- final legacyRows = await database.getAll(- 'SELECT $projection FROM ${FilesDB.filesTable} '- 'ORDER BY ${FilesDB.columnCreationTime} DESC',- );-- expect(_ids(paged), _rowIDs(totalOrderRows));- expect(- paged.map(_fileSnapshot).toList(),- totalOrderRows.map(_rowSnapshot).toList(),- );- expect(- paged.map((file) => file.creationTime).toList(),- legacyRows.map((row) => row[FilesDB.columnCreationTime]).toList(),- );- expect(_ids(paged).toSet(), _rowIDs(legacyRows).toSet());- expect(- _sortedRowSnapshots(legacyRows),- _sortedRowSnapshots(totalOrderRows),- );- },- );-- test('Search preserves dedupe, ignored collection, and tie winner', () async {- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 1,- );-- final all = await filesDB.getAllFilesFromDB(- const {},- dedupeByUploadId: false,- );- final deduped = await filesDB.getAllFilesFromDB(const {});- final ignored = await filesDB.getAllFilesFromDB(const {- _ignoredCollectionID,- }, dedupeByUploadId: false);-- expect(all, hasLength(20));- expect(deduped, hasLength(18));- expect(_ids(deduped), contains(10));- expect(_ids(deduped), isNot(contains(9)));- expect(_ids(deduped), contains(8));- expect(_ids(deduped), isNot(contains(7)));- expect(_ids(ignored), isNot(anyOf(contains(7), contains(8))));- });-- test('pending or uploaded preserves predicates and both orders', () async {- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 2,- );- final options = DBFilterOptions(dedupeUploadID: false);-- final descending = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: options,- );- final ascending = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- asc: true,- filterOptions: options,- );- final ownerChecked = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- applyOwnerCheck: true,- filterOptions: options,- );- final hidden = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- visibility: hiddenVisibility,- filterOptions: options,- );-- expect(_ids(descending.files), [17, 16, 14, 15, 13, 10, 9, 8, 7, 6, 5, 2]);- expect(_ids(ascending.files), _ids(descending.files).reversed.toList());- expect(_ids(ownerChecked.files), [17, 16, 14, 15, 10, 9, 8, 7, 5, 2]);- expect(_ids(hidden.files), [11]);- expect(descending.hasMore, isFalse);- });-- test('pending or uploaded matches a one-shot total-order query', () async {- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 3,- );- final result = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- final projection = FilesDB.materializedFileColumns.join(', ');- final rows = await database.getAll(- 'SELECT $projection FROM ${FilesDB.filesTable} '- 'WHERE ${FilesDB.columnCreationTime} >= ? '- 'AND ${FilesDB.columnCreationTime} <= ? '- 'AND (${FilesDB.columnCollectionID} IS NOT NULL '- 'AND ${FilesDB.columnCollectionID} IS NOT -1) '- 'AND ${FilesDB.columnMMdVisibility} = ? '- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnModificationTime} DESC, '- '${FilesDB.columnGeneratedID} DESC',- [100, 300, visibleVisibility],- );-- expect(_ids(result.files), _rowIDs(rows));- expect(- result.files.map(_fileSnapshot).toList(),- rows.map(_rowSnapshot).toList(),- );- });-- test(- 'local and uploaded preserves local, visibility, owner, and saved filters',- () async {- final identityFilterDB = FilesDB.forTesting(- database: database,- materializationPageSize: 3,- filter: (files, _) async => files,- );- final descending = await identityFilterDB.getAllLocalAndUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- final ascending = await identityFilterDB.getAllLocalAndUploadedFiles(- 100,- 300,- _ownerID,- asc: true,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- final ownedOnly = await identityFilterDB.getAllLocalAndUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(- dedupeUploadID: false,- ignoreSharedItems: true,- ),- );- final savedFiltered =- await FilesDB.forTesting(- database: database,- materializationPageSize: 1,- ).getAllLocalAndUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(- dedupeUploadID: false,- ignoreSavedFiles: true,- ),- );-- expect(_ids(descending.files), [- 17,- 20,- 19,- 16,- 14,- 15,- 13,- 12,- 10,- 9,- 8,- 7,- 6,- 5,- 4,- 3,- 2,- ]);- expect(_ids(ascending.files), _ids(descending.files).reversed.toList());- expect(_ids(ownedOnly.files), [- 17,- 20,- 19,- 16,- 14,- 15,- 12,- 10,- 9,- 8,- 7,- 5,- 4,- 3,- 2,- ]);- expect(_ids(savedFiltered.files), isNot(contains(6)));- expect(_ids(savedFiltered.files), containsAll([5, 13]));- for (final generatedID in [3, 4]) {- final pending = descending.files.singleWhere(- (file) => file.generatedID == generatedID,- );- expect(pending.uploadedFileID, isNull);- expect(pending.collectionID, isNull);- expect(pending.localID, isNotNull);- }-- final projection = FilesDB.materializedFileColumns.join(', ');- final baseQuery =- 'SELECT $projection FROM ${FilesDB.filesTable} '- 'WHERE ${FilesDB.columnCreationTime} >= ? '- 'AND ${FilesDB.columnCreationTime} <= ? '- 'AND (${FilesDB.columnMMdVisibility} IS NULL '- 'OR ${FilesDB.columnMMdVisibility} = ?) '- 'AND (${FilesDB.columnLocalID} IS NOT NULL OR '- '(${FilesDB.columnCollectionID} IS NOT NULL '- 'AND ${FilesDB.columnCollectionID} IS NOT -1)) ';- final oneShotDescending = await database.getAll(- '$baseQuery ORDER BY ${FilesDB.columnCreationTime} DESC, '- '${FilesDB.columnModificationTime} DESC, '- '${FilesDB.columnGeneratedID} DESC',- [100, 300, visibleVisibility],- );- final oneShotAscending = await database.getAll(- '$baseQuery ORDER BY ${FilesDB.columnCreationTime} ASC, '- '${FilesDB.columnModificationTime} ASC, '- '${FilesDB.columnGeneratedID} ASC',- [100, 300, visibleVisibility],- );- expect(_ids(descending.files), _rowIDs(oneShotDescending));- expect(_ids(ascending.files), _rowIDs(oneShotAscending));- expect(- descending.files.map(_fileSnapshot).toList(),- oneShotDescending.map(_rowSnapshot).toList(),- );- expect(- ascending.files.map(_fileSnapshot).toList(),- oneShotAscending.map(_rowSnapshot).toList(),- );- },- );-- test('limited loads preserve raw-row hasMore semantics', () async {- Future<(int, bool, FilesDBMaterializationMetrics)> load(int? limit) async {- late FilesDBMaterializationMetrics metrics;- final result =- await FilesDB.forTesting(- database: database,- materializationPageSize: 3,- materializationHooks: FilesDBMaterializationHooks(- onComplete: (value) => metrics = value,- ),- ).getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- limit: limit,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- return (result.files.length, result.hasMore, metrics);- }-- final zero = await load(0);- expect(- (zero.$1, zero.$2, zero.$3.rawRows, zero.$3.pageCount),- (0, true, 0, 0),- );-- for (final limit in [2, 3, 6, 7]) {- final value = await load(limit);- expect(value.$1, limit);- expect(value.$2, isTrue);- expect(value.$3.rawRows, limit);- expect(value.$3.maxRawPage, lessThanOrEqualTo(3));- }-- final aboveResultSize = await load(50);- expect((aboveResultSize.$1, aboveResultSize.$2), (12, false));-- final unbounded = await load(null);- expect((unbounded.$1, unbounded.$2), (12, false));- expect(unbounded.$3.pageCount, 5);-- final negative = await load(-1);- expect((negative.$1, negative.$2), (12, false));- expect(negative.$3.maxRawPage, lessThanOrEqualTo(3));-- late FilesDBMaterializationMetrics filteredMetrics;- final filtered =- await FilesDB.forTesting(- database: database,- materializationPageSize: 3,- materializationHooks: FilesDBMaterializationHooks(- onComplete: (value) => filteredMetrics = value,- ),- ).getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- limit: 7,- filterOptions: DBFilterOptions(dedupeUploadID: true),- );- expect(filtered.hasMore, isTrue);- expect(filteredMetrics.rawRows, 7);- expect(filteredMetrics.finalRows, 6);- expect(filtered.files, hasLength(6));- });-- test(- 'queries and conversions are sequential and globally filtered once',- () async {- var activePageWork = 0;- var maximumActivePageWork = 0;- var filterCalls = 0;- final pageLengths = <int>[];- final searchConversionLengths = <int>[];- late FilesDBMaterializationMetrics metrics;-- void startWork() {- activePageWork++;- maximumActivePageWork = activePageWork > maximumActivePageWork- ? activePageWork- : maximumActivePageWork;- }-- void finishWork() {- activePageWork--;- }-- final result = await FilesDB.forTesting(- database: database,- materializationPageSize: 3,- filter: (files, _) async {- filterCalls++;- expect(files, hasLength(20));- return files;- },- materializationHooks: FilesDBMaterializationHooks(- beforePageQuery: (page, transaction) async => startWork(),- afterPageQuery: (page, rows, transaction) async {- pageLengths.add(rows);- finishWork();- },- beforePageConversion: (page, rows) async {- searchConversionLengths.add(rows);- startWork();- },- afterPageConversion: (page, rows) async => finishWork(),- onComplete: (value) => metrics = value,- ),- ).getAllFilesFromDB(const {}, dedupeByUploadId: false);-- expect(result, hasLength(20));- expect(filterCalls, 1);- expect(maximumActivePageWork, 1);- expect(activePageWork, 0);- expect(pageLengths.every((length) => length <= 3), isTrue);- expect(searchConversionLengths.every((length) => length <= 3), isTrue);- expect(- searchConversionLengths,- pageLengths.where((length) => length > 0).toList(),- );- expect(metrics.maxRawPage, 3);- expect(metrics.maxRawPage, lessThanOrEqualTo(metrics.pageSize));- expect(metrics.rawRows, 20);- expect(metrics.finalRows, 20);- expect(metrics.success, isTrue);- expect(metrics.startCurrentRss, isNotNull);- expect(metrics.endMaxRss, isNotNull);-- final galleryConversionLengths = <int>[];- await FilesDB.forTesting(- database: database,- materializationPageSize: 3,- materializationHooks: FilesDBMaterializationHooks(- beforePageConversion: (page, rows) async {- galleryConversionLengths.add(rows);- },- ),- ).getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- expect(galleryConversionLengths, isNotEmpty);- expect(galleryConversionLengths.every((length) => length <= 3), isTrue);- },- );-- test(- 'exact Gallery ties use low ID ascending and high ID descending',- () async {- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 2,- );- final options = DBFilterOptions(dedupeUploadID: true);-- final descending = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: options,- );- final ascending = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- asc: true,- filterOptions: options,- );-- expect(_ids(descending.files), contains(10));- expect(_ids(descending.files), isNot(contains(9)));- expect(_ids(ascending.files), contains(9));- expect(_ids(ascending.files), isNot(contains(10)));- },- );-- test(- 'page query failure closes the transaction and a retry succeeds',- () async {- var shouldFail = true;- var filterCalls = 0;- SqliteReadContext? failedTransaction;- final metrics = <FilesDBMaterializationMetrics>[];- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 2,- filter: (files, _) async {- filterCalls++;- return files;- },- materializationHooks: FilesDBMaterializationHooks(- beforePageQuery: (page, transaction) async {- if (shouldFail && page == 2) {- failedTransaction = transaction;- shouldFail = false;- await transaction.getAll(- 'SELECT missing_test_column FROM ${FilesDB.filesTable}',- );- }- },- onComplete: metrics.add,- ),- );-- await expectLater(- filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- ),- throwsA(isA<Exception>()),- );- expect(filterCalls, 0);- expect(failedTransaction?.closed, isTrue);- expect(metrics.single.success, isFalse);- expect(metrics.single.errorType, 'SqliteException');-- final retry = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- expect(retry.files, hasLength(12));- expect(filterCalls, 1);- expect(metrics.last.success, isTrue);- },- );-- test(- 'page conversion failure returns no partial result and permits retry',- () async {- var conversionPage = 0;- var shouldFail = true;- var filterCalls = 0;- SqliteReadContext? failedTransaction;- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 2,- galleryPageConverter: (rows) async {- conversionPage++;- if (shouldFail && conversionPage == 2) {- shouldFail = false;- throw StateError('injected page conversion failure');- }- return _convertRowsForTest(rows);- },- filter: (files, _) async {- filterCalls++;- return files;- },- materializationHooks: FilesDBMaterializationHooks(- beforePageQuery: (page, transaction) async {- failedTransaction = transaction;- },- ),- );-- await expectLater(- filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- ),- throwsStateError,- );- expect(filterCalls, 0);- expect(failedTransaction?.closed, isTrue);-- final retry = await filesDB.getAllPendingOrUploadedFiles(- 100,- 300,- _ownerID,- filterOptions: DBFilterOptions(dedupeUploadID: false),- );- expect(retry.files, hasLength(12));- expect(filterCalls, 1);- },- );-- test(- 'final filter failure cannot escape partial data and permits retry',- () async {- var shouldFail = true;- var filterInputs = 0;- SqliteReadContext? finalTransaction;- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 3,- filter: (files, _) async {- filterInputs = files.length;- if (shouldFail) {- shouldFail = false;- throw StateError('injected final filter failure');- }- return files;- },- materializationHooks: FilesDBMaterializationHooks(- beforePageQuery: (page, transaction) async {- finalTransaction = transaction;- },- ),- );-- await expectLater(- filesDB.getAllFilesFromDB(const {}, dedupeByUploadId: false),- throwsStateError,- );- expect(filterInputs, 20);- expect(finalTransaction?.closed, isTrue);-- final retry = await filesDB.getAllFilesFromDB(- const {},- dedupeByUploadId: false,- );- expect(retry, hasLength(20));- },- );-- test(- 'all pages share one snapshot while a concurrent writer proceeds',- () async {- final firstPageRead = Completer<void>();- final releaseReader = Completer<void>();- final filesDB = FilesDB.forTesting(- database: database,- materializationPageSize: 2,- materializationHooks: FilesDBMaterializationHooks(- afterPageQuery: (page, rows, transaction) async {- if (page == 1 && !firstPageRead.isCompleted) {- firstPageRead.complete();- await releaseReader.future;- }- },- ),- );-- final activeRead = filesDB.getAllFilesFromDB(- const {},- dedupeByUploadId: false,- );- await firstPageRead.future;- try {- await _insertRows(database, [- _row(- id: 100,- creationTime: 250,- modificationTime: 50,- uploadedFileID: 9100,- collectionID: 910,- ),- ]);- } finally {- releaseReader.complete();- }-- final snapshot = await activeRead;- expect(snapshot, hasLength(20));- expect(_ids(snapshot), isNot(contains(100)));-- final nextRead = await filesDB.getAllFilesFromDB(- const {},- dedupeByUploadId: false,- );- expect(nextRead, hasLength(21));- expect(_ids(nextRead), contains(100));- },- );-}--List<Map<String, Object?>> _fixtureRows() => [- _row(id: 1, creationTime: 99, modificationTime: 1),- _row(id: 2, creationTime: 100, modificationTime: 10),- _row(- id: 3,- creationTime: 110,- modificationTime: 11,- uploadedFileID: null,- collectionID: null,- localID: 'pending-null',- ),- _row(- id: 4,- creationTime: 120,- modificationTime: 12,- uploadedFileID: -1,- collectionID: -1,- localID: 'pending-legacy',- ),- _row(- id: 5,- creationTime: 130,- modificationTime: 13,- uploadedFileID: 105,- collectionID: 15,- ownerID: _ownerID,- hash: 'owned-shared-hash',- ),- _row(- id: 6,- creationTime: 130,- modificationTime: 13,- uploadedFileID: 106,- collectionID: 16,- ownerID: _sharedOwnerID,- hash: 'owned-shared-hash',- ),- _row(- id: 7,- creationTime: 140,- modificationTime: 14,- uploadedFileID: 600,- collectionID: 100,- ),- _row(- id: 8,- creationTime: 150,- modificationTime: 15,- uploadedFileID: 600,- collectionID: _ignoredCollectionID,- ),- _row(- id: 9,- creationTime: 160,- modificationTime: 16,- uploadedFileID: 500,- collectionID: 201,- ),- _row(- id: 10,- creationTime: 160,- modificationTime: 16,- uploadedFileID: 500,- collectionID: 202,- ),- _row(- id: 11,- creationTime: 170,- modificationTime: 20,- visibility: hiddenVisibility,- ),- _row(id: 12, creationTime: 180, modificationTime: 21, visibility: null),- _row(- id: 13,- creationTime: 190,- modificationTime: 22,- ownerID: _sharedOwnerID,- ),- _row(id: 14, creationTime: 200, modificationTime: 31, ownerID: null),- _row(id: 15, creationTime: 200, modificationTime: 30),- _row(- id: 16,- creationTime: 210,- modificationTime: 29,- uploadedFileID: 116,- collectionID: 26,- localID: 'sentinel-local',- title: 'sentinel-title',- deviceFolder: 'sentinel-folder',- latitude: 12.25,- longitude: 77.75,- fileType: 1,- updationTime: 211,- addedTime: 212,- encryptedKey: _largeValue('sentinel'),- keyDecryptionNonce: 'sentinel-nonce',- fileDecryptionHeader: _largeValue('sentinel'),- thumbnailDecryptionHeader: _largeValue('sentinel'),- metadataDecryptionHeader: _largeValue('sentinel'),- fileSubType: 7,- duration: 1234,- exif: _largeValue('sentinel'),- hash: 'sentinel-hash',- metadataVersion: 3,- mMdVersion: 4,- pubMmdEncodedJson: '{"caption":"sentinel","w":8000,"h":4000}',- pubMmdVersion: 5,- fileSize: 987654321,- ),- _row(id: 17, creationTime: 300, modificationTime: 40),- _row(id: 18, creationTime: 301, modificationTime: 41),- _row(- id: 19,- creationTime: 220,- modificationTime: 31,- uploadedFileID: null,- collectionID: null,- localID: 'pending-null-late',- ),- _row(- id: 20,- creationTime: 230,- modificationTime: 32,- uploadedFileID: -1,- collectionID: -1,- localID: 'pending-legacy-late',- ),-];--Map<String, Object?> _row({- required int id,- required int creationTime,- required int modificationTime,- Object? uploadedFileID = _useGeneratedValue,- Object? collectionID = _useGeneratedValue,- Object? ownerID = _ownerID,- String? localID,- String? title,- String? deviceFolder,- double? latitude,- double? longitude,- int fileType = 0,- int? updationTime,- int? addedTime,- String? encryptedKey,- String? keyDecryptionNonce,- String? fileDecryptionHeader,- String? thumbnailDecryptionHeader,- String? metadataDecryptionHeader,- int? fileSubType,- int? duration,- String? exif,- String? hash,- int? metadataVersion,- String? mMdEncodedJson,- int? mMdVersion,- Object? visibility = visibleVisibility,- String? pubMmdEncodedJson,- int? pubMmdVersion,- int? fileSize,-}) {- final resolvedUploadID = identical(uploadedFileID, _useGeneratedValue)- ? 1000 + id- : uploadedFileID;- final resolvedCollectionID = identical(collectionID, _useGeneratedValue)- ? 100 + id- : collectionID;- return {- FilesDB.columnGeneratedID: id,- FilesDB.columnLocalID: localID,- FilesDB.columnUploadedFileID: resolvedUploadID,- FilesDB.columnOwnerID: ownerID,- FilesDB.columnCollectionID: resolvedCollectionID,- FilesDB.columnTitle: title ?? 'fixture-$id',- FilesDB.columnDeviceFolder: deviceFolder,- FilesDB.columnLatitude: latitude,- FilesDB.columnLongitude: longitude,- FilesDB.columnFileType: fileType,- FilesDB.columnModificationTime: modificationTime,- FilesDB.columnEncryptedKey: encryptedKey,- FilesDB.columnKeyDecryptionNonce: keyDecryptionNonce,- FilesDB.columnFileDecryptionHeader: fileDecryptionHeader,- FilesDB.columnThumbnailDecryptionHeader: thumbnailDecryptionHeader,- FilesDB.columnMetadataDecryptionHeader: metadataDecryptionHeader,- FilesDB.columnCreationTime: creationTime,- FilesDB.columnUpdationTime: updationTime,- FilesDB.columnFileSubType: fileSubType,- FilesDB.columnDuration: duration,- FilesDB.columnExif: exif,- FilesDB.columnHash: hash ?? 'hash-$id',- FilesDB.columnMetadataVersion: metadataVersion,- FilesDB.columnMMdEncodedJson:- mMdEncodedJson ?? '{"visibility":${visibility ?? visibleVisibility}}',- FilesDB.columnMMdVersion: mMdVersion,- FilesDB.columnMMdVisibility: visibility,- FilesDB.columnPubMMdEncodedJson: pubMmdEncodedJson ?? '{}',- FilesDB.columnPubMMdVersion: pubMmdVersion,- FilesDB.columnFileSize: fileSize,- FilesDB.columnAddedTime: addedTime ?? -1,- };-}--Future<void> _insertRows(- SqliteDatabase database,- List<Map<String, Object?>> rows,-) async {- final columns = FilesDB.materializedFileColumns;- final placeholders = List.filled(columns.length, '?').join(', ');- await database.executeBatch(- 'INSERT INTO ${FilesDB.filesTable} (${columns.join(', ')}) '- 'VALUES ($placeholders)',- rows- .map((row) => columns.map<Object?>((column) => row[column]).toList())- .toList(),- );-}--List<EnteFile> _convertRowsForTest(List<Map<String, dynamic>> rows) =>- rows.map((row) {- final file = EnteFile();- file.generatedID = row[FilesDB.columnGeneratedID];- file.localID = row[FilesDB.columnLocalID];- final uploadedID = row[FilesDB.columnUploadedFileID];- file.uploadedFileID = uploadedID == -1 ? null : uploadedID;- file.ownerID = row[FilesDB.columnOwnerID];- final collectionID = row[FilesDB.columnCollectionID];- file.collectionID = collectionID == -1 ? null : collectionID;- file.title = row[FilesDB.columnTitle];- file.fileType = getFileType(row[FilesDB.columnFileType]);- file.creationTime = row[FilesDB.columnCreationTime];- file.modificationTime = row[FilesDB.columnModificationTime];- file.hash = row[FilesDB.columnHash];- file.mMdEncodedJson = row[FilesDB.columnMMdEncodedJson];- file.pubMmdEncodedJson = row[FilesDB.columnPubMMdEncodedJson];- return file;- }).toList();--List<int> _ids(Iterable<EnteFile> files) =>- files.map((file) => file.generatedID!).toList();--List<int> _rowIDs(Iterable<Map<String, dynamic>> rows) =>- rows.map((row) => row[FilesDB.columnGeneratedID] as int).toList();--Map<String, Object?> _fileSnapshot(EnteFile file) => {- 'generatedID': file.generatedID,- 'localID': file.localID,- 'uploadedFileID': file.uploadedFileID,- 'ownerID': file.ownerID,- 'collectionID': file.collectionID,- 'title': file.title,- 'deviceFolder': file.deviceFolder,- 'latitude': file.location?.latitude,- 'longitude': file.location?.longitude,- 'fileType': file.fileType.name,- 'creationTime': file.creationTime,- 'modificationTime': file.modificationTime,- 'updationTime': file.updationTime,- 'addedTime': file.addedTime,- 'encryptedKey': file.encryptedKey,- 'keyDecryptionNonce': file.keyDecryptionNonce,- 'fileDecryptionHeader': file.fileDecryptionHeader,- 'thumbnailDecryptionHeader': file.thumbnailDecryptionHeader,- 'metadataDecryptionHeader': file.metadataDecryptionHeader,- 'fileSubType': file.fileSubType,- 'duration': file.duration,- 'exif': file.exif,- 'hash': file.hash,- 'metadataVersion': file.metadataVersion,- 'fileSize': file.fileSize,- 'mMdEncodedJson': file.mMdEncodedJson,- 'mMdVersion': file.mMdVersion,- 'pubMmdEncodedJson': file.pubMmdEncodedJson,- 'pubMmdVersion': file.pubMmdVersion,-};--Map<String, Object?> _rowSnapshot(Map<String, dynamic> row) {- final uploadedID = row[FilesDB.columnUploadedFileID];- final collectionID = row[FilesDB.columnCollectionID];- final hasLocation =- row[FilesDB.columnLatitude] != null &&- row[FilesDB.columnLongitude] != null;- return {- 'generatedID': row[FilesDB.columnGeneratedID],- 'localID': row[FilesDB.columnLocalID],- 'uploadedFileID': uploadedID == -1 ? null : uploadedID,- 'ownerID': row[FilesDB.columnOwnerID],- 'collectionID': collectionID == -1 ? null : collectionID,- 'title': row[FilesDB.columnTitle],- 'deviceFolder': row[FilesDB.columnDeviceFolder],- 'latitude': hasLocation ? row[FilesDB.columnLatitude] : null,- 'longitude': hasLocation ? row[FilesDB.columnLongitude] : null,- 'fileType': getFileType(row[FilesDB.columnFileType]).name,- 'creationTime': row[FilesDB.columnCreationTime],- 'modificationTime': row[FilesDB.columnModificationTime],- 'updationTime': row[FilesDB.columnUpdationTime] ?? -1,- 'addedTime': row[FilesDB.columnAddedTime],- 'encryptedKey': row[FilesDB.columnEncryptedKey],- 'keyDecryptionNonce': row[FilesDB.columnKeyDecryptionNonce],- 'fileDecryptionHeader': row[FilesDB.columnFileDecryptionHeader],- 'thumbnailDecryptionHeader': row[FilesDB.columnThumbnailDecryptionHeader],- 'metadataDecryptionHeader': row[FilesDB.columnMetadataDecryptionHeader],- 'fileSubType': row[FilesDB.columnFileSubType] ?? -1,- 'duration': row[FilesDB.columnDuration] ?? 0,- 'exif': row[FilesDB.columnExif],- 'hash': row[FilesDB.columnHash],- 'metadataVersion': row[FilesDB.columnMetadataVersion] ?? 0,- 'fileSize': row[FilesDB.columnFileSize],- 'mMdEncodedJson': row[FilesDB.columnMMdEncodedJson] ?? '{}',- 'mMdVersion': row[FilesDB.columnMMdVersion] ?? 0,- 'pubMmdEncodedJson': row[FilesDB.columnPubMMdEncodedJson] ?? '{}',- 'pubMmdVersion': row[FilesDB.columnPubMMdVersion] ?? 0,- };-}--List<Map<String, Object?>> _sortedRowSnapshots(- Iterable<Map<String, dynamic>> rows,-) {- final snapshots = rows.map(_rowSnapshot).toList();- snapshots.sort(- (left, right) =>- (left['generatedID'] as int).compareTo(right['generatedID'] as int),- );- return snapshots;-}--String _largeValue(String prefix) => '$prefix-${List.filled(4096, 'x').join()}';--class _FakePathProvider extends PathProviderPlatform {- final String rootPath;-- _FakePathProvider(this.rootPath);-- @override- Future<String?> getApplicationDocumentsPath() async => rootPath;-- @override- Future<String?> getApplicationSupportPath() async => rootPath;-- @override- Future<String?> getTemporaryPath() async => rootPath;-}