Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[photos] Keep FilesDB materialization branch production-only

ashil-pilot7390b9dd39bc
diff --git a/mobile/apps/photos/benchmark/run_files_db_materialization_benchmark.sh b/mobile/apps/photos/benchmark/run_files_db_materialization_benchmark.sh
deleted file mode 100755
index 366809d480..0000000000
--- a/mobile/apps/photos/benchmark/run_files_db_materialization_benchmark.sh
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-if [[ $# -ne 1 || -z "$1" || "$1" == "/" ]]; then
- printf 'Usage: %s <persistent-result-directory>\n' "$0" >&2
- exit 64
-fi
-
-result_dir="$1"
-mkdir -p "$result_dir"
-result_dir="$(cd "$result_dir" && pwd -P)"
-if [[ "$result_dir" == "/" ]]; then
- printf 'Refusing to use the filesystem root as the result directory\n' >&2
- exit 64
-fi
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
-photos_dir="$(cd "$script_dir/.." && pwd -P)"
-database_path="$result_dir/files-db-100000.sqlite"
-raw_log="$result_dir/raw-runs.log"
-measurements="$result_dir/measurements.jsonl"
-report="$result_dir/benchmark-report.md"
-benchmark_test="test/benchmark/files_db_materialization_benchmark_test.dart"
-
-for output in "$database_path" "$raw_log" "$measurements" "$report"; do
- if [[ -e "$output" ]]; then
- printf 'Refusing to overwrite existing benchmark output: %s\n' "$output" >&2
- exit 1
- fi
-done
-
-cleanup_fixture() {
- if [[ "${FILESDB_BENCHMARK_KEEP_FIXTURE:-0}" == "1" ]]; then
- return
- fi
- for generated_database_file in \
- "$database_path" \
- "$database_path-shm" \
- "$database_path-wal"; do
- if [[ -e "$generated_database_file" ]]; then
- find "$generated_database_file" -delete
- fi
- done
-}
-
-trap cleanup_fixture EXIT
-trap 'exit 130' INT
-trap 'exit 143' TERM
-
-cd "$photos_dir"
-touch "$raw_log" "$measurements"
-
-run_worker() {
- local variant="$1"
- local run_kind="$2"
- local page_size="$3"
- local family="$4"
- FILESDB_BENCHMARK_MODE=run \
- FILESDB_BENCHMARK_DB="$database_path" \
- FILESDB_BENCHMARK_VARIANT="$variant" \
- FILESDB_BENCHMARK_RUN_KIND="$run_kind" \
- FILESDB_BENCHMARK_PAGE_SIZE="$page_size" \
- FILESDB_BENCHMARK_FAMILY="$family" \
- flutter test --no-pub "$benchmark_test" 2>&1 \
- | tee -a "$raw_log" \
- | tr '\r' '\n' \
- | sed -n 's/^.*FILESDB_BENCHMARK_JSON //p' >> "$measurements"
-}
-
-FILESDB_BENCHMARK_MODE=build \
-FILESDB_BENCHMARK_DB="$database_path" \
- flutter test --no-pub "$benchmark_test" 2>&1 \
- | tee -a "$raw_log" \
- | tr '\r' '\n' \
- | sed -n 's/^.*FILESDB_BENCHMARK_JSON //p' >> "$measurements"
-
-for family in \
- search_all_files \
- gallery_pending_or_uploaded \
- gallery_local_and_uploaded; do
- run_worker legacy warmup 2000 "$family"
- for run in 1 2 3 4 5; do
- run_worker legacy measured 2000 "$family"
- done
- for page_size in 500 1000 2000 5000; do
- run_worker paged warmup "$page_size" "$family"
- for run in 1 2 3 4 5; do
- run_worker paged measured "$page_size" "$family"
- done
- done
-done
-
-dart benchmark/summarize_files_db_materialization_benchmark.dart \
- "$measurements" > "$report"
-
-printf 'Benchmark report: %s\n' "$report"
diff --git a/mobile/apps/photos/benchmark/summarize_files_db_materialization_benchmark.dart b/mobile/apps/photos/benchmark/summarize_files_db_materialization_benchmark.dart
deleted file mode 100644
index b93b2d494b..0000000000
--- a/mobile/apps/photos/benchmark/summarize_files_db_materialization_benchmark.dart
+++ /dev/null
@@ -1,196 +0,0 @@
-import 'dart:convert';
-import 'dart:io';
-
-void main(List<String> arguments) {
- if (arguments.length != 1) {
- stderr.writeln('Usage: dart run <script> <measurements.jsonl>');
- exitCode = 64;
- return;
- }
-
- final allRecords = File(arguments.single)
- .readAsLinesSync()
- .where((line) => line.trim().isNotEmpty)
- .map((line) => jsonDecode(line) as Map<String, dynamic>)
- .toList();
- final records = allRecords
- .where((record) => record['event'] == 'measurement')
- .where((record) => record['runKind'] == 'measured')
- .toList();
- final grouped = <String, List<Map<String, dynamic>>>{};
- for (final record in records) {
- final variant = record['variant'] as String;
- final family = record['benchmarkFamily'] as String;
- final label = variant == 'legacy'
- ? 'legacy-$family'
- : '$family-paged-${record['pageSize'] as int}';
- grouped.putIfAbsent(label, () => []).add(record);
- }
-
- const expectedLabels = [
- 'legacy-search_all_files',
- 'search_all_files-paged-500',
- 'search_all_files-paged-1000',
- 'search_all_files-paged-2000',
- 'search_all_files-paged-5000',
- 'legacy-gallery_pending_or_uploaded',
- 'gallery_pending_or_uploaded-paged-500',
- 'gallery_pending_or_uploaded-paged-1000',
- 'gallery_pending_or_uploaded-paged-2000',
- 'gallery_pending_or_uploaded-paged-5000',
- 'legacy-gallery_local_and_uploaded',
- 'gallery_local_and_uploaded-paged-500',
- 'gallery_local_and_uploaded-paged-1000',
- 'gallery_local_and_uploaded-paged-2000',
- 'gallery_local_and_uploaded-paged-5000',
- ];
- for (final label in expectedLabels) {
- final runs = grouped[label] ?? const [];
- if (runs.length < 5) {
- stderr.writeln(
- '$label has ${runs.length} measured runs; expected at least 5',
- );
- exitCode = 1;
- return;
- }
- }
-
- stdout.writeln('# FilesDB 100,000-row materialization benchmark');
- stdout.writeln();
- stdout.writeln('- Fixture rows: 100,000');
- stdout.writeln('- Fixture seed: 20260825');
- stdout.writeln(
- '- One fresh-process warm-up and five fresh-process measured runs per query family/page size',
- );
- stdout.writeln(
- '- RSS uses `dart:io` `ProcessInfo.currentRss`/`maxRss`, sampled '
- 'immediately before and after materialization in each fresh process; it '
- 'is process-wide, informational, and not an acceptance gate',
- );
- stdout.writeln();
- stdout.writeln(
- '| Variant | Runs | Median wall ms | Median transaction ms | Median query ms | Median conversion ms | Median filter ms | Max raw page | End current RSS MiB values | End max RSS MiB values |',
- );
- stdout.writeln('|---|---:|---:|---:|---:|---:|---:|---:|---|---|');
-
- for (final label in expectedLabels) {
- final runs = grouped[label]!;
- final wall = _median(_values(runs, 'wallMicros')) / 1000;
- final transaction = _median(_values(runs, 'transactionMicros')) / 1000;
- final query = _median(_values(runs, 'queryMicros')) / 1000;
- final conversion = _median(_values(runs, 'conversionMicros')) / 1000;
- final filter = _median(_values(runs, 'filterMicros')) / 1000;
- final maxRawPage = runs
- .map((run) => run['maxRawPage'] as int)
- .reduce((left, right) => left > right ? left : right);
- final currentRss = runs
- .map((run) => _mib(run['endCurrentRss'] as int))
- .map((value) => value.toStringAsFixed(1))
- .join(', ');
- final maxRss = runs
- .map((run) => _mib(run['endMaxRss'] as int))
- .map((value) => value.toStringAsFixed(1))
- .join(', ');
-
- stdout.writeln(
- '| $label | ${runs.length} | ${wall.toStringAsFixed(1)} | '
- '${transaction.toStringAsFixed(1)} | ${query.toStringAsFixed(1)} | '
- '${conversion.toStringAsFixed(1)} | ${filter.toStringAsFixed(1)} | '
- '$maxRawPage | $currentRss | $maxRss |',
- );
- }
-
- stdout.writeln();
- for (final family in [
- 'search_all_files',
- 'gallery_pending_or_uploaded',
- 'gallery_local_and_uploaded',
- ]) {
- final legacyMax = grouped['legacy-$family']!
- .map((run) => run['maxRawPage'] as int)
- .reduce((left, right) => left > right ? left : right);
- final productionMax = grouped['$family-paged-2000']!
- .map((run) => run['maxRawPage'] as int)
- .reduce((left, right) => left > right ? left : right);
- final reduction = (1 - (productionMax / legacyMax)) * 100;
- stdout.writeln(
- 'The selected 2,000-row `$family` load bounds each raw payload at '
- '$productionMax rows versus $legacyMax for the legacy one-shot load '
- '(${reduction.toStringAsFixed(1)}% fewer rows per maximum payload).',
- );
- }
-
- stdout.writeln();
- stdout.writeln('## Gallery paging investigation');
- stdout.writeln();
- for (final family in [
- 'gallery_pending_or_uploaded',
- 'gallery_local_and_uploaded',
- ]) {
- final legacyRuns = grouped['legacy-$family']!;
- final selectedRuns = grouped['$family-paged-2000']!;
- final legacyWall = _median(_values(legacyRuns, 'wallMicros')) / 1000;
- final selectedWall = _median(_values(selectedRuns, 'wallMicros')) / 1000;
- final legacyTransaction =
- _median(_values(legacyRuns, 'transactionMicros')) / 1000;
- final selectedTransaction =
- _median(_values(selectedRuns, 'transactionMicros')) / 1000;
- stdout.writeln(
- '- `$family`: selected 2,000-row paging median wall time '
- '${selectedWall.toStringAsFixed(1)} ms versus '
- '${legacyWall.toStringAsFixed(1)} ms one-shot '
- '(${_percentDelta(selectedWall, legacyWall)}); median transaction time '
- '${selectedTransaction.toStringAsFixed(1)} ms versus '
- '${legacyTransaction.toStringAsFixed(1)} ms '
- '(${_percentDelta(selectedTransaction, legacyTransaction)}).',
- );
- }
- stdout.writeln(
- '- An unindexed diagnostic run exposed `USE TEMP B-TREE FOR LAST 2 TERMS '
- 'OF ORDER BY` and selected-page medians of 1,479.8 ms (pending) and '
- '1,408.6 ms (local). The selected composite '
- '`(creation_time, modification_time, _id)` index removes that repeated '
- 'sort from the plans below.',
- );
- stdout.writeln(
- '- On this fixed fixture, creating the candidate index took 0.10 seconds '
- 'and increased the database from 98,906,112 to 101,859,328 bytes '
- '(2,953,216 bytes, 3.0%). This startup migration and storage cost was '
- 'accepted to avoid the measured Gallery regression while retaining the '
- '98% payload reduction.',
- );
-
- stdout.writeln();
- stdout.writeln('## Query plans');
- stdout.writeln();
- final plansByFamily = <String, Set<String>>{};
- for (final record in allRecords.where(
- (record) => record['event'] == 'queryPlan',
- )) {
- final family = record['family'] as String;
- final details = (record['details'] as List<dynamic>).cast<String>();
- plansByFamily.putIfAbsent(family, () => {}).addAll(details);
- }
- for (final family in plansByFamily.keys.toList()..sort()) {
- stdout.writeln('- `$family`: ${plansByFamily[family]!.join('; ')}');
- }
-}
-
-List<int> _values(List<Map<String, dynamic>> records, String key) =>
- records.map((record) => record[key] as int).toList();
-
-double _median(List<int> values) {
- values.sort();
- final middle = values.length ~/ 2;
- if (values.length.isOdd) {
- return values[middle].toDouble();
- }
- return (values[middle - 1] + values[middle]) / 2;
-}
-
-double _mib(int bytes) => bytes / (1024 * 1024);
-
-String _percentDelta(double selected, double legacy) {
- final delta = ((selected / legacy) - 1) * 100;
- return '${delta >= 0 ? '+' : ''}${delta.toStringAsFixed(1)}%';
-}
diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dart
index b3d176ccd8..bcf174aaa2 100644
--- a/mobile/apps/photos/lib/db/files_db.dart
+++ b/mobile/apps/photos/lib/db/files_db.dart
@@ -130,56 +130,17 @@ class FilesDB with SqlDbBase {
static final String _materializedFileProjection = _columnNames.join(", ");
static int _nextMaterializationOperationID = 0;
- final Future<SqliteDatabase>? _databaseOverride;
- final int materializationPageSize;
- final FilesDBMaterializationHooks? _materializationHooks;
- final FilesDBPageConverter? _searchPageConverterOverride;
- final FilesDBPageConverter? _galleryPageConverterOverride;
- final FilesDBFilter? _filterOverride;
-
- FilesDB._privateConstructor()
- : _databaseOverride = null,
- materializationPageSize = defaultMaterializationPageSize,
- _materializationHooks = null,
- _searchPageConverterOverride = null,
- _galleryPageConverterOverride = null,
- _filterOverride = null;
-
- @visibleForTesting
- FilesDB.forTesting({
- required SqliteDatabase database,
- this.materializationPageSize = defaultMaterializationPageSize,
- FilesDBMaterializationHooks? materializationHooks,
- FilesDBPageConverter? searchPageConverter,
- FilesDBPageConverter? galleryPageConverter,
- FilesDBFilter? filter,
- }) : assert(materializationPageSize > 0),
- _databaseOverride = Future.value(database),
- _materializationHooks = materializationHooks,
- _searchPageConverterOverride = searchPageConverter,
- _galleryPageConverterOverride = galleryPageConverter,
- _filterOverride = filter;
+ FilesDB._privateConstructor();
static final FilesDB instance = FilesDB._privateConstructor();
static Future<SqliteDatabase>? _sqliteAsyncDBFuture;
Future<SqliteDatabase> get sqliteAsyncDB async {
- if (_databaseOverride != null) {
- return _databaseOverride;
- }
_sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase();
return _sqliteAsyncDBFuture!;
}
- @visibleForTesting
- static List<String> get materializedFileColumns =>
- List.unmodifiable(_columnNames);
-
- @visibleForTesting
- static Future<void> migrateTestDatabase(SqliteDatabase database) =>
- instance.migrate(database, _migrationScripts);
-
Future<SqliteDatabase> _initSqliteAsyncDatabase() async {
final Directory documentsDirectory =
await getApplicationDocumentsDirectory();
@@ -1969,10 +1930,6 @@ class FilesDB with SqlDbBase {
}
Future<List<EnteFile>> _convertSearchPage(List<Map<String, dynamic>> rows) {
- final override = _searchPageConverterOverride;
- if (override != null) {
- return override(rows);
- }
return Computer.shared().compute(
_convertFilesDBRowsForIsolate,
param: {"result": rows},
@@ -1983,10 +1940,6 @@ class FilesDB with SqlDbBase {
Future<List<EnteFile>> _convertGalleryPage(
List<Map<String, dynamic>> rows,
) async {
- final override = _galleryPageConverterOverride;
- if (override != null) {
- return override(rows);
- }
return convertToFiles(rows);
}
@@ -1995,14 +1948,14 @@ class FilesDB with SqlDbBase {
required String whereClause,
required List<Object?> whereArguments,
required _MaterializedFileOrder order,
- required FilesDBPageConverter converter,
+ required _FilesDBPageConverter converter,
DBFilterOptions? filterOptions,
int? limit,
}) async {
final operation = _FilesDBMaterializationOperation(
id: ++_nextMaterializationOperationID,
family: family,
- pageSize: materializationPageSize,
+ pageSize: defaultMaterializationPageSize,
);
final files = <EnteFile>[];
final boundedLimit = limit != null && limit >= 0 ? limit : null;
@@ -2018,13 +1971,9 @@ class FilesDB with SqlDbBase {
while (remaining == null || remaining > 0) {
final requestedRows = remaining == null
- ? materializationPageSize
- : min(materializationPageSize, remaining);
+ ? defaultMaterializationPageSize
+ : min(defaultMaterializationPageSize, remaining);
final pageNumber = operation.pageCount + 1;
- await _materializationHooks?.beforePageQuery?.call(
- pageNumber,
- transaction,
- );
final queryWatch = Stopwatch()..start();
late final List<Map<String, dynamic>> rows;
@@ -2050,7 +1999,7 @@ class FilesDB with SqlDbBase {
operation.rawRows += rows.length;
operation.maxRawPage = max(operation.maxRawPage, rows.length);
if (rows.length > requestedRows ||
- rows.length > materializationPageSize) {
+ rows.length > defaultMaterializationPageSize) {
throw StateError(
'FilesDB materialization page exceeded its configured bound',
);
@@ -2065,21 +2014,11 @@ class FilesDB with SqlDbBase {
);
}
- await _materializationHooks?.afterPageQuery?.call(
- pageNumber,
- rows.length,
- transaction,
- );
-
if (rows.isEmpty) {
break;
}
cursor = order.cursorFrom(rows.last);
- await _materializationHooks?.beforePageConversion?.call(
- pageNumber,
- rows.length,
- );
final conversionWatch = Stopwatch()..start();
late final List<EnteFile> convertedPage;
try {
@@ -2088,10 +2027,6 @@ class FilesDB with SqlDbBase {
conversionWatch.stop();
operation.recordConversion(conversionWatch.elapsed);
}
- await _materializationHooks?.afterPageConversion?.call(
- pageNumber,
- rows.length,
- );
files.addAll(convertedPage);
if (remaining != null) {
@@ -2109,8 +2044,7 @@ class FilesDB with SqlDbBase {
operation.filterWatch.start();
late final List<EnteFile> filteredFiles;
try {
- final filter = _filterOverride ?? applyDBFilters;
- filteredFiles = await filter(files, filterOptions);
+ filteredFiles = await applyDBFilters(files, filterOptions);
} finally {
operation.filterWatch.stop();
}
@@ -2155,15 +2089,6 @@ class FilesDB with SqlDbBase {
} else {
_logger.warning(message);
}
- try {
- _materializationHooks?.onComplete?.call(metrics);
- } catch (hookError, hookStackTrace) {
- _logger.warning(
- 'FilesDBMaterialization metrics hook failed',
- hookError,
- hookStackTrace,
- );
- }
}
Future<bool> hasAnyFile() async {
@@ -2446,42 +2371,10 @@ class FilesDB with SqlDbBase {
}
}
-typedef FilesDBPageConverter =
+typedef _FilesDBPageConverter =
Future<List<EnteFile>> Function(List<Map<String, dynamic>> rows);
-typedef FilesDBFilter =
- Future<List<EnteFile>> Function(
- List<EnteFile> files,
- DBFilterOptions? options,
- );
-
-@visibleForTesting
-class FilesDBMaterializationHooks {
- final Future<void> Function(int pageNumber, SqliteReadContext transaction)?
- beforePageQuery;
- final Future<void> Function(
- int pageNumber,
- int rowCount,
- SqliteReadContext transaction,
- )?
- afterPageQuery;
- final Future<void> Function(int pageNumber, int rowCount)?
- beforePageConversion;
- final Future<void> Function(int pageNumber, int rowCount)?
- afterPageConversion;
- final void Function(FilesDBMaterializationMetrics metrics)? onComplete;
-
- const FilesDBMaterializationHooks({
- this.beforePageQuery,
- this.afterPageQuery,
- this.beforePageConversion,
- this.afterPageConversion,
- this.onComplete,
- });
-}
-
-@visibleForTesting
-class FilesDBMaterializationMetrics {
+class _FilesDBMaterializationMetrics {
final int operationID;
final String family;
final int pageSize;
@@ -2503,7 +2396,7 @@ class FilesDBMaterializationMetrics {
final int? startMaxRss;
final int? endMaxRss;
- const FilesDBMaterializationMetrics({
+ const _FilesDBMaterializationMetrics({
required this.operationID,
required this.family,
required this.pageSize,
@@ -2548,29 +2441,6 @@ class FilesDBMaterializationMetrics {
'startMaxRss=${startMaxRss ?? 'unavailable'}',
'endMaxRss=${endMaxRss ?? 'unavailable'}',
].join(' ');
-
- Map<String, Object?> toJson() => {
- 'operationID': operationID,
- 'family': family,
- 'status': success ? 'success' : 'error',
- 'errorType': errorType,
- 'pageSize': pageSize,
- 'pageCount': pageCount,
- 'rawRows': rawRows,
- 'finalRows': finalRows,
- 'maxRawPage': maxRawPage,
- 'queryMicros': queryDuration.inMicroseconds,
- 'maxQueryMicros': maxQueryDuration.inMicroseconds,
- 'conversionMicros': conversionDuration.inMicroseconds,
- 'maxConversionMicros': maxConversionDuration.inMicroseconds,
- 'filterMicros': filterDuration.inMicroseconds,
- 'transactionMicros': transactionDuration.inMicroseconds,
- 'totalMicros': totalDuration.inMicroseconds,
- 'startCurrentRss': startCurrentRss,
- 'endCurrentRss': endCurrentRss,
- 'startMaxRss': startMaxRss,
- 'endMaxRss': endMaxRss,
- };
}
enum _MaterializedFileOrder {
@@ -2662,7 +2532,7 @@ class _FilesDBMaterializationOperation {
endMaxRss = _readMaxRss();
}
- FilesDBMaterializationMetrics get metrics => FilesDBMaterializationMetrics(
+ _FilesDBMaterializationMetrics get metrics => _FilesDBMaterializationMetrics(
operationID: id,
family: family,
pageSize: pageSize,
diff --git a/mobile/apps/photos/test/benchmark/files_db_materialization_benchmark_test.dart b/mobile/apps/photos/test/benchmark/files_db_materialization_benchmark_test.dart
deleted file mode 100644
index 27a432807d..0000000000
--- a/mobile/apps/photos/test/benchmark/files_db_materialization_benchmark_test.dart
+++ /dev/null
@@ -1,428 +0,0 @@
-import 'dart:convert';
-import 'dart:io';
-import 'dart:math';
-
-import 'package:computer/computer.dart';
-import 'package:flutter_test/flutter_test.dart';
-import 'package:photos/db/files_db.dart';
-import 'package:photos/models/file/file.dart';
-import 'package:photos/services/filter/db_filters.dart';
-import 'package:sqlite_async/sqlite_async.dart';
-
-const _fixtureRows = 100000;
-const _fixtureSeed = 20260825;
-const _marker = 'FILESDB_BENCHMARK_JSON ';
-const _searchFamily = 'search_all_files';
-const _pendingFamily = 'gallery_pending_or_uploaded';
-const _localFamily = 'gallery_local_and_uploaded';
-
-void main() {
- final environment = Platform.environment;
- final mode = environment['FILESDB_BENCHMARK_MODE'];
- final databasePath = environment['FILESDB_BENCHMARK_DB'];
-
- if (mode == null || databasePath == null) {
- test(
- 'FilesDB materialization benchmark worker',
- () {},
- skip: 'non-CI benchmark; run through the benchmark script',
- );
- return;
- }
-
- test(
- 'FilesDB materialization benchmark worker',
- () async {
- if (mode == 'build') {
- await _buildFixture(databasePath);
- return;
- }
- if (mode != 'run') {
- fail('Unknown FILESDB_BENCHMARK_MODE: $mode');
- }
-
- final variant = environment['FILESDB_BENCHMARK_VARIANT'] ?? 'paged';
- final runKind = environment['FILESDB_BENCHMARK_RUN_KIND'] ?? 'measured';
- final family = environment['FILESDB_BENCHMARK_FAMILY'] ?? _searchFamily;
- final pageSize = int.parse(
- environment['FILESDB_BENCHMARK_PAGE_SIZE'] ??
- FilesDB.defaultMaterializationPageSize.toString(),
- );
- await _runBenchmark(
- databasePath: databasePath,
- variant: variant,
- runKind: runKind,
- family: family,
- pageSize: pageSize,
- );
- },
- timeout: const Timeout(Duration(minutes: 15)),
- );
-}
-
-Future<void> _buildFixture(String databasePath) async {
- final file = File(databasePath);
- await file.parent.create(recursive: true);
- if (await file.exists()) {
- await file.delete();
- }
- final database = SqliteDatabase(path: databasePath);
- try {
- await FilesDB.migrateTestDatabase(database);
- final columns = FilesDB.materializedFileColumns;
- final placeholders = List.filled(columns.length, '?').join(', ');
- final insertStatement =
- 'INSERT INTO ${FilesDB.filesTable} (${columns.join(', ')}) '
- 'VALUES ($placeholders)';
- final random = Random(_fixtureSeed);
- const batchSize = 1000;
-
- for (var first = 1; first <= _fixtureRows; first += batchSize) {
- final last = min(_fixtureRows + 1, first + batchSize);
- final parameterSets = <List<Object?>>[];
- for (var id = first; id < last; id++) {
- final creationTime = 2000000000000000 - (id ~/ 4);
- final modificationTime = creationTime + random.nextInt(7);
- final values = _fixtureValues(
- id: id,
- creationTime: creationTime,
- modificationTime: modificationTime,
- );
- parameterSets.add(
- columns.map<Object?>((column) => values[column]).toList(),
- );
- }
- await database.executeBatch(insertStatement, parameterSets);
- }
-
- final countRows = await database.getAll(
- 'SELECT COUNT(*) AS count FROM ${FilesDB.filesTable}',
- );
- expect(countRows.single['count'], _fixtureRows);
- await database.execute('PRAGMA wal_checkpoint(TRUNCATE)');
- stdout.writeln(
- '$_marker${jsonEncode({'event': 'fixture', 'rows': _fixtureRows, 'seed': _fixtureSeed, 'databaseBytes': await file.length()})}',
- );
- } finally {
- await database.close();
- }
-}
-
-Future<void> _runBenchmark({
- required String databasePath,
- required String variant,
- required String runKind,
- required String family,
- required int pageSize,
-}) async {
- final database = SqliteDatabase(path: databasePath);
- await database.initialize();
- await Computer.shared().turnOn(workersCount: 1);
- try {
- if (runKind == 'warmup') {
- await _writeQueryPlan(database, family);
- }
- final startCurrentRss = ProcessInfo.currentRss;
- final startMaxRss = ProcessInfo.maxRss;
- final wallWatch = Stopwatch()..start();
-
- late final Map<String, Object?> measurement;
- if (variant == 'legacy') {
- measurement = await _runLegacy(database, family);
- } else if (variant == 'paged') {
- measurement = await _runPaged(database, pageSize, family);
- } else {
- fail('Unknown FILESDB_BENCHMARK_VARIANT: $variant');
- }
-
- wallWatch.stop();
- measurement.addAll({
- 'event': 'measurement',
- 'variant': variant,
- 'benchmarkFamily': family,
- 'runKind': runKind,
- 'seed': _fixtureSeed,
- 'fixtureRows': _fixtureRows,
- 'pageSize': variant == 'legacy' ? null : pageSize,
- 'wallMicros': wallWatch.elapsedMicroseconds,
- 'startCurrentRss': startCurrentRss,
- 'endCurrentRss': ProcessInfo.currentRss,
- 'startMaxRss': startMaxRss,
- 'endMaxRss': ProcessInfo.maxRss,
- });
- expect(measurement['rawRows'], _fixtureRows);
- expect(measurement['finalRows'], _fixtureRows);
- if (variant == 'legacy') {
- expect(measurement['maxRawPage'], _fixtureRows);
- } else {
- expect(measurement['maxRawPage'], lessThanOrEqualTo(pageSize));
- expect(measurement['maxRawPage'], lessThanOrEqualTo(5000));
- }
- stdout.writeln('$_marker${jsonEncode(measurement)}');
- } finally {
- await Computer.shared().turnOff();
- await database.close();
- }
-}
-
-Future<Map<String, Object?>> _runPaged(
- SqliteDatabase database,
- int pageSize,
- String family,
-) async {
- late FilesDBMaterializationMetrics metrics;
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: pageSize,
- materializationHooks: FilesDBMaterializationHooks(
- onComplete: (value) => metrics = value,
- ),
- );
- final options = DBFilterOptions(dedupeUploadID: false);
- final int finalRows;
- switch (family) {
- case _searchFamily:
- finalRows = (await filesDB.getAllFilesFromDB(
- const {},
- dedupeByUploadId: false,
- )).length;
- case _pendingFamily:
- finalRows = (await filesDB.getAllPendingOrUploadedFiles(
- 0,
- 3000000000000000,
- 1,
- filterOptions: options,
- )).files.length;
- case _localFamily:
- finalRows = (await filesDB.getAllLocalAndUploadedFiles(
- 0,
- 3000000000000000,
- 1,
- filterOptions: options,
- )).files.length;
- default:
- fail('Unknown FILESDB_BENCHMARK_FAMILY: $family');
- }
- expect(finalRows, _fixtureRows);
- return metrics.toJson();
-}
-
-Future<Map<String, Object?>> _runLegacy(
- SqliteDatabase database,
- String family,
-) async {
- late final String query;
- late final List<Object?> arguments;
- late final bool convertInWorker;
- switch (family) {
- case _searchFamily:
- query =
- 'SELECT * FROM ${FilesDB.filesTable} '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC';
- arguments = const [];
- convertInWorker = true;
- case _pendingFamily:
- query =
- 'SELECT * FROM ${FilesDB.filesTable} '
- 'WHERE ${FilesDB.columnCreationTime} >= ? '
- 'AND ${FilesDB.columnCreationTime} <= ? '
- 'AND (${FilesDB.columnCollectionID} IS NOT NULL '
- 'AND ${FilesDB.columnCollectionID} IS NOT -1) '
- 'AND ${FilesDB.columnMMdVisibility} = ? '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnModificationTime} DESC';
- arguments = const [0, 3000000000000000, 0];
- convertInWorker = false;
- case _localFamily:
- query =
- 'SELECT * FROM ${FilesDB.filesTable} '
- 'WHERE ${FilesDB.columnCreationTime} >= ? '
- 'AND ${FilesDB.columnCreationTime} <= ? '
- 'AND (${FilesDB.columnMMdVisibility} IS NULL '
- 'OR ${FilesDB.columnMMdVisibility} = ?) '
- 'AND (${FilesDB.columnLocalID} IS NOT NULL OR '
- '(${FilesDB.columnCollectionID} IS NOT NULL '
- 'AND ${FilesDB.columnCollectionID} IS NOT -1)) '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnModificationTime} DESC';
- arguments = const [0, 3000000000000000, 0];
- convertInWorker = false;
- default:
- fail('Unknown FILESDB_BENCHMARK_FAMILY: $family');
- }
-
- final transactionWatch = Stopwatch()..start();
- final queryWatch = Stopwatch()..start();
- late final List<Map<String, dynamic>> rows;
- try {
- rows = await database.readTransaction(
- (transaction) => transaction.getAll(query, arguments),
- );
- } finally {
- queryWatch.stop();
- transactionWatch.stop();
- }
-
- final conversionWatch = Stopwatch()..start();
- late final List<EnteFile> files;
- try {
- files = convertInWorker
- ? await Computer.shared().compute(
- _convertLegacyRows,
- param: {'result': rows},
- taskName: 'FilesDB.legacyBenchmark',
- )
- : FilesDB.instance.convertToFiles(rows);
- } finally {
- conversionWatch.stop();
- }
-
- final filterWatch = Stopwatch()..start();
- late final List<EnteFile> filteredFiles;
- try {
- filteredFiles = await applyDBFilters(
- files,
- DBFilterOptions(dedupeUploadID: false),
- );
- } finally {
- filterWatch.stop();
- }
-
- return {
- 'pageCount': 1,
- 'rawRows': rows.length,
- 'finalRows': filteredFiles.length,
- 'maxRawPage': rows.length,
- 'queryMicros': queryWatch.elapsedMicroseconds,
- 'maxQueryMicros': queryWatch.elapsedMicroseconds,
- 'conversionMicros': conversionWatch.elapsedMicroseconds,
- 'maxConversionMicros': conversionWatch.elapsedMicroseconds,
- 'filterMicros': filterWatch.elapsedMicroseconds,
- 'transactionMicros': transactionWatch.elapsedMicroseconds,
- };
-}
-
-Future<void> _writeQueryPlan(SqliteDatabase database, String family) async {
- final projection = FilesDB.materializedFileColumns.join(', ');
- late final String query;
- late final List<Object?> arguments;
- switch (family) {
- case _searchFamily:
- query =
- 'EXPLAIN QUERY PLAN SELECT $projection '
- 'FROM ${FilesDB.filesTable} '
- 'WHERE (${FilesDB.columnCreationTime}, ${FilesDB.columnGeneratedID}) '
- '< (?, ?) '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnGeneratedID} DESC LIMIT ?';
- arguments = [2000000000000000, _fixtureRows, 2000];
- case _pendingFamily:
- query =
- 'EXPLAIN QUERY PLAN SELECT $projection '
- 'FROM ${FilesDB.filesTable} '
- 'WHERE ${FilesDB.columnCreationTime} >= ? '
- 'AND ${FilesDB.columnCreationTime} <= ? '
- 'AND (${FilesDB.columnCollectionID} IS NOT NULL '
- 'AND ${FilesDB.columnCollectionID} IS NOT -1) '
- 'AND ${FilesDB.columnMMdVisibility} = ? '
- 'AND (${FilesDB.columnCreationTime}, '
- '${FilesDB.columnModificationTime}, ${FilesDB.columnGeneratedID}) '
- '< (?, ?, ?) '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnModificationTime} DESC, '
- '${FilesDB.columnGeneratedID} DESC LIMIT ?';
- arguments = [
- 0,
- 3000000000000000,
- 0,
- 2000000000000000,
- 2000000000000000,
- _fixtureRows,
- 2000,
- ];
- case _localFamily:
- query =
- 'EXPLAIN QUERY PLAN SELECT $projection '
- 'FROM ${FilesDB.filesTable} '
- 'WHERE ${FilesDB.columnCreationTime} >= ? '
- 'AND ${FilesDB.columnCreationTime} <= ? '
- 'AND (${FilesDB.columnMMdVisibility} IS NULL '
- 'OR ${FilesDB.columnMMdVisibility} = ?) '
- 'AND (${FilesDB.columnLocalID} IS NOT NULL OR '
- '(${FilesDB.columnCollectionID} IS NOT NULL '
- 'AND ${FilesDB.columnCollectionID} IS NOT -1)) '
- 'AND (${FilesDB.columnCreationTime}, '
- '${FilesDB.columnModificationTime}, ${FilesDB.columnGeneratedID}) '
- '< (?, ?, ?) '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnModificationTime} DESC, '
- '${FilesDB.columnGeneratedID} DESC LIMIT ?';
- arguments = [
- 0,
- 3000000000000000,
- 0,
- 2000000000000000,
- 2000000000000000,
- _fixtureRows,
- 2000,
- ];
- default:
- fail('Unknown FILESDB_BENCHMARK_FAMILY: $family');
- }
- final rows = await database.getAll(query, arguments);
- stdout.writeln(
- '$_marker${jsonEncode({'event': 'queryPlan', 'family': family, 'details': rows.map((row) => row['detail']).toList()})}',
- );
-}
-
-Map<String, Object?> _fixtureValues({
- required int id,
- required int creationTime,
- required int modificationTime,
-}) {
- const cryptoValue =
- '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef';
- const headerValue =
- 'header-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
- '0123456789abcdef0123456789abcdef';
- const exifValue =
- '{"Make":"Deterministic","Model":"Benchmark","payload":"'
- '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
- '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}';
- return {
- FilesDB.columnGeneratedID: id,
- FilesDB.columnLocalID: id % 10 == 0 ? 'local-$id' : null,
- FilesDB.columnUploadedFileID: id,
- FilesDB.columnOwnerID: id.isEven ? 1 : 2,
- FilesDB.columnCollectionID: 100 + (id % 997),
- FilesDB.columnTitle: 'deterministic-$id.jpg',
- FilesDB.columnDeviceFolder: id % 3 == 0 ? 'camera' : null,
- FilesDB.columnLatitude: id % 17 == 0 ? 12.25 : null,
- FilesDB.columnLongitude: id % 17 == 0 ? 77.75 : null,
- FilesDB.columnFileType: id % 8 == 0 ? 1 : 0,
- FilesDB.columnModificationTime: modificationTime,
- FilesDB.columnEncryptedKey: cryptoValue,
- FilesDB.columnKeyDecryptionNonce: cryptoValue,
- FilesDB.columnFileDecryptionHeader: headerValue,
- FilesDB.columnThumbnailDecryptionHeader: headerValue,
- FilesDB.columnMetadataDecryptionHeader: headerValue,
- FilesDB.columnCreationTime: creationTime,
- FilesDB.columnUpdationTime: creationTime + 1,
- FilesDB.columnFileSubType: 0,
- FilesDB.columnDuration: id % 8 == 0 ? 12000 : 0,
- FilesDB.columnExif: exifValue,
- FilesDB.columnHash: 'hash-${id % 99991}',
- FilesDB.columnMetadataVersion: 3,
- FilesDB.columnMMdEncodedJson: '{"visibility":0}',
- FilesDB.columnMMdVersion: 1,
- FilesDB.columnMMdVisibility: 0,
- FilesDB.columnPubMMdEncodedJson:
- '{"caption":"benchmark","w":4032,"h":3024}',
- FilesDB.columnPubMMdVersion: 1,
- FilesDB.columnFileSize: 1000000 + id,
- FilesDB.columnAddedTime: creationTime + 2,
- };
-}
-
-List<EnteFile> _convertLegacyRows(Map<dynamic, dynamic> arguments) =>
- FilesDB.instance.convertToFilesForIsolate(arguments);
diff --git a/mobile/apps/photos/test/db/files_db_materialization_test.dart b/mobile/apps/photos/test/db/files_db_materialization_test.dart
deleted file mode 100644
index c87edcb59c..0000000000
--- a/mobile/apps/photos/test/db/files_db_materialization_test.dart
+++ /dev/null
@@ -1,1123 +0,0 @@
-import 'dart:async';
-import 'dart:io';
-
-import 'package:computer/computer.dart';
-import 'package:flutter_secure_storage/flutter_secure_storage.dart';
-import 'package:flutter_test/flutter_test.dart';
-import 'package:path/path.dart' as path;
-import 'package:path_provider_platform_interface/path_provider_platform_interface.dart';
-import 'package:photos/core/configuration.dart';
-import 'package:photos/db/files_db.dart';
-import 'package:photos/models/file/file.dart';
-import 'package:photos/models/file/file_type.dart';
-import 'package:photos/models/metadata/common_keys.dart';
-import 'package:photos/services/filter/db_filters.dart';
-import 'package:shared_preferences/shared_preferences.dart';
-import 'package:sqlite_async/sqlite_async.dart';
-
-const _ownerID = 0;
-const _sharedOwnerID = 99;
-const _ignoredCollectionID = 900;
-const _useGeneratedValue = Object();
-
-void main() {
- late Directory temporaryDirectory;
- late SqliteDatabase database;
- late PathProviderPlatform previousPathProvider;
- var startedComputer = false;
-
- setUpAll(() async {
- temporaryDirectory = await Directory.systemTemp.createTemp(
- 'files-db-materialization-test-',
- );
- previousPathProvider = PathProviderPlatform.instance;
- PathProviderPlatform.instance = _FakePathProvider(temporaryDirectory.path);
- FlutterSecureStorage.setMockInitialValues({});
- SharedPreferences.setMockInitialValues({Configuration.userIDKey: _ownerID});
- await Configuration.instance.init(await SharedPreferences.getInstance());
- database = SqliteDatabase(
- path: path.join(temporaryDirectory.path, 'files.sqlite'),
- );
- await FilesDB.migrateTestDatabase(database);
- if (!Computer.shared().isRunning) {
- await Computer.shared().turnOn(workersCount: 1);
- startedComputer = true;
- }
- });
-
- setUp(() async {
- await database.execute('DELETE FROM ${FilesDB.filesTable}');
- await _insertRows(database, _fixtureRows());
- });
-
- tearDownAll(() async {
- if (startedComputer) {
- await Computer.shared().turnOff();
- }
- await database.close();
- PathProviderPlatform.instance = previousPathProvider;
- await temporaryDirectory.delete(recursive: true);
- });
-
- test(
- 'explicit projection preserves the complete EnteFile contract',
- () async {
- expect(FilesDB.materializedFileColumns, <String>[
- FilesDB.columnGeneratedID,
- FilesDB.columnLocalID,
- FilesDB.columnUploadedFileID,
- FilesDB.columnOwnerID,
- FilesDB.columnCollectionID,
- FilesDB.columnTitle,
- FilesDB.columnDeviceFolder,
- FilesDB.columnLatitude,
- FilesDB.columnLongitude,
- FilesDB.columnFileType,
- FilesDB.columnModificationTime,
- FilesDB.columnEncryptedKey,
- FilesDB.columnKeyDecryptionNonce,
- FilesDB.columnFileDecryptionHeader,
- FilesDB.columnThumbnailDecryptionHeader,
- FilesDB.columnMetadataDecryptionHeader,
- FilesDB.columnCreationTime,
- FilesDB.columnUpdationTime,
- FilesDB.columnFileSubType,
- FilesDB.columnDuration,
- FilesDB.columnExif,
- FilesDB.columnHash,
- FilesDB.columnMetadataVersion,
- FilesDB.columnMMdEncodedJson,
- FilesDB.columnMMdVersion,
- FilesDB.columnMMdVisibility,
- FilesDB.columnPubMMdEncodedJson,
- FilesDB.columnPubMMdVersion,
- FilesDB.columnFileSize,
- FilesDB.columnAddedTime,
- ]);
-
- final result =
- await FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- ).getAllLocalAndUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- final file = result.files.singleWhere((file) => file.generatedID == 16);
- final largeValue = _largeValue('sentinel');
-
- expect(file.localID, 'sentinel-local');
- expect(file.uploadedFileID, 116);
- expect(file.ownerID, _ownerID);
- expect(file.collectionID, 26);
- expect(file.title, 'sentinel-title');
- expect(file.deviceFolder, 'sentinel-folder');
- expect(file.location?.latitude, 12.25);
- expect(file.location?.longitude, 77.75);
- expect(file.fileType, FileType.video);
- expect(file.creationTime, 210);
- expect(file.modificationTime, 29);
- expect(file.updationTime, 211);
- expect(file.addedTime, 212);
- expect(file.encryptedKey, largeValue);
- expect(file.keyDecryptionNonce, 'sentinel-nonce');
- expect(file.fileDecryptionHeader, largeValue);
- expect(file.thumbnailDecryptionHeader, largeValue);
- expect(file.metadataDecryptionHeader, largeValue);
- expect(file.fileSubType, 7);
- expect(file.duration, 1234);
- expect(file.exif, largeValue);
- expect(file.hash, 'sentinel-hash');
- expect(file.metadataVersion, 3);
- expect(file.fileSize, 987654321);
- expect(file.mMdEncodedJson, '{"visibility":0}');
- expect(file.mMdVersion, 4);
- expect(
- file.pubMmdEncodedJson,
- '{"caption":"sentinel","w":8000,"h":4000}',
- );
- expect(file.pubMmdVersion, 5);
- },
- );
-
- test(
- 'Search matches one-shot total order and defined legacy order',
- () async {
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- );
- final paged = await filesDB.getAllFilesFromDB(
- const {},
- dedupeByUploadId: false,
- );
- final projection = FilesDB.materializedFileColumns.join(', ');
- final totalOrderRows = await database.getAll(
- 'SELECT $projection FROM ${FilesDB.filesTable} '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnGeneratedID} DESC',
- );
- final legacyRows = await database.getAll(
- 'SELECT $projection FROM ${FilesDB.filesTable} '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC',
- );
-
- expect(_ids(paged), _rowIDs(totalOrderRows));
- expect(
- paged.map(_fileSnapshot).toList(),
- totalOrderRows.map(_rowSnapshot).toList(),
- );
- expect(
- paged.map((file) => file.creationTime).toList(),
- legacyRows.map((row) => row[FilesDB.columnCreationTime]).toList(),
- );
- expect(_ids(paged).toSet(), _rowIDs(legacyRows).toSet());
- expect(
- _sortedRowSnapshots(legacyRows),
- _sortedRowSnapshots(totalOrderRows),
- );
- },
- );
-
- test('Search preserves dedupe, ignored collection, and tie winner', () async {
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 1,
- );
-
- final all = await filesDB.getAllFilesFromDB(
- const {},
- dedupeByUploadId: false,
- );
- final deduped = await filesDB.getAllFilesFromDB(const {});
- final ignored = await filesDB.getAllFilesFromDB(const {
- _ignoredCollectionID,
- }, dedupeByUploadId: false);
-
- expect(all, hasLength(20));
- expect(deduped, hasLength(18));
- expect(_ids(deduped), contains(10));
- expect(_ids(deduped), isNot(contains(9)));
- expect(_ids(deduped), contains(8));
- expect(_ids(deduped), isNot(contains(7)));
- expect(_ids(ignored), isNot(anyOf(contains(7), contains(8))));
- });
-
- test('pending or uploaded preserves predicates and both orders', () async {
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 2,
- );
- final options = DBFilterOptions(dedupeUploadID: false);
-
- final descending = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: options,
- );
- final ascending = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- asc: true,
- filterOptions: options,
- );
- final ownerChecked = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- applyOwnerCheck: true,
- filterOptions: options,
- );
- final hidden = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- visibility: hiddenVisibility,
- filterOptions: options,
- );
-
- expect(_ids(descending.files), [17, 16, 14, 15, 13, 10, 9, 8, 7, 6, 5, 2]);
- expect(_ids(ascending.files), _ids(descending.files).reversed.toList());
- expect(_ids(ownerChecked.files), [17, 16, 14, 15, 10, 9, 8, 7, 5, 2]);
- expect(_ids(hidden.files), [11]);
- expect(descending.hasMore, isFalse);
- });
-
- test('pending or uploaded matches a one-shot total-order query', () async {
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- );
- final result = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- final projection = FilesDB.materializedFileColumns.join(', ');
- final rows = await database.getAll(
- 'SELECT $projection FROM ${FilesDB.filesTable} '
- 'WHERE ${FilesDB.columnCreationTime} >= ? '
- 'AND ${FilesDB.columnCreationTime} <= ? '
- 'AND (${FilesDB.columnCollectionID} IS NOT NULL '
- 'AND ${FilesDB.columnCollectionID} IS NOT -1) '
- 'AND ${FilesDB.columnMMdVisibility} = ? '
- 'ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnModificationTime} DESC, '
- '${FilesDB.columnGeneratedID} DESC',
- [100, 300, visibleVisibility],
- );
-
- expect(_ids(result.files), _rowIDs(rows));
- expect(
- result.files.map(_fileSnapshot).toList(),
- rows.map(_rowSnapshot).toList(),
- );
- });
-
- test(
- 'local and uploaded preserves local, visibility, owner, and saved filters',
- () async {
- final identityFilterDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- filter: (files, _) async => files,
- );
- final descending = await identityFilterDB.getAllLocalAndUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- final ascending = await identityFilterDB.getAllLocalAndUploadedFiles(
- 100,
- 300,
- _ownerID,
- asc: true,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- final ownedOnly = await identityFilterDB.getAllLocalAndUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(
- dedupeUploadID: false,
- ignoreSharedItems: true,
- ),
- );
- final savedFiltered =
- await FilesDB.forTesting(
- database: database,
- materializationPageSize: 1,
- ).getAllLocalAndUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(
- dedupeUploadID: false,
- ignoreSavedFiles: true,
- ),
- );
-
- expect(_ids(descending.files), [
- 17,
- 20,
- 19,
- 16,
- 14,
- 15,
- 13,
- 12,
- 10,
- 9,
- 8,
- 7,
- 6,
- 5,
- 4,
- 3,
- 2,
- ]);
- expect(_ids(ascending.files), _ids(descending.files).reversed.toList());
- expect(_ids(ownedOnly.files), [
- 17,
- 20,
- 19,
- 16,
- 14,
- 15,
- 12,
- 10,
- 9,
- 8,
- 7,
- 5,
- 4,
- 3,
- 2,
- ]);
- expect(_ids(savedFiltered.files), isNot(contains(6)));
- expect(_ids(savedFiltered.files), containsAll([5, 13]));
- for (final generatedID in [3, 4]) {
- final pending = descending.files.singleWhere(
- (file) => file.generatedID == generatedID,
- );
- expect(pending.uploadedFileID, isNull);
- expect(pending.collectionID, isNull);
- expect(pending.localID, isNotNull);
- }
-
- final projection = FilesDB.materializedFileColumns.join(', ');
- final baseQuery =
- 'SELECT $projection FROM ${FilesDB.filesTable} '
- 'WHERE ${FilesDB.columnCreationTime} >= ? '
- 'AND ${FilesDB.columnCreationTime} <= ? '
- 'AND (${FilesDB.columnMMdVisibility} IS NULL '
- 'OR ${FilesDB.columnMMdVisibility} = ?) '
- 'AND (${FilesDB.columnLocalID} IS NOT NULL OR '
- '(${FilesDB.columnCollectionID} IS NOT NULL '
- 'AND ${FilesDB.columnCollectionID} IS NOT -1)) ';
- final oneShotDescending = await database.getAll(
- '$baseQuery ORDER BY ${FilesDB.columnCreationTime} DESC, '
- '${FilesDB.columnModificationTime} DESC, '
- '${FilesDB.columnGeneratedID} DESC',
- [100, 300, visibleVisibility],
- );
- final oneShotAscending = await database.getAll(
- '$baseQuery ORDER BY ${FilesDB.columnCreationTime} ASC, '
- '${FilesDB.columnModificationTime} ASC, '
- '${FilesDB.columnGeneratedID} ASC',
- [100, 300, visibleVisibility],
- );
- expect(_ids(descending.files), _rowIDs(oneShotDescending));
- expect(_ids(ascending.files), _rowIDs(oneShotAscending));
- expect(
- descending.files.map(_fileSnapshot).toList(),
- oneShotDescending.map(_rowSnapshot).toList(),
- );
- expect(
- ascending.files.map(_fileSnapshot).toList(),
- oneShotAscending.map(_rowSnapshot).toList(),
- );
- },
- );
-
- test('limited loads preserve raw-row hasMore semantics', () async {
- Future<(int, bool, FilesDBMaterializationMetrics)> load(int? limit) async {
- late FilesDBMaterializationMetrics metrics;
- final result =
- await FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- materializationHooks: FilesDBMaterializationHooks(
- onComplete: (value) => metrics = value,
- ),
- ).getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- limit: limit,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- return (result.files.length, result.hasMore, metrics);
- }
-
- final zero = await load(0);
- expect(
- (zero.$1, zero.$2, zero.$3.rawRows, zero.$3.pageCount),
- (0, true, 0, 0),
- );
-
- for (final limit in [2, 3, 6, 7]) {
- final value = await load(limit);
- expect(value.$1, limit);
- expect(value.$2, isTrue);
- expect(value.$3.rawRows, limit);
- expect(value.$3.maxRawPage, lessThanOrEqualTo(3));
- }
-
- final aboveResultSize = await load(50);
- expect((aboveResultSize.$1, aboveResultSize.$2), (12, false));
-
- final unbounded = await load(null);
- expect((unbounded.$1, unbounded.$2), (12, false));
- expect(unbounded.$3.pageCount, 5);
-
- final negative = await load(-1);
- expect((negative.$1, negative.$2), (12, false));
- expect(negative.$3.maxRawPage, lessThanOrEqualTo(3));
-
- late FilesDBMaterializationMetrics filteredMetrics;
- final filtered =
- await FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- materializationHooks: FilesDBMaterializationHooks(
- onComplete: (value) => filteredMetrics = value,
- ),
- ).getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- limit: 7,
- filterOptions: DBFilterOptions(dedupeUploadID: true),
- );
- expect(filtered.hasMore, isTrue);
- expect(filteredMetrics.rawRows, 7);
- expect(filteredMetrics.finalRows, 6);
- expect(filtered.files, hasLength(6));
- });
-
- test(
- 'queries and conversions are sequential and globally filtered once',
- () async {
- var activePageWork = 0;
- var maximumActivePageWork = 0;
- var filterCalls = 0;
- final pageLengths = <int>[];
- final searchConversionLengths = <int>[];
- late FilesDBMaterializationMetrics metrics;
-
- void startWork() {
- activePageWork++;
- maximumActivePageWork = activePageWork > maximumActivePageWork
- ? activePageWork
- : maximumActivePageWork;
- }
-
- void finishWork() {
- activePageWork--;
- }
-
- final result = await FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- filter: (files, _) async {
- filterCalls++;
- expect(files, hasLength(20));
- return files;
- },
- materializationHooks: FilesDBMaterializationHooks(
- beforePageQuery: (page, transaction) async => startWork(),
- afterPageQuery: (page, rows, transaction) async {
- pageLengths.add(rows);
- finishWork();
- },
- beforePageConversion: (page, rows) async {
- searchConversionLengths.add(rows);
- startWork();
- },
- afterPageConversion: (page, rows) async => finishWork(),
- onComplete: (value) => metrics = value,
- ),
- ).getAllFilesFromDB(const {}, dedupeByUploadId: false);
-
- expect(result, hasLength(20));
- expect(filterCalls, 1);
- expect(maximumActivePageWork, 1);
- expect(activePageWork, 0);
- expect(pageLengths.every((length) => length <= 3), isTrue);
- expect(searchConversionLengths.every((length) => length <= 3), isTrue);
- expect(
- searchConversionLengths,
- pageLengths.where((length) => length > 0).toList(),
- );
- expect(metrics.maxRawPage, 3);
- expect(metrics.maxRawPage, lessThanOrEqualTo(metrics.pageSize));
- expect(metrics.rawRows, 20);
- expect(metrics.finalRows, 20);
- expect(metrics.success, isTrue);
- expect(metrics.startCurrentRss, isNotNull);
- expect(metrics.endMaxRss, isNotNull);
-
- final galleryConversionLengths = <int>[];
- await FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- materializationHooks: FilesDBMaterializationHooks(
- beforePageConversion: (page, rows) async {
- galleryConversionLengths.add(rows);
- },
- ),
- ).getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- expect(galleryConversionLengths, isNotEmpty);
- expect(galleryConversionLengths.every((length) => length <= 3), isTrue);
- },
- );
-
- test(
- 'exact Gallery ties use low ID ascending and high ID descending',
- () async {
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 2,
- );
- final options = DBFilterOptions(dedupeUploadID: true);
-
- final descending = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: options,
- );
- final ascending = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- asc: true,
- filterOptions: options,
- );
-
- expect(_ids(descending.files), contains(10));
- expect(_ids(descending.files), isNot(contains(9)));
- expect(_ids(ascending.files), contains(9));
- expect(_ids(ascending.files), isNot(contains(10)));
- },
- );
-
- test(
- 'page query failure closes the transaction and a retry succeeds',
- () async {
- var shouldFail = true;
- var filterCalls = 0;
- SqliteReadContext? failedTransaction;
- final metrics = <FilesDBMaterializationMetrics>[];
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 2,
- filter: (files, _) async {
- filterCalls++;
- return files;
- },
- materializationHooks: FilesDBMaterializationHooks(
- beforePageQuery: (page, transaction) async {
- if (shouldFail && page == 2) {
- failedTransaction = transaction;
- shouldFail = false;
- await transaction.getAll(
- 'SELECT missing_test_column FROM ${FilesDB.filesTable}',
- );
- }
- },
- onComplete: metrics.add,
- ),
- );
-
- await expectLater(
- filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- ),
- throwsA(isA<Exception>()),
- );
- expect(filterCalls, 0);
- expect(failedTransaction?.closed, isTrue);
- expect(metrics.single.success, isFalse);
- expect(metrics.single.errorType, 'SqliteException');
-
- final retry = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- expect(retry.files, hasLength(12));
- expect(filterCalls, 1);
- expect(metrics.last.success, isTrue);
- },
- );
-
- test(
- 'page conversion failure returns no partial result and permits retry',
- () async {
- var conversionPage = 0;
- var shouldFail = true;
- var filterCalls = 0;
- SqliteReadContext? failedTransaction;
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 2,
- galleryPageConverter: (rows) async {
- conversionPage++;
- if (shouldFail && conversionPage == 2) {
- shouldFail = false;
- throw StateError('injected page conversion failure');
- }
- return _convertRowsForTest(rows);
- },
- filter: (files, _) async {
- filterCalls++;
- return files;
- },
- materializationHooks: FilesDBMaterializationHooks(
- beforePageQuery: (page, transaction) async {
- failedTransaction = transaction;
- },
- ),
- );
-
- await expectLater(
- filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- ),
- throwsStateError,
- );
- expect(filterCalls, 0);
- expect(failedTransaction?.closed, isTrue);
-
- final retry = await filesDB.getAllPendingOrUploadedFiles(
- 100,
- 300,
- _ownerID,
- filterOptions: DBFilterOptions(dedupeUploadID: false),
- );
- expect(retry.files, hasLength(12));
- expect(filterCalls, 1);
- },
- );
-
- test(
- 'final filter failure cannot escape partial data and permits retry',
- () async {
- var shouldFail = true;
- var filterInputs = 0;
- SqliteReadContext? finalTransaction;
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 3,
- filter: (files, _) async {
- filterInputs = files.length;
- if (shouldFail) {
- shouldFail = false;
- throw StateError('injected final filter failure');
- }
- return files;
- },
- materializationHooks: FilesDBMaterializationHooks(
- beforePageQuery: (page, transaction) async {
- finalTransaction = transaction;
- },
- ),
- );
-
- await expectLater(
- filesDB.getAllFilesFromDB(const {}, dedupeByUploadId: false),
- throwsStateError,
- );
- expect(filterInputs, 20);
- expect(finalTransaction?.closed, isTrue);
-
- final retry = await filesDB.getAllFilesFromDB(
- const {},
- dedupeByUploadId: false,
- );
- expect(retry, hasLength(20));
- },
- );
-
- test(
- 'all pages share one snapshot while a concurrent writer proceeds',
- () async {
- final firstPageRead = Completer<void>();
- final releaseReader = Completer<void>();
- final filesDB = FilesDB.forTesting(
- database: database,
- materializationPageSize: 2,
- materializationHooks: FilesDBMaterializationHooks(
- afterPageQuery: (page, rows, transaction) async {
- if (page == 1 && !firstPageRead.isCompleted) {
- firstPageRead.complete();
- await releaseReader.future;
- }
- },
- ),
- );
-
- final activeRead = filesDB.getAllFilesFromDB(
- const {},
- dedupeByUploadId: false,
- );
- await firstPageRead.future;
- try {
- await _insertRows(database, [
- _row(
- id: 100,
- creationTime: 250,
- modificationTime: 50,
- uploadedFileID: 9100,
- collectionID: 910,
- ),
- ]);
- } finally {
- releaseReader.complete();
- }
-
- final snapshot = await activeRead;
- expect(snapshot, hasLength(20));
- expect(_ids(snapshot), isNot(contains(100)));
-
- final nextRead = await filesDB.getAllFilesFromDB(
- const {},
- dedupeByUploadId: false,
- );
- expect(nextRead, hasLength(21));
- expect(_ids(nextRead), contains(100));
- },
- );
-}
-
-List<Map<String, Object?>> _fixtureRows() => [
- _row(id: 1, creationTime: 99, modificationTime: 1),
- _row(id: 2, creationTime: 100, modificationTime: 10),
- _row(
- id: 3,
- creationTime: 110,
- modificationTime: 11,
- uploadedFileID: null,
- collectionID: null,
- localID: 'pending-null',
- ),
- _row(
- id: 4,
- creationTime: 120,
- modificationTime: 12,
- uploadedFileID: -1,
- collectionID: -1,
- localID: 'pending-legacy',
- ),
- _row(
- id: 5,
- creationTime: 130,
- modificationTime: 13,
- uploadedFileID: 105,
- collectionID: 15,
- ownerID: _ownerID,
- hash: 'owned-shared-hash',
- ),
- _row(
- id: 6,
- creationTime: 130,
- modificationTime: 13,
- uploadedFileID: 106,
- collectionID: 16,
- ownerID: _sharedOwnerID,
- hash: 'owned-shared-hash',
- ),
- _row(
- id: 7,
- creationTime: 140,
- modificationTime: 14,
- uploadedFileID: 600,
- collectionID: 100,
- ),
- _row(
- id: 8,
- creationTime: 150,
- modificationTime: 15,
- uploadedFileID: 600,
- collectionID: _ignoredCollectionID,
- ),
- _row(
- id: 9,
- creationTime: 160,
- modificationTime: 16,
- uploadedFileID: 500,
- collectionID: 201,
- ),
- _row(
- id: 10,
- creationTime: 160,
- modificationTime: 16,
- uploadedFileID: 500,
- collectionID: 202,
- ),
- _row(
- id: 11,
- creationTime: 170,
- modificationTime: 20,
- visibility: hiddenVisibility,
- ),
- _row(id: 12, creationTime: 180, modificationTime: 21, visibility: null),
- _row(
- id: 13,
- creationTime: 190,
- modificationTime: 22,
- ownerID: _sharedOwnerID,
- ),
- _row(id: 14, creationTime: 200, modificationTime: 31, ownerID: null),
- _row(id: 15, creationTime: 200, modificationTime: 30),
- _row(
- id: 16,
- creationTime: 210,
- modificationTime: 29,
- uploadedFileID: 116,
- collectionID: 26,
- localID: 'sentinel-local',
- title: 'sentinel-title',
- deviceFolder: 'sentinel-folder',
- latitude: 12.25,
- longitude: 77.75,
- fileType: 1,
- updationTime: 211,
- addedTime: 212,
- encryptedKey: _largeValue('sentinel'),
- keyDecryptionNonce: 'sentinel-nonce',
- fileDecryptionHeader: _largeValue('sentinel'),
- thumbnailDecryptionHeader: _largeValue('sentinel'),
- metadataDecryptionHeader: _largeValue('sentinel'),
- fileSubType: 7,
- duration: 1234,
- exif: _largeValue('sentinel'),
- hash: 'sentinel-hash',
- metadataVersion: 3,
- mMdVersion: 4,
- pubMmdEncodedJson: '{"caption":"sentinel","w":8000,"h":4000}',
- pubMmdVersion: 5,
- fileSize: 987654321,
- ),
- _row(id: 17, creationTime: 300, modificationTime: 40),
- _row(id: 18, creationTime: 301, modificationTime: 41),
- _row(
- id: 19,
- creationTime: 220,
- modificationTime: 31,
- uploadedFileID: null,
- collectionID: null,
- localID: 'pending-null-late',
- ),
- _row(
- id: 20,
- creationTime: 230,
- modificationTime: 32,
- uploadedFileID: -1,
- collectionID: -1,
- localID: 'pending-legacy-late',
- ),
-];
-
-Map<String, Object?> _row({
- required int id,
- required int creationTime,
- required int modificationTime,
- Object? uploadedFileID = _useGeneratedValue,
- Object? collectionID = _useGeneratedValue,
- Object? ownerID = _ownerID,
- String? localID,
- String? title,
- String? deviceFolder,
- double? latitude,
- double? longitude,
- int fileType = 0,
- int? updationTime,
- int? addedTime,
- String? encryptedKey,
- String? keyDecryptionNonce,
- String? fileDecryptionHeader,
- String? thumbnailDecryptionHeader,
- String? metadataDecryptionHeader,
- int? fileSubType,
- int? duration,
- String? exif,
- String? hash,
- int? metadataVersion,
- String? mMdEncodedJson,
- int? mMdVersion,
- Object? visibility = visibleVisibility,
- String? pubMmdEncodedJson,
- int? pubMmdVersion,
- int? fileSize,
-}) {
- final resolvedUploadID = identical(uploadedFileID, _useGeneratedValue)
- ? 1000 + id
- : uploadedFileID;
- final resolvedCollectionID = identical(collectionID, _useGeneratedValue)
- ? 100 + id
- : collectionID;
- return {
- FilesDB.columnGeneratedID: id,
- FilesDB.columnLocalID: localID,
- FilesDB.columnUploadedFileID: resolvedUploadID,
- FilesDB.columnOwnerID: ownerID,
- FilesDB.columnCollectionID: resolvedCollectionID,
- FilesDB.columnTitle: title ?? 'fixture-$id',
- FilesDB.columnDeviceFolder: deviceFolder,
- FilesDB.columnLatitude: latitude,
- FilesDB.columnLongitude: longitude,
- FilesDB.columnFileType: fileType,
- FilesDB.columnModificationTime: modificationTime,
- FilesDB.columnEncryptedKey: encryptedKey,
- FilesDB.columnKeyDecryptionNonce: keyDecryptionNonce,
- FilesDB.columnFileDecryptionHeader: fileDecryptionHeader,
- FilesDB.columnThumbnailDecryptionHeader: thumbnailDecryptionHeader,
- FilesDB.columnMetadataDecryptionHeader: metadataDecryptionHeader,
- FilesDB.columnCreationTime: creationTime,
- FilesDB.columnUpdationTime: updationTime,
- FilesDB.columnFileSubType: fileSubType,
- FilesDB.columnDuration: duration,
- FilesDB.columnExif: exif,
- FilesDB.columnHash: hash ?? 'hash-$id',
- FilesDB.columnMetadataVersion: metadataVersion,
- FilesDB.columnMMdEncodedJson:
- mMdEncodedJson ?? '{"visibility":${visibility ?? visibleVisibility}}',
- FilesDB.columnMMdVersion: mMdVersion,
- FilesDB.columnMMdVisibility: visibility,
- FilesDB.columnPubMMdEncodedJson: pubMmdEncodedJson ?? '{}',
- FilesDB.columnPubMMdVersion: pubMmdVersion,
- FilesDB.columnFileSize: fileSize,
- FilesDB.columnAddedTime: addedTime ?? -1,
- };
-}
-
-Future<void> _insertRows(
- SqliteDatabase database,
- List<Map<String, Object?>> rows,
-) async {
- final columns = FilesDB.materializedFileColumns;
- final placeholders = List.filled(columns.length, '?').join(', ');
- await database.executeBatch(
- 'INSERT INTO ${FilesDB.filesTable} (${columns.join(', ')}) '
- 'VALUES ($placeholders)',
- rows
- .map((row) => columns.map<Object?>((column) => row[column]).toList())
- .toList(),
- );
-}
-
-List<EnteFile> _convertRowsForTest(List<Map<String, dynamic>> rows) =>
- rows.map((row) {
- final file = EnteFile();
- file.generatedID = row[FilesDB.columnGeneratedID];
- file.localID = row[FilesDB.columnLocalID];
- final uploadedID = row[FilesDB.columnUploadedFileID];
- file.uploadedFileID = uploadedID == -1 ? null : uploadedID;
- file.ownerID = row[FilesDB.columnOwnerID];
- final collectionID = row[FilesDB.columnCollectionID];
- file.collectionID = collectionID == -1 ? null : collectionID;
- file.title = row[FilesDB.columnTitle];
- file.fileType = getFileType(row[FilesDB.columnFileType]);
- file.creationTime = row[FilesDB.columnCreationTime];
- file.modificationTime = row[FilesDB.columnModificationTime];
- file.hash = row[FilesDB.columnHash];
- file.mMdEncodedJson = row[FilesDB.columnMMdEncodedJson];
- file.pubMmdEncodedJson = row[FilesDB.columnPubMMdEncodedJson];
- return file;
- }).toList();
-
-List<int> _ids(Iterable<EnteFile> files) =>
- files.map((file) => file.generatedID!).toList();
-
-List<int> _rowIDs(Iterable<Map<String, dynamic>> rows) =>
- rows.map((row) => row[FilesDB.columnGeneratedID] as int).toList();
-
-Map<String, Object?> _fileSnapshot(EnteFile file) => {
- 'generatedID': file.generatedID,
- 'localID': file.localID,
- 'uploadedFileID': file.uploadedFileID,
- 'ownerID': file.ownerID,
- 'collectionID': file.collectionID,
- 'title': file.title,
- 'deviceFolder': file.deviceFolder,
- 'latitude': file.location?.latitude,
- 'longitude': file.location?.longitude,
- 'fileType': file.fileType.name,
- 'creationTime': file.creationTime,
- 'modificationTime': file.modificationTime,
- 'updationTime': file.updationTime,
- 'addedTime': file.addedTime,
- 'encryptedKey': file.encryptedKey,
- 'keyDecryptionNonce': file.keyDecryptionNonce,
- 'fileDecryptionHeader': file.fileDecryptionHeader,
- 'thumbnailDecryptionHeader': file.thumbnailDecryptionHeader,
- 'metadataDecryptionHeader': file.metadataDecryptionHeader,
- 'fileSubType': file.fileSubType,
- 'duration': file.duration,
- 'exif': file.exif,
- 'hash': file.hash,
- 'metadataVersion': file.metadataVersion,
- 'fileSize': file.fileSize,
- 'mMdEncodedJson': file.mMdEncodedJson,
- 'mMdVersion': file.mMdVersion,
- 'pubMmdEncodedJson': file.pubMmdEncodedJson,
- 'pubMmdVersion': file.pubMmdVersion,
-};
-
-Map<String, Object?> _rowSnapshot(Map<String, dynamic> row) {
- final uploadedID = row[FilesDB.columnUploadedFileID];
- final collectionID = row[FilesDB.columnCollectionID];
- final hasLocation =
- row[FilesDB.columnLatitude] != null &&
- row[FilesDB.columnLongitude] != null;
- return {
- 'generatedID': row[FilesDB.columnGeneratedID],
- 'localID': row[FilesDB.columnLocalID],
- 'uploadedFileID': uploadedID == -1 ? null : uploadedID,
- 'ownerID': row[FilesDB.columnOwnerID],
- 'collectionID': collectionID == -1 ? null : collectionID,
- 'title': row[FilesDB.columnTitle],
- 'deviceFolder': row[FilesDB.columnDeviceFolder],
- 'latitude': hasLocation ? row[FilesDB.columnLatitude] : null,
- 'longitude': hasLocation ? row[FilesDB.columnLongitude] : null,
- 'fileType': getFileType(row[FilesDB.columnFileType]).name,
- 'creationTime': row[FilesDB.columnCreationTime],
- 'modificationTime': row[FilesDB.columnModificationTime],
- 'updationTime': row[FilesDB.columnUpdationTime] ?? -1,
- 'addedTime': row[FilesDB.columnAddedTime],
- 'encryptedKey': row[FilesDB.columnEncryptedKey],
- 'keyDecryptionNonce': row[FilesDB.columnKeyDecryptionNonce],
- 'fileDecryptionHeader': row[FilesDB.columnFileDecryptionHeader],
- 'thumbnailDecryptionHeader': row[FilesDB.columnThumbnailDecryptionHeader],
- 'metadataDecryptionHeader': row[FilesDB.columnMetadataDecryptionHeader],
- 'fileSubType': row[FilesDB.columnFileSubType] ?? -1,
- 'duration': row[FilesDB.columnDuration] ?? 0,
- 'exif': row[FilesDB.columnExif],
- 'hash': row[FilesDB.columnHash],
- 'metadataVersion': row[FilesDB.columnMetadataVersion] ?? 0,
- 'fileSize': row[FilesDB.columnFileSize],
- 'mMdEncodedJson': row[FilesDB.columnMMdEncodedJson] ?? '{}',
- 'mMdVersion': row[FilesDB.columnMMdVersion] ?? 0,
- 'pubMmdEncodedJson': row[FilesDB.columnPubMMdEncodedJson] ?? '{}',
- 'pubMmdVersion': row[FilesDB.columnPubMMdVersion] ?? 0,
- };
-}
-
-List<Map<String, Object?>> _sortedRowSnapshots(
- Iterable<Map<String, dynamic>> rows,
-) {
- final snapshots = rows.map(_rowSnapshot).toList();
- snapshots.sort(
- (left, right) =>
- (left['generatedID'] as int).compareTo(right['generatedID'] as int),
- );
- return snapshots;
-}
-
-String _largeValue(String prefix) => '$prefix-${List.filled(4096, 'x').join()}';
-
-class _FakePathProvider extends PathProviderPlatform {
- final String rootPath;
-
- _FakePathProvider(this.rootPath);
-
- @override
- Future<String?> getApplicationDocumentsPath() async => rootPath;
-
- @override
- Future<String?> getApplicationSupportPath() async => rootPath;
-
- @override
- Future<String?> getTemporaryPath() async => rootPath;
-}