Skip to content

High performance self-hosted photo and video management solution.

refactor(mobile): device asset entity to use modified time (#17064)

shenlong97e52c5156c4
diff --git a/mobile/analysis_options.yaml b/mobile/analysis_options.yaml
index 07c6f65b71..04f3145908 100644
--- a/mobile/analysis_options.yaml
+++ b/mobile/analysis_options.yaml
@@ -61,6 +61,7 @@ custom_lint:
# refactor to make the providers and services testable
- lib/providers/backup/{backup,manual_upload}.provider.dart # uses only PMProgressHandler
- lib/services/{background,backup}.service.dart # uses only PMProgressHandler
+ - test/**.dart
- import_rule_isar:
message: isar must only be used in entities and repositories
restrict: package:isar
@@ -150,7 +151,6 @@ dart_code_metrics:
- avoid-unnecessary-continue
- avoid-unnecessary-nullable-return-type: false
- binary-expression-operand-order
- - move-variable-outside-iteration
- pattern-fields-ordering
- prefer-abstract-final-static-class
- prefer-commenting-future-delayed
diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart
index 868b036d1b..83d540d54c 100644
--- a/mobile/lib/constants/constants.dart
+++ b/mobile/lib/constants/constants.dart
@@ -4,3 +4,6 @@ const double downloadFailed = -2;
// Number of log entries to retain on app start
const int kLogTruncateLimit = 250;
+
+const int kBatchHashFileLimit = 128;
+const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB
diff --git a/mobile/lib/domain/interfaces/device_asset.interface.dart b/mobile/lib/domain/interfaces/device_asset.interface.dart
new file mode 100644
index 0000000000..1df8cc2250
--- /dev/null
+++ b/mobile/lib/domain/interfaces/device_asset.interface.dart
@@ -0,0 +1,12 @@
+import 'dart:async';
+
+import 'package:immich_mobile/domain/interfaces/db.interface.dart';
+import 'package:immich_mobile/domain/models/device_asset.model.dart';
+
+abstract interface class IDeviceAssetRepository implements IDatabaseRepository {
+ Future<bool> updateAll(List<DeviceAsset> assetHash);
+
+ Future<List<DeviceAsset>> getByIds(List<String> localIds);
+
+ Future<void> deleteIds(List<String> ids);
+}
diff --git a/mobile/lib/domain/models/device_asset.model.dart b/mobile/lib/domain/models/device_asset.model.dart
new file mode 100644
index 0000000000..2ec56b0d80
--- /dev/null
+++ b/mobile/lib/domain/models/device_asset.model.dart
@@ -0,0 +1,44 @@
+import 'dart:typed_data';
+
+class DeviceAsset {
+ final String assetId;
+ final Uint8List hash;
+ final DateTime modifiedTime;
+
+ const DeviceAsset({
+ required this.assetId,
+ required this.hash,
+ required this.modifiedTime,
+ });
+
+ @override
+ bool operator ==(covariant DeviceAsset other) {
+ if (identical(this, other)) return true;
+
+ return other.assetId == assetId &&
+ other.hash == hash &&
+ other.modifiedTime == modifiedTime;
+ }
+
+ @override
+ int get hashCode {
+ return assetId.hashCode ^ hash.hashCode ^ modifiedTime.hashCode;
+ }
+
+ @override
+ String toString() {
+ return 'DeviceAsset(assetId: $assetId, hash: $hash, modifiedTime: $modifiedTime)';
+ }
+
+ DeviceAsset copyWith({
+ String? assetId,
+ Uint8List? hash,
+ DateTime? modifiedTime,
+ }) {
+ return DeviceAsset(
+ assetId: assetId ?? this.assetId,
+ hash: hash ?? this.hash,
+ modifiedTime: modifiedTime ?? this.modifiedTime,
+ );
+ }
+}
diff --git a/mobile/lib/entities/asset.entity.dart b/mobile/lib/entities/asset.entity.dart
index 048068ad3d..084cd1ee5d 100644
--- a/mobile/lib/entities/asset.entity.dart
+++ b/mobile/lib/entities/asset.entity.dart
@@ -6,6 +6,7 @@ import 'package:immich_mobile/extensions/string_extensions.dart';
import 'package:immich_mobile/infrastructure/entities/exif.entity.dart'
as entity;
import 'package:immich_mobile/infrastructure/utils/exif.converter.dart';
+import 'package:immich_mobile/utils/diff.dart';
import 'package:immich_mobile/utils/hash.dart';
import 'package:isar/isar.dart';
import 'package:openapi/api.dart';
@@ -358,7 +359,7 @@ class Asset {
// take most values from newer asset
// keep vales that can never be set by the asset not in DB
if (a.isRemote) {
- return a._copyWith(
+ return a.copyWith(
id: id,
localId: localId,
width: a.width ?? width,
@@ -366,7 +367,7 @@ class Asset {
exifInfo: a.exifInfo?.copyWith(assetId: id) ?? exifInfo,
);
} else if (isRemote) {
- return _copyWith(
+ return copyWith(
localId: localId ?? a.localId,
width: width ?? a.width,
height: height ?? a.height,
@@ -374,7 +375,7 @@ class Asset {
);
} else {
// TODO: Revisit this and remove all bool field assignments
- return a._copyWith(
+ return a.copyWith(
id: id,
remoteId: remoteId,
livePhotoVideoId: livePhotoVideoId,
@@ -394,7 +395,7 @@ class Asset {
// fill in potentially missing values, i.e. merge assets
if (a.isRemote) {
// values from remote take precedence
- return _copyWith(
+ return copyWith(
remoteId: a.remoteId,
width: a.width,
height: a.height,
@@ -416,7 +417,7 @@ class Asset {
);
} else {
// add only missing values (and set isLocal to true)
- return _copyWith(
+ return copyWith(
localId: localId ?? a.localId,
width: width ?? a.width,
height: height ?? a.height,
@@ -427,7 +428,7 @@ class Asset {
}
}
- Asset _copyWith({
+ Asset copyWith({
Id? id,
String? checksum,
String? remoteId,
@@ -488,6 +489,9 @@ class Asset {
static int compareById(Asset a, Asset b) => a.id.compareTo(b.id);
+ static int compareByLocalId(Asset a, Asset b) =>
+ compareToNullable(a.localId, b.localId);
+
static int compareByChecksum(Asset a, Asset b) =>
a.checksum.compareTo(b.checksum);
diff --git a/mobile/lib/infrastructure/entities/device_asset.entity.dart b/mobile/lib/infrastructure/entities/device_asset.entity.dart
new file mode 100644
index 0000000000..d8bfb2aa45
--- /dev/null
+++ b/mobile/lib/infrastructure/entities/device_asset.entity.dart
@@ -0,0 +1,36 @@
+import 'dart:typed_data';
+
+import 'package:immich_mobile/domain/models/device_asset.model.dart';
+import 'package:immich_mobile/utils/hash.dart';
+import 'package:isar/isar.dart';
+
+part 'device_asset.entity.g.dart';
+
+@Collection(inheritance: false)
+class DeviceAssetEntity {
+ Id get id => fastHash(assetId);
+
+ @Index(replace: true, unique: true, type: IndexType.hash)
+ final String assetId;
+ @Index(unique: false, type: IndexType.hash)
+ final List<byte> hash;
+ final DateTime modifiedTime;
+
+ const DeviceAssetEntity({
+ required this.assetId,
+ required this.hash,
+ required this.modifiedTime,
+ });
+
+ DeviceAsset toModel() => DeviceAsset(
+ assetId: assetId,
+ hash: Uint8List.fromList(hash),
+ modifiedTime: modifiedTime,
+ );
+
+ static DeviceAssetEntity fromDto(DeviceAsset dto) => DeviceAssetEntity(
+ assetId: dto.assetId,
+ hash: dto.hash,
+ modifiedTime: dto.modifiedTime,
+ );
+}
diff --git a/mobile/lib/infrastructure/entities/device_asset.entity.g.dart b/mobile/lib/infrastructure/entities/device_asset.entity.g.dart
new file mode 100644
index 0000000000..a66f8288ef
--- /dev/null
+++ b/mobile/lib/infrastructure/entities/device_asset.entity.g.dart
@@ -0,0 +1,895 @@
+// GENERATED CODE - DO NOT MODIFY BY HAND
+
+part of 'device_asset.entity.dart';
+
+// **************************************************************************
+// IsarCollectionGenerator
+// **************************************************************************
+
+// coverage:ignore-file
+// ignore_for_file: duplicate_ignore, non_constant_identifier_names, constant_identifier_names, invalid_use_of_protected_member, unnecessary_cast, prefer_const_constructors, lines_longer_than_80_chars, require_trailing_commas, inference_failure_on_function_invocation, unnecessary_parenthesis, unnecessary_raw_strings, unnecessary_null_checks, join_return_with_assignment, prefer_final_locals, avoid_js_rounded_ints, avoid_positional_boolean_parameters, always_specify_types
+
+extension GetDeviceAssetEntityCollection on Isar {
+ IsarCollection<DeviceAssetEntity> get deviceAssetEntitys => this.collection();
+}
+
+const DeviceAssetEntitySchema = CollectionSchema(
+ name: r'DeviceAssetEntity',
+ id: 6967030785073446271,
+ properties: {
+ r'assetId': PropertySchema(
+ id: 0,
+ name: r'assetId',
+ type: IsarType.string,
+ ),
+ r'hash': PropertySchema(
+ id: 1,
+ name: r'hash',
+ type: IsarType.byteList,
+ ),
+ r'modifiedTime': PropertySchema(
+ id: 2,
+ name: r'modifiedTime',
+ type: IsarType.dateTime,
+ )
+ },
+ estimateSize: _deviceAssetEntityEstimateSize,
+ serialize: _deviceAssetEntitySerialize,
+ deserialize: _deviceAssetEntityDeserialize,
+ deserializeProp: _deviceAssetEntityDeserializeProp,
+ idName: r'id',
+ indexes: {
+ r'assetId': IndexSchema(
+ id: 174362542210192109,
+ name: r'assetId',
+ unique: true,
+ replace: true,
+ properties: [
+ IndexPropertySchema(
+ name: r'assetId',
+ type: IndexType.hash,
+ caseSensitive: true,
+ )
+ ],
+ ),
+ r'hash': IndexSchema(
+ id: -7973251393006690288,
+ name: r'hash',
+ unique: false,
+ replace: false,
+ properties: [
+ IndexPropertySchema(
+ name: r'hash',
+ type: IndexType.hash,
+ caseSensitive: false,
+ )
+ ],
+ )
+ },
+ links: {},
+ embeddedSchemas: {},
+ getId: _deviceAssetEntityGetId,
+ getLinks: _deviceAssetEntityGetLinks,
+ attach: _deviceAssetEntityAttach,
+ version: '3.1.8',
+);
+
+int _deviceAssetEntityEstimateSize(
+ DeviceAssetEntity object,
+ List<int> offsets,
+ Map<Type, List<int>> allOffsets,
+) {
+ var bytesCount = offsets.last;
+ bytesCount += 3 + object.assetId.length * 3;
+ bytesCount += 3 + object.hash.length;
+ return bytesCount;
+}
+
+void _deviceAssetEntitySerialize(
+ DeviceAssetEntity object,
+ IsarWriter writer,
+ List<int> offsets,
+ Map<Type, List<int>> allOffsets,
+) {
+ writer.writeString(offsets[0], object.assetId);
+ writer.writeByteList(offsets[1], object.hash);
+ writer.writeDateTime(offsets[2], object.modifiedTime);
+}
+
+DeviceAssetEntity _deviceAssetEntityDeserialize(
+ Id id,
+ IsarReader reader,
+ List<int> offsets,
+ Map<Type, List<int>> allOffsets,
+) {
+ final object = DeviceAssetEntity(
+ assetId: reader.readString(offsets[0]),
+ hash: reader.readByteList(offsets[1]) ?? [],
+ modifiedTime: reader.readDateTime(offsets[2]),
+ );
+ return object;
+}
+
+P _deviceAssetEntityDeserializeProp<P>(
+ IsarReader reader,
+ int propertyId,
+ int offset,
+ Map<Type, List<int>> allOffsets,
+) {
+ switch (propertyId) {
+ case 0:
+ return (reader.readString(offset)) as P;
+ case 1:
+ return (reader.readByteList(offset) ?? []) as P;
+ case 2:
+ return (reader.readDateTime(offset)) as P;
+ default:
+ throw IsarError('Unknown property with id $propertyId');
+ }
+}
+
+Id _deviceAssetEntityGetId(DeviceAssetEntity object) {
+ return object.id;
+}
+
+List<IsarLinkBase<dynamic>> _deviceAssetEntityGetLinks(
+ DeviceAssetEntity object) {
+ return [];
+}
+
+void _deviceAssetEntityAttach(
+ IsarCollection<dynamic> col, Id id, DeviceAssetEntity object) {}
+
+extension DeviceAssetEntityByIndex on IsarCollection<DeviceAssetEntity> {
+ Future<DeviceAssetEntity?> getByAssetId(String assetId) {
+ return getByIndex(r'assetId', [assetId]);
+ }
+
+ DeviceAssetEntity? getByAssetIdSync(String assetId) {
+ return getByIndexSync(r'assetId', [assetId]);
+ }
+
+ Future<bool> deleteByAssetId(String assetId) {
+ return deleteByIndex(r'assetId', [assetId]);
+ }
+
+ bool deleteByAssetIdSync(String assetId) {
+ return deleteByIndexSync(r'assetId', [assetId]);
+ }
+
+ Future<List<DeviceAssetEntity?>> getAllByAssetId(List<String> assetIdValues) {
+ final values = assetIdValues.map((e) => [e]).toList();
+ return getAllByIndex(r'assetId', values);
+ }
+
+ List<DeviceAssetEntity?> getAllByAssetIdSync(List<String> assetIdValues) {
+ final values = assetIdValues.map((e) => [e]).toList();
+ return getAllByIndexSync(r'assetId', values);
+ }
+
+ Future<int> deleteAllByAssetId(List<String> assetIdValues) {
+ final values = assetIdValues.map((e) => [e]).toList();
+ return deleteAllByIndex(r'assetId', values);
+ }
+
+ int deleteAllByAssetIdSync(List<String> assetIdValues) {
+ final values = assetIdValues.map((e) => [e]).toList();
+ return deleteAllByIndexSync(r'assetId', values);
+ }
+
+ Future<Id> putByAssetId(DeviceAssetEntity object) {
+ return putByIndex(r'assetId', object);
+ }
+
+ Id putByAssetIdSync(DeviceAssetEntity object, {bool saveLinks = true}) {
+ return putByIndexSync(r'assetId', object, saveLinks: saveLinks);
+ }
+
+ Future<List<Id>> putAllByAssetId(List<DeviceAssetEntity> objects) {
+ return putAllByIndex(r'assetId', objects);
+ }
+
+ List<Id> putAllByAssetIdSync(List<DeviceAssetEntity> objects,
+ {bool saveLinks = true}) {
+ return putAllByIndexSync(r'assetId', objects, saveLinks: saveLinks);
+ }
+}
+
+extension DeviceAssetEntityQueryWhereSort
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QWhere> {
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhere> anyId() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addWhereClause(const IdWhereClause.any());
+ });
+ }
+}
+
+extension DeviceAssetEntityQueryWhere
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QWhereClause> {
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ idEqualTo(Id id) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addWhereClause(IdWhereClause.between(
+ lower: id,
+ upper: id,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ idNotEqualTo(Id id) {
+ return QueryBuilder.apply(this, (query) {
+ if (query.whereSort == Sort.asc) {
+ return query
+ .addWhereClause(
+ IdWhereClause.lessThan(upper: id, includeUpper: false),
+ )
+ .addWhereClause(
+ IdWhereClause.greaterThan(lower: id, includeLower: false),
+ );
+ } else {
+ return query
+ .addWhereClause(
+ IdWhereClause.greaterThan(lower: id, includeLower: false),
+ )
+ .addWhereClause(
+ IdWhereClause.lessThan(upper: id, includeUpper: false),
+ );
+ }
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ idGreaterThan(Id id, {bool include = false}) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addWhereClause(
+ IdWhereClause.greaterThan(lower: id, includeLower: include),
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ idLessThan(Id id, {bool include = false}) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addWhereClause(
+ IdWhereClause.lessThan(upper: id, includeUpper: include),
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ idBetween(
+ Id lowerId,
+ Id upperId, {
+ bool includeLower = true,
+ bool includeUpper = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addWhereClause(IdWhereClause.between(
+ lower: lowerId,
+ includeLower: includeLower,
+ upper: upperId,
+ includeUpper: includeUpper,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ assetIdEqualTo(String assetId) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addWhereClause(IndexWhereClause.equalTo(
+ indexName: r'assetId',
+ value: [assetId],
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ assetIdNotEqualTo(String assetId) {
+ return QueryBuilder.apply(this, (query) {
+ if (query.whereSort == Sort.asc) {
+ return query
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'assetId',
+ lower: [],
+ upper: [assetId],
+ includeUpper: false,
+ ))
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'assetId',
+ lower: [assetId],
+ includeLower: false,
+ upper: [],
+ ));
+ } else {
+ return query
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'assetId',
+ lower: [assetId],
+ includeLower: false,
+ upper: [],
+ ))
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'assetId',
+ lower: [],
+ upper: [assetId],
+ includeUpper: false,
+ ));
+ }
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ hashEqualTo(List<int> hash) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addWhereClause(IndexWhereClause.equalTo(
+ indexName: r'hash',
+ value: [hash],
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterWhereClause>
+ hashNotEqualTo(List<int> hash) {
+ return QueryBuilder.apply(this, (query) {
+ if (query.whereSort == Sort.asc) {
+ return query
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'hash',
+ lower: [],
+ upper: [hash],
+ includeUpper: false,
+ ))
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'hash',
+ lower: [hash],
+ includeLower: false,
+ upper: [],
+ ));
+ } else {
+ return query
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'hash',
+ lower: [hash],
+ includeLower: false,
+ upper: [],
+ ))
+ .addWhereClause(IndexWhereClause.between(
+ indexName: r'hash',
+ lower: [],
+ upper: [hash],
+ includeUpper: false,
+ ));
+ }
+ });
+ }
+}
+
+extension DeviceAssetEntityQueryFilter
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QFilterCondition> {
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdEqualTo(
+ String value, {
+ bool caseSensitive = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.equalTo(
+ property: r'assetId',
+ value: value,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdGreaterThan(
+ String value, {
+ bool include = false,
+ bool caseSensitive = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.greaterThan(
+ include: include,
+ property: r'assetId',
+ value: value,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdLessThan(
+ String value, {
+ bool include = false,
+ bool caseSensitive = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.lessThan(
+ include: include,
+ property: r'assetId',
+ value: value,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdBetween(
+ String lower,
+ String upper, {
+ bool includeLower = true,
+ bool includeUpper = true,
+ bool caseSensitive = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.between(
+ property: r'assetId',
+ lower: lower,
+ includeLower: includeLower,
+ upper: upper,
+ includeUpper: includeUpper,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdStartsWith(
+ String value, {
+ bool caseSensitive = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.startsWith(
+ property: r'assetId',
+ value: value,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdEndsWith(
+ String value, {
+ bool caseSensitive = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.endsWith(
+ property: r'assetId',
+ value: value,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdContains(String value, {bool caseSensitive = true}) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.contains(
+ property: r'assetId',
+ value: value,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdMatches(String pattern, {bool caseSensitive = true}) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.matches(
+ property: r'assetId',
+ wildcard: pattern,
+ caseSensitive: caseSensitive,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdIsEmpty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.equalTo(
+ property: r'assetId',
+ value: '',
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ assetIdIsNotEmpty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.greaterThan(
+ property: r'assetId',
+ value: '',
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashElementEqualTo(int value) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.equalTo(
+ property: r'hash',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashElementGreaterThan(
+ int value, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.greaterThan(
+ include: include,
+ property: r'hash',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashElementLessThan(
+ int value, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.lessThan(
+ include: include,
+ property: r'hash',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashElementBetween(
+ int lower,
+ int upper, {
+ bool includeLower = true,
+ bool includeUpper = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.between(
+ property: r'hash',
+ lower: lower,
+ includeLower: includeLower,
+ upper: upper,
+ includeUpper: includeUpper,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashLengthEqualTo(int length) {
+ return QueryBuilder.apply(this, (query) {
+ return query.listLength(
+ r'hash',
+ length,
+ true,
+ length,
+ true,
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashIsEmpty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.listLength(
+ r'hash',
+ 0,
+ true,
+ 0,
+ true,
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashIsNotEmpty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.listLength(
+ r'hash',
+ 0,
+ false,
+ 999999,
+ true,
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashLengthLessThan(
+ int length, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.listLength(
+ r'hash',
+ 0,
+ true,
+ length,
+ include,
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashLengthGreaterThan(
+ int length, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.listLength(
+ r'hash',
+ length,
+ include,
+ 999999,
+ true,
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ hashLengthBetween(
+ int lower,
+ int upper, {
+ bool includeLower = true,
+ bool includeUpper = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.listLength(
+ r'hash',
+ lower,
+ includeLower,
+ upper,
+ includeUpper,
+ );
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ idEqualTo(Id value) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.equalTo(
+ property: r'id',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ idGreaterThan(
+ Id value, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.greaterThan(
+ include: include,
+ property: r'id',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ idLessThan(
+ Id value, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.lessThan(
+ include: include,
+ property: r'id',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ idBetween(
+ Id lower,
+ Id upper, {
+ bool includeLower = true,
+ bool includeUpper = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.between(
+ property: r'id',
+ lower: lower,
+ includeLower: includeLower,
+ upper: upper,
+ includeUpper: includeUpper,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ modifiedTimeEqualTo(DateTime value) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.equalTo(
+ property: r'modifiedTime',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ modifiedTimeGreaterThan(
+ DateTime value, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.greaterThan(
+ include: include,
+ property: r'modifiedTime',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ modifiedTimeLessThan(
+ DateTime value, {
+ bool include = false,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.lessThan(
+ include: include,
+ property: r'modifiedTime',
+ value: value,
+ ));
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterFilterCondition>
+ modifiedTimeBetween(
+ DateTime lower,
+ DateTime upper, {
+ bool includeLower = true,
+ bool includeUpper = true,
+ }) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addFilterCondition(FilterCondition.between(
+ property: r'modifiedTime',
+ lower: lower,
+ includeLower: includeLower,
+ upper: upper,
+ includeUpper: includeUpper,
+ ));
+ });
+ }
+}
+
+extension DeviceAssetEntityQueryObject
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QFilterCondition> {}
+
+extension DeviceAssetEntityQueryLinks
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QFilterCondition> {}
+
+extension DeviceAssetEntityQuerySortBy
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QSortBy> {
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ sortByAssetId() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'assetId', Sort.asc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ sortByAssetIdDesc() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'assetId', Sort.desc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ sortByModifiedTime() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'modifiedTime', Sort.asc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ sortByModifiedTimeDesc() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'modifiedTime', Sort.desc);
+ });
+ }
+}
+
+extension DeviceAssetEntityQuerySortThenBy
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QSortThenBy> {
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ thenByAssetId() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'assetId', Sort.asc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ thenByAssetIdDesc() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'assetId', Sort.desc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy> thenById() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'id', Sort.asc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ thenByIdDesc() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'id', Sort.desc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ thenByModifiedTime() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'modifiedTime', Sort.asc);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QAfterSortBy>
+ thenByModifiedTimeDesc() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addSortBy(r'modifiedTime', Sort.desc);
+ });
+ }
+}
+
+extension DeviceAssetEntityQueryWhereDistinct
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QDistinct> {
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QDistinct>
+ distinctByAssetId({bool caseSensitive = true}) {
+ return QueryBuilder.apply(this, (query) {
+ return query.addDistinctBy(r'assetId', caseSensitive: caseSensitive);
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QDistinct>
+ distinctByHash() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addDistinctBy(r'hash');
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QDistinct>
+ distinctByModifiedTime() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addDistinctBy(r'modifiedTime');
+ });
+ }
+}
+
+extension DeviceAssetEntityQueryProperty
+ on QueryBuilder<DeviceAssetEntity, DeviceAssetEntity, QQueryProperty> {
+ QueryBuilder<DeviceAssetEntity, int, QQueryOperations> idProperty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addPropertyName(r'id');
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, String, QQueryOperations> assetIdProperty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addPropertyName(r'assetId');
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, List<int>, QQueryOperations> hashProperty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addPropertyName(r'hash');
+ });
+ }
+
+ QueryBuilder<DeviceAssetEntity, DateTime, QQueryOperations>
+ modifiedTimeProperty() {
+ return QueryBuilder.apply(this, (query) {
+ return query.addPropertyName(r'modifiedTime');
+ });
+ }
+}
diff --git a/mobile/lib/infrastructure/repositories/device_asset.repository.dart b/mobile/lib/infrastructure/repositories/device_asset.repository.dart
new file mode 100644
index 0000000000..87784ecaab
--- /dev/null
+++ b/mobile/lib/infrastructure/repositories/device_asset.repository.dart
@@ -0,0 +1,37 @@
+import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart';
+import 'package:immich_mobile/domain/models/device_asset.model.dart';
+import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart';
+import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
+import 'package:isar/isar.dart';
+
+class IsarDeviceAssetRepository extends IsarDatabaseRepository
+ implements IDeviceAssetRepository {
+ final Isar _db;
+
+ const IsarDeviceAssetRepository(this._db) : super(_db);
+
+ @override
+ Future<void> deleteIds(List<String> ids) {
+ return transaction(() async {
+ await _db.deviceAssetEntitys.deleteAllByAssetId(ids.toList());
+ });
+ }
+
+ @override
+ Future<List<DeviceAsset>> getByIds(List<String> localIds) {
+ return _db.deviceAssetEntitys
+ .where()
+ .anyOf(localIds, (query, id) => query.assetIdEqualTo(id))
+ .findAll()
+ .then((value) => value.map((e) => e.toModel()).toList());
+ }
+
+ @override
+ Future<bool> updateAll(List<DeviceAsset> assetHash) {
+ return transaction(() async {
+ await _db.deviceAssetEntitys
+ .putAll(assetHash.map(DeviceAssetEntity.fromDto).toList());
+ return true;
+ });
+ }
+}
diff --git a/mobile/lib/interfaces/asset.interface.dart b/mobile/lib/interfaces/asset.interface.dart
index ed524c4f35..76744c9172 100644
--- a/mobile/lib/interfaces/asset.interface.dart
+++ b/mobile/lib/interfaces/asset.interface.dart
@@ -1,6 +1,5 @@
import 'package:immich_mobile/entities/album.entity.dart';
import 'package:immich_mobile/entities/asset.entity.dart';
-import 'package:immich_mobile/entities/device_asset.entity.dart';
import 'package:immich_mobile/interfaces/database.interface.dart';
abstract interface class IAssetRepository implements IDatabaseRepository {
@@ -50,10 +49,6 @@ abstract interface class IAssetRepository implements IDatabaseRepository {
int limit = 100,
});
- Future<List<DeviceAsset?>> getDeviceAssetsById(List<Object> ids);
-
- Future<void> upsertDeviceAssets(List<DeviceAsset> deviceAssets);
-
Future<void> upsertDuplicatedAssets(Iterable<String> duplicatedAssets);
Future<List<String>> getAllDuplicatedAssetIds();
diff --git a/mobile/lib/providers/infrastructure/device_asset.provider.dart b/mobile/lib/providers/infrastructure/device_asset.provider.dart
new file mode 100644
index 0000000000..5fa532b9ec
--- /dev/null
+++ b/mobile/lib/providers/infrastructure/device_asset.provider.dart
@@ -0,0 +1,8 @@
+import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart';
+import 'package:immich_mobile/infrastructure/repositories/device_asset.repository.dart';
+import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
+
+final deviceAssetRepositoryProvider = Provider<IDeviceAssetRepository>(
+ (ref) => IsarDeviceAssetRepository(ref.watch(isarProvider)),
+);
diff --git a/mobile/lib/repositories/asset.repository.dart b/mobile/lib/repositories/asset.repository.dart
index d9e8897e97..cda2b25e4d 100644
--- a/mobile/lib/repositories/asset.repository.dart
+++ b/mobile/lib/repositories/asset.repository.dart
@@ -1,12 +1,7 @@
-import 'dart:io';
-
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/entities/album.entity.dart';
-import 'package:immich_mobile/entities/android_device_asset.entity.dart';
import 'package:immich_mobile/entities/asset.entity.dart';
-import 'package:immich_mobile/entities/device_asset.entity.dart';
import 'package:immich_mobile/entities/duplicated_asset.entity.dart';
-import 'package:immich_mobile/entities/ios_device_asset.entity.dart';
import 'package:immich_mobile/infrastructure/entities/exif.entity.dart';
import 'package:immich_mobile/interfaces/asset.interface.dart';
import 'package:immich_mobile/providers/db.provider.dart';
@@ -158,19 +153,6 @@ class AssetRepository extends DatabaseRepository implements IAssetRepository {
return _getMatchesImpl(query, fastHash(ownerId), assets, limit);
}
- @override
- Future<List<DeviceAsset?>> getDeviceAssetsById(List<Object> ids) =>
- Platform.isAndroid
- ? db.androidDeviceAssets.getAll(ids.cast())
- : db.iOSDeviceAssets.getAllById(ids.cast());
-
- @override
- Future<void> upsertDeviceAssets(List<DeviceAsset> deviceAssets) => txn(
- () => Platform.isAndroid
- ? db.androidDeviceAssets.putAll(deviceAssets.cast())
- : db.iOSDeviceAssets.putAll(deviceAssets.cast()),
- );
-
@override
Future<Asset> update(Asset asset) async {
await txn(() => asset.put(db));
diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart
index bb19340d2f..ca2b0ee37e 100644
--- a/mobile/lib/services/hash.service.dart
+++ b/mobile/lib/services/hash.service.dart
@@ -1,172 +1,205 @@
+// ignore_for_file: avoid-unsafe-collection-methods
+
+import 'dart:convert';
import 'dart:io';
import 'package:flutter/foundation.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
-import 'package:immich_mobile/entities/album.entity.dart';
-import 'package:immich_mobile/interfaces/album_media.interface.dart';
-import 'package:immich_mobile/interfaces/asset.interface.dart';
-import 'package:immich_mobile/repositories/album_media.repository.dart';
-import 'package:immich_mobile/repositories/asset.repository.dart';
-import 'package:immich_mobile/services/background.service.dart';
-import 'package:immich_mobile/entities/android_device_asset.entity.dart';
+import 'package:immich_mobile/constants/constants.dart';
+import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart';
+import 'package:immich_mobile/domain/models/device_asset.model.dart';
import 'package:immich_mobile/entities/asset.entity.dart';
-import 'package:immich_mobile/entities/device_asset.entity.dart';
-import 'package:immich_mobile/entities/ios_device_asset.entity.dart';
-import 'package:immich_mobile/extensions/string_extensions.dart';
+import 'package:immich_mobile/providers/infrastructure/device_asset.provider.dart';
+import 'package:immich_mobile/services/background.service.dart';
import 'package:logging/logging.dart';
class HashService {
- HashService(
- this._assetRepository,
- this._backgroundService,
- this._albumMediaRepository,
- );
- final IAssetRepository _assetRepository;
+ HashService({
+ required IDeviceAssetRepository deviceAssetRepository,
+ required BackgroundService backgroundService,
+ this.batchSizeLimit = kBatchHashSizeLimit,
+ this.batchFileLimit = kBatchHashFileLimit,
+ }) : _deviceAssetRepository = deviceAssetRepository,
+ _backgroundService = backgroundService;
+
+ final IDeviceAssetRepository _deviceAssetRepository;
final BackgroundService _backgroundService;
- final IAlbumMediaRepository _albumMediaRepository;
+ final int batchSizeLimit;
+ final int batchFileLimit;
final _log = Logger('HashService');
- /// Returns all assets that were successfully hashed
- Future<List<Asset>> getHashedAssets(
- Album album, {
- int start = 0,
- int end = 0x7fffffffffffffff,
- DateTime? modifiedFrom,
- DateTime? modifiedUntil,
- Set<String>? excludedAssets,
- }) async {
- final entities = await _albumMediaRepository.getAssets(
- album.localId!,
- start: start,
- end: end,
- modifiedFrom: modifiedFrom,
- modifiedUntil: modifiedUntil,
- );
- final filtered = excludedAssets == null
- ? entities
- : entities.where((e) => !excludedAssets.contains(e.localId!)).toList();
- return _hashAssets(filtered);
- }
-
/// Processes a list of local [Asset]s, storing their hash and returning only those
/// that were successfully hashed. Hashes are looked up in a DB table
- /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing
- /// entries are newly hashed and added to the DB table.
- Future<List<Asset>> _hashAssets(List<Asset> assets) async {
- const int batchFileCount = 128;
- const int batchDataSize = 1024 * 1024 * 1024; // 1GB
-
- final ids = assets
- .map(Platform.isAndroid ? (a) => a.localId!.toInt() : (a) => a.localId!)
- .toList();
- final List<DeviceAsset?> hashes =
- await _assetRepository.getDeviceAssetsById(ids);
- final List<DeviceAsset> toAdd = [];
- final List<String> toHash = [];
-
- int bytes = 0;
-
- for (int i = 0; i < assets.length; i++) {
- if (hashes[i] != null) {
- continue;
- }
+ /// [DeviceAsset] by local id. Only missing entries are newly hashed and added to the DB table.
+ Future<List<Asset>> hashAssets(List<Asset> assets) async {
+ assets.sort(Asset.compareByLocalId);
- File? file;
+ // Get and sort DB entries - guaranteed to be a subset of assets
+ final hashesInDB = await _deviceAssetRepository.getByIds(
+ assets.map((a) => a.localId!).toList(),
+ );
+ hashesInDB.sort((a, b) => a.assetId.compareTo(b.assetId));
+
+ int dbIndex = 0;
+ int bytesProcessed = 0;
+ final hashedAssets = <Asset>[];
+ final toBeHashed = <_AssetPath>[];
+ final toBeDeleted = <String>[];
+
+ for (int assetIndex = 0; assetIndex < assets.length; assetIndex++) {
+ final asset = assets[assetIndex];
+ DeviceAsset? matchingDbEntry;
+
+ if (dbIndex < hashesInDB.length) {
+ final deviceAsset = hashesInDB[dbIndex];
+ if (deviceAsset.assetId == asset.localId) {
+ matchingDbEntry = deviceAsset;
+ dbIndex++;
+ }
+ }
- try {
- file = await assets[i].local!.originFile;
- } catch (error, stackTrace) {
- _log.warning(
- "Error getting file to hash for asset ${assets[i].localId}, name: ${assets[i].fileName}, created on: ${assets[i].fileCreatedAt}, skipping",
- error,
- stackTrace,
+ if (matchingDbEntry != null &&
+ matchingDbEntry.hash.isNotEmpty &&
+ matchingDbEntry.modifiedTime.isAtSameMomentAs(asset.fileModifiedAt)) {
+ // Reuse the existing hash
+ hashedAssets.add(
+ asset.copyWith(checksum: base64.encode(matchingDbEntry.hash)),
);
+ continue;
}
+ final file = await _tryGetAssetFile(asset);
if (file == null) {
- final fileName = assets[i].fileName;
-
- _log.warning(
- "Failed to get file for asset ${assets[i].localId}, name: $fileName, created on: ${assets[i].fileCreatedAt}, skipping",
- );
+ // Can't access file, delete any DB entry
+ if (matchingDbEntry != null) {
+ toBeDeleted.add(matchingDbEntry.assetId);
+ }
continue;
}
- bytes += await file.length();
- toHash.add(file.path);
- final deviceAsset = Platform.isAndroid
- ? AndroidDeviceAsset(id: ids[i] as int, hash: const [])
- : IOSDeviceAsset(id: ids[i] as String, hash: const []);
- toAdd.add(deviceAsset);
- hashes[i] = deviceAsset;
- if (toHash.length == batchFileCount || bytes >= batchDataSize) {
- await _processBatch(toHash, toAdd);
- toAdd.clear();
- toHash.clear();
- bytes = 0;
+
+ bytesProcessed += await file.length();
+ toBeHashed.add(_AssetPath(asset: asset, path: file.path));
+
+ if (_shouldProcessBatch(toBeHashed.length, bytesProcessed)) {
+ hashedAssets.addAll(await _processBatch(toBeHashed, toBeDeleted));
+ toBeHashed.clear();
+ toBeDeleted.clear();
+ bytesProcessed = 0;
}
}
- if (toHash.isNotEmpty) {
- await _processBatch(toHash, toAdd);
+ assert(dbIndex == hashesInDB.length, "All hashes should've been processed");
+
+ // Process any remaining files
+ if (toBeHashed.isNotEmpty) {
+ hashedAssets.addAll(await _processBatch(toBeHashed, toBeDeleted));
+ }
+
+ // Clean up deleted references
+ if (toBeDeleted.isNotEmpty) {
+ await _deviceAssetRepository.deleteIds(toBeDeleted);
+ }
+
+ return hashedAssets;
+ }
+
+ bool _shouldProcessBatch(int assetCount, int bytesProcessed) =>
+ assetCount >= batchFileLimit || bytesProcessed >= batchSizeLimit;
+
+ Future<File?> _tryGetAssetFile(Asset asset) async {
+ try {
+ final file = await asset.local!.originFile;
+ if (file == null) {
+ _log.warning(
+ "Failed to get file for asset ${asset.localId ?? '<N/A>'}, name: ${asset.fileName}, created on: ${asset.fileCreatedAt}, skipping",
+ );
+ return null;
+ }
+ return file;
+ } catch (error, stackTrace) {
+ _log.warning(
+ "Error getting file to hash for asset ${asset.localId ?? '<N/A>'}, name: ${asset.fileName}, created on: ${asset.fileCreatedAt}, skipping",
+ error,
+ stackTrace,
+ );
+ return null;
}
- return _getHashedAssets(assets, hashes);
}
- /// Processes a batch of files and saves any successfully hashed
- /// values to the DB table.
- Future<void> _processBatch(
- final List<String> toHash,
- final List<DeviceAsset> toAdd,
+ /// Processes a batch of files and returns a list of successfully hashed assets after saving
+ /// them in [DeviceAssetToHash] for future retrieval
+ Future<List<Asset>> _processBatch(
+ List<_AssetPath> toBeHashed,
+ List<String> toBeDeleted,
) async {
- final hashes = await _hashFiles(toHash);
- bool anyNull = false;
- for (int j = 0; j < hashes.length; j++) {
- if (hashes[j]?.length == 20) {
- toAdd[j].hash = hashes[j]!;
+ _log.info("Hashing ${toBeHashed.length} files");
+ final hashes = await _hashFiles(toBeHashed.map((e) => e.path).toList());
+ assert(
+ hashes.length == toBeHashed.length,
+ "Number of Hashes returned from platform should be the same as the input",
+ );
+
+ final hashedAssets = <Asset>[];
+ final toBeAdded = <DeviceAsset>[];
+
+ for (final (index, hash) in hashes.indexed) {
+ final asset = toBeHashed.elementAtOrNull(index)?.asset;
+ if (asset != null && hash?.length == 20) {
+ hashedAssets.add(asset.copyWith(checksum: base64.encode(hash!)));
+ toBeAdded.add(
+ DeviceAsset(
+ assetId: asset.localId!,
+ hash: hash,
+ modifiedTime: asset.fileModifiedAt,
+ ),
+ );
} else {
- _log.warning("Failed to hash file ${toHash[j]}, skipping");
- anyNull = true;
+ _log.warning("Failed to hash file ${asset?.localId ?? '<null>'}");
+ if (asset != null) {
+ toBeDeleted.add(asset.localId!);
+ }
}
}
- final validHashes = anyNull
- ? toAdd.where((e) => e.hash.length == 20).toList(growable: false)
- : toAdd;
- await _assetRepository
- .transaction(() => _assetRepository.upsertDeviceAssets(validHashes));
- _log.fine("Hashed ${validHashes.length}/${toHash.length} assets");
+ // Update the DB for future retrieval
+ await _deviceAssetRepository.transaction(() async {
+ await _deviceAssetRepository.updateAll(toBeAdded);
+ await _deviceAssetRepository.deleteIds(toBeDeleted);
+ });
+
+ _log.fine("Hashed ${hashedAssets.length}/${toBeHashed.length} assets");
+ return hashedAssets;
}
- /// Hashes the given files and returns a list of the same length
- /// files that could not be hashed have a `null` value
+ /// Hashes the given files and returns a list of the same length.
+ /// Files that could not be hashed will have a `null` value
Future<List<Uint8List?>> _hashFiles(List<String> paths) async {
- final List<Uint8List?>? hashes =
- await _backgroundService.digestFiles(paths);
- if (hashes == null) {
- throw Exception("Hashing ${paths.length} files failed");
+ try {
+ final hashes = await _backgroundService.digestFiles(paths);
+ if (hashes != null) {
+ return hashes;
+ }
+ _log.severe("Hashing ${paths.length} files failed");
+ } catch (e, s) {
+ _log.severe("Error occurred while hashing assets", e, s);
}
- return hashes;
+ return List.filled(paths.length, null);
}
+}
- /// Returns all successfully hashed [Asset]s with their hash value set
- List<Asset> _getHashedAssets(
- List<Asset> assets,
- List<DeviceAsset?> hashes,
- ) {
- final List<Asset> result = [];
- for (int i = 0; i < assets.length; i++) {
- if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) {
- assets[i].byteHash = hashes[i]!.hash;
- result.add(assets[i]);
- }
- }
- return result;
+class _AssetPath {
+ final Asset asset;
+ final String path;
+
+ const _AssetPath({required this.asset, required this.path});
+
+ _AssetPath copyWith({Asset? asset, String? path}) {
+ return _AssetPath(asset: asset ?? this.asset, path: path ?? this.path);
}
}
final hashServiceProvider = Provider(
(ref) => HashService(
- ref.watch(assetRepositoryProvider),
- ref.watch(backgroundServiceProvider),
- ref.watch(albumMediaRepositoryProvider),
+ deviceAssetRepository: ref.watch(deviceAssetRepositoryProvider),
+ backgroundService: ref.watch(backgroundServiceProvider),
),
);
diff --git a/mobile/lib/services/sync.service.dart b/mobile/lib/services/sync.service.dart
index a05d4b648e..f2b16b080a 100644
--- a/mobile/lib/services/sync.service.dart
+++ b/mobile/lib/services/sync.service.dart
@@ -577,15 +577,18 @@ class SyncService {
Set<String>? excludedAssets,
bool forceRefresh = false,
]) async {
+ _log.info("Syncing a local album to DB: ${deviceAlbum.name}");
if (!forceRefresh && !await _hasAlbumChangeOnDevice(deviceAlbum, dbAlbum)) {
- _log.fine(
+ _log.info(
"Local album ${deviceAlbum.name} has not changed. Skipping sync.",
);
return false;
}
+ _log.info("Local album ${deviceAlbum.name} has changed. Syncing...");
if (!forceRefresh &&
excludedAssets == null &&
await _syncDeviceAlbumFast(deviceAlbum, dbAlbum)) {
+ _log.info("Fast synced local album ${deviceAlbum.name} to DB");
return true;
}
// general case, e.g. some assets have been deleted or there are excluded albums on iOS
@@ -598,7 +601,7 @@ class SyncService {
assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
final int assetCountOnDevice =
await _albumMediaRepository.getAssetCount(deviceAlbum.localId!);
- final List<Asset> onDevice = await _hashService.getHashedAssets(
+ final List<Asset> onDevice = await _getHashedAssets(
deviceAlbum,
excludedAssets: excludedAssets,
);
@@ -611,7 +614,7 @@ class SyncService {
dbAlbum.name == deviceAlbum.name &&
dbAlbum.modifiedAt.isAtSameMomentAs(deviceAlbum.modifiedAt)) {
// changes only affeted excluded albums
- _log.fine(
+ _log.info(
"Only excluded assets in local album ${deviceAlbum.name} changed. Stopping sync.",
);
if (assetCountOnDevice !=
@@ -626,11 +629,11 @@ class SyncService {
}
return false;
}
- _log.fine(
+ _log.info(
"Syncing local album ${deviceAlbum.name}. ${toAdd.length} assets to add, ${toUpdate.length} to update, ${toDelete.length} to delete",
);
final (existingInDb, updated) = await _linkWithExistingFromDb(toAdd);
- _log.fine(
+ _log.info(
"Linking assets to add with existing from db. ${existingInDb.length} existing, ${updated.length} to update",
);
deleteCandidates.addAll(toDelete);
@@ -667,6 +670,9 @@ class SyncService {
/// returns `true` if successful, else `false`
Future<bool> _syncDeviceAlbumFast(Album deviceAlbum, Album dbAlbum) async {
if (!deviceAlbum.modifiedAt.isAfter(dbAlbum.modifiedAt)) {
+ _log.info(
+ "Local album ${deviceAlbum.name} has not changed. Skipping sync.",
+ );
return false;
}
final int totalOnDevice =
@@ -676,15 +682,21 @@ class SyncService {
?.assetCount ??
0;
if (totalOnDevice <= lastKnownTotal) {
+ _log.info(
+ "Local album ${deviceAlbum.name} totalOnDevice is less than lastKnownTotal. Skipping sync.",
+ );
return false;
}
- final List<Asset> newAssets = await _hashService.getHashedAssets(
+ final List<Asset> newAssets = await _getHashedAssets(
deviceAlbum,
modifiedFrom: dbAlbum.modifiedAt.add(const Duration(seconds: 1)),
modifiedUntil: deviceAlbum.modifiedAt,
);
if (totalOnDevice != lastKnownTotal + newAssets.length) {
+ _log.info(
+ "Local album ${deviceAlbum.name} totalOnDevice is not equal to lastKnownTotal + newAssets.length. Skipping sync.",
+ );
return false;
}
dbAlbum.modifiedAt = deviceAlbum.modifiedAt;
@@ -719,8 +731,8 @@ class SyncService {
List<Asset> existing, [
Set<String>? excludedAssets,
]) async {
- _log.info("Syncing a new local album to DB: ${album.name}");
- final assets = await _hashService.getHashedAssets(
+ _log.info("Adding a new local album to DB: ${album.name}");
+ final assets = await _getHashedAssets(
album,
excludedAssets: excludedAssets,
);
@@ -824,6 +836,28 @@ class SyncService {
}
}
+ /// Returns all assets that were successfully hashed
+ Future<List<Asset>> _getHashedAssets(
+ Album album, {
+ int start = 0,
+ int end = 0x7fffffffffffffff,
+ DateTime? modifiedFrom,
+ DateTime? modifiedUntil,
+ Set<String>? excludedAssets,
+ }) async {
+ final entities = await _albumMediaRepository.getAssets(
+ album.localId!,
+ start: start,
+ end: end,
+ modifiedFrom: modifiedFrom,
+ modifiedUntil: modifiedUntil,
+ );
+ final filtered = excludedAssets == null
+ ? entities
+ : entities.where((e) => !excludedAssets.contains(e.localId!)).toList();
+ return _hashService.hashAssets(filtered);
+ }
+
List<Asset> _removeDuplicates(List<Asset> assets) {
final int before = assets.length;
assets.sort(Asset.compareByOwnerChecksumCreatedModified);
diff --git a/mobile/lib/utils/bootstrap.dart b/mobile/lib/utils/bootstrap.dart
index 21231becf6..dec48582b3 100644
--- a/mobile/lib/utils/bootstrap.dart
+++ b/mobile/lib/utils/bootstrap.dart
@@ -10,6 +10,7 @@ import 'package:immich_mobile/entities/backup_album.entity.dart';
import 'package:immich_mobile/entities/duplicated_asset.entity.dart';
import 'package:immich_mobile/entities/etag.entity.dart';
import 'package:immich_mobile/entities/ios_device_asset.entity.dart';
+import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart';
import 'package:immich_mobile/infrastructure/entities/exif.entity.dart';
import 'package:immich_mobile/infrastructure/entities/log.entity.dart';
import 'package:immich_mobile/infrastructure/entities/store.entity.dart';
@@ -39,6 +40,7 @@ abstract final class Bootstrap {
ETagSchema,
if (Platform.isAndroid) AndroidDeviceAssetSchema,
if (Platform.isIOS) IOSDeviceAssetSchema,
+ DeviceAssetEntitySchema,
],
directory: dir.path,
maxSizeMiB: 1024,
diff --git a/mobile/lib/utils/diff.dart b/mobile/lib/utils/diff.dart
index a36902d8c7..ea20de16cc 100644
--- a/mobile/lib/utils/diff.dart
+++ b/mobile/lib/utils/diff.dart
@@ -75,3 +75,17 @@ bool diffSortedListsSync<T>(
}
return diff;
}
+
+int compareToNullable<T extends Comparable>(T? a, T? b) {
+ if (a == null && b == null) {
+ return 0;
+ }
+
+ if (a == null) {
+ return 1;
+ }
+ if (b == null) {
+ return -1;
+ }
+ return a.compareTo(b);
+}
diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart
index 3e73ab445b..bebd7a027b 100644
--- a/mobile/lib/utils/migration.dart
+++ b/mobile/lib/utils/migration.dart
@@ -1,40 +1,51 @@
+// ignore_for_file: avoid-unsafe-collection-methods
+
import 'dart:async';
+import 'dart:io';
+import 'package:flutter/widgets.dart';
import 'package:immich_mobile/domain/models/store.model.dart';
import 'package:immich_mobile/entities/album.entity.dart';
+import 'package:immich_mobile/entities/android_device_asset.entity.dart';
import 'package:immich_mobile/entities/asset.entity.dart';
import 'package:immich_mobile/entities/etag.entity.dart';
+import 'package:immich_mobile/entities/ios_device_asset.entity.dart';
import 'package:immich_mobile/entities/store.entity.dart';
+import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart';
import 'package:immich_mobile/infrastructure/entities/exif.entity.dart';
import 'package:immich_mobile/infrastructure/entities/store.entity.dart';
import 'package:immich_mobile/infrastructure/entities/user.entity.dart';
+import 'package:immich_mobile/utils/diff.dart';
import 'package:isar/isar.dart';
-const int targetVersion = 9;
+const int targetVersion = 10;
Future<void> migrateDatabaseIfNeeded(Isar db) async {
- final int version = Store.get(StoreKey.version, 1);
+ final int version = Store.get(StoreKey.version, targetVersion);
if (version < 9) {
- await Store.put(StoreKey.version, version);
+ await Store.put(StoreKey.version, targetVersion);
final value = await db.storeValues.get(StoreKey.currentUser.id);
if (value != null) {
final id = value.intValue;
- if (id == null) {
- return;
+ if (id != null) {
+ await db.writeTxn(() async {
+ final user = await db.users.get(id);
+ await db.storeValues
+ .put(StoreValue(StoreKey.currentUser.id, strValue: user?.id));
+ });
}
- await db.writeTxn(() async {
- final user = await db.users.get(id);
- await db.storeValues
- .put(StoreValue(StoreKey.currentUser.id, strValue: user?.id));
- });
}
- // Do not clear other entities
- return;
}
- if (version < targetVersion) {
- _migrateTo(db, targetVersion);
+ if (version < 10) {
+ await Store.put(StoreKey.version, targetVersion);
+ await _migrateDeviceAsset(db);
+ }
+
+ final shouldTruncate = version < 8 && version < targetVersion;
+ if (shouldTruncate) {
+ await _migrateTo(db, targetVersion);
}
}
@@ -49,3 +60,59 @@ Future<void> _migrateTo(Isar db, int version) async {
});
await Store.put(StoreKey.version, version);
}
+
+Future<void> _migrateDeviceAsset(Isar db) async {
+ final ids = Platform.isAndroid
+ ? (await db.androidDeviceAssets.where().findAll())
+ .map((a) => _DeviceAsset(assetId: a.id.toString(), hash: a.hash))
+ .toList()
+ : (await db.iOSDeviceAssets.where().findAll())
+ .map((i) => _DeviceAsset(assetId: i.id, hash: i.hash))
+ .toList();
+ final localAssets = (await db.assets
+ .where()
+ .anyOf(ids, (query, id) => query.localIdEqualTo(id.assetId))
+ .findAll())
+ .map((a) => _DeviceAsset(assetId: a.localId!, dateTime: a.fileModifiedAt))
+ .toList();
+ debugPrint("Device Asset Ids length - ${ids.length}");
+ debugPrint("Local Asset Ids length - ${localAssets.length}");
+ ids.sort((a, b) => a.assetId.compareTo(b.assetId));
+ localAssets.sort((a, b) => a.assetId.compareTo(b.assetId));
+ final List<DeviceAssetEntity> toAdd = [];
+ await diffSortedLists(
+ ids,
+ localAssets,
+ compare: (a, b) => a.assetId.compareTo(b.assetId),
+ both: (deviceAsset, asset) {
+ toAdd.add(
+ DeviceAssetEntity(
+ assetId: deviceAsset.assetId,
+ hash: deviceAsset.hash!,
+ modifiedTime: asset.dateTime!,
+ ),
+ );
+ return false;
+ },
+ onlyFirst: (deviceAsset) {
+ debugPrint(
+ 'DeviceAsset not found in local assets: ${deviceAsset.assetId}',
+ );
+ },
+ onlySecond: (asset) {
+ debugPrint('Local asset not found in DeviceAsset: ${asset.assetId}');
+ },
+ );
+ debugPrint("Total number of device assets migrated - ${toAdd.length}");
+ await db.writeTxn(() async {
+ await db.deviceAssetEntitys.putAll(toAdd);
+ });
+}
+
+class _DeviceAsset {
+ final String assetId;
+ final List<int>? hash;
+ final DateTime? dateTime;
+
+ const _DeviceAsset({required this.assetId, this.hash, this.dateTime});
+}
diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock
index e79d9f4084..7c8348726f 100644
--- a/mobile/pubspec.lock
+++ b/mobile/pubspec.lock
@@ -463,7 +463,7 @@ packages:
source: hosted
version: "2.1.4"
file:
- dependency: transitive
+ dependency: "direct dev"
description:
name: file
sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4
diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml
index d4ab110a3e..e939c65836 100644
--- a/mobile/pubspec.yaml
+++ b/mobile/pubspec.yaml
@@ -102,6 +102,7 @@ dev_dependencies:
immich_mobile_immich_lint:
path: './immich_lint'
fake_async: ^1.3.1
+ file: ^7.0.1 # for MemoryFileSystem
# Drift generator
drift_dev: ^2.23.1
diff --git a/mobile/test/domain/services/hash_service_test.dart b/mobile/test/domain/services/hash_service_test.dart
new file mode 100644
index 0000000000..2da41cd704
--- /dev/null
+++ b/mobile/test/domain/services/hash_service_test.dart
@@ -0,0 +1,425 @@
+import 'dart:convert';
+import 'dart:io';
+import 'dart:math';
+
+import 'package:collection/collection.dart';
+import 'package:file/memory.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart';
+import 'package:immich_mobile/domain/models/device_asset.model.dart';
+import 'package:immich_mobile/entities/asset.entity.dart';
+import 'package:immich_mobile/services/background.service.dart';
+import 'package:immich_mobile/services/hash.service.dart';
+import 'package:mocktail/mocktail.dart';
+import 'package:photo_manager/photo_manager.dart';
+
+import '../../fixtures/asset.stub.dart';
+import '../../infrastructure/repository.mock.dart';
+import '../../service.mocks.dart';
+
+class MockAsset extends Mock implements Asset {}
+
+class MockAssetEntity extends Mock implements AssetEntity {}
+
+void main() {
+ late HashService sut;
+ late BackgroundService mockBackgroundService;
+ late IDeviceAssetRepository mockDeviceAssetRepository;
+
+ setUp(() {
+ mockBackgroundService = MockBackgroundService();
+ mockDeviceAssetRepository = MockDeviceAssetRepository();
+
+ sut = HashService(
+ deviceAssetRepository: mockDeviceAssetRepository,
+ backgroundService: mockBackgroundService,
+ );
+
+ when(() => mockDeviceAssetRepository.transaction<Null>(any()))
+ .thenAnswer((_) async {
+ final capturedCallback = verify(
+ () => mockDeviceAssetRepository.transaction<Null>(captureAny()),
+ ).captured;
+ // Invoke the transaction callback
+ await (capturedCallback.firstOrNull as Future<Null> Function()?)?.call();
+ });
+ when(() => mockDeviceAssetRepository.updateAll(any()))
+ .thenAnswer((_) async => true);
+ when(() => mockDeviceAssetRepository.deleteIds(any()))
+ .thenAnswer((_) async => true);
+ });
+
+ group("HashService: No DeviceAsset entry", () {
+ test("hash successfully", () async {
+ final (mockAsset, file, deviceAsset, hash) =
+ await _createAssetMock(AssetStub.image1);
+
+ when(() => mockBackgroundService.digestFiles([file.path]))
+ .thenAnswer((_) async => [hash]);
+ // No DB entries for this asset
+ when(
+ () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]),
+ ).thenAnswer((_) async => []);
+
+ final result = await sut.hashAssets([mockAsset]);
+
+ // Verify we stored the new hash in DB
+ when(() => mockDeviceAssetRepository.transaction<Null>(any()))
+ .thenAnswer((_) async {
+ final capturedCallback = verify(
+ () => mockDeviceAssetRepository.transaction<Null>(captureAny()),
+ ).captured;
+ // Invoke the transaction callback
+ await (capturedCallback.firstOrNull as Future<Null> Function()?)
+ ?.call();
+ verify(
+ () => mockDeviceAssetRepository.updateAll([
+ deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt),
+ ]),
+ ).called(1);
+ verify(() => mockDeviceAssetRepository.deleteIds([])).called(1);
+ });
+ expect(
+ result,
+ [AssetStub.image1.copyWith(checksum: base64.encode(hash))],
+ );
+ });
+ });
+
+ group("HashService: Has DeviceAsset entry", () {
+ test("when the asset is not modified", () async {
+ final hash = utf8.encode("image1-hash");
+
+ when(
+ () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]),
+ ).thenAnswer(
+ (_) async => [
+ DeviceAsset(
+ assetId: AssetStub.image1.localId!,
+ hash: hash,
+ modifiedTime: AssetStub.image1.fileModifiedAt,
+ ),
+ ],
+ );
+ final result = await sut.hashAssets([AssetStub.image1]);
+
+ verifyNever(() => mockBackgroundService.digestFiles(any()));
+ verifyNever(() => mockBackgroundService.digestFile(any()));
+ verifyNever(() => mockDeviceAssetRepository.updateAll(any()));
+ verifyNever(() => mockDeviceAssetRepository.deleteIds(any()));
+
+ expect(result, [
+ AssetStub.image1.copyWith(checksum: base64.encode(hash)),
+ ]);
+ });
+
+ test("hashed successful when asset is modified", () async {
+ final (mockAsset, file, deviceAsset, hash) =
+ await _createAssetMock(AssetStub.image1);
+
+ when(() => mockBackgroundService.digestFiles([file.path]))
+ .thenAnswer((_) async => [hash]);
+ when(
+ () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]),
+ ).thenAnswer((_) async => [deviceAsset]);
+
+ final result = await sut.hashAssets([mockAsset]);
+
+ when(() => mockDeviceAssetRepository.transaction<Null>(any()))
+ .thenAnswer((_) async {
+ final capturedCallback = verify(
+ () => mockDeviceAssetRepository.transaction<Null>(captureAny()),
+ ).captured;
+ // Invoke the transaction callback
+ await (capturedCallback.firstOrNull as Future<Null> Function()?)
+ ?.call();
+ verify(
+ () => mockDeviceAssetRepository.updateAll([
+ deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt),
+ ]),
+ ).called(1);
+ verify(() => mockDeviceAssetRepository.deleteIds([])).called(1);
+ });
+
+ verify(() => mockBackgroundService.digestFiles([file.path])).called(1);
+
+ expect(result, [
+ AssetStub.image1.copyWith(checksum: base64.encode(hash)),
+ ]);
+ });
+ });
+
+ group("HashService: Cleanup", () {
+ late Asset mockAsset;
+ late Uint8List hash;
+ late DeviceAsset deviceAsset;
+ late File file;
+
+ setUp(() async {
+ (mockAsset, file, deviceAsset, hash) =
+ await _createAssetMock(AssetStub.image1);
+
+ when(() => mockBackgroundService.digestFiles([file.path]))
+ .thenAnswer((_) async => [hash]);
+ when(
+ () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]),
+ ).thenAnswer((_) async => [deviceAsset]);
+ });
+
+ test("cleanups DeviceAsset when local file cannot be obtained", () async {
+ when(() => mockAsset.local).thenThrow(Exception("File not found"));
+ final result = await sut.hashAssets([mockAsset]);
+
+ verifyNever(() => mockBackgroundService.digestFiles(any()));
+ verifyNever(() => mockBackgroundService.digestFile(any()));
+ verifyNever(() => mockDeviceAssetRepository.updateAll(any()));
+ verify(
+ () => mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]),
+ ).called(1);
+
+ expect(result, isEmpty);
+ });
+
+ test("cleanups DeviceAsset when hashing failed", () async {
+ when(() => mockDeviceAssetRepository.transaction<Null>(any()))
+ .thenAnswer((_) async {
+ final capturedCallback = verify(
+ () => mockDeviceAssetRepository.transaction<Null>(captureAny()),
+ ).captured;
+ // Invoke the transaction callback
+ await (capturedCallback.firstOrNull as Future<Null> Function()?)
+ ?.call();
+
+ // Verify the callback inside the transaction because, doing it outside results
+ // in a small delay before the callback is invoked, resulting in other LOCs getting executed
+ // resulting in an incorrect state
+ //
+ // i.e, consider the following piece of code
+ // await _deviceAssetRepository.transaction(() async {
+ // await _deviceAssetRepository.updateAll(toBeAdded);
+ // await _deviceAssetRepository.deleteIds(toBeDeleted);
+ // });
+ // toBeDeleted.clear();
+ // since the transaction method is mocked, the callback is not invoked until it is captured
+ // and executed manually in the next event loop. However, the toBeDeleted.clear() is executed
+ // immediately once the transaction stub is executed, resulting in the deleteIds method being
+ // called with an empty list.
+ //
+ // To avoid this, we capture the callback and execute it within the transaction stub itself
+ // and verify the results inside the transaction stub
+ verify(() => mockDeviceAssetRepository.updateAll([])).called(1);
+ verify(
+ () =>
+ mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]),
+ ).called(1);
+ });
+
+ when(() => mockBackgroundService.digestFiles([file.path])).thenAnswer(
+ // Invalid hash, length != 20
+ (_) async => [Uint8List.fromList(hash.slice(2).toList())],
+ );
+
+ final result = await sut.hashAssets([mockAsset]);
+
+ verify(() => mockBackgroundService.digestFiles([file.path])).called(1);
+ expect(result, isEmpty);
+ });
+ });
+
+ group("HashService: Batch processing", () {
+ test("processes assets in batches when size limit is reached", () async {
+ // Setup multiple assets with large file sizes
+ final (mock1, mock2, mock3) = await (
+ _createAssetMock(AssetStub.image1),
+ _createAssetMock(AssetStub.image2),
+ _createAssetMock(AssetStub.image3),
+ ).wait;
+
+ final (asset1, file1, deviceAsset1, hash1) = mock1;
+ final (asset2, file2, deviceAsset2, hash2) = mock2;
+ final (asset3, file3, deviceAsset3, hash3) = mock3;
+
+ when(() => mockDeviceAssetRepository.getByIds(any()))
+ .thenAnswer((_) async => []);
+
+ // Setup for multiple batch processing calls
+ when(() => mockBackgroundService.digestFiles([file1.path, file2.path]))
+ .thenAnswer((_) async => [hash1, hash2]);
+ when(() => mockBackgroundService.digestFiles([file3.path]))
+ .thenAnswer((_) async => [hash3]);
+
+ final size = await file1.length() + await file2.length();
+
+ sut = HashService(
+ deviceAssetRepository: mockDeviceAssetRepository,
+ backgroundService: mockBackgroundService,
+ batchSizeLimit: size,
+ );
+ final result = await sut.hashAssets([asset1, asset2, asset3]);
+
+ // Verify multiple batch process calls
+ verify(() => mockBackgroundService.digestFiles([file1.path, file2.path]))
+ .called(1);
+ verify(() => mockBackgroundService.digestFiles([file3.path])).called(1);
+
+ expect(
+ result,
+ [
+ AssetStub.image1.copyWith(checksum: base64.encode(hash1)),
+ AssetStub.image2.copyWith(checksum: base64.encode(hash2)),
+ AssetStub.image3.copyWith(checksum: base64.encode(hash3)),
+ ],
+ );
+ });
+
+ test("processes assets in batches when file limit is reached", () async {
+ // Setup multiple assets with large file sizes
+ final (mock1, mock2, mock3) = await (
+ _createAssetMock(AssetStub.image1),
+ _createAssetMock(AssetStub.image2),
+ _createAssetMock(AssetStub.image3),
+ ).wait;
+
+ final (asset1, file1, deviceAsset1, hash1) = mock1;
+ final (asset2, file2, deviceAsset2, hash2) = mock2;
+ final (asset3, file3, deviceAsset3, hash3) = mock3;
+
+ when(() => mockDeviceAssetRepository.getByIds(any()))
+ .thenAnswer((_) async => []);
+
+ when(() => mockBackgroundService.digestFiles([file1.path]))
+ .thenAnswer((_) async => [hash1]);
+ when(() => mockBackgroundService.digestFiles([file2.path]))
+ .thenAnswer((_) async => [hash2]);
+ when(() => mockBackgroundService.digestFiles([file3.path]))
+ .thenAnswer((_) async => [hash3]);
+
+ sut = HashService(
+ deviceAssetRepository: mockDeviceAssetRepository,
+ backgroundService: mockBackgroundService,
+ batchFileLimit: 1,
+ );
+ final result = await sut.hashAssets([asset1, asset2, asset3]);
+
+ // Verify multiple batch process calls
+ verify(() => mockBackgroundService.digestFiles([file1.path])).called(1);
+ verify(() => mockBackgroundService.digestFiles([file2.path])).called(1);
+ verify(() => mockBackgroundService.digestFiles([file3.path])).called(1);
+
+ expect(
+ result,
+ [
+ AssetStub.image1.copyWith(checksum: base64.encode(hash1)),
+ AssetStub.image2.copyWith(checksum: base64.encode(hash2)),
+ AssetStub.image3.copyWith(checksum: base64.encode(hash3)),
+ ],
+ );
+ });
+
+ test("HashService: Sort & Process different states", () async {
+ final (asset1, file1, deviceAsset1, hash1) =
+ await _createAssetMock(AssetStub.image1); // Will need rehashing
+ final (asset2, file2, deviceAsset2, hash2) =
+ await _createAssetMock(AssetStub.image2); // Will have matching hash
+ final (asset3, file3, deviceAsset3, hash3) =
+ await _createAssetMock(AssetStub.image3); // No DB entry
+ final asset4 =
+ AssetStub.image3.copyWith(localId: "image4"); // Cannot be hashed
+
+ when(() => mockBackgroundService.digestFiles([file1.path, file3.path]))
+ .thenAnswer((_) async => [hash1, hash3]);
+ // DB entries are not sorted and a dummy entry added
+ when(
+ () => mockDeviceAssetRepository.getByIds([
+ AssetStub.image1.localId!,
+ AssetStub.image2.localId!,
+ AssetStub.image3.localId!,
+ asset4.localId!,
+ ]),
+ ).thenAnswer(
+ (_) async => [
+ // Same timestamp to reuse deviceAsset
+ deviceAsset2.copyWith(modifiedTime: asset2.fileModifiedAt),
+ deviceAsset1,
+ deviceAsset3.copyWith(assetId: asset4.localId!),
+ ],
+ );
+
+ final result = await sut.hashAssets([asset1, asset2, asset3, asset4]);
+
+ // Verify correct processing of all assets
+ verify(() => mockBackgroundService.digestFiles([file1.path, file3.path]))
+ .called(1);
+ expect(result.length, 3);
+ expect(result, [
+ AssetStub.image2.copyWith(checksum: base64.encode(hash2)),
+ AssetStub.image1.copyWith(checksum: base64.encode(hash1)),
+ AssetStub.image3.copyWith(checksum: base64.encode(hash3)),
+ ]);
+ });
+
+ group("HashService: Edge cases", () {
+ test("handles empty list of assets", () async {
+ when(() => mockDeviceAssetRepository.getByIds(any()))
+ .thenAnswer((_) async => []);
+
+ final result = await sut.hashAssets([]);
+
+ verifyNever(() => mockBackgroundService.digestFiles(any()));
+ verifyNever(() => mockDeviceAssetRepository.updateAll(any()));
+ verifyNever(() => mockDeviceAssetRepository.deleteIds(any()));
+
+ expect(result, isEmpty);
+ });
+
+ test("handles all file access failures", () async {
+ // No DB entries
+ when(
+ () => mockDeviceAssetRepository.getByIds(
+ [AssetStub.image1.localId!, AssetStub.image2.localId!],
+ ),
+ ).thenAnswer((_) async => []);
+
+ final result = await sut.hashAssets([
+ AssetStub.image1,
+ AssetStub.image2,
+ ]);
+
+ verifyNever(() => mockBackgroundService.digestFiles(any()));
+ verifyNever(() => mockDeviceAssetRepository.updateAll(any()));
+ expect(result, isEmpty);
+ });
+ });
+ });
+}
+
+Future<(Asset, File, DeviceAsset, Uint8List)> _createAssetMock(
+ Asset asset,
+) async {
+ final random = Random();
+ final hash =
+ Uint8List.fromList(List.generate(20, (i) => random.nextInt(255)));
+ final mockAsset = MockAsset();
+ final mockAssetEntity = MockAssetEntity();
+ final fs = MemoryFileSystem();
+ final deviceAsset = DeviceAsset(
+ assetId: asset.localId!,
+ hash: Uint8List.fromList(hash),
+ modifiedTime: DateTime.now(),
+ );
+ final tmp = await fs.systemTempDirectory.createTemp();
+ final file = tmp.childFile("${asset.fileName}-path");
+ await file.writeAsString("${asset.fileName}-content");
+
+ when(() => mockAsset.localId).thenReturn(asset.localId);
+ when(() => mockAsset.fileName).thenReturn(asset.fileName);
+ when(() => mockAsset.fileCreatedAt).thenReturn(asset.fileCreatedAt);
+ when(() => mockAsset.fileModifiedAt).thenReturn(asset.fileModifiedAt);
+ when(() => mockAsset.copyWith(checksum: any(named: "checksum")))
+ .thenReturn(asset.copyWith(checksum: base64.encode(hash)));
+ when(() => mockAsset.local).thenAnswer((_) => mockAssetEntity);
+ when(() => mockAssetEntity.originFile).thenAnswer((_) async => file);
+
+ return (mockAsset, file, deviceAsset, hash);
+}
diff --git a/mobile/test/fixtures/asset.stub.dart b/mobile/test/fixtures/asset.stub.dart
index 26108d63b2..b69b392129 100644
--- a/mobile/test/fixtures/asset.stub.dart
+++ b/mobile/test/fixtures/asset.stub.dart
@@ -8,8 +8,8 @@ final class AssetStub {
localId: "image1",
remoteId: 'image1-remote',
ownerId: 1,
- fileCreatedAt: DateTime.now(),
- fileModifiedAt: DateTime.now(),
+ fileCreatedAt: DateTime(2019),
+ fileModifiedAt: DateTime(2020),
updatedAt: DateTime.now(),
durationInSeconds: 0,
type: AssetType.image,
@@ -34,4 +34,19 @@ final class AssetStub {
isArchived: false,
isTrashed: false,
);
+
+ static final image3 = Asset(
+ checksum: "image3-checksum",
+ localId: "image3",
+ ownerId: 1,
+ fileCreatedAt: DateTime(2025),
+ fileModifiedAt: DateTime(2025),
+ updatedAt: DateTime.now(),
+ durationInSeconds: 60,
+ type: AssetType.image,
+ fileName: "image3.jpg",
+ isFavorite: true,
+ isArchived: false,
+ isTrashed: false,
+ );
}
diff --git a/mobile/test/infrastructure/repository.mock.dart b/mobile/test/infrastructure/repository.mock.dart
index c9287bfb1c..192858adff 100644
--- a/mobile/test/infrastructure/repository.mock.dart
+++ b/mobile/test/infrastructure/repository.mock.dart
@@ -1,3 +1,4 @@
+import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart';
import 'package:immich_mobile/domain/interfaces/log.interface.dart';
import 'package:immich_mobile/domain/interfaces/store.interface.dart';
import 'package:immich_mobile/domain/interfaces/user.interface.dart';
@@ -10,5 +11,8 @@ class MockLogRepository extends Mock implements ILogRepository {}
class MockUserRepository extends Mock implements IUserRepository {}
+class MockDeviceAssetRepository extends Mock
+ implements IDeviceAssetRepository {}
+
// API Repos
class MockUserApiRepository extends Mock implements IUserApiRepository {}
diff --git a/mobile/test/service.mocks.dart b/mobile/test/service.mocks.dart
index 33c325b105..8ee1c58609 100644
--- a/mobile/test/service.mocks.dart
+++ b/mobile/test/service.mocks.dart
@@ -1,4 +1,5 @@
import 'package:immich_mobile/services/api.service.dart';
+import 'package:immich_mobile/services/background.service.dart';
import 'package:immich_mobile/services/entity.service.dart';
import 'package:immich_mobile/services/hash.service.dart';
import 'package:immich_mobile/services/network.service.dart';
@@ -17,3 +18,5 @@ class MockEntityService extends Mock implements EntityService {}
class MockNetworkService extends Mock implements NetworkService {}
class MockSearchApi extends Mock implements SearchApi {}
+
+class MockBackgroundService extends Mock implements BackgroundService {}
diff --git a/mobile/test/test_utils.dart b/mobile/test/test_utils.dart
index a5a89a2440..c0f789795c 100644
--- a/mobile/test/test_utils.dart
+++ b/mobile/test/test_utils.dart
@@ -12,6 +12,7 @@ import 'package:immich_mobile/entities/backup_album.entity.dart';
import 'package:immich_mobile/entities/duplicated_asset.entity.dart';
import 'package:immich_mobile/entities/etag.entity.dart';
import 'package:immich_mobile/entities/ios_device_asset.entity.dart';
+import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart';
import 'package:immich_mobile/infrastructure/entities/exif.entity.dart';
import 'package:immich_mobile/infrastructure/entities/log.entity.dart';
import 'package:immich_mobile/infrastructure/entities/store.entity.dart';
@@ -52,6 +53,7 @@ abstract final class TestUtils {
ETagSchema,
AndroidDeviceAssetSchema,
IOSDeviceAssetSchema,
+ DeviceAssetEntitySchema,
],
directory: "test/",
maxSizeMiB: 1024,