High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor: hashing service (#21997)
shenlong532ec10d5f8d
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.ktindex 9c618d9ed0..28400c803f 100644--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt@@ -209,6 +209,40 @@ data class SyncDelta (override fun hashCode(): Int = toList().hashCode()}++/** Generated class from Pigeon that represents data sent in messages. */+data class HashResult (+ val assetId: String,+ val error: String? = null,+ val hash: String? = null+)+ {+ companion object {+ fun fromList(pigeonVar_list: List<Any?>): HashResult {+ val assetId = pigeonVar_list[0] as String+ val error = pigeonVar_list[1] as String?+ val hash = pigeonVar_list[2] as String?+ return HashResult(assetId, error, hash)+ }+ }+ fun toList(): List<Any?> {+ return listOf(+ assetId,+ error,+ hash,+ )+ }+ override fun equals(other: Any?): Boolean {+ if (other !is HashResult) {+ return false+ }+ if (this === other) {+ return true+ }+ return MessagesPigeonUtils.deepEquals(toList(), other.toList()) }++ override fun hashCode(): Int = toList().hashCode()+}private open class MessagesPigeonCodec : StandardMessageCodec() {override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? {return when (type) {@@ -227,6 +261,11 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {SyncDelta.fromList(it)}}+ 132.toByte() -> {+ return (readValue(buffer) as? List<Any?>)?.let {+ HashResult.fromList(it)+ }+ }else -> super.readValueOfType(type, buffer)}}@@ -244,11 +283,16 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {stream.write(131)writeValue(stream, value.toList())}+ is HashResult -> {+ stream.write(132)+ writeValue(stream, value.toList())+ }else -> super.writeValue(stream, value)}}}+/** Generated interface from Pigeon that represents a handler of messages from Flutter. */interface NativeSyncApi {fun shouldFullSync(): Boolean@@ -259,7 +303,8 @@ interface NativeSyncApi {fun getAlbums(): List<PlatformAlbum>fun getAssetsCountSince(albumId: String, timestamp: Long): Longfun getAssetsForAlbum(albumId: String, updatedTimeCond: Long?): List<PlatformAsset>- fun hashPaths(paths: List<String>): List<ByteArray?>+ fun hashAssets(assetIds: List<String>, allowNetworkAccess: Boolean, callback: (Result<List<HashResult>>) -> Unit)+ fun cancelHashing()companion object {/** The codec used by NativeSyncApi. */@@ -402,13 +447,33 @@ interface NativeSyncApi {}}run {- val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$separatedMessageChannelSuffix", codec, taskQueue)+ val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets$separatedMessageChannelSuffix", codec, taskQueue)if (api != null) {channel.setMessageHandler { message, reply ->val args = message as List<Any?>- val pathsArg = args[0] as List<String>+ val assetIdsArg = args[0] as List<String>+ val allowNetworkAccessArg = args[1] as Boolean+ api.hashAssets(assetIdsArg, allowNetworkAccessArg) { result: Result<List<HashResult>> ->+ val error = result.exceptionOrNull()+ if (error != null) {+ reply.reply(MessagesPigeonUtils.wrapError(error))+ } else {+ val data = result.getOrNull()+ reply.reply(MessagesPigeonUtils.wrapResult(data))+ }+ }+ }+ } else {+ channel.setMessageHandler(null)+ }+ }+ run {+ val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing$separatedMessageChannelSuffix", codec)+ if (api != null) {+ channel.setMessageHandler { _, reply ->val wrapped: List<Any?> = try {- listOf(api.hashPaths(pathsArg))+ api.cancelHashing()+ listOf(null)} catch (exception: Throwable) {MessagesPigeonUtils.wrapError(exception)}diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.ktindex b2ceb8a9f2..868f3c6cdd 100644--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt@@ -1,14 +1,25 @@package app.alextran.immich.syncimport android.annotation.SuppressLint+import android.content.ContentUrisimport android.content.Contextimport android.database.Cursorimport android.provider.MediaStore-import android.util.Log+import android.util.Base64import androidx.core.database.getStringOrNull+import kotlinx.coroutines.CoroutineScope+import kotlinx.coroutines.Dispatchers+import kotlinx.coroutines.Job+import kotlinx.coroutines.async+import kotlinx.coroutines.awaitAll+import kotlinx.coroutines.ensureActive+import kotlinx.coroutines.launch+import kotlinx.coroutines.sync.Semaphore+import kotlinx.coroutines.sync.withPermitimport java.io.File-import java.io.FileInputStreamimport java.security.MessageDigest+import kotlin.coroutines.cancellation.CancellationException+import kotlin.coroutines.coroutineContextsealed class AssetResult {data class ValidAsset(val asset: PlatformAsset, val albumId: String) : AssetResult()@@ -19,8 +30,12 @@ sealed class AssetResult {open class NativeSyncApiImplBase(context: Context) {private val ctx: Context = context.applicationContext+ private var hashTask: Job? = null+companion object {- private const val TAG = "NativeSyncApiImplBase"+ private const val MAX_CONCURRENT_HASH_OPERATIONS = 16+ private val hashSemaphore = Semaphore(MAX_CONCURRENT_HASH_OPERATIONS)+ private const val HASHING_CANCELLED_CODE = "HASH_CANCELLED"const val MEDIA_SELECTION ="(${MediaStore.Files.FileColumns.MEDIA_TYPE} = ? OR ${MediaStore.Files.FileColumns.MEDIA_TYPE} = ?)"@@ -215,23 +230,74 @@ open class NativeSyncApiImplBase(context: Context) {.toList()}- fun hashPaths(paths: List<String>): List<ByteArray?> {- val buffer = ByteArray(HASH_BUFFER_SIZE)- val digest = MessageDigest.getInstance("SHA-1")+ fun hashAssets(+ assetIds: List<String>,+ // allowNetworkAccess is only used on the iOS implementation+ @Suppress("UNUSED_PARAMETER") allowNetworkAccess: Boolean,+ callback: (Result<List<HashResult>>) -> Unit+ ) {+ if (assetIds.isEmpty()) {+ callback(Result.success(emptyList()))+ return+ }- return paths.map { path ->+ hashTask?.cancel()+ hashTask = CoroutineScope(Dispatchers.IO).launch {try {- FileInputStream(path).use { file ->- var bytesRead: Int- while (file.read(buffer).also { bytesRead = it } > 0) {- digest.update(buffer, 0, bytesRead)+ val results = assetIds.map { assetId ->+ async {+ hashSemaphore.withPermit {+ ensureActive()+ hashAsset(assetId)+ }}- }- digest.digest()+ }.awaitAll()++ callback(Result.success(results))+ } catch (e: CancellationException) {+ callback(+ Result.failure(+ FlutterError(+ HASHING_CANCELLED_CODE,+ "Hashing operation was cancelled",+ null+ )+ )+ )} catch (e: Exception) {- Log.w(TAG, "Failed to hash file $path: $e")- null+ callback(Result.failure(e))}}}++ private suspend fun hashAsset(assetId: String): HashResult {+ return try {+ val assetUri = ContentUris.withAppendedId(+ MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL),+ assetId.toLong()+ )++ val digest = MessageDigest.getInstance("SHA-1")+ ctx.contentResolver.openInputStream(assetUri)?.use { inputStream ->+ var bytesRead: Int+ val buffer = ByteArray(HASH_BUFFER_SIZE)+ while (inputStream.read(buffer).also { bytesRead = it } > 0) {+ coroutineContext.ensureActive()+ digest.update(buffer, 0, bytesRead)+ }+ } ?: return HashResult(assetId, "Cannot open input stream for asset", null)++ val hashString = Base64.encodeToString(digest.digest(), Base64.NO_WRAP)+ HashResult(assetId, null, hashString)+ } catch (e: SecurityException) {+ HashResult(assetId, "Permission denied accessing asset: ${e.message}", null)+ } catch (e: Exception) {+ HashResult(assetId, "Failed to hash asset: ${e.message}", null)+ }+ }++ fun cancelHashing() {+ hashTask?.cancel()+ hashTask = null+ }}diff --git a/mobile/ios/.gitignore b/mobile/ios/.gitignoreindex f312f249a3..e32cadbf68 100644--- a/mobile/ios/.gitignore+++ b/mobile/ios/.gitignore@@ -4,7 +4,6 @@*.moved-aside*.pbxuser*.perspectivev3-**/*sync/.sconsign.dblite.tags***/.vagrant/diff --git a/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift b/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swiftindex ca2453404c..f7f8f69989 100644--- a/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift+++ b/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift@@ -5,7 +5,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi {func enable() throws {BackgroundWorkerApiImpl.scheduleRefreshWorker()BackgroundWorkerApiImpl.scheduleProcessingWorker()- print("BackgroundUploadImpl:enbale Background worker scheduled")+ print("BackgroundWorkerApiImpl:enable Background worker scheduled")}func configure(settings: BackgroundWorkerSettings) throws {@@ -15,7 +15,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi {func disable() throws {BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.refreshTaskID);BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.processingTaskID);- print("BackgroundUploadImpl:disableUploadWorker Disabled background workers")+ print("BackgroundWorkerApiImpl:disableUploadWorker Disabled background workers")}private static let refreshTaskID = "app.alextran.immich.background.refreshUpload"diff --git a/mobile/ios/Runner/Sync/Messages.g.swift b/mobile/ios/Runner/Sync/Messages.g.swiftindex 19f4384672..305aca5266 100644--- a/mobile/ios/Runner/Sync/Messages.g.swift+++ b/mobile/ios/Runner/Sync/Messages.g.swift@@ -267,6 +267,39 @@ struct SyncDelta: Hashable {}}+/// Generated class from Pigeon that represents data sent in messages.+struct HashResult: Hashable {+ var assetId: String+ var error: String? = nil+ var hash: String? = nil+++ // swift-format-ignore: AlwaysUseLowerCamelCase+ static func fromList(_ pigeonVar_list: [Any?]) -> HashResult? {+ let assetId = pigeonVar_list[0] as! String+ let error: String? = nilOrValue(pigeonVar_list[1])+ let hash: String? = nilOrValue(pigeonVar_list[2])++ return HashResult(+ assetId: assetId,+ error: error,+ hash: hash+ )+ }+ func toList() -> [Any?] {+ return [+ assetId,+ error,+ hash,+ ]+ }+ static func == (lhs: HashResult, rhs: HashResult) -> Bool {+ return deepEqualsMessages(lhs.toList(), rhs.toList()) }+ func hash(into hasher: inout Hasher) {+ deepHashMessages(value: toList(), hasher: &hasher)+ }+}+private class MessagesPigeonCodecReader: FlutterStandardReader {override func readValue(ofType type: UInt8) -> Any? {switch type {@@ -276,6 +309,8 @@ private class MessagesPigeonCodecReader: FlutterStandardReader {return PlatformAlbum.fromList(self.readValue() as! [Any?])case 131:return SyncDelta.fromList(self.readValue() as! [Any?])+ case 132:+ return HashResult.fromList(self.readValue() as! [Any?])default:return super.readValue(ofType: type)}@@ -293,6 +328,9 @@ private class MessagesPigeonCodecWriter: FlutterStandardWriter {} else if let value = value as? SyncDelta {super.writeByte(131)super.writeValue(value.toList())+ } else if let value = value as? HashResult {+ super.writeByte(132)+ super.writeValue(value.toList())} else {super.writeValue(value)}@@ -313,6 +351,7 @@ class MessagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {static let shared = MessagesPigeonCodec(readerWriter: MessagesPigeonCodecReaderWriter())}+/// Generated protocol from Pigeon that represents a handler of messages from Flutter.protocol NativeSyncApi {func shouldFullSync() throws -> Bool@@ -323,7 +362,8 @@ protocol NativeSyncApi {func getAlbums() throws -> [PlatformAlbum]func getAssetsCountSince(albumId: String, timestamp: Int64) throws -> Int64func getAssetsForAlbum(albumId: String, updatedTimeCond: Int64?) throws -> [PlatformAsset]- func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?]+ func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void)+ func cancelHashing() throws}/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.@@ -459,22 +499,38 @@ class NativeSyncApiSetup {} else {getAssetsForAlbumChannel.setMessageHandler(nil)}- let hashPathsChannel = taskQueue == nil- ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)- : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue)+ let hashAssetsChannel = taskQueue == nil+ ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)+ : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue)if let api = api {- hashPathsChannel.setMessageHandler { message, reply in+ hashAssetsChannel.setMessageHandler { message, reply inlet args = message as! [Any?]- let pathsArg = args[0] as! [String]+ let assetIdsArg = args[0] as! [String]+ let allowNetworkAccessArg = args[1] as! Bool+ api.hashAssets(assetIds: assetIdsArg, allowNetworkAccess: allowNetworkAccessArg) { result in+ switch result {+ case .success(let res):+ reply(wrapResult(res))+ case .failure(let error):+ reply(wrapError(error))+ }+ }+ }+ } else {+ hashAssetsChannel.setMessageHandler(nil)+ }+ let cancelHashingChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)+ if let api = api {+ cancelHashingChannel.setMessageHandler { _, reply indo {- let result = try api.hashPaths(paths: pathsArg)- reply(wrapResult(result))+ try api.cancelHashing()+ reply(wrapResult(nil))} catch {reply(wrapError(error))}}} else {- hashPathsChannel.setMessageHandler(nil)+ cancelHashingChannel.setMessageHandler(nil)}}}diff --git a/mobile/ios/Runner/Sync/MessagesImpl.swift b/mobile/ios/Runner/Sync/MessagesImpl.swiftindex 2810dee7c1..bb23bae6b6 100644--- a/mobile/ios/Runner/Sync/MessagesImpl.swift+++ b/mobile/ios/Runner/Sync/MessagesImpl.swift@@ -17,30 +17,16 @@ struct AssetWrapper: Hashable, Equatable {}}-extension PHAsset {- func toPlatformAsset() -> PlatformAsset {- return PlatformAsset(- id: localIdentifier,- name: title(),- type: Int64(mediaType.rawValue),- createdAt: creationDate.map { Int64($0.timeIntervalSince1970) },- updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) },- width: Int64(pixelWidth),- height: Int64(pixelHeight),- durationInSeconds: Int64(duration),- orientation: 0,- isFavorite: isFavorite- )- }-}-class NativeSyncApiImpl: NativeSyncApi {private let defaults: UserDefaultsprivate let changeTokenKey = "immich:changeToken"private let albumTypes: [PHAssetCollectionType] = [.album, .smartAlbum]private let recoveredAlbumSubType = 1000000219- private let hashBufferSize = 2 * 1024 * 1024+ private var hashTask: Task<Void, Error>?+ private static let hashCancelledCode = "HASH_CANCELLED"+ private static let hashCancelled = Result<[HashResult], Error>.failure(PigeonError(code: hashCancelledCode, message: "Hashing cancelled", details: nil))+init(with defaults: UserDefaults = .standard) {self.defaults = defaults@@ -96,7 +82,7 @@ class NativeSyncApiImpl: NativeSyncApi {let collections = PHAssetCollection.fetchAssetCollections(with: type, subtype: .any, options: nil)for i in 0..<collections.count {let album = collections.object(at: i)-+// Ignore recovered albumif(album.assetCollectionSubtype.rawValue == self.recoveredAlbumSubType) {continue;@@ -254,7 +240,7 @@ class NativeSyncApiImpl: NativeSyncApi {let date = NSDate(timeIntervalSince1970: TimeInterval(updatedTimeCond!))options.predicate = NSPredicate(format: "creationDate > %@ OR modificationDate > %@", date, date)}-+let result = PHAsset.fetchAssets(in: album, options: options)if(result.count == 0) {return []@@ -267,23 +253,114 @@ class NativeSyncApiImpl: NativeSyncApi {return assets}- func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] {- return paths.map { path in- guard let file = FileHandle(forReadingAtPath: path) else {- print("Cannot open file: \(path)")- return nil+ func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void) {+ if let prevTask = hashTask {+ prevTask.cancel()+ hashTask = nil+ }+ hashTask = Task { [weak self] in+ var missingAssetIds = Set(assetIds)+ var assets = [PHAsset]()+ assets.reserveCapacity(assetIds.count)+ PHAsset.fetchAssets(withLocalIdentifiers: assetIds, options: nil).enumerateObjects { (asset, _, stop) in+ if Task.isCancelled {+ stop.pointee = true+ return+ }+ missingAssetIds.remove(asset.localIdentifier)+ assets.append(asset)+ }++ if Task.isCancelled {+ return completion(Self.hashCancelled)+ }++ await withTaskGroup(of: HashResult?.self) { taskGroup in+ var results = [HashResult]()+ results.reserveCapacity(assets.count)+ for asset in assets {+ if Task.isCancelled {+ return completion(Self.hashCancelled)}-- var hasher = Insecure.SHA1()- while autoreleasepool(invoking: {- let chunk = file.readData(ofLength: hashBufferSize)- guard !chunk.isEmpty else { return false }- hasher.update(data: chunk)- return true- }) { }-- let digest = hasher.finalize()- return FlutterStandardTypedData(bytes: Data(digest))+ taskGroup.addTask {+ guard let self = self else { return nil }+ return await self.hashAsset(asset, allowNetworkAccess: allowNetworkAccess)+ }+ }++ for await result in taskGroup {+ guard let result = result else {+ return completion(Self.hashCancelled)+ }+ results.append(result)+ }++ for missing in missingAssetIds {+ results.append(HashResult(assetId: missing, error: "Asset not found in library", hash: nil))+ }++ completion(.success(results))+ }+ }+ }++ func cancelHashing() {+ hashTask?.cancel()+ hashTask = nil+ }++ private func hashAsset(_ asset: PHAsset, allowNetworkAccess: Bool) async -> HashResult? {+ class RequestRef {+ var id: PHAssetResourceDataRequestID?+ }+ let requestRef = RequestRef()+ return await withTaskCancellationHandler(operation: {+ if Task.isCancelled {+ return nil+ }++ guard let resource = asset.getResource() else {+ return HashResult(assetId: asset.localIdentifier, error: "Cannot get asset resource", hash: nil)+ }++ if Task.isCancelled {+ return nil+ }++ let options = PHAssetResourceRequestOptions()+ options.isNetworkAccessAllowed = allowNetworkAccess++ return await withCheckedContinuation { continuation in+ var hasher = Insecure.SHA1()++ requestRef.id = PHAssetResourceManager.default().requestData(+ for: resource,+ options: options,+ dataReceivedHandler: { data in+ hasher.update(data: data)+ },+ completionHandler: { error in+ let result: HashResult? = switch (error) {+ case let e as PHPhotosError where e.code == .userCancelled: nil+ case let .some(e): HashResult(+ assetId: asset.localIdentifier,+ error: "Failed to hash asset: \(e.localizedDescription)",+ hash: nil+ )+ case .none:+ HashResult(+ assetId: asset.localIdentifier,+ error: nil,+ hash: Data(hasher.finalize()).base64EncodedString()+ )+ }+ continuation.resume(returning: result)+ }+ )}+ }, onCancel: {+ guard let requestId = requestRef.id else { return }+ PHAssetResourceManager.default().cancelDataRequest(requestId)+ })}}diff --git a/mobile/ios/Runner/Sync/PHAssetExtensions.swift b/mobile/ios/Runner/Sync/PHAssetExtensions.swiftnew file mode 100644index 0000000000..2b1ef6ac88--- /dev/null+++ b/mobile/ios/Runner/Sync/PHAssetExtensions.swift@@ -0,0 +1,77 @@+import Photos++extension PHAsset {+ func toPlatformAsset() -> PlatformAsset {+ return PlatformAsset(+ id: localIdentifier,+ name: title,+ type: Int64(mediaType.rawValue),+ createdAt: creationDate.map { Int64($0.timeIntervalSince1970) },+ updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) },+ width: Int64(pixelWidth),+ height: Int64(pixelHeight),+ durationInSeconds: Int64(duration),+ orientation: 0,+ isFavorite: isFavorite+ )+ }++ var title: String {+ return filename ?? originalFilename ?? "<unknown>"+ }++ var filename: String? {+ return value(forKey: "filename") as? String+ }++ // This method is expected to be slow as it goes through the asset resources to fetch the originalFilename+ var originalFilename: String? {+ return getResource()?.originalFilename+ }++ func getResource() -> PHAssetResource? {+ let resources = PHAssetResource.assetResources(for: self)++ let filteredResources = resources.filter { $0.isMediaResource && isValidResourceType($0.type) }++ guard !filteredResources.isEmpty else {+ return nil+ }++ if filteredResources.count == 1 {+ return filteredResources.first+ }++ if let currentResource = filteredResources.first(where: { $0.isCurrent }) {+ return currentResource+ }++ if let fullSizeResource = filteredResources.first(where: { isFullSizeResourceType($0.type) }) {+ return fullSizeResource+ }++ return nil+ }++ private func isValidResourceType(_ type: PHAssetResourceType) -> Bool {+ switch mediaType {+ case .image:+ return [.photo, .alternatePhoto, .fullSizePhoto].contains(type)+ case .video:+ return [.video, .fullSizeVideo, .fullSizePairedVideo].contains(type)+ default:+ return false+ }+ }++ private func isFullSizeResourceType(_ type: PHAssetResourceType) -> Bool {+ switch mediaType {+ case .image:+ return type == .fullSizePhoto+ case .video:+ return type == .fullSizeVideo+ default:+ return false+ }+ }+}diff --git a/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift b/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swiftnew file mode 100644index 0000000000..699d55a98d--- /dev/null+++ b/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift@@ -0,0 +1,16 @@++import Photos++extension PHAssetResource {+ var isCurrent: Bool {+ return value(forKey: "isCurrent") as? Bool ?? false+ }++ var isMediaResource: Bool {+ var isMedia = type != .adjustmentData+ if #available(iOS 17, *) {+ isMedia = isMedia && type != .photoProxy+ }+ return isMedia+ }+}diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dartindex 60c9b92cf9..7429616f14 100644--- a/mobile/lib/constants/constants.dart+++ b/mobile/lib/constants/constants.dart@@ -1,3 +1,5 @@+import 'dart:io';+const int noDbId = -9223372036854775808; // from Isarconst double downloadCompleted = -1;const double downloadFailed = -2;@@ -10,7 +12,7 @@ const int kSyncEventBatchSize = 5000;const int kFetchLocalAssetsBatchSize = 40000;// Hash batch limits-const int kBatchHashFileLimit = 256;+final int kBatchHashFileLimit = Platform.isIOS ? 32 : 512;const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB// Secure storage keysdiff --git a/mobile/lib/domain/services/background_worker.service.dart b/mobile/lib/domain/services/background_worker.service.dartindex d57fe507d9..78dd1e980f 100644--- a/mobile/lib/domain/services/background_worker.service.dart+++ b/mobile/lib/domain/services/background_worker.service.dart@@ -184,6 +184,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {try {final backgroundSyncManager = _ref.read(backgroundSyncProvider);+ final nativeSyncApi = _ref.read(nativeSyncApiProvider);_isCleanedUp = true;_ref.dispose();@@ -199,7 +200,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {_drift.close(),_driftLogger.close(),backgroundSyncManager.cancel(),- backgroundSyncManager.cancelLocal(),+ nativeSyncApi.cancelHashing(),];if (_isar.isOpen) {diff --git a/mobile/lib/domain/services/hash.service.dart b/mobile/lib/domain/services/hash.service.dartindex 8044b298d3..90f29b8bc1 100644--- a/mobile/lib/domain/services/hash.service.dart+++ b/mobile/lib/domain/services/hash.service.dart@@ -1,20 +1,18 @@-import 'dart:convert';-+import 'package:flutter/services.dart';import 'package:immich_mobile/constants/constants.dart';import 'package:immich_mobile/domain/models/album/local_album.model.dart';import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';-import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';import 'package:immich_mobile/platform/native_sync_api.g.dart';import 'package:logging/logging.dart';+const String _kHashCancelledCode = "HASH_CANCELLED";+class HashService {- final int batchSizeLimit;- final int batchFileLimit;+ final int _batchSize;final DriftLocalAlbumRepository _localAlbumRepository;final DriftLocalAssetRepository _localAssetRepository;- final StorageRepository _storageRepository;final NativeSyncApi _nativeSyncApi;final bool Function()? _cancelChecker;final _log = Logger('HashService');@@ -22,37 +20,42 @@ class HashService {HashService({required DriftLocalAlbumRepository localAlbumRepository,required DriftLocalAssetRepository localAssetRepository,- required StorageRepository storageRepository,required NativeSyncApi nativeSyncApi,bool Function()? cancelChecker,- this.batchSizeLimit = kBatchHashSizeLimit,- this.batchFileLimit = kBatchHashFileLimit,+ int? batchSize,}) : _localAlbumRepository = localAlbumRepository,_localAssetRepository = localAssetRepository,- _storageRepository = storageRepository,_cancelChecker = cancelChecker,- _nativeSyncApi = nativeSyncApi;+ _nativeSyncApi = nativeSyncApi,+ _batchSize = batchSize ?? kBatchHashFileLimit;bool get isCancelled => _cancelChecker?.call() ?? false;Future<void> hashAssets() async {_log.info("Starting hashing of assets");final Stopwatch stopwatch = Stopwatch()..start();- // Sorted by backupSelection followed by isCloud- final localAlbums = await _localAlbumRepository.getAll(- sortBy: {SortLocalAlbumsBy.backupSelection, SortLocalAlbumsBy.isIosSharedAlbum},- );-- for (final album in localAlbums) {- if (isCancelled) {- _log.warning("Hashing cancelled. Stopped processing albums.");- break;+ try {+ // Sorted by backupSelection followed by isCloud+ final localAlbums = await _localAlbumRepository.getBackupAlbums();++ for (final album in localAlbums) {+ if (isCancelled) {+ _log.warning("Hashing cancelled. Stopped processing albums.");+ break;+ }++ final assetsToHash = await _localAlbumRepository.getAssetsToHash(album.id);+ if (assetsToHash.isNotEmpty) {+ await _hashAssets(album, assetsToHash);+ }}-- final assetsToHash = await _localAlbumRepository.getAssetsToHash(album.id);- if (assetsToHash.isNotEmpty) {- await _hashAssets(album, assetsToHash);+ } on PlatformException catch (e) {+ if (e.code == _kHashCancelledCode) {+ _log.warning("Hashing cancelled by platform");+ return;}+ } catch (e, s) {+ _log.severe("Error during hashing", e, s);}stopwatch.stop();@@ -63,8 +66,7 @@ class HashService {/// with hash for those that were successfully hashed. Hashes are looked up in a table/// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB.Future<void> _hashAssets(LocalAlbum album, List<LocalAsset> assetsToHash) async {- int bytesProcessed = 0;- final toHash = <_AssetToPath>[];+ final toHash = <String, LocalAsset>{};for (final asset in assetsToHash) {if (isCancelled) {@@ -72,21 +74,10 @@ class HashService {return;}- final file = await _storageRepository.getFileForAsset(asset.id);- if (file == null) {- _log.warning(- "Cannot get file for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt} from album: ${album.name}",- );- continue;- }-- bytesProcessed += await file.length();- toHash.add(_AssetToPath(asset: asset, path: file.path));-- if (toHash.length >= batchFileLimit || bytesProcessed >= batchSizeLimit) {+ toHash[asset.id] = asset;+ if (toHash.length == _batchSize) {await _processBatch(album, toHash);toHash.clear();- bytesProcessed = 0;}}@@ -94,33 +85,36 @@ class HashService {}/// Processes a batch of assets.- Future<void> _processBatch(LocalAlbum album, List<_AssetToPath> toHash) async {+ Future<void> _processBatch(LocalAlbum album, Map<String, LocalAsset> toHash) async {if (toHash.isEmpty) {return;}_log.fine("Hashing ${toHash.length} files");- final hashed = <LocalAsset>[];- final hashes = await _nativeSyncApi.hashPaths(toHash.map((e) => e.path).toList());+ final hashed = <String, String>{};+ final hashResults = await _nativeSyncApi.hashAssets(+ toHash.keys.toList(),+ allowNetworkAccess: album.backupSelection == BackupSelection.selected,+ );assert(- hashes.length == toHash.length,- "Hashes length does not match toHash length: ${hashes.length} != ${toHash.length}",+ hashResults.length == toHash.length,+ "Hashes length does not match toHash length: ${hashResults.length} != ${toHash.length}",);- for (int i = 0; i < hashes.length; i++) {+ for (int i = 0; i < hashResults.length; i++) {if (isCancelled) {_log.warning("Hashing cancelled. Stopped processing batch.");return;}- final hash = hashes[i];- final asset = toHash[i].asset;- if (hash?.length == 20) {- hashed.add(asset.copyWith(checksum: base64.encode(hash!)));+ final hashResult = hashResults[i];+ if (hashResult.hash != null) {+ hashed[hashResult.assetId] = hashResult.hash!;} else {+ final asset = toHash[hashResult.assetId];_log.warning(- "Failed to hash file for ${asset.id}: ${asset.name} created at ${asset.createdAt} from album: ${album.name}",+ "Failed to hash asset with id: ${hashResult.assetId}, name: ${asset?.name}, createdAt: ${asset?.createdAt}, from album: ${album.name}. Error: ${hashResult.error ?? "unknown"}",);}}@@ -128,13 +122,5 @@ class HashService {_log.fine("Hashed ${hashed.length}/${toHash.length} assets");await _localAssetRepository.updateHashes(hashed);- await _storageRepository.clearCache();}}--class _AssetToPath {- final LocalAsset asset;- final String path;-- const _AssetToPath({required this.asset, required this.path});-}diff --git a/mobile/lib/infrastructure/repositories/local_asset.repository.dart b/mobile/lib/infrastructure/repositories/local_asset.repository.dartindex 05c8e06678..c4f5221966 100644--- a/mobile/lib/infrastructure/repositories/local_asset.repository.dart+++ b/mobile/lib/infrastructure/repositories/local_asset.repository.dart@@ -36,17 +36,17 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository {Stream<LocalAsset?> watch(String id) => _assetSelectable(id).watchSingleOrNull();- Future<void> updateHashes(Iterable<LocalAsset> hashes) {+ Future<void> updateHashes(Map<String, String> hashes) {if (hashes.isEmpty) {return Future.value();}return _db.batch((batch) async {- for (final asset in hashes) {+ for (final entry in hashes.entries) {batch.update(_db.localAssetEntity,- LocalAssetEntityCompanion(checksum: Value(asset.checksum)),- where: (e) => e.id.equals(asset.id),+ LocalAssetEntityCompanion(checksum: Value(entry.value)),+ where: (e) => e.id.equals(entry.key),);}});diff --git a/mobile/lib/pages/backup/drift_backup_album_selection.page.dart b/mobile/lib/pages/backup/drift_backup_album_selection.page.dartindex 8cfdd2db95..368341f24a 100644--- a/mobile/lib/pages/backup/drift_backup_album_selection.page.dart+++ b/mobile/lib/pages/backup/drift_backup_album_selection.page.dart@@ -1,3 +1,4 @@+import 'dart:async';import 'dart:io';import 'package:auto_route/auto_route.dart';@@ -5,12 +6,14 @@ import 'package:easy_localization/easy_localization.dart';import 'package:flutter/material.dart';import 'package:hooks_riverpod/hooks_riverpod.dart';import 'package:immich_mobile/domain/models/album/local_album.model.dart';+import 'package:immich_mobile/domain/services/sync_linked_album.service.dart';import 'package:immich_mobile/extensions/build_context_extensions.dart';import 'package:immich_mobile/extensions/translate_extensions.dart';import 'package:immich_mobile/providers/app_settings.provider.dart';-import 'package:immich_mobile/domain/services/sync_linked_album.service.dart';+import 'package:immich_mobile/providers/background_sync.provider.dart';import 'package:immich_mobile/providers/backup/backup_album.provider.dart';import 'package:immich_mobile/providers/backup/drift_backup.provider.dart';+import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';import 'package:immich_mobile/providers/user.provider.dart';import 'package:immich_mobile/services/app_settings.service.dart';import 'package:immich_mobile/widgets/backup/drift_album_info_list_tile.dart';@@ -64,16 +67,6 @@ class _DriftBackupAlbumSelectionPageState extends ConsumerState<DriftBackupAlbum});await _handleLinkedAlbumFuture;}-- // Restart backup if total count changed and backup is enabled- final currentTotalAssetCount = ref.read(driftBackupProvider.select((p) => p.totalCount));- final totalChanged = currentTotalAssetCount != _initialTotalAssetCount;- final isBackupEnabled = ref.read(appSettingsServiceProvider).getSetting(AppSettingsEnum.enableBackup);-- if (totalChanged && isBackupEnabled) {- await ref.read(driftBackupProvider.notifier).cancel();- await ref.read(driftBackupProvider.notifier).startBackup(user.id);- }}@override@@ -102,6 +95,27 @@ class _DriftBackupAlbumSelectionPageState extends ConsumerState<DriftBackupAlbumonPopInvokedWithResult: (didPop, _) async {if (!didPop) {await _handlePagePopped();++ final user = ref.read(currentUserProvider);+ if (user == null) {+ return;+ }++ final isBackupEnabled = ref.read(appSettingsServiceProvider).getSetting(AppSettingsEnum.enableBackup);+ await ref.read(driftBackupProvider.notifier).getBackupStatus(user.id);+ final currentTotalAssetCount = ref.read(driftBackupProvider.select((p) => p.totalCount));+ final totalChanged = currentTotalAssetCount != _initialTotalAssetCount;+ final backupNotifier = ref.read(driftBackupProvider.notifier);+ final backgroundSync = ref.read(backgroundSyncProvider);+ final nativeSync = ref.read(nativeSyncApiProvider);+ if (totalChanged) {+ // Waits for hashing to be cancelled before starting a new one+ unawaited(nativeSync.cancelHashing().whenComplete(() => backgroundSync.hashAssets()));+ if (isBackupEnabled) {+ unawaited(backupNotifier.cancel().whenComplete(() => backupNotifier.startBackup(user.id)));+ }+ }+Navigator.of(context).pop();}},diff --git a/mobile/lib/platform/native_sync_api.g.dart b/mobile/lib/platform/native_sync_api.g.dartindex 6fc96f5046..01237f8c19 100644--- a/mobile/lib/platform/native_sync_api.g.dart+++ b/mobile/lib/platform/native_sync_api.g.dart@@ -205,6 +205,45 @@ class SyncDelta {int get hashCode => Object.hashAll(_toList());}+class HashResult {+ HashResult({required this.assetId, this.error, this.hash});++ String assetId;++ String? error;++ String? hash;++ List<Object?> _toList() {+ return <Object?>[assetId, error, hash];+ }++ Object encode() {+ return _toList();+ }++ static HashResult decode(Object result) {+ result as List<Object?>;+ return HashResult(assetId: result[0]! as String, error: result[1] as String?, hash: result[2] as String?);+ }++ @override+ // ignore: avoid_equals_and_hash_code_on_mutable_classes+ bool operator ==(Object other) {+ if (other is! HashResult || other.runtimeType != runtimeType) {+ return false;+ }+ if (identical(this, other)) {+ return true;+ }+ return _deepEquals(encode(), other.encode());+ }++ @override+ // ignore: avoid_equals_and_hash_code_on_mutable_classes+ int get hashCode => Object.hashAll(_toList());+}+class _PigeonCodec extends StandardMessageCodec {const _PigeonCodec();@override@@ -221,6 +260,9 @@ class _PigeonCodec extends StandardMessageCodec {} else if (value is SyncDelta) {buffer.putUint8(131);writeValue(buffer, value.encode());+ } else if (value is HashResult) {+ buffer.putUint8(132);+ writeValue(buffer, value.encode());} else {super.writeValue(buffer, value);}@@ -235,6 +277,8 @@ class _PigeonCodec extends StandardMessageCodec {return PlatformAlbum.decode(readValue(buffer)!);case 131:return SyncDelta.decode(readValue(buffer)!);+ case 132:+ return HashResult.decode(readValue(buffer)!);default:return super.readValueOfType(type, buffer);}@@ -468,15 +512,15 @@ class NativeSyncApi {}}- Future<List<Uint8List?>> hashPaths(List<String> paths) async {+ Future<List<HashResult>> hashAssets(List<String> assetIds, {bool allowNetworkAccess = false}) async {final String pigeonVar_channelName =- 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$pigeonVar_messageChannelSuffix';+ 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets$pigeonVar_messageChannelSuffix';final BasicMessageChannel<Object?> pigeonVar_channel = BasicMessageChannel<Object?>(pigeonVar_channelName,pigeonChannelCodec,binaryMessenger: pigeonVar_binaryMessenger,);- final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[paths]);+ final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[assetIds, allowNetworkAccess]);final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;if (pigeonVar_replyList == null) {throw _createConnectionError(pigeonVar_channelName);@@ -492,7 +536,30 @@ class NativeSyncApi {message: 'Host platform returned null value for non-null return value.',);} else {- return (pigeonVar_replyList[0] as List<Object?>?)!.cast<Uint8List?>();+ return (pigeonVar_replyList[0] as List<Object?>?)!.cast<HashResult>();+ }+ }++ Future<void> cancelHashing() async {+ final String pigeonVar_channelName =+ 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing$pigeonVar_messageChannelSuffix';+ final BasicMessageChannel<Object?> pigeonVar_channel = BasicMessageChannel<Object?>(+ pigeonVar_channelName,+ pigeonChannelCodec,+ binaryMessenger: pigeonVar_binaryMessenger,+ );+ final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(null);+ final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;+ if (pigeonVar_replyList == null) {+ throw _createConnectionError(pigeonVar_channelName);+ } else if (pigeonVar_replyList.length > 1) {+ throw PlatformException(+ code: pigeonVar_replyList[0]! as String,+ message: pigeonVar_replyList[1] as String?,+ details: pigeonVar_replyList[2],+ );+ } else {+ return;}}}diff --git a/mobile/lib/providers/infrastructure/sync.provider.dart b/mobile/lib/providers/infrastructure/sync.provider.dartindex ddc6eed441..f03754505c 100644--- a/mobile/lib/providers/infrastructure/sync.provider.dart+++ b/mobile/lib/providers/infrastructure/sync.provider.dart@@ -10,7 +10,6 @@ import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart';import 'package:immich_mobile/providers/infrastructure/db.provider.dart';import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';-import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';final syncStreamServiceProvider = Provider((ref) => SyncStreamService(@@ -35,7 +34,6 @@ final hashServiceProvider = Provider((ref) => HashService(localAlbumRepository: ref.watch(localAlbumRepository),localAssetRepository: ref.watch(localAssetRepository),- storageRepository: ref.watch(storageRepositoryProvider),nativeSyncApi: ref.watch(nativeSyncApiProvider),),);diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dartindex 48302be79c..9d1f4e51e8 100644--- a/mobile/lib/services/hash.service.dart+++ b/mobile/lib/services/hash.service.dart@@ -16,9 +16,10 @@ class HashService {required IsarDeviceAssetRepository deviceAssetRepository,required BackgroundService backgroundService,this.batchSizeLimit = kBatchHashSizeLimit,- this.batchFileLimit = kBatchHashFileLimit,+ int? batchFileLimit,}) : _deviceAssetRepository = deviceAssetRepository,- _backgroundService = backgroundService;+ _backgroundService = backgroundService,+ batchFileLimit = batchFileLimit ?? kBatchHashFileLimit;final IsarDeviceAssetRepository _deviceAssetRepository;final BackgroundService _backgroundService;diff --git a/mobile/pigeon/native_sync_api.dart b/mobile/pigeon/native_sync_api.dartindex e84c814c3d..ac08a68ca3 100644--- a/mobile/pigeon/native_sync_api.dart+++ b/mobile/pigeon/native_sync_api.dart@@ -71,6 +71,14 @@ class SyncDelta {});}+class HashResult {+ final String assetId;+ final String? error;+ final String? hash;++ const HashResult({required this.assetId, this.error, this.hash});+}+@HostApi()abstract class NativeSyncApi {bool shouldFullSync();@@ -94,6 +102,9 @@ abstract class NativeSyncApi {@TaskQueue(type: TaskQueueType.serialBackgroundThread)List<PlatformAsset> getAssetsForAlbum(String albumId, {int? updatedTimeCond});+ @async@TaskQueue(type: TaskQueueType.serialBackgroundThread)- List<Uint8List?> hashPaths(List<String> paths);+ List<HashResult> hashAssets(List<String> assetIds, {bool allowNetworkAccess = false});++ void cancelHashing();}diff --git a/mobile/test/domain/services/hash_service_test.dart b/mobile/test/domain/services/hash_service_test.dartindex 7969131e7f..20d60b6866 100644--- a/mobile/test/domain/services/hash_service_test.dart+++ b/mobile/test/domain/services/hash_service_test.dart@@ -1,11 +1,7 @@-import 'dart:convert';-import 'dart:io';-import 'dart:typed_data';-import 'package:flutter_test/flutter_test.dart';-import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';+import 'package:immich_mobile/domain/models/album/local_album.model.dart';import 'package:immich_mobile/domain/services/hash.service.dart';-import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';+import 'package:immich_mobile/platform/native_sync_api.g.dart';import 'package:mocktail/mocktail.dart';import '../../fixtures/album.stub.dart';@@ -13,220 +9,181 @@ import '../../fixtures/asset.stub.dart';import '../../infrastructure/repository.mock.dart';import '../service.mock.dart';-class MockFile extends Mock implements File {}-void main() {late HashService sut;late MockLocalAlbumRepository mockAlbumRepo;late MockLocalAssetRepository mockAssetRepo;- late MockStorageRepository mockStorageRepo;late MockNativeSyncApi mockNativeApi;- final sortBy = {SortLocalAlbumsBy.backupSelection, SortLocalAlbumsBy.isIosSharedAlbum};setUp(() {mockAlbumRepo = MockLocalAlbumRepository();mockAssetRepo = MockLocalAssetRepository();- mockStorageRepo = MockStorageRepository();mockNativeApi = MockNativeSyncApi();sut = HashService(localAlbumRepository: mockAlbumRepo,localAssetRepository: mockAssetRepo,- storageRepository: mockStorageRepo,nativeSyncApi: mockNativeApi,);registerFallbackValue(LocalAlbumStub.recent);registerFallbackValue(LocalAssetStub.image1);+ registerFallbackValue(<String, String>{});when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});- when(() => mockStorageRepo.clearCache()).thenAnswer((_) async => {});});group('HashService hashAssets', () {test('skips albums with no assets to hash', () async {when(- () => mockAlbumRepo.getAll(sortBy: sortBy),+ () => mockAlbumRepo.getBackupAlbums(),).thenAnswer((_) async => [LocalAlbumStub.recent.copyWith(assetCount: 0)]);when(() => mockAlbumRepo.getAssetsToHash(LocalAlbumStub.recent.id)).thenAnswer((_) async => []);await sut.hashAssets();- verifyNever(() => mockStorageRepo.getFileForAsset(any()));- verifyNever(() => mockNativeApi.hashPaths(any()));+ verifyNever(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess')));});});group('HashService _hashAssets', () {- test('skips assets without files', () async {+ test('skips empty batches', () async {final album = LocalAlbumStub.recent;- final asset = LocalAssetStub.image1;- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => null);+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);+ when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => []);await sut.hashAssets();- verifyNever(() => mockNativeApi.hashPaths(any()));+ verifyNever(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess')));});test('processes assets when available', () async {final album = LocalAlbumStub.recent;final asset = LocalAssetStub.image1;- final mockFile = MockFile();- final hash = Uint8List.fromList(List.generate(20, (i) => i));-- when(() => mockFile.length()).thenAnswer((_) async => 1000);- when(() => mockFile.path).thenReturn('image-path');- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile);- when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [hash]);+ when(+ () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false),+ ).thenAnswer((_) async => [HashResult(assetId: asset.id, hash: 'test-hash')]);await sut.hashAssets();- verify(() => mockNativeApi.hashPaths(['image-path'])).called(1);- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;+ verify(() => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false)).called(1);+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;expect(captured.length, 1);- expect(captured[0].checksum, base64.encode(hash));+ expect(captured[asset.id], 'test-hash');});test('handles failed hashes', () async {final album = LocalAlbumStub.recent;final asset = LocalAssetStub.image1;- final mockFile = MockFile();- when(() => mockFile.length()).thenAnswer((_) async => 1000);- when(() => mockFile.path).thenReturn('image-path');- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile);- when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [null]);- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});+ when(+ () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false),+ ).thenAnswer((_) async => [HashResult(assetId: asset.id, error: 'Failed to hash')]);await sut.hashAssets();- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;expect(captured.length, 0);});- test('handles invalid hash length', () async {+ test('handles null hash results', () async {final album = LocalAlbumStub.recent;final asset = LocalAssetStub.image1;- final mockFile = MockFile();- when(() => mockFile.length()).thenAnswer((_) async => 1000);- when(() => mockFile.path).thenReturn('image-path');- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile);-- final invalidHash = Uint8List.fromList([1, 2, 3]);- when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [invalidHash]);- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});+ when(+ () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false),+ ).thenAnswer((_) async => [HashResult(assetId: asset.id, hash: null)]);await sut.hashAssets();- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;expect(captured.length, 0);});- test('batches by file count limit', () async {+ test('batches by size limit', () async {+ const batchSize = 2;final sut = HashService(localAlbumRepository: mockAlbumRepo,localAssetRepository: mockAssetRepo,- storageRepository: mockStorageRepo,nativeSyncApi: mockNativeApi,- batchFileLimit: 1,+ batchSize: batchSize,);final album = LocalAlbumStub.recent;final asset1 = LocalAssetStub.image1;final asset2 = LocalAssetStub.image2;- final mockFile1 = MockFile();- final mockFile2 = MockFile();- when(() => mockFile1.length()).thenAnswer((_) async => 100);- when(() => mockFile1.path).thenReturn('path-1');- when(() => mockFile2.length()).thenAnswer((_) async => 100);- when(() => mockFile2.path).thenReturn('path-2');-- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);- when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1);- when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2);+ final asset3 = LocalAssetStub.image1.copyWith(id: 'image3', name: 'image3.jpg');++ final capturedCalls = <List<String>>[];- final hash = Uint8List.fromList(List.generate(20, (i) => i));- when(() => mockNativeApi.hashPaths(any())).thenAnswer((_) async => [hash]);when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);+ when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2, asset3]);+ when(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))).thenAnswer((+ invocation,+ ) async {+ final assetIds = invocation.positionalArguments[0] as List<String>;+ capturedCalls.add(List<String>.from(assetIds));+ return assetIds.map((id) => HashResult(assetId: id, hash: '$id-hash')).toList();+ });await sut.hashAssets();- verify(() => mockNativeApi.hashPaths(['path-1'])).called(1);- verify(() => mockNativeApi.hashPaths(['path-2'])).called(1);+ expect(capturedCalls.length, 2, reason: 'Should make exactly 2 calls to hashAssets');+ expect(capturedCalls[0], [asset1.id, asset2.id], reason: 'First call should batch the first two assets');+ expect(capturedCalls[1], [asset3.id], reason: 'Second call should have the remaining asset');+verify(() => mockAssetRepo.updateHashes(any())).called(2);});- test('batches by size limit', () async {- final sut = HashService(- localAlbumRepository: mockAlbumRepo,- localAssetRepository: mockAssetRepo,- storageRepository: mockStorageRepo,- nativeSyncApi: mockNativeApi,- batchSizeLimit: 80,- );-+ test('handles mixed success and failure in batch', () async {final album = LocalAlbumStub.recent;final asset1 = LocalAssetStub.image1;final asset2 = LocalAssetStub.image2;- final mockFile1 = MockFile();- final mockFile2 = MockFile();- when(() => mockFile1.length()).thenAnswer((_) async => 100);- when(() => mockFile1.path).thenReturn('path-1');- when(() => mockFile2.length()).thenAnswer((_) async => 100);- when(() => mockFile2.path).thenReturn('path-2');-- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);- when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1);- when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2);- final hash = Uint8List.fromList(List.generate(20, (i) => i));- when(() => mockNativeApi.hashPaths(any())).thenAnswer((_) async => [hash]);- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);+ when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);+ when(() => mockNativeApi.hashAssets([asset1.id, asset2.id], allowNetworkAccess: false)).thenAnswer(+ (_) async => [+ HashResult(assetId: asset1.id, hash: 'asset1-hash'),+ HashResult(assetId: asset2.id, error: 'Failed to hash asset2'),+ ],+ );await sut.hashAssets();- verify(() => mockNativeApi.hashPaths(['path-1'])).called(1);- verify(() => mockNativeApi.hashPaths(['path-2'])).called(1);- verify(() => mockAssetRepo.updateHashes(any())).called(2);+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;+ expect(captured.length, 1);+ expect(captured[asset1.id], 'asset1-hash');});- test('handles mixed success and failure in batch', () async {- final album = LocalAlbumStub.recent;+ test('uses allowNetworkAccess based on album backup selection', () async {+ final selectedAlbum = LocalAlbumStub.recent.copyWith(backupSelection: BackupSelection.selected);+ final nonSelectedAlbum = LocalAlbumStub.recent.copyWith(id: 'album2', backupSelection: BackupSelection.excluded);final asset1 = LocalAssetStub.image1;final asset2 = LocalAssetStub.image2;- final mockFile1 = MockFile();- final mockFile2 = MockFile();- when(() => mockFile1.length()).thenAnswer((_) async => 100);- when(() => mockFile1.path).thenReturn('path-1');- when(() => mockFile2.length()).thenAnswer((_) async => 100);- when(() => mockFile2.path).thenReturn('path-2');-- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);- when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1);- when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2);- final validHash = Uint8List.fromList(List.generate(20, (i) => i));- when(() => mockNativeApi.hashPaths(['path-1', 'path-2'])).thenAnswer((_) async => [validHash, null]);- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [selectedAlbum, nonSelectedAlbum]);+ when(() => mockAlbumRepo.getAssetsToHash(selectedAlbum.id)).thenAnswer((_) async => [asset1]);+ when(() => mockAlbumRepo.getAssetsToHash(nonSelectedAlbum.id)).thenAnswer((_) async => [asset2]);+ when(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))).thenAnswer((+ invocation,+ ) async {+ final assetIds = invocation.positionalArguments[0] as List<String>;+ return assetIds.map((id) => HashResult(assetId: id, hash: '$id-hash')).toList();+ });await sut.hashAssets();- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;- expect(captured.length, 1);- expect(captured.first.id, asset1.id);+ verify(() => mockNativeApi.hashAssets([asset1.id], allowNetworkAccess: true)).called(1);+ verify(() => mockNativeApi.hashAssets([asset2.id], allowNetworkAccess: false)).called(1);});});}