Skip to content

High performance self-hosted photo and video management solution.

refactor: hashing service (#21997)

shenlong532ec10d5f8d
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt
index 9c618d9ed0..28400c803f 100644
--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt
@@ -209,6 +209,40 @@ data class SyncDelta (
override fun hashCode(): Int = toList().hashCode()
}
+
+/** Generated class from Pigeon that represents data sent in messages. */
+data class HashResult (
+ val assetId: String,
+ val error: String? = null,
+ val hash: String? = null
+)
+ {
+ companion object {
+ fun fromList(pigeonVar_list: List<Any?>): HashResult {
+ val assetId = pigeonVar_list[0] as String
+ val error = pigeonVar_list[1] as String?
+ val hash = pigeonVar_list[2] as String?
+ return HashResult(assetId, error, hash)
+ }
+ }
+ fun toList(): List<Any?> {
+ return listOf(
+ assetId,
+ error,
+ hash,
+ )
+ }
+ override fun equals(other: Any?): Boolean {
+ if (other !is HashResult) {
+ return false
+ }
+ if (this === other) {
+ return true
+ }
+ return MessagesPigeonUtils.deepEquals(toList(), other.toList()) }
+
+ override fun hashCode(): Int = toList().hashCode()
+}
private open class MessagesPigeonCodec : StandardMessageCodec() {
override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? {
return when (type) {
@@ -227,6 +261,11 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {
SyncDelta.fromList(it)
}
}
+ 132.toByte() -> {
+ return (readValue(buffer) as? List<Any?>)?.let {
+ HashResult.fromList(it)
+ }
+ }
else -> super.readValueOfType(type, buffer)
}
}
@@ -244,11 +283,16 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {
stream.write(131)
writeValue(stream, value.toList())
}
+ is HashResult -> {
+ stream.write(132)
+ writeValue(stream, value.toList())
+ }
else -> super.writeValue(stream, value)
}
}
}
+
/** Generated interface from Pigeon that represents a handler of messages from Flutter. */
interface NativeSyncApi {
fun shouldFullSync(): Boolean
@@ -259,7 +303,8 @@ interface NativeSyncApi {
fun getAlbums(): List<PlatformAlbum>
fun getAssetsCountSince(albumId: String, timestamp: Long): Long
fun getAssetsForAlbum(albumId: String, updatedTimeCond: Long?): List<PlatformAsset>
- fun hashPaths(paths: List<String>): List<ByteArray?>
+ fun hashAssets(assetIds: List<String>, allowNetworkAccess: Boolean, callback: (Result<List<HashResult>>) -> Unit)
+ fun cancelHashing()
companion object {
/** The codec used by NativeSyncApi. */
@@ -402,13 +447,33 @@ interface NativeSyncApi {
}
}
run {
- val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$separatedMessageChannelSuffix", codec, taskQueue)
+ val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets$separatedMessageChannelSuffix", codec, taskQueue)
if (api != null) {
channel.setMessageHandler { message, reply ->
val args = message as List<Any?>
- val pathsArg = args[0] as List<String>
+ val assetIdsArg = args[0] as List<String>
+ val allowNetworkAccessArg = args[1] as Boolean
+ api.hashAssets(assetIdsArg, allowNetworkAccessArg) { result: Result<List<HashResult>> ->
+ val error = result.exceptionOrNull()
+ if (error != null) {
+ reply.reply(MessagesPigeonUtils.wrapError(error))
+ } else {
+ val data = result.getOrNull()
+ reply.reply(MessagesPigeonUtils.wrapResult(data))
+ }
+ }
+ }
+ } else {
+ channel.setMessageHandler(null)
+ }
+ }
+ run {
+ val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing$separatedMessageChannelSuffix", codec)
+ if (api != null) {
+ channel.setMessageHandler { _, reply ->
val wrapped: List<Any?> = try {
- listOf(api.hashPaths(pathsArg))
+ api.cancelHashing()
+ listOf(null)
} catch (exception: Throwable) {
MessagesPigeonUtils.wrapError(exception)
}
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt
index b2ceb8a9f2..868f3c6cdd 100644
--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt
@@ -1,14 +1,25 @@
package app.alextran.immich.sync
import android.annotation.SuppressLint
+import android.content.ContentUris
import android.content.Context
import android.database.Cursor
import android.provider.MediaStore
-import android.util.Log
+import android.util.Base64
import androidx.core.database.getStringOrNull
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
import java.io.File
-import java.io.FileInputStream
import java.security.MessageDigest
+import kotlin.coroutines.cancellation.CancellationException
+import kotlin.coroutines.coroutineContext
sealed class AssetResult {
data class ValidAsset(val asset: PlatformAsset, val albumId: String) : AssetResult()
@@ -19,8 +30,12 @@ sealed class AssetResult {
open class NativeSyncApiImplBase(context: Context) {
private val ctx: Context = context.applicationContext
+ private var hashTask: Job? = null
+
companion object {
- private const val TAG = "NativeSyncApiImplBase"
+ private const val MAX_CONCURRENT_HASH_OPERATIONS = 16
+ private val hashSemaphore = Semaphore(MAX_CONCURRENT_HASH_OPERATIONS)
+ private const val HASHING_CANCELLED_CODE = "HASH_CANCELLED"
const val MEDIA_SELECTION =
"(${MediaStore.Files.FileColumns.MEDIA_TYPE} = ? OR ${MediaStore.Files.FileColumns.MEDIA_TYPE} = ?)"
@@ -215,23 +230,74 @@ open class NativeSyncApiImplBase(context: Context) {
.toList()
}
- fun hashPaths(paths: List<String>): List<ByteArray?> {
- val buffer = ByteArray(HASH_BUFFER_SIZE)
- val digest = MessageDigest.getInstance("SHA-1")
+ fun hashAssets(
+ assetIds: List<String>,
+ // allowNetworkAccess is only used on the iOS implementation
+ @Suppress("UNUSED_PARAMETER") allowNetworkAccess: Boolean,
+ callback: (Result<List<HashResult>>) -> Unit
+ ) {
+ if (assetIds.isEmpty()) {
+ callback(Result.success(emptyList()))
+ return
+ }
- return paths.map { path ->
+ hashTask?.cancel()
+ hashTask = CoroutineScope(Dispatchers.IO).launch {
try {
- FileInputStream(path).use { file ->
- var bytesRead: Int
- while (file.read(buffer).also { bytesRead = it } > 0) {
- digest.update(buffer, 0, bytesRead)
+ val results = assetIds.map { assetId ->
+ async {
+ hashSemaphore.withPermit {
+ ensureActive()
+ hashAsset(assetId)
+ }
}
- }
- digest.digest()
+ }.awaitAll()
+
+ callback(Result.success(results))
+ } catch (e: CancellationException) {
+ callback(
+ Result.failure(
+ FlutterError(
+ HASHING_CANCELLED_CODE,
+ "Hashing operation was cancelled",
+ null
+ )
+ )
+ )
} catch (e: Exception) {
- Log.w(TAG, "Failed to hash file $path: $e")
- null
+ callback(Result.failure(e))
}
}
}
+
+ private suspend fun hashAsset(assetId: String): HashResult {
+ return try {
+ val assetUri = ContentUris.withAppendedId(
+ MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL),
+ assetId.toLong()
+ )
+
+ val digest = MessageDigest.getInstance("SHA-1")
+ ctx.contentResolver.openInputStream(assetUri)?.use { inputStream ->
+ var bytesRead: Int
+ val buffer = ByteArray(HASH_BUFFER_SIZE)
+ while (inputStream.read(buffer).also { bytesRead = it } > 0) {
+ coroutineContext.ensureActive()
+ digest.update(buffer, 0, bytesRead)
+ }
+ } ?: return HashResult(assetId, "Cannot open input stream for asset", null)
+
+ val hashString = Base64.encodeToString(digest.digest(), Base64.NO_WRAP)
+ HashResult(assetId, null, hashString)
+ } catch (e: SecurityException) {
+ HashResult(assetId, "Permission denied accessing asset: ${e.message}", null)
+ } catch (e: Exception) {
+ HashResult(assetId, "Failed to hash asset: ${e.message}", null)
+ }
+ }
+
+ fun cancelHashing() {
+ hashTask?.cancel()
+ hashTask = null
+ }
}
diff --git a/mobile/ios/.gitignore b/mobile/ios/.gitignore
index f312f249a3..e32cadbf68 100644
--- a/mobile/ios/.gitignore
+++ b/mobile/ios/.gitignore
@@ -4,7 +4,6 @@
*.moved-aside
*.pbxuser
*.perspectivev3
-**/*sync/
.sconsign.dblite
.tags*
**/.vagrant/
diff --git a/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift b/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift
index ca2453404c..f7f8f69989 100644
--- a/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift
+++ b/mobile/ios/Runner/Background/BackgroundWorkerApiImpl.swift
@@ -5,7 +5,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi {
func enable() throws {
BackgroundWorkerApiImpl.scheduleRefreshWorker()
BackgroundWorkerApiImpl.scheduleProcessingWorker()
- print("BackgroundUploadImpl:enbale Background worker scheduled")
+ print("BackgroundWorkerApiImpl:enable Background worker scheduled")
}
func configure(settings: BackgroundWorkerSettings) throws {
@@ -15,7 +15,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi {
func disable() throws {
BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.refreshTaskID);
BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.processingTaskID);
- print("BackgroundUploadImpl:disableUploadWorker Disabled background workers")
+ print("BackgroundWorkerApiImpl:disableUploadWorker Disabled background workers")
}
private static let refreshTaskID = "app.alextran.immich.background.refreshUpload"
diff --git a/mobile/ios/Runner/Sync/Messages.g.swift b/mobile/ios/Runner/Sync/Messages.g.swift
index 19f4384672..305aca5266 100644
--- a/mobile/ios/Runner/Sync/Messages.g.swift
+++ b/mobile/ios/Runner/Sync/Messages.g.swift
@@ -267,6 +267,39 @@ struct SyncDelta: Hashable {
}
}
+/// Generated class from Pigeon that represents data sent in messages.
+struct HashResult: Hashable {
+ var assetId: String
+ var error: String? = nil
+ var hash: String? = nil
+
+
+ // swift-format-ignore: AlwaysUseLowerCamelCase
+ static func fromList(_ pigeonVar_list: [Any?]) -> HashResult? {
+ let assetId = pigeonVar_list[0] as! String
+ let error: String? = nilOrValue(pigeonVar_list[1])
+ let hash: String? = nilOrValue(pigeonVar_list[2])
+
+ return HashResult(
+ assetId: assetId,
+ error: error,
+ hash: hash
+ )
+ }
+ func toList() -> [Any?] {
+ return [
+ assetId,
+ error,
+ hash,
+ ]
+ }
+ static func == (lhs: HashResult, rhs: HashResult) -> Bool {
+ return deepEqualsMessages(lhs.toList(), rhs.toList()) }
+ func hash(into hasher: inout Hasher) {
+ deepHashMessages(value: toList(), hasher: &hasher)
+ }
+}
+
private class MessagesPigeonCodecReader: FlutterStandardReader {
override func readValue(ofType type: UInt8) -> Any? {
switch type {
@@ -276,6 +309,8 @@ private class MessagesPigeonCodecReader: FlutterStandardReader {
return PlatformAlbum.fromList(self.readValue() as! [Any?])
case 131:
return SyncDelta.fromList(self.readValue() as! [Any?])
+ case 132:
+ return HashResult.fromList(self.readValue() as! [Any?])
default:
return super.readValue(ofType: type)
}
@@ -293,6 +328,9 @@ private class MessagesPigeonCodecWriter: FlutterStandardWriter {
} else if let value = value as? SyncDelta {
super.writeByte(131)
super.writeValue(value.toList())
+ } else if let value = value as? HashResult {
+ super.writeByte(132)
+ super.writeValue(value.toList())
} else {
super.writeValue(value)
}
@@ -313,6 +351,7 @@ class MessagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
static let shared = MessagesPigeonCodec(readerWriter: MessagesPigeonCodecReaderWriter())
}
+
/// Generated protocol from Pigeon that represents a handler of messages from Flutter.
protocol NativeSyncApi {
func shouldFullSync() throws -> Bool
@@ -323,7 +362,8 @@ protocol NativeSyncApi {
func getAlbums() throws -> [PlatformAlbum]
func getAssetsCountSince(albumId: String, timestamp: Int64) throws -> Int64
func getAssetsForAlbum(albumId: String, updatedTimeCond: Int64?) throws -> [PlatformAsset]
- func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?]
+ func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void)
+ func cancelHashing() throws
}
/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
@@ -459,22 +499,38 @@ class NativeSyncApiSetup {
} else {
getAssetsForAlbumChannel.setMessageHandler(nil)
}
- let hashPathsChannel = taskQueue == nil
- ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
- : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue)
+ let hashAssetsChannel = taskQueue == nil
+ ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
+ : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue)
if let api = api {
- hashPathsChannel.setMessageHandler { message, reply in
+ hashAssetsChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
- let pathsArg = args[0] as! [String]
+ let assetIdsArg = args[0] as! [String]
+ let allowNetworkAccessArg = args[1] as! Bool
+ api.hashAssets(assetIds: assetIdsArg, allowNetworkAccess: allowNetworkAccessArg) { result in
+ switch result {
+ case .success(let res):
+ reply(wrapResult(res))
+ case .failure(let error):
+ reply(wrapError(error))
+ }
+ }
+ }
+ } else {
+ hashAssetsChannel.setMessageHandler(nil)
+ }
+ let cancelHashingChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
+ if let api = api {
+ cancelHashingChannel.setMessageHandler { _, reply in
do {
- let result = try api.hashPaths(paths: pathsArg)
- reply(wrapResult(result))
+ try api.cancelHashing()
+ reply(wrapResult(nil))
} catch {
reply(wrapError(error))
}
}
} else {
- hashPathsChannel.setMessageHandler(nil)
+ cancelHashingChannel.setMessageHandler(nil)
}
}
}
diff --git a/mobile/ios/Runner/Sync/MessagesImpl.swift b/mobile/ios/Runner/Sync/MessagesImpl.swift
index 2810dee7c1..bb23bae6b6 100644
--- a/mobile/ios/Runner/Sync/MessagesImpl.swift
+++ b/mobile/ios/Runner/Sync/MessagesImpl.swift
@@ -17,30 +17,16 @@ struct AssetWrapper: Hashable, Equatable {
}
}
-extension PHAsset {
- func toPlatformAsset() -> PlatformAsset {
- return PlatformAsset(
- id: localIdentifier,
- name: title(),
- type: Int64(mediaType.rawValue),
- createdAt: creationDate.map { Int64($0.timeIntervalSince1970) },
- updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) },
- width: Int64(pixelWidth),
- height: Int64(pixelHeight),
- durationInSeconds: Int64(duration),
- orientation: 0,
- isFavorite: isFavorite
- )
- }
-}
-
class NativeSyncApiImpl: NativeSyncApi {
private let defaults: UserDefaults
private let changeTokenKey = "immich:changeToken"
private let albumTypes: [PHAssetCollectionType] = [.album, .smartAlbum]
private let recoveredAlbumSubType = 1000000219
- private let hashBufferSize = 2 * 1024 * 1024
+ private var hashTask: Task<Void, Error>?
+ private static let hashCancelledCode = "HASH_CANCELLED"
+ private static let hashCancelled = Result<[HashResult], Error>.failure(PigeonError(code: hashCancelledCode, message: "Hashing cancelled", details: nil))
+
init(with defaults: UserDefaults = .standard) {
self.defaults = defaults
@@ -96,7 +82,7 @@ class NativeSyncApiImpl: NativeSyncApi {
let collections = PHAssetCollection.fetchAssetCollections(with: type, subtype: .any, options: nil)
for i in 0..<collections.count {
let album = collections.object(at: i)
-
+
// Ignore recovered album
if(album.assetCollectionSubtype.rawValue == self.recoveredAlbumSubType) {
continue;
@@ -254,7 +240,7 @@ class NativeSyncApiImpl: NativeSyncApi {
let date = NSDate(timeIntervalSince1970: TimeInterval(updatedTimeCond!))
options.predicate = NSPredicate(format: "creationDate > %@ OR modificationDate > %@", date, date)
}
-
+
let result = PHAsset.fetchAssets(in: album, options: options)
if(result.count == 0) {
return []
@@ -267,23 +253,114 @@ class NativeSyncApiImpl: NativeSyncApi {
return assets
}
- func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] {
- return paths.map { path in
- guard let file = FileHandle(forReadingAtPath: path) else {
- print("Cannot open file: \(path)")
- return nil
+ func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void) {
+ if let prevTask = hashTask {
+ prevTask.cancel()
+ hashTask = nil
+ }
+ hashTask = Task { [weak self] in
+ var missingAssetIds = Set(assetIds)
+ var assets = [PHAsset]()
+ assets.reserveCapacity(assetIds.count)
+ PHAsset.fetchAssets(withLocalIdentifiers: assetIds, options: nil).enumerateObjects { (asset, _, stop) in
+ if Task.isCancelled {
+ stop.pointee = true
+ return
+ }
+ missingAssetIds.remove(asset.localIdentifier)
+ assets.append(asset)
+ }
+
+ if Task.isCancelled {
+ return completion(Self.hashCancelled)
+ }
+
+ await withTaskGroup(of: HashResult?.self) { taskGroup in
+ var results = [HashResult]()
+ results.reserveCapacity(assets.count)
+ for asset in assets {
+ if Task.isCancelled {
+ return completion(Self.hashCancelled)
}
-
- var hasher = Insecure.SHA1()
- while autoreleasepool(invoking: {
- let chunk = file.readData(ofLength: hashBufferSize)
- guard !chunk.isEmpty else { return false }
- hasher.update(data: chunk)
- return true
- }) { }
-
- let digest = hasher.finalize()
- return FlutterStandardTypedData(bytes: Data(digest))
+ taskGroup.addTask {
+ guard let self = self else { return nil }
+ return await self.hashAsset(asset, allowNetworkAccess: allowNetworkAccess)
+ }
+ }
+
+ for await result in taskGroup {
+ guard let result = result else {
+ return completion(Self.hashCancelled)
+ }
+ results.append(result)
+ }
+
+ for missing in missingAssetIds {
+ results.append(HashResult(assetId: missing, error: "Asset not found in library", hash: nil))
+ }
+
+ completion(.success(results))
+ }
+ }
+ }
+
+ func cancelHashing() {
+ hashTask?.cancel()
+ hashTask = nil
+ }
+
+ private func hashAsset(_ asset: PHAsset, allowNetworkAccess: Bool) async -> HashResult? {
+ class RequestRef {
+ var id: PHAssetResourceDataRequestID?
+ }
+ let requestRef = RequestRef()
+ return await withTaskCancellationHandler(operation: {
+ if Task.isCancelled {
+ return nil
+ }
+
+ guard let resource = asset.getResource() else {
+ return HashResult(assetId: asset.localIdentifier, error: "Cannot get asset resource", hash: nil)
+ }
+
+ if Task.isCancelled {
+ return nil
+ }
+
+ let options = PHAssetResourceRequestOptions()
+ options.isNetworkAccessAllowed = allowNetworkAccess
+
+ return await withCheckedContinuation { continuation in
+ var hasher = Insecure.SHA1()
+
+ requestRef.id = PHAssetResourceManager.default().requestData(
+ for: resource,
+ options: options,
+ dataReceivedHandler: { data in
+ hasher.update(data: data)
+ },
+ completionHandler: { error in
+ let result: HashResult? = switch (error) {
+ case let e as PHPhotosError where e.code == .userCancelled: nil
+ case let .some(e): HashResult(
+ assetId: asset.localIdentifier,
+ error: "Failed to hash asset: \(e.localizedDescription)",
+ hash: nil
+ )
+ case .none:
+ HashResult(
+ assetId: asset.localIdentifier,
+ error: nil,
+ hash: Data(hasher.finalize()).base64EncodedString()
+ )
+ }
+ continuation.resume(returning: result)
+ }
+ )
}
+ }, onCancel: {
+ guard let requestId = requestRef.id else { return }
+ PHAssetResourceManager.default().cancelDataRequest(requestId)
+ })
}
}
diff --git a/mobile/ios/Runner/Sync/PHAssetExtensions.swift b/mobile/ios/Runner/Sync/PHAssetExtensions.swift
new file mode 100644
index 0000000000..2b1ef6ac88
--- /dev/null
+++ b/mobile/ios/Runner/Sync/PHAssetExtensions.swift
@@ -0,0 +1,77 @@
+import Photos
+
+extension PHAsset {
+ func toPlatformAsset() -> PlatformAsset {
+ return PlatformAsset(
+ id: localIdentifier,
+ name: title,
+ type: Int64(mediaType.rawValue),
+ createdAt: creationDate.map { Int64($0.timeIntervalSince1970) },
+ updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) },
+ width: Int64(pixelWidth),
+ height: Int64(pixelHeight),
+ durationInSeconds: Int64(duration),
+ orientation: 0,
+ isFavorite: isFavorite
+ )
+ }
+
+ var title: String {
+ return filename ?? originalFilename ?? "<unknown>"
+ }
+
+ var filename: String? {
+ return value(forKey: "filename") as? String
+ }
+
+ // This method is expected to be slow as it goes through the asset resources to fetch the originalFilename
+ var originalFilename: String? {
+ return getResource()?.originalFilename
+ }
+
+ func getResource() -> PHAssetResource? {
+ let resources = PHAssetResource.assetResources(for: self)
+
+ let filteredResources = resources.filter { $0.isMediaResource && isValidResourceType($0.type) }
+
+ guard !filteredResources.isEmpty else {
+ return nil
+ }
+
+ if filteredResources.count == 1 {
+ return filteredResources.first
+ }
+
+ if let currentResource = filteredResources.first(where: { $0.isCurrent }) {
+ return currentResource
+ }
+
+ if let fullSizeResource = filteredResources.first(where: { isFullSizeResourceType($0.type) }) {
+ return fullSizeResource
+ }
+
+ return nil
+ }
+
+ private func isValidResourceType(_ type: PHAssetResourceType) -> Bool {
+ switch mediaType {
+ case .image:
+ return [.photo, .alternatePhoto, .fullSizePhoto].contains(type)
+ case .video:
+ return [.video, .fullSizeVideo, .fullSizePairedVideo].contains(type)
+ default:
+ return false
+ }
+ }
+
+ private func isFullSizeResourceType(_ type: PHAssetResourceType) -> Bool {
+ switch mediaType {
+ case .image:
+ return type == .fullSizePhoto
+ case .video:
+ return type == .fullSizeVideo
+ default:
+ return false
+ }
+ }
+}
diff --git a/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift b/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift
new file mode 100644
index 0000000000..699d55a98d
--- /dev/null
+++ b/mobile/ios/Runner/Sync/PHAssetResourceExtensions.swift
@@ -0,0 +1,16 @@
+
+import Photos
+
+extension PHAssetResource {
+ var isCurrent: Bool {
+ return value(forKey: "isCurrent") as? Bool ?? false
+ }
+
+ var isMediaResource: Bool {
+ var isMedia = type != .adjustmentData
+ if #available(iOS 17, *) {
+ isMedia = isMedia && type != .photoProxy
+ }
+ return isMedia
+ }
+}
diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart
index 60c9b92cf9..7429616f14 100644
--- a/mobile/lib/constants/constants.dart
+++ b/mobile/lib/constants/constants.dart
@@ -1,3 +1,5 @@
+import 'dart:io';
+
const int noDbId = -9223372036854775808; // from Isar
const double downloadCompleted = -1;
const double downloadFailed = -2;
@@ -10,7 +12,7 @@ const int kSyncEventBatchSize = 5000;
const int kFetchLocalAssetsBatchSize = 40000;
// Hash batch limits
-const int kBatchHashFileLimit = 256;
+final int kBatchHashFileLimit = Platform.isIOS ? 32 : 512;
const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB
// Secure storage keys
diff --git a/mobile/lib/domain/services/background_worker.service.dart b/mobile/lib/domain/services/background_worker.service.dart
index d57fe507d9..78dd1e980f 100644
--- a/mobile/lib/domain/services/background_worker.service.dart
+++ b/mobile/lib/domain/services/background_worker.service.dart
@@ -184,6 +184,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
try {
final backgroundSyncManager = _ref.read(backgroundSyncProvider);
+ final nativeSyncApi = _ref.read(nativeSyncApiProvider);
_isCleanedUp = true;
_ref.dispose();
@@ -199,7 +200,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
_drift.close(),
_driftLogger.close(),
backgroundSyncManager.cancel(),
- backgroundSyncManager.cancelLocal(),
+ nativeSyncApi.cancelHashing(),
];
if (_isar.isOpen) {
diff --git a/mobile/lib/domain/services/hash.service.dart b/mobile/lib/domain/services/hash.service.dart
index 8044b298d3..90f29b8bc1 100644
--- a/mobile/lib/domain/services/hash.service.dart
+++ b/mobile/lib/domain/services/hash.service.dart
@@ -1,20 +1,18 @@
-import 'dart:convert';
-
+import 'package:flutter/services.dart';
import 'package:immich_mobile/constants/constants.dart';
import 'package:immich_mobile/domain/models/album/local_album.model.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
-import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
import 'package:immich_mobile/platform/native_sync_api.g.dart';
import 'package:logging/logging.dart';
+const String _kHashCancelledCode = "HASH_CANCELLED";
+
class HashService {
- final int batchSizeLimit;
- final int batchFileLimit;
+ final int _batchSize;
final DriftLocalAlbumRepository _localAlbumRepository;
final DriftLocalAssetRepository _localAssetRepository;
- final StorageRepository _storageRepository;
final NativeSyncApi _nativeSyncApi;
final bool Function()? _cancelChecker;
final _log = Logger('HashService');
@@ -22,37 +20,42 @@ class HashService {
HashService({
required DriftLocalAlbumRepository localAlbumRepository,
required DriftLocalAssetRepository localAssetRepository,
- required StorageRepository storageRepository,
required NativeSyncApi nativeSyncApi,
bool Function()? cancelChecker,
- this.batchSizeLimit = kBatchHashSizeLimit,
- this.batchFileLimit = kBatchHashFileLimit,
+ int? batchSize,
}) : _localAlbumRepository = localAlbumRepository,
_localAssetRepository = localAssetRepository,
- _storageRepository = storageRepository,
_cancelChecker = cancelChecker,
- _nativeSyncApi = nativeSyncApi;
+ _nativeSyncApi = nativeSyncApi,
+ _batchSize = batchSize ?? kBatchHashFileLimit;
bool get isCancelled => _cancelChecker?.call() ?? false;
Future<void> hashAssets() async {
_log.info("Starting hashing of assets");
final Stopwatch stopwatch = Stopwatch()..start();
- // Sorted by backupSelection followed by isCloud
- final localAlbums = await _localAlbumRepository.getAll(
- sortBy: {SortLocalAlbumsBy.backupSelection, SortLocalAlbumsBy.isIosSharedAlbum},
- );
-
- for (final album in localAlbums) {
- if (isCancelled) {
- _log.warning("Hashing cancelled. Stopped processing albums.");
- break;
+ try {
+ // Sorted by backupSelection followed by isCloud
+ final localAlbums = await _localAlbumRepository.getBackupAlbums();
+
+ for (final album in localAlbums) {
+ if (isCancelled) {
+ _log.warning("Hashing cancelled. Stopped processing albums.");
+ break;
+ }
+
+ final assetsToHash = await _localAlbumRepository.getAssetsToHash(album.id);
+ if (assetsToHash.isNotEmpty) {
+ await _hashAssets(album, assetsToHash);
+ }
}
-
- final assetsToHash = await _localAlbumRepository.getAssetsToHash(album.id);
- if (assetsToHash.isNotEmpty) {
- await _hashAssets(album, assetsToHash);
+ } on PlatformException catch (e) {
+ if (e.code == _kHashCancelledCode) {
+ _log.warning("Hashing cancelled by platform");
+ return;
}
+ } catch (e, s) {
+ _log.severe("Error during hashing", e, s);
}
stopwatch.stop();
@@ -63,8 +66,7 @@ class HashService {
/// with hash for those that were successfully hashed. Hashes are looked up in a table
/// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB.
Future<void> _hashAssets(LocalAlbum album, List<LocalAsset> assetsToHash) async {
- int bytesProcessed = 0;
- final toHash = <_AssetToPath>[];
+ final toHash = <String, LocalAsset>{};
for (final asset in assetsToHash) {
if (isCancelled) {
@@ -72,21 +74,10 @@ class HashService {
return;
}
- final file = await _storageRepository.getFileForAsset(asset.id);
- if (file == null) {
- _log.warning(
- "Cannot get file for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt} from album: ${album.name}",
- );
- continue;
- }
-
- bytesProcessed += await file.length();
- toHash.add(_AssetToPath(asset: asset, path: file.path));
-
- if (toHash.length >= batchFileLimit || bytesProcessed >= batchSizeLimit) {
+ toHash[asset.id] = asset;
+ if (toHash.length == _batchSize) {
await _processBatch(album, toHash);
toHash.clear();
- bytesProcessed = 0;
}
}
@@ -94,33 +85,36 @@ class HashService {
}
/// Processes a batch of assets.
- Future<void> _processBatch(LocalAlbum album, List<_AssetToPath> toHash) async {
+ Future<void> _processBatch(LocalAlbum album, Map<String, LocalAsset> toHash) async {
if (toHash.isEmpty) {
return;
}
_log.fine("Hashing ${toHash.length} files");
- final hashed = <LocalAsset>[];
- final hashes = await _nativeSyncApi.hashPaths(toHash.map((e) => e.path).toList());
+ final hashed = <String, String>{};
+ final hashResults = await _nativeSyncApi.hashAssets(
+ toHash.keys.toList(),
+ allowNetworkAccess: album.backupSelection == BackupSelection.selected,
+ );
assert(
- hashes.length == toHash.length,
- "Hashes length does not match toHash length: ${hashes.length} != ${toHash.length}",
+ hashResults.length == toHash.length,
+ "Hashes length does not match toHash length: ${hashResults.length} != ${toHash.length}",
);
- for (int i = 0; i < hashes.length; i++) {
+ for (int i = 0; i < hashResults.length; i++) {
if (isCancelled) {
_log.warning("Hashing cancelled. Stopped processing batch.");
return;
}
- final hash = hashes[i];
- final asset = toHash[i].asset;
- if (hash?.length == 20) {
- hashed.add(asset.copyWith(checksum: base64.encode(hash!)));
+ final hashResult = hashResults[i];
+ if (hashResult.hash != null) {
+ hashed[hashResult.assetId] = hashResult.hash!;
} else {
+ final asset = toHash[hashResult.assetId];
_log.warning(
- "Failed to hash file for ${asset.id}: ${asset.name} created at ${asset.createdAt} from album: ${album.name}",
+ "Failed to hash asset with id: ${hashResult.assetId}, name: ${asset?.name}, createdAt: ${asset?.createdAt}, from album: ${album.name}. Error: ${hashResult.error ?? "unknown"}",
);
}
}
@@ -128,13 +122,5 @@ class HashService {
_log.fine("Hashed ${hashed.length}/${toHash.length} assets");
await _localAssetRepository.updateHashes(hashed);
- await _storageRepository.clearCache();
}
}
-
-class _AssetToPath {
- final LocalAsset asset;
- final String path;
-
- const _AssetToPath({required this.asset, required this.path});
-}
diff --git a/mobile/lib/infrastructure/repositories/local_asset.repository.dart b/mobile/lib/infrastructure/repositories/local_asset.repository.dart
index 05c8e06678..c4f5221966 100644
--- a/mobile/lib/infrastructure/repositories/local_asset.repository.dart
+++ b/mobile/lib/infrastructure/repositories/local_asset.repository.dart
@@ -36,17 +36,17 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository {
Stream<LocalAsset?> watch(String id) => _assetSelectable(id).watchSingleOrNull();
- Future<void> updateHashes(Iterable<LocalAsset> hashes) {
+ Future<void> updateHashes(Map<String, String> hashes) {
if (hashes.isEmpty) {
return Future.value();
}
return _db.batch((batch) async {
- for (final asset in hashes) {
+ for (final entry in hashes.entries) {
batch.update(
_db.localAssetEntity,
- LocalAssetEntityCompanion(checksum: Value(asset.checksum)),
- where: (e) => e.id.equals(asset.id),
+ LocalAssetEntityCompanion(checksum: Value(entry.value)),
+ where: (e) => e.id.equals(entry.key),
);
}
});
diff --git a/mobile/lib/pages/backup/drift_backup_album_selection.page.dart b/mobile/lib/pages/backup/drift_backup_album_selection.page.dart
index 8cfdd2db95..368341f24a 100644
--- a/mobile/lib/pages/backup/drift_backup_album_selection.page.dart
+++ b/mobile/lib/pages/backup/drift_backup_album_selection.page.dart
@@ -1,3 +1,4 @@
+import 'dart:async';
import 'dart:io';
import 'package:auto_route/auto_route.dart';
@@ -5,12 +6,14 @@ import 'package:easy_localization/easy_localization.dart';
import 'package:flutter/material.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/domain/models/album/local_album.model.dart';
+import 'package:immich_mobile/domain/services/sync_linked_album.service.dart';
import 'package:immich_mobile/extensions/build_context_extensions.dart';
import 'package:immich_mobile/extensions/translate_extensions.dart';
import 'package:immich_mobile/providers/app_settings.provider.dart';
-import 'package:immich_mobile/domain/services/sync_linked_album.service.dart';
+import 'package:immich_mobile/providers/background_sync.provider.dart';
import 'package:immich_mobile/providers/backup/backup_album.provider.dart';
import 'package:immich_mobile/providers/backup/drift_backup.provider.dart';
+import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
import 'package:immich_mobile/providers/user.provider.dart';
import 'package:immich_mobile/services/app_settings.service.dart';
import 'package:immich_mobile/widgets/backup/drift_album_info_list_tile.dart';
@@ -64,16 +67,6 @@ class _DriftBackupAlbumSelectionPageState extends ConsumerState<DriftBackupAlbum
});
await _handleLinkedAlbumFuture;
}
-
- // Restart backup if total count changed and backup is enabled
- final currentTotalAssetCount = ref.read(driftBackupProvider.select((p) => p.totalCount));
- final totalChanged = currentTotalAssetCount != _initialTotalAssetCount;
- final isBackupEnabled = ref.read(appSettingsServiceProvider).getSetting(AppSettingsEnum.enableBackup);
-
- if (totalChanged && isBackupEnabled) {
- await ref.read(driftBackupProvider.notifier).cancel();
- await ref.read(driftBackupProvider.notifier).startBackup(user.id);
- }
}
@override
@@ -102,6 +95,27 @@ class _DriftBackupAlbumSelectionPageState extends ConsumerState<DriftBackupAlbum
onPopInvokedWithResult: (didPop, _) async {
if (!didPop) {
await _handlePagePopped();
+
+ final user = ref.read(currentUserProvider);
+ if (user == null) {
+ return;
+ }
+
+ final isBackupEnabled = ref.read(appSettingsServiceProvider).getSetting(AppSettingsEnum.enableBackup);
+ await ref.read(driftBackupProvider.notifier).getBackupStatus(user.id);
+ final currentTotalAssetCount = ref.read(driftBackupProvider.select((p) => p.totalCount));
+ final totalChanged = currentTotalAssetCount != _initialTotalAssetCount;
+ final backupNotifier = ref.read(driftBackupProvider.notifier);
+ final backgroundSync = ref.read(backgroundSyncProvider);
+ final nativeSync = ref.read(nativeSyncApiProvider);
+ if (totalChanged) {
+ // Waits for hashing to be cancelled before starting a new one
+ unawaited(nativeSync.cancelHashing().whenComplete(() => backgroundSync.hashAssets()));
+ if (isBackupEnabled) {
+ unawaited(backupNotifier.cancel().whenComplete(() => backupNotifier.startBackup(user.id)));
+ }
+ }
+
Navigator.of(context).pop();
}
},
diff --git a/mobile/lib/platform/native_sync_api.g.dart b/mobile/lib/platform/native_sync_api.g.dart
index 6fc96f5046..01237f8c19 100644
--- a/mobile/lib/platform/native_sync_api.g.dart
+++ b/mobile/lib/platform/native_sync_api.g.dart
@@ -205,6 +205,45 @@ class SyncDelta {
int get hashCode => Object.hashAll(_toList());
}
+class HashResult {
+ HashResult({required this.assetId, this.error, this.hash});
+
+ String assetId;
+
+ String? error;
+
+ String? hash;
+
+ List<Object?> _toList() {
+ return <Object?>[assetId, error, hash];
+ }
+
+ Object encode() {
+ return _toList();
+ }
+
+ static HashResult decode(Object result) {
+ result as List<Object?>;
+ return HashResult(assetId: result[0]! as String, error: result[1] as String?, hash: result[2] as String?);
+ }
+
+ @override
+ // ignore: avoid_equals_and_hash_code_on_mutable_classes
+ bool operator ==(Object other) {
+ if (other is! HashResult || other.runtimeType != runtimeType) {
+ return false;
+ }
+ if (identical(this, other)) {
+ return true;
+ }
+ return _deepEquals(encode(), other.encode());
+ }
+
+ @override
+ // ignore: avoid_equals_and_hash_code_on_mutable_classes
+ int get hashCode => Object.hashAll(_toList());
+}
+
class _PigeonCodec extends StandardMessageCodec {
const _PigeonCodec();
@override
@@ -221,6 +260,9 @@ class _PigeonCodec extends StandardMessageCodec {
} else if (value is SyncDelta) {
buffer.putUint8(131);
writeValue(buffer, value.encode());
+ } else if (value is HashResult) {
+ buffer.putUint8(132);
+ writeValue(buffer, value.encode());
} else {
super.writeValue(buffer, value);
}
@@ -235,6 +277,8 @@ class _PigeonCodec extends StandardMessageCodec {
return PlatformAlbum.decode(readValue(buffer)!);
case 131:
return SyncDelta.decode(readValue(buffer)!);
+ case 132:
+ return HashResult.decode(readValue(buffer)!);
default:
return super.readValueOfType(type, buffer);
}
@@ -468,15 +512,15 @@ class NativeSyncApi {
}
}
- Future<List<Uint8List?>> hashPaths(List<String> paths) async {
+ Future<List<HashResult>> hashAssets(List<String> assetIds, {bool allowNetworkAccess = false}) async {
final String pigeonVar_channelName =
- 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$pigeonVar_messageChannelSuffix';
+ 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets$pigeonVar_messageChannelSuffix';
final BasicMessageChannel<Object?> pigeonVar_channel = BasicMessageChannel<Object?>(
pigeonVar_channelName,
pigeonChannelCodec,
binaryMessenger: pigeonVar_binaryMessenger,
);
- final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[paths]);
+ final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(<Object?>[assetIds, allowNetworkAccess]);
final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;
if (pigeonVar_replyList == null) {
throw _createConnectionError(pigeonVar_channelName);
@@ -492,7 +536,30 @@ class NativeSyncApi {
message: 'Host platform returned null value for non-null return value.',
);
} else {
- return (pigeonVar_replyList[0] as List<Object?>?)!.cast<Uint8List?>();
+ return (pigeonVar_replyList[0] as List<Object?>?)!.cast<HashResult>();
+ }
+ }
+
+ Future<void> cancelHashing() async {
+ final String pigeonVar_channelName =
+ 'dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing$pigeonVar_messageChannelSuffix';
+ final BasicMessageChannel<Object?> pigeonVar_channel = BasicMessageChannel<Object?>(
+ pigeonVar_channelName,
+ pigeonChannelCodec,
+ binaryMessenger: pigeonVar_binaryMessenger,
+ );
+ final Future<Object?> pigeonVar_sendFuture = pigeonVar_channel.send(null);
+ final List<Object?>? pigeonVar_replyList = await pigeonVar_sendFuture as List<Object?>?;
+ if (pigeonVar_replyList == null) {
+ throw _createConnectionError(pigeonVar_channelName);
+ } else if (pigeonVar_replyList.length > 1) {
+ throw PlatformException(
+ code: pigeonVar_replyList[0]! as String,
+ message: pigeonVar_replyList[1] as String?,
+ details: pigeonVar_replyList[2],
+ );
+ } else {
+ return;
}
}
}
diff --git a/mobile/lib/providers/infrastructure/sync.provider.dart b/mobile/lib/providers/infrastructure/sync.provider.dart
index ddc6eed441..f03754505c 100644
--- a/mobile/lib/providers/infrastructure/sync.provider.dart
+++ b/mobile/lib/providers/infrastructure/sync.provider.dart
@@ -10,7 +10,6 @@ import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart';
import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
-import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
final syncStreamServiceProvider = Provider(
(ref) => SyncStreamService(
@@ -35,7 +34,6 @@ final hashServiceProvider = Provider(
(ref) => HashService(
localAlbumRepository: ref.watch(localAlbumRepository),
localAssetRepository: ref.watch(localAssetRepository),
- storageRepository: ref.watch(storageRepositoryProvider),
nativeSyncApi: ref.watch(nativeSyncApiProvider),
),
);
diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart
index 48302be79c..9d1f4e51e8 100644
--- a/mobile/lib/services/hash.service.dart
+++ b/mobile/lib/services/hash.service.dart
@@ -16,9 +16,10 @@ class HashService {
required IsarDeviceAssetRepository deviceAssetRepository,
required BackgroundService backgroundService,
this.batchSizeLimit = kBatchHashSizeLimit,
- this.batchFileLimit = kBatchHashFileLimit,
+ int? batchFileLimit,
}) : _deviceAssetRepository = deviceAssetRepository,
- _backgroundService = backgroundService;
+ _backgroundService = backgroundService,
+ batchFileLimit = batchFileLimit ?? kBatchHashFileLimit;
final IsarDeviceAssetRepository _deviceAssetRepository;
final BackgroundService _backgroundService;
diff --git a/mobile/pigeon/native_sync_api.dart b/mobile/pigeon/native_sync_api.dart
index e84c814c3d..ac08a68ca3 100644
--- a/mobile/pigeon/native_sync_api.dart
+++ b/mobile/pigeon/native_sync_api.dart
@@ -71,6 +71,14 @@ class SyncDelta {
});
}
+class HashResult {
+ final String assetId;
+ final String? error;
+ final String? hash;
+
+ const HashResult({required this.assetId, this.error, this.hash});
+}
+
@HostApi()
abstract class NativeSyncApi {
bool shouldFullSync();
@@ -94,6 +102,9 @@ abstract class NativeSyncApi {
@TaskQueue(type: TaskQueueType.serialBackgroundThread)
List<PlatformAsset> getAssetsForAlbum(String albumId, {int? updatedTimeCond});
+ @async
@TaskQueue(type: TaskQueueType.serialBackgroundThread)
- List<Uint8List?> hashPaths(List<String> paths);
+ List<HashResult> hashAssets(List<String> assetIds, {bool allowNetworkAccess = false});
+
+ void cancelHashing();
}
diff --git a/mobile/test/domain/services/hash_service_test.dart b/mobile/test/domain/services/hash_service_test.dart
index 7969131e7f..20d60b6866 100644
--- a/mobile/test/domain/services/hash_service_test.dart
+++ b/mobile/test/domain/services/hash_service_test.dart
@@ -1,11 +1,7 @@
-import 'dart:convert';
-import 'dart:io';
-import 'dart:typed_data';
-
import 'package:flutter_test/flutter_test.dart';
-import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
+import 'package:immich_mobile/domain/models/album/local_album.model.dart';
import 'package:immich_mobile/domain/services/hash.service.dart';
-import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
+import 'package:immich_mobile/platform/native_sync_api.g.dart';
import 'package:mocktail/mocktail.dart';
import '../../fixtures/album.stub.dart';
@@ -13,220 +9,181 @@ import '../../fixtures/asset.stub.dart';
import '../../infrastructure/repository.mock.dart';
import '../service.mock.dart';
-class MockFile extends Mock implements File {}
-
void main() {
late HashService sut;
late MockLocalAlbumRepository mockAlbumRepo;
late MockLocalAssetRepository mockAssetRepo;
- late MockStorageRepository mockStorageRepo;
late MockNativeSyncApi mockNativeApi;
- final sortBy = {SortLocalAlbumsBy.backupSelection, SortLocalAlbumsBy.isIosSharedAlbum};
setUp(() {
mockAlbumRepo = MockLocalAlbumRepository();
mockAssetRepo = MockLocalAssetRepository();
- mockStorageRepo = MockStorageRepository();
mockNativeApi = MockNativeSyncApi();
sut = HashService(
localAlbumRepository: mockAlbumRepo,
localAssetRepository: mockAssetRepo,
- storageRepository: mockStorageRepo,
nativeSyncApi: mockNativeApi,
);
registerFallbackValue(LocalAlbumStub.recent);
registerFallbackValue(LocalAssetStub.image1);
+ registerFallbackValue(<String, String>{});
when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
- when(() => mockStorageRepo.clearCache()).thenAnswer((_) async => {});
});
group('HashService hashAssets', () {
test('skips albums with no assets to hash', () async {
when(
- () => mockAlbumRepo.getAll(sortBy: sortBy),
+ () => mockAlbumRepo.getBackupAlbums(),
).thenAnswer((_) async => [LocalAlbumStub.recent.copyWith(assetCount: 0)]);
when(() => mockAlbumRepo.getAssetsToHash(LocalAlbumStub.recent.id)).thenAnswer((_) async => []);
await sut.hashAssets();
- verifyNever(() => mockStorageRepo.getFileForAsset(any()));
- verifyNever(() => mockNativeApi.hashPaths(any()));
+ verifyNever(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess')));
});
});
group('HashService _hashAssets', () {
- test('skips assets without files', () async {
+ test('skips empty batches', () async {
final album = LocalAlbumStub.recent;
- final asset = LocalAssetStub.image1;
- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);
- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);
- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => null);
+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);
+ when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => []);
await sut.hashAssets();
- verifyNever(() => mockNativeApi.hashPaths(any()));
+ verifyNever(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess')));
});
test('processes assets when available', () async {
final album = LocalAlbumStub.recent;
final asset = LocalAssetStub.image1;
- final mockFile = MockFile();
- final hash = Uint8List.fromList(List.generate(20, (i) => i));
-
- when(() => mockFile.length()).thenAnswer((_) async => 1000);
- when(() => mockFile.path).thenReturn('image-path');
- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);
+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);
when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);
- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile);
- when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [hash]);
+ when(
+ () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false),
+ ).thenAnswer((_) async => [HashResult(assetId: asset.id, hash: 'test-hash')]);
await sut.hashAssets();
- verify(() => mockNativeApi.hashPaths(['image-path'])).called(1);
- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
+ verify(() => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false)).called(1);
+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;
expect(captured.length, 1);
- expect(captured[0].checksum, base64.encode(hash));
+ expect(captured[asset.id], 'test-hash');
});
test('handles failed hashes', () async {
final album = LocalAlbumStub.recent;
final asset = LocalAssetStub.image1;
- final mockFile = MockFile();
- when(() => mockFile.length()).thenAnswer((_) async => 1000);
- when(() => mockFile.path).thenReturn('image-path');
- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);
+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);
when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);
- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile);
- when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [null]);
- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
+ when(
+ () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false),
+ ).thenAnswer((_) async => [HashResult(assetId: asset.id, error: 'Failed to hash')]);
await sut.hashAssets();
- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;
expect(captured.length, 0);
});
- test('handles invalid hash length', () async {
+ test('handles null hash results', () async {
final album = LocalAlbumStub.recent;
final asset = LocalAssetStub.image1;
- final mockFile = MockFile();
- when(() => mockFile.length()).thenAnswer((_) async => 1000);
- when(() => mockFile.path).thenReturn('image-path');
- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);
+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);
when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset]);
- when(() => mockStorageRepo.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile);
-
- final invalidHash = Uint8List.fromList([1, 2, 3]);
- when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer((_) async => [invalidHash]);
- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
+ when(
+ () => mockNativeApi.hashAssets([asset.id], allowNetworkAccess: false),
+ ).thenAnswer((_) async => [HashResult(assetId: asset.id, hash: null)]);
await sut.hashAssets();
- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;
expect(captured.length, 0);
});
- test('batches by file count limit', () async {
+ test('batches by size limit', () async {
+ const batchSize = 2;
final sut = HashService(
localAlbumRepository: mockAlbumRepo,
localAssetRepository: mockAssetRepo,
- storageRepository: mockStorageRepo,
nativeSyncApi: mockNativeApi,
- batchFileLimit: 1,
+ batchSize: batchSize,
);
final album = LocalAlbumStub.recent;
final asset1 = LocalAssetStub.image1;
final asset2 = LocalAssetStub.image2;
- final mockFile1 = MockFile();
- final mockFile2 = MockFile();
- when(() => mockFile1.length()).thenAnswer((_) async => 100);
- when(() => mockFile1.path).thenReturn('path-1');
- when(() => mockFile2.length()).thenAnswer((_) async => 100);
- when(() => mockFile2.path).thenReturn('path-2');
-
- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);
- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);
- when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1);
- when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2);
+ final asset3 = LocalAssetStub.image1.copyWith(id: 'image3', name: 'image3.jpg');
+
+ final capturedCalls = <List<String>>[];
- final hash = Uint8List.fromList(List.generate(20, (i) => i));
- when(() => mockNativeApi.hashPaths(any())).thenAnswer((_) async => [hash]);
when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);
+ when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2, asset3]);
+ when(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))).thenAnswer((
+ invocation,
+ ) async {
+ final assetIds = invocation.positionalArguments[0] as List<String>;
+ capturedCalls.add(List<String>.from(assetIds));
+ return assetIds.map((id) => HashResult(assetId: id, hash: '$id-hash')).toList();
+ });
await sut.hashAssets();
- verify(() => mockNativeApi.hashPaths(['path-1'])).called(1);
- verify(() => mockNativeApi.hashPaths(['path-2'])).called(1);
+ expect(capturedCalls.length, 2, reason: 'Should make exactly 2 calls to hashAssets');
+ expect(capturedCalls[0], [asset1.id, asset2.id], reason: 'First call should batch the first two assets');
+ expect(capturedCalls[1], [asset3.id], reason: 'Second call should have the remaining asset');
+
verify(() => mockAssetRepo.updateHashes(any())).called(2);
});
- test('batches by size limit', () async {
- final sut = HashService(
- localAlbumRepository: mockAlbumRepo,
- localAssetRepository: mockAssetRepo,
- storageRepository: mockStorageRepo,
- nativeSyncApi: mockNativeApi,
- batchSizeLimit: 80,
- );
-
+ test('handles mixed success and failure in batch', () async {
final album = LocalAlbumStub.recent;
final asset1 = LocalAssetStub.image1;
final asset2 = LocalAssetStub.image2;
- final mockFile1 = MockFile();
- final mockFile2 = MockFile();
- when(() => mockFile1.length()).thenAnswer((_) async => 100);
- when(() => mockFile1.path).thenReturn('path-1');
- when(() => mockFile2.length()).thenAnswer((_) async => 100);
- when(() => mockFile2.path).thenReturn('path-2');
-
- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);
- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);
- when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1);
- when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2);
- final hash = Uint8List.fromList(List.generate(20, (i) => i));
- when(() => mockNativeApi.hashPaths(any())).thenAnswer((_) async => [hash]);
- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [album]);
+ when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);
+ when(() => mockNativeApi.hashAssets([asset1.id, asset2.id], allowNetworkAccess: false)).thenAnswer(
+ (_) async => [
+ HashResult(assetId: asset1.id, hash: 'asset1-hash'),
+ HashResult(assetId: asset2.id, error: 'Failed to hash asset2'),
+ ],
+ );
await sut.hashAssets();
- verify(() => mockNativeApi.hashPaths(['path-1'])).called(1);
- verify(() => mockNativeApi.hashPaths(['path-2'])).called(1);
- verify(() => mockAssetRepo.updateHashes(any())).called(2);
+ final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as Map<String, String>;
+ expect(captured.length, 1);
+ expect(captured[asset1.id], 'asset1-hash');
});
- test('handles mixed success and failure in batch', () async {
- final album = LocalAlbumStub.recent;
+ test('uses allowNetworkAccess based on album backup selection', () async {
+ final selectedAlbum = LocalAlbumStub.recent.copyWith(backupSelection: BackupSelection.selected);
+ final nonSelectedAlbum = LocalAlbumStub.recent.copyWith(id: 'album2', backupSelection: BackupSelection.excluded);
final asset1 = LocalAssetStub.image1;
final asset2 = LocalAssetStub.image2;
- final mockFile1 = MockFile();
- final mockFile2 = MockFile();
- when(() => mockFile1.length()).thenAnswer((_) async => 100);
- when(() => mockFile1.path).thenReturn('path-1');
- when(() => mockFile2.length()).thenAnswer((_) async => 100);
- when(() => mockFile2.path).thenReturn('path-2');
-
- when(() => mockAlbumRepo.getAll(sortBy: sortBy)).thenAnswer((_) async => [album]);
- when(() => mockAlbumRepo.getAssetsToHash(album.id)).thenAnswer((_) async => [asset1, asset2]);
- when(() => mockStorageRepo.getFileForAsset(asset1.id)).thenAnswer((_) async => mockFile1);
- when(() => mockStorageRepo.getFileForAsset(asset2.id)).thenAnswer((_) async => mockFile2);
- final validHash = Uint8List.fromList(List.generate(20, (i) => i));
- when(() => mockNativeApi.hashPaths(['path-1', 'path-2'])).thenAnswer((_) async => [validHash, null]);
- when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
+ when(() => mockAlbumRepo.getBackupAlbums()).thenAnswer((_) async => [selectedAlbum, nonSelectedAlbum]);
+ when(() => mockAlbumRepo.getAssetsToHash(selectedAlbum.id)).thenAnswer((_) async => [asset1]);
+ when(() => mockAlbumRepo.getAssetsToHash(nonSelectedAlbum.id)).thenAnswer((_) async => [asset2]);
+ when(() => mockNativeApi.hashAssets(any(), allowNetworkAccess: any(named: 'allowNetworkAccess'))).thenAnswer((
+ invocation,
+ ) async {
+ final assetIds = invocation.positionalArguments[0] as List<String>;
+ return assetIds.map((id) => HashResult(assetId: id, hash: '$id-hash')).toList();
+ });
await sut.hashAssets();
- final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
- expect(captured.length, 1);
- expect(captured.first.id, asset1.id);
+ verify(() => mockNativeApi.hashAssets([asset1.id], allowNetworkAccess: true)).called(1);
+ verify(() => mockNativeApi.hashAssets([asset2.id], allowNetworkAccess: false)).called(1);
});
});
}