π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Unify downloaded assets
Manav Rathi16cefb5b76be
diff --git a/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xml b/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xmlindex 1f4f111870..31cbb53937 100644--- a/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xml+++ b/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xml@@ -27,7 +27,7 @@</activity><service- android:name=".llm.ModelDownloadJobService"+ android:name=".assets.AssetDownloadJobService"android:exported="false"android:permission="android.permission.BIND_JOB_SERVICE" />diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex 6c8286de8c..b15b638e02 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -11,7 +11,7 @@ import io.ente.ensu.bindings.Transcriberimport io.ente.ensu.device.AndroidDeviceCapabilityProviderimport io.ente.ensu.llm.DownloadPhaseimport io.ente.ensu.llm.LlmProvider-import io.ente.ensu.llm.ModelDownloader+import io.ente.ensu.assets.AssetStoreimport io.ente.ensu.logging.FileLogRepositoryimport io.ente.ensu.chat.Attachmentimport io.ente.ensu.chat.ChatMessage@@ -35,7 +35,7 @@ class AppStore(private val chatRepository: ChatRepository,private val llmProvider: LlmProvider,knowledgeProvider: KnowledgeProvider,- val modelDownloader: ModelDownloader,+ val assetStore: AssetStore,val transcriber: Transcriber,private val deviceCapabilityProvider: AndroidDeviceCapabilityProvider,val configDefaults: ConfigDefaults,@@ -55,14 +55,13 @@ class AppStore(logRepository = logRepository)private val modelSettingsActions =- ModelSettingsActions(_state, sessionPreferences, llmProvider, modelDownloader, logRepository, configDefaults)+ ModelSettingsActions(_state, sessionPreferences, llmProvider, logRepository, configDefaults)private val chatActions = ChatStoreActions(state = _state,sessionPreferences = sessionPreferences,chatRepository = chatRepository,llmProvider = llmProvider,knowledgeProvider = knowledgeProvider,- modelDownloader = modelDownloader,clock = clock,logRepository = logRepository,messageStore = messageStore,diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.ktindex a528fa908c..4d66c7c4c2 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt@@ -12,7 +12,7 @@ import io.ente.ensu.settings.SessionPreferencesDataStoreimport io.ente.ensu.chat.ChatRepositoryimport io.ente.ensu.config.loadConfigDefaultsimport io.ente.ensu.llm.LlmProvider-import io.ente.ensu.llm.ModelDownloader+import io.ente.ensu.assets.AssetStoreimport io.ente.ensu.llm.ModelSettingsStateimport io.ente.ensu.logging.FileLogRepositoryimport io.ente.ensu.storage.CredentialStore@@ -37,11 +37,11 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {val configDefaults = loadConfigDefaults()val logRepository = FileLogRepository(application)- private val modelDownloader = (application as EnsuApplication).modelDownloader- private val _isReady = MutableStateFlow(!modelDownloader.needsMigration())+ private val assetStore = (application as EnsuApplication).assetStore+ private val _isReady = MutableStateFlow(!assetStore.needsMigration())val isReady = _isReady.asStateFlow()private val llmProvider = LlmProvider(- downloader = modelDownloader,+ assetStore = assetStore,transcriber = transcriber,deviceCapabilityProvider = deviceCapabilityProvider,knowledgeEmbedding = configDefaults.knowledgeEmbedding@@ -55,7 +55,7 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {chatRepository = chatRepository,llmProvider = llmProvider,knowledgeProvider = knowledgeProvider,- modelDownloader = modelDownloader,+ assetStore = assetStore,transcriber = transcriber,deviceCapabilityProvider = deviceCapabilityProvider,configDefaults = configDefaults,@@ -69,7 +69,7 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {runCatching {advancedSettingsDataStore.migrateLegacyModelSelection { url, mmproj ->withContext(Dispatchers.IO) {- modelDownloader.migrate(url, mmproj)+ assetStore.migrate(url, mmproj)}}}diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.ktindex 62694336f8..e2aeb57aed 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt@@ -2,16 +2,21 @@ package io.ente.ensuimport android.app.Applicationimport io.ente.ensu.bindings.Transcriber+import io.ente.ensu.bindings.transcriptionModelAsset+import io.ente.ensu.bindings.voiceActivityModelAsset+import io.ente.ensu.assets.AssetStoreimport io.ente.ensu.knowledge.KnowledgeProvider-import io.ente.ensu.llm.ModelDownloaderclass EnsuApplication : Application() {- val modelDownloader by lazy { ModelDownloader(this) }- val knowledgeProvider by lazy { KnowledgeProvider(noBackupFilesDir.resolve("knowledge")) }+ val assetStore by lazy { AssetStore(this) }+ val knowledgeProvider by lazy { KnowledgeProvider(assetStore) }val transcriber by lazy {+ val store = assetStore+ val transcription = transcriptionModelAsset()+ val voiceActivity = voiceActivityModelAsset()Transcriber(- modelDownloader.modelDir(modelDownloader.transcriptionTarget).absolutePath,- modelDownloader.voiceActivityModelPath().absolutePath+ store.assetDir(transcription).absolutePath,+ store.voiceActivityModelPath(voiceActivity).absolutePath)}}diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.ktindex 734237348f..2a0c966097 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt@@ -118,7 +118,7 @@ internal fun HomeNavigation() {ChatView(chatState = appState.chat,- modelDownloader = store.modelDownloader,+ assetStore = store.assetStore,transcriber = store.transcriber,isDrawerOpen = drawerState.currentValue == DrawerValue.Open,onMessageChange = store::updateMessageText,diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloadJobService.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetDownloadJobService.ktsimilarity index 91%rename from mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloadJobService.ktrename to mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetDownloadJobService.ktindex aff8906d07..f68981e1f7 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloadJobService.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetDownloadJobService.kt@@ -1,4 +1,4 @@-package io.ente.ensu.llm+package io.ente.ensu.assetsimport android.app.Notificationimport android.app.job.JobInfo@@ -16,7 +16,7 @@ import androidx.core.app.NotificationCompatimport androidx.core.app.NotificationManagerCompat@RequiresApi(Build.VERSION_CODES.UPSIDE_DOWN_CAKE)-class ModelDownloadJobService : JobService() {+class AssetDownloadJobService : JobService() {override fun onStartJob(params: JobParameters): Boolean {synchronized(lock) {if (!downloadActive) return false@@ -45,8 +45,8 @@ class ModelDownloadJobService : JobService() {}companion object {- private const val TAG = "ModelDownloadJob"- private const val CHANNEL_ID = "model-download"+ private const val TAG = "AssetDownloadJob"+ private const val CHANNEL_ID = "asset-download"private const val NOTIFICATION_ID = 1private const val JOB_ID = 1private const val NOTIFY_INTERVAL_MS = 1000L@@ -56,7 +56,7 @@ class ModelDownloadJobService : JobService() {private var onCancel: (() -> Unit)? = nullprivate var downloadActive = falseprivate var lastNotifyMs = 0L- private var runningJob: ModelDownloadJobService? = null+ private var runningJob: AssetDownloadJobService? = nullprivate var runningParams: JobParameters? = nullfun attach(context: Context) {@@ -75,7 +75,7 @@ class ModelDownloadJobService : JobService() {Log.w(TAG, "JobScheduler unavailable")return}- val job = JobInfo.Builder(JOB_ID, ComponentName(appContext, ModelDownloadJobService::class.java))+ val job = JobInfo.Builder(JOB_ID, ComponentName(appContext, AssetDownloadJobService::class.java)).setUserInitiated(true).setRequiredNetworkType(JobInfo.NETWORK_TYPE_ANY).build()@@ -90,7 +90,7 @@ class ModelDownloadJobService : JobService() {fun update(percent: Int, indeterminate: Boolean) {if (Build.VERSION.SDK_INT < Build.VERSION_CODES.UPSIDE_DOWN_CAKE) return- val job: ModelDownloadJobService+ val job: AssetDownloadJobServiceval params: JobParameterssynchronized(lock) {if (!downloadActive) return@@ -110,7 +110,7 @@ class ModelDownloadJobService : JobService() {fun end() {if (Build.VERSION.SDK_INT < Build.VERSION_CODES.UPSIDE_DOWN_CAKE) return- val job: ModelDownloadJobService?+ val job: AssetDownloadJobService?val params: JobParameters?synchronized(lock) {downloadActive = false@@ -134,12 +134,12 @@ class ModelDownloadJobService : JobService() {): Notification {NotificationManagerCompat.from(context).createNotificationChannel(NotificationChannelCompat.Builder(CHANNEL_ID, NotificationManagerCompat.IMPORTANCE_LOW)- .setName("Model downloads")+ .setName("Asset downloads").build())return NotificationCompat.Builder(context, CHANNEL_ID).setSmallIcon(android.R.drawable.stat_sys_download)- .setContentTitle("Downloading model")+ .setContentTitle("Downloading assets").setProgress(100, percent, indeterminate).setOngoing(true).setOnlyAlertOnce(true)diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.ktnew file mode 100644index 0000000000..73ab0b8db6--- /dev/null+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt@@ -0,0 +1,135 @@+package io.ente.ensu.assets++import android.content.Context+import android.os.Environment+import android.util.Log+import io.ente.ensu.bindings.AssetDownloadCallback+import io.ente.ensu.bindings.AssetDownloadProgress+import io.ente.ensu.bindings.Asset+import io.ente.ensu.bindings.AssetStoreCore+import io.ente.ensu.bindings.CancellationToken+import io.ente.ensu.bindings.LegacyAssets+import io.ente.ensu.bindings.KnowledgeReconciliation+import io.ente.ensu.bindings.cleanupObsoleteKnowledgePackRevisions+import io.ente.ensu.bindings.migrateEnsuAssets+import io.ente.ensu.bindings.needsAssetMigration+import io.ente.ensu.bindings.reconcileKnowledgePack+import io.ente.ensu.bindings.uniffiEnsureInitialized+import java.io.File+import kotlinx.coroutines.CancellationException+import kotlinx.coroutines.Dispatchers+import kotlinx.coroutines.async+import kotlinx.coroutines.coroutineScope+import kotlinx.coroutines.sync.Mutex+import kotlinx.coroutines.withContext++class AssetStore(context: Context) {+ private val appContext = context.applicationContext+ private val assetsDir = File(appContext.noBackupFilesDir, "assets")+ private val legacyLlmDir = appContext.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS)+ ?.let { File(it, "llm") }+ private val legacyTranscriptionDir = File(appContext.dataDir, "app_ensu_transcription_models")+ private val downloadMutex = Mutex()+ private val core: AssetStoreCore++ init {+ uniffiEnsureInitialized()+ core = AssetStoreCore(assetsDir.absolutePath)+ AssetDownloadJobService.attach(appContext)+ }++ fun needsMigration(): Boolean = needsAssetMigration(legacyAssets(null, null))++ fun migrate(legacyModelUrl: String?, legacyMmprojUrl: String?): String? {+ File(appContext.filesDir, "llm").deleteRecursively()+ return migrateEnsuAssets(+ assetsDir.absolutePath,+ legacyAssets(legacyModelUrl, legacyMmprojUrl)+ )+ }++ private fun legacyAssets(modelUrl: String?, mmprojUrl: String?) = LegacyAssets(+ llmDir = legacyLlmDir?.absolutePath,+ transcriptionDir = legacyTranscriptionDir.absolutePath,+ modelUrl = modelUrl,+ mmprojUrl = mmprojUrl+ )++ val isDownloadActive: Boolean get() = core.isDownloadActive()++ fun assetDir(asset: Asset): File = File(core.assetDir(asset))++ fun llmModelPath(asset: Asset): File? =+ core.llmModelPath(asset)?.let(::File)++ fun llmMmprojPath(asset: Asset): File? =+ core.llmMmprojPath(asset)?.let(::File)++ fun voiceActivityModelPath(asset: Asset): File = File(core.voiceActivityModelPath(asset))++ fun isDownloaded(asset: Asset): Boolean = core.isDownloaded(asset)++ fun removeDownloaded(asset: Asset): Boolean = core.removeDownloaded(asset)++ internal fun reconcileKnowledge(stableId: String): KnowledgeReconciliation =+ reconcileKnowledgePack(core, stableId)++ internal fun cleanupKnowledgeRevisions(stableId: String, activeIdentity: String) =+ cleanupObsoleteKnowledgePackRevisions(core, stableId, activeIdentity)++ suspend fun estimateDownloadSize(asset: Asset): Long? = withContext(Dispatchers.IO) {+ core.estimatedDownloadSize(asset)+ }++ suspend fun download(+ assets: List<Asset>,+ onProgress: (AssetDownloadProgress) -> Unit+ ): Unit = withContext(Dispatchers.IO) {+ if (!downloadMutex.tryLock()) throw DownloadAlreadyActiveException()+ try {+ downloadLocked(assets, onProgress)+ } finally {+ downloadMutex.unlock()+ }+ }++ private suspend fun downloadLocked(+ assets: List<Asset>,+ onProgress: (AssetDownloadProgress) -> Unit+ ) {+ if (assets.all { core.isDownloaded(it) }) return++ val token = CancellationToken()+ AssetDownloadJobService.begin { token.cancel() }+ try {+ coroutineScope {+ val download = async {+ core.download(+ assets,+ object : AssetDownloadCallback {+ override fun onProgress(progress: AssetDownloadProgress) {+ progress.logLine?.let { Log.i("AssetStore", it) }+ AssetDownloadJobService.update(+ progress.percentage.toInt(),+ progress.totalBytes == null+ )+ onProgress(progress)+ }+ },+ token+ )+ }+ try {+ download.await()+ } catch (e: CancellationException) {+ token.cancel()+ throw e+ }+ }+ } finally {+ AssetDownloadJobService.end()+ }+ }+}++private class DownloadAlreadyActiveException : Exception("An asset download is already active")diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex f36d6a082f..e5a4f0233b 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -9,11 +9,11 @@ import io.ente.ensu.llm.DownloadPhaseimport io.ente.ensu.llm.LlmMessageimport io.ente.ensu.llm.LlmMessageRoleimport io.ente.ensu.llm.LlmModelSelection-import io.ente.ensu.llm.ModelDownloaderimport io.ente.ensu.llm.LlmProviderimport io.ente.ensu.chat.Attachmentimport io.ente.ensu.chat.ChatMessageimport io.ente.ensu.chat.ChatSession+import io.ente.ensu.bindings.AssetDownloadExceptionimport io.ente.ensu.bindings.DbExceptionimport io.ente.ensu.bindings.LlmExceptionimport io.ente.ensu.bindings.ConfigDefaults@@ -50,7 +50,6 @@ internal class ChatStoreActions(private val chatRepository: ChatRepository,private val llmProvider: LlmProvider,private val knowledgeProvider: KnowledgeProvider,- private val modelDownloader: ModelDownloader,private val clock: () -> Long,private val logRepository: FileLogRepository,private val messageStore: MutableMap<String, MutableList<ChatMessage>>,@@ -533,7 +532,8 @@ internal class ChatStoreActions(}} catch (err: Throwable) {val cancelled = err is kotlinx.coroutines.CancellationException ||- err is LlmException.Cancelled+ err is LlmException.Cancelled ||+ err is AssetDownloadException.Cancelledif (!isActive()) return@launchstreamingParentId = nullstate.update { appState ->@@ -915,7 +915,7 @@ internal class ChatStoreActions(if (!state.value.chat.deviceCapability.isChatSupported()) {return sessionTitleFromText(fallback, fallback = fallback)}- if (!modelDownloader.isDownloaded(selection.modelTarget)) {+ if (!llmProvider.isChatModelReady(selection)) {return sessionTitleFromText(fallback, fallback = fallback)}val cleanedInput = sanitizeTitleText(input)diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.ktindex e4aee15fbe..b5f45a9a34 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt@@ -46,7 +46,7 @@ import androidx.compose.ui.platform.LocalFocusManagerimport androidx.compose.ui.unit.dpimport androidx.core.content.ContextCompatimport io.ente.ensu.bindings.Transcriber-import io.ente.ensu.llm.ModelDownloader+import io.ente.ensu.assets.AssetStoreimport io.ente.ensu.designsystem.EnsuColorimport io.ente.ensu.designsystem.EnsuSpacingimport io.ente.ensu.device.ChatDeviceCapability@@ -59,7 +59,7 @@ import kotlinx.coroutines.delay@Composablefun ChatView(chatState: ChatState,- modelDownloader: ModelDownloader,+ assetStore: AssetStore,transcriber: Transcriber,isDrawerOpen: Boolean,onMessageChange: (String) -> Unit,@@ -89,7 +89,7 @@ fun ChatView(val isChatUnsupported = unsupportedCapability != nullvar pendingVoiceSessionKey by remember { mutableStateOf<String?>(null) }val voiceController = rememberVoiceTranscriptionController(- modelDownloader = modelDownloader,+ assetStore = assetStore,transcriber = transcriber,onTranscript = { transcript ->latestOnMessageChange(appendVoiceTranscript(latestMessageText, transcript))diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.ktindex f6efd855ce..417b699c0b 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt@@ -14,11 +14,13 @@ import androidx.compose.runtime.setValueimport androidx.compose.ui.platform.LocalLifecycleOwnerimport androidx.lifecycle.Lifecycleimport androidx.lifecycle.LifecycleEventObserver+import io.ente.ensu.bindings.AssetDownloadExceptionimport io.ente.ensu.bindings.LlmExceptionimport io.ente.ensu.bindings.Transcriberimport io.ente.ensu.bindings.TranscriptionException-import io.ente.ensu.bindings.uniffiEnsureInitialized-import io.ente.ensu.llm.ModelDownloader+import io.ente.ensu.bindings.transcriptionModelAsset+import io.ente.ensu.bindings.voiceActivityModelAsset+import io.ente.ensu.assets.AssetStoreimport kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.CoroutineScopeimport kotlinx.coroutines.Dispatchers@@ -67,13 +69,13 @@ internal fun VoiceInputState.statusText(): String? = when (this) {@Composableinternal fun rememberVoiceTranscriptionController(- modelDownloader: ModelDownloader,+ assetStore: AssetStore,transcriber: Transcriber,onTranscript: (String) -> Unit): VoiceTranscriptionController {val lifecycleOwner = LocalLifecycleOwner.currentval controller = remember(transcriber) {- VoiceTranscriptionController(modelDownloader, transcriber, onTranscript)+ VoiceTranscriptionController(assetStore, transcriber, onTranscript)}DisposableEffect(controller, lifecycleOwner) {val observer = LifecycleEventObserver { _, event ->@@ -93,13 +95,13 @@ internal fun rememberVoiceTranscriptionController(}internal class VoiceTranscriptionController(- private val modelDownloader: ModelDownloader,+ private val assetStore: AssetStore,private val transcriber: Transcriber,private val onTranscript: (String) -> Unit) {- private val modelTargets = listOf(- modelDownloader.transcriptionTarget,- modelDownloader.voiceActivityTarget+ private val modelAssets = listOf(+ transcriptionModelAsset(),+ voiceActivityModelAsset())private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)@@ -146,6 +148,13 @@ internal class VoiceTranscriptionController(} catch (error: TranscriptionException) {Log.w(TAG, "Voice model preparation failed: ${error.message}", error)state = VoiceInputState.Error(transcriptionErrorMessage(error))+ } catch (error: AssetDownloadException) {+ if (error is AssetDownloadException.Cancelled) {+ state = VoiceInputState.Idle+ return@launch+ }+ Log.w(TAG, "Voice model download failed: ${error.message}", error)+ state = VoiceInputState.Error(downloadErrorMessage())} catch (error: LlmException) {if (error is LlmException.Cancelled) {state = VoiceInputState.Idle@@ -276,8 +285,6 @@ internal class VoiceTranscriptionController(ensureTranscriptionModelDownloaded()awaitTranscriptionModelPreload()val transcript = withContext(Dispatchers.IO) {- uniffiEnsureInitialized()-withContext(Dispatchers.Main.immediate) {state = VoiceInputState.Transcribing}@@ -301,6 +308,13 @@ internal class VoiceTranscriptionController(} catch (error: TranscriptionException) {Log.w(TAG, "Voice transcription failed: ${error.message}", error)state = VoiceInputState.Error(transcriptionErrorMessage(error))+ } catch (error: AssetDownloadException) {+ if (error is AssetDownloadException.Cancelled) {+ state = VoiceInputState.Idle+ return+ }+ Log.w(TAG, "Voice model download failed: ${error.message}", error)+ state = VoiceInputState.Error(downloadErrorMessage())} catch (error: LlmException) {if (error is LlmException.Cancelled) {state = VoiceInputState.Idle@@ -322,7 +336,6 @@ internal class VoiceTranscriptionController(transcriptionPreloadJob?.cancel()transcriptionPreloadJob = scope.launch(Dispatchers.IO) {try {- uniffiEnsureInitialized()transcriber.loadModel()} catch (error: CancellationException) {throw error@@ -345,15 +358,17 @@ internal class VoiceTranscriptionController(private suspend fun ensureTranscriptionModelDownloaded() {val activeJob = coroutineContext[Job]- uniffiEnsureInitialized()- if (withContext(Dispatchers.IO) { modelTargets.all { modelDownloader.isDownloaded(it) } }) {+ if (withContext(Dispatchers.IO) { modelAssets.all { assetStore.isDownloaded(it) } }) {return}setStateIfJobActive(activeJob, VoiceInputState.Downloading(null))- modelDownloader.download(modelTargets) { progress ->- postStateIfJobActive(activeJob, VoiceInputState.Downloading(progress.percent?.coerceIn(0, 100)))+ assetStore.download(modelAssets) { progress ->+ postStateIfJobActive(+ activeJob,+ VoiceInputState.Downloading(progress.percentage.toInt().coerceIn(0, 100))+ )}}diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.ktindex e00084e202..f490c07af8 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt@@ -1,16 +1,11 @@package io.ente.ensu.knowledge-import io.ente.ensu.bindings.CancellationToken+import io.ente.ensu.assets.AssetStoreimport io.ente.ensu.bindings.KnowledgeDatasetConfig-import io.ente.ensu.bindings.KnowledgeDownloadCallback-import io.ente.ensu.bindings.KnowledgeDownloadProgressimport io.ente.ensu.bindings.KnowledgePromptHitimport io.ente.ensu.bindings.KnowledgeReconciliationimport io.ente.ensu.bindings.RetrievalIndex-import io.ente.ensu.bindings.cleanupObsoleteKnowledgePackRevisions-import io.ente.ensu.bindings.downloadKnowledgePack-import io.ente.ensu.bindings.reconcileKnowledgePack-import java.io.File+import io.ente.ensu.bindings.knowledgePackAssetimport kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.Deferredimport kotlinx.coroutines.Dispatchers@@ -24,10 +19,9 @@ import kotlinx.coroutines.sync.withLockimport kotlinx.coroutines.withContextclass KnowledgeProvider(- private val knowledgeRoot: File+ private val assetStore: AssetStore) {private data class ActiveMutation(- val cancellation: CancellationToken,val task: Deferred<KnowledgeReconciliation>)@@ -74,19 +68,11 @@ class KnowledgeProvider(mutationGate.withLock {mutations[dataset.stableId] ?: run {ownsMutation = true- val cancellation = CancellationToken()val task = async(Dispatchers.IO) {try {- downloadKnowledgePack(- packRoot(dataset).absolutePath,- dataset.stableId,- object : KnowledgeDownloadCallback {- override fun onProgress(progress: KnowledgeDownloadProgress) {- onProgress(progress)- }- },- cancellation- )+ assetStore.download(listOf(knowledgePackAsset(dataset.stableId))) {+ onProgress(KnowledgeDownloadProgress(it.label, it.percentage))+ }indexGate.withLock {val result = reconcileAndOpenLocked(dataset)check(result.activeIdentity == dataset.currentDownloadIdentity) {@@ -97,14 +83,13 @@ class KnowledgeProvider(} catch (error: Throwable) {withContext(NonCancellable) {indexGate.withLock {- incomingRevision(dataset).deleteRecursively()runCatching { reconcileAndOpenLocked(dataset) }}}throw error}}- ActiveMutation(cancellation, task).also {+ ActiveMutation(task).also {mutations[dataset.stableId] = it}}@@ -114,7 +99,7 @@ class KnowledgeProvider(try {mutation.task.await()} catch (error: CancellationException) {- if (ownsMutation) mutation.cancellation.cancel()+ if (ownsMutation) mutation.task.cancel()throw error} finally {withContext(NonCancellable) {@@ -132,7 +117,7 @@ class KnowledgeProvider(suspend fun cancel(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {val mutation = mutationGate.withLock { mutations[dataset.stableId] }- mutation?.cancellation?.cancel()+ mutation?.task?.cancel()runCatching { mutation?.task?.await() }mutationGate.withLock {if (mutations[dataset.stableId] === mutation) {@@ -166,8 +151,7 @@ class KnowledgeProvider(private fun reconcileAndOpenLocked(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {val previous = indexes[dataset.stableId]- val root = packRoot(dataset).absolutePath- val result = reconcileKnowledgePack(root, dataset.stableId)+ val result = assetStore.reconcileKnowledge(dataset.stableId)val directory = result.activeDirectoryif (directory == null) {indexes.remove(dataset.stableId)@@ -179,18 +163,17 @@ class KnowledgeProvider(}result.activeIdentity?.let { activeIdentity ->runCatching {- cleanupObsoleteKnowledgePackRevisions(root, dataset.stableId, activeIdentity)+ assetStore.cleanupKnowledgeRevisions(dataset.stableId, activeIdentity)}}return result}- private fun packRoot(dataset: KnowledgeDatasetConfig): File =- File(knowledgeRoot, dataset.stableId)-- private fun incomingRevision(dataset: KnowledgeDatasetConfig): File =- File(packRoot(dataset), dataset.currentDownloadIdentity)-private suspend fun lifecycleGate(stableId: String): Mutex =mutationGate.withLock { lifecycleGates.getOrPut(stableId) { Mutex() } }}++data class KnowledgeDownloadProgress(+ val label: String,+ val percentage: Double+)diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.ktindex 165d8fbe18..48ecd90754 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt@@ -5,10 +5,8 @@ import androidx.datastore.preferences.core.editimport androidx.datastore.preferences.core.stringSetPreferencesKeyimport androidx.datastore.preferences.preferencesDataStoreimport io.ente.ensu.AppState-import io.ente.ensu.bindings.DownloadError+import io.ente.ensu.bindings.AssetDownloadExceptionimport io.ente.ensu.bindings.KnowledgeDatasetConfig-import io.ente.ensu.bindings.KnowledgeDownloadException-import io.ente.ensu.bindings.KnowledgeDownloadProgressimport io.ente.ensu.bindings.KnowledgeReconciliationimport io.ente.ensu.bindings.KnowledgeReconciliationStatusimport io.ente.ensu.device.isChatSupported@@ -225,23 +223,22 @@ class KnowledgeStore()private fun userFacingKnowledgeError(error: Throwable): String {- val downloadError = (error as? KnowledgeDownloadException.Download)?.error- return when (downloadError) {- DownloadError.Cancelled -> "Download cancelled"- DownloadError.StorageFull ->+ return when (error) {+ is AssetDownloadException.Cancelled -> "Download cancelled"+ is AssetDownloadException.StorageFull ->"Not enough storage space to download this knowledge pack."- is DownloadError.Network ->+ is AssetDownloadException.Network ->"Couldn't download the knowledge pack. Check your connection and try again."- is DownloadError.Http ->+ is AssetDownloadException.Http ->"The knowledge pack is currently unavailable. Please try again later."- is DownloadError.Validation,- is DownloadError.SizeMismatch,- is DownloadError.Protocol,- is DownloadError.InvalidTarget ->+ is AssetDownloadException.Validation,+ is AssetDownloadException.SizeMismatch,+ is AssetDownloadException.Protocol,+ is AssetDownloadException.InvalidTarget ->"The knowledge pack couldn't be verified. Please try again."- is DownloadError.Io ->+ is AssetDownloadException.Io ->"Couldn't save the knowledge pack. Please try again."- null -> "Knowledge pack setup failed. Please try again."+ else -> "Knowledge pack setup failed. Please try again."}}}diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.ktindex 0b76ba6cf6..ed478e9941 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt@@ -1,10 +1,7 @@package io.ente.ensu.llm-import io.ente.ensu.bindings.ModelTarget-data class LlmModelSelection(val id: String,- val modelTarget: ModelTarget,val contextLength: Int? = null,val maxTokens: Int? = null)diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.ktindex e24386fdcf..84177f186a 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt@@ -1,6 +1,8 @@package io.ente.ensu.llmimport android.util.Log+import io.ente.ensu.assets.AssetStore+import io.ente.ensu.bindings.Assetimport io.ente.ensu.bindings.LlmChatMessage as NativeChatMessageimport io.ente.ensu.bindings.LlmChatRequestimport io.ente.ensu.bindings.LlmContext@@ -12,7 +14,8 @@ import io.ente.ensu.bindings.LlmModelimport io.ente.ensu.bindings.LlmModelLoadParamsimport io.ente.ensu.bindings.KnowledgeEmbeddingConfigimport io.ente.ensu.bindings.Transcriber-import io.ente.ensu.bindings.knowledgeEmbeddingModelTarget+import io.ente.ensu.bindings.knowledgeEmbeddingModelAsset+import io.ente.ensu.bindings.llmAssetimport io.ente.ensu.bindings.llmCancelimport io.ente.ensu.bindings.llmInitBackendimport io.ente.ensu.device.AndroidDeviceCapabilityProvider@@ -26,11 +29,11 @@ import kotlinx.coroutines.sync.withLockimport kotlinx.coroutines.withContextclass RequiredModelValidationError(- val targetId: String-) : Exception("Downloaded model failed validation: $targetId")+ val modelId: String+) : Exception("Downloaded model failed validation: $modelId")class LlmProvider(- private val downloader: ModelDownloader,+ private val assetStore: AssetStore,private val transcriber: Transcriber,private val deviceCapabilityProvider: AndroidDeviceCapabilityProvider,private val knowledgeEmbedding: KnowledgeEmbeddingConfig,@@ -48,18 +51,24 @@ class LlmProvider(@Volatile private var currentJobId: Long? = nullprivate var backendInitialized = falseprivate val modelLoadMutex = Mutex()- private val embeddingDownloadTarget = knowledgeEmbeddingModelTarget()+ private val embeddingAsset = knowledgeEmbeddingModelAsset()class EmbeddingAssetInvalid : Exception("Embedding model asset is invalid")fun isChatModelReady(selection: LlmModelSelection): Boolean =- downloader.isDownloaded(selection.modelTarget)+ assetStore.isDownloaded(chatAsset(selection))fun isEmbeddingModelReady(): Boolean =- downloader.isDownloaded(embeddingDownloadTarget)+ assetStore.isDownloaded(embeddingAsset)suspend fun estimateEmbeddingDownloadSize(): Long? =- downloader.estimateDownloadSize(embeddingDownloadTarget)+ assetStore.estimateDownloadSize(embeddingAsset)++ suspend fun estimateChatModelDownloadSize(selection: LlmModelSelection): Long? =+ assetStore.estimateDownloadSize(chatAsset(selection))++ val isDownloadActive: Boolean+ get() = assetStore.isDownloadActivesuspend fun ensureModelReady(selection: LlmModelSelection,@@ -78,29 +87,30 @@ class LlmProvider() {withContext(ioDispatcher) {deviceCapabilityProvider.chatCapability().requireChatSupported()- val missingTargets = modelLoadMutex.withLock {+ val asset = chatAsset(selection)+ val missingAssets = modelLoadMutex.withLock {val embeddingReady = isEmbeddingModelReady()if (IS_ENSU_PACKS_ENABLED && !embeddingReady) {- val embeddingFile = downloader.llmModelPath(embeddingDownloadTarget)+ val embeddingFile = assetStore.llmModelPath(embeddingAsset)if (embeddingFile?.exists() == true) {- downloader.removeDownloaded(embeddingDownloadTarget)+ assetStore.removeDownloaded(embeddingAsset)}}buildList {- if (!isChatModelReady(selection)) add(selection.modelTarget)- if (IS_ENSU_PACKS_ENABLED && !isEmbeddingModelReady()) add(embeddingDownloadTarget)+ if (!assetStore.isDownloaded(asset)) add(asset)+ if (IS_ENSU_PACKS_ENABLED && !isEmbeddingModelReady()) add(embeddingAsset)}}- if (missingTargets.isNotEmpty()) {- downloader.download(missingTargets, onProgress)+ if (missingAssets.isNotEmpty()) {+ downloadAssets(missingAssets, onProgress)}modelLoadMutex.withLock {if (IS_ENSU_PACKS_ENABLED && !isEmbeddingModelReady()) {- downloader.removeDownloaded(embeddingDownloadTarget)+ assetStore.removeDownloaded(embeddingAsset)throw RequiredModelValidationError(knowledgeEmbedding.targetId)}- if (!isChatModelReady(selection)) {+ if (!assetStore.isDownloaded(asset)) {throw RequiredModelValidationError(selection.id)}ensureModelReadyLocked(@@ -128,7 +138,7 @@ class LlmProvider(val mmprojPath = if (imageFiles.isEmpty()) {null} else {- downloader.llmMmprojPath(selection.modelTarget)?.absolutePath+ assetStore.llmMmprojPath(chatAsset(selection))?.absolutePath}val clampedTemperature = temperature.coerceIn(0.35f, 0.7f)@@ -174,7 +184,7 @@ class LlmProvider(val embeddingModel = LlmModel.load(LlmModelLoadParams(- modelPath = requireNotNull(downloader.llmModelPath(embeddingDownloadTarget)).absolutePath,+ modelPath = requireNotNull(assetStore.llmModelPath(embeddingAsset)).absolutePath,nGpuLayers = 0,useMmap = true,useMlock = false@@ -196,9 +206,10 @@ class LlmProvider(withContext(ioDispatcher) {runCatching {modelLoadMutex.withLock {- if (!downloader.isDownloaded(selection.modelTarget)) return@withLock+ val asset = chatAsset(selection)+ if (!assetStore.isDownloaded(asset)) return@withLockval mmprojPath =- downloader.llmMmprojPath(selection.modelTarget)?.absolutePath+ assetStore.llmMmprojPath(asset)?.absolutePath?.takeIf { File(it).exists() }?: return@withLockensureModelReadyLocked(selection, onProgress = {})@@ -289,19 +300,20 @@ class LlmProvider(unloadModel()- val wasAlreadyDownloaded = downloader.isDownloaded(selection.modelTarget)+ val asset = chatAsset(selection)+ val wasAlreadyDownloaded = assetStore.isDownloaded(asset)if (shouldDownload) {- downloader.download(listOf(selection.modelTarget), onProgress)+ downloadAssets(listOf(asset), onProgress)}onProgress(DownloadProgress(100, "Loading model...", phase = DownloadPhase.Loading))try {loadWithFallbacks(selection,- requireNotNull(downloader.llmModelPath(selection.modelTarget))+ requireNotNull(assetStore.llmModelPath(asset)))} catch (error: Throwable) {- if (allowRecovery && wasAlreadyDownloaded && downloader.removeDownloaded(selection.modelTarget)) {+ if (allowRecovery && wasAlreadyDownloaded && assetStore.removeDownloaded(asset)) {onProgress(DownloadProgress(0, "Starting download...", phase = DownloadPhase.Downloading))ensureModelReadyLocked(selection, onProgress, allowRecovery = false)return@@ -311,6 +323,22 @@ class LlmProvider(onProgress(DownloadProgress(100, "Ready", phase = DownloadPhase.Ready))}+ private suspend fun downloadAssets(+ assets: List<Asset>,+ onProgress: (DownloadProgress) -> Unit+ ) {+ assetStore.download(assets) { progress ->+ onProgress(+ DownloadProgress(+ progress.percentage.toInt().coerceIn(0, 99),+ progress.status+ )+ )+ }+ }++ private fun chatAsset(selection: LlmModelSelection): Asset = llmAsset(selection.id)+private fun loadWithFallbacks(selection: LlmModelSelection, modelFile: File) {val desiredCtx = selection.contextLength ?: 12000val contexts = listOf(desiredCtx, 12000, 8192, 4096, 2048, 1024).distinct().filter { it > 0 }diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloader.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloader.ktdeleted file mode 100644index 1d0a6db553..0000000000--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloader.kt+++ /dev/null@@ -1,126 +0,0 @@-package io.ente.ensu.llm--import android.content.Context-import android.os.Environment-import android.util.Log-import io.ente.ensu.bindings.CancellationToken-import io.ente.ensu.bindings.ModelDownloadCallback-import io.ente.ensu.bindings.ModelDownloadCore-import io.ente.ensu.bindings.ModelDownloadProgress-import io.ente.ensu.bindings.LlmException-import io.ente.ensu.bindings.ModelTarget-import io.ente.ensu.bindings.LegacyModels-import io.ente.ensu.bindings.migrateMobileModels-import io.ente.ensu.bindings.transcriptionModelTarget-import io.ente.ensu.bindings.voiceActivityModelTarget-import io.ente.ensu.bindings.uniffiEnsureInitialized-import java.io.File-import kotlinx.coroutines.CancellationException-import kotlinx.coroutines.Dispatchers-import kotlinx.coroutines.async-import kotlinx.coroutines.coroutineScope-import kotlinx.coroutines.sync.Mutex-import kotlinx.coroutines.withContext--class ModelDownloader(context: Context) {- private val appContext = context.applicationContext- private val modelsDir = File(appContext.noBackupFilesDir, "models")- private val legacyDir = appContext.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS)- ?.let { File(it, "llm") }- private val legacyTranscriptionDir = File(appContext.dataDir, "app_ensu_transcription_models")- private val downloadMutex = Mutex()- private val core: ModelDownloadCore- val transcriptionTarget: ModelTarget- val voiceActivityTarget: ModelTarget-- init {- uniffiEnsureInitialized()- ModelDownloadJobService.attach(appContext)- core = ModelDownloadCore(modelsDir.absolutePath)- transcriptionTarget = transcriptionModelTarget()- voiceActivityTarget = voiceActivityModelTarget()- }-- fun needsMigration(): Boolean =- legacyDir?.exists() == true || legacyTranscriptionDir.exists()-- fun migrate(legacyModelUrl: String?, legacyMmprojUrl: String?): String? {- File(appContext.filesDir, "llm").deleteRecursively()- return migrateMobileModels(- modelsDir.absolutePath,- LegacyModels(- legacyDir?.absolutePath,- legacyTranscriptionDir.absolutePath,- legacyModelUrl,- legacyMmprojUrl- )- )- }-- val isDownloadActive: Boolean get() = core.isDownloadActive()-- fun modelDir(target: ModelTarget): File = File(core.modelDir(target))-- fun llmModelPath(target: ModelTarget): File? =- core.llmModelPath(target)?.let(::File)-- fun llmMmprojPath(target: ModelTarget): File? =- core.llmMmprojPath(target)?.let(::File)-- fun voiceActivityModelPath(): File = File(core.voiceActivityModelPath())-- fun isDownloaded(target: ModelTarget): Boolean = core.isDownloaded(target)-- fun removeDownloaded(target: ModelTarget): Boolean = core.removeDownloaded(target)-- suspend fun estimateDownloadSize(target: ModelTarget): Long? = withContext(Dispatchers.IO) {- core.estimatedDownloadSize(target)- }-- suspend fun download(- targets: List<ModelTarget>,- onProgress: (DownloadProgress) -> Unit- ): Unit = withContext(Dispatchers.IO) {- if (!downloadMutex.tryLock()) throw LlmException.Cancelled()- try {- downloadLocked(targets, onProgress)- } finally {- downloadMutex.unlock()- }- }-- private suspend fun downloadLocked(- targets: List<ModelTarget>,- onProgress: (DownloadProgress) -> Unit- ) {- if (targets.all { core.isDownloaded(it) }) return-- val token = CancellationToken()- ModelDownloadJobService.begin { token.cancel() }- try {- coroutineScope {- val download = async {- core.download(- targets,- object : ModelDownloadCallback {- override fun onProgress(progress: ModelDownloadProgress) {- progress.logLine?.let { Log.i("ModelDownloader", it) }- ModelDownloadJobService.update(progress.percent, progress.totalBytes == null)- onProgress(DownloadProgress(progress.percent, progress.status))- }- },- token- )- }- try {- download.await()- } catch (e: CancellationException) {- token.cancel()- throw e- }- }- } finally {- ModelDownloadJobService.end()- }- }-}diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex 06b41a85ad..6c1b954343 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@ -3,10 +3,9 @@ package io.ente.ensu.llmimport android.system.ErrnoExceptionimport android.system.OsConstantsimport io.ente.ensu.AppState+import io.ente.ensu.bindings.AssetDownloadExceptionimport io.ente.ensu.bindings.ConfigDefaults-import io.ente.ensu.bindings.DownloadErrorimport io.ente.ensu.bindings.LlmException-import io.ente.ensu.bindings.mobileLlmTargetimport io.ente.ensu.device.isChatSupportedimport io.ente.ensu.logging.FileLogRepositoryimport io.ente.ensu.logging.LogLevel@@ -23,7 +22,6 @@ internal class ModelSettingsActions(private val state: MutableStateFlow<AppState>,private val sessionPreferences: SessionPreferencesDataStore,private val llmProvider: LlmProvider,- private val modelDownloader: ModelDownloader,private val logRepository: FileLogRepository,private val configDefaults: ConfigDefaults) {@@ -104,7 +102,7 @@ internal class ModelSettingsActions(val scope = scope ?: returnscope.launch {- if (!modelDownloader.isDownloadActive && modelDownloadJob?.isActive != true) {+ if (!llmProvider.isDownloadActive && modelDownloadJob?.isActive != true) {persistModelDownloadRequested(false)state.update { appState ->appState.copy(@@ -119,7 +117,7 @@ internal class ModelSettingsActions(}}- val chatSize = if (chatReady) 0L else modelDownloader.estimateDownloadSize(selection.modelTarget)+ val chatSize = if (chatReady) 0L else llmProvider.estimateChatModelDownloadSize(selection)val embeddingSize = if (!IS_ENSU_PACKS_ENABLED || embeddingReady) {0L} else {@@ -231,7 +229,8 @@ internal class ModelSettingsActions(}} catch (err: Throwable) {val cancelled = err is kotlinx.coroutines.CancellationException ||- err is LlmException.Cancelled+ err is LlmException.Cancelled ||+ err is AssetDownloadException.Cancelledval failureMessage = if (cancelled) {"Download cancelled"} else {@@ -276,7 +275,7 @@ internal class ModelSettingsActions(if (!currentState.chat.deviceCapability.isChatSupported()) returnval selection = resolveSelection(currentState.modelSettings)- if (!modelDownloader.isDownloaded(selection.modelTarget)) return+ if (!llmProvider.isChatModelReady(selection)) returnscope.launch {try {@@ -335,7 +334,6 @@ internal class ModelSettingsActions(return LlmModelSelection(id = preset.id,- modelTarget = mobileLlmTarget(preset.id),contextLength = contextLength,maxTokens = maxTokens)@@ -366,14 +364,14 @@ internal class ModelSettingsActions(if (retryCount >= MAX_DOWNLOAD_RETRIES) return falseif (err is kotlinx.coroutines.CancellationException) return falseif (err is LlmException.Cancelled) return false+ if (err is AssetDownloadException.Cancelled) return falseif (isOutOfStorageError(err)) return falseif (err is RequiredModelValidationError) return false- if (err is LlmException.Download) {- when (val error = err.error) {- is DownloadError.Validation -> return false- is DownloadError.Http -> if (error.status.toInt() in NON_RETRYABLE_HTTP) return false- else -> {}- }+ when (err) {+ is AssetDownloadException.Validation -> return false+ is AssetDownloadException.Http ->+ if (err.status.toInt() in NON_RETRYABLE_HTTP) return false+ else -> {}}return true}@@ -393,7 +391,7 @@ internal class ModelSettingsActions(private fun isOutOfStorageError(err: Throwable): Boolean {var current: Throwable? = errwhile (current != null) {- if (current is LlmException.Download && current.error is DownloadError.StorageFull) {+ if (current is AssetDownloadException.StorageFull) {return true}if (current is ErrnoException && current.errno == OsConstants.ENOSPC) return truediff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex 78e99fb3aa..35f7dd027f 100644--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -77,7 +77,6 @@ final class ChatViewModel: ObservableObject {let knowledgeStore: KnowledgeStoreprivate let provider: LlmProvider- private let downloader: ModelDownloaderprivate let knowledgeEmbedding: KnowledgeEmbeddingConfigprivate let knowledgeProvider: KnowledgeProviderprivate let voiceTranscriber: VoiceTranscriptionService@@ -108,7 +107,7 @@ final class ChatViewModel: ObservableObject {private var pendingOverflow: PendingOverflow?private var overflowBypassMessageId: UUID?- init() {+ init(assetStore: AssetStore) {logger.info("Initializing")let summaries = Self.loadSessionSummaries().reduce(into: [String: String]()) { result, item inresult[item.key.lowercased()] = item.value@@ -117,16 +116,17 @@ final class ChatViewModel: ObservableObject {?? FileManager.default.temporaryDirectorylet config = ConfigDefaults.shared- let downloader = ModelDownloader()- // Must initialize after the downloader migrates the persisted selection.+ // Must initialize after the asset store migrates the persisted selection.self.modelSettings = ModelSettingsStore.shared+ let transcription = transcriptionModelAsset()+ let voiceActivity = voiceActivityModelAsset()let transcriber = Transcriber(- modelDir: downloader.modelDir(downloader.transcriptionTarget).path,- vadModelPath: downloader.voiceActivityModelPath().path+ modelDir: assetStore.assetDir(transcription).path,+ vadModelPath: assetStore.voiceActivityModelPath(voiceActivity).path)- let provider = LlmProvider(downloader: downloader, transcriber: transcriber, knowledgeEmbedding: config.knowledgeEmbedding)- let voiceTranscriber = VoiceTranscriptionService(transcriber: transcriber, downloader: downloader)- let knowledgeProvider = KnowledgeProvider(root: baseDir.appendingPathComponent("knowledge", isDirectory: true))+ let provider = LlmProvider(assetStore: assetStore, transcriber: transcriber, knowledgeEmbedding: config.knowledgeEmbedding)+ let voiceTranscriber = VoiceTranscriptionService(transcriber: transcriber, assetStore: assetStore)+ let knowledgeProvider = KnowledgeProvider(assetStore: assetStore)let knowledgeStore = KnowledgeStore(datasets: config.knowledgeDatasets, provider: knowledgeProvider)// Chat DB + attachments.@@ -166,7 +166,6 @@ final class ChatViewModel: ObservableObject {// Stored properties.self.provider = provider- self.downloader = downloaderself.knowledgeEmbedding = config.knowledgeEmbeddingself.knowledgeProvider = knowledgeProviderself.knowledgeStore = knowledgeStore@@ -223,7 +222,6 @@ final class ChatViewModel: ObservableObject {editingMessageId = nildraftText = ""draftAttachments = []- downloader.cancel()}func dismissUnsupportedDeviceDialog() {@@ -475,7 +473,7 @@ final class ChatViewModel: ObservableObject {guard !isGenerating && !isDownloading else { return }guard !isChatUnsupported else { return }let selection = modelSettings.currentSelection()- guard downloader.isDownloaded(selection.modelTarget) else { return }+ guard provider.isChatModelReady(selection) else { return }Task { [weak self] inguard let self else { return }@@ -766,7 +764,6 @@ final class ChatViewModel: ObservableObject {if let existingTask = sharedModelReadyTask, sharedModelReadyKey != modelKey {existingTask.cancel()- downloader.cancel()clearSharedModelReadyTask()}@@ -905,7 +902,6 @@ final class ChatViewModel: ObservableObject {modelDownloadTask = nilsharedModelReadyTask?.cancel()clearSharedModelReadyTask()- downloader.cancel()if modelDownloadLoggedStart {logger.info("Model download cancelled")modelDownloadLoggedStart = false@@ -1058,7 +1054,6 @@ final class ChatViewModel: ObservableObject {} catch {if isCancellation(error) {self.sharedModelReadyTask?.cancel()- self.downloader.cancel()self.clearSharedModelReadyTask()if self.activeGenerationId == generationId {isGenerating = false@@ -1077,7 +1072,7 @@ final class ChatViewModel: ObservableObject {downloadToast = DownloadToastState(phase: .errorDownload,percent: nil,- status: self.userFacingModelReadyError(error, wasDownloaded: self.downloader.isDownloaded(selection.modelTarget)),+ status: self.userFacingModelReadyError(error, wasDownloaded: self.provider.isChatModelReady(selection)),offerRetryDownload: true)activeGenerationId = nil@@ -1887,6 +1882,7 @@ final class ChatViewModel: ObservableObject {private func isCancellation(_ error: Error) -> Bool {if error is CancellationError { return true }if case LlmError.Cancelled = error { return true }+ if case AssetDownloadError.Cancelled = error { return true }return (error as? URLError)?.code == .cancelled}@@ -1906,15 +1902,13 @@ final class ChatViewModel: ObservableObject {return false}- if case let LlmError.Download(inner) = error {- switch inner {- case .validation:- return false- case let .http(status):- if status == 401 || status == 403 || status == 404 { return false }- default:- break- }+ switch error {+ case AssetDownloadError.Validation:+ return false+ case let AssetDownloadError.Http(status):+ if status == 401 || status == 403 || status == 404 { return false }+ default:+ break}return true@@ -1938,7 +1932,7 @@ final class ChatViewModel: ObservableObject {}private func isOutOfStorageError(_ error: Error) -> Bool {- if case LlmError.Download(.storageFull) = error { return true }+ if case AssetDownloadError.StorageFull = error { return true }return error.isOutOfDiskSpace}diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swiftindex 9ba888a0c5..7dac4dbfa4 100644--- a/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift@@ -83,7 +83,8 @@ final class VoiceTranscriptionService {typealias TranscriptHandler = @MainActor @Sendable (String) -> Voidprivate let transcriber: Transcriber- private let downloader: ModelDownloader+ private let assetStore: AssetStore+ private let modelAssets: [Asset]private var transcriptionTask: Task<Void, Never>?private var preloadTask: Task<Void, Never>?private var activeVoiceTaskId = UUID()@@ -91,9 +92,10 @@ final class VoiceTranscriptionService {private let recorder = PcmAudioRecorder()- init(transcriber: Transcriber, downloader: ModelDownloader) {+ init(transcriber: Transcriber, assetStore: AssetStore) {self.transcriber = transcriber- self.downloader = downloader+ self.assetStore = assetStore+ self.modelAssets = [transcriptionModelAsset(), voiceActivityModelAsset()]}func startRecording(@@ -143,8 +145,6 @@ final class VoiceTranscriptionService {let taskId = beginVoiceTask()let downloadId = beginDownload(taskId: taskId)let transcriber = transcriber- let downloader = downloader- let targets = [downloader.transcriptionTarget, downloader.voiceActivityTarget]let sampleRate = recording.sampleRatelet pcm = recording.pcm@@ -188,6 +188,7 @@ final class VoiceTranscriptionService {} catch is CancellationError {return} catch {+ if case AssetDownloadError.Cancelled = error { return }if case LlmError.Cancelled = error { return }await MainActor.run { [weak self] inself?.finishDownload(downloadId: downloadId)@@ -239,6 +240,7 @@ final class VoiceTranscriptionService {} catch is CancellationError {return} catch {+ if case AssetDownloadError.Cancelled = error { return }if case LlmError.Cancelled = error { return }await MainActor.run { [weak self] inself?.finishDownload(downloadId: downloadId)@@ -254,17 +256,17 @@ final class VoiceTranscriptionService {downloadId: UUID,onState: @escaping StateHandler) async throws {- let targets = await [downloader.transcriptionTarget, downloader.voiceActivityTarget]- let downloader = self.downloader- if targets.allSatisfy({ downloader.isDownloaded($0) }) {+ let assets = await modelAssets+ let assetStore = self.assetStore+ if assets.allSatisfy({ assetStore.isDownloaded($0) }) {return}await MainActor.run { [weak self] inguard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else { return }onState(.downloading(percent: nil))}- try await downloader.download(targets: targets) { [weak self] progress in- let percent = progress.percent.map { min(max($0, 0), 100) }+ try await assetStore.download(assets: assets) { [weak self] progress in+ let percent = min(max(Int(progress.percentage), 0), 100)Task { @MainActor [weak self] inguard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else { return }onState(.downloading(percent: percent))diff --git a/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swift b/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swiftindex 13f1a0ca9a..fc391fa240 100644--- a/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swift+++ b/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swift@@ -2,14 +2,17 @@ import SwiftUI@mainstruct EnsuApp: App {+ private let assetStore: AssetStore+init() {EnsuLogging.shared.start()- ModelDownloader.registerBackgroundTask()+ AssetStore.registerBackgroundTask()+ assetStore = AssetStore()}var body: some Scene {WindowGroup {- RootView()+ RootView(assetStore: assetStore)}}}diff --git a/mobile/native/apple/apps/ensu/Ensu/HomeView.swift b/mobile/native/apple/apps/ensu/Ensu/HomeView.swiftindex 06a3569ef5..f2b9d13997 100644--- a/mobile/native/apple/apps/ensu/Ensu/HomeView.swift+++ b/mobile/native/apple/apps/ensu/Ensu/HomeView.swift@@ -1,7 +1,11 @@import SwiftUIstruct HomeView: View {- @StateObject private var chatModel = ChatViewModel()+ @StateObject private var chatModel: ChatViewModel++ init(assetStore: AssetStore) {+ _chatModel = StateObject(wrappedValue: ChatViewModel(assetStore: assetStore))+ }var body: some View {ChatView(viewModel: chatModel)diff --git a/mobile/native/apple/apps/ensu/Ensu/Info.plist b/mobile/native/apple/apps/ensu/Ensu/Info.plistindex 138f535fb6..083557b865 100644--- a/mobile/native/apple/apps/ensu/Ensu/Info.plist+++ b/mobile/native/apple/apps/ensu/Ensu/Info.plist@@ -64,7 +64,7 @@<key>BGTaskSchedulerPermittedIdentifiers</key><array>- <string>io.ente.ensu.model-download</string>+ <string>io.ente.ensu.asset-download</string></array><key>NSAppTransportSecurity</key>diff --git a/mobile/native/apple/apps/ensu/Ensu/RootView.swift b/mobile/native/apple/apps/ensu/Ensu/RootView.swiftindex 473a19f754..586ff0becd 100644--- a/mobile/native/apple/apps/ensu/Ensu/RootView.swift+++ b/mobile/native/apple/apps/ensu/Ensu/RootView.swift@@ -1,12 +1,14 @@import SwiftUIstruct RootView: View {+ let assetStore: AssetStore+var body: some View {ZStack {EnsuColor.backgroundBase.ignoresSafeArea()- HomeView()+ HomeView(assetStore: assetStore)}.frame(maxWidth: .infinity, maxHeight: .infinity).tint(EnsuColor.action)diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swiftsimilarity index 64%rename from mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swiftrename to mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swiftindex 13f050e605..a0ae8b69cc 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift@@ -1,35 +1,31 @@import BackgroundTasksimport Foundation-private let logger = EnsuLogging.shared.logger("ModelDownloader")+private let logger = EnsuLogging.shared.logger("AssetStore")-final class ModelDownloader {- private let core: ModelDownloadCore+final class AssetStore: @unchecked Sendable {+ private let core: AssetStoreCoreprivate let activeLock = NSLock()- private var activeToken: CancellationToken?- let transcriptionTarget: ModelTarget- let voiceActivityTarget: ModelTarget+ private var downloadActive = false@MainActorinit() {let baseDir = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first?? FileManager.default.temporaryDirectory- var modelsDir = baseDir.appendingPathComponent("models", isDirectory: true)- try? FileManager.default.createDirectory(at: modelsDir, withIntermediateDirectories: true)+ var assetsDir = baseDir.appendingPathComponent("assets", isDirectory: true)+ try? FileManager.default.createDirectory(at: assetsDir, withIntermediateDirectories: true)var values = URLResourceValues()values.isExcludedFromBackup = true- try? modelsDir.setResourceValues(values)- core = ModelDownloadCore(modelsDir: modelsDir.path)- transcriptionTarget = transcriptionModelTarget()- voiceActivityTarget = voiceActivityModelTarget()+ try? assetsDir.setResourceValues(values)+ core = AssetStoreCore(assetsDir: assetsDir.path)let settings = UserDefaults.standardlet pendingSelection = settings.object(forKey: "ensu.model.id") == nillet legacyModelUrl = pendingSelection && settings.bool(forKey: "ensu.model.use_custom")? settings.string(forKey: "ensu.model.url"): nil- let presetId = migrateMobileModels(- modelsDir: modelsDir.path,- legacy: LegacyModels(+ let presetId = migrateEnsuAssets(+ assetsDir: assetsDir.path,+ legacy: LegacyAssets(llmDir: baseDir.appendingPathComponent("llm", isDirectory: true).path,transcriptionDir: baseDir.appendingPathComponent("transcription", isDirectory: true).path,modelUrl: legacyModelUrl,@@ -46,91 +42,97 @@ final class ModelDownloader {static func registerBackgroundTask() {if #available(iOS 26.0, *) {- ModelDownloadBackgroundTask.register()+ AssetDownloadBackgroundTask.register()}}- func modelDir(_ target: ModelTarget) -> URL {- URL(fileURLWithPath: core.modelDir(target: target))+ func assetDir(_ asset: Asset) -> URL {+ URL(fileURLWithPath: core.assetDir(asset: asset))}- func llmModelPath(_ target: ModelTarget) -> URL? {- core.llmModelPath(target: target).map { URL(fileURLWithPath: $0) }+ func llmModelPath(_ asset: Asset) -> URL? {+ core.llmModelPath(asset: asset).map { URL(fileURLWithPath: $0) }}- func llmMmprojPath(_ target: ModelTarget) -> URL? {- core.llmMmprojPath(target: target).map { URL(fileURLWithPath: $0) }+ func llmMmprojPath(_ asset: Asset) -> URL? {+ core.llmMmprojPath(asset: asset).map { URL(fileURLWithPath: $0) }}- func voiceActivityModelPath() -> URL {- URL(fileURLWithPath: core.voiceActivityModelPath())+ func voiceActivityModelPath(_ asset: Asset) -> URL {+ URL(fileURLWithPath: core.voiceActivityModelPath(asset: asset))}- func isDownloaded(_ target: ModelTarget) -> Bool {- core.isDownloaded(target: target)+ func isDownloaded(_ asset: Asset) -> Bool {+ core.isDownloaded(asset: asset)}- func removeDownloaded(_ target: ModelTarget) -> Bool {- core.removeDownloaded(target: target)+ func removeDownloaded(_ asset: Asset) -> Bool {+ core.removeDownloaded(asset: asset)}- func cancel() {- activeLock.withLock { activeToken }?.cancel()+ func reconcileKnowledge(_ stableId: String) throws -> KnowledgeReconciliation {+ try reconcileKnowledgePack(store: core, stableId: stableId)}- func estimateDownloadSize(_ target: ModelTarget) async -> Int64? {+ func cleanupKnowledgeRevisions(_ stableId: String, activeIdentity: String) throws {+ try cleanupObsoleteKnowledgePackRevisions(+ store: core,+ stableId: stableId,+ activeIdentity: activeIdentity+ )+ }++ func estimateDownloadSize(_ asset: Asset) async -> Int64? {await Task.detached(priority: .utility) { [core] in- core.estimatedDownloadSize(target: target)+ core.estimatedDownloadSize(asset: asset)}.value}func download(- targets: [ModelTarget],- onProgress: @escaping (DownloadProgress) -> Void+ assets: [Asset],+ onProgress: @escaping (AssetDownloadProgress) -> Void) async throws {let token = CancellationToken()let started = activeLock.withLock { () -> Bool in- if activeToken != nil { return false }- activeToken = token+ if downloadActive { return false }+ downloadActive = truereturn true}guard started else { throw DownloadAlreadyActiveError() }- defer { activeLock.withLock { activeToken = nil } }+ defer { activeLock.withLock { downloadActive = false } }- if targets.allSatisfy({ self.core.isDownloaded(target: $0) }) {+ if assets.allSatisfy({ self.core.isDownloaded(asset: $0) }) {return}- onProgress(DownloadProgress(percent: 0, status: "Starting download..."))-if #available(iOS 26.0, *) {- ModelDownloadBackgroundTask.begin {+ AssetDownloadBackgroundTask.begin {token.cancel()}}var succeeded = falsedefer {if #available(iOS 26.0, *) {- ModelDownloadBackgroundTask.end(success: succeeded)+ AssetDownloadBackgroundTask.end(success: succeeded)}}let core = corelet downloadTask = Task.detached(priority: .utility) {try Task.checkCancellation()- let callback = ModelDownloadCallbackSink { progress in+ let callback = AssetDownloadCallbackSink { progress inif let line = progress.logLine {logger.info(line)}if #available(iOS 26.0, *) {- ModelDownloadBackgroundTask.update(+ AssetDownloadBackgroundTask.update(downloadedBytes: progress.downloadedBytes,totalBytes: progress.totalBytes)}- onProgress(DownloadProgress(percent: Int(progress.percent), status: progress.status))+ onProgress(progress)}- try core.download(targets: targets, callback: callback, cancellation: token)+ try core.download(assets: assets, callback: callback, cancellation: token)}try await withTaskCancellationHandler {@@ -145,21 +147,21 @@ final class ModelDownloader {private struct DownloadAlreadyActiveError: Error {}-private final class ModelDownloadCallbackSink: ModelDownloadCallback, @unchecked Sendable {- private let onProgressHandler: (ModelDownloadProgress) -> Void+private final class AssetDownloadCallbackSink: AssetDownloadCallback, @unchecked Sendable {+ private let onProgressHandler: (AssetDownloadProgress) -> Void- init(onProgress: @escaping (ModelDownloadProgress) -> Void) {+ init(onProgress: @escaping (AssetDownloadProgress) -> Void) {self.onProgressHandler = onProgress}- func onProgress(progress: ModelDownloadProgress) {+ func onProgress(progress: AssetDownloadProgress) {onProgressHandler(progress)}}@available(iOS 26.0, *)-private enum ModelDownloadBackgroundTask {- private static let identifier = "io.ente.ensu.model-download"+private enum AssetDownloadBackgroundTask {+ private static let identifier = "io.ente.ensu.asset-download"private static let lock = NSLock()private static var task: BGContinuedProcessingTask?private static var onExpiration: (() -> Void)?@@ -183,7 +185,7 @@ private enum ModelDownloadBackgroundTask {let request = BGContinuedProcessingTaskRequest(identifier: identifier,- title: "Downloading model",+ title: "Downloading assets",subtitle: "")request.strategy = .faildiff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swiftindex 2ff621ba49..c75881899c 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift@@ -5,6 +5,11 @@ struct KnowledgeSearchHit: Sendable {let hit: RetrievalHit}+struct KnowledgeDownloadProgress: Sendable {+ let label: String+ let percentage: Double+}+private struct KnowledgePackValidationError: LocalizedError {var errorDescription: String? {"Downloaded knowledge pack failed current revision validation"@@ -13,29 +18,25 @@ private struct KnowledgePackValidationError: LocalizedError {actor KnowledgeProvider {private final class Mutation: @unchecked Sendable {- let cancellation: CancellationTokenlet task: Task<Void, Error>init(- packRoot: URL,+ assetStore: AssetStore,dataset: KnowledgeDatasetConfig,onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void- ) {- let cancellation = CancellationToken()- self.cancellation = cancellation+ ) throws {+ let asset = try knowledgePackAsset(stableId: dataset.stableId)task = Task.detached(priority: .utility) {- let callback = KnowledgeDownloadCallbackSink(onProgress: onProgress)- try downloadKnowledgePack(- packRoot: packRoot.path,- stableId: dataset.stableId,- callback: callback,- cancellation: cancellation- )+ try await assetStore.download(assets: [asset]) {+ onProgress(KnowledgeDownloadProgress(+ label: $0.label,+ percentage: $0.percentage+ ))+ }}}func cancel() {- cancellation.cancel()task.cancel()}}@@ -45,22 +46,14 @@ actor KnowledgeProvider {let index: RetrievalIndex}- private let root: URL+ private let assetStore: AssetStoreprivate var mutations: [String: Mutation] = [:]private var lifecycleGates: [String: AsyncSerialGate] = [:]private var indexes: [String: OpenIndex] = [:]private let indexGate = AsyncSerialGate()- init(root: URL) {- self.root = root- try? FileManager.default.createDirectory(- at: root,- withIntermediateDirectories: true- )- var excludedRoot = root- var values = URLResourceValues()- values.isExcludedFromBackup = true- try? excludedRoot.setResourceValues(values)+ init(assetStore: AssetStore) {+ self.assetStore = assetStore}func reconcile(dataset: KnowledgeDatasetConfig) async throws -> KnowledgeReconciliation {@@ -84,8 +77,8 @@ actor KnowledgeProvider {if let current = mutations[dataset.stableId] {return current}- let created = Mutation(- packRoot: packRoot(dataset),+ let created = try Mutation(+ assetStore: assetStore,dataset: dataset,onProgress: onProgress)@@ -99,9 +92,7 @@ actor KnowledgeProvider {if mutations[dataset.stableId] === mutation {mutations.removeValue(forKey: dataset.stableId)}- _ = try? await withIndexGate {- try await reconcileAndActivateLocked(dataset: dataset, removeIncoming: true)- }+ _ = try? await withIndexGate { try await reconcileAndActivateLocked(dataset: dataset) }throw error}@@ -125,21 +116,12 @@ actor KnowledgeProvider {func cancel(dataset: KnowledgeDatasetConfig) async -> KnowledgeReconciliation? {let mutation = mutations[dataset.stableId]mutation?.cancel()- let transferFailed: Bool- do {- try await mutation?.task.value- transferFailed = false- } catch {- transferFailed = true- }+ _ = try? await mutation?.task.valueif let mutation, mutations[dataset.stableId] === mutation {mutations.removeValue(forKey: dataset.stableId)}return try? await withIndexGate {- try await reconcileAndActivateLocked(- dataset: dataset,- removeIncoming: transferFailed- )+ try await reconcileAndActivateLocked(dataset: dataset)}}@@ -190,30 +172,17 @@ actor KnowledgeProvider {}private func reconcileAndActivateLocked(- dataset: KnowledgeDatasetConfig,- removeIncoming: Bool = false+ dataset: KnowledgeDatasetConfig) async throws -> KnowledgeReconciliation {- let packRoot = packRoot(dataset)+ let assetStore = assetStorelet result = try await Task.detached(priority: .utility) {- if removeIncoming {- try? FileManager.default.removeItem(- at: packRoot.appendingPathComponent(- dataset.currentDownloadIdentity,- isDirectory: true- )- )- }- return try reconcileKnowledgePack(- packRoot: packRoot.path,- stableId: dataset.stableId- )+ try assetStore.reconcileKnowledge(dataset.stableId)}.valuetry activate(result: result, dataset: dataset)if let activeIdentity = result.activeIdentity {_ = try? await Task.detached(priority: .utility) {- try cleanupObsoleteKnowledgePackRevisions(- packRoot: packRoot.path,- stableId: dataset.stableId,+ try assetStore.cleanupKnowledgeRevisions(+ dataset.stableId,activeIdentity: activeIdentity)}.value@@ -249,20 +218,4 @@ actor KnowledgeProvider {lifecycleGates[stableId] = gatereturn gate}-- private func packRoot(_ dataset: KnowledgeDatasetConfig) -> URL {- root.appendingPathComponent(dataset.stableId, isDirectory: true)- }-}--private final class KnowledgeDownloadCallbackSink: KnowledgeDownloadCallback, @unchecked Sendable {- private let onProgressHandler: @Sendable (KnowledgeDownloadProgress) -> Void-- init(onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void) {- onProgressHandler = onProgress- }-- func onProgress(progress: KnowledgeDownloadProgress) {- onProgressHandler(progress)- }}diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swiftindex a3cf381f22..a44adb622b 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift@@ -200,22 +200,24 @@ final class KnowledgeStore: ObservableObject {}private func userFacingKnowledgeError(_ error: Error) -> String {- guard case let KnowledgeDownloadError.Download(downloadError) = error else {- return "Knowledge pack setup failed. Please try again."- }- switch downloadError {- case .cancelled:+ switch error {+ case AssetDownloadError.Cancelled:return "Download cancelled"- case .storageFull:+ case AssetDownloadError.StorageFull:return "Not enough storage space to download this knowledge pack."- case .network:+ case AssetDownloadError.Network:return "Couldn't download the knowledge pack. Check your connection and try again."- case .http:+ case AssetDownloadError.Http:return "The knowledge pack is currently unavailable. Please try again later."- case .validation, .sizeMismatch, .`protocol`, .invalidTarget:+ case AssetDownloadError.Validation,+ AssetDownloadError.SizeMismatch,+ AssetDownloadError.Protocol,+ AssetDownloadError.InvalidTarget:return "The knowledge pack couldn't be verified. Please try again."- case .io:+ case AssetDownloadError.Io:return "Couldn't save the knowledge pack. Please try again."+ default:+ return "Knowledge pack setup failed. Please try again."}}}diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swiftindex 5c859f9ef9..d1ab9cba39 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift@@ -2,7 +2,6 @@ import Foundationstruct LlmModelSelection: Equatable {let id: String- let modelTarget: ModelTargetlet contextLength: Int?let maxTokens: Int?}@@ -65,10 +64,10 @@ struct GenerationSummary {}struct RequiredModelValidationError: LocalizedError {- let targetId: String+ let modelId: Stringvar errorDescription: String? {- "Downloaded model failed validation: \(targetId)"+ "Downloaded model failed validation: \(modelId)"}}@@ -115,9 +114,10 @@ final class LlmProvider {let requestedContextLength: Int?}- private let downloader: ModelDownloader+ private let assetStore: AssetStoreprivate let transcriber: Transcriberprivate let knowledgeEmbedding: KnowledgeEmbeddingConfig+ private let embeddingAsset: Assetprivate var loadedModel: LlmModel?private var loadedContext: LlmContext?private var currentModelKey: LoadedModelKey?@@ -126,35 +126,37 @@ final class LlmProvider {private var currentJobId: Int64?private let modelLoadGate = AsyncSerialGate()- init(downloader: ModelDownloader, transcriber: Transcriber, knowledgeEmbedding: KnowledgeEmbeddingConfig) {- self.downloader = downloader+ init(assetStore: AssetStore, transcriber: Transcriber, knowledgeEmbedding: KnowledgeEmbeddingConfig) {+ self.assetStore = assetStoreself.transcriber = transcriberself.knowledgeEmbedding = knowledgeEmbedding+ self.embeddingAsset = knowledgeEmbeddingModelAsset()}- private var embeddingDownloadTarget: ModelTarget {- knowledgeEmbeddingModelTarget()+ func isEmbeddingModelReady() -> Bool {+ assetStore.isDownloaded(embeddingAsset)}- func isEmbeddingModelReady() -> Bool {- downloader.isDownloaded(embeddingDownloadTarget)+ func isChatModelReady(_ selection: LlmModelSelection) -> Bool {+ assetStore.isDownloaded(chatAsset(selection))}func isModelDownloaded(_ selection: LlmModelSelection) -> Bool {- downloader.isDownloaded(selection.modelTarget) &&+ isChatModelReady(selection) &&(!isEnsuPacksEnabled || isEmbeddingModelReady())}func missingModelDownloadSize(_ selection: LlmModelSelection) async -> Int64? {var total: Int64 = 0- if !downloader.isDownloaded(selection.modelTarget) {- guard let chatSize = await downloader.estimateDownloadSize(selection.modelTarget) else {+ let asset = chatAsset(selection)+ if !assetStore.isDownloaded(asset) {+ guard let chatSize = await assetStore.estimateDownloadSize(asset) else {return nil}total += chatSize}if isEnsuPacksEnabled && !isEmbeddingModelReady() {- guard let embeddingSize = await downloader.estimateDownloadSize(embeddingDownloadTarget) else {+ guard let embeddingSize = await assetStore.estimateDownloadSize(embeddingAsset) else {return nil}total += embeddingSize@@ -171,26 +173,27 @@ final class LlmProvider {throw UnsupportedDeviceMemoryError(capability: capability)}- let missingTargets: [ModelTarget] = try await modelLoadGate.withLock {+ let asset = chatAsset(selection)+ let missingAssets: [Asset] = try await modelLoadGate.withLock {let embeddingReady = isEmbeddingModelReady()if isEnsuPacksEnabled && !embeddingReady {- _ = downloader.removeDownloaded(embeddingDownloadTarget)+ _ = assetStore.removeDownloaded(embeddingAsset)}return [- downloader.isDownloaded(selection.modelTarget) ? nil : selection.modelTarget,- (!isEnsuPacksEnabled || embeddingReady) ? nil : embeddingDownloadTarget,+ assetStore.isDownloaded(asset) ? nil : asset,+ (!isEnsuPacksEnabled || embeddingReady) ? nil : embeddingAsset,].compactMap { $0 }}- if !missingTargets.isEmpty {- try await downloader.download(targets: missingTargets, onProgress: onProgress)+ if !missingAssets.isEmpty {+ try await downloadAssets(missingAssets, onProgress: onProgress)}try await modelLoadGate.withLock {guard !isEnsuPacksEnabled || isEmbeddingModelReady() else {- _ = downloader.removeDownloaded(embeddingDownloadTarget)- throw RequiredModelValidationError(targetId: knowledgeEmbedding.targetId)+ _ = assetStore.removeDownloaded(embeddingAsset)+ throw RequiredModelValidationError(modelId: knowledgeEmbedding.targetId)}try await ensureModelReadyLocked(selection,@@ -198,8 +201,8 @@ final class LlmProvider {allowRecovery: false,shouldDownload: false)- guard downloader.isDownloaded(selection.modelTarget) else {- throw RequiredModelValidationError(targetId: selection.id)+ guard assetStore.isDownloaded(asset) else {+ throw RequiredModelValidationError(modelId: selection.id)}}}@@ -235,19 +238,20 @@ final class LlmProvider {backendInitialized = true}- let wasAlreadyDownloaded = downloader.isDownloaded(selection.modelTarget)+ let asset = chatAsset(selection)+ let wasAlreadyDownloaded = assetStore.isDownloaded(asset)if shouldDownload {- try await downloader.download(targets: [selection.modelTarget], onProgress: onProgress)+ try await downloadAssets([asset], onProgress: onProgress)}onProgress(DownloadProgress(percent: 100, status: "Loading model...", phase: .loading))do {try loadModel(selection,- modelPath: downloader.llmModelPath(selection.modelTarget)!+ modelPath: assetStore.llmModelPath(asset)!)} catch {- if allowRecovery, wasAlreadyDownloaded, downloader.removeDownloaded(selection.modelTarget) {+ if allowRecovery, wasAlreadyDownloaded, assetStore.removeDownloaded(asset) {onProgress(DownloadProgress(percent: 0, status: "Starting download..."))try await ensureModelReadyLocked(selection, onProgress: onProgress, allowRecovery: false)return@@ -257,6 +261,20 @@ final class LlmProvider {onProgress(DownloadProgress(percent: 100, status: "Ready", phase: .ready))}+ private func downloadAssets(+ _ assets: [Asset],+ onProgress: @escaping (DownloadProgress) -> Void+ ) async throws {+ try await assetStore.download(assets: assets) { progress in+ onProgress(+ DownloadProgress(+ percent: min(max(Int(progress.percentage), 0), 99),+ status: progress.status+ )+ )+ }+ }+func generateChat(_ selection: LlmModelSelection,messages: [LlmMessage],@@ -298,9 +316,10 @@ final class LlmProvider {LlmChatMessage(role: $0.role.roleString, content: $0.text)}+ let asset = chatAsset(selection)let mmprojPath = imageFiles.isEmpty? nil- : downloader.llmMmprojPath(selection.modelTarget)?.path+ : assetStore.llmMmprojPath(asset)?.pathlet clampedTemperature = min(max(temperature, 0.35), 0.7)let request = LlmChatRequest(@@ -371,7 +390,7 @@ final class LlmProvider {backendInitialized = true}- guard let embeddingModelPath = downloader.llmModelPath(embeddingDownloadTarget) else {+ guard let embeddingModelPath = assetStore.llmModelPath(embeddingAsset) else {throw EmbeddingAssetInvalidError()}var embeddingModel: LlmModel? = try LlmModel.load(@@ -412,14 +431,15 @@ final class LlmProvider {}func prewarmImageInference(_ selection: LlmModelSelection) async {- guard downloader.isDownloaded(selection.modelTarget) else { return }+ guard isChatModelReady(selection) else { return }do {try await Task.detached(priority: .utility) { [weak self] inguard let self else { return }try await self.modelLoadGate.withLock {- guard self.downloader.isDownloaded(selection.modelTarget) else { return }- guard let mmprojPath = self.downloader.llmMmprojPath(selection.modelTarget),+ let asset = self.chatAsset(selection)+ guard self.assetStore.isDownloaded(asset) else { return }+ guard let mmprojPath = self.assetStore.llmMmprojPath(asset),FileManager.default.fileExists(atPath: mmprojPath.path) else {return}@@ -469,6 +489,10 @@ final class LlmProvider {transcriber.unloadModel()}+ private func chatAsset(_ selection: LlmModelSelection) -> Asset {+ llmAsset(modelId: selection.id)+ }+private func loadModel(_ selection: LlmModelSelection, modelPath: URL) throws {let params = LlmModelLoadParams(modelPath: modelPath.path, nGpuLayers: 0, useMmap: true, useMlock: false)let model = try LlmModel.load(params: params)diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swiftindex 776ead4909..5dac12ad3c 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift@@ -100,7 +100,6 @@ final class ModelSettingsStore: ObservableObject {let preset = presets.first(where: { $0.id == modelId }) ?? defaultModelreturn LlmModelSelection(id: preset.id,- modelTarget: try! mobileLlmTarget(modelId: preset.id),contextLength: Int(contextLength),maxTokens: Int(maxTokens).flatMap { $0 > 0 ? $0 : nil })diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 56971dab38..aa6c551cf7 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2268,9 +2268,9 @@ checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099"name = "ensu-desktop"version = "0.1.18-beta"dependencies = [+ "ente-assets","ente-core","ente-ensu",- "ente-model-download","keyring","libc","serde",@@ -2311,6 +2311,21 @@ dependencies = ["zeroize",]+[[package]]+name = "ente-assets"+version = "0.0.0"+dependencies = [+ "flate2",+ "futures-util",+ "reqwest 0.12.28",+ "serde",+ "serde_json",+ "sha2 0.11.0",+ "tar",+ "thiserror 2.0.18",+ "tokio",+]+[[package]]name = "ente-cast-uniffi"version = "0.0.0"@@ -2373,9 +2388,9 @@ dependencies = [name = "ente-ensu"version = "0.0.0"dependencies = [+ "ente-assets","ente-core","ente-image",- "ente-model-download","llama-cpp-2","memmap2","ort",@@ -2386,7 +2401,6 @@ dependencies = ["self_cell","serde","serde_json",- "sha2 0.11.0","tempfile","thiserror 2.0.18","tokio",@@ -2401,9 +2415,10 @@ dependencies = [name = "ente-ensu-uniffi"version = "0.0.0"dependencies = [+ "ente-assets","ente-ensu",- "ente-model-download","thiserror 2.0.18",+ "tokio","uniffi","uuid",]@@ -2422,21 +2437,6 @@ dependencies = ["tiff 0.10.3",]-[[package]]-name = "ente-model-download"-version = "0.0.0"-dependencies = [- "flate2",- "futures-util",- "reqwest 0.12.28",- "serde",- "serde_json",- "sha2 0.11.0",- "tar",- "thiserror 2.0.18",- "tokio",-]-[[package]]name = "ente-paste"version = "0.0.0"@@ -2456,8 +2456,8 @@ version = "0.0.0"dependencies = ["anyhow","ash",+ "ente-assets","ente-image",- "ente-model-download","fast_image_resize","flate2","half",diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex abd9a2321f..222d71b6b0 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -12,24 +12,24 @@ members = ["bindings/wasm/core","bindings/wasm/space","crates/accounts",+ "crates/assets","crates/contacts","crates/core","crates/ensu",- "crates/paste","crates/image",- "crates/model-download",+ "crates/paste","crates/photos","crates/test-support","space",][workspace.dependencies]-ente-ensu = { path = "crates/ensu" }ente-accounts = { path = "crates/accounts" }+ente-assets = { path = "crates/assets" }ente-contacts = { path = "crates/contacts" }ente-core = { path = "crates/core" }+ente-ensu = { path = "crates/ensu" }ente-image = { path = "crates/image" }-ente-model-download = { path = "crates/model-download" }ente-paste = { path = "crates/paste" }ente-photos = { path = "crates/photos" }ente-rs = { path = "apps/cli" }diff --git a/rust/apps/ensu/src-tauri/Cargo.toml b/rust/apps/ensu/src-tauri/Cargo.tomlindex 49a424d66e..8f65f247ce 100644--- a/rust/apps/ensu/src-tauri/Cargo.toml+++ b/rust/apps/ensu/src-tauri/Cargo.toml@@ -7,19 +7,19 @@ edition = "2024"tauri-build = { version = "2.6.2", features = [] }[dependencies]+ente-assets.workspace = true+ente-core.workspace = true+ente-ensu.workspace = true+keyring = { version = "3", default-features = false, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }+serde.workspace = true+serde_json.workspace = truetauri = { version = "2.11.2", features = [] }tauri-plugin-dialog = "2.7.1"tauri-plugin-fs = "2.5.1"tauri-plugin-opener = "2.5.4"tauri-plugin-process = "2.3.1"tauri-plugin-updater = "2.10.1"-serde.workspace = true-serde_json.workspace = true-ente-ensu.workspace = true-ente-model-download.workspace = true-ente-core.workspace = trueuuid = { version = "1", features = ["v4"] }-keyring = { version = "3", default-features = false, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }[target.'cfg(unix)'.dependencies]libc = "0.2"diff --git a/rust/apps/ensu/src-tauri/src/commands/llm.rs b/rust/apps/ensu/src-tauri/src/commands/llm.rsindex 7c3e7e1f5a..31fdc7017a 100644--- a/rust/apps/ensu/src-tauri/src/commands/llm.rs+++ b/rust/apps/ensu/src-tauri/src/commands/llm.rs@@ -3,9 +3,8 @@ use std::path::PathBuf;use std::sync::{Arc, Mutex, PoisonError};use std::time::{Duration, Instant};+use ente_assets::{Asset, AssetStore, download};use ente_ensu::llm;-use ente_model_download::download;-use ente_model_download::{ModelDownloader, ModelTarget};use serde::Serialize;use tauri::async_runtime;use tauri::{AppHandle, Emitter, Manager, State as TauriState, WebviewWindow};@@ -20,16 +19,19 @@ pub struct State {}pub struct ModelDownloadState {- downloader: Arc<ModelDownloader>,- models_dir: PathBuf,+ store: Arc<AssetStore>,+ assets_dir: PathBuf,+ legacy_models_dir: PathBuf,active_token: Mutex<Option<download::CancellationToken>>,}impl ModelDownloadState {- pub fn new(models_dir: PathBuf) -> Self {+ pub fn new(app_data_dir: PathBuf) -> Self {+ let assets_dir = app_data_dir.join("assets");Self {- downloader: Arc::new(ModelDownloader::new(&models_dir)),- models_dir,+ store: Arc::new(AssetStore::new(&assets_dir)),+ assets_dir,+ legacy_models_dir: app_data_dir.join("models"),active_token: Mutex::new(None),}}@@ -94,8 +96,8 @@ fn fs_thread_error() -> ApiError {ApiError::new("io_thread", "FS task failed")}-fn desktop_target(model_id: &str) -> Result<ModelTarget, ApiError> {- ente_ensu::model::desktop_llm_target(model_id)+fn desktop_asset(model_id: &str) -> Result<Asset, ApiError> {+ ente_ensu::model::desktop_llm_asset(model_id).map_err(|err| ApiError::new("invalid_target", err.to_string()))}@@ -245,14 +247,14 @@ pub fn llm_model_status(state: TauriState<'_, ModelDownloadState>,model_id: String,) -> Result<ModelStatus, ApiError> {- let target = desktop_target(&model_id)?;+ let asset = desktop_asset(&model_id)?;Ok(ModelStatus {- model_path: ente_ensu::model::llm_model_path(&state.downloader, &target)+ model_path: ente_ensu::model::llm_model_path(&state.store, &asset).map(|path| path.display().to_string()).unwrap_or_default(),- mmproj_path: ente_ensu::model::llm_mmproj_path(&state.downloader, &target)+ mmproj_path: ente_ensu::model::llm_mmproj_path(&state.store, &asset).map(|path| path.display().to_string()),- downloaded: state.downloader.is_downloaded(&target),+ downloaded: state.store.is_downloaded(&asset),})}@@ -262,12 +264,18 @@ pub async fn llm_migrate_models(legacy_model_url: Option<String>,legacy_mmproj_url: Option<String>,) -> Result<Option<String>, ApiError> {- let models_dir = state.models_dir.clone();+ let assets_dir = state.assets_dir.clone();+ let legacy_models_dir = state.legacy_models_dir.clone();async_runtime::spawn_blocking(move || {- ente_ensu::model::migrations::migrate_desktop_models(- &models_dir,- legacy_model_url.as_deref(),- legacy_mmproj_url.as_deref(),+ ente_ensu::model::migrations::migrate_ensu_assets(+ &assets_dir,+ ente_ensu::model::migrations::LegacyAssets {+ llm_dir: None,+ transcription_dir: None,+ models_dir: Some(legacy_models_dir),+ model_url: legacy_model_url,+ mmproj_url: legacy_mmproj_url,+ },)}).await@@ -280,8 +288,8 @@ pub async fn llm_download_model(state: TauriState<'_, ModelDownloadState>,model_id: String,) -> Result<(), ApiError> {- let downloader = Arc::clone(&state.downloader);- let target = desktop_target(&model_id)?;+ let store = Arc::clone(&state.store);+ let asset = desktop_asset(&model_id)?;let token = {let mut slot = state.active_token@@ -297,35 +305,33 @@ pub async fn llm_download_model(*slot = Some(token.clone());token};- let result = async_runtime::spawn_blocking(move || {- downloader- .download(- std::slice::from_ref(&target),- |progress| {- if let Some(line) = &progress.log_line {- logging::log("LLMDownload", line.clone());- }- let _ = window.emit(- "llm-download-progress",- DownloadProgress {- percent: progress.percent,- status: progress.status,- bytes_downloaded: progress.downloaded_bytes,- total_bytes: progress.total_bytes,- },- );- },- &token,- )- .map_err(|err| ApiError::new(download_code(&err), err.to_string()))- })- .await- .map_err(|_| fs_thread_error());+ let result = store+ .download(+ &asset,+ |progress| {+ let progress = ente_ensu::model::display_progress(progress);+ if let Some(line) = &progress.log_line {+ logging::log("LLMDownload", line.clone());+ }+ let _ = window.emit(+ "llm-download-progress",+ DownloadProgress {+ percent: progress.percent,+ status: progress.status,+ bytes_downloaded: progress.downloaded_bytes,+ total_bytes: progress.total_bytes,+ },+ );+ },+ token,+ )+ .await+ .map_err(|err| ApiError::new(download_code(&err), err.to_string()));*state.active_token.lock().unwrap_or_else(PoisonError::into_inner) = None;- result?+ result}#[tauri::command]diff --git a/rust/apps/ensu/src-tauri/src/main.rs b/rust/apps/ensu/src-tauri/src/main.rsindex 0abe0301ed..663f11e3b1 100644--- a/rust/apps/ensu/src-tauri/src/main.rs+++ b/rust/apps/ensu/src-tauri/src/main.rs@@ -21,8 +21,9 @@ fn main() {logging::init_logging(app.handle());logging::log("App", "setup started");- let models_dir = app.path().app_data_dir()?.join("models");- app.manage(commands::llm::ModelDownloadState::new(models_dir));+ app.manage(commands::llm::ModelDownloadState::new(+ app.path().app_data_dir()?,+ ));// Show the main window after setup is completeif let Some(window) = app.get_webview_window("main")diff --git a/rust/bindings/uniffi/ensu/Cargo.toml b/rust/bindings/uniffi/ensu/Cargo.tomlindex c906a327a3..1567596d84 100644--- a/rust/bindings/uniffi/ensu/Cargo.toml+++ b/rust/bindings/uniffi/ensu/Cargo.toml@@ -7,8 +7,9 @@ name = "ensu"crate-type = ["staticlib", "cdylib"][dependencies]+ente-assets.workspace = trueente-ensu = { workspace = true, features = ["transcription"] }-ente-model-download.workspace = truethiserror.workspace = true+tokio = { version = "1", features = ["rt-multi-thread"] }uniffi.workspace = trueuuid = { version = "1" }diff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rsnew file mode 100644index 0000000000..f1ce142241--- /dev/null+++ b/rust/bindings/uniffi/ensu/src/assets.rs@@ -0,0 +1,268 @@+use std::path::PathBuf;+use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::{Arc, OnceLock};++use ente_assets::{Asset as RustAsset, AssetStore as RustAssetStore, download};+use thiserror::Error;++#[derive(uniffi::Object)]+pub struct Asset {+ pub(crate) inner: RustAsset,+}++impl Asset {+ pub(crate) fn new(inner: RustAsset) -> Arc<Self> {+ Arc::new(Self { inner })+ }+}++#[derive(Debug, Error, uniffi::Error)]+pub enum AssetDownloadError {+ #[error("download cancelled")]+ Cancelled,+ #[error("{message}")]+ Validation { message: String },+ #[error("HTTP {status}")]+ Http { status: u16 },+ #[error("network: {message}")]+ Network { message: String },+ #[error("size mismatch: expected {expected} bytes, got {actual}")]+ SizeMismatch { expected: u64, actual: u64 },+ #[error("range protocol violation: {message}")]+ Protocol { message: String },+ #[error("invalid download target: {message}")]+ InvalidTarget { message: String },+ #[error("not enough storage space")]+ StorageFull,+ #[error("{message}")]+ Io { message: String },+}++impl From<download::Error> for AssetDownloadError {+ fn from(value: download::Error) -> Self {+ match value {+ download::Error::Cancelled => Self::Cancelled,+ download::Error::Target { source, .. } => Self::from(*source),+ download::Error::Fallback { single, .. } => Self::from(*single),+ download::Error::Validation(message) => Self::Validation { message },+ download::Error::Http(status) => Self::Http { status },+ download::Error::Network(message) => Self::Network { message },+ download::Error::SizeMismatch { expected, actual } => {+ Self::SizeMismatch { expected, actual }+ }+ download::Error::Protocol(message) => Self::Protocol { message },+ download::Error::InvalidTarget(message) => Self::InvalidTarget { message },+ download::Error::StorageFull => Self::StorageFull,+ download::Error::Io(error) => Self::Io {+ message: error.to_string(),+ },+ download::Error::Json(error) => Self::Io {+ message: error.to_string(),+ },+ }+ }+}++#[uniffi::export]+pub fn llm_asset(model_id: String) -> Arc<Asset> {+ Asset::new(ente_ensu::model::mobile_llm_asset(&model_id).expect("valid mobile model catalog"))+}++#[uniffi::export]+pub fn transcription_model_asset() -> Arc<Asset> {+ Asset::new(ente_ensu::model::transcription_model_asset())+}++#[uniffi::export]+pub fn voice_activity_model_asset() -> Arc<Asset> {+ Asset::new(ente_ensu::model::voice_activity_model_asset())+}++#[uniffi::export]+pub fn knowledge_embedding_model_asset() -> Arc<Asset> {+ Asset::new(ente_ensu::model::knowledge_embedding_model_asset())+}++#[derive(Debug, Clone, uniffi::Record)]+pub struct AssetDownloadProgress {+ pub label: String,+ pub downloaded_bytes: i64,+ pub total_bytes: Option<i64>,+ pub percentage: f64,+ pub status: String,+ pub log_line: Option<String>,+}++impl From<ente_assets::download::Progress> for AssetDownloadProgress {+ fn from(value: ente_assets::download::Progress) -> Self {+ let display = ente_ensu::model::display_progress(value.clone());+ Self {+ label: value.label,+ downloaded_bytes: i64::try_from(value.downloaded_bytes).unwrap_or(i64::MAX),+ total_bytes: value+ .total_bytes+ .map(|total| i64::try_from(total).unwrap_or(i64::MAX)),+ percentage: value.percentage,+ status: display.status,+ log_line: display.log_line,+ }+ }+}++#[uniffi::export(callback_interface)]+pub trait AssetDownloadCallback: Send + Sync {+ fn on_progress(&self, progress: AssetDownloadProgress);+}++#[derive(Clone, uniffi::Record)]+pub struct LegacyAssets {+ pub llm_dir: Option<String>,+ pub transcription_dir: Option<String>,+ pub model_url: Option<String>,+ pub mmproj_url: Option<String>,+}++impl From<LegacyAssets> for ente_ensu::model::migrations::LegacyAssets {+ fn from(value: LegacyAssets) -> Self {+ Self {+ llm_dir: value.llm_dir.map(PathBuf::from),+ transcription_dir: value.transcription_dir.map(PathBuf::from),+ models_dir: None,+ model_url: value.model_url,+ mmproj_url: value.mmproj_url,+ }+ }+}++#[uniffi::export]+pub fn needs_asset_migration(legacy: LegacyAssets) -> bool {+ ente_ensu::model::migrations::needs_migration(&legacy.into())+}++#[uniffi::export]+pub fn migrate_ensu_assets(assets_dir: String, legacy: LegacyAssets) -> Option<String> {+ ente_ensu::model::migrations::migrate_ensu_assets(+ PathBuf::from(assets_dir).as_path(),+ legacy.into(),+ )+}++#[derive(uniffi::Object)]+pub struct AssetStoreCore {+ inner: RustAssetStore,+ runtime: OnceLock<tokio::runtime::Runtime>,+ active_downloads: AtomicUsize,+}++#[uniffi::export]+impl AssetStoreCore {+ #[uniffi::constructor]+ pub fn new(assets_dir: String) -> Arc<Self> {+ Arc::new(Self {+ inner: RustAssetStore::new(assets_dir),+ runtime: OnceLock::new(),+ active_downloads: AtomicUsize::new(0),+ })+ }++ pub fn asset_dir(&self, asset: Arc<Asset>) -> String {+ self.inner.asset_dir(&asset.inner).display().to_string()+ }++ pub fn llm_model_path(&self, asset: Arc<Asset>) -> Option<String> {+ ente_ensu::model::llm_model_path(&self.inner, &asset.inner)+ .map(|path| path.display().to_string())+ }++ pub fn llm_mmproj_path(&self, asset: Arc<Asset>) -> Option<String> {+ ente_ensu::model::llm_mmproj_path(&self.inner, &asset.inner)+ .map(|path| path.display().to_string())+ }++ pub fn voice_activity_model_path(&self, asset: Arc<Asset>) -> String {+ ente_ensu::model::voice_activity_model_path(&self.inner, &asset.inner)+ .display()+ .to_string()+ }++ pub fn is_downloaded(&self, asset: Arc<Asset>) -> bool {+ self.inner.is_downloaded(&asset.inner)+ }++ pub fn is_download_active(&self) -> bool {+ self.active_downloads.load(Ordering::SeqCst) > 0+ }++ pub fn estimated_download_size(&self, asset: Arc<Asset>) -> Option<i64> {+ self.runtime()+ .ok()?+ .block_on(self.inner.estimated_download_size(&asset.inner))+ .map(|size| i64::try_from(size).unwrap_or(i64::MAX))+ }++ pub fn remove_downloaded(&self, asset: Arc<Asset>) -> bool {+ let existed = self.inner.is_downloaded(&asset.inner);+ let removed = self+ .runtime()+ .is_ok_and(|runtime| runtime.block_on(self.inner.remove(&asset.inner)).is_ok());+ existed && removed+ }++ pub fn download(+ &self,+ assets: Vec<Arc<Asset>>,+ callback: Box<dyn AssetDownloadCallback>,+ cancellation: Arc<CancellationToken>,+ ) -> Result<(), AssetDownloadError> {+ self.active_downloads.fetch_add(1, Ordering::SeqCst);+ let result = self.runtime().and_then(|runtime| {+ runtime.block_on(async {+ for asset in assets {+ self.inner+ .download(+ &asset.inner,+ |progress| callback.on_progress(progress.into()),+ cancellation.inner.clone(),+ )+ .await?;+ }+ Ok(())+ })+ });+ self.active_downloads.fetch_sub(1, Ordering::SeqCst);+ result.map_err(Into::into)+ }+}++impl AssetStoreCore {+ pub(crate) fn store(&self) -> &RustAssetStore {+ &self.inner+ }++ pub(crate) fn runtime(&self) -> Result<&tokio::runtime::Runtime, ente_assets::download::Error> {+ if let Some(runtime) = self.runtime.get() {+ return Ok(runtime);+ }+ let runtime = tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?;+ Ok(self.runtime.get_or_init(|| runtime))+ }+}++#[derive(Default, uniffi::Object)]+pub struct CancellationToken {+ pub(crate) inner: ente_assets::download::CancellationToken,+}++#[uniffi::export]+impl CancellationToken {+ #[uniffi::constructor]+ pub fn new() -> Arc<Self> {+ Arc::new(Self::default())+ }++ pub fn cancel(&self) {+ self.inner.cancel();+ }+}diff --git a/rust/bindings/uniffi/ensu/src/download.rs b/rust/bindings/uniffi/ensu/src/download.rsdeleted file mode 100644index 3211752e56..0000000000--- a/rust/bindings/uniffi/ensu/src/download.rs+++ /dev/null@@ -1,39 +0,0 @@-use ente_model_download::download;--#[derive(Debug, Clone, uniffi::Enum)]-pub enum DownloadError {- Cancelled,- Validation { message: String },- Http { status: u16 },- Network { message: String },- SizeMismatch { expected: u64, actual: u64 },- Protocol { message: String },- InvalidTarget { message: String },- StorageFull,- Io { message: String },-}--impl From<download::Error> for DownloadError {- fn from(value: download::Error) -> Self {- match value {- download::Error::Cancelled => Self::Cancelled,- download::Error::Target { source, .. } => Self::from(*source),- download::Error::Fallback { single, .. } => Self::from(*single),- download::Error::Validation(message) => Self::Validation { message },- download::Error::Http(status) => Self::Http { status },- download::Error::Network(message) => Self::Network { message },- download::Error::SizeMismatch { expected, actual } => {- Self::SizeMismatch { expected, actual }- }- download::Error::Protocol(message) => Self::Protocol { message },- download::Error::InvalidTarget(message) => Self::InvalidTarget { message },- download::Error::StorageFull => Self::StorageFull,- download::Error::Io(err) => Self::Io {- message: err.to_string(),- },- download::Error::Json(err) => Self::Io {- message: err.to_string(),- },- }- }-}diff --git a/rust/bindings/uniffi/ensu/src/lib.rs b/rust/bindings/uniffi/ensu/src/lib.rsindex 52e1a939ed..32a99e8f84 100644--- a/rust/bindings/uniffi/ensu/src/lib.rs+++ b/rust/bindings/uniffi/ensu/src/lib.rs@@ -1,10 +1,9 @@uniffi::setup_scaffolding!("ensu");+pub mod assets;pub mod config;pub mod db;-pub mod download;pub mod image;pub mod llm;-pub mod model;pub mod retrieval;pub mod transcription;diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rsindex 7f5ab0703f..ebe8d9cf2e 100644--- a/rust/bindings/uniffi/ensu/src/llm.rs+++ b/rust/bindings/uniffi/ensu/src/llm.rs@@ -1,6 +1,5 @@use ente_ensu::llm;-use crate::download::DownloadError;use std::sync::Arc;use thiserror::Error;@@ -20,8 +19,6 @@ pub enum LlmError {PromptTooLong { tokens: u64, context_size: u32 },#[error("{op}: {detail}")]Llama { op: String, detail: String },- #[error("download failed")]- Download { error: DownloadError },}impl From<llm::Error> for LlmError {diff --git a/rust/bindings/uniffi/ensu/src/model.rs b/rust/bindings/uniffi/ensu/src/model.rsdeleted file mode 100644index 2e3809ec3f..0000000000--- a/rust/bindings/uniffi/ensu/src/model.rs+++ /dev/null@@ -1,234 +0,0 @@-use std::path::{Path, PathBuf};-use std::sync::Arc;--use crate::download::DownloadError;-use crate::llm::LlmError;--#[derive(Debug, Clone, uniffi::Enum)]-pub enum ModelTarget {- Files {- id: String,- files: Vec<ModelFile>,- },- TarGz {- id: String,- url: String,- sha256: String,- },-}--#[derive(Debug, Clone, uniffi::Record)]-pub struct ModelFile {- pub name: String,- pub url: String,- pub sha256: String,-}--impl From<ModelTarget> for ente_model_download::ModelTarget {- fn from(value: ModelTarget) -> Self {- match value {- ModelTarget::Files { id, files } => Self::Files {- id,- files: files- .into_iter()- .map(|file| ente_model_download::ModelFile {- name: file.name,- url: file.url,- sha256: file.sha256,- })- .collect(),- },- ModelTarget::TarGz { id, url, sha256 } => Self::TarGz { id, url, sha256 },- }- }-}--impl From<ente_model_download::ModelTarget> for ModelTarget {- fn from(value: ente_model_download::ModelTarget) -> Self {- match value {- ente_model_download::ModelTarget::Files { id, files } => Self::Files {- id,- files: files- .into_iter()- .map(|file| ModelFile {- name: file.name,- url: file.url,- sha256: file.sha256,- })- .collect(),- },- ente_model_download::ModelTarget::TarGz { id, url, sha256 } => {- Self::TarGz { id, url, sha256 }- }- }- }-}--#[uniffi::export]-pub fn mobile_llm_target(model_id: String) -> Result<ModelTarget, LlmError> {- ente_ensu::model::mobile_llm_target(&model_id)- .map(Into::into)- .map_err(|err| LlmError::Download {- error: DownloadError::InvalidTarget {- message: err.to_string(),- },- })-}--#[uniffi::export]-pub fn transcription_model_target() -> ModelTarget {- ente_ensu::model::transcription_target().into()-}--#[uniffi::export]-pub fn voice_activity_model_target() -> ModelTarget {- ente_ensu::model::voice_activity_target().into()-}--#[uniffi::export]-pub fn knowledge_embedding_model_target() -> ModelTarget {- ente_ensu::model::knowledge_embedding_target().into()-}--#[derive(Debug, Clone, uniffi::Record)]-pub struct ModelDownloadProgress {- pub downloaded_bytes: i64,- pub total_bytes: Option<i64>,- pub percent: i32,- pub status: String,- pub log_line: Option<String>,-}--impl From<ente_model_download::ModelDownloadProgress> for ModelDownloadProgress {- fn from(value: ente_model_download::ModelDownloadProgress) -> Self {- Self {- downloaded_bytes: i64::try_from(value.downloaded_bytes).unwrap_or(i64::MAX),- total_bytes: value- .total_bytes- .map(|total| i64::try_from(total).unwrap_or(i64::MAX)),- percent: value.percent,- status: value.status,- log_line: value.log_line,- }- }-}--#[uniffi::export(callback_interface)]-pub trait ModelDownloadCallback: Send + Sync {- fn on_progress(&self, progress: ModelDownloadProgress);-}--#[derive(uniffi::Record)]-pub struct LegacyModels {- pub llm_dir: Option<String>,- pub transcription_dir: String,- pub model_url: Option<String>,- pub mmproj_url: Option<String>,-}--#[uniffi::export]-pub fn migrate_mobile_models(models_dir: String, legacy: LegacyModels) -> Option<String> {- ente_ensu::model::migrations::migrate_mobile_models(- Path::new(&models_dir),- ente_ensu::model::migrations::LegacyModels {- llm_dir: legacy.llm_dir.map(PathBuf::from),- transcription_dir: PathBuf::from(legacy.transcription_dir),- model_url: legacy.model_url,- mmproj_url: legacy.mmproj_url,- },- )-}--#[derive(uniffi::Object)]-pub struct ModelDownloadCore {- inner: ente_model_download::ModelDownloader,-}--#[uniffi::export]-impl ModelDownloadCore {- #[uniffi::constructor]- pub fn new(models_dir: String) -> Arc<Self> {- Arc::new(Self {- inner: ente_model_download::ModelDownloader::new(models_dir),- })- }-- pub fn model_dir(&self, target: ModelTarget) -> String {- self.inner.model_dir(&target.into()).display().to_string()- }-- pub fn llm_model_path(&self, target: ModelTarget) -> Option<String> {- ente_ensu::model::llm_model_path(&self.inner, &target.into())- .map(|path| path.display().to_string())- }-- pub fn llm_mmproj_path(&self, target: ModelTarget) -> Option<String> {- ente_ensu::model::llm_mmproj_path(&self.inner, &target.into())- .map(|path| path.display().to_string())- }-- pub fn voice_activity_model_path(&self) -> String {- ente_ensu::model::voice_activity_model_path(&self.inner)- .display()- .to_string()- }-- pub fn is_downloaded(&self, target: ModelTarget) -> bool {- self.inner.is_downloaded(&target.into())- }-- pub fn is_download_active(&self) -> bool {- self.inner.is_download_active()- }-- pub fn estimated_download_size(&self, target: ModelTarget) -> Option<i64> {- self.inner- .estimated_download_size(&target.into())- .map(|size| i64::try_from(size).unwrap_or(i64::MAX))- }-- pub fn remove_downloaded(&self, target: ModelTarget) -> bool {- self.inner.remove_downloaded(&target.into())- }-- pub fn download(- &self,- targets: Vec<ModelTarget>,- callback: Box<dyn ModelDownloadCallback>,- cancellation: Arc<CancellationToken>,- ) -> Result<(), LlmError> {- let targets: Vec<ente_model_download::ModelTarget> =- targets.into_iter().map(Into::into).collect();- self.inner- .download(- &targets,- move |progress| callback.on_progress(progress.into()),- &cancellation.inner,- )- .map_err(map_download_error)- }-}--#[derive(Default, uniffi::Object)]-pub struct CancellationToken {- pub(crate) inner: ente_model_download::download::CancellationToken,-}--#[uniffi::export]-impl CancellationToken {- #[uniffi::constructor]- pub fn new() -> Arc<Self> {- Arc::new(Self::default())- }-- pub fn cancel(&self) {- self.inner.cancel();- }-}--fn map_download_error(err: ente_model_download::download::Error) -> LlmError {- match DownloadError::from(err) {- DownloadError::Cancelled => LlmError::Cancelled,- error => LlmError::Download { error },- }-}diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rsindex d6d997c7eb..8d2145d0ba 100644--- a/rust/bindings/uniffi/ensu/src/retrieval.rs+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs@@ -3,8 +3,7 @@ use std::sync::Arc;use ente_ensu::retrieval as core;use thiserror::Error;-use crate::download::DownloadError;-use crate::model::CancellationToken;+use crate::assets::{Asset, AssetStoreCore};#[derive(Debug, Error, uniffi::Error)]pub enum KnowledgeRetrievalError {@@ -46,12 +45,6 @@ fn knowledge_dataset(})}-#[derive(Debug, Error, uniffi::Error)]-pub enum KnowledgeDownloadError {- #[error("knowledge download failed")]- Download { error: DownloadError },-}-#[derive(Debug, Clone, uniffi::Record)]pub struct RetrievalHit {pub score: f32,@@ -165,26 +158,34 @@ impl From<core::KnowledgeReconciliation> for KnowledgeReconciliation {#[uniffi::export]pub fn reconcile_knowledge_pack(- pack_root: String,+ store: Arc<AssetStoreCore>,stable_id: String,) -> Result<KnowledgeReconciliation, KnowledgeRetrievalError> {- core::reconcile_knowledge_pack(pack_root, &knowledge_dataset(&stable_id)?)+ let dataset = knowledge_dataset(&stable_id)?;+ let pack_root = core::knowledge_pack_root(store.store(), &dataset).map_err(|error| {+ KnowledgeRetrievalError::InvalidInput {+ detail: error.to_string(),+ }+ })?;+ core::reconcile_knowledge_pack(pack_root, &dataset).map(Into::into).map_err(Into::into)}#[uniffi::export]pub fn cleanup_obsolete_knowledge_pack_revisions(- pack_root: String,+ store: Arc<AssetStoreCore>,stable_id: String,active_identity: String,) -> Result<(), KnowledgeRetrievalError> {- core::cleanup_obsolete_knowledge_pack_revisions(- pack_root,- &knowledge_dataset(&stable_id)?,- &active_identity,- )- .map_err(Into::into)+ let dataset = knowledge_dataset(&stable_id)?;+ let pack_root = core::knowledge_pack_root(store.store(), &dataset).map_err(|error| {+ KnowledgeRetrievalError::InvalidInput {+ detail: error.to_string(),+ }+ })?;+ core::cleanup_obsolete_knowledge_pack_revisions(pack_root, &dataset, &active_identity)+ .map_err(Into::into)}#[derive(Debug, Clone, uniffi::Record)]@@ -289,47 +290,12 @@ pub fn clean_assistant_text(stored_text: String) -> String {core::clean_assistant_text(&stored_text)}-#[derive(Debug, Clone, uniffi::Record)]-pub struct KnowledgeDownloadProgress {- pub label: String,- pub percentage: f64,-}--impl From<ente_model_download::download::Progress> for KnowledgeDownloadProgress {- fn from(value: ente_model_download::download::Progress) -> Self {- Self {- label: value.label,- percentage: value.percentage,- }- }-}--#[uniffi::export(callback_interface)]-pub trait KnowledgeDownloadCallback: Send + Sync {- fn on_progress(&self, progress: KnowledgeDownloadProgress);-}-#[uniffi::export]-pub fn download_knowledge_pack(- pack_root: String,- stable_id: String,- callback: Box<dyn KnowledgeDownloadCallback>,- cancellation: Arc<CancellationToken>,-) -> Result<(), KnowledgeDownloadError> {- let expected_dataset = ente_ensu::config::knowledge_dataset(&stable_id).ok_or_else(|| {- KnowledgeDownloadError::Download {- error: DownloadError::InvalidTarget {- message: format!("unknown knowledge dataset ID: {stable_id}"),- },- }- })?;- core::download_knowledge_pack(- pack_root,- &expected_dataset,- move |progress| callback.on_progress(progress.into()),- cancellation.inner.clone(),- )- .map_err(|error| KnowledgeDownloadError::Download {- error: error.into(),- })+pub fn knowledge_pack_asset(stable_id: String) -> Result<Arc<Asset>, KnowledgeRetrievalError> {+ let dataset = knowledge_dataset(&stable_id)?;+ core::knowledge_asset(&dataset)+ .map(Asset::new)+ .map_err(|error| KnowledgeRetrievalError::InvalidInput {+ detail: error.to_string(),+ })}diff --git a/rust/crates/model-download/Cargo.toml b/rust/crates/assets/Cargo.tomlsimilarity index 69%rename from rust/crates/model-download/Cargo.tomlrename to rust/crates/assets/Cargo.tomlindex 0e696435dc..0c1cf76068 100644--- a/rust/crates/model-download/Cargo.toml+++ b/rust/crates/assets/Cargo.toml@@ -1,5 +1,5 @@[package]-name = "ente-model-download"+name = "ente-assets"edition = "2024"[dependencies]@@ -11,4 +11,7 @@ serde_json.workspace = truesha2 = "0.11.0"tar = "0.4.46"thiserror.workspace = true-tokio = { version = "1", features = ["rt", "time"] }+tokio = { version = "1", features = ["sync"] }++[dev-dependencies]+tokio = { version = "1", features = ["macros", "rt"] }diff --git a/rust/crates/assets/src/archive.rs b/rust/crates/assets/src/archive.rsnew file mode 100644index 0000000000..261c66ac40--- /dev/null+++ b/rust/crates/assets/src/archive.rs@@ -0,0 +1,75 @@+use std::fs;+use std::fs::File;+use std::io::{Read, Write};+use std::path::{Component, Path, PathBuf};++use flate2::read::GzDecoder;+use tar::Archive;++use crate::download::{CancellationToken, Error};++pub(crate) fn extract_tar_gz(+ archive_path: &Path,+ staging_dir: &Path,+ cancellation: &CancellationToken,+) -> Result<PathBuf, Error> {+ if staging_dir.exists() {+ fs::remove_dir_all(staging_dir)?;+ }+ fs::create_dir_all(staging_dir)?;++ let tar_gz = File::open(archive_path)?;+ let mut archive = Archive::new(GzDecoder::new(tar_gz));+ for entry in archive.entries()? {+ if cancellation.is_cancelled() {+ return Err(Error::Cancelled);+ }+ let mut entry = entry?;+ let entry_type = entry.header().entry_type();+ if !entry_type.is_file() && !entry_type.is_dir() {+ return Err(Error::Validation(+ "tar archive contains a link or special file".to_string(),+ ));+ }+ let path = entry.path()?.into_owned();+ if path.components().any(|component| {+ matches!(+ component,+ Component::ParentDir | Component::RootDir | Component::Prefix(_)+ )+ }) {+ return Err(Error::Validation(+ "tar archive contains an unsafe path".to_string(),+ ));+ }+ let destination = staging_dir.join(path);+ if entry_type.is_dir() {+ fs::create_dir_all(destination)?;+ continue;+ }+ if let Some(parent) = destination.parent() {+ fs::create_dir_all(parent)?;+ }+ let mut output = File::create(destination)?;+ let mut buffer = [0; 1024 * 1024];+ loop {+ if cancellation.is_cancelled() {+ return Err(Error::Cancelled);+ }+ let count = entry.read(&mut buffer)?;+ if count == 0 {+ break;+ }+ output.write_all(&buffer[..count])?;+ }+ }++ let entries = fs::read_dir(staging_dir)?.collect::<Result<Vec<_>, _>>()?;+ if entries.is_empty() {+ return Err(Error::Validation("tar archive is empty".to_string()));+ }+ Ok(match entries.as_slice() {+ [only] if only.path().is_dir() => only.path(),+ _ => staging_dir.to_path_buf(),+ })+}diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/assets/src/download.rssimilarity index 87%rename from rust/crates/model-download/src/download.rsrename to rust/crates/assets/src/download.rsindex 1a9a6328a1..35268e3b56 100644--- a/rust/crates/model-download/src/download.rs+++ b/rust/crates/assets/src/download.rs@@ -1,10 +1,8 @@-use std::cell::RefCell;use std::fs::{self, File, OpenOptions};use std::io::{ErrorKind, Write};use std::path::{Path, PathBuf};-use std::rc::Rc;-use std::sync::Arc;use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};use futures_util::future::try_join_all;@@ -12,16 +10,13 @@ use reqwest::header::{ACCEPT_RANGES, CONTENT_RANGE, ETAG, IF_RANGE, LAST_MODIFIEuse reqwest::{Client, Response, StatusCode};use serde::{Deserialize, Serialize};use sha2::{Digest, Sha256};-use tokio::runtime::Builder;-use tokio::time::timeout;const MIN_RANGE_DOWNLOAD_BYTES: u64 = 1024 * 1024;const MAX_ATTEMPTS: usize = 3;const RANGE_DOWNLOAD_CONCURRENCY: usize = 4;const PROGRESS_INTERVAL: Duration = Duration::from_millis(250);const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);-const RESPONSE_START_TIMEOUT: Duration = Duration::from_secs(30);-const READ_STALL_TIMEOUT: Duration = Duration::from_secs(30);+const READ_TIMEOUT: Duration = Duration::from_secs(30);#[derive(Debug, thiserror::Error)]pub enum Error {@@ -69,7 +64,7 @@ impl From<std::io::Error> for Error {}#[derive(Debug, Clone)]-pub struct Target {+pub struct DownloadTarget {pub label: String,pub url: String,pub sha256: String,@@ -124,6 +119,7 @@ struct FileDownloadReport {#[derive(Debug, Clone, Serialize, Deserialize)]struct PartialDownloadMetadata {url: String,+ sha256: String,etag: Option<String>,last_modified: Option<String>,}@@ -153,6 +149,7 @@ struct FileProgress {#[derive(Debug, Clone, Serialize, Deserialize)]struct RangeDownloadMetadata {url: String,+ sha256: String,size_bytes: u64,etag: Option<String>,last_modified: Option<String>,@@ -192,13 +189,9 @@ impl Downloader {})}- pub(crate) fn client(&self) -> &Client {- &self.client- }-pub async fn download(&self,- targets: Vec<Target>,+ targets: Vec<DownloadTarget>,on_progress: impl FnMut(Progress) + Send,cancellation: CancellationToken,) -> Result<(), Error> {@@ -207,50 +200,43 @@ impl Downloader {}).await}++ pub(crate) async fn estimated_download_size(&self, targets: &[DownloadTarget]) -> Option<u64> {+ let mut remaining = 0u64;+ for target in targets {+ if target.destination.is_file() {+ continue;+ }+ let probe = fetch_download_probe(&self.client, &target.url).await;+ let total = probe.content_length?;+ let downloaded =+ existing_download_bytes(target, &target.destination, &probe, false).min(total);+ remaining = remaining.saturating_add(total - downloaded);+ }+ Some(remaining)+ }}fn build_client() -> Result<Client, Error> {Client::builder().connect_timeout(CONNECT_TIMEOUT)+ .read_timeout(READ_TIMEOUT).build().map_err(|err| Error::Network(err.to_string()))}-pub(crate) fn fetch(- downloader: &Downloader,- targets: Vec<Target>,- on_progress: impl FnMut(Progress),- is_cancelled: impl Fn() -> bool,-) -> Result<(), Error> {- let runtime = Builder::new_current_thread()- .enable_io()- .enable_time()- .build()?;- runtime.block_on(fetch_async(- &downloader.client,- targets,- on_progress,- is_cancelled,- ))-}-async fn fetch_async(client: &Client,- targets: Vec<Target>,- on_progress: impl FnMut(Progress),- is_cancelled: impl Fn() -> bool,+ targets: Vec<DownloadTarget>,+ mut on_progress: impl FnMut(Progress) + Send,+ is_cancelled: impl Fn() -> bool + Sync,) -> Result<(), Error> {if targets.is_empty() {return Ok(());}for target in &targets {- let sha256 = &target.sha256;- if sha256.len() != 64 || !sha256.chars().all(|c| c.is_ascii_hexdigit()) {- return Err(Error::InvalidTarget(format!(- "'{sha256}' is not a SHA-256 digest"- )));- }+ validate_sha256(&target.sha256)?;}if is_cancelled() {@@ -289,7 +275,7 @@ async fn fetch_async(None};- let file_states = targets+ let mut file_states = targets.iter().zip(&download_probes).zip(&cached)@@ -306,15 +292,12 @@ async fn fetch_async(}}).collect::<Vec<_>>();- let file_states = Rc::new(RefCell::new(file_states));- let on_progress = Rc::new(RefCell::new(on_progress));-emit_progress(ProgressPhase::Preparing,Duration::ZERO,total_bytes,&file_states,- &on_progress,+ &mut on_progress,);for (index, target) in targets.iter().enumerate() {@@ -337,11 +320,8 @@ async fn fetch_async(&destination,&download_probe,|file_progress| {- {- let mut states = file_states.borrow_mut();- if let Some(state) = states.get_mut(index) {- *state = file_progress;- }+ if let Some(state) = file_states.get_mut(index) {+ *state = file_progress;}emit_progress(ProgressPhase::File {@@ -352,7 +332,7 @@ async fn fetch_async(download_started_at.elapsed(),total_bytes,&file_states,- &on_progress,+ &mut on_progress,);},&is_cancelled,@@ -366,15 +346,12 @@ async fn fetch_async(},})?;- {- let mut states = file_states.borrow_mut();- if let Some(state) = states.get_mut(index) {- state.downloaded_bytes = file_report.final_size;- state.total_bytes = expected_file_total.or(Some(file_report.final_size));- state.network_downloaded_bytes = file_report.network_downloaded_bytes;- state.elapsed = file_report.elapsed;- state.retry_count = file_report.retry_count;- }+ if let Some(state) = file_states.get_mut(index) {+ state.downloaded_bytes = file_report.final_size;+ state.total_bytes = expected_file_total.or(Some(file_report.final_size));+ state.network_downloaded_bytes = file_report.network_downloaded_bytes;+ state.elapsed = file_report.elapsed;+ state.retry_count = file_report.retry_count;}emit_progress(@@ -386,7 +363,7 @@ async fn fetch_async(download_started_at.elapsed(),total_bytes,&file_states,- &on_progress,+ &mut on_progress,);}emit_progress(@@ -394,19 +371,32 @@ async fn fetch_async(download_started_at.elapsed(),total_bytes,&file_states,- &on_progress,+ &mut on_progress,);Ok(())}+pub(crate) fn validate_sha256(sha256: &str) -> Result<(), Error> {+ if sha256.len() == 64+ && sha256+ .chars()+ .all(|character| character.is_ascii_hexdigit())+ {+ return Ok(());+ }+ Err(Error::InvalidTarget(format!(+ "'{sha256}' is not a SHA-256 digest"+ )))+}+async fn download_file(client: &Client,- target: &Target,+ target: &DownloadTarget,destination: &Path,download_probe: &DownloadProbe,- mut on_progress: impl FnMut(FileProgress),- is_cancelled: &impl Fn() -> bool,+ mut on_progress: impl FnMut(FileProgress) + Send,+ is_cancelled: &(impl Fn() -> bool + Sync),) -> Result<FileDownloadReport, Error> {let parent = destination.parent()@@ -465,11 +455,11 @@ async fn download_file(async fn download_file_single(client: &Client,- target: &Target,+ target: &DownloadTarget,destination: &Path,expected_file_total: Option<u64>,- on_progress: &mut dyn FnMut(FileProgress),- is_cancelled: &impl Fn() -> bool,+ on_progress: &mut (dyn FnMut(FileProgress) + Send),+ is_cancelled: &(impl Fn() -> bool + Sync),) -> Result<FileDownloadReport, Error> {let tmp_path = tmp_path_for(destination);let partial_metadata_path = partial_metadata_path_for(destination);@@ -484,7 +474,7 @@ async fn download_file_single(let tmp_size = file_size(&tmp_path).unwrap_or(0);let resume_validator = if tmp_size > 0 {- partial_download_validator(destination, &target.url)+ partial_download_validator(destination, target)} else {None};@@ -579,9 +569,9 @@ async fn download_file_single(return Err(Error::Cancelled);}- let chunk = match timeout(READ_STALL_TIMEOUT, response.chunk()).await {- Ok(Ok(chunk)) => chunk,- Ok(Err(err)) => {+ let chunk = match response.chunk().await {+ Ok(chunk) => chunk,+ Err(err) => {file.flush().ok();if attempt == MAX_ATTEMPTS {return Err(Error::Network(err.to_string()));@@ -590,18 +580,6 @@ async fn download_file_single(retry_attempt = true;break;}- Err(_) => {- file.flush().ok();- if attempt == MAX_ATTEMPTS {- return Err(Error::Network(format!(- "stalled for {} seconds",- READ_STALL_TIMEOUT.as_secs()- )));- }- retry_count = retry_count.saturating_add(1);- retry_attempt = true;- break;- }};let Some(chunk) = chunk else {break;@@ -691,12 +669,12 @@ async fn download_file_single(async fn download_file_ranged(client: &Client,- target: &Target,+ target: &DownloadTarget,destination: &Path,total: u64,response_metadata: Option<ResponseMetadata>,- on_progress: &mut dyn FnMut(FileProgress),- is_cancelled: &impl Fn() -> bool,+ on_progress: &mut (dyn FnMut(FileProgress) + Send),+ is_cancelled: &(impl Fn() -> bool + Sync),) -> Result<FileDownloadReport, Error> {let tmp_path = tmp_path_for(destination);let range_metadata_path = range_metadata_path_for(destination);@@ -711,7 +689,7 @@ async fn download_file_ranged(.truncate(false).open(&tmp_path)?;file.set_len(total)?;- let file = Rc::new(file);+ let file = Arc::new(file);let range_states = range_metadata.ranges@@ -726,26 +704,26 @@ async fn download_file_ranged(retry_count: 0,}).collect::<Vec<_>>();- let range_states = Rc::new(RefCell::new(range_states));- let range_metadata = Rc::new(RefCell::new(range_metadata));- let on_progress = Rc::new(RefCell::new(on_progress));+ let range_states = Arc::new(Mutex::new(range_states));+ let range_metadata = Arc::new(Mutex::new(range_metadata));+ let on_progress = Arc::new(Mutex::new(on_progress));emit_range_file_progress(total, file_started_at, &range_states, &on_progress);let mut downloads = Vec::new();{- let metadata = range_metadata.borrow();+ let metadata = range_metadata.lock().expect("range metadata lock");for (part_index, range) in metadata.ranges.iter().copied().enumerate() {if range.complete {continue;}- let file = Rc::clone(&file);- let range_states = Rc::clone(&range_states);- let range_metadata = Rc::clone(&range_metadata);+ let file = Arc::clone(&file);+ let range_states = Arc::clone(&range_states);+ let range_metadata = Arc::clone(&range_metadata);let range_metadata_path = range_metadata_path.clone();let response_metadata = response_metadata.clone();- let on_progress = Rc::clone(&on_progress);+ let on_progress = Arc::clone(&on_progress);downloads.push(async move {download_range_part(@@ -795,13 +773,12 @@ async fn download_file_ranged(});}- let network_downloaded_bytes = range_states- .borrow()+ let states = range_states.lock().expect("range state lock");+ let network_downloaded_bytes = states.iter().map(|state| state.network_downloaded_bytes).sum::<u64>();- let retry_count = range_states- .borrow()+ let retry_count = states.iter().map(|state| state.retry_count).fold(0u32, u32::saturating_add);@@ -817,18 +794,18 @@ async fn download_file_ranged(#[allow(clippy::too_many_arguments)]async fn download_range_part(client: &Client,- target: &Target,- file: Rc<File>,+ target: &DownloadTarget,+ file: Arc<File>,part_index: usize,range: RangeDownloadPartMetadata,total: u64,response_metadata: Option<ResponseMetadata>,- range_states: Rc<RefCell<Vec<RangePartState>>>,- range_metadata: Rc<RefCell<RangeDownloadMetadata>>,+ range_states: Arc<Mutex<Vec<RangePartState>>>,+ range_metadata: Arc<Mutex<RangeDownloadMetadata>>,range_metadata_path: PathBuf,file_started_at: Instant,- on_progress: Rc<RefCell<&mut dyn FnMut(FileProgress)>>,- is_cancelled: &impl Fn() -> bool,+ on_progress: Arc<Mutex<&mut (dyn FnMut(FileProgress) + Send)>>,+ is_cancelled: &(impl Fn() -> bool + Sync),) -> Result<(), Error> {let range_len = range_download_len(range);@@ -838,7 +815,7 @@ async fn download_range_part(}{- let mut states = range_states.borrow_mut();+ let mut states = range_states.lock().expect("range state lock");if let Some(state) = states.get_mut(part_index) {state.downloaded_bytes = 0;}@@ -884,9 +861,9 @@ async fn download_range_part(return Err(Error::Cancelled);}- let chunk = match timeout(READ_STALL_TIMEOUT, response.chunk()).await {- Ok(Ok(chunk)) => chunk,- Ok(Err(err)) => {+ let chunk = match response.chunk().await {+ Ok(chunk) => chunk,+ Err(err) => {if attempt == MAX_ATTEMPTS {return Err(Error::Network(err.to_string()));}@@ -894,17 +871,6 @@ async fn download_range_part(retry_attempt = true;break;}- Err(_) => {- if attempt == MAX_ATTEMPTS {- return Err(Error::Network(format!(- "stalled for {} seconds",- READ_STALL_TIMEOUT.as_secs()- )));- }- increment_range_retry(&range_states, part_index);- retry_attempt = true;- break;- }};let Some(chunk) = chunk else {break;@@ -925,7 +891,7 @@ async fn download_range_part(downloaded_in_range = downloaded_in_range.saturating_add(chunk_len);{- let mut states = range_states.borrow_mut();+ let mut states = range_states.lock().expect("range state lock");if let Some(state) = states.get_mut(part_index) {state.downloaded_bytes = downloaded_in_range;state.network_downloaded_bytes =@@ -957,7 +923,7 @@ async fn download_range_part(}{- let mut states = range_states.borrow_mut();+ let mut states = range_states.lock().expect("range state lock");if let Some(state) = states.get_mut(part_index) {state.downloaded_bytes = range_len;}@@ -984,14 +950,9 @@ async fn request_file(.header(RANGE, format!("bytes={resume_from}-")).header(IF_RANGE, if_range);}- timeout(RESPONSE_START_TIMEOUT, request.send())+ request+ .send().await- .map_err(|_| {- Error::Network(format!(- "request did not receive a response within {} seconds",- RESPONSE_START_TIMEOUT.as_secs()- ))- })?.map_err(|err| Error::Network(err.to_string()))}@@ -1007,22 +968,14 @@ async fn request_model_range(if let Some(if_range) = if_range_header_value(response_metadata) {request = request.header(IF_RANGE, if_range);}- timeout(RESPONSE_START_TIMEOUT, request.send())+ request+ .send().await- .map_err(|_| {- Error::Network(format!(- "request did not receive a response within {} seconds",- RESPONSE_START_TIMEOUT.as_secs()- ))- })?.map_err(|err| Error::Network(err.to_string()))}async fn fetch_download_probe(client: &Client, url: &str) -> DownloadProbe {- let response = timeout(RESPONSE_START_TIMEOUT, client.head(url).send())- .await- .ok()- .and_then(Result::ok);+ let response = client.head(url).send().await.ok();let Some(response) = response else {return DownloadProbe::default();};@@ -1054,16 +1007,12 @@ async fn fetch_download_probe(client: &Client, url: &str) -> DownloadProbe {}}-pub(crate) async fn probe_content_length(client: &Client, url: &str) -> Option<u64> {- fetch_download_probe(client, url).await.content_length-}-fn should_use_range_download(total: u64, probe: &DownloadProbe) -> bool {total >= MIN_RANGE_DOWNLOAD_BYTES && probe.supports_ranges}fn prepare_range_download_metadata(- target: &Target,+ target: &DownloadTarget,destination: &Path,total: u64,response_metadata: Option<ResponseMetadata>,@@ -1088,6 +1037,7 @@ fn prepare_range_download_metadata(cleanup_range_download(destination);let metadata = RangeDownloadMetadata {url: target.url.clone(),+ sha256: target.sha256.clone(),size_bytes: total,etag: response_metadata.as_ref()@@ -1130,12 +1080,13 @@ fn range_download_parts(total: u64, concurrency: usize) -> Vec<RangeDownloadPartfn range_download_metadata_matches(metadata: &RangeDownloadMetadata,- target: &Target,+ target: &DownloadTarget,total: u64,response_metadata: Option<&ResponseMetadata>,ranges: &[RangeDownloadPartMetadata],) -> bool {metadata.url == target.url+ && metadata.sha256.eq_ignore_ascii_case(&target.sha256)&& metadata.size_bytes == total&& range_download_validators_match(metadata, response_metadata)&& metadata.ranges.len() == ranges.len()@@ -1171,10 +1122,10 @@ fn range_download_len(range: RangeDownloadPartMetadata) -> u64 {fn emit_range_file_progress(total_bytes: u64,file_started_at: Instant,- range_states: &Rc<RefCell<Vec<RangePartState>>>,- on_progress: &Rc<RefCell<&mut dyn FnMut(FileProgress)>>,+ range_states: &Arc<Mutex<Vec<RangePartState>>>,+ on_progress: &Arc<Mutex<&mut (dyn FnMut(FileProgress) + Send)>>,) {- let states = range_states.borrow();+ let states = range_states.lock().expect("range state lock");let downloaded_bytes = states.iter().map(|state| state.downloaded_bytes)@@ -1190,7 +1141,7 @@ fn emit_range_file_progress(.fold(0u32, u32::saturating_add);drop(states);- let mut callback = on_progress.borrow_mut();+ let mut callback = on_progress.lock().expect("progress lock");(**callback)(FileProgress {downloaded_bytes,total_bytes: Some(total_bytes),@@ -1200,19 +1151,19 @@ fn emit_range_file_progress(});}-fn increment_range_retry(range_states: &Rc<RefCell<Vec<RangePartState>>>, part_index: usize) {- let mut states = range_states.borrow_mut();+fn increment_range_retry(range_states: &Arc<Mutex<Vec<RangePartState>>>, part_index: usize) {+ let mut states = range_states.lock().expect("range state lock");if let Some(state) = states.get_mut(part_index) {state.retry_count = state.retry_count.saturating_add(1);}}fn mark_range_complete(- range_metadata: &Rc<RefCell<RangeDownloadMetadata>>,+ range_metadata: &Arc<Mutex<RangeDownloadMetadata>>,range_metadata_path: &Path,part_index: usize,) -> Result<(), Error> {- let mut metadata = range_metadata.borrow_mut();+ let mut metadata = range_metadata.lock().expect("range metadata lock");if let Some(range) = metadata.ranges.get_mut(part_index) {range.complete = true;}@@ -1330,37 +1281,35 @@ fn emit_progress<F: FnMut(Progress)>(phase: ProgressPhase,elapsed: Duration,total_bytes: Option<u64>,- file_states: &Rc<RefCell<Vec<FileProgress>>>,- on_progress: &Rc<RefCell<F>>,+ file_states: &[FileProgress],+ on_progress: &mut F,) {- let states = file_states.borrow();- let downloaded_bytes = states+ let downloaded_bytes = file_states.iter().map(|state| state.downloaded_bytes).sum::<u64>();- let network_downloaded_bytes = states+ let network_downloaded_bytes = file_states.iter().map(|state| state.network_downloaded_bytes).sum::<u64>();- let retry_count = states+ let retry_count = file_states.iter().map(|state| state.retry_count).fold(0u32, u32::saturating_add);let file_state = match &phase {- ProgressPhase::File { index, .. } => states.get(*index).copied(),+ ProgressPhase::File { index, .. } => file_states.get(*index).copied(),_ => None,};let total_bytes = match phase {ProgressPhase::Complete => total_bytes.or(Some(downloaded_bytes)),_ => total_bytes.or_else(|| {- let total = states+ let total = file_states.iter().filter_map(|state| state.total_bytes).sum::<u64>();(total > 0).then_some(total)}),};- drop(states);let (label, file_complete, complete) = match phase {ProgressPhase::Preparing => ("Preparing downloads", false, false),@@ -1377,7 +1326,7 @@ fn emit_progress<F: FnMut(Progress)>(.map(|state| (state.downloaded_bytes, state.total_bytes)).unwrap_or((0, None));- on_progress.borrow_mut()(Progress {+ on_progress(Progress {label: label.to_string(),downloaded_bytes,total_bytes,@@ -1412,7 +1361,7 @@ fn bytes_per_second(bytes: u64, elapsed: Duration) -> f64 {}}-fn prepare_cached_download(target: &Target, destination: &Path) -> bool {+fn prepare_cached_download(target: &DownloadTarget, destination: &Path) -> bool {if !destination.exists() || check_sha256(destination, &target.sha256, &target.label).is_err() {return false;}@@ -1486,10 +1435,10 @@ fn partial_metadata_path_for(path: &Path) -> PathBuf {PathBuf::from(format!("{}.tmp.partial.json", path.display()))}-fn partial_download_validator(destination: &Path, url: &str) -> Option<String> {+fn partial_download_validator(destination: &Path, target: &DownloadTarget) -> Option<String> {let text = fs::read_to_string(partial_metadata_path_for(destination)).ok()?;let metadata: PartialDownloadMetadata = serde_json::from_str(&text).ok()?;- if metadata.url != url {+ if metadata.url != target.url || !metadata.sha256.eq_ignore_ascii_case(&target.sha256) {return None;}metadata.etag.or(metadata.last_modified)@@ -1497,11 +1446,12 @@ fn partial_download_validator(destination: &Path, url: &str) -> Option<String> {fn write_partial_download_metadata(destination: &Path,- target: &Target,+ target: &DownloadTarget,response_metadata: &ResponseMetadata,) -> Result<(), Error> {let metadata = PartialDownloadMetadata {url: target.url.clone(),+ sha256: target.sha256.clone(),etag: response_metadata.etag.clone(),last_modified: response_metadata.last_modified.clone(),};@@ -1526,7 +1476,7 @@ fn now_ms() -> u64 {}fn existing_download_bytes(- target: &Target,+ target: &DownloadTarget,destination: &Path,probe: &DownloadProbe,cached: bool,@@ -1562,7 +1512,9 @@ fn existing_download_bytes(return 0;}- 0+ partial_download_validator(destination, target)+ .and_then(|_| file_size(&tmp_path_for(destination)))+ .unwrap_or(0)}fn file_size(path: &Path) -> Option<u64> {@@ -1629,6 +1581,20 @@ mod tests {use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};use std::thread;+ async fn download(+ targets: Vec<DownloadTarget>,+ on_progress: impl FnMut(Progress) + Send,+ is_cancelled: bool,+ ) -> Result<(), Error> {+ let cancellation = CancellationToken::default();+ if is_cancelled {+ cancellation.cancel();+ }+ Downloader::new()?+ .download(targets, on_progress, cancellation)+ .await+ }+#[test]fn range_download_parts_split_file_into_four_ranges() {let ranges = range_download_parts(10, 4);@@ -1702,8 +1668,8 @@ mod tests {assert_eq!(parse_content_range("bytes 10-19/not-a-number"), None);}- #[test]- fn download_uses_four_ranges_when_server_supports_ranges() {+ #[tokio::test]+ async fn download_uses_four_ranges_when_server_supports_ranges() {let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));let head_count = Arc::new(AtomicUsize::new(0));let range_get_count = Arc::new(AtomicUsize::new(0));@@ -1753,27 +1719,17 @@ mod tests {let destination = test_dir.join("model.bin");let url = format!("http://{address}/model.bin");- let probe_client = Client::builder().build().expect("build probe client");- let probe_runtime = Builder::new_current_thread()- .enable_io()- .enable_time()- .build()- .expect("build probe runtime");- let probe = probe_runtime.block_on(fetch_download_probe(&probe_client, &url));- assert_eq!(probe.content_length, Some(bytes.len() as u64));- assert!(probe.supports_ranges);-- let result = fetch(- &Downloader::new().unwrap(),- vec![Target {+ let result = download(+ vec![DownloadTarget {label: "Model".to_string(),url,destination: destination.clone(),sha256: sha_hex(&bytes),}],|_| {},- || false,- );+ false,+ )+ .await;running.store(false, Ordering::SeqCst);let _ = TcpStream::connect(address);@@ -1784,7 +1740,7 @@ mod tests {fs::read(&destination).expect("read downloaded file"),*bytes);- assert_eq!(head_count.load(Ordering::SeqCst), 2);+ assert_eq!(head_count.load(Ordering::SeqCst), 1);assert_eq!(range_get_count.load(Ordering::SeqCst),RANGE_DOWNLOAD_CONCURRENCY,@@ -1797,8 +1753,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn download_uses_single_stream_when_server_lacks_range_support() {+ #[tokio::test]+ async fn download_uses_single_stream_when_server_lacks_range_support() {let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));let get_count = Arc::new(AtomicUsize::new(0));@@ -1813,17 +1769,17 @@ mod tests {let test_dir = scratch_dir("single-download");let destination = test_dir.join("model.bin");- let result = fetch(- &Downloader::new().unwrap(),- vec![Target {+ let result = download(+ vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: sha_hex(&bytes),}],|_| {},- || false,- );+ false,+ )+ .await;result.expect("single-stream download succeeds");assert_eq!(@@ -1835,8 +1791,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn downloads_files_sequentially() {+ #[tokio::test]+ async fn downloads_files_sequentially() {let bytes = Arc::new(sample_bytes(512));let active = Arc::new(AtomicUsize::new(0));let max_active = Arc::new(AtomicUsize::new(0));@@ -1859,27 +1815,23 @@ mod tests {};let test_dir = scratch_dir("sequential-downloads");- let targets = ["model", "projector"].map(|label| Target {+ let targets = ["model", "projector"].map(|label| DownloadTarget {label: label.to_string(),url: server.url(&format!("/{label}.bin")),destination: test_dir.join(format!("{label}.bin")),sha256: sha_hex(&bytes),});- fetch(- &Downloader::new().unwrap(),- targets.into(),- |_| {},- || false,- )- .expect("downloads succeed");+ download(targets.into(), |_| {}, false)+ .await+ .expect("downloads succeed");assert_eq!(max_active.load(Ordering::SeqCst), 1);let _ = fs::remove_dir_all(test_dir);}- #[test]- fn completed_download_is_served_from_cache() {+ #[tokio::test]+ async fn completed_download_is_served_from_cache() {let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));let head_count = Arc::new(AtomicUsize::new(0));let range_get_count = Arc::new(AtomicUsize::new(0));@@ -1903,25 +1855,22 @@ mod tests {let test_dir = scratch_dir("cache-skip-download");let destination = test_dir.join("model.bin");- let target = Target {+ let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: sha_hex(&bytes),};- fetch(- &Downloader::new().unwrap(),- vec![target.clone()],- |_| {},- || false,- )- .expect("first download succeeds");+ download(vec![target.clone()], |_| {}, false)+ .await+ .expect("first download succeeds");let requests_after_first =head_count.load(Ordering::SeqCst) + range_get_count.load(Ordering::SeqCst);assert!(requests_after_first > 0);- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)+ download(vec![target], |_| {}, false)+ .await.expect("second download succeeds");assert_eq!(head_count.load(Ordering::SeqCst) + range_get_count.load(Ordering::SeqCst),@@ -1933,8 +1882,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn cache_hit_is_revalidated_and_reheals_on_corruption() {+ #[tokio::test]+ async fn cache_hit_is_revalidated_and_reheals_on_corruption() {let bytes = Arc::new(sample_bytes(512));let get_count = Arc::new(AtomicUsize::new(0));@@ -1948,27 +1897,24 @@ mod tests {let test_dir = scratch_dir("revalidate-download");let destination = test_dir.join("model.bin");- let target = Target {+ let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: sha_hex(&bytes),};- fetch(- &Downloader::new().unwrap(),- vec![target.clone()],- |_| {},- || false,- )- .expect("first download succeeds");+ download(vec![target.clone()], |_| {}, false)+ .await+ .expect("first download succeeds");assert_eq!(get_count.load(Ordering::SeqCst), 1);let mut data = fs::read(&destination).expect("read cached file");data[0] = data[0].wrapping_add(1);fs::write(&destination, &data).expect("corrupt cached file");- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)+ download(vec![target], |_| {}, false)+ .await.expect("second download re-heals the corrupt cache");assert_eq!(get_count.load(Ordering::SeqCst),@@ -1980,8 +1926,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn existing_valid_file_without_sidecar_is_adopted() {+ #[tokio::test]+ async fn existing_valid_file_without_sidecar_is_adopted() {let bytes = Arc::new(sample_bytes(512));let get_count = Arc::new(AtomicUsize::new(0));@@ -1997,14 +1943,15 @@ mod tests {let destination = test_dir.join("model.bin");fs::write(&destination, bytes.as_slice()).expect("place existing file");- let target = Target {+ let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: sha_hex(&bytes),};- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)+ download(vec![target], |_| {}, false)+ .await.expect("fetch adopts existing file");assert_eq!(@@ -2017,8 +1964,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn single_stream_download_resumes_from_partial_tmp() {+ #[tokio::test]+ async fn single_stream_download_resumes_from_partial_tmp() {let bytes = Arc::new(sample_bytes(512));let head_count = Arc::new(AtomicUsize::new(0));let range_get_count = Arc::new(AtomicUsize::new(0));@@ -2042,7 +1989,7 @@ mod tests {let test_dir = scratch_dir("resume-download");let destination = test_dir.join("model.bin");- let target = Target {+ let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),@@ -2059,7 +2006,8 @@ mod tests {).expect("write partial sidecar");- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)+ download(vec![target], |_| {}, false)+ .await.expect("resume succeeds");assert_eq!(@@ -2078,8 +2026,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn single_stream_download_restarts_when_resume_validator_is_stale() {+ #[tokio::test]+ async fn single_stream_download_restarts_when_resume_validator_is_stale() {let bytes = Arc::new(sample_bytes(512));let head_count = Arc::new(AtomicUsize::new(0));let range_get_count = Arc::new(AtomicUsize::new(0));@@ -2103,7 +2051,7 @@ mod tests {let test_dir = scratch_dir("stale-resume-download");let destination = test_dir.join("model.bin");- let target = Target {+ let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),@@ -2120,7 +2068,8 @@ mod tests {).expect("write stale partial sidecar");- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)+ download(vec![target], |_| {}, false)+ .await.expect("restart succeeds");assert_eq!(@@ -2137,8 +2086,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn single_stream_download_ignores_partial_without_sidecar() {+ #[tokio::test]+ async fn single_stream_download_ignores_partial_without_sidecar() {let bytes = Arc::new(sample_bytes(512));let get_count = Arc::new(AtomicUsize::new(0));@@ -2154,14 +2103,15 @@ mod tests {let destination = test_dir.join("model.bin");fs::write(tmp_path_for(&destination), &bytes[..256]).expect("place orphan tmp");- let target = Target {+ let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: sha_hex(&bytes),};- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)+ download(vec![target], |_| {}, false)+ .await.expect("fresh download succeeds");assert_eq!(@@ -2177,8 +2127,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn single_stream_resume_restarts_when_server_resumes_from_wrong_offset() {+ #[tokio::test]+ async fn single_stream_resume_restarts_when_server_resumes_from_wrong_offset() {let bytes = Arc::new(sample_bytes(512));let bad_resume_count = Arc::new(AtomicUsize::new(0));let full_get_count = Arc::new(AtomicUsize::new(0));@@ -2199,7 +2149,7 @@ mod tests {let test_dir = scratch_dir("wrong-offset-resume");let destination = test_dir.join("model.bin");- let target = Target {+ let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),@@ -2216,7 +2166,8 @@ mod tests {).expect("write partial sidecar");- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)+ download(vec![target], |_| {}, false)+ .await.expect("restart after bad 206 succeeds");assert_eq!(@@ -2287,30 +2238,30 @@ mod tests {}}- #[test]- fn cancelled_fetch_returns_the_cancelled_variant() {+ #[tokio::test]+ async fn cancelled_fetch_returns_the_cancelled_variant() {let test_dir = scratch_dir("cancel-download");let destination = test_dir.join("model.bin");- let result = fetch(- &Downloader::new().unwrap(),- vec![Target {+ let result = download(+ vec![DownloadTarget {label: "Model".to_string(),url: "http://127.0.0.1:1/model.bin".to_string(),destination: destination.clone(),sha256: "0".repeat(64),}],|_| {},- || true,- );+ true,+ )+ .await;assert!(matches!(result, Err(Error::Cancelled)));let _ = fs::remove_dir_all(test_dir);}- #[test]- fn cached_download_adoption_removes_stale_partials() {+ #[tokio::test]+ async fn cached_download_adoption_removes_stale_partials() {let test_dir = scratch_dir("cached-adoption");let destination = test_dir.join("model.bin");fs::write(&destination, b"complete-model").expect("write model");@@ -2318,17 +2269,17 @@ mod tests {fs::write(range_metadata_path_for(&destination), "{}").expect("write ranges sidecar");fs::write(partial_metadata_path_for(&destination), "{}").expect("write partial sidecar");- let result = fetch(- &Downloader::new().unwrap(),- vec![Target {+ let result = download(+ vec![DownloadTarget {label: "Model".to_string(),url: "http://127.0.0.1:1/model.bin".to_string(),destination: destination.clone(),sha256: sha_hex(b"complete-model"),}],|_| {},- || false,- );+ false,+ )+ .await;assert!(result.is_ok());assert!(destination.exists());@@ -2339,8 +2290,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn sha_pinned_download_verifies_content() {+ #[tokio::test]+ async fn sha_pinned_download_verifies_content() {let bytes = Arc::new(sample_bytes(2048));let get_count = Arc::new(AtomicUsize::new(0));let server = {@@ -2358,17 +2309,17 @@ mod tests {expected.push_str(&format!("{byte:02x}"));}- fetch(- &Downloader::new().unwrap(),- vec![Target {+ download(+ vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: expected,}],|_| {},- || false,+ false,)+ .await.expect("sha-pinned download succeeds");assert_eq!(fs::read(&destination).expect("read downloaded file"),@@ -2378,8 +2329,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn sha_mismatch_fails_immediately_without_installing() {+ #[tokio::test]+ async fn sha_mismatch_fails_immediately_without_installing() {let bytes = Arc::new(sample_bytes(2048));let get_count = Arc::new(AtomicUsize::new(0));let server = {@@ -2393,17 +2344,17 @@ mod tests {let test_dir = scratch_dir("sha-mismatch");let destination = test_dir.join("model.bin");- let result = fetch(- &Downloader::new().unwrap(),- vec![Target {+ let result = download(+ vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: "0".repeat(64),}],|_| {},- || false,- );+ false,+ )+ .await;assert!(result.is_err());assert!(!destination.exists());@@ -2412,8 +2363,8 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn ranged_sha_mismatch_fails_without_single_stream_fallback() {+ #[tokio::test]+ async fn ranged_sha_mismatch_fails_without_single_stream_fallback() {let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));let head_count = Arc::new(AtomicUsize::new(0));let range_get_count = Arc::new(AtomicUsize::new(0));@@ -2437,17 +2388,17 @@ mod tests {let test_dir = scratch_dir("ranged-sha-mismatch");let destination = test_dir.join("model.bin");- let result = fetch(- &Downloader::new().unwrap(),- vec![Target {+ let result = download(+ vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),destination: destination.clone(),sha256: "0".repeat(64),}],|_| {},- || false,- );+ false,+ )+ .await;assert!(result.is_err());assert!(!destination.exists());@@ -2461,40 +2412,40 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}- #[test]- fn sha_pinned_target_adopts_cached_file_regardless_of_url() {+ #[tokio::test]+ async fn sha_pinned_target_adopts_cached_file_regardless_of_url() {let test_dir = scratch_dir("sha-cached");let destination = test_dir.join("model.bin");fs::write(&destination, b"model-bytes").expect("write cached file");let sha = sha256_file(&destination).expect("hash cached file");- let old_target = Target {+ let old_target = DownloadTarget {label: "Model".to_string(),url: "http://cache.example.org/old-location".to_string(),destination: destination.clone(),sha256: sha.clone(),};- fetch(- &Downloader::new().unwrap(),- vec![Target {+ download(+ vec![DownloadTarget {url: "http://127.0.0.1:9/new-location".to_string(),..old_target.clone()}],|_| {},- || false,+ false,)+ .await.expect("cached file adopted by checksum despite changed URL");fs::write(&destination, b"corrupted").expect("corrupt cached file");- let result = fetch(- &Downloader::new().unwrap(),- vec![Target {+ let result = download(+ vec![DownloadTarget {url: "http://127.0.0.1:9/new-location".to_string(),..old_target}],|_| {},- || false,- );+ false,+ )+ .await;assert!(result.is_err());let _ = fs::remove_dir_all(test_dir);diff --git a/rust/crates/assets/src/lib.rs b/rust/crates/assets/src/lib.rsnew file mode 100644index 0000000000..a4db6103d2--- /dev/null+++ b/rust/crates/assets/src/lib.rs@@ -0,0 +1,5 @@+mod archive;+pub mod download;+mod store;++pub use store::{Asset, AssetFile, AssetStore};diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rsnew file mode 100644index 0000000000..b5f16fa380--- /dev/null+++ b/rust/crates/assets/src/store.rs@@ -0,0 +1,605 @@+use std::collections::HashSet;+use std::fs;+use std::path::{Path, PathBuf};+use std::sync::OnceLock;++use tokio::sync::Mutex;++use crate::archive;+use crate::download::{self, CancellationToken, DownloadTarget, Downloader, Error, Progress};++const STAGING_DIR: &str = ".staging";+const ARCHIVE_FILE: &str = ".archive.tar.gz";+const EXTRACTION_DIR: &str = ".extracted";++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Asset {+ components: Vec<String>,+ kind: AssetKind,+}++#[derive(Debug, Clone, PartialEq, Eq)]+enum AssetKind {+ Files(Vec<AssetFile>),+ TarGz { url: String, sha256: String },+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct AssetFile {+ pub name: String,+ pub url: String,+ pub sha256: String,+}++impl Asset {+ pub fn files(components: Vec<String>, files: Vec<AssetFile>) -> Result<Self, Error> {+ validate_components(&components)?;+ if files.is_empty() {+ return Err(Error::InvalidTarget("asset has no files".to_string()));+ }+ let mut names = HashSet::new();+ for file in &files {+ validate_component(&file.name)?;+ download::validate_sha256(&file.sha256)?;+ if !names.insert(file.name.as_str()) {+ return Err(Error::InvalidTarget(format!(+ "asset has a duplicate file name {}",+ file.name+ )));+ }+ }+ Ok(Self {+ components,+ kind: AssetKind::Files(files),+ })+ }++ pub fn tar_gz(components: Vec<String>, url: String, sha256: String) -> Result<Self, Error> {+ validate_components(&components)?;+ download::validate_sha256(&sha256)?;+ Ok(Self {+ components,+ kind: AssetKind::TarGz { url, sha256 },+ })+ }+}++pub struct AssetStore {+ root: PathBuf,+ downloader: OnceLock<Downloader>,+ mutation_gate: Mutex<()>,+}++impl AssetStore {+ pub fn new(root: impl Into<PathBuf>) -> Self {+ Self {+ root: root.into(),+ downloader: OnceLock::new(),+ mutation_gate: Mutex::new(()),+ }+ }++ pub fn asset_dir(&self, asset: &Asset) -> PathBuf {+ join_components(&self.root, &asset.components)+ }++ pub fn file_path(&self, asset: &Asset, name: &str) -> Option<PathBuf> {+ match &asset.kind {+ AssetKind::Files(files) if files.iter().any(|file| file.name == name) => {+ Some(self.asset_dir(asset).join(name))+ }+ _ => None,+ }+ }++ pub fn is_downloaded(&self, asset: &Asset) -> bool {+ let directory = self.asset_dir(asset);+ match &asset.kind {+ AssetKind::Files(files) => files+ .iter()+ .all(|file| is_regular_file(&directory.join(&file.name))),+ AssetKind::TarGz { .. } => is_directory(&directory),+ }+ }++ pub async fn estimated_download_size(&self, asset: &Asset) -> Option<u64> {+ if self.is_downloaded(asset) {+ return Some(0);+ }+ self.downloader()+ .ok()?+ .estimated_download_size(&self.download_targets(asset))+ .await+ }++ pub async fn download(+ &self,+ asset: &Asset,+ mut on_progress: impl FnMut(Progress) + Send,+ cancellation: CancellationToken,+ ) -> Result<(), Error> {+ let _guard = self.mutation_gate.lock().await;+ if self.is_downloaded(asset) {+ return Ok(());+ }+ if cancellation.is_cancelled() {+ return Err(Error::Cancelled);+ }++ let mut last_progress = None;+ self.downloader()?+ .download(+ self.download_targets(asset),+ |mut progress| {+ progress.complete = false;+ last_progress = Some(progress.clone());+ on_progress(progress);+ },+ cancellation.clone(),+ )+ .await?;++ match &asset.kind {+ AssetKind::Files(files) => self.publish_files(asset, files)?,+ AssetKind::TarGz { .. } => self.publish_archive(asset, cancellation).await?,+ }++ if let Some(mut progress) = last_progress {+ progress.percentage = 100.0;+ progress.file_complete = false;+ progress.complete = true;+ on_progress(progress);+ }+ Ok(())+ }++ pub async fn remove(&self, asset: &Asset) -> Result<(), Error> {+ let _guard = self.mutation_gate.lock().await;+ remove_path(&self.asset_dir(asset))?;+ remove_path(&self.staging_dir(asset))?;+ Ok(())+ }++ fn downloader(&self) -> Result<&Downloader, Error> {+ if let Some(downloader) = self.downloader.get() {+ return Ok(downloader);+ }+ let downloader = Downloader::new()?;+ Ok(self.downloader.get_or_init(|| downloader))+ }++ fn staging_dir(&self, asset: &Asset) -> PathBuf {+ join_components(&self.root.join(STAGING_DIR), &asset.components)+ }++ fn download_targets(&self, asset: &Asset) -> Vec<DownloadTarget> {+ let staging = self.staging_dir(asset);+ match &asset.kind {+ AssetKind::Files(files) => files+ .iter()+ .map(|file| DownloadTarget {+ label: file.name.clone(),+ url: file.url.clone(),+ sha256: file.sha256.clone(),+ destination: staging.join(&file.name),+ })+ .collect(),+ AssetKind::TarGz { url, sha256 } => vec![DownloadTarget {+ label: asset.components.last().cloned().unwrap_or_default(),+ url: url.clone(),+ sha256: sha256.clone(),+ destination: staging.join(ARCHIVE_FILE),+ }],+ }+ }++ fn publish_files(&self, asset: &Asset, files: &[AssetFile]) -> Result<(), Error> {+ let staging = self.staging_dir(asset);+ for file in files {+ if !is_regular_file(&staging.join(&file.name)) {+ return Err(Error::Validation(format!(+ "{} is not a regular file",+ staging.join(&file.name).display()+ )));+ }+ }+ let expected = files+ .iter()+ .map(|file| file.name.as_str())+ .collect::<HashSet<_>>();+ for entry in fs::read_dir(&staging)? {+ let entry = entry?;+ let name = entry.file_name();+ if name.to_str().is_some_and(|name| expected.contains(name)) {+ if !is_regular_file(&entry.path()) {+ return Err(Error::Validation(format!(+ "{} is not a regular file",+ entry.path().display()+ )));+ }+ } else {+ remove_path(&entry.path())?;+ }+ }+ self.publish_directory(&staging, &self.asset_dir(asset))+ }++ async fn publish_archive(+ &self,+ asset: &Asset,+ cancellation: CancellationToken,+ ) -> Result<(), Error> {+ let staging = self.staging_dir(asset);+ let archive_path = staging.join(ARCHIVE_FILE);+ let extraction_dir = staging.join(EXTRACTION_DIR);+ let extraction_for_task = extraction_dir.clone();+ let extraction_cancellation = cancellation.clone();+ let source = tokio::task::spawn_blocking(move || {+ let result = archive::extract_tar_gz(+ &archive_path,+ &extraction_for_task,+ &extraction_cancellation,+ );+ if result.is_err() {+ let _ = fs::remove_dir_all(&extraction_for_task);+ }+ result+ })+ .await+ .map_err(|error| Error::Io(std::io::Error::other(error)))??;+ if cancellation.is_cancelled() {+ remove_path(&extraction_dir)?;+ return Err(Error::Cancelled);+ }+ self.publish_directory(&source, &self.asset_dir(asset))?;+ remove_path(&staging)?;+ Ok(())+ }++ fn publish_directory(&self, source: &Path, destination: &Path) -> Result<(), Error> {+ let parent = destination+ .parent()+ .ok_or_else(|| Error::InvalidTarget(destination.display().to_string()))?;+ fs::create_dir_all(parent)?;+ remove_path(destination)?;+ fs::rename(source, destination)?;+ Ok(())+ }+}++fn validate_components(components: &[String]) -> Result<(), Error> {+ if components.is_empty() {+ return Err(Error::InvalidTarget(+ "asset key has no components".to_string(),+ ));+ }+ for component in components {+ validate_component(component)?;+ }+ if components[0] == STAGING_DIR {+ return Err(Error::InvalidTarget(format!("'{STAGING_DIR}' is reserved")));+ }+ Ok(())+}++fn validate_component(component: &str) -> Result<(), Error> {+ if !component.is_empty()+ && component != "."+ && component != ".."+ && !component.contains('/')+ && !component.contains('\\')+ {+ return Ok(());+ }+ Err(Error::InvalidTarget(format!(+ "'{component}' is not a safe path component"+ )))+}++fn join_components(root: &Path, components: &[String]) -> PathBuf {+ components+ .iter()+ .fold(root.to_path_buf(), |path, component| path.join(component))+}++fn is_regular_file(path: &Path) -> bool {+ fs::symlink_metadata(path)+ .map(|metadata| metadata.file_type().is_file())+ .unwrap_or(false)+}++fn is_directory(path: &Path) -> bool {+ fs::symlink_metadata(path)+ .map(|metadata| metadata.file_type().is_dir())+ .unwrap_or(false)+}++fn remove_path(path: &Path) -> Result<(), Error> {+ let metadata = match fs::symlink_metadata(path) {+ Ok(metadata) => metadata,+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),+ Err(error) => return Err(error.into()),+ };+ if metadata.file_type().is_dir() {+ fs::remove_dir_all(path)?;+ } else {+ fs::remove_file(path)?;+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use std::io::Cursor;+ use std::time::{SystemTime, UNIX_EPOCH};++ use flate2::Compression;+ use flate2::write::GzEncoder;+ use sha2::{Digest, Sha256};+ use tar::{Builder, EntryType, Header};++ use super::*;++ #[test]+ fn store_and_operations_are_send_and_sync() {+ fn assert_send<T: Send>(_: T) {}+ fn assert_send_sync<T: Send + Sync>() {}++ assert_send_sync::<AssetStore>();+ assert_send_sync::<CancellationToken>();++ let store = AssetStore::new("assets");+ let asset =+ Asset::files(key(&["models", "clip"]), vec![file("model.onnx", b"model")]).unwrap();+ assert_send(store.download(&asset, |_| {}, CancellationToken::default()));+ assert_send(store.estimated_download_size(&asset));+ assert_send(store.remove(&asset));+ }++ #[test]+ fn rejects_invalid_assets() {+ assert!(Asset::files(Vec::new(), vec![file("model", b"model")]).is_err());+ assert!(Asset::files(key(&[".staging", "model"]), vec![file("model", b"model")]).is_err());+ assert!(Asset::files(key(&["models", ".."]), vec![file("model", b"model")]).is_err());+ assert!(Asset::files(key(&["models", "model"]), Vec::new()).is_err());+ assert!(+ Asset::files(+ key(&["models", "model"]),+ vec![file("model", b"a"), file("model", b"b")]+ )+ .is_err()+ );+ let mut invalid_sha = file("model", b"model");+ invalid_sha.sha256 = "invalid".to_string();+ assert!(Asset::files(key(&["models", "model"]), vec![invalid_sha]).is_err());+ }++ #[tokio::test]+ async fn publishes_all_files_before_reporting_completion() {+ let root = scratch_dir("files");+ let store = AssetStore::new(&root);+ let asset = Asset::files(+ key(&["models", "clip"]),+ vec![file("model.onnx", b"model"), file("vocab.txt", b"vocab")],+ )+ .unwrap();+ stage_files(+ &store,+ &asset,+ &[("model.onnx", b"model"), ("vocab.txt", b"vocab")],+ );++ let final_dir = store.asset_dir(&asset);+ let mut completion_observed_after_publish = false;+ store+ .download(+ &asset,+ |progress| {+ if progress.complete {+ completion_observed_after_publish = final_dir.is_dir();+ }+ },+ CancellationToken::default(),+ )+ .await+ .unwrap();++ assert!(completion_observed_after_publish);+ assert!(store.is_downloaded(&asset));+ assert_eq!(fs::read(final_dir.join("model.onnx")).unwrap(), b"model");+ assert_eq!(fs::read(final_dir.join("vocab.txt")).unwrap(), b"vocab");+ assert!(!store.staging_dir(&asset).exists());+ let _ = fs::remove_dir_all(root);+ }++ #[tokio::test]+ async fn ignores_derived_files_and_discards_them_when_rebuilding() {+ let root = scratch_dir("derived");+ let store = AssetStore::new(&root);+ let asset = Asset::files(+ key(&["onnx-runtime", "macos"]),+ vec![file("runtime.tgz", b"archive")],+ )+ .unwrap();+ stage_files(&store, &asset, &[("runtime.tgz", b"archive")]);+ store+ .download(&asset, |_| {}, CancellationToken::default())+ .await+ .unwrap();++ let directory = store.asset_dir(&asset);+ fs::write(directory.join("libonnxruntime.dylib"), b"derived").unwrap();+ assert!(store.is_downloaded(&asset));++ fs::remove_file(directory.join("runtime.tgz")).unwrap();+ assert!(!store.is_downloaded(&asset));+ stage_files(&store, &asset, &[("runtime.tgz", b"archive")]);+ store+ .download(&asset, |_| {}, CancellationToken::default())+ .await+ .unwrap();++ assert!(!directory.join("libonnxruntime.dylib").exists());+ assert!(store.is_downloaded(&asset));+ let _ = fs::remove_dir_all(root);+ }++ #[tokio::test]+ async fn extracts_and_publishes_tar_gz() {+ let root = scratch_dir("archive");+ let store = AssetStore::new(&root);+ let archive = tar_gz(+ "parakeet",+ &[("encoder.onnx", b"encoder"), ("tokens.txt", b"tokens")],+ );+ let asset = Asset::tar_gz(+ key(&["models", "parakeet"]),+ "https://example.invalid/parakeet.tar.gz".to_string(),+ sha(&archive),+ )+ .unwrap();+ let staging = store.staging_dir(&asset);+ fs::create_dir_all(&staging).unwrap();+ fs::write(staging.join(ARCHIVE_FILE), archive).unwrap();++ store+ .download(&asset, |_| {}, CancellationToken::default())+ .await+ .unwrap();++ let directory = store.asset_dir(&asset);+ assert_eq!(+ fs::read(directory.join("encoder.onnx")).unwrap(),+ b"encoder"+ );+ assert_eq!(fs::read(directory.join("tokens.txt")).unwrap(), b"tokens");+ assert!(!staging.exists());+ let _ = fs::remove_dir_all(root);+ }++ #[tokio::test]+ async fn rejects_tar_links_without_publishing() {+ let root = scratch_dir("archive-link");+ let store = AssetStore::new(&root);+ let archive = tar_gz_with_link();+ let asset = Asset::tar_gz(+ key(&["models", "linked"]),+ "https://example.invalid/linked.tar.gz".to_string(),+ sha(&archive),+ )+ .unwrap();+ let staging = store.staging_dir(&asset);+ fs::create_dir_all(&staging).unwrap();+ fs::write(staging.join(ARCHIVE_FILE), archive).unwrap();++ assert!(+ store+ .download(&asset, |_| {}, CancellationToken::default())+ .await+ .is_err()+ );+ assert!(!store.asset_dir(&asset).exists());+ assert!(staging.join(ARCHIVE_FILE).is_file());+ let _ = fs::remove_dir_all(root);+ }++ #[tokio::test]+ async fn remove_deletes_published_and_staged_data() {+ let root = scratch_dir("remove");+ let store = AssetStore::new(&root);+ let asset =+ Asset::files(key(&["models", "clip"]), vec![file("model.onnx", b"model")]).unwrap();+ let final_dir = store.asset_dir(&asset);+ let staging = store.staging_dir(&asset);+ fs::create_dir_all(&final_dir).unwrap();+ fs::write(final_dir.join("model.onnx"), b"model").unwrap();+ fs::create_dir_all(&staging).unwrap();+ fs::write(staging.join("model.onnx.tmp"), b"partial").unwrap();++ store.remove(&asset).await.unwrap();+ assert!(!final_dir.exists());+ assert!(!staging.exists());+ store.remove(&asset).await.unwrap();+ let _ = fs::remove_dir_all(root);+ }++ fn key(components: &[&str]) -> Vec<String> {+ components+ .iter()+ .map(|component| component.to_string())+ .collect()+ }++ fn file(name: &str, bytes: &[u8]) -> AssetFile {+ AssetFile {+ name: name.to_string(),+ url: format!("https://example.invalid/{name}"),+ sha256: sha(bytes),+ }+ }++ fn stage_files(store: &AssetStore, asset: &Asset, files: &[(&str, &[u8])]) {+ let directory = store.staging_dir(asset);+ fs::create_dir_all(&directory).unwrap();+ for (name, bytes) in files {+ fs::write(directory.join(name), bytes).unwrap();+ }+ }++ fn sha(bytes: &[u8]) -> String {+ Sha256::digest(bytes)+ .iter()+ .map(|byte| format!("{byte:02x}"))+ .collect()+ }++ fn tar_gz(root: &str, files: &[(&str, &[u8])]) -> Vec<u8> {+ let mut bytes = Vec::new();+ {+ let encoder = GzEncoder::new(&mut bytes, Compression::default());+ let mut archive = Builder::new(encoder);+ for (name, contents) in files {+ let mut header = Header::new_gnu();+ header.set_size(contents.len() as u64);+ header.set_mode(0o644);+ header.set_cksum();+ archive+ .append_data(+ &mut header,+ format!("{root}/{name}"),+ Cursor::new(*contents),+ )+ .unwrap();+ }+ archive.into_inner().unwrap().finish().unwrap();+ }+ bytes+ }++ fn tar_gz_with_link() -> Vec<u8> {+ let mut bytes = Vec::new();+ {+ let encoder = GzEncoder::new(&mut bytes, Compression::default());+ let mut archive = Builder::new(encoder);+ let mut header = Header::new_gnu();+ header.set_entry_type(EntryType::Symlink);+ header.set_size(0);+ header.set_mode(0o777);+ header.set_link_name("../outside").unwrap();+ header.set_cksum();+ archive+ .append_data(&mut header, "model/link", Cursor::new(Vec::<u8>::new()))+ .unwrap();+ archive.into_inner().unwrap().finish().unwrap();+ }+ bytes+ }++ fn scratch_dir(name: &str) -> PathBuf {+ let suffix = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap()+ .as_nanos();+ std::env::temp_dir().join(format!("ente-assets-{name}-{suffix}"))+ }+}diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex 23dda9264a..01dd009119 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -3,9 +3,9 @@ name = "ente-ensu"edition = "2024"[dependencies]+ente-assets.workspace = trueente-core.workspace = trueente-image.workspace = true-ente-model-download.workspace = truememmap2 = "0.9.9"regex = { version = "1", optional = true }rubato = { version = "0.16", optional = true }@@ -13,7 +13,6 @@ rusqlite = { version = "0.38", features = ["bundled"], optional = true }self_cell = "1"serde.workspace = trueserde_json.workspace = true-sha2 = "0.11.0"thiserror.workspace = truetokio = { version = "1", features = ["rt", "time"] }transcribe-rs = { version = "0.3.11", default-features = false, features = ["onnx", "vad-silero"], optional = true }diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rsindex 88a4d51582..b98c44d9bf 100644--- a/rust/crates/ensu/src/config.rs+++ b/rust/crates/ensu/src/config.rs@@ -209,6 +209,7 @@ mod tests {preset.id);}+ assert!(seen.insert(knowledge::knowledge_embedding_config().target_id));}#[test]diff --git a/rust/crates/ensu/src/config/knowledge.rs b/rust/crates/ensu/src/config/knowledge.rsindex 35ae281cf5..41cb2d2c6d 100644--- a/rust/crates/ensu/src/config/knowledge.rs+++ b/rust/crates/ensu/src/config/knowledge.rs@@ -183,6 +183,8 @@ pub(crate) fn is_path_safe_component(value: &str) -> bool {#[cfg(test)]mod tests {+ use std::collections::HashSet;+use super::*;#[test]@@ -204,6 +206,14 @@ mod tests {let embedding = knowledge_embedding_config();assert_eq!(datasets.len(), 2);+ assert_eq!(+ datasets+ .iter()+ .map(|dataset| dataset.stable_id.as_str())+ .collect::<HashSet<_>>()+ .len(),+ datasets.len()+ );assert_eq!(datasets[0].stable_id, "simplewiki");assert_eq!(datasets[1].stable_id, "wikibooks");assert!(diff --git a/rust/crates/ensu/src/model/migrations.rs b/rust/crates/ensu/src/model/migrations.rsindex 425005a7ee..531d943fb4 100644--- a/rust/crates/ensu/src/model/migrations.rs+++ b/rust/crates/ensu/src/model/migrations.rs@@ -3,51 +3,83 @@ use std::fs::{self, File};use std::io::Read;use std::path::{Path, PathBuf};-use ente_model_download::{ModelDownloader, ModelTarget};-use sha2::{Digest, Sha256};+use ente_assets::{Asset, AssetStore};use crate::config;use crate::model::{self, InvalidPreset, trimmed};-pub struct LegacyModels {+pub struct LegacyAssets {pub llm_dir: Option<PathBuf>,- pub transcription_dir: PathBuf,+ pub transcription_dir: Option<PathBuf>,+ pub models_dir: Option<PathBuf>,pub model_url: Option<String>,pub mmproj_url: Option<String>,}-pub fn migrate_mobile_models(models_dir: &Path, legacy: LegacyModels) -> Option<String> {- let store = ModelDownloader::new(models_dir);- if let Some(llm_dir) = &legacy.llm_dir- && let Ok(targets) = llm_targets()- {- migrate_legacy_dir(&store, llm_dir, &targets);+pub fn needs_migration(legacy: &LegacyAssets) -> bool {+ [+ legacy.llm_dir.as_deref(),+ legacy.transcription_dir.as_deref(),+ legacy.models_dir.as_deref(),+ ]+ .into_iter()+ .flatten()+ .any(Path::exists)+}++pub fn migrate_ensu_assets(assets_dir: &Path, legacy: LegacyAssets) -> Option<String> {+ let store = AssetStore::new(assets_dir);+ let Ok(llm_migrations) = llm_migrations() else {+ return None;+ };++ if let Some(directory) = legacy.llm_dir {+ migrate_flat_models(+ &store,+ &llm_migrations,+ &directory.join("models"),+ &directory,+ );+ }+ if let Some(directory) = legacy.transcription_dir {+ migrate_transcription(&store, &directory);}- migrate_legacy_transcription_dir(- &store,- &legacy.transcription_dir,- &model::transcription_target(),- &model::voice_activity_target(),- );+ if let Some(directory) = legacy.models_dir {+ migrate_flat_models(&store, &llm_migrations, &directory, &directory);+ }+legacy_selected_preset_id(legacy.model_url.as_deref()?, legacy.mmproj_url.as_deref())}-pub fn migrate_desktop_models(- models_dir: &Path,- model_url: Option<&str>,- mmproj_url: Option<&str>,-) -> Option<String> {- let selected = model_url.and_then(|model_url| legacy_selected_preset_id(model_url, mmproj_url));- if let Ok(targets) = llm_targets() {- migrate_flat_models_dir(models_dir, &targets);- }- selected+struct LlmMigration {+ asset: Asset,+ files: Vec<LlmFile>,}-fn llm_targets() -> Result<Vec<ModelTarget>, InvalidPreset> {+struct LlmFile {+ url: String,+ name: &'static str,+}++fn llm_migrations() -> Result<Vec<LlmMigration>, InvalidPreset> {config::llm_catalog().iter()- .map(model::llm_target)+ .map(|preset| {+ let mut files = vec![LlmFile {+ url: preset.url.clone(),+ name: "model.gguf",+ }];+ if let Some(url) = trimmed(preset.mmproj_url.as_deref()) {+ files.push(LlmFile {+ url: url.to_string(),+ name: "mmproj.gguf",+ });+ }+ Ok(LlmMigration {+ asset: model::llm_asset(preset)?,+ files,+ })+ }).collect()}@@ -61,192 +93,121 @@ fn legacy_selected_preset_id(model_url: &str, mmproj_url: Option<&str>) -> Optio.map(|preset| preset.id)}-fn migrate_legacy_dir(store: &ModelDownloader, legacy_dir: &Path, targets: &[ModelTarget]) {- if !legacy_dir.exists() {- return;- }- if adopt_targets(store, targets, &[], &legacy_dir.join("models")) {- let _ = fs::remove_dir_all(legacy_dir);- }-}--fn migrate_flat_models_dir(models_dir: &Path, targets: &[ModelTarget]) {- if !models_dir.exists() {- return;- }- let store = ModelDownloader::new(models_dir);- let hashed_dirs = [models_dir.to_path_buf()];- if !adopt_targets(&store, targets, &hashed_dirs, models_dir) {- return;- }- let Ok(entries) = fs::read_dir(models_dir) else {- return;- };- for entry in entries.flatten() {- let path = entry.path();- if path.is_file() {- let _ = fs::remove_file(&path);- } else if path.file_name().is_some_and(|name| name == "custom") {- let _ = fs::remove_dir_all(&path);- }+fn migrate_flat_models(+ store: &AssetStore,+ targets: &[LlmMigration],+ source: &Path,+ cleanup: &Path,+) {+ if source.exists() && adopt_flat_targets(store, targets, source) {+ let _ = fs::remove_dir_all(cleanup);}}-fn migrate_legacy_transcription_dir(- store: &ModelDownloader,- legacy_dir: &Path,- model: &ModelTarget,- vad: &ModelTarget,-) {- const LEGACY_MODEL_DIR_NAME: &str = "parakeet-tdt-0.6b-v3-int8";- const LEGACY_MODEL_ID: &str = "parakeet-v3-int8";- const LEGACY_VAD_FILE_NAME: &str = "silero_vad_v4.onnx";- const LEGACY_VAD_ID: &str = "silero-vad-v4";+fn migrate_transcription(store: &AssetStore, legacy_dir: &Path) {+ const LEGACY_MODEL_DIR: &str = "parakeet-tdt-0.6b-v3-int8";+ const LEGACY_VAD_FILE: &str = "silero_vad_v4.onnx";if !legacy_dir.exists() {return;}- let mut all_moved = true;-- if matches!(model, ModelTarget::TarGz { id, .. } if id == LEGACY_MODEL_ID) {- let dest = store.model_dir(model);- if !dest.exists() {- let source = legacy_dir.join(LEGACY_MODEL_DIR_NAME);- if source.is_dir() {- if let Some(parent) = dest.parent() {- let _ = fs::create_dir_all(parent);- }- if fs::rename(&source, &dest).is_err() {- all_moved = false;- }+ let mut complete = true;++ let model = model::transcription_model_asset();+ let model_destination = store.asset_dir(&model);+ if !store.is_downloaded(&model) {+ let source = legacy_dir.join(LEGACY_MODEL_DIR);+ if source.is_dir() {+ if let Some(parent) = model_destination.parent() {+ let _ = fs::create_dir_all(parent);}+ complete &= fs::rename(source, model_destination).is_ok();}}- if let ModelTarget::Files { id, files } = vad- && id == LEGACY_VAD_ID- && let [file] = files.as_slice()- {- let dest = store.model_dir(vad).join(&file.name);- if !dest.exists() {- let source = legacy_dir.join(LEGACY_VAD_FILE_NAME);- if is_non_empty_file(&source) && !move_file(&source, &dest) {- all_moved = false;- }+ let vad = model::voice_activity_model_asset();+ let vad_destination = model::voice_activity_model_path(store, &vad);+ if !store.is_downloaded(&vad) {+ let source = legacy_dir.join(LEGACY_VAD_FILE);+ if is_non_empty_file(&source) {+ complete &= move_file(&source, &vad_destination);}}-- if all_moved {+ if complete {let _ = fs::remove_dir_all(legacy_dir);}}-fn sidecar_url(path: &Path) -> Option<String> {- let sidecar = PathBuf::from(format!("{}.metadata.json", path.display()));- let text = fs::read_to_string(sidecar).ok()?;- let value: serde_json::Value = serde_json::from_str(&text).ok()?;- value["url"].as_str().map(String::from)-}--fn adopt_targets(- store: &ModelDownloader,- targets: &[ModelTarget],- hashed_dirs: &[PathBuf],- flat_dir: &Path,-) -> bool {- let mut plans: Vec<Vec<(&str, &str, PathBuf)>> = Vec::new();- for target in targets {- let ModelTarget::Files { files, .. } = target else {- continue;- };- let dir = store.model_dir(target);- plans.push(- files- .iter()- .map(|file| (file.url.as_str(), file.name.as_str(), dir.join(&file.name)))- .collect(),- );- }-+fn adopt_flat_targets(store: &AssetStore, targets: &[LlmMigration], flat_dir: &Path) -> bool {let mut basename_urls: HashMap<String, HashSet<&str>> = HashMap::new();+ let mut complete = true;for target in targets {- let ModelTarget::Files { files, .. } = target else {- continue;- };- for file in files {+ for file in &target.files {basename_urls- .entry(filename_for_url(&file.url, &file.name))+ .entry(filename_for_url(&file.url, file.name)).or_default()- .insert(file.url.as_str());+ .insert(&file.url);}}- let mut all_moved = true;- for plan in &plans {- let sources: Vec<Option<Option<PathBuf>>> = plan+ for target in targets {+ let destination = store.asset_dir(&target.asset);+ let sources = target+ .files.iter()- .map(|(url, fallback, dest)| {- if dest.exists() {+ .map(|file| {+ let final_path = destination.join(file.name);+ if final_path.exists() {return Some(None);}- let basename = filename_for_url(url, fallback);- let url = *url;- for dir in hashed_dirs {- let hashed = dir.join(format!("{}_{basename}", sha256_hex(url)));- if looks_like_gguf(&hashed) {- return Some(Some(hashed));- }+ let basename = filename_for_url(&file.url, file.name);+ let source = flat_dir.join(&basename);+ if !looks_like_gguf(&source) {+ return None;}- let flat = flat_dir.join(&basename);- if looks_like_gguf(&flat) {- match sidecar_url(&flat) {- Some(sidecar) => {- if sidecar == url {- return Some(Some(flat));- }- }- None => {- if basename_urls[&basename].len() == 1 {- return Some(Some(flat));- }- }- }+ match sidecar_url(&source) {+ Some(url) if url == file.url => Some(Some(source)),+ None if basename_urls[&basename].len() == 1 => Some(Some(source)),+ _ => None,}- None})- .collect();+ .collect::<Vec<_>>();if sources.iter().any(Option::is_none) {continue;}- for ((_, _, dest), source) in plan.iter().zip(sources) {- let Some(Some(source)) = source else {- continue;- };- if !move_file(&source, dest) {- all_moved = false;+ for (file, source) in target.files.iter().zip(sources) {+ if let Some(Some(source)) = source {+ complete &= move_file(&source, &destination.join(file.name));}}}- all_moved+ complete}-fn move_file(source: &Path, dest: &Path) -> bool {- if let Some(parent) = dest.parent() {+fn move_file(source: &Path, destination: &Path) -> bool {+ if let Some(parent) = destination.parent() {let _ = fs::create_dir_all(parent);}- if fs::rename(source, dest).is_ok() {+ if fs::rename(source, destination).is_ok() {return true;}- let staged = PathBuf::from(format!("{}.migrating", dest.display()));- if fs::copy(source, &staged)- .and_then(|_| fs::rename(&staged, dest))+ let staging = PathBuf::from(format!("{}.migrating", destination.display()));+ if fs::copy(source, &staging)+ .and_then(|_| fs::rename(&staging, destination)).is_err(){- let _ = fs::remove_file(&staged);+ let _ = fs::remove_file(staging);return false;}true}+fn sidecar_url(path: &Path) -> Option<String> {+ let sidecar = PathBuf::from(format!("{}.metadata.json", path.display()));+ let text = fs::read_to_string(sidecar).ok()?;+ let value: serde_json::Value = serde_json::from_str(&text).ok()?;+ value["url"].as_str().map(String::from)+}+fn is_non_empty_file(path: &Path) -> bool {path.metadata().map(|metadata| metadata.is_file() && metadata.len() > 0)@@ -272,20 +233,10 @@ fn filename_for_url(url: &str, fallback: &str) -> String {}}-fn sha256_hex(value: &str) -> String {- Sha256::digest(value.as_bytes())- .iter()- .map(|byte| format!("{byte:02x}"))- .collect()-}-#[cfg(test)]mod tests {- use ente_model_download::ModelFile;-use super::*;use crate::config::ModelPreset;- use crate::model;fn scratch_dir(name: &str) -> PathBuf {let dir = std::env::temp_dir().join(format!(@@ -295,468 +246,122 @@ mod tests {.unwrap().as_nanos()));- fs::create_dir_all(&dir).expect("create test dir");+ fs::create_dir_all(&dir).unwrap();dir}- fn test_sha() -> String {- "0".repeat(64)- }-- fn gguf_target(id: &str, url: String, mmproj_url: String) -> ModelTarget {- model::llm_target(&ModelPreset {+ fn target(id: &str, model_url: &str, mmproj_url: &str) -> LlmMigration {+ let preset = ModelPreset {id: id.to_string(),title: String::new(),- url,- sha256: test_sha(),- mmproj_url: Some(mmproj_url),- mmproj_sha256: Some(test_sha()),- })- .unwrap()- }-- fn target(id: &str) -> ModelTarget {- gguf_target(- id,- "https://example.org/models/main.gguf?download=true".to_string(),- "https://example.org/models/mmproj.gguf".to_string(),- )- }-- fn tar_gz_target(id: &str) -> ModelTarget {- ModelTarget::TarGz {- id: id.to_string(),- url: format!("https://models.example.org/{id}.tar.gz"),- sha256: test_sha(),- }- }-- fn vad_target(id: &str, url: String) -> ModelTarget {- ModelTarget::Files {- id: id.to_string(),- files: vec![ModelFile {- name: "model.onnx".to_string(),- url,- sha256: test_sha(),- }],+ url: model_url.to_string(),+ sha256: "0".repeat(64),+ mmproj_url: Some(mmproj_url.to_string()),+ mmproj_sha256: Some("0".repeat(64)),+ };+ LlmMigration {+ asset: model::llm_asset(&preset).unwrap(),+ files: vec![+ LlmFile {+ url: model_url.to_string(),+ name: "model.gguf",+ },+ LlmFile {+ url: mmproj_url.to_string(),+ name: "mmproj.gguf",+ },+ ],}}- fn file_target(id: &str) -> ModelTarget {- vad_target(id, format!("https://models.example.org/{id}.onnx"))- }-- fn write_gguf(path: &Path, content: &[u8]) {- fs::create_dir_all(path.parent().unwrap()).unwrap();- fs::write(path, content).unwrap();- }-#[test]- fn mobile_migration_resolves_preset_selection() {- let base = scratch_dir("mobile-selection");+ fn resolves_only_catalog_selections() {let defaults = config::defaults();let preset = &defaults.mobile_model_presets[0];- let selected = migrate_mobile_models(- &base.join("models"),- LegacyModels {- llm_dir: None,- transcription_dir: base.join("transcription"),- model_url: Some(preset.url.clone()),- mmproj_url: preset.mmproj_url.clone(),- },- );- assert_eq!(selected.as_deref(), Some(preset.id.as_str()));-- let unknown = migrate_mobile_models(- &base.join("models"),- LegacyModels {- llm_dir: None,- transcription_dir: base.join("transcription"),- model_url: Some("https://example.org/custom.gguf".to_string()),- mmproj_url: None,- },- );- assert_eq!(unknown, None);-- let _ = fs::remove_dir_all(base);- }-- #[test]- fn desktop_migration_resolves_preset_selection() {- let base = scratch_dir("desktop-selection");- let defaults = config::defaults();- let preset = &defaults.desktop_model_presets[0];- let selected = migrate_desktop_models(- &base.join("models"),- Some(&preset.url),- preset.mmproj_url.as_deref(),+ assert_eq!(+ legacy_selected_preset_id(&preset.url, preset.mmproj_url.as_deref()).as_deref(),+ Some(preset.id.as_str()));- assert_eq!(selected.as_deref(), Some(preset.id.as_str()));-assert_eq!(- migrate_desktop_models(&base.join("models"), None, None),+ legacy_selected_preset_id("https://example.org/custom.gguf", None),None);-- let _ = fs::remove_dir_all(base);}#[test]- fn migrate_transcription_moves_model_dir_and_vad_file() {- let base = scratch_dir("migrate-transcription");- let legacy = base.join("transcription");- fs::create_dir_all(legacy.join("parakeet-tdt-0.6b-v3-int8")).unwrap();- fs::write(- legacy.join("parakeet-tdt-0.6b-v3-int8/encoder.onnx"),- b"enc",- )- .unwrap();- fs::write(legacy.join("silero_vad_v4.onnx"), b"vad").unwrap();- fs::write(legacy.join("stale.partial"), b"junk").unwrap();-- let models_dir = base.join("models");- let model = tar_gz_target("parakeet-v3-int8");- let vad = vad_target(- "silero-vad-v4",- "https://models.example.org/silero_vad_v4.onnx".to_string(),+ fn adopts_released_flat_model_all_or_nothing() {+ let base = scratch_dir("flat");+ let legacy = base.join("llm/models");+ fs::create_dir_all(&legacy).unwrap();+ fs::write(legacy.join("model.gguf"), b"GGUFmodel").unwrap();+ fs::write(legacy.join("mmproj.gguf"), b"GGUFmmproj").unwrap();+ let target = target(+ "model",+ "https://example.org/model.gguf",+ "https://example.org/mmproj.gguf",);- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);+ let store = AssetStore::new(base.join("assets"));++ adopt_flat_targets(&store, &[target], &legacy);- let downloader = ModelDownloader::new(&models_dir);- assert!(downloader.is_downloaded(&model));- assert!(downloader.is_downloaded(&vad));assert_eq!(- fs::read(models_dir.join("parakeet-v3-int8/encoder.onnx")).unwrap(),- b"enc"+ fs::read(base.join("assets/models/model/model.gguf")).unwrap(),+ b"GGUFmodel");assert_eq!(- fs::read(models_dir.join("silero-vad-v4/model.onnx")).unwrap(),- b"vad"+ fs::read(base.join("assets/models/model/mmproj.gguf")).unwrap(),+ b"GGUFmmproj");- assert!(!legacy.exists());-- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);- assert!(downloader.is_downloaded(&model));-let _ = fs::remove_dir_all(base);}#[test]- fn migrate_transcription_keeps_existing_destinations() {- let base = scratch_dir("migrate-transcription-existing");- let legacy = base.join("transcription");- fs::create_dir_all(legacy.join("parakeet-tdt-0.6b-v3-int8")).unwrap();- fs::write(- legacy.join("parakeet-tdt-0.6b-v3-int8/encoder.onnx"),- b"old",- )- .unwrap();-- let models_dir = base.join("models");- let model = tar_gz_target("parakeet-v3-int8");- let vad = file_target("silero-vad-v4");- fs::create_dir_all(models_dir.join("parakeet-v3-int8")).unwrap();- fs::write(models_dir.join("parakeet-v3-int8/encoder.onnx"), b"new").unwrap();-- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);-- assert_eq!(- fs::read(models_dir.join("parakeet-v3-int8/encoder.onnx")).unwrap(),- b"new"- );- assert!(!legacy.exists());-+ fn rejects_ambiguous_flat_projector() {+ let base = scratch_dir("ambiguous");+ let legacy = base.join("models");+ fs::create_dir_all(&legacy).unwrap();+ fs::write(legacy.join("a.gguf"), b"GGUFa").unwrap();+ fs::write(legacy.join("b.gguf"), b"GGUFb").unwrap();+ fs::write(legacy.join("mmproj.gguf"), b"GGUFshared").unwrap();+ let targets = [+ target(+ "a",+ "https://example.org/a.gguf",+ "https://example.org/a/mmproj.gguf",+ ),+ target(+ "b",+ "https://example.org/b.gguf",+ "https://example.org/b/mmproj.gguf",+ ),+ ];+ let store = AssetStore::new(base.join("assets"));++ adopt_flat_targets(&store, &targets, &legacy);++ assert!(!base.join("assets/models/a").exists());+ assert!(!base.join("assets/models/b").exists());let _ = fs::remove_dir_all(base);}#[test]- fn migrate_transcription_adopts_nothing_when_target_ids_differ() {- let base = scratch_dir("migrate-transcription-mismatch");+ fn adopts_released_transcription() {+ let base = scratch_dir("transcription");let legacy = base.join("transcription");fs::create_dir_all(legacy.join("parakeet-tdt-0.6b-v3-int8")).unwrap();fs::write(legacy.join("parakeet-tdt-0.6b-v3-int8/encoder.onnx"),- b"enc",+ b"model",).unwrap();fs::write(legacy.join("silero_vad_v4.onnx"), b"vad").unwrap();+ let store = AssetStore::new(base.join("assets"));- let models_dir = base.join("models");- let model = tar_gz_target("parakeet-v4-int8");- let vad = file_target("silero-vad-v5");- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);-- let downloader = ModelDownloader::new(&models_dir);- assert!(!downloader.is_downloaded(&model));- assert!(!downloader.is_downloaded(&vad));- assert!(!legacy.exists());-- let _ = fs::remove_dir_all(base);- }+ migrate_transcription(&store, &legacy);- #[test]- fn migrate_adopts_known_files_and_discards_legacy_dir() {- let base = scratch_dir("migrate");- let legacy = base.join("llm");- let legacy_models = legacy.join("models");- fs::create_dir_all(&legacy_models).unwrap();-- fs::write(legacy_models.join("main.gguf"), b"GGUFmain").unwrap();- fs::write(legacy_models.join("mmproj.gguf"), b"GGUFmmproj").unwrap();- fs::write(legacy_models.join("main.gguf.tmp"), b"GGUFpartial").unwrap();- fs::write(legacy_models.join("main.gguf.metadata.json"), b"{}").unwrap();- fs::write(legacy_models.join("orphan.gguf"), b"GGUForphan").unwrap();-- let models_dir = base.join("models");- let preset = target("qwen-2b-q8");- migrate_legacy_dir(- &ModelDownloader::new(&models_dir),- &legacy,- std::slice::from_ref(&preset),- );-- let downloader = ModelDownloader::new(&models_dir);- assert_eq!(- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmain"- );- assert_eq!(- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmmproj"- );- assert_eq!(fs::read_dir(&models_dir).unwrap().count(), 1);+ assert!(store.is_downloaded(&model::transcription_model_asset()));+ assert!(store.is_downloaded(&model::voice_activity_model_asset()));assert!(!legacy.exists());-- let _ = fs::remove_dir_all(base);- }-- #[test]- fn migrate_adopts_targets_only_when_every_file_resolves() {- let base = scratch_dir("migrate-ambiguous");- let legacy = base.join("llm");- let legacy_models = legacy.join("models");- fs::create_dir_all(&legacy_models).unwrap();- fs::write(legacy_models.join("a.gguf"), b"GGUFa").unwrap();- fs::write(legacy_models.join("b.gguf"), b"GGUFb").unwrap();- fs::write(legacy_models.join("mmproj-F16.gguf"), b"GGUFshared").unwrap();-- let targets = ["a", "b"].map(|name| {- gguf_target(- &format!("preset-{name}"),- format!("https://example.org/{name}/{name}.gguf"),- format!("https://example.org/{name}/mmproj-F16.gguf"),- )- });-- let models_dir = base.join("models");- migrate_legacy_dir(&ModelDownloader::new(&models_dir), &legacy, &targets);-- assert!(!models_dir.exists() || fs::read_dir(&models_dir).unwrap().count() == 0);- assert!(!legacy.exists());-- let _ = fs::remove_dir_all(base);- }-- #[test]- fn migrate_flat_models_dir_adopts_known_files_and_cleans_root() {- let models_dir = scratch_dir("flat-migrate");- fs::create_dir_all(models_dir.join("custom")).unwrap();-- let preset = target("qwen-2b-q8");- fs::write(models_dir.join("main.gguf"), b"GGUFmain").unwrap();- let mmproj_hashed = format!(- "{}_mmproj.gguf",- sha256_hex("https://example.org/models/mmproj.gguf")- );- fs::write(models_dir.join(mmproj_hashed), b"GGUFmm").unwrap();-- fs::write(models_dir.join("orphan.gguf"), b"GGUForphan").unwrap();- fs::write(models_dir.join("main.gguf.tmp"), b"GGUFpartial").unwrap();- let downloader = ModelDownloader::new(&models_dir);- write_gguf(- &models_dir.join("other-key").join("model.gguf"),- b"GGUFkeep",- );-- migrate_flat_models_dir(&models_dir, std::slice::from_ref(&preset));-- assert_eq!(- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmain"- );- assert_eq!(- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmm"- );- assert_eq!(- fs::read(models_dir.join("other-key/model.gguf")).unwrap(),- b"GGUFkeep"- );- assert!(!models_dir.join("custom").exists());- let root_files = fs::read_dir(&models_dir)- .unwrap()- .flatten()- .filter(|entry| entry.path().is_file())- .count();- assert_eq!(root_files, 0);-- migrate_flat_models_dir(&models_dir, std::slice::from_ref(&preset));- assert_eq!(- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmain"- );-- let _ = fs::remove_dir_all(models_dir);- }-- #[test]- fn migrate_flat_models_dir_adopts_sidecarless_files_for_duplicated_targets() {- let models_dir = scratch_dir("flat-duplicate-targets");- fs::create_dir_all(&models_dir).unwrap();- fs::write(models_dir.join("main.gguf"), b"GGUFmain").unwrap();- fs::write(models_dir.join("mmproj.gguf"), b"GGUFmm").unwrap();-- let preset = target("lfm-vl-1.6b");- migrate_flat_models_dir(&models_dir, &[preset.clone(), preset.clone()]);-- let downloader = ModelDownloader::new(&models_dir);- assert_eq!(- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmain"- );- assert_eq!(- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmm"- );-- let _ = fs::remove_dir_all(models_dir);- }-- #[test]- fn migrate_flat_models_dir_resolves_ambiguity_via_metadata_sidecars() {- let models_dir = scratch_dir("flat-sidecar");- fs::create_dir_all(&models_dir).unwrap();-- let targets = ["a", "b"].map(|name| {- gguf_target(- &format!("preset-{name}"),- format!("https://example.org/{name}/{name}.gguf"),- format!("https://example.org/{name}/mmproj-F16.gguf"),- )- });-- fs::write(models_dir.join("a.gguf"), b"GGUFa").unwrap();- fs::write(- models_dir.join("a.gguf.metadata.json"),- br#"{"url": "https://example.org/a/a.gguf"}"#,- )- .unwrap();- fs::write(models_dir.join("mmproj-F16.gguf"), b"GGUFmma").unwrap();- fs::write(- models_dir.join("mmproj-F16.gguf.metadata.json"),- br#"{"url": "https://example.org/a/mmproj-F16.gguf"}"#,- )- .unwrap();-- migrate_flat_models_dir(&models_dir, &targets);-- let downloader = ModelDownloader::new(&models_dir);- assert_eq!(- fs::read(model::llm_model_path(&downloader, &targets[0]).unwrap()).unwrap(),- b"GGUFa"- );- assert_eq!(- fs::read(model::llm_mmproj_path(&downloader, &targets[0]).unwrap()).unwrap(),- b"GGUFmma"- );- assert!(- !model::llm_model_path(&downloader, &targets[1])- .unwrap()- .exists()- );-- let _ = fs::remove_dir_all(models_dir);- }-- #[test]- fn migrate_flat_models_dir_rejects_files_whose_sidecar_disagrees() {- let models_dir = scratch_dir("flat-sidecar-mismatch");- fs::create_dir_all(&models_dir).unwrap();-- let preset = target("qwen-2b-q8");- fs::write(models_dir.join("main.gguf"), b"GGUFstale").unwrap();- fs::write(- models_dir.join("main.gguf.metadata.json"),- br#"{"url": "https://example.org/elsewhere/main.gguf"}"#,- )- .unwrap();-- migrate_flat_models_dir(&models_dir, std::slice::from_ref(&preset));-- let downloader = ModelDownloader::new(&models_dir);- assert!(- !model::llm_model_path(&downloader, &preset)- .unwrap()- .exists()- );-- let _ = fs::remove_dir_all(models_dir);- }-- #[test]- fn migrate_flat_models_dir_drops_targets_with_ambiguous_basenames() {- let models_dir = scratch_dir("flat-ambiguous");- fs::create_dir_all(&models_dir).unwrap();- fs::write(models_dir.join("a.gguf"), b"GGUFa").unwrap();- fs::write(models_dir.join("mmproj-F16.gguf"), b"GGUFshared").unwrap();-- let targets = ["a", "b"].map(|name| {- gguf_target(- &format!("preset-{name}"),- format!("https://example.org/{name}/{name}.gguf"),- format!("https://example.org/{name}/mmproj-F16.gguf"),- )- });-- migrate_flat_models_dir(&models_dir, &targets);-- assert_eq!(fs::read_dir(&models_dir).unwrap().count(), 0);-- let _ = fs::remove_dir_all(models_dir);- }-- #[test]- fn migrate_keeps_existing_destination_files() {- let base = scratch_dir("migrate-existing");- let legacy = base.join("llm");- fs::create_dir_all(legacy.join("models")).unwrap();- fs::write(legacy.join("models/main.gguf"), b"GGUFold").unwrap();- fs::write(legacy.join("models/mmproj.gguf"), b"GGUFmm").unwrap();-- let models_dir = base.join("models");- let preset = target("qwen-2b-q8");- let downloader = ModelDownloader::new(&models_dir);- write_gguf(- &model::llm_model_path(&downloader, &preset).unwrap(),- b"GGUFnew",- );-- migrate_legacy_dir(- &ModelDownloader::new(&models_dir),- &legacy,- std::slice::from_ref(&preset),- );-- assert_eq!(- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFnew"- );- assert_eq!(- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),- b"GGUFmm"- );- assert!(!legacy.exists());-let _ = fs::remove_dir_all(base);}}diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rsindex 91fd1a1529..0fd9304bb0 100644--- a/rust/crates/ensu/src/model/mod.rs+++ b/rust/crates/ensu/src/model/mod.rs@@ -2,22 +2,32 @@ pub mod migrations;use std::path::PathBuf;-use ente_model_download::{ModelDownloader, ModelFile, ModelTarget};+use ente_assets::{Asset, AssetFile, AssetStore, download::Progress};use crate::config::{self, ModelPreset};+const MODELS: &str = "models";const LLM_MODEL_FILE: &str = "model.gguf";const LLM_MMPROJ_FILE: &str = "mmproj.gguf";const VOICE_ACTIVITY_MODEL_FILE: &str = "model.onnx";#[derive(Debug, thiserror::Error)]-#[error("preset {id} pairs only half of the mmproj URL and checksum")]+#[error("invalid preset {id}: {reason}")]pub struct InvalidPreset {pub id: String,+ reason: String,}-pub(crate) fn llm_target(preset: &ModelPreset) -> Result<ModelTarget, InvalidPreset> {- let mut files = vec![ModelFile {+pub struct ModelDownloadProgress {+ pub downloaded_bytes: u64,+ pub total_bytes: Option<u64>,+ pub percent: i32,+ pub status: String,+ pub log_line: Option<String>,+}++pub(crate) fn llm_asset(preset: &ModelPreset) -> Result<Asset, InvalidPreset> {+ let mut files = vec![AssetFile {name: LLM_MODEL_FILE.to_string(),url: preset.url.clone(),sha256: preset.sha256.clone(),@@ -26,91 +36,169 @@ pub(crate) fn llm_target(preset: &ModelPreset) -> Result<ModelTarget, InvalidPretrimmed(preset.mmproj_url.as_deref()),trimmed(preset.mmproj_sha256.as_deref()),) {- (Some(url), Some(sha256)) => files.push(ModelFile {+ (Some(url), Some(sha256)) => files.push(AssetFile {name: LLM_MMPROJ_FILE.to_string(),url: url.to_string(),sha256: sha256.to_string(),}),(None, None) => {}_ => {- return Err(InvalidPreset {- id: preset.id.clone(),- });+ return Err(invalid_preset(+ preset,+ "mmproj URL and checksum must be paired",+ ));}}- Ok(ModelTarget::Files {- id: preset.id.clone(),- files,- })+ Asset::files(model_key(&preset.id), files)+ .map_err(|error| invalid_preset(preset, error.to_string()))}-pub fn mobile_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {+pub fn mobile_llm_asset(model_id: &str) -> Result<Asset, InvalidPreset> {let preset = config::llm_catalog().into_iter().find(|preset| preset.id == model_id).unwrap_or_else(|| config::defaults().mobile_default_model);- llm_target(&preset)+ llm_asset(&preset)}-pub fn desktop_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {+pub fn desktop_llm_asset(model_id: &str) -> Result<Asset, InvalidPreset> {let preset = config::llm_catalog().into_iter().find(|preset| preset.id == model_id).unwrap_or_else(|| config::defaults().desktop_default_model);- llm_target(&preset)+ llm_asset(&preset)}-pub fn transcription_target() -> ModelTarget {+pub fn transcription_model_asset() -> Asset {let preset = config::defaults().transcription_model;- ModelTarget::TarGz {- id: preset.id,- url: preset.url,- sha256: preset.sha256,- }+ Asset::tar_gz(model_key(&preset.id), preset.url, preset.sha256)+ .expect("transcription asset config")}-pub fn voice_activity_target() -> ModelTarget {+pub fn voice_activity_model_asset() -> Asset {let preset = config::defaults().voice_activity_model;- ModelTarget::Files {- id: preset.id,- files: vec![ModelFile {+ Asset::files(+ model_key(&preset.id),+ vec![AssetFile {name: VOICE_ACTIVITY_MODEL_FILE.to_string(),url: preset.url,sha256: preset.sha256,}],- }+ )+ .expect("voice activity asset config")}-pub fn knowledge_embedding_target() -> ModelTarget {+pub fn knowledge_embedding_model_asset() -> Asset {let embedding = config::knowledge_embedding_config();- ModelTarget::Files {- id: embedding.target_id,- files: vec![ModelFile {+ Asset::files(+ model_key(&embedding.target_id),+ vec![AssetFile {name: LLM_MODEL_FILE.to_string(),url: embedding.model_url,sha256: embedding.model_sha256,}],- }+ )+ .expect("knowledge embedding asset config")+}++pub fn llm_model_path(store: &AssetStore, asset: &Asset) -> Option<PathBuf> {+ store.file_path(asset, LLM_MODEL_FILE)}-pub fn llm_model_path(downloader: &ModelDownloader, target: &ModelTarget) -> Option<PathBuf> {- downloader.file_path(target, LLM_MODEL_FILE)+pub fn llm_mmproj_path(store: &AssetStore, asset: &Asset) -> Option<PathBuf> {+ store.file_path(asset, LLM_MMPROJ_FILE)}-pub fn llm_mmproj_path(downloader: &ModelDownloader, target: &ModelTarget) -> Option<PathBuf> {- downloader.file_path(target, LLM_MMPROJ_FILE)+pub fn voice_activity_model_path(store: &AssetStore, asset: &Asset) -> PathBuf {+ store+ .file_path(asset, VOICE_ACTIVITY_MODEL_FILE)+ .expect("voice activity model file")}-pub fn voice_activity_model_path(downloader: &ModelDownloader) -> PathBuf {- downloader- .model_dir(&voice_activity_target())- .join(VOICE_ACTIVITY_MODEL_FILE)+pub(crate) fn model_key(id: &str) -> Vec<String> {+ vec![MODELS.to_string(), id.to_string()]}pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {value.map(str::trim).filter(|value| !value.is_empty())}+pub fn display_progress(progress: Progress) -> ModelDownloadProgress {+ let total = progress.total_bytes.filter(|total| *total > 0);+ let percent = total+ .map(|total| ((progress.downloaded_bytes * 100 / total) as i32).clamp(0, 99))+ .unwrap_or(0);+ let status = if let Some(total) = total {+ format!(+ "Downloading... {} / {}",+ format_bytes(progress.downloaded_bytes),+ format_bytes(total)+ )+ } else if progress.file_downloaded_bytes > 0 {+ format!(+ "Downloading {}... {}",+ progress.label.to_lowercase(),+ format_bytes(progress.file_downloaded_bytes)+ )+ } else {+ format!("Downloading {}...", progress.label.to_lowercase())+ };+ let log_line = if progress.file_complete {+ Some(format!(+ "Asset download file complete label={} bytes={} elapsedMs={} rate={}/s retries={}",+ progress.label,+ progress.file_downloaded_bytes,+ progress.file_elapsed_ms,+ format_bytes(rate_bytes(progress.file_bytes_per_second)),+ progress.file_retry_count+ ))+ } else if progress.complete {+ Some(format!(+ "Asset download complete bytes={} elapsedMs={} rate={}/s retries={}",+ progress.downloaded_bytes,+ progress.elapsed_ms,+ format_bytes(rate_bytes(progress.bytes_per_second)),+ progress.retry_count+ ))+ } else {+ None+ };++ ModelDownloadProgress {+ downloaded_bytes: progress.downloaded_bytes,+ total_bytes: progress.total_bytes,+ percent,+ status,+ log_line,+ }+}++fn invalid_preset(preset: &ModelPreset, reason: impl Into<String>) -> InvalidPreset {+ InvalidPreset {+ id: preset.id.clone(),+ reason: reason.into(),+ }+}++fn rate_bytes(bytes_per_second: f64) -> u64 {+ if bytes_per_second.is_finite() && bytes_per_second > 0.0 {+ bytes_per_second as u64+ } else {+ 0+ }+}++fn format_bytes(bytes: u64) -> String {+ const UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];+ let mut size = bytes as f64;+ let mut unit = 0;+ while size >= 1024.0 && unit < UNITS.len() - 1 {+ size /= 1024.0;+ unit += 1;+ }+ format!("{size:.1} {}", UNITS[unit])+}+#[cfg(test)]mod tests {use super::*;@@ -127,82 +215,77 @@ mod tests {}#[test]- fn llm_target_names_model_and_mmproj_files() {- let target = llm_target(&preset(+ fn llm_asset_names_model_and_mmproj_files() {+ let asset = llm_asset(&preset(Some("https://example.org/mmproj.gguf"),- Some("abc"),+ Some(&"a".repeat(64)),)).expect("paired preset");- let downloader = ModelDownloader::new(std::env::temp_dir().join("ensu-models-test"));+ let store = AssetStore::new(std::env::temp_dir().join("ensu-assets-test"));assert_eq!(- llm_model_path(&downloader, &target).unwrap(),- downloader.model_dir(&target).join("model.gguf")+ llm_model_path(&store, &asset).unwrap(),+ store.asset_dir(&asset).join("model.gguf"));assert_eq!(- llm_mmproj_path(&downloader, &target).unwrap(),- downloader.model_dir(&target).join("mmproj.gguf")+ llm_mmproj_path(&store, &asset).unwrap(),+ store.asset_dir(&asset).join("mmproj.gguf"));}#[test]- fn llm_target_without_mmproj_has_no_mmproj_path() {- let target = llm_target(&preset(None, None)).expect("model-only preset");- let downloader = ModelDownloader::new(std::env::temp_dir().join("ensu-models-test"));- assert!(llm_model_path(&downloader, &target).is_some());- assert!(llm_mmproj_path(&downloader, &target).is_none());+ fn llm_asset_without_mmproj_has_no_mmproj_path() {+ let asset = llm_asset(&preset(None, None)).expect("model-only preset");+ let store = AssetStore::new(std::env::temp_dir().join("ensu-assets-test"));+ assert!(llm_model_path(&store, &asset).is_some());+ assert!(llm_mmproj_path(&store, &asset).is_none());}#[test]- fn llm_target_rejects_half_specified_mmproj() {- assert!(llm_target(&preset(Some("https://example.org/mmproj.gguf"), None)).is_err());- assert!(llm_target(&preset(None, Some("abc"))).is_err());- assert!(llm_target(&preset(Some(" "), Some("abc"))).is_err());+ fn llm_asset_rejects_half_specified_mmproj() {+ assert!(llm_asset(&preset(Some("https://example.org/mmproj.gguf"), None)).is_err());+ assert!(llm_asset(&preset(None, Some(&"a".repeat(64)))).is_err());+ assert!(llm_asset(&preset(Some(" "), Some(&"a".repeat(64)))).is_err());}#[test]- fn llm_targets_resolve_catalog_ids_or_fall_back_to_defaults() {+ fn llm_assets_resolve_catalog_ids_or_fall_back_to_defaults() {let defaults = crate::config::defaults();for preset in crate::config::llm_catalog() {assert_eq!(- mobile_llm_target(&preset.id).unwrap(),- llm_target(&preset).unwrap(),+ mobile_llm_asset(&preset.id).unwrap(),+ llm_asset(&preset).unwrap(),"{}",preset.id);assert_eq!(- desktop_llm_target(&preset.id).unwrap(),- llm_target(&preset).unwrap(),+ desktop_llm_asset(&preset.id).unwrap(),+ llm_asset(&preset).unwrap(),"{}",preset.id);}- let mobile_default = llm_target(&defaults.mobile_default_model).unwrap();- assert_eq!(mobile_llm_target("").unwrap(), mobile_default);- assert_eq!(mobile_llm_target("no-such-model").unwrap(), mobile_default);- let desktop_default = llm_target(&defaults.desktop_default_model).unwrap();- assert_eq!(desktop_llm_target("").unwrap(), desktop_default);- assert_eq!(- desktop_llm_target("no-such-model").unwrap(),- desktop_default- );+ let mobile_default = llm_asset(&defaults.mobile_default_model).unwrap();+ assert_eq!(mobile_llm_asset("").unwrap(), mobile_default);+ assert_eq!(mobile_llm_asset("no-such-model").unwrap(), mobile_default);+ let desktop_default = llm_asset(&defaults.desktop_default_model).unwrap();+ assert_eq!(desktop_llm_asset("").unwrap(), desktop_default);+ assert_eq!(desktop_llm_asset("no-such-model").unwrap(), desktop_default);}#[test]- fn all_catalog_presets_produce_valid_targets() {+ fn all_catalog_presets_produce_valid_assets() {for preset in crate::config::llm_catalog() {- llm_target(&preset).expect("catalog preset");+ llm_asset(&preset).expect("catalog preset");}}#[test]- fn knowledge_embedding_target_is_checksum_backed() {- let target = knowledge_embedding_target();- let ModelTarget::Files { id, files } = target else {- panic!("embedding target must use files");- };- assert_eq!(id, "embeddinggemma-300m-q8-0");- assert_eq!(files.len(), 1);- assert_eq!(files[0].name, "model.gguf");- assert_eq!(files[0].sha256.len(), 64);+ fn knowledge_embedding_asset_is_checksum_backed() {+ let store = AssetStore::new(std::env::temp_dir().join("ensu-assets-test"));+ let asset = knowledge_embedding_model_asset();+ assert_eq!(+ store.file_path(&asset, "model.gguf"),+ Some(store.asset_dir(&asset).join("model.gguf"))+ );}}diff --git a/rust/crates/ensu/src/retrieval.rs b/rust/crates/ensu/src/retrieval.rsindex 923701f0f3..b1d3e31896 100644--- a/rust/crates/ensu/src/retrieval.rs+++ b/rust/crates/ensu/src/retrieval.rs@@ -45,6 +45,7 @@ pub use citation::{pub use index::{RetrievalHit, RetrievalIndex};pub use pack::{KnowledgeReconciliation, KnowledgeReconciliationStatus,- cleanup_obsolete_knowledge_pack_revisions, download_knowledge_pack, reconcile_knowledge_pack,+ cleanup_obsolete_knowledge_pack_revisions, knowledge_asset, knowledge_pack_root,+ reconcile_knowledge_pack,};pub use prompt::{KnowledgePromptContext, KnowledgePromptHit, build_knowledge_prompt_context};diff --git a/rust/crates/ensu/src/retrieval/pack.rs b/rust/crates/ensu/src/retrieval/pack.rsindex a0add74dd0..65a9fc4572 100644--- a/rust/crates/ensu/src/retrieval/pack.rs+++ b/rust/crates/ensu/src/retrieval/pack.rs@@ -1,7 +1,7 @@use std::fs;use std::path::{Path, PathBuf};-use ente_model_download::download::{self, Downloader, Progress, Target};+use ente_assets::{Asset, AssetFile, AssetStore, download};use crate::config::{KNOWLEDGE_ARTIFACT_FILENAMES, KnowledgeDatasetConfig, is_path_safe_component,@@ -10,67 +10,39 @@ use crate::config::{use super::{RetrievalError, index::RetrievalIndex};-pub fn download_knowledge_pack(- pack_root: impl AsRef<Path>,- expected_pack: &KnowledgeDatasetConfig,- on_progress: impl FnMut(Progress) + Send,- cancellation: download::CancellationToken,-) -> Result<(), download::Error> {- let pack_root = pack_root.as_ref();- let targets = knowledge_download_targets(pack_root, expected_pack)?;- tokio::runtime::Builder::new_current_thread()- .enable_io()- .enable_time()- .build()?- .block_on(Downloader::new()?.download(targets, on_progress, cancellation))-}--fn knowledge_download_targets(- pack_root: &Path,- expected_pack: &KnowledgeDatasetConfig,-) -> Result<Vec<Target>, download::Error> {- let root_identity = pack_root- .file_name()- .and_then(|name| name.to_str())- .ok_or_else(|| {- download::Error::InvalidTarget(- "knowledge pack root must have a UTF-8 stable ID".to_string(),- )- })?;- if root_identity != expected_pack.stable_id {- return Err(download::Error::InvalidTarget(- "knowledge pack root must end with the selected stable ID".to_string(),- ));- }- fs::create_dir_all(pack_root)?;- if !fs::symlink_metadata(pack_root)?.file_type().is_dir() {- return Err(download::Error::InvalidTarget(- "knowledge pack root must be a directory, not a symlink".to_string(),- ));- }- let revision_directory = pack_root.join(&expected_pack.current_download_identity);- fs::create_dir_all(&revision_directory)?;- if !fs::symlink_metadata(&revision_directory)?- .file_type()- .is_dir()- {- return Err(download::Error::InvalidTarget(- "knowledge revision path must be a directory, not a symlink".to_string(),- ));- }-+pub fn knowledge_asset(expected_pack: &KnowledgeDatasetConfig) -> Result<Asset, download::Error> {let urls = knowledge_artifact_urls(expected_pack);- Ok(KNOWLEDGE_ARTIFACT_FILENAMES+ let files = KNOWLEDGE_ARTIFACT_FILENAMES.into_iter().zip(urls).zip(expected_pack.artifact_sha256.iter().cloned())- .map(|((filename, url), sha256)| Target {- label: filename.to_owned(),+ .map(|((filename, url), sha256)| AssetFile {+ name: filename.to_owned(),url,sha256,- destination: revision_directory.join(filename),})- .collect())+ .collect();+ Asset::files(+ vec![+ "knowledge".to_string(),+ expected_pack.stable_id.clone(),+ expected_pack.current_download_identity.clone(),+ ],+ files,+ )+}++pub fn knowledge_pack_root(+ store: &AssetStore,+ expected_pack: &KnowledgeDatasetConfig,+) -> Result<PathBuf, download::Error> {+ store+ .asset_dir(&knowledge_asset(expected_pack)?)+ .parent()+ .map(Path::to_path_buf)+ .ok_or_else(|| {+ download::Error::InvalidTarget("knowledge asset has no pack directory".to_string())+ })}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -241,23 +213,21 @@ mod tests {use crate::retrieval::index::tests::synthetic_pack;#[test]- fn constructs_only_the_four_fixed_download_targets() {- let temp = tempfile::tempdir().unwrap();+ fn constructs_only_the_four_fixed_download_files() {let expected = defaults().knowledge_datasets.remove(0);- let pack_root = temp.path().join(&expected.stable_id);- let targets = knowledge_download_targets(&pack_root, &expected).unwrap();- assert_eq!(targets.len(), 4);- for (target, filename) in targets.iter().zip(KNOWLEDGE_ARTIFACT_FILENAMES) {- assert!(target.url.ends_with(filename));- assert!(target.destination.ends_with(filename));- assert!(- target- .destination- .to_string_lossy()- .contains(&expected.current_download_identity)- );- assert_eq!(target.sha256.len(), 64);+ let store = AssetStore::new("assets");+ let asset = knowledge_asset(&expected).unwrap();+ assert_eq!(+ store.asset_dir(&asset),+ Path::new("assets")+ .join("knowledge")+ .join(&expected.stable_id)+ .join(&expected.current_download_identity)+ );+ for name in KNOWLEDGE_ARTIFACT_FILENAMES {+ assert!(store.file_path(&asset, name).is_some());}+ assert!(!store.is_downloaded(&asset));}#[test]diff --git a/rust/crates/model-download/src/archive.rs b/rust/crates/model-download/src/archive.rsdeleted file mode 100644index 146ee9e06f..0000000000--- a/rust/crates/model-download/src/archive.rs+++ /dev/null@@ -1,35 +0,0 @@-use std::fs;-use std::fs::File;-use std::path::Path;--use flate2::read::GzDecoder;-use tar::Archive;--use crate::download;--pub(crate) fn extract_tar_gz(- archive_path: &Path,- staging_dir: &Path,- dest_dir: &Path,-) -> Result<(), download::Error> {- if staging_dir.exists() {- fs::remove_dir_all(staging_dir)?;- }- fs::create_dir_all(staging_dir)?;-- let tar_gz = File::open(archive_path)?;- let mut archive = Archive::new(GzDecoder::new(tar_gz));- archive.unpack(staging_dir)?;-- let entries = fs::read_dir(staging_dir)?.flatten().collect::<Vec<_>>();- let source = match entries.as_slice() {- [only] if only.path().is_dir() => only.path(),- _ => staging_dir.to_path_buf(),- };- if dest_dir.exists() {- fs::remove_dir_all(dest_dir)?;- }- fs::rename(&source, dest_dir)?;- let _ = fs::remove_dir_all(staging_dir);- Ok(())-}diff --git a/rust/crates/model-download/src/lib.rs b/rust/crates/model-download/src/lib.rsdeleted file mode 100644index 6af18cdd83..0000000000--- a/rust/crates/model-download/src/lib.rs+++ /dev/null@@ -1,5 +0,0 @@-mod archive;-pub mod download;-mod model;--pub use model::{ModelDownloadProgress, ModelDownloader, ModelFile, ModelTarget};diff --git a/rust/crates/model-download/src/model.rs b/rust/crates/model-download/src/model.rsdeleted file mode 100644index 9b96045788..0000000000--- a/rust/crates/model-download/src/model.rs+++ /dev/null@@ -1,648 +0,0 @@-use std::collections::HashSet;-use std::fs;-use std::path::{Path, PathBuf};-use std::sync::atomic::{AtomicBool, Ordering};-use std::sync::{Mutex, OnceLock, PoisonError};--use tokio::runtime::Builder;--use crate::archive;-use crate::download::{self, Progress, Target};--#[derive(Debug, Clone, PartialEq, Eq)]-pub enum ModelTarget {- Files {- id: String,- files: Vec<ModelFile>,- },- TarGz {- id: String,- url: String,- sha256: String,- },-}--#[derive(Debug, Clone, PartialEq, Eq)]-pub struct ModelFile {- pub name: String,- pub url: String,- pub sha256: String,-}--#[derive(Debug, Clone)]-pub struct ModelDownloadProgress {- pub downloaded_bytes: u64,- pub total_bytes: Option<u64>,- pub percent: i32,- pub status: String,- pub log_line: Option<String>,-}--pub struct ModelDownloader {- downloader: OnceLock<download::Downloader>,- models_dir: PathBuf,- active: AtomicBool,- download_lock: Mutex<()>,-}--impl ModelDownloader {- pub fn new(models_dir: impl Into<PathBuf>) -> Self {- Self {- downloader: OnceLock::new(),- models_dir: models_dir.into(),- active: AtomicBool::new(false),- download_lock: Mutex::new(()),- }- }-- pub fn downloader(&self) -> Result<&download::Downloader, download::Error> {- if let Some(downloader) = self.downloader.get() {- return Ok(downloader);- }- let downloader = download::Downloader::new()?;- Ok(self.downloader.get_or_init(|| downloader))- }-- pub fn model_dir(&self, target: &ModelTarget) -> PathBuf {- storage_dir(&self.models_dir, target)- }-- pub fn file_path(&self, target: &ModelTarget, name: &str) -> Option<PathBuf> {- match target {- ModelTarget::Files { files, .. } if files.iter().any(|file| file.name == name) => {- Some(storage_dir(&self.models_dir, target).join(name))- }- _ => None,- }- }-- pub fn is_downloaded(&self, target: &ModelTarget) -> bool {- if let ModelTarget::TarGz { .. } = target {- return storage_dir(&self.models_dir, target).is_dir();- }- let paths = expected_paths(&self.models_dir, target);- !paths.is_empty() && paths.iter().all(|path| path.is_file())- }-- pub fn is_download_active(&self) -> bool {- self.active.load(Ordering::SeqCst)- }-- pub fn remove_downloaded(&self, target: &ModelTarget) -> bool {- let Ok(_guard) = self.download_lock.try_lock() else {- return false;- };- if checked_file_name(storage_key(target)).is_err() {- return false;- }- let dir = storage_dir(&self.models_dir, target);- dir.exists() && fs::remove_dir_all(dir).is_ok()- }-- pub fn download(- &self,- targets: &[ModelTarget],- mut on_progress: impl FnMut(ModelDownloadProgress),- cancellation: &download::CancellationToken,- ) -> Result<(), download::Error> {- let _guard = self- .download_lock- .lock()- .unwrap_or_else(PoisonError::into_inner);- let pending = targets- .iter()- .filter(|target| !self.is_downloaded(target))- .collect::<Vec<_>>();- if pending.is_empty() {- return Ok(());- }-- let mut transfers = Vec::new();- for target in &pending {- transfers.extend(download_targets(&self.models_dir, target)?);- }-- if cancellation.is_cancelled() {- return Err(download::Error::Cancelled);- }- let downloader = self.downloader()?;- self.active.store(true, Ordering::SeqCst);- let result = download::fetch(- downloader,- transfers,- |progress| on_progress(display_progress(progress)),- || cancellation.is_cancelled(),- )- .and_then(|()| {- for target in &pending {- if cancellation.is_cancelled() {- return Err(download::Error::Cancelled);- }- self.extract_if_archive(target, &mut on_progress)?;- }- Ok(())- });- self.active.store(false, Ordering::SeqCst);- result- }-- fn extract_if_archive(- &self,- target: &ModelTarget,- on_progress: &mut impl FnMut(ModelDownloadProgress),- ) -> Result<(), download::Error> {- let ModelTarget::TarGz { id, .. } = target else {- return Ok(());- };- on_progress(ModelDownloadProgress {- downloaded_bytes: 0,- total_bytes: None,- percent: 99,- status: "Extracting...".to_string(),- log_line: Some(format!("Extracting model archive id={id}")),- });- let archive_path = self.staging_dir().join(format!("{id}.tar.gz"));- let extraction_dir = self.staging_dir().join(id);- let result =- archive::extract_tar_gz(&archive_path, &extraction_dir, &self.model_dir(target));- if result.is_err() {- let _ = fs::remove_dir_all(extraction_dir);- } else {- let _ = fs::remove_file(&archive_path);- }- result- }-- fn staging_dir(&self) -> PathBuf {- self.models_dir.join(".staging")- }-- pub fn estimated_download_size(&self, target: &ModelTarget) -> Option<u64> {- let runtime = Builder::new_current_thread()- .enable_io()- .enable_time()- .build()- .ok()?;- let downloader = self.downloader().ok()?;- runtime.block_on(async {- let mut total: u64 = 0;- let mut any = false;- for entry in download_targets(&self.models_dir, target).ok()? {- let path = &entry.destination;- let size = if path.exists() {- fs::metadata(path).ok().map(|m| m.len()).filter(|s| *s > 0)- } else {- download::probe_content_length(downloader.client(), &entry.url).await- };- if let Some(size) = size {- any = true;- total = total.saturating_add(size);- }- }- any.then_some(total)- })- }-}--fn storage_key(target: &ModelTarget) -> &str {- match target {- ModelTarget::Files { id, .. } | ModelTarget::TarGz { id, .. } => id,- }-}--fn storage_dir(models_dir: &Path, target: &ModelTarget) -> PathBuf {- models_dir.join(storage_key(target))-}--fn expected_paths(models_dir: &Path, target: &ModelTarget) -> Vec<PathBuf> {- let dir = storage_dir(models_dir, target);- match target {- ModelTarget::TarGz { .. } => vec![dir],- ModelTarget::Files { files, .. } => files.iter().map(|file| dir.join(&file.name)).collect(),- }-}--fn download_targets(- models_dir: &Path,- target: &ModelTarget,-) -> Result<Vec<Target>, download::Error> {- let dir = storage_dir(models_dir, target);- match target {- ModelTarget::TarGz { id, url, sha256 } => {- checked_file_name(id)?;- Ok(vec![Target {- label: "Model".to_string(),- url: url.clone(),- sha256: sha256.clone(),- destination: models_dir.join(".staging").join(format!("{id}.tar.gz")),- }])- }- ModelTarget::Files { id, files } => {- checked_file_name(id)?;- if files.is_empty() {- return Err(download::Error::InvalidTarget(format!("{id} has no files")));- }- let mut names = HashSet::new();- files- .iter()- .map(|file| {- checked_file_name(&file.name)?;- if !names.insert(file.name.as_str()) {- return Err(download::Error::InvalidTarget(format!(- "{id} has a duplicate file name {}",- file.name- )));- }- Ok(Target {- label: file.name.clone(),- url: file.url.clone(),- sha256: file.sha256.clone(),- destination: dir.join(&file.name),- })- })- .collect()- }- }-}--fn checked_file_name(name: &str) -> Result<(), download::Error> {- if !name.is_empty()- && name != "."- && name != ".."- && !name.contains('/')- && !name.contains('\\')- {- return Ok(());- }- Err(download::Error::InvalidTarget(format!(- "'{name}' is not a safe file name"- )))-}--fn display_progress(progress: Progress) -> ModelDownloadProgress {- let total = progress.total_bytes.filter(|total| *total > 0);- let percent = total- .map(|total| ((progress.downloaded_bytes * 100 / total) as i32).clamp(0, 99))- .unwrap_or(0);- let status = if let Some(total) = total {- format!(- "Downloading... {} / {}",- format_bytes(progress.downloaded_bytes),- format_bytes(total)- )- } else if progress.file_downloaded_bytes > 0 {- format!(- "Downloading {}... {}",- progress.label.to_lowercase(),- format_bytes(progress.file_downloaded_bytes)- )- } else {- format!("Downloading {}...", progress.label.to_lowercase())- };- let log_line = if progress.file_complete {- Some(format!(- "Model download file complete label={} bytes={} elapsedMs={} rate={}/s retries={}",- progress.label,- progress.file_downloaded_bytes,- progress.file_elapsed_ms,- format_bytes(rate_bytes(progress.file_bytes_per_second)),- progress.file_retry_count- ))- } else if progress.complete {- Some(format!(- "Model download complete bytes={} elapsedMs={} rate={}/s retries={}",- progress.downloaded_bytes,- progress.elapsed_ms,- format_bytes(rate_bytes(progress.bytes_per_second)),- progress.retry_count- ))- } else {- None- };-- ModelDownloadProgress {- downloaded_bytes: progress.downloaded_bytes,- total_bytes: progress.total_bytes,- percent,- status,- log_line,- }-}--fn rate_bytes(bytes_per_second: f64) -> u64 {- if bytes_per_second.is_finite() && bytes_per_second > 0.0 {- bytes_per_second as u64- } else {- 0- }-}--fn format_bytes(bytes: u64) -> String {- const UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];- let mut size = bytes as f64;- let mut unit = 0;- while size >= 1024.0 && unit < UNITS.len() - 1 {- size /= 1024.0;- unit += 1;- }- format!("{size:.1} {}", UNITS[unit])-}--#[cfg(test)]-mod tests {- use super::*;-- fn scratch_dir(name: &str) -> PathBuf {- let dir = std::env::temp_dir().join(format!(- "ente-model-download-{name}-{}",- std::time::SystemTime::now()- .duration_since(std::time::UNIX_EPOCH)- .unwrap()- .as_nanos()- ));- fs::create_dir_all(&dir).expect("create test dir");- dir- }-- fn test_sha() -> String {- "0".repeat(64)- }-- fn target(id: &str) -> ModelTarget {- ModelTarget::Files {- id: id.to_string(),- files: vec![- ModelFile {- name: "model.gguf".to_string(),- url: "https://example.org/models/main.gguf?download=true".to_string(),- sha256: test_sha(),- },- ModelFile {- name: "mmproj.gguf".to_string(),- url: "https://example.org/models/mmproj.gguf".to_string(),- sha256: test_sha(),- },- ],- }- }-- fn write_gguf(path: &Path, content: &[u8]) {- fs::create_dir_all(path.parent().unwrap()).unwrap();- fs::write(path, content).unwrap();- }-- #[test]- fn paths_are_keyed_by_preset_id() {- let dir = scratch_dir("paths");- let downloader = ModelDownloader::new(&dir);-- assert_eq!(- downloader- .file_path(&target("qwen-2b-q8"), "model.gguf")- .unwrap(),- dir.join("qwen-2b-q8/model.gguf")- );- assert_eq!(- downloader.file_path(&target("qwen-2b-q8"), "mmproj.gguf"),- Some(dir.join("qwen-2b-q8/mmproj.gguf"))- );-- let _ = fs::remove_dir_all(dir);- }-- #[test]- fn is_downloaded_requires_every_member_file() {- let dir = scratch_dir("is-downloaded");- let downloader = ModelDownloader::new(&dir);- let target = target("qwen-2b-q8");- let model = downloader.file_path(&target, "model.gguf").unwrap();- let mmproj = downloader.file_path(&target, "mmproj.gguf").unwrap();-- assert!(!downloader.is_downloaded(&target));-- write_gguf(&model, b"GGUFdata");- assert!(!downloader.is_downloaded(&target));-- write_gguf(&mmproj, b"GGUFdata");- assert!(downloader.is_downloaded(&target));-- let _ = fs::remove_dir_all(dir);- }-- #[test]- fn download_skips_when_already_present() {- let dir = scratch_dir("skip");- let downloader = ModelDownloader::new(&dir);- let target = target("qwen-2b-q8");- write_gguf(- &downloader.file_path(&target, "model.gguf").unwrap(),- b"GGUFdata",- );- write_gguf(- &downloader.file_path(&target, "mmproj.gguf").unwrap(),- b"GGUFdata",- );-- downloader- .download(- std::slice::from_ref(&target),- |_| {},- &download::CancellationToken::new(),- )- .expect("skip without network");-- let _ = fs::remove_dir_all(dir);- }-- fn tar_gz_target(id: &str) -> ModelTarget {- ModelTarget::TarGz {- id: id.to_string(),- url: format!("https://models.example.org/{id}.tar.gz"),- sha256: test_sha(),- }- }-- fn file_target(id: &str) -> ModelTarget {- ModelTarget::Files {- id: id.to_string(),- files: vec![ModelFile {- name: "model.onnx".to_string(),- url: format!("https://models.example.org/{id}.onnx"),- sha256: test_sha(),- }],- }- }-- fn write_tar_gz(path: &Path, dir_name: &str, files: &[(&str, &[u8])]) {- let file = fs::File::create(path).unwrap();- let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());- let mut builder = tar::Builder::new(encoder);- for (name, content) in files {- let mut header = tar::Header::new_gnu();- header.set_size(content.len() as u64);- header.set_mode(0o644);- header.set_cksum();- builder- .append_data(&mut header, format!("{dir_name}/{name}"), *content)- .unwrap();- }- builder.into_inner().unwrap().finish().unwrap();- }-- #[test]- fn archive_and_onnx_targets_use_per_model_dirs() {- let dir = scratch_dir("variant-paths");- let downloader = ModelDownloader::new(&dir);-- let archive = tar_gz_target("parakeet-v3-int8");- assert_eq!(downloader.model_dir(&archive), dir.join("parakeet-v3-int8"));- assert_eq!(downloader.file_path(&archive, "mmproj.gguf"), None);-- let onnx = file_target("silero-vad-v4");- assert_eq!(- downloader.file_path(&onnx, "model.onnx").unwrap(),- dir.join("silero-vad-v4/model.onnx")- );- assert_eq!(- downloader.file_path(&onnx, "model.onnx"),- Some(dir.join("silero-vad-v4/model.onnx"))- );- assert_eq!(downloader.file_path(&onnx, "missing.onnx"), None);- assert_eq!(downloader.file_path(&onnx, "mmproj.gguf"), None);-- let _ = fs::remove_dir_all(dir);- }-- #[test]- fn archive_download_extracts_into_model_dir() {- let dir = scratch_dir("variant-extract");- let downloader = ModelDownloader::new(&dir);- let target = tar_gz_target("parakeet-v3-int8");- assert!(!downloader.is_downloaded(&target));-- fs::create_dir_all(dir.join(".staging")).unwrap();- write_tar_gz(- &dir.join(".staging/parakeet-v3-int8.tar.gz"),- "parakeet-tdt-0.6b-v3-int8",- &[("encoder.onnx", b"enc"), ("tokens.txt", b"tok")],- );- let mut statuses = Vec::new();- downloader- .extract_if_archive(&target, &mut |progress| statuses.push(progress.status))- .expect("extract");-- assert!(downloader.is_downloaded(&target));- assert_eq!(- fs::read(dir.join("parakeet-v3-int8/encoder.onnx")).unwrap(),- b"enc"- );- assert_eq!(statuses, ["Extracting..."]);- assert!(!dir.join(".staging/parakeet-v3-int8.tar.gz").exists());-- assert!(downloader.remove_downloaded(&target));- assert!(!downloader.is_downloaded(&target));-- let _ = fs::remove_dir_all(dir);- }-- #[test]- fn failed_archive_extraction_cleans_staging() {- let dir = scratch_dir("variant-extract-failure");- let downloader = ModelDownloader::new(&dir);- let target = tar_gz_target("parakeet-v3-int8");- let staging_dir = dir.join(".staging");- fs::create_dir_all(&staging_dir).unwrap();- fs::write(staging_dir.join("parakeet-v3-int8.tar.gz"), b"invalid").unwrap();-- assert!(downloader.extract_if_archive(&target, &mut |_| {}).is_err());- assert!(!staging_dir.join("parakeet-v3-int8").exists());- assert!(staging_dir.join("parakeet-v3-int8.tar.gz").exists());-- let _ = fs::remove_dir_all(dir);- }-- #[test]- fn onnx_target_downloaded_and_removed_via_model_file() {- let dir = scratch_dir("variant-onnx");- let downloader = ModelDownloader::new(&dir);- let target = file_target("silero-vad-v4");- assert!(!downloader.is_downloaded(&target));-- write_gguf(- &downloader.file_path(&target, "model.onnx").unwrap(),- b"onnxbytes",- );- assert!(downloader.is_downloaded(&target));-- assert!(downloader.remove_downloaded(&target));- assert!(!downloader.is_downloaded(&target));- assert!(!dir.join("silero-vad-v4").exists());-- let _ = fs::remove_dir_all(dir);- }-- #[test]- fn display_progress_composes_percent_and_status() {- let progress = Progress {- label: "Model".to_string(),- downloaded_bytes: 512 * 1024 * 1024,- total_bytes: Some(2 * 1024 * 1024 * 1024),- file_downloaded_bytes: 512 * 1024 * 1024,- file_total_bytes: Some(2 * 1024 * 1024 * 1024),- percentage: 25.0,- elapsed_ms: 0,- bytes_per_second: 0.0,- file_elapsed_ms: 0,- file_bytes_per_second: 0.0,- retry_count: 0,- file_retry_count: 0,- file_complete: false,- complete: false,- };-- let display = display_progress(progress);- assert_eq!(display.percent, 25);- assert_eq!(display.status, "Downloading... 512.0 MB / 2.0 GB");- assert_eq!(display.log_line, None);-- let unknown_total = Progress {- total_bytes: None,- ..display_to_progress_stub()- };- let display = display_progress(unknown_total);- assert_eq!(display.percent, 0);- assert_eq!(display.status, "Downloading model... 100.0 B");-- let file_complete = Progress {- file_complete: true,- file_bytes_per_second: 2048.0,- ..display_to_progress_stub()- };- let display = display_progress(file_complete);- assert_eq!(- display.log_line.as_deref(),- Some(- "Model download file complete label=Model bytes=100 elapsedMs=0 rate=2.0 KB/s retries=0"- )- );- }-- fn display_to_progress_stub() -> Progress {- Progress {- label: "Model".to_string(),- downloaded_bytes: 100,- total_bytes: None,- file_downloaded_bytes: 100,- file_total_bytes: None,- percentage: 0.0,- elapsed_ms: 0,- bytes_per_second: 0.0,- file_elapsed_ms: 0,- file_bytes_per_second: 0.0,- retry_count: 0,- file_retry_count: 0,- file_complete: false,- complete: false,- }- }-}diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex 53ec14e8e4..980da9e89d 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -61,11 +61,11 @@ ort = { workspace = true, features = [[dev-dependencies]anyhow = "1"-ente-model-download.workspace = true+ente-assets.workspace = trueflate2 = "1.1"serde = { version = "1", features = ["derive"] }serde_json.workspace = truetar = "0.4"tempfile = "3.12"-tokio = { version = "1", features = ["rt"] }+tokio = { version = "1", features = ["macros", "rt"] }url = "2.5.8"diff --git a/rust/crates/photos/examples/ml_goldens.rs b/rust/crates/photos/examples/ml_goldens.rsindex fa561d6bf1..d2e372eb47 100644--- a/rust/crates/photos/examples/ml_goldens.rs+++ b/rust/crates/photos/examples/ml_goldens.rs@@ -16,7 +16,8 @@ use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};const CLIP_TEXT_GOLDEN_PHRASE: &str = "a photo of a dog playing on a sunny beach";-fn main() -> Result<()> {+#[tokio::main(flavor = "current_thread")]+async fn main() -> Result<()> {let mut args = std::env::args().skip(1);let command = args.next();if args.next().is_some() {@@ -24,13 +25,18 @@ fn main() -> Result<()> {}match command.as_deref() {- Some("generate") => run_with_large_stack("generate_goldens", generate),+ Some("generate") => {+ let assets = GoldenTestAssets::load().await?;+ run_with_large_stack("generate_goldens", move || generate(assets))+ }Some("validate-zero-separation") => {- run_with_large_stack("validate_zero_separation", validate_zero_separation)+ let assets = GoldenTestAssets::load().await?;+ run_with_large_stack("validate_zero_separation", move || {+ validate_zero_separation(assets)+ })}_ => bail!(usage()),}- Ok(())}fn usage() -> &'static str {@@ -73,8 +79,7 @@ fn production_model_specs(assets: &GoldenTestAssets) -> Vec<GoldenModelSpec> {]}-fn generate() -> Result<()> {- let assets = GoldenTestAssets::load()?;+fn generate(assets: GoldenTestAssets) -> Result<()> {let source = render_golden_data(&production_model_specs(&assets)).map_err(|error| anyhow::anyhow!("generating goldens: {error}"))?;let path = GoldenTestAssets::golden_data_path()?;@@ -92,8 +97,7 @@ fn generate() -> Result<()> {Ok(())}-fn validate_zero_separation() -> Result<()> {- let assets = GoldenTestAssets::load()?;+fn validate_zero_separation(assets: GoldenTestAssets) -> Result<()> {for model_path in [&assets.face_detection.path,&assets.face_embedding.path,diff --git a/rust/crates/photos/tests/ml_indexing.rs b/rust/crates/photos/tests/ml_indexing.rsindex 19f785b96b..aefa062f87 100644--- a/rust/crates/photos/tests/ml_indexing.rs+++ b/rust/crates/photos/tests/ml_indexing.rs@@ -7,13 +7,15 @@ use support::ml_indexing::{ComparisonStats, MlIndexingTestContext, fail_if_any, run_with_large_stack,};-#[test]-fn rust_ml_matches_python_goldens() {- run_with_large_stack("rust_ml_matches_python_goldens", run_ml_indexing_test);+#[tokio::test]+async fn rust_ml_matches_python_goldens() -> Result<()> {+ let context = MlIndexingTestContext::load().await?;+ run_with_large_stack("rust_ml_matches_python_goldens", move || {+ run_ml_indexing_test(context)+ })}-fn run_ml_indexing_test() -> Result<()> {- let context = MlIndexingTestContext::load()?;+fn run_ml_indexing_test(context: MlIndexingTestContext) -> Result<()> {let runtime = context.prepare_runtime();let mut failures = context.validate_manifest_expectations()?;diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rsindex f9a2d30e62..aa9fbdc43f 100644--- a/rust/crates/photos/tests/support/ml_indexing.rs+++ b/rust/crates/photos/tests/support/ml_indexing.rs@@ -1,11 +1,11 @@use std::{collections::{BTreeSet, HashMap, HashSet},fs,- path::{Path, PathBuf},+ path::{Component, Path, PathBuf},};use anyhow::{Context, Result, bail};-use ente_model_download::download;+use ente_assets::{Asset, AssetFile, AssetStore, download};use ente_photos::ml::{error::MlError,indexing::{@@ -24,17 +24,19 @@ const FACE_EMBEDDING_DIM: usize = 192;const FLOAT_TOLERANCE: f64 = 1e-8;const PRINT_STATS_ENV: &str = "ENTE_ML_INDEXING_PRINT_STATS";-pub(crate) fn run_with_large_stack(name: &str, test: fn() -> Result<()>) {+pub(crate) fn run_with_large_stack(+ name: &str,+ test: impl FnOnce() -> Result<()> + Send + 'static,+) -> Result<()> {let result = std::thread::Builder::new().name(name.to_string()).stack_size(64 * 1024 * 1024).spawn(test)- .expect("spawn ML indexing test thread")+ .with_context(|| format!("spawn {name} thread"))?.join();match result {- Ok(Ok(())) => {}- Ok(Err(error)) => panic!("{error:#}"),+ Ok(result) => result,Err(panic) => std::panic::resume_unwind(panic),}}@@ -163,73 +165,38 @@ fn should_print_stats() -> bool {pub(crate) struct MlIndexingTestContext {asset_lock: AssetLock,- cache_dir: PathBuf,+ store: AssetStore,golden_results: HashMap<String, ComparableResult>,manifest: FixtureManifest,model_paths: ModelPaths,}-struct AssetFetcher {- runtime: tokio::runtime::Runtime,- downloader: download::Downloader,-}--impl AssetFetcher {- fn new() -> Result<Self> {- Ok(Self {- runtime: tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .context("build download runtime")?,- downloader: download::Downloader::new().context("build downloader")?,- })- }-- fn fetch(&self, targets: Vec<download::Target>) -> Result<()> {- self.runtime.block_on(self.downloader.download(- targets,- |_| {},- download::CancellationToken::default(),- ))?;- Ok(())- }-}-impl MlIndexingTestContext {- pub(crate) fn load() -> Result<Self> {+ pub(crate) async fn load() -> Result<Self> {let repo_root = repo_root()?;let asset_lock = load_asset_lock(&repo_root)?;let cache_dir = cache_dir(&repo_root);- let fetcher = AssetFetcher::new()?;+ let store = AssetStore::new(&cache_dir);let manifest_path =- resolve_document_asset(&fetcher, &cache_dir, "manifest", &asset_lock.manifest)?;- let golden_path = resolve_document_asset(- &fetcher,- &cache_dir,- "python-golden",- &asset_lock.python_golden,- )?;+ resolve_document_asset(&store, "manifest", &asset_lock.manifest).await?;+ let golden_path =+ resolve_document_asset(&store, "python-golden", &asset_lock.python_golden).await?;let manifest = load_manifest(&manifest_path)?;let golden_results = load_golden_results(&golden_path)?;- fetch_fixtures(- &fetcher,- &cache_dir,- &asset_lock.fixture_base_url,- &manifest,- )?;+ fetch_fixtures(&store, &asset_lock.fixture_base_url, &manifest).await?;let onnx_runtime_library =- resolve_onnx_runtime_library(&fetcher, &cache_dir, &asset_lock.onnx_runtime)?;+ resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;let _ = ort::init_from(&onnx_runtime_library).context("load ONNX Runtime dynamic library")?.commit();- let model_paths = resolve_model_paths(&fetcher, &cache_dir, &asset_lock.models)?;+ let model_paths = resolve_model_paths(&store, &asset_lock.models).await?;Ok(Self {asset_lock,- cache_dir,+ store,golden_results,manifest,model_paths,@@ -368,7 +335,17 @@ impl MlIndexingTestContext {}fn resolve_fixture(&self, fixture: &FixtureFile) -> Result<PathBuf> {- cache_path_for(&self.cache_dir, "fixtures", &fixture.path, &fixture.sha256)+ let label = file_id_for_manifest_path(&fixture.path)?;+ let asset = file_asset(+ "fixtures",+ &label,+ &fixture.path,+ &fixture_url(&self.asset_lock.fixture_base_url, &fixture.path)?,+ &fixture.sha256,+ )?;+ self.store+ .file_path(&asset, &label)+ .context("fixture asset has no declared file")}fn unsupported_decode_file_ids(&self) -> HashSet<String> {@@ -401,37 +378,30 @@ pub(crate) struct GoldenTestAssets {#[allow(dead_code)]impl GoldenTestAssets {- pub(crate) fn load() -> Result<Self> {+ pub(crate) async fn load() -> Result<Self> {let repo_root = repo_root()?;let asset_lock = load_asset_lock(&repo_root)?;let cache_dir = cache_dir(&repo_root);- let fetcher = AssetFetcher::new()?;+ let store = AssetStore::new(&cache_dir);let onnx_runtime_library =- resolve_onnx_runtime_library(&fetcher, &cache_dir, &asset_lock.onnx_runtime)?;+ resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;let _ = ort::init_from(&onnx_runtime_library).context("load ONNX Runtime dynamic library")?.commit();- let golden_model = |label: &str, model: &ModelAsset| -> Result<GoldenModelAsset> {- Ok(GoldenModelAsset {- path: resolve_model_asset(&fetcher, &cache_dir, label, model)?,- sha256: model.sha256.clone(),- })- };-let models = &asset_lock.models;Ok(Self {- face_detection: golden_model("face-detection", &models.face_detection)?,- face_embedding: golden_model("face-embedding", &models.face_embedding)?,- clip_image: golden_model("clip-image", &models.clip_image)?,- clip_text: golden_model("clip-text", &models.clip_text)?,+ face_detection: golden_model(&store, "face-detection", &models.face_detection).await?,+ face_embedding: golden_model(&store, "face-embedding", &models.face_embedding).await?,+ clip_image: golden_model(&store, "clip-image", &models.clip_image).await?,+ clip_text: golden_model(&store, "clip-text", &models.clip_text).await?,clip_text_vocab: resolve_model_asset(- &fetcher,- &cache_dir,+ &store,"clip-text-vocab",&models.clip_text_vocab,- )?,+ )+ .await?,})}@@ -562,7 +532,7 @@ fn repo_root() -> Result<PathBuf> {}fn cache_dir(repo_root: &Path) -> PathBuf {- repo_root.join("rust/.cache")+ repo_root.join("rust/.cache/assets")}fn load_asset_lock(repo_root: &Path) -> Result<AssetLock> {@@ -614,14 +584,21 @@ fn file_id_for_manifest_path(path: &str) -> Result<String> {.with_context(|| format!("derive file_id from manifest path '{path}'"))}-fn resolve_document_asset(- fetcher: &AssetFetcher,- cache_dir: &Path,+async fn resolve_document_asset(+ store: &AssetStore,label: &str,asset: &DocumentAsset,) -> Result<PathBuf> {- let target = cache_path_for(cache_dir, "documents", &asset.path, &asset.sha256)?;- ensure_remote_asset(fetcher, &asset.url, &asset.sha256, &target, label)+ download_asset(+ store,+ "documents",+ label,+ &asset.path,+ &asset.url,+ &asset.sha256,+ label,+ )+ .await}fn fixture_url(fixture_base_url: &str, path: &str) -> Result<String> {@@ -632,30 +609,29 @@ fn fixture_url(fixture_base_url: &str, path: &str) -> Result<String> {.to_string())}-fn fetch_fixtures(- fetcher: &AssetFetcher,- cache_dir: &Path,+async fn fetch_fixtures(+ store: &AssetStore,fixture_base_url: &str,manifest: &FixtureManifest,) -> Result<()> {- let targets = manifest- .files- .iter()- .map(|fixture| {- Ok(download::Target {- label: file_id_for_manifest_path(&fixture.path)?,- url: fixture_url(fixture_base_url, &fixture.path)?,- sha256: normalize_sha256(&fixture.sha256),- destination: cache_path_for(cache_dir, "fixtures", &fixture.path, &fixture.sha256)?,- })- })- .collect::<Result<Vec<_>>>()?;- fetcher.fetch(targets).context("fetch fixtures")+ for fixture in &manifest.files {+ let label = file_id_for_manifest_path(&fixture.path)?;+ download_asset(+ store,+ "fixtures",+ &label,+ &fixture.path,+ &fixture_url(fixture_base_url, &fixture.path)?,+ &fixture.sha256,+ &label,+ )+ .await?;+ }+ Ok(())}-fn resolve_onnx_runtime_library(- fetcher: &AssetFetcher,- cache_dir: &Path,+async fn resolve_onnx_runtime_library(+ store: &AssetStore,assets: &OnnxRuntimeAssets,) -> Result<PathBuf> {let target_key = onnx_runtime_target_key()?;@@ -672,44 +648,43 @@ fn resolve_onnx_runtime_library(})?;let archive_name = url_file_name(&archive.url)?;- let archive_path = cache_path_for(- cache_dir,- "onnx-runtime-archives",- &archive_name,- &archive.sha256,- )?;- let archive_path = ensure_remote_asset(- fetcher,+ let archive_asset = file_asset(+ "onnx-runtime",+ &target_key,+ &format!("{target_key}/{archive_name}"),&archive.url,&archive.sha256,- &archive_path,- "onnx-runtime-archive",)?;+ store+ .download(+ &archive_asset,+ |_| {},+ download::CancellationToken::default(),+ )+ .await+ .context("download ONNX Runtime archive")?;+ let archive_path = store.asset_dir(&archive_asset).join(&archive_name);let library_name = file_id_for_manifest_path(&archive.library_path)?;- let library_cache_path = format!("{target_key}/{library_name}");- let library_target = cache_path_for(- cache_dir,- "onnx-runtime-libraries",- &library_cache_path,- &archive.library_sha256,- )?;- if library_target.is_file() {- verify_file_sha256(+ let library_target = store.asset_dir(&archive_asset).join(library_name);+ if library_target.is_file()+ && verify_file_sha256(&library_target,&archive.library_sha256,"onnx-runtime-library",- )?;+ )+ .is_ok()+ {return Ok(library_target);}- extract_tgz_member(&archive_path, &archive.library_path, &library_target)- .with_context(|| format!("extract ONNX Runtime library for {target_key}"))?;- verify_file_sha256(+ extract_tgz_member(+ &archive_path,+ &archive.library_path,&library_target,&archive.library_sha256,- "onnx-runtime-library",- )?;+ )+ .with_context(|| format!("extract ONNX Runtime library for {target_key}"))?;Ok(library_target)}@@ -736,9 +711,18 @@ fn url_file_name(url: &str) -> Result<String> {.with_context(|| format!("derive file name from URL '{url}'"))}-fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) -> Result<()> {+fn extract_tgz_member(+ archive_path: &Path,+ member_path: &str,+ target: &Path,+ expected_sha256: &str,+) -> Result<()> {let expected_member = Path::new(member_path);- if expected_member.is_absolute() {+ if expected_member.is_absolute()+ || expected_member+ .components()+ .any(|component| !matches!(component, Component::CurDir | Component::Normal(_)))+ {bail!("archive member path must be relative: {member_path}");}@@ -758,9 +742,25 @@ fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) ->if entry_path.as_ref() != expected_member {continue;}- entry- .unpack(target)- .with_context(|| format!("unpack {member_path} to {}", target.display()))?;+ if !entry.header().entry_type().is_file() {+ bail!("archive member is not a regular file: {member_path}");+ }+ let temporary = PathBuf::from(format!("{}.tmp", target.display()));+ let _ = fs::remove_file(&temporary);+ let result: Result<()> = (|| {+ entry+ .unpack(&temporary)+ .with_context(|| format!("unpack {member_path} to {}", temporary.display()))?;+ verify_file_sha256(&temporary, expected_sha256, "onnx-runtime-library")?;+ let _ = fs::remove_file(target);+ fs::rename(&temporary, target)+ .with_context(|| format!("publish {}", target.display()))?;+ Ok(())+ })();+ if result.is_err() {+ let _ = fs::remove_file(temporary);+ }+ result?;return Ok(());}@@ -770,16 +770,12 @@ fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) ->);}-fn resolve_model_paths(- fetcher: &AssetFetcher,- cache_dir: &Path,- models: &ModelAssets,-) -> Result<ModelPaths> {+async fn resolve_model_paths(store: &AssetStore, models: &ModelAssets) -> Result<ModelPaths> {let face_detection =- resolve_model_asset(fetcher, cache_dir, "face-detection", &models.face_detection)?;+ resolve_model_asset(store, "face-detection", &models.face_detection).await?;let face_embedding =- resolve_model_asset(fetcher, cache_dir, "face-embedding", &models.face_embedding)?;- let clip_image = resolve_model_asset(fetcher, cache_dir, "clip-image", &models.clip_image)?;+ resolve_model_asset(store, "face-embedding", &models.face_embedding).await?;+ let clip_image = resolve_model_asset(store, "clip-image", &models.clip_image).await?;Ok(ModelPaths {face_detection: face_detection.to_string_lossy().into_owned(),@@ -795,49 +791,94 @@ fn resolve_model_paths(})}-fn resolve_model_asset(- fetcher: &AssetFetcher,- cache_dir: &Path,+async fn golden_model(+ store: &AssetStore,+ label: &str,+ model: &ModelAsset,+) -> Result<GoldenModelAsset> {+ Ok(GoldenModelAsset {+ path: download_asset(+ store,+ "models",+ label,+ &model.file_name,+ &model.url,+ &model.sha256,+ label,+ )+ .await?,+ sha256: model.sha256.clone(),+ })+}++async fn resolve_model_asset(+ store: &AssetStore,label: &str,asset: &ModelAsset,) -> Result<PathBuf> {- let target = cache_path_for(cache_dir, "models", &asset.file_name, &asset.sha256)?;- ensure_remote_asset(fetcher, &asset.url, &asset.sha256, &target, label)+ download_asset(+ store,+ "models",+ label,+ &asset.file_name,+ &asset.url,+ &asset.sha256,+ &asset.file_name,+ )+ .await}-fn ensure_remote_asset(- fetcher: &AssetFetcher,+async fn download_asset(+ store: &AssetStore,+ namespace: &str,+ key: &str,+ relative_path: &str,url: &str,expected_sha256: &str,- target: &Path,label: &str,) -> Result<PathBuf> {- fetcher- .fetch(vec![download::Target {- label: label.to_string(),- url: url.to_string(),- sha256: normalize_sha256(expected_sha256),- destination: target.to_path_buf(),- }])+ let asset = file_asset(namespace, key, relative_path, url, expected_sha256)?;+ store+ .download(&asset, |_| {}, download::CancellationToken::default())+ .await.with_context(|| format!("download {label} from {url}"))?;- Ok(target.to_path_buf())+ let name = Path::new(relative_path)+ .file_name()+ .with_context(|| format!("asset path has no file name: {relative_path}"))?;+ Ok(store.asset_dir(&asset).join(name))}-fn cache_path_for(- cache_dir: &Path,+fn file_asset(namespace: &str,+ key: &str,relative_path: &str,+ url: &str,expected_sha256: &str,-) -> Result<PathBuf> {+) -> Result<Asset> {let relative = Path::new(relative_path);if relative.is_absolute() {bail!("asset path must be relative: {relative_path}");}- let sha = normalize_sha256(expected_sha256);- let prefix = sha- .get(..16)- .with_context(|| format!("SHA-256 is too short for {relative_path}"))?;- Ok(cache_dir.join(namespace).join(prefix).join(relative))+ let sha256 = normalize_sha256(expected_sha256);+ let name = relative+ .file_name()+ .and_then(|name| name.to_str())+ .with_context(|| format!("asset path has no UTF-8 file name: {relative_path}"))?;+ if relative+ .components()+ .any(|component| !matches!(component, Component::Normal(_)))+ {+ bail!("asset path contains an unsafe component: {relative_path}");+ }+ Asset::files(+ vec![namespace.to_string(), key.to_string(), sha256.clone()],+ vec![AssetFile {+ name: name.to_string(),+ url: url.to_string(),+ sha256,+ }],+ )+ .map_err(Into::into)}fn verify_file_sha256(path: &Path, expected_sha256: &str, label: &str) -> Result<()> {