Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Unify downloaded assets

Manav Rathi16cefb5b76be
diff --git a/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xml b/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xml
index 1f4f111870..31cbb53937 100644
--- a/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xml
+++ b/mobile/native/android/apps/ensu/app/src/main/AndroidManifest.xml
@@ -27,7 +27,7 @@
</activity>
<service
- android:name=".llm.ModelDownloadJobService"
+ android:name=".assets.AssetDownloadJobService"
android:exported="false"
android:permission="android.permission.BIND_JOB_SERVICE" />
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index 6c8286de8c..b15b638e02 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -11,7 +11,7 @@ import io.ente.ensu.bindings.Transcriber
import io.ente.ensu.device.AndroidDeviceCapabilityProvider
import io.ente.ensu.llm.DownloadPhase
import io.ente.ensu.llm.LlmProvider
-import io.ente.ensu.llm.ModelDownloader
+import io.ente.ensu.assets.AssetStore
import io.ente.ensu.logging.FileLogRepository
import io.ente.ensu.chat.Attachment
import io.ente.ensu.chat.ChatMessage
@@ -35,7 +35,7 @@ class AppStore(
private val chatRepository: ChatRepository,
private val llmProvider: LlmProvider,
knowledgeProvider: KnowledgeProvider,
- val modelDownloader: ModelDownloader,
+ val assetStore: AssetStore,
val transcriber: Transcriber,
private val deviceCapabilityProvider: AndroidDeviceCapabilityProvider,
val configDefaults: ConfigDefaults,
@@ -55,14 +55,13 @@ class AppStore(
logRepository = logRepository
)
private val modelSettingsActions =
- ModelSettingsActions(_state, sessionPreferences, llmProvider, modelDownloader, logRepository, configDefaults)
+ ModelSettingsActions(_state, sessionPreferences, llmProvider, logRepository, configDefaults)
private val chatActions = ChatStoreActions(
state = _state,
sessionPreferences = sessionPreferences,
chatRepository = chatRepository,
llmProvider = llmProvider,
knowledgeProvider = knowledgeProvider,
- modelDownloader = modelDownloader,
clock = clock,
logRepository = logRepository,
messageStore = messageStore,
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
index a528fa908c..4d66c7c4c2 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
@@ -12,7 +12,7 @@ import io.ente.ensu.settings.SessionPreferencesDataStore
import io.ente.ensu.chat.ChatRepository
import io.ente.ensu.config.loadConfigDefaults
import io.ente.ensu.llm.LlmProvider
-import io.ente.ensu.llm.ModelDownloader
+import io.ente.ensu.assets.AssetStore
import io.ente.ensu.llm.ModelSettingsState
import io.ente.ensu.logging.FileLogRepository
import io.ente.ensu.storage.CredentialStore
@@ -37,11 +37,11 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {
val configDefaults = loadConfigDefaults()
val logRepository = FileLogRepository(application)
- private val modelDownloader = (application as EnsuApplication).modelDownloader
- private val _isReady = MutableStateFlow(!modelDownloader.needsMigration())
+ private val assetStore = (application as EnsuApplication).assetStore
+ private val _isReady = MutableStateFlow(!assetStore.needsMigration())
val isReady = _isReady.asStateFlow()
private val llmProvider = LlmProvider(
- downloader = modelDownloader,
+ assetStore = assetStore,
transcriber = transcriber,
deviceCapabilityProvider = deviceCapabilityProvider,
knowledgeEmbedding = configDefaults.knowledgeEmbedding
@@ -55,7 +55,7 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {
chatRepository = chatRepository,
llmProvider = llmProvider,
knowledgeProvider = knowledgeProvider,
- modelDownloader = modelDownloader,
+ assetStore = assetStore,
transcriber = transcriber,
deviceCapabilityProvider = deviceCapabilityProvider,
configDefaults = configDefaults,
@@ -69,7 +69,7 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {
runCatching {
advancedSettingsDataStore.migrateLegacyModelSelection { url, mmproj ->
withContext(Dispatchers.IO) {
- modelDownloader.migrate(url, mmproj)
+ assetStore.migrate(url, mmproj)
}
}
}
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt
index 62694336f8..e2aeb57aed 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt
@@ -2,16 +2,21 @@ package io.ente.ensu
import android.app.Application
import io.ente.ensu.bindings.Transcriber
+import io.ente.ensu.bindings.transcriptionModelAsset
+import io.ente.ensu.bindings.voiceActivityModelAsset
+import io.ente.ensu.assets.AssetStore
import io.ente.ensu.knowledge.KnowledgeProvider
-import io.ente.ensu.llm.ModelDownloader
class EnsuApplication : Application() {
- val modelDownloader by lazy { ModelDownloader(this) }
- val knowledgeProvider by lazy { KnowledgeProvider(noBackupFilesDir.resolve("knowledge")) }
+ val assetStore by lazy { AssetStore(this) }
+ val knowledgeProvider by lazy { KnowledgeProvider(assetStore) }
val transcriber by lazy {
+ val store = assetStore
+ val transcription = transcriptionModelAsset()
+ val voiceActivity = voiceActivityModelAsset()
Transcriber(
- modelDownloader.modelDir(modelDownloader.transcriptionTarget).absolutePath,
- modelDownloader.voiceActivityModelPath().absolutePath
+ store.assetDir(transcription).absolutePath,
+ store.voiceActivityModelPath(voiceActivity).absolutePath
)
}
}
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
index 734237348f..2a0c966097 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
@@ -118,7 +118,7 @@ internal fun HomeNavigation(
) {
ChatView(
chatState = appState.chat,
- modelDownloader = store.modelDownloader,
+ assetStore = store.assetStore,
transcriber = store.transcriber,
isDrawerOpen = drawerState.currentValue == DrawerValue.Open,
onMessageChange = store::updateMessageText,
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloadJobService.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetDownloadJobService.kt
similarity index 91%
rename from mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloadJobService.kt
rename to mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetDownloadJobService.kt
index aff8906d07..f68981e1f7 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloadJobService.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetDownloadJobService.kt
@@ -1,4 +1,4 @@
-package io.ente.ensu.llm
+package io.ente.ensu.assets
import android.app.Notification
import android.app.job.JobInfo
@@ -16,7 +16,7 @@ import androidx.core.app.NotificationCompat
import androidx.core.app.NotificationManagerCompat
@RequiresApi(Build.VERSION_CODES.UPSIDE_DOWN_CAKE)
-class ModelDownloadJobService : JobService() {
+class AssetDownloadJobService : JobService() {
override fun onStartJob(params: JobParameters): Boolean {
synchronized(lock) {
if (!downloadActive) return false
@@ -45,8 +45,8 @@ class ModelDownloadJobService : JobService() {
}
companion object {
- private const val TAG = "ModelDownloadJob"
- private const val CHANNEL_ID = "model-download"
+ private const val TAG = "AssetDownloadJob"
+ private const val CHANNEL_ID = "asset-download"
private const val NOTIFICATION_ID = 1
private const val JOB_ID = 1
private const val NOTIFY_INTERVAL_MS = 1000L
@@ -56,7 +56,7 @@ class ModelDownloadJobService : JobService() {
private var onCancel: (() -> Unit)? = null
private var downloadActive = false
private var lastNotifyMs = 0L
- private var runningJob: ModelDownloadJobService? = null
+ private var runningJob: AssetDownloadJobService? = null
private var runningParams: JobParameters? = null
fun attach(context: Context) {
@@ -75,7 +75,7 @@ class ModelDownloadJobService : JobService() {
Log.w(TAG, "JobScheduler unavailable")
return
}
- val job = JobInfo.Builder(JOB_ID, ComponentName(appContext, ModelDownloadJobService::class.java))
+ val job = JobInfo.Builder(JOB_ID, ComponentName(appContext, AssetDownloadJobService::class.java))
.setUserInitiated(true)
.setRequiredNetworkType(JobInfo.NETWORK_TYPE_ANY)
.build()
@@ -90,7 +90,7 @@ class ModelDownloadJobService : JobService() {
fun update(percent: Int, indeterminate: Boolean) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.UPSIDE_DOWN_CAKE) return
- val job: ModelDownloadJobService
+ val job: AssetDownloadJobService
val params: JobParameters
synchronized(lock) {
if (!downloadActive) return
@@ -110,7 +110,7 @@ class ModelDownloadJobService : JobService() {
fun end() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.UPSIDE_DOWN_CAKE) return
- val job: ModelDownloadJobService?
+ val job: AssetDownloadJobService?
val params: JobParameters?
synchronized(lock) {
downloadActive = false
@@ -134,12 +134,12 @@ class ModelDownloadJobService : JobService() {
): Notification {
NotificationManagerCompat.from(context).createNotificationChannel(
NotificationChannelCompat.Builder(CHANNEL_ID, NotificationManagerCompat.IMPORTANCE_LOW)
- .setName("Model downloads")
+ .setName("Asset downloads")
.build()
)
return NotificationCompat.Builder(context, CHANNEL_ID)
.setSmallIcon(android.R.drawable.stat_sys_download)
- .setContentTitle("Downloading model")
+ .setContentTitle("Downloading assets")
.setProgress(100, percent, indeterminate)
.setOngoing(true)
.setOnlyAlertOnce(true)
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
new file mode 100644
index 0000000000..73ab0b8db6
--- /dev/null
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/assets/AssetStore.kt
@@ -0,0 +1,135 @@
+package io.ente.ensu.assets
+
+import android.content.Context
+import android.os.Environment
+import android.util.Log
+import io.ente.ensu.bindings.AssetDownloadCallback
+import io.ente.ensu.bindings.AssetDownloadProgress
+import io.ente.ensu.bindings.Asset
+import io.ente.ensu.bindings.AssetStoreCore
+import io.ente.ensu.bindings.CancellationToken
+import io.ente.ensu.bindings.LegacyAssets
+import io.ente.ensu.bindings.KnowledgeReconciliation
+import io.ente.ensu.bindings.cleanupObsoleteKnowledgePackRevisions
+import io.ente.ensu.bindings.migrateEnsuAssets
+import io.ente.ensu.bindings.needsAssetMigration
+import io.ente.ensu.bindings.reconcileKnowledgePack
+import io.ente.ensu.bindings.uniffiEnsureInitialized
+import java.io.File
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.withContext
+
+class AssetStore(context: Context) {
+ private val appContext = context.applicationContext
+ private val assetsDir = File(appContext.noBackupFilesDir, "assets")
+ private val legacyLlmDir = appContext.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS)
+ ?.let { File(it, "llm") }
+ private val legacyTranscriptionDir = File(appContext.dataDir, "app_ensu_transcription_models")
+ private val downloadMutex = Mutex()
+ private val core: AssetStoreCore
+
+ init {
+ uniffiEnsureInitialized()
+ core = AssetStoreCore(assetsDir.absolutePath)
+ AssetDownloadJobService.attach(appContext)
+ }
+
+ fun needsMigration(): Boolean = needsAssetMigration(legacyAssets(null, null))
+
+ fun migrate(legacyModelUrl: String?, legacyMmprojUrl: String?): String? {
+ File(appContext.filesDir, "llm").deleteRecursively()
+ return migrateEnsuAssets(
+ assetsDir.absolutePath,
+ legacyAssets(legacyModelUrl, legacyMmprojUrl)
+ )
+ }
+
+ private fun legacyAssets(modelUrl: String?, mmprojUrl: String?) = LegacyAssets(
+ llmDir = legacyLlmDir?.absolutePath,
+ transcriptionDir = legacyTranscriptionDir.absolutePath,
+ modelUrl = modelUrl,
+ mmprojUrl = mmprojUrl
+ )
+
+ val isDownloadActive: Boolean get() = core.isDownloadActive()
+
+ fun assetDir(asset: Asset): File = File(core.assetDir(asset))
+
+ fun llmModelPath(asset: Asset): File? =
+ core.llmModelPath(asset)?.let(::File)
+
+ fun llmMmprojPath(asset: Asset): File? =
+ core.llmMmprojPath(asset)?.let(::File)
+
+ fun voiceActivityModelPath(asset: Asset): File = File(core.voiceActivityModelPath(asset))
+
+ fun isDownloaded(asset: Asset): Boolean = core.isDownloaded(asset)
+
+ fun removeDownloaded(asset: Asset): Boolean = core.removeDownloaded(asset)
+
+ internal fun reconcileKnowledge(stableId: String): KnowledgeReconciliation =
+ reconcileKnowledgePack(core, stableId)
+
+ internal fun cleanupKnowledgeRevisions(stableId: String, activeIdentity: String) =
+ cleanupObsoleteKnowledgePackRevisions(core, stableId, activeIdentity)
+
+ suspend fun estimateDownloadSize(asset: Asset): Long? = withContext(Dispatchers.IO) {
+ core.estimatedDownloadSize(asset)
+ }
+
+ suspend fun download(
+ assets: List<Asset>,
+ onProgress: (AssetDownloadProgress) -> Unit
+ ): Unit = withContext(Dispatchers.IO) {
+ if (!downloadMutex.tryLock()) throw DownloadAlreadyActiveException()
+ try {
+ downloadLocked(assets, onProgress)
+ } finally {
+ downloadMutex.unlock()
+ }
+ }
+
+ private suspend fun downloadLocked(
+ assets: List<Asset>,
+ onProgress: (AssetDownloadProgress) -> Unit
+ ) {
+ if (assets.all { core.isDownloaded(it) }) return
+
+ val token = CancellationToken()
+ AssetDownloadJobService.begin { token.cancel() }
+ try {
+ coroutineScope {
+ val download = async {
+ core.download(
+ assets,
+ object : AssetDownloadCallback {
+ override fun onProgress(progress: AssetDownloadProgress) {
+ progress.logLine?.let { Log.i("AssetStore", it) }
+ AssetDownloadJobService.update(
+ progress.percentage.toInt(),
+ progress.totalBytes == null
+ )
+ onProgress(progress)
+ }
+ },
+ token
+ )
+ }
+ try {
+ download.await()
+ } catch (e: CancellationException) {
+ token.cancel()
+ throw e
+ }
+ }
+ } finally {
+ AssetDownloadJobService.end()
+ }
+ }
+}
+
+private class DownloadAlreadyActiveException : Exception("An asset download is already active")
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index f36d6a082f..e5a4f0233b 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -9,11 +9,11 @@ import io.ente.ensu.llm.DownloadPhase
import io.ente.ensu.llm.LlmMessage
import io.ente.ensu.llm.LlmMessageRole
import io.ente.ensu.llm.LlmModelSelection
-import io.ente.ensu.llm.ModelDownloader
import io.ente.ensu.llm.LlmProvider
import io.ente.ensu.chat.Attachment
import io.ente.ensu.chat.ChatMessage
import io.ente.ensu.chat.ChatSession
+import io.ente.ensu.bindings.AssetDownloadException
import io.ente.ensu.bindings.DbException
import io.ente.ensu.bindings.LlmException
import io.ente.ensu.bindings.ConfigDefaults
@@ -50,7 +50,6 @@ internal class ChatStoreActions(
private val chatRepository: ChatRepository,
private val llmProvider: LlmProvider,
private val knowledgeProvider: KnowledgeProvider,
- private val modelDownloader: ModelDownloader,
private val clock: () -> Long,
private val logRepository: FileLogRepository,
private val messageStore: MutableMap<String, MutableList<ChatMessage>>,
@@ -533,7 +532,8 @@ internal class ChatStoreActions(
}
} catch (err: Throwable) {
val cancelled = err is kotlinx.coroutines.CancellationException ||
- err is LlmException.Cancelled
+ err is LlmException.Cancelled ||
+ err is AssetDownloadException.Cancelled
if (!isActive()) return@launch
streamingParentId = null
state.update { appState ->
@@ -915,7 +915,7 @@ internal class ChatStoreActions(
if (!state.value.chat.deviceCapability.isChatSupported()) {
return sessionTitleFromText(fallback, fallback = fallback)
}
- if (!modelDownloader.isDownloaded(selection.modelTarget)) {
+ if (!llmProvider.isChatModelReady(selection)) {
return sessionTitleFromText(fallback, fallback = fallback)
}
val cleanedInput = sanitizeTitleText(input)
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
index e4aee15fbe..b5f45a9a34 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
@@ -46,7 +46,7 @@ import androidx.compose.ui.platform.LocalFocusManager
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import io.ente.ensu.bindings.Transcriber
-import io.ente.ensu.llm.ModelDownloader
+import io.ente.ensu.assets.AssetStore
import io.ente.ensu.designsystem.EnsuColor
import io.ente.ensu.designsystem.EnsuSpacing
import io.ente.ensu.device.ChatDeviceCapability
@@ -59,7 +59,7 @@ import kotlinx.coroutines.delay
@Composable
fun ChatView(
chatState: ChatState,
- modelDownloader: ModelDownloader,
+ assetStore: AssetStore,
transcriber: Transcriber,
isDrawerOpen: Boolean,
onMessageChange: (String) -> Unit,
@@ -89,7 +89,7 @@ fun ChatView(
val isChatUnsupported = unsupportedCapability != null
var pendingVoiceSessionKey by remember { mutableStateOf<String?>(null) }
val voiceController = rememberVoiceTranscriptionController(
- modelDownloader = modelDownloader,
+ assetStore = assetStore,
transcriber = transcriber,
onTranscript = { transcript ->
latestOnMessageChange(appendVoiceTranscript(latestMessageText, transcript))
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
index f6efd855ce..417b699c0b 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/VoiceInput.kt
@@ -14,11 +14,13 @@ import androidx.compose.runtime.setValue
import androidx.compose.ui.platform.LocalLifecycleOwner
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleEventObserver
+import io.ente.ensu.bindings.AssetDownloadException
import io.ente.ensu.bindings.LlmException
import io.ente.ensu.bindings.Transcriber
import io.ente.ensu.bindings.TranscriptionException
-import io.ente.ensu.bindings.uniffiEnsureInitialized
-import io.ente.ensu.llm.ModelDownloader
+import io.ente.ensu.bindings.transcriptionModelAsset
+import io.ente.ensu.bindings.voiceActivityModelAsset
+import io.ente.ensu.assets.AssetStore
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -67,13 +69,13 @@ internal fun VoiceInputState.statusText(): String? = when (this) {
@Composable
internal fun rememberVoiceTranscriptionController(
- modelDownloader: ModelDownloader,
+ assetStore: AssetStore,
transcriber: Transcriber,
onTranscript: (String) -> Unit
): VoiceTranscriptionController {
val lifecycleOwner = LocalLifecycleOwner.current
val controller = remember(transcriber) {
- VoiceTranscriptionController(modelDownloader, transcriber, onTranscript)
+ VoiceTranscriptionController(assetStore, transcriber, onTranscript)
}
DisposableEffect(controller, lifecycleOwner) {
val observer = LifecycleEventObserver { _, event ->
@@ -93,13 +95,13 @@ internal fun rememberVoiceTranscriptionController(
}
internal class VoiceTranscriptionController(
- private val modelDownloader: ModelDownloader,
+ private val assetStore: AssetStore,
private val transcriber: Transcriber,
private val onTranscript: (String) -> Unit
) {
- private val modelTargets = listOf(
- modelDownloader.transcriptionTarget,
- modelDownloader.voiceActivityTarget
+ private val modelAssets = listOf(
+ transcriptionModelAsset(),
+ voiceActivityModelAsset()
)
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)
@@ -146,6 +148,13 @@ internal class VoiceTranscriptionController(
} catch (error: TranscriptionException) {
Log.w(TAG, "Voice model preparation failed: ${error.message}", error)
state = VoiceInputState.Error(transcriptionErrorMessage(error))
+ } catch (error: AssetDownloadException) {
+ if (error is AssetDownloadException.Cancelled) {
+ state = VoiceInputState.Idle
+ return@launch
+ }
+ Log.w(TAG, "Voice model download failed: ${error.message}", error)
+ state = VoiceInputState.Error(downloadErrorMessage())
} catch (error: LlmException) {
if (error is LlmException.Cancelled) {
state = VoiceInputState.Idle
@@ -276,8 +285,6 @@ internal class VoiceTranscriptionController(
ensureTranscriptionModelDownloaded()
awaitTranscriptionModelPreload()
val transcript = withContext(Dispatchers.IO) {
- uniffiEnsureInitialized()
-
withContext(Dispatchers.Main.immediate) {
state = VoiceInputState.Transcribing
}
@@ -301,6 +308,13 @@ internal class VoiceTranscriptionController(
} catch (error: TranscriptionException) {
Log.w(TAG, "Voice transcription failed: ${error.message}", error)
state = VoiceInputState.Error(transcriptionErrorMessage(error))
+ } catch (error: AssetDownloadException) {
+ if (error is AssetDownloadException.Cancelled) {
+ state = VoiceInputState.Idle
+ return
+ }
+ Log.w(TAG, "Voice model download failed: ${error.message}", error)
+ state = VoiceInputState.Error(downloadErrorMessage())
} catch (error: LlmException) {
if (error is LlmException.Cancelled) {
state = VoiceInputState.Idle
@@ -322,7 +336,6 @@ internal class VoiceTranscriptionController(
transcriptionPreloadJob?.cancel()
transcriptionPreloadJob = scope.launch(Dispatchers.IO) {
try {
- uniffiEnsureInitialized()
transcriber.loadModel()
} catch (error: CancellationException) {
throw error
@@ -345,15 +358,17 @@ internal class VoiceTranscriptionController(
private suspend fun ensureTranscriptionModelDownloaded() {
val activeJob = coroutineContext[Job]
- uniffiEnsureInitialized()
- if (withContext(Dispatchers.IO) { modelTargets.all { modelDownloader.isDownloaded(it) } }) {
+ if (withContext(Dispatchers.IO) { modelAssets.all { assetStore.isDownloaded(it) } }) {
return
}
setStateIfJobActive(activeJob, VoiceInputState.Downloading(null))
- modelDownloader.download(modelTargets) { progress ->
- postStateIfJobActive(activeJob, VoiceInputState.Downloading(progress.percent?.coerceIn(0, 100)))
+ assetStore.download(modelAssets) { progress ->
+ postStateIfJobActive(
+ activeJob,
+ VoiceInputState.Downloading(progress.percentage.toInt().coerceIn(0, 100))
+ )
}
}
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt
index e00084e202..f490c07af8 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt
@@ -1,16 +1,11 @@
package io.ente.ensu.knowledge
-import io.ente.ensu.bindings.CancellationToken
+import io.ente.ensu.assets.AssetStore
import io.ente.ensu.bindings.KnowledgeDatasetConfig
-import io.ente.ensu.bindings.KnowledgeDownloadCallback
-import io.ente.ensu.bindings.KnowledgeDownloadProgress
import io.ente.ensu.bindings.KnowledgePromptHit
import io.ente.ensu.bindings.KnowledgeReconciliation
import io.ente.ensu.bindings.RetrievalIndex
-import io.ente.ensu.bindings.cleanupObsoleteKnowledgePackRevisions
-import io.ente.ensu.bindings.downloadKnowledgePack
-import io.ente.ensu.bindings.reconcileKnowledgePack
-import java.io.File
+import io.ente.ensu.bindings.knowledgePackAsset
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers
@@ -24,10 +19,9 @@ import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
class KnowledgeProvider(
- private val knowledgeRoot: File
+ private val assetStore: AssetStore
) {
private data class ActiveMutation(
- val cancellation: CancellationToken,
val task: Deferred<KnowledgeReconciliation>
)
@@ -74,19 +68,11 @@ class KnowledgeProvider(
mutationGate.withLock {
mutations[dataset.stableId] ?: run {
ownsMutation = true
- val cancellation = CancellationToken()
val task = async(Dispatchers.IO) {
try {
- downloadKnowledgePack(
- packRoot(dataset).absolutePath,
- dataset.stableId,
- object : KnowledgeDownloadCallback {
- override fun onProgress(progress: KnowledgeDownloadProgress) {
- onProgress(progress)
- }
- },
- cancellation
- )
+ assetStore.download(listOf(knowledgePackAsset(dataset.stableId))) {
+ onProgress(KnowledgeDownloadProgress(it.label, it.percentage))
+ }
indexGate.withLock {
val result = reconcileAndOpenLocked(dataset)
check(result.activeIdentity == dataset.currentDownloadIdentity) {
@@ -97,14 +83,13 @@ class KnowledgeProvider(
} catch (error: Throwable) {
withContext(NonCancellable) {
indexGate.withLock {
- incomingRevision(dataset).deleteRecursively()
runCatching { reconcileAndOpenLocked(dataset) }
}
}
throw error
}
}
- ActiveMutation(cancellation, task).also {
+ ActiveMutation(task).also {
mutations[dataset.stableId] = it
}
}
@@ -114,7 +99,7 @@ class KnowledgeProvider(
try {
mutation.task.await()
} catch (error: CancellationException) {
- if (ownsMutation) mutation.cancellation.cancel()
+ if (ownsMutation) mutation.task.cancel()
throw error
} finally {
withContext(NonCancellable) {
@@ -132,7 +117,7 @@ class KnowledgeProvider(
suspend fun cancel(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {
val mutation = mutationGate.withLock { mutations[dataset.stableId] }
- mutation?.cancellation?.cancel()
+ mutation?.task?.cancel()
runCatching { mutation?.task?.await() }
mutationGate.withLock {
if (mutations[dataset.stableId] === mutation) {
@@ -166,8 +151,7 @@ class KnowledgeProvider(
private fun reconcileAndOpenLocked(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {
val previous = indexes[dataset.stableId]
- val root = packRoot(dataset).absolutePath
- val result = reconcileKnowledgePack(root, dataset.stableId)
+ val result = assetStore.reconcileKnowledge(dataset.stableId)
val directory = result.activeDirectory
if (directory == null) {
indexes.remove(dataset.stableId)
@@ -179,18 +163,17 @@ class KnowledgeProvider(
}
result.activeIdentity?.let { activeIdentity ->
runCatching {
- cleanupObsoleteKnowledgePackRevisions(root, dataset.stableId, activeIdentity)
+ assetStore.cleanupKnowledgeRevisions(dataset.stableId, activeIdentity)
}
}
return result
}
- private fun packRoot(dataset: KnowledgeDatasetConfig): File =
- File(knowledgeRoot, dataset.stableId)
-
- private fun incomingRevision(dataset: KnowledgeDatasetConfig): File =
- File(packRoot(dataset), dataset.currentDownloadIdentity)
-
private suspend fun lifecycleGate(stableId: String): Mutex =
mutationGate.withLock { lifecycleGates.getOrPut(stableId) { Mutex() } }
}
+
+data class KnowledgeDownloadProgress(
+ val label: String,
+ val percentage: Double
+)
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt
index 165d8fbe18..48ecd90754 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt
@@ -5,10 +5,8 @@ import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.stringSetPreferencesKey
import androidx.datastore.preferences.preferencesDataStore
import io.ente.ensu.AppState
-import io.ente.ensu.bindings.DownloadError
+import io.ente.ensu.bindings.AssetDownloadException
import io.ente.ensu.bindings.KnowledgeDatasetConfig
-import io.ente.ensu.bindings.KnowledgeDownloadException
-import io.ente.ensu.bindings.KnowledgeDownloadProgress
import io.ente.ensu.bindings.KnowledgeReconciliation
import io.ente.ensu.bindings.KnowledgeReconciliationStatus
import io.ente.ensu.device.isChatSupported
@@ -225,23 +223,22 @@ class KnowledgeStore(
)
private fun userFacingKnowledgeError(error: Throwable): String {
- val downloadError = (error as? KnowledgeDownloadException.Download)?.error
- return when (downloadError) {
- DownloadError.Cancelled -> "Download cancelled"
- DownloadError.StorageFull ->
+ return when (error) {
+ is AssetDownloadException.Cancelled -> "Download cancelled"
+ is AssetDownloadException.StorageFull ->
"Not enough storage space to download this knowledge pack."
- is DownloadError.Network ->
+ is AssetDownloadException.Network ->
"Couldn't download the knowledge pack. Check your connection and try again."
- is DownloadError.Http ->
+ is AssetDownloadException.Http ->
"The knowledge pack is currently unavailable. Please try again later."
- is DownloadError.Validation,
- is DownloadError.SizeMismatch,
- is DownloadError.Protocol,
- is DownloadError.InvalidTarget ->
+ is AssetDownloadException.Validation,
+ is AssetDownloadException.SizeMismatch,
+ is AssetDownloadException.Protocol,
+ is AssetDownloadException.InvalidTarget ->
"The knowledge pack couldn't be verified. Please try again."
- is DownloadError.Io ->
+ is AssetDownloadException.Io ->
"Couldn't save the knowledge pack. Please try again."
- null -> "Knowledge pack setup failed. Please try again."
+ else -> "Knowledge pack setup failed. Please try again."
}
}
}
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt
index 0b76ba6cf6..ed478e9941 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt
@@ -1,10 +1,7 @@
package io.ente.ensu.llm
-import io.ente.ensu.bindings.ModelTarget
-
data class LlmModelSelection(
val id: String,
- val modelTarget: ModelTarget,
val contextLength: Int? = null,
val maxTokens: Int? = null
)
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
index e24386fdcf..84177f186a 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
@@ -1,6 +1,8 @@
package io.ente.ensu.llm
import android.util.Log
+import io.ente.ensu.assets.AssetStore
+import io.ente.ensu.bindings.Asset
import io.ente.ensu.bindings.LlmChatMessage as NativeChatMessage
import io.ente.ensu.bindings.LlmChatRequest
import io.ente.ensu.bindings.LlmContext
@@ -12,7 +14,8 @@ import io.ente.ensu.bindings.LlmModel
import io.ente.ensu.bindings.LlmModelLoadParams
import io.ente.ensu.bindings.KnowledgeEmbeddingConfig
import io.ente.ensu.bindings.Transcriber
-import io.ente.ensu.bindings.knowledgeEmbeddingModelTarget
+import io.ente.ensu.bindings.knowledgeEmbeddingModelAsset
+import io.ente.ensu.bindings.llmAsset
import io.ente.ensu.bindings.llmCancel
import io.ente.ensu.bindings.llmInitBackend
import io.ente.ensu.device.AndroidDeviceCapabilityProvider
@@ -26,11 +29,11 @@ import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
class RequiredModelValidationError(
- val targetId: String
-) : Exception("Downloaded model failed validation: $targetId")
+ val modelId: String
+) : Exception("Downloaded model failed validation: $modelId")
class LlmProvider(
- private val downloader: ModelDownloader,
+ private val assetStore: AssetStore,
private val transcriber: Transcriber,
private val deviceCapabilityProvider: AndroidDeviceCapabilityProvider,
private val knowledgeEmbedding: KnowledgeEmbeddingConfig,
@@ -48,18 +51,24 @@ class LlmProvider(
@Volatile private var currentJobId: Long? = null
private var backendInitialized = false
private val modelLoadMutex = Mutex()
- private val embeddingDownloadTarget = knowledgeEmbeddingModelTarget()
+ private val embeddingAsset = knowledgeEmbeddingModelAsset()
class EmbeddingAssetInvalid : Exception("Embedding model asset is invalid")
fun isChatModelReady(selection: LlmModelSelection): Boolean =
- downloader.isDownloaded(selection.modelTarget)
+ assetStore.isDownloaded(chatAsset(selection))
fun isEmbeddingModelReady(): Boolean =
- downloader.isDownloaded(embeddingDownloadTarget)
+ assetStore.isDownloaded(embeddingAsset)
suspend fun estimateEmbeddingDownloadSize(): Long? =
- downloader.estimateDownloadSize(embeddingDownloadTarget)
+ assetStore.estimateDownloadSize(embeddingAsset)
+
+ suspend fun estimateChatModelDownloadSize(selection: LlmModelSelection): Long? =
+ assetStore.estimateDownloadSize(chatAsset(selection))
+
+ val isDownloadActive: Boolean
+ get() = assetStore.isDownloadActive
suspend fun ensureModelReady(
selection: LlmModelSelection,
@@ -78,29 +87,30 @@ class LlmProvider(
) {
withContext(ioDispatcher) {
deviceCapabilityProvider.chatCapability().requireChatSupported()
- val missingTargets = modelLoadMutex.withLock {
+ val asset = chatAsset(selection)
+ val missingAssets = modelLoadMutex.withLock {
val embeddingReady = isEmbeddingModelReady()
if (IS_ENSU_PACKS_ENABLED && !embeddingReady) {
- val embeddingFile = downloader.llmModelPath(embeddingDownloadTarget)
+ val embeddingFile = assetStore.llmModelPath(embeddingAsset)
if (embeddingFile?.exists() == true) {
- downloader.removeDownloaded(embeddingDownloadTarget)
+ assetStore.removeDownloaded(embeddingAsset)
}
}
buildList {
- if (!isChatModelReady(selection)) add(selection.modelTarget)
- if (IS_ENSU_PACKS_ENABLED && !isEmbeddingModelReady()) add(embeddingDownloadTarget)
+ if (!assetStore.isDownloaded(asset)) add(asset)
+ if (IS_ENSU_PACKS_ENABLED && !isEmbeddingModelReady()) add(embeddingAsset)
}
}
- if (missingTargets.isNotEmpty()) {
- downloader.download(missingTargets, onProgress)
+ if (missingAssets.isNotEmpty()) {
+ downloadAssets(missingAssets, onProgress)
}
modelLoadMutex.withLock {
if (IS_ENSU_PACKS_ENABLED && !isEmbeddingModelReady()) {
- downloader.removeDownloaded(embeddingDownloadTarget)
+ assetStore.removeDownloaded(embeddingAsset)
throw RequiredModelValidationError(knowledgeEmbedding.targetId)
}
- if (!isChatModelReady(selection)) {
+ if (!assetStore.isDownloaded(asset)) {
throw RequiredModelValidationError(selection.id)
}
ensureModelReadyLocked(
@@ -128,7 +138,7 @@ class LlmProvider(
val mmprojPath = if (imageFiles.isEmpty()) {
null
} else {
- downloader.llmMmprojPath(selection.modelTarget)?.absolutePath
+ assetStore.llmMmprojPath(chatAsset(selection))?.absolutePath
}
val clampedTemperature = temperature.coerceIn(0.35f, 0.7f)
@@ -174,7 +184,7 @@ class LlmProvider(
val embeddingModel = LlmModel.load(
LlmModelLoadParams(
- modelPath = requireNotNull(downloader.llmModelPath(embeddingDownloadTarget)).absolutePath,
+ modelPath = requireNotNull(assetStore.llmModelPath(embeddingAsset)).absolutePath,
nGpuLayers = 0,
useMmap = true,
useMlock = false
@@ -196,9 +206,10 @@ class LlmProvider(
withContext(ioDispatcher) {
runCatching {
modelLoadMutex.withLock {
- if (!downloader.isDownloaded(selection.modelTarget)) return@withLock
+ val asset = chatAsset(selection)
+ if (!assetStore.isDownloaded(asset)) return@withLock
val mmprojPath =
- downloader.llmMmprojPath(selection.modelTarget)?.absolutePath
+ assetStore.llmMmprojPath(asset)?.absolutePath
?.takeIf { File(it).exists() }
?: return@withLock
ensureModelReadyLocked(selection, onProgress = {})
@@ -289,19 +300,20 @@ class LlmProvider(
unloadModel()
- val wasAlreadyDownloaded = downloader.isDownloaded(selection.modelTarget)
+ val asset = chatAsset(selection)
+ val wasAlreadyDownloaded = assetStore.isDownloaded(asset)
if (shouldDownload) {
- downloader.download(listOf(selection.modelTarget), onProgress)
+ downloadAssets(listOf(asset), onProgress)
}
onProgress(DownloadProgress(100, "Loading model...", phase = DownloadPhase.Loading))
try {
loadWithFallbacks(
selection,
- requireNotNull(downloader.llmModelPath(selection.modelTarget))
+ requireNotNull(assetStore.llmModelPath(asset))
)
} catch (error: Throwable) {
- if (allowRecovery && wasAlreadyDownloaded && downloader.removeDownloaded(selection.modelTarget)) {
+ if (allowRecovery && wasAlreadyDownloaded && assetStore.removeDownloaded(asset)) {
onProgress(DownloadProgress(0, "Starting download...", phase = DownloadPhase.Downloading))
ensureModelReadyLocked(selection, onProgress, allowRecovery = false)
return
@@ -311,6 +323,22 @@ class LlmProvider(
onProgress(DownloadProgress(100, "Ready", phase = DownloadPhase.Ready))
}
+ private suspend fun downloadAssets(
+ assets: List<Asset>,
+ onProgress: (DownloadProgress) -> Unit
+ ) {
+ assetStore.download(assets) { progress ->
+ onProgress(
+ DownloadProgress(
+ progress.percentage.toInt().coerceIn(0, 99),
+ progress.status
+ )
+ )
+ }
+ }
+
+ private fun chatAsset(selection: LlmModelSelection): Asset = llmAsset(selection.id)
+
private fun loadWithFallbacks(selection: LlmModelSelection, modelFile: File) {
val desiredCtx = selection.contextLength ?: 12000
val contexts = listOf(desiredCtx, 12000, 8192, 4096, 2048, 1024).distinct().filter { it > 0 }
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloader.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloader.kt
deleted file mode 100644
index 1d0a6db553..0000000000
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelDownloader.kt
+++ /dev/null
@@ -1,126 +0,0 @@
-package io.ente.ensu.llm
-
-import android.content.Context
-import android.os.Environment
-import android.util.Log
-import io.ente.ensu.bindings.CancellationToken
-import io.ente.ensu.bindings.ModelDownloadCallback
-import io.ente.ensu.bindings.ModelDownloadCore
-import io.ente.ensu.bindings.ModelDownloadProgress
-import io.ente.ensu.bindings.LlmException
-import io.ente.ensu.bindings.ModelTarget
-import io.ente.ensu.bindings.LegacyModels
-import io.ente.ensu.bindings.migrateMobileModels
-import io.ente.ensu.bindings.transcriptionModelTarget
-import io.ente.ensu.bindings.voiceActivityModelTarget
-import io.ente.ensu.bindings.uniffiEnsureInitialized
-import java.io.File
-import kotlinx.coroutines.CancellationException
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.async
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.withContext
-
-class ModelDownloader(context: Context) {
- private val appContext = context.applicationContext
- private val modelsDir = File(appContext.noBackupFilesDir, "models")
- private val legacyDir = appContext.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS)
- ?.let { File(it, "llm") }
- private val legacyTranscriptionDir = File(appContext.dataDir, "app_ensu_transcription_models")
- private val downloadMutex = Mutex()
- private val core: ModelDownloadCore
- val transcriptionTarget: ModelTarget
- val voiceActivityTarget: ModelTarget
-
- init {
- uniffiEnsureInitialized()
- ModelDownloadJobService.attach(appContext)
- core = ModelDownloadCore(modelsDir.absolutePath)
- transcriptionTarget = transcriptionModelTarget()
- voiceActivityTarget = voiceActivityModelTarget()
- }
-
- fun needsMigration(): Boolean =
- legacyDir?.exists() == true || legacyTranscriptionDir.exists()
-
- fun migrate(legacyModelUrl: String?, legacyMmprojUrl: String?): String? {
- File(appContext.filesDir, "llm").deleteRecursively()
- return migrateMobileModels(
- modelsDir.absolutePath,
- LegacyModels(
- legacyDir?.absolutePath,
- legacyTranscriptionDir.absolutePath,
- legacyModelUrl,
- legacyMmprojUrl
- )
- )
- }
-
- val isDownloadActive: Boolean get() = core.isDownloadActive()
-
- fun modelDir(target: ModelTarget): File = File(core.modelDir(target))
-
- fun llmModelPath(target: ModelTarget): File? =
- core.llmModelPath(target)?.let(::File)
-
- fun llmMmprojPath(target: ModelTarget): File? =
- core.llmMmprojPath(target)?.let(::File)
-
- fun voiceActivityModelPath(): File = File(core.voiceActivityModelPath())
-
- fun isDownloaded(target: ModelTarget): Boolean = core.isDownloaded(target)
-
- fun removeDownloaded(target: ModelTarget): Boolean = core.removeDownloaded(target)
-
- suspend fun estimateDownloadSize(target: ModelTarget): Long? = withContext(Dispatchers.IO) {
- core.estimatedDownloadSize(target)
- }
-
- suspend fun download(
- targets: List<ModelTarget>,
- onProgress: (DownloadProgress) -> Unit
- ): Unit = withContext(Dispatchers.IO) {
- if (!downloadMutex.tryLock()) throw LlmException.Cancelled()
- try {
- downloadLocked(targets, onProgress)
- } finally {
- downloadMutex.unlock()
- }
- }
-
- private suspend fun downloadLocked(
- targets: List<ModelTarget>,
- onProgress: (DownloadProgress) -> Unit
- ) {
- if (targets.all { core.isDownloaded(it) }) return
-
- val token = CancellationToken()
- ModelDownloadJobService.begin { token.cancel() }
- try {
- coroutineScope {
- val download = async {
- core.download(
- targets,
- object : ModelDownloadCallback {
- override fun onProgress(progress: ModelDownloadProgress) {
- progress.logLine?.let { Log.i("ModelDownloader", it) }
- ModelDownloadJobService.update(progress.percent, progress.totalBytes == null)
- onProgress(DownloadProgress(progress.percent, progress.status))
- }
- },
- token
- )
- }
- try {
- download.await()
- } catch (e: CancellationException) {
- token.cancel()
- throw e
- }
- }
- } finally {
- ModelDownloadJobService.end()
- }
- }
-}
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index 06b41a85ad..6c1b954343 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -3,10 +3,9 @@ package io.ente.ensu.llm
import android.system.ErrnoException
import android.system.OsConstants
import io.ente.ensu.AppState
+import io.ente.ensu.bindings.AssetDownloadException
import io.ente.ensu.bindings.ConfigDefaults
-import io.ente.ensu.bindings.DownloadError
import io.ente.ensu.bindings.LlmException
-import io.ente.ensu.bindings.mobileLlmTarget
import io.ente.ensu.device.isChatSupported
import io.ente.ensu.logging.FileLogRepository
import io.ente.ensu.logging.LogLevel
@@ -23,7 +22,6 @@ internal class ModelSettingsActions(
private val state: MutableStateFlow<AppState>,
private val sessionPreferences: SessionPreferencesDataStore,
private val llmProvider: LlmProvider,
- private val modelDownloader: ModelDownloader,
private val logRepository: FileLogRepository,
private val configDefaults: ConfigDefaults
) {
@@ -104,7 +102,7 @@ internal class ModelSettingsActions(
val scope = scope ?: return
scope.launch {
- if (!modelDownloader.isDownloadActive && modelDownloadJob?.isActive != true) {
+ if (!llmProvider.isDownloadActive && modelDownloadJob?.isActive != true) {
persistModelDownloadRequested(false)
state.update { appState ->
appState.copy(
@@ -119,7 +117,7 @@ internal class ModelSettingsActions(
}
}
- val chatSize = if (chatReady) 0L else modelDownloader.estimateDownloadSize(selection.modelTarget)
+ val chatSize = if (chatReady) 0L else llmProvider.estimateChatModelDownloadSize(selection)
val embeddingSize = if (!IS_ENSU_PACKS_ENABLED || embeddingReady) {
0L
} else {
@@ -231,7 +229,8 @@ internal class ModelSettingsActions(
}
} catch (err: Throwable) {
val cancelled = err is kotlinx.coroutines.CancellationException ||
- err is LlmException.Cancelled
+ err is LlmException.Cancelled ||
+ err is AssetDownloadException.Cancelled
val failureMessage = if (cancelled) {
"Download cancelled"
} else {
@@ -276,7 +275,7 @@ internal class ModelSettingsActions(
if (!currentState.chat.deviceCapability.isChatSupported()) return
val selection = resolveSelection(currentState.modelSettings)
- if (!modelDownloader.isDownloaded(selection.modelTarget)) return
+ if (!llmProvider.isChatModelReady(selection)) return
scope.launch {
try {
@@ -335,7 +334,6 @@ internal class ModelSettingsActions(
return LlmModelSelection(
id = preset.id,
- modelTarget = mobileLlmTarget(preset.id),
contextLength = contextLength,
maxTokens = maxTokens
)
@@ -366,14 +364,14 @@ internal class ModelSettingsActions(
if (retryCount >= MAX_DOWNLOAD_RETRIES) return false
if (err is kotlinx.coroutines.CancellationException) return false
if (err is LlmException.Cancelled) return false
+ if (err is AssetDownloadException.Cancelled) return false
if (isOutOfStorageError(err)) return false
if (err is RequiredModelValidationError) return false
- if (err is LlmException.Download) {
- when (val error = err.error) {
- is DownloadError.Validation -> return false
- is DownloadError.Http -> if (error.status.toInt() in NON_RETRYABLE_HTTP) return false
- else -> {}
- }
+ when (err) {
+ is AssetDownloadException.Validation -> return false
+ is AssetDownloadException.Http ->
+ if (err.status.toInt() in NON_RETRYABLE_HTTP) return false
+ else -> {}
}
return true
}
@@ -393,7 +391,7 @@ internal class ModelSettingsActions(
private fun isOutOfStorageError(err: Throwable): Boolean {
var current: Throwable? = err
while (current != null) {
- if (current is LlmException.Download && current.error is DownloadError.StorageFull) {
+ if (current is AssetDownloadException.StorageFull) {
return true
}
if (current is ErrnoException && current.errno == OsConstants.ENOSPC) return true
diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index 78e99fb3aa..35f7dd027f 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -77,7 +77,6 @@ final class ChatViewModel: ObservableObject {
let knowledgeStore: KnowledgeStore
private let provider: LlmProvider
- private let downloader: ModelDownloader
private let knowledgeEmbedding: KnowledgeEmbeddingConfig
private let knowledgeProvider: KnowledgeProvider
private let voiceTranscriber: VoiceTranscriptionService
@@ -108,7 +107,7 @@ final class ChatViewModel: ObservableObject {
private var pendingOverflow: PendingOverflow?
private var overflowBypassMessageId: UUID?
- init() {
+ init(assetStore: AssetStore) {
logger.info("Initializing")
let summaries = Self.loadSessionSummaries().reduce(into: [String: String]()) { result, item in
result[item.key.lowercased()] = item.value
@@ -117,16 +116,17 @@ final class ChatViewModel: ObservableObject {
?? FileManager.default.temporaryDirectory
let config = ConfigDefaults.shared
- let downloader = ModelDownloader()
- // Must initialize after the downloader migrates the persisted selection.
+ // Must initialize after the asset store migrates the persisted selection.
self.modelSettings = ModelSettingsStore.shared
+ let transcription = transcriptionModelAsset()
+ let voiceActivity = voiceActivityModelAsset()
let transcriber = Transcriber(
- modelDir: downloader.modelDir(downloader.transcriptionTarget).path,
- vadModelPath: downloader.voiceActivityModelPath().path
+ modelDir: assetStore.assetDir(transcription).path,
+ vadModelPath: assetStore.voiceActivityModelPath(voiceActivity).path
)
- let provider = LlmProvider(downloader: downloader, transcriber: transcriber, knowledgeEmbedding: config.knowledgeEmbedding)
- let voiceTranscriber = VoiceTranscriptionService(transcriber: transcriber, downloader: downloader)
- let knowledgeProvider = KnowledgeProvider(root: baseDir.appendingPathComponent("knowledge", isDirectory: true))
+ let provider = LlmProvider(assetStore: assetStore, transcriber: transcriber, knowledgeEmbedding: config.knowledgeEmbedding)
+ let voiceTranscriber = VoiceTranscriptionService(transcriber: transcriber, assetStore: assetStore)
+ let knowledgeProvider = KnowledgeProvider(assetStore: assetStore)
let knowledgeStore = KnowledgeStore(datasets: config.knowledgeDatasets, provider: knowledgeProvider)
// Chat DB + attachments.
@@ -166,7 +166,6 @@ final class ChatViewModel: ObservableObject {
// Stored properties.
self.provider = provider
- self.downloader = downloader
self.knowledgeEmbedding = config.knowledgeEmbedding
self.knowledgeProvider = knowledgeProvider
self.knowledgeStore = knowledgeStore
@@ -223,7 +222,6 @@ final class ChatViewModel: ObservableObject {
editingMessageId = nil
draftText = ""
draftAttachments = []
- downloader.cancel()
}
func dismissUnsupportedDeviceDialog() {
@@ -475,7 +473,7 @@ final class ChatViewModel: ObservableObject {
guard !isGenerating && !isDownloading else { return }
guard !isChatUnsupported else { return }
let selection = modelSettings.currentSelection()
- guard downloader.isDownloaded(selection.modelTarget) else { return }
+ guard provider.isChatModelReady(selection) else { return }
Task { [weak self] in
guard let self else { return }
@@ -766,7 +764,6 @@ final class ChatViewModel: ObservableObject {
if let existingTask = sharedModelReadyTask, sharedModelReadyKey != modelKey {
existingTask.cancel()
- downloader.cancel()
clearSharedModelReadyTask()
}
@@ -905,7 +902,6 @@ final class ChatViewModel: ObservableObject {
modelDownloadTask = nil
sharedModelReadyTask?.cancel()
clearSharedModelReadyTask()
- downloader.cancel()
if modelDownloadLoggedStart {
logger.info("Model download cancelled")
modelDownloadLoggedStart = false
@@ -1058,7 +1054,6 @@ final class ChatViewModel: ObservableObject {
} catch {
if isCancellation(error) {
self.sharedModelReadyTask?.cancel()
- self.downloader.cancel()
self.clearSharedModelReadyTask()
if self.activeGenerationId == generationId {
isGenerating = false
@@ -1077,7 +1072,7 @@ final class ChatViewModel: ObservableObject {
downloadToast = DownloadToastState(
phase: .errorDownload,
percent: nil,
- status: self.userFacingModelReadyError(error, wasDownloaded: self.downloader.isDownloaded(selection.modelTarget)),
+ status: self.userFacingModelReadyError(error, wasDownloaded: self.provider.isChatModelReady(selection)),
offerRetryDownload: true
)
activeGenerationId = nil
@@ -1887,6 +1882,7 @@ final class ChatViewModel: ObservableObject {
private func isCancellation(_ error: Error) -> Bool {
if error is CancellationError { return true }
if case LlmError.Cancelled = error { return true }
+ if case AssetDownloadError.Cancelled = error { return true }
return (error as? URLError)?.code == .cancelled
}
@@ -1906,15 +1902,13 @@ final class ChatViewModel: ObservableObject {
return false
}
- if case let LlmError.Download(inner) = error {
- switch inner {
- case .validation:
- return false
- case let .http(status):
- if status == 401 || status == 403 || status == 404 { return false }
- default:
- break
- }
+ switch error {
+ case AssetDownloadError.Validation:
+ return false
+ case let AssetDownloadError.Http(status):
+ if status == 401 || status == 403 || status == 404 { return false }
+ default:
+ break
}
return true
@@ -1938,7 +1932,7 @@ final class ChatViewModel: ObservableObject {
}
private func isOutOfStorageError(_ error: Error) -> Bool {
- if case LlmError.Download(.storageFull) = error { return true }
+ if case AssetDownloadError.StorageFull = error { return true }
return error.isOutOfDiskSpace
}
diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
index 9ba888a0c5..7dac4dbfa4 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/VoiceTranscriptionService.swift
@@ -83,7 +83,8 @@ final class VoiceTranscriptionService {
typealias TranscriptHandler = @MainActor @Sendable (String) -> Void
private let transcriber: Transcriber
- private let downloader: ModelDownloader
+ private let assetStore: AssetStore
+ private let modelAssets: [Asset]
private var transcriptionTask: Task<Void, Never>?
private var preloadTask: Task<Void, Never>?
private var activeVoiceTaskId = UUID()
@@ -91,9 +92,10 @@ final class VoiceTranscriptionService {
private let recorder = PcmAudioRecorder()
- init(transcriber: Transcriber, downloader: ModelDownloader) {
+ init(transcriber: Transcriber, assetStore: AssetStore) {
self.transcriber = transcriber
- self.downloader = downloader
+ self.assetStore = assetStore
+ self.modelAssets = [transcriptionModelAsset(), voiceActivityModelAsset()]
}
func startRecording(
@@ -143,8 +145,6 @@ final class VoiceTranscriptionService {
let taskId = beginVoiceTask()
let downloadId = beginDownload(taskId: taskId)
let transcriber = transcriber
- let downloader = downloader
- let targets = [downloader.transcriptionTarget, downloader.voiceActivityTarget]
let sampleRate = recording.sampleRate
let pcm = recording.pcm
@@ -188,6 +188,7 @@ final class VoiceTranscriptionService {
} catch is CancellationError {
return
} catch {
+ if case AssetDownloadError.Cancelled = error { return }
if case LlmError.Cancelled = error { return }
await MainActor.run { [weak self] in
self?.finishDownload(downloadId: downloadId)
@@ -239,6 +240,7 @@ final class VoiceTranscriptionService {
} catch is CancellationError {
return
} catch {
+ if case AssetDownloadError.Cancelled = error { return }
if case LlmError.Cancelled = error { return }
await MainActor.run { [weak self] in
self?.finishDownload(downloadId: downloadId)
@@ -254,17 +256,17 @@ final class VoiceTranscriptionService {
downloadId: UUID,
onState: @escaping StateHandler
) async throws {
- let targets = await [downloader.transcriptionTarget, downloader.voiceActivityTarget]
- let downloader = self.downloader
- if targets.allSatisfy({ downloader.isDownloaded($0) }) {
+ let assets = await modelAssets
+ let assetStore = self.assetStore
+ if assets.allSatisfy({ assetStore.isDownloaded($0) }) {
return
}
await MainActor.run { [weak self] in
guard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else { return }
onState(.downloading(percent: nil))
}
- try await downloader.download(targets: targets) { [weak self] progress in
- let percent = progress.percent.map { min(max($0, 0), 100) }
+ try await assetStore.download(assets: assets) { [weak self] progress in
+ let percent = min(max(Int(progress.percentage), 0), 100)
Task { @MainActor [weak self] in
guard self?.isDownloadActive(taskId: taskId, downloadId: downloadId) == true else { return }
onState(.downloading(percent: percent))
diff --git a/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swift b/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swift
index 13f1a0ca9a..fc391fa240 100644
--- a/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/EnsuApp.swift
@@ -2,14 +2,17 @@ import SwiftUI
@main
struct EnsuApp: App {
+ private let assetStore: AssetStore
+
init() {
EnsuLogging.shared.start()
- ModelDownloader.registerBackgroundTask()
+ AssetStore.registerBackgroundTask()
+ assetStore = AssetStore()
}
var body: some Scene {
WindowGroup {
- RootView()
+ RootView(assetStore: assetStore)
}
}
}
diff --git a/mobile/native/apple/apps/ensu/Ensu/HomeView.swift b/mobile/native/apple/apps/ensu/Ensu/HomeView.swift
index 06a3569ef5..f2b9d13997 100644
--- a/mobile/native/apple/apps/ensu/Ensu/HomeView.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/HomeView.swift
@@ -1,7 +1,11 @@
import SwiftUI
struct HomeView: View {
- @StateObject private var chatModel = ChatViewModel()
+ @StateObject private var chatModel: ChatViewModel
+
+ init(assetStore: AssetStore) {
+ _chatModel = StateObject(wrappedValue: ChatViewModel(assetStore: assetStore))
+ }
var body: some View {
ChatView(viewModel: chatModel)
diff --git a/mobile/native/apple/apps/ensu/Ensu/Info.plist b/mobile/native/apple/apps/ensu/Ensu/Info.plist
index 138f535fb6..083557b865 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Info.plist
+++ b/mobile/native/apple/apps/ensu/Ensu/Info.plist
@@ -64,7 +64,7 @@
<key>BGTaskSchedulerPermittedIdentifiers</key>
<array>
- <string>io.ente.ensu.model-download</string>
+ <string>io.ente.ensu.asset-download</string>
</array>
<key>NSAppTransportSecurity</key>
diff --git a/mobile/native/apple/apps/ensu/Ensu/RootView.swift b/mobile/native/apple/apps/ensu/Ensu/RootView.swift
index 473a19f754..586ff0becd 100644
--- a/mobile/native/apple/apps/ensu/Ensu/RootView.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/RootView.swift
@@ -1,12 +1,14 @@
import SwiftUI
struct RootView: View {
+ let assetStore: AssetStore
+
var body: some View {
ZStack {
EnsuColor.backgroundBase
.ignoresSafeArea()
- HomeView()
+ HomeView(assetStore: assetStore)
}
.frame(maxWidth: .infinity, maxHeight: .infinity)
.tint(EnsuColor.action)
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift
similarity index 64%
rename from mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift
rename to mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift
index 13f050e605..a0ae8b69cc 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/AssetStore.swift
@@ -1,35 +1,31 @@
import BackgroundTasks
import Foundation
-private let logger = EnsuLogging.shared.logger("ModelDownloader")
+private let logger = EnsuLogging.shared.logger("AssetStore")
-final class ModelDownloader {
- private let core: ModelDownloadCore
+final class AssetStore: @unchecked Sendable {
+ private let core: AssetStoreCore
private let activeLock = NSLock()
- private var activeToken: CancellationToken?
- let transcriptionTarget: ModelTarget
- let voiceActivityTarget: ModelTarget
+ private var downloadActive = false
@MainActor
init() {
let baseDir = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first
?? FileManager.default.temporaryDirectory
- var modelsDir = baseDir.appendingPathComponent("models", isDirectory: true)
- try? FileManager.default.createDirectory(at: modelsDir, withIntermediateDirectories: true)
+ var assetsDir = baseDir.appendingPathComponent("assets", isDirectory: true)
+ try? FileManager.default.createDirectory(at: assetsDir, withIntermediateDirectories: true)
var values = URLResourceValues()
values.isExcludedFromBackup = true
- try? modelsDir.setResourceValues(values)
- core = ModelDownloadCore(modelsDir: modelsDir.path)
- transcriptionTarget = transcriptionModelTarget()
- voiceActivityTarget = voiceActivityModelTarget()
+ try? assetsDir.setResourceValues(values)
+ core = AssetStoreCore(assetsDir: assetsDir.path)
let settings = UserDefaults.standard
let pendingSelection = settings.object(forKey: "ensu.model.id") == nil
let legacyModelUrl = pendingSelection && settings.bool(forKey: "ensu.model.use_custom")
? settings.string(forKey: "ensu.model.url")
: nil
- let presetId = migrateMobileModels(
- modelsDir: modelsDir.path,
- legacy: LegacyModels(
+ let presetId = migrateEnsuAssets(
+ assetsDir: assetsDir.path,
+ legacy: LegacyAssets(
llmDir: baseDir.appendingPathComponent("llm", isDirectory: true).path,
transcriptionDir: baseDir.appendingPathComponent("transcription", isDirectory: true).path,
modelUrl: legacyModelUrl,
@@ -46,91 +42,97 @@ final class ModelDownloader {
static func registerBackgroundTask() {
if #available(iOS 26.0, *) {
- ModelDownloadBackgroundTask.register()
+ AssetDownloadBackgroundTask.register()
}
}
- func modelDir(_ target: ModelTarget) -> URL {
- URL(fileURLWithPath: core.modelDir(target: target))
+ func assetDir(_ asset: Asset) -> URL {
+ URL(fileURLWithPath: core.assetDir(asset: asset))
}
- func llmModelPath(_ target: ModelTarget) -> URL? {
- core.llmModelPath(target: target).map { URL(fileURLWithPath: $0) }
+ func llmModelPath(_ asset: Asset) -> URL? {
+ core.llmModelPath(asset: asset).map { URL(fileURLWithPath: $0) }
}
- func llmMmprojPath(_ target: ModelTarget) -> URL? {
- core.llmMmprojPath(target: target).map { URL(fileURLWithPath: $0) }
+ func llmMmprojPath(_ asset: Asset) -> URL? {
+ core.llmMmprojPath(asset: asset).map { URL(fileURLWithPath: $0) }
}
- func voiceActivityModelPath() -> URL {
- URL(fileURLWithPath: core.voiceActivityModelPath())
+ func voiceActivityModelPath(_ asset: Asset) -> URL {
+ URL(fileURLWithPath: core.voiceActivityModelPath(asset: asset))
}
- func isDownloaded(_ target: ModelTarget) -> Bool {
- core.isDownloaded(target: target)
+ func isDownloaded(_ asset: Asset) -> Bool {
+ core.isDownloaded(asset: asset)
}
- func removeDownloaded(_ target: ModelTarget) -> Bool {
- core.removeDownloaded(target: target)
+ func removeDownloaded(_ asset: Asset) -> Bool {
+ core.removeDownloaded(asset: asset)
}
- func cancel() {
- activeLock.withLock { activeToken }?.cancel()
+ func reconcileKnowledge(_ stableId: String) throws -> KnowledgeReconciliation {
+ try reconcileKnowledgePack(store: core, stableId: stableId)
}
- func estimateDownloadSize(_ target: ModelTarget) async -> Int64? {
+ func cleanupKnowledgeRevisions(_ stableId: String, activeIdentity: String) throws {
+ try cleanupObsoleteKnowledgePackRevisions(
+ store: core,
+ stableId: stableId,
+ activeIdentity: activeIdentity
+ )
+ }
+
+ func estimateDownloadSize(_ asset: Asset) async -> Int64? {
await Task.detached(priority: .utility) { [core] in
- core.estimatedDownloadSize(target: target)
+ core.estimatedDownloadSize(asset: asset)
}.value
}
func download(
- targets: [ModelTarget],
- onProgress: @escaping (DownloadProgress) -> Void
+ assets: [Asset],
+ onProgress: @escaping (AssetDownloadProgress) -> Void
) async throws {
let token = CancellationToken()
let started = activeLock.withLock { () -> Bool in
- if activeToken != nil { return false }
- activeToken = token
+ if downloadActive { return false }
+ downloadActive = true
return true
}
guard started else { throw DownloadAlreadyActiveError() }
- defer { activeLock.withLock { activeToken = nil } }
+ defer { activeLock.withLock { downloadActive = false } }
- if targets.allSatisfy({ self.core.isDownloaded(target: $0) }) {
+ if assets.allSatisfy({ self.core.isDownloaded(asset: $0) }) {
return
}
- onProgress(DownloadProgress(percent: 0, status: "Starting download..."))
-
if #available(iOS 26.0, *) {
- ModelDownloadBackgroundTask.begin {
+ AssetDownloadBackgroundTask.begin {
token.cancel()
}
}
var succeeded = false
defer {
if #available(iOS 26.0, *) {
- ModelDownloadBackgroundTask.end(success: succeeded)
+ AssetDownloadBackgroundTask.end(success: succeeded)
}
}
let core = core
let downloadTask = Task.detached(priority: .utility) {
try Task.checkCancellation()
- let callback = ModelDownloadCallbackSink { progress in
+ let callback = AssetDownloadCallbackSink { progress in
if let line = progress.logLine {
logger.info(line)
}
if #available(iOS 26.0, *) {
- ModelDownloadBackgroundTask.update(
+ AssetDownloadBackgroundTask.update(
downloadedBytes: progress.downloadedBytes,
totalBytes: progress.totalBytes
)
}
- onProgress(DownloadProgress(percent: Int(progress.percent), status: progress.status))
+ onProgress(progress)
}
- try core.download(targets: targets, callback: callback, cancellation: token)
+ try core.download(assets: assets, callback: callback, cancellation: token)
}
try await withTaskCancellationHandler {
@@ -145,21 +147,21 @@ final class ModelDownloader {
private struct DownloadAlreadyActiveError: Error {}
-private final class ModelDownloadCallbackSink: ModelDownloadCallback, @unchecked Sendable {
- private let onProgressHandler: (ModelDownloadProgress) -> Void
+private final class AssetDownloadCallbackSink: AssetDownloadCallback, @unchecked Sendable {
+ private let onProgressHandler: (AssetDownloadProgress) -> Void
- init(onProgress: @escaping (ModelDownloadProgress) -> Void) {
+ init(onProgress: @escaping (AssetDownloadProgress) -> Void) {
self.onProgressHandler = onProgress
}
- func onProgress(progress: ModelDownloadProgress) {
+ func onProgress(progress: AssetDownloadProgress) {
onProgressHandler(progress)
}
}
@available(iOS 26.0, *)
-private enum ModelDownloadBackgroundTask {
- private static let identifier = "io.ente.ensu.model-download"
+private enum AssetDownloadBackgroundTask {
+ private static let identifier = "io.ente.ensu.asset-download"
private static let lock = NSLock()
private static var task: BGContinuedProcessingTask?
private static var onExpiration: (() -> Void)?
@@ -183,7 +185,7 @@ private enum ModelDownloadBackgroundTask {
let request = BGContinuedProcessingTaskRequest(
identifier: identifier,
- title: "Downloading model",
+ title: "Downloading assets",
subtitle: ""
)
request.strategy = .fail
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift
index 2ff621ba49..c75881899c 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift
@@ -5,6 +5,11 @@ struct KnowledgeSearchHit: Sendable {
let hit: RetrievalHit
}
+struct KnowledgeDownloadProgress: Sendable {
+ let label: String
+ let percentage: Double
+}
+
private struct KnowledgePackValidationError: LocalizedError {
var errorDescription: String? {
"Downloaded knowledge pack failed current revision validation"
@@ -13,29 +18,25 @@ private struct KnowledgePackValidationError: LocalizedError {
actor KnowledgeProvider {
private final class Mutation: @unchecked Sendable {
- let cancellation: CancellationToken
let task: Task<Void, Error>
init(
- packRoot: URL,
+ assetStore: AssetStore,
dataset: KnowledgeDatasetConfig,
onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void
- ) {
- let cancellation = CancellationToken()
- self.cancellation = cancellation
+ ) throws {
+ let asset = try knowledgePackAsset(stableId: dataset.stableId)
task = Task.detached(priority: .utility) {
- let callback = KnowledgeDownloadCallbackSink(onProgress: onProgress)
- try downloadKnowledgePack(
- packRoot: packRoot.path,
- stableId: dataset.stableId,
- callback: callback,
- cancellation: cancellation
- )
+ try await assetStore.download(assets: [asset]) {
+ onProgress(KnowledgeDownloadProgress(
+ label: $0.label,
+ percentage: $0.percentage
+ ))
+ }
}
}
func cancel() {
- cancellation.cancel()
task.cancel()
}
}
@@ -45,22 +46,14 @@ actor KnowledgeProvider {
let index: RetrievalIndex
}
- private let root: URL
+ private let assetStore: AssetStore
private var mutations: [String: Mutation] = [:]
private var lifecycleGates: [String: AsyncSerialGate] = [:]
private var indexes: [String: OpenIndex] = [:]
private let indexGate = AsyncSerialGate()
- init(root: URL) {
- self.root = root
- try? FileManager.default.createDirectory(
- at: root,
- withIntermediateDirectories: true
- )
- var excludedRoot = root
- var values = URLResourceValues()
- values.isExcludedFromBackup = true
- try? excludedRoot.setResourceValues(values)
+ init(assetStore: AssetStore) {
+ self.assetStore = assetStore
}
func reconcile(dataset: KnowledgeDatasetConfig) async throws -> KnowledgeReconciliation {
@@ -84,8 +77,8 @@ actor KnowledgeProvider {
if let current = mutations[dataset.stableId] {
return current
}
- let created = Mutation(
- packRoot: packRoot(dataset),
+ let created = try Mutation(
+ assetStore: assetStore,
dataset: dataset,
onProgress: onProgress
)
@@ -99,9 +92,7 @@ actor KnowledgeProvider {
if mutations[dataset.stableId] === mutation {
mutations.removeValue(forKey: dataset.stableId)
}
- _ = try? await withIndexGate {
- try await reconcileAndActivateLocked(dataset: dataset, removeIncoming: true)
- }
+ _ = try? await withIndexGate { try await reconcileAndActivateLocked(dataset: dataset) }
throw error
}
@@ -125,21 +116,12 @@ actor KnowledgeProvider {
func cancel(dataset: KnowledgeDatasetConfig) async -> KnowledgeReconciliation? {
let mutation = mutations[dataset.stableId]
mutation?.cancel()
- let transferFailed: Bool
- do {
- try await mutation?.task.value
- transferFailed = false
- } catch {
- transferFailed = true
- }
+ _ = try? await mutation?.task.value
if let mutation, mutations[dataset.stableId] === mutation {
mutations.removeValue(forKey: dataset.stableId)
}
return try? await withIndexGate {
- try await reconcileAndActivateLocked(
- dataset: dataset,
- removeIncoming: transferFailed
- )
+ try await reconcileAndActivateLocked(dataset: dataset)
}
}
@@ -190,30 +172,17 @@ actor KnowledgeProvider {
}
private func reconcileAndActivateLocked(
- dataset: KnowledgeDatasetConfig,
- removeIncoming: Bool = false
+ dataset: KnowledgeDatasetConfig
) async throws -> KnowledgeReconciliation {
- let packRoot = packRoot(dataset)
+ let assetStore = assetStore
let result = try await Task.detached(priority: .utility) {
- if removeIncoming {
- try? FileManager.default.removeItem(
- at: packRoot.appendingPathComponent(
- dataset.currentDownloadIdentity,
- isDirectory: true
- )
- )
- }
- return try reconcileKnowledgePack(
- packRoot: packRoot.path,
- stableId: dataset.stableId
- )
+ try assetStore.reconcileKnowledge(dataset.stableId)
}.value
try activate(result: result, dataset: dataset)
if let activeIdentity = result.activeIdentity {
_ = try? await Task.detached(priority: .utility) {
- try cleanupObsoleteKnowledgePackRevisions(
- packRoot: packRoot.path,
- stableId: dataset.stableId,
+ try assetStore.cleanupKnowledgeRevisions(
+ dataset.stableId,
activeIdentity: activeIdentity
)
}.value
@@ -249,20 +218,4 @@ actor KnowledgeProvider {
lifecycleGates[stableId] = gate
return gate
}
-
- private func packRoot(_ dataset: KnowledgeDatasetConfig) -> URL {
- root.appendingPathComponent(dataset.stableId, isDirectory: true)
- }
-}
-
-private final class KnowledgeDownloadCallbackSink: KnowledgeDownloadCallback, @unchecked Sendable {
- private let onProgressHandler: @Sendable (KnowledgeDownloadProgress) -> Void
-
- init(onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void) {
- onProgressHandler = onProgress
- }
-
- func onProgress(progress: KnowledgeDownloadProgress) {
- onProgressHandler(progress)
- }
}
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift
index a3cf381f22..a44adb622b 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift
@@ -200,22 +200,24 @@ final class KnowledgeStore: ObservableObject {
}
private func userFacingKnowledgeError(_ error: Error) -> String {
- guard case let KnowledgeDownloadError.Download(downloadError) = error else {
- return "Knowledge pack setup failed. Please try again."
- }
- switch downloadError {
- case .cancelled:
+ switch error {
+ case AssetDownloadError.Cancelled:
return "Download cancelled"
- case .storageFull:
+ case AssetDownloadError.StorageFull:
return "Not enough storage space to download this knowledge pack."
- case .network:
+ case AssetDownloadError.Network:
return "Couldn't download the knowledge pack. Check your connection and try again."
- case .http:
+ case AssetDownloadError.Http:
return "The knowledge pack is currently unavailable. Please try again later."
- case .validation, .sizeMismatch, .`protocol`, .invalidTarget:
+ case AssetDownloadError.Validation,
+ AssetDownloadError.SizeMismatch,
+ AssetDownloadError.Protocol,
+ AssetDownloadError.InvalidTarget:
return "The knowledge pack couldn't be verified. Please try again."
- case .io:
+ case AssetDownloadError.Io:
return "Couldn't save the knowledge pack. Please try again."
+ default:
+ return "Knowledge pack setup failed. Please try again."
}
}
}
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift
index 5c859f9ef9..d1ab9cba39 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift
@@ -2,7 +2,6 @@ import Foundation
struct LlmModelSelection: Equatable {
let id: String
- let modelTarget: ModelTarget
let contextLength: Int?
let maxTokens: Int?
}
@@ -65,10 +64,10 @@ struct GenerationSummary {
}
struct RequiredModelValidationError: LocalizedError {
- let targetId: String
+ let modelId: String
var errorDescription: String? {
- "Downloaded model failed validation: \(targetId)"
+ "Downloaded model failed validation: \(modelId)"
}
}
@@ -115,9 +114,10 @@ final class LlmProvider {
let requestedContextLength: Int?
}
- private let downloader: ModelDownloader
+ private let assetStore: AssetStore
private let transcriber: Transcriber
private let knowledgeEmbedding: KnowledgeEmbeddingConfig
+ private let embeddingAsset: Asset
private var loadedModel: LlmModel?
private var loadedContext: LlmContext?
private var currentModelKey: LoadedModelKey?
@@ -126,35 +126,37 @@ final class LlmProvider {
private var currentJobId: Int64?
private let modelLoadGate = AsyncSerialGate()
- init(downloader: ModelDownloader, transcriber: Transcriber, knowledgeEmbedding: KnowledgeEmbeddingConfig) {
- self.downloader = downloader
+ init(assetStore: AssetStore, transcriber: Transcriber, knowledgeEmbedding: KnowledgeEmbeddingConfig) {
+ self.assetStore = assetStore
self.transcriber = transcriber
self.knowledgeEmbedding = knowledgeEmbedding
+ self.embeddingAsset = knowledgeEmbeddingModelAsset()
}
- private var embeddingDownloadTarget: ModelTarget {
- knowledgeEmbeddingModelTarget()
+ func isEmbeddingModelReady() -> Bool {
+ assetStore.isDownloaded(embeddingAsset)
}
- func isEmbeddingModelReady() -> Bool {
- downloader.isDownloaded(embeddingDownloadTarget)
+ func isChatModelReady(_ selection: LlmModelSelection) -> Bool {
+ assetStore.isDownloaded(chatAsset(selection))
}
func isModelDownloaded(_ selection: LlmModelSelection) -> Bool {
- downloader.isDownloaded(selection.modelTarget) &&
+ isChatModelReady(selection) &&
(!isEnsuPacksEnabled || isEmbeddingModelReady())
}
func missingModelDownloadSize(_ selection: LlmModelSelection) async -> Int64? {
var total: Int64 = 0
- if !downloader.isDownloaded(selection.modelTarget) {
- guard let chatSize = await downloader.estimateDownloadSize(selection.modelTarget) else {
+ let asset = chatAsset(selection)
+ if !assetStore.isDownloaded(asset) {
+ guard let chatSize = await assetStore.estimateDownloadSize(asset) else {
return nil
}
total += chatSize
}
if isEnsuPacksEnabled && !isEmbeddingModelReady() {
- guard let embeddingSize = await downloader.estimateDownloadSize(embeddingDownloadTarget) else {
+ guard let embeddingSize = await assetStore.estimateDownloadSize(embeddingAsset) else {
return nil
}
total += embeddingSize
@@ -171,26 +173,27 @@ final class LlmProvider {
throw UnsupportedDeviceMemoryError(capability: capability)
}
- let missingTargets: [ModelTarget] = try await modelLoadGate.withLock {
+ let asset = chatAsset(selection)
+ let missingAssets: [Asset] = try await modelLoadGate.withLock {
let embeddingReady = isEmbeddingModelReady()
if isEnsuPacksEnabled && !embeddingReady {
- _ = downloader.removeDownloaded(embeddingDownloadTarget)
+ _ = assetStore.removeDownloaded(embeddingAsset)
}
return [
- downloader.isDownloaded(selection.modelTarget) ? nil : selection.modelTarget,
- (!isEnsuPacksEnabled || embeddingReady) ? nil : embeddingDownloadTarget,
+ assetStore.isDownloaded(asset) ? nil : asset,
+ (!isEnsuPacksEnabled || embeddingReady) ? nil : embeddingAsset,
].compactMap { $0 }
}
- if !missingTargets.isEmpty {
- try await downloader.download(targets: missingTargets, onProgress: onProgress)
+ if !missingAssets.isEmpty {
+ try await downloadAssets(missingAssets, onProgress: onProgress)
}
try await modelLoadGate.withLock {
guard !isEnsuPacksEnabled || isEmbeddingModelReady() else {
- _ = downloader.removeDownloaded(embeddingDownloadTarget)
- throw RequiredModelValidationError(targetId: knowledgeEmbedding.targetId)
+ _ = assetStore.removeDownloaded(embeddingAsset)
+ throw RequiredModelValidationError(modelId: knowledgeEmbedding.targetId)
}
try await ensureModelReadyLocked(
selection,
@@ -198,8 +201,8 @@ final class LlmProvider {
allowRecovery: false,
shouldDownload: false
)
- guard downloader.isDownloaded(selection.modelTarget) else {
- throw RequiredModelValidationError(targetId: selection.id)
+ guard assetStore.isDownloaded(asset) else {
+ throw RequiredModelValidationError(modelId: selection.id)
}
}
}
@@ -235,19 +238,20 @@ final class LlmProvider {
backendInitialized = true
}
- let wasAlreadyDownloaded = downloader.isDownloaded(selection.modelTarget)
+ let asset = chatAsset(selection)
+ let wasAlreadyDownloaded = assetStore.isDownloaded(asset)
if shouldDownload {
- try await downloader.download(targets: [selection.modelTarget], onProgress: onProgress)
+ try await downloadAssets([asset], onProgress: onProgress)
}
onProgress(DownloadProgress(percent: 100, status: "Loading model...", phase: .loading))
do {
try loadModel(
selection,
- modelPath: downloader.llmModelPath(selection.modelTarget)!
+ modelPath: assetStore.llmModelPath(asset)!
)
} catch {
- if allowRecovery, wasAlreadyDownloaded, downloader.removeDownloaded(selection.modelTarget) {
+ if allowRecovery, wasAlreadyDownloaded, assetStore.removeDownloaded(asset) {
onProgress(DownloadProgress(percent: 0, status: "Starting download..."))
try await ensureModelReadyLocked(selection, onProgress: onProgress, allowRecovery: false)
return
@@ -257,6 +261,20 @@ final class LlmProvider {
onProgress(DownloadProgress(percent: 100, status: "Ready", phase: .ready))
}
+ private func downloadAssets(
+ _ assets: [Asset],
+ onProgress: @escaping (DownloadProgress) -> Void
+ ) async throws {
+ try await assetStore.download(assets: assets) { progress in
+ onProgress(
+ DownloadProgress(
+ percent: min(max(Int(progress.percentage), 0), 99),
+ status: progress.status
+ )
+ )
+ }
+ }
+
func generateChat(
_ selection: LlmModelSelection,
messages: [LlmMessage],
@@ -298,9 +316,10 @@ final class LlmProvider {
LlmChatMessage(role: $0.role.roleString, content: $0.text)
}
+ let asset = chatAsset(selection)
let mmprojPath = imageFiles.isEmpty
? nil
- : downloader.llmMmprojPath(selection.modelTarget)?.path
+ : assetStore.llmMmprojPath(asset)?.path
let clampedTemperature = min(max(temperature, 0.35), 0.7)
let request = LlmChatRequest(
@@ -371,7 +390,7 @@ final class LlmProvider {
backendInitialized = true
}
- guard let embeddingModelPath = downloader.llmModelPath(embeddingDownloadTarget) else {
+ guard let embeddingModelPath = assetStore.llmModelPath(embeddingAsset) else {
throw EmbeddingAssetInvalidError()
}
var embeddingModel: LlmModel? = try LlmModel.load(
@@ -412,14 +431,15 @@ final class LlmProvider {
}
func prewarmImageInference(_ selection: LlmModelSelection) async {
- guard downloader.isDownloaded(selection.modelTarget) else { return }
+ guard isChatModelReady(selection) else { return }
do {
try await Task.detached(priority: .utility) { [weak self] in
guard let self else { return }
try await self.modelLoadGate.withLock {
- guard self.downloader.isDownloaded(selection.modelTarget) else { return }
- guard let mmprojPath = self.downloader.llmMmprojPath(selection.modelTarget),
+ let asset = self.chatAsset(selection)
+ guard self.assetStore.isDownloaded(asset) else { return }
+ guard let mmprojPath = self.assetStore.llmMmprojPath(asset),
FileManager.default.fileExists(atPath: mmprojPath.path) else {
return
}
@@ -469,6 +489,10 @@ final class LlmProvider {
transcriber.unloadModel()
}
+ private func chatAsset(_ selection: LlmModelSelection) -> Asset {
+ llmAsset(modelId: selection.id)
+ }
+
private func loadModel(_ selection: LlmModelSelection, modelPath: URL) throws {
let params = LlmModelLoadParams(modelPath: modelPath.path, nGpuLayers: 0, useMmap: true, useMlock: false)
let model = try LlmModel.load(params: params)
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
index 776ead4909..5dac12ad3c 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
@@ -100,7 +100,6 @@ final class ModelSettingsStore: ObservableObject {
let preset = presets.first(where: { $0.id == modelId }) ?? defaultModel
return LlmModelSelection(
id: preset.id,
- modelTarget: try! mobileLlmTarget(modelId: preset.id),
contextLength: Int(contextLength),
maxTokens: Int(maxTokens).flatMap { $0 > 0 ? $0 : nil }
)
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 56971dab38..aa6c551cf7 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2268,9 +2268,9 @@ checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099"
name = "ensu-desktop"
version = "0.1.18-beta"
dependencies = [
+ "ente-assets",
"ente-core",
"ente-ensu",
- "ente-model-download",
"keyring",
"libc",
"serde",
@@ -2311,6 +2311,21 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "ente-assets"
+version = "0.0.0"
+dependencies = [
+ "flate2",
+ "futures-util",
+ "reqwest 0.12.28",
+ "serde",
+ "serde_json",
+ "sha2 0.11.0",
+ "tar",
+ "thiserror 2.0.18",
+ "tokio",
+]
+
[[package]]
name = "ente-cast-uniffi"
version = "0.0.0"
@@ -2373,9 +2388,9 @@ dependencies = [
name = "ente-ensu"
version = "0.0.0"
dependencies = [
+ "ente-assets",
"ente-core",
"ente-image",
- "ente-model-download",
"llama-cpp-2",
"memmap2",
"ort",
@@ -2386,7 +2401,6 @@ dependencies = [
"self_cell",
"serde",
"serde_json",
- "sha2 0.11.0",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -2401,9 +2415,10 @@ dependencies = [
name = "ente-ensu-uniffi"
version = "0.0.0"
dependencies = [
+ "ente-assets",
"ente-ensu",
- "ente-model-download",
"thiserror 2.0.18",
+ "tokio",
"uniffi",
"uuid",
]
@@ -2422,21 +2437,6 @@ dependencies = [
"tiff 0.10.3",
]
-[[package]]
-name = "ente-model-download"
-version = "0.0.0"
-dependencies = [
- "flate2",
- "futures-util",
- "reqwest 0.12.28",
- "serde",
- "serde_json",
- "sha2 0.11.0",
- "tar",
- "thiserror 2.0.18",
- "tokio",
-]
-
[[package]]
name = "ente-paste"
version = "0.0.0"
@@ -2456,8 +2456,8 @@ version = "0.0.0"
dependencies = [
"anyhow",
"ash",
+ "ente-assets",
"ente-image",
- "ente-model-download",
"fast_image_resize",
"flate2",
"half",
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index abd9a2321f..222d71b6b0 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -12,24 +12,24 @@ members = [
"bindings/wasm/core",
"bindings/wasm/space",
"crates/accounts",
+ "crates/assets",
"crates/contacts",
"crates/core",
"crates/ensu",
- "crates/paste",
"crates/image",
- "crates/model-download",
+ "crates/paste",
"crates/photos",
"crates/test-support",
"space",
]
[workspace.dependencies]
-ente-ensu = { path = "crates/ensu" }
ente-accounts = { path = "crates/accounts" }
+ente-assets = { path = "crates/assets" }
ente-contacts = { path = "crates/contacts" }
ente-core = { path = "crates/core" }
+ente-ensu = { path = "crates/ensu" }
ente-image = { path = "crates/image" }
-ente-model-download = { path = "crates/model-download" }
ente-paste = { path = "crates/paste" }
ente-photos = { path = "crates/photos" }
ente-rs = { path = "apps/cli" }
diff --git a/rust/apps/ensu/src-tauri/Cargo.toml b/rust/apps/ensu/src-tauri/Cargo.toml
index 49a424d66e..8f65f247ce 100644
--- a/rust/apps/ensu/src-tauri/Cargo.toml
+++ b/rust/apps/ensu/src-tauri/Cargo.toml
@@ -7,19 +7,19 @@ edition = "2024"
tauri-build = { version = "2.6.2", features = [] }
[dependencies]
+ente-assets.workspace = true
+ente-core.workspace = true
+ente-ensu.workspace = true
+keyring = { version = "3", default-features = false, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }
+serde.workspace = true
+serde_json.workspace = true
tauri = { version = "2.11.2", features = [] }
tauri-plugin-dialog = "2.7.1"
tauri-plugin-fs = "2.5.1"
tauri-plugin-opener = "2.5.4"
tauri-plugin-process = "2.3.1"
tauri-plugin-updater = "2.10.1"
-serde.workspace = true
-serde_json.workspace = true
-ente-ensu.workspace = true
-ente-model-download.workspace = true
-ente-core.workspace = true
uuid = { version = "1", features = ["v4"] }
-keyring = { version = "3", default-features = false, features = ["apple-native", "windows-native", "linux-native-sync-persistent", "crypto-rust"] }
[target.'cfg(unix)'.dependencies]
libc = "0.2"
diff --git a/rust/apps/ensu/src-tauri/src/commands/llm.rs b/rust/apps/ensu/src-tauri/src/commands/llm.rs
index 7c3e7e1f5a..31fdc7017a 100644
--- a/rust/apps/ensu/src-tauri/src/commands/llm.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/llm.rs
@@ -3,9 +3,8 @@ use std::path::PathBuf;
use std::sync::{Arc, Mutex, PoisonError};
use std::time::{Duration, Instant};
+use ente_assets::{Asset, AssetStore, download};
use ente_ensu::llm;
-use ente_model_download::download;
-use ente_model_download::{ModelDownloader, ModelTarget};
use serde::Serialize;
use tauri::async_runtime;
use tauri::{AppHandle, Emitter, Manager, State as TauriState, WebviewWindow};
@@ -20,16 +19,19 @@ pub struct State {
}
pub struct ModelDownloadState {
- downloader: Arc<ModelDownloader>,
- models_dir: PathBuf,
+ store: Arc<AssetStore>,
+ assets_dir: PathBuf,
+ legacy_models_dir: PathBuf,
active_token: Mutex<Option<download::CancellationToken>>,
}
impl ModelDownloadState {
- pub fn new(models_dir: PathBuf) -> Self {
+ pub fn new(app_data_dir: PathBuf) -> Self {
+ let assets_dir = app_data_dir.join("assets");
Self {
- downloader: Arc::new(ModelDownloader::new(&models_dir)),
- models_dir,
+ store: Arc::new(AssetStore::new(&assets_dir)),
+ assets_dir,
+ legacy_models_dir: app_data_dir.join("models"),
active_token: Mutex::new(None),
}
}
@@ -94,8 +96,8 @@ fn fs_thread_error() -> ApiError {
ApiError::new("io_thread", "FS task failed")
}
-fn desktop_target(model_id: &str) -> Result<ModelTarget, ApiError> {
- ente_ensu::model::desktop_llm_target(model_id)
+fn desktop_asset(model_id: &str) -> Result<Asset, ApiError> {
+ ente_ensu::model::desktop_llm_asset(model_id)
.map_err(|err| ApiError::new("invalid_target", err.to_string()))
}
@@ -245,14 +247,14 @@ pub fn llm_model_status(
state: TauriState<'_, ModelDownloadState>,
model_id: String,
) -> Result<ModelStatus, ApiError> {
- let target = desktop_target(&model_id)?;
+ let asset = desktop_asset(&model_id)?;
Ok(ModelStatus {
- model_path: ente_ensu::model::llm_model_path(&state.downloader, &target)
+ model_path: ente_ensu::model::llm_model_path(&state.store, &asset)
.map(|path| path.display().to_string())
.unwrap_or_default(),
- mmproj_path: ente_ensu::model::llm_mmproj_path(&state.downloader, &target)
+ mmproj_path: ente_ensu::model::llm_mmproj_path(&state.store, &asset)
.map(|path| path.display().to_string()),
- downloaded: state.downloader.is_downloaded(&target),
+ downloaded: state.store.is_downloaded(&asset),
})
}
@@ -262,12 +264,18 @@ pub async fn llm_migrate_models(
legacy_model_url: Option<String>,
legacy_mmproj_url: Option<String>,
) -> Result<Option<String>, ApiError> {
- let models_dir = state.models_dir.clone();
+ let assets_dir = state.assets_dir.clone();
+ let legacy_models_dir = state.legacy_models_dir.clone();
async_runtime::spawn_blocking(move || {
- ente_ensu::model::migrations::migrate_desktop_models(
- &models_dir,
- legacy_model_url.as_deref(),
- legacy_mmproj_url.as_deref(),
+ ente_ensu::model::migrations::migrate_ensu_assets(
+ &assets_dir,
+ ente_ensu::model::migrations::LegacyAssets {
+ llm_dir: None,
+ transcription_dir: None,
+ models_dir: Some(legacy_models_dir),
+ model_url: legacy_model_url,
+ mmproj_url: legacy_mmproj_url,
+ },
)
})
.await
@@ -280,8 +288,8 @@ pub async fn llm_download_model(
state: TauriState<'_, ModelDownloadState>,
model_id: String,
) -> Result<(), ApiError> {
- let downloader = Arc::clone(&state.downloader);
- let target = desktop_target(&model_id)?;
+ let store = Arc::clone(&state.store);
+ let asset = desktop_asset(&model_id)?;
let token = {
let mut slot = state
.active_token
@@ -297,35 +305,33 @@ pub async fn llm_download_model(
*slot = Some(token.clone());
token
};
- let result = async_runtime::spawn_blocking(move || {
- downloader
- .download(
- std::slice::from_ref(&target),
- |progress| {
- if let Some(line) = &progress.log_line {
- logging::log("LLMDownload", line.clone());
- }
- let _ = window.emit(
- "llm-download-progress",
- DownloadProgress {
- percent: progress.percent,
- status: progress.status,
- bytes_downloaded: progress.downloaded_bytes,
- total_bytes: progress.total_bytes,
- },
- );
- },
- &token,
- )
- .map_err(|err| ApiError::new(download_code(&err), err.to_string()))
- })
- .await
- .map_err(|_| fs_thread_error());
+ let result = store
+ .download(
+ &asset,
+ |progress| {
+ let progress = ente_ensu::model::display_progress(progress);
+ if let Some(line) = &progress.log_line {
+ logging::log("LLMDownload", line.clone());
+ }
+ let _ = window.emit(
+ "llm-download-progress",
+ DownloadProgress {
+ percent: progress.percent,
+ status: progress.status,
+ bytes_downloaded: progress.downloaded_bytes,
+ total_bytes: progress.total_bytes,
+ },
+ );
+ },
+ token,
+ )
+ .await
+ .map_err(|err| ApiError::new(download_code(&err), err.to_string()));
*state
.active_token
.lock()
.unwrap_or_else(PoisonError::into_inner) = None;
- result?
+ result
}
#[tauri::command]
diff --git a/rust/apps/ensu/src-tauri/src/main.rs b/rust/apps/ensu/src-tauri/src/main.rs
index 0abe0301ed..663f11e3b1 100644
--- a/rust/apps/ensu/src-tauri/src/main.rs
+++ b/rust/apps/ensu/src-tauri/src/main.rs
@@ -21,8 +21,9 @@ fn main() {
logging::init_logging(app.handle());
logging::log("App", "setup started");
- let models_dir = app.path().app_data_dir()?.join("models");
- app.manage(commands::llm::ModelDownloadState::new(models_dir));
+ app.manage(commands::llm::ModelDownloadState::new(
+ app.path().app_data_dir()?,
+ ));
// Show the main window after setup is complete
if let Some(window) = app.get_webview_window("main")
diff --git a/rust/bindings/uniffi/ensu/Cargo.toml b/rust/bindings/uniffi/ensu/Cargo.toml
index c906a327a3..1567596d84 100644
--- a/rust/bindings/uniffi/ensu/Cargo.toml
+++ b/rust/bindings/uniffi/ensu/Cargo.toml
@@ -7,8 +7,9 @@ name = "ensu"
crate-type = ["staticlib", "cdylib"]
[dependencies]
+ente-assets.workspace = true
ente-ensu = { workspace = true, features = ["transcription"] }
-ente-model-download.workspace = true
thiserror.workspace = true
+tokio = { version = "1", features = ["rt-multi-thread"] }
uniffi.workspace = true
uuid = { version = "1" }
diff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rs
new file mode 100644
index 0000000000..f1ce142241
--- /dev/null
+++ b/rust/bindings/uniffi/ensu/src/assets.rs
@@ -0,0 +1,268 @@
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, OnceLock};
+
+use ente_assets::{Asset as RustAsset, AssetStore as RustAssetStore, download};
+use thiserror::Error;
+
+#[derive(uniffi::Object)]
+pub struct Asset {
+ pub(crate) inner: RustAsset,
+}
+
+impl Asset {
+ pub(crate) fn new(inner: RustAsset) -> Arc<Self> {
+ Arc::new(Self { inner })
+ }
+}
+
+#[derive(Debug, Error, uniffi::Error)]
+pub enum AssetDownloadError {
+ #[error("download cancelled")]
+ Cancelled,
+ #[error("{message}")]
+ Validation { message: String },
+ #[error("HTTP {status}")]
+ Http { status: u16 },
+ #[error("network: {message}")]
+ Network { message: String },
+ #[error("size mismatch: expected {expected} bytes, got {actual}")]
+ SizeMismatch { expected: u64, actual: u64 },
+ #[error("range protocol violation: {message}")]
+ Protocol { message: String },
+ #[error("invalid download target: {message}")]
+ InvalidTarget { message: String },
+ #[error("not enough storage space")]
+ StorageFull,
+ #[error("{message}")]
+ Io { message: String },
+}
+
+impl From<download::Error> for AssetDownloadError {
+ fn from(value: download::Error) -> Self {
+ match value {
+ download::Error::Cancelled => Self::Cancelled,
+ download::Error::Target { source, .. } => Self::from(*source),
+ download::Error::Fallback { single, .. } => Self::from(*single),
+ download::Error::Validation(message) => Self::Validation { message },
+ download::Error::Http(status) => Self::Http { status },
+ download::Error::Network(message) => Self::Network { message },
+ download::Error::SizeMismatch { expected, actual } => {
+ Self::SizeMismatch { expected, actual }
+ }
+ download::Error::Protocol(message) => Self::Protocol { message },
+ download::Error::InvalidTarget(message) => Self::InvalidTarget { message },
+ download::Error::StorageFull => Self::StorageFull,
+ download::Error::Io(error) => Self::Io {
+ message: error.to_string(),
+ },
+ download::Error::Json(error) => Self::Io {
+ message: error.to_string(),
+ },
+ }
+ }
+}
+
+#[uniffi::export]
+pub fn llm_asset(model_id: String) -> Arc<Asset> {
+ Asset::new(ente_ensu::model::mobile_llm_asset(&model_id).expect("valid mobile model catalog"))
+}
+
+#[uniffi::export]
+pub fn transcription_model_asset() -> Arc<Asset> {
+ Asset::new(ente_ensu::model::transcription_model_asset())
+}
+
+#[uniffi::export]
+pub fn voice_activity_model_asset() -> Arc<Asset> {
+ Asset::new(ente_ensu::model::voice_activity_model_asset())
+}
+
+#[uniffi::export]
+pub fn knowledge_embedding_model_asset() -> Arc<Asset> {
+ Asset::new(ente_ensu::model::knowledge_embedding_model_asset())
+}
+
+#[derive(Debug, Clone, uniffi::Record)]
+pub struct AssetDownloadProgress {
+ pub label: String,
+ pub downloaded_bytes: i64,
+ pub total_bytes: Option<i64>,
+ pub percentage: f64,
+ pub status: String,
+ pub log_line: Option<String>,
+}
+
+impl From<ente_assets::download::Progress> for AssetDownloadProgress {
+ fn from(value: ente_assets::download::Progress) -> Self {
+ let display = ente_ensu::model::display_progress(value.clone());
+ Self {
+ label: value.label,
+ downloaded_bytes: i64::try_from(value.downloaded_bytes).unwrap_or(i64::MAX),
+ total_bytes: value
+ .total_bytes
+ .map(|total| i64::try_from(total).unwrap_or(i64::MAX)),
+ percentage: value.percentage,
+ status: display.status,
+ log_line: display.log_line,
+ }
+ }
+}
+
+#[uniffi::export(callback_interface)]
+pub trait AssetDownloadCallback: Send + Sync {
+ fn on_progress(&self, progress: AssetDownloadProgress);
+}
+
+#[derive(Clone, uniffi::Record)]
+pub struct LegacyAssets {
+ pub llm_dir: Option<String>,
+ pub transcription_dir: Option<String>,
+ pub model_url: Option<String>,
+ pub mmproj_url: Option<String>,
+}
+
+impl From<LegacyAssets> for ente_ensu::model::migrations::LegacyAssets {
+ fn from(value: LegacyAssets) -> Self {
+ Self {
+ llm_dir: value.llm_dir.map(PathBuf::from),
+ transcription_dir: value.transcription_dir.map(PathBuf::from),
+ models_dir: None,
+ model_url: value.model_url,
+ mmproj_url: value.mmproj_url,
+ }
+ }
+}
+
+#[uniffi::export]
+pub fn needs_asset_migration(legacy: LegacyAssets) -> bool {
+ ente_ensu::model::migrations::needs_migration(&legacy.into())
+}
+
+#[uniffi::export]
+pub fn migrate_ensu_assets(assets_dir: String, legacy: LegacyAssets) -> Option<String> {
+ ente_ensu::model::migrations::migrate_ensu_assets(
+ PathBuf::from(assets_dir).as_path(),
+ legacy.into(),
+ )
+}
+
+#[derive(uniffi::Object)]
+pub struct AssetStoreCore {
+ inner: RustAssetStore,
+ runtime: OnceLock<tokio::runtime::Runtime>,
+ active_downloads: AtomicUsize,
+}
+
+#[uniffi::export]
+impl AssetStoreCore {
+ #[uniffi::constructor]
+ pub fn new(assets_dir: String) -> Arc<Self> {
+ Arc::new(Self {
+ inner: RustAssetStore::new(assets_dir),
+ runtime: OnceLock::new(),
+ active_downloads: AtomicUsize::new(0),
+ })
+ }
+
+ pub fn asset_dir(&self, asset: Arc<Asset>) -> String {
+ self.inner.asset_dir(&asset.inner).display().to_string()
+ }
+
+ pub fn llm_model_path(&self, asset: Arc<Asset>) -> Option<String> {
+ ente_ensu::model::llm_model_path(&self.inner, &asset.inner)
+ .map(|path| path.display().to_string())
+ }
+
+ pub fn llm_mmproj_path(&self, asset: Arc<Asset>) -> Option<String> {
+ ente_ensu::model::llm_mmproj_path(&self.inner, &asset.inner)
+ .map(|path| path.display().to_string())
+ }
+
+ pub fn voice_activity_model_path(&self, asset: Arc<Asset>) -> String {
+ ente_ensu::model::voice_activity_model_path(&self.inner, &asset.inner)
+ .display()
+ .to_string()
+ }
+
+ pub fn is_downloaded(&self, asset: Arc<Asset>) -> bool {
+ self.inner.is_downloaded(&asset.inner)
+ }
+
+ pub fn is_download_active(&self) -> bool {
+ self.active_downloads.load(Ordering::SeqCst) > 0
+ }
+
+ pub fn estimated_download_size(&self, asset: Arc<Asset>) -> Option<i64> {
+ self.runtime()
+ .ok()?
+ .block_on(self.inner.estimated_download_size(&asset.inner))
+ .map(|size| i64::try_from(size).unwrap_or(i64::MAX))
+ }
+
+ pub fn remove_downloaded(&self, asset: Arc<Asset>) -> bool {
+ let existed = self.inner.is_downloaded(&asset.inner);
+ let removed = self
+ .runtime()
+ .is_ok_and(|runtime| runtime.block_on(self.inner.remove(&asset.inner)).is_ok());
+ existed && removed
+ }
+
+ pub fn download(
+ &self,
+ assets: Vec<Arc<Asset>>,
+ callback: Box<dyn AssetDownloadCallback>,
+ cancellation: Arc<CancellationToken>,
+ ) -> Result<(), AssetDownloadError> {
+ self.active_downloads.fetch_add(1, Ordering::SeqCst);
+ let result = self.runtime().and_then(|runtime| {
+ runtime.block_on(async {
+ for asset in assets {
+ self.inner
+ .download(
+ &asset.inner,
+ |progress| callback.on_progress(progress.into()),
+ cancellation.inner.clone(),
+ )
+ .await?;
+ }
+ Ok(())
+ })
+ });
+ self.active_downloads.fetch_sub(1, Ordering::SeqCst);
+ result.map_err(Into::into)
+ }
+}
+
+impl AssetStoreCore {
+ pub(crate) fn store(&self) -> &RustAssetStore {
+ &self.inner
+ }
+
+ pub(crate) fn runtime(&self) -> Result<&tokio::runtime::Runtime, ente_assets::download::Error> {
+ if let Some(runtime) = self.runtime.get() {
+ return Ok(runtime);
+ }
+ let runtime = tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?;
+ Ok(self.runtime.get_or_init(|| runtime))
+ }
+}
+
+#[derive(Default, uniffi::Object)]
+pub struct CancellationToken {
+ pub(crate) inner: ente_assets::download::CancellationToken,
+}
+
+#[uniffi::export]
+impl CancellationToken {
+ #[uniffi::constructor]
+ pub fn new() -> Arc<Self> {
+ Arc::new(Self::default())
+ }
+
+ pub fn cancel(&self) {
+ self.inner.cancel();
+ }
+}
diff --git a/rust/bindings/uniffi/ensu/src/download.rs b/rust/bindings/uniffi/ensu/src/download.rs
deleted file mode 100644
index 3211752e56..0000000000
--- a/rust/bindings/uniffi/ensu/src/download.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use ente_model_download::download;
-
-#[derive(Debug, Clone, uniffi::Enum)]
-pub enum DownloadError {
- Cancelled,
- Validation { message: String },
- Http { status: u16 },
- Network { message: String },
- SizeMismatch { expected: u64, actual: u64 },
- Protocol { message: String },
- InvalidTarget { message: String },
- StorageFull,
- Io { message: String },
-}
-
-impl From<download::Error> for DownloadError {
- fn from(value: download::Error) -> Self {
- match value {
- download::Error::Cancelled => Self::Cancelled,
- download::Error::Target { source, .. } => Self::from(*source),
- download::Error::Fallback { single, .. } => Self::from(*single),
- download::Error::Validation(message) => Self::Validation { message },
- download::Error::Http(status) => Self::Http { status },
- download::Error::Network(message) => Self::Network { message },
- download::Error::SizeMismatch { expected, actual } => {
- Self::SizeMismatch { expected, actual }
- }
- download::Error::Protocol(message) => Self::Protocol { message },
- download::Error::InvalidTarget(message) => Self::InvalidTarget { message },
- download::Error::StorageFull => Self::StorageFull,
- download::Error::Io(err) => Self::Io {
- message: err.to_string(),
- },
- download::Error::Json(err) => Self::Io {
- message: err.to_string(),
- },
- }
- }
-}
diff --git a/rust/bindings/uniffi/ensu/src/lib.rs b/rust/bindings/uniffi/ensu/src/lib.rs
index 52e1a939ed..32a99e8f84 100644
--- a/rust/bindings/uniffi/ensu/src/lib.rs
+++ b/rust/bindings/uniffi/ensu/src/lib.rs
@@ -1,10 +1,9 @@
uniffi::setup_scaffolding!("ensu");
+pub mod assets;
pub mod config;
pub mod db;
-pub mod download;
pub mod image;
pub mod llm;
-pub mod model;
pub mod retrieval;
pub mod transcription;
diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rs
index 7f5ab0703f..ebe8d9cf2e 100644
--- a/rust/bindings/uniffi/ensu/src/llm.rs
+++ b/rust/bindings/uniffi/ensu/src/llm.rs
@@ -1,6 +1,5 @@
use ente_ensu::llm;
-use crate::download::DownloadError;
use std::sync::Arc;
use thiserror::Error;
@@ -20,8 +19,6 @@ pub enum LlmError {
PromptTooLong { tokens: u64, context_size: u32 },
#[error("{op}: {detail}")]
Llama { op: String, detail: String },
- #[error("download failed")]
- Download { error: DownloadError },
}
impl From<llm::Error> for LlmError {
diff --git a/rust/bindings/uniffi/ensu/src/model.rs b/rust/bindings/uniffi/ensu/src/model.rs
deleted file mode 100644
index 2e3809ec3f..0000000000
--- a/rust/bindings/uniffi/ensu/src/model.rs
+++ /dev/null
@@ -1,234 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use crate::download::DownloadError;
-use crate::llm::LlmError;
-
-#[derive(Debug, Clone, uniffi::Enum)]
-pub enum ModelTarget {
- Files {
- id: String,
- files: Vec<ModelFile>,
- },
- TarGz {
- id: String,
- url: String,
- sha256: String,
- },
-}
-
-#[derive(Debug, Clone, uniffi::Record)]
-pub struct ModelFile {
- pub name: String,
- pub url: String,
- pub sha256: String,
-}
-
-impl From<ModelTarget> for ente_model_download::ModelTarget {
- fn from(value: ModelTarget) -> Self {
- match value {
- ModelTarget::Files { id, files } => Self::Files {
- id,
- files: files
- .into_iter()
- .map(|file| ente_model_download::ModelFile {
- name: file.name,
- url: file.url,
- sha256: file.sha256,
- })
- .collect(),
- },
- ModelTarget::TarGz { id, url, sha256 } => Self::TarGz { id, url, sha256 },
- }
- }
-}
-
-impl From<ente_model_download::ModelTarget> for ModelTarget {
- fn from(value: ente_model_download::ModelTarget) -> Self {
- match value {
- ente_model_download::ModelTarget::Files { id, files } => Self::Files {
- id,
- files: files
- .into_iter()
- .map(|file| ModelFile {
- name: file.name,
- url: file.url,
- sha256: file.sha256,
- })
- .collect(),
- },
- ente_model_download::ModelTarget::TarGz { id, url, sha256 } => {
- Self::TarGz { id, url, sha256 }
- }
- }
- }
-}
-
-#[uniffi::export]
-pub fn mobile_llm_target(model_id: String) -> Result<ModelTarget, LlmError> {
- ente_ensu::model::mobile_llm_target(&model_id)
- .map(Into::into)
- .map_err(|err| LlmError::Download {
- error: DownloadError::InvalidTarget {
- message: err.to_string(),
- },
- })
-}
-
-#[uniffi::export]
-pub fn transcription_model_target() -> ModelTarget {
- ente_ensu::model::transcription_target().into()
-}
-
-#[uniffi::export]
-pub fn voice_activity_model_target() -> ModelTarget {
- ente_ensu::model::voice_activity_target().into()
-}
-
-#[uniffi::export]
-pub fn knowledge_embedding_model_target() -> ModelTarget {
- ente_ensu::model::knowledge_embedding_target().into()
-}
-
-#[derive(Debug, Clone, uniffi::Record)]
-pub struct ModelDownloadProgress {
- pub downloaded_bytes: i64,
- pub total_bytes: Option<i64>,
- pub percent: i32,
- pub status: String,
- pub log_line: Option<String>,
-}
-
-impl From<ente_model_download::ModelDownloadProgress> for ModelDownloadProgress {
- fn from(value: ente_model_download::ModelDownloadProgress) -> Self {
- Self {
- downloaded_bytes: i64::try_from(value.downloaded_bytes).unwrap_or(i64::MAX),
- total_bytes: value
- .total_bytes
- .map(|total| i64::try_from(total).unwrap_or(i64::MAX)),
- percent: value.percent,
- status: value.status,
- log_line: value.log_line,
- }
- }
-}
-
-#[uniffi::export(callback_interface)]
-pub trait ModelDownloadCallback: Send + Sync {
- fn on_progress(&self, progress: ModelDownloadProgress);
-}
-
-#[derive(uniffi::Record)]
-pub struct LegacyModels {
- pub llm_dir: Option<String>,
- pub transcription_dir: String,
- pub model_url: Option<String>,
- pub mmproj_url: Option<String>,
-}
-
-#[uniffi::export]
-pub fn migrate_mobile_models(models_dir: String, legacy: LegacyModels) -> Option<String> {
- ente_ensu::model::migrations::migrate_mobile_models(
- Path::new(&models_dir),
- ente_ensu::model::migrations::LegacyModels {
- llm_dir: legacy.llm_dir.map(PathBuf::from),
- transcription_dir: PathBuf::from(legacy.transcription_dir),
- model_url: legacy.model_url,
- mmproj_url: legacy.mmproj_url,
- },
- )
-}
-
-#[derive(uniffi::Object)]
-pub struct ModelDownloadCore {
- inner: ente_model_download::ModelDownloader,
-}
-
-#[uniffi::export]
-impl ModelDownloadCore {
- #[uniffi::constructor]
- pub fn new(models_dir: String) -> Arc<Self> {
- Arc::new(Self {
- inner: ente_model_download::ModelDownloader::new(models_dir),
- })
- }
-
- pub fn model_dir(&self, target: ModelTarget) -> String {
- self.inner.model_dir(&target.into()).display().to_string()
- }
-
- pub fn llm_model_path(&self, target: ModelTarget) -> Option<String> {
- ente_ensu::model::llm_model_path(&self.inner, &target.into())
- .map(|path| path.display().to_string())
- }
-
- pub fn llm_mmproj_path(&self, target: ModelTarget) -> Option<String> {
- ente_ensu::model::llm_mmproj_path(&self.inner, &target.into())
- .map(|path| path.display().to_string())
- }
-
- pub fn voice_activity_model_path(&self) -> String {
- ente_ensu::model::voice_activity_model_path(&self.inner)
- .display()
- .to_string()
- }
-
- pub fn is_downloaded(&self, target: ModelTarget) -> bool {
- self.inner.is_downloaded(&target.into())
- }
-
- pub fn is_download_active(&self) -> bool {
- self.inner.is_download_active()
- }
-
- pub fn estimated_download_size(&self, target: ModelTarget) -> Option<i64> {
- self.inner
- .estimated_download_size(&target.into())
- .map(|size| i64::try_from(size).unwrap_or(i64::MAX))
- }
-
- pub fn remove_downloaded(&self, target: ModelTarget) -> bool {
- self.inner.remove_downloaded(&target.into())
- }
-
- pub fn download(
- &self,
- targets: Vec<ModelTarget>,
- callback: Box<dyn ModelDownloadCallback>,
- cancellation: Arc<CancellationToken>,
- ) -> Result<(), LlmError> {
- let targets: Vec<ente_model_download::ModelTarget> =
- targets.into_iter().map(Into::into).collect();
- self.inner
- .download(
- &targets,
- move |progress| callback.on_progress(progress.into()),
- &cancellation.inner,
- )
- .map_err(map_download_error)
- }
-}
-
-#[derive(Default, uniffi::Object)]
-pub struct CancellationToken {
- pub(crate) inner: ente_model_download::download::CancellationToken,
-}
-
-#[uniffi::export]
-impl CancellationToken {
- #[uniffi::constructor]
- pub fn new() -> Arc<Self> {
- Arc::new(Self::default())
- }
-
- pub fn cancel(&self) {
- self.inner.cancel();
- }
-}
-
-fn map_download_error(err: ente_model_download::download::Error) -> LlmError {
- match DownloadError::from(err) {
- DownloadError::Cancelled => LlmError::Cancelled,
- error => LlmError::Download { error },
- }
-}
diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rs
index d6d997c7eb..8d2145d0ba 100644
--- a/rust/bindings/uniffi/ensu/src/retrieval.rs
+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs
@@ -3,8 +3,7 @@ use std::sync::Arc;
use ente_ensu::retrieval as core;
use thiserror::Error;
-use crate::download::DownloadError;
-use crate::model::CancellationToken;
+use crate::assets::{Asset, AssetStoreCore};
#[derive(Debug, Error, uniffi::Error)]
pub enum KnowledgeRetrievalError {
@@ -46,12 +45,6 @@ fn knowledge_dataset(
})
}
-#[derive(Debug, Error, uniffi::Error)]
-pub enum KnowledgeDownloadError {
- #[error("knowledge download failed")]
- Download { error: DownloadError },
-}
-
#[derive(Debug, Clone, uniffi::Record)]
pub struct RetrievalHit {
pub score: f32,
@@ -165,26 +158,34 @@ impl From<core::KnowledgeReconciliation> for KnowledgeReconciliation {
#[uniffi::export]
pub fn reconcile_knowledge_pack(
- pack_root: String,
+ store: Arc<AssetStoreCore>,
stable_id: String,
) -> Result<KnowledgeReconciliation, KnowledgeRetrievalError> {
- core::reconcile_knowledge_pack(pack_root, &knowledge_dataset(&stable_id)?)
+ let dataset = knowledge_dataset(&stable_id)?;
+ let pack_root = core::knowledge_pack_root(store.store(), &dataset).map_err(|error| {
+ KnowledgeRetrievalError::InvalidInput {
+ detail: error.to_string(),
+ }
+ })?;
+ core::reconcile_knowledge_pack(pack_root, &dataset)
.map(Into::into)
.map_err(Into::into)
}
#[uniffi::export]
pub fn cleanup_obsolete_knowledge_pack_revisions(
- pack_root: String,
+ store: Arc<AssetStoreCore>,
stable_id: String,
active_identity: String,
) -> Result<(), KnowledgeRetrievalError> {
- core::cleanup_obsolete_knowledge_pack_revisions(
- pack_root,
- &knowledge_dataset(&stable_id)?,
- &active_identity,
- )
- .map_err(Into::into)
+ let dataset = knowledge_dataset(&stable_id)?;
+ let pack_root = core::knowledge_pack_root(store.store(), &dataset).map_err(|error| {
+ KnowledgeRetrievalError::InvalidInput {
+ detail: error.to_string(),
+ }
+ })?;
+ core::cleanup_obsolete_knowledge_pack_revisions(pack_root, &dataset, &active_identity)
+ .map_err(Into::into)
}
#[derive(Debug, Clone, uniffi::Record)]
@@ -289,47 +290,12 @@ pub fn clean_assistant_text(stored_text: String) -> String {
core::clean_assistant_text(&stored_text)
}
-#[derive(Debug, Clone, uniffi::Record)]
-pub struct KnowledgeDownloadProgress {
- pub label: String,
- pub percentage: f64,
-}
-
-impl From<ente_model_download::download::Progress> for KnowledgeDownloadProgress {
- fn from(value: ente_model_download::download::Progress) -> Self {
- Self {
- label: value.label,
- percentage: value.percentage,
- }
- }
-}
-
-#[uniffi::export(callback_interface)]
-pub trait KnowledgeDownloadCallback: Send + Sync {
- fn on_progress(&self, progress: KnowledgeDownloadProgress);
-}
-
#[uniffi::export]
-pub fn download_knowledge_pack(
- pack_root: String,
- stable_id: String,
- callback: Box<dyn KnowledgeDownloadCallback>,
- cancellation: Arc<CancellationToken>,
-) -> Result<(), KnowledgeDownloadError> {
- let expected_dataset = ente_ensu::config::knowledge_dataset(&stable_id).ok_or_else(|| {
- KnowledgeDownloadError::Download {
- error: DownloadError::InvalidTarget {
- message: format!("unknown knowledge dataset ID: {stable_id}"),
- },
- }
- })?;
- core::download_knowledge_pack(
- pack_root,
- &expected_dataset,
- move |progress| callback.on_progress(progress.into()),
- cancellation.inner.clone(),
- )
- .map_err(|error| KnowledgeDownloadError::Download {
- error: error.into(),
- })
+pub fn knowledge_pack_asset(stable_id: String) -> Result<Arc<Asset>, KnowledgeRetrievalError> {
+ let dataset = knowledge_dataset(&stable_id)?;
+ core::knowledge_asset(&dataset)
+ .map(Asset::new)
+ .map_err(|error| KnowledgeRetrievalError::InvalidInput {
+ detail: error.to_string(),
+ })
}
diff --git a/rust/crates/model-download/Cargo.toml b/rust/crates/assets/Cargo.toml
similarity index 69%
rename from rust/crates/model-download/Cargo.toml
rename to rust/crates/assets/Cargo.toml
index 0e696435dc..0c1cf76068 100644
--- a/rust/crates/model-download/Cargo.toml
+++ b/rust/crates/assets/Cargo.toml
@@ -1,5 +1,5 @@
[package]
-name = "ente-model-download"
+name = "ente-assets"
edition = "2024"
[dependencies]
@@ -11,4 +11,7 @@ serde_json.workspace = true
sha2 = "0.11.0"
tar = "0.4.46"
thiserror.workspace = true
-tokio = { version = "1", features = ["rt", "time"] }
+tokio = { version = "1", features = ["sync"] }
+
+[dev-dependencies]
+tokio = { version = "1", features = ["macros", "rt"] }
diff --git a/rust/crates/assets/src/archive.rs b/rust/crates/assets/src/archive.rs
new file mode 100644
index 0000000000..261c66ac40
--- /dev/null
+++ b/rust/crates/assets/src/archive.rs
@@ -0,0 +1,75 @@
+use std::fs;
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::{Component, Path, PathBuf};
+
+use flate2::read::GzDecoder;
+use tar::Archive;
+
+use crate::download::{CancellationToken, Error};
+
+pub(crate) fn extract_tar_gz(
+ archive_path: &Path,
+ staging_dir: &Path,
+ cancellation: &CancellationToken,
+) -> Result<PathBuf, Error> {
+ if staging_dir.exists() {
+ fs::remove_dir_all(staging_dir)?;
+ }
+ fs::create_dir_all(staging_dir)?;
+
+ let tar_gz = File::open(archive_path)?;
+ let mut archive = Archive::new(GzDecoder::new(tar_gz));
+ for entry in archive.entries()? {
+ if cancellation.is_cancelled() {
+ return Err(Error::Cancelled);
+ }
+ let mut entry = entry?;
+ let entry_type = entry.header().entry_type();
+ if !entry_type.is_file() && !entry_type.is_dir() {
+ return Err(Error::Validation(
+ "tar archive contains a link or special file".to_string(),
+ ));
+ }
+ let path = entry.path()?.into_owned();
+ if path.components().any(|component| {
+ matches!(
+ component,
+ Component::ParentDir | Component::RootDir | Component::Prefix(_)
+ )
+ }) {
+ return Err(Error::Validation(
+ "tar archive contains an unsafe path".to_string(),
+ ));
+ }
+ let destination = staging_dir.join(path);
+ if entry_type.is_dir() {
+ fs::create_dir_all(destination)?;
+ continue;
+ }
+ if let Some(parent) = destination.parent() {
+ fs::create_dir_all(parent)?;
+ }
+ let mut output = File::create(destination)?;
+ let mut buffer = [0; 1024 * 1024];
+ loop {
+ if cancellation.is_cancelled() {
+ return Err(Error::Cancelled);
+ }
+ let count = entry.read(&mut buffer)?;
+ if count == 0 {
+ break;
+ }
+ output.write_all(&buffer[..count])?;
+ }
+ }
+
+ let entries = fs::read_dir(staging_dir)?.collect::<Result<Vec<_>, _>>()?;
+ if entries.is_empty() {
+ return Err(Error::Validation("tar archive is empty".to_string()));
+ }
+ Ok(match entries.as_slice() {
+ [only] if only.path().is_dir() => only.path(),
+ _ => staging_dir.to_path_buf(),
+ })
+}
diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/assets/src/download.rs
similarity index 87%
rename from rust/crates/model-download/src/download.rs
rename to rust/crates/assets/src/download.rs
index 1a9a6328a1..35268e3b56 100644
--- a/rust/crates/model-download/src/download.rs
+++ b/rust/crates/assets/src/download.rs
@@ -1,10 +1,8 @@
-use std::cell::RefCell;
use std::fs::{self, File, OpenOptions};
use std::io::{ErrorKind, Write};
use std::path::{Path, PathBuf};
-use std::rc::Rc;
-use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use futures_util::future::try_join_all;
@@ -12,16 +10,13 @@ use reqwest::header::{ACCEPT_RANGES, CONTENT_RANGE, ETAG, IF_RANGE, LAST_MODIFIE
use reqwest::{Client, Response, StatusCode};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
-use tokio::runtime::Builder;
-use tokio::time::timeout;
const MIN_RANGE_DOWNLOAD_BYTES: u64 = 1024 * 1024;
const MAX_ATTEMPTS: usize = 3;
const RANGE_DOWNLOAD_CONCURRENCY: usize = 4;
const PROGRESS_INTERVAL: Duration = Duration::from_millis(250);
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
-const RESPONSE_START_TIMEOUT: Duration = Duration::from_secs(30);
-const READ_STALL_TIMEOUT: Duration = Duration::from_secs(30);
+const READ_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Debug, thiserror::Error)]
pub enum Error {
@@ -69,7 +64,7 @@ impl From<std::io::Error> for Error {
}
#[derive(Debug, Clone)]
-pub struct Target {
+pub struct DownloadTarget {
pub label: String,
pub url: String,
pub sha256: String,
@@ -124,6 +119,7 @@ struct FileDownloadReport {
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PartialDownloadMetadata {
url: String,
+ sha256: String,
etag: Option<String>,
last_modified: Option<String>,
}
@@ -153,6 +149,7 @@ struct FileProgress {
#[derive(Debug, Clone, Serialize, Deserialize)]
struct RangeDownloadMetadata {
url: String,
+ sha256: String,
size_bytes: u64,
etag: Option<String>,
last_modified: Option<String>,
@@ -192,13 +189,9 @@ impl Downloader {
})
}
- pub(crate) fn client(&self) -> &Client {
- &self.client
- }
-
pub async fn download(
&self,
- targets: Vec<Target>,
+ targets: Vec<DownloadTarget>,
on_progress: impl FnMut(Progress) + Send,
cancellation: CancellationToken,
) -> Result<(), Error> {
@@ -207,50 +200,43 @@ impl Downloader {
})
.await
}
+
+ pub(crate) async fn estimated_download_size(&self, targets: &[DownloadTarget]) -> Option<u64> {
+ let mut remaining = 0u64;
+ for target in targets {
+ if target.destination.is_file() {
+ continue;
+ }
+ let probe = fetch_download_probe(&self.client, &target.url).await;
+ let total = probe.content_length?;
+ let downloaded =
+ existing_download_bytes(target, &target.destination, &probe, false).min(total);
+ remaining = remaining.saturating_add(total - downloaded);
+ }
+ Some(remaining)
+ }
}
fn build_client() -> Result<Client, Error> {
Client::builder()
.connect_timeout(CONNECT_TIMEOUT)
+ .read_timeout(READ_TIMEOUT)
.build()
.map_err(|err| Error::Network(err.to_string()))
}
-pub(crate) fn fetch(
- downloader: &Downloader,
- targets: Vec<Target>,
- on_progress: impl FnMut(Progress),
- is_cancelled: impl Fn() -> bool,
-) -> Result<(), Error> {
- let runtime = Builder::new_current_thread()
- .enable_io()
- .enable_time()
- .build()?;
- runtime.block_on(fetch_async(
- &downloader.client,
- targets,
- on_progress,
- is_cancelled,
- ))
-}
-
async fn fetch_async(
client: &Client,
- targets: Vec<Target>,
- on_progress: impl FnMut(Progress),
- is_cancelled: impl Fn() -> bool,
+ targets: Vec<DownloadTarget>,
+ mut on_progress: impl FnMut(Progress) + Send,
+ is_cancelled: impl Fn() -> bool + Sync,
) -> Result<(), Error> {
if targets.is_empty() {
return Ok(());
}
for target in &targets {
- let sha256 = &target.sha256;
- if sha256.len() != 64 || !sha256.chars().all(|c| c.is_ascii_hexdigit()) {
- return Err(Error::InvalidTarget(format!(
- "'{sha256}' is not a SHA-256 digest"
- )));
- }
+ validate_sha256(&target.sha256)?;
}
if is_cancelled() {
@@ -289,7 +275,7 @@ async fn fetch_async(
None
};
- let file_states = targets
+ let mut file_states = targets
.iter()
.zip(&download_probes)
.zip(&cached)
@@ -306,15 +292,12 @@ async fn fetch_async(
}
})
.collect::<Vec<_>>();
- let file_states = Rc::new(RefCell::new(file_states));
- let on_progress = Rc::new(RefCell::new(on_progress));
-
emit_progress(
ProgressPhase::Preparing,
Duration::ZERO,
total_bytes,
&file_states,
- &on_progress,
+ &mut on_progress,
);
for (index, target) in targets.iter().enumerate() {
@@ -337,11 +320,8 @@ async fn fetch_async(
&destination,
&download_probe,
|file_progress| {
- {
- let mut states = file_states.borrow_mut();
- if let Some(state) = states.get_mut(index) {
- *state = file_progress;
- }
+ if let Some(state) = file_states.get_mut(index) {
+ *state = file_progress;
}
emit_progress(
ProgressPhase::File {
@@ -352,7 +332,7 @@ async fn fetch_async(
download_started_at.elapsed(),
total_bytes,
&file_states,
- &on_progress,
+ &mut on_progress,
);
},
&is_cancelled,
@@ -366,15 +346,12 @@ async fn fetch_async(
},
})?;
- {
- let mut states = file_states.borrow_mut();
- if let Some(state) = states.get_mut(index) {
- state.downloaded_bytes = file_report.final_size;
- state.total_bytes = expected_file_total.or(Some(file_report.final_size));
- state.network_downloaded_bytes = file_report.network_downloaded_bytes;
- state.elapsed = file_report.elapsed;
- state.retry_count = file_report.retry_count;
- }
+ if let Some(state) = file_states.get_mut(index) {
+ state.downloaded_bytes = file_report.final_size;
+ state.total_bytes = expected_file_total.or(Some(file_report.final_size));
+ state.network_downloaded_bytes = file_report.network_downloaded_bytes;
+ state.elapsed = file_report.elapsed;
+ state.retry_count = file_report.retry_count;
}
emit_progress(
@@ -386,7 +363,7 @@ async fn fetch_async(
download_started_at.elapsed(),
total_bytes,
&file_states,
- &on_progress,
+ &mut on_progress,
);
}
emit_progress(
@@ -394,19 +371,32 @@ async fn fetch_async(
download_started_at.elapsed(),
total_bytes,
&file_states,
- &on_progress,
+ &mut on_progress,
);
Ok(())
}
+pub(crate) fn validate_sha256(sha256: &str) -> Result<(), Error> {
+ if sha256.len() == 64
+ && sha256
+ .chars()
+ .all(|character| character.is_ascii_hexdigit())
+ {
+ return Ok(());
+ }
+ Err(Error::InvalidTarget(format!(
+ "'{sha256}' is not a SHA-256 digest"
+ )))
+}
+
async fn download_file(
client: &Client,
- target: &Target,
+ target: &DownloadTarget,
destination: &Path,
download_probe: &DownloadProbe,
- mut on_progress: impl FnMut(FileProgress),
- is_cancelled: &impl Fn() -> bool,
+ mut on_progress: impl FnMut(FileProgress) + Send,
+ is_cancelled: &(impl Fn() -> bool + Sync),
) -> Result<FileDownloadReport, Error> {
let parent = destination
.parent()
@@ -465,11 +455,11 @@ async fn download_file(
async fn download_file_single(
client: &Client,
- target: &Target,
+ target: &DownloadTarget,
destination: &Path,
expected_file_total: Option<u64>,
- on_progress: &mut dyn FnMut(FileProgress),
- is_cancelled: &impl Fn() -> bool,
+ on_progress: &mut (dyn FnMut(FileProgress) + Send),
+ is_cancelled: &(impl Fn() -> bool + Sync),
) -> Result<FileDownloadReport, Error> {
let tmp_path = tmp_path_for(destination);
let partial_metadata_path = partial_metadata_path_for(destination);
@@ -484,7 +474,7 @@ async fn download_file_single(
let tmp_size = file_size(&tmp_path).unwrap_or(0);
let resume_validator = if tmp_size > 0 {
- partial_download_validator(destination, &target.url)
+ partial_download_validator(destination, target)
} else {
None
};
@@ -579,9 +569,9 @@ async fn download_file_single(
return Err(Error::Cancelled);
}
- let chunk = match timeout(READ_STALL_TIMEOUT, response.chunk()).await {
- Ok(Ok(chunk)) => chunk,
- Ok(Err(err)) => {
+ let chunk = match response.chunk().await {
+ Ok(chunk) => chunk,
+ Err(err) => {
file.flush().ok();
if attempt == MAX_ATTEMPTS {
return Err(Error::Network(err.to_string()));
@@ -590,18 +580,6 @@ async fn download_file_single(
retry_attempt = true;
break;
}
- Err(_) => {
- file.flush().ok();
- if attempt == MAX_ATTEMPTS {
- return Err(Error::Network(format!(
- "stalled for {} seconds",
- READ_STALL_TIMEOUT.as_secs()
- )));
- }
- retry_count = retry_count.saturating_add(1);
- retry_attempt = true;
- break;
- }
};
let Some(chunk) = chunk else {
break;
@@ -691,12 +669,12 @@ async fn download_file_single(
async fn download_file_ranged(
client: &Client,
- target: &Target,
+ target: &DownloadTarget,
destination: &Path,
total: u64,
response_metadata: Option<ResponseMetadata>,
- on_progress: &mut dyn FnMut(FileProgress),
- is_cancelled: &impl Fn() -> bool,
+ on_progress: &mut (dyn FnMut(FileProgress) + Send),
+ is_cancelled: &(impl Fn() -> bool + Sync),
) -> Result<FileDownloadReport, Error> {
let tmp_path = tmp_path_for(destination);
let range_metadata_path = range_metadata_path_for(destination);
@@ -711,7 +689,7 @@ async fn download_file_ranged(
.truncate(false)
.open(&tmp_path)?;
file.set_len(total)?;
- let file = Rc::new(file);
+ let file = Arc::new(file);
let range_states = range_metadata
.ranges
@@ -726,26 +704,26 @@ async fn download_file_ranged(
retry_count: 0,
})
.collect::<Vec<_>>();
- let range_states = Rc::new(RefCell::new(range_states));
- let range_metadata = Rc::new(RefCell::new(range_metadata));
- let on_progress = Rc::new(RefCell::new(on_progress));
+ let range_states = Arc::new(Mutex::new(range_states));
+ let range_metadata = Arc::new(Mutex::new(range_metadata));
+ let on_progress = Arc::new(Mutex::new(on_progress));
emit_range_file_progress(total, file_started_at, &range_states, &on_progress);
let mut downloads = Vec::new();
{
- let metadata = range_metadata.borrow();
+ let metadata = range_metadata.lock().expect("range metadata lock");
for (part_index, range) in metadata.ranges.iter().copied().enumerate() {
if range.complete {
continue;
}
- let file = Rc::clone(&file);
- let range_states = Rc::clone(&range_states);
- let range_metadata = Rc::clone(&range_metadata);
+ let file = Arc::clone(&file);
+ let range_states = Arc::clone(&range_states);
+ let range_metadata = Arc::clone(&range_metadata);
let range_metadata_path = range_metadata_path.clone();
let response_metadata = response_metadata.clone();
- let on_progress = Rc::clone(&on_progress);
+ let on_progress = Arc::clone(&on_progress);
downloads.push(async move {
download_range_part(
@@ -795,13 +773,12 @@ async fn download_file_ranged(
});
}
- let network_downloaded_bytes = range_states
- .borrow()
+ let states = range_states.lock().expect("range state lock");
+ let network_downloaded_bytes = states
.iter()
.map(|state| state.network_downloaded_bytes)
.sum::<u64>();
- let retry_count = range_states
- .borrow()
+ let retry_count = states
.iter()
.map(|state| state.retry_count)
.fold(0u32, u32::saturating_add);
@@ -817,18 +794,18 @@ async fn download_file_ranged(
#[allow(clippy::too_many_arguments)]
async fn download_range_part(
client: &Client,
- target: &Target,
- file: Rc<File>,
+ target: &DownloadTarget,
+ file: Arc<File>,
part_index: usize,
range: RangeDownloadPartMetadata,
total: u64,
response_metadata: Option<ResponseMetadata>,
- range_states: Rc<RefCell<Vec<RangePartState>>>,
- range_metadata: Rc<RefCell<RangeDownloadMetadata>>,
+ range_states: Arc<Mutex<Vec<RangePartState>>>,
+ range_metadata: Arc<Mutex<RangeDownloadMetadata>>,
range_metadata_path: PathBuf,
file_started_at: Instant,
- on_progress: Rc<RefCell<&mut dyn FnMut(FileProgress)>>,
- is_cancelled: &impl Fn() -> bool,
+ on_progress: Arc<Mutex<&mut (dyn FnMut(FileProgress) + Send)>>,
+ is_cancelled: &(impl Fn() -> bool + Sync),
) -> Result<(), Error> {
let range_len = range_download_len(range);
@@ -838,7 +815,7 @@ async fn download_range_part(
}
{
- let mut states = range_states.borrow_mut();
+ let mut states = range_states.lock().expect("range state lock");
if let Some(state) = states.get_mut(part_index) {
state.downloaded_bytes = 0;
}
@@ -884,9 +861,9 @@ async fn download_range_part(
return Err(Error::Cancelled);
}
- let chunk = match timeout(READ_STALL_TIMEOUT, response.chunk()).await {
- Ok(Ok(chunk)) => chunk,
- Ok(Err(err)) => {
+ let chunk = match response.chunk().await {
+ Ok(chunk) => chunk,
+ Err(err) => {
if attempt == MAX_ATTEMPTS {
return Err(Error::Network(err.to_string()));
}
@@ -894,17 +871,6 @@ async fn download_range_part(
retry_attempt = true;
break;
}
- Err(_) => {
- if attempt == MAX_ATTEMPTS {
- return Err(Error::Network(format!(
- "stalled for {} seconds",
- READ_STALL_TIMEOUT.as_secs()
- )));
- }
- increment_range_retry(&range_states, part_index);
- retry_attempt = true;
- break;
- }
};
let Some(chunk) = chunk else {
break;
@@ -925,7 +891,7 @@ async fn download_range_part(
downloaded_in_range = downloaded_in_range.saturating_add(chunk_len);
{
- let mut states = range_states.borrow_mut();
+ let mut states = range_states.lock().expect("range state lock");
if let Some(state) = states.get_mut(part_index) {
state.downloaded_bytes = downloaded_in_range;
state.network_downloaded_bytes =
@@ -957,7 +923,7 @@ async fn download_range_part(
}
{
- let mut states = range_states.borrow_mut();
+ let mut states = range_states.lock().expect("range state lock");
if let Some(state) = states.get_mut(part_index) {
state.downloaded_bytes = range_len;
}
@@ -984,14 +950,9 @@ async fn request_file(
.header(RANGE, format!("bytes={resume_from}-"))
.header(IF_RANGE, if_range);
}
- timeout(RESPONSE_START_TIMEOUT, request.send())
+ request
+ .send()
.await
- .map_err(|_| {
- Error::Network(format!(
- "request did not receive a response within {} seconds",
- RESPONSE_START_TIMEOUT.as_secs()
- ))
- })?
.map_err(|err| Error::Network(err.to_string()))
}
@@ -1007,22 +968,14 @@ async fn request_model_range(
if let Some(if_range) = if_range_header_value(response_metadata) {
request = request.header(IF_RANGE, if_range);
}
- timeout(RESPONSE_START_TIMEOUT, request.send())
+ request
+ .send()
.await
- .map_err(|_| {
- Error::Network(format!(
- "request did not receive a response within {} seconds",
- RESPONSE_START_TIMEOUT.as_secs()
- ))
- })?
.map_err(|err| Error::Network(err.to_string()))
}
async fn fetch_download_probe(client: &Client, url: &str) -> DownloadProbe {
- let response = timeout(RESPONSE_START_TIMEOUT, client.head(url).send())
- .await
- .ok()
- .and_then(Result::ok);
+ let response = client.head(url).send().await.ok();
let Some(response) = response else {
return DownloadProbe::default();
};
@@ -1054,16 +1007,12 @@ async fn fetch_download_probe(client: &Client, url: &str) -> DownloadProbe {
}
}
-pub(crate) async fn probe_content_length(client: &Client, url: &str) -> Option<u64> {
- fetch_download_probe(client, url).await.content_length
-}
-
fn should_use_range_download(total: u64, probe: &DownloadProbe) -> bool {
total >= MIN_RANGE_DOWNLOAD_BYTES && probe.supports_ranges
}
fn prepare_range_download_metadata(
- target: &Target,
+ target: &DownloadTarget,
destination: &Path,
total: u64,
response_metadata: Option<ResponseMetadata>,
@@ -1088,6 +1037,7 @@ fn prepare_range_download_metadata(
cleanup_range_download(destination);
let metadata = RangeDownloadMetadata {
url: target.url.clone(),
+ sha256: target.sha256.clone(),
size_bytes: total,
etag: response_metadata
.as_ref()
@@ -1130,12 +1080,13 @@ fn range_download_parts(total: u64, concurrency: usize) -> Vec<RangeDownloadPart
fn range_download_metadata_matches(
metadata: &RangeDownloadMetadata,
- target: &Target,
+ target: &DownloadTarget,
total: u64,
response_metadata: Option<&ResponseMetadata>,
ranges: &[RangeDownloadPartMetadata],
) -> bool {
metadata.url == target.url
+ && metadata.sha256.eq_ignore_ascii_case(&target.sha256)
&& metadata.size_bytes == total
&& range_download_validators_match(metadata, response_metadata)
&& metadata.ranges.len() == ranges.len()
@@ -1171,10 +1122,10 @@ fn range_download_len(range: RangeDownloadPartMetadata) -> u64 {
fn emit_range_file_progress(
total_bytes: u64,
file_started_at: Instant,
- range_states: &Rc<RefCell<Vec<RangePartState>>>,
- on_progress: &Rc<RefCell<&mut dyn FnMut(FileProgress)>>,
+ range_states: &Arc<Mutex<Vec<RangePartState>>>,
+ on_progress: &Arc<Mutex<&mut (dyn FnMut(FileProgress) + Send)>>,
) {
- let states = range_states.borrow();
+ let states = range_states.lock().expect("range state lock");
let downloaded_bytes = states
.iter()
.map(|state| state.downloaded_bytes)
@@ -1190,7 +1141,7 @@ fn emit_range_file_progress(
.fold(0u32, u32::saturating_add);
drop(states);
- let mut callback = on_progress.borrow_mut();
+ let mut callback = on_progress.lock().expect("progress lock");
(**callback)(FileProgress {
downloaded_bytes,
total_bytes: Some(total_bytes),
@@ -1200,19 +1151,19 @@ fn emit_range_file_progress(
});
}
-fn increment_range_retry(range_states: &Rc<RefCell<Vec<RangePartState>>>, part_index: usize) {
- let mut states = range_states.borrow_mut();
+fn increment_range_retry(range_states: &Arc<Mutex<Vec<RangePartState>>>, part_index: usize) {
+ let mut states = range_states.lock().expect("range state lock");
if let Some(state) = states.get_mut(part_index) {
state.retry_count = state.retry_count.saturating_add(1);
}
}
fn mark_range_complete(
- range_metadata: &Rc<RefCell<RangeDownloadMetadata>>,
+ range_metadata: &Arc<Mutex<RangeDownloadMetadata>>,
range_metadata_path: &Path,
part_index: usize,
) -> Result<(), Error> {
- let mut metadata = range_metadata.borrow_mut();
+ let mut metadata = range_metadata.lock().expect("range metadata lock");
if let Some(range) = metadata.ranges.get_mut(part_index) {
range.complete = true;
}
@@ -1330,37 +1281,35 @@ fn emit_progress<F: FnMut(Progress)>(
phase: ProgressPhase,
elapsed: Duration,
total_bytes: Option<u64>,
- file_states: &Rc<RefCell<Vec<FileProgress>>>,
- on_progress: &Rc<RefCell<F>>,
+ file_states: &[FileProgress],
+ on_progress: &mut F,
) {
- let states = file_states.borrow();
- let downloaded_bytes = states
+ let downloaded_bytes = file_states
.iter()
.map(|state| state.downloaded_bytes)
.sum::<u64>();
- let network_downloaded_bytes = states
+ let network_downloaded_bytes = file_states
.iter()
.map(|state| state.network_downloaded_bytes)
.sum::<u64>();
- let retry_count = states
+ let retry_count = file_states
.iter()
.map(|state| state.retry_count)
.fold(0u32, u32::saturating_add);
let file_state = match &phase {
- ProgressPhase::File { index, .. } => states.get(*index).copied(),
+ ProgressPhase::File { index, .. } => file_states.get(*index).copied(),
_ => None,
};
let total_bytes = match phase {
ProgressPhase::Complete => total_bytes.or(Some(downloaded_bytes)),
_ => total_bytes.or_else(|| {
- let total = states
+ let total = file_states
.iter()
.filter_map(|state| state.total_bytes)
.sum::<u64>();
(total > 0).then_some(total)
}),
};
- drop(states);
let (label, file_complete, complete) = match phase {
ProgressPhase::Preparing => ("Preparing downloads", false, false),
@@ -1377,7 +1326,7 @@ fn emit_progress<F: FnMut(Progress)>(
.map(|state| (state.downloaded_bytes, state.total_bytes))
.unwrap_or((0, None));
- on_progress.borrow_mut()(Progress {
+ on_progress(Progress {
label: label.to_string(),
downloaded_bytes,
total_bytes,
@@ -1412,7 +1361,7 @@ fn bytes_per_second(bytes: u64, elapsed: Duration) -> f64 {
}
}
-fn prepare_cached_download(target: &Target, destination: &Path) -> bool {
+fn prepare_cached_download(target: &DownloadTarget, destination: &Path) -> bool {
if !destination.exists() || check_sha256(destination, &target.sha256, &target.label).is_err() {
return false;
}
@@ -1486,10 +1435,10 @@ fn partial_metadata_path_for(path: &Path) -> PathBuf {
PathBuf::from(format!("{}.tmp.partial.json", path.display()))
}
-fn partial_download_validator(destination: &Path, url: &str) -> Option<String> {
+fn partial_download_validator(destination: &Path, target: &DownloadTarget) -> Option<String> {
let text = fs::read_to_string(partial_metadata_path_for(destination)).ok()?;
let metadata: PartialDownloadMetadata = serde_json::from_str(&text).ok()?;
- if metadata.url != url {
+ if metadata.url != target.url || !metadata.sha256.eq_ignore_ascii_case(&target.sha256) {
return None;
}
metadata.etag.or(metadata.last_modified)
@@ -1497,11 +1446,12 @@ fn partial_download_validator(destination: &Path, url: &str) -> Option<String> {
fn write_partial_download_metadata(
destination: &Path,
- target: &Target,
+ target: &DownloadTarget,
response_metadata: &ResponseMetadata,
) -> Result<(), Error> {
let metadata = PartialDownloadMetadata {
url: target.url.clone(),
+ sha256: target.sha256.clone(),
etag: response_metadata.etag.clone(),
last_modified: response_metadata.last_modified.clone(),
};
@@ -1526,7 +1476,7 @@ fn now_ms() -> u64 {
}
fn existing_download_bytes(
- target: &Target,
+ target: &DownloadTarget,
destination: &Path,
probe: &DownloadProbe,
cached: bool,
@@ -1562,7 +1512,9 @@ fn existing_download_bytes(
return 0;
}
- 0
+ partial_download_validator(destination, target)
+ .and_then(|_| file_size(&tmp_path_for(destination)))
+ .unwrap_or(0)
}
fn file_size(path: &Path) -> Option<u64> {
@@ -1629,6 +1581,20 @@ mod tests {
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::thread;
+ async fn download(
+ targets: Vec<DownloadTarget>,
+ on_progress: impl FnMut(Progress) + Send,
+ is_cancelled: bool,
+ ) -> Result<(), Error> {
+ let cancellation = CancellationToken::default();
+ if is_cancelled {
+ cancellation.cancel();
+ }
+ Downloader::new()?
+ .download(targets, on_progress, cancellation)
+ .await
+ }
+
#[test]
fn range_download_parts_split_file_into_four_ranges() {
let ranges = range_download_parts(10, 4);
@@ -1702,8 +1668,8 @@ mod tests {
assert_eq!(parse_content_range("bytes 10-19/not-a-number"), None);
}
- #[test]
- fn download_uses_four_ranges_when_server_supports_ranges() {
+ #[tokio::test]
+ async fn download_uses_four_ranges_when_server_supports_ranges() {
let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));
let head_count = Arc::new(AtomicUsize::new(0));
let range_get_count = Arc::new(AtomicUsize::new(0));
@@ -1753,27 +1719,17 @@ mod tests {
let destination = test_dir.join("model.bin");
let url = format!("http://{address}/model.bin");
- let probe_client = Client::builder().build().expect("build probe client");
- let probe_runtime = Builder::new_current_thread()
- .enable_io()
- .enable_time()
- .build()
- .expect("build probe runtime");
- let probe = probe_runtime.block_on(fetch_download_probe(&probe_client, &url));
- assert_eq!(probe.content_length, Some(bytes.len() as u64));
- assert!(probe.supports_ranges);
-
- let result = fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ let result = download(
+ vec![DownloadTarget {
label: "Model".to_string(),
url,
destination: destination.clone(),
sha256: sha_hex(&bytes),
}],
|_| {},
- || false,
- );
+ false,
+ )
+ .await;
running.store(false, Ordering::SeqCst);
let _ = TcpStream::connect(address);
@@ -1784,7 +1740,7 @@ mod tests {
fs::read(&destination).expect("read downloaded file"),
*bytes
);
- assert_eq!(head_count.load(Ordering::SeqCst), 2);
+ assert_eq!(head_count.load(Ordering::SeqCst), 1);
assert_eq!(
range_get_count.load(Ordering::SeqCst),
RANGE_DOWNLOAD_CONCURRENCY,
@@ -1797,8 +1753,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn download_uses_single_stream_when_server_lacks_range_support() {
+ #[tokio::test]
+ async fn download_uses_single_stream_when_server_lacks_range_support() {
let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));
let get_count = Arc::new(AtomicUsize::new(0));
@@ -1813,17 +1769,17 @@ mod tests {
let test_dir = scratch_dir("single-download");
let destination = test_dir.join("model.bin");
- let result = fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ let result = download(
+ vec![DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: sha_hex(&bytes),
}],
|_| {},
- || false,
- );
+ false,
+ )
+ .await;
result.expect("single-stream download succeeds");
assert_eq!(
@@ -1835,8 +1791,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn downloads_files_sequentially() {
+ #[tokio::test]
+ async fn downloads_files_sequentially() {
let bytes = Arc::new(sample_bytes(512));
let active = Arc::new(AtomicUsize::new(0));
let max_active = Arc::new(AtomicUsize::new(0));
@@ -1859,27 +1815,23 @@ mod tests {
};
let test_dir = scratch_dir("sequential-downloads");
- let targets = ["model", "projector"].map(|label| Target {
+ let targets = ["model", "projector"].map(|label| DownloadTarget {
label: label.to_string(),
url: server.url(&format!("/{label}.bin")),
destination: test_dir.join(format!("{label}.bin")),
sha256: sha_hex(&bytes),
});
- fetch(
- &Downloader::new().unwrap(),
- targets.into(),
- |_| {},
- || false,
- )
- .expect("downloads succeed");
+ download(targets.into(), |_| {}, false)
+ .await
+ .expect("downloads succeed");
assert_eq!(max_active.load(Ordering::SeqCst), 1);
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn completed_download_is_served_from_cache() {
+ #[tokio::test]
+ async fn completed_download_is_served_from_cache() {
let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));
let head_count = Arc::new(AtomicUsize::new(0));
let range_get_count = Arc::new(AtomicUsize::new(0));
@@ -1903,25 +1855,22 @@ mod tests {
let test_dir = scratch_dir("cache-skip-download");
let destination = test_dir.join("model.bin");
- let target = Target {
+ let target = DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: sha_hex(&bytes),
};
- fetch(
- &Downloader::new().unwrap(),
- vec![target.clone()],
- |_| {},
- || false,
- )
- .expect("first download succeeds");
+ download(vec![target.clone()], |_| {}, false)
+ .await
+ .expect("first download succeeds");
let requests_after_first =
head_count.load(Ordering::SeqCst) + range_get_count.load(Ordering::SeqCst);
assert!(requests_after_first > 0);
- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)
+ download(vec![target], |_| {}, false)
+ .await
.expect("second download succeeds");
assert_eq!(
head_count.load(Ordering::SeqCst) + range_get_count.load(Ordering::SeqCst),
@@ -1933,8 +1882,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn cache_hit_is_revalidated_and_reheals_on_corruption() {
+ #[tokio::test]
+ async fn cache_hit_is_revalidated_and_reheals_on_corruption() {
let bytes = Arc::new(sample_bytes(512));
let get_count = Arc::new(AtomicUsize::new(0));
@@ -1948,27 +1897,24 @@ mod tests {
let test_dir = scratch_dir("revalidate-download");
let destination = test_dir.join("model.bin");
- let target = Target {
+ let target = DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: sha_hex(&bytes),
};
- fetch(
- &Downloader::new().unwrap(),
- vec![target.clone()],
- |_| {},
- || false,
- )
- .expect("first download succeeds");
+ download(vec![target.clone()], |_| {}, false)
+ .await
+ .expect("first download succeeds");
assert_eq!(get_count.load(Ordering::SeqCst), 1);
let mut data = fs::read(&destination).expect("read cached file");
data[0] = data[0].wrapping_add(1);
fs::write(&destination, &data).expect("corrupt cached file");
- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)
+ download(vec![target], |_| {}, false)
+ .await
.expect("second download re-heals the corrupt cache");
assert_eq!(
get_count.load(Ordering::SeqCst),
@@ -1980,8 +1926,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn existing_valid_file_without_sidecar_is_adopted() {
+ #[tokio::test]
+ async fn existing_valid_file_without_sidecar_is_adopted() {
let bytes = Arc::new(sample_bytes(512));
let get_count = Arc::new(AtomicUsize::new(0));
@@ -1997,14 +1943,15 @@ mod tests {
let destination = test_dir.join("model.bin");
fs::write(&destination, bytes.as_slice()).expect("place existing file");
- let target = Target {
+ let target = DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: sha_hex(&bytes),
};
- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)
+ download(vec![target], |_| {}, false)
+ .await
.expect("fetch adopts existing file");
assert_eq!(
@@ -2017,8 +1964,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn single_stream_download_resumes_from_partial_tmp() {
+ #[tokio::test]
+ async fn single_stream_download_resumes_from_partial_tmp() {
let bytes = Arc::new(sample_bytes(512));
let head_count = Arc::new(AtomicUsize::new(0));
let range_get_count = Arc::new(AtomicUsize::new(0));
@@ -2042,7 +1989,7 @@ mod tests {
let test_dir = scratch_dir("resume-download");
let destination = test_dir.join("model.bin");
- let target = Target {
+ let target = DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
@@ -2059,7 +2006,8 @@ mod tests {
)
.expect("write partial sidecar");
- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)
+ download(vec![target], |_| {}, false)
+ .await
.expect("resume succeeds");
assert_eq!(
@@ -2078,8 +2026,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn single_stream_download_restarts_when_resume_validator_is_stale() {
+ #[tokio::test]
+ async fn single_stream_download_restarts_when_resume_validator_is_stale() {
let bytes = Arc::new(sample_bytes(512));
let head_count = Arc::new(AtomicUsize::new(0));
let range_get_count = Arc::new(AtomicUsize::new(0));
@@ -2103,7 +2051,7 @@ mod tests {
let test_dir = scratch_dir("stale-resume-download");
let destination = test_dir.join("model.bin");
- let target = Target {
+ let target = DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
@@ -2120,7 +2068,8 @@ mod tests {
)
.expect("write stale partial sidecar");
- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)
+ download(vec![target], |_| {}, false)
+ .await
.expect("restart succeeds");
assert_eq!(
@@ -2137,8 +2086,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn single_stream_download_ignores_partial_without_sidecar() {
+ #[tokio::test]
+ async fn single_stream_download_ignores_partial_without_sidecar() {
let bytes = Arc::new(sample_bytes(512));
let get_count = Arc::new(AtomicUsize::new(0));
@@ -2154,14 +2103,15 @@ mod tests {
let destination = test_dir.join("model.bin");
fs::write(tmp_path_for(&destination), &bytes[..256]).expect("place orphan tmp");
- let target = Target {
+ let target = DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: sha_hex(&bytes),
};
- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)
+ download(vec![target], |_| {}, false)
+ .await
.expect("fresh download succeeds");
assert_eq!(
@@ -2177,8 +2127,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn single_stream_resume_restarts_when_server_resumes_from_wrong_offset() {
+ #[tokio::test]
+ async fn single_stream_resume_restarts_when_server_resumes_from_wrong_offset() {
let bytes = Arc::new(sample_bytes(512));
let bad_resume_count = Arc::new(AtomicUsize::new(0));
let full_get_count = Arc::new(AtomicUsize::new(0));
@@ -2199,7 +2149,7 @@ mod tests {
let test_dir = scratch_dir("wrong-offset-resume");
let destination = test_dir.join("model.bin");
- let target = Target {
+ let target = DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
@@ -2216,7 +2166,8 @@ mod tests {
)
.expect("write partial sidecar");
- fetch(&Downloader::new().unwrap(), vec![target], |_| {}, || false)
+ download(vec![target], |_| {}, false)
+ .await
.expect("restart after bad 206 succeeds");
assert_eq!(
@@ -2287,30 +2238,30 @@ mod tests {
}
}
- #[test]
- fn cancelled_fetch_returns_the_cancelled_variant() {
+ #[tokio::test]
+ async fn cancelled_fetch_returns_the_cancelled_variant() {
let test_dir = scratch_dir("cancel-download");
let destination = test_dir.join("model.bin");
- let result = fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ let result = download(
+ vec![DownloadTarget {
label: "Model".to_string(),
url: "http://127.0.0.1:1/model.bin".to_string(),
destination: destination.clone(),
sha256: "0".repeat(64),
}],
|_| {},
- || true,
- );
+ true,
+ )
+ .await;
assert!(matches!(result, Err(Error::Cancelled)));
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn cached_download_adoption_removes_stale_partials() {
+ #[tokio::test]
+ async fn cached_download_adoption_removes_stale_partials() {
let test_dir = scratch_dir("cached-adoption");
let destination = test_dir.join("model.bin");
fs::write(&destination, b"complete-model").expect("write model");
@@ -2318,17 +2269,17 @@ mod tests {
fs::write(range_metadata_path_for(&destination), "{}").expect("write ranges sidecar");
fs::write(partial_metadata_path_for(&destination), "{}").expect("write partial sidecar");
- let result = fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ let result = download(
+ vec![DownloadTarget {
label: "Model".to_string(),
url: "http://127.0.0.1:1/model.bin".to_string(),
destination: destination.clone(),
sha256: sha_hex(b"complete-model"),
}],
|_| {},
- || false,
- );
+ false,
+ )
+ .await;
assert!(result.is_ok());
assert!(destination.exists());
@@ -2339,8 +2290,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn sha_pinned_download_verifies_content() {
+ #[tokio::test]
+ async fn sha_pinned_download_verifies_content() {
let bytes = Arc::new(sample_bytes(2048));
let get_count = Arc::new(AtomicUsize::new(0));
let server = {
@@ -2358,17 +2309,17 @@ mod tests {
expected.push_str(&format!("{byte:02x}"));
}
- fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ download(
+ vec![DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: expected,
}],
|_| {},
- || false,
+ false,
)
+ .await
.expect("sha-pinned download succeeds");
assert_eq!(
fs::read(&destination).expect("read downloaded file"),
@@ -2378,8 +2329,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn sha_mismatch_fails_immediately_without_installing() {
+ #[tokio::test]
+ async fn sha_mismatch_fails_immediately_without_installing() {
let bytes = Arc::new(sample_bytes(2048));
let get_count = Arc::new(AtomicUsize::new(0));
let server = {
@@ -2393,17 +2344,17 @@ mod tests {
let test_dir = scratch_dir("sha-mismatch");
let destination = test_dir.join("model.bin");
- let result = fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ let result = download(
+ vec![DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: "0".repeat(64),
}],
|_| {},
- || false,
- );
+ false,
+ )
+ .await;
assert!(result.is_err());
assert!(!destination.exists());
@@ -2412,8 +2363,8 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn ranged_sha_mismatch_fails_without_single_stream_fallback() {
+ #[tokio::test]
+ async fn ranged_sha_mismatch_fails_without_single_stream_fallback() {
let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));
let head_count = Arc::new(AtomicUsize::new(0));
let range_get_count = Arc::new(AtomicUsize::new(0));
@@ -2437,17 +2388,17 @@ mod tests {
let test_dir = scratch_dir("ranged-sha-mismatch");
let destination = test_dir.join("model.bin");
- let result = fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ let result = download(
+ vec![DownloadTarget {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination: destination.clone(),
sha256: "0".repeat(64),
}],
|_| {},
- || false,
- );
+ false,
+ )
+ .await;
assert!(result.is_err());
assert!(!destination.exists());
@@ -2461,40 +2412,40 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
- #[test]
- fn sha_pinned_target_adopts_cached_file_regardless_of_url() {
+ #[tokio::test]
+ async fn sha_pinned_target_adopts_cached_file_regardless_of_url() {
let test_dir = scratch_dir("sha-cached");
let destination = test_dir.join("model.bin");
fs::write(&destination, b"model-bytes").expect("write cached file");
let sha = sha256_file(&destination).expect("hash cached file");
- let old_target = Target {
+ let old_target = DownloadTarget {
label: "Model".to_string(),
url: "http://cache.example.org/old-location".to_string(),
destination: destination.clone(),
sha256: sha.clone(),
};
- fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ download(
+ vec![DownloadTarget {
url: "http://127.0.0.1:9/new-location".to_string(),
..old_target.clone()
}],
|_| {},
- || false,
+ false,
)
+ .await
.expect("cached file adopted by checksum despite changed URL");
fs::write(&destination, b"corrupted").expect("corrupt cached file");
- let result = fetch(
- &Downloader::new().unwrap(),
- vec![Target {
+ let result = download(
+ vec![DownloadTarget {
url: "http://127.0.0.1:9/new-location".to_string(),
..old_target
}],
|_| {},
- || false,
- );
+ false,
+ )
+ .await;
assert!(result.is_err());
let _ = fs::remove_dir_all(test_dir);
diff --git a/rust/crates/assets/src/lib.rs b/rust/crates/assets/src/lib.rs
new file mode 100644
index 0000000000..a4db6103d2
--- /dev/null
+++ b/rust/crates/assets/src/lib.rs
@@ -0,0 +1,5 @@
+mod archive;
+pub mod download;
+mod store;
+
+pub use store::{Asset, AssetFile, AssetStore};
diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rs
new file mode 100644
index 0000000000..b5f16fa380
--- /dev/null
+++ b/rust/crates/assets/src/store.rs
@@ -0,0 +1,605 @@
+use std::collections::HashSet;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
+
+use tokio::sync::Mutex;
+
+use crate::archive;
+use crate::download::{self, CancellationToken, DownloadTarget, Downloader, Error, Progress};
+
+const STAGING_DIR: &str = ".staging";
+const ARCHIVE_FILE: &str = ".archive.tar.gz";
+const EXTRACTION_DIR: &str = ".extracted";
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Asset {
+ components: Vec<String>,
+ kind: AssetKind,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum AssetKind {
+ Files(Vec<AssetFile>),
+ TarGz { url: String, sha256: String },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AssetFile {
+ pub name: String,
+ pub url: String,
+ pub sha256: String,
+}
+
+impl Asset {
+ pub fn files(components: Vec<String>, files: Vec<AssetFile>) -> Result<Self, Error> {
+ validate_components(&components)?;
+ if files.is_empty() {
+ return Err(Error::InvalidTarget("asset has no files".to_string()));
+ }
+ let mut names = HashSet::new();
+ for file in &files {
+ validate_component(&file.name)?;
+ download::validate_sha256(&file.sha256)?;
+ if !names.insert(file.name.as_str()) {
+ return Err(Error::InvalidTarget(format!(
+ "asset has a duplicate file name {}",
+ file.name
+ )));
+ }
+ }
+ Ok(Self {
+ components,
+ kind: AssetKind::Files(files),
+ })
+ }
+
+ pub fn tar_gz(components: Vec<String>, url: String, sha256: String) -> Result<Self, Error> {
+ validate_components(&components)?;
+ download::validate_sha256(&sha256)?;
+ Ok(Self {
+ components,
+ kind: AssetKind::TarGz { url, sha256 },
+ })
+ }
+}
+
+pub struct AssetStore {
+ root: PathBuf,
+ downloader: OnceLock<Downloader>,
+ mutation_gate: Mutex<()>,
+}
+
+impl AssetStore {
+ pub fn new(root: impl Into<PathBuf>) -> Self {
+ Self {
+ root: root.into(),
+ downloader: OnceLock::new(),
+ mutation_gate: Mutex::new(()),
+ }
+ }
+
+ pub fn asset_dir(&self, asset: &Asset) -> PathBuf {
+ join_components(&self.root, &asset.components)
+ }
+
+ pub fn file_path(&self, asset: &Asset, name: &str) -> Option<PathBuf> {
+ match &asset.kind {
+ AssetKind::Files(files) if files.iter().any(|file| file.name == name) => {
+ Some(self.asset_dir(asset).join(name))
+ }
+ _ => None,
+ }
+ }
+
+ pub fn is_downloaded(&self, asset: &Asset) -> bool {
+ let directory = self.asset_dir(asset);
+ match &asset.kind {
+ AssetKind::Files(files) => files
+ .iter()
+ .all(|file| is_regular_file(&directory.join(&file.name))),
+ AssetKind::TarGz { .. } => is_directory(&directory),
+ }
+ }
+
+ pub async fn estimated_download_size(&self, asset: &Asset) -> Option<u64> {
+ if self.is_downloaded(asset) {
+ return Some(0);
+ }
+ self.downloader()
+ .ok()?
+ .estimated_download_size(&self.download_targets(asset))
+ .await
+ }
+
+ pub async fn download(
+ &self,
+ asset: &Asset,
+ mut on_progress: impl FnMut(Progress) + Send,
+ cancellation: CancellationToken,
+ ) -> Result<(), Error> {
+ let _guard = self.mutation_gate.lock().await;
+ if self.is_downloaded(asset) {
+ return Ok(());
+ }
+ if cancellation.is_cancelled() {
+ return Err(Error::Cancelled);
+ }
+
+ let mut last_progress = None;
+ self.downloader()?
+ .download(
+ self.download_targets(asset),
+ |mut progress| {
+ progress.complete = false;
+ last_progress = Some(progress.clone());
+ on_progress(progress);
+ },
+ cancellation.clone(),
+ )
+ .await?;
+
+ match &asset.kind {
+ AssetKind::Files(files) => self.publish_files(asset, files)?,
+ AssetKind::TarGz { .. } => self.publish_archive(asset, cancellation).await?,
+ }
+
+ if let Some(mut progress) = last_progress {
+ progress.percentage = 100.0;
+ progress.file_complete = false;
+ progress.complete = true;
+ on_progress(progress);
+ }
+ Ok(())
+ }
+
+ pub async fn remove(&self, asset: &Asset) -> Result<(), Error> {
+ let _guard = self.mutation_gate.lock().await;
+ remove_path(&self.asset_dir(asset))?;
+ remove_path(&self.staging_dir(asset))?;
+ Ok(())
+ }
+
+ fn downloader(&self) -> Result<&Downloader, Error> {
+ if let Some(downloader) = self.downloader.get() {
+ return Ok(downloader);
+ }
+ let downloader = Downloader::new()?;
+ Ok(self.downloader.get_or_init(|| downloader))
+ }
+
+ fn staging_dir(&self, asset: &Asset) -> PathBuf {
+ join_components(&self.root.join(STAGING_DIR), &asset.components)
+ }
+
+ fn download_targets(&self, asset: &Asset) -> Vec<DownloadTarget> {
+ let staging = self.staging_dir(asset);
+ match &asset.kind {
+ AssetKind::Files(files) => files
+ .iter()
+ .map(|file| DownloadTarget {
+ label: file.name.clone(),
+ url: file.url.clone(),
+ sha256: file.sha256.clone(),
+ destination: staging.join(&file.name),
+ })
+ .collect(),
+ AssetKind::TarGz { url, sha256 } => vec![DownloadTarget {
+ label: asset.components.last().cloned().unwrap_or_default(),
+ url: url.clone(),
+ sha256: sha256.clone(),
+ destination: staging.join(ARCHIVE_FILE),
+ }],
+ }
+ }
+
+ fn publish_files(&self, asset: &Asset, files: &[AssetFile]) -> Result<(), Error> {
+ let staging = self.staging_dir(asset);
+ for file in files {
+ if !is_regular_file(&staging.join(&file.name)) {
+ return Err(Error::Validation(format!(
+ "{} is not a regular file",
+ staging.join(&file.name).display()
+ )));
+ }
+ }
+ let expected = files
+ .iter()
+ .map(|file| file.name.as_str())
+ .collect::<HashSet<_>>();
+ for entry in fs::read_dir(&staging)? {
+ let entry = entry?;
+ let name = entry.file_name();
+ if name.to_str().is_some_and(|name| expected.contains(name)) {
+ if !is_regular_file(&entry.path()) {
+ return Err(Error::Validation(format!(
+ "{} is not a regular file",
+ entry.path().display()
+ )));
+ }
+ } else {
+ remove_path(&entry.path())?;
+ }
+ }
+ self.publish_directory(&staging, &self.asset_dir(asset))
+ }
+
+ async fn publish_archive(
+ &self,
+ asset: &Asset,
+ cancellation: CancellationToken,
+ ) -> Result<(), Error> {
+ let staging = self.staging_dir(asset);
+ let archive_path = staging.join(ARCHIVE_FILE);
+ let extraction_dir = staging.join(EXTRACTION_DIR);
+ let extraction_for_task = extraction_dir.clone();
+ let extraction_cancellation = cancellation.clone();
+ let source = tokio::task::spawn_blocking(move || {
+ let result = archive::extract_tar_gz(
+ &archive_path,
+ &extraction_for_task,
+ &extraction_cancellation,
+ );
+ if result.is_err() {
+ let _ = fs::remove_dir_all(&extraction_for_task);
+ }
+ result
+ })
+ .await
+ .map_err(|error| Error::Io(std::io::Error::other(error)))??;
+ if cancellation.is_cancelled() {
+ remove_path(&extraction_dir)?;
+ return Err(Error::Cancelled);
+ }
+ self.publish_directory(&source, &self.asset_dir(asset))?;
+ remove_path(&staging)?;
+ Ok(())
+ }
+
+ fn publish_directory(&self, source: &Path, destination: &Path) -> Result<(), Error> {
+ let parent = destination
+ .parent()
+ .ok_or_else(|| Error::InvalidTarget(destination.display().to_string()))?;
+ fs::create_dir_all(parent)?;
+ remove_path(destination)?;
+ fs::rename(source, destination)?;
+ Ok(())
+ }
+}
+
+fn validate_components(components: &[String]) -> Result<(), Error> {
+ if components.is_empty() {
+ return Err(Error::InvalidTarget(
+ "asset key has no components".to_string(),
+ ));
+ }
+ for component in components {
+ validate_component(component)?;
+ }
+ if components[0] == STAGING_DIR {
+ return Err(Error::InvalidTarget(format!("'{STAGING_DIR}' is reserved")));
+ }
+ Ok(())
+}
+
+fn validate_component(component: &str) -> Result<(), Error> {
+ if !component.is_empty()
+ && component != "."
+ && component != ".."
+ && !component.contains('/')
+ && !component.contains('\\')
+ {
+ return Ok(());
+ }
+ Err(Error::InvalidTarget(format!(
+ "'{component}' is not a safe path component"
+ )))
+}
+
+fn join_components(root: &Path, components: &[String]) -> PathBuf {
+ components
+ .iter()
+ .fold(root.to_path_buf(), |path, component| path.join(component))
+}
+
+fn is_regular_file(path: &Path) -> bool {
+ fs::symlink_metadata(path)
+ .map(|metadata| metadata.file_type().is_file())
+ .unwrap_or(false)
+}
+
+fn is_directory(path: &Path) -> bool {
+ fs::symlink_metadata(path)
+ .map(|metadata| metadata.file_type().is_dir())
+ .unwrap_or(false)
+}
+
+fn remove_path(path: &Path) -> Result<(), Error> {
+ let metadata = match fs::symlink_metadata(path) {
+ Ok(metadata) => metadata,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
+ Err(error) => return Err(error.into()),
+ };
+ if metadata.file_type().is_dir() {
+ fs::remove_dir_all(path)?;
+ } else {
+ fs::remove_file(path)?;
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use std::io::Cursor;
+ use std::time::{SystemTime, UNIX_EPOCH};
+
+ use flate2::Compression;
+ use flate2::write::GzEncoder;
+ use sha2::{Digest, Sha256};
+ use tar::{Builder, EntryType, Header};
+
+ use super::*;
+
+ #[test]
+ fn store_and_operations_are_send_and_sync() {
+ fn assert_send<T: Send>(_: T) {}
+ fn assert_send_sync<T: Send + Sync>() {}
+
+ assert_send_sync::<AssetStore>();
+ assert_send_sync::<CancellationToken>();
+
+ let store = AssetStore::new("assets");
+ let asset =
+ Asset::files(key(&["models", "clip"]), vec![file("model.onnx", b"model")]).unwrap();
+ assert_send(store.download(&asset, |_| {}, CancellationToken::default()));
+ assert_send(store.estimated_download_size(&asset));
+ assert_send(store.remove(&asset));
+ }
+
+ #[test]
+ fn rejects_invalid_assets() {
+ assert!(Asset::files(Vec::new(), vec![file("model", b"model")]).is_err());
+ assert!(Asset::files(key(&[".staging", "model"]), vec![file("model", b"model")]).is_err());
+ assert!(Asset::files(key(&["models", ".."]), vec![file("model", b"model")]).is_err());
+ assert!(Asset::files(key(&["models", "model"]), Vec::new()).is_err());
+ assert!(
+ Asset::files(
+ key(&["models", "model"]),
+ vec![file("model", b"a"), file("model", b"b")]
+ )
+ .is_err()
+ );
+ let mut invalid_sha = file("model", b"model");
+ invalid_sha.sha256 = "invalid".to_string();
+ assert!(Asset::files(key(&["models", "model"]), vec![invalid_sha]).is_err());
+ }
+
+ #[tokio::test]
+ async fn publishes_all_files_before_reporting_completion() {
+ let root = scratch_dir("files");
+ let store = AssetStore::new(&root);
+ let asset = Asset::files(
+ key(&["models", "clip"]),
+ vec![file("model.onnx", b"model"), file("vocab.txt", b"vocab")],
+ )
+ .unwrap();
+ stage_files(
+ &store,
+ &asset,
+ &[("model.onnx", b"model"), ("vocab.txt", b"vocab")],
+ );
+
+ let final_dir = store.asset_dir(&asset);
+ let mut completion_observed_after_publish = false;
+ store
+ .download(
+ &asset,
+ |progress| {
+ if progress.complete {
+ completion_observed_after_publish = final_dir.is_dir();
+ }
+ },
+ CancellationToken::default(),
+ )
+ .await
+ .unwrap();
+
+ assert!(completion_observed_after_publish);
+ assert!(store.is_downloaded(&asset));
+ assert_eq!(fs::read(final_dir.join("model.onnx")).unwrap(), b"model");
+ assert_eq!(fs::read(final_dir.join("vocab.txt")).unwrap(), b"vocab");
+ assert!(!store.staging_dir(&asset).exists());
+ let _ = fs::remove_dir_all(root);
+ }
+
+ #[tokio::test]
+ async fn ignores_derived_files_and_discards_them_when_rebuilding() {
+ let root = scratch_dir("derived");
+ let store = AssetStore::new(&root);
+ let asset = Asset::files(
+ key(&["onnx-runtime", "macos"]),
+ vec![file("runtime.tgz", b"archive")],
+ )
+ .unwrap();
+ stage_files(&store, &asset, &[("runtime.tgz", b"archive")]);
+ store
+ .download(&asset, |_| {}, CancellationToken::default())
+ .await
+ .unwrap();
+
+ let directory = store.asset_dir(&asset);
+ fs::write(directory.join("libonnxruntime.dylib"), b"derived").unwrap();
+ assert!(store.is_downloaded(&asset));
+
+ fs::remove_file(directory.join("runtime.tgz")).unwrap();
+ assert!(!store.is_downloaded(&asset));
+ stage_files(&store, &asset, &[("runtime.tgz", b"archive")]);
+ store
+ .download(&asset, |_| {}, CancellationToken::default())
+ .await
+ .unwrap();
+
+ assert!(!directory.join("libonnxruntime.dylib").exists());
+ assert!(store.is_downloaded(&asset));
+ let _ = fs::remove_dir_all(root);
+ }
+
+ #[tokio::test]
+ async fn extracts_and_publishes_tar_gz() {
+ let root = scratch_dir("archive");
+ let store = AssetStore::new(&root);
+ let archive = tar_gz(
+ "parakeet",
+ &[("encoder.onnx", b"encoder"), ("tokens.txt", b"tokens")],
+ );
+ let asset = Asset::tar_gz(
+ key(&["models", "parakeet"]),
+ "https://example.invalid/parakeet.tar.gz".to_string(),
+ sha(&archive),
+ )
+ .unwrap();
+ let staging = store.staging_dir(&asset);
+ fs::create_dir_all(&staging).unwrap();
+ fs::write(staging.join(ARCHIVE_FILE), archive).unwrap();
+
+ store
+ .download(&asset, |_| {}, CancellationToken::default())
+ .await
+ .unwrap();
+
+ let directory = store.asset_dir(&asset);
+ assert_eq!(
+ fs::read(directory.join("encoder.onnx")).unwrap(),
+ b"encoder"
+ );
+ assert_eq!(fs::read(directory.join("tokens.txt")).unwrap(), b"tokens");
+ assert!(!staging.exists());
+ let _ = fs::remove_dir_all(root);
+ }
+
+ #[tokio::test]
+ async fn rejects_tar_links_without_publishing() {
+ let root = scratch_dir("archive-link");
+ let store = AssetStore::new(&root);
+ let archive = tar_gz_with_link();
+ let asset = Asset::tar_gz(
+ key(&["models", "linked"]),
+ "https://example.invalid/linked.tar.gz".to_string(),
+ sha(&archive),
+ )
+ .unwrap();
+ let staging = store.staging_dir(&asset);
+ fs::create_dir_all(&staging).unwrap();
+ fs::write(staging.join(ARCHIVE_FILE), archive).unwrap();
+
+ assert!(
+ store
+ .download(&asset, |_| {}, CancellationToken::default())
+ .await
+ .is_err()
+ );
+ assert!(!store.asset_dir(&asset).exists());
+ assert!(staging.join(ARCHIVE_FILE).is_file());
+ let _ = fs::remove_dir_all(root);
+ }
+
+ #[tokio::test]
+ async fn remove_deletes_published_and_staged_data() {
+ let root = scratch_dir("remove");
+ let store = AssetStore::new(&root);
+ let asset =
+ Asset::files(key(&["models", "clip"]), vec![file("model.onnx", b"model")]).unwrap();
+ let final_dir = store.asset_dir(&asset);
+ let staging = store.staging_dir(&asset);
+ fs::create_dir_all(&final_dir).unwrap();
+ fs::write(final_dir.join("model.onnx"), b"model").unwrap();
+ fs::create_dir_all(&staging).unwrap();
+ fs::write(staging.join("model.onnx.tmp"), b"partial").unwrap();
+
+ store.remove(&asset).await.unwrap();
+ assert!(!final_dir.exists());
+ assert!(!staging.exists());
+ store.remove(&asset).await.unwrap();
+ let _ = fs::remove_dir_all(root);
+ }
+
+ fn key(components: &[&str]) -> Vec<String> {
+ components
+ .iter()
+ .map(|component| component.to_string())
+ .collect()
+ }
+
+ fn file(name: &str, bytes: &[u8]) -> AssetFile {
+ AssetFile {
+ name: name.to_string(),
+ url: format!("https://example.invalid/{name}"),
+ sha256: sha(bytes),
+ }
+ }
+
+ fn stage_files(store: &AssetStore, asset: &Asset, files: &[(&str, &[u8])]) {
+ let directory = store.staging_dir(asset);
+ fs::create_dir_all(&directory).unwrap();
+ for (name, bytes) in files {
+ fs::write(directory.join(name), bytes).unwrap();
+ }
+ }
+
+ fn sha(bytes: &[u8]) -> String {
+ Sha256::digest(bytes)
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
+ }
+
+ fn tar_gz(root: &str, files: &[(&str, &[u8])]) -> Vec<u8> {
+ let mut bytes = Vec::new();
+ {
+ let encoder = GzEncoder::new(&mut bytes, Compression::default());
+ let mut archive = Builder::new(encoder);
+ for (name, contents) in files {
+ let mut header = Header::new_gnu();
+ header.set_size(contents.len() as u64);
+ header.set_mode(0o644);
+ header.set_cksum();
+ archive
+ .append_data(
+ &mut header,
+ format!("{root}/{name}"),
+ Cursor::new(*contents),
+ )
+ .unwrap();
+ }
+ archive.into_inner().unwrap().finish().unwrap();
+ }
+ bytes
+ }
+
+ fn tar_gz_with_link() -> Vec<u8> {
+ let mut bytes = Vec::new();
+ {
+ let encoder = GzEncoder::new(&mut bytes, Compression::default());
+ let mut archive = Builder::new(encoder);
+ let mut header = Header::new_gnu();
+ header.set_entry_type(EntryType::Symlink);
+ header.set_size(0);
+ header.set_mode(0o777);
+ header.set_link_name("../outside").unwrap();
+ header.set_cksum();
+ archive
+ .append_data(&mut header, "model/link", Cursor::new(Vec::<u8>::new()))
+ .unwrap();
+ archive.into_inner().unwrap().finish().unwrap();
+ }
+ bytes
+ }
+
+ fn scratch_dir(name: &str) -> PathBuf {
+ let suffix = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_nanos();
+ std::env::temp_dir().join(format!("ente-assets-{name}-{suffix}"))
+ }
+}
diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.toml
index 23dda9264a..01dd009119 100644
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@ -3,9 +3,9 @@ name = "ente-ensu"
edition = "2024"
[dependencies]
+ente-assets.workspace = true
ente-core.workspace = true
ente-image.workspace = true
-ente-model-download.workspace = true
memmap2 = "0.9.9"
regex = { version = "1", optional = true }
rubato = { version = "0.16", optional = true }
@@ -13,7 +13,6 @@ rusqlite = { version = "0.38", features = ["bundled"], optional = true }
self_cell = "1"
serde.workspace = true
serde_json.workspace = true
-sha2 = "0.11.0"
thiserror.workspace = true
tokio = { version = "1", features = ["rt", "time"] }
transcribe-rs = { version = "0.3.11", default-features = false, features = ["onnx", "vad-silero"], optional = true }
diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rs
index 88a4d51582..b98c44d9bf 100644
--- a/rust/crates/ensu/src/config.rs
+++ b/rust/crates/ensu/src/config.rs
@@ -209,6 +209,7 @@ mod tests {
preset.id
);
}
+ assert!(seen.insert(knowledge::knowledge_embedding_config().target_id));
}
#[test]
diff --git a/rust/crates/ensu/src/config/knowledge.rs b/rust/crates/ensu/src/config/knowledge.rs
index 35ae281cf5..41cb2d2c6d 100644
--- a/rust/crates/ensu/src/config/knowledge.rs
+++ b/rust/crates/ensu/src/config/knowledge.rs
@@ -183,6 +183,8 @@ pub(crate) fn is_path_safe_component(value: &str) -> bool {
#[cfg(test)]
mod tests {
+ use std::collections::HashSet;
+
use super::*;
#[test]
@@ -204,6 +206,14 @@ mod tests {
let embedding = knowledge_embedding_config();
assert_eq!(datasets.len(), 2);
+ assert_eq!(
+ datasets
+ .iter()
+ .map(|dataset| dataset.stable_id.as_str())
+ .collect::<HashSet<_>>()
+ .len(),
+ datasets.len()
+ );
assert_eq!(datasets[0].stable_id, "simplewiki");
assert_eq!(datasets[1].stable_id, "wikibooks");
assert!(
diff --git a/rust/crates/ensu/src/model/migrations.rs b/rust/crates/ensu/src/model/migrations.rs
index 425005a7ee..531d943fb4 100644
--- a/rust/crates/ensu/src/model/migrations.rs
+++ b/rust/crates/ensu/src/model/migrations.rs
@@ -3,51 +3,83 @@ use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
-use ente_model_download::{ModelDownloader, ModelTarget};
-use sha2::{Digest, Sha256};
+use ente_assets::{Asset, AssetStore};
use crate::config;
use crate::model::{self, InvalidPreset, trimmed};
-pub struct LegacyModels {
+pub struct LegacyAssets {
pub llm_dir: Option<PathBuf>,
- pub transcription_dir: PathBuf,
+ pub transcription_dir: Option<PathBuf>,
+ pub models_dir: Option<PathBuf>,
pub model_url: Option<String>,
pub mmproj_url: Option<String>,
}
-pub fn migrate_mobile_models(models_dir: &Path, legacy: LegacyModels) -> Option<String> {
- let store = ModelDownloader::new(models_dir);
- if let Some(llm_dir) = &legacy.llm_dir
- && let Ok(targets) = llm_targets()
- {
- migrate_legacy_dir(&store, llm_dir, &targets);
+pub fn needs_migration(legacy: &LegacyAssets) -> bool {
+ [
+ legacy.llm_dir.as_deref(),
+ legacy.transcription_dir.as_deref(),
+ legacy.models_dir.as_deref(),
+ ]
+ .into_iter()
+ .flatten()
+ .any(Path::exists)
+}
+
+pub fn migrate_ensu_assets(assets_dir: &Path, legacy: LegacyAssets) -> Option<String> {
+ let store = AssetStore::new(assets_dir);
+ let Ok(llm_migrations) = llm_migrations() else {
+ return None;
+ };
+
+ if let Some(directory) = legacy.llm_dir {
+ migrate_flat_models(
+ &store,
+ &llm_migrations,
+ &directory.join("models"),
+ &directory,
+ );
+ }
+ if let Some(directory) = legacy.transcription_dir {
+ migrate_transcription(&store, &directory);
}
- migrate_legacy_transcription_dir(
- &store,
- &legacy.transcription_dir,
- &model::transcription_target(),
- &model::voice_activity_target(),
- );
+ if let Some(directory) = legacy.models_dir {
+ migrate_flat_models(&store, &llm_migrations, &directory, &directory);
+ }
+
legacy_selected_preset_id(legacy.model_url.as_deref()?, legacy.mmproj_url.as_deref())
}
-pub fn migrate_desktop_models(
- models_dir: &Path,
- model_url: Option<&str>,
- mmproj_url: Option<&str>,
-) -> Option<String> {
- let selected = model_url.and_then(|model_url| legacy_selected_preset_id(model_url, mmproj_url));
- if let Ok(targets) = llm_targets() {
- migrate_flat_models_dir(models_dir, &targets);
- }
- selected
+struct LlmMigration {
+ asset: Asset,
+ files: Vec<LlmFile>,
}
-fn llm_targets() -> Result<Vec<ModelTarget>, InvalidPreset> {
+struct LlmFile {
+ url: String,
+ name: &'static str,
+}
+
+fn llm_migrations() -> Result<Vec<LlmMigration>, InvalidPreset> {
config::llm_catalog()
.iter()
- .map(model::llm_target)
+ .map(|preset| {
+ let mut files = vec![LlmFile {
+ url: preset.url.clone(),
+ name: "model.gguf",
+ }];
+ if let Some(url) = trimmed(preset.mmproj_url.as_deref()) {
+ files.push(LlmFile {
+ url: url.to_string(),
+ name: "mmproj.gguf",
+ });
+ }
+ Ok(LlmMigration {
+ asset: model::llm_asset(preset)?,
+ files,
+ })
+ })
.collect()
}
@@ -61,192 +93,121 @@ fn legacy_selected_preset_id(model_url: &str, mmproj_url: Option<&str>) -> Optio
.map(|preset| preset.id)
}
-fn migrate_legacy_dir(store: &ModelDownloader, legacy_dir: &Path, targets: &[ModelTarget]) {
- if !legacy_dir.exists() {
- return;
- }
- if adopt_targets(store, targets, &[], &legacy_dir.join("models")) {
- let _ = fs::remove_dir_all(legacy_dir);
- }
-}
-
-fn migrate_flat_models_dir(models_dir: &Path, targets: &[ModelTarget]) {
- if !models_dir.exists() {
- return;
- }
- let store = ModelDownloader::new(models_dir);
- let hashed_dirs = [models_dir.to_path_buf()];
- if !adopt_targets(&store, targets, &hashed_dirs, models_dir) {
- return;
- }
- let Ok(entries) = fs::read_dir(models_dir) else {
- return;
- };
- for entry in entries.flatten() {
- let path = entry.path();
- if path.is_file() {
- let _ = fs::remove_file(&path);
- } else if path.file_name().is_some_and(|name| name == "custom") {
- let _ = fs::remove_dir_all(&path);
- }
+fn migrate_flat_models(
+ store: &AssetStore,
+ targets: &[LlmMigration],
+ source: &Path,
+ cleanup: &Path,
+) {
+ if source.exists() && adopt_flat_targets(store, targets, source) {
+ let _ = fs::remove_dir_all(cleanup);
}
}
-fn migrate_legacy_transcription_dir(
- store: &ModelDownloader,
- legacy_dir: &Path,
- model: &ModelTarget,
- vad: &ModelTarget,
-) {
- const LEGACY_MODEL_DIR_NAME: &str = "parakeet-tdt-0.6b-v3-int8";
- const LEGACY_MODEL_ID: &str = "parakeet-v3-int8";
- const LEGACY_VAD_FILE_NAME: &str = "silero_vad_v4.onnx";
- const LEGACY_VAD_ID: &str = "silero-vad-v4";
+fn migrate_transcription(store: &AssetStore, legacy_dir: &Path) {
+ const LEGACY_MODEL_DIR: &str = "parakeet-tdt-0.6b-v3-int8";
+ const LEGACY_VAD_FILE: &str = "silero_vad_v4.onnx";
if !legacy_dir.exists() {
return;
}
- let mut all_moved = true;
-
- if matches!(model, ModelTarget::TarGz { id, .. } if id == LEGACY_MODEL_ID) {
- let dest = store.model_dir(model);
- if !dest.exists() {
- let source = legacy_dir.join(LEGACY_MODEL_DIR_NAME);
- if source.is_dir() {
- if let Some(parent) = dest.parent() {
- let _ = fs::create_dir_all(parent);
- }
- if fs::rename(&source, &dest).is_err() {
- all_moved = false;
- }
+ let mut complete = true;
+
+ let model = model::transcription_model_asset();
+ let model_destination = store.asset_dir(&model);
+ if !store.is_downloaded(&model) {
+ let source = legacy_dir.join(LEGACY_MODEL_DIR);
+ if source.is_dir() {
+ if let Some(parent) = model_destination.parent() {
+ let _ = fs::create_dir_all(parent);
}
+ complete &= fs::rename(source, model_destination).is_ok();
}
}
- if let ModelTarget::Files { id, files } = vad
- && id == LEGACY_VAD_ID
- && let [file] = files.as_slice()
- {
- let dest = store.model_dir(vad).join(&file.name);
- if !dest.exists() {
- let source = legacy_dir.join(LEGACY_VAD_FILE_NAME);
- if is_non_empty_file(&source) && !move_file(&source, &dest) {
- all_moved = false;
- }
+ let vad = model::voice_activity_model_asset();
+ let vad_destination = model::voice_activity_model_path(store, &vad);
+ if !store.is_downloaded(&vad) {
+ let source = legacy_dir.join(LEGACY_VAD_FILE);
+ if is_non_empty_file(&source) {
+ complete &= move_file(&source, &vad_destination);
}
}
-
- if all_moved {
+ if complete {
let _ = fs::remove_dir_all(legacy_dir);
}
}
-fn sidecar_url(path: &Path) -> Option<String> {
- let sidecar = PathBuf::from(format!("{}.metadata.json", path.display()));
- let text = fs::read_to_string(sidecar).ok()?;
- let value: serde_json::Value = serde_json::from_str(&text).ok()?;
- value["url"].as_str().map(String::from)
-}
-
-fn adopt_targets(
- store: &ModelDownloader,
- targets: &[ModelTarget],
- hashed_dirs: &[PathBuf],
- flat_dir: &Path,
-) -> bool {
- let mut plans: Vec<Vec<(&str, &str, PathBuf)>> = Vec::new();
- for target in targets {
- let ModelTarget::Files { files, .. } = target else {
- continue;
- };
- let dir = store.model_dir(target);
- plans.push(
- files
- .iter()
- .map(|file| (file.url.as_str(), file.name.as_str(), dir.join(&file.name)))
- .collect(),
- );
- }
-
+fn adopt_flat_targets(store: &AssetStore, targets: &[LlmMigration], flat_dir: &Path) -> bool {
let mut basename_urls: HashMap<String, HashSet<&str>> = HashMap::new();
+ let mut complete = true;
for target in targets {
- let ModelTarget::Files { files, .. } = target else {
- continue;
- };
- for file in files {
+ for file in &target.files {
basename_urls
- .entry(filename_for_url(&file.url, &file.name))
+ .entry(filename_for_url(&file.url, file.name))
.or_default()
- .insert(file.url.as_str());
+ .insert(&file.url);
}
}
- let mut all_moved = true;
- for plan in &plans {
- let sources: Vec<Option<Option<PathBuf>>> = plan
+ for target in targets {
+ let destination = store.asset_dir(&target.asset);
+ let sources = target
+ .files
.iter()
- .map(|(url, fallback, dest)| {
- if dest.exists() {
+ .map(|file| {
+ let final_path = destination.join(file.name);
+ if final_path.exists() {
return Some(None);
}
- let basename = filename_for_url(url, fallback);
- let url = *url;
- for dir in hashed_dirs {
- let hashed = dir.join(format!("{}_{basename}", sha256_hex(url)));
- if looks_like_gguf(&hashed) {
- return Some(Some(hashed));
- }
+ let basename = filename_for_url(&file.url, file.name);
+ let source = flat_dir.join(&basename);
+ if !looks_like_gguf(&source) {
+ return None;
}
- let flat = flat_dir.join(&basename);
- if looks_like_gguf(&flat) {
- match sidecar_url(&flat) {
- Some(sidecar) => {
- if sidecar == url {
- return Some(Some(flat));
- }
- }
- None => {
- if basename_urls[&basename].len() == 1 {
- return Some(Some(flat));
- }
- }
- }
+ match sidecar_url(&source) {
+ Some(url) if url == file.url => Some(Some(source)),
+ None if basename_urls[&basename].len() == 1 => Some(Some(source)),
+ _ => None,
}
- None
})
- .collect();
+ .collect::<Vec<_>>();
if sources.iter().any(Option::is_none) {
continue;
}
- for ((_, _, dest), source) in plan.iter().zip(sources) {
- let Some(Some(source)) = source else {
- continue;
- };
- if !move_file(&source, dest) {
- all_moved = false;
+ for (file, source) in target.files.iter().zip(sources) {
+ if let Some(Some(source)) = source {
+ complete &= move_file(&source, &destination.join(file.name));
}
}
}
- all_moved
+ complete
}
-fn move_file(source: &Path, dest: &Path) -> bool {
- if let Some(parent) = dest.parent() {
+fn move_file(source: &Path, destination: &Path) -> bool {
+ if let Some(parent) = destination.parent() {
let _ = fs::create_dir_all(parent);
}
- if fs::rename(source, dest).is_ok() {
+ if fs::rename(source, destination).is_ok() {
return true;
}
- let staged = PathBuf::from(format!("{}.migrating", dest.display()));
- if fs::copy(source, &staged)
- .and_then(|_| fs::rename(&staged, dest))
+ let staging = PathBuf::from(format!("{}.migrating", destination.display()));
+ if fs::copy(source, &staging)
+ .and_then(|_| fs::rename(&staging, destination))
.is_err()
{
- let _ = fs::remove_file(&staged);
+ let _ = fs::remove_file(staging);
return false;
}
true
}
+fn sidecar_url(path: &Path) -> Option<String> {
+ let sidecar = PathBuf::from(format!("{}.metadata.json", path.display()));
+ let text = fs::read_to_string(sidecar).ok()?;
+ let value: serde_json::Value = serde_json::from_str(&text).ok()?;
+ value["url"].as_str().map(String::from)
+}
+
fn is_non_empty_file(path: &Path) -> bool {
path.metadata()
.map(|metadata| metadata.is_file() && metadata.len() > 0)
@@ -272,20 +233,10 @@ fn filename_for_url(url: &str, fallback: &str) -> String {
}
}
-fn sha256_hex(value: &str) -> String {
- Sha256::digest(value.as_bytes())
- .iter()
- .map(|byte| format!("{byte:02x}"))
- .collect()
-}
-
#[cfg(test)]
mod tests {
- use ente_model_download::ModelFile;
-
use super::*;
use crate::config::ModelPreset;
- use crate::model;
fn scratch_dir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
@@ -295,468 +246,122 @@ mod tests {
.unwrap()
.as_nanos()
));
- fs::create_dir_all(&dir).expect("create test dir");
+ fs::create_dir_all(&dir).unwrap();
dir
}
- fn test_sha() -> String {
- "0".repeat(64)
- }
-
- fn gguf_target(id: &str, url: String, mmproj_url: String) -> ModelTarget {
- model::llm_target(&ModelPreset {
+ fn target(id: &str, model_url: &str, mmproj_url: &str) -> LlmMigration {
+ let preset = ModelPreset {
id: id.to_string(),
title: String::new(),
- url,
- sha256: test_sha(),
- mmproj_url: Some(mmproj_url),
- mmproj_sha256: Some(test_sha()),
- })
- .unwrap()
- }
-
- fn target(id: &str) -> ModelTarget {
- gguf_target(
- id,
- "https://example.org/models/main.gguf?download=true".to_string(),
- "https://example.org/models/mmproj.gguf".to_string(),
- )
- }
-
- fn tar_gz_target(id: &str) -> ModelTarget {
- ModelTarget::TarGz {
- id: id.to_string(),
- url: format!("https://models.example.org/{id}.tar.gz"),
- sha256: test_sha(),
- }
- }
-
- fn vad_target(id: &str, url: String) -> ModelTarget {
- ModelTarget::Files {
- id: id.to_string(),
- files: vec![ModelFile {
- name: "model.onnx".to_string(),
- url,
- sha256: test_sha(),
- }],
+ url: model_url.to_string(),
+ sha256: "0".repeat(64),
+ mmproj_url: Some(mmproj_url.to_string()),
+ mmproj_sha256: Some("0".repeat(64)),
+ };
+ LlmMigration {
+ asset: model::llm_asset(&preset).unwrap(),
+ files: vec![
+ LlmFile {
+ url: model_url.to_string(),
+ name: "model.gguf",
+ },
+ LlmFile {
+ url: mmproj_url.to_string(),
+ name: "mmproj.gguf",
+ },
+ ],
}
}
- fn file_target(id: &str) -> ModelTarget {
- vad_target(id, format!("https://models.example.org/{id}.onnx"))
- }
-
- fn write_gguf(path: &Path, content: &[u8]) {
- fs::create_dir_all(path.parent().unwrap()).unwrap();
- fs::write(path, content).unwrap();
- }
-
#[test]
- fn mobile_migration_resolves_preset_selection() {
- let base = scratch_dir("mobile-selection");
+ fn resolves_only_catalog_selections() {
let defaults = config::defaults();
let preset = &defaults.mobile_model_presets[0];
- let selected = migrate_mobile_models(
- &base.join("models"),
- LegacyModels {
- llm_dir: None,
- transcription_dir: base.join("transcription"),
- model_url: Some(preset.url.clone()),
- mmproj_url: preset.mmproj_url.clone(),
- },
- );
- assert_eq!(selected.as_deref(), Some(preset.id.as_str()));
-
- let unknown = migrate_mobile_models(
- &base.join("models"),
- LegacyModels {
- llm_dir: None,
- transcription_dir: base.join("transcription"),
- model_url: Some("https://example.org/custom.gguf".to_string()),
- mmproj_url: None,
- },
- );
- assert_eq!(unknown, None);
-
- let _ = fs::remove_dir_all(base);
- }
-
- #[test]
- fn desktop_migration_resolves_preset_selection() {
- let base = scratch_dir("desktop-selection");
- let defaults = config::defaults();
- let preset = &defaults.desktop_model_presets[0];
- let selected = migrate_desktop_models(
- &base.join("models"),
- Some(&preset.url),
- preset.mmproj_url.as_deref(),
+ assert_eq!(
+ legacy_selected_preset_id(&preset.url, preset.mmproj_url.as_deref()).as_deref(),
+ Some(preset.id.as_str())
);
- assert_eq!(selected.as_deref(), Some(preset.id.as_str()));
-
assert_eq!(
- migrate_desktop_models(&base.join("models"), None, None),
+ legacy_selected_preset_id("https://example.org/custom.gguf", None),
None
);
-
- let _ = fs::remove_dir_all(base);
}
#[test]
- fn migrate_transcription_moves_model_dir_and_vad_file() {
- let base = scratch_dir("migrate-transcription");
- let legacy = base.join("transcription");
- fs::create_dir_all(legacy.join("parakeet-tdt-0.6b-v3-int8")).unwrap();
- fs::write(
- legacy.join("parakeet-tdt-0.6b-v3-int8/encoder.onnx"),
- b"enc",
- )
- .unwrap();
- fs::write(legacy.join("silero_vad_v4.onnx"), b"vad").unwrap();
- fs::write(legacy.join("stale.partial"), b"junk").unwrap();
-
- let models_dir = base.join("models");
- let model = tar_gz_target("parakeet-v3-int8");
- let vad = vad_target(
- "silero-vad-v4",
- "https://models.example.org/silero_vad_v4.onnx".to_string(),
+ fn adopts_released_flat_model_all_or_nothing() {
+ let base = scratch_dir("flat");
+ let legacy = base.join("llm/models");
+ fs::create_dir_all(&legacy).unwrap();
+ fs::write(legacy.join("model.gguf"), b"GGUFmodel").unwrap();
+ fs::write(legacy.join("mmproj.gguf"), b"GGUFmmproj").unwrap();
+ let target = target(
+ "model",
+ "https://example.org/model.gguf",
+ "https://example.org/mmproj.gguf",
);
- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);
+ let store = AssetStore::new(base.join("assets"));
+
+ adopt_flat_targets(&store, &[target], &legacy);
- let downloader = ModelDownloader::new(&models_dir);
- assert!(downloader.is_downloaded(&model));
- assert!(downloader.is_downloaded(&vad));
assert_eq!(
- fs::read(models_dir.join("parakeet-v3-int8/encoder.onnx")).unwrap(),
- b"enc"
+ fs::read(base.join("assets/models/model/model.gguf")).unwrap(),
+ b"GGUFmodel"
);
assert_eq!(
- fs::read(models_dir.join("silero-vad-v4/model.onnx")).unwrap(),
- b"vad"
+ fs::read(base.join("assets/models/model/mmproj.gguf")).unwrap(),
+ b"GGUFmmproj"
);
- assert!(!legacy.exists());
-
- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);
- assert!(downloader.is_downloaded(&model));
-
let _ = fs::remove_dir_all(base);
}
#[test]
- fn migrate_transcription_keeps_existing_destinations() {
- let base = scratch_dir("migrate-transcription-existing");
- let legacy = base.join("transcription");
- fs::create_dir_all(legacy.join("parakeet-tdt-0.6b-v3-int8")).unwrap();
- fs::write(
- legacy.join("parakeet-tdt-0.6b-v3-int8/encoder.onnx"),
- b"old",
- )
- .unwrap();
-
- let models_dir = base.join("models");
- let model = tar_gz_target("parakeet-v3-int8");
- let vad = file_target("silero-vad-v4");
- fs::create_dir_all(models_dir.join("parakeet-v3-int8")).unwrap();
- fs::write(models_dir.join("parakeet-v3-int8/encoder.onnx"), b"new").unwrap();
-
- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);
-
- assert_eq!(
- fs::read(models_dir.join("parakeet-v3-int8/encoder.onnx")).unwrap(),
- b"new"
- );
- assert!(!legacy.exists());
-
+ fn rejects_ambiguous_flat_projector() {
+ let base = scratch_dir("ambiguous");
+ let legacy = base.join("models");
+ fs::create_dir_all(&legacy).unwrap();
+ fs::write(legacy.join("a.gguf"), b"GGUFa").unwrap();
+ fs::write(legacy.join("b.gguf"), b"GGUFb").unwrap();
+ fs::write(legacy.join("mmproj.gguf"), b"GGUFshared").unwrap();
+ let targets = [
+ target(
+ "a",
+ "https://example.org/a.gguf",
+ "https://example.org/a/mmproj.gguf",
+ ),
+ target(
+ "b",
+ "https://example.org/b.gguf",
+ "https://example.org/b/mmproj.gguf",
+ ),
+ ];
+ let store = AssetStore::new(base.join("assets"));
+
+ adopt_flat_targets(&store, &targets, &legacy);
+
+ assert!(!base.join("assets/models/a").exists());
+ assert!(!base.join("assets/models/b").exists());
let _ = fs::remove_dir_all(base);
}
#[test]
- fn migrate_transcription_adopts_nothing_when_target_ids_differ() {
- let base = scratch_dir("migrate-transcription-mismatch");
+ fn adopts_released_transcription() {
+ let base = scratch_dir("transcription");
let legacy = base.join("transcription");
fs::create_dir_all(legacy.join("parakeet-tdt-0.6b-v3-int8")).unwrap();
fs::write(
legacy.join("parakeet-tdt-0.6b-v3-int8/encoder.onnx"),
- b"enc",
+ b"model",
)
.unwrap();
fs::write(legacy.join("silero_vad_v4.onnx"), b"vad").unwrap();
+ let store = AssetStore::new(base.join("assets"));
- let models_dir = base.join("models");
- let model = tar_gz_target("parakeet-v4-int8");
- let vad = file_target("silero-vad-v5");
- migrate_legacy_transcription_dir(&ModelDownloader::new(&models_dir), &legacy, &model, &vad);
-
- let downloader = ModelDownloader::new(&models_dir);
- assert!(!downloader.is_downloaded(&model));
- assert!(!downloader.is_downloaded(&vad));
- assert!(!legacy.exists());
-
- let _ = fs::remove_dir_all(base);
- }
+ migrate_transcription(&store, &legacy);
- #[test]
- fn migrate_adopts_known_files_and_discards_legacy_dir() {
- let base = scratch_dir("migrate");
- let legacy = base.join("llm");
- let legacy_models = legacy.join("models");
- fs::create_dir_all(&legacy_models).unwrap();
-
- fs::write(legacy_models.join("main.gguf"), b"GGUFmain").unwrap();
- fs::write(legacy_models.join("mmproj.gguf"), b"GGUFmmproj").unwrap();
- fs::write(legacy_models.join("main.gguf.tmp"), b"GGUFpartial").unwrap();
- fs::write(legacy_models.join("main.gguf.metadata.json"), b"{}").unwrap();
- fs::write(legacy_models.join("orphan.gguf"), b"GGUForphan").unwrap();
-
- let models_dir = base.join("models");
- let preset = target("qwen-2b-q8");
- migrate_legacy_dir(
- &ModelDownloader::new(&models_dir),
- &legacy,
- std::slice::from_ref(&preset),
- );
-
- let downloader = ModelDownloader::new(&models_dir);
- assert_eq!(
- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmain"
- );
- assert_eq!(
- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmmproj"
- );
- assert_eq!(fs::read_dir(&models_dir).unwrap().count(), 1);
+ assert!(store.is_downloaded(&model::transcription_model_asset()));
+ assert!(store.is_downloaded(&model::voice_activity_model_asset()));
assert!(!legacy.exists());
-
- let _ = fs::remove_dir_all(base);
- }
-
- #[test]
- fn migrate_adopts_targets_only_when_every_file_resolves() {
- let base = scratch_dir("migrate-ambiguous");
- let legacy = base.join("llm");
- let legacy_models = legacy.join("models");
- fs::create_dir_all(&legacy_models).unwrap();
- fs::write(legacy_models.join("a.gguf"), b"GGUFa").unwrap();
- fs::write(legacy_models.join("b.gguf"), b"GGUFb").unwrap();
- fs::write(legacy_models.join("mmproj-F16.gguf"), b"GGUFshared").unwrap();
-
- let targets = ["a", "b"].map(|name| {
- gguf_target(
- &format!("preset-{name}"),
- format!("https://example.org/{name}/{name}.gguf"),
- format!("https://example.org/{name}/mmproj-F16.gguf"),
- )
- });
-
- let models_dir = base.join("models");
- migrate_legacy_dir(&ModelDownloader::new(&models_dir), &legacy, &targets);
-
- assert!(!models_dir.exists() || fs::read_dir(&models_dir).unwrap().count() == 0);
- assert!(!legacy.exists());
-
- let _ = fs::remove_dir_all(base);
- }
-
- #[test]
- fn migrate_flat_models_dir_adopts_known_files_and_cleans_root() {
- let models_dir = scratch_dir("flat-migrate");
- fs::create_dir_all(models_dir.join("custom")).unwrap();
-
- let preset = target("qwen-2b-q8");
- fs::write(models_dir.join("main.gguf"), b"GGUFmain").unwrap();
- let mmproj_hashed = format!(
- "{}_mmproj.gguf",
- sha256_hex("https://example.org/models/mmproj.gguf")
- );
- fs::write(models_dir.join(mmproj_hashed), b"GGUFmm").unwrap();
-
- fs::write(models_dir.join("orphan.gguf"), b"GGUForphan").unwrap();
- fs::write(models_dir.join("main.gguf.tmp"), b"GGUFpartial").unwrap();
- let downloader = ModelDownloader::new(&models_dir);
- write_gguf(
- &models_dir.join("other-key").join("model.gguf"),
- b"GGUFkeep",
- );
-
- migrate_flat_models_dir(&models_dir, std::slice::from_ref(&preset));
-
- assert_eq!(
- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmain"
- );
- assert_eq!(
- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmm"
- );
- assert_eq!(
- fs::read(models_dir.join("other-key/model.gguf")).unwrap(),
- b"GGUFkeep"
- );
- assert!(!models_dir.join("custom").exists());
- let root_files = fs::read_dir(&models_dir)
- .unwrap()
- .flatten()
- .filter(|entry| entry.path().is_file())
- .count();
- assert_eq!(root_files, 0);
-
- migrate_flat_models_dir(&models_dir, std::slice::from_ref(&preset));
- assert_eq!(
- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmain"
- );
-
- let _ = fs::remove_dir_all(models_dir);
- }
-
- #[test]
- fn migrate_flat_models_dir_adopts_sidecarless_files_for_duplicated_targets() {
- let models_dir = scratch_dir("flat-duplicate-targets");
- fs::create_dir_all(&models_dir).unwrap();
- fs::write(models_dir.join("main.gguf"), b"GGUFmain").unwrap();
- fs::write(models_dir.join("mmproj.gguf"), b"GGUFmm").unwrap();
-
- let preset = target("lfm-vl-1.6b");
- migrate_flat_models_dir(&models_dir, &[preset.clone(), preset.clone()]);
-
- let downloader = ModelDownloader::new(&models_dir);
- assert_eq!(
- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmain"
- );
- assert_eq!(
- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmm"
- );
-
- let _ = fs::remove_dir_all(models_dir);
- }
-
- #[test]
- fn migrate_flat_models_dir_resolves_ambiguity_via_metadata_sidecars() {
- let models_dir = scratch_dir("flat-sidecar");
- fs::create_dir_all(&models_dir).unwrap();
-
- let targets = ["a", "b"].map(|name| {
- gguf_target(
- &format!("preset-{name}"),
- format!("https://example.org/{name}/{name}.gguf"),
- format!("https://example.org/{name}/mmproj-F16.gguf"),
- )
- });
-
- fs::write(models_dir.join("a.gguf"), b"GGUFa").unwrap();
- fs::write(
- models_dir.join("a.gguf.metadata.json"),
- br#"{"url": "https://example.org/a/a.gguf"}"#,
- )
- .unwrap();
- fs::write(models_dir.join("mmproj-F16.gguf"), b"GGUFmma").unwrap();
- fs::write(
- models_dir.join("mmproj-F16.gguf.metadata.json"),
- br#"{"url": "https://example.org/a/mmproj-F16.gguf"}"#,
- )
- .unwrap();
-
- migrate_flat_models_dir(&models_dir, &targets);
-
- let downloader = ModelDownloader::new(&models_dir);
- assert_eq!(
- fs::read(model::llm_model_path(&downloader, &targets[0]).unwrap()).unwrap(),
- b"GGUFa"
- );
- assert_eq!(
- fs::read(model::llm_mmproj_path(&downloader, &targets[0]).unwrap()).unwrap(),
- b"GGUFmma"
- );
- assert!(
- !model::llm_model_path(&downloader, &targets[1])
- .unwrap()
- .exists()
- );
-
- let _ = fs::remove_dir_all(models_dir);
- }
-
- #[test]
- fn migrate_flat_models_dir_rejects_files_whose_sidecar_disagrees() {
- let models_dir = scratch_dir("flat-sidecar-mismatch");
- fs::create_dir_all(&models_dir).unwrap();
-
- let preset = target("qwen-2b-q8");
- fs::write(models_dir.join("main.gguf"), b"GGUFstale").unwrap();
- fs::write(
- models_dir.join("main.gguf.metadata.json"),
- br#"{"url": "https://example.org/elsewhere/main.gguf"}"#,
- )
- .unwrap();
-
- migrate_flat_models_dir(&models_dir, std::slice::from_ref(&preset));
-
- let downloader = ModelDownloader::new(&models_dir);
- assert!(
- !model::llm_model_path(&downloader, &preset)
- .unwrap()
- .exists()
- );
-
- let _ = fs::remove_dir_all(models_dir);
- }
-
- #[test]
- fn migrate_flat_models_dir_drops_targets_with_ambiguous_basenames() {
- let models_dir = scratch_dir("flat-ambiguous");
- fs::create_dir_all(&models_dir).unwrap();
- fs::write(models_dir.join("a.gguf"), b"GGUFa").unwrap();
- fs::write(models_dir.join("mmproj-F16.gguf"), b"GGUFshared").unwrap();
-
- let targets = ["a", "b"].map(|name| {
- gguf_target(
- &format!("preset-{name}"),
- format!("https://example.org/{name}/{name}.gguf"),
- format!("https://example.org/{name}/mmproj-F16.gguf"),
- )
- });
-
- migrate_flat_models_dir(&models_dir, &targets);
-
- assert_eq!(fs::read_dir(&models_dir).unwrap().count(), 0);
-
- let _ = fs::remove_dir_all(models_dir);
- }
-
- #[test]
- fn migrate_keeps_existing_destination_files() {
- let base = scratch_dir("migrate-existing");
- let legacy = base.join("llm");
- fs::create_dir_all(legacy.join("models")).unwrap();
- fs::write(legacy.join("models/main.gguf"), b"GGUFold").unwrap();
- fs::write(legacy.join("models/mmproj.gguf"), b"GGUFmm").unwrap();
-
- let models_dir = base.join("models");
- let preset = target("qwen-2b-q8");
- let downloader = ModelDownloader::new(&models_dir);
- write_gguf(
- &model::llm_model_path(&downloader, &preset).unwrap(),
- b"GGUFnew",
- );
-
- migrate_legacy_dir(
- &ModelDownloader::new(&models_dir),
- &legacy,
- std::slice::from_ref(&preset),
- );
-
- assert_eq!(
- fs::read(model::llm_model_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFnew"
- );
- assert_eq!(
- fs::read(model::llm_mmproj_path(&downloader, &preset).unwrap()).unwrap(),
- b"GGUFmm"
- );
- assert!(!legacy.exists());
-
let _ = fs::remove_dir_all(base);
}
}
diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rs
index 91fd1a1529..0fd9304bb0 100644
--- a/rust/crates/ensu/src/model/mod.rs
+++ b/rust/crates/ensu/src/model/mod.rs
@@ -2,22 +2,32 @@ pub mod migrations;
use std::path::PathBuf;
-use ente_model_download::{ModelDownloader, ModelFile, ModelTarget};
+use ente_assets::{Asset, AssetFile, AssetStore, download::Progress};
use crate::config::{self, ModelPreset};
+const MODELS: &str = "models";
const LLM_MODEL_FILE: &str = "model.gguf";
const LLM_MMPROJ_FILE: &str = "mmproj.gguf";
const VOICE_ACTIVITY_MODEL_FILE: &str = "model.onnx";
#[derive(Debug, thiserror::Error)]
-#[error("preset {id} pairs only half of the mmproj URL and checksum")]
+#[error("invalid preset {id}: {reason}")]
pub struct InvalidPreset {
pub id: String,
+ reason: String,
}
-pub(crate) fn llm_target(preset: &ModelPreset) -> Result<ModelTarget, InvalidPreset> {
- let mut files = vec![ModelFile {
+pub struct ModelDownloadProgress {
+ pub downloaded_bytes: u64,
+ pub total_bytes: Option<u64>,
+ pub percent: i32,
+ pub status: String,
+ pub log_line: Option<String>,
+}
+
+pub(crate) fn llm_asset(preset: &ModelPreset) -> Result<Asset, InvalidPreset> {
+ let mut files = vec![AssetFile {
name: LLM_MODEL_FILE.to_string(),
url: preset.url.clone(),
sha256: preset.sha256.clone(),
@@ -26,91 +36,169 @@ pub(crate) fn llm_target(preset: &ModelPreset) -> Result<ModelTarget, InvalidPre
trimmed(preset.mmproj_url.as_deref()),
trimmed(preset.mmproj_sha256.as_deref()),
) {
- (Some(url), Some(sha256)) => files.push(ModelFile {
+ (Some(url), Some(sha256)) => files.push(AssetFile {
name: LLM_MMPROJ_FILE.to_string(),
url: url.to_string(),
sha256: sha256.to_string(),
}),
(None, None) => {}
_ => {
- return Err(InvalidPreset {
- id: preset.id.clone(),
- });
+ return Err(invalid_preset(
+ preset,
+ "mmproj URL and checksum must be paired",
+ ));
}
}
- Ok(ModelTarget::Files {
- id: preset.id.clone(),
- files,
- })
+ Asset::files(model_key(&preset.id), files)
+ .map_err(|error| invalid_preset(preset, error.to_string()))
}
-pub fn mobile_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {
+pub fn mobile_llm_asset(model_id: &str) -> Result<Asset, InvalidPreset> {
let preset = config::llm_catalog()
.into_iter()
.find(|preset| preset.id == model_id)
.unwrap_or_else(|| config::defaults().mobile_default_model);
- llm_target(&preset)
+ llm_asset(&preset)
}
-pub fn desktop_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {
+pub fn desktop_llm_asset(model_id: &str) -> Result<Asset, InvalidPreset> {
let preset = config::llm_catalog()
.into_iter()
.find(|preset| preset.id == model_id)
.unwrap_or_else(|| config::defaults().desktop_default_model);
- llm_target(&preset)
+ llm_asset(&preset)
}
-pub fn transcription_target() -> ModelTarget {
+pub fn transcription_model_asset() -> Asset {
let preset = config::defaults().transcription_model;
- ModelTarget::TarGz {
- id: preset.id,
- url: preset.url,
- sha256: preset.sha256,
- }
+ Asset::tar_gz(model_key(&preset.id), preset.url, preset.sha256)
+ .expect("transcription asset config")
}
-pub fn voice_activity_target() -> ModelTarget {
+pub fn voice_activity_model_asset() -> Asset {
let preset = config::defaults().voice_activity_model;
- ModelTarget::Files {
- id: preset.id,
- files: vec![ModelFile {
+ Asset::files(
+ model_key(&preset.id),
+ vec![AssetFile {
name: VOICE_ACTIVITY_MODEL_FILE.to_string(),
url: preset.url,
sha256: preset.sha256,
}],
- }
+ )
+ .expect("voice activity asset config")
}
-pub fn knowledge_embedding_target() -> ModelTarget {
+pub fn knowledge_embedding_model_asset() -> Asset {
let embedding = config::knowledge_embedding_config();
- ModelTarget::Files {
- id: embedding.target_id,
- files: vec![ModelFile {
+ Asset::files(
+ model_key(&embedding.target_id),
+ vec![AssetFile {
name: LLM_MODEL_FILE.to_string(),
url: embedding.model_url,
sha256: embedding.model_sha256,
}],
- }
+ )
+ .expect("knowledge embedding asset config")
+}
+
+pub fn llm_model_path(store: &AssetStore, asset: &Asset) -> Option<PathBuf> {
+ store.file_path(asset, LLM_MODEL_FILE)
}
-pub fn llm_model_path(downloader: &ModelDownloader, target: &ModelTarget) -> Option<PathBuf> {
- downloader.file_path(target, LLM_MODEL_FILE)
+pub fn llm_mmproj_path(store: &AssetStore, asset: &Asset) -> Option<PathBuf> {
+ store.file_path(asset, LLM_MMPROJ_FILE)
}
-pub fn llm_mmproj_path(downloader: &ModelDownloader, target: &ModelTarget) -> Option<PathBuf> {
- downloader.file_path(target, LLM_MMPROJ_FILE)
+pub fn voice_activity_model_path(store: &AssetStore, asset: &Asset) -> PathBuf {
+ store
+ .file_path(asset, VOICE_ACTIVITY_MODEL_FILE)
+ .expect("voice activity model file")
}
-pub fn voice_activity_model_path(downloader: &ModelDownloader) -> PathBuf {
- downloader
- .model_dir(&voice_activity_target())
- .join(VOICE_ACTIVITY_MODEL_FILE)
+pub(crate) fn model_key(id: &str) -> Vec<String> {
+ vec![MODELS.to_string(), id.to_string()]
}
pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {
value.map(str::trim).filter(|value| !value.is_empty())
}
+pub fn display_progress(progress: Progress) -> ModelDownloadProgress {
+ let total = progress.total_bytes.filter(|total| *total > 0);
+ let percent = total
+ .map(|total| ((progress.downloaded_bytes * 100 / total) as i32).clamp(0, 99))
+ .unwrap_or(0);
+ let status = if let Some(total) = total {
+ format!(
+ "Downloading... {} / {}",
+ format_bytes(progress.downloaded_bytes),
+ format_bytes(total)
+ )
+ } else if progress.file_downloaded_bytes > 0 {
+ format!(
+ "Downloading {}... {}",
+ progress.label.to_lowercase(),
+ format_bytes(progress.file_downloaded_bytes)
+ )
+ } else {
+ format!("Downloading {}...", progress.label.to_lowercase())
+ };
+ let log_line = if progress.file_complete {
+ Some(format!(
+ "Asset download file complete label={} bytes={} elapsedMs={} rate={}/s retries={}",
+ progress.label,
+ progress.file_downloaded_bytes,
+ progress.file_elapsed_ms,
+ format_bytes(rate_bytes(progress.file_bytes_per_second)),
+ progress.file_retry_count
+ ))
+ } else if progress.complete {
+ Some(format!(
+ "Asset download complete bytes={} elapsedMs={} rate={}/s retries={}",
+ progress.downloaded_bytes,
+ progress.elapsed_ms,
+ format_bytes(rate_bytes(progress.bytes_per_second)),
+ progress.retry_count
+ ))
+ } else {
+ None
+ };
+
+ ModelDownloadProgress {
+ downloaded_bytes: progress.downloaded_bytes,
+ total_bytes: progress.total_bytes,
+ percent,
+ status,
+ log_line,
+ }
+}
+
+fn invalid_preset(preset: &ModelPreset, reason: impl Into<String>) -> InvalidPreset {
+ InvalidPreset {
+ id: preset.id.clone(),
+ reason: reason.into(),
+ }
+}
+
+fn rate_bytes(bytes_per_second: f64) -> u64 {
+ if bytes_per_second.is_finite() && bytes_per_second > 0.0 {
+ bytes_per_second as u64
+ } else {
+ 0
+ }
+}
+
+fn format_bytes(bytes: u64) -> String {
+ const UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
+ let mut size = bytes as f64;
+ let mut unit = 0;
+ while size >= 1024.0 && unit < UNITS.len() - 1 {
+ size /= 1024.0;
+ unit += 1;
+ }
+ format!("{size:.1} {}", UNITS[unit])
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -127,82 +215,77 @@ mod tests {
}
#[test]
- fn llm_target_names_model_and_mmproj_files() {
- let target = llm_target(&preset(
+ fn llm_asset_names_model_and_mmproj_files() {
+ let asset = llm_asset(&preset(
Some("https://example.org/mmproj.gguf"),
- Some("abc"),
+ Some(&"a".repeat(64)),
))
.expect("paired preset");
- let downloader = ModelDownloader::new(std::env::temp_dir().join("ensu-models-test"));
+ let store = AssetStore::new(std::env::temp_dir().join("ensu-assets-test"));
assert_eq!(
- llm_model_path(&downloader, &target).unwrap(),
- downloader.model_dir(&target).join("model.gguf")
+ llm_model_path(&store, &asset).unwrap(),
+ store.asset_dir(&asset).join("model.gguf")
);
assert_eq!(
- llm_mmproj_path(&downloader, &target).unwrap(),
- downloader.model_dir(&target).join("mmproj.gguf")
+ llm_mmproj_path(&store, &asset).unwrap(),
+ store.asset_dir(&asset).join("mmproj.gguf")
);
}
#[test]
- fn llm_target_without_mmproj_has_no_mmproj_path() {
- let target = llm_target(&preset(None, None)).expect("model-only preset");
- let downloader = ModelDownloader::new(std::env::temp_dir().join("ensu-models-test"));
- assert!(llm_model_path(&downloader, &target).is_some());
- assert!(llm_mmproj_path(&downloader, &target).is_none());
+ fn llm_asset_without_mmproj_has_no_mmproj_path() {
+ let asset = llm_asset(&preset(None, None)).expect("model-only preset");
+ let store = AssetStore::new(std::env::temp_dir().join("ensu-assets-test"));
+ assert!(llm_model_path(&store, &asset).is_some());
+ assert!(llm_mmproj_path(&store, &asset).is_none());
}
#[test]
- fn llm_target_rejects_half_specified_mmproj() {
- assert!(llm_target(&preset(Some("https://example.org/mmproj.gguf"), None)).is_err());
- assert!(llm_target(&preset(None, Some("abc"))).is_err());
- assert!(llm_target(&preset(Some(" "), Some("abc"))).is_err());
+ fn llm_asset_rejects_half_specified_mmproj() {
+ assert!(llm_asset(&preset(Some("https://example.org/mmproj.gguf"), None)).is_err());
+ assert!(llm_asset(&preset(None, Some(&"a".repeat(64)))).is_err());
+ assert!(llm_asset(&preset(Some(" "), Some(&"a".repeat(64)))).is_err());
}
#[test]
- fn llm_targets_resolve_catalog_ids_or_fall_back_to_defaults() {
+ fn llm_assets_resolve_catalog_ids_or_fall_back_to_defaults() {
let defaults = crate::config::defaults();
for preset in crate::config::llm_catalog() {
assert_eq!(
- mobile_llm_target(&preset.id).unwrap(),
- llm_target(&preset).unwrap(),
+ mobile_llm_asset(&preset.id).unwrap(),
+ llm_asset(&preset).unwrap(),
"{}",
preset.id
);
assert_eq!(
- desktop_llm_target(&preset.id).unwrap(),
- llm_target(&preset).unwrap(),
+ desktop_llm_asset(&preset.id).unwrap(),
+ llm_asset(&preset).unwrap(),
"{}",
preset.id
);
}
- let mobile_default = llm_target(&defaults.mobile_default_model).unwrap();
- assert_eq!(mobile_llm_target("").unwrap(), mobile_default);
- assert_eq!(mobile_llm_target("no-such-model").unwrap(), mobile_default);
- let desktop_default = llm_target(&defaults.desktop_default_model).unwrap();
- assert_eq!(desktop_llm_target("").unwrap(), desktop_default);
- assert_eq!(
- desktop_llm_target("no-such-model").unwrap(),
- desktop_default
- );
+ let mobile_default = llm_asset(&defaults.mobile_default_model).unwrap();
+ assert_eq!(mobile_llm_asset("").unwrap(), mobile_default);
+ assert_eq!(mobile_llm_asset("no-such-model").unwrap(), mobile_default);
+ let desktop_default = llm_asset(&defaults.desktop_default_model).unwrap();
+ assert_eq!(desktop_llm_asset("").unwrap(), desktop_default);
+ assert_eq!(desktop_llm_asset("no-such-model").unwrap(), desktop_default);
}
#[test]
- fn all_catalog_presets_produce_valid_targets() {
+ fn all_catalog_presets_produce_valid_assets() {
for preset in crate::config::llm_catalog() {
- llm_target(&preset).expect("catalog preset");
+ llm_asset(&preset).expect("catalog preset");
}
}
#[test]
- fn knowledge_embedding_target_is_checksum_backed() {
- let target = knowledge_embedding_target();
- let ModelTarget::Files { id, files } = target else {
- panic!("embedding target must use files");
- };
- assert_eq!(id, "embeddinggemma-300m-q8-0");
- assert_eq!(files.len(), 1);
- assert_eq!(files[0].name, "model.gguf");
- assert_eq!(files[0].sha256.len(), 64);
+ fn knowledge_embedding_asset_is_checksum_backed() {
+ let store = AssetStore::new(std::env::temp_dir().join("ensu-assets-test"));
+ let asset = knowledge_embedding_model_asset();
+ assert_eq!(
+ store.file_path(&asset, "model.gguf"),
+ Some(store.asset_dir(&asset).join("model.gguf"))
+ );
}
}
diff --git a/rust/crates/ensu/src/retrieval.rs b/rust/crates/ensu/src/retrieval.rs
index 923701f0f3..b1d3e31896 100644
--- a/rust/crates/ensu/src/retrieval.rs
+++ b/rust/crates/ensu/src/retrieval.rs
@@ -45,6 +45,7 @@ pub use citation::{
pub use index::{RetrievalHit, RetrievalIndex};
pub use pack::{
KnowledgeReconciliation, KnowledgeReconciliationStatus,
- cleanup_obsolete_knowledge_pack_revisions, download_knowledge_pack, reconcile_knowledge_pack,
+ cleanup_obsolete_knowledge_pack_revisions, knowledge_asset, knowledge_pack_root,
+ reconcile_knowledge_pack,
};
pub use prompt::{KnowledgePromptContext, KnowledgePromptHit, build_knowledge_prompt_context};
diff --git a/rust/crates/ensu/src/retrieval/pack.rs b/rust/crates/ensu/src/retrieval/pack.rs
index a0add74dd0..65a9fc4572 100644
--- a/rust/crates/ensu/src/retrieval/pack.rs
+++ b/rust/crates/ensu/src/retrieval/pack.rs
@@ -1,7 +1,7 @@
use std::fs;
use std::path::{Path, PathBuf};
-use ente_model_download::download::{self, Downloader, Progress, Target};
+use ente_assets::{Asset, AssetFile, AssetStore, download};
use crate::config::{
KNOWLEDGE_ARTIFACT_FILENAMES, KnowledgeDatasetConfig, is_path_safe_component,
@@ -10,67 +10,39 @@ use crate::config::{
use super::{RetrievalError, index::RetrievalIndex};
-pub fn download_knowledge_pack(
- pack_root: impl AsRef<Path>,
- expected_pack: &KnowledgeDatasetConfig,
- on_progress: impl FnMut(Progress) + Send,
- cancellation: download::CancellationToken,
-) -> Result<(), download::Error> {
- let pack_root = pack_root.as_ref();
- let targets = knowledge_download_targets(pack_root, expected_pack)?;
- tokio::runtime::Builder::new_current_thread()
- .enable_io()
- .enable_time()
- .build()?
- .block_on(Downloader::new()?.download(targets, on_progress, cancellation))
-}
-
-fn knowledge_download_targets(
- pack_root: &Path,
- expected_pack: &KnowledgeDatasetConfig,
-) -> Result<Vec<Target>, download::Error> {
- let root_identity = pack_root
- .file_name()
- .and_then(|name| name.to_str())
- .ok_or_else(|| {
- download::Error::InvalidTarget(
- "knowledge pack root must have a UTF-8 stable ID".to_string(),
- )
- })?;
- if root_identity != expected_pack.stable_id {
- return Err(download::Error::InvalidTarget(
- "knowledge pack root must end with the selected stable ID".to_string(),
- ));
- }
- fs::create_dir_all(pack_root)?;
- if !fs::symlink_metadata(pack_root)?.file_type().is_dir() {
- return Err(download::Error::InvalidTarget(
- "knowledge pack root must be a directory, not a symlink".to_string(),
- ));
- }
- let revision_directory = pack_root.join(&expected_pack.current_download_identity);
- fs::create_dir_all(&revision_directory)?;
- if !fs::symlink_metadata(&revision_directory)?
- .file_type()
- .is_dir()
- {
- return Err(download::Error::InvalidTarget(
- "knowledge revision path must be a directory, not a symlink".to_string(),
- ));
- }
-
+pub fn knowledge_asset(expected_pack: &KnowledgeDatasetConfig) -> Result<Asset, download::Error> {
let urls = knowledge_artifact_urls(expected_pack);
- Ok(KNOWLEDGE_ARTIFACT_FILENAMES
+ let files = KNOWLEDGE_ARTIFACT_FILENAMES
.into_iter()
.zip(urls)
.zip(expected_pack.artifact_sha256.iter().cloned())
- .map(|((filename, url), sha256)| Target {
- label: filename.to_owned(),
+ .map(|((filename, url), sha256)| AssetFile {
+ name: filename.to_owned(),
url,
sha256,
- destination: revision_directory.join(filename),
})
- .collect())
+ .collect();
+ Asset::files(
+ vec![
+ "knowledge".to_string(),
+ expected_pack.stable_id.clone(),
+ expected_pack.current_download_identity.clone(),
+ ],
+ files,
+ )
+}
+
+pub fn knowledge_pack_root(
+ store: &AssetStore,
+ expected_pack: &KnowledgeDatasetConfig,
+) -> Result<PathBuf, download::Error> {
+ store
+ .asset_dir(&knowledge_asset(expected_pack)?)
+ .parent()
+ .map(Path::to_path_buf)
+ .ok_or_else(|| {
+ download::Error::InvalidTarget("knowledge asset has no pack directory".to_string())
+ })
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -241,23 +213,21 @@ mod tests {
use crate::retrieval::index::tests::synthetic_pack;
#[test]
- fn constructs_only_the_four_fixed_download_targets() {
- let temp = tempfile::tempdir().unwrap();
+ fn constructs_only_the_four_fixed_download_files() {
let expected = defaults().knowledge_datasets.remove(0);
- let pack_root = temp.path().join(&expected.stable_id);
- let targets = knowledge_download_targets(&pack_root, &expected).unwrap();
- assert_eq!(targets.len(), 4);
- for (target, filename) in targets.iter().zip(KNOWLEDGE_ARTIFACT_FILENAMES) {
- assert!(target.url.ends_with(filename));
- assert!(target.destination.ends_with(filename));
- assert!(
- target
- .destination
- .to_string_lossy()
- .contains(&expected.current_download_identity)
- );
- assert_eq!(target.sha256.len(), 64);
+ let store = AssetStore::new("assets");
+ let asset = knowledge_asset(&expected).unwrap();
+ assert_eq!(
+ store.asset_dir(&asset),
+ Path::new("assets")
+ .join("knowledge")
+ .join(&expected.stable_id)
+ .join(&expected.current_download_identity)
+ );
+ for name in KNOWLEDGE_ARTIFACT_FILENAMES {
+ assert!(store.file_path(&asset, name).is_some());
}
+ assert!(!store.is_downloaded(&asset));
}
#[test]
diff --git a/rust/crates/model-download/src/archive.rs b/rust/crates/model-download/src/archive.rs
deleted file mode 100644
index 146ee9e06f..0000000000
--- a/rust/crates/model-download/src/archive.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-use std::fs;
-use std::fs::File;
-use std::path::Path;
-
-use flate2::read::GzDecoder;
-use tar::Archive;
-
-use crate::download;
-
-pub(crate) fn extract_tar_gz(
- archive_path: &Path,
- staging_dir: &Path,
- dest_dir: &Path,
-) -> Result<(), download::Error> {
- if staging_dir.exists() {
- fs::remove_dir_all(staging_dir)?;
- }
- fs::create_dir_all(staging_dir)?;
-
- let tar_gz = File::open(archive_path)?;
- let mut archive = Archive::new(GzDecoder::new(tar_gz));
- archive.unpack(staging_dir)?;
-
- let entries = fs::read_dir(staging_dir)?.flatten().collect::<Vec<_>>();
- let source = match entries.as_slice() {
- [only] if only.path().is_dir() => only.path(),
- _ => staging_dir.to_path_buf(),
- };
- if dest_dir.exists() {
- fs::remove_dir_all(dest_dir)?;
- }
- fs::rename(&source, dest_dir)?;
- let _ = fs::remove_dir_all(staging_dir);
- Ok(())
-}
diff --git a/rust/crates/model-download/src/lib.rs b/rust/crates/model-download/src/lib.rs
deleted file mode 100644
index 6af18cdd83..0000000000
--- a/rust/crates/model-download/src/lib.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-mod archive;
-pub mod download;
-mod model;
-
-pub use model::{ModelDownloadProgress, ModelDownloader, ModelFile, ModelTarget};
diff --git a/rust/crates/model-download/src/model.rs b/rust/crates/model-download/src/model.rs
deleted file mode 100644
index 9b96045788..0000000000
--- a/rust/crates/model-download/src/model.rs
+++ /dev/null
@@ -1,648 +0,0 @@
-use std::collections::HashSet;
-use std::fs;
-use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Mutex, OnceLock, PoisonError};
-
-use tokio::runtime::Builder;
-
-use crate::archive;
-use crate::download::{self, Progress, Target};
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ModelTarget {
- Files {
- id: String,
- files: Vec<ModelFile>,
- },
- TarGz {
- id: String,
- url: String,
- sha256: String,
- },
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ModelFile {
- pub name: String,
- pub url: String,
- pub sha256: String,
-}
-
-#[derive(Debug, Clone)]
-pub struct ModelDownloadProgress {
- pub downloaded_bytes: u64,
- pub total_bytes: Option<u64>,
- pub percent: i32,
- pub status: String,
- pub log_line: Option<String>,
-}
-
-pub struct ModelDownloader {
- downloader: OnceLock<download::Downloader>,
- models_dir: PathBuf,
- active: AtomicBool,
- download_lock: Mutex<()>,
-}
-
-impl ModelDownloader {
- pub fn new(models_dir: impl Into<PathBuf>) -> Self {
- Self {
- downloader: OnceLock::new(),
- models_dir: models_dir.into(),
- active: AtomicBool::new(false),
- download_lock: Mutex::new(()),
- }
- }
-
- pub fn downloader(&self) -> Result<&download::Downloader, download::Error> {
- if let Some(downloader) = self.downloader.get() {
- return Ok(downloader);
- }
- let downloader = download::Downloader::new()?;
- Ok(self.downloader.get_or_init(|| downloader))
- }
-
- pub fn model_dir(&self, target: &ModelTarget) -> PathBuf {
- storage_dir(&self.models_dir, target)
- }
-
- pub fn file_path(&self, target: &ModelTarget, name: &str) -> Option<PathBuf> {
- match target {
- ModelTarget::Files { files, .. } if files.iter().any(|file| file.name == name) => {
- Some(storage_dir(&self.models_dir, target).join(name))
- }
- _ => None,
- }
- }
-
- pub fn is_downloaded(&self, target: &ModelTarget) -> bool {
- if let ModelTarget::TarGz { .. } = target {
- return storage_dir(&self.models_dir, target).is_dir();
- }
- let paths = expected_paths(&self.models_dir, target);
- !paths.is_empty() && paths.iter().all(|path| path.is_file())
- }
-
- pub fn is_download_active(&self) -> bool {
- self.active.load(Ordering::SeqCst)
- }
-
- pub fn remove_downloaded(&self, target: &ModelTarget) -> bool {
- let Ok(_guard) = self.download_lock.try_lock() else {
- return false;
- };
- if checked_file_name(storage_key(target)).is_err() {
- return false;
- }
- let dir = storage_dir(&self.models_dir, target);
- dir.exists() && fs::remove_dir_all(dir).is_ok()
- }
-
- pub fn download(
- &self,
- targets: &[ModelTarget],
- mut on_progress: impl FnMut(ModelDownloadProgress),
- cancellation: &download::CancellationToken,
- ) -> Result<(), download::Error> {
- let _guard = self
- .download_lock
- .lock()
- .unwrap_or_else(PoisonError::into_inner);
- let pending = targets
- .iter()
- .filter(|target| !self.is_downloaded(target))
- .collect::<Vec<_>>();
- if pending.is_empty() {
- return Ok(());
- }
-
- let mut transfers = Vec::new();
- for target in &pending {
- transfers.extend(download_targets(&self.models_dir, target)?);
- }
-
- if cancellation.is_cancelled() {
- return Err(download::Error::Cancelled);
- }
- let downloader = self.downloader()?;
- self.active.store(true, Ordering::SeqCst);
- let result = download::fetch(
- downloader,
- transfers,
- |progress| on_progress(display_progress(progress)),
- || cancellation.is_cancelled(),
- )
- .and_then(|()| {
- for target in &pending {
- if cancellation.is_cancelled() {
- return Err(download::Error::Cancelled);
- }
- self.extract_if_archive(target, &mut on_progress)?;
- }
- Ok(())
- });
- self.active.store(false, Ordering::SeqCst);
- result
- }
-
- fn extract_if_archive(
- &self,
- target: &ModelTarget,
- on_progress: &mut impl FnMut(ModelDownloadProgress),
- ) -> Result<(), download::Error> {
- let ModelTarget::TarGz { id, .. } = target else {
- return Ok(());
- };
- on_progress(ModelDownloadProgress {
- downloaded_bytes: 0,
- total_bytes: None,
- percent: 99,
- status: "Extracting...".to_string(),
- log_line: Some(format!("Extracting model archive id={id}")),
- });
- let archive_path = self.staging_dir().join(format!("{id}.tar.gz"));
- let extraction_dir = self.staging_dir().join(id);
- let result =
- archive::extract_tar_gz(&archive_path, &extraction_dir, &self.model_dir(target));
- if result.is_err() {
- let _ = fs::remove_dir_all(extraction_dir);
- } else {
- let _ = fs::remove_file(&archive_path);
- }
- result
- }
-
- fn staging_dir(&self) -> PathBuf {
- self.models_dir.join(".staging")
- }
-
- pub fn estimated_download_size(&self, target: &ModelTarget) -> Option<u64> {
- let runtime = Builder::new_current_thread()
- .enable_io()
- .enable_time()
- .build()
- .ok()?;
- let downloader = self.downloader().ok()?;
- runtime.block_on(async {
- let mut total: u64 = 0;
- let mut any = false;
- for entry in download_targets(&self.models_dir, target).ok()? {
- let path = &entry.destination;
- let size = if path.exists() {
- fs::metadata(path).ok().map(|m| m.len()).filter(|s| *s > 0)
- } else {
- download::probe_content_length(downloader.client(), &entry.url).await
- };
- if let Some(size) = size {
- any = true;
- total = total.saturating_add(size);
- }
- }
- any.then_some(total)
- })
- }
-}
-
-fn storage_key(target: &ModelTarget) -> &str {
- match target {
- ModelTarget::Files { id, .. } | ModelTarget::TarGz { id, .. } => id,
- }
-}
-
-fn storage_dir(models_dir: &Path, target: &ModelTarget) -> PathBuf {
- models_dir.join(storage_key(target))
-}
-
-fn expected_paths(models_dir: &Path, target: &ModelTarget) -> Vec<PathBuf> {
- let dir = storage_dir(models_dir, target);
- match target {
- ModelTarget::TarGz { .. } => vec![dir],
- ModelTarget::Files { files, .. } => files.iter().map(|file| dir.join(&file.name)).collect(),
- }
-}
-
-fn download_targets(
- models_dir: &Path,
- target: &ModelTarget,
-) -> Result<Vec<Target>, download::Error> {
- let dir = storage_dir(models_dir, target);
- match target {
- ModelTarget::TarGz { id, url, sha256 } => {
- checked_file_name(id)?;
- Ok(vec![Target {
- label: "Model".to_string(),
- url: url.clone(),
- sha256: sha256.clone(),
- destination: models_dir.join(".staging").join(format!("{id}.tar.gz")),
- }])
- }
- ModelTarget::Files { id, files } => {
- checked_file_name(id)?;
- if files.is_empty() {
- return Err(download::Error::InvalidTarget(format!("{id} has no files")));
- }
- let mut names = HashSet::new();
- files
- .iter()
- .map(|file| {
- checked_file_name(&file.name)?;
- if !names.insert(file.name.as_str()) {
- return Err(download::Error::InvalidTarget(format!(
- "{id} has a duplicate file name {}",
- file.name
- )));
- }
- Ok(Target {
- label: file.name.clone(),
- url: file.url.clone(),
- sha256: file.sha256.clone(),
- destination: dir.join(&file.name),
- })
- })
- .collect()
- }
- }
-}
-
-fn checked_file_name(name: &str) -> Result<(), download::Error> {
- if !name.is_empty()
- && name != "."
- && name != ".."
- && !name.contains('/')
- && !name.contains('\\')
- {
- return Ok(());
- }
- Err(download::Error::InvalidTarget(format!(
- "'{name}' is not a safe file name"
- )))
-}
-
-fn display_progress(progress: Progress) -> ModelDownloadProgress {
- let total = progress.total_bytes.filter(|total| *total > 0);
- let percent = total
- .map(|total| ((progress.downloaded_bytes * 100 / total) as i32).clamp(0, 99))
- .unwrap_or(0);
- let status = if let Some(total) = total {
- format!(
- "Downloading... {} / {}",
- format_bytes(progress.downloaded_bytes),
- format_bytes(total)
- )
- } else if progress.file_downloaded_bytes > 0 {
- format!(
- "Downloading {}... {}",
- progress.label.to_lowercase(),
- format_bytes(progress.file_downloaded_bytes)
- )
- } else {
- format!("Downloading {}...", progress.label.to_lowercase())
- };
- let log_line = if progress.file_complete {
- Some(format!(
- "Model download file complete label={} bytes={} elapsedMs={} rate={}/s retries={}",
- progress.label,
- progress.file_downloaded_bytes,
- progress.file_elapsed_ms,
- format_bytes(rate_bytes(progress.file_bytes_per_second)),
- progress.file_retry_count
- ))
- } else if progress.complete {
- Some(format!(
- "Model download complete bytes={} elapsedMs={} rate={}/s retries={}",
- progress.downloaded_bytes,
- progress.elapsed_ms,
- format_bytes(rate_bytes(progress.bytes_per_second)),
- progress.retry_count
- ))
- } else {
- None
- };
-
- ModelDownloadProgress {
- downloaded_bytes: progress.downloaded_bytes,
- total_bytes: progress.total_bytes,
- percent,
- status,
- log_line,
- }
-}
-
-fn rate_bytes(bytes_per_second: f64) -> u64 {
- if bytes_per_second.is_finite() && bytes_per_second > 0.0 {
- bytes_per_second as u64
- } else {
- 0
- }
-}
-
-fn format_bytes(bytes: u64) -> String {
- const UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
- let mut size = bytes as f64;
- let mut unit = 0;
- while size >= 1024.0 && unit < UNITS.len() - 1 {
- size /= 1024.0;
- unit += 1;
- }
- format!("{size:.1} {}", UNITS[unit])
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn scratch_dir(name: &str) -> PathBuf {
- let dir = std::env::temp_dir().join(format!(
- "ente-model-download-{name}-{}",
- std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .unwrap()
- .as_nanos()
- ));
- fs::create_dir_all(&dir).expect("create test dir");
- dir
- }
-
- fn test_sha() -> String {
- "0".repeat(64)
- }
-
- fn target(id: &str) -> ModelTarget {
- ModelTarget::Files {
- id: id.to_string(),
- files: vec![
- ModelFile {
- name: "model.gguf".to_string(),
- url: "https://example.org/models/main.gguf?download=true".to_string(),
- sha256: test_sha(),
- },
- ModelFile {
- name: "mmproj.gguf".to_string(),
- url: "https://example.org/models/mmproj.gguf".to_string(),
- sha256: test_sha(),
- },
- ],
- }
- }
-
- fn write_gguf(path: &Path, content: &[u8]) {
- fs::create_dir_all(path.parent().unwrap()).unwrap();
- fs::write(path, content).unwrap();
- }
-
- #[test]
- fn paths_are_keyed_by_preset_id() {
- let dir = scratch_dir("paths");
- let downloader = ModelDownloader::new(&dir);
-
- assert_eq!(
- downloader
- .file_path(&target("qwen-2b-q8"), "model.gguf")
- .unwrap(),
- dir.join("qwen-2b-q8/model.gguf")
- );
- assert_eq!(
- downloader.file_path(&target("qwen-2b-q8"), "mmproj.gguf"),
- Some(dir.join("qwen-2b-q8/mmproj.gguf"))
- );
-
- let _ = fs::remove_dir_all(dir);
- }
-
- #[test]
- fn is_downloaded_requires_every_member_file() {
- let dir = scratch_dir("is-downloaded");
- let downloader = ModelDownloader::new(&dir);
- let target = target("qwen-2b-q8");
- let model = downloader.file_path(&target, "model.gguf").unwrap();
- let mmproj = downloader.file_path(&target, "mmproj.gguf").unwrap();
-
- assert!(!downloader.is_downloaded(&target));
-
- write_gguf(&model, b"GGUFdata");
- assert!(!downloader.is_downloaded(&target));
-
- write_gguf(&mmproj, b"GGUFdata");
- assert!(downloader.is_downloaded(&target));
-
- let _ = fs::remove_dir_all(dir);
- }
-
- #[test]
- fn download_skips_when_already_present() {
- let dir = scratch_dir("skip");
- let downloader = ModelDownloader::new(&dir);
- let target = target("qwen-2b-q8");
- write_gguf(
- &downloader.file_path(&target, "model.gguf").unwrap(),
- b"GGUFdata",
- );
- write_gguf(
- &downloader.file_path(&target, "mmproj.gguf").unwrap(),
- b"GGUFdata",
- );
-
- downloader
- .download(
- std::slice::from_ref(&target),
- |_| {},
- &download::CancellationToken::new(),
- )
- .expect("skip without network");
-
- let _ = fs::remove_dir_all(dir);
- }
-
- fn tar_gz_target(id: &str) -> ModelTarget {
- ModelTarget::TarGz {
- id: id.to_string(),
- url: format!("https://models.example.org/{id}.tar.gz"),
- sha256: test_sha(),
- }
- }
-
- fn file_target(id: &str) -> ModelTarget {
- ModelTarget::Files {
- id: id.to_string(),
- files: vec![ModelFile {
- name: "model.onnx".to_string(),
- url: format!("https://models.example.org/{id}.onnx"),
- sha256: test_sha(),
- }],
- }
- }
-
- fn write_tar_gz(path: &Path, dir_name: &str, files: &[(&str, &[u8])]) {
- let file = fs::File::create(path).unwrap();
- let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
- let mut builder = tar::Builder::new(encoder);
- for (name, content) in files {
- let mut header = tar::Header::new_gnu();
- header.set_size(content.len() as u64);
- header.set_mode(0o644);
- header.set_cksum();
- builder
- .append_data(&mut header, format!("{dir_name}/{name}"), *content)
- .unwrap();
- }
- builder.into_inner().unwrap().finish().unwrap();
- }
-
- #[test]
- fn archive_and_onnx_targets_use_per_model_dirs() {
- let dir = scratch_dir("variant-paths");
- let downloader = ModelDownloader::new(&dir);
-
- let archive = tar_gz_target("parakeet-v3-int8");
- assert_eq!(downloader.model_dir(&archive), dir.join("parakeet-v3-int8"));
- assert_eq!(downloader.file_path(&archive, "mmproj.gguf"), None);
-
- let onnx = file_target("silero-vad-v4");
- assert_eq!(
- downloader.file_path(&onnx, "model.onnx").unwrap(),
- dir.join("silero-vad-v4/model.onnx")
- );
- assert_eq!(
- downloader.file_path(&onnx, "model.onnx"),
- Some(dir.join("silero-vad-v4/model.onnx"))
- );
- assert_eq!(downloader.file_path(&onnx, "missing.onnx"), None);
- assert_eq!(downloader.file_path(&onnx, "mmproj.gguf"), None);
-
- let _ = fs::remove_dir_all(dir);
- }
-
- #[test]
- fn archive_download_extracts_into_model_dir() {
- let dir = scratch_dir("variant-extract");
- let downloader = ModelDownloader::new(&dir);
- let target = tar_gz_target("parakeet-v3-int8");
- assert!(!downloader.is_downloaded(&target));
-
- fs::create_dir_all(dir.join(".staging")).unwrap();
- write_tar_gz(
- &dir.join(".staging/parakeet-v3-int8.tar.gz"),
- "parakeet-tdt-0.6b-v3-int8",
- &[("encoder.onnx", b"enc"), ("tokens.txt", b"tok")],
- );
- let mut statuses = Vec::new();
- downloader
- .extract_if_archive(&target, &mut |progress| statuses.push(progress.status))
- .expect("extract");
-
- assert!(downloader.is_downloaded(&target));
- assert_eq!(
- fs::read(dir.join("parakeet-v3-int8/encoder.onnx")).unwrap(),
- b"enc"
- );
- assert_eq!(statuses, ["Extracting..."]);
- assert!(!dir.join(".staging/parakeet-v3-int8.tar.gz").exists());
-
- assert!(downloader.remove_downloaded(&target));
- assert!(!downloader.is_downloaded(&target));
-
- let _ = fs::remove_dir_all(dir);
- }
-
- #[test]
- fn failed_archive_extraction_cleans_staging() {
- let dir = scratch_dir("variant-extract-failure");
- let downloader = ModelDownloader::new(&dir);
- let target = tar_gz_target("parakeet-v3-int8");
- let staging_dir = dir.join(".staging");
- fs::create_dir_all(&staging_dir).unwrap();
- fs::write(staging_dir.join("parakeet-v3-int8.tar.gz"), b"invalid").unwrap();
-
- assert!(downloader.extract_if_archive(&target, &mut |_| {}).is_err());
- assert!(!staging_dir.join("parakeet-v3-int8").exists());
- assert!(staging_dir.join("parakeet-v3-int8.tar.gz").exists());
-
- let _ = fs::remove_dir_all(dir);
- }
-
- #[test]
- fn onnx_target_downloaded_and_removed_via_model_file() {
- let dir = scratch_dir("variant-onnx");
- let downloader = ModelDownloader::new(&dir);
- let target = file_target("silero-vad-v4");
- assert!(!downloader.is_downloaded(&target));
-
- write_gguf(
- &downloader.file_path(&target, "model.onnx").unwrap(),
- b"onnxbytes",
- );
- assert!(downloader.is_downloaded(&target));
-
- assert!(downloader.remove_downloaded(&target));
- assert!(!downloader.is_downloaded(&target));
- assert!(!dir.join("silero-vad-v4").exists());
-
- let _ = fs::remove_dir_all(dir);
- }
-
- #[test]
- fn display_progress_composes_percent_and_status() {
- let progress = Progress {
- label: "Model".to_string(),
- downloaded_bytes: 512 * 1024 * 1024,
- total_bytes: Some(2 * 1024 * 1024 * 1024),
- file_downloaded_bytes: 512 * 1024 * 1024,
- file_total_bytes: Some(2 * 1024 * 1024 * 1024),
- percentage: 25.0,
- elapsed_ms: 0,
- bytes_per_second: 0.0,
- file_elapsed_ms: 0,
- file_bytes_per_second: 0.0,
- retry_count: 0,
- file_retry_count: 0,
- file_complete: false,
- complete: false,
- };
-
- let display = display_progress(progress);
- assert_eq!(display.percent, 25);
- assert_eq!(display.status, "Downloading... 512.0 MB / 2.0 GB");
- assert_eq!(display.log_line, None);
-
- let unknown_total = Progress {
- total_bytes: None,
- ..display_to_progress_stub()
- };
- let display = display_progress(unknown_total);
- assert_eq!(display.percent, 0);
- assert_eq!(display.status, "Downloading model... 100.0 B");
-
- let file_complete = Progress {
- file_complete: true,
- file_bytes_per_second: 2048.0,
- ..display_to_progress_stub()
- };
- let display = display_progress(file_complete);
- assert_eq!(
- display.log_line.as_deref(),
- Some(
- "Model download file complete label=Model bytes=100 elapsedMs=0 rate=2.0 KB/s retries=0"
- )
- );
- }
-
- fn display_to_progress_stub() -> Progress {
- Progress {
- label: "Model".to_string(),
- downloaded_bytes: 100,
- total_bytes: None,
- file_downloaded_bytes: 100,
- file_total_bytes: None,
- percentage: 0.0,
- elapsed_ms: 0,
- bytes_per_second: 0.0,
- file_elapsed_ms: 0,
- file_bytes_per_second: 0.0,
- retry_count: 0,
- file_retry_count: 0,
- file_complete: false,
- complete: false,
- }
- }
-}
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index 53ec14e8e4..980da9e89d 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -61,11 +61,11 @@ ort = { workspace = true, features = [
[dev-dependencies]
anyhow = "1"
-ente-model-download.workspace = true
+ente-assets.workspace = true
flate2 = "1.1"
serde = { version = "1", features = ["derive"] }
serde_json.workspace = true
tar = "0.4"
tempfile = "3.12"
-tokio = { version = "1", features = ["rt"] }
+tokio = { version = "1", features = ["macros", "rt"] }
url = "2.5.8"
diff --git a/rust/crates/photos/examples/ml_goldens.rs b/rust/crates/photos/examples/ml_goldens.rs
index fa561d6bf1..d2e372eb47 100644
--- a/rust/crates/photos/examples/ml_goldens.rs
+++ b/rust/crates/photos/examples/ml_goldens.rs
@@ -16,7 +16,8 @@ use support::ml_indexing::{GoldenTestAssets, run_with_large_stack};
const CLIP_TEXT_GOLDEN_PHRASE: &str = "a photo of a dog playing on a sunny beach";
-fn main() -> Result<()> {
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> Result<()> {
let mut args = std::env::args().skip(1);
let command = args.next();
if args.next().is_some() {
@@ -24,13 +25,18 @@ fn main() -> Result<()> {
}
match command.as_deref() {
- Some("generate") => run_with_large_stack("generate_goldens", generate),
+ Some("generate") => {
+ let assets = GoldenTestAssets::load().await?;
+ run_with_large_stack("generate_goldens", move || generate(assets))
+ }
Some("validate-zero-separation") => {
- run_with_large_stack("validate_zero_separation", validate_zero_separation)
+ let assets = GoldenTestAssets::load().await?;
+ run_with_large_stack("validate_zero_separation", move || {
+ validate_zero_separation(assets)
+ })
}
_ => bail!(usage()),
}
- Ok(())
}
fn usage() -> &'static str {
@@ -73,8 +79,7 @@ fn production_model_specs(assets: &GoldenTestAssets) -> Vec<GoldenModelSpec> {
]
}
-fn generate() -> Result<()> {
- let assets = GoldenTestAssets::load()?;
+fn generate(assets: GoldenTestAssets) -> Result<()> {
let source = render_golden_data(&production_model_specs(&assets))
.map_err(|error| anyhow::anyhow!("generating goldens: {error}"))?;
let path = GoldenTestAssets::golden_data_path()?;
@@ -92,8 +97,7 @@ fn generate() -> Result<()> {
Ok(())
}
-fn validate_zero_separation() -> Result<()> {
- let assets = GoldenTestAssets::load()?;
+fn validate_zero_separation(assets: GoldenTestAssets) -> Result<()> {
for model_path in [
&assets.face_detection.path,
&assets.face_embedding.path,
diff --git a/rust/crates/photos/tests/ml_indexing.rs b/rust/crates/photos/tests/ml_indexing.rs
index 19f785b96b..aefa062f87 100644
--- a/rust/crates/photos/tests/ml_indexing.rs
+++ b/rust/crates/photos/tests/ml_indexing.rs
@@ -7,13 +7,15 @@ use support::ml_indexing::{
ComparisonStats, MlIndexingTestContext, fail_if_any, run_with_large_stack,
};
-#[test]
-fn rust_ml_matches_python_goldens() {
- run_with_large_stack("rust_ml_matches_python_goldens", run_ml_indexing_test);
+#[tokio::test]
+async fn rust_ml_matches_python_goldens() -> Result<()> {
+ let context = MlIndexingTestContext::load().await?;
+ run_with_large_stack("rust_ml_matches_python_goldens", move || {
+ run_ml_indexing_test(context)
+ })
}
-fn run_ml_indexing_test() -> Result<()> {
- let context = MlIndexingTestContext::load()?;
+fn run_ml_indexing_test(context: MlIndexingTestContext) -> Result<()> {
let runtime = context.prepare_runtime();
let mut failures = context.validate_manifest_expectations()?;
diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rs
index f9a2d30e62..aa9fbdc43f 100644
--- a/rust/crates/photos/tests/support/ml_indexing.rs
+++ b/rust/crates/photos/tests/support/ml_indexing.rs
@@ -1,11 +1,11 @@
use std::{
collections::{BTreeSet, HashMap, HashSet},
fs,
- path::{Path, PathBuf},
+ path::{Component, Path, PathBuf},
};
use anyhow::{Context, Result, bail};
-use ente_model_download::download;
+use ente_assets::{Asset, AssetFile, AssetStore, download};
use ente_photos::ml::{
error::MlError,
indexing::{
@@ -24,17 +24,19 @@ const FACE_EMBEDDING_DIM: usize = 192;
const FLOAT_TOLERANCE: f64 = 1e-8;
const PRINT_STATS_ENV: &str = "ENTE_ML_INDEXING_PRINT_STATS";
-pub(crate) fn run_with_large_stack(name: &str, test: fn() -> Result<()>) {
+pub(crate) fn run_with_large_stack(
+ name: &str,
+ test: impl FnOnce() -> Result<()> + Send + 'static,
+) -> Result<()> {
let result = std::thread::Builder::new()
.name(name.to_string())
.stack_size(64 * 1024 * 1024)
.spawn(test)
- .expect("spawn ML indexing test thread")
+ .with_context(|| format!("spawn {name} thread"))?
.join();
match result {
- Ok(Ok(())) => {}
- Ok(Err(error)) => panic!("{error:#}"),
+ Ok(result) => result,
Err(panic) => std::panic::resume_unwind(panic),
}
}
@@ -163,73 +165,38 @@ fn should_print_stats() -> bool {
pub(crate) struct MlIndexingTestContext {
asset_lock: AssetLock,
- cache_dir: PathBuf,
+ store: AssetStore,
golden_results: HashMap<String, ComparableResult>,
manifest: FixtureManifest,
model_paths: ModelPaths,
}
-struct AssetFetcher {
- runtime: tokio::runtime::Runtime,
- downloader: download::Downloader,
-}
-
-impl AssetFetcher {
- fn new() -> Result<Self> {
- Ok(Self {
- runtime: tokio::runtime::Builder::new_current_thread()
- .enable_all()
- .build()
- .context("build download runtime")?,
- downloader: download::Downloader::new().context("build downloader")?,
- })
- }
-
- fn fetch(&self, targets: Vec<download::Target>) -> Result<()> {
- self.runtime.block_on(self.downloader.download(
- targets,
- |_| {},
- download::CancellationToken::default(),
- ))?;
- Ok(())
- }
-}
-
impl MlIndexingTestContext {
- pub(crate) fn load() -> Result<Self> {
+ pub(crate) async fn load() -> Result<Self> {
let repo_root = repo_root()?;
let asset_lock = load_asset_lock(&repo_root)?;
let cache_dir = cache_dir(&repo_root);
- let fetcher = AssetFetcher::new()?;
+ let store = AssetStore::new(&cache_dir);
let manifest_path =
- resolve_document_asset(&fetcher, &cache_dir, "manifest", &asset_lock.manifest)?;
- let golden_path = resolve_document_asset(
- &fetcher,
- &cache_dir,
- "python-golden",
- &asset_lock.python_golden,
- )?;
+ resolve_document_asset(&store, "manifest", &asset_lock.manifest).await?;
+ let golden_path =
+ resolve_document_asset(&store, "python-golden", &asset_lock.python_golden).await?;
let manifest = load_manifest(&manifest_path)?;
let golden_results = load_golden_results(&golden_path)?;
- fetch_fixtures(
- &fetcher,
- &cache_dir,
- &asset_lock.fixture_base_url,
- &manifest,
- )?;
+ fetch_fixtures(&store, &asset_lock.fixture_base_url, &manifest).await?;
let onnx_runtime_library =
- resolve_onnx_runtime_library(&fetcher, &cache_dir, &asset_lock.onnx_runtime)?;
+ resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;
let _ = ort::init_from(&onnx_runtime_library)
.context("load ONNX Runtime dynamic library")?
.commit();
- let model_paths = resolve_model_paths(&fetcher, &cache_dir, &asset_lock.models)?;
+ let model_paths = resolve_model_paths(&store, &asset_lock.models).await?;
Ok(Self {
asset_lock,
- cache_dir,
+ store,
golden_results,
manifest,
model_paths,
@@ -368,7 +335,17 @@ impl MlIndexingTestContext {
}
fn resolve_fixture(&self, fixture: &FixtureFile) -> Result<PathBuf> {
- cache_path_for(&self.cache_dir, "fixtures", &fixture.path, &fixture.sha256)
+ let label = file_id_for_manifest_path(&fixture.path)?;
+ let asset = file_asset(
+ "fixtures",
+ &label,
+ &fixture.path,
+ &fixture_url(&self.asset_lock.fixture_base_url, &fixture.path)?,
+ &fixture.sha256,
+ )?;
+ self.store
+ .file_path(&asset, &label)
+ .context("fixture asset has no declared file")
}
fn unsupported_decode_file_ids(&self) -> HashSet<String> {
@@ -401,37 +378,30 @@ pub(crate) struct GoldenTestAssets {
#[allow(dead_code)]
impl GoldenTestAssets {
- pub(crate) fn load() -> Result<Self> {
+ pub(crate) async fn load() -> Result<Self> {
let repo_root = repo_root()?;
let asset_lock = load_asset_lock(&repo_root)?;
let cache_dir = cache_dir(&repo_root);
- let fetcher = AssetFetcher::new()?;
+ let store = AssetStore::new(&cache_dir);
let onnx_runtime_library =
- resolve_onnx_runtime_library(&fetcher, &cache_dir, &asset_lock.onnx_runtime)?;
+ resolve_onnx_runtime_library(&store, &asset_lock.onnx_runtime).await?;
let _ = ort::init_from(&onnx_runtime_library)
.context("load ONNX Runtime dynamic library")?
.commit();
- let golden_model = |label: &str, model: &ModelAsset| -> Result<GoldenModelAsset> {
- Ok(GoldenModelAsset {
- path: resolve_model_asset(&fetcher, &cache_dir, label, model)?,
- sha256: model.sha256.clone(),
- })
- };
-
let models = &asset_lock.models;
Ok(Self {
- face_detection: golden_model("face-detection", &models.face_detection)?,
- face_embedding: golden_model("face-embedding", &models.face_embedding)?,
- clip_image: golden_model("clip-image", &models.clip_image)?,
- clip_text: golden_model("clip-text", &models.clip_text)?,
+ face_detection: golden_model(&store, "face-detection", &models.face_detection).await?,
+ face_embedding: golden_model(&store, "face-embedding", &models.face_embedding).await?,
+ clip_image: golden_model(&store, "clip-image", &models.clip_image).await?,
+ clip_text: golden_model(&store, "clip-text", &models.clip_text).await?,
clip_text_vocab: resolve_model_asset(
- &fetcher,
- &cache_dir,
+ &store,
"clip-text-vocab",
&models.clip_text_vocab,
- )?,
+ )
+ .await?,
})
}
@@ -562,7 +532,7 @@ fn repo_root() -> Result<PathBuf> {
}
fn cache_dir(repo_root: &Path) -> PathBuf {
- repo_root.join("rust/.cache")
+ repo_root.join("rust/.cache/assets")
}
fn load_asset_lock(repo_root: &Path) -> Result<AssetLock> {
@@ -614,14 +584,21 @@ fn file_id_for_manifest_path(path: &str) -> Result<String> {
.with_context(|| format!("derive file_id from manifest path '{path}'"))
}
-fn resolve_document_asset(
- fetcher: &AssetFetcher,
- cache_dir: &Path,
+async fn resolve_document_asset(
+ store: &AssetStore,
label: &str,
asset: &DocumentAsset,
) -> Result<PathBuf> {
- let target = cache_path_for(cache_dir, "documents", &asset.path, &asset.sha256)?;
- ensure_remote_asset(fetcher, &asset.url, &asset.sha256, &target, label)
+ download_asset(
+ store,
+ "documents",
+ label,
+ &asset.path,
+ &asset.url,
+ &asset.sha256,
+ label,
+ )
+ .await
}
fn fixture_url(fixture_base_url: &str, path: &str) -> Result<String> {
@@ -632,30 +609,29 @@ fn fixture_url(fixture_base_url: &str, path: &str) -> Result<String> {
.to_string())
}
-fn fetch_fixtures(
- fetcher: &AssetFetcher,
- cache_dir: &Path,
+async fn fetch_fixtures(
+ store: &AssetStore,
fixture_base_url: &str,
manifest: &FixtureManifest,
) -> Result<()> {
- let targets = manifest
- .files
- .iter()
- .map(|fixture| {
- Ok(download::Target {
- label: file_id_for_manifest_path(&fixture.path)?,
- url: fixture_url(fixture_base_url, &fixture.path)?,
- sha256: normalize_sha256(&fixture.sha256),
- destination: cache_path_for(cache_dir, "fixtures", &fixture.path, &fixture.sha256)?,
- })
- })
- .collect::<Result<Vec<_>>>()?;
- fetcher.fetch(targets).context("fetch fixtures")
+ for fixture in &manifest.files {
+ let label = file_id_for_manifest_path(&fixture.path)?;
+ download_asset(
+ store,
+ "fixtures",
+ &label,
+ &fixture.path,
+ &fixture_url(fixture_base_url, &fixture.path)?,
+ &fixture.sha256,
+ &label,
+ )
+ .await?;
+ }
+ Ok(())
}
-fn resolve_onnx_runtime_library(
- fetcher: &AssetFetcher,
- cache_dir: &Path,
+async fn resolve_onnx_runtime_library(
+ store: &AssetStore,
assets: &OnnxRuntimeAssets,
) -> Result<PathBuf> {
let target_key = onnx_runtime_target_key()?;
@@ -672,44 +648,43 @@ fn resolve_onnx_runtime_library(
})?;
let archive_name = url_file_name(&archive.url)?;
- let archive_path = cache_path_for(
- cache_dir,
- "onnx-runtime-archives",
- &archive_name,
- &archive.sha256,
- )?;
- let archive_path = ensure_remote_asset(
- fetcher,
+ let archive_asset = file_asset(
+ "onnx-runtime",
+ &target_key,
+ &format!("{target_key}/{archive_name}"),
&archive.url,
&archive.sha256,
- &archive_path,
- "onnx-runtime-archive",
)?;
+ store
+ .download(
+ &archive_asset,
+ |_| {},
+ download::CancellationToken::default(),
+ )
+ .await
+ .context("download ONNX Runtime archive")?;
+ let archive_path = store.asset_dir(&archive_asset).join(&archive_name);
let library_name = file_id_for_manifest_path(&archive.library_path)?;
- let library_cache_path = format!("{target_key}/{library_name}");
- let library_target = cache_path_for(
- cache_dir,
- "onnx-runtime-libraries",
- &library_cache_path,
- &archive.library_sha256,
- )?;
- if library_target.is_file() {
- verify_file_sha256(
+ let library_target = store.asset_dir(&archive_asset).join(library_name);
+ if library_target.is_file()
+ && verify_file_sha256(
&library_target,
&archive.library_sha256,
"onnx-runtime-library",
- )?;
+ )
+ .is_ok()
+ {
return Ok(library_target);
}
- extract_tgz_member(&archive_path, &archive.library_path, &library_target)
- .with_context(|| format!("extract ONNX Runtime library for {target_key}"))?;
- verify_file_sha256(
+ extract_tgz_member(
+ &archive_path,
+ &archive.library_path,
&library_target,
&archive.library_sha256,
- "onnx-runtime-library",
- )?;
+ )
+ .with_context(|| format!("extract ONNX Runtime library for {target_key}"))?;
Ok(library_target)
}
@@ -736,9 +711,18 @@ fn url_file_name(url: &str) -> Result<String> {
.with_context(|| format!("derive file name from URL '{url}'"))
}
-fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) -> Result<()> {
+fn extract_tgz_member(
+ archive_path: &Path,
+ member_path: &str,
+ target: &Path,
+ expected_sha256: &str,
+) -> Result<()> {
let expected_member = Path::new(member_path);
- if expected_member.is_absolute() {
+ if expected_member.is_absolute()
+ || expected_member
+ .components()
+ .any(|component| !matches!(component, Component::CurDir | Component::Normal(_)))
+ {
bail!("archive member path must be relative: {member_path}");
}
@@ -758,9 +742,25 @@ fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) ->
if entry_path.as_ref() != expected_member {
continue;
}
- entry
- .unpack(target)
- .with_context(|| format!("unpack {member_path} to {}", target.display()))?;
+ if !entry.header().entry_type().is_file() {
+ bail!("archive member is not a regular file: {member_path}");
+ }
+ let temporary = PathBuf::from(format!("{}.tmp", target.display()));
+ let _ = fs::remove_file(&temporary);
+ let result: Result<()> = (|| {
+ entry
+ .unpack(&temporary)
+ .with_context(|| format!("unpack {member_path} to {}", temporary.display()))?;
+ verify_file_sha256(&temporary, expected_sha256, "onnx-runtime-library")?;
+ let _ = fs::remove_file(target);
+ fs::rename(&temporary, target)
+ .with_context(|| format!("publish {}", target.display()))?;
+ Ok(())
+ })();
+ if result.is_err() {
+ let _ = fs::remove_file(temporary);
+ }
+ result?;
return Ok(());
}
@@ -770,16 +770,12 @@ fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) ->
);
}
-fn resolve_model_paths(
- fetcher: &AssetFetcher,
- cache_dir: &Path,
- models: &ModelAssets,
-) -> Result<ModelPaths> {
+async fn resolve_model_paths(store: &AssetStore, models: &ModelAssets) -> Result<ModelPaths> {
let face_detection =
- resolve_model_asset(fetcher, cache_dir, "face-detection", &models.face_detection)?;
+ resolve_model_asset(store, "face-detection", &models.face_detection).await?;
let face_embedding =
- resolve_model_asset(fetcher, cache_dir, "face-embedding", &models.face_embedding)?;
- let clip_image = resolve_model_asset(fetcher, cache_dir, "clip-image", &models.clip_image)?;
+ resolve_model_asset(store, "face-embedding", &models.face_embedding).await?;
+ let clip_image = resolve_model_asset(store, "clip-image", &models.clip_image).await?;
Ok(ModelPaths {
face_detection: face_detection.to_string_lossy().into_owned(),
@@ -795,49 +791,94 @@ fn resolve_model_paths(
})
}
-fn resolve_model_asset(
- fetcher: &AssetFetcher,
- cache_dir: &Path,
+async fn golden_model(
+ store: &AssetStore,
+ label: &str,
+ model: &ModelAsset,
+) -> Result<GoldenModelAsset> {
+ Ok(GoldenModelAsset {
+ path: download_asset(
+ store,
+ "models",
+ label,
+ &model.file_name,
+ &model.url,
+ &model.sha256,
+ label,
+ )
+ .await?,
+ sha256: model.sha256.clone(),
+ })
+}
+
+async fn resolve_model_asset(
+ store: &AssetStore,
label: &str,
asset: &ModelAsset,
) -> Result<PathBuf> {
- let target = cache_path_for(cache_dir, "models", &asset.file_name, &asset.sha256)?;
- ensure_remote_asset(fetcher, &asset.url, &asset.sha256, &target, label)
+ download_asset(
+ store,
+ "models",
+ label,
+ &asset.file_name,
+ &asset.url,
+ &asset.sha256,
+ &asset.file_name,
+ )
+ .await
}
-fn ensure_remote_asset(
- fetcher: &AssetFetcher,
+async fn download_asset(
+ store: &AssetStore,
+ namespace: &str,
+ key: &str,
+ relative_path: &str,
url: &str,
expected_sha256: &str,
- target: &Path,
label: &str,
) -> Result<PathBuf> {
- fetcher
- .fetch(vec![download::Target {
- label: label.to_string(),
- url: url.to_string(),
- sha256: normalize_sha256(expected_sha256),
- destination: target.to_path_buf(),
- }])
+ let asset = file_asset(namespace, key, relative_path, url, expected_sha256)?;
+ store
+ .download(&asset, |_| {}, download::CancellationToken::default())
+ .await
.with_context(|| format!("download {label} from {url}"))?;
- Ok(target.to_path_buf())
+ let name = Path::new(relative_path)
+ .file_name()
+ .with_context(|| format!("asset path has no file name: {relative_path}"))?;
+ Ok(store.asset_dir(&asset).join(name))
}
-fn cache_path_for(
- cache_dir: &Path,
+fn file_asset(
namespace: &str,
+ key: &str,
relative_path: &str,
+ url: &str,
expected_sha256: &str,
-) -> Result<PathBuf> {
+) -> Result<Asset> {
let relative = Path::new(relative_path);
if relative.is_absolute() {
bail!("asset path must be relative: {relative_path}");
}
- let sha = normalize_sha256(expected_sha256);
- let prefix = sha
- .get(..16)
- .with_context(|| format!("SHA-256 is too short for {relative_path}"))?;
- Ok(cache_dir.join(namespace).join(prefix).join(relative))
+ let sha256 = normalize_sha256(expected_sha256);
+ let name = relative
+ .file_name()
+ .and_then(|name| name.to_str())
+ .with_context(|| format!("asset path has no UTF-8 file name: {relative_path}"))?;
+ if relative
+ .components()
+ .any(|component| !matches!(component, Component::Normal(_)))
+ {
+ bail!("asset path contains an unsafe component: {relative_path}");
+ }
+ Asset::files(
+ vec![namespace.to_string(), key.to_string(), sha256.clone()],
+ vec![AssetFile {
+ name: name.to_string(),
+ url: url.to_string(),
+ sha256,
+ }],
+ )
+ .map_err(Into::into)
}
fn verify_file_sha256(path: &Path, expected_sha256: &str, label: &str) -> Result<()> {