π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Adapt knowledge packs to shared model downloads
setalp0e6083a22e99
diff --cc mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.ktindex 229adcf778,7bc5256cea..f43cb02c80--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/EnsuApplication.kt@@@ -7,13 -6,10 +7,13 @@@ import io.ente.ensu.llm.ModelDownloadeclass EnsuApplication : Application() {val modelDownloader by lazy { ModelDownloader(this) }+ val knowledgeProvider by lazy {+ KnowledgeProvider(noBackupFilesDir.resolve("knowledge"))+ }val transcriber by lazy {Transcriber(- modelDownloader.modelPath(modelDownloader.transcriptionModelTarget).absolutePath,- modelDownloader.modelPath(modelDownloader.voiceActivityModelTarget).absolutePath+ modelDownloader.modelDir(modelDownloader.transcriptionTarget).absolutePath,+ modelDownloader.voiceActivityModelPath().absolutePath)}}diff --cc mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex a2ed7e1022,1d130ae690..a2693741e4--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@@ -480,42 -459,11 +480,42 @@@ internal class ChatStoreActions}rebuildChatState(sessionId)- generationJob = scope.launch {+ val activeJob = scope.launch {+ priorGeneration?.join()+ priorSummary?.join()val isActive = { isGenerationActive(generationToken, sessionId) }+ if (!isActive()) return@launchval progressTracker = DownloadProgressTracker()+ var embeddingAssetInvalid = false+ val enabledDatasets = state.value.knowledge.enabledReadyDatasets+ val knowledgeHits: List<KnowledgeSearchHit> = if (+ userMessage.text.isNotBlank() && enabledDatasets.isNotEmpty()+ ) {+ try {+ val hits = llmProvider.withChatModelReleasedForRetrieval { embed ->+ if (!isActive()) throw kotlinx.coroutines.CancellationException()+ val query = embed(userMessage.text.trim())+ if (!isActive()) throw kotlinx.coroutines.CancellationException()+ knowledgeProvider.search(+ datasets = enabledDatasets,+ query = query,+ maxHits = configDefaults.knowledgeEmbedding.maxHits+ )+ }+ if (!isActive()) return@launch+ hits+ } catch (error: Throwable) {+ if (error is kotlinx.coroutines.CancellationException) return@launch+ embeddingAssetInvalid = error is LlmProvider.EmbeddingAssetInvalid+ emptyList()+ }+ } else {+ emptyList()+ }++ if (!isActive()) return@launchtry {- llmProvider.ensureModelReady(target) { progress ->+ llmProvider.ensureModelReady(selection) { progress ->if (!isActive()) return@ensureModelReadyval resolvedProgress = progressTracker.resolve(progress)state.update { appState ->@@@ -563,9 -511,12 +563,9 @@@if (!isActive()) return@launch- val generationLimits = resolveGenerationLimits(target)- state.update { appState ->- appState.copy(chat = appState.chat.copy(isModelDownloaded = true))- }-+ val generationLimits = resolveGenerationLimits(selection)- val historySelection = buildHistorySelection(+ val normalSystemPrompt = buildSystemPrompt()+ val normalHistorySelection = buildHistorySelection(sessionId = sessionId,promptText = prompt.text,promptImageCount = prompt.imageFiles.size,@@@ -656,20 -566,18 +656,20 @@@var tokenCount = 0try {- val summary = llmProvider.generateChat(- selection = selection,- messages = llmMessages,- imageFiles = prompt.imageFiles,- temperature = modelSettingsActions.resolveTemperature(settings),- maxTokens = generationLimits.maxOutput- ) { token ->- buffer.append(token)- tokenCount += estimateTokens(token)- if (isActive()) {- state.update { appState ->- appState.copy(chat = appState.chat.copy(streamingResponse = buffer.toString()))+ val generate: suspend () -> io.ente.ensu.llm.GenerationSummary = {+ llmProvider.generateChat(- target = target,++ selection = selection,+ messages = llmMessages,+ imageFiles = prompt.imageFiles,+ temperature = modelSettingsActions.resolveTemperature(settings),+ maxTokens = generationLimits.maxOutput+ ) { token ->+ buffer.append(token)+ tokenCount += estimateTokens(token)+ if (isActive()) {+ state.update { appState ->+ appState.copy(chat = appState.chat.copy(streamingResponse = buffer.toString()))+ }}}}diff --cc mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.ktindex 56b965f33c,7717d48cfd..025c34c85c--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt@@@ -11,9 -10,7 +11,9 @@@ import io.ente.ensu.bindings.LlmGeneratimport io.ente.ensu.bindings.LlmGenerationSummary as NativeSummaryimport io.ente.ensu.bindings.LlmModelimport io.ente.ensu.bindings.LlmModelLoadParams+import io.ente.ensu.bindings.KnowledgeEmbeddingConfig- import io.ente.ensu.bindings.ModelDownloadTargetimport io.ente.ensu.bindings.Transcriber++import io.ente.ensu.bindings.knowledgeEmbeddingModelTargetimport io.ente.ensu.bindings.llmCancelimport io.ente.ensu.bindings.llmInitBackendimport io.ente.ensu.device.AndroidDeviceCapabilityProvider@@@ -44,25 -40,9 +44,21 @@@ class LlmProvider@Volatile private var currentJobId: Long? = nullprivate var backendInitialized = falseprivate val modelLoadMutex = Mutex()- private val embeddingDownloadTarget = ModelDownloadTarget.Gguf(- knowledgeEmbedding.targetId,- knowledgeEmbedding.modelUrl,- null- )++ private val embeddingDownloadTarget = knowledgeEmbeddingModelTarget()++ class EmbeddingAssetInvalid : Exception("Embedding model asset is invalid")+- fun isChatModelReady(target: LlmModelTarget): Boolean =- downloader.isDownloaded(target.downloadTarget)++ fun isChatModelReady(selection: LlmModelSelection): Boolean =++ downloader.isDownloaded(selection.modelTarget)++ fun isEmbeddingModelReady(): Boolean =+ downloader.isDownloaded(embeddingDownloadTarget)++ suspend fun estimateEmbeddingDownloadSize(): Long? =+ downloader.estimateDownloadSize(embeddingDownloadTarget)suspend fun ensureModelReady(- target: LlmModelTarget,+ selection: LlmModelSelection,onProgress: (DownloadProgress) -> Unit) {withContext(ioDispatcher) {@@@ -72,137 -52,51 +68,139 @@@}}+ suspend fun ensureRequiredModelsReady(- target: LlmModelTarget,++ selection: LlmModelSelection,+ onProgress: (DownloadProgress) -> Unit+ ) {+ withContext(ioDispatcher) {+ deviceCapabilityProvider.chatCapability().requireChatSupported()+ val missingTargets = modelLoadMutex.withLock {+ val embeddingReady = isEmbeddingModelReady()+ if (!embeddingReady) {- val embeddingFile = downloader.modelPath(embeddingDownloadTarget)- if (embeddingFile.exists()) {++ val embeddingFile = downloader.llmModelPath(embeddingDownloadTarget)++ if (embeddingFile?.exists() == true) {+ downloader.removeDownloaded(embeddingDownloadTarget)+ }+ }++ buildList {- if (!isChatModelReady(target)) add(target.downloadTarget)++ if (!isChatModelReady(selection)) add(selection.modelTarget)+ if (!isEmbeddingModelReady()) add(embeddingDownloadTarget)+ }+ }+ if (missingTargets.isNotEmpty()) {+ downloader.download(missingTargets, onProgress)+ }+ modelLoadMutex.withLock {+ if (!isEmbeddingModelReady()) {+ downloader.removeDownloaded(embeddingDownloadTarget)+ throw IllegalStateException("Embedding model failed exact readiness validation")+ }- if (!isChatModelReady(target)) {++ if (!isChatModelReady(selection)) {+ throw IllegalStateException("Chat model failed readiness validation")+ }+ ensureModelReadyLocked(- target,++ selection,+ onProgress,+ allowRecovery = false,+ shouldDownload = false+ )+ }+ }+ }+suspend fun generateChat(- target: LlmModelTarget,+ selection: LlmModelSelection,messages: List<LlmMessage>,imageFiles: List<File>,temperature: Float,maxTokens: Int?,onToken: (String) -> Unit): GenerationSummary = withContext(ioDispatcher) {- deviceCapabilityProvider.chatCapability().requireChatSupported()- val context = loadedContext ?: throw IllegalStateException("Model context not loaded")- currentJobId = null- val mmprojPath = if (imageFiles.isEmpty()) {- null- } else {- downloader.llmMmprojPath(selection.modelTarget)?.absolutePath+ modelLoadMutex.withLock {+ deviceCapabilityProvider.chatCapability().requireChatSupported()+ val context = loadedContext ?: throw IllegalStateException("Model context not loaded")+ currentJobId = null+ val mmprojPath = if (imageFiles.isEmpty()) {+ null+ } else {- downloader.mmprojPath(target.downloadTarget)++ downloader.llmMmprojPath(selection.modelTarget)?.absolutePath+ }+ val clampedTemperature = temperature.coerceIn(0.35f, 0.7f)++ val request = LlmChatRequest(+ messages = messages.map { msg ->+ NativeChatMessage(msg.roleString(), msg.text)+ },+ templateOverride = null,+ addAssistant = true,+ imagePaths = imageFiles.map { it.absolutePath },+ mmprojPath = mmprojPath,+ mediaMarker = null,+ maxTokens = maxTokens,+ temperature = clampedTemperature,+ topP = 0.9f,+ topK = 50,+ repeatPenalty = 1.18f,+ frequencyPenalty = 0f,+ presencePenalty = 0f,+ seed = null,+ stopSequences = null,+ grammar = null+ )++ unloadTranscriptionModelIfLoaded()+ val summary = generateStreamWithCallback(context, request, onToken)+ GenerationSummary(summary.jobId, summary.generatedTokens ?: 0, summary.totalTimeMs)+ }+ }++ suspend fun <T> withChatModelReleasedForRetrieval(+ block: suspend (embed: (String) -> List<Float>) -> T+ ): T = withContext(ioDispatcher) {+ modelLoadMutex.withLock {+ deviceCapabilityProvider.chatCapability().requireChatSupported()+ if (!isEmbeddingModelReady()) throw EmbeddingAssetInvalid()+ unloadTranscriptionModelIfLoaded()+ unloadModel()+ if (!backendInitialized) {+ llmInitBackend()+ backendInitialized = true+ }++ val embeddingModel = LlmModel.load(+ LlmModelLoadParams(- modelPath = downloader.modelPath(embeddingDownloadTarget).absolutePath,++ modelPath = requireNotNull(++ downloader.llmModelPath(embeddingDownloadTarget)++ ).absolutePath,+ nGpuLayers = 0,+ useMmap = true,+ useMlock = false+ )+ )+ var embeddingContext: LlmContext? = null+ try {+ val threads = max(1, Runtime.getRuntime().availableProcessors() - 1)+ embeddingContext = embeddingModel.newEmbeddingContext(+ LlmEmbeddingContextParams(+ contextSize = knowledgeEmbedding.contextSize,+ nThreads = threads,+ batchSize = knowledgeEmbedding.batchSize,+ microBatchSize = knowledgeEmbedding.microBatchSize,+ sourceDim = knowledgeEmbedding.sourceDim,+ dim = knowledgeEmbedding.dim,+ queryPrompt = knowledgeEmbedding.queryPrompt+ )+ )+ block { text -> embeddingContext.embed(text) }+ } finally {+ embeddingContext?.destroy()+ embeddingModel.destroy()+ }}- val clampedTemperature = temperature.coerceIn(0.35f, 0.7f)-- val request = LlmChatRequest(- messages = messages.map { msg ->- NativeChatMessage(msg.roleString(), msg.text)- },- templateOverride = null,- addAssistant = true,- imagePaths = imageFiles.map { it.absolutePath },- mmprojPath = mmprojPath,- mediaMarker = null,- maxTokens = maxTokens,- temperature = clampedTemperature,- topP = 0.9f,- topK = 50,- repeatPenalty = 1.18f,- frequencyPenalty = 0f,- presencePenalty = 0f,- seed = null,- stopSequences = null,- grammar = null- )-- unloadTranscriptionModelIfLoaded()- val summary = generateStreamWithCallback(context, request, onToken)- GenerationSummary(summary.jobId, summary.generatedTokens ?: 0, summary.totalTimeMs)}- suspend fun prewarmImageInference(target: LlmModelTarget) {+ suspend fun prewarmImageInference(selection: LlmModelSelection) {withContext(ioDispatcher) {runCatching {modelLoadMutex.withLock {@@@ -280,13 -171,12 +279,13 @@@}private suspend fun ensureModelReadyLocked(- target: LlmModelTarget,+ selection: LlmModelSelection,onProgress: (DownloadProgress) -> Unit,- allowRecovery: Boolean = true+ allowRecovery: Boolean = true,+ shouldDownload: Boolean = true) {deviceCapabilityProvider.chatCapability().requireChatSupported()- val modelKey = LoadedModelKey(target.id, target.contextLength)+ val modelKey = LoadedModelKey(selection.id, selection.contextLength)if (!backendInitialized) {llmInitBackend()backendInitialized = true@@@ -298,11 -188,8 +297,10 @@@unloadModel()- val downloaded = if (shouldDownload) {- downloader.download(listOf(target.downloadTarget), onProgress)- } else {- false+ val wasAlreadyDownloaded = downloader.isDownloaded(selection.modelTarget)- downloader.download(listOf(selection.modelTarget), onProgress)++ if (shouldDownload) {++ downloader.download(listOf(selection.modelTarget), onProgress)+ }onProgress(DownloadProgress(100, "Loading model...", phase = DownloadPhase.Loading))try {diff --cc mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex e3d1cded4e,368c76fbd7..96f374a5d4--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@@ -79,11 -78,9 +78,11 @@@ internal class ModelSettingsActions}return}- val target = resolveTarget(state.value.modelSettings)- val chatReady = llmProvider.isChatModelReady(target)+ val selection = resolveSelection(state.value.modelSettings)- val isDownloaded = modelDownloader.isDownloaded(selection.modelTarget)- if (isDownloaded) {++ val chatReady = llmProvider.isChatModelReady(selection)+ val embeddingReady = llmProvider.isEmbeddingModelReady()+ val requiredModelsReady = chatReady && embeddingReady+ if (requiredModelsReady) {persistModelDownloadRequested(false)}state.update { appState ->@@@ -117,13 -116,7 +116,13 @@@}}- val chatSize = if (chatReady) 0L else modelDownloader.estimateDownloadSize(target.downloadTarget)- val size = modelDownloader.estimateDownloadSize(selection.modelTarget)++ val chatSize = if (chatReady) 0L else modelDownloader.estimateDownloadSize(selection.modelTarget)+ val embeddingSize = if (embeddingReady) 0L else llmProvider.estimateEmbeddingDownloadSize()+ val size = if (chatSize == null || embeddingSize == null) {+ null+ } else {+ chatSize + embeddingSize+ }state.update { appState ->appState.copy(chat = appState.chat.copy(@@@ -142,11 -135,9 +141,11 @@@if (currentState.chat.isDownloading || currentState.chat.isGenerating) returnif (!userInitiated && !currentState.chat.hasRequestedModelDownload) return- val target = resolveTarget(currentState.modelSettings)- val wasChatReady = llmProvider.isChatModelReady(target)+ val selection = resolveSelection(currentState.modelSettings)- val isDownloaded = modelDownloader.isDownloaded(selection.modelTarget)- if (isDownloaded) {++ val wasChatReady = llmProvider.isChatModelReady(selection)+ val wasEmbeddingReady = llmProvider.isEmbeddingModelReady()+ val requiredModelsReady = wasChatReady && wasEmbeddingReady+ if (requiredModelsReady) {state.update { appState ->appState.copy(chat = appState.chat.copy(@@@ -190,11 -181,9 +189,11 @@@var retryCount = 0while (true) {try {- llmProvider.ensureModelReady(selection) { progress ->+ llmProvider.ensureRequiredModelsReady(- target++ selection+ ) { progress ->val resolvedProgress = progressTracker.resolve(progress)- if (!isDownloaded && resolvedProgress.isFinished && !loggedComplete) {+ if (!requiredModelsReady && resolvedProgress.isFinished && !loggedComplete) {loggedComplete = truelogRepository.log(LogLevel.Info,diff --cc mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex 49913cbfb4,1a316a5358..33c1ca0aae--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@@ -116,25 -113,15 +116,27 @@@ final class ChatViewModel: ObservableOblet baseDir = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first?? FileManager.default.temporaryDirectory+ let config = ConfigDefaults.sharedlet downloader = ModelDownloader()+ // Must initialize after the downloader migrates the persisted selection.+ self.modelSettings = ModelSettingsStore.sharedlet transcriber = Transcriber(- modelDir: downloader.modelPath(target: downloader.transcriptionModelTarget).path,- vadModelPath: downloader.modelPath(target: downloader.voiceActivityModelTarget).path+ modelDir: downloader.modelDir(downloader.transcriptionTarget).path,+ vadModelPath: downloader.voiceActivityModelPath().path)- let provider = LlmProvider(downloader: downloader, transcriber: transcriber)+ let provider = LlmProvider(+ downloader: downloader,+ transcriber: transcriber,+ knowledgeEmbedding: config.knowledgeEmbedding+ )let voiceTranscriber = VoiceTranscriptionService(transcriber: transcriber, downloader: downloader)+ let knowledgeProvider = KnowledgeProvider(+ root: baseDir.appendingPathComponent("knowledge", isDirectory: true)+ )+ let knowledgeStore = KnowledgeStore(+ datasets: config.knowledgeDatasets,+ provider: knowledgeProvider+ )// Chat DB + attachments.let dbDir = baseDir.appendingPathComponent("llmchat", isDirectory: true)@@@ -585,7 -566,8 +587,7 @@@self.hasRequestedModelDownload = truedo {- try await self.ensureChatModelReady(target: target)- try await self.ensureModelReadyShared(selection)- self.isModelDownloaded = true++ try await self.ensureChatModelReady(selection)} catch {if self.isCancellation(error) {return@@@ -698,12 -680,11 +700,12 @@@downloadToast = nilmodelDownloadSizeBytes = nilhasRequestedModelDownload = false+ requiredModelsReady = falsereturn}- let target = modelSettings.currentTarget()- requiredModelsReady = provider.requiredModelsReady(target: target)+ let selection = modelSettings.currentSelection()- isModelDownloaded = downloader.isDownloaded(selection.modelTarget)- if isModelDownloaded {++ requiredModelsReady = provider.requiredModelsReady(selection)+ if requiredModelsReady {clearDownloadProgressMemory()modelDownloadSizeBytes = nilreturn@@@ -711,9 -692,9 +713,9 @@@Task { [weak self] inguard let self else { return }- let size = await provider.missingRequiredDownloadSize(target: target)- let size = await downloader.estimateDownloadSize(selection.modelTarget)++ let size = await provider.missingRequiredDownloadSize(selection)await MainActor.run {- guard self.modelReadyKey(for: self.modelSettings.currentTarget()) == self.modelReadyKey(for: target) else {+ guard self.modelReadyKey(for: self.modelSettings.currentSelection()) == self.modelReadyKey(for: selection) else {return}if self.sharedModelReadyTask == nil &&@@@ -733,34 -714,11 +735,34 @@@sharedModelReadyKey = nil}- private func modelReadyKey(for target: LlmModelTarget) -> ModelReadyKey {- ModelReadyKey(id: target.id, requestedContextLength: target.contextLength)+ private func modelReadyKey(for selection: LlmModelSelection) -> ModelReadyKey {+ ModelReadyKey(id: selection.id, requestedContextLength: selection.contextLength)}- private func ensureChatModelReady(target: LlmModelTarget) async throws {- private func ensureModelReadyShared(_ selection: LlmModelSelection) async throws {++ private func ensureChatModelReady(_ selection: LlmModelSelection) async throws {+ if isChatUnsupported {+ throw UnsupportedDeviceMemoryError(capability: deviceCapability)+ }+ var retryCount = 0+ while true {+ do {- try await provider.ensureModelReady(target: target) { progress in++ try await provider.ensureModelReady(selection) { progress in+ Task { @MainActor in+ self.handleProgress(progress)+ }+ }+ return+ } catch {+ if isCancellation(error) || !shouldRetryModelDownload(error, retryCount: retryCount) {+ throw error+ }+ retryCount += 1+ try await Task.sleep(nanoseconds: retryDelayNanoseconds(for: retryCount))+ }+ }+ }+- private func ensureRequiredModelsReadyShared(target: LlmModelTarget) async throws {++ private func ensureRequiredModelsReadyShared(_ selection: LlmModelSelection) async throws {if isChatUnsupported {throw UnsupportedDeviceMemoryError(capability: deviceCapability)}@@@ -781,9 -739,7 +783,9 @@@var retryCount = 0while true {do {- try await provider.ensureModelReady(selection) { progress in+ try await provider.ensureRequiredModelsReady(- target: target++ selection+ ) { progress inTask { @MainActor inself.handleProgress(progress)}@@@ -830,9 -786,10 +832,9 @@@hasRequestedModelDownload = true}- let target = modelSettings.currentTarget()- if provider.requiredModelsReady(target: target) {+ let selection = modelSettings.currentSelection()- let isDownloaded = downloader.isDownloaded(selection.modelTarget)- if isDownloaded {- isModelDownloaded = true++ if provider.requiredModelsReady(selection) {+ requiredModelsReady = truemodelDownloadSizeBytes = nilreturn}@@@ -845,10 -802,7 +847,10 @@@modelDownloadTask?.cancel()modelDownloadTask = Task {do {- try await self.ensureRequiredModelsReadyShared(target: target)- try await self.ensureModelReadyShared(selection)++ try await self.ensureRequiredModelsReadyShared(selection)+ self.handleProgress(+ DownloadProgress(percent: 100, status: "Ready", phase: .ready)+ )} catch {if isCancellation(error) {return@@@ -880,9 -834,10 +882,9 @@@downloadToast = nilreturn}- let target = modelSettings.currentTarget()- requiredModelsReady = provider.requiredModelsReady(target: target)+ let selection = modelSettings.currentSelection()- let isDownloaded = downloader.isDownloaded(selection.modelTarget)- isModelDownloaded = isDownloaded- if !isDownloaded {++ requiredModelsReady = provider.requiredModelsReady(selection)+ if !requiredModelsReady {return}modelDownloadSizeBytes = nil@@@ -890,7 -845,7 +892,7 @@@modelDownloadTask = Task { [weak self] inguard let self else { return }do {- try await self.ensureRequiredModelsReadyShared(target: target)- try await self.ensureModelReadyShared(selection)++ try await self.ensureRequiredModelsReadyShared(selection)} catch {if self.isCancellation(error) {return@@@ -1004,14 -949,11 +1006,14 @@@showUnsupportedDeviceDialog = truereturn}- generationTask?.cancel()- sessionSummaryTask?.cancel()+ let priorGeneration = generationTask+ let priorSummary = sessionSummaryTask+ priorGeneration?.cancel()+ priorSummary?.cancel()+ provider.stopGeneration()stopRequested = false- let target = modelSettings.currentTarget()+ let selection = modelSettings.currentSelection()let prompt = buildPrompt(text: userNode.text, attachments: userNode.attachments)let generationId = UUID()@@@ -1027,42 -969,8 +1029,42 @@@rebuildMessages(for: userNode.sessionId)generationTask = Task {+ defer {+ settleGenerationIfActive(generationId: generationId, sessionId: userNode.sessionId)+ }+ _ = await priorGeneration?.result+ _ = await priorSummary?.result+ guard !Task.isCancelled, activeGenerationId == generationId else { return }+ var embeddingAssetInvalid = false+ var knowledgeHits: [KnowledgeSearchHit] = []+ let enabledDatasets = knowledgeStore.enabledReadyDatasets+ if !userNode.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty,+ !enabledDatasets.isEmpty {+ do {+ let hits = try await provider.withChatModelReleasedForRetrieval { embed in+ try Task.checkCancellation()+ let query = try embed(+ userNode.text.trimmingCharacters(in: .whitespacesAndNewlines)+ )+ try Task.checkCancellation()+ return await knowledgeProvider.search(+ datasets: enabledDatasets,+ query: query,+ maxHits: knowledgeEmbedding.maxHits+ )+ }+ try Task.checkCancellation()+ knowledgeHits = hits+ } catch {+ if isCancellation(error) {+ return+ }+ embeddingAssetInvalid = error is EmbeddingAssetInvalidError+ }+ }+do {- try await self.ensureChatModelReady(target: target)- try await self.ensureModelReadyShared(selection)++ try await self.ensureChatModelReady(selection)} catch {if isCancellation(error) {self.sharedModelReadyTask?.cancel()@@@ -1095,9 -1003,8 +1097,9 @@@return}- let generationLimits = resolveGenerationLimits(target: target)+ let generationLimits = resolveGenerationLimits(selection)- let historySelection = buildHistorySelection(+ let normalSystemPrompt = systemPrompt()+ let normalHistorySelection = buildHistorySelection(sessionId: userNode.sessionId,promptText: prompt.text,promptImageCount: prompt.imageFiles.count,@@@ -1208,14 -1073,13 +1210,14 @@@}do {- let summary = try await provider.generateChat(- selection,- messages: messages,- imageFiles: prompt.imageFiles,- temperature: resolveTemperature(),- maxTokens: generationLimits.maxOutput,- onToken: { token in+ let runGeneration: () async throws -> GenerationSummary = {+ try await self.provider.generateChat(- target: target,++ selection,+ messages: messages,+ imageFiles: prompt.imageFiles,+ temperature: self.resolveTemperature(),+ maxTokens: generationLimits.maxOutput,+ onToken: { token inlet tokenEstimate = max(1, token.count / 4)var snapshot = ""var shouldUpdateNow = false@@@ -1443,9 -1225,7 +1445,7 @@@}downloadToast = DownloadToastState(phase: .ready, percent: 100, status: progress.status, offerRetryDownload: false)isDownloading = false- requiredModelsReady = provider.requiredModelsReady(- target: modelSettings.currentTarget()- )- isModelDownloaded = true++ requiredModelsReady = provider.requiredModelsReady(modelSettings.currentSelection())modelDownloadSizeBytes = nilclearDownloadProgressMemory()Task { @MainActor indiff --cc mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swiftindex 2171aa2b00,820dedda9c..d9cf6093ba--- a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift@@@ -131,95 -113,9 +126,91 @@@ final class LlmProviderprivate var currentJobId: Int64?private let modelLoadGate = AsyncSerialGate()- init(downloader: ModelDownloader, transcriber: Transcriber) {+ init(+ downloader: ModelDownloader,+ transcriber: Transcriber,+ knowledgeEmbedding: KnowledgeEmbeddingConfig+ ) {self.downloader = downloaderself.transcriber = transcriber+ self.knowledgeEmbedding = knowledgeEmbedding+ }+- private var embeddingDownloadTarget: ModelDownloadTarget {- .gguf(- id: knowledgeEmbedding.targetId,- url: knowledgeEmbedding.modelUrl,- mmprojUrl: nil- )++ private var embeddingDownloadTarget: ModelTarget {++ knowledgeEmbeddingModelTarget()+ }++ func isEmbeddingModelReady() -> Bool {- downloader.isDownloaded(target: embeddingDownloadTarget)++ downloader.isDownloaded(embeddingDownloadTarget)+ }+- func requiredModelsReady(target: LlmModelTarget) -> Bool {- downloader.isDownloaded(target: target.downloadTarget) && isEmbeddingModelReady()++ func requiredModelsReady(_ selection: LlmModelSelection) -> Bool {++ downloader.isDownloaded(selection.modelTarget) && isEmbeddingModelReady()+ }+- func missingRequiredDownloadSize(target: LlmModelTarget) async -> Int64? {++ func missingRequiredDownloadSize(_ selection: LlmModelSelection) async -> Int64? {+ var total: Int64 = 0- let chatMissing = !downloader.isDownloaded(target: target.downloadTarget)++ let chatMissing = !downloader.isDownloaded(selection.modelTarget)+ if chatMissing {+ guard let chatSize = await downloader.estimateDownloadSize(- target: target.downloadTarget++ selection.modelTarget+ ) else {+ return nil+ }+ total += chatSize+ }+ if !isEmbeddingModelReady() {+ guard let embeddingSize = await downloader.estimateDownloadSize(- target: embeddingDownloadTarget++ embeddingDownloadTarget+ ) else {+ return nil+ }+ total += embeddingSize+ }+ return total > 0 ? total : nil+ }++ func ensureRequiredModelsReady(- target: LlmModelTarget,++ _ selection: LlmModelSelection,+ onProgress: @escaping (DownloadProgress) -> Void+ ) async throws {+ let capability = currentChatDeviceCapability()+ if !capability.isChatSupported {+ throw UnsupportedDeviceMemoryError(capability: capability)+ }+- let missingTargets: [ModelDownloadTarget] = try await modelLoadGate.withLock {- let chatReady = downloader.isDownloaded(target: target.downloadTarget)++ let missingTargets: [ModelTarget] = try await modelLoadGate.withLock {++ let chatReady = downloader.isDownloaded(selection.modelTarget)+ let embeddingReady = isEmbeddingModelReady()+ if !embeddingReady {- _ = downloader.removeDownloaded(target: embeddingDownloadTarget)++ _ = downloader.removeDownloaded(embeddingDownloadTarget)+ }++ return [- chatReady ? nil : target.downloadTarget,++ chatReady ? nil : selection.modelTarget,+ embeddingReady ? nil : embeddingDownloadTarget,+ ].compactMap { $0 }+ }++ if !missingTargets.isEmpty {- _ = try await downloader.download(targets: missingTargets, onProgress: onProgress)++ try await downloader.download(targets: missingTargets, onProgress: onProgress)+ }++ try await modelLoadGate.withLock {+ guard isEmbeddingModelReady() else {- _ = downloader.removeDownloaded(target: embeddingDownloadTarget)++ _ = downloader.removeDownloaded(embeddingDownloadTarget)+ throw RequiredModelValidationError(targetId: knowledgeEmbedding.targetId)+ }+ try await ensureModelReadyLocked(- target: target,++ selection,+ onProgress: onProgress,+ allowRecovery: false,+ shouldDownload: false+ )- guard downloader.isDownloaded(target: target.downloadTarget) else {- throw RequiredModelValidationError(targetId: target.id)++ guard downloader.isDownloaded(selection.modelTarget) else {++ throw RequiredModelValidationError(targetId: selection.id)+ }+ }}func ensureModelReady(@@@ -232,10 -128,9 +223,10 @@@}private func ensureModelReadyLocked(- target: LlmModelTarget,+ _ selection: LlmModelSelection,onProgress: @escaping (DownloadProgress) -> Void,- allowRecovery: Bool+ allowRecovery: Bool,+ shouldDownload: Bool = true) async throws {let capability = currentChatDeviceCapability()if !capability.isChatSupported {@@@ -253,14 -148,8 +244,10 @@@backendInitialized = true}- let downloaded = if shouldDownload {- try await downloader.download(- targets: [target.downloadTarget],- onProgress: onProgress- )- } else {- false+ let wasAlreadyDownloaded = downloader.isDownloaded(selection.modelTarget)- try await downloader.download(targets: [selection.modelTarget], onProgress: onProgress)++ if shouldDownload {++ try await downloader.download(targets: [selection.modelTarget], onProgress: onProgress)+ }onProgress(DownloadProgress(percent: 100, status: "Loading model...", phase: .loading))do {@@@ -283,26 -175,6 +273,26 @@@temperature: Float,maxTokens: Int?,onToken: @escaping (String) -> Void+ ) async throws -> GenerationSummary {+ try await modelLoadGate.withLock {+ try await generateChatLocked(- target: target,++ selection,+ messages: messages,+ imageFiles: imageFiles,+ temperature: temperature,+ maxTokens: maxTokens,+ onToken: onToken+ )+ }+ }++ private func generateChatLocked(- target: LlmModelTarget,++ _ selection: LlmModelSelection,+ messages: [LlmMessage],+ imageFiles: [URL],+ temperature: Float,+ maxTokens: Int?,+ onToken: @escaping (String) -> Void) async throws -> GenerationSummary {let capability = currentChatDeviceCapability()if !capability.isChatSupported {@@@ -369,62 -243,6 +361,62 @@@)}+ func withChatModelReleasedForRetrieval<T>(+ _ operation: (_ embed: (String) throws -> [Float]) async throws -> T+ ) async throws -> T {+ try await modelLoadGate.withLock {+ let capability = currentChatDeviceCapability()+ if !capability.isChatSupported {+ throw UnsupportedDeviceMemoryError(capability: capability)+ }+ guard isEmbeddingModelReady() else {+ throw EmbeddingAssetInvalidError()+ }++ unloadTranscriptionModelIfLoaded()+ unloadModel()+ if !backendInitialized {+ try llmInitBackend()+ backendInitialized = true+ }++ var embeddingModel: LlmModel? = try LlmModel.load(+ params: LlmModelLoadParams(- modelPath: downloader.modelPath(target: embeddingDownloadTarget).path,++ modelPath: downloader.llmModelPath(embeddingDownloadTarget)!.path,+ nGpuLayers: 0,+ useMmap: true,+ useMlock: false+ )+ )+ var embeddingContext: LlmContext?+ defer {+ embeddingContext = nil+ embeddingModel = nil+ }+ guard let model = embeddingModel else {+ throw EmbeddingAssetInvalidError()+ }+ let threadCount = max(1, ProcessInfo.processInfo.activeProcessorCount - 1)+ embeddingContext = try model.newEmbeddingContext(+ params: LlmEmbeddingContextParams(+ contextSize: knowledgeEmbedding.contextSize,+ nThreads: Int32(threadCount),+ batchSize: knowledgeEmbedding.batchSize,+ microBatchSize: knowledgeEmbedding.microBatchSize,+ sourceDim: knowledgeEmbedding.sourceDim,+ dim: knowledgeEmbedding.dim,+ queryPrompt: knowledgeEmbedding.queryPrompt+ )+ )+ guard let context = embeddingContext else {+ throw EmbeddingAssetInvalidError()+ }+ return try await operation { text in+ try context.embed(text: text)+ }+ }+ }+func stopGeneration() {if let jobId = currentJobId {llmCancel(jobId: jobId)@@@ -463,21 -281,15 +455,21 @@@}}- func resetContext() {- guard let model = loadedModel else { return }- let contextParams = LlmContextParams(contextSize: currentContextLength.map(Int32.init), nThreads: nil, nBatch: nil)- loadedContext = nil- loadedContext = try? model.newContext(params: contextParams)+ func resetContext() async {+ try? await modelLoadGate.withLock {+ guard let model = loadedModel else { return }+ let contextParams = LlmContextParams(+ contextSize: currentContextLength.map(Int32.init),+ nThreads: nil,+ nBatch: nil+ )+ loadedContext = nil+ loadedContext = try? model.newContext(params: contextParams)+ }}- func loadedContextLength(target: LlmModelTarget) -> Int? {- let modelKey = LoadedModelKey(id: target.id, requestedContextLength: target.contextLength)+ func loadedContextLength(_ selection: LlmModelSelection) -> Int? {+ let modelKey = LoadedModelKey(id: selection.id, requestedContextLength: selection.contextLength)guard currentModelKey == modelKey, loadedModel != nil, loadedContext != nil else {return nil}diff --cc rust/Cargo.lockindex 3449148891,2ca8fc1747..e0e9fe30af--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@@ -2389,7 -2388,7 +2389,8 @@@ dependencies ="self_cell","serde","serde_json",+ "sha2 0.11.0",+ "tempfile","thiserror 2.0.18","tokio","transcribe-rs",diff --cc rust/bindings/uniffi/ensu/src/config.rsindex 59d96ad629,fa6ae09049..739bca9154--- a/rust/bindings/uniffi/ensu/src/config.rs+++ b/rust/bindings/uniffi/ensu/src/config.rs@@@ -5,46 -5,11 +5,50 @@@ pub struct ConfigModelPresetpub id: String,pub title: String,pub url: String,+ pub sha256: String,pub mmproj_url: Option<String>,+ pub mmproj_sha256: Option<String>,}+#[derive(Debug, Clone, uniffi::Record)]+pub struct KnowledgeEmbeddingConfig {+ pub model_url: String,++ pub model_sha256: String,+ pub target_id: String,+ pub source_dim: u32,+ pub dim: u32,+ pub query_prompt: String,+ pub context_size: u32,+ pub batch_size: u32,+ pub micro_batch_size: u32,+ pub max_hits: u32,+ pub max_context_utf8_bytes: u32,+}++#[derive(Debug, Clone, uniffi::Record)]+pub struct AttributionConfig {+ pub credit: String,+ pub license_label: String,+ pub license_url: String,+ pub public_pack_url: String,+ pub build_provenance: String,+ pub modification_notice: String,+}++#[derive(Debug, Clone, uniffi::Record)]+pub struct KnowledgeDatasetConfig {+ pub stable_id: String,+ pub label: String,+ pub current_download_identity: String,+ pub artifact_base_url: String,++ pub artifact_sha256: Vec<String>,+ pub download_size_bytes: i64,+ pub max_chars: u32,+ pub source_url_template: String,+ pub relevance_threshold: f32,+ pub attribution: AttributionConfig,+}+#[derive(Debug, Clone, uniffi::Record)]pub struct ConfigDefaults {pub mobile_system_prompt_body: String,@@@ -72,81 -37,6 +78,84 @@@ impl From<config::ModelPreset> for Conf}}+impl From<config::KnowledgeEmbeddingConfig> for KnowledgeEmbeddingConfig {+ fn from(value: config::KnowledgeEmbeddingConfig) -> Self {+ Self {+ model_url: value.model_url,++ model_sha256: value.model_sha256,+ target_id: value.target_id,+ source_dim: value.source_dim,+ dim: value.dim,+ query_prompt: value.query_prompt,+ context_size: value.context_size,+ batch_size: value.batch_size,+ micro_batch_size: value.micro_batch_size,+ max_hits: value.max_hits,+ max_context_utf8_bytes: value.max_context_utf8_bytes,+ }+ }+}++impl From<config::AttributionConfig> for AttributionConfig {+ fn from(value: config::AttributionConfig) -> Self {+ Self {+ credit: value.credit,+ license_label: value.license_label,+ license_url: value.license_url,+ public_pack_url: value.public_pack_url,+ build_provenance: value.build_provenance,+ modification_notice: value.modification_notice,+ }+ }+}++impl From<AttributionConfig> for config::AttributionConfig {+ fn from(value: AttributionConfig) -> Self {+ Self {+ credit: value.credit,+ license_label: value.license_label,+ license_url: value.license_url,+ public_pack_url: value.public_pack_url,+ build_provenance: value.build_provenance,+ modification_notice: value.modification_notice,+ }+ }+}++impl From<config::KnowledgeDatasetConfig> for KnowledgeDatasetConfig {+ fn from(value: config::KnowledgeDatasetConfig) -> Self {+ Self {+ stable_id: value.stable_id,+ label: value.label,+ current_download_identity: value.current_download_identity,+ artifact_base_url: value.artifact_base_url,++ artifact_sha256: value.artifact_sha256,+ download_size_bytes: value.download_size_bytes,+ max_chars: value.max_chars,+ source_url_template: value.source_url_template,+ relevance_threshold: value.relevance_threshold,+ attribution: value.attribution.into(),+ }+ }+}++impl From<KnowledgeDatasetConfig> for config::KnowledgeDatasetConfig {+ fn from(value: KnowledgeDatasetConfig) -> Self {+ Self {+ stable_id: value.stable_id,+ label: value.label,+ current_download_identity: value.current_download_identity,+ artifact_base_url: value.artifact_base_url,++ artifact_sha256: value.artifact_sha256,+ download_size_bytes: value.download_size_bytes,+ max_chars: value.max_chars,+ source_url_template: value.source_url_template,+ relevance_threshold: value.relevance_threshold,+ attribution: value.attribution.into(),+ }+ }+}+impl From<config::Defaults> for ConfigDefaults {fn from(value: config::Defaults) -> Self {Self {diff --cc rust/bindings/uniffi/ensu/src/lib.rsindex 0dbe156fc1,c435d8bfaf..52e1a939ed--- a/rust/bindings/uniffi/ensu/src/lib.rs+++ b/rust/bindings/uniffi/ensu/src/lib.rs@@@ -5,6 -5,5 +5,6 @@@ pub mod dbpub mod download;pub mod image;pub mod llm;- pub mod model_downloader;+ pub mod model;+pub mod retrieval;pub mod transcription;diff --cc rust/bindings/uniffi/ensu/src/model.rsindex 0000000000,e9d1fb7452..bd32856f46mode 000000,100644..100644--- a/rust/bindings/uniffi/ensu/src/model.rs+++ b/rust/bindings/uniffi/ensu/src/model.rs@@@ -1,0 -1,229 +1,234 @@@+ use std::path::{Path, PathBuf};+ use std::sync::Arc;++ use crate::download::DownloadError;+ use crate::llm::LlmError;++ #[derive(Debug, Clone, uniffi::Enum)]+ pub enum ModelTarget {+ Files {+ id: String,+ files: Vec<ModelFile>,+ },+ TarGz {+ id: String,+ url: String,+ sha256: String,+ },+ }++ #[derive(Debug, Clone, uniffi::Record)]+ pub struct ModelFile {+ pub name: String,+ pub url: String,+ pub sha256: String,+ }++ impl From<ModelTarget> for ente_model_download::ModelTarget {+ fn from(value: ModelTarget) -> Self {+ match value {+ ModelTarget::Files { id, files } => Self::Files {+ id,+ files: files+ .into_iter()+ .map(|file| ente_model_download::ModelFile {+ name: file.name,+ url: file.url,+ sha256: file.sha256,+ })+ .collect(),+ },+ ModelTarget::TarGz { id, url, sha256 } => Self::TarGz { id, url, sha256 },+ }+ }+ }++ impl From<ente_model_download::ModelTarget> for ModelTarget {+ fn from(value: ente_model_download::ModelTarget) -> Self {+ match value {+ ente_model_download::ModelTarget::Files { id, files } => Self::Files {+ id,+ files: files+ .into_iter()+ .map(|file| ModelFile {+ name: file.name,+ url: file.url,+ sha256: file.sha256,+ })+ .collect(),+ },+ ente_model_download::ModelTarget::TarGz { id, url, sha256 } => {+ Self::TarGz { id, url, sha256 }+ }+ }+ }+ }++ #[uniffi::export]+ pub fn mobile_llm_target(model_id: String) -> Result<ModelTarget, LlmError> {+ ente_ensu::model::mobile_llm_target(&model_id)+ .map(Into::into)+ .map_err(|err| LlmError::Download {+ error: DownloadError::InvalidTarget {+ message: err.to_string(),+ },+ })+ }++ #[uniffi::export]+ pub fn transcription_model_target() -> ModelTarget {+ ente_ensu::model::transcription_target().into()+ }++ #[uniffi::export]+ pub fn voice_activity_model_target() -> ModelTarget {+ ente_ensu::model::voice_activity_target().into()+ }+++#[uniffi::export]++pub fn knowledge_embedding_model_target() -> ModelTarget {++ ente_ensu::model::knowledge_embedding_target().into()++}+++ #[derive(Debug, Clone, uniffi::Record)]+ pub struct ModelDownloadProgress {+ pub downloaded_bytes: i64,+ pub total_bytes: Option<i64>,+ pub percent: i32,+ pub status: String,+ pub log_line: Option<String>,+ }++ impl From<ente_model_download::ModelDownloadProgress> for ModelDownloadProgress {+ fn from(value: ente_model_download::ModelDownloadProgress) -> Self {+ Self {+ downloaded_bytes: i64::try_from(value.downloaded_bytes).unwrap_or(i64::MAX),+ total_bytes: value+ .total_bytes+ .map(|total| i64::try_from(total).unwrap_or(i64::MAX)),+ percent: value.percent,+ status: value.status,+ log_line: value.log_line,+ }+ }+ }++ #[uniffi::export(callback_interface)]+ pub trait ModelDownloadCallback: Send + Sync {+ fn on_progress(&self, progress: ModelDownloadProgress);+ }++ #[derive(uniffi::Record)]+ pub struct LegacyModels {+ pub llm_dir: Option<String>,+ pub transcription_dir: String,+ pub model_url: Option<String>,+ pub mmproj_url: Option<String>,+ }++ #[uniffi::export]+ pub fn migrate_mobile_models(models_dir: String, legacy: LegacyModels) -> Option<String> {+ ente_ensu::model::migrations::migrate_mobile_models(+ Path::new(&models_dir),+ ente_ensu::model::migrations::LegacyModels {+ llm_dir: legacy.llm_dir.map(PathBuf::from),+ transcription_dir: PathBuf::from(legacy.transcription_dir),+ model_url: legacy.model_url,+ mmproj_url: legacy.mmproj_url,+ },+ )+ }++ #[derive(uniffi::Object)]+ pub struct ModelDownloadCore {+ inner: ente_model_download::ModelDownloader,+ }++ #[uniffi::export]+ impl ModelDownloadCore {+ #[uniffi::constructor]+ pub fn new(models_dir: String) -> Arc<Self> {+ Arc::new(Self {+ inner: ente_model_download::ModelDownloader::new(models_dir),+ })+ }++ pub fn model_dir(&self, target: ModelTarget) -> String {+ self.inner.model_dir(&target.into()).display().to_string()+ }++ pub fn llm_model_path(&self, target: ModelTarget) -> Option<String> {+ ente_ensu::model::llm_model_path(&self.inner, &target.into())+ .map(|path| path.display().to_string())+ }++ pub fn llm_mmproj_path(&self, target: ModelTarget) -> Option<String> {+ ente_ensu::model::llm_mmproj_path(&self.inner, &target.into())+ .map(|path| path.display().to_string())+ }++ pub fn voice_activity_model_path(&self) -> String {+ ente_ensu::model::voice_activity_model_path(&self.inner)+ .display()+ .to_string()+ }++ pub fn is_downloaded(&self, target: ModelTarget) -> bool {+ self.inner.is_downloaded(&target.into())+ }++ pub fn is_download_active(&self) -> bool {+ self.inner.is_download_active()+ }++ pub fn estimated_download_size(&self, target: ModelTarget) -> Option<i64> {+ self.inner+ .estimated_download_size(&target.into())+ .map(|size| i64::try_from(size).unwrap_or(i64::MAX))+ }++ pub fn remove_downloaded(&self, target: ModelTarget) -> bool {+ self.inner.remove_downloaded(&target.into())+ }++ pub fn download(+ &self,+ targets: Vec<ModelTarget>,+ callback: Box<dyn ModelDownloadCallback>,+ cancellation: Arc<CancellationToken>,+ ) -> Result<(), LlmError> {+ let targets: Vec<ente_model_download::ModelTarget> =+ targets.into_iter().map(Into::into).collect();+ self.inner+ .download(+ &targets,+ move |progress| callback.on_progress(progress.into()),+ &cancellation.inner,+ )+ .map_err(map_download_error)+ }+ }++ #[derive(Default, uniffi::Object)]+ pub struct CancellationToken {+ inner: ente_model_download::download::CancellationToken,+ }++ #[uniffi::export]+ impl CancellationToken {+ #[uniffi::constructor]+ pub fn new() -> Arc<Self> {+ Arc::new(Self::default())+ }++ pub fn cancel(&self) {+ self.inner.cancel();+ }+ }++ fn map_download_error(err: ente_model_download::download::Error) -> LlmError {+ match DownloadError::from(err) {+ DownloadError::Cancelled => LlmError::Cancelled,+ error => LlmError::Download { error },+ }+ }diff --cc rust/crates/ensu/src/config.rsindex 89b020bc0b,b27399803d..10a75fdcb0--- a/rust/crates/ensu/src/config.rs+++ b/rust/crates/ensu/src/config.rs@@@ -136,11 -133,27 +145,27 @@@ fn silero_vad_v4() -> ModelPreset}}- pub fn defaults() -> Defaults {- let mobile_default_model = lfm_vl_1_6b();- let desktop_default_model = gemma_4_e4b_q4km();+ pub(crate) fn llm_catalog() -> Vec<ModelPreset> {+ vec![+ lfm_vl_1_6b(),+ qwen_0_8b(),+ qwen_2b_q8(),+ qwen_4b_q4km(),+ gemma_4_e4b_q4km(),+ gemma_4_e2b_q4km(),+ ]+ }+ pub fn defaults() -> Defaults {+ let catalog = llm_catalog();+ let preset = |id: &str| -> ModelPreset {+ catalog+ .iter()+ .find(|preset| preset.id == id)+ .expect("preset id is in the catalog")+ .clone()+ };- Defaults {+ let defaults = Defaults {mobile_system_prompt_body: MOBILE_SYSTEM_PROMPT_BODY.to_string(),desktop_system_prompt_body: DESKTOP_SYSTEM_PROMPT_BODY.to_string(),system_prompt_date_placeholder: SYSTEM_PROMPT_DATE_PLACEHOLDER.to_string(),diff --cc rust/crates/ensu/src/config/knowledge.rsindex 67c22f6678,0000000000..c5a90f44fcmode 100644,000000..100644--- a/rust/crates/ensu/src/config/knowledge.rs+++ b/rust/crates/ensu/src/config/knowledge.rs@@@ -1,214 -1,0 +1,239 @@@+mod validation;++use thiserror::Error;++pub(crate) use validation::is_path_safe_component;+pub(crate) use validation::{validate_knowledge_datasets, validate_knowledge_embedding};++pub(crate) const KNOWLEDGE_LICENSE_LABEL: &str = "CC BY-SA 4.0";+pub(crate) const KNOWLEDGE_LICENSE_URL: &str = "https://creativecommons.org/licenses/by-sa/4.0/";++pub(crate) const KNOWLEDGE_MANIFEST_FILE: &str = "manifest.json";+pub(crate) const KNOWLEDGE_VECTORS_FILE: &str = "vectors.i8";+pub(crate) const KNOWLEDGE_META_FILE: &str = "meta.zst";+pub(crate) const KNOWLEDGE_OFFSETS_FILE: &str = "meta.offsets";+pub(crate) const KNOWLEDGE_ARTIFACT_FILENAMES: [&str; 4] = [+ KNOWLEDGE_MANIFEST_FILE,+ KNOWLEDGE_VECTORS_FILE,+ KNOWLEDGE_META_FILE,+ KNOWLEDGE_OFFSETS_FILE,+];++const EMBEDDING_MODEL_IDENTITY: &str =+ "ggml-org/embeddinggemma-300M-GGUF:embeddinggemma-300M-Q8_0.gguf";+const DOCUMENT_PROMPT: &str = "title: {title} | text: {text}";+const QUERY_PROMPT: &str = "task: search result | query: {query}";+const EMBEDDING_MODEL_URL: &str = "https://huggingface.co/ente-ai/embeddinggemma-300m-gguf/resolve/957b55764bd672f51240ac026e3a23ac9459ee3c/embeddinggemma-300M-Q8_0.gguf";++const EMBEDDING_MODEL_SHA256: &str =++ "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63";+const EMBEDDING_TARGET_ID: &str = "embeddinggemma-300m-q8-0";+const SOURCE_DIM: u32 = 768;+const RETRIEVAL_DIM: u32 = 512;+const EMBEDDING_CONTEXT_SIZE: u32 = 2_048;+const MAX_HITS: u32 = 3;+const MAX_CONTEXT_UTF8_BYTES: u32 = 6_000;+const MODIFICATION_NOTICE: &str = "Adapted by Ente";++#[derive(Debug, Clone, PartialEq)]+pub struct KnowledgeEmbeddingConfig {+ pub model_url: String,++ pub model_sha256: String,+ pub target_id: String,+ pub source_dim: u32,+ pub dim: u32,+ pub matryoshka: bool,+ pub query_prompt: String,+ pub context_size: u32,+ pub batch_size: u32,+ pub micro_batch_size: u32,+ pub max_hits: u32,+ pub max_context_utf8_bytes: u32,+}++#[derive(Debug, Clone, PartialEq)]+pub struct AttributionConfig {+ pub credit: String,+ pub license_label: String,+ pub license_url: String,+ pub public_pack_url: String,+ pub build_provenance: String,+ pub modification_notice: String,+}++#[derive(Debug, Clone, PartialEq)]+pub struct KnowledgeDatasetConfig {+ pub stable_id: String,+ pub label: String,+ pub current_download_identity: String,+ pub artifact_base_url: String,++ pub artifact_sha256: Vec<String>,+ pub download_size_bytes: i64,+ pub max_chars: u32,+ pub source_url_template: String,+ pub relevance_threshold: f32,+ pub attribution: AttributionConfig,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct KnowledgeIndexContract {+ pub model: String,+ pub source_dim: u32,+ pub matryoshka: bool,+ pub doc_prompt: String,+ pub query_prompt: String,+ pub dim: u32,+ pub quant: String,+ pub scale: u32,+ pub meta_codec: String,+ pub meta_rows_per_block: u32,+}++#[derive(Debug, Clone, PartialEq, Eq, Error)]+#[error("{0}")]+pub struct KnowledgeConfigError(String);++impl KnowledgeConfigError {+ fn invalid(field: &str, reason: impl AsRef<str>) -> Self {+ Self(format!("invalid {field}: {}", reason.as_ref()))+ }+}++pub(crate) fn knowledge_index_contract() -> KnowledgeIndexContract {+ KnowledgeIndexContract {+ model: EMBEDDING_MODEL_IDENTITY.to_owned(),+ source_dim: SOURCE_DIM,+ matryoshka: true,+ doc_prompt: DOCUMENT_PROMPT.to_owned(),+ query_prompt: QUERY_PROMPT.to_owned(),+ dim: RETRIEVAL_DIM,+ quant: "int8".to_owned(),+ scale: 127,+ meta_codec: "zstd-blocks".to_owned(),+ meta_rows_per_block: 128,+ }+}++pub(crate) fn knowledge_embedding_config() -> KnowledgeEmbeddingConfig {+ let contract = knowledge_index_contract();+ KnowledgeEmbeddingConfig {+ model_url: EMBEDDING_MODEL_URL.to_owned(),++ model_sha256: EMBEDDING_MODEL_SHA256.to_owned(),+ target_id: EMBEDDING_TARGET_ID.to_owned(),+ source_dim: contract.source_dim,+ dim: contract.dim,+ matryoshka: contract.matryoshka,+ query_prompt: contract.query_prompt,+ context_size: EMBEDDING_CONTEXT_SIZE,+ batch_size: EMBEDDING_CONTEXT_SIZE,+ micro_batch_size: EMBEDDING_CONTEXT_SIZE,+ max_hits: MAX_HITS,+ max_context_utf8_bytes: MAX_CONTEXT_UTF8_BYTES,+ }+}++pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {+ vec![+ KnowledgeDatasetConfig {+ stable_id: "simplewiki".to_owned(),+ label: "Simple English Wikipedia".to_owned(),+ current_download_identity: "simplewiki-2026-07-02".to_owned(),+ artifact_base_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/simplewiki/data/".to_owned(),++ artifact_sha256: vec![++ "118dfc77aef7c7186c8b15551ec94e8b8878fde0b3ca20c97cc8222e5b70b309".to_owned(),++ "ed90f60e757784fd4752652139299be792948628be5d7455847333f362d30cd5".to_owned(),++ "d16e84f734fb5f216acd89ed4451c35f6b9777d4d85e1a1f883ec4c44a984471".to_owned(),++ "51530bed206223347f12b7cf931df0172746c5732da76ebae3ae355f85031e3f".to_owned(),++ ],+ download_size_bytes: 167_849_446,+ max_chars: 600,+ source_url_template: "https://simple.wikipedia.org/wiki/{title}".to_owned(),+ relevance_threshold: 0.55,+ attribution: AttributionConfig {+ credit: "Simple English Wikipedia contributors".to_owned(),+ license_label: KNOWLEDGE_LICENSE_LABEL.to_owned(),+ license_url: KNOWLEDGE_LICENSE_URL.to_owned(),+ public_pack_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/tree/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/simplewiki".to_owned(),+ build_provenance: "Simple English Wikipedia dump dated 2026-07-02".to_owned(),+ modification_notice: MODIFICATION_NOTICE.to_owned(),+ },+ },+ KnowledgeDatasetConfig {+ stable_id: "wikibooks".to_owned(),+ label: "Wikibooks".to_owned(),+ current_download_identity: "enwikibooks-2026-07-02".to_owned(),+ artifact_base_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/wikibooks/data/".to_owned(),++ artifact_sha256: vec![++ "2076386cdc215ca4950aa92c4b6bcac605a63e76fda4989677eb5f9a637bba14".to_owned(),++ "bcf4f5d15b78470def1908aaddcb2abae17fbe7752d60c90d11fd4d951b0b591".to_owned(),++ "6c071ac25729f43f69c614489e38e3a7a2406febf679ad6272609ad48447daf2".to_owned(),++ "58f0c01aff935c04cc301fa94633235ab8aa1dd31e0320855ecc1d57bd2f606f".to_owned(),++ ],+ download_size_bytes: 202_595_475,+ max_chars: 1_400,+ source_url_template: "https://en.wikibooks.org/wiki/{title}".to_owned(),+ relevance_threshold: 0.55,+ attribution: AttributionConfig {+ credit: "Wikibooks contributors".to_owned(),+ license_label: KNOWLEDGE_LICENSE_LABEL.to_owned(),+ license_url: KNOWLEDGE_LICENSE_URL.to_owned(),+ public_pack_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/tree/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/wikibooks".to_owned(),+ build_provenance: "English Wikibooks dump dated 2026-07-02".to_owned(),+ modification_notice: MODIFICATION_NOTICE.to_owned(),+ },+ },+ ]+}++pub(crate) fn knowledge_artifact_urls(+ dataset: &KnowledgeDatasetConfig,+) -> Result<[String; 4], KnowledgeConfigError> {+ validation::validate_artifact_base_url(&dataset.artifact_base_url, "artifact_base_url")?;+ Ok(KNOWLEDGE_ARTIFACT_FILENAMES.map(|name| format!("{}{name}", dataset.artifact_base_url)))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn artifact_urls_append_only_fixed_names() {+ let urls = knowledge_artifact_urls(&knowledge_datasets()[0]).unwrap();+ assert_eq!(+ urls.each_ref()+ .map(|url| url.rsplit('/').next().unwrap().to_owned()),+ KNOWLEDGE_ARTIFACT_FILENAMES.map(str::to_owned)+ );+ assert!(urls.iter().all(|url| {+ url.contains("/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/simplewiki/data/")+ }));+ }++ #[test]+ fn pinned_catalog_and_embedding_are_valid() {+ let datasets = knowledge_datasets();+ validate_knowledge_datasets(&datasets).unwrap();+ let embedding = knowledge_embedding_config();+ validate_knowledge_embedding(&embedding, &[]).unwrap();++ assert_eq!(datasets.len(), 2);+ assert_eq!(datasets[0].stable_id, "simplewiki");+ assert_eq!(datasets[1].stable_id, "wikibooks");+ assert!(+ datasets+ .iter()+ .all(|dataset| dataset.attribution.modification_notice == "Adapted by Ente")+ );+ assert_eq!(datasets[0].download_size_bytes, 167_849_446);+ assert_eq!(datasets[1].download_size_bytes, 202_595_475);++ assert!(datasets.iter().all(|dataset| {++ dataset.artifact_sha256.len() == KNOWLEDGE_ARTIFACT_FILENAMES.len()++ && dataset++ .artifact_sha256++ .iter()++ .all(|sha256| sha256.len() == 64)++ }));++ assert_eq!(embedding.model_sha256.len(), 64);+ assert_eq!(knowledge_index_contract().source_dim, 768);+ assert_eq!(knowledge_index_contract().dim, 512);+ }+}diff --cc rust/crates/ensu/src/config/knowledge/validation.rsindex 218a60fbd1,0000000000..d7851b40eamode 100644,000000..100644--- a/rust/crates/ensu/src/config/knowledge/validation.rs+++ b/rust/crates/ensu/src/config/knowledge/validation.rs@@@ -1,413 -1,0 +1,443 @@@+use std::collections::HashSet;++use url::Url;++use super::{- EMBEDDING_CONTEXT_SIZE, KNOWLEDGE_LICENSE_LABEL, KNOWLEDGE_LICENSE_URL, KnowledgeConfigError,- KnowledgeDatasetConfig, KnowledgeEmbeddingConfig, MAX_CONTEXT_UTF8_BYTES, MAX_HITS,- knowledge_index_contract,++ EMBEDDING_CONTEXT_SIZE, KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL,++ KNOWLEDGE_LICENSE_URL, KnowledgeConfigError, KnowledgeDatasetConfig, KnowledgeEmbeddingConfig,++ MAX_CONTEXT_UTF8_BYTES, MAX_HITS, knowledge_index_contract,+};++const MAX_DATASETS: usize = 32;+const MAX_PATH_COMPONENT_BYTES: usize = 96;+const MAX_DISPLAY_BYTES: usize = 256;+const MAX_URL_BYTES: usize = 2_048;+const MAX_PROVENANCE_BYTES: usize = 4_096;++pub fn validate_knowledge_datasets(+ datasets: &[KnowledgeDatasetConfig],+) -> Result<(), KnowledgeConfigError> {+ if !(1..=MAX_DATASETS).contains(&datasets.len()) {+ return Err(KnowledgeConfigError::invalid(+ "knowledge_datasets",+ format!("must contain between 1 and {MAX_DATASETS} records"),+ ));+ }++ let mut stable_ids = HashSet::with_capacity(datasets.len());+ let mut current_identities = HashSet::with_capacity(datasets.len());+ for (index, dataset) in datasets.iter().enumerate() {+ let prefix = format!("knowledge_datasets[{index}]");+ validate_path_component(&dataset.stable_id, &format!("{prefix}.stable_id"))?;+ if !stable_ids.insert(dataset.stable_id.as_str()) {+ return Err(KnowledgeConfigError::invalid(+ &format!("{prefix}.stable_id"),+ "must be unique",+ ));+ }++ validate_path_component(+ &dataset.current_download_identity,+ &format!("{prefix}.current_download_identity"),+ )?;+ if !current_identities.insert(dataset.current_download_identity.as_str()) {+ return Err(KnowledgeConfigError::invalid(+ &format!("{prefix}.current_download_identity"),+ "must be unique",+ ));+ }++ validate_display_text(&dataset.label, &format!("{prefix}.label"))?;+ validate_artifact_base_url(+ &dataset.artifact_base_url,+ &format!("{prefix}.artifact_base_url"),+ )?;++ if dataset.artifact_sha256.len() != KNOWLEDGE_ARTIFACT_FILENAMES.len()++ || dataset.artifact_sha256.iter().any(|sha256| {++ sha256.len() != 64 || !sha256.chars().all(|ch| ch.is_ascii_hexdigit())++ })++ {++ return Err(KnowledgeConfigError::invalid(++ &format!("{prefix}.artifact_sha256"),++ "must provide one SHA-256 digest per knowledge artifact",++ ));++ }+ if dataset.download_size_bytes <= 0 {+ return Err(KnowledgeConfigError::invalid(+ &format!("{prefix}.download_size_bytes"),+ "must be greater than zero",+ ));+ }+ validate_source_url_template(+ &dataset.source_url_template,+ &format!("{prefix}.source_url_template"),+ )?;+ if !(1..=4_096).contains(&dataset.max_chars) {+ return Err(KnowledgeConfigError::invalid(+ &format!("{prefix}.max_chars"),+ "must be between 1 and 4096",+ ));+ }+ if !dataset.relevance_threshold.is_finite()+ || !(0.0..=1.0).contains(&dataset.relevance_threshold)+ {+ return Err(KnowledgeConfigError::invalid(+ &format!("{prefix}.relevance_threshold"),+ "must be finite and between 0 and 1",+ ));+ }++ let attribution = &dataset.attribution;+ validate_display_text(&attribution.credit, &format!("{prefix}.attribution.credit"))?;+ validate_display_text(+ &attribution.license_label,+ &format!("{prefix}.attribution.license_label"),+ )?;+ validate_https_url(+ &attribution.license_url,+ &format!("{prefix}.attribution.license_url"),+ )?;+ if attribution.license_label != KNOWLEDGE_LICENSE_LABEL+ || attribution.license_url != KNOWLEDGE_LICENSE_URL+ {+ return Err(KnowledgeConfigError::invalid(+ &format!("{prefix}.attribution"),+ "must use the v1 CC BY-SA 4.0 license",+ ));+ }+ validate_https_url(+ &attribution.public_pack_url,+ &format!("{prefix}.attribution.public_pack_url"),+ )?;+ validate_provenance_text(+ &attribution.build_provenance,+ &format!("{prefix}.attribution.build_provenance"),+ )?;+ validate_provenance_text(+ &attribution.modification_notice,+ &format!("{prefix}.attribution.modification_notice"),+ )?;+ }++ Ok(())+}++pub fn validate_knowledge_embedding(+ config: &KnowledgeEmbeddingConfig,+ reserved_model_ids: &[String],+) -> Result<(), KnowledgeConfigError> {+ validate_path_component(&config.target_id, "knowledge_embedding.target_id")?;+ if config.target_id.starts_with("custom") {+ return Err(KnowledgeConfigError::invalid(+ "knowledge_embedding.target_id",+ "must not start with reserved prefix 'custom'",+ ));+ }+ if reserved_model_ids.iter().any(|id| id == &config.target_id) {+ return Err(KnowledgeConfigError::invalid(+ "knowledge_embedding.target_id",+ "must not alias another model target",+ ));+ }+ validate_embedding_model_url(&config.model_url, "knowledge_embedding.model_url")?;++ if config.model_sha256.len() != 64++ || !config.model_sha256.chars().all(|ch| ch.is_ascii_hexdigit())++ {++ return Err(KnowledgeConfigError::invalid(++ "knowledge_embedding.model_sha256",++ "must be a SHA-256 digest",++ ));++ }+ let contract = knowledge_index_contract();+ let exact_fields_match = config.source_dim == contract.source_dim+ && config.dim == contract.dim+ && config.matryoshka == contract.matryoshka+ && config.query_prompt == contract.query_prompt+ && config.context_size == EMBEDDING_CONTEXT_SIZE+ && config.batch_size == EMBEDDING_CONTEXT_SIZE+ && config.micro_batch_size == EMBEDDING_CONTEXT_SIZE+ && config.max_hits == MAX_HITS+ && config.max_context_utf8_bytes == MAX_CONTEXT_UTF8_BYTES;+ if !exact_fields_match {+ return Err(KnowledgeConfigError::invalid(+ "knowledge_embedding",+ "must match the v1 embedding and retrieval contract",+ ));+ }++ Ok(())+}++fn validate_path_component(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {+ if is_path_safe_component(value) {+ Ok(())+ } else {+ Err(KnowledgeConfigError::invalid(+ field,+ "must be a path-safe 1-96 byte ASCII component",+ ))+ }+}++pub(crate) fn is_path_safe_component(value: &str) -> bool {+ let bytes = value.as_bytes();+ let valid_edge = |byte: u8| byte.is_ascii_lowercase() || byte.is_ascii_digit();+ let valid_body = |byte: u8| valid_edge(byte) || matches!(byte, b'.' | b'_' | b'-');+ !bytes.is_empty()+ && bytes.len() <= MAX_PATH_COMPONENT_BYTES+ && valid_edge(bytes[0])+ && valid_edge(bytes[bytes.len() - 1])+ && bytes.iter().copied().all(valid_body)+}++fn validate_display_text(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {+ validate_text(value, field, MAX_DISPLAY_BYTES)+}++fn validate_provenance_text(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {+ validate_text(value, field, MAX_PROVENANCE_BYTES)+}++fn validate_text(value: &str, field: &str, max_bytes: usize) -> Result<(), KnowledgeConfigError> {+ if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) {+ Err(KnowledgeConfigError::invalid(+ field,+ format!("must be nonempty, control-free, and at most {max_bytes} UTF-8 bytes"),+ ))+ } else {+ Ok(())+ }+}++fn parse_https_url(value: &str, field: &str) -> Result<Url, KnowledgeConfigError> {+ if value.len() > MAX_URL_BYTES || value.chars().any(char::is_control) {+ return Err(KnowledgeConfigError::invalid(+ field,+ format!("must be control-free and at most {MAX_URL_BYTES} UTF-8 bytes"),+ ));+ }+ let url = Url::parse(value)+ .map_err(|error| KnowledgeConfigError::invalid(field, error.to_string()))?;+ if url.scheme() != "https"+ || url.host_str().is_none()+ || !url.username().is_empty()+ || url.password().is_some()+ {+ return Err(KnowledgeConfigError::invalid(+ field,+ "must be an HTTPS URL without credentials",+ ));+ }+ Ok(url)+}++fn validate_https_url(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {+ parse_https_url(value, field).map(|_| ())+}++fn validate_source_url_template(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {+ if value.len() > MAX_URL_BYTES {+ return Err(KnowledgeConfigError::invalid(+ field,+ format!("must be at most {MAX_URL_BYTES} UTF-8 bytes"),+ ));+ }+ if value.match_indices("{title}").count() != 1 {+ return Err(KnowledgeConfigError::invalid(+ field,+ "must contain exactly one {title} placeholder",+ ));+ }+ let without_title = value.replacen("{title}", "", 1);+ if without_title.contains(['{', '}']) {+ return Err(KnowledgeConfigError::invalid(+ field,+ "must not contain another brace-delimited placeholder",+ ));+ }+ parse_https_url(&value.replacen("{title}", "title", 1), field).map(|_| ())+}++pub(super) fn validate_artifact_base_url(+ value: &str,+ field: &str,+) -> Result<(), KnowledgeConfigError> {+ let url = parse_https_url(value, field)?;+ let segments = url+ .path_segments()+ .map(|segments| segments.collect::<Vec<_>>())+ .ok_or_else(|| KnowledgeConfigError::invalid(field, "must have path segments"))?;+ let pinned = url.host_str() == Some("huggingface.co")+ && url.port().is_none()+ && url.query().is_none()+ && url.fragment().is_none()+ && value.ends_with('/')+ && segments.len() >= 8+ && segments[..4] == ["datasets", "ente-ai", "ensu-knowledge-packs", "resolve"]+ && is_commit_hash(segments[4]);+ if pinned {+ Ok(())+ } else {+ Err(KnowledgeConfigError::invalid(+ field,+ "must be an immutable Ente knowledge-pack resolver base URL",+ ))+ }+}++fn validate_embedding_model_url(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {+ let url = parse_https_url(value, field)?;+ let segments = url+ .path_segments()+ .map(|segments| segments.collect::<Vec<_>>())+ .ok_or_else(|| KnowledgeConfigError::invalid(field, "must have path segments"))?;+ let pinned = url.host_str() == Some("huggingface.co")+ && url.port().is_none()+ && url.query().is_none()+ && url.fragment().is_none()+ && segments.len() == 5+ && segments[..3] == ["ente-ai", "embeddinggemma-300m-gguf", "resolve"]+ && is_commit_hash(segments[3])+ && !segments[4].is_empty();+ if pinned {+ Ok(())+ } else {+ Err(KnowledgeConfigError::invalid(+ field,+ "must be an immutable Ente EmbeddingGemma resolver URL",+ ))+ }+}++fn is_commit_hash(value: &str) -> bool {+ value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit())+}++#[cfg(test)]+mod tests {+ use super::super::{EMBEDDING_TARGET_ID, knowledge_datasets, knowledge_embedding_config};+ use super::*;++ fn valid_datasets() -> Vec<KnowledgeDatasetConfig> {+ knowledge_datasets()+ }++ #[test]+ fn rejects_empty_oversized_and_duplicate_catalogs() {+ assert!(validate_knowledge_datasets(&[]).is_err());++ let mut too_many = Vec::new();+ for index in 0..=MAX_DATASETS {+ let mut dataset = valid_datasets().remove(0);+ dataset.stable_id = format!("pack-{index}");+ dataset.current_download_identity = format!("revision-{index}");+ too_many.push(dataset);+ }+ assert!(validate_knowledge_datasets(&too_many).is_err());++ let mut duplicate = valid_datasets();+ duplicate[1].stable_id = duplicate[0].stable_id.clone();+ assert!(validate_knowledge_datasets(&duplicate).is_err());+ duplicate[1].stable_id = "wikibooks".to_owned();+ duplicate[1].current_download_identity = duplicate[0].current_download_identity.clone();+ assert!(validate_knowledge_datasets(&duplicate).is_err());+ }++ #[test]+ fn rejects_path_unsafe_ids_and_boundary_overflow() {+ for invalid in [+ "",+ ".",+ "..",+ "UPPER",+ "-starts",+ "ends-",+ "has/slash",+ " spaced ",+ ] {+ let mut datasets = valid_datasets();+ datasets[0].stable_id = invalid.to_owned();+ assert!(+ validate_knowledge_datasets(&datasets).is_err(),+ "accepted {invalid:?}"+ );+ }++ let mut datasets = valid_datasets();+ datasets[0].stable_id = format!("a{}z", "b".repeat(MAX_PATH_COMPONENT_BYTES - 2));+ validate_knowledge_datasets(&datasets).unwrap();+ datasets[0].stable_id.push('z');+ assert!(validate_knowledge_datasets(&datasets).is_err());+ }++ #[test]+ fn rejects_invalid_dataset_strings_numbers_and_urls() {+ let mut datasets = valid_datasets();+ datasets[0].label = "\u{0000}".to_owned();+ assert!(validate_knowledge_datasets(&datasets).is_err());++ let mut datasets = valid_datasets();+ datasets[0].attribution.build_provenance = "x".repeat(MAX_PROVENANCE_BYTES + 1);+ assert!(validate_knowledge_datasets(&datasets).is_err());++ let mut datasets = valid_datasets();+ datasets[0].max_chars = 0;+ assert!(validate_knowledge_datasets(&datasets).is_err());+ datasets[0].max_chars = 4_097;+ assert!(validate_knowledge_datasets(&datasets).is_err());++ let mut datasets = valid_datasets();+ datasets[0].download_size_bytes = 0;+ assert!(validate_knowledge_datasets(&datasets).is_err());+ datasets[0].download_size_bytes = -1;+ assert!(validate_knowledge_datasets(&datasets).is_err());+++ let mut datasets = valid_datasets();++ datasets[0].artifact_sha256.pop();++ assert!(validate_knowledge_datasets(&datasets).is_err());++ datasets[0]++ .artifact_sha256++ .push("not-a-checksum".to_owned());++ assert!(validate_knowledge_datasets(&datasets).is_err());+++ let mut datasets = valid_datasets();+ datasets[0].relevance_threshold = f32::NAN;+ assert!(validate_knowledge_datasets(&datasets).is_err());+ datasets[0].relevance_threshold = 1.01;+ assert!(validate_knowledge_datasets(&datasets).is_err());++ let mut datasets = valid_datasets();+ datasets[0].artifact_base_url = datasets[0].artifact_base_url.replace(+ "/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/",+ "/resolve/main/",+ );+ assert!(validate_knowledge_datasets(&datasets).is_err());++ let mut datasets = valid_datasets();+ datasets[0].source_url_template = "http://example.com/{title}/{section}".to_owned();+ assert!(validate_knowledge_datasets(&datasets).is_err());+ }++ #[test]+ fn rejects_invalid_embedding_contract_and_model_aliases() {+ let mut config = knowledge_embedding_config();+ config.target_id = "custom-embedding".to_owned();+ assert!(validate_knowledge_embedding(&config, &[]).is_err());++ let mut config = knowledge_embedding_config();+ assert!(validate_knowledge_embedding(&config, &[config.target_id.clone()]).is_err());+ config.target_id = EMBEDDING_TARGET_ID.to_owned();+ config.model_url = config+ .model_url+ .replace("957b55764bd672f51240ac026e3a23ac9459ee3c", "main");+ assert!(validate_knowledge_embedding(&config, &[]).is_err());+++ let mut config = knowledge_embedding_config();++ config.model_sha256 = "not-a-checksum".to_owned();++ assert!(validate_knowledge_embedding(&config, &[]).is_err());+++ let mut config = knowledge_embedding_config();+ config.dim = 768;+ assert!(validate_knowledge_embedding(&config, &[]).is_err());+ }+}diff --cc rust/crates/ensu/src/lib.rsindex 5e8677ca3b,959af5920a..6589452b27--- a/rust/crates/ensu/src/lib.rs+++ b/rust/crates/ensu/src/lib.rs@@@ -2,7 -2,7 +2,8 @@@ pub mod configpub mod db;pub mod image;pub mod llm;+ pub mod model;+pub mod retrieval;#[cfg(feature = "transcription")]pub mod transcription;diff --cc rust/crates/ensu/src/model/mod.rsindex 0000000000,c970fa9412..3fd869eedamode 000000,100644..100644--- a/rust/crates/ensu/src/model/mod.rs+++ b/rust/crates/ensu/src/model/mod.rs@@@ -1,0 -1,184 +1,209 @@@+ pub mod migrations;++ use std::path::PathBuf;++ use ente_model_download::{ModelDownloader, ModelFile, ModelTarget};++ use crate::config::{self, ModelPreset};++ const LLM_MODEL_FILE: &str = "model.gguf";+ const LLM_MMPROJ_FILE: &str = "mmproj.gguf";++const EMBEDDING_MODEL_FILE: &str = "model.gguf";+ const VOICE_ACTIVITY_MODEL_FILE: &str = "model.onnx";++ #[derive(Debug, thiserror::Error)]+ #[error("preset {id} pairs only half of the mmproj URL and checksum")]+ pub struct InvalidPreset {+ pub id: String,+ }++ pub(crate) fn llm_target(preset: &ModelPreset) -> Result<ModelTarget, InvalidPreset> {+ let mut files = vec![ModelFile {+ name: LLM_MODEL_FILE.to_string(),+ url: preset.url.clone(),+ sha256: preset.sha256.clone(),+ }];+ match (+ trimmed(preset.mmproj_url.as_deref()),+ trimmed(preset.mmproj_sha256.as_deref()),+ ) {+ (Some(url), Some(sha256)) => files.push(ModelFile {+ name: LLM_MMPROJ_FILE.to_string(),+ url: url.to_string(),+ sha256: sha256.to_string(),+ }),+ (None, None) => {}+ _ => {+ return Err(InvalidPreset {+ id: preset.id.clone(),+ });+ }+ }+ Ok(ModelTarget::Files {+ id: preset.id.clone(),+ files,+ })+ }++ pub fn mobile_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {+ let preset = config::llm_catalog()+ .into_iter()+ .find(|preset| preset.id == model_id)+ .unwrap_or_else(|| config::defaults().mobile_default_model);+ llm_target(&preset)+ }++ pub fn desktop_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {+ let preset = config::llm_catalog()+ .into_iter()+ .find(|preset| preset.id == model_id)+ .unwrap_or_else(|| config::defaults().desktop_default_model);+ llm_target(&preset)+ }++ pub fn transcription_target() -> ModelTarget {+ let preset = config::defaults().transcription_model;+ ModelTarget::TarGz {+ id: preset.id,+ url: preset.url,+ sha256: preset.sha256,+ }+ }++ pub fn voice_activity_target() -> ModelTarget {+ let preset = config::defaults().voice_activity_model;+ ModelTarget::Files {+ id: preset.id,+ files: vec![ModelFile {+ name: VOICE_ACTIVITY_MODEL_FILE.to_string(),+ url: preset.url,+ sha256: preset.sha256,+ }],+ }+ }+++pub fn knowledge_embedding_target() -> ModelTarget {++ let embedding = config::defaults().knowledge_embedding;++ ModelTarget::Files {++ id: embedding.target_id,++ files: vec![ModelFile {++ name: EMBEDDING_MODEL_FILE.to_string(),++ url: embedding.model_url,++ sha256: embedding.model_sha256,++ }],++ }++}+++ pub fn llm_model_path(downloader: &ModelDownloader, target: &ModelTarget) -> Option<PathBuf> {+ downloader.file_path(target, LLM_MODEL_FILE)+ }++ pub fn llm_mmproj_path(downloader: &ModelDownloader, target: &ModelTarget) -> Option<PathBuf> {+ downloader.file_path(target, LLM_MMPROJ_FILE)+ }++ pub fn voice_activity_model_path(downloader: &ModelDownloader) -> PathBuf {+ downloader+ .model_dir(&voice_activity_target())+ .join(VOICE_ACTIVITY_MODEL_FILE)+ }++ pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {+ value.map(str::trim).filter(|value| !value.is_empty())+ }++ #[cfg(test)]+ mod tests {+ use super::*;++ fn preset(mmproj_url: Option<&str>, mmproj_sha256: Option<&str>) -> ModelPreset {+ ModelPreset {+ id: "qwen-2b-q8".to_string(),+ title: "Qwen".to_string(),+ url: "https://example.org/main.gguf".to_string(),+ sha256: "0".repeat(64),+ mmproj_url: mmproj_url.map(Into::into),+ mmproj_sha256: mmproj_sha256.map(Into::into),+ }+ }++ #[test]+ fn llm_target_names_model_and_mmproj_files() {+ let target = llm_target(&preset(+ Some("https://example.org/mmproj.gguf"),+ Some("abc"),+ ))+ .expect("paired preset");+ let downloader = ModelDownloader::new(std::env::temp_dir().join("ensu-models-test"));+ assert_eq!(+ llm_model_path(&downloader, &target).unwrap(),+ downloader.model_dir(&target).join("model.gguf")+ );+ assert_eq!(+ llm_mmproj_path(&downloader, &target).unwrap(),+ downloader.model_dir(&target).join("mmproj.gguf")+ );+ }++ #[test]+ fn llm_target_without_mmproj_has_no_mmproj_path() {+ let target = llm_target(&preset(None, None)).expect("model-only preset");+ let downloader = ModelDownloader::new(std::env::temp_dir().join("ensu-models-test"));+ assert!(llm_model_path(&downloader, &target).is_some());+ assert!(llm_mmproj_path(&downloader, &target).is_none());+ }++ #[test]+ fn llm_target_rejects_half_specified_mmproj() {+ assert!(llm_target(&preset(Some("https://example.org/mmproj.gguf"), None)).is_err());+ assert!(llm_target(&preset(None, Some("abc"))).is_err());+ assert!(llm_target(&preset(Some(" "), Some("abc"))).is_err());+ }++ #[test]+ fn llm_targets_resolve_catalog_ids_or_fall_back_to_defaults() {+ let defaults = crate::config::defaults();+ for preset in crate::config::llm_catalog() {+ assert_eq!(+ mobile_llm_target(&preset.id).unwrap(),+ llm_target(&preset).unwrap(),+ "{}",+ preset.id+ );+ assert_eq!(+ desktop_llm_target(&preset.id).unwrap(),+ llm_target(&preset).unwrap(),+ "{}",+ preset.id+ );+ }+ let mobile_default = llm_target(&defaults.mobile_default_model).unwrap();+ assert_eq!(mobile_llm_target("").unwrap(), mobile_default);+ assert_eq!(mobile_llm_target("no-such-model").unwrap(), mobile_default);+ let desktop_default = llm_target(&defaults.desktop_default_model).unwrap();+ assert_eq!(desktop_llm_target("").unwrap(), desktop_default);+ assert_eq!(+ desktop_llm_target("no-such-model").unwrap(),+ desktop_default+ );+ }++ #[test]+ fn all_catalog_presets_produce_valid_targets() {+ for preset in crate::config::llm_catalog() {+ llm_target(&preset).expect("catalog preset");+ }+ }++++ #[test]++ fn knowledge_embedding_target_is_checksum_backed() {++ let target = knowledge_embedding_target();++ let ModelTarget::Files { id, files } = target else {++ panic!("embedding target must use files");++ };++ assert_eq!(id, "embeddinggemma-300m-q8-0");++ assert_eq!(files.len(), 1);++ assert_eq!(files[0].name, "model.gguf");++ assert_eq!(files[0].sha256.len(), 64);++ }+ }diff --cc rust/crates/ensu/src/retrieval/pack.rsindex 9e8e5803b1,0000000000..7a97283c8amode 100644,000000..100644--- a/rust/crates/ensu/src/retrieval/pack.rs+++ b/rust/crates/ensu/src/retrieval/pack.rs@@@ -1,347 -1,0 +1,339 @@@+use std::fs;+use std::path::{Path, PathBuf};+- use ente_model_download::download::{self, Progress, Target};++use ente_model_download::download::{self, Downloader, Progress, Target};++use crate::config::{+ KNOWLEDGE_ARTIFACT_FILENAMES, KnowledgeDatasetConfig, is_path_safe_component,+ knowledge_artifact_urls, validate_knowledge_datasets,+};++use super::{RetrievalError, index::RetrievalIndex};++pub fn download_knowledge_pack(+ pack_root: impl AsRef<Path>,+ expected_pack: &KnowledgeDatasetConfig,+ on_progress: impl FnMut(Progress),+ is_cancelled: impl Fn() -> bool,+) -> Result<(), download::Error> {+ let pack_root = pack_root.as_ref();+ let targets = knowledge_download_targets(pack_root, expected_pack)?;- download::fetch(targets, validate_staged_file, on_progress, is_cancelled)++ Downloader::new()?.download_blocking(targets, on_progress, is_cancelled)+}++fn knowledge_download_targets(+ pack_root: &Path,+ expected_pack: &KnowledgeDatasetConfig,+) -> Result<Vec<Target>, download::Error> {+ validate_knowledge_datasets(std::slice::from_ref(expected_pack))+ .map_err(|error| download::Error::InvalidTarget(error.to_string()))?;+ let root_identity = pack_root+ .file_name()+ .and_then(|name| name.to_str())+ .ok_or_else(|| {+ download::Error::InvalidTarget(+ "knowledge pack root must have a UTF-8 stable ID".to_string(),+ )+ })?;+ if root_identity != expected_pack.stable_id {+ return Err(download::Error::InvalidTarget(+ "knowledge pack root must end with the selected stable ID".to_string(),+ ));+ }+ fs::create_dir_all(pack_root)?;+ if !fs::symlink_metadata(pack_root)?.file_type().is_dir() {+ return Err(download::Error::InvalidTarget(+ "knowledge pack root must be a directory, not a symlink".to_string(),+ ));+ }+ let revision_directory = pack_root.join(&expected_pack.current_download_identity);+ fs::create_dir_all(&revision_directory)?;+ if !fs::symlink_metadata(&revision_directory)?+ .file_type()+ .is_dir()+ {+ return Err(download::Error::InvalidTarget(+ "knowledge revision path must be a directory, not a symlink".to_string(),+ ));+ }++ let urls = knowledge_artifact_urls(expected_pack)+ .map_err(|error| download::Error::InvalidTarget(error.to_string()))?;+ Ok(KNOWLEDGE_ARTIFACT_FILENAMES+ .into_iter()+ .zip(urls)- .map(|(filename, url)| Target {++ .zip(expected_pack.artifact_sha256.iter().cloned())++ .map(|((filename, url), sha256)| Target {+ label: filename.to_owned(),+ url,- destination_path: revision_directory.join(filename).display().to_string(),++ sha256,++ destination: revision_directory.join(filename),+ })+ .collect())+}+- fn validate_staged_file(_target: &Target, path: &Path) -> Result<(), download::Error> {- let metadata = fs::symlink_metadata(path)?;- if metadata.file_type().is_file() && metadata.len() > 0 {- Ok(())- } else {- Err(download::Error::Validation(format!(- "{} must be a nonempty regular file",- path.display()- )))- }- }-+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum KnowledgeReconciliationStatus {+ Download,+ Ready,+ UpdateAvailable,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct KnowledgeReconciliation {+ pub status: KnowledgeReconciliationStatus,+ pub active_identity: Option<String>,+ pub active_directory: Option<String>,+}++pub fn reconcile_knowledge_pack(+ pack_root: impl AsRef<Path>,+ expected_pack: &KnowledgeDatasetConfig,+) -> Result<KnowledgeReconciliation, RetrievalError> {+ let pack_root = pack_root.as_ref();+ validate_pack_root(pack_root, expected_pack, true)?;++ let mut valid = Vec::<(String, PathBuf)>::new();+ let mut invalid = Vec::<PathBuf>::new();+ for entry in fs::read_dir(pack_root)? {+ let entry = match entry {+ Ok(entry) => entry,+ Err(_) => continue,+ };+ let file_type = match entry.file_type() {+ Ok(file_type) => file_type,+ Err(_) => continue,+ };+ if !file_type.is_dir() {+ continue;+ }+ let path = entry.path();+ match RetrievalIndex::open(&path, expected_pack) {+ Ok(index) => valid.push((index.dataset_identity().to_owned(), path)),+ Err(_) => invalid.push(path),+ }+ }++ let current_position = valid+ .iter()+ .position(|(identity, _)| identity == &expected_pack.current_download_identity);+ let active = if let Some(position) = current_position {+ Some(valid.swap_remove(position))+ } else {+ valid.sort_by(|left, right| left.0.cmp(&right.0));+ (!valid.is_empty()).then(|| valid.remove(0))+ };++ for path in &invalid {+ let _ = fs::remove_dir_all(path);+ }++ match active {+ Some((identity, directory)) => {+ cleanup_revision_artifacts(&directory);+ let status = if identity == expected_pack.current_download_identity {+ KnowledgeReconciliationStatus::Ready+ } else {+ KnowledgeReconciliationStatus::UpdateAvailable+ };+ Ok(KnowledgeReconciliation {+ status,+ active_identity: Some(identity),+ active_directory: Some(directory.display().to_string()),+ })+ }+ None => Ok(KnowledgeReconciliation {+ status: KnowledgeReconciliationStatus::Download,+ active_identity: None,+ active_directory: None,+ }),+ }+}++pub fn cleanup_obsolete_knowledge_pack_revisions(+ pack_root: impl AsRef<Path>,+ expected_pack: &KnowledgeDatasetConfig,+ active_identity: &str,+) -> Result<(), RetrievalError> {+ if !is_path_safe_component(active_identity) {+ return Err(RetrievalError::InvalidInput(+ "active knowledge revision identity is not path-safe".to_string(),+ ));+ }++ let pack_root = pack_root.as_ref();+ validate_pack_root(pack_root, expected_pack, false)?;++ let active_directory = pack_root.join(active_identity);+ RetrievalIndex::open(&active_directory, expected_pack)?;++ for entry in fs::read_dir(pack_root)?.flatten() {+ let Ok(file_type) = entry.file_type() else {+ continue;+ };+ if file_type.is_dir() && entry.file_name().to_str() != Some(active_identity) {+ let _ = fs::remove_dir_all(entry.path());+ }+ }+ cleanup_revision_artifacts(&active_directory);+ Ok(())+}++fn validate_pack_root(+ pack_root: &Path,+ expected_pack: &KnowledgeDatasetConfig,+ create: bool,+) -> Result<(), RetrievalError> {+ validate_knowledge_datasets(std::slice::from_ref(expected_pack))+ .map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;+ let root_identity = pack_root+ .file_name()+ .and_then(|name| name.to_str())+ .ok_or_else(|| {+ RetrievalError::InvalidInput("pack root must have a UTF-8 stable ID".to_string())+ })?;+ if root_identity != expected_pack.stable_id || !is_path_safe_component(root_identity) {+ return Err(RetrievalError::InvalidInput(+ "pack root must end with the selected stable dataset ID".to_string(),+ ));+ }+ if create {+ fs::create_dir_all(pack_root)?;+ }+ if !fs::symlink_metadata(pack_root)?.file_type().is_dir() {+ return Err(RetrievalError::InvalidInput(+ "pack root must be a directory, not a symlink".to_string(),+ ));+ }+ Ok(())+}++fn cleanup_revision_artifacts(directory: &Path) {+ let Ok(entries) = fs::read_dir(directory) else {+ return;+ };+ for entry in entries.flatten() {+ if entry+ .file_name()+ .to_str()+ .is_some_and(|name| KNOWLEDGE_ARTIFACT_FILENAMES.contains(&name))+ {+ continue;+ }+ let path = entry.path();+ match entry.file_type() {+ Ok(file_type) if file_type.is_dir() => {+ let _ = fs::remove_dir_all(path);+ }+ Ok(_) => {+ let _ = fs::remove_file(path);+ }+ Err(_) => {}+ }+ }+}++#[cfg(test)]+mod tests {+ use std::fs;++ use super::*;+ use crate::config::defaults;+ use crate::retrieval::index::tests::synthetic_pack;++ #[test]+ fn constructs_only_the_four_fixed_download_targets() {+ let temp = tempfile::tempdir().unwrap();+ let expected = defaults().knowledge_datasets.remove(0);+ let pack_root = temp.path().join(&expected.stable_id);+ let targets = knowledge_download_targets(&pack_root, &expected).unwrap();+ assert_eq!(targets.len(), 4);+ for (target, filename) in targets.iter().zip(KNOWLEDGE_ARTIFACT_FILENAMES) {+ assert!(target.url.ends_with(filename));- assert!(target.destination_path.ends_with(filename));++ assert!(target.destination.ends_with(filename));+ assert!(+ target- .destination_path++ .destination++ .to_string_lossy()+ .contains(&expected.current_download_identity)+ );++ assert_eq!(target.sha256.len(), 64);+ }+ }++ #[test]+ fn retains_a_non_current_revision_and_reports_update() {+ let pack = synthetic_pack("simplewiki-old");+ let invalid_current = pack+ .pack_root+ .join(&pack.expected.current_download_identity);+ fs::create_dir_all(&invalid_current).unwrap();++ let result = reconcile_knowledge_pack(&pack.pack_root, &pack.expected).unwrap();+ assert_eq!(+ result.status,+ KnowledgeReconciliationStatus::UpdateAvailable+ );+ assert_eq!(result.active_identity.as_deref(), Some("simplewiki-old"));+ assert!(!invalid_current.exists());+ assert!(pack.revision.exists());+ }++ #[test]+ fn current_revision_wins_and_artifacts_are_removed() {+ let old = synthetic_pack("simplewiki-old");+ let current_identity = old.expected.current_download_identity.clone();+ let current = old.pack_root.join(¤t_identity);+ copy_dir(&old.revision, ¤t);+ let mut manifest: serde_json::Value =+ serde_json::from_slice(&fs::read(current.join("manifest.json")).unwrap()).unwrap();+ manifest["dataset"] = serde_json::Value::String(current_identity.clone());+ fs::write(+ current.join("manifest.json"),+ serde_json::to_vec(&manifest).unwrap(),+ )+ .unwrap();+ fs::write(current.join("manifest.json.metadata.json"), "sidecar").unwrap();+ fs::create_dir(current.join("orphan")).unwrap();++ let result = reconcile_knowledge_pack(&old.pack_root, &old.expected).unwrap();+ assert_eq!(result.status, KnowledgeReconciliationStatus::Ready);+ assert_eq!(+ result.active_identity.as_deref(),+ Some(current_identity.as_str())+ );+ cleanup_obsolete_knowledge_pack_revisions(&old.pack_root, &old.expected, ¤t_identity)+ .unwrap();+ assert!(!old.revision.exists());+ assert!(!current.join("manifest.json.metadata.json").exists());+ assert!(!current.join("orphan").exists());+ let retained = fs::read_dir(current)+ .unwrap()+ .map(|entry| entry.unwrap().file_name().into_string().unwrap())+ .collect::<std::collections::HashSet<_>>();+ assert_eq!(+ retained,+ KNOWLEDGE_ARTIFACT_FILENAMES+ .map(str::to_owned)+ .into_iter()+ .collect()+ );+ }++ #[test]+ fn no_valid_revision_reports_download_and_cleans_invalid_directories() {+ let pack = synthetic_pack("simplewiki-old");+ fs::remove_file(pack.revision.join("vectors.i8")).unwrap();+ let result = reconcile_knowledge_pack(&pack.pack_root, &pack.expected).unwrap();+ assert_eq!(result.status, KnowledgeReconciliationStatus::Download);+ assert!(!pack.revision.exists());+ }++ fn copy_dir(source: &Path, destination: &Path) {+ fs::create_dir_all(destination).unwrap();+ for entry in fs::read_dir(source).unwrap() {+ let entry = entry.unwrap();+ fs::copy(entry.path(), destination.join(entry.file_name())).unwrap();+ }+ }+}diff --cc rust/crates/model-download/src/download.rsindex be8984fa8a,1a9a6328a1..96933a381b--- a/rust/crates/model-download/src/download.rs+++ b/rust/crates/model-download/src/download.rs@@@ -188,9 -180,45 +180,54 @@@ struct ContentRangetotal: Option<u64>,}- pub fn fetch(+ #[derive(Clone)]+ pub struct Downloader {+ client: Client,+ }++ impl Downloader {+ pub fn new() -> Result<Self, Error> {+ Ok(Self {+ client: build_client()?,+ })+ }++ pub(crate) fn client(&self) -> &Client {+ &self.client+ }++ pub async fn download(+ &self,+ targets: Vec<Target>,+ on_progress: impl FnMut(Progress) + Send,+ cancellation: CancellationToken,+ ) -> Result<(), Error> {+ fetch_async(&self.client, targets, on_progress, move || {+ cancellation.is_cancelled()+ })+ .await+ }++++ pub fn download_blocking(++ &self,++ targets: Vec<Target>,++ on_progress: impl FnMut(Progress),++ is_cancelled: impl Fn() -> bool,++ ) -> Result<(), Error> {++ fetch(self, targets, on_progress, is_cancelled)++ }+ }++ fn build_client() -> Result<Client, Error> {+ Client::builder()+ .connect_timeout(CONNECT_TIMEOUT)+ .build()+ .map_err(|err| Error::Network(err.to_string()))+ }++ pub(crate) fn fetch(+ downloader: &Downloader,targets: Vec<Target>,- validate: impl Fn(&Target, &Path) -> Result<(), Error>,on_progress: impl FnMut(Progress),is_cancelled: impl Fn() -> bool,) -> Result<(), Error> {diff --cc rust/crates/model-download/src/model.rsindex 0000000000,9b96045788..4674c998e4mode 000000,100644..100644--- a/rust/crates/model-download/src/model.rs+++ b/rust/crates/model-download/src/model.rs@@@ -1,0 -1,648 +1,679 @@@+ use std::collections::HashSet;+ use std::fs;+ use std::path::{Path, PathBuf};+ use std::sync::atomic::{AtomicBool, Ordering};+ use std::sync::{Mutex, OnceLock, PoisonError};++ use tokio::runtime::Builder;++ use crate::archive;+ use crate::download::{self, Progress, Target};++ #[derive(Debug, Clone, PartialEq, Eq)]+ pub enum ModelTarget {+ Files {+ id: String,+ files: Vec<ModelFile>,+ },+ TarGz {+ id: String,+ url: String,+ sha256: String,+ },+ }++ #[derive(Debug, Clone, PartialEq, Eq)]+ pub struct ModelFile {+ pub name: String,+ pub url: String,+ pub sha256: String,+ }++ #[derive(Debug, Clone)]+ pub struct ModelDownloadProgress {+ pub downloaded_bytes: u64,+ pub total_bytes: Option<u64>,+ pub percent: i32,+ pub status: String,+ pub log_line: Option<String>,+ }++ pub struct ModelDownloader {+ downloader: OnceLock<download::Downloader>,+ models_dir: PathBuf,+ active: AtomicBool,+ download_lock: Mutex<()>,+ }++ impl ModelDownloader {+ pub fn new(models_dir: impl Into<PathBuf>) -> Self {+ Self {+ downloader: OnceLock::new(),+ models_dir: models_dir.into(),+ active: AtomicBool::new(false),+ download_lock: Mutex::new(()),+ }+ }++ pub fn downloader(&self) -> Result<&download::Downloader, download::Error> {+ if let Some(downloader) = self.downloader.get() {+ return Ok(downloader);+ }+ let downloader = download::Downloader::new()?;+ Ok(self.downloader.get_or_init(|| downloader))+ }++ pub fn model_dir(&self, target: &ModelTarget) -> PathBuf {+ storage_dir(&self.models_dir, target)+ }++ pub fn file_path(&self, target: &ModelTarget, name: &str) -> Option<PathBuf> {+ match target {+ ModelTarget::Files { files, .. } if files.iter().any(|file| file.name == name) => {+ Some(storage_dir(&self.models_dir, target).join(name))+ }+ _ => None,+ }+ }++ pub fn is_downloaded(&self, target: &ModelTarget) -> bool {+ if let ModelTarget::TarGz { .. } = target {+ return storage_dir(&self.models_dir, target).is_dir();+ }+ let paths = expected_paths(&self.models_dir, target);+ !paths.is_empty() && paths.iter().all(|path| path.is_file())+ }++ pub fn is_download_active(&self) -> bool {+ self.active.load(Ordering::SeqCst)+ }++ pub fn remove_downloaded(&self, target: &ModelTarget) -> bool {+ let Ok(_guard) = self.download_lock.try_lock() else {+ return false;+ };+ if checked_file_name(storage_key(target)).is_err() {+ return false;+ }+ let dir = storage_dir(&self.models_dir, target);+ dir.exists() && fs::remove_dir_all(dir).is_ok()+ }++ pub fn download(+ &self,+ targets: &[ModelTarget],+ mut on_progress: impl FnMut(ModelDownloadProgress),+ cancellation: &download::CancellationToken,+ ) -> Result<(), download::Error> {+ let _guard = self+ .download_lock+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ let pending = targets+ .iter()+ .filter(|target| !self.is_downloaded(target))+ .collect::<Vec<_>>();+ if pending.is_empty() {+ return Ok(());+ }++ let mut transfers = Vec::new();+ for target in &pending {+ transfers.extend(download_targets(&self.models_dir, target)?);+ }++ if cancellation.is_cancelled() {+ return Err(download::Error::Cancelled);+ }+ let downloader = self.downloader()?;+ self.active.store(true, Ordering::SeqCst);+ let result = download::fetch(+ downloader,+ transfers,+ |progress| on_progress(display_progress(progress)),+ || cancellation.is_cancelled(),+ )+ .and_then(|()| {+ for target in &pending {+ if cancellation.is_cancelled() {+ return Err(download::Error::Cancelled);+ }+ self.extract_if_archive(target, &mut on_progress)?;+ }+ Ok(())+ });+ self.active.store(false, Ordering::SeqCst);+ result+ }++ fn extract_if_archive(+ &self,+ target: &ModelTarget,+ on_progress: &mut impl FnMut(ModelDownloadProgress),+ ) -> Result<(), download::Error> {+ let ModelTarget::TarGz { id, .. } = target else {+ return Ok(());+ };+ on_progress(ModelDownloadProgress {+ downloaded_bytes: 0,+ total_bytes: None,+ percent: 99,+ status: "Extracting...".to_string(),+ log_line: Some(format!("Extracting model archive id={id}")),+ });+ let archive_path = self.staging_dir().join(format!("{id}.tar.gz"));+ let extraction_dir = self.staging_dir().join(id);+ let result =+ archive::extract_tar_gz(&archive_path, &extraction_dir, &self.model_dir(target));+ if result.is_err() {+ let _ = fs::remove_dir_all(extraction_dir);+ } else {+ let _ = fs::remove_file(&archive_path);+ }+ result+ }++ fn staging_dir(&self) -> PathBuf {+ self.models_dir.join(".staging")+ }++ pub fn estimated_download_size(&self, target: &ModelTarget) -> Option<u64> {+ let runtime = Builder::new_current_thread()+ .enable_io()+ .enable_time()+ .build()+ .ok()?;+ let downloader = self.downloader().ok()?;+ runtime.block_on(async {+ let mut total: u64 = 0;+ let mut any = false;+ for entry in download_targets(&self.models_dir, target).ok()? {+ let path = &entry.destination;- let size = if path.exists() {- fs::metadata(path).ok().map(|m| m.len()).filter(|s| *s > 0)++ let size = if let Some(size) = valid_local_size(&entry, path) {++ Some(size)+ } else {+ download::probe_content_length(downloader.client(), &entry.url).await+ };+ if let Some(size) = size {+ any = true;+ total = total.saturating_add(size);+ }+ }+ any.then_some(total)+ })+ }+ }+++fn valid_local_size(target: &Target, path: &Path) -> Option<u64> {++ let actual = download::sha256_file(path).ok()?;++ if actual != target.sha256.trim().to_ascii_lowercase() {++ return None;++ }++ fs::metadata(path)++ .ok()++ .filter(|metadata| metadata.is_file())++ .map(|metadata| metadata.len())++ .filter(|size| *size > 0)++}+++ fn storage_key(target: &ModelTarget) -> &str {+ match target {+ ModelTarget::Files { id, .. } | ModelTarget::TarGz { id, .. } => id,+ }+ }++ fn storage_dir(models_dir: &Path, target: &ModelTarget) -> PathBuf {+ models_dir.join(storage_key(target))+ }++ fn expected_paths(models_dir: &Path, target: &ModelTarget) -> Vec<PathBuf> {+ let dir = storage_dir(models_dir, target);+ match target {+ ModelTarget::TarGz { .. } => vec![dir],+ ModelTarget::Files { files, .. } => files.iter().map(|file| dir.join(&file.name)).collect(),+ }+ }++ fn download_targets(+ models_dir: &Path,+ target: &ModelTarget,+ ) -> Result<Vec<Target>, download::Error> {+ let dir = storage_dir(models_dir, target);+ match target {+ ModelTarget::TarGz { id, url, sha256 } => {+ checked_file_name(id)?;+ Ok(vec![Target {+ label: "Model".to_string(),+ url: url.clone(),+ sha256: sha256.clone(),+ destination: models_dir.join(".staging").join(format!("{id}.tar.gz")),+ }])+ }+ ModelTarget::Files { id, files } => {+ checked_file_name(id)?;+ if files.is_empty() {+ return Err(download::Error::InvalidTarget(format!("{id} has no files")));+ }+ let mut names = HashSet::new();+ files+ .iter()+ .map(|file| {+ checked_file_name(&file.name)?;+ if !names.insert(file.name.as_str()) {+ return Err(download::Error::InvalidTarget(format!(+ "{id} has a duplicate file name {}",+ file.name+ )));+ }+ Ok(Target {+ label: file.name.clone(),+ url: file.url.clone(),+ sha256: file.sha256.clone(),+ destination: dir.join(&file.name),+ })+ })+ .collect()+ }+ }+ }++ fn checked_file_name(name: &str) -> Result<(), download::Error> {+ if !name.is_empty()+ && name != "."+ && name != ".."+ && !name.contains('/')+ && !name.contains('\\')+ {+ return Ok(());+ }+ Err(download::Error::InvalidTarget(format!(+ "'{name}' is not a safe file name"+ )))+ }++ fn display_progress(progress: Progress) -> ModelDownloadProgress {+ let total = progress.total_bytes.filter(|total| *total > 0);+ let percent = total+ .map(|total| ((progress.downloaded_bytes * 100 / total) as i32).clamp(0, 99))+ .unwrap_or(0);+ let status = if let Some(total) = total {+ format!(+ "Downloading... {} / {}",+ format_bytes(progress.downloaded_bytes),+ format_bytes(total)+ )+ } else if progress.file_downloaded_bytes > 0 {+ format!(+ "Downloading {}... {}",+ progress.label.to_lowercase(),+ format_bytes(progress.file_downloaded_bytes)+ )+ } else {+ format!("Downloading {}...", progress.label.to_lowercase())+ };+ let log_line = if progress.file_complete {+ Some(format!(+ "Model download file complete label={} bytes={} elapsedMs={} rate={}/s retries={}",+ progress.label,+ progress.file_downloaded_bytes,+ progress.file_elapsed_ms,+ format_bytes(rate_bytes(progress.file_bytes_per_second)),+ progress.file_retry_count+ ))+ } else if progress.complete {+ Some(format!(+ "Model download complete bytes={} elapsedMs={} rate={}/s retries={}",+ progress.downloaded_bytes,+ progress.elapsed_ms,+ format_bytes(rate_bytes(progress.bytes_per_second)),+ progress.retry_count+ ))+ } else {+ None+ };++ ModelDownloadProgress {+ downloaded_bytes: progress.downloaded_bytes,+ total_bytes: progress.total_bytes,+ percent,+ status,+ log_line,+ }+ }++ fn rate_bytes(bytes_per_second: f64) -> u64 {+ if bytes_per_second.is_finite() && bytes_per_second > 0.0 {+ bytes_per_second as u64+ } else {+ 0+ }+ }++ fn format_bytes(bytes: u64) -> String {+ const UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];+ let mut size = bytes as f64;+ let mut unit = 0;+ while size >= 1024.0 && unit < UNITS.len() - 1 {+ size /= 1024.0;+ unit += 1;+ }+ format!("{size:.1} {}", UNITS[unit])+ }++ #[cfg(test)]+ mod tests {+ use super::*;++ fn scratch_dir(name: &str) -> PathBuf {+ let dir = std::env::temp_dir().join(format!(+ "ente-model-download-{name}-{}",+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap()+ .as_nanos()+ ));+ fs::create_dir_all(&dir).expect("create test dir");+ dir+ }++ fn test_sha() -> String {+ "0".repeat(64)+ }++ fn target(id: &str) -> ModelTarget {+ ModelTarget::Files {+ id: id.to_string(),+ files: vec![+ ModelFile {+ name: "model.gguf".to_string(),+ url: "https://example.org/models/main.gguf?download=true".to_string(),+ sha256: test_sha(),+ },+ ModelFile {+ name: "mmproj.gguf".to_string(),+ url: "https://example.org/models/mmproj.gguf".to_string(),+ sha256: test_sha(),+ },+ ],+ }+ }++ fn write_gguf(path: &Path, content: &[u8]) {+ fs::create_dir_all(path.parent().unwrap()).unwrap();+ fs::write(path, content).unwrap();+ }++ #[test]+ fn paths_are_keyed_by_preset_id() {+ let dir = scratch_dir("paths");+ let downloader = ModelDownloader::new(&dir);++ assert_eq!(+ downloader+ .file_path(&target("qwen-2b-q8"), "model.gguf")+ .unwrap(),+ dir.join("qwen-2b-q8/model.gguf")+ );+ assert_eq!(+ downloader.file_path(&target("qwen-2b-q8"), "mmproj.gguf"),+ Some(dir.join("qwen-2b-q8/mmproj.gguf"))+ );++ let _ = fs::remove_dir_all(dir);+ }++ #[test]+ fn is_downloaded_requires_every_member_file() {+ let dir = scratch_dir("is-downloaded");+ let downloader = ModelDownloader::new(&dir);+ let target = target("qwen-2b-q8");+ let model = downloader.file_path(&target, "model.gguf").unwrap();+ let mmproj = downloader.file_path(&target, "mmproj.gguf").unwrap();++ assert!(!downloader.is_downloaded(&target));++ write_gguf(&model, b"GGUFdata");+ assert!(!downloader.is_downloaded(&target));++ write_gguf(&mmproj, b"GGUFdata");+ assert!(downloader.is_downloaded(&target));++ let _ = fs::remove_dir_all(dir);+ }+++ #[test]++ fn local_size_ignores_files_that_fail_checksum_validation() {++ let dir = scratch_dir("local-size-validation");++ let path = dir.join("model.gguf");++ write_gguf(&path, b"GGUFdata");++ let mut target = Target {++ label: "Model".to_string(),++ url: "https://example.org/model.gguf".to_string(),++ sha256: test_sha(),++ destination: path.clone(),++ };++++ assert_eq!(valid_local_size(&target, &path), None);++ target.sha256 = download::sha256_file(&path).unwrap();++ assert_eq!(valid_local_size(&target, &path), Some(8));++++ let _ = fs::remove_dir_all(dir);++ }+++ #[test]+ fn download_skips_when_already_present() {+ let dir = scratch_dir("skip");+ let downloader = ModelDownloader::new(&dir);+ let target = target("qwen-2b-q8");+ write_gguf(+ &downloader.file_path(&target, "model.gguf").unwrap(),+ b"GGUFdata",+ );+ write_gguf(+ &downloader.file_path(&target, "mmproj.gguf").unwrap(),+ b"GGUFdata",+ );++ downloader+ .download(+ std::slice::from_ref(&target),+ |_| {},+ &download::CancellationToken::new(),+ )+ .expect("skip without network");++ let _ = fs::remove_dir_all(dir);+ }++ fn tar_gz_target(id: &str) -> ModelTarget {+ ModelTarget::TarGz {+ id: id.to_string(),+ url: format!("https://models.example.org/{id}.tar.gz"),+ sha256: test_sha(),+ }+ }++ fn file_target(id: &str) -> ModelTarget {+ ModelTarget::Files {+ id: id.to_string(),+ files: vec![ModelFile {+ name: "model.onnx".to_string(),+ url: format!("https://models.example.org/{id}.onnx"),+ sha256: test_sha(),+ }],+ }+ }++ fn write_tar_gz(path: &Path, dir_name: &str, files: &[(&str, &[u8])]) {+ let file = fs::File::create(path).unwrap();+ let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());+ let mut builder = tar::Builder::new(encoder);+ for (name, content) in files {+ let mut header = tar::Header::new_gnu();+ header.set_size(content.len() as u64);+ header.set_mode(0o644);+ header.set_cksum();+ builder+ .append_data(&mut header, format!("{dir_name}/{name}"), *content)+ .unwrap();+ }+ builder.into_inner().unwrap().finish().unwrap();+ }++ #[test]+ fn archive_and_onnx_targets_use_per_model_dirs() {+ let dir = scratch_dir("variant-paths");+ let downloader = ModelDownloader::new(&dir);++ let archive = tar_gz_target("parakeet-v3-int8");+ assert_eq!(downloader.model_dir(&archive), dir.join("parakeet-v3-int8"));+ assert_eq!(downloader.file_path(&archive, "mmproj.gguf"), None);++ let onnx = file_target("silero-vad-v4");+ assert_eq!(+ downloader.file_path(&onnx, "model.onnx").unwrap(),+ dir.join("silero-vad-v4/model.onnx")+ );+ assert_eq!(+ downloader.file_path(&onnx, "model.onnx"),+ Some(dir.join("silero-vad-v4/model.onnx"))+ );+ assert_eq!(downloader.file_path(&onnx, "missing.onnx"), None);+ assert_eq!(downloader.file_path(&onnx, "mmproj.gguf"), None);++ let _ = fs::remove_dir_all(dir);+ }++ #[test]+ fn archive_download_extracts_into_model_dir() {+ let dir = scratch_dir("variant-extract");+ let downloader = ModelDownloader::new(&dir);+ let target = tar_gz_target("parakeet-v3-int8");+ assert!(!downloader.is_downloaded(&target));++ fs::create_dir_all(dir.join(".staging")).unwrap();+ write_tar_gz(+ &dir.join(".staging/parakeet-v3-int8.tar.gz"),+ "parakeet-tdt-0.6b-v3-int8",+ &[("encoder.onnx", b"enc"), ("tokens.txt", b"tok")],+ );+ let mut statuses = Vec::new();+ downloader+ .extract_if_archive(&target, &mut |progress| statuses.push(progress.status))+ .expect("extract");++ assert!(downloader.is_downloaded(&target));+ assert_eq!(+ fs::read(dir.join("parakeet-v3-int8/encoder.onnx")).unwrap(),+ b"enc"+ );+ assert_eq!(statuses, ["Extracting..."]);+ assert!(!dir.join(".staging/parakeet-v3-int8.tar.gz").exists());++ assert!(downloader.remove_downloaded(&target));+ assert!(!downloader.is_downloaded(&target));++ let _ = fs::remove_dir_all(dir);+ }++ #[test]+ fn failed_archive_extraction_cleans_staging() {+ let dir = scratch_dir("variant-extract-failure");+ let downloader = ModelDownloader::new(&dir);+ let target = tar_gz_target("parakeet-v3-int8");+ let staging_dir = dir.join(".staging");+ fs::create_dir_all(&staging_dir).unwrap();+ fs::write(staging_dir.join("parakeet-v3-int8.tar.gz"), b"invalid").unwrap();++ assert!(downloader.extract_if_archive(&target, &mut |_| {}).is_err());+ assert!(!staging_dir.join("parakeet-v3-int8").exists());+ assert!(staging_dir.join("parakeet-v3-int8.tar.gz").exists());++ let _ = fs::remove_dir_all(dir);+ }++ #[test]+ fn onnx_target_downloaded_and_removed_via_model_file() {+ let dir = scratch_dir("variant-onnx");+ let downloader = ModelDownloader::new(&dir);+ let target = file_target("silero-vad-v4");+ assert!(!downloader.is_downloaded(&target));++ write_gguf(+ &downloader.file_path(&target, "model.onnx").unwrap(),+ b"onnxbytes",+ );+ assert!(downloader.is_downloaded(&target));++ assert!(downloader.remove_downloaded(&target));+ assert!(!downloader.is_downloaded(&target));+ assert!(!dir.join("silero-vad-v4").exists());++ let _ = fs::remove_dir_all(dir);+ }++ #[test]+ fn display_progress_composes_percent_and_status() {+ let progress = Progress {+ label: "Model".to_string(),+ downloaded_bytes: 512 * 1024 * 1024,+ total_bytes: Some(2 * 1024 * 1024 * 1024),+ file_downloaded_bytes: 512 * 1024 * 1024,+ file_total_bytes: Some(2 * 1024 * 1024 * 1024),+ percentage: 25.0,+ elapsed_ms: 0,+ bytes_per_second: 0.0,+ file_elapsed_ms: 0,+ file_bytes_per_second: 0.0,+ retry_count: 0,+ file_retry_count: 0,+ file_complete: false,+ complete: false,+ };++ let display = display_progress(progress);+ assert_eq!(display.percent, 25);+ assert_eq!(display.status, "Downloading... 512.0 MB / 2.0 GB");+ assert_eq!(display.log_line, None);++ let unknown_total = Progress {+ total_bytes: None,+ ..display_to_progress_stub()+ };+ let display = display_progress(unknown_total);+ assert_eq!(display.percent, 0);+ assert_eq!(display.status, "Downloading model... 100.0 B");++ let file_complete = Progress {+ file_complete: true,+ file_bytes_per_second: 2048.0,+ ..display_to_progress_stub()+ };+ let display = display_progress(file_complete);+ assert_eq!(+ display.log_line.as_deref(),+ Some(+ "Model download file complete label=Model bytes=100 elapsedMs=0 rate=2.0 KB/s retries=0"+ )+ );+ }++ fn display_to_progress_stub() -> Progress {+ Progress {+ label: "Model".to_string(),+ downloaded_bytes: 100,+ total_bytes: None,+ file_downloaded_bytes: 100,+ file_total_bytes: None,+ percentage: 0.0,+ elapsed_ms: 0,+ bytes_per_second: 0.0,+ file_elapsed_ms: 0,+ file_bytes_per_second: 0.0,+ retry_count: 0,+ file_retry_count: 0,+ file_complete: false,+ complete: false,+ }+ }+ }