π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Align mobile knowledge pack lifecycle
setalpc35b00a61045
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex 595e187b40..6c8286de8c 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -68,7 +68,8 @@ class AppStore(messageStore = messageStore,attachmentActions = attachmentActions,modelSettingsActions = modelSettingsActions,- configDefaults = configDefaults+ configDefaults = configDefaults,+ awaitKnowledgeReady = knowledgeStore::awaitEnabledPacksReady)fun bootstrap(scope: CoroutineScope) {chatActions.setScope(scope)diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.ktindex 844ab5d6c7..734237348f 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt@@ -38,7 +38,6 @@ import io.ente.ensu.logging.LogViewerScreenimport io.ente.ensu.settings.SettingsScreenimport io.ente.ensu.settings.SystemPromptSettingsScreenimport io.ente.ensu.settings.KnowledgeSettingsScreen-import io.ente.ensu.settings.TermsAndConditionsScreenimport io.ente.ensu.device.isChatSupported@Composable@@ -96,9 +95,6 @@ internal fun HomeNavigation(HomeRoute.Knowledge -> {SimpleTopBar(title = "Ensu Packs") { navController.popBackStack() }}- HomeRoute.Terms -> {- SimpleTopBar(title = "Terms and Conditions") { navController.popBackStack() }- }else -> Unit}}@@ -156,7 +152,6 @@ internal fun HomeNavigation(isAdvancedUnlocked = appState.developerSettings.isAdvancedUnlocked,onOpenLogs = { navController.navigate(HomeRoute.Logs) },onOpenKnowledge = { navController.navigate(HomeRoute.Knowledge) },- onOpenTerms = { navController.navigate(HomeRoute.Terms) },onOpenModelSettings = { navController.navigate(HomeRoute.ModelSettings) },onOpenSystemPromptSettings = { navController.navigate(HomeRoute.SystemPromptSettings) },onUnlockAdvanced = {@@ -181,15 +176,6 @@ internal fun HomeNavigation(onSetEnabled = store::setKnowledgePackEnabled)}- composable(- route = HomeRoute.Terms,- enterTransition = { forwardEnter() },- exitTransition = { forwardExit() },- popEnterTransition = { backEnter() },- popExitTransition = { backExit() }- ) {- TermsAndConditionsScreen()- }composable(route = HomeRoute.Logs,enterTransition = { forwardEnter() },@@ -258,7 +244,6 @@ internal object HomeRoute {const val ModelSettings = "model-settings"const val SystemPromptSettings = "system-prompt-settings"const val Knowledge = "knowledge"- const val Terms = "terms"}internal fun AnimatedContentTransitionScope<NavBackStackEntry>.forwardEnter() =diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.ktindex 88890adfd9..8608cc0a5d 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt@@ -795,13 +795,6 @@ private fun KnowledgeSourcesDialog(style = EnsuTypography.large,color = EnsuColor.textPrimary())- citation.section?.trim()?.takeIf { it.isNotEmpty() }?.let { section ->- Text(- text = section,- style = EnsuTypography.small,- color = EnsuColor.textMuted()- )- }HorizontalDivider(color = EnsuColor.border())Text(text = "Attribution",diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex 2b9cdea940..f36d6a082f 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -17,7 +17,6 @@ import io.ente.ensu.chat.ChatSessionimport io.ente.ensu.bindings.DbExceptionimport io.ente.ensu.bindings.LlmExceptionimport io.ente.ensu.bindings.ConfigDefaults-import io.ente.ensu.bindings.KnowledgePromptHitimport io.ente.ensu.logging.LogLevelimport io.ente.ensu.chat.MessageAuthorimport io.ente.ensu.chat.sessionTitleFromText@@ -28,7 +27,6 @@ import io.ente.ensu.bindings.buildKnowledgePromptContextimport io.ente.ensu.bindings.cleanAssistantTextimport io.ente.ensu.bindings.finalizeAssistantTextimport io.ente.ensu.knowledge.KnowledgeProvider-import io.ente.ensu.knowledge.KnowledgeSearchHitimport java.text.SimpleDateFormatimport java.util.Dateimport java.util.Locale@@ -58,7 +56,8 @@ internal class ChatStoreActions(private val messageStore: MutableMap<String, MutableList<ChatMessage>>,private val attachmentActions: AttachmentStoreActions,private val modelSettingsActions: ModelSettingsActions,- private val configDefaults: ConfigDefaults+ private val configDefaults: ConfigDefaults,+ private val awaitKnowledgeReady: suspend () -> Unit) {private val branchSelections = mutableMapOf<String, MutableMap<String, String>>()private val sessionSummaries = mutableMapOf<String, String>()@@ -485,10 +484,12 @@ internal class ChatStoreActions(priorSummary?.join()val isActive = { isGenerationActive(generationToken, sessionId) }if (!isActive()) return@launch+ awaitKnowledgeReady()+ if (!isActive()) return@launchval progressTracker = DownloadProgressTracker()var embeddingAssetInvalid = falseval enabledDatasets = state.value.knowledge.enabledReadyDatasets- val knowledgeHits: List<KnowledgeSearchHit> = if (+ val knowledgeHits = if (userMessage.text.isNotBlank() && enabledDatasets.isNotEmpty()) {try {@@ -607,12 +608,7 @@ internal class ChatStoreActions().coerceAtLeast(0)val knowledgeContext = runCatching {buildKnowledgePromptContext(- hits = knowledgeHits.map { item ->- KnowledgePromptHit(- datasetId = item.dataset.stableId,- hit = item.hit- )- },+ hits = knowledgeHits,maxUtf8Bytes = min(configDefaults.knowledgeEmbedding.maxContextUtf8Bytes.toInt(),remainingKnowledgeBytes@@ -632,9 +628,7 @@ internal class ChatStoreActions(systemPrompt = it)}- val useKnowledge = knowledgeContext != null &&- knowledgeHistorySelection != null &&- !knowledgeHistorySelection.wasTrimmed+ val useKnowledge = knowledgeHistorySelection?.wasTrimmed == falseval historySelection = if (useKnowledge) {knowledgeHistorySelection ?: normalHistorySelection} else {diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.ktindex 8f60ab0ceb..e00084e202 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt@@ -1,38 +1,33 @@package io.ente.ensu.knowledge+import io.ente.ensu.bindings.CancellationTokenimport io.ente.ensu.bindings.KnowledgeDatasetConfigimport io.ente.ensu.bindings.KnowledgeDownloadCallbackimport io.ente.ensu.bindings.KnowledgeDownloadProgress+import io.ente.ensu.bindings.KnowledgePromptHitimport io.ente.ensu.bindings.KnowledgeReconciliation-import io.ente.ensu.bindings.RetrievalHitimport io.ente.ensu.bindings.RetrievalIndeximport io.ente.ensu.bindings.cleanupObsoleteKnowledgePackRevisionsimport io.ente.ensu.bindings.downloadKnowledgePackimport io.ente.ensu.bindings.reconcileKnowledgePackimport java.io.File-import java.util.concurrent.atomic.AtomicBoolean-import kotlinx.coroutines.CoroutineScope+import kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.Deferredimport kotlinx.coroutines.Dispatchers-import kotlinx.coroutines.SupervisorJob+import kotlinx.coroutines.NonCancellableimport kotlinx.coroutines.async+import kotlinx.coroutines.coroutineScopeimport kotlinx.coroutines.currentCoroutineContextimport kotlinx.coroutines.ensureActiveimport kotlinx.coroutines.sync.Muteximport kotlinx.coroutines.sync.withLockimport kotlinx.coroutines.withContext-data class KnowledgeSearchHit(- val dataset: KnowledgeDatasetConfig,- val hit: RetrievalHit-)-class KnowledgeProvider(- private val knowledgeRoot: File,- private val ioScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)+ private val knowledgeRoot: File) {private data class ActiveMutation(- val cancelled: AtomicBoolean,+ val cancellation: CancellationToken,val task: Deferred<KnowledgeReconciliation>)@@ -73,23 +68,24 @@ class KnowledgeProvider(suspend fun download(dataset: KnowledgeDatasetConfig,onProgress: (KnowledgeDownloadProgress) -> Unit- ): KnowledgeReconciliation {+ ): KnowledgeReconciliation = coroutineScope {+ var ownsMutation = falseval mutation = lifecycleGate(dataset.stableId).withLock {mutationGate.withLock {mutations[dataset.stableId] ?: run {- val cancelled = AtomicBoolean(false)- val task = ioScope.async {+ ownsMutation = true+ val cancellation = CancellationToken()+ val task = async(Dispatchers.IO) {try {downloadKnowledgePack(packRoot(dataset).absolutePath,- dataset,+ dataset.stableId,object : KnowledgeDownloadCallback {override fun onProgress(progress: KnowledgeDownloadProgress) {onProgress(progress)}-- override fun isCancelled(): Boolean = cancelled.get()- }+ },+ cancellation)indexGate.withLock {val result = reconcileAndOpenLocked(dataset)@@ -99,26 +95,36 @@ class KnowledgeProvider(result}} catch (error: Throwable) {- indexGate.withLock {- incomingRevision(dataset).deleteRecursively()- runCatching { reconcileAndOpenLocked(dataset) }+ withContext(NonCancellable) {+ indexGate.withLock {+ incomingRevision(dataset).deleteRecursively()+ runCatching { reconcileAndOpenLocked(dataset) }+ }}throw error}}- ActiveMutation(cancelled, task).also {+ ActiveMutation(cancellation, task).also {mutations[dataset.stableId] = it}}}}- return try {+ try {mutation.task.await()+ } catch (error: CancellationException) {+ if (ownsMutation) mutation.cancellation.cancel()+ throw error} finally {- mutationGate.withLock {- if (mutations[dataset.stableId] === mutation && mutation.task.isCompleted) {- mutations.remove(dataset.stableId)+ withContext(NonCancellable) {+ if (ownsMutation && !mutation.task.isCompleted) {+ mutation.task.join()+ }+ mutationGate.withLock {+ if (mutations[dataset.stableId] === mutation && mutation.task.isCompleted) {+ mutations.remove(dataset.stableId)+ }}}}@@ -126,7 +132,7 @@ class KnowledgeProvider(suspend fun cancel(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {val mutation = mutationGate.withLock { mutations[dataset.stableId] }- mutation?.cancelled?.set(true)+ mutation?.cancellation?.cancel()runCatching { mutation?.task?.await() }mutationGate.withLock {if (mutations[dataset.stableId] === mutation) {@@ -140,9 +146,9 @@ class KnowledgeProvider(datasets: List<KnowledgeDatasetConfig>,query: List<Float>,maxHits: UInt- ): List<KnowledgeSearchHit> = withContext(Dispatchers.IO) {+ ): List<KnowledgePromptHit> = withContext(Dispatchers.IO) {indexGate.withLock {- val merged = mutableListOf<KnowledgeSearchHit>()+ val merged = mutableListOf<KnowledgePromptHit>()for (dataset in datasets) {currentCoroutineContext().ensureActive()val open = indexes[dataset.stableId] ?: continue@@ -152,7 +158,7 @@ class KnowledgeProvider(continue}currentCoroutineContext().ensureActive()- merged += hits.map { hit -> KnowledgeSearchHit(dataset, hit) }+ merged += hits.map { hit -> KnowledgePromptHit(dataset.stableId, hit) }}merged.sortedByDescending { it.hit.score }.take(maxHits.toInt())}@@ -161,19 +167,19 @@ class KnowledgeProvider(private fun reconcileAndOpenLocked(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {val previous = indexes[dataset.stableId]val root = packRoot(dataset).absolutePath- val result = reconcileKnowledgePack(root, dataset)+ val result = reconcileKnowledgePack(root, dataset.stableId)val directory = result.activeDirectoryif (directory == null) {indexes.remove(dataset.stableId)previous?.index?.destroy()} else if (previous?.directory != directory) {- val replacement = RetrievalIndex.open(directory, dataset)+ val replacement = RetrievalIndex.open(directory, dataset.stableId)indexes[dataset.stableId] = OpenIndex(directory, replacement)previous?.index?.destroy()}result.activeIdentity?.let { activeIdentity ->runCatching {- cleanupObsoleteKnowledgePackRevisions(root, dataset, activeIdentity)+ cleanupObsoleteKnowledgePackRevisions(root, dataset.stableId, activeIdentity)}}return resultdiff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.ktindex 185e73fbc6..165d8fbe18 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt@@ -8,11 +8,13 @@ import io.ente.ensu.AppStateimport io.ente.ensu.bindings.DownloadErrorimport io.ente.ensu.bindings.KnowledgeDatasetConfigimport io.ente.ensu.bindings.KnowledgeDownloadException+import io.ente.ensu.bindings.KnowledgeDownloadProgressimport io.ente.ensu.bindings.KnowledgeReconciliationimport io.ente.ensu.bindings.KnowledgeReconciliationStatusimport io.ente.ensu.device.isChatSupportedimport io.ente.ensu.logging.FileLogRepositoryimport io.ente.ensu.logging.LogLevel+import kotlinx.coroutines.CompletableDeferredimport kotlinx.coroutines.CoroutineScopeimport kotlinx.coroutines.Jobimport kotlinx.coroutines.flow.Flow@@ -24,14 +26,12 @@ import kotlinx.coroutines.launchprivate val Context.knowledgePreferences by preferencesDataStore("ensu_knowledge_preferences")-data class KnowledgeMutationProgress(val percent: Int, val label: String)-data class KnowledgePackState(val config: KnowledgeDatasetConfig,val status: KnowledgeReconciliationStatus? = null,val activeIdentity: String? = null,val enabled: Boolean = false,- val mutationProgress: KnowledgeMutationProgress? = null,+ val mutationProgress: KnowledgeDownloadProgress? = null,val errorMessage: String? = null) {val isMutating: Boolean get() = mutationProgress != null@@ -61,9 +61,12 @@ class KnowledgeStore(private val catalog = datasets.associateBy { it.stableId }private val jobs = mutableMapOf<String, Job>()private var scope: CoroutineScope? = null+ private var bootstrapJob: Job? = null+ private val enabledPacksReady = CompletableDeferred<Unit>()fun bootstrap(scope: CoroutineScope) {this.scope = scope+ if (bootstrapJob != null) returnstate.update { appState ->appState.copy(knowledge = KnowledgeState(@@ -71,33 +74,30 @@ class KnowledgeStore())}- scope.launch {+ bootstrapJob = scope.launch {val requestedEnabled = runCatching {preferences.enabledDatasetIds.first()}.getOrDefault(emptySet())- catalog.values.forEach { dataset ->- val result = runCatching { provider.reconcile(dataset) }- updatePack(dataset.stableId) { current ->- result.fold(- onSuccess = { reconciliation ->- current.fromReconciliation(- reconciliation,- enabled = dataset.stableId in requestedEnabled- )- },- onFailure = { error ->- current.copy(- status = KnowledgeReconciliationStatus.DOWNLOAD,- enabled = false,- errorMessage = userFacingKnowledgeError(error)- )- }- )+ val (enabled, disabled) = catalog.values.partition {+ it.stableId in requestedEnabled+ }+ try {+ enabled.forEach { dataset ->+ reconcileAndUpdate(dataset, enabled = true)}+ } finally {+ enabledPacksReady.complete(Unit)+ }+ disabled.forEach { dataset ->+ reconcileAndUpdate(dataset, enabled = false)}}}+ suspend fun awaitEnabledPacksReady() {+ enabledPacksReady.await()+ }+fun downloadOrUpdate(stableId: String) {if (!state.value.chat.deviceCapability.isChatSupported()) returnval dataset = catalog[stableId] ?: return@@ -106,7 +106,7 @@ class KnowledgeStore(val wasInstalled = state.value.knowledge.packs[stableId]?.activeIdentity != nullupdatePack(stableId) {it.copy(- mutationProgress = KnowledgeMutationProgress(0, "Starting download..."),+ mutationProgress = KnowledgeDownloadProgress("Starting download...", 0.0),errorMessage = null)}@@ -114,12 +114,7 @@ class KnowledgeStore(try {val reconciliation = provider.download(dataset) { progress ->updatePack(stableId) { current ->- current.copy(- mutationProgress = KnowledgeMutationProgress(- progress.percentage.toInt().coerceIn(0, 100),- progress.label- )- )+ current.copy(mutationProgress = progress)}}val shouldEnable = !wasInstalled &&@@ -198,6 +193,27 @@ class KnowledgeStore(}}+ private suspend fun reconcileAndUpdate(+ dataset: KnowledgeDatasetConfig,+ enabled: Boolean+ ) {+ val result = runCatching { provider.reconcile(dataset) }+ updatePack(dataset.stableId) { current ->+ result.fold(+ onSuccess = { reconciliation ->+ current.fromReconciliation(reconciliation, enabled)+ },+ onFailure = { error ->+ current.copy(+ status = KnowledgeReconciliationStatus.DOWNLOAD,+ enabled = false,+ errorMessage = userFacingKnowledgeError(error)+ )+ }+ )+ }+ }+private fun KnowledgePackState.fromReconciliation(result: KnowledgeReconciliation,enabled: Booleandiff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.ktindex 30a70ae9aa..114edf00b6 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt@@ -5,7 +5,6 @@ import io.ente.ensu.bindings.LlmChatMessage as NativeChatMessageimport io.ente.ensu.bindings.LlmChatRequestimport io.ente.ensu.bindings.LlmContextimport io.ente.ensu.bindings.LlmContextParams-import io.ente.ensu.bindings.LlmEmbeddingContextParamsimport io.ente.ensu.bindings.LlmGenerationEventimport io.ente.ensu.bindings.LlmGenerationEventCallbackimport io.ente.ensu.bindings.LlmGenerationSummary as NativeSummary@@ -25,6 +24,10 @@ import kotlinx.coroutines.sync.Muteximport kotlinx.coroutines.sync.withLockimport kotlinx.coroutines.withContext+class RequiredModelValidationError(+ val targetId: String+) : Exception("Downloaded model failed validation: $targetId")+class LlmProvider(private val downloader: ModelDownloader,private val transcriber: Transcriber,@@ -94,10 +97,10 @@ class LlmProvider(modelLoadMutex.withLock {if (!isEmbeddingModelReady()) {downloader.removeDownloaded(embeddingDownloadTarget)- throw IllegalStateException("Embedding model failed exact readiness validation")+ throw RequiredModelValidationError(knowledgeEmbedding.targetId)}if (!isChatModelReady(selection)) {- throw IllegalStateException("Chat model failed readiness validation")+ throw RequiredModelValidationError(selection.id)}ensureModelReadyLocked(selection,@@ -179,17 +182,7 @@ class LlmProvider(var embeddingContext: LlmContext? = nulltry {val threads = max(1, Runtime.getRuntime().availableProcessors() - 1)- embeddingContext = embeddingModel.newEmbeddingContext(- LlmEmbeddingContextParams(- contextSize = knowledgeEmbedding.contextSize,- nThreads = threads,- batchSize = knowledgeEmbedding.batchSize,- microBatchSize = knowledgeEmbedding.microBatchSize,- sourceDim = knowledgeEmbedding.sourceDim,- dim = knowledgeEmbedding.dim,- queryPrompt = knowledgeEmbedding.queryPrompt- )- )+ embeddingContext = embeddingModel.newEmbeddingContext(threads)block { text -> embeddingContext.embed(text) }} finally {embeddingContext?.destroy()diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex ef1ee5aac6..7d15ac0579 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@ -144,9 +144,7 @@ internal class ModelSettingsActions(if (!userInitiated && !currentState.chat.hasRequestedModelDownload) returnval selection = resolveSelection(currentState.modelSettings)- val wasChatReady = llmProvider.isChatModelReady(selection)- val wasEmbeddingReady = llmProvider.isEmbeddingModelReady()- val isDownloaded = wasChatReady && wasEmbeddingReady+ val isDownloaded = llmProvider.isChatModelReady(selection) && llmProvider.isEmbeddingModelReady()if (isDownloaded) {state.update { appState ->appState.copy(@@ -363,6 +361,7 @@ internal class ModelSettingsActions(if (err is kotlinx.coroutines.CancellationException) return falseif (err is LlmException.Cancelled) return falseif (isOutOfStorageError(err)) return false+ if (err is RequiredModelValidationError) return falseif (err is LlmException.Download) {when (val error = err.error) {is DownloadError.Validation -> return falsediff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.ktindex adafc2bb8a..873b2ac384 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.kt@@ -118,7 +118,7 @@ private fun KnowledgePackCard(if (pack.isMutating) {Spacer(Modifier.height(EnsuSpacing.md.dp))LinearProgressIndicator(- progress = { (pack.mutationProgress?.percent ?: 0).coerceIn(0, 100) / 100f },+ progress = { (pack.mutationProgress?.percentage ?: 0.0).coerceIn(0.0, 100.0).toFloat() / 100f },modifier = Modifier.fillMaxWidth(),color = EnsuColor.accent(),trackColor = EnsuColor.border()diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/LegalSettingsScreens.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/LegalSettingsScreens.ktdeleted file mode 100644index 45c5464704..0000000000--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/LegalSettingsScreens.kt+++ /dev/null@@ -1,51 +0,0 @@-package io.ente.ensu.settings--import androidx.compose.foundation.layout.Arrangement-import androidx.compose.foundation.layout.PaddingValues-import androidx.compose.foundation.layout.padding-import androidx.compose.foundation.lazy.LazyColumn-import androidx.compose.material.icons.Icons-import androidx.compose.material.icons.outlined.Description-import androidx.compose.runtime.Composable-import androidx.compose.ui.platform.LocalUriHandler-import androidx.compose.ui.Modifier-import androidx.compose.ui.unit.dp-import io.ente.ensu.designsystem.EnsuSpacing-import io.ente.ensu.designsystem.HugeIcons--object EnsuLegalDocuments {- const val PRIVACY_TITLE = "Privacy Policy"- const val PRIVACY_URL = "https://ente.com/privacy/"- const val ENTE_TERMS_TITLE = "Ente Terms and Conditions"- const val ENTE_TERMS_URL = "https://ente.com/terms/"-}--@Composable-fun TermsAndConditionsScreen() {- val uriHandler = LocalUriHandler.current-- LazyColumn(- modifier = Modifier.padding(horizontal = EnsuSpacing.pageHorizontal.dp),- contentPadding = PaddingValues(vertical = EnsuSpacing.lg.dp),- verticalArrangement = Arrangement.spacedBy(EnsuSpacing.sm.dp)- ) {- item(key = "privacy") {- SettingsRow(- SettingsItem(- title = EnsuLegalDocuments.PRIVACY_TITLE,- iconRes = HugeIcons.ViewIcon,- onClick = { uriHandler.openUri(EnsuLegalDocuments.PRIVACY_URL) }- )- )- }- item(key = "ente-terms") {- SettingsRow(- SettingsItem(- title = EnsuLegalDocuments.ENTE_TERMS_TITLE,- iconVector = Icons.Outlined.Description,- onClick = { uriHandler.openUri(EnsuLegalDocuments.ENTE_TERMS_URL) }- )- )- }- }-}diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.ktindex e71436526c..189c3c5e1e 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.kt@@ -49,7 +49,6 @@ fun SettingsScreen(isAdvancedUnlocked: Boolean,onOpenLogs: () -> Unit,onOpenKnowledge: () -> Unit,- onOpenTerms: () -> Unit,onOpenModelSettings: () -> Unit,onOpenSystemPromptSettings: () -> Unit,onUnlockAdvanced: () -> Unit,@@ -60,7 +59,7 @@ fun SettingsScreen(var lastBuildVersionTapAt by remember { mutableStateOf<Long?>(null) }val context = LocalContext.current- val allItems = remember(context, onOpenLogs, onOpenKnowledge, onOpenTerms, onSignIn) {+ val allItems = remember(context, onOpenLogs, onOpenKnowledge, onSignIn) {buildList {add(SettingsItem(@@ -96,14 +95,16 @@ fun SettingsScreen(add(SettingsItem(- title = "Terms and Conditions",+ title = "Privacy Policy",+ iconRes = HugeIcons.ViewIcon,+ onClick = { context.openExternalLink("https://ente.com/privacy") }+ )+ )+ add(+ SettingsItem(+ title = "Terms of Service",iconVector = Icons.Outlined.Description,- searchTerms = listOf(- EnsuLegalDocuments.PRIVACY_TITLE,- EnsuLegalDocuments.ENTE_TERMS_TITLE,- "Terms of Service"- ),- onClick = onOpenTerms+ onClick = { context.openExternalLink("https://ente.com/terms") }))}@@ -113,8 +114,7 @@ fun SettingsScreen(val q = query.trim().lowercase()if (q.isEmpty()) return@remember allItemsallItems.filter { item ->- item.title.lowercase().contains(q) ||- item.searchTerms.any { it.lowercase().contains(q) }+ item.title.lowercase().contains(q)}}@@ -206,17 +206,16 @@ fun SettingsScreen(}}-internal data class SettingsItem(+private data class SettingsItem(val title: String,val iconRes: Int? = null,val iconVector: ImageVector? = null,- val searchTerms: List<String> = emptyList(),val onClick: () -> Unit,val isDestructive: Boolean = false)@Composable-internal fun SettingsRow(item: SettingsItem) {+private fun SettingsRow(item: SettingsItem) {val iconAndTextColor = if (item.isDestructive) EnsuColor.error else EnsuColor.textPrimary()Row(diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex b9c89b6633..78e99fb3aa 100644--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -1023,6 +1023,8 @@ final class ChatViewModel: ObservableObject {_ = await priorGeneration?.result_ = await priorSummary?.resultguard !Task.isCancelled, activeGenerationId == generationId else { return }+ await knowledgeStore.bootstrap()+ guard !Task.isCancelled, activeGenerationId == generationId else { return }var embeddingAssetInvalid = falsevar knowledgeHits: [KnowledgeSearchHit] = []let enabledDatasets = knowledgeStore.enabledReadyDatasets@@ -1148,8 +1150,7 @@ final class ChatViewModel: ObservableObject {systemPrompt: candidate)}- let useKnowledge = knowledgeContext != nil &&- knowledgeHistorySelection?.wasTrimmed == false+ let useKnowledge = knowledgeHistorySelection?.wasTrimmed == falselet historySelection = useKnowledge ?(knowledgeHistorySelection ?? normalHistorySelection) : normalHistorySelectionvar activeCitations = useKnowledge ? (knowledgeContext?.citations ?? []) : []diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swiftindex 6ca7d8e49d..9e0fb2d391 100644--- a/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift@@ -340,13 +340,6 @@ private struct KnowledgeSourcesSheet: View {.font(EnsuTypography.large).foregroundStyle(EnsuColor.textPrimary)- if let section = citation.section?.trimmingCharacters(in: .whitespacesAndNewlines),- !section.isEmpty {- Text(section)- .font(EnsuTypography.small)- .foregroundStyle(EnsuColor.textMuted)- }-Divider()Text("Attribution")diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swiftindex ebf4014fd4..2ff621ba49 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift@@ -13,6 +13,7 @@ private struct KnowledgePackValidationError: LocalizedError {actor KnowledgeProvider {private final class Mutation: @unchecked Sendable {+ let cancellation: CancellationTokenlet task: Task<Void, Error>init(@@ -20,20 +21,23 @@ actor KnowledgeProvider {dataset: KnowledgeDatasetConfig,onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void) {+ let cancellation = CancellationToken()+ self.cancellation = cancellationtask = Task.detached(priority: .utility) {- let callback = KnowledgeDownloadCallbackSink(- onProgress: onProgress,- isCancelled: { Task.isCancelled }- )+ let callback = KnowledgeDownloadCallbackSink(onProgress: onProgress)try downloadKnowledgePack(packRoot: packRoot.path,- expectedDataset: dataset,- callback: callback+ stableId: dataset.stableId,+ callback: callback,+ cancellation: cancellation)}}- func cancel() { task.cancel() }+ func cancel() {+ cancellation.cancel()+ task.cancel()+ }}private struct OpenIndex {@@ -43,6 +47,7 @@ actor KnowledgeProvider {private let root: URLprivate var mutations: [String: Mutation] = [:]+ private var lifecycleGates: [String: AsyncSerialGate] = [:]private var indexes: [String: OpenIndex] = [:]private let indexGate = AsyncSerialGate()@@ -59,8 +64,14 @@ actor KnowledgeProvider {}func reconcile(dataset: KnowledgeDatasetConfig) async throws -> KnowledgeReconciliation {- try await withIndexGate {- try await reconcileAndActivateLocked(dataset: dataset)+ let gate = lifecycleGate(dataset.stableId)+ return try await gate.withLock {+ if let mutation = mutations[dataset.stableId] {+ _ = try? await mutation.task.value+ }+ return try await withIndexGate {+ try await reconcileAndActivateLocked(dataset: dataset)+ }}}@@ -68,17 +79,18 @@ actor KnowledgeProvider {dataset: KnowledgeDatasetConfig,onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void) async throws -> KnowledgeReconciliation {- let mutation: Mutation- if let current = mutations[dataset.stableId] {- mutation = current- } else {+ let gate = lifecycleGate(dataset.stableId)+ let mutation = try await gate.withLock {+ if let current = mutations[dataset.stableId] {+ return current+ }let created = Mutation(packRoot: packRoot(dataset),dataset: dataset,onProgress: onProgress)mutations[dataset.stableId] = created- mutation = created+ return created}do {@@ -193,7 +205,7 @@ actor KnowledgeProvider {}return try reconcileKnowledgePack(packRoot: packRoot.path,- expectedDataset: dataset+ stableId: dataset.stableId)}.valuetry activate(result: result, dataset: dataset)@@ -201,7 +213,7 @@ actor KnowledgeProvider {_ = try? await Task.detached(priority: .utility) {try cleanupObsoleteKnowledgePackRevisions(packRoot: packRoot.path,- expectedDataset: dataset,+ stableId: dataset.stableId,activeIdentity: activeIdentity)}.value@@ -220,7 +232,7 @@ actor KnowledgeProvider {guard indexes[dataset.stableId]?.directory != directory else { return }let replacement = try RetrievalIndex.open(directory: directory,- expectedDataset: dataset+ stableId: dataset.stableId)indexes[dataset.stableId] = OpenIndex(directory: directory, index: replacement)}@@ -229,6 +241,15 @@ actor KnowledgeProvider {try await indexGate.withLock(operation)}+ private func lifecycleGate(_ stableId: String) -> AsyncSerialGate {+ if let gate = lifecycleGates[stableId] {+ return gate+ }+ let gate = AsyncSerialGate()+ lifecycleGates[stableId] = gate+ return gate+ }+private func packRoot(_ dataset: KnowledgeDatasetConfig) -> URL {root.appendingPathComponent(dataset.stableId, isDirectory: true)}@@ -236,21 +257,12 @@ actor KnowledgeProvider {private final class KnowledgeDownloadCallbackSink: KnowledgeDownloadCallback, @unchecked Sendable {private let onProgressHandler: @Sendable (KnowledgeDownloadProgress) -> Void- private let isCancelledHandler: @Sendable () -> Bool- init(- onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void,- isCancelled: @escaping @Sendable () -> Bool- ) {+ init(onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void) {onProgressHandler = onProgress- isCancelledHandler = isCancelled}func onProgress(progress: KnowledgeDownloadProgress) {onProgressHandler(progress)}-- func isCancelled() -> Bool {- isCancelledHandler()- }}diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swiftindex aad3cb4080..a3cf381f22 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift@@ -8,6 +8,7 @@ final class KnowledgeStore: ObservableObject {private let provider: KnowledgeProviderprivate let preferences: KnowledgePreferences+ private var bootstrapTask: Task<Void, Never>?private var mutationTasks: [String: Task<Void, Never>] = [:]init(@@ -31,21 +32,50 @@ final class KnowledgeStore: ObservableObject {}func bootstrap() async {+ if let bootstrapTask {+ await bootstrapTask.value+ return+ }++ let task = Task { @MainActor [weak self] in+ guard let self else { return }+ await bootstrapEnabledDatasets()+ }+ bootstrapTask = task+ await task.value+ }++ private func bootstrapEnabledDatasets() async {let requestedEnabled = preferences.enabledDatasetIds- for dataset in packs.map(\.config) {- do {- let result = try await provider.reconcile(dataset: dataset)- applyReconciliation(- result,- stableId: dataset.stableId,- enabled: requestedEnabled.contains(dataset.stableId)- )- } catch {- updatePack(dataset.stableId) { pack in- pack.status = .download- pack.enabled = false- pack.errorMessage = userFacingKnowledgeError(error)- }+ let datasets = packs.map(\.config)+ for dataset in datasets where requestedEnabled.contains(dataset.stableId) {+ await reconcileForBootstrap(dataset, enabled: true)+ }++ Task { @MainActor [weak self] in+ guard let self else { return }+ for dataset in datasets where !requestedEnabled.contains(dataset.stableId) {+ await reconcileForBootstrap(dataset, enabled: false)+ }+ }+ }++ private func reconcileForBootstrap(+ _ dataset: KnowledgeDatasetConfig,+ enabled: Bool+ ) async {+ do {+ let result = try await provider.reconcile(dataset: dataset)+ applyReconciliation(+ result,+ stableId: dataset.stableId,+ enabled: enabled+ )+ } catch {+ updatePack(dataset.stableId) { pack in+ pack.status = .download+ pack.enabled = false+ pack.errorMessage = userFacingKnowledgeError(error)}}}diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swiftindex 621f25ee31..e9feeae763 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift@@ -146,8 +146,7 @@ final class LlmProvider {func missingModelDownloadSize(_ selection: LlmModelSelection) async -> Int64? {var total: Int64 = 0- let chatMissing = !downloader.isDownloaded(selection.modelTarget)- if chatMissing {+ if !downloader.isDownloaded(selection.modelTarget) {guard let chatSize = await downloader.estimateDownloadSize(selection.modelTarget) else {return nil}@@ -172,14 +171,13 @@ final class LlmProvider {}let missingTargets: [ModelTarget] = try await modelLoadGate.withLock {- let chatReady = downloader.isDownloaded(selection.modelTarget)let embeddingReady = isEmbeddingModelReady()if !embeddingReady {_ = downloader.removeDownloaded(embeddingDownloadTarget)}return [- chatReady ? nil : selection.modelTarget,+ downloader.isDownloaded(selection.modelTarget) ? nil : selection.modelTarget,embeddingReady ? nil : embeddingDownloadTarget,].compactMap { $0 }}@@ -393,15 +391,7 @@ final class LlmProvider {}let threadCount = max(1, ProcessInfo.processInfo.activeProcessorCount - 1)embeddingContext = try model.newEmbeddingContext(- params: LlmEmbeddingContextParams(- contextSize: knowledgeEmbedding.contextSize,- nThreads: Int32(threadCount),- batchSize: knowledgeEmbedding.batchSize,- microBatchSize: knowledgeEmbedding.microBatchSize,- sourceDim: knowledgeEmbedding.sourceDim,- dim: knowledgeEmbedding.dim,- queryPrompt: knowledgeEmbedding.queryPrompt- )+ nThreads: Int32(threadCount))guard let context = embeddingContext else {throw EmbeddingAssetInvalidError()diff --git a/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swift b/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swiftindex 51da91ed1b..588173833f 100644--- a/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swift@@ -101,7 +101,7 @@ struct KnowledgeSettingsView: View {if let error = pack.errorMessage {Text(error).font(EnsuTypography.small)- .foregroundStyle(.red)+ .foregroundStyle(EnsuColor.error)}}.padding(EnsuSpacing.lg)diff --git a/mobile/native/apple/apps/ensu/Ensu/Settings/LegalSettingsViews.swift b/mobile/native/apple/apps/ensu/Ensu/Settings/LegalSettingsViews.swiftdeleted file mode 100644index 52a0cf17ab..0000000000--- a/mobile/native/apple/apps/ensu/Ensu/Settings/LegalSettingsViews.swift+++ /dev/null@@ -1,44 +0,0 @@-import SwiftUI--enum EnsuLegalDocuments {- static let privacyTitle = "Privacy Policy"- static let privacyURL = URL(string: "https://ente.com/privacy/")!- static let enteTermsTitle = "Ente Terms and Conditions"- static let enteTermsURL = URL(string: "https://ente.com/terms/")!-- static let searchTerms = [- privacyTitle,- enteTermsTitle,- "Terms of Service",- ]-}--struct TermsAndConditionsView: View {- var body: some View {- ScrollView {- VStack(alignment: .leading, spacing: EnsuSpacing.lg) {- Link(destination: EnsuLegalDocuments.privacyURL) {- SettingsCard(- title: EnsuLegalDocuments.privacyTitle,- iconName: "ViewIcon",- showsChevron: true- )- }- .buttonStyle(.plain)-- Link(destination: EnsuLegalDocuments.enteTermsURL) {- SettingsCard(- title: EnsuLegalDocuments.enteTermsTitle,- iconName: "DescriptionIcon",- showsChevron: true- )- }- .buttonStyle(.plain)- }- .padding(EnsuSpacing.lg)- }- .background(EnsuColor.backgroundBase)- .navigationTitle("Terms and Conditions")- .navigationBarTitleDisplayMode(.inline)- }-}diff --git a/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swift b/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swiftindex 75baae4018..f905623047 100644--- a/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swift@@ -21,7 +21,7 @@ struct SettingsView: View {VStack(alignment: .leading, spacing: EnsuSpacing.lg) {if let aboutItem = filteredAboutItem {Button(action: aboutItem.action) {- SettingsCard(title: aboutItem.title, iconName: aboutItem.iconName, showsChevron: true)+ settingsCard(title: aboutItem.title, iconName: aboutItem.iconName, showsChevron: true)}.buttonStyle(.plain)}@@ -30,23 +30,21 @@ struct SettingsView: View {NavigationLink {item.destination} label: {- SettingsCard(title: item.title, iconName: item.iconName, showsChevron: true)+ settingsCard(title: item.title, iconName: item.iconName, showsChevron: true)}.buttonStyle(.plain)}if shouldShowSignInRow {Button(action: onSignIn) {- SettingsCard(title: signInTitle, iconName: "Upload01Icon", showsChevron: true)+ settingsCard(title: signInTitle, iconName: "Upload01Icon", showsChevron: true)}.buttonStyle(.plain)}- ForEach(filteredTermsItems) { item in- NavigationLink {- item.destination- } label: {- SettingsCard(title: item.title, iconName: item.iconName, showsChevron: true)+ ForEach(filteredLegalLinkItems) { item in+ Button(action: item.action) {+ settingsCard(title: item.title, iconName: item.iconName, showsChevron: true)}.buttonStyle(.plain)}@@ -61,7 +59,7 @@ struct SettingsView: View {NavigationLink {item.destination} label: {- SettingsCard(title: item.title, iconName: item.iconName, showsChevron: true)+ settingsCard(title: item.title, iconName: item.iconName, showsChevron: true)}.buttonStyle(.plain)}@@ -110,15 +108,6 @@ struct SettingsView: View {}}- private var filteredTermsItems: [SettingsItem] {- guard !trimmedQuery.isEmpty else { return termsItems }- let q = trimmedQuery.lowercased()- return termsItems.filter { item in- item.title.lowercased().contains(q) ||- item.searchTerms.contains { $0.lowercased().contains(q) }- }- }-private var filteredAdvancedItems: [SettingsItem] {guard isAdvancedUnlocked else { return [] }guard !trimmedQuery.isEmpty else { return advancedItems }@@ -136,12 +125,35 @@ struct SettingsView: View {)}+ private var legalLinkItems: [SettingsActionItem] {+ [+ SettingsActionItem(+ title: "Privacy Policy",+ iconName: "ViewIcon",+ action: { openExternalLink("https://ente.com/privacy") }+ ),+ SettingsActionItem(+ title: "Terms of Service",+ iconName: "DescriptionIcon",+ action: { openExternalLink("https://ente.com/terms") }+ )+ ]+ }+private var filteredAboutItem: SettingsActionItem? {guard !trimmedQuery.isEmpty else { return aboutItem }let q = trimmedQuery.lowercased()return aboutItem.title.lowercased().contains(q) ? aboutItem : nil}+ private var filteredLegalLinkItems: [SettingsActionItem] {+ guard !trimmedQuery.isEmpty else { return legalLinkItems }+ let q = trimmedQuery.lowercased()+ return legalLinkItems.filter { item in+ item.title.lowercased().contains(q)+ }+ }+private var shouldShowSignInRow: Bool {guard !trimmedQuery.isEmpty else { return true }let q = trimmedQuery.lowercased()@@ -165,17 +177,6 @@ struct SettingsView: View {]}- private var termsItems: [SettingsItem] {- [- SettingsItem(- title: "Terms and Conditions",- iconName: "DescriptionIcon",- destination: AnyView(TermsAndConditionsView()),- searchTerms: EnsuLegalDocuments.searchTerms- )- ]- }-private var advancedItems: [SettingsItem] {[SettingsItem(@@ -228,14 +229,8 @@ struct SettingsView: View {guard let url = URL(string: urlString) else { return }openURL(url)}-}--struct SettingsCard: View {- let title: String- let iconName: String- let showsChevron: Bool- var body: some View {+ private func settingsCard(title: String, iconName: String, showsChevron: Bool) -> some View {HStack(spacing: EnsuSpacing.md) {Image(iconName).resizable()@@ -397,14 +392,6 @@ private struct SettingsItem: Identifiable {let title: Stringlet iconName: Stringlet destination: AnyView- let searchTerms: [String]-- init(title: String, iconName: String, destination: AnyView, searchTerms: [String] = []) {- self.title = title- self.iconName = iconName- self.destination = destination- self.searchTerms = searchTerms- }}private struct SettingsActionItem: Identifiable {diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex e0e9fe30af..429f215f0b 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2394,7 +2394,6 @@ dependencies = ["thiserror 2.0.18","tokio","transcribe-rs",- "unicode-segmentation","url","uuid","zeroize",diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 6938a675f4..d59402326d 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -37,14 +37,11 @@ ente-rs = { path = "apps/cli" }ente-space = { path = "space" }ente-test-support = { path = "crates/test-support" }ort = { git = "https://github.com/pykeio/ort.git", rev = "31c8f1443bb579bfddb82705288347697e468fb3", default-features = false }-memmap2 = "0.9.9"serde = { version = "1.0.228", features = ["derive"] }serde_json = "1.0.149"thiserror = "2.0.18"uniffi = { version = "0.31.0" }-url = "2.5.8"wasm-bindgen = "0.2.125"-zstd = "0.13.3"# Keep transitive ORT users (notably transcribe-rs) on the same revision as direct ORT dependencies.[patch.crates-io]diff --git a/rust/bindings/uniffi/ensu/src/config.rs b/rust/bindings/uniffi/ensu/src/config.rsindex 739bca9154..c17d28aa78 100644--- a/rust/bindings/uniffi/ensu/src/config.rs+++ b/rust/bindings/uniffi/ensu/src/config.rs@@ -12,15 +12,7 @@ pub struct ConfigModelPreset {#[derive(Debug, Clone, uniffi::Record)]pub struct KnowledgeEmbeddingConfig {- pub model_url: String,- pub model_sha256: String,pub target_id: String,- pub source_dim: u32,- pub dim: u32,- pub query_prompt: String,- pub context_size: u32,- pub batch_size: u32,- pub micro_batch_size: u32,pub max_hits: u32,pub max_context_utf8_bytes: u32,}@@ -31,7 +23,6 @@ pub struct AttributionConfig {pub license_label: String,pub license_url: String,pub public_pack_url: String,- pub build_provenance: String,pub modification_notice: String,}@@ -40,11 +31,7 @@ pub struct KnowledgeDatasetConfig {pub stable_id: String,pub label: String,pub current_download_identity: String,- pub artifact_base_url: String,- pub artifact_sha256: Vec<String>,pub download_size_bytes: i64,- pub max_chars: u32,- pub source_url_template: String,pub relevance_threshold: f32,pub attribution: AttributionConfig,}@@ -81,15 +68,7 @@ impl From<config::ModelPreset> for ConfigModelPreset {impl From<config::KnowledgeEmbeddingConfig> for KnowledgeEmbeddingConfig {fn from(value: config::KnowledgeEmbeddingConfig) -> Self {Self {- model_url: value.model_url,- model_sha256: value.model_sha256,target_id: value.target_id,- source_dim: value.source_dim,- dim: value.dim,- query_prompt: value.query_prompt,- context_size: value.context_size,- batch_size: value.batch_size,- micro_batch_size: value.micro_batch_size,max_hits: value.max_hits,max_context_utf8_bytes: value.max_context_utf8_bytes,}@@ -103,20 +82,6 @@ impl From<config::AttributionConfig> for AttributionConfig {license_label: value.license_label,license_url: value.license_url,public_pack_url: value.public_pack_url,- build_provenance: value.build_provenance,- modification_notice: value.modification_notice,- }- }-}--impl From<AttributionConfig> for config::AttributionConfig {- fn from(value: AttributionConfig) -> Self {- Self {- credit: value.credit,- license_label: value.license_label,- license_url: value.license_url,- public_pack_url: value.public_pack_url,- build_provenance: value.build_provenance,modification_notice: value.modification_notice,}}@@ -128,28 +93,7 @@ impl From<config::KnowledgeDatasetConfig> for KnowledgeDatasetConfig {stable_id: value.stable_id,label: value.label,current_download_identity: value.current_download_identity,- artifact_base_url: value.artifact_base_url,- artifact_sha256: value.artifact_sha256,- download_size_bytes: value.download_size_bytes,- max_chars: value.max_chars,- source_url_template: value.source_url_template,- relevance_threshold: value.relevance_threshold,- attribution: value.attribution.into(),- }- }-}--impl From<KnowledgeDatasetConfig> for config::KnowledgeDatasetConfig {- fn from(value: KnowledgeDatasetConfig) -> Self {- Self {- stable_id: value.stable_id,- label: value.label,- current_download_identity: value.current_download_identity,- artifact_base_url: value.artifact_base_url,- artifact_sha256: value.artifact_sha256,download_size_bytes: value.download_size_bytes,- max_chars: value.max_chars,- source_url_template: value.source_url_template,relevance_threshold: value.relevance_threshold,attribution: value.attribution.into(),}diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rsindex 9db3728cbd..7f5ab0703f 100644--- a/rust/bindings/uniffi/ensu/src/llm.rs+++ b/rust/bindings/uniffi/ensu/src/llm.rs@@ -67,17 +67,6 @@ pub struct LlmContextParams {pub n_batch: Option<i32>,}-#[derive(Debug, Clone, uniffi::Record)]-pub struct LlmEmbeddingContextParams {- pub context_size: u32,- pub n_threads: Option<i32>,- pub batch_size: u32,- pub micro_batch_size: u32,- pub source_dim: u32,- pub dim: u32,- pub query_prompt: String,-}-#[derive(Debug, Clone, uniffi::Record)]pub struct LlmChatMessage {pub role: String,@@ -144,10 +133,10 @@ impl LlmModel {pub fn new_embedding_context(&self,- params: LlmEmbeddingContextParams,+ n_threads: Option<i32>,) -> Result<Arc<LlmContext>, LlmError> {- let handle =- llm::Context::new_embedding(&self.handle, params.into()).map_err(LlmError::from)?;+ let handle = llm::Context::new_knowledge_embedding(&self.handle, n_threads)+ .map_err(LlmError::from)?;Ok(Arc::new(LlmContext { handle }))}}@@ -212,20 +201,6 @@ impl From<LlmContextParams> for llm::ContextParams {}}-impl From<LlmEmbeddingContextParams> for llm::EmbeddingContextParams {- fn from(value: LlmEmbeddingContextParams) -> Self {- Self {- context_size: value.context_size,- n_threads: value.n_threads,- batch_size: value.batch_size,- micro_batch_size: value.micro_batch_size,- source_dim: value.source_dim,- dim: value.dim,- query_prompt: value.query_prompt,- }- }-}-impl From<LlmChatMessage> for llm::ChatMessage {fn from(value: LlmChatMessage) -> Self {Self {diff --git a/rust/bindings/uniffi/ensu/src/model.rs b/rust/bindings/uniffi/ensu/src/model.rsindex bd32856f46..2e3809ec3f 100644--- a/rust/bindings/uniffi/ensu/src/model.rs+++ b/rust/bindings/uniffi/ensu/src/model.rs@@ -211,7 +211,7 @@ impl ModelDownloadCore {#[derive(Default, uniffi::Object)]pub struct CancellationToken {- inner: ente_model_download::download::CancellationToken,+ pub(crate) inner: ente_model_download::download::CancellationToken,}#[uniffi::export]diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rsindex b03d839da5..d6d997c7eb 100644--- a/rust/bindings/uniffi/ensu/src/retrieval.rs+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs@@ -3,8 +3,8 @@ use std::sync::Arc;use ente_ensu::retrieval as core;use thiserror::Error;-use crate::config::KnowledgeDatasetConfig;use crate::download::DownloadError;+use crate::model::CancellationToken;#[derive(Debug, Error, uniffi::Error)]pub enum KnowledgeRetrievalError {@@ -36,6 +36,16 @@ impl From<core::RetrievalError> for KnowledgeRetrievalError {}}+fn knowledge_dataset(+ stable_id: &str,+) -> Result<ente_ensu::config::KnowledgeDatasetConfig, KnowledgeRetrievalError> {+ ente_ensu::config::knowledge_dataset(stable_id).ok_or_else(|| {+ KnowledgeRetrievalError::InvalidInput {+ detail: format!("unknown knowledge dataset ID: {stable_id}"),+ }+ })+}+#[derive(Debug, Error, uniffi::Error)]pub enum KnowledgeDownloadError {#[error("knowledge download failed")]@@ -100,9 +110,9 @@ impl RetrievalIndex {#[uniffi::constructor]pub fn open(directory: String,- expected_dataset: KnowledgeDatasetConfig,+ stable_id: String,) -> Result<Arc<Self>, KnowledgeRetrievalError> {- let inner = core::RetrievalIndex::open(directory, &expected_dataset.into())?;+ let inner = core::RetrievalIndex::open(directory, &knowledge_dataset(&stable_id)?)?;Ok(Arc::new(Self { inner }))}@@ -156,9 +166,9 @@ impl From<core::KnowledgeReconciliation> for KnowledgeReconciliation {#[uniffi::export]pub fn reconcile_knowledge_pack(pack_root: String,- expected_dataset: KnowledgeDatasetConfig,+ stable_id: String,) -> Result<KnowledgeReconciliation, KnowledgeRetrievalError> {- core::reconcile_knowledge_pack(pack_root, &expected_dataset.into())+ core::reconcile_knowledge_pack(pack_root, &knowledge_dataset(&stable_id)?).map(Into::into).map_err(Into::into)}@@ -166,12 +176,12 @@ pub fn reconcile_knowledge_pack(#[uniffi::export]pub fn cleanup_obsolete_knowledge_pack_revisions(pack_root: String,- expected_dataset: KnowledgeDatasetConfig,+ stable_id: String,active_identity: String,) -> Result<(), KnowledgeRetrievalError> {core::cleanup_obsolete_knowledge_pack_revisions(pack_root,- &expected_dataset.into(),+ &knowledge_dataset(&stable_id)?,&active_identity,).map_err(Into::into)@@ -183,7 +193,6 @@ pub struct SourceCitation {pub dataset_label: String,pub credit: String,pub title: String,- pub section: Option<String>,pub source_url: String,pub license_label: String,pub license_url: String,@@ -196,7 +205,6 @@ impl From<SourceCitation> for core::SourceCitation {dataset_label: value.dataset_label,credit: value.credit,title: value.title,- section: value.section,source_url: value.source_url,license_label: value.license_label,license_url: value.license_url,@@ -211,7 +219,6 @@ impl From<core::SourceCitation> for SourceCitation {dataset_label: value.dataset_label,credit: value.credit,title: value.title,- section: value.section,source_url: value.source_url,license_label: value.license_label,license_url: value.license_url,@@ -300,57 +307,29 @@ impl From<ente_model_download::download::Progress> for KnowledgeDownloadProgress#[uniffi::export(callback_interface)]pub trait KnowledgeDownloadCallback: Send + Sync {fn on_progress(&self, progress: KnowledgeDownloadProgress);- fn is_cancelled(&self) -> bool;}#[uniffi::export]pub fn download_knowledge_pack(pack_root: String,- expected_dataset: KnowledgeDatasetConfig,+ stable_id: String,callback: Box<dyn KnowledgeDownloadCallback>,+ cancellation: Arc<CancellationToken>,) -> Result<(), KnowledgeDownloadError> {- let callback: Arc<dyn KnowledgeDownloadCallback> = Arc::from(callback);- let progress_callback = Arc::clone(&callback);- let cancel_callback = Arc::clone(&callback);+ let expected_dataset = ente_ensu::config::knowledge_dataset(&stable_id).ok_or_else(|| {+ KnowledgeDownloadError::Download {+ error: DownloadError::InvalidTarget {+ message: format!("unknown knowledge dataset ID: {stable_id}"),+ },+ }+ })?;core::download_knowledge_pack(pack_root,- &expected_dataset.into(),- move |progress| progress_callback.on_progress(progress.into()),- move || cancel_callback.is_cancelled(),+ &expected_dataset,+ move |progress| callback.on_progress(progress.into()),+ cancellation.inner.clone(),).map_err(|error| KnowledgeDownloadError::Download {error: error.into(),})}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn builds_a_prompt_context_through_the_binding_types() {- let context = build_knowledge_prompt_context(- vec![KnowledgePromptHit {- dataset_id: "simplewiki".to_string(),- hit: RetrievalHit {- score: 0.9,- text: "A useful passage".to_string(),- title: "Example".to_string(),- section: None,- source_url: "https://simple.wikipedia.org/wiki/Example".to_string(),- },- }],- 6_000,- )- .unwrap()- .unwrap();-- assert!(- context- .text- .contains("# Example (Simple English Wikipedia)")- );- assert_eq!(context.citations.len(), 1);- assert_eq!(context.citations[0].dataset_id, "simplewiki");- }-}diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex 294ae4e0dd..5d04f6a385 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -6,7 +6,7 @@ edition = "2024"ente-core.workspace = trueente-image.workspace = trueente-model-download.workspace = true-memmap2.workspace = true+memmap2 = "0.9.9"parking_lot = "0.12"regex = { version = "1", optional = true }rubato = { version = "0.16", optional = true }@@ -18,11 +18,10 @@ sha2 = "0.11.0"thiserror.workspace = truetokio = { version = "1", features = ["rt", "time"] }transcribe-rs = { version = "0.3.11", default-features = false, features = ["onnx", "vad-silero"], optional = true }-unicode-segmentation = "1.13.2"-url.workspace = true+url = "2.5.8"uuid = { version = "1", features = ["v4", "rng-getrandom"] }zeroize = { version = "1.8", features = ["derive"] }-zstd.workspace = true+zstd = "0.13.3"[dev-dependencies]tempfile = "3.27.0"diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rsindex 10a75fdcb0..88a4d51582 100644--- a/rust/crates/ensu/src/config.rs+++ b/rust/crates/ensu/src/config.rs@@ -1,11 +1,12 @@mod knowledge;+pub use knowledge::knowledge_dataset;pub use knowledge::{AttributionConfig, KnowledgeDatasetConfig, KnowledgeEmbeddingConfig};pub(crate) use knowledge::{KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL, KNOWLEDGE_LICENSE_URL,KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE, KNOWLEDGE_VECTORS_FILE,- KnowledgeConfigError, is_path_safe_component, knowledge_artifact_urls, knowledge_datasets,- knowledge_index_contract, validate_knowledge_datasets, validate_knowledge_embedding,+ is_path_safe_component, knowledge_artifact_urls, knowledge_datasets,+ knowledge_embedding_config, knowledge_index_contract,};#[derive(Debug, Clone)]@@ -165,7 +166,7 @@ pub fn defaults() -> Defaults {.expect("preset id is in the catalog").clone()};- let defaults = Defaults {+ Defaults {mobile_system_prompt_body: MOBILE_SYSTEM_PROMPT_BODY.to_string(),desktop_system_prompt_body: DESKTOP_SYSTEM_PROMPT_BODY.to_string(),system_prompt_date_placeholder: SYSTEM_PROMPT_DATE_PLACEHOLDER.to_string(),@@ -187,26 +188,7 @@ pub fn defaults() -> Defaults {voice_activity_model: silero_vad_v4(),knowledge_embedding: knowledge::knowledge_embedding_config(),knowledge_datasets: knowledge::knowledge_datasets(),- };-- validate_defaults(&defaults).expect("bundled Ensu defaults must be valid");- defaults-}--fn validate_defaults(defaults: &Defaults) -> Result<(), KnowledgeConfigError> {- validate_knowledge_datasets(&defaults.knowledge_datasets)?;-- let reserved_model_ids = std::iter::once(&defaults.mobile_default_model)- .chain(defaults.mobile_model_presets.iter())- .chain(std::iter::once(&defaults.desktop_default_model))- .chain(defaults.desktop_model_presets.iter())- .chain([- &defaults.transcription_model,- &defaults.voice_activity_model,- ])- .map(|preset| preset.id.clone())- .collect::<Vec<_>>();- validate_knowledge_embedding(&defaults.knowledge_embedding, &reserved_model_ids)+ }}#[cfg(test)]diff --git a/rust/crates/ensu/src/config/knowledge.rs b/rust/crates/ensu/src/config/knowledge.rsindex c5a90f44fc..f0c44d88da 100644--- a/rust/crates/ensu/src/config/knowledge.rs+++ b/rust/crates/ensu/src/config/knowledge.rs@@ -1,10 +1,3 @@-mod validation;--use thiserror::Error;--pub(crate) use validation::is_path_safe_component;-pub(crate) use validation::{validate_knowledge_datasets, validate_knowledge_embedding};-pub(crate) const KNOWLEDGE_LICENSE_LABEL: &str = "CC BY-SA 4.0";pub(crate) const KNOWLEDGE_LICENSE_URL: &str = "https://creativecommons.org/licenses/by-sa/4.0/";@@ -41,7 +34,6 @@ pub struct KnowledgeEmbeddingConfig {pub target_id: String,pub source_dim: u32,pub dim: u32,- pub matryoshka: bool,pub query_prompt: String,pub context_size: u32,pub batch_size: u32,@@ -56,7 +48,6 @@ pub struct AttributionConfig {pub license_label: String,pub license_url: String,pub public_pack_url: String,- pub build_provenance: String,pub modification_notice: String,}@@ -88,16 +79,6 @@ pub struct KnowledgeIndexContract {pub meta_rows_per_block: u32,}-#[derive(Debug, Clone, PartialEq, Eq, Error)]-#[error("{0}")]-pub struct KnowledgeConfigError(String);--impl KnowledgeConfigError {- fn invalid(field: &str, reason: impl AsRef<str>) -> Self {- Self(format!("invalid {field}: {}", reason.as_ref()))- }-}-pub(crate) fn knowledge_index_contract() -> KnowledgeIndexContract {KnowledgeIndexContract {model: EMBEDDING_MODEL_IDENTITY.to_owned(),@@ -114,15 +95,13 @@ pub(crate) fn knowledge_index_contract() -> KnowledgeIndexContract {}pub(crate) fn knowledge_embedding_config() -> KnowledgeEmbeddingConfig {- let contract = knowledge_index_contract();KnowledgeEmbeddingConfig {model_url: EMBEDDING_MODEL_URL.to_owned(),model_sha256: EMBEDDING_MODEL_SHA256.to_owned(),target_id: EMBEDDING_TARGET_ID.to_owned(),- source_dim: contract.source_dim,- dim: contract.dim,- matryoshka: contract.matryoshka,- query_prompt: contract.query_prompt,+ source_dim: SOURCE_DIM,+ dim: RETRIEVAL_DIM,+ query_prompt: QUERY_PROMPT.to_owned(),context_size: EMBEDDING_CONTEXT_SIZE,batch_size: EMBEDDING_CONTEXT_SIZE,micro_batch_size: EMBEDDING_CONTEXT_SIZE,@@ -153,7 +132,6 @@ pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {license_label: KNOWLEDGE_LICENSE_LABEL.to_owned(),license_url: KNOWLEDGE_LICENSE_URL.to_owned(),public_pack_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/tree/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/simplewiki".to_owned(),- build_provenance: "Simple English Wikipedia dump dated 2026-07-02".to_owned(),modification_notice: MODIFICATION_NOTICE.to_owned(),},},@@ -177,18 +155,30 @@ pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {license_label: KNOWLEDGE_LICENSE_LABEL.to_owned(),license_url: KNOWLEDGE_LICENSE_URL.to_owned(),public_pack_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/tree/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/wikibooks".to_owned(),- build_provenance: "English Wikibooks dump dated 2026-07-02".to_owned(),modification_notice: MODIFICATION_NOTICE.to_owned(),},},]}-pub(crate) fn knowledge_artifact_urls(- dataset: &KnowledgeDatasetConfig,-) -> Result<[String; 4], KnowledgeConfigError> {- validation::validate_artifact_base_url(&dataset.artifact_base_url, "artifact_base_url")?;- Ok(KNOWLEDGE_ARTIFACT_FILENAMES.map(|name| format!("{}{name}", dataset.artifact_base_url)))+pub fn knowledge_dataset(stable_id: &str) -> Option<KnowledgeDatasetConfig> {+ knowledge_datasets()+ .into_iter()+ .find(|dataset| dataset.stable_id == stable_id)+}++pub(crate) fn knowledge_artifact_urls(dataset: &KnowledgeDatasetConfig) -> [String; 4] {+ KNOWLEDGE_ARTIFACT_FILENAMES.map(|name| format!("{}{name}", dataset.artifact_base_url))+}++pub(crate) fn is_path_safe_component(value: &str) -> bool {+ !value.is_empty()+ && value.len() <= 255+ && value != "."+ && value != ".."+ && !value.chars().any(|character| {+ character.is_control() || character == '/' || character == '\\' || character == ':'+ })}#[cfg(test)]@@ -197,7 +187,7 @@ mod tests {#[test]fn artifact_urls_append_only_fixed_names() {- let urls = knowledge_artifact_urls(&knowledge_datasets()[0]).unwrap();+ let urls = knowledge_artifact_urls(&knowledge_datasets()[0]);assert_eq!(urls.each_ref().map(|url| url.rsplit('/').next().unwrap().to_owned()),@@ -211,9 +201,7 @@ mod tests {#[test]fn pinned_catalog_and_embedding_are_valid() {let datasets = knowledge_datasets();- validate_knowledge_datasets(&datasets).unwrap();let embedding = knowledge_embedding_config();- validate_knowledge_embedding(&embedding, &[]).unwrap();assert_eq!(datasets.len(), 2);assert_eq!(datasets[0].stable_id, "simplewiki");@@ -235,5 +223,10 @@ mod tests {assert_eq!(embedding.model_sha256.len(), 64);assert_eq!(knowledge_index_contract().source_dim, 768);assert_eq!(knowledge_index_contract().dim, 512);+ assert!(datasets.iter().all(|dataset| {+ is_path_safe_component(&dataset.stable_id)+ && is_path_safe_component(&dataset.current_download_identity)+ }));+ assert!(knowledge_dataset("missing").is_none());}}diff --git a/rust/crates/ensu/src/config/knowledge/validation.rs b/rust/crates/ensu/src/config/knowledge/validation.rsdeleted file mode 100644index d7851b40ea..0000000000--- a/rust/crates/ensu/src/config/knowledge/validation.rs+++ /dev/null@@ -1,443 +0,0 @@-use std::collections::HashSet;--use url::Url;--use super::{- EMBEDDING_CONTEXT_SIZE, KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL,- KNOWLEDGE_LICENSE_URL, KnowledgeConfigError, KnowledgeDatasetConfig, KnowledgeEmbeddingConfig,- MAX_CONTEXT_UTF8_BYTES, MAX_HITS, knowledge_index_contract,-};--const MAX_DATASETS: usize = 32;-const MAX_PATH_COMPONENT_BYTES: usize = 96;-const MAX_DISPLAY_BYTES: usize = 256;-const MAX_URL_BYTES: usize = 2_048;-const MAX_PROVENANCE_BYTES: usize = 4_096;--pub fn validate_knowledge_datasets(- datasets: &[KnowledgeDatasetConfig],-) -> Result<(), KnowledgeConfigError> {- if !(1..=MAX_DATASETS).contains(&datasets.len()) {- return Err(KnowledgeConfigError::invalid(- "knowledge_datasets",- format!("must contain between 1 and {MAX_DATASETS} records"),- ));- }-- let mut stable_ids = HashSet::with_capacity(datasets.len());- let mut current_identities = HashSet::with_capacity(datasets.len());- for (index, dataset) in datasets.iter().enumerate() {- let prefix = format!("knowledge_datasets[{index}]");- validate_path_component(&dataset.stable_id, &format!("{prefix}.stable_id"))?;- if !stable_ids.insert(dataset.stable_id.as_str()) {- return Err(KnowledgeConfigError::invalid(- &format!("{prefix}.stable_id"),- "must be unique",- ));- }-- validate_path_component(- &dataset.current_download_identity,- &format!("{prefix}.current_download_identity"),- )?;- if !current_identities.insert(dataset.current_download_identity.as_str()) {- return Err(KnowledgeConfigError::invalid(- &format!("{prefix}.current_download_identity"),- "must be unique",- ));- }-- validate_display_text(&dataset.label, &format!("{prefix}.label"))?;- validate_artifact_base_url(- &dataset.artifact_base_url,- &format!("{prefix}.artifact_base_url"),- )?;- if dataset.artifact_sha256.len() != KNOWLEDGE_ARTIFACT_FILENAMES.len()- || dataset.artifact_sha256.iter().any(|sha256| {- sha256.len() != 64 || !sha256.chars().all(|ch| ch.is_ascii_hexdigit())- })- {- return Err(KnowledgeConfigError::invalid(- &format!("{prefix}.artifact_sha256"),- "must provide one SHA-256 digest per knowledge artifact",- ));- }- if dataset.download_size_bytes <= 0 {- return Err(KnowledgeConfigError::invalid(- &format!("{prefix}.download_size_bytes"),- "must be greater than zero",- ));- }- validate_source_url_template(- &dataset.source_url_template,- &format!("{prefix}.source_url_template"),- )?;- if !(1..=4_096).contains(&dataset.max_chars) {- return Err(KnowledgeConfigError::invalid(- &format!("{prefix}.max_chars"),- "must be between 1 and 4096",- ));- }- if !dataset.relevance_threshold.is_finite()- || !(0.0..=1.0).contains(&dataset.relevance_threshold)- {- return Err(KnowledgeConfigError::invalid(- &format!("{prefix}.relevance_threshold"),- "must be finite and between 0 and 1",- ));- }-- let attribution = &dataset.attribution;- validate_display_text(&attribution.credit, &format!("{prefix}.attribution.credit"))?;- validate_display_text(- &attribution.license_label,- &format!("{prefix}.attribution.license_label"),- )?;- validate_https_url(- &attribution.license_url,- &format!("{prefix}.attribution.license_url"),- )?;- if attribution.license_label != KNOWLEDGE_LICENSE_LABEL- || attribution.license_url != KNOWLEDGE_LICENSE_URL- {- return Err(KnowledgeConfigError::invalid(- &format!("{prefix}.attribution"),- "must use the v1 CC BY-SA 4.0 license",- ));- }- validate_https_url(- &attribution.public_pack_url,- &format!("{prefix}.attribution.public_pack_url"),- )?;- validate_provenance_text(- &attribution.build_provenance,- &format!("{prefix}.attribution.build_provenance"),- )?;- validate_provenance_text(- &attribution.modification_notice,- &format!("{prefix}.attribution.modification_notice"),- )?;- }-- Ok(())-}--pub fn validate_knowledge_embedding(- config: &KnowledgeEmbeddingConfig,- reserved_model_ids: &[String],-) -> Result<(), KnowledgeConfigError> {- validate_path_component(&config.target_id, "knowledge_embedding.target_id")?;- if config.target_id.starts_with("custom") {- return Err(KnowledgeConfigError::invalid(- "knowledge_embedding.target_id",- "must not start with reserved prefix 'custom'",- ));- }- if reserved_model_ids.iter().any(|id| id == &config.target_id) {- return Err(KnowledgeConfigError::invalid(- "knowledge_embedding.target_id",- "must not alias another model target",- ));- }- validate_embedding_model_url(&config.model_url, "knowledge_embedding.model_url")?;- if config.model_sha256.len() != 64- || !config.model_sha256.chars().all(|ch| ch.is_ascii_hexdigit())- {- return Err(KnowledgeConfigError::invalid(- "knowledge_embedding.model_sha256",- "must be a SHA-256 digest",- ));- }- let contract = knowledge_index_contract();- let exact_fields_match = config.source_dim == contract.source_dim- && config.dim == contract.dim- && config.matryoshka == contract.matryoshka- && config.query_prompt == contract.query_prompt- && config.context_size == EMBEDDING_CONTEXT_SIZE- && config.batch_size == EMBEDDING_CONTEXT_SIZE- && config.micro_batch_size == EMBEDDING_CONTEXT_SIZE- && config.max_hits == MAX_HITS- && config.max_context_utf8_bytes == MAX_CONTEXT_UTF8_BYTES;- if !exact_fields_match {- return Err(KnowledgeConfigError::invalid(- "knowledge_embedding",- "must match the v1 embedding and retrieval contract",- ));- }-- Ok(())-}--fn validate_path_component(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {- if is_path_safe_component(value) {- Ok(())- } else {- Err(KnowledgeConfigError::invalid(- field,- "must be a path-safe 1-96 byte ASCII component",- ))- }-}--pub(crate) fn is_path_safe_component(value: &str) -> bool {- let bytes = value.as_bytes();- let valid_edge = |byte: u8| byte.is_ascii_lowercase() || byte.is_ascii_digit();- let valid_body = |byte: u8| valid_edge(byte) || matches!(byte, b'.' | b'_' | b'-');- !bytes.is_empty()- && bytes.len() <= MAX_PATH_COMPONENT_BYTES- && valid_edge(bytes[0])- && valid_edge(bytes[bytes.len() - 1])- && bytes.iter().copied().all(valid_body)-}--fn validate_display_text(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {- validate_text(value, field, MAX_DISPLAY_BYTES)-}--fn validate_provenance_text(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {- validate_text(value, field, MAX_PROVENANCE_BYTES)-}--fn validate_text(value: &str, field: &str, max_bytes: usize) -> Result<(), KnowledgeConfigError> {- if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) {- Err(KnowledgeConfigError::invalid(- field,- format!("must be nonempty, control-free, and at most {max_bytes} UTF-8 bytes"),- ))- } else {- Ok(())- }-}--fn parse_https_url(value: &str, field: &str) -> Result<Url, KnowledgeConfigError> {- if value.len() > MAX_URL_BYTES || value.chars().any(char::is_control) {- return Err(KnowledgeConfigError::invalid(- field,- format!("must be control-free and at most {MAX_URL_BYTES} UTF-8 bytes"),- ));- }- let url = Url::parse(value)- .map_err(|error| KnowledgeConfigError::invalid(field, error.to_string()))?;- if url.scheme() != "https"- || url.host_str().is_none()- || !url.username().is_empty()- || url.password().is_some()- {- return Err(KnowledgeConfigError::invalid(- field,- "must be an HTTPS URL without credentials",- ));- }- Ok(url)-}--fn validate_https_url(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {- parse_https_url(value, field).map(|_| ())-}--fn validate_source_url_template(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {- if value.len() > MAX_URL_BYTES {- return Err(KnowledgeConfigError::invalid(- field,- format!("must be at most {MAX_URL_BYTES} UTF-8 bytes"),- ));- }- if value.match_indices("{title}").count() != 1 {- return Err(KnowledgeConfigError::invalid(- field,- "must contain exactly one {title} placeholder",- ));- }- let without_title = value.replacen("{title}", "", 1);- if without_title.contains(['{', '}']) {- return Err(KnowledgeConfigError::invalid(- field,- "must not contain another brace-delimited placeholder",- ));- }- parse_https_url(&value.replacen("{title}", "title", 1), field).map(|_| ())-}--pub(super) fn validate_artifact_base_url(- value: &str,- field: &str,-) -> Result<(), KnowledgeConfigError> {- let url = parse_https_url(value, field)?;- let segments = url- .path_segments()- .map(|segments| segments.collect::<Vec<_>>())- .ok_or_else(|| KnowledgeConfigError::invalid(field, "must have path segments"))?;- let pinned = url.host_str() == Some("huggingface.co")- && url.port().is_none()- && url.query().is_none()- && url.fragment().is_none()- && value.ends_with('/')- && segments.len() >= 8- && segments[..4] == ["datasets", "ente-ai", "ensu-knowledge-packs", "resolve"]- && is_commit_hash(segments[4]);- if pinned {- Ok(())- } else {- Err(KnowledgeConfigError::invalid(- field,- "must be an immutable Ente knowledge-pack resolver base URL",- ))- }-}--fn validate_embedding_model_url(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {- let url = parse_https_url(value, field)?;- let segments = url- .path_segments()- .map(|segments| segments.collect::<Vec<_>>())- .ok_or_else(|| KnowledgeConfigError::invalid(field, "must have path segments"))?;- let pinned = url.host_str() == Some("huggingface.co")- && url.port().is_none()- && url.query().is_none()- && url.fragment().is_none()- && segments.len() == 5- && segments[..3] == ["ente-ai", "embeddinggemma-300m-gguf", "resolve"]- && is_commit_hash(segments[3])- && !segments[4].is_empty();- if pinned {- Ok(())- } else {- Err(KnowledgeConfigError::invalid(- field,- "must be an immutable Ente EmbeddingGemma resolver URL",- ))- }-}--fn is_commit_hash(value: &str) -> bool {- value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit())-}--#[cfg(test)]-mod tests {- use super::super::{EMBEDDING_TARGET_ID, knowledge_datasets, knowledge_embedding_config};- use super::*;-- fn valid_datasets() -> Vec<KnowledgeDatasetConfig> {- knowledge_datasets()- }-- #[test]- fn rejects_empty_oversized_and_duplicate_catalogs() {- assert!(validate_knowledge_datasets(&[]).is_err());-- let mut too_many = Vec::new();- for index in 0..=MAX_DATASETS {- let mut dataset = valid_datasets().remove(0);- dataset.stable_id = format!("pack-{index}");- dataset.current_download_identity = format!("revision-{index}");- too_many.push(dataset);- }- assert!(validate_knowledge_datasets(&too_many).is_err());-- let mut duplicate = valid_datasets();- duplicate[1].stable_id = duplicate[0].stable_id.clone();- assert!(validate_knowledge_datasets(&duplicate).is_err());- duplicate[1].stable_id = "wikibooks".to_owned();- duplicate[1].current_download_identity = duplicate[0].current_download_identity.clone();- assert!(validate_knowledge_datasets(&duplicate).is_err());- }-- #[test]- fn rejects_path_unsafe_ids_and_boundary_overflow() {- for invalid in [- "",- ".",- "..",- "UPPER",- "-starts",- "ends-",- "has/slash",- " spaced ",- ] {- let mut datasets = valid_datasets();- datasets[0].stable_id = invalid.to_owned();- assert!(- validate_knowledge_datasets(&datasets).is_err(),- "accepted {invalid:?}"- );- }-- let mut datasets = valid_datasets();- datasets[0].stable_id = format!("a{}z", "b".repeat(MAX_PATH_COMPONENT_BYTES - 2));- validate_knowledge_datasets(&datasets).unwrap();- datasets[0].stable_id.push('z');- assert!(validate_knowledge_datasets(&datasets).is_err());- }-- #[test]- fn rejects_invalid_dataset_strings_numbers_and_urls() {- let mut datasets = valid_datasets();- datasets[0].label = "\u{0000}".to_owned();- assert!(validate_knowledge_datasets(&datasets).is_err());-- let mut datasets = valid_datasets();- datasets[0].attribution.build_provenance = "x".repeat(MAX_PROVENANCE_BYTES + 1);- assert!(validate_knowledge_datasets(&datasets).is_err());-- let mut datasets = valid_datasets();- datasets[0].max_chars = 0;- assert!(validate_knowledge_datasets(&datasets).is_err());- datasets[0].max_chars = 4_097;- assert!(validate_knowledge_datasets(&datasets).is_err());-- let mut datasets = valid_datasets();- datasets[0].download_size_bytes = 0;- assert!(validate_knowledge_datasets(&datasets).is_err());- datasets[0].download_size_bytes = -1;- assert!(validate_knowledge_datasets(&datasets).is_err());-- let mut datasets = valid_datasets();- datasets[0].artifact_sha256.pop();- assert!(validate_knowledge_datasets(&datasets).is_err());- datasets[0]- .artifact_sha256- .push("not-a-checksum".to_owned());- assert!(validate_knowledge_datasets(&datasets).is_err());-- let mut datasets = valid_datasets();- datasets[0].relevance_threshold = f32::NAN;- assert!(validate_knowledge_datasets(&datasets).is_err());- datasets[0].relevance_threshold = 1.01;- assert!(validate_knowledge_datasets(&datasets).is_err());-- let mut datasets = valid_datasets();- datasets[0].artifact_base_url = datasets[0].artifact_base_url.replace(- "/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/",- "/resolve/main/",- );- assert!(validate_knowledge_datasets(&datasets).is_err());-- let mut datasets = valid_datasets();- datasets[0].source_url_template = "http://example.com/{title}/{section}".to_owned();- assert!(validate_knowledge_datasets(&datasets).is_err());- }-- #[test]- fn rejects_invalid_embedding_contract_and_model_aliases() {- let mut config = knowledge_embedding_config();- config.target_id = "custom-embedding".to_owned();- assert!(validate_knowledge_embedding(&config, &[]).is_err());-- let mut config = knowledge_embedding_config();- assert!(validate_knowledge_embedding(&config, &[config.target_id.clone()]).is_err());- config.target_id = EMBEDDING_TARGET_ID.to_owned();- config.model_url = config- .model_url- .replace("957b55764bd672f51240ac026e3a23ac9459ee3c", "main");- assert!(validate_knowledge_embedding(&config, &[]).is_err());-- let mut config = knowledge_embedding_config();- config.model_sha256 = "not-a-checksum".to_owned();- assert!(validate_knowledge_embedding(&config, &[]).is_err());-- let mut config = knowledge_embedding_config();- config.dim = 768;- assert!(validate_knowledge_embedding(&config, &[]).is_err());- }-}diff --git a/rust/crates/ensu/src/llm/context.rs b/rust/crates/ensu/src/llm/context.rsindex 5585b84758..fdf3707573 100644--- a/rust/crates/ensu/src/llm/context.rs+++ b/rust/crates/ensu/src/llm/context.rs@@ -272,6 +272,25 @@ impl Context {Ok(Arc::new(context))}+ pub fn new_knowledge_embedding(+ model: &ModelRef,+ n_threads: Option<i32>,+ ) -> Result<ContextRef, Error> {+ let config = crate::config::knowledge_embedding_config();+ Self::new_embedding(+ model,+ EmbeddingContextParams {+ context_size: config.context_size,+ n_threads,+ batch_size: config.batch_size,+ micro_batch_size: config.micro_batch_size,+ source_dim: config.source_dim,+ dim: config.dim,+ query_prompt: config.query_prompt,+ },+ )+ }+pub fn prewarm_multimodal(&self,mmproj_path: String,diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rsindex 4aba9e0dbb..91fd1a1529 100644--- a/rust/crates/ensu/src/model/mod.rs+++ b/rust/crates/ensu/src/model/mod.rs@@ -82,7 +82,7 @@ pub fn voice_activity_target() -> ModelTarget {}pub fn knowledge_embedding_target() -> ModelTarget {- let embedding = config::defaults().knowledge_embedding;+ let embedding = config::knowledge_embedding_config();ModelTarget::Files {id: embedding.target_id,files: vec![ModelFile {diff --git a/rust/crates/ensu/src/retrieval/citation.rs b/rust/crates/ensu/src/retrieval/citation.rsindex a2f472f70f..e74e097840 100644--- a/rust/crates/ensu/src/retrieval/citation.rs+++ b/rust/crates/ensu/src/retrieval/citation.rs@@ -20,7 +20,6 @@ pub struct SourceCitation {pub dataset_label: String,pub credit: String,pub title: String,- pub section: Option<String>,pub source_url: String,pub license_label: String,pub license_url: String,@@ -85,16 +84,7 @@ fn normalize_and_deduplicate(}let dataset_label = normalize_required(&citation.dataset_label, "dataset label")?;let credit = normalize_required(&citation.credit, "credit")?;- let mut title = normalize_required(&citation.title, "title")?;- if let Some(section) = citation- .section- .as_deref()- .map(normalize_single_line)- .filter(|section| !section.is_empty())- {- title.push_str(" β ");- title.push_str(§ion);- }+ let title = normalize_required(&citation.title, "title")?;let license_label = normalize_required(&citation.license_label, "license label")?;if license_label != LICENSE_LABEL || citation.license_url != LICENSE_URL {return Err(RetrievalError::InvalidInput(@@ -112,7 +102,6 @@ fn normalize_and_deduplicate(dataset_label,credit,title,- section: None,source_url: citation.source_url.clone(),license_label,license_url: citation.license_url.clone(),@@ -206,7 +195,6 @@ fn parse_footer(footer: &str) -> Option<Vec<SourceCitation>> {dataset_label: dataset_label.to_owned(),credit: credit.to_owned(),title: title.to_owned(),- section: None,source_url: source_url.to_owned(),license_label: license_label.to_owned(),license_url: license_url.to_owned(),@@ -288,7 +276,6 @@ mod tests {dataset_label: dataset_label.to_owned(),credit: format!("{dataset_label} contributors"),title: "Example title".to_owned(),- section: None,source_url: url.to_owned(),license_label: LICENSE_LABEL.to_owned(),license_url: LICENSE_URL.to_owned(),@@ -302,19 +289,18 @@ mod tests {"Simple English Wikipedia","https://simple.wikipedia.org/wiki/Example",);- let mut second = citation(+ let second = citation("wikibooks","Wikibooks","https://en.wikibooks.org/wiki/Tea",);- second.section = Some("Brewing".to_owned());let finalized = finalize_assistant_text("An answer", &[first, second]).unwrap();assert!(!finalized.ends_with('\n'));let parsed = parse_assistant_text(&finalized);assert_eq!(parsed.text, "An answer");assert_eq!(parsed.citations.len(), 2);- assert_eq!(parsed.citations[1].title, "Example title β Brewing");+ assert_eq!(parsed.citations[1].title, "Example title");assert_eq!(knowledge_source_chip_label(&parsed.citations).as_deref(),Some("Simple English Wikipedia +1")diff --git a/rust/crates/ensu/src/retrieval/index.rs b/rust/crates/ensu/src/retrieval/index.rsindex 3e931f9d98..aedcabff1a 100644--- a/rust/crates/ensu/src/retrieval/index.rs+++ b/rust/crates/ensu/src/retrieval/index.rs@@ -11,7 +11,6 @@ use url::Url;use crate::config::{KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE, KNOWLEDGE_VECTORS_FILE,KnowledgeDatasetConfig, is_path_safe_component, knowledge_index_contract,- validate_knowledge_datasets,};use super::{BEGIN_CONTEXT_SENTINEL, END_CONTEXT_SENTINEL, RetrievalError, normalize_single_line};@@ -30,7 +29,6 @@ pub struct RetrievalHit {#[derive(Debug, Deserialize)]struct Manifest {dataset: String,- granularity: String,max_chars: u32,model: String,source_dim: u32,@@ -48,7 +46,6 @@ struct Manifest {#[derive(Debug, Deserialize)]struct MetadataRow {- id: serde_json::Value,title: String,text: String,#[serde(default)]@@ -73,8 +70,6 @@ impl RetrievalIndex {directory: impl AsRef<Path>,expected_pack: &KnowledgeDatasetConfig,) -> Result<Self, RetrievalError> {- validate_knowledge_datasets(std::slice::from_ref(expected_pack))- .map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;let directory = directory.as_ref();let directory_metadata = fs::symlink_metadata(directory)?;if !directory_metadata.file_type().is_dir() {@@ -270,11 +265,6 @@ impl RetrievalIndex {))})?;let row: MetadataRow = serde_json::from_str(line)?;- if row.id.is_null() {- return Err(RetrievalError::InvalidPack(- "metadata row id must not be null".to_string(),- ));- }if row.title.trim().is_empty() {return Err(RetrievalError::InvalidPack("metadata title must not be empty".to_string(),@@ -406,14 +396,6 @@ fn validate_manifest("manifest dataset must equal its path-safe revision directory".to_string(),));}- if manifest.granularity.trim().is_empty()- || manifest.granularity.len() > 256- || manifest.granularity.chars().any(char::is_control)- {- return Err(RetrievalError::InvalidPack(- "manifest granularity is invalid".to_string(),- ));- }if manifest.count == 0 || manifest.dim == 0 || manifest.meta_rows_per_block == 0 {return Err(RetrievalError::InvalidPack("manifest count, dim, and metadata rows per block must be positive".to_string(),diff --git a/rust/crates/ensu/src/retrieval/pack.rs b/rust/crates/ensu/src/retrieval/pack.rsindex 7a97283c8a..a0add74dd0 100644--- a/rust/crates/ensu/src/retrieval/pack.rs+++ b/rust/crates/ensu/src/retrieval/pack.rs@@ -5,7 +5,7 @@ use ente_model_download::download::{self, Downloader, Progress, Target};use crate::config::{KNOWLEDGE_ARTIFACT_FILENAMES, KnowledgeDatasetConfig, is_path_safe_component,- knowledge_artifact_urls, validate_knowledge_datasets,+ knowledge_artifact_urls,};use super::{RetrievalError, index::RetrievalIndex};@@ -13,20 +13,22 @@ use super::{RetrievalError, index::RetrievalIndex};pub fn download_knowledge_pack(pack_root: impl AsRef<Path>,expected_pack: &KnowledgeDatasetConfig,- on_progress: impl FnMut(Progress),- is_cancelled: impl Fn() -> bool,+ on_progress: impl FnMut(Progress) + Send,+ cancellation: download::CancellationToken,) -> Result<(), download::Error> {let pack_root = pack_root.as_ref();let targets = knowledge_download_targets(pack_root, expected_pack)?;- Downloader::new()?.download_blocking(targets, on_progress, is_cancelled)+ tokio::runtime::Builder::new_current_thread()+ .enable_io()+ .enable_time()+ .build()?+ .block_on(Downloader::new()?.download(targets, on_progress, cancellation))}fn knowledge_download_targets(pack_root: &Path,expected_pack: &KnowledgeDatasetConfig,) -> Result<Vec<Target>, download::Error> {- validate_knowledge_datasets(std::slice::from_ref(expected_pack))- .map_err(|error| download::Error::InvalidTarget(error.to_string()))?;let root_identity = pack_root.file_name().and_then(|name| name.to_str())@@ -57,8 +59,7 @@ fn knowledge_download_targets());}- let urls = knowledge_artifact_urls(expected_pack)- .map_err(|error| download::Error::InvalidTarget(error.to_string()))?;+ let urls = knowledge_artifact_urls(expected_pack);Ok(KNOWLEDGE_ARTIFACT_FILENAMES.into_iter().zip(urls)@@ -120,7 +121,7 @@ pub fn reconcile_knowledge_pack(let active = if let Some(position) = current_position {Some(valid.swap_remove(position))} else {- valid.sort_by(|left, right| left.0.cmp(&right.0));+ valid.sort_by(|left, right| right.0.cmp(&left.0));(!valid.is_empty()).then(|| valid.remove(0))};@@ -184,8 +185,6 @@ fn validate_pack_root(expected_pack: &KnowledgeDatasetConfig,create: bool,) -> Result<(), RetrievalError> {- validate_knowledge_datasets(std::slice::from_ref(expected_pack))- .map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;let root_identity = pack_root.file_name().and_then(|name| name.to_str())diff --git a/rust/crates/ensu/src/retrieval/prompt.rs b/rust/crates/ensu/src/retrieval/prompt.rsindex 018ae33e64..c6493aeb29 100644--- a/rust/crates/ensu/src/retrieval/prompt.rs+++ b/rust/crates/ensu/src/retrieval/prompt.rs@@ -1,5 +1,3 @@-use unicode_segmentation::UnicodeSegmentation;-use crate::config::knowledge_datasets;use super::{@@ -69,7 +67,7 @@ pub fn build_knowledge_prompt_context(break;}- let passage = truncate_utf8_graphemes(&item.hit.text, available).trim();+ let passage = truncate_utf8(&item.hit.text, available).trim();if passage.is_empty() {break;}@@ -80,8 +78,7 @@ pub fn build_knowledge_prompt_context(dataset_id: dataset.stable_id.clone(),dataset_label: dataset.label.clone(),credit: dataset.attribution.credit.clone(),- title: item.hit.title.clone(),- section: item.hit.section.clone(),+ title: display_title,source_url: item.hit.source_url.clone(),license_label: dataset.attribution.license_label.clone(),license_url: dataset.attribution.license_url.clone(),@@ -96,18 +93,14 @@ pub fn build_knowledge_prompt_context(Ok(Some(KnowledgePromptContext { text, citations }))}-fn truncate_utf8_graphemes(value: &str, max_bytes: usize) -> &str {+fn truncate_utf8(value: &str, max_bytes: usize) -> &str {if value.len() <= max_bytes {return value;}- let mut end = 0;- for (start, grapheme) in value.grapheme_indices(true) {- let next = start + grapheme.len();- if next > max_bytes {- break;- }- end = next;+ let mut end = max_bytes;+ while !value.is_char_boundary(end) {+ end -= 1;}&value[..end]}@@ -170,8 +163,7 @@ A useful passage\n\context.citations[0].credit,"Simple English Wikipedia contributors");- assert_eq!(context.citations[0].title, "Example title");- assert_eq!(context.citations[0].section.as_deref(), Some("Details"));+ assert_eq!(context.citations[0].title, "Example title β Details");assert_eq!(context.citations[0].source_url,"https://example.com/source"@@ -199,42 +191,14 @@ A useful passage\n\assert!(context.text.len() <= budget);}- #[test]- fn truncates_only_at_extended_grapheme_boundaries() {- let item = hit("simplewiki", "Aπ¨βπ©βπ§βπ¦B");- let budget_inside_family_emoji = fixed_bytes(&item) + 1 + "π¨".len();- let context = build_knowledge_prompt_context(&[item], budget_inside_family_emoji)- .unwrap()- .unwrap();-- assert!(- context- .text- .contains("\nA\n----- END KNOWLEDGE CONTEXT -----")- );- assert!(!context.text.contains('π¨'));-- let item = hit("simplewiki", "Ae\u{301}B");- let budget_inside_combining_grapheme = fixed_bytes(&item) + 2;- let context = build_knowledge_prompt_context(&[item], budget_inside_combining_grapheme)- .unwrap()- .unwrap();- assert!(- context- .text- .contains("\nA\n----- END KNOWLEDGE CONTEXT -----")- );- assert!(!context.text.contains("e\u{301}"));- }-#[test]fn respects_two_three_and_four_byte_scalar_boundaries() {- assert_eq!(truncate_utf8_graphemes("Γ©", 1), "");- assert_eq!(truncate_utf8_graphemes("Γ©", 2), "Γ©");- assert_eq!(truncate_utf8_graphemes("β¬", 2), "");- assert_eq!(truncate_utf8_graphemes("β¬", 3), "β¬");- assert_eq!(truncate_utf8_graphemes("π", 3), "");- assert_eq!(truncate_utf8_graphemes("π", 4), "π");+ assert_eq!(truncate_utf8("Γ©", 1), "");+ assert_eq!(truncate_utf8("Γ©", 2), "Γ©");+ assert_eq!(truncate_utf8("β¬", 2), "");+ assert_eq!(truncate_utf8("β¬", 3), "β¬");+ assert_eq!(truncate_utf8("π", 3), "");+ assert_eq!(truncate_utf8("π", 4), "π");}#[test]@@ -249,7 +213,7 @@ A useful passage\n\.text.contains("# Example title (Simple English Wikipedia)\npassage"));- assert_eq!(context.citations[0].section.as_deref(), Some(" \n "));+ assert_eq!(context.citations[0].title, "Example title");let blank = hit("simplewiki", " \n\t ");assert!(diff --git a/rust/crates/ensu/tests/knowledge_real_assets.rs b/rust/crates/ensu/tests/knowledge_real_assets.rsdeleted file mode 100644index a592d09ecc..0000000000--- a/rust/crates/ensu/tests/knowledge_real_assets.rs+++ /dev/null@@ -1,80 +0,0 @@-use std::env;-use std::path::Path;--use ente_ensu::config::defaults;-use ente_ensu::llm::{Context, EmbeddingContextParams, Model, ModelLoadParams};-use ente_ensu::retrieval::RetrievalIndex;--const COMPATIBILITY_QUERY: &str = "how tall is mount everest";-const EXPECTED_TOP_TITLE: &str = "Mount Everest";--#[test]-fn pinned_embedding_and_pack_match_builder_top_hit() {- let Some(model_path) = required_env("ENSU_KNOWLEDGE_EMBEDDING_GGUF") else {- eprintln!("skipping real-asset test: ENSU_KNOWLEDGE_EMBEDDING_GGUF is unset");- return;- };- let Some(pack_directory) = required_env("ENSU_KNOWLEDGE_PACK_DIR") else {- eprintln!("skipping real-asset test: ENSU_KNOWLEDGE_PACK_DIR is unset");- return;- };- let defaults = defaults();- let embedding = defaults.knowledge_embedding;- let pack_identity = Path::new(&pack_directory)- .file_name()- .and_then(|name| name.to_str())- .expect("pack directory has a UTF-8 identity");- let dataset = defaults- .knowledge_datasets- .into_iter()- .find(|dataset| {- pack_identity == dataset.current_download_identity- || pack_identity.starts_with(&format!("{}-", dataset.stable_id))- })- .expect("pack directory belongs to a bundled dataset");-- let model = Model::load(ModelLoadParams {- model_path,- n_gpu_layers: Some(0),- use_mmap: Some(true),- use_mlock: Some(false),- })- .expect("load pinned EmbeddingGemma");- let context = Context::new_embedding(- &model,- EmbeddingContextParams {- context_size: embedding.context_size,- n_threads: None,- batch_size: embedding.batch_size,- micro_batch_size: embedding.micro_batch_size,- source_dim: embedding.source_dim,- dim: embedding.dim,- query_prompt: embedding.query_prompt,- },- )- .expect("create mean-pooled embedding context");- let vector = context- .embed(COMPATIBILITY_QUERY)- .expect("embed real query");- assert_eq!(vector.len(), 512);- assert!(vector.iter().all(|value| value.is_finite()));- let norm = vector- .iter()- .map(|value| f64::from(*value) * f64::from(*value))- .sum::<f64>()- .sqrt();- assert!((norm - 1.0).abs() < 1e-5, "embedding norm was {norm}");-- let index = RetrievalIndex::open(pack_directory, &dataset).expect("open published pack");- let hits = index- .search(&vector, embedding.max_hits, dataset.relevance_threshold)- .expect("search published pack");- assert_eq!(- hits.first().map(|hit| hit.title.as_str()),- Some(EXPECTED_TOP_TITLE)- );-}--fn required_env(name: &str) -> Option<String> {- env::var(name).ok().filter(|value| !value.trim().is_empty())-}diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/model-download/src/download.rsindex 96933a381b..1a9a6328a1 100644--- a/rust/crates/model-download/src/download.rs+++ b/rust/crates/model-download/src/download.rs@@ -207,15 +207,6 @@ impl Downloader {}).await}-- pub fn download_blocking(- &self,- targets: Vec<Target>,- on_progress: impl FnMut(Progress),- is_cancelled: impl Fn() -> bool,- ) -> Result<(), Error> {- fetch(self, targets, on_progress, is_cancelled)- }}fn build_client() -> Result<Client, Error> {diff --git a/rust/crates/model-download/src/model.rs b/rust/crates/model-download/src/model.rsindex 4674c998e4..9b96045788 100644--- a/rust/crates/model-download/src/model.rs+++ b/rust/crates/model-download/src/model.rs@@ -189,8 +189,8 @@ impl ModelDownloader {let mut any = false;for entry in download_targets(&self.models_dir, target).ok()? {let path = &entry.destination;- let size = if let Some(size) = valid_local_size(&entry, path) {- Some(size)+ let size = if path.exists() {+ fs::metadata(path).ok().map(|m| m.len()).filter(|s| *s > 0)} else {download::probe_content_length(downloader.client(), &entry.url).await};@@ -204,18 +204,6 @@ impl ModelDownloader {}}-fn valid_local_size(target: &Target, path: &Path) -> Option<u64> {- let actual = download::sha256_file(path).ok()?;- if actual != target.sha256.trim().to_ascii_lowercase() {- return None;- }- fs::metadata(path)- .ok()- .filter(|metadata| metadata.is_file())- .map(|metadata| metadata.len())- .filter(|size| *size > 0)-}-fn storage_key(target: &ModelTarget) -> &str {match target {ModelTarget::Files { id, .. } | ModelTarget::TarGz { id, .. } => id,@@ -441,25 +429,6 @@ mod tests {let _ = fs::remove_dir_all(dir);}- #[test]- fn local_size_ignores_files_that_fail_checksum_validation() {- let dir = scratch_dir("local-size-validation");- let path = dir.join("model.gguf");- write_gguf(&path, b"GGUFdata");- let mut target = Target {- label: "Model".to_string(),- url: "https://example.org/model.gguf".to_string(),- sha256: test_sha(),- destination: path.clone(),- };-- assert_eq!(valid_local_size(&target, &path), None);- target.sha256 = download::sha256_file(&path).unwrap();- assert_eq!(valid_local_size(&target, &path), Some(8));-- let _ = fs::remove_dir_all(dir);- }-#[test]fn download_skips_when_already_present() {let dir = scratch_dir("skip");