Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[ensu] Align mobile knowledge pack lifecycle

setalpc35b00a61045
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index 595e187b40..6c8286de8c 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -68,7 +68,8 @@ class AppStore(
messageStore = messageStore,
attachmentActions = attachmentActions,
modelSettingsActions = modelSettingsActions,
- configDefaults = configDefaults
+ configDefaults = configDefaults,
+ awaitKnowledgeReady = knowledgeStore::awaitEnabledPacksReady
)
fun bootstrap(scope: CoroutineScope) {
chatActions.setScope(scope)
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
index 844ab5d6c7..734237348f 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
@@ -38,7 +38,6 @@ import io.ente.ensu.logging.LogViewerScreen
import io.ente.ensu.settings.SettingsScreen
import io.ente.ensu.settings.SystemPromptSettingsScreen
import io.ente.ensu.settings.KnowledgeSettingsScreen
-import io.ente.ensu.settings.TermsAndConditionsScreen
import io.ente.ensu.device.isChatSupported
@Composable
@@ -96,9 +95,6 @@ internal fun HomeNavigation(
HomeRoute.Knowledge -> {
SimpleTopBar(title = "Ensu Packs") { navController.popBackStack() }
}
- HomeRoute.Terms -> {
- SimpleTopBar(title = "Terms and Conditions") { navController.popBackStack() }
- }
else -> Unit
}
}
@@ -156,7 +152,6 @@ internal fun HomeNavigation(
isAdvancedUnlocked = appState.developerSettings.isAdvancedUnlocked,
onOpenLogs = { navController.navigate(HomeRoute.Logs) },
onOpenKnowledge = { navController.navigate(HomeRoute.Knowledge) },
- onOpenTerms = { navController.navigate(HomeRoute.Terms) },
onOpenModelSettings = { navController.navigate(HomeRoute.ModelSettings) },
onOpenSystemPromptSettings = { navController.navigate(HomeRoute.SystemPromptSettings) },
onUnlockAdvanced = {
@@ -181,15 +176,6 @@ internal fun HomeNavigation(
onSetEnabled = store::setKnowledgePackEnabled
)
}
- composable(
- route = HomeRoute.Terms,
- enterTransition = { forwardEnter() },
- exitTransition = { forwardExit() },
- popEnterTransition = { backEnter() },
- popExitTransition = { backExit() }
- ) {
- TermsAndConditionsScreen()
- }
composable(
route = HomeRoute.Logs,
enterTransition = { forwardEnter() },
@@ -258,7 +244,6 @@ internal object HomeRoute {
const val ModelSettings = "model-settings"
const val SystemPromptSettings = "system-prompt-settings"
const val Knowledge = "knowledge"
- const val Terms = "terms"
}
internal fun AnimatedContentTransitionScope<NavBackStackEntry>.forwardEnter() =
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
index 88890adfd9..8608cc0a5d 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
@@ -795,13 +795,6 @@ private fun KnowledgeSourcesDialog(
style = EnsuTypography.large,
color = EnsuColor.textPrimary()
)
- citation.section?.trim()?.takeIf { it.isNotEmpty() }?.let { section ->
- Text(
- text = section,
- style = EnsuTypography.small,
- color = EnsuColor.textMuted()
- )
- }
HorizontalDivider(color = EnsuColor.border())
Text(
text = "Attribution",
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index 2b9cdea940..f36d6a082f 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -17,7 +17,6 @@ import io.ente.ensu.chat.ChatSession
import io.ente.ensu.bindings.DbException
import io.ente.ensu.bindings.LlmException
import io.ente.ensu.bindings.ConfigDefaults
-import io.ente.ensu.bindings.KnowledgePromptHit
import io.ente.ensu.logging.LogLevel
import io.ente.ensu.chat.MessageAuthor
import io.ente.ensu.chat.sessionTitleFromText
@@ -28,7 +27,6 @@ import io.ente.ensu.bindings.buildKnowledgePromptContext
import io.ente.ensu.bindings.cleanAssistantText
import io.ente.ensu.bindings.finalizeAssistantText
import io.ente.ensu.knowledge.KnowledgeProvider
-import io.ente.ensu.knowledge.KnowledgeSearchHit
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
@@ -58,7 +56,8 @@ internal class ChatStoreActions(
private val messageStore: MutableMap<String, MutableList<ChatMessage>>,
private val attachmentActions: AttachmentStoreActions,
private val modelSettingsActions: ModelSettingsActions,
- private val configDefaults: ConfigDefaults
+ private val configDefaults: ConfigDefaults,
+ private val awaitKnowledgeReady: suspend () -> Unit
) {
private val branchSelections = mutableMapOf<String, MutableMap<String, String>>()
private val sessionSummaries = mutableMapOf<String, String>()
@@ -485,10 +484,12 @@ internal class ChatStoreActions(
priorSummary?.join()
val isActive = { isGenerationActive(generationToken, sessionId) }
if (!isActive()) return@launch
+ awaitKnowledgeReady()
+ if (!isActive()) return@launch
val progressTracker = DownloadProgressTracker()
var embeddingAssetInvalid = false
val enabledDatasets = state.value.knowledge.enabledReadyDatasets
- val knowledgeHits: List<KnowledgeSearchHit> = if (
+ val knowledgeHits = if (
userMessage.text.isNotBlank() && enabledDatasets.isNotEmpty()
) {
try {
@@ -607,12 +608,7 @@ internal class ChatStoreActions(
).coerceAtLeast(0)
val knowledgeContext = runCatching {
buildKnowledgePromptContext(
- hits = knowledgeHits.map { item ->
- KnowledgePromptHit(
- datasetId = item.dataset.stableId,
- hit = item.hit
- )
- },
+ hits = knowledgeHits,
maxUtf8Bytes = min(
configDefaults.knowledgeEmbedding.maxContextUtf8Bytes.toInt(),
remainingKnowledgeBytes
@@ -632,9 +628,7 @@ internal class ChatStoreActions(
systemPrompt = it
)
}
- val useKnowledge = knowledgeContext != null &&
- knowledgeHistorySelection != null &&
- !knowledgeHistorySelection.wasTrimmed
+ val useKnowledge = knowledgeHistorySelection?.wasTrimmed == false
val historySelection = if (useKnowledge) {
knowledgeHistorySelection ?: normalHistorySelection
} else {
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt
index 8f60ab0ceb..e00084e202 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeProvider.kt
@@ -1,38 +1,33 @@
package io.ente.ensu.knowledge
+import io.ente.ensu.bindings.CancellationToken
import io.ente.ensu.bindings.KnowledgeDatasetConfig
import io.ente.ensu.bindings.KnowledgeDownloadCallback
import io.ente.ensu.bindings.KnowledgeDownloadProgress
+import io.ente.ensu.bindings.KnowledgePromptHit
import io.ente.ensu.bindings.KnowledgeReconciliation
-import io.ente.ensu.bindings.RetrievalHit
import io.ente.ensu.bindings.RetrievalIndex
import io.ente.ensu.bindings.cleanupObsoleteKnowledgePackRevisions
import io.ente.ensu.bindings.downloadKnowledgePack
import io.ente.ensu.bindings.reconcileKnowledgePack
import java.io.File
-import java.util.concurrent.atomic.AtomicBoolean
-import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
-data class KnowledgeSearchHit(
- val dataset: KnowledgeDatasetConfig,
- val hit: RetrievalHit
-)
-
class KnowledgeProvider(
- private val knowledgeRoot: File,
- private val ioScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
+ private val knowledgeRoot: File
) {
private data class ActiveMutation(
- val cancelled: AtomicBoolean,
+ val cancellation: CancellationToken,
val task: Deferred<KnowledgeReconciliation>
)
@@ -73,23 +68,24 @@ class KnowledgeProvider(
suspend fun download(
dataset: KnowledgeDatasetConfig,
onProgress: (KnowledgeDownloadProgress) -> Unit
- ): KnowledgeReconciliation {
+ ): KnowledgeReconciliation = coroutineScope {
+ var ownsMutation = false
val mutation = lifecycleGate(dataset.stableId).withLock {
mutationGate.withLock {
mutations[dataset.stableId] ?: run {
- val cancelled = AtomicBoolean(false)
- val task = ioScope.async {
+ ownsMutation = true
+ val cancellation = CancellationToken()
+ val task = async(Dispatchers.IO) {
try {
downloadKnowledgePack(
packRoot(dataset).absolutePath,
- dataset,
+ dataset.stableId,
object : KnowledgeDownloadCallback {
override fun onProgress(progress: KnowledgeDownloadProgress) {
onProgress(progress)
}
-
- override fun isCancelled(): Boolean = cancelled.get()
- }
+ },
+ cancellation
)
indexGate.withLock {
val result = reconcileAndOpenLocked(dataset)
@@ -99,26 +95,36 @@ class KnowledgeProvider(
result
}
} catch (error: Throwable) {
- indexGate.withLock {
- incomingRevision(dataset).deleteRecursively()
- runCatching { reconcileAndOpenLocked(dataset) }
+ withContext(NonCancellable) {
+ indexGate.withLock {
+ incomingRevision(dataset).deleteRecursively()
+ runCatching { reconcileAndOpenLocked(dataset) }
+ }
}
throw error
}
}
- ActiveMutation(cancelled, task).also {
+ ActiveMutation(cancellation, task).also {
mutations[dataset.stableId] = it
}
}
}
}
- return try {
+ try {
mutation.task.await()
+ } catch (error: CancellationException) {
+ if (ownsMutation) mutation.cancellation.cancel()
+ throw error
} finally {
- mutationGate.withLock {
- if (mutations[dataset.stableId] === mutation && mutation.task.isCompleted) {
- mutations.remove(dataset.stableId)
+ withContext(NonCancellable) {
+ if (ownsMutation && !mutation.task.isCompleted) {
+ mutation.task.join()
+ }
+ mutationGate.withLock {
+ if (mutations[dataset.stableId] === mutation && mutation.task.isCompleted) {
+ mutations.remove(dataset.stableId)
+ }
}
}
}
@@ -126,7 +132,7 @@ class KnowledgeProvider(
suspend fun cancel(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {
val mutation = mutationGate.withLock { mutations[dataset.stableId] }
- mutation?.cancelled?.set(true)
+ mutation?.cancellation?.cancel()
runCatching { mutation?.task?.await() }
mutationGate.withLock {
if (mutations[dataset.stableId] === mutation) {
@@ -140,9 +146,9 @@ class KnowledgeProvider(
datasets: List<KnowledgeDatasetConfig>,
query: List<Float>,
maxHits: UInt
- ): List<KnowledgeSearchHit> = withContext(Dispatchers.IO) {
+ ): List<KnowledgePromptHit> = withContext(Dispatchers.IO) {
indexGate.withLock {
- val merged = mutableListOf<KnowledgeSearchHit>()
+ val merged = mutableListOf<KnowledgePromptHit>()
for (dataset in datasets) {
currentCoroutineContext().ensureActive()
val open = indexes[dataset.stableId] ?: continue
@@ -152,7 +158,7 @@ class KnowledgeProvider(
continue
}
currentCoroutineContext().ensureActive()
- merged += hits.map { hit -> KnowledgeSearchHit(dataset, hit) }
+ merged += hits.map { hit -> KnowledgePromptHit(dataset.stableId, hit) }
}
merged.sortedByDescending { it.hit.score }.take(maxHits.toInt())
}
@@ -161,19 +167,19 @@ class KnowledgeProvider(
private fun reconcileAndOpenLocked(dataset: KnowledgeDatasetConfig): KnowledgeReconciliation {
val previous = indexes[dataset.stableId]
val root = packRoot(dataset).absolutePath
- val result = reconcileKnowledgePack(root, dataset)
+ val result = reconcileKnowledgePack(root, dataset.stableId)
val directory = result.activeDirectory
if (directory == null) {
indexes.remove(dataset.stableId)
previous?.index?.destroy()
} else if (previous?.directory != directory) {
- val replacement = RetrievalIndex.open(directory, dataset)
+ val replacement = RetrievalIndex.open(directory, dataset.stableId)
indexes[dataset.stableId] = OpenIndex(directory, replacement)
previous?.index?.destroy()
}
result.activeIdentity?.let { activeIdentity ->
runCatching {
- cleanupObsoleteKnowledgePackRevisions(root, dataset, activeIdentity)
+ cleanupObsoleteKnowledgePackRevisions(root, dataset.stableId, activeIdentity)
}
}
return result
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt
index 185e73fbc6..165d8fbe18 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/knowledge/KnowledgeStore.kt
@@ -8,11 +8,13 @@ import io.ente.ensu.AppState
import io.ente.ensu.bindings.DownloadError
import io.ente.ensu.bindings.KnowledgeDatasetConfig
import io.ente.ensu.bindings.KnowledgeDownloadException
+import io.ente.ensu.bindings.KnowledgeDownloadProgress
import io.ente.ensu.bindings.KnowledgeReconciliation
import io.ente.ensu.bindings.KnowledgeReconciliationStatus
import io.ente.ensu.device.isChatSupported
import io.ente.ensu.logging.FileLogRepository
import io.ente.ensu.logging.LogLevel
+import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.Flow
@@ -24,14 +26,12 @@ import kotlinx.coroutines.launch
private val Context.knowledgePreferences by preferencesDataStore("ensu_knowledge_preferences")
-data class KnowledgeMutationProgress(val percent: Int, val label: String)
-
data class KnowledgePackState(
val config: KnowledgeDatasetConfig,
val status: KnowledgeReconciliationStatus? = null,
val activeIdentity: String? = null,
val enabled: Boolean = false,
- val mutationProgress: KnowledgeMutationProgress? = null,
+ val mutationProgress: KnowledgeDownloadProgress? = null,
val errorMessage: String? = null
) {
val isMutating: Boolean get() = mutationProgress != null
@@ -61,9 +61,12 @@ class KnowledgeStore(
private val catalog = datasets.associateBy { it.stableId }
private val jobs = mutableMapOf<String, Job>()
private var scope: CoroutineScope? = null
+ private var bootstrapJob: Job? = null
+ private val enabledPacksReady = CompletableDeferred<Unit>()
fun bootstrap(scope: CoroutineScope) {
this.scope = scope
+ if (bootstrapJob != null) return
state.update { appState ->
appState.copy(
knowledge = KnowledgeState(
@@ -71,33 +74,30 @@ class KnowledgeStore(
)
)
}
- scope.launch {
+ bootstrapJob = scope.launch {
val requestedEnabled = runCatching {
preferences.enabledDatasetIds.first()
}.getOrDefault(emptySet())
- catalog.values.forEach { dataset ->
- val result = runCatching { provider.reconcile(dataset) }
- updatePack(dataset.stableId) { current ->
- result.fold(
- onSuccess = { reconciliation ->
- current.fromReconciliation(
- reconciliation,
- enabled = dataset.stableId in requestedEnabled
- )
- },
- onFailure = { error ->
- current.copy(
- status = KnowledgeReconciliationStatus.DOWNLOAD,
- enabled = false,
- errorMessage = userFacingKnowledgeError(error)
- )
- }
- )
+ val (enabled, disabled) = catalog.values.partition {
+ it.stableId in requestedEnabled
+ }
+ try {
+ enabled.forEach { dataset ->
+ reconcileAndUpdate(dataset, enabled = true)
}
+ } finally {
+ enabledPacksReady.complete(Unit)
+ }
+ disabled.forEach { dataset ->
+ reconcileAndUpdate(dataset, enabled = false)
}
}
}
+ suspend fun awaitEnabledPacksReady() {
+ enabledPacksReady.await()
+ }
+
fun downloadOrUpdate(stableId: String) {
if (!state.value.chat.deviceCapability.isChatSupported()) return
val dataset = catalog[stableId] ?: return
@@ -106,7 +106,7 @@ class KnowledgeStore(
val wasInstalled = state.value.knowledge.packs[stableId]?.activeIdentity != null
updatePack(stableId) {
it.copy(
- mutationProgress = KnowledgeMutationProgress(0, "Starting download..."),
+ mutationProgress = KnowledgeDownloadProgress("Starting download...", 0.0),
errorMessage = null
)
}
@@ -114,12 +114,7 @@ class KnowledgeStore(
try {
val reconciliation = provider.download(dataset) { progress ->
updatePack(stableId) { current ->
- current.copy(
- mutationProgress = KnowledgeMutationProgress(
- progress.percentage.toInt().coerceIn(0, 100),
- progress.label
- )
- )
+ current.copy(mutationProgress = progress)
}
}
val shouldEnable = !wasInstalled &&
@@ -198,6 +193,27 @@ class KnowledgeStore(
}
}
+ private suspend fun reconcileAndUpdate(
+ dataset: KnowledgeDatasetConfig,
+ enabled: Boolean
+ ) {
+ val result = runCatching { provider.reconcile(dataset) }
+ updatePack(dataset.stableId) { current ->
+ result.fold(
+ onSuccess = { reconciliation ->
+ current.fromReconciliation(reconciliation, enabled)
+ },
+ onFailure = { error ->
+ current.copy(
+ status = KnowledgeReconciliationStatus.DOWNLOAD,
+ enabled = false,
+ errorMessage = userFacingKnowledgeError(error)
+ )
+ }
+ )
+ }
+ }
+
private fun KnowledgePackState.fromReconciliation(
result: KnowledgeReconciliation,
enabled: Boolean
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
index 30a70ae9aa..114edf00b6 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
@@ -5,7 +5,6 @@ import io.ente.ensu.bindings.LlmChatMessage as NativeChatMessage
import io.ente.ensu.bindings.LlmChatRequest
import io.ente.ensu.bindings.LlmContext
import io.ente.ensu.bindings.LlmContextParams
-import io.ente.ensu.bindings.LlmEmbeddingContextParams
import io.ente.ensu.bindings.LlmGenerationEvent
import io.ente.ensu.bindings.LlmGenerationEventCallback
import io.ente.ensu.bindings.LlmGenerationSummary as NativeSummary
@@ -25,6 +24,10 @@ import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
+class RequiredModelValidationError(
+ val targetId: String
+) : Exception("Downloaded model failed validation: $targetId")
+
class LlmProvider(
private val downloader: ModelDownloader,
private val transcriber: Transcriber,
@@ -94,10 +97,10 @@ class LlmProvider(
modelLoadMutex.withLock {
if (!isEmbeddingModelReady()) {
downloader.removeDownloaded(embeddingDownloadTarget)
- throw IllegalStateException("Embedding model failed exact readiness validation")
+ throw RequiredModelValidationError(knowledgeEmbedding.targetId)
}
if (!isChatModelReady(selection)) {
- throw IllegalStateException("Chat model failed readiness validation")
+ throw RequiredModelValidationError(selection.id)
}
ensureModelReadyLocked(
selection,
@@ -179,17 +182,7 @@ class LlmProvider(
var embeddingContext: LlmContext? = null
try {
val threads = max(1, Runtime.getRuntime().availableProcessors() - 1)
- embeddingContext = embeddingModel.newEmbeddingContext(
- LlmEmbeddingContextParams(
- contextSize = knowledgeEmbedding.contextSize,
- nThreads = threads,
- batchSize = knowledgeEmbedding.batchSize,
- microBatchSize = knowledgeEmbedding.microBatchSize,
- sourceDim = knowledgeEmbedding.sourceDim,
- dim = knowledgeEmbedding.dim,
- queryPrompt = knowledgeEmbedding.queryPrompt
- )
- )
+ embeddingContext = embeddingModel.newEmbeddingContext(threads)
block { text -> embeddingContext.embed(text) }
} finally {
embeddingContext?.destroy()
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index ef1ee5aac6..7d15ac0579 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -144,9 +144,7 @@ internal class ModelSettingsActions(
if (!userInitiated && !currentState.chat.hasRequestedModelDownload) return
val selection = resolveSelection(currentState.modelSettings)
- val wasChatReady = llmProvider.isChatModelReady(selection)
- val wasEmbeddingReady = llmProvider.isEmbeddingModelReady()
- val isDownloaded = wasChatReady && wasEmbeddingReady
+ val isDownloaded = llmProvider.isChatModelReady(selection) && llmProvider.isEmbeddingModelReady()
if (isDownloaded) {
state.update { appState ->
appState.copy(
@@ -363,6 +361,7 @@ internal class ModelSettingsActions(
if (err is kotlinx.coroutines.CancellationException) return false
if (err is LlmException.Cancelled) return false
if (isOutOfStorageError(err)) return false
+ if (err is RequiredModelValidationError) return false
if (err is LlmException.Download) {
when (val error = err.error) {
is DownloadError.Validation -> return false
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.kt
index adafc2bb8a..873b2ac384 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/KnowledgeSettingsScreen.kt
@@ -118,7 +118,7 @@ private fun KnowledgePackCard(
if (pack.isMutating) {
Spacer(Modifier.height(EnsuSpacing.md.dp))
LinearProgressIndicator(
- progress = { (pack.mutationProgress?.percent ?: 0).coerceIn(0, 100) / 100f },
+ progress = { (pack.mutationProgress?.percentage ?: 0.0).coerceIn(0.0, 100.0).toFloat() / 100f },
modifier = Modifier.fillMaxWidth(),
color = EnsuColor.accent(),
trackColor = EnsuColor.border()
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/LegalSettingsScreens.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/LegalSettingsScreens.kt
deleted file mode 100644
index 45c5464704..0000000000
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/LegalSettingsScreens.kt
+++ /dev/null
@@ -1,51 +0,0 @@
-package io.ente.ensu.settings
-
-import androidx.compose.foundation.layout.Arrangement
-import androidx.compose.foundation.layout.PaddingValues
-import androidx.compose.foundation.layout.padding
-import androidx.compose.foundation.lazy.LazyColumn
-import androidx.compose.material.icons.Icons
-import androidx.compose.material.icons.outlined.Description
-import androidx.compose.runtime.Composable
-import androidx.compose.ui.platform.LocalUriHandler
-import androidx.compose.ui.Modifier
-import androidx.compose.ui.unit.dp
-import io.ente.ensu.designsystem.EnsuSpacing
-import io.ente.ensu.designsystem.HugeIcons
-
-object EnsuLegalDocuments {
- const val PRIVACY_TITLE = "Privacy Policy"
- const val PRIVACY_URL = "https://ente.com/privacy/"
- const val ENTE_TERMS_TITLE = "Ente Terms and Conditions"
- const val ENTE_TERMS_URL = "https://ente.com/terms/"
-}
-
-@Composable
-fun TermsAndConditionsScreen() {
- val uriHandler = LocalUriHandler.current
-
- LazyColumn(
- modifier = Modifier.padding(horizontal = EnsuSpacing.pageHorizontal.dp),
- contentPadding = PaddingValues(vertical = EnsuSpacing.lg.dp),
- verticalArrangement = Arrangement.spacedBy(EnsuSpacing.sm.dp)
- ) {
- item(key = "privacy") {
- SettingsRow(
- SettingsItem(
- title = EnsuLegalDocuments.PRIVACY_TITLE,
- iconRes = HugeIcons.ViewIcon,
- onClick = { uriHandler.openUri(EnsuLegalDocuments.PRIVACY_URL) }
- )
- )
- }
- item(key = "ente-terms") {
- SettingsRow(
- SettingsItem(
- title = EnsuLegalDocuments.ENTE_TERMS_TITLE,
- iconVector = Icons.Outlined.Description,
- onClick = { uriHandler.openUri(EnsuLegalDocuments.ENTE_TERMS_URL) }
- )
- )
- }
- }
-}
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.kt
index e71436526c..189c3c5e1e 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/SettingsScreen.kt
@@ -49,7 +49,6 @@ fun SettingsScreen(
isAdvancedUnlocked: Boolean,
onOpenLogs: () -> Unit,
onOpenKnowledge: () -> Unit,
- onOpenTerms: () -> Unit,
onOpenModelSettings: () -> Unit,
onOpenSystemPromptSettings: () -> Unit,
onUnlockAdvanced: () -> Unit,
@@ -60,7 +59,7 @@ fun SettingsScreen(
var lastBuildVersionTapAt by remember { mutableStateOf<Long?>(null) }
val context = LocalContext.current
- val allItems = remember(context, onOpenLogs, onOpenKnowledge, onOpenTerms, onSignIn) {
+ val allItems = remember(context, onOpenLogs, onOpenKnowledge, onSignIn) {
buildList {
add(
SettingsItem(
@@ -96,14 +95,16 @@ fun SettingsScreen(
add(
SettingsItem(
- title = "Terms and Conditions",
+ title = "Privacy Policy",
+ iconRes = HugeIcons.ViewIcon,
+ onClick = { context.openExternalLink("https://ente.com/privacy") }
+ )
+ )
+ add(
+ SettingsItem(
+ title = "Terms of Service",
iconVector = Icons.Outlined.Description,
- searchTerms = listOf(
- EnsuLegalDocuments.PRIVACY_TITLE,
- EnsuLegalDocuments.ENTE_TERMS_TITLE,
- "Terms of Service"
- ),
- onClick = onOpenTerms
+ onClick = { context.openExternalLink("https://ente.com/terms") }
)
)
}
@@ -113,8 +114,7 @@ fun SettingsScreen(
val q = query.trim().lowercase()
if (q.isEmpty()) return@remember allItems
allItems.filter { item ->
- item.title.lowercase().contains(q) ||
- item.searchTerms.any { it.lowercase().contains(q) }
+ item.title.lowercase().contains(q)
}
}
@@ -206,17 +206,16 @@ fun SettingsScreen(
}
}
-internal data class SettingsItem(
+private data class SettingsItem(
val title: String,
val iconRes: Int? = null,
val iconVector: ImageVector? = null,
- val searchTerms: List<String> = emptyList(),
val onClick: () -> Unit,
val isDestructive: Boolean = false
)
@Composable
-internal fun SettingsRow(item: SettingsItem) {
+private fun SettingsRow(item: SettingsItem) {
val iconAndTextColor = if (item.isDestructive) EnsuColor.error else EnsuColor.textPrimary()
Row(
diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index b9c89b6633..78e99fb3aa 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -1023,6 +1023,8 @@ final class ChatViewModel: ObservableObject {
_ = await priorGeneration?.result
_ = await priorSummary?.result
guard !Task.isCancelled, activeGenerationId == generationId else { return }
+ await knowledgeStore.bootstrap()
+ guard !Task.isCancelled, activeGenerationId == generationId else { return }
var embeddingAssetInvalid = false
var knowledgeHits: [KnowledgeSearchHit] = []
let enabledDatasets = knowledgeStore.enabledReadyDatasets
@@ -1148,8 +1150,7 @@ final class ChatViewModel: ObservableObject {
systemPrompt: candidate
)
}
- let useKnowledge = knowledgeContext != nil &&
- knowledgeHistorySelection?.wasTrimmed == false
+ let useKnowledge = knowledgeHistorySelection?.wasTrimmed == false
let historySelection = useKnowledge ?
(knowledgeHistorySelection ?? normalHistorySelection) : normalHistorySelection
var activeCitations = useKnowledge ? (knowledgeContext?.citations ?? []) : []
diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift
index 6ca7d8e49d..9e0fb2d391 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift
@@ -340,13 +340,6 @@ private struct KnowledgeSourcesSheet: View {
.font(EnsuTypography.large)
.foregroundStyle(EnsuColor.textPrimary)
- if let section = citation.section?.trimmingCharacters(in: .whitespacesAndNewlines),
- !section.isEmpty {
- Text(section)
- .font(EnsuTypography.small)
- .foregroundStyle(EnsuColor.textMuted)
- }
-
Divider()
Text("Attribution")
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift
index ebf4014fd4..2ff621ba49 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeProvider.swift
@@ -13,6 +13,7 @@ private struct KnowledgePackValidationError: LocalizedError {
actor KnowledgeProvider {
private final class Mutation: @unchecked Sendable {
+ let cancellation: CancellationToken
let task: Task<Void, Error>
init(
@@ -20,20 +21,23 @@ actor KnowledgeProvider {
dataset: KnowledgeDatasetConfig,
onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void
) {
+ let cancellation = CancellationToken()
+ self.cancellation = cancellation
task = Task.detached(priority: .utility) {
- let callback = KnowledgeDownloadCallbackSink(
- onProgress: onProgress,
- isCancelled: { Task.isCancelled }
- )
+ let callback = KnowledgeDownloadCallbackSink(onProgress: onProgress)
try downloadKnowledgePack(
packRoot: packRoot.path,
- expectedDataset: dataset,
- callback: callback
+ stableId: dataset.stableId,
+ callback: callback,
+ cancellation: cancellation
)
}
}
- func cancel() { task.cancel() }
+ func cancel() {
+ cancellation.cancel()
+ task.cancel()
+ }
}
private struct OpenIndex {
@@ -43,6 +47,7 @@ actor KnowledgeProvider {
private let root: URL
private var mutations: [String: Mutation] = [:]
+ private var lifecycleGates: [String: AsyncSerialGate] = [:]
private var indexes: [String: OpenIndex] = [:]
private let indexGate = AsyncSerialGate()
@@ -59,8 +64,14 @@ actor KnowledgeProvider {
}
func reconcile(dataset: KnowledgeDatasetConfig) async throws -> KnowledgeReconciliation {
- try await withIndexGate {
- try await reconcileAndActivateLocked(dataset: dataset)
+ let gate = lifecycleGate(dataset.stableId)
+ return try await gate.withLock {
+ if let mutation = mutations[dataset.stableId] {
+ _ = try? await mutation.task.value
+ }
+ return try await withIndexGate {
+ try await reconcileAndActivateLocked(dataset: dataset)
+ }
}
}
@@ -68,17 +79,18 @@ actor KnowledgeProvider {
dataset: KnowledgeDatasetConfig,
onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void
) async throws -> KnowledgeReconciliation {
- let mutation: Mutation
- if let current = mutations[dataset.stableId] {
- mutation = current
- } else {
+ let gate = lifecycleGate(dataset.stableId)
+ let mutation = try await gate.withLock {
+ if let current = mutations[dataset.stableId] {
+ return current
+ }
let created = Mutation(
packRoot: packRoot(dataset),
dataset: dataset,
onProgress: onProgress
)
mutations[dataset.stableId] = created
- mutation = created
+ return created
}
do {
@@ -193,7 +205,7 @@ actor KnowledgeProvider {
}
return try reconcileKnowledgePack(
packRoot: packRoot.path,
- expectedDataset: dataset
+ stableId: dataset.stableId
)
}.value
try activate(result: result, dataset: dataset)
@@ -201,7 +213,7 @@ actor KnowledgeProvider {
_ = try? await Task.detached(priority: .utility) {
try cleanupObsoleteKnowledgePackRevisions(
packRoot: packRoot.path,
- expectedDataset: dataset,
+ stableId: dataset.stableId,
activeIdentity: activeIdentity
)
}.value
@@ -220,7 +232,7 @@ actor KnowledgeProvider {
guard indexes[dataset.stableId]?.directory != directory else { return }
let replacement = try RetrievalIndex.open(
directory: directory,
- expectedDataset: dataset
+ stableId: dataset.stableId
)
indexes[dataset.stableId] = OpenIndex(directory: directory, index: replacement)
}
@@ -229,6 +241,15 @@ actor KnowledgeProvider {
try await indexGate.withLock(operation)
}
+ private func lifecycleGate(_ stableId: String) -> AsyncSerialGate {
+ if let gate = lifecycleGates[stableId] {
+ return gate
+ }
+ let gate = AsyncSerialGate()
+ lifecycleGates[stableId] = gate
+ return gate
+ }
+
private func packRoot(_ dataset: KnowledgeDatasetConfig) -> URL {
root.appendingPathComponent(dataset.stableId, isDirectory: true)
}
@@ -236,21 +257,12 @@ actor KnowledgeProvider {
private final class KnowledgeDownloadCallbackSink: KnowledgeDownloadCallback, @unchecked Sendable {
private let onProgressHandler: @Sendable (KnowledgeDownloadProgress) -> Void
- private let isCancelledHandler: @Sendable () -> Bool
- init(
- onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void,
- isCancelled: @escaping @Sendable () -> Bool
- ) {
+ init(onProgress: @escaping @Sendable (KnowledgeDownloadProgress) -> Void) {
onProgressHandler = onProgress
- isCancelledHandler = isCancelled
}
func onProgress(progress: KnowledgeDownloadProgress) {
onProgressHandler(progress)
}
-
- func isCancelled() -> Bool {
- isCancelledHandler()
- }
}
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift
index aad3cb4080..a3cf381f22 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/KnowledgeStore.swift
@@ -8,6 +8,7 @@ final class KnowledgeStore: ObservableObject {
private let provider: KnowledgeProvider
private let preferences: KnowledgePreferences
+ private var bootstrapTask: Task<Void, Never>?
private var mutationTasks: [String: Task<Void, Never>] = [:]
init(
@@ -31,21 +32,50 @@ final class KnowledgeStore: ObservableObject {
}
func bootstrap() async {
+ if let bootstrapTask {
+ await bootstrapTask.value
+ return
+ }
+
+ let task = Task { @MainActor [weak self] in
+ guard let self else { return }
+ await bootstrapEnabledDatasets()
+ }
+ bootstrapTask = task
+ await task.value
+ }
+
+ private func bootstrapEnabledDatasets() async {
let requestedEnabled = preferences.enabledDatasetIds
- for dataset in packs.map(\.config) {
- do {
- let result = try await provider.reconcile(dataset: dataset)
- applyReconciliation(
- result,
- stableId: dataset.stableId,
- enabled: requestedEnabled.contains(dataset.stableId)
- )
- } catch {
- updatePack(dataset.stableId) { pack in
- pack.status = .download
- pack.enabled = false
- pack.errorMessage = userFacingKnowledgeError(error)
- }
+ let datasets = packs.map(\.config)
+ for dataset in datasets where requestedEnabled.contains(dataset.stableId) {
+ await reconcileForBootstrap(dataset, enabled: true)
+ }
+
+ Task { @MainActor [weak self] in
+ guard let self else { return }
+ for dataset in datasets where !requestedEnabled.contains(dataset.stableId) {
+ await reconcileForBootstrap(dataset, enabled: false)
+ }
+ }
+ }
+
+ private func reconcileForBootstrap(
+ _ dataset: KnowledgeDatasetConfig,
+ enabled: Bool
+ ) async {
+ do {
+ let result = try await provider.reconcile(dataset: dataset)
+ applyReconciliation(
+ result,
+ stableId: dataset.stableId,
+ enabled: enabled
+ )
+ } catch {
+ updatePack(dataset.stableId) { pack in
+ pack.status = .download
+ pack.enabled = false
+ pack.errorMessage = userFacingKnowledgeError(error)
}
}
}
diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift
index 621f25ee31..e9feeae763 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift
@@ -146,8 +146,7 @@ final class LlmProvider {
func missingModelDownloadSize(_ selection: LlmModelSelection) async -> Int64? {
var total: Int64 = 0
- let chatMissing = !downloader.isDownloaded(selection.modelTarget)
- if chatMissing {
+ if !downloader.isDownloaded(selection.modelTarget) {
guard let chatSize = await downloader.estimateDownloadSize(selection.modelTarget) else {
return nil
}
@@ -172,14 +171,13 @@ final class LlmProvider {
}
let missingTargets: [ModelTarget] = try await modelLoadGate.withLock {
- let chatReady = downloader.isDownloaded(selection.modelTarget)
let embeddingReady = isEmbeddingModelReady()
if !embeddingReady {
_ = downloader.removeDownloaded(embeddingDownloadTarget)
}
return [
- chatReady ? nil : selection.modelTarget,
+ downloader.isDownloaded(selection.modelTarget) ? nil : selection.modelTarget,
embeddingReady ? nil : embeddingDownloadTarget,
].compactMap { $0 }
}
@@ -393,15 +391,7 @@ final class LlmProvider {
}
let threadCount = max(1, ProcessInfo.processInfo.activeProcessorCount - 1)
embeddingContext = try model.newEmbeddingContext(
- params: LlmEmbeddingContextParams(
- contextSize: knowledgeEmbedding.contextSize,
- nThreads: Int32(threadCount),
- batchSize: knowledgeEmbedding.batchSize,
- microBatchSize: knowledgeEmbedding.microBatchSize,
- sourceDim: knowledgeEmbedding.sourceDim,
- dim: knowledgeEmbedding.dim,
- queryPrompt: knowledgeEmbedding.queryPrompt
- )
+ nThreads: Int32(threadCount)
)
guard let context = embeddingContext else {
throw EmbeddingAssetInvalidError()
diff --git a/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swift b/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swift
index 51da91ed1b..588173833f 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Settings/KnowledgeSettingsView.swift
@@ -101,7 +101,7 @@ struct KnowledgeSettingsView: View {
if let error = pack.errorMessage {
Text(error)
.font(EnsuTypography.small)
- .foregroundStyle(.red)
+ .foregroundStyle(EnsuColor.error)
}
}
.padding(EnsuSpacing.lg)
diff --git a/mobile/native/apple/apps/ensu/Ensu/Settings/LegalSettingsViews.swift b/mobile/native/apple/apps/ensu/Ensu/Settings/LegalSettingsViews.swift
deleted file mode 100644
index 52a0cf17ab..0000000000
--- a/mobile/native/apple/apps/ensu/Ensu/Settings/LegalSettingsViews.swift
+++ /dev/null
@@ -1,44 +0,0 @@
-import SwiftUI
-
-enum EnsuLegalDocuments {
- static let privacyTitle = "Privacy Policy"
- static let privacyURL = URL(string: "https://ente.com/privacy/")!
- static let enteTermsTitle = "Ente Terms and Conditions"
- static let enteTermsURL = URL(string: "https://ente.com/terms/")!
-
- static let searchTerms = [
- privacyTitle,
- enteTermsTitle,
- "Terms of Service",
- ]
-}
-
-struct TermsAndConditionsView: View {
- var body: some View {
- ScrollView {
- VStack(alignment: .leading, spacing: EnsuSpacing.lg) {
- Link(destination: EnsuLegalDocuments.privacyURL) {
- SettingsCard(
- title: EnsuLegalDocuments.privacyTitle,
- iconName: "ViewIcon",
- showsChevron: true
- )
- }
- .buttonStyle(.plain)
-
- Link(destination: EnsuLegalDocuments.enteTermsURL) {
- SettingsCard(
- title: EnsuLegalDocuments.enteTermsTitle,
- iconName: "DescriptionIcon",
- showsChevron: true
- )
- }
- .buttonStyle(.plain)
- }
- .padding(EnsuSpacing.lg)
- }
- .background(EnsuColor.backgroundBase)
- .navigationTitle("Terms and Conditions")
- .navigationBarTitleDisplayMode(.inline)
- }
-}
diff --git a/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swift b/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swift
index 75baae4018..f905623047 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Settings/SettingsView.swift
@@ -21,7 +21,7 @@ struct SettingsView: View {
VStack(alignment: .leading, spacing: EnsuSpacing.lg) {
if let aboutItem = filteredAboutItem {
Button(action: aboutItem.action) {
- SettingsCard(title: aboutItem.title, iconName: aboutItem.iconName, showsChevron: true)
+ settingsCard(title: aboutItem.title, iconName: aboutItem.iconName, showsChevron: true)
}
.buttonStyle(.plain)
}
@@ -30,23 +30,21 @@ struct SettingsView: View {
NavigationLink {
item.destination
} label: {
- SettingsCard(title: item.title, iconName: item.iconName, showsChevron: true)
+ settingsCard(title: item.title, iconName: item.iconName, showsChevron: true)
}
.buttonStyle(.plain)
}
if shouldShowSignInRow {
Button(action: onSignIn) {
- SettingsCard(title: signInTitle, iconName: "Upload01Icon", showsChevron: true)
+ settingsCard(title: signInTitle, iconName: "Upload01Icon", showsChevron: true)
}
.buttonStyle(.plain)
}
- ForEach(filteredTermsItems) { item in
- NavigationLink {
- item.destination
- } label: {
- SettingsCard(title: item.title, iconName: item.iconName, showsChevron: true)
+ ForEach(filteredLegalLinkItems) { item in
+ Button(action: item.action) {
+ settingsCard(title: item.title, iconName: item.iconName, showsChevron: true)
}
.buttonStyle(.plain)
}
@@ -61,7 +59,7 @@ struct SettingsView: View {
NavigationLink {
item.destination
} label: {
- SettingsCard(title: item.title, iconName: item.iconName, showsChevron: true)
+ settingsCard(title: item.title, iconName: item.iconName, showsChevron: true)
}
.buttonStyle(.plain)
}
@@ -110,15 +108,6 @@ struct SettingsView: View {
}
}
- private var filteredTermsItems: [SettingsItem] {
- guard !trimmedQuery.isEmpty else { return termsItems }
- let q = trimmedQuery.lowercased()
- return termsItems.filter { item in
- item.title.lowercased().contains(q) ||
- item.searchTerms.contains { $0.lowercased().contains(q) }
- }
- }
-
private var filteredAdvancedItems: [SettingsItem] {
guard isAdvancedUnlocked else { return [] }
guard !trimmedQuery.isEmpty else { return advancedItems }
@@ -136,12 +125,35 @@ struct SettingsView: View {
)
}
+ private var legalLinkItems: [SettingsActionItem] {
+ [
+ SettingsActionItem(
+ title: "Privacy Policy",
+ iconName: "ViewIcon",
+ action: { openExternalLink("https://ente.com/privacy") }
+ ),
+ SettingsActionItem(
+ title: "Terms of Service",
+ iconName: "DescriptionIcon",
+ action: { openExternalLink("https://ente.com/terms") }
+ )
+ ]
+ }
+
private var filteredAboutItem: SettingsActionItem? {
guard !trimmedQuery.isEmpty else { return aboutItem }
let q = trimmedQuery.lowercased()
return aboutItem.title.lowercased().contains(q) ? aboutItem : nil
}
+ private var filteredLegalLinkItems: [SettingsActionItem] {
+ guard !trimmedQuery.isEmpty else { return legalLinkItems }
+ let q = trimmedQuery.lowercased()
+ return legalLinkItems.filter { item in
+ item.title.lowercased().contains(q)
+ }
+ }
+
private var shouldShowSignInRow: Bool {
guard !trimmedQuery.isEmpty else { return true }
let q = trimmedQuery.lowercased()
@@ -165,17 +177,6 @@ struct SettingsView: View {
]
}
- private var termsItems: [SettingsItem] {
- [
- SettingsItem(
- title: "Terms and Conditions",
- iconName: "DescriptionIcon",
- destination: AnyView(TermsAndConditionsView()),
- searchTerms: EnsuLegalDocuments.searchTerms
- )
- ]
- }
-
private var advancedItems: [SettingsItem] {
[
SettingsItem(
@@ -228,14 +229,8 @@ struct SettingsView: View {
guard let url = URL(string: urlString) else { return }
openURL(url)
}
-}
-
-struct SettingsCard: View {
- let title: String
- let iconName: String
- let showsChevron: Bool
- var body: some View {
+ private func settingsCard(title: String, iconName: String, showsChevron: Bool) -> some View {
HStack(spacing: EnsuSpacing.md) {
Image(iconName)
.resizable()
@@ -397,14 +392,6 @@ private struct SettingsItem: Identifiable {
let title: String
let iconName: String
let destination: AnyView
- let searchTerms: [String]
-
- init(title: String, iconName: String, destination: AnyView, searchTerms: [String] = []) {
- self.title = title
- self.iconName = iconName
- self.destination = destination
- self.searchTerms = searchTerms
- }
}
private struct SettingsActionItem: Identifiable {
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index e0e9fe30af..429f215f0b 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2394,7 +2394,6 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"transcribe-rs",
- "unicode-segmentation",
"url",
"uuid",
"zeroize",
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 6938a675f4..d59402326d 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -37,14 +37,11 @@ ente-rs = { path = "apps/cli" }
ente-space = { path = "space" }
ente-test-support = { path = "crates/test-support" }
ort = { git = "https://github.com/pykeio/ort.git", rev = "31c8f1443bb579bfddb82705288347697e468fb3", default-features = false }
-memmap2 = "0.9.9"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.149"
thiserror = "2.0.18"
uniffi = { version = "0.31.0" }
-url = "2.5.8"
wasm-bindgen = "0.2.125"
-zstd = "0.13.3"
# Keep transitive ORT users (notably transcribe-rs) on the same revision as direct ORT dependencies.
[patch.crates-io]
diff --git a/rust/bindings/uniffi/ensu/src/config.rs b/rust/bindings/uniffi/ensu/src/config.rs
index 739bca9154..c17d28aa78 100644
--- a/rust/bindings/uniffi/ensu/src/config.rs
+++ b/rust/bindings/uniffi/ensu/src/config.rs
@@ -12,15 +12,7 @@ pub struct ConfigModelPreset {
#[derive(Debug, Clone, uniffi::Record)]
pub struct KnowledgeEmbeddingConfig {
- pub model_url: String,
- pub model_sha256: String,
pub target_id: String,
- pub source_dim: u32,
- pub dim: u32,
- pub query_prompt: String,
- pub context_size: u32,
- pub batch_size: u32,
- pub micro_batch_size: u32,
pub max_hits: u32,
pub max_context_utf8_bytes: u32,
}
@@ -31,7 +23,6 @@ pub struct AttributionConfig {
pub license_label: String,
pub license_url: String,
pub public_pack_url: String,
- pub build_provenance: String,
pub modification_notice: String,
}
@@ -40,11 +31,7 @@ pub struct KnowledgeDatasetConfig {
pub stable_id: String,
pub label: String,
pub current_download_identity: String,
- pub artifact_base_url: String,
- pub artifact_sha256: Vec<String>,
pub download_size_bytes: i64,
- pub max_chars: u32,
- pub source_url_template: String,
pub relevance_threshold: f32,
pub attribution: AttributionConfig,
}
@@ -81,15 +68,7 @@ impl From<config::ModelPreset> for ConfigModelPreset {
impl From<config::KnowledgeEmbeddingConfig> for KnowledgeEmbeddingConfig {
fn from(value: config::KnowledgeEmbeddingConfig) -> Self {
Self {
- model_url: value.model_url,
- model_sha256: value.model_sha256,
target_id: value.target_id,
- source_dim: value.source_dim,
- dim: value.dim,
- query_prompt: value.query_prompt,
- context_size: value.context_size,
- batch_size: value.batch_size,
- micro_batch_size: value.micro_batch_size,
max_hits: value.max_hits,
max_context_utf8_bytes: value.max_context_utf8_bytes,
}
@@ -103,20 +82,6 @@ impl From<config::AttributionConfig> for AttributionConfig {
license_label: value.license_label,
license_url: value.license_url,
public_pack_url: value.public_pack_url,
- build_provenance: value.build_provenance,
- modification_notice: value.modification_notice,
- }
- }
-}
-
-impl From<AttributionConfig> for config::AttributionConfig {
- fn from(value: AttributionConfig) -> Self {
- Self {
- credit: value.credit,
- license_label: value.license_label,
- license_url: value.license_url,
- public_pack_url: value.public_pack_url,
- build_provenance: value.build_provenance,
modification_notice: value.modification_notice,
}
}
@@ -128,28 +93,7 @@ impl From<config::KnowledgeDatasetConfig> for KnowledgeDatasetConfig {
stable_id: value.stable_id,
label: value.label,
current_download_identity: value.current_download_identity,
- artifact_base_url: value.artifact_base_url,
- artifact_sha256: value.artifact_sha256,
- download_size_bytes: value.download_size_bytes,
- max_chars: value.max_chars,
- source_url_template: value.source_url_template,
- relevance_threshold: value.relevance_threshold,
- attribution: value.attribution.into(),
- }
- }
-}
-
-impl From<KnowledgeDatasetConfig> for config::KnowledgeDatasetConfig {
- fn from(value: KnowledgeDatasetConfig) -> Self {
- Self {
- stable_id: value.stable_id,
- label: value.label,
- current_download_identity: value.current_download_identity,
- artifact_base_url: value.artifact_base_url,
- artifact_sha256: value.artifact_sha256,
download_size_bytes: value.download_size_bytes,
- max_chars: value.max_chars,
- source_url_template: value.source_url_template,
relevance_threshold: value.relevance_threshold,
attribution: value.attribution.into(),
}
diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rs
index 9db3728cbd..7f5ab0703f 100644
--- a/rust/bindings/uniffi/ensu/src/llm.rs
+++ b/rust/bindings/uniffi/ensu/src/llm.rs
@@ -67,17 +67,6 @@ pub struct LlmContextParams {
pub n_batch: Option<i32>,
}
-#[derive(Debug, Clone, uniffi::Record)]
-pub struct LlmEmbeddingContextParams {
- pub context_size: u32,
- pub n_threads: Option<i32>,
- pub batch_size: u32,
- pub micro_batch_size: u32,
- pub source_dim: u32,
- pub dim: u32,
- pub query_prompt: String,
-}
-
#[derive(Debug, Clone, uniffi::Record)]
pub struct LlmChatMessage {
pub role: String,
@@ -144,10 +133,10 @@ impl LlmModel {
pub fn new_embedding_context(
&self,
- params: LlmEmbeddingContextParams,
+ n_threads: Option<i32>,
) -> Result<Arc<LlmContext>, LlmError> {
- let handle =
- llm::Context::new_embedding(&self.handle, params.into()).map_err(LlmError::from)?;
+ let handle = llm::Context::new_knowledge_embedding(&self.handle, n_threads)
+ .map_err(LlmError::from)?;
Ok(Arc::new(LlmContext { handle }))
}
}
@@ -212,20 +201,6 @@ impl From<LlmContextParams> for llm::ContextParams {
}
}
-impl From<LlmEmbeddingContextParams> for llm::EmbeddingContextParams {
- fn from(value: LlmEmbeddingContextParams) -> Self {
- Self {
- context_size: value.context_size,
- n_threads: value.n_threads,
- batch_size: value.batch_size,
- micro_batch_size: value.micro_batch_size,
- source_dim: value.source_dim,
- dim: value.dim,
- query_prompt: value.query_prompt,
- }
- }
-}
-
impl From<LlmChatMessage> for llm::ChatMessage {
fn from(value: LlmChatMessage) -> Self {
Self {
diff --git a/rust/bindings/uniffi/ensu/src/model.rs b/rust/bindings/uniffi/ensu/src/model.rs
index bd32856f46..2e3809ec3f 100644
--- a/rust/bindings/uniffi/ensu/src/model.rs
+++ b/rust/bindings/uniffi/ensu/src/model.rs
@@ -211,7 +211,7 @@ impl ModelDownloadCore {
#[derive(Default, uniffi::Object)]
pub struct CancellationToken {
- inner: ente_model_download::download::CancellationToken,
+ pub(crate) inner: ente_model_download::download::CancellationToken,
}
#[uniffi::export]
diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rs
index b03d839da5..d6d997c7eb 100644
--- a/rust/bindings/uniffi/ensu/src/retrieval.rs
+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs
@@ -3,8 +3,8 @@ use std::sync::Arc;
use ente_ensu::retrieval as core;
use thiserror::Error;
-use crate::config::KnowledgeDatasetConfig;
use crate::download::DownloadError;
+use crate::model::CancellationToken;
#[derive(Debug, Error, uniffi::Error)]
pub enum KnowledgeRetrievalError {
@@ -36,6 +36,16 @@ impl From<core::RetrievalError> for KnowledgeRetrievalError {
}
}
+fn knowledge_dataset(
+ stable_id: &str,
+) -> Result<ente_ensu::config::KnowledgeDatasetConfig, KnowledgeRetrievalError> {
+ ente_ensu::config::knowledge_dataset(stable_id).ok_or_else(|| {
+ KnowledgeRetrievalError::InvalidInput {
+ detail: format!("unknown knowledge dataset ID: {stable_id}"),
+ }
+ })
+}
+
#[derive(Debug, Error, uniffi::Error)]
pub enum KnowledgeDownloadError {
#[error("knowledge download failed")]
@@ -100,9 +110,9 @@ impl RetrievalIndex {
#[uniffi::constructor]
pub fn open(
directory: String,
- expected_dataset: KnowledgeDatasetConfig,
+ stable_id: String,
) -> Result<Arc<Self>, KnowledgeRetrievalError> {
- let inner = core::RetrievalIndex::open(directory, &expected_dataset.into())?;
+ let inner = core::RetrievalIndex::open(directory, &knowledge_dataset(&stable_id)?)?;
Ok(Arc::new(Self { inner }))
}
@@ -156,9 +166,9 @@ impl From<core::KnowledgeReconciliation> for KnowledgeReconciliation {
#[uniffi::export]
pub fn reconcile_knowledge_pack(
pack_root: String,
- expected_dataset: KnowledgeDatasetConfig,
+ stable_id: String,
) -> Result<KnowledgeReconciliation, KnowledgeRetrievalError> {
- core::reconcile_knowledge_pack(pack_root, &expected_dataset.into())
+ core::reconcile_knowledge_pack(pack_root, &knowledge_dataset(&stable_id)?)
.map(Into::into)
.map_err(Into::into)
}
@@ -166,12 +176,12 @@ pub fn reconcile_knowledge_pack(
#[uniffi::export]
pub fn cleanup_obsolete_knowledge_pack_revisions(
pack_root: String,
- expected_dataset: KnowledgeDatasetConfig,
+ stable_id: String,
active_identity: String,
) -> Result<(), KnowledgeRetrievalError> {
core::cleanup_obsolete_knowledge_pack_revisions(
pack_root,
- &expected_dataset.into(),
+ &knowledge_dataset(&stable_id)?,
&active_identity,
)
.map_err(Into::into)
@@ -183,7 +193,6 @@ pub struct SourceCitation {
pub dataset_label: String,
pub credit: String,
pub title: String,
- pub section: Option<String>,
pub source_url: String,
pub license_label: String,
pub license_url: String,
@@ -196,7 +205,6 @@ impl From<SourceCitation> for core::SourceCitation {
dataset_label: value.dataset_label,
credit: value.credit,
title: value.title,
- section: value.section,
source_url: value.source_url,
license_label: value.license_label,
license_url: value.license_url,
@@ -211,7 +219,6 @@ impl From<core::SourceCitation> for SourceCitation {
dataset_label: value.dataset_label,
credit: value.credit,
title: value.title,
- section: value.section,
source_url: value.source_url,
license_label: value.license_label,
license_url: value.license_url,
@@ -300,57 +307,29 @@ impl From<ente_model_download::download::Progress> for KnowledgeDownloadProgress
#[uniffi::export(callback_interface)]
pub trait KnowledgeDownloadCallback: Send + Sync {
fn on_progress(&self, progress: KnowledgeDownloadProgress);
- fn is_cancelled(&self) -> bool;
}
#[uniffi::export]
pub fn download_knowledge_pack(
pack_root: String,
- expected_dataset: KnowledgeDatasetConfig,
+ stable_id: String,
callback: Box<dyn KnowledgeDownloadCallback>,
+ cancellation: Arc<CancellationToken>,
) -> Result<(), KnowledgeDownloadError> {
- let callback: Arc<dyn KnowledgeDownloadCallback> = Arc::from(callback);
- let progress_callback = Arc::clone(&callback);
- let cancel_callback = Arc::clone(&callback);
+ let expected_dataset = ente_ensu::config::knowledge_dataset(&stable_id).ok_or_else(|| {
+ KnowledgeDownloadError::Download {
+ error: DownloadError::InvalidTarget {
+ message: format!("unknown knowledge dataset ID: {stable_id}"),
+ },
+ }
+ })?;
core::download_knowledge_pack(
pack_root,
- &expected_dataset.into(),
- move |progress| progress_callback.on_progress(progress.into()),
- move || cancel_callback.is_cancelled(),
+ &expected_dataset,
+ move |progress| callback.on_progress(progress.into()),
+ cancellation.inner.clone(),
)
.map_err(|error| KnowledgeDownloadError::Download {
error: error.into(),
})
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn builds_a_prompt_context_through_the_binding_types() {
- let context = build_knowledge_prompt_context(
- vec![KnowledgePromptHit {
- dataset_id: "simplewiki".to_string(),
- hit: RetrievalHit {
- score: 0.9,
- text: "A useful passage".to_string(),
- title: "Example".to_string(),
- section: None,
- source_url: "https://simple.wikipedia.org/wiki/Example".to_string(),
- },
- }],
- 6_000,
- )
- .unwrap()
- .unwrap();
-
- assert!(
- context
- .text
- .contains("# Example (Simple English Wikipedia)")
- );
- assert_eq!(context.citations.len(), 1);
- assert_eq!(context.citations[0].dataset_id, "simplewiki");
- }
-}
diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.toml
index 294ae4e0dd..5d04f6a385 100644
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2024"
ente-core.workspace = true
ente-image.workspace = true
ente-model-download.workspace = true
-memmap2.workspace = true
+memmap2 = "0.9.9"
parking_lot = "0.12"
regex = { version = "1", optional = true }
rubato = { version = "0.16", optional = true }
@@ -18,11 +18,10 @@ sha2 = "0.11.0"
thiserror.workspace = true
tokio = { version = "1", features = ["rt", "time"] }
transcribe-rs = { version = "0.3.11", default-features = false, features = ["onnx", "vad-silero"], optional = true }
-unicode-segmentation = "1.13.2"
-url.workspace = true
+url = "2.5.8"
uuid = { version = "1", features = ["v4", "rng-getrandom"] }
zeroize = { version = "1.8", features = ["derive"] }
-zstd.workspace = true
+zstd = "0.13.3"
[dev-dependencies]
tempfile = "3.27.0"
diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rs
index 10a75fdcb0..88a4d51582 100644
--- a/rust/crates/ensu/src/config.rs
+++ b/rust/crates/ensu/src/config.rs
@@ -1,11 +1,12 @@
mod knowledge;
+pub use knowledge::knowledge_dataset;
pub use knowledge::{AttributionConfig, KnowledgeDatasetConfig, KnowledgeEmbeddingConfig};
pub(crate) use knowledge::{
KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL, KNOWLEDGE_LICENSE_URL,
KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE, KNOWLEDGE_VECTORS_FILE,
- KnowledgeConfigError, is_path_safe_component, knowledge_artifact_urls, knowledge_datasets,
- knowledge_index_contract, validate_knowledge_datasets, validate_knowledge_embedding,
+ is_path_safe_component, knowledge_artifact_urls, knowledge_datasets,
+ knowledge_embedding_config, knowledge_index_contract,
};
#[derive(Debug, Clone)]
@@ -165,7 +166,7 @@ pub fn defaults() -> Defaults {
.expect("preset id is in the catalog")
.clone()
};
- let defaults = Defaults {
+ Defaults {
mobile_system_prompt_body: MOBILE_SYSTEM_PROMPT_BODY.to_string(),
desktop_system_prompt_body: DESKTOP_SYSTEM_PROMPT_BODY.to_string(),
system_prompt_date_placeholder: SYSTEM_PROMPT_DATE_PLACEHOLDER.to_string(),
@@ -187,26 +188,7 @@ pub fn defaults() -> Defaults {
voice_activity_model: silero_vad_v4(),
knowledge_embedding: knowledge::knowledge_embedding_config(),
knowledge_datasets: knowledge::knowledge_datasets(),
- };
-
- validate_defaults(&defaults).expect("bundled Ensu defaults must be valid");
- defaults
-}
-
-fn validate_defaults(defaults: &Defaults) -> Result<(), KnowledgeConfigError> {
- validate_knowledge_datasets(&defaults.knowledge_datasets)?;
-
- let reserved_model_ids = std::iter::once(&defaults.mobile_default_model)
- .chain(defaults.mobile_model_presets.iter())
- .chain(std::iter::once(&defaults.desktop_default_model))
- .chain(defaults.desktop_model_presets.iter())
- .chain([
- &defaults.transcription_model,
- &defaults.voice_activity_model,
- ])
- .map(|preset| preset.id.clone())
- .collect::<Vec<_>>();
- validate_knowledge_embedding(&defaults.knowledge_embedding, &reserved_model_ids)
+ }
}
#[cfg(test)]
diff --git a/rust/crates/ensu/src/config/knowledge.rs b/rust/crates/ensu/src/config/knowledge.rs
index c5a90f44fc..f0c44d88da 100644
--- a/rust/crates/ensu/src/config/knowledge.rs
+++ b/rust/crates/ensu/src/config/knowledge.rs
@@ -1,10 +1,3 @@
-mod validation;
-
-use thiserror::Error;
-
-pub(crate) use validation::is_path_safe_component;
-pub(crate) use validation::{validate_knowledge_datasets, validate_knowledge_embedding};
-
pub(crate) const KNOWLEDGE_LICENSE_LABEL: &str = "CC BY-SA 4.0";
pub(crate) const KNOWLEDGE_LICENSE_URL: &str = "https://creativecommons.org/licenses/by-sa/4.0/";
@@ -41,7 +34,6 @@ pub struct KnowledgeEmbeddingConfig {
pub target_id: String,
pub source_dim: u32,
pub dim: u32,
- pub matryoshka: bool,
pub query_prompt: String,
pub context_size: u32,
pub batch_size: u32,
@@ -56,7 +48,6 @@ pub struct AttributionConfig {
pub license_label: String,
pub license_url: String,
pub public_pack_url: String,
- pub build_provenance: String,
pub modification_notice: String,
}
@@ -88,16 +79,6 @@ pub struct KnowledgeIndexContract {
pub meta_rows_per_block: u32,
}
-#[derive(Debug, Clone, PartialEq, Eq, Error)]
-#[error("{0}")]
-pub struct KnowledgeConfigError(String);
-
-impl KnowledgeConfigError {
- fn invalid(field: &str, reason: impl AsRef<str>) -> Self {
- Self(format!("invalid {field}: {}", reason.as_ref()))
- }
-}
-
pub(crate) fn knowledge_index_contract() -> KnowledgeIndexContract {
KnowledgeIndexContract {
model: EMBEDDING_MODEL_IDENTITY.to_owned(),
@@ -114,15 +95,13 @@ pub(crate) fn knowledge_index_contract() -> KnowledgeIndexContract {
}
pub(crate) fn knowledge_embedding_config() -> KnowledgeEmbeddingConfig {
- let contract = knowledge_index_contract();
KnowledgeEmbeddingConfig {
model_url: EMBEDDING_MODEL_URL.to_owned(),
model_sha256: EMBEDDING_MODEL_SHA256.to_owned(),
target_id: EMBEDDING_TARGET_ID.to_owned(),
- source_dim: contract.source_dim,
- dim: contract.dim,
- matryoshka: contract.matryoshka,
- query_prompt: contract.query_prompt,
+ source_dim: SOURCE_DIM,
+ dim: RETRIEVAL_DIM,
+ query_prompt: QUERY_PROMPT.to_owned(),
context_size: EMBEDDING_CONTEXT_SIZE,
batch_size: EMBEDDING_CONTEXT_SIZE,
micro_batch_size: EMBEDDING_CONTEXT_SIZE,
@@ -153,7 +132,6 @@ pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {
license_label: KNOWLEDGE_LICENSE_LABEL.to_owned(),
license_url: KNOWLEDGE_LICENSE_URL.to_owned(),
public_pack_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/tree/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/simplewiki".to_owned(),
- build_provenance: "Simple English Wikipedia dump dated 2026-07-02".to_owned(),
modification_notice: MODIFICATION_NOTICE.to_owned(),
},
},
@@ -177,18 +155,30 @@ pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {
license_label: KNOWLEDGE_LICENSE_LABEL.to_owned(),
license_url: KNOWLEDGE_LICENSE_URL.to_owned(),
public_pack_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/tree/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/wikibooks".to_owned(),
- build_provenance: "English Wikibooks dump dated 2026-07-02".to_owned(),
modification_notice: MODIFICATION_NOTICE.to_owned(),
},
},
]
}
-pub(crate) fn knowledge_artifact_urls(
- dataset: &KnowledgeDatasetConfig,
-) -> Result<[String; 4], KnowledgeConfigError> {
- validation::validate_artifact_base_url(&dataset.artifact_base_url, "artifact_base_url")?;
- Ok(KNOWLEDGE_ARTIFACT_FILENAMES.map(|name| format!("{}{name}", dataset.artifact_base_url)))
+pub fn knowledge_dataset(stable_id: &str) -> Option<KnowledgeDatasetConfig> {
+ knowledge_datasets()
+ .into_iter()
+ .find(|dataset| dataset.stable_id == stable_id)
+}
+
+pub(crate) fn knowledge_artifact_urls(dataset: &KnowledgeDatasetConfig) -> [String; 4] {
+ KNOWLEDGE_ARTIFACT_FILENAMES.map(|name| format!("{}{name}", dataset.artifact_base_url))
+}
+
+pub(crate) fn is_path_safe_component(value: &str) -> bool {
+ !value.is_empty()
+ && value.len() <= 255
+ && value != "."
+ && value != ".."
+ && !value.chars().any(|character| {
+ character.is_control() || character == '/' || character == '\\' || character == ':'
+ })
}
#[cfg(test)]
@@ -197,7 +187,7 @@ mod tests {
#[test]
fn artifact_urls_append_only_fixed_names() {
- let urls = knowledge_artifact_urls(&knowledge_datasets()[0]).unwrap();
+ let urls = knowledge_artifact_urls(&knowledge_datasets()[0]);
assert_eq!(
urls.each_ref()
.map(|url| url.rsplit('/').next().unwrap().to_owned()),
@@ -211,9 +201,7 @@ mod tests {
#[test]
fn pinned_catalog_and_embedding_are_valid() {
let datasets = knowledge_datasets();
- validate_knowledge_datasets(&datasets).unwrap();
let embedding = knowledge_embedding_config();
- validate_knowledge_embedding(&embedding, &[]).unwrap();
assert_eq!(datasets.len(), 2);
assert_eq!(datasets[0].stable_id, "simplewiki");
@@ -235,5 +223,10 @@ mod tests {
assert_eq!(embedding.model_sha256.len(), 64);
assert_eq!(knowledge_index_contract().source_dim, 768);
assert_eq!(knowledge_index_contract().dim, 512);
+ assert!(datasets.iter().all(|dataset| {
+ is_path_safe_component(&dataset.stable_id)
+ && is_path_safe_component(&dataset.current_download_identity)
+ }));
+ assert!(knowledge_dataset("missing").is_none());
}
}
diff --git a/rust/crates/ensu/src/config/knowledge/validation.rs b/rust/crates/ensu/src/config/knowledge/validation.rs
deleted file mode 100644
index d7851b40ea..0000000000
--- a/rust/crates/ensu/src/config/knowledge/validation.rs
+++ /dev/null
@@ -1,443 +0,0 @@
-use std::collections::HashSet;
-
-use url::Url;
-
-use super::{
- EMBEDDING_CONTEXT_SIZE, KNOWLEDGE_ARTIFACT_FILENAMES, KNOWLEDGE_LICENSE_LABEL,
- KNOWLEDGE_LICENSE_URL, KnowledgeConfigError, KnowledgeDatasetConfig, KnowledgeEmbeddingConfig,
- MAX_CONTEXT_UTF8_BYTES, MAX_HITS, knowledge_index_contract,
-};
-
-const MAX_DATASETS: usize = 32;
-const MAX_PATH_COMPONENT_BYTES: usize = 96;
-const MAX_DISPLAY_BYTES: usize = 256;
-const MAX_URL_BYTES: usize = 2_048;
-const MAX_PROVENANCE_BYTES: usize = 4_096;
-
-pub fn validate_knowledge_datasets(
- datasets: &[KnowledgeDatasetConfig],
-) -> Result<(), KnowledgeConfigError> {
- if !(1..=MAX_DATASETS).contains(&datasets.len()) {
- return Err(KnowledgeConfigError::invalid(
- "knowledge_datasets",
- format!("must contain between 1 and {MAX_DATASETS} records"),
- ));
- }
-
- let mut stable_ids = HashSet::with_capacity(datasets.len());
- let mut current_identities = HashSet::with_capacity(datasets.len());
- for (index, dataset) in datasets.iter().enumerate() {
- let prefix = format!("knowledge_datasets[{index}]");
- validate_path_component(&dataset.stable_id, &format!("{prefix}.stable_id"))?;
- if !stable_ids.insert(dataset.stable_id.as_str()) {
- return Err(KnowledgeConfigError::invalid(
- &format!("{prefix}.stable_id"),
- "must be unique",
- ));
- }
-
- validate_path_component(
- &dataset.current_download_identity,
- &format!("{prefix}.current_download_identity"),
- )?;
- if !current_identities.insert(dataset.current_download_identity.as_str()) {
- return Err(KnowledgeConfigError::invalid(
- &format!("{prefix}.current_download_identity"),
- "must be unique",
- ));
- }
-
- validate_display_text(&dataset.label, &format!("{prefix}.label"))?;
- validate_artifact_base_url(
- &dataset.artifact_base_url,
- &format!("{prefix}.artifact_base_url"),
- )?;
- if dataset.artifact_sha256.len() != KNOWLEDGE_ARTIFACT_FILENAMES.len()
- || dataset.artifact_sha256.iter().any(|sha256| {
- sha256.len() != 64 || !sha256.chars().all(|ch| ch.is_ascii_hexdigit())
- })
- {
- return Err(KnowledgeConfigError::invalid(
- &format!("{prefix}.artifact_sha256"),
- "must provide one SHA-256 digest per knowledge artifact",
- ));
- }
- if dataset.download_size_bytes <= 0 {
- return Err(KnowledgeConfigError::invalid(
- &format!("{prefix}.download_size_bytes"),
- "must be greater than zero",
- ));
- }
- validate_source_url_template(
- &dataset.source_url_template,
- &format!("{prefix}.source_url_template"),
- )?;
- if !(1..=4_096).contains(&dataset.max_chars) {
- return Err(KnowledgeConfigError::invalid(
- &format!("{prefix}.max_chars"),
- "must be between 1 and 4096",
- ));
- }
- if !dataset.relevance_threshold.is_finite()
- || !(0.0..=1.0).contains(&dataset.relevance_threshold)
- {
- return Err(KnowledgeConfigError::invalid(
- &format!("{prefix}.relevance_threshold"),
- "must be finite and between 0 and 1",
- ));
- }
-
- let attribution = &dataset.attribution;
- validate_display_text(&attribution.credit, &format!("{prefix}.attribution.credit"))?;
- validate_display_text(
- &attribution.license_label,
- &format!("{prefix}.attribution.license_label"),
- )?;
- validate_https_url(
- &attribution.license_url,
- &format!("{prefix}.attribution.license_url"),
- )?;
- if attribution.license_label != KNOWLEDGE_LICENSE_LABEL
- || attribution.license_url != KNOWLEDGE_LICENSE_URL
- {
- return Err(KnowledgeConfigError::invalid(
- &format!("{prefix}.attribution"),
- "must use the v1 CC BY-SA 4.0 license",
- ));
- }
- validate_https_url(
- &attribution.public_pack_url,
- &format!("{prefix}.attribution.public_pack_url"),
- )?;
- validate_provenance_text(
- &attribution.build_provenance,
- &format!("{prefix}.attribution.build_provenance"),
- )?;
- validate_provenance_text(
- &attribution.modification_notice,
- &format!("{prefix}.attribution.modification_notice"),
- )?;
- }
-
- Ok(())
-}
-
-pub fn validate_knowledge_embedding(
- config: &KnowledgeEmbeddingConfig,
- reserved_model_ids: &[String],
-) -> Result<(), KnowledgeConfigError> {
- validate_path_component(&config.target_id, "knowledge_embedding.target_id")?;
- if config.target_id.starts_with("custom") {
- return Err(KnowledgeConfigError::invalid(
- "knowledge_embedding.target_id",
- "must not start with reserved prefix 'custom'",
- ));
- }
- if reserved_model_ids.iter().any(|id| id == &config.target_id) {
- return Err(KnowledgeConfigError::invalid(
- "knowledge_embedding.target_id",
- "must not alias another model target",
- ));
- }
- validate_embedding_model_url(&config.model_url, "knowledge_embedding.model_url")?;
- if config.model_sha256.len() != 64
- || !config.model_sha256.chars().all(|ch| ch.is_ascii_hexdigit())
- {
- return Err(KnowledgeConfigError::invalid(
- "knowledge_embedding.model_sha256",
- "must be a SHA-256 digest",
- ));
- }
- let contract = knowledge_index_contract();
- let exact_fields_match = config.source_dim == contract.source_dim
- && config.dim == contract.dim
- && config.matryoshka == contract.matryoshka
- && config.query_prompt == contract.query_prompt
- && config.context_size == EMBEDDING_CONTEXT_SIZE
- && config.batch_size == EMBEDDING_CONTEXT_SIZE
- && config.micro_batch_size == EMBEDDING_CONTEXT_SIZE
- && config.max_hits == MAX_HITS
- && config.max_context_utf8_bytes == MAX_CONTEXT_UTF8_BYTES;
- if !exact_fields_match {
- return Err(KnowledgeConfigError::invalid(
- "knowledge_embedding",
- "must match the v1 embedding and retrieval contract",
- ));
- }
-
- Ok(())
-}
-
-fn validate_path_component(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {
- if is_path_safe_component(value) {
- Ok(())
- } else {
- Err(KnowledgeConfigError::invalid(
- field,
- "must be a path-safe 1-96 byte ASCII component",
- ))
- }
-}
-
-pub(crate) fn is_path_safe_component(value: &str) -> bool {
- let bytes = value.as_bytes();
- let valid_edge = |byte: u8| byte.is_ascii_lowercase() || byte.is_ascii_digit();
- let valid_body = |byte: u8| valid_edge(byte) || matches!(byte, b'.' | b'_' | b'-');
- !bytes.is_empty()
- && bytes.len() <= MAX_PATH_COMPONENT_BYTES
- && valid_edge(bytes[0])
- && valid_edge(bytes[bytes.len() - 1])
- && bytes.iter().copied().all(valid_body)
-}
-
-fn validate_display_text(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {
- validate_text(value, field, MAX_DISPLAY_BYTES)
-}
-
-fn validate_provenance_text(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {
- validate_text(value, field, MAX_PROVENANCE_BYTES)
-}
-
-fn validate_text(value: &str, field: &str, max_bytes: usize) -> Result<(), KnowledgeConfigError> {
- if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) {
- Err(KnowledgeConfigError::invalid(
- field,
- format!("must be nonempty, control-free, and at most {max_bytes} UTF-8 bytes"),
- ))
- } else {
- Ok(())
- }
-}
-
-fn parse_https_url(value: &str, field: &str) -> Result<Url, KnowledgeConfigError> {
- if value.len() > MAX_URL_BYTES || value.chars().any(char::is_control) {
- return Err(KnowledgeConfigError::invalid(
- field,
- format!("must be control-free and at most {MAX_URL_BYTES} UTF-8 bytes"),
- ));
- }
- let url = Url::parse(value)
- .map_err(|error| KnowledgeConfigError::invalid(field, error.to_string()))?;
- if url.scheme() != "https"
- || url.host_str().is_none()
- || !url.username().is_empty()
- || url.password().is_some()
- {
- return Err(KnowledgeConfigError::invalid(
- field,
- "must be an HTTPS URL without credentials",
- ));
- }
- Ok(url)
-}
-
-fn validate_https_url(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {
- parse_https_url(value, field).map(|_| ())
-}
-
-fn validate_source_url_template(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {
- if value.len() > MAX_URL_BYTES {
- return Err(KnowledgeConfigError::invalid(
- field,
- format!("must be at most {MAX_URL_BYTES} UTF-8 bytes"),
- ));
- }
- if value.match_indices("{title}").count() != 1 {
- return Err(KnowledgeConfigError::invalid(
- field,
- "must contain exactly one {title} placeholder",
- ));
- }
- let without_title = value.replacen("{title}", "", 1);
- if without_title.contains(['{', '}']) {
- return Err(KnowledgeConfigError::invalid(
- field,
- "must not contain another brace-delimited placeholder",
- ));
- }
- parse_https_url(&value.replacen("{title}", "title", 1), field).map(|_| ())
-}
-
-pub(super) fn validate_artifact_base_url(
- value: &str,
- field: &str,
-) -> Result<(), KnowledgeConfigError> {
- let url = parse_https_url(value, field)?;
- let segments = url
- .path_segments()
- .map(|segments| segments.collect::<Vec<_>>())
- .ok_or_else(|| KnowledgeConfigError::invalid(field, "must have path segments"))?;
- let pinned = url.host_str() == Some("huggingface.co")
- && url.port().is_none()
- && url.query().is_none()
- && url.fragment().is_none()
- && value.ends_with('/')
- && segments.len() >= 8
- && segments[..4] == ["datasets", "ente-ai", "ensu-knowledge-packs", "resolve"]
- && is_commit_hash(segments[4]);
- if pinned {
- Ok(())
- } else {
- Err(KnowledgeConfigError::invalid(
- field,
- "must be an immutable Ente knowledge-pack resolver base URL",
- ))
- }
-}
-
-fn validate_embedding_model_url(value: &str, field: &str) -> Result<(), KnowledgeConfigError> {
- let url = parse_https_url(value, field)?;
- let segments = url
- .path_segments()
- .map(|segments| segments.collect::<Vec<_>>())
- .ok_or_else(|| KnowledgeConfigError::invalid(field, "must have path segments"))?;
- let pinned = url.host_str() == Some("huggingface.co")
- && url.port().is_none()
- && url.query().is_none()
- && url.fragment().is_none()
- && segments.len() == 5
- && segments[..3] == ["ente-ai", "embeddinggemma-300m-gguf", "resolve"]
- && is_commit_hash(segments[3])
- && !segments[4].is_empty();
- if pinned {
- Ok(())
- } else {
- Err(KnowledgeConfigError::invalid(
- field,
- "must be an immutable Ente EmbeddingGemma resolver URL",
- ))
- }
-}
-
-fn is_commit_hash(value: &str) -> bool {
- value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit())
-}
-
-#[cfg(test)]
-mod tests {
- use super::super::{EMBEDDING_TARGET_ID, knowledge_datasets, knowledge_embedding_config};
- use super::*;
-
- fn valid_datasets() -> Vec<KnowledgeDatasetConfig> {
- knowledge_datasets()
- }
-
- #[test]
- fn rejects_empty_oversized_and_duplicate_catalogs() {
- assert!(validate_knowledge_datasets(&[]).is_err());
-
- let mut too_many = Vec::new();
- for index in 0..=MAX_DATASETS {
- let mut dataset = valid_datasets().remove(0);
- dataset.stable_id = format!("pack-{index}");
- dataset.current_download_identity = format!("revision-{index}");
- too_many.push(dataset);
- }
- assert!(validate_knowledge_datasets(&too_many).is_err());
-
- let mut duplicate = valid_datasets();
- duplicate[1].stable_id = duplicate[0].stable_id.clone();
- assert!(validate_knowledge_datasets(&duplicate).is_err());
- duplicate[1].stable_id = "wikibooks".to_owned();
- duplicate[1].current_download_identity = duplicate[0].current_download_identity.clone();
- assert!(validate_knowledge_datasets(&duplicate).is_err());
- }
-
- #[test]
- fn rejects_path_unsafe_ids_and_boundary_overflow() {
- for invalid in [
- "",
- ".",
- "..",
- "UPPER",
- "-starts",
- "ends-",
- "has/slash",
- " spaced ",
- ] {
- let mut datasets = valid_datasets();
- datasets[0].stable_id = invalid.to_owned();
- assert!(
- validate_knowledge_datasets(&datasets).is_err(),
- "accepted {invalid:?}"
- );
- }
-
- let mut datasets = valid_datasets();
- datasets[0].stable_id = format!("a{}z", "b".repeat(MAX_PATH_COMPONENT_BYTES - 2));
- validate_knowledge_datasets(&datasets).unwrap();
- datasets[0].stable_id.push('z');
- assert!(validate_knowledge_datasets(&datasets).is_err());
- }
-
- #[test]
- fn rejects_invalid_dataset_strings_numbers_and_urls() {
- let mut datasets = valid_datasets();
- datasets[0].label = "\u{0000}".to_owned();
- assert!(validate_knowledge_datasets(&datasets).is_err());
-
- let mut datasets = valid_datasets();
- datasets[0].attribution.build_provenance = "x".repeat(MAX_PROVENANCE_BYTES + 1);
- assert!(validate_knowledge_datasets(&datasets).is_err());
-
- let mut datasets = valid_datasets();
- datasets[0].max_chars = 0;
- assert!(validate_knowledge_datasets(&datasets).is_err());
- datasets[0].max_chars = 4_097;
- assert!(validate_knowledge_datasets(&datasets).is_err());
-
- let mut datasets = valid_datasets();
- datasets[0].download_size_bytes = 0;
- assert!(validate_knowledge_datasets(&datasets).is_err());
- datasets[0].download_size_bytes = -1;
- assert!(validate_knowledge_datasets(&datasets).is_err());
-
- let mut datasets = valid_datasets();
- datasets[0].artifact_sha256.pop();
- assert!(validate_knowledge_datasets(&datasets).is_err());
- datasets[0]
- .artifact_sha256
- .push("not-a-checksum".to_owned());
- assert!(validate_knowledge_datasets(&datasets).is_err());
-
- let mut datasets = valid_datasets();
- datasets[0].relevance_threshold = f32::NAN;
- assert!(validate_knowledge_datasets(&datasets).is_err());
- datasets[0].relevance_threshold = 1.01;
- assert!(validate_knowledge_datasets(&datasets).is_err());
-
- let mut datasets = valid_datasets();
- datasets[0].artifact_base_url = datasets[0].artifact_base_url.replace(
- "/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/",
- "/resolve/main/",
- );
- assert!(validate_knowledge_datasets(&datasets).is_err());
-
- let mut datasets = valid_datasets();
- datasets[0].source_url_template = "http://example.com/{title}/{section}".to_owned();
- assert!(validate_knowledge_datasets(&datasets).is_err());
- }
-
- #[test]
- fn rejects_invalid_embedding_contract_and_model_aliases() {
- let mut config = knowledge_embedding_config();
- config.target_id = "custom-embedding".to_owned();
- assert!(validate_knowledge_embedding(&config, &[]).is_err());
-
- let mut config = knowledge_embedding_config();
- assert!(validate_knowledge_embedding(&config, &[config.target_id.clone()]).is_err());
- config.target_id = EMBEDDING_TARGET_ID.to_owned();
- config.model_url = config
- .model_url
- .replace("957b55764bd672f51240ac026e3a23ac9459ee3c", "main");
- assert!(validate_knowledge_embedding(&config, &[]).is_err());
-
- let mut config = knowledge_embedding_config();
- config.model_sha256 = "not-a-checksum".to_owned();
- assert!(validate_knowledge_embedding(&config, &[]).is_err());
-
- let mut config = knowledge_embedding_config();
- config.dim = 768;
- assert!(validate_knowledge_embedding(&config, &[]).is_err());
- }
-}
diff --git a/rust/crates/ensu/src/llm/context.rs b/rust/crates/ensu/src/llm/context.rs
index 5585b84758..fdf3707573 100644
--- a/rust/crates/ensu/src/llm/context.rs
+++ b/rust/crates/ensu/src/llm/context.rs
@@ -272,6 +272,25 @@ impl Context {
Ok(Arc::new(context))
}
+ pub fn new_knowledge_embedding(
+ model: &ModelRef,
+ n_threads: Option<i32>,
+ ) -> Result<ContextRef, Error> {
+ let config = crate::config::knowledge_embedding_config();
+ Self::new_embedding(
+ model,
+ EmbeddingContextParams {
+ context_size: config.context_size,
+ n_threads,
+ batch_size: config.batch_size,
+ micro_batch_size: config.micro_batch_size,
+ source_dim: config.source_dim,
+ dim: config.dim,
+ query_prompt: config.query_prompt,
+ },
+ )
+ }
+
pub fn prewarm_multimodal(
&self,
mmproj_path: String,
diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rs
index 4aba9e0dbb..91fd1a1529 100644
--- a/rust/crates/ensu/src/model/mod.rs
+++ b/rust/crates/ensu/src/model/mod.rs
@@ -82,7 +82,7 @@ pub fn voice_activity_target() -> ModelTarget {
}
pub fn knowledge_embedding_target() -> ModelTarget {
- let embedding = config::defaults().knowledge_embedding;
+ let embedding = config::knowledge_embedding_config();
ModelTarget::Files {
id: embedding.target_id,
files: vec![ModelFile {
diff --git a/rust/crates/ensu/src/retrieval/citation.rs b/rust/crates/ensu/src/retrieval/citation.rs
index a2f472f70f..e74e097840 100644
--- a/rust/crates/ensu/src/retrieval/citation.rs
+++ b/rust/crates/ensu/src/retrieval/citation.rs
@@ -20,7 +20,6 @@ pub struct SourceCitation {
pub dataset_label: String,
pub credit: String,
pub title: String,
- pub section: Option<String>,
pub source_url: String,
pub license_label: String,
pub license_url: String,
@@ -85,16 +84,7 @@ fn normalize_and_deduplicate(
}
let dataset_label = normalize_required(&citation.dataset_label, "dataset label")?;
let credit = normalize_required(&citation.credit, "credit")?;
- let mut title = normalize_required(&citation.title, "title")?;
- if let Some(section) = citation
- .section
- .as_deref()
- .map(normalize_single_line)
- .filter(|section| !section.is_empty())
- {
- title.push_str(" β€” ");
- title.push_str(&section);
- }
+ let title = normalize_required(&citation.title, "title")?;
let license_label = normalize_required(&citation.license_label, "license label")?;
if license_label != LICENSE_LABEL || citation.license_url != LICENSE_URL {
return Err(RetrievalError::InvalidInput(
@@ -112,7 +102,6 @@ fn normalize_and_deduplicate(
dataset_label,
credit,
title,
- section: None,
source_url: citation.source_url.clone(),
license_label,
license_url: citation.license_url.clone(),
@@ -206,7 +195,6 @@ fn parse_footer(footer: &str) -> Option<Vec<SourceCitation>> {
dataset_label: dataset_label.to_owned(),
credit: credit.to_owned(),
title: title.to_owned(),
- section: None,
source_url: source_url.to_owned(),
license_label: license_label.to_owned(),
license_url: license_url.to_owned(),
@@ -288,7 +276,6 @@ mod tests {
dataset_label: dataset_label.to_owned(),
credit: format!("{dataset_label} contributors"),
title: "Example title".to_owned(),
- section: None,
source_url: url.to_owned(),
license_label: LICENSE_LABEL.to_owned(),
license_url: LICENSE_URL.to_owned(),
@@ -302,19 +289,18 @@ mod tests {
"Simple English Wikipedia",
"https://simple.wikipedia.org/wiki/Example",
);
- let mut second = citation(
+ let second = citation(
"wikibooks",
"Wikibooks",
"https://en.wikibooks.org/wiki/Tea",
);
- second.section = Some("Brewing".to_owned());
let finalized = finalize_assistant_text("An answer", &[first, second]).unwrap();
assert!(!finalized.ends_with('\n'));
let parsed = parse_assistant_text(&finalized);
assert_eq!(parsed.text, "An answer");
assert_eq!(parsed.citations.len(), 2);
- assert_eq!(parsed.citations[1].title, "Example title β€” Brewing");
+ assert_eq!(parsed.citations[1].title, "Example title");
assert_eq!(
knowledge_source_chip_label(&parsed.citations).as_deref(),
Some("Simple English Wikipedia +1")
diff --git a/rust/crates/ensu/src/retrieval/index.rs b/rust/crates/ensu/src/retrieval/index.rs
index 3e931f9d98..aedcabff1a 100644
--- a/rust/crates/ensu/src/retrieval/index.rs
+++ b/rust/crates/ensu/src/retrieval/index.rs
@@ -11,7 +11,6 @@ use url::Url;
use crate::config::{
KNOWLEDGE_MANIFEST_FILE, KNOWLEDGE_META_FILE, KNOWLEDGE_OFFSETS_FILE, KNOWLEDGE_VECTORS_FILE,
KnowledgeDatasetConfig, is_path_safe_component, knowledge_index_contract,
- validate_knowledge_datasets,
};
use super::{BEGIN_CONTEXT_SENTINEL, END_CONTEXT_SENTINEL, RetrievalError, normalize_single_line};
@@ -30,7 +29,6 @@ pub struct RetrievalHit {
#[derive(Debug, Deserialize)]
struct Manifest {
dataset: String,
- granularity: String,
max_chars: u32,
model: String,
source_dim: u32,
@@ -48,7 +46,6 @@ struct Manifest {
#[derive(Debug, Deserialize)]
struct MetadataRow {
- id: serde_json::Value,
title: String,
text: String,
#[serde(default)]
@@ -73,8 +70,6 @@ impl RetrievalIndex {
directory: impl AsRef<Path>,
expected_pack: &KnowledgeDatasetConfig,
) -> Result<Self, RetrievalError> {
- validate_knowledge_datasets(std::slice::from_ref(expected_pack))
- .map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;
let directory = directory.as_ref();
let directory_metadata = fs::symlink_metadata(directory)?;
if !directory_metadata.file_type().is_dir() {
@@ -270,11 +265,6 @@ impl RetrievalIndex {
))
})?;
let row: MetadataRow = serde_json::from_str(line)?;
- if row.id.is_null() {
- return Err(RetrievalError::InvalidPack(
- "metadata row id must not be null".to_string(),
- ));
- }
if row.title.trim().is_empty() {
return Err(RetrievalError::InvalidPack(
"metadata title must not be empty".to_string(),
@@ -406,14 +396,6 @@ fn validate_manifest(
"manifest dataset must equal its path-safe revision directory".to_string(),
));
}
- if manifest.granularity.trim().is_empty()
- || manifest.granularity.len() > 256
- || manifest.granularity.chars().any(char::is_control)
- {
- return Err(RetrievalError::InvalidPack(
- "manifest granularity is invalid".to_string(),
- ));
- }
if manifest.count == 0 || manifest.dim == 0 || manifest.meta_rows_per_block == 0 {
return Err(RetrievalError::InvalidPack(
"manifest count, dim, and metadata rows per block must be positive".to_string(),
diff --git a/rust/crates/ensu/src/retrieval/pack.rs b/rust/crates/ensu/src/retrieval/pack.rs
index 7a97283c8a..a0add74dd0 100644
--- a/rust/crates/ensu/src/retrieval/pack.rs
+++ b/rust/crates/ensu/src/retrieval/pack.rs
@@ -5,7 +5,7 @@ use ente_model_download::download::{self, Downloader, Progress, Target};
use crate::config::{
KNOWLEDGE_ARTIFACT_FILENAMES, KnowledgeDatasetConfig, is_path_safe_component,
- knowledge_artifact_urls, validate_knowledge_datasets,
+ knowledge_artifact_urls,
};
use super::{RetrievalError, index::RetrievalIndex};
@@ -13,20 +13,22 @@ use super::{RetrievalError, index::RetrievalIndex};
pub fn download_knowledge_pack(
pack_root: impl AsRef<Path>,
expected_pack: &KnowledgeDatasetConfig,
- on_progress: impl FnMut(Progress),
- is_cancelled: impl Fn() -> bool,
+ on_progress: impl FnMut(Progress) + Send,
+ cancellation: download::CancellationToken,
) -> Result<(), download::Error> {
let pack_root = pack_root.as_ref();
let targets = knowledge_download_targets(pack_root, expected_pack)?;
- Downloader::new()?.download_blocking(targets, on_progress, is_cancelled)
+ tokio::runtime::Builder::new_current_thread()
+ .enable_io()
+ .enable_time()
+ .build()?
+ .block_on(Downloader::new()?.download(targets, on_progress, cancellation))
}
fn knowledge_download_targets(
pack_root: &Path,
expected_pack: &KnowledgeDatasetConfig,
) -> Result<Vec<Target>, download::Error> {
- validate_knowledge_datasets(std::slice::from_ref(expected_pack))
- .map_err(|error| download::Error::InvalidTarget(error.to_string()))?;
let root_identity = pack_root
.file_name()
.and_then(|name| name.to_str())
@@ -57,8 +59,7 @@ fn knowledge_download_targets(
));
}
- let urls = knowledge_artifact_urls(expected_pack)
- .map_err(|error| download::Error::InvalidTarget(error.to_string()))?;
+ let urls = knowledge_artifact_urls(expected_pack);
Ok(KNOWLEDGE_ARTIFACT_FILENAMES
.into_iter()
.zip(urls)
@@ -120,7 +121,7 @@ pub fn reconcile_knowledge_pack(
let active = if let Some(position) = current_position {
Some(valid.swap_remove(position))
} else {
- valid.sort_by(|left, right| left.0.cmp(&right.0));
+ valid.sort_by(|left, right| right.0.cmp(&left.0));
(!valid.is_empty()).then(|| valid.remove(0))
};
@@ -184,8 +185,6 @@ fn validate_pack_root(
expected_pack: &KnowledgeDatasetConfig,
create: bool,
) -> Result<(), RetrievalError> {
- validate_knowledge_datasets(std::slice::from_ref(expected_pack))
- .map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;
let root_identity = pack_root
.file_name()
.and_then(|name| name.to_str())
diff --git a/rust/crates/ensu/src/retrieval/prompt.rs b/rust/crates/ensu/src/retrieval/prompt.rs
index 018ae33e64..c6493aeb29 100644
--- a/rust/crates/ensu/src/retrieval/prompt.rs
+++ b/rust/crates/ensu/src/retrieval/prompt.rs
@@ -1,5 +1,3 @@
-use unicode_segmentation::UnicodeSegmentation;
-
use crate::config::knowledge_datasets;
use super::{
@@ -69,7 +67,7 @@ pub fn build_knowledge_prompt_context(
break;
}
- let passage = truncate_utf8_graphemes(&item.hit.text, available).trim();
+ let passage = truncate_utf8(&item.hit.text, available).trim();
if passage.is_empty() {
break;
}
@@ -80,8 +78,7 @@ pub fn build_knowledge_prompt_context(
dataset_id: dataset.stable_id.clone(),
dataset_label: dataset.label.clone(),
credit: dataset.attribution.credit.clone(),
- title: item.hit.title.clone(),
- section: item.hit.section.clone(),
+ title: display_title,
source_url: item.hit.source_url.clone(),
license_label: dataset.attribution.license_label.clone(),
license_url: dataset.attribution.license_url.clone(),
@@ -96,18 +93,14 @@ pub fn build_knowledge_prompt_context(
Ok(Some(KnowledgePromptContext { text, citations }))
}
-fn truncate_utf8_graphemes(value: &str, max_bytes: usize) -> &str {
+fn truncate_utf8(value: &str, max_bytes: usize) -> &str {
if value.len() <= max_bytes {
return value;
}
- let mut end = 0;
- for (start, grapheme) in value.grapheme_indices(true) {
- let next = start + grapheme.len();
- if next > max_bytes {
- break;
- }
- end = next;
+ let mut end = max_bytes;
+ while !value.is_char_boundary(end) {
+ end -= 1;
}
&value[..end]
}
@@ -170,8 +163,7 @@ A useful passage\n\
context.citations[0].credit,
"Simple English Wikipedia contributors"
);
- assert_eq!(context.citations[0].title, "Example title");
- assert_eq!(context.citations[0].section.as_deref(), Some("Details"));
+ assert_eq!(context.citations[0].title, "Example title β€” Details");
assert_eq!(
context.citations[0].source_url,
"https://example.com/source"
@@ -199,42 +191,14 @@ A useful passage\n\
assert!(context.text.len() <= budget);
}
- #[test]
- fn truncates_only_at_extended_grapheme_boundaries() {
- let item = hit("simplewiki", "AπŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦B");
- let budget_inside_family_emoji = fixed_bytes(&item) + 1 + "πŸ‘¨".len();
- let context = build_knowledge_prompt_context(&[item], budget_inside_family_emoji)
- .unwrap()
- .unwrap();
-
- assert!(
- context
- .text
- .contains("\nA\n----- END KNOWLEDGE CONTEXT -----")
- );
- assert!(!context.text.contains('πŸ‘¨'));
-
- let item = hit("simplewiki", "Ae\u{301}B");
- let budget_inside_combining_grapheme = fixed_bytes(&item) + 2;
- let context = build_knowledge_prompt_context(&[item], budget_inside_combining_grapheme)
- .unwrap()
- .unwrap();
- assert!(
- context
- .text
- .contains("\nA\n----- END KNOWLEDGE CONTEXT -----")
- );
- assert!(!context.text.contains("e\u{301}"));
- }
-
#[test]
fn respects_two_three_and_four_byte_scalar_boundaries() {
- assert_eq!(truncate_utf8_graphemes("Γ©", 1), "");
- assert_eq!(truncate_utf8_graphemes("Γ©", 2), "Γ©");
- assert_eq!(truncate_utf8_graphemes("€", 2), "");
- assert_eq!(truncate_utf8_graphemes("€", 3), "€");
- assert_eq!(truncate_utf8_graphemes("πŸ™‚", 3), "");
- assert_eq!(truncate_utf8_graphemes("πŸ™‚", 4), "πŸ™‚");
+ assert_eq!(truncate_utf8("Γ©", 1), "");
+ assert_eq!(truncate_utf8("Γ©", 2), "Γ©");
+ assert_eq!(truncate_utf8("€", 2), "");
+ assert_eq!(truncate_utf8("€", 3), "€");
+ assert_eq!(truncate_utf8("πŸ™‚", 3), "");
+ assert_eq!(truncate_utf8("πŸ™‚", 4), "πŸ™‚");
}
#[test]
@@ -249,7 +213,7 @@ A useful passage\n\
.text
.contains("# Example title (Simple English Wikipedia)\npassage")
);
- assert_eq!(context.citations[0].section.as_deref(), Some(" \n "));
+ assert_eq!(context.citations[0].title, "Example title");
let blank = hit("simplewiki", " \n\t ");
assert!(
diff --git a/rust/crates/ensu/tests/knowledge_real_assets.rs b/rust/crates/ensu/tests/knowledge_real_assets.rs
deleted file mode 100644
index a592d09ecc..0000000000
--- a/rust/crates/ensu/tests/knowledge_real_assets.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-use std::env;
-use std::path::Path;
-
-use ente_ensu::config::defaults;
-use ente_ensu::llm::{Context, EmbeddingContextParams, Model, ModelLoadParams};
-use ente_ensu::retrieval::RetrievalIndex;
-
-const COMPATIBILITY_QUERY: &str = "how tall is mount everest";
-const EXPECTED_TOP_TITLE: &str = "Mount Everest";
-
-#[test]
-fn pinned_embedding_and_pack_match_builder_top_hit() {
- let Some(model_path) = required_env("ENSU_KNOWLEDGE_EMBEDDING_GGUF") else {
- eprintln!("skipping real-asset test: ENSU_KNOWLEDGE_EMBEDDING_GGUF is unset");
- return;
- };
- let Some(pack_directory) = required_env("ENSU_KNOWLEDGE_PACK_DIR") else {
- eprintln!("skipping real-asset test: ENSU_KNOWLEDGE_PACK_DIR is unset");
- return;
- };
- let defaults = defaults();
- let embedding = defaults.knowledge_embedding;
- let pack_identity = Path::new(&pack_directory)
- .file_name()
- .and_then(|name| name.to_str())
- .expect("pack directory has a UTF-8 identity");
- let dataset = defaults
- .knowledge_datasets
- .into_iter()
- .find(|dataset| {
- pack_identity == dataset.current_download_identity
- || pack_identity.starts_with(&format!("{}-", dataset.stable_id))
- })
- .expect("pack directory belongs to a bundled dataset");
-
- let model = Model::load(ModelLoadParams {
- model_path,
- n_gpu_layers: Some(0),
- use_mmap: Some(true),
- use_mlock: Some(false),
- })
- .expect("load pinned EmbeddingGemma");
- let context = Context::new_embedding(
- &model,
- EmbeddingContextParams {
- context_size: embedding.context_size,
- n_threads: None,
- batch_size: embedding.batch_size,
- micro_batch_size: embedding.micro_batch_size,
- source_dim: embedding.source_dim,
- dim: embedding.dim,
- query_prompt: embedding.query_prompt,
- },
- )
- .expect("create mean-pooled embedding context");
- let vector = context
- .embed(COMPATIBILITY_QUERY)
- .expect("embed real query");
- assert_eq!(vector.len(), 512);
- assert!(vector.iter().all(|value| value.is_finite()));
- let norm = vector
- .iter()
- .map(|value| f64::from(*value) * f64::from(*value))
- .sum::<f64>()
- .sqrt();
- assert!((norm - 1.0).abs() < 1e-5, "embedding norm was {norm}");
-
- let index = RetrievalIndex::open(pack_directory, &dataset).expect("open published pack");
- let hits = index
- .search(&vector, embedding.max_hits, dataset.relevance_threshold)
- .expect("search published pack");
- assert_eq!(
- hits.first().map(|hit| hit.title.as_str()),
- Some(EXPECTED_TOP_TITLE)
- );
-}
-
-fn required_env(name: &str) -> Option<String> {
- env::var(name).ok().filter(|value| !value.trim().is_empty())
-}
diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/model-download/src/download.rs
index 96933a381b..1a9a6328a1 100644
--- a/rust/crates/model-download/src/download.rs
+++ b/rust/crates/model-download/src/download.rs
@@ -207,15 +207,6 @@ impl Downloader {
})
.await
}
-
- pub fn download_blocking(
- &self,
- targets: Vec<Target>,
- on_progress: impl FnMut(Progress),
- is_cancelled: impl Fn() -> bool,
- ) -> Result<(), Error> {
- fetch(self, targets, on_progress, is_cancelled)
- }
}
fn build_client() -> Result<Client, Error> {
diff --git a/rust/crates/model-download/src/model.rs b/rust/crates/model-download/src/model.rs
index 4674c998e4..9b96045788 100644
--- a/rust/crates/model-download/src/model.rs
+++ b/rust/crates/model-download/src/model.rs
@@ -189,8 +189,8 @@ impl ModelDownloader {
let mut any = false;
for entry in download_targets(&self.models_dir, target).ok()? {
let path = &entry.destination;
- let size = if let Some(size) = valid_local_size(&entry, path) {
- Some(size)
+ let size = if path.exists() {
+ fs::metadata(path).ok().map(|m| m.len()).filter(|s| *s > 0)
} else {
download::probe_content_length(downloader.client(), &entry.url).await
};
@@ -204,18 +204,6 @@ impl ModelDownloader {
}
}
-fn valid_local_size(target: &Target, path: &Path) -> Option<u64> {
- let actual = download::sha256_file(path).ok()?;
- if actual != target.sha256.trim().to_ascii_lowercase() {
- return None;
- }
- fs::metadata(path)
- .ok()
- .filter(|metadata| metadata.is_file())
- .map(|metadata| metadata.len())
- .filter(|size| *size > 0)
-}
-
fn storage_key(target: &ModelTarget) -> &str {
match target {
ModelTarget::Files { id, .. } | ModelTarget::TarGz { id, .. } => id,
@@ -441,25 +429,6 @@ mod tests {
let _ = fs::remove_dir_all(dir);
}
- #[test]
- fn local_size_ignores_files_that_fail_checksum_validation() {
- let dir = scratch_dir("local-size-validation");
- let path = dir.join("model.gguf");
- write_gguf(&path, b"GGUFdata");
- let mut target = Target {
- label: "Model".to_string(),
- url: "https://example.org/model.gguf".to_string(),
- sha256: test_sha(),
- destination: path.clone(),
- };
-
- assert_eq!(valid_local_size(&target, &path), None);
- target.sha256 = download::sha256_file(&path).unwrap();
- assert_eq!(valid_local_size(&target, &path), Some(8));
-
- let _ = fs::remove_dir_all(dir);
- }
-
#[test]
fn download_skips_when_already_present() {
let dir = scratch_dir("skip");