💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Refine historical passage reuse across clients
setalp94bca31a118c
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.ktindex 128426d052..5646f47e57 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt@@ -193,8 +193,7 @@ class AppStore(modelSettingsActions.cancelModelDownload()}- fun retryAssistantMessage(messageId: String, searchAsWritten: Boolean = false) =- chatActions.retryAssistantMessage(messageId, searchAsWritten)+ fun retryAssistantMessage(messageId: String) = chatActions.retryAssistantMessage(messageId)fun sendMessage() = chatActions.sendMessage()diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.ktindex a0efe20cc9..8a15c355a9 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt@@ -15,7 +15,6 @@ import androidx.compose.material3.DrawerStateimport androidx.compose.material3.DrawerValueimport androidx.compose.material3.Scaffoldimport androidx.compose.runtime.Composable-import androidx.compose.runtime.collectAsStateimport androidx.compose.runtime.getValueimport androidx.compose.ui.Modifierimport androidx.navigation.NavBackStackEntry@@ -55,7 +54,6 @@ internal fun HomeNavigation(onAttachmentSelected: (AttachmentType) -> Unit,onOpenAttachment: (Attachment) -> Unit,) {- val notesState by store.notesStore.state.collectAsState()Scaffold(containerColor = EnsuColor.backgroundBase(),contentWindowInsets = WindowInsets(0, 0, 0, 0),@@ -125,15 +123,6 @@ internal fun HomeNavigation(onRemoveAttachment = store::removeAttachment,onEditMessage = { message -> store.beginEditing(message.id) },onRetryMessage = { message -> store.retryAssistantMessage(message.id) },- onSearchMessage =- if (- appState.knowledge.enabledReadyDatasets.isNotEmpty() ||- notesState.collections.isNotEmpty()- ) {- { message ->- store.retryAssistantMessage(message.id, searchAsWritten = true)- }- } else null,onCancelEdit = store::cancelEditing,onBranchChange = store::updateBranchSelection,onOpenAttachment = onOpenAttachment,diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.ktindex 671bc38395..deba524767 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt@@ -107,7 +107,6 @@ internal fun MessageList(branchSelections: Map<String, Int>,onEditMessage: (ChatMessage) -> Unit,onRetryMessage: (ChatMessage) -> Unit,- onSearchMessage: ((ChatMessage) -> Unit)?,onBranchChange: (String, Int) -> Unit,onOpenAttachment: (Attachment) -> Unit,onStartDownload: (Boolean) -> Unit,@@ -320,7 +319,6 @@ internal fun MessageList(isLastMessage = message == messages.last(),branchSelections = branchSelections,onRetry = { onRetryMessage(message) },- onSearch = onSearchMessage?.let { search -> { search(message) } },onBranchChange = onBranchChange,showsMetadata = true,)@@ -641,7 +639,6 @@ private fun AssistantMessageBubble(isLastMessage: Boolean,branchSelections: Map<String, Int>,onRetry: () -> Unit,- onSearch: (() -> Unit)?,onBranchChange: (String, Int) -> Unit,showsMetadata: Boolean,) {@@ -671,14 +668,6 @@ private fun AssistantMessageBubble(onRetry()})- onSearch?.let { search ->- add(- MessageAction("Search as written", HugeIcons.Search01Icon) {- haptic.perform(HapticFeedbackType.TextHandleMove)- search()- }- )- }}}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.ktindex c76fb2e984..1ea54e5864 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.kt@@ -138,7 +138,7 @@ class ChatRepository(internal fun startFollowup(sessionId: String, path: List<String>, userText: String) =withDbRecovery {- db.startConversationFollowup(sessionId, path, userText, userText)+ db.startConversationFollowup(sessionId, path, userText)}internal fun prepareConversation(diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex d6d5f90f72..0ac6a4ff22 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -10,7 +10,6 @@ import io.ente.ensu.bindings.ConversationRequestimport io.ente.ensu.bindings.DbExceptionimport io.ente.ensu.bindings.GroundedExcerptimport io.ente.ensu.bindings.GroundedSource-import io.ente.ensu.bindings.IncludedPassageimport io.ente.ensu.bindings.LlmExceptionimport io.ente.ensu.bindings.PassageLocatorimport io.ente.ensu.bindings.buildGroundedPromptContext@@ -318,7 +317,7 @@ internal class ChatStoreActions(generationJob?.cancel()}- fun retryAssistantMessage(messageId: String, searchAsWritten: Boolean = false) {+ fun retryAssistantMessage(messageId: String) {val priorGeneration = generationJobval priorSummary = sessionSummaryJobif (state.value.chat.isGenerating) {@@ -347,7 +346,7 @@ internal class ChatStoreActions(priorGeneration?.join()priorSummary?.join()llmProvider.resetContext()- startGeneration(sessionId, parent, searchAsWritten)+ startGeneration(sessionId, parent)}}@@ -483,11 +482,7 @@ internal class ChatStoreActions(}}- private fun startGeneration(- sessionId: String,- userMessage: ChatMessage,- searchAsWritten: Boolean = false,- ) {+ private fun startGeneration(sessionId: String, userMessage: ChatMessage) {val scope = scope ?: returnif (!state.value.chat.deviceCapability.isChatSupported()) returnval prompt = buildPrompt(userMessage.text, userMessage.attachments)@@ -544,23 +539,21 @@ internal class ChatStoreActions(val enabledDatasets = state.value.knowledge.enabledReadyDatasetstry {if (prompt.imageFiles.isEmpty()) {- val history = buildSelectedPath(sessionId).takeWhile { it.id != userMessage.id }- val hasCurrentSources =- searchAsWritten ||- enabledDatasets.isNotEmpty() ||- notesStore.state.value.collections.isNotEmpty()+ val messages = messageStore[sessionId].orEmpty()+ val path = conversationPath(messages, userMessage)+ val pathIds = path.toSet()val turn =withContext(Dispatchers.IO) {if (- hasCurrentSources ||- history.any {+ messages.any {+ it.id in pathIds &&it.author == MessageAuthor.Assistant &&- parseGroundedAssistantText(it.text).sources.isNotEmpty()- }+ parseGroundedAssistantText(it.text).sources.isNotEmpty()+ }) {chatRepository.startFollowup(sessionId,- history.map { it.id } + userMessage.id,+ path,userMessage.text,)} else null@@ -645,13 +638,12 @@ internal class ChatStoreActions(try {followup?.let { turn ->- val resolution =+ val locators =withContext(Dispatchers.IO) {- if (searchAsWritten) turn.searchAsWritten()- else turn.searchWithHistory(knowledgeHits)+ turn.searchWithHistory(knowledgeHits)}if (!isActive() || stopRequested) return@launch- resolution.referencedPassages.mapNotNull { reloadPassage(it) }+ locators.mapNotNull { reloadPassage(it) }}.orEmpty()} catch (error: kotlinx.coroutines.CancellationException) {@@ -954,21 +946,15 @@ internal class ChatStoreActions(return true}- private suspend fun reloadPassage(reference: IncludedPassage): GroundedExcerpt? {- return try {- when (val locator = reference.locator) {+ private suspend fun reloadPassage(locator: PassageLocator): GroundedExcerpt? =+ try {+ when (locator) {is PassageLocator.EnsuPack ->knowledgeProvider.reload(locator, state.value.knowledge.enabledReadyDatasets)is PassageLocator.LocalNote -> notesStore.reload(locator)}} catch (error: kotlinx.coroutines.CancellationException) {throw error- } catch (error: ConversationException.Cancelled) {- throw error- } catch (error: ConversationException.Stale) {- throw error- } catch (error: DbException) {- throw error} catch (error: Exception) {logRepository.log(LogLevel.Warning,@@ -979,7 +965,6 @@ internal class ChatStoreActions()null}- }private suspend fun generateTextConversation(sessionId: String,diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.ktindex fb48753507..b8a9e4b0d4 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt@@ -66,7 +66,6 @@ fun ChatView(onRemoveAttachment: (Attachment) -> Unit,onEditMessage: (ChatMessage) -> Unit,onRetryMessage: (ChatMessage) -> Unit,- onSearchMessage: ((ChatMessage) -> Unit)?,onCancelEdit: () -> Unit,onBranchChange: (String, Int) -> Unit,onOpenAttachment: (Attachment) -> Unit,@@ -237,7 +236,6 @@ fun ChatView(branchSelections = chatState.branchSelections,onEditMessage = onEditMessage,onRetryMessage = onRetryMessage,- onSearchMessage = onSearchMessage,onBranchChange = onBranchChange,onOpenAttachment = onOpenAttachment,onStartDownload = onStartDownload,diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.ktindex ca4970c808..46278812be 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt@@ -273,10 +273,7 @@ class NotesStore(} catch (error: CancellationException) {throw error} catch (_: NotesException.RebuildRequired) {- enqueue(record.id, rebuild = true, immediate = true)- update(record.id) {- it.copy(status = NotesStatus.Pending, indexAvailable = false)- }+ requestRebuild(record.id)} catch (error: Exception) {fail(record.id, error)}@@ -292,10 +289,7 @@ class NotesStore(val hit = provider.reload(locator) ?: return@withContext nullverify(selectMixedGroundingCandidates(emptyList(), listOf(hit), 1u)).singleOrNull()} catch (_: NotesException.RebuildRequired) {- enqueue(locator.collectionId, rebuild = true, immediate = true)- update(locator.collectionId) {- it.copy(status = NotesStatus.Pending, indexAvailable = false)- }+ requestRebuild(locator.collectionId)null}}@@ -365,6 +359,11 @@ class NotesStore(}}+ private fun requestRebuild(id: String) {+ enqueue(id, rebuild = true, immediate = true)+ update(id) { it.copy(status = NotesStatus.Pending, indexAvailable = false) }+ }+private fun enqueue(id: String,rebuild: Boolean = false,@@ -544,13 +543,7 @@ class NotesStore(}is NotesException.RebuildRequired -> {if (snapshot == null) {- enqueue(id, rebuild = true, immediate = true)- update(id) {- it.copy(- status = NotesStatus.Pending,- indexAvailable = false,- )- }+ requestRebuild(id)} else fail(id, failure)}is Exception -> fail(id, failure)diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swiftindex 09c0fc5ac2..e9488c39af 100644--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift@@ -302,10 +302,6 @@ struct ChatView: View {onRetry: { message inviewModel.retryAssistantResponse(message)},- onSearch: viewModel.canSearchSources- ? { message in- viewModel.retryAssistantResponse(message, searchAsWritten: true)- } : nil,onBranchChange: { message, delta inviewModel.changeBranch(for: message, delta: delta)},diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex 1f51be1a64..cfae99ff4d 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -980,11 +980,7 @@ final class ChatViewModel: ObservableObject {startModelDownload(userInitiated: true)}- var canSearchSources: Bool {- !knowledgeStore.enabledReadyDatasets.isEmpty || !notesStore.collections.isEmpty- }-- func retryAssistantResponse(_ message: RenderedChatMessage, searchAsWritten: Bool = false) {+ func retryAssistantResponse(_ message: RenderedChatMessage) {let priorGeneration = generationTasklet priorSummary = sessionSummaryTaskguard message.role == .assistant else { return }@@ -1021,7 +1017,7 @@ final class ChatViewModel: ObservableObject {_ = await priorSummary?.resultguard let self else { return }await provider.resetContext()- startGeneration(for: userNode, searchAsWritten: searchAsWritten)+ startGeneration(for: userNode)}}@@ -1064,7 +1060,7 @@ final class ChatViewModel: ObservableObject {return session.id}- private func startGeneration(for userNode: MessageNode, searchAsWritten: Bool = false) {+ private func startGeneration(for userNode: MessageNode) {guard !isChatUnsupported else {showUnsupportedDeviceDialog = truereturn@@ -1112,27 +1108,21 @@ final class ChatViewModel: ObservableObject {defer { followup?.cancel() }do {if prompt.imageFiles.isEmpty {- let history = Array(- buildSelectedPath(- for: userNode.sessionId,- childrenMap: childrenByParent(sessionId: userNode.sessionId)- ).prefix { $0.id != userNode.id })- let path = history.map { $0.id.uuidString } + [userNode.id.uuidString]- let assistantTexts = history.filter { $0.role == .assistant }.map { $0.text }- let hasCurrentSources =- searchAsWritten || !enabledDatasets.isEmpty- || !notesStore.collections.isEmpty+ let path = buildConversationPath(for: userNode)+ let pathIds = Set(path)+ let assistantTexts = (messageStore[userNode.sessionId] ?? [])+ .filter { pathIds.contains($0.id.uuidString) && $0.role == .assistant }+ .map { $0.text }followup = try await Task.detached {[chatDb] () throws -> ConversationFollowup? inguard- hasCurrentSources- || assistantTexts.contains(where: {- !parseGroundedAssistantText(storedText: $0).sources.isEmpty- })+ assistantTexts.contains(where: {+ !parseGroundedAssistantText(storedText: $0).sources.isEmpty+ })else { return nil }return try chatDb.startConversationFollowup(sessionUuid: userNode.sessionId.uuidString, path: path,- question: userNode.text, expectedUserText: userNode.text+ question: userNode.text)}.valuetry Task.checkCancellation()@@ -1183,15 +1173,16 @@ final class ChatViewModel: ObservableObject {do {if let turn = followup {let searched = knowledgeHits- let resolution = try await Task.detached {- try searchAsWritten- ? turn.searchAsWritten() : turn.searchWithHistory(searched: searched)+ let locators = try await Task.detached {+ try turn.searchWithHistory(searched: searched)}.valuetry Task.checkCancellation()guard activeGenerationId == generationId else { return }- for reference in resolution.referencedPassages {+ for locator in locators {do {- if let hit = try await reloadPassage(reference) { reloaded.append(hit) }+ if let hit = try await reloadPassage(locator) {+ reloaded.append(hit)+ }} catch {if isCancellation(error) { throw error }logger.warning("Source reload failed", details: "\(error)")@@ -1518,13 +1509,13 @@ final class ChatViewModel: ObservableObject {}}- private func reloadPassage(_ reference: IncludedPassage) async throws -> GroundedExcerpt? {- switch reference.locator {+ private func reloadPassage(_ locator: PassageLocator) async throws -> GroundedExcerpt? {+ switch locator {case .ensuPack:return try await knowledgeProvider.reload(- reference.locator, datasets: knowledgeStore.enabledReadyDatasets)+ locator, datasets: knowledgeStore.enabledReadyDatasets)case .localNote:- return try await notesStore.reload(reference.locator)+ return try await notesStore.reload(locator)}}diff --git a/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift b/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swiftindex ae32450803..25908d5ab5 100644--- a/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift+++ b/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift@@ -246,7 +246,6 @@ struct AssistantMessageBubbleView: View {let isLastMessage: Boollet onCopy: () -> Voidlet onRetry: () -> Void- let onSearch: (() -> Void)?let onBranchChange: (Int) -> Voidlet onOpenAttachment: (ChatAttachment) -> Voidlet showsMetadata: Bool@@ -291,12 +290,6 @@ struct AssistantMessageBubbleView: View {hapticMedium()onRetry()}- if let onSearch {- Button("Search as written") {- hapticMedium()- onSearch()- }- }}}diff --git a/apple/apps/ensu/Ensu/Chat/MessageListView.swift b/apple/apps/ensu/Ensu/Chat/MessageListView.swiftindex 72b243ae7f..7ea5e43077 100644--- a/apple/apps/ensu/Ensu/Chat/MessageListView.swift+++ b/apple/apps/ensu/Ensu/Chat/MessageListView.swift@@ -14,7 +14,6 @@ struct MessageListView: View {let onEdit: (RenderedChatMessage) -> Voidlet onCopy: (RenderedChatMessage) -> Voidlet onRetry: (RenderedChatMessage) -> Void- let onSearch: ((RenderedChatMessage) -> Void)?let onBranchChange: (RenderedChatMessage, Int) -> Voidlet onDismissKeyboard: () -> Void@@ -205,7 +204,6 @@ struct MessageListView: View {isLastMessage: message.id == messages.last?.id,onCopy: { onCopy(message) },onRetry: { onRetry(message) },- onSearch: onSearch.map { search in { search(message) } },onBranchChange: { delta in onBranchChange(message, delta) },onOpenAttachment: openAttachment,showsMetadata: truediff --git a/apple/apps/ensu/Ensu/Services/NotesStore.swift b/apple/apps/ensu/Ensu/Services/NotesStore.swiftindex 2b0c3dcf63..23546ed3e1 100644--- a/apple/apps/ensu/Ensu/Services/NotesStore.swift+++ b/apple/apps/ensu/Ensu/Services/NotesStore.swift@@ -188,11 +188,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {do {hits += try await provider.search(collection.id, query: query)} catch NotesError.RebuildRequired {- enqueue(collection.id, rebuild: true, due: .distantPast)- update(collection.id) {- $0.status = .pending- $0.indexAvailable = false- }+ requestRebuild(collection.id)} catch is CancellationError {throw CancellationError()} catch {@@ -212,11 +208,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {selectMixedGroundingCandidates(packHits: [], notesHits: [hit], notesLimit: 1)).first} catch NotesError.RebuildRequired {- enqueue(collectionId, rebuild: true, due: .distantPast)- update(collectionId) {- $0.status = .pending- $0.indexAvailable = false- }+ requestRebuild(collectionId)return nil}}@@ -271,6 +263,14 @@ final class NotesStore: ObservableObject, ModelMaintenance {}}+ private func requestRebuild(_ id: String) {+ enqueue(id, rebuild: true, due: .distantPast)+ update(id) {+ $0.status = .pending+ $0.indexAvailable = false+ }+ }+private func enqueue(_ id: String, forced: Set<String> = [], rebuild: Bool = false, due: Date? = nil) {@@ -409,11 +409,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {}} catch NotesError.RebuildRequired {if snapshot == nil {- enqueue(id, rebuild: true, due: .distantPast)- update(id) {- $0.status = .pending- $0.indexAvailable = false- }+ requestRebuild(id)} else {fail(id, NotesError.RebuildRequired)}diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex e98c5ba014..c97d1165bb 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2536,7 +2536,6 @@ dependencies = ["self_cell","serde","serde_json",- "sha2 0.11.0","tempfile","thiserror 2.0.18","tokio",diff --git a/rust/apps/ensu/src-tauri/src/commands/chat_db.rs b/rust/apps/ensu/src-tauri/src/commands/chat_db.rsindex 4a916fc14d..90a5ec2b22 100644--- a/rust/apps/ensu/src-tauri/src/commands/chat_db.rs+++ b/rust/apps/ensu/src-tauri/src/commands/chat_db.rs@@ -172,8 +172,6 @@ pub struct ChatMessageDto {created_at: i64,attachments: Vec<ChatAttachmentDto>,sources: Vec<GroundedSourceDto>,- #[serde(skip_serializing_if = "Option::is_none")]- evidence_saved: Option<bool>,}impl From<db::Message> for ChatMessageDto {@@ -191,7 +189,6 @@ impl From<db::Message> for ChatMessageDto {}};Self {- evidence_saved: None,message_uuid: message.uuid.to_string(),session_uuid: message.session_uuid.to_string(),parent_message_uuid: message.parent_message_uuid.map(|value| value.to_string()),@@ -388,26 +385,20 @@ pub async fn chat_db_insert_message(let snapshot = prepared.as_ref().map(|(snapshot, _)| snapshot);let message =db.insert_message_guarded(session_uuid, sender, &text, parent, attachments, snapshot)?;- let evidence_saved =- prepared- .filter(|(_, passages)| !passages.is_empty())- .map(|(snapshot, passages)| {- match db.save_answer_evidence(&snapshot, message.uuid, passages) {- Ok(saved) => saved,- Err(error) => {- logging::log(- "ChatDb",- format!(- "Reply saved but source reference persistence failed: {error}"- ),- );- false- }- }- });- let mut result = ChatMessageDto::from(message);- result.evidence_saved = evidence_saved;- Ok(result)+ if let Some((snapshot, passages)) = prepared.filter(|(_, passages)| !passages.is_empty()) {+ match db.save_answer_evidence(&snapshot, message.uuid, passages) {+ Ok(true) => {}+ Ok(false) => logging::log(+ "ChatDb",+ "Reply saved, but its source references could not be retained",+ ),+ Err(error) => logging::log(+ "ChatDb",+ format!("Reply saved but source reference persistence failed: {error}"),+ ),+ }+ }+ Ok(ChatMessageDto::from(message))}).await}diff --git a/rust/apps/ensu/src-tauri/src/commands/conversation.rs b/rust/apps/ensu/src-tauri/src/commands/conversation.rsindex 0917d9fcad..f935772c61 100644--- a/rust/apps/ensu/src-tauri/src/commands/conversation.rs+++ b/rust/apps/ensu/src-tauri/src/commands/conversation.rs@@ -23,7 +23,7 @@ struct Slot {token: String,snapshot: ConversationSnapshot,db: Arc<ChatDb<SqliteBackend>>,- grounded: Option<ente_ensu::retrieval::GroundedPromptContext>,+ passages: Vec<IncludedPassage>,model_epoch: u64,cancel_epoch: u64,}@@ -80,14 +80,7 @@ impl State {}slots.remove(window)- .map(|slot| {- (- slot.snapshot,- slot.grounded- .map(|context| context.included_passages)- .unwrap_or_default(),- )- })+ .map(|slot| (slot.snapshot, slot.passages)).ok_or_else(|| error("Preparation has expired"))}}@@ -253,9 +246,6 @@ pub async fn conversation_prepare());}let model_epoch = llm_state.model_state_epoch();- if let Some(candidates) = &input.grounding_candidates {- candidates.validate().map_err(|e| error(e.to_string()))?;- }state.slots.lock()@@ -268,19 +258,21 @@ pub async fn conversation_prepare(.conversation_snapshot(input.session_uuid, &input.path).map_err(ApiError::from)?;if let Some(token) = &input.resolution_token {- app.state::<super::followup::State>().validate(+ let selection = app.state::<super::followup::State>().take_selection(work_window.label(),token,&db,&snapshot,- input.grounding_candidates.as_ref(),)?;+ input.history_query = Some(selection.question);+ input.grounding_candidates = Some(selection.candidates);}let candidates = match input.grounding_candidates.take() {Some(candidates) => candidates,None => conversation::GroundingCandidates::new(vec![], vec![], conversation::MAX_GROUNDING_BYTES).map_err(preparation_error)?,};+ candidates.validate().map_err(|e| error(e.to_string()))?;let turn = conversation::TurnInput {session: input.session_uuid,messages: snapshot.messages().to_vec(),@@ -312,18 +304,21 @@ pub async fn conversation_prepare());let result = prepared.preparation.run(&mut effects).map_err(preparation_error)?;effects.check_current().map_err(preparation_error)?;- let grounded = prepared.grounding.context;+ let mut grounded = prepared.grounding.context;+ let passages = grounded.as_mut()+ .map(|context| std::mem::take(&mut context.included_passages))+ .unwrap_or_default();Ok((Response {preparation_token: token.clone(),messages: result.messages,- grounded_context: grounded.clone().map(Into::into),+ grounded_context: grounded.map(Into::into),},Slot {token,snapshot,db,- grounded,+ passages,model_epoch,cancel_epoch: expected_epoch,},diff --git a/rust/apps/ensu/src-tauri/src/commands/followup.rs b/rust/apps/ensu/src-tauri/src/commands/followup.rsindex 444cafc6a7..4e6967c6e7 100644--- a/rust/apps/ensu/src-tauri/src/commands/followup.rs+++ b/rust/apps/ensu/src-tauri/src/commands/followup.rs@@ -3,10 +3,10 @@ use ente_ensu::{conversation as core,db::{ChatDb, SqliteBackend, chat::ConversationSnapshot},};-use serde::{Deserialize, Serialize};+use serde::Deserialize;use std::collections::HashMap;use std::sync::{Arc, Mutex, atomic::Ordering};-use tauri::{AppHandle, Manager, State as TauriState, WebviewWindow};+use tauri::{AppHandle, Manager, State as TauriState, WebviewWindow, async_runtime};use uuid::Uuid;#[derive(Default)]@@ -15,9 +15,13 @@ pub struct State {}struct Guard {token: String,- fingerprint: String,db: Arc<ChatDb<SqliteBackend>>,- candidates: Option<String>,+ selection: Option<Selection>,+}+pub(crate) struct Selection {+ fingerprint: String,+ pub question: String,+ pub candidates: core::GroundingCandidates,}fn error(message: impl Into<String>) -> ApiError {@@ -25,36 +29,35 @@ fn error(message: impl Into<String>) -> ApiError {}impl State {- pub(crate) fn validate(+ pub(crate) fn take_selection(&self,window: &str,token: &str,db: &Arc<ChatDb<SqliteBackend>>,snapshot: &ConversationSnapshot,- candidates: Option<&core::GroundingCandidates>,- ) -> Result<(), ApiError> {- let guards = self+ ) -> Result<Selection, ApiError> {+ let fingerprint = core::fingerprint(snapshot.messages());+ let mut guards = self.guards.lock().map_err(|_| error("Follow-up state is unavailable"))?;- if guards.get(window).is_some_and(|guard| {- guard.token == token- && guard.candidates.as_ref().is_some_and(|expected| {- candidates- .and_then(|c| serde_json::to_string(c).ok())+ guards+ .get_mut(window)+ .filter(|guard| {+ guard.token == token+ && Arc::ptr_eq(&guard.db, db)+ && guard+ .selection.as_ref()- == Some(expected)- })- && Arc::ptr_eq(&guard.db, db)- && guard.fingerprint == core::fingerprint(snapshot.messages())- }) {- Ok(())- } else {- Err(ApiError::new(- "stale",- "Conversation changed after selecting sources; retry the reply",- ))- }+ .is_some_and(|selection| selection.fingerprint == fingerprint)+ })+ .and_then(|guard| guard.selection.take())+ .ok_or_else(|| {+ ApiError::new(+ "stale",+ "Source selection expired or conversation changed; retry the reply",+ )+ })}}@@ -69,13 +72,6 @@ pub struct Request {cancellation_epoch: u64,candidates: Option<core::GroundingCandidates>,}-#[derive(Serialize)]-#[serde(rename_all = "camelCase")]-pub struct Response {- token: String,- candidates: core::GroundingCandidates,-}-#[tauri::command]pub async fn conversation_resolve_followup(app: AppHandle,@@ -83,7 +79,7 @@ pub async fn conversation_resolve_followup(state: TauriState<'_, State>,llm_state: TauriState<'_, super::llm::State>,input: Request,-) -> Result<Response, ApiError> {+) -> Result<String, ApiError> {if Uuid::parse_str(&input.token).is_err()|| input.path.is_empty()|| input.path.len() > core::MAX_HISTORY_MESSAGES@@ -94,23 +90,25 @@ pub async fn conversation_resolve_followup());}let db = app.state::<ChatDbState>().database()?;- let snapshot = db- .conversation_snapshot(input.session_uuid, &input.path)- .map_err(ApiError::from)?;- let question = snapshot- .messages()- .last()- .ok_or_else(|| error("Missing current question"))?;- if question.sender != ente_ensu::db::Sender::SelfUser {- return Err(error("Follow-up must end at a user question"));- }+ let window = window.label().to_owned();+ let epoch = llm_state.retrieval_epoch();+ let expected = input.cancellation_epoch;+ let check = move || {+ if epoch.load(Ordering::Relaxed) == expected {+ Ok(())+ } else {+ Err(ApiError::new("cancelled", "Follow-up cancelled"))+ }+ };+ check()?;{let mut guards = state.guards.lock().map_err(|_| error("Follow-up state is unavailable"))?;+ check()?;if guards- .get(window.label())+ .get(&window).is_some_and(|guard| guard.token == input.token){return Err(error(@@ -118,40 +116,53 @@ pub async fn conversation_resolve_followup());}guards.insert(- window.label().to_owned(),+ window.clone(),Guard {token: input.token.clone(),- fingerprint: core::fingerprint(snapshot.messages()),db: db.clone(),- candidates: None,+ selection: None,},);}- let epoch = llm_state.retrieval_epoch();- let expected = input.cancellation_epoch;- let check = || {- if epoch.load(Ordering::Relaxed) == expected {- Ok(())- } else {- Err(ApiError::new("cancelled", "Follow-up cancelled"))+ let work_db = db.clone();+ let check_work = check.clone();+ let (snapshot, mut selection, references) = async_runtime::spawn_blocking(move || {+ check_work()?;+ let snapshot = work_db+ .conversation_snapshot(input.session_uuid, &input.path)+ .map_err(ApiError::from)?;+ let question = snapshot+ .messages()+ .last()+ .ok_or_else(|| error("Missing current question"))?;+ if question.sender != ente_ensu::db::Sender::SelfUser {+ return Err(error("Follow-up must end at a user question"));}- };- check()?;- let evidence: Vec<_> = snapshot.evidence().into_iter().cloned().collect();- let (searched, max_utf8_bytes) = input- .candidates- .map_or((Vec::new(), core::MAX_GROUNDING_BYTES), |fresh| {- (fresh.searched, fresh.max_utf8_bytes)- });- let mut candidates = core::GroundingCandidates::new(Vec::new(), searched, max_utf8_bytes)- .map_err(|cause| error(cause.to_string()))?;- let references = core::direct_followup_references(- snapshot.messages(),- &evidence,- &input.question,- &candidates.searched,- )- .unwrap_or_default();+ check_work()?;+ let evidence = snapshot.evidence();+ let (searched, max_utf8_bytes) = input+ .candidates+ .map_or((Vec::new(), core::MAX_GROUNDING_BYTES), |fresh| {+ (fresh.searched, fresh.max_utf8_bytes)+ });+ let candidates = core::GroundingCandidates::new(Vec::new(), searched, max_utf8_bytes)+ .map_err(|cause| error(cause.to_string()))?;+ let references = core::direct_followup_references(+ snapshot.messages(),+ &evidence,+ &input.question,+ &candidates.searched,+ );+ let selection = Selection {+ fingerprint: core::fingerprint(snapshot.messages()),+ question: input.question,+ candidates,+ };+ check_work()?;+ Ok((snapshot, selection, references))+ })+ .await+ .map_err(|_| error("Source selection task failed"))??;for reference in references {check()?;match super::knowledge::reload_for_followup(@@ -162,7 +173,7 @@ pub async fn conversation_resolve_followup().await{- Ok(Some(passage)) => candidates.referenced.push(passage),+ Ok(Some(passage)) => selection.candidates.referenced.push(passage),Ok(None) => {}Err(error) if error.name == Some("cancelled") => return Err(error),Err(error) => crate::logging::log(@@ -171,42 +182,43 @@ pub async fn conversation_resolve_followup(),}}- candidates- .validate()- .map_err(|cause| error(cause.to_string()))?;- check()?;- if !db- .conversation_snapshot_matches(&snapshot)- .map_err(ApiError::from)?- || !Arc::ptr_eq(&db, &app.state::<ChatDbState>().database()?)- {- return Err(ApiError::new(- "stale",- "Conversation changed while selecting sources; retry the reply",- ));- }- let mut guards = state- .guards- .lock()- .map_err(|_| error("Follow-up state is unavailable"))?;- let guard = guards- .get_mut(window.label())- .filter(|guard| guard.token == input.token)- .ok_or_else(|| ApiError::new("stale", "Source question was superseded"))?;- guard.candidates =- Some(serde_json::to_string(&candidates).map_err(|cause| error(cause.to_string()))?);- Ok(Response {- token: input.token,- candidates,+ async_runtime::spawn_blocking(move || {+ check()?;+ selection+ .candidates+ .validate()+ .map_err(|cause| error(cause.to_string()))?;+ if !db+ .conversation_snapshot_matches(&snapshot)+ .map_err(ApiError::from)?+ || !Arc::ptr_eq(&db, &app.state::<ChatDbState>().database()?)+ {+ return Err(ApiError::new(+ "stale",+ "Conversation changed while selecting sources; retry the reply",+ ));+ }+ let state = app.state::<State>();+ let mut guards = state+ .guards+ .lock()+ .map_err(|_| error("Follow-up state is unavailable"))?;+ check()?;+ let guard = guards+ .get_mut(&window)+ .filter(|guard| guard.token == input.token)+ .ok_or_else(|| ApiError::new("stale", "Source question was superseded"))?;+ guard.selection = Some(selection);+ Ok(input.token)})+ .await+ .map_err(|_| error("Source selection validation task failed"))?}#[cfg(test)]mod tests {use super::*;- fn candidates() -> core::GroundingCandidates {- core::GroundingCandidates::new(vec![], vec![], 6000).unwrap()- }+fn fixture() -> (Arc<ChatDb<SqliteBackend>>, ConversationSnapshot) {let db = Arc::new(ChatDb::open_in_memory(vec![4; 32]).unwrap());let session = db.create_session("Synthetic source question").unwrap();@@ -225,22 +237,20 @@ mod tests {(db, snapshot)}#[test]- fn preparation_requires_matching_completed_source_selection() {+ fn preparation_requires_matching_source_selection_and_consumes_it_once() {let (db, snapshot) = fixture();let state = State::default();- let valid = candidates();state.guards.lock().unwrap().insert("window".into(),Guard {token: "token".into(),- fingerprint: core::fingerprint(snapshot.messages()),db: db.clone(),- candidates: None,+ selection: None,},);assert!(state- .validate("window", "token", &db, &snapshot, Some(&valid))+ .take_selection("window", "token", &db, &snapshot).is_err());state@@ -249,36 +259,28 @@ mod tests {.unwrap().get_mut("window").unwrap()- .candidates = Some(serde_json::to_string(&valid).unwrap());- assert!(- state- .validate("window", "token", &db, &snapshot, Some(&valid))- .is_ok()- );- let mut changed = candidates();- changed.max_utf8_bytes = 3000;- let (other_db, _) = fixture();- for (token, db, candidates) in [- ("token", &db, Some(&changed)),- ("token", &db, None),- ("another-token", &db, Some(&valid)),- ("token", &other_db, Some(&valid)),+ .selection = Some(Selection {+ fingerprint: core::fingerprint(snapshot.messages()),+ question: "Where does she track it?".into(),+ candidates: core::GroundingCandidates::new(vec![], vec![], 6000).unwrap(),+ });+ let (other_db, other_snapshot) = fixture();+ for (window, token, db, snapshot) in [+ ("another-window", "token", &db, &snapshot),+ ("window", "another-token", &db, &snapshot),+ ("window", "token", &other_db, &snapshot),+ ("window", "token", &db, &other_snapshot),] {- assert!(- state- .validate("window", token, db, &snapshot, candidates)- .is_err()- );+ assert!(state.take_selection(window, token, db, snapshot).is_err());}- let question = snapshot.messages()[0].uuid;- db.update_message_text(question, "Actually, use another source")- .unwrap();- let edited = db- .conversation_snapshot(snapshot.session_uuid(), &[question])+ let selection = state+ .take_selection("window", "token", &db, &snapshot).unwrap();+ assert_eq!(selection.question, "Where does she track it?");+ assert_eq!(selection.candidates.max_utf8_bytes, 6000);assert!(state- .validate("window", "token", &db, &edited, Some(&valid))+ .take_selection("window", "token", &db, &snapshot).is_err());}diff --git a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs b/rust/apps/ensu/src-tauri/src/commands/knowledge.rsindex a1778ee741..cbacb1a35b 100644--- a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs+++ b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs@@ -662,14 +662,14 @@ pub async fn knowledge_retrieve(check_cancelled()?;let candidates = ente_ensu::conversation::GroundingCandidates::new(vec![], excerpts, context_budget as usize)- .map_err(|error| ApiError::new("followup", error.to_string()))?;+ .map_err(|error| ApiError::new("conversation", error.to_string()))?;candidates.pack(context_budget as usize).map(|context| context.map(|context| {let mut result = GroundedPromptContextDto::from(context);result.candidates = Some(candidates);result}))- .map_err(|error| ApiError::new("followup", error.to_string()))+ .map_err(|error| ApiError::new("conversation", error.to_string()))}).await.map_err(|_| ApiError::new("llm_thread", "Knowledge retrieval task failed"))?diff --git a/rust/apps/ensu/src-tauri/src/commands/mod.rs b/rust/apps/ensu/src-tauri/src/commands/mod.rsindex f365f4e84b..af4b64ca95 100644--- a/rust/apps/ensu/src-tauri/src/commands/mod.rs+++ b/rust/apps/ensu/src-tauri/src/commands/mod.rs@@ -4,6 +4,7 @@ mod common;pub(crate) mod config;pub(crate) mod conversation;pub(crate) mod crypto;+pub(crate) mod followup;pub(crate) mod knowledge;pub(crate) mod llm;pub(crate) mod notes;@@ -22,5 +23,3 @@ pub fn cleanup_for_exit(app: &AppHandle) {chat_db::clear_for_exit(app);logging::log("App", "cleanup_for_exit complete");}--pub mod followup;diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rsindex cfc74f8b83..57fc7bfb5a 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs@@ -169,15 +169,11 @@ impl IndexCache {}}- fn search(- &mut self,- collection_id: &str,- query: &[f32],- ) -> Option<Result<Vec<ente_ensu::notes::NotesSearchHit>, NotesError>> {+ fn get(&mut self, collection_id: &str) -> Option<&NotesCollectionIndex> {self.clock = self.clock.wrapping_add(1);let entry = self.entries.get_mut(collection_id)?;entry.last_used = self.clock;- Some(entry.index.search(query))+ Some(&entry.index)}fn insert(&mut self, collection_id: String, index: NotesCollectionIndex) {@@ -681,21 +677,7 @@ impl RetrievalHandle {collection_id: &str,query: &[f32],) -> Result<Vec<ente_ensu::notes::NotesSearchHit>, NotesError> {- if let Some(result) = self- .cached_indexes- .lock()- .unwrap_or_else(PoisonError::into_inner)- .search(collection_id, query)- {- return result;- }- let index = NotesCollectionIndex::open(&self.index_root, collection_id.to_string())?;- let hits = index.search(query)?;- self.cached_indexes- .lock()- .unwrap_or_else(PoisonError::into_inner)- .insert(collection_id.to_string(), index);- Ok(hits)+ self.with_index(collection_id, |index| index.search(query))}pub(crate) fn reload_passage(@@ -705,25 +687,29 @@ impl RetrievalHandle {let ente_ensu::retrieval::PassageLocator::LocalNote { collection_id, .. } = locator else {return Ok(None);};+ self.with_index(collection_id, |index| index.reload_passage(locator))+ }++ fn with_index<T>(+ &self,+ collection_id: &str,+ read: impl FnOnce(&NotesCollectionIndex) -> Result<T, NotesError>,+ ) -> Result<T, NotesError> {+ if let Some(index) = self+ .cached_indexes+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .get(collection_id){- let mut cache = self- .cached_indexes- .lock()- .unwrap_or_else(PoisonError::into_inner);- cache.clock = cache.clock.wrapping_add(1);- let clock = cache.clock;- if let Some(entry) = cache.entries.get_mut(collection_id) {- entry.last_used = clock;- return entry.index.reload_passage(locator);- }+ return read(index);}- let index = NotesCollectionIndex::open(&self.index_root, collection_id.clone())?;- let hit = index.reload_passage(locator)?;+ let index = NotesCollectionIndex::open(&self.index_root, collection_id.to_owned())?;+ let result = read(&index)?;self.cached_indexes.lock().unwrap_or_else(PoisonError::into_inner)- .insert(collection_id.clone(), index);- Ok(hit)+ .insert(collection_id.to_owned(), index);+ Ok(result)}pub(crate) fn verify_source_reference(&self, reference: &NoteSourceReference) -> bool {diff --git a/rust/bindings/uniffi/ensu/src/conversation.rs b/rust/bindings/uniffi/ensu/src/conversation.rsindex f7f0914464..b121306a78 100644--- a/rust/bindings/uniffi/ensu/src/conversation.rs+++ b/rust/bindings/uniffi/ensu/src/conversation.rs@@ -264,12 +264,24 @@ impl ConversationPreparation {).map_err(error)?;work.saved = true;- if let Some(grounded) = &work.grounded {- let _ = self.db.save_answer_evidence(+ if let Some(grounded) = work.grounded.take()+ && !grounded.included_passages.is_empty()+ {+ match self.db.save_answer_evidence(&work.snapshot,answer.uuid,- grounded.included_passages.clone(),- );+ grounded.included_passages,+ ) {+ Ok(true) => {}+ Ok(false) => ente_uniffi_log::warn!(+ target: "Conversation",+ "Reply saved, but its source references could not be retained"+ ),+ Err(error) => ente_uniffi_log::warn!(+ target: "Conversation",+ "Reply saved but source reference persistence failed: {error}"+ ),+ }}Ok(to_message(answer))}diff --git a/rust/bindings/uniffi/ensu/src/followup.rs b/rust/bindings/uniffi/ensu/src/followup.rsindex ae64860e1a..17edc75f0c 100644--- a/rust/bindings/uniffi/ensu/src/followup.rs+++ b/rust/bindings/uniffi/ensu/src/followup.rs@@ -1,7 +1,7 @@use crate::{conversation::{ConversationError, error},db::EnsuDb,- retrieval::{GroundedExcerpt, IncludedPassage},+ retrieval::{GroundedExcerpt, PassageLocator},};use ente_ensu::{conversation as core, db, retrieval};use std::sync::{@@ -10,11 +10,6 @@ use std::sync::{};use uuid::Uuid;-#[derive(Clone, uniffi::Record)]-pub struct ConversationFollowupResult {- pub referenced_passages: Vec<IncludedPassage>,-}-#[derive(uniffi::Object)]pub struct ConversationFollowup {db: Arc<db::ChatDb<db::SqliteBackend>>,@@ -26,7 +21,19 @@ pub struct ConversationFollowup {struct SourceSelection {references: Vec<retrieval::IncludedPassage>,- searched: Option<Vec<retrieval::GroundedExcerpt>>,+ searched: Vec<retrieval::GroundedExcerpt>,+}++fn search_excerpts(+ hits: impl IntoIterator<Item = GroundedExcerpt>,+) -> Result<Vec<retrieval::GroundedExcerpt>, ConversationError> {+ core::GroundingCandidates::new(+ Vec::new(),+ hits.into_iter().map(Into::into).collect(),+ core::MAX_GROUNDING_BYTES,+ )+ .map(|candidates| candidates.searched)+ .map_err(error)}#[uniffi::export]@@ -36,7 +43,6 @@ impl EnsuDb {session_uuid: String,path: Vec<String>,question: String,- expected_user_text: String,) -> Result<Arc<ConversationFollowup>, ConversationError> {if path.is_empty()|| path.len() > core::MAX_HISTORY_MESSAGES@@ -58,7 +64,7 @@ impl EnsuDb {if !snapshot.messages().last()- .is_some_and(|m| m.sender == db::Sender::SelfUser && m.text == expected_user_text)+ .is_some_and(|m| m.sender == db::Sender::SelfUser && m.text == question){return Err(ConversationError::Stale);}@@ -104,69 +110,13 @@ impl ConversationFollowup {let selected = stored.as_ref().ok_or_else(|| error("Source selection is not complete"))?;- if let Some(expected) = &selected.searched {- let actual = core::GroundingCandidates::new(- Vec::new(),- searched.iter().cloned().map(Into::into).collect(),- core::MAX_GROUNDING_BYTES,- )- .map_err(error)?- .searched;- if actual != *expected {- return Err(error(- "Source search changed after selecting earlier passages; retry the reply",- ));- }+ if search_excerpts(searched.iter().cloned())? != selected.searched {+ return Err(error(+ "Source search changed after selecting earlier passages; retry the reply",+ ));}Ok((self.question.clone(), selected.references.clone()))}-- fn direct(- &self,- searched: Option<Vec<GroundedExcerpt>>,- ) -> Result<ConversationFollowupResult, ConversationError> {- self.check()?;- let mut stored = self.selection.lock().map_err(error)?;- if stored.is_some() {- return Err(error("Source selection was already attempted"));- }- let searched = searched- .map(|hits| {- core::GroundingCandidates::new(- Vec::new(),- hits.into_iter().map(Into::into).collect(),- core::MAX_GROUNDING_BYTES,- )- .map(|candidates| candidates.searched)- .map_err(error)- })- .transpose()?;- let references = if let Some(searched) = &searched {- let evidence = self- .snapshot- .evidence()- .into_iter()- .cloned()- .collect::<Vec<_>>();- core::direct_followup_references(- self.snapshot.messages(),- &evidence,- &self.question,- searched,- )- .unwrap_or_default()- } else {- Vec::new()- };- let result = ConversationFollowupResult {- referenced_passages: references.iter().cloned().map(Into::into).collect(),- };- *stored = Some(SourceSelection {- references,- searched,- });- Ok(result)- }}#[uniffi::export]@@ -175,145 +125,31 @@ impl ConversationFollowup {self.cancelled.store(true, Ordering::Release);}- pub fn search_as_written(&self) -> Result<ConversationFollowupResult, ConversationError> {- self.direct(None)- }-pub fn search_with_history(&self,searched: Vec<GroundedExcerpt>,- ) -> Result<ConversationFollowupResult, ConversationError> {- self.direct(Some(searched))- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn source_selection_is_bound_to_search_history_and_cancellation() {- let db = EnsuDb {- inner: Arc::new(db::ChatDb::open_in_memory(vec![7; 32]).unwrap()),- };- let session = db.inner.create_session("Direct follow-up").unwrap();- let user = db- .inner- .insert_message(session.uuid, "self", "What does Cedar cover?", None, vec![])- .unwrap();- let source = retrieval::GroundedExcerpt {- locator: retrieval::PassageLocator::LocalNote {- collection_id: Uuid::from_u128(10).to_string(),- document_id: "cedar.md".into(),- indexed_revision: "a".repeat(64),- shard_sha256: "b".repeat(64),- chunk_index: 0,- },- score: 0.9,- text: "Cedar requires 30 days.".into(),- source: retrieval::GroundedSource::LocalNote {- reference: ente_ensu::notes::NoteSourceReference {- collection_id: Uuid::from_u128(10).to_string(),- collection_label: None,- document_id: "cedar.md".into(),- indexed_revision: "a".repeat(64),- title: "Cedar".into(),- section: None,- },- },- };- let grounded =- retrieval::build_grounded_prompt_context(std::slice::from_ref(&source), 6000)- .unwrap()- .unwrap();- let before = db- .inner- .conversation_snapshot(session.uuid, &[user.uuid])- .unwrap();- let text =- retrieval::finalize_grounded_assistant_text("Cedar covers returns.", &grounded.sources)- .unwrap();- let answer = db- .inner- .insert_message(session.uuid, "other", &text, Some(user.uuid), vec![])- .unwrap();- assert!(- db.inner- .save_answer_evidence(&before, answer.uuid, grounded.included_passages.clone())- .unwrap()- );- let current = db- .inner- .insert_message(- session.uuid,- "self",- "And the deadline?",- Some(answer.uuid),- vec![],- )- .unwrap();- let snapshot = db- .inner- .conversation_snapshot(session.uuid, &[user.uuid, answer.uuid, current.uuid])- .unwrap();- let start = || {- db.start_conversation_followup(- session.uuid.to_string(),- vec![- user.uuid.to_string(),- answer.uuid.to_string(),- current.uuid.to_string(),- ],- current.text.clone(),- current.text.clone(),- )- .unwrap()- };- let turn = start();- assert!(turn.prepare(&db.inner, &snapshot, &[]).is_err());- let result = turn.search_with_history(vec![]).unwrap();- assert_eq!(- turn.prepare(&db.inner, &snapshot, &[]).unwrap().1,- grounded.included_passages- );- assert_eq!(result.referenced_passages.len(), 1);- assert!(- turn.prepare(&db.inner, &snapshot, &[source.into()])- .is_err()- );- assert!(turn.search_with_history(vec![]).is_err());- let direct = start();- assert!(- direct- .search_as_written()- .unwrap()- .referenced_passages- .is_empty()- );- assert!(- direct- .prepare(&db.inner, &snapshot, &[])- .unwrap()- .1- .is_empty()+ ) -> Result<Vec<PassageLocator>, ConversationError> {+ self.check()?;+ let mut stored = self.selection.lock().map_err(error)?;+ if stored.is_some() {+ return Err(error("Source selection was already attempted"));+ }+ let searched = search_excerpts(searched)?;+ let evidence = self.snapshot.evidence();+ let references = core::direct_followup_references(+ self.snapshot.messages(),+ &evidence,+ &self.question,+ &searched,);- let other = Arc::new(db::ChatDb::open_in_memory(vec![7; 32]).unwrap());- assert!(matches!(- turn.prepare(&other, &snapshot, &[]),- Err(ConversationError::Stale)- ));- let cancelled = start();- cancelled.cancel();- assert!(matches!(- cancelled.search_with_history(vec![]),- Err(ConversationError::Cancelled)- ));- db.inner- .update_message_text(user.uuid, "Edited original")- .unwrap();- assert!(matches!(- turn.prepare(&db.inner, &snapshot, &[]),- Err(ConversationError::Stale)- ));+ let locators = references+ .iter()+ .map(|reference| reference.locator.clone().into())+ .collect();+ *stored = Some(SourceSelection {+ references,+ searched,+ });+ Ok(locators)}}diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rsindex d8eda806c7..e737ac970e 100644--- a/rust/bindings/uniffi/ensu/src/retrieval.rs+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs@@ -101,60 +101,6 @@ impl From<PassageLocator> for core::PassageLocator {}}-#[derive(Debug, Clone, uniffi::Record)]-pub struct PassageSpan {- pub start_utf8: u32,- pub end_utf8: u32,- pub text_sha256: String,-}--impl From<core::PassageSpan> for PassageSpan {- fn from(value: core::PassageSpan) -> Self {- Self {- start_utf8: value.start_utf8,- end_utf8: value.end_utf8,- text_sha256: value.text_sha256,- }- }-}--impl From<PassageSpan> for core::PassageSpan {- fn from(value: PassageSpan) -> Self {- Self {- start_utf8: value.start_utf8,- end_utf8: value.end_utf8,- text_sha256: value.text_sha256,- }- }-}--#[derive(Debug, Clone, uniffi::Record)]-pub struct IncludedPassage {- pub locator: PassageLocator,- pub cleaning_version: u32,- pub spans: Vec<PassageSpan>,-}--impl From<core::IncludedPassage> for IncludedPassage {- fn from(value: core::IncludedPassage) -> Self {- Self {- locator: value.locator.into(),- cleaning_version: value.cleaning_version,- spans: value.spans.into_iter().map(Into::into).collect(),- }- }-}--impl From<IncludedPassage> for core::IncludedPassage {- fn from(value: IncludedPassage) -> Self {- Self {- locator: value.locator.into(),- cleaning_version: value.cleaning_version,- spans: value.spans.into_iter().map(Into::into).collect(),- }- }-}-#[derive(Debug, Clone, uniffi::Record)]pub struct RetrievalHit {pub locator: PassageLocator,diff --git a/rust/bindings/uniffi/log/src/lib.rs b/rust/bindings/uniffi/log/src/lib.rsindex 4313ac670a..e4829774d4 100644--- a/rust/bindings/uniffi/log/src/lib.rs+++ b/rust/bindings/uniffi/log/src/lib.rs@@ -1,3 +1,5 @@+pub use log::warn;+#[macro_export]macro_rules! setup {() => {diff --git a/rust/crates/ensu-crypto/src/lib.rs b/rust/crates/ensu-crypto/src/lib.rsindex 87bbb4fbbc..75ae9bd599 100644--- a/rust/crates/ensu-crypto/src/lib.rs+++ b/rust/crates/ensu-crypto/src/lib.rs@@ -34,7 +34,15 @@ pub struct EncryptedChatPayload {}pub fn sha256(bytes: &[u8]) -> [u8; 32] {- Sha256::digest(bytes).into()+ sha256_parts(&[bytes])+}++pub fn sha256_parts(parts: &[&[u8]]) -> [u8; 32] {+ let mut hash = Sha256::new();+ for bytes in parts {+ hash.update(bytes);+ }+ hash.finalize().into()}pub fn generate_chat_key() -> String {diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex da2d21135b..087b289d1c 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -14,7 +14,6 @@ rusqlite = { workspace = true, default-features = true, optional = true }self_cell = "1.0.0"serde.workspace = trueserde_json.workspace = true-sha2.workspace = truethiserror.workspace = truetokio = { workspace = true, features = ["rt", "time"] }transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx", "vad-silero"], optional = true }diff --git a/rust/crates/ensu/src/conversation.rs b/rust/crates/ensu/src/conversation.rsindex 570dedda7b..24d4ebbb80 100644--- a/rust/crates/ensu/src/conversation.rs+++ b/rust/crates/ensu/src/conversation.rs@@ -15,8 +15,6 @@ use lookup::{HistoryLookup, lookup_history, requested_history};pub use prepare::{Effects, Preparation, PreparationResult, PrepareError};pub use state::AnswerEvidence;pub(crate) use state::{ConversationEnvelope, valid_answer_passages};-#[cfg(test)]-pub(crate) use state::{MAX_ANSWER_PASSAGES, MAX_EVIDENCE_ANSWERS, MAX_EVIDENCE_BYTES};pub use turn::{PreparedTurn, TurnInput, prepare_turn};#[cfg(test)]diff --git a/rust/crates/ensu/src/conversation/followup.rs b/rust/crates/ensu/src/conversation/followup.rsindex 37edf4dcde..b98e304c6f 100644--- a/rust/crates/ensu/src/conversation/followup.rs+++ b/rust/crates/ensu/src/conversation/followup.rs@@ -2,37 +2,70 @@ use super::{AnswerEvidence, valid_answer_passages};use crate::db::Message;use crate::retrieval::IncludedPassage;+fn refers_back(question: &str) -> bool {+ question.split(|c: char| !c.is_alphanumeric()).any(|word| {+ matches!(+ word.to_lowercase().as_str(),+ "it" | "its"+ | "this"+ | "that"+ | "these"+ | "those"+ | "my"+ | "mine"+ | "our"+ | "ours"+ | "they"+ | "them"+ | "their"+ | "theirs"+ | "she"+ | "her"+ | "hers"+ | "he"+ | "him"+ | "his"+ | "former"+ | "latter"+ | "above"+ | "previous"+ | "earlier"+ )+ })+}+pub fn direct_followup_references(history: &[Message],evidence: &[AnswerEvidence],question: &str,searched: &[crate::retrieval::GroundedExcerpt],-) -> Option<Vec<IncludedPassage>> {+) -> Vec<IncludedPassage> {let Some((message, entry)) = history.iter().rev().skip(1).find_map(|message| {evidence.iter().find(|entry| entry.assistant_message_uuid == message.uuid && entry.matches(history)).map(|entry| (message, entry))}) else {- return Some(Vec::new());+ return Vec::new();};if !valid_answer_passages(&entry.passages) {- return None;+ return Vec::new();}let prior = crate::retrieval::parse_grounded_assistant_text(&message.text).sources;let named = super::grounding::named_source_keys(question,prior.iter().chain(searched.iter().map(|hit| &hit.source)),);- if !named.is_empty()+ if !refers_back(question)+ && !named.is_empty()&& named.iter().all(|key| {searched.iter().any(|hit| super::grounding::source_key(&hit.source) == *key)}){- Some(Vec::new())+ Vec::new()} else {- Some(entry.passages.clone())+ entry.passages.clone()}}diff --git a/rust/crates/ensu/src/conversation/grounding.rs b/rust/crates/ensu/src/conversation/grounding.rsindex 3568fdfba8..11695c5c69 100644--- a/rust/crates/ensu/src/conversation/grounding.rs+++ b/rust/crates/ensu/src/conversation/grounding.rs@@ -51,7 +51,8 @@ pub(super) fn named_source_keys<'a>(retrieval::GroundedSource::LocalNote { reference } => &reference.title,retrieval::GroundedSource::EnsuPack { citation } => &citation.title,};- if !words(title).is_empty() {+ let mut label_words = words(title);+ if !label_words.is_empty() {let title = tokens(title);for (start, window) in query_tokens.windows(title.len()).enumerate() {if window == title {@@ -59,16 +60,13 @@ pub(super) fn named_source_keys<'a>(}}}- let label = match source {- retrieval::GroundedSource::LocalNote { reference } => {- format!("{} {}", reference.title, reference.document_id)- }- retrieval::GroundedSource::EnsuPack { citation } => citation.title.clone(),- };+ if let retrieval::GroundedSource::LocalNote { reference } = source {+ label_words.extend(words(&reference.document_id));+ }documents.entry(source_key(source)).or_default()- .extend(words(&label));+ .extend(label_words);}let query_words = words(query);documents@@ -95,7 +93,7 @@ pub(super) fn named_source_keys<'a>(}impl GroundingCandidates {- pub fn protect_named_sources(&mut self, query: &str) -> Result<(), PrepareError> {+ pub(super) fn protect_named_sources(&mut self, query: &str) -> Result<(), PrepareError> {self.validate()?;if query.len() > super::lookup::LOOKUP_MAX_QUERY_BYTES {return Ok(());@@ -107,34 +105,86 @@ impl GroundingCandidates {.map(|r| &r.source).chain(self.searched.iter().map(|r| &r.source)),);+ let fresh_index = if !self.referenced.is_empty() && named.is_empty() {+ self.searched.iter().position(|hit| {+ !hit.text.trim().is_empty()+ && !self+ .referenced+ .iter()+ .any(|r| r.reference.locator == hit.locator)+ })+ } else {+ None+ };+ if let Some(index) = fresh_index {+ self.searched[..=index].rotate_right(1);+ }let mut required: std::collections::BTreeSet<_> = self.referenced.iter().map(|r| source_key(&r.source)).collect();- for hit in &self.searched {+ let mut promoted = Vec::new();+ for (index, hit) in self.searched.iter().enumerate() {+ let fresh = fresh_index.is_some() && index == 0;let id = source_key(&hit.source);- if !named.contains(&id) || !required.insert(id) {+ if !fresh && (!named.contains(&id) || required.contains(&id)) {continue;}- let Some(reference) =- retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), usize::MAX)+ let budgets = if fresh {+ [self.max_utf8_bytes / 2, self.max_utf8_bytes]+ } else {+ [usize::MAX; 2]+ };+ let Some(reference) = budgets.into_iter().find_map(|budget| {+ retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), budget).ok()- .flatten()- .and_then(|context| context.included_passages.into_iter().next())- else {+ .flatten()?+ .included_passages+ .into_iter()+ .next()+ }) else {continue;};let Some(passages) = reference.verified_spans(&hit.text) else {continue;};- self.referenced.push(ReferencedPassage {+ required.insert(id);+ promoted.push(ReferencedPassage {reference,source: hit.source.clone(),passages,});+ }++ let (mut named_references, incidental): (Vec<_>, Vec<_>) =+ std::mem::take(&mut self.referenced)+ .into_iter()+ .partition(|r| named.contains(&source_key(&r.source)));+ named_references.extend(promoted);+ let mut seen = std::collections::BTreeSet::new();+ let (distinct, additional): (Vec<_>, Vec<_>) = named_references+ .into_iter()+ .partition(|r| seen.insert(source_key(&r.source)));+ let mut packs = 0;+ let mut notes = 0;+ for reference in distinct.into_iter().chain(additional).chain(incidental) {+ let (count, limit) = match &reference.source {+ retrieval::GroundedSource::EnsuPack { .. } => {+ (&mut packs, retrieval::MAX_PACK_HITS)+ }+ retrieval::GroundedSource::LocalNote { .. } => {+ (&mut notes, retrieval::MAX_NOTES_GROUNDING_HITS)+ }+ };+ if *count == limit {+ continue;+ }+ self.referenced.push(reference);if self.validate().is_err() {self.referenced.pop();+ } else {+ *count += 1;}}Ok(())diff --git a/rust/crates/ensu/src/conversation/grounding_tests.rs b/rust/crates/ensu/src/conversation/grounding_tests.rsindex 1f7276582b..5624796852 100644--- a/rust/crates/ensu/src/conversation/grounding_tests.rs+++ b/rust/crates/ensu/src/conversation/grounding_tests.rs@@ -28,6 +28,305 @@ fn hit(text: &str, id: &str) -> GroundedExcerpt {fn measure(messages: &[ChatMessage]) -> Result<usize, PrepareError> {Ok(messages.iter().map(|m| m.content.chars().count() + 8).sum())}++fn pack_hit(text: &str, title: &str, row: u64) -> GroundedExcerpt {+ GroundedExcerpt {+ locator: PassageLocator::EnsuPack {+ dataset_id: "simplewiki".into(),+ revision_sha256: "a".repeat(64),+ row,+ },+ source: GroundedSource::EnsuPack {+ citation: retrieval::SourceCitation {+ dataset_id: "simplewiki".into(),+ dataset_label: "Wikipedia".into(),+ credit: "Wikipedia".into(),+ title: title.into(),+ source_url: format!("https://example.com/{title}"),+ license_label: "CC BY-SA 4.0".into(),+ license_url: "https://creativecommons.org/licenses/by-sa/4.0/".into(),+ },+ },+ score: 0.9,+ text: text.into(),+ }+}++fn followup_history(prior: &[GroundedExcerpt]) -> (Vec<Message>, Vec<AnswerEvidence>) {+ let old = retrieval::build_grounded_prompt_context(prior, MAX_GROUNDING_BYTES)+ .unwrap()+ .unwrap();+ let mut history: Vec<_> = (1..=3)+ .map(|i| Message {+ uuid: Uuid::from_u128(i),+ session_uuid: Uuid::from_u128(100),+ parent_message_uuid: (i > 1).then(|| Uuid::from_u128(i - 1)),+ sender: if i == 2 {+ Sender::Other+ } else {+ Sender::SelfUser+ },+ text: "Explain the sources.".into(),+ attachments: vec![],+ created_at: i as i64,+ })+ .collect();+ history[1].text =+ retrieval::finalize_grounded_assistant_text("See the cited sources.", &old.sources)+ .unwrap();+ let evidence = vec![AnswerEvidence {+ assistant_message_uuid: history[1].uuid,+ prefix_fingerprint: fingerprint(&history[..2]),+ passages: old.included_passages,+ }];+ (history, evidence)+}++fn fit_followup(+ prior: &[GroundedExcerpt],+ searched: Vec<GroundedExcerpt>,+ query: &str,+ input_budget: usize,+) -> GroundedPromptContext {+ let (history, evidence) = followup_history(prior);+ let reloaded = direct_followup_references(&history, &evidence, query, &searched)+ .into_iter()+ .map(|reference| {+ let hit = prior+ .iter()+ .find(|hit| hit.locator == reference.locator)+ .unwrap();+ ReferencedPassage {+ passages: reference.verified_spans(&hit.text).unwrap(),+ source: hit.source.clone(),+ reference,+ }+ })+ .collect();+ let mut candidates = GroundingCandidates::new(reloaded, searched, MAX_GROUNDING_BYTES).unwrap();+ candidates.protect_named_sources(query).unwrap();+ let context = fit_grounding(&candidates, "System", query, input_budget, measure)+ .unwrap()+ .context+ .unwrap();+ assert!(context.text.len() <= MAX_GROUNDING_BYTES);+ assert!(+ measure(&answer_messages(+ &grounded_system("System", Some(&context)),+ None,+ &[],+ query+ ))+ .unwrap()+ <= input_budget+ );+ context+}++#[test]+fn followup_fitting_keeps_implicit_antecedents_and_allows_named_switches() {+ let cedar = hit("Cedar allows 30 days.", "cedar.md");+ let juniper = hit("Juniper allows 21 days.", "juniper.md");+ let old = retrieval::build_grounded_prompt_context(std::slice::from_ref(&cedar), 6000)+ .unwrap()+ .unwrap();+ for query in [+ "Is Juniper longer than that?",+ "Compare its deadline with Juniper.",+ "Is Juniper's return period longer than ours?",+ "Is Juniper's return period longer than mine?",+ "Compare Juniper's return period with our policy.",+ "Compare Juniper's return period with my policy.",+ "Does she get more time than Juniper?",+ "Compare her deadline with Juniper.",+ "Is Juniper's deadline longer than hers?",+ "Does he get more time than Juniper?",+ "Does Juniper get more time than him?",+ "Compare his deadline with Juniper.",+ ] {+ let context = fit_followup(+ std::slice::from_ref(&cedar),+ vec![juniper.clone()],+ query,+ 7000,+ );+ assert!(context.text.contains(&cedar.text), "{query}");+ assert!(context.text.contains(&juniper.text), "{query}");+ assert!(+ context+ .included_passages+ .contains(&old.included_passages[0])+ );+ }+ let updated = hit("Cedar now requires seven days.", "cedar.md");+ for (query, fresh) in [+ ("What is Juniper's return period?", &juniper),+ ("What is within Juniper's coverage?", &juniper),+ ("What is Cedar's current deadline?", &updated),+ ] {+ let context = fit_followup(+ std::slice::from_ref(&cedar),+ vec![fresh.clone()],+ query,+ 7000,+ );+ assert_eq!(context.sources, vec![fresh.source.clone()], "{query}");+ assert!(context.text.contains(&fresh.text), "{query}");+ }+}++#[test]+fn followup_fitting_prioritizes_named_sources_at_source_and_byte_limits() {+ let mut prior: Vec<_> = ["pine", "birch", "oak", "maple", "cedar"]+ .iter()+ .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))+ .collect();+ let atlas = hit(&"Atlas allows 21 days. ".repeat(40), "atlas.md");+ let note_count = prior.len();+ prior.extend([+ pack_hit("Mars is the fourth planet.", "Mars", 0),+ pack_hit("Venus is the second planet.", "Venus", 1),+ ]);+ let byte_limited = [+ prior[note_count - 1].clone(),+ hit(&"Unrelated detail. ".repeat(400), "pine.md"),+ ];+ for prior in [&prior[..note_count], &prior[..], &byte_limited] {+ let context = fit_followup(prior, vec![atlas.clone()], "Compare Cedar and Atlas", 7000);+ assert!(context.text.contains("cedar allows 30 days."));+ assert!(context.text.contains(atlas.text.trim()));+ assert!(context.sources.len() <= retrieval::MAX_GROUNDING_HITS);+ }++ let mars = pack_hit("Mars is the fourth planet.", "Mars", 0);+ let venus = pack_hit("Venus is the second planet.", "Venus", 1);+ let jupiter = pack_hit("Jupiter is the fifth planet.", "Jupiter", 2);+ let another_mars_chunk = pack_hit("Mars also has two moons.", "Mars", 3);+ for prior in [[venus, mars.clone()], [mars.clone(), another_mars_chunk]] {+ let context = fit_followup(+ &prior,+ vec![jupiter.clone()],+ "Compare Mars and Jupiter",+ 7000,+ );+ assert!(context.text.contains(&mars.text));+ assert!(context.text.contains(&jupiter.text));+ assert_eq!(context.sources.len(), 2);+ }+}++#[test]+fn unnamed_questions_keep_fresh_results_and_available_historical_passages() {+ let notes: Vec<_> = ["cedar", "pine", "birch", "oak", "maple"]+ .iter()+ .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))+ .collect();+ let biology = hit(+ "Photosynthesis converts light into energy.",+ "biology-chapter-seven.md",+ );+ let mut another_chunk = notes[0].clone();+ if let PassageLocator::LocalNote { chunk_index, .. } = &mut another_chunk.locator {+ *chunk_index += 1;+ }+ another_chunk.text = "International shipping has an extended deadline.".into();+ let old = retrieval::build_grounded_prompt_context(¬es, MAX_GROUNDING_BYTES)+ .unwrap()+ .unwrap();+ for (query, fresh) in [+ ("How does photosynthesis work?", &biology),+ ("And the deadline?", &biology),+ ("And the deadline?", &another_chunk),+ ] {+ let context = fit_followup(¬es, vec![fresh.clone()], query, 7000);+ assert!(context.text.contains(&fresh.text), "{query}");+ assert!(context.text.contains(¬es[0].text), "{query}");+ assert!(+ context+ .included_passages+ .contains(&old.included_passages[0])+ );+ assert!(context.sources.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);+ }++ let packs = [+ pack_hit("Mars is the fourth planet.", "Mars", 0),+ pack_hit("Venus is the second planet.", "Venus", 1),+ ];+ let fresh = pack_hit("Plants use sunlight to produce sugar.", "Botany", 2);+ let context = fit_followup(+ &packs,+ vec![fresh.clone()],+ "How does photosynthesis work?",+ 7000,+ );+ assert!(context.text.contains(&fresh.text));+ assert!(context.text.contains(&packs[0].text));+ assert_eq!(context.sources.len(), retrieval::MAX_PACK_HITS);++ let context = fit_followup(¬es, vec![], "And the deadline?", 7000);+ assert_eq!(context.included_passages, old.included_passages);+}++#[test]+fn fresh_followup_results_survive_byte_and_token_repacking() {+ let prefix = hit("Price: 100 rupees.", "cedar.md");+ let mut full = prefix.clone();+ full.text.push_str(" Warranty duration: 24 months.");+ let context = fit_followup(+ &[prefix],+ vec![full.clone()],+ "And what's its warranty duration?",+ 7000,+ );+ assert!(context.text.contains(&full.text));+ assert_eq!(context.sources, vec![full.source]);+ assert_eq!(context.included_passages.len(), 1);+ assert_eq!(+ context.included_passages[0]+ .verified_spans(&full.text)+ .unwrap(),+ vec![full.text]+ );++ let old = hit(&"Cedar allows 30 days. ".repeat(280), "cedar.md");+ let fresh = hit(+ &format!(+ "Plants use light. {}",+ "Photosynthesis details. ".repeat(300)+ ),+ "biology-chapter-seven.md",+ );+ for input_budget in [7000, 1600] {+ let context = fit_followup(+ std::slice::from_ref(&old),+ vec![old.clone(), fresh.clone()],+ "How does photosynthesis work?",+ input_budget,+ );+ assert!(context.text.contains("Plants use light."));+ let reference = context+ .included_passages+ .iter()+ .find(|reference| reference.locator == fresh.locator)+ .unwrap();+ let spans = reference.verified_spans(&fresh.text).unwrap();+ assert!(context.text.contains(&spans[0]));+ }++ let mut long_label = hit("Plants use light.", "biology.md");+ if let GroundedSource::LocalNote { reference } = &mut long_label.source {+ reference.title = "Biology ".repeat(400);+ }+ let context = fit_followup(+ &[old],+ vec![long_label.clone()],+ "How does photosynthesis work?",+ 7000,+ );+ assert!(context.text.contains(&long_label.text));+}+#[test]fn measured_repack_keeps_text_citations_and_exact_spans_together() {for (raw, prefix) in [@@ -44,19 +343,15 @@ fn measured_repack_keeps_text_citations_and_exact_spans_together() {),] {let candidates = GroundingCandidates::new(vec![], vec![hit(&raw, "one.md")], 6000).unwrap();- let original = serde_json::to_string(&candidates).unwrap();let fit = fit_grounding(&candidates, "System", "Question", 4096, measure).unwrap();assert!(fit.repacked);assert!(fit.evidence_tokens <= 1024);let context = fit.context.unwrap();- assert_eq!(context.sources.len(), context.included_passages.len());+ assert_eq!(context.sources, vec![candidates.searched[0].source.clone()]);assert_eq!(context.included_passages.len(), 1);- for reference in &context.included_passages {- let spans = reference.verified_spans(&raw).unwrap();- assert!(context.text.contains(&spans[0]));- assert!(spans[0].starts_with(prefix));- }- assert_eq!(serde_json::to_string(&candidates).unwrap(), original);+ let spans = context.included_passages[0].verified_spans(&raw).unwrap();+ assert!(context.text.contains(&spans[0]));+ assert!(spans[0].starts_with(prefix));}}#[test]@@ -82,11 +377,6 @@ fn requested_old_spans_raise_soft_allowance_and_are_never_cut() {let fit = fit_grounding(&candidates, "System", "Question", 3000, measure).unwrap();assert!(fit.repacked && fit.evidence_tokens > 750);assert_eq!(fit.context.unwrap().included_passages[0], reference);- assert!(- fit_grounding(&candidates, "System", "Question", 700, measure)- .unwrap()- .repacked- );}#[test]fn mandatory_input_and_insufficient_evidence_budget_are_explicit() {@@ -134,7 +424,6 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {candidates.protect_named_sources("Is the Juniper period longer than Cedar?").unwrap();- assert_eq!(candidates.referenced.len(), 2);let fit = fit_grounding(&candidates,"System",@@ -157,42 +446,10 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {vec![hit.text.clone()]);}- assert!(- fit_grounding(- &candidates,- "System",- "Compare Cedar and Juniper.",- 900,- measure- )- .unwrap()- .repacked- );}#[test]fn ordinary_questions_and_shared_label_words_do_not_promote_every_source() {- let mut partial = GroundingCandidates::new(- vec![],- vec![hit(" ", "cedar.md"), hit("21 days", "juniper.md")],- 6000,- )- .unwrap();- partial- .protect_named_sources("Compare Cedar and Juniper")- .unwrap();- let context = fit_grounding(- &partial,- "System",- "Compare Cedar and Juniper",- 4096,- measure,- )- .unwrap()- .context- .unwrap();- assert_eq!(context.sources.len(), 1);- assert!(context.text.contains("21 days"));let base = GroundingCandidates::new(vec![],vec![@@ -210,34 +467,6 @@ fn ordinary_questions_and_shared_label_words_do_not_promote_every_source() {}}-#[test]-fn an_old_reference_and_new_named_document_are_both_protected() {- let old = hit("Cedar 30 days.", "cedar.md");- let packed = retrieval::build_grounded_prompt_context(std::slice::from_ref(&old), 6000)- .unwrap()- .unwrap();- let reference = packed.included_passages[0].clone();- let required = ReferencedPassage {- passages: reference.verified_spans(&old.text).unwrap(),- source: old.source,- reference: reference.clone(),- };- let juniper = hit("Juniper 21 days.", "juniper.md");- let mut second_chunk = juniper.clone();- if let PassageLocator::LocalNote { chunk_index, .. } = &mut second_chunk.locator {- *chunk_index = 1;- }- let mut c =- GroundingCandidates::new(vec![required], vec![juniper, second_chunk], 6000).unwrap();- c.protect_named_sources("Is Juniper longer than that?")- .unwrap();- assert_eq!(c.referenced.len(), 2);- assert_eq!(c.referenced[0].reference, reference);- c.protect_named_sources("Compare Cedar and Juniper.")- .unwrap();- assert_eq!(c.referenced.len(), 2);-}-fn prior_history() -> Vec<Message> {vec![Message {uuid: Uuid::from_u128(1),@@ -326,23 +555,7 @@ fn source_system_and_required_history_are_measured_in_final_roles() {#[test]fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switches() {- let mut pack = hit("Mars is the fourth planet.", "unused.md");- pack.locator = PassageLocator::EnsuPack {- dataset_id: "wiki".into(),- revision_sha256: "a".repeat(64),- row: 0,- };- pack.source = GroundedSource::EnsuPack {- citation: crate::retrieval::SourceCitation {- dataset_id: "wiki".into(),- dataset_label: "Wiki".into(),- credit: "Wiki".into(),- title: "Mars".into(),- source_url: "https://example.com/mars".into(),- license_label: "CC BY-SA 4.0".into(),- license_url: "https://creativecommons.org/licenses/by-sa/4.0/".into(),- },- };+ let pack = pack_hit("Mars is the fourth planet.", "Mars", 0);let mut note = hit("Check-in code COPPER-29.", "mars-expedition.md");if let GroundedSource::LocalNote { reference } = &mut note.source {reference.title = "Mars expedition checklist".into();@@ -352,37 +565,11 @@ fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switcc.protect_named_sources(query).unwrap();assert_eq!(c.referenced.len(), 2);assert_eq!(c.pack(6000).unwrap().unwrap().sources.len(), 2);- assert!(c.pack(100).is_err());- let context = retrieval::build_grounded_prompt_context(&[pack], 6000)- .unwrap()- .unwrap();- let mut history: Vec<_> = (1..=3)- .map(|i| Message {- uuid: Uuid::from_u128(i),- session_uuid: Uuid::from_u128(100),- parent_message_uuid: (i > 1).then(|| Uuid::from_u128(i - 1)),- sender: if i == 2 {- Sender::Other- } else {- Sender::SelfUser- },- text: query.into(),- attachments: vec![],- created_at: i as i64,- })- .collect();- history[1].text =- retrieval::finalize_grounded_assistant_text("Fourth planet", &context.sources).unwrap();- let evidence = vec![AnswerEvidence {- assistant_message_uuid: history[1].uuid,- prefix_fingerprint: fingerprint(&history[..2]),- passages: context.included_passages.clone(),- }];+ let (history, evidence) = followup_history(&[pack]);assert_eq!(- direct_followup_references(&history, &evidence, query, std::slice::from_ref(¬e))- .unwrap(),- context.included_passages+ direct_followup_references(&history, &evidence, query, std::slice::from_ref(¬e)),+ evidence[0].passages);assert!(direct_followup_references(@@ -391,90 +578,29 @@ fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switc"What is the check-in code in the Mars expedition checklist?",&[note])- .unwrap().is_empty());}#[test]-fn direct_followups_reuse_saved_spans_but_named_search_can_change_sources() {- let prior: Vec<_> = ["cedar", "pine", "birch", "oak", "maple"]- .iter()- .map(|name| hit("Archived policy fact.", &format!("{name}.md")))- .collect();- let context = retrieval::build_grounded_prompt_context(&prior, 6000)- .unwrap()- .unwrap();- let mut history: Vec<_> = (1..=5)- .map(|i| Message {+fn followups_use_last_available_evidence_until_its_history_changes() {+ let (mut history, evidence) = followup_history(&[hit("Archived policy fact.", "cedar.md")]);+ for i in 4..=5 {+ history.push(Message {uuid: Uuid::from_u128(i),- session_uuid: Uuid::from_u128(100),- parent_message_uuid: (i > 1).then(|| Uuid::from_u128(i - 1)),- sender: if i % 2 == 0 {+ parent_message_uuid: Some(Uuid::from_u128(i - 1)),+ sender: if i == 4 {Sender::Other} else {Sender::SelfUser},- text: "Please clarify the subject.".into(),- attachments: vec![],- created_at: i as i64,- })- .collect();- history[1].text =- retrieval::finalize_grounded_assistant_text("Saved facts", &context.sources).unwrap();- let evidence = vec![AnswerEvidence {- assistant_message_uuid: history[1].uuid,- prefix_fingerprint: fingerprint(&history[..2]),- passages: context.included_passages.clone(),- }];- let atlas = hit("Warranty lasts two years.", "atlas.md");- assert_eq!(- direct_followup_references(&history, &evidence, "And the deadline?", &[]).unwrap(),- context.included_passages- );- assert!(- direct_followup_references(- &history,- &evidence,- "How long is the Atlas warranty?",- std::slice::from_ref(&atlas)- )- .unwrap()- .is_empty()- );- let mut candidates = GroundingCandidates::new(vec![], vec![atlas.clone()], 6000).unwrap();- candidates- .protect_named_sources("How long is the Atlas warranty?")- .unwrap();- assert!(- candidates- .pack(6000)- .unwrap()- .unwrap()- .text- .contains(&atlas.text)- );- let mut updated = prior[0].clone();- updated.text = "Cedar now requires seven days.".into();- assert!(- direct_followup_references(- &history,- &evidence,- "What is Cedar's current deadline?",- &[updated]- )- .unwrap()- .is_empty()- );+ ..history[2].clone()+ });+ }assert_eq!(- direct_followup_references(&history, &evidence, "Compare Cedar and Atlas", &[atlas])- .unwrap(),- context.included_passages+ direct_followup_references(&history, &evidence, "And the deadline?", &[]),+ evidence[0].passages);history[0].text = "Edited history".into();- assert!(- direct_followup_references(&history, &evidence, "And the deadline?", &[])- .unwrap()- .is_empty()- );+ assert!(direct_followup_references(&history, &evidence, "And the deadline?", &[]).is_empty());}diff --git a/rust/crates/ensu/src/conversation/state.rs b/rust/crates/ensu/src/conversation/state.rsindex 105bc2a8a0..5873549475 100644--- a/rust/crates/ensu/src/conversation/state.rs+++ b/rust/crates/ensu/src/conversation/state.rs@@ -19,7 +19,7 @@ pub struct AnswerEvidence {}impl AnswerEvidence {- pub fn matches(&self, history: &[Message]) -> bool {+ pub(crate) fn matches(&self, history: &[Message]) -> bool {history.iter().position(|m| m.uuid == self.assistant_message_uuid)@@ -30,14 +30,13 @@ impl AnswerEvidence {}fn valid(&self) -> bool {- self.prefix_fingerprint.len() == 64- && !self.passages.is_empty()- && valid_answer_passages(&self.passages)+ self.prefix_fingerprint.len() == 64 && valid_answer_passages(&self.passages)}}pub(crate) fn valid_answer_passages(passages: &[IncludedPassage]) -> bool {- passages.len() <= MAX_ANSWER_PASSAGES+ !passages.is_empty()+ && passages.len() <= MAX_ANSWER_PASSAGES&& passages.iter().all(IncludedPassage::valid_metadata)&& serde_json::to_vec(passages).is_ok_and(|bytes| bytes.len() <= MAX_EVIDENCE_BYTES)}@@ -117,7 +116,6 @@ impl ConversationEnvelope {if history.len() < 2|| answer.sender != Sender::Other|| answer.session_uuid != self.session_uuid- || passages.is_empty()|| !valid_answer_passages(&passages)|| super::validate_path(history).is_err(){diff --git a/rust/crates/ensu/src/conversation/tests.rs b/rust/crates/ensu/src/conversation/tests.rsindex d327029078..743b8bbd85 100644--- a/rust/crates/ensu/src/conversation/tests.rs+++ b/rust/crates/ensu/src/conversation/tests.rs@@ -1,3 +1,4 @@+use super::state::{MAX_ANSWER_PASSAGES, MAX_EVIDENCE_ANSWERS, MAX_EVIDENCE_BYTES};use super::*;fn message(index: u128, sender: Sender, text: &str) -> Message {@@ -76,6 +77,69 @@ fn saved_memory_preserves_uuid_strings() {);}+fn evidence_passages() -> Vec<crate::retrieval::IncludedPassage> {+ vec![crate::retrieval::passage_fixture().0]+}++#[test]+fn invalid_summary_and_evidence_are_discarded_independently() {+ let history = vec![+ message(1, Sender::SelfUser, "Question"),+ message(2, Sender::Other, "Answer"),+ ];+ let session = history[0].session_uuid;+ let memory = ConversationState::new(session, &history, "Memory".into()).unwrap();+ let mut state = ConversationEnvelope::empty(session);+ state.summary = Some(memory.summary.clone());+ assert!(state.add_evidence(&history, evidence_passages()));+ let original = serde_json::to_value(&state).unwrap();+ let decode = |value: &serde_json::Value| {+ ConversationEnvelope::decode(&serde_json::to_vec(value).unwrap(), session)+ };+ let mut value = original.clone();+ value["summary"] = serde_json::json!({"bad": true});+ let decoded = decode(&value).unwrap();+ assert!(decoded.summary.is_none());+ assert_eq!(decoded.evidence, state.evidence);++ value = original;+ value["evidence"][0]["passages"][0]["spans"][0]["textSha256"] = "bad".into();+ let decoded = decode(&value).unwrap();+ assert!(decoded.evidence.is_empty());+ assert_eq!(decoded.summary_state().unwrap(), memory);+ value["format_version"] = 1.into();+ assert!(decode(&value).is_none());+ assert!(+ ConversationEnvelope::decode(&serde_json::to_vec(&state).unwrap(), Uuid::nil()).is_none()+ );+}++#[test]+fn evidence_retention_obeys_answer_and_byte_limits() {+ for passage_count in [1, MAX_ANSWER_PASSAGES] {+ let passages = vec![evidence_passages().remove(0); passage_count];+ let mut state = ConversationEnvelope::empty(Uuid::from_u128(100));+ let mut history = Vec::new();+ for index in 1..=10 {+ history.push(message(index * 2 - 1, Sender::SelfUser, "Question"));+ history.push(message(index * 2, Sender::Other, "Answer"));+ assert!(state.add_evidence(&history, passages.clone()));+ }+ assert_eq!(+ state.evidence.last().unwrap().assistant_message_uuid,+ Uuid::from_u128(20)+ );+ assert_ne!(state.evidence[0].assistant_message_uuid, Uuid::from_u128(2));+ assert!(serde_json::to_vec(&state.evidence).unwrap().len() <= MAX_EVIDENCE_BYTES);+ if passage_count == 1 {+ assert_eq!(state.evidence.len(), MAX_EVIDENCE_ANSWERS);+ } else {+ assert!(state.evidence.len() < MAX_EVIDENCE_ANSWERS);+ }+ assert!(!state.add_evidence(&history, vec![passages[0].clone(); MAX_ANSWER_PASSAGES + 1]));+ }+}+#[test]fn coverage_tracks_original_text_and_branch_but_not_recent_tail() {let mut history = vec![diff --git a/rust/crates/ensu/src/db/chat/conversation.rs b/rust/crates/ensu/src/db/chat/conversation.rsindex 59ed836bed..9d94b50b89 100644--- a/rust/crates/ensu/src/db/chat/conversation.rs+++ b/rust/crates/ensu/src/db/chat/conversation.rs@@ -24,11 +24,12 @@ impl ConversationSnapshot {pub fn state(&self) -> Option<&ConversationState> {self.state.as_ref()}- pub fn evidence(&self) -> Vec<&AnswerEvidence> {+ pub fn evidence(&self) -> Vec<AnswerEvidence> {self.derived.evidence.iter().filter(|entry| entry.matches(&self.messages))+ .cloned().collect()}pub fn session_uuid(&self) -> Uuid {@@ -165,12 +166,12 @@ impl<B: Backend> ChatDb<B> {answer: Uuid,passages: Vec<crate::retrieval::IncludedPassage>,) -> Result<bool> {- if !conversation::valid_answer_passages(&passages) || passages.is_empty() {+ if !conversation::valid_answer_passages(&passages) {return Ok(false);}let mut path: Vec<_> = snapshot.messages.iter().map(|m| m.uuid).collect();path.push(answer);- let current = self.conversation_snapshot(snapshot.session_uuid, &path)?;+ let mut current = self.conversation_snapshot(snapshot.session_uuid, &path)?;if current.envelope != snapshot.envelope|| current.rows[..snapshot.rows.len()] != snapshot.rows{@@ -188,11 +189,10 @@ impl<B: Backend> ChatDb<B> {}) {return Ok(false);}- let mut derived = current.derived.clone();- if !derived.add_evidence(¤t.messages, passages) {+ if !current.derived.add_evidence(¤t.messages, passages) {return Ok(false);}- let envelope = self.encode_conversation(&derived)?;+ let envelope = self.encode_conversation(¤t.derived)?;self.replace_conversation_envelope(¤t, envelope)}@@ -419,11 +419,7 @@ mod tests {}fn passages() -> Vec<crate::retrieval::IncludedPassage> {- let fixtures: serde_json::Value = serde_json::from_str(include_str!(- "../../../tests/fixtures/passage-references-v1.json"- ))- .unwrap();- vec![serde_json::from_value(fixtures[0]["reference"].clone()).unwrap()]+ vec![crate::retrieval::passage_fixture().0]}fn sourced_answer(@@ -742,133 +738,6 @@ mod tests {assert!(db.get_session(session).unwrap().is_none());}- #[test]- fn malformed_components_do_not_destroy_valid_siblings() {- let (db, session, messages) = seeded();- let memory = state(session, &messages, "Memory");- let mut captured = snapshot(&db, session, &messages);- assert!(- db.replace_conversation_state(&mut captured, Some(memory.clone()))- .unwrap()- );- let answer = sourced_answer(&db, &captured);- assert!(- db.save_answer_evidence(&captured, answer.uuid, passages())- .unwrap()- );- let mut path = messages;- path.push(answer);- let restored = snapshot(&db, session, &path);- let mut value = serde_json::to_value(&restored.derived).unwrap();- assert!(- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), Uuid::new_v4())- .is_none()- );- value["format_version"] = 1.into();- assert!(- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), session).is_none()- );- value = serde_json::to_value(&restored.derived).unwrap();- value["summary"] = serde_json::json!({"bad": true});- let decoded =- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), session).unwrap();- assert!(decoded.summary.is_none());- assert_eq!(decoded.evidence.len(), 1);- value = serde_json::to_value(&restored.derived).unwrap();- value["evidence"][0]["passages"][0]["spans"][0]["textSha256"] = "bad".into();- let decoded =- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), session).unwrap();- assert!(decoded.evidence.is_empty());- assert_eq!(decoded.summary_state().unwrap(), memory);- }-- #[test]- fn evidence_retention_is_bounded_and_keeps_newest_answers() {- let (db, session, mut messages) = seeded();- let mut first_answer = None;- for index in 0..10 {- let captured = snapshot(&db, session, &messages);- let answer = sourced_answer(&db, &captured);- first_answer.get_or_insert(answer.uuid);- assert!(- db.save_answer_evidence(&captured, answer.uuid, passages())- .unwrap()- );- messages.push(answer);- let current = snapshot(&db, session, &messages);- assert!(current.derived.evidence.len() <= conversation::MAX_EVIDENCE_ANSWERS);- assert!(serde_json::to_vec(¤t.derived).unwrap().len() <= MAX_STATE_BYTES);- if index == 9 {- assert_eq!(current.evidence().len(), 8);- assert!(- !current- .evidence()- .iter()- .any(|entry| Some(entry.assistant_message_uuid) == first_answer)- );- } else {- let question = db- .insert_message(- session,- "self",- "Next",- Some(messages.last().unwrap().uuid),- vec![],- )- .unwrap();- messages.push(question);- }- }- }-- #[test]- fn evidence_byte_limit_evicts_older_records_and_rejects_oversized_inputs() {- let (db, session, mut messages) = seeded();- let mut large = Vec::new();- for chunk in 0..conversation::MAX_ANSWER_PASSAGES {- let mut reference = passages().remove(0);- if let crate::retrieval::PassageLocator::LocalNote { chunk_index, .. } =- &mut reference.locator- {- *chunk_index = chunk as u64;- }- large.push(reference);- }- assert!(!conversation::valid_answer_passages(&vec![- large[0].clone();- 8- ]));- let mut future = large.clone();- future[0].cleaning_version = 999;- assert!(!conversation::valid_answer_passages(&future));- for _ in 0..6 {- let captured = snapshot(&db, session, &messages);- let answer = sourced_answer(&db, &captured);- assert!(- db.save_answer_evidence(&captured, answer.uuid, large.clone())- .unwrap()- );- messages.push(answer);- let question = db- .insert_message(- session,- "self",- "Next",- Some(messages.last().unwrap().uuid),- vec![],- )- .unwrap();- messages.push(question);- }- let current = snapshot(&db, session, &messages);- assert!(current.evidence().len() < 6);- assert!(!current.evidence().is_empty());- assert!(- serde_json::to_vec(¤t.derived.evidence).unwrap().len()- <= conversation::MAX_EVIDENCE_BYTES- );- }-#[test]fn encrypted_memory_survives_titles_appends_and_recent_edits() {let (db, session, messages) = seeded();diff --git a/rust/crates/ensu/src/notes/index.rs b/rust/crates/ensu/src/notes/index.rsindex a95496dac3..46468f2695 100644--- a/rust/crates/ensu/src/notes/index.rs+++ b/rust/crates/ensu/src/notes/index.rs@@ -6,6 +6,7 @@ use super::manifest::{};use super::shard::{NotesShard, load_and_validate_shard_metadata, load_and_validate_vectors};use super::{NotesCollectionReadPin, NotesError, pinned_revision_key, validate_collection_id};+use crate::retrieval::PassageLocator;const RELEVANCE_THRESHOLD: f32 = 0.50;const MAX_HITS_PER_COLLECTION: usize = 5;@@ -13,7 +14,7 @@ const MAX_HITS_PER_DOCUMENT: usize = 3;#[derive(Debug, Clone, PartialEq)]pub struct NotesSearchHit {- pub locator: crate::retrieval::PassageLocator,+ pub locator: PassageLocator,pub collection_id: String,pub document_id: String,pub revision: String,@@ -166,25 +167,28 @@ impl NotesCollectionIndex {}))}- fn passage_locator(- &self,- document: &IndexedDocument,- chunk_index: usize,- ) -> crate::retrieval::PassageLocator {- crate::retrieval::PassageLocator::LocalNote {- collection_id: self.collection_id.clone(),- document_id: document.document_id.clone(),- indexed_revision: document.revision.clone(),- shard_sha256: document.shard_sha256.clone(),- chunk_index: chunk_index as u64,- }+ fn load_shard(&self, document: &IndexedDocument) -> Result<NotesShard, NotesError> {+ let directory = shard_directory(+ self.read_pin.collection_directory(),+ &document.document_id,+ &document.revision,+ );+ load_and_validate_shard_metadata(+ &directory,+ &self.collection_id,+ &document.document_id,+ &document.revision,+ document.chunk_count,+ &document.shard_sha256,+ None,+ )}pub fn reload_passage(&self,- locator: &crate::retrieval::PassageLocator,+ locator: &PassageLocator,) -> Result<Option<NotesSearchHit>, NotesError> {- let crate::retrieval::PassageLocator::LocalNote {+ let PassageLocator::LocalNote {collection_id,document_id,indexed_revision,@@ -210,20 +214,7 @@ impl NotesCollectionIndex {if chunk_index >= document.chunk_count {return Ok(None);}- let directory = shard_directory(- self.read_pin.collection_directory(),- document_id,- indexed_revision,- );- let shard = load_and_validate_shard_metadata(- &directory,- collection_id,- document_id,- indexed_revision,- document.chunk_count,- shard_sha256,- None,- )?;+ let shard = self.load_shard(document)?;let chunk = &shard.chunks[chunk_index];Ok(Some(NotesSearchHit {locator: locator.clone(),@@ -295,28 +286,19 @@ impl NotesCollectionIndex {let document = &self.documents[candidate.document_index];let shard = match loaded_shards.entry(candidate.document_index) {Entry::Occupied(entry) => entry.into_mut(),- Entry::Vacant(entry) => {- let directory = shard_directory(- self.read_pin.collection_directory(),- &document.document_id,- &document.revision,- );- entry.insert(load_and_validate_shard_metadata(- &directory,- &self.collection_id,- &document.document_id,- &document.revision,- document.chunk_count,- &document.shard_sha256,- None,- )?)- }+ Entry::Vacant(entry) => entry.insert(self.load_shard(document)?),};let chunk = shard.chunks.get(candidate.chunk_index).ok_or_else(|| {NotesError::InvalidIndex("selected shard chunk is missing".to_string())})?;hits.push(NotesSearchHit {- locator: self.passage_locator(document, candidate.chunk_index),+ locator: PassageLocator::LocalNote {+ collection_id: self.collection_id.clone(),+ document_id: document.document_id.clone(),+ indexed_revision: document.revision.clone(),+ shard_sha256: document.shard_sha256.clone(),+ chunk_index: candidate.chunk_index as u64,+ },collection_id: self.collection_id.clone(),document_id: document.document_id.clone(),revision: document.revision.clone(),@@ -419,7 +401,6 @@ mod tests {#[test]fn passage_reload_checks_revision_chunk_and_shard_integrity() {- use crate::retrieval::PassageLocator;let temp = tempfile::tempdir().unwrap();let doc = document("trip.md", 2);let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();diff --git a/rust/crates/ensu/src/retrieval.rs b/rust/crates/ensu/src/retrieval.rsindex 720131fad6..6fd0445f9f 100644--- a/rust/crates/ensu/src/retrieval.rs+++ b/rust/crates/ensu/src/retrieval.rs@@ -61,3 +61,6 @@ pub use prompt::{KnowledgePromptContext, KnowledgePromptHit, build_knowledge_propub(crate) use grounded::{MAX_GROUNDING_HITS, MAX_PACK_HITS, build_followup_prompt_context};pub(crate) use passage::clean_passage_text;pub use passage::{IncludedPassage, PassageLocator, PassageSpan};++#[cfg(test)]+pub(crate) use passage::tests::passage_fixture;diff --git a/rust/crates/ensu/src/retrieval/grounded.rs b/rust/crates/ensu/src/retrieval/grounded.rsindex a14343d2c7..c5e608eb54 100644--- a/rust/crates/ensu/src/retrieval/grounded.rs+++ b/rust/crates/ensu/src/retrieval/grounded.rs@@ -265,6 +265,8 @@ pub(crate) fn build_followup_prompt_context(}let mut text = format!("{BEGIN_CONTEXT_SENTINEL}\n{CONTEXT_WARNING}");let footer = format!("\n{END_CONTEXT_SENTINEL}");+ let mut sections = Vec::new();+ let mut used_bytes = text.len() + footer.len();let mut sources = Vec::new();let mut included_passages = Vec::new();let mut packs = 0;@@ -295,25 +297,48 @@ pub(crate) fn build_followup_prompt_context(}let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);let passages = item.passages.join("\n[omitted source text]\n");- if text- .len()+ if used_bytes.saturating_add(prefix.len()).saturating_add(passages.len())- .saturating_add(footer.len())> max_utf8_bytes{return Err(invalid());}- text.push_str(&prefix);- text.push_str(&passages);+ used_bytes += prefix.len() + passages.len();+ sections.push(format!("{prefix}{passages}"));sources.push(item.source.clone());included_passages.push(item.reference.clone());}for item in searched {- if included_passages+ if let Some(index) = included_passages.iter()- .any(|reference| reference.locator == item.locator)+ .position(|reference| reference.locator == item.locator){+ let reference = &included_passages[index];+ if !item.score.is_finite()+ || !item.locator.matches_source(&item.source)+ || reference.verified_spans(&item.text).is_none()+ {+ continue;+ }+ // Fresh search may add text beyond the saved spans, but every old+ // span must still fit. Keep one source entry for the entire chunk.+ let prefix = format!("\n\n# {}\n", grounded_label(&sources[index])?);+ let available = max_utf8_bytes - used_bytes + sections[index].len() - prefix.len();+ let cleaned = super::clean_passage_text(&item.text);+ let passage = truncate_utf8(&cleaned, available).trim_end();+ if reference+ .spans+ .iter()+ .any(|span| span.end_utf8 as usize > passage.len())+ {+ continue;+ }+ let expanded = super::IncludedPassage::prefix(item.locator.clone(), passage)?;+ used_bytes -= sections[index].len();+ sections[index] = format!("{prefix}{passage}");+ used_bytes += sections[index].len();+ included_passages[index] = expanded;continue;}let is_pack = matches!(&item.source, GroundedSource::EnsuPack { .. });@@ -321,8 +346,7 @@ pub(crate) fn build_followup_prompt_context(continue;}let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);- let Some(available) = max_utf8_bytes.checked_sub(text.len() + prefix.len() + footer.len())- else {+ let Some(available) = max_utf8_bytes.checked_sub(used_bytes + prefix.len()) else {continue;};if !item.score.is_finite() || !item.locator.matches_source(&item.source) {@@ -333,8 +357,8 @@ pub(crate) fn build_followup_prompt_context(if passage.is_empty() {continue;}- text.push_str(&prefix);- text.push_str(passage);+ used_bytes += prefix.len() + passage.len();+ sections.push(format!("{prefix}{passage}"));sources.push(item.source.clone());included_passages.push(super::IncludedPassage::prefix(item.locator.clone(),@@ -346,6 +370,9 @@ pub(crate) fn build_followup_prompt_context(notes += 1;}}+ for section in sections {+ text.push_str(§ion);+ }text.push_str(&footer);Ok(Some(GroundedPromptContext {text,@@ -399,11 +426,13 @@ mod tests {}#[test]- fn mismatched_source_and_locator_are_rejected() {- let candidates = select_mixed_grounding_candidates(&[], &[note("one.md", 0.9)], 5).unwrap();- let mut wrong = candidates;+ fn mismatched_passage_sources_and_text_are_rejected() {+ let mut wrong = select_mixed_grounding_candidates(&[], &[note("one.md", 0.9)], 5).unwrap();wrong[0].locator = note("other.md", 0.9).locator;assert!(build_grounded_prompt_context(&wrong, 6000).is_err());+ let mut old = previous();+ old.passages[0] = "Cafe 🙂".into();+ assert!(build_followup_prompt_context(&[old], &[], 6000).is_err());}#[test]@@ -474,12 +503,7 @@ mod tests {}fn previous() -> ReferencedPassage {- let fixtures: serde_json::Value = serde_json::from_str(include_str!(- "../../tests/fixtures/passage-references-v1.json"- ))- .unwrap();- let reference: super::super::IncludedPassage =- serde_json::from_value(fixtures[0]["reference"].clone()).unwrap();+ let (reference, raw) = super::super::passage_fixture();let source = GroundedSource::LocalNote {reference: NoteSourceReference {collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),@@ -491,9 +515,7 @@ mod tests {},};ReferencedPassage {- passages: reference- .verified_spans(fixtures[0]["raw"].as_str().unwrap())- .unwrap(),+ passages: reference.verified_spans(raw).unwrap(),reference,source,}@@ -541,12 +563,54 @@ mod tests {assert!(packed.text.len() <= budget);assert_eq!(packed.included_passages[0], old.reference);}++ let raw = super::super::passage_fixture().1;+ let matching = GroundedExcerpt {+ locator: old.reference.locator.clone(),+ source: old.source.clone(),+ score: 1.0,+ text: raw.into(),+ };+ let expanded = build_followup_prompt_context(+ std::slice::from_ref(&old),+ std::slice::from_ref(&matching),+ 6000,+ )+ .unwrap()+ .unwrap();+ assert_eq!(expanded.sources, vec![old.source.clone()]);+ assert_eq!(expanded.included_passages.len(), 1);+ assert!(+ expanded+ .text+ .contains(&super::super::clean_passage_text(raw))+ );+ assert_eq!(+ expanded.included_passages[0].verified_spans(raw).unwrap(),+ vec![super::super::clean_passage_text(raw)]+ );++ let mut prefix = old.clone();+ prefix.reference.spans.truncate(1);+ prefix.passages.truncate(1);+ let required_prefix =+ build_followup_prompt_context(std::slice::from_ref(&prefix), &[], 6000)+ .unwrap()+ .unwrap();+ for budget in required_prefix.text.len()..=expanded.text.len() {+ let packed = build_followup_prompt_context(+ std::slice::from_ref(&prefix),+ std::slice::from_ref(&matching),+ budget,+ )+ .unwrap()+ .unwrap();+ assert!(packed.text.len() <= budget);+ assert_eq!(packed.included_passages.len(), 1);+ let spans = packed.included_passages[0].verified_spans(raw).unwrap();+ assert!(spans[0].starts_with(&prefix.passages[0]));+ assert!(packed.text.contains(&spans[0]));+ }assert!(build_followup_prompt_context(&[old], &[], required.text.len() - 1).is_err());}- #[test]- fn referenced_text_must_match_the_saved_digest() {- let mut old = previous();- old.passages[0] = "Changed text".into();- assert!(build_followup_prompt_context(&[old], &[], 6000).is_err());- }}diff --git a/rust/crates/ensu/src/retrieval/index.rs b/rust/crates/ensu/src/retrieval/index.rsindex e29b096fdf..8cecc68fc9 100644--- a/rust/crates/ensu/src/retrieval/index.rs+++ b/rust/crates/ensu/src/retrieval/index.rs@@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BinaryHeap};use std::fs::{self, File};use std::io::Read;use std::path::Path;+use std::sync::OnceLock;use memmap2::{Mmap, MmapOptions};use serde::Deserialize;@@ -56,7 +57,8 @@ struct MetadataRow {pub struct RetrievalIndex {dataset_identity: String,stable_id: String,- revision_sha256: String,+ revision_sha256: OnceLock<String>,+ manifest_bytes: Vec<u8>,count: usize,dim: usize,scale: f32,@@ -165,6 +167,7 @@ impl RetrievalIndex {.collect::<Vec<_>>();validate_offsets(&offsets, metadata_len)?;+ // Native callers prevent file swaps or deletion while mapped.#[expect(unsafe_code,reason = "File-backed memory mapping requires an unsafe call"@@ -178,7 +181,8 @@ impl RetrievalIndex {Ok(Self {stable_id: expected_pack.stable_id.clone(),- revision_sha256: text_revision_digest(&manifest_bytes, &metadata, &offsets_raw),+ revision_sha256: OnceLock::new(),+ manifest_bytes,dataset_identity: manifest.dataset,count,dim,@@ -196,6 +200,17 @@ impl RetrievalIndex {&self.dataset_identity}+ fn revision_sha256(&self) -> &str {+ self.revision_sha256.get_or_init(|| {+ let offsets = self+ .offsets+ .iter()+ .flat_map(|offset| offset.to_le_bytes())+ .collect::<Vec<_>>();+ text_revision_digest(&self.manifest_bytes, &self.metadata, &offsets)+ })+ }+pub fn search(&self,query: &[f32],@@ -271,13 +286,13 @@ impl RetrievalIndex {else {return Ok(None);};- if dataset_id != &self.stable_id || revision_sha256 != &self.revision_sha256 {- return Ok(None);- }let Ok(row) = usize::try_from(*row) else {return Ok(None);};- if row >= self.count {+ if dataset_id != &self.stable_id+ || row >= self.count+ || revision_sha256 != self.revision_sha256()+ {return Ok(None);}Ok(self.load_hits(&[RankedRow { score: 0.0, row }])?.pop())@@ -337,7 +352,7 @@ impl RetrievalIndex {hits[rank] = Some(RetrievalHit {locator: super::PassageLocator::EnsuPack {dataset_id: self.stable_id.clone(),- revision_sha256: self.revision_sha256.clone(),+ revision_sha256: self.revision_sha256().to_owned(),row: selected.row as u64,},score: selected.score,@@ -542,18 +557,18 @@ fn python_quote(value: &str) -> String {}fn text_revision_digest(manifest: &[u8], metadata: &[u8], offsets: &[u8]) -> String {- use sha2::{Digest, Sha256};- let mut revision = Sha256::new();- revision.update(b"ensu-pack-text-revision-v1\0");- for bytes in [manifest, metadata, offsets] {- revision.update((bytes.len() as u64).to_le_bytes());- revision.update(bytes);- }- revision- .finalize()- .iter()- .map(|byte| format!("{byte:02x}"))- .collect()+ ente_ensu_crypto::sha256_parts(&[+ b"ensu-pack-text-revision-v1\0",+ &(manifest.len() as u64).to_le_bytes(),+ manifest,+ &(metadata.len() as u64).to_le_bytes(),+ metadata,+ &(offsets.len() as u64).to_le_bytes(),+ offsets,+ ])+ .iter()+ .map(|byte| format!("{byte:02x}"))+ .collect()}fn sanitize_prompt_text(value: &str) -> String {@@ -665,22 +680,25 @@ pub(super) mod tests {}#[test]- fn passage_reload_survives_reopen_and_rejects_changed_text_artifacts() {+ fn passage_revisions_are_lazy_and_reloads_reject_changed_artifacts() {use crate::retrieval::PassageLocator;let pack = synthetic_pack("simplewiki-test");let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();+ assert!(index.revision_sha256.get().is_none());let mut query = vec![0.0; 512];+ assert!(index.search(&query, 3, 1.0).unwrap().is_empty());+ assert!(index.revision_sha256.get().is_none());query[0] = 1.0;let hits = index.search(&query, 3, 0.0).unwrap();assert_ne!(hits[0].locator, hits[1].locator);+ let revision = text_revision_digest(+ &fs::read(pack.revision.join(KNOWLEDGE_MANIFEST_FILE)).unwrap(),+ &fs::read(pack.revision.join(KNOWLEDGE_META_FILE)).unwrap(),+ &fs::read(pack.revision.join(KNOWLEDGE_OFFSETS_FILE)).unwrap(),+ );+ assert_eq!(index.revision_sha256.get(), Some(&revision));drop(index);let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();- for hit in &hits {- let reloaded = index.reload_passage(&hit.locator).unwrap().unwrap();- assert_eq!(reloaded.locator, hit.locator);- assert_eq!(reloaded.text, hit.text);- assert_eq!(reloaded.source_url, hit.source_url);- }let mut wrong = hits[0].locator.clone();if let PassageLocator::EnsuPack { row, .. } = &mut wrong {*row = u64::MAX;@@ -691,6 +709,14 @@ pub(super) mod tests {*dataset_id = "wikibooks".to_string();}assert!(index.reload_passage(&wrong).unwrap().is_none());+ assert!(index.revision_sha256.get().is_none());+ for hit in &hits {+ let reloaded = index.reload_passage(&hit.locator).unwrap().unwrap();+ assert_eq!(reloaded.locator, hit.locator);+ assert_eq!(reloaded.text, hit.text);+ assert_eq!(reloaded.source_url, hit.source_url);+ }+ assert_eq!(index.revision_sha256.get(), Some(&revision));drop(index);let metadata_path = pack.revision.join(KNOWLEDGE_META_FILE);diff --git a/rust/crates/ensu/src/retrieval/passage.rs b/rust/crates/ensu/src/retrieval/passage.rsindex 4d66d3b9a8..f957cf2e97 100644--- a/rust/crates/ensu/src/retrieval/passage.rs+++ b/rust/crates/ensu/src/retrieval/passage.rs@@ -1,5 +1,4 @@use serde::{Deserialize, Serialize};-use sha2::{Digest, Sha256};use super::{BEGIN_CONTEXT_SENTINEL, END_CONTEXT_SENTINEL, GroundedSource, RetrievalError};@@ -129,35 +128,22 @@ impl IncludedPassage {}pub fn verified_spans(&self, reloaded_text: &str) -> Option<Vec<String>> {- if self.cleaning_version != PASSAGE_CLEANING_VERSION- || self.spans.is_empty()- || self.spans.len() > MAX_PASSAGE_SPANS- || reloaded_text.len() > MAX_PASSAGE_UTF8_BYTES- {+ if !self.valid_metadata() || reloaded_text.len() > MAX_PASSAGE_UTF8_BYTES {return None;}let cleaned = clean_passage_text(reloaded_text);- let mut end = 0;- let mut texts = Vec::with_capacity(self.spans.len());- for span in &self.spans {- let start = usize::try_from(span.start_utf8).ok()?;- let next_end = usize::try_from(span.end_utf8).ok()?;- if start < end || start >= next_end {- return None;- }- let text = cleaned.get(start..next_end)?;- if digest(text.as_bytes()) != span.text_sha256 {- return None;- }- texts.push(text.to_owned());- end = next_end;- }- Some(texts)+ self.spans+ .iter()+ .map(|span| {+ let text = cleaned.get(span.start_utf8 as usize..span.end_utf8 as usize)?;+ (digest(text.as_bytes()) == span.text_sha256).then(|| text.to_owned())+ })+ .collect()}}pub(super) fn digest(bytes: &[u8]) -> String {- Sha256::digest(bytes)+ ente_ensu_crypto::sha256(bytes).iter().map(|byte| format!("{byte:02x}")).collect()@@ -194,66 +180,95 @@ pub(crate) fn clean_passage_text(value: &str) -> String {}#[cfg(test)]-mod tests {+pub(super) mod tests {use super::*;- #[derive(Deserialize)]- struct Fixture {- name: String,- raw: String,- cleaned: String,- reference: IncludedPassage,- texts: Vec<String>,- }-- #[test]- fn shared_passage_reference_fixtures() {- let fixtures: Vec<Fixture> = serde_json::from_str(include_str!(- "../../tests/fixtures/passage-references-v1.json"- ))+ pub(crate) fn passage_fixture() -> (IncludedPassage, &'static str) {+ let reference = serde_json::from_value(serde_json::json!({+ "locator": {+ "type": "localNote",+ "collectionId": "123e4567-e89b-12d3-a456-426614174000",+ "documentId": "trip.md",+ "indexedRevision": "a".repeat(64),+ "shardSha256": "b".repeat(64),+ "chunkIndex": 2+ },+ "cleaningVersion": 1,+ "spans": [+ {+ "startUtf8": 0,+ "endUtf8": 9,+ "textSha256": "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"+ },+ {+ "startUtf8": 20,+ "endUtf8": 23,+ "textSha256": "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"+ }+ ]+ })).unwrap();- for fixture in fixtures {- assert_eq!(- clean_passage_text(&fixture.raw),- fixture.cleaned,- "{}",- fixture.name- );- assert_eq!(- fixture.reference.verified_spans(&fixture.raw),- Some(fixture.texts),- "{}",- fixture.name- );- }+ (reference, " \r\nCafé🙂\r\nबीच\tend \0\r\n")}#[test]- fn refuses_changed_text_invalid_offsets_and_unknown_versions() {- let fixtures: Vec<Fixture> = serde_json::from_str(include_str!(- "../../tests/fixtures/passage-references-v1.json"- ))- .unwrap();- let fixture = &fixtures[0];+ fn passage_references_verify_cleaned_text_and_reject_invalid_spans() {+ let (fixture, raw) = passage_fixture();+ for (raw, cleaned, spans, texts) in [+ (+ raw,+ "Café🙂\nबीच\tend",+ fixture.spans.clone(),+ vec!["Café🙂", "end"],+ ),+ (+ "\r\n----- BEGIN KNOWLEDGE CONTEXT -----\r\nx\r\n",+ "[source] ----- BEGIN KNOWLEDGE CONTEXT -----\nx",+ vec![PassageSpan {+ start_utf8: 0,+ end_utf8: 43,+ text_sha256: "73c093471df3c5f2f0e96999d8d5ebc745a803f55d3f36c4523c502c2e055e02"+ .into(),+ }],+ vec!["[source] ----- BEGIN KNOWLEDGE CONTEXT ----"],+ ),+ (+ " a\u{1}b\tc\rd ",+ "a b\tc\nd",+ vec![PassageSpan {+ start_utf8: 0,+ end_utf8: 7,+ text_sha256: "1205bffe81cb4550cdf063c07e75e290c2ebaf60d77780a338e18eb99d26bd4c"+ .into(),+ }],+ vec!["a b\tc\nd"],+ ),+ ] {+ let reference = IncludedPassage {+ spans,+ ..fixture.clone()+ };+ assert_eq!(clean_passage_text(raw), cleaned);+ assert_eq!(reference.verified_spans(raw).unwrap(), texts);+ }assert!(fixture- .reference- .verified_spans(&fixture.raw.replace("Café", "Cafe"))+ .verified_spans(&raw.replace("Café", "Cafe ")).is_none());for (start, end) in [(0, 4), (4, 9), (9, 0), (0, u32::MAX)] {- let mut reference = fixture.reference.clone();+ let mut reference = fixture.clone();reference.spans[0].start_utf8 = start;reference.spans[0].end_utf8 = end;- assert!(reference.verified_spans(&fixture.raw).is_none());+ assert!(reference.verified_spans(raw).is_none());}- let mut reference = fixture.reference.clone();+ let mut reference = fixture.clone();reference.cleaning_version = 2;- assert!(reference.verified_spans(&fixture.raw).is_none());- reference = fixture.reference.clone();+ assert!(reference.verified_spans(raw).is_none());+ reference = fixture;reference.spans[1] = reference.spans[0].clone();- assert!(reference.verified_spans(&fixture.raw).is_none());+ assert!(reference.verified_spans(raw).is_none());reference.spans = vec![reference.spans[0].clone(); 9];- assert!(reference.verified_spans(&fixture.raw).is_none());+ assert!(reference.verified_spans(raw).is_none());}}diff --git a/rust/crates/ensu/tests/fixtures/passage-references-v1.json b/rust/crates/ensu/tests/fixtures/passage-references-v1.jsondeleted file mode 100644index ad039dbc94..0000000000--- a/rust/crates/ensu/tests/fixtures/passage-references-v1.json+++ /dev/null@@ -1,86 +0,0 @@-[- {- "name": "multilingual-disjoint",- "raw": " \r\nCafé🙂\r\nबीच\tend \u0000\r\n",- "cleaned": "Café🙂\nबीच\tend",- "reference": {- "locator": {- "type": "localNote",- "collectionId": "123e4567-e89b-12d3-a456-426614174000",- "documentId": "trip.md",- "indexedRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",- "shardSha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",- "chunkIndex": 2- },- "cleaningVersion": 1,- "spans": [- {- "startUtf8": 0,- "endUtf8": 9,- "textSha256": "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"- },- {- "startUtf8": 20,- "endUtf8": 23,- "textSha256": "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"- }- ]- },- "texts": [- "Café🙂",- "end"- ]- },- {- "name": "sentinel",- "raw": "\r\n----- BEGIN KNOWLEDGE CONTEXT -----\r\nx\r\n",- "cleaned": "[source] ----- BEGIN KNOWLEDGE CONTEXT -----\nx",- "reference": {- "locator": {- "type": "localNote",- "collectionId": "123e4567-e89b-12d3-a456-426614174000",- "documentId": "trip.md",- "indexedRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",- "shardSha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",- "chunkIndex": 2- },- "cleaningVersion": 1,- "spans": [- {- "startUtf8": 0,- "endUtf8": 43,- "textSha256": "73c093471df3c5f2f0e96999d8d5ebc745a803f55d3f36c4523c502c2e055e02"- }- ]- },- "texts": [- "[source] ----- BEGIN KNOWLEDGE CONTEXT ----"- ]- },- {- "name": "controls",- "raw": " a\u0001b\tc\rd ",- "cleaned": "a b\tc\nd",- "reference": {- "locator": {- "type": "localNote",- "collectionId": "123e4567-e89b-12d3-a456-426614174000",- "documentId": "trip.md",- "indexedRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",- "shardSha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",- "chunkIndex": 2- },- "cleaningVersion": 1,- "spans": [- {- "startUtf8": 0,- "endUtf8": 7,- "textSha256": "1205bffe81cb4550cdf063c07e75e290c2ebaf60d77780a338e18eb99d26bd4c"- }- ]- },- "texts": [- "a b\tc\nd"- ]- }-]diff --git a/web/apps/ensu/src/components/chat/ChatMessageList.tsx b/web/apps/ensu/src/components/chat/ChatMessageList.tsxindex 508c6580c0..3009a31270 100644--- a/web/apps/ensu/src/components/chat/ChatMessageList.tsx+++ b/web/apps/ensu/src/components/chat/ChatMessageList.tsx@@ -18,7 +18,6 @@ import {Copy01Icon,Edit01Icon,RepeatIcon,- Search01Icon,} from "@hugeicons/core-free-icons";import { HugeiconsIcon } from "@hugeicons/react";import {@@ -103,8 +102,7 @@ export interface ChatMessageListProps {) => void;onEditMessage: (message: ChatMessage) => void;onCopyMessage: (text: string) => void;- onRetryMessage: (message: ChatMessage, searchAsWritten?: boolean) => void;- canSearchSources: boolean;+ onRetryMessage: (message: ChatMessage) => void;onPrevBranch: (switcher: BranchSwitcher) => void;onNextBranch: (switcher: BranchSwitcher) => void;onRequestPreview: (attachment: ChatAttachment, sessionUuid: string) => void;@@ -419,8 +417,7 @@ interface MessageRowProps {) => void;onEditMessage: (message: ChatMessage) => void;onCopyMessage: (text: string) => void;- onRetryMessage: (message: ChatMessage, searchAsWritten?: boolean) => void;- canSearchSources: boolean;+ onRetryMessage: (message: ChatMessage) => void;onPrevBranch: (switcher: BranchSwitcher) => void;onNextBranch: (switcher: BranchSwitcher) => void;attachmentPreviews: Record<string, string>;@@ -454,7 +451,6 @@ const MessageRow = memo(onEditMessage,onCopyMessage,onRetryMessage,- canSearchSources,onPrevBranch,onNextBranch,attachmentPreviews,@@ -780,24 +776,6 @@ const MessageRow = memo(</IconButton></>)}- {!isStreaming &&- !isSelf &&- (!isSynthetic || isLastMessage) &&- canSearchSources && (- <IconButton- aria-label="Search as written"- title="Search as written"- sx={actionButtonSx}- onClick={() =>- onRetryMessage(message, true)- }- >- <HugeiconsIcon- icon={Search01Icon}- {...actionIconProps}- />- </IconButton>- )}</Stack>{isStreaming || isSynthetic ? null : (@@ -959,7 +937,6 @@ export const ChatMessageList = memo(onEditMessage,onCopyMessage,onRetryMessage,- canSearchSources,onPrevBranch,onNextBranch,onRequestPreview,@@ -1026,7 +1003,6 @@ export const ChatMessageList = memo(onEditMessage={onEditMessage}onCopyMessage={onCopyMessage}onRetryMessage={onRetryMessage}- canSearchSources={canSearchSources}onPrevBranch={onPrevBranch}onNextBranch={onNextBranch}attachmentPreviews={attachmentPreviews}@@ -1053,7 +1029,6 @@ export const ChatMessageList = memo(assistantMarkdownSx,attachmentPreviews,branchSwitchers,- canSearchSources,dialogCloseButtonSx,dialogCloseIconProps,dialogTitleSx,diff --git a/web/apps/ensu/src/pages/chat.tsx b/web/apps/ensu/src/pages/chat.tsxindex add8c1c578..cdbf17cfa4 100644--- a/web/apps/ensu/src/pages/chat.tsx+++ b/web/apps/ensu/src/pages/chat.tsx@@ -54,7 +54,6 @@ import {prepareDesktopConversation,resolveDesktopSourceFollowup,type PreparedReply,- type ResolvedSourceFollowup,} from "@/services/llm/conversation";import { stripHiddenPartsText } from "@/services/llm/history-text";import {@@ -2850,12 +2849,6 @@ const Page: React.FC = () => {activeKnowledgeSources,));- if (assistantMessage.evidenceSaved === false) {- log.warn(- "Reply saved, but its source references could not be retained",- );- }-await updateBranchSelectionState(parentMessageUuid,assistantMessage.messageUuid,@@ -2940,7 +2933,6 @@ const Page: React.FC = () => {sessionUuid,imagePaths,mediaMarker,- searchAsWritten = false,}: {promptText: string;parentMessage: ChatMessage;@@ -2950,7 +2942,6 @@ const Page: React.FC = () => {sessionUuid?: string;imagePaths?: string[];mediaMarker?: string;- searchAsWritten?: boolean;}): Promise<void> => {const parentMessageUuid = parentMessage.messageUuid;const activeSessionId =@@ -3106,12 +3097,7 @@ const Page: React.FC = () => {historyPath,stopAtMessageUuid,);- const selectedPath = selectedMessages.map(- (message) => message.messageUuid,- );- if (selectedPath.at(-1) !== parentMessageUuid)- selectedPath.push(parentMessageUuid);- let followup: ResolvedSourceFollowup | undefined;+ let resolutionToken: string | undefined;const remainingKnowledgeBytes = useConversationMemory? 6000: Math.max(@@ -3160,12 +3146,16 @@ const Page: React.FC = () => {if (useConversationMemory &&- !searchAsWritten &&selectedMessages.some((message) => message.sources?.length)) {+ const selectedPath = selectedMessages.map(+ (message) => message.messageUuid,+ );+ if (selectedPath.at(-1) !== parentMessageUuid)+ selectedPath.push(parentMessageUuid);setConversationStatus("Finding sources");try {- followup = await provider.withKnowledgeRetrieval(+ resolutionToken = await provider.withKnowledgeRetrieval((cancellationEpoch) =>resolveDesktopSourceFollowup({sessionUuid: activeSessionId,@@ -3178,11 +3168,6 @@ const Page: React.FC = () => {isActiveGeneration,);if (!isActiveGeneration()) return;- knowledgeContext = {- text: "",- sources: [],- candidates: followup.candidates,- };} catch (error) {const { name } = tauriCommandError(error);if (@@ -3265,10 +3250,14 @@ const Page: React.FC = () => {),system: normalSystemPrompt,current: promptText,- historyQuery: knowledgeQuery,+ historyQuery: resolutionToken+ ? undefined+ : knowledgeQuery,maxTokens,- groundingCandidates: knowledgeContext?.candidates,- resolutionToken: followup?.token,+ groundingCandidates: resolutionToken+ ? undefined+ : knowledgeContext?.candidates,+ resolutionToken,},provider,{@@ -3417,12 +3406,6 @@ const Page: React.FC = () => {));if (!isActiveGeneration()) return;- if (assistantMessage.evidenceSaved === false) {- log.warn(- "Reply saved, but its source references could not be retained",- );- }-void updateBranchSelectionState(parentMessageUuid,assistantMessage.messageUuid,@@ -3506,7 +3489,7 @@ const Page: React.FC = () => {);const handleRetryMessage = useCallback(- async (message: ChatMessage, searchAsWritten = false) => {+ async (message: ChatMessage) => {if (message.sender !== "assistant") return;if (!chatKey || !currentSessionId) return;if (isDownloading) {@@ -3549,7 +3532,6 @@ const Page: React.FC = () => {stopAtMessageUuid: parentUuid,resetContext: true,sessionUuid: currentSessionId,- searchAsWritten,});},[@@ -4724,19 +4706,6 @@ const Page: React.FC = () => {onEditMessage={handleEditMessage}onCopyMessage={handleCopyMessage}onRetryMessage={handleRetryMessage}- canSearchSources={- isTauriRuntime &&- (notesCollections.length > 0 ||- knowledgePacks.some(- (pack) =>- enabledKnowledgePackIds.has(- pack.stableId,- ) &&- (pack.status === "ready" ||- pack.status ===- "updateAvailable"),- ))- }onPrevBranch={handlePrevBranch}onNextBranch={handleNextBranch}onRequestPreview={loadAttachmentPreview}diff --git a/web/apps/ensu/src/services/chat/store.ts b/web/apps/ensu/src/services/chat/store.tsindex a5ee07c269..4b7ae133a2 100644--- a/web/apps/ensu/src/services/chat/store.ts+++ b/web/apps/ensu/src/services/chat/store.ts@@ -116,7 +116,6 @@ export interface ChatMessage {createdAt: number;attachments?: ChatAttachment[];sources?: GroundedSource[];- evidenceSaved?: boolean;isSynthetic?: boolean;}@@ -144,7 +143,6 @@ interface NativeMessage {createdAt: number;attachments?: NativeAttachment[];sources?: GroundedSource[];- evidenceSaved?: boolean;}const nowMicros = () => Date.now() * 1000;@@ -430,7 +428,6 @@ const listMessagesNative = async (size: attachment.size,})),sources: message.sources,- evidenceSaved: message.evidenceSaved,}));};@@ -503,7 +500,6 @@ const addMessageNative = async (size: attachment.size,})),sources: message.sources,- evidenceSaved: message.evidenceSaved,};};diff --git a/web/apps/ensu/src/services/llm/conversation.ts b/web/apps/ensu/src/services/llm/conversation.tsindex 932c5e38b1..567ceb7c5a 100644--- a/web/apps/ensu/src/services/llm/conversation.ts+++ b/web/apps/ensu/src/services/llm/conversation.ts@@ -86,11 +86,6 @@ export const prepareDesktopConversation = async (}};-export interface ResolvedSourceFollowup {- token: string;- candidates: unknown;-}-export const resolveDesktopSourceFollowup = (input: {sessionUuid: string;path: string[];@@ -98,7 +93,7 @@ export const resolveDesktopSourceFollowup = (input: {enabledStableIds: string[];cancellationEpoch: number;candidates?: unknown;-}): Promise<ResolvedSourceFollowup> =>- invoke<ResolvedSourceFollowup>("conversation_resolve_followup", {+}): Promise<string> =>+ invoke<string>("conversation_resolve_followup", {input: { ...input, token: crypto.randomUUID() },});