Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

[ensu] Refine historical passage reuse across clients

setalp94bca31a118c
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
index 128426d052..5646f47e57 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppStore.kt
@@ -193,8 +193,7 @@ class AppStore(
modelSettingsActions.cancelModelDownload()
}
- fun retryAssistantMessage(messageId: String, searchAsWritten: Boolean = false) =
- chatActions.retryAssistantMessage(messageId, searchAsWritten)
+ fun retryAssistantMessage(messageId: String) = chatActions.retryAssistantMessage(messageId)
fun sendMessage() = chatActions.sendMessage()
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
index a0efe20cc9..8a15c355a9 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/HomeNavigation.kt
@@ -15,7 +15,6 @@ import androidx.compose.material3.DrawerState
import androidx.compose.material3.DrawerValue
import androidx.compose.material3.Scaffold
import androidx.compose.runtime.Composable
-import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.ui.Modifier
import androidx.navigation.NavBackStackEntry
@@ -55,7 +54,6 @@ internal fun HomeNavigation(
onAttachmentSelected: (AttachmentType) -> Unit,
onOpenAttachment: (Attachment) -> Unit,
) {
- val notesState by store.notesStore.state.collectAsState()
Scaffold(
containerColor = EnsuColor.backgroundBase(),
contentWindowInsets = WindowInsets(0, 0, 0, 0),
@@ -125,15 +123,6 @@ internal fun HomeNavigation(
onRemoveAttachment = store::removeAttachment,
onEditMessage = { message -> store.beginEditing(message.id) },
onRetryMessage = { message -> store.retryAssistantMessage(message.id) },
- onSearchMessage =
- if (
- appState.knowledge.enabledReadyDatasets.isNotEmpty() ||
- notesState.collections.isNotEmpty()
- ) {
- { message ->
- store.retryAssistantMessage(message.id, searchAsWritten = true)
- }
- } else null,
onCancelEdit = store::cancelEditing,
onBranchChange = store::updateBranchSelection,
onOpenAttachment = onOpenAttachment,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
index 671bc38395..deba524767 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
@@ -107,7 +107,6 @@ internal fun MessageList(
branchSelections: Map<String, Int>,
onEditMessage: (ChatMessage) -> Unit,
onRetryMessage: (ChatMessage) -> Unit,
- onSearchMessage: ((ChatMessage) -> Unit)?,
onBranchChange: (String, Int) -> Unit,
onOpenAttachment: (Attachment) -> Unit,
onStartDownload: (Boolean) -> Unit,
@@ -320,7 +319,6 @@ internal fun MessageList(
isLastMessage = message == messages.last(),
branchSelections = branchSelections,
onRetry = { onRetryMessage(message) },
- onSearch = onSearchMessage?.let { search -> { search(message) } },
onBranchChange = onBranchChange,
showsMetadata = true,
)
@@ -641,7 +639,6 @@ private fun AssistantMessageBubble(
isLastMessage: Boolean,
branchSelections: Map<String, Int>,
onRetry: () -> Unit,
- onSearch: (() -> Unit)?,
onBranchChange: (String, Int) -> Unit,
showsMetadata: Boolean,
) {
@@ -671,14 +668,6 @@ private fun AssistantMessageBubble(
onRetry()
}
)
- onSearch?.let { search ->
- add(
- MessageAction("Search as written", HugeIcons.Search01Icon) {
- haptic.perform(HapticFeedbackType.TextHandleMove)
- search()
- }
- )
- }
}
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.kt
index c76fb2e984..1ea54e5864 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatRepository.kt
@@ -138,7 +138,7 @@ class ChatRepository(
internal fun startFollowup(sessionId: String, path: List<String>, userText: String) =
withDbRecovery {
- db.startConversationFollowup(sessionId, path, userText, userText)
+ db.startConversationFollowup(sessionId, path, userText)
}
internal fun prepareConversation(
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index d6d5f90f72..0ac6a4ff22 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -10,7 +10,6 @@ import io.ente.ensu.bindings.ConversationRequest
import io.ente.ensu.bindings.DbException
import io.ente.ensu.bindings.GroundedExcerpt
import io.ente.ensu.bindings.GroundedSource
-import io.ente.ensu.bindings.IncludedPassage
import io.ente.ensu.bindings.LlmException
import io.ente.ensu.bindings.PassageLocator
import io.ente.ensu.bindings.buildGroundedPromptContext
@@ -318,7 +317,7 @@ internal class ChatStoreActions(
generationJob?.cancel()
}
- fun retryAssistantMessage(messageId: String, searchAsWritten: Boolean = false) {
+ fun retryAssistantMessage(messageId: String) {
val priorGeneration = generationJob
val priorSummary = sessionSummaryJob
if (state.value.chat.isGenerating) {
@@ -347,7 +346,7 @@ internal class ChatStoreActions(
priorGeneration?.join()
priorSummary?.join()
llmProvider.resetContext()
- startGeneration(sessionId, parent, searchAsWritten)
+ startGeneration(sessionId, parent)
}
}
@@ -483,11 +482,7 @@ internal class ChatStoreActions(
}
}
- private fun startGeneration(
- sessionId: String,
- userMessage: ChatMessage,
- searchAsWritten: Boolean = false,
- ) {
+ private fun startGeneration(sessionId: String, userMessage: ChatMessage) {
val scope = scope ?: return
if (!state.value.chat.deviceCapability.isChatSupported()) return
val prompt = buildPrompt(userMessage.text, userMessage.attachments)
@@ -544,23 +539,21 @@ internal class ChatStoreActions(
val enabledDatasets = state.value.knowledge.enabledReadyDatasets
try {
if (prompt.imageFiles.isEmpty()) {
- val history = buildSelectedPath(sessionId).takeWhile { it.id != userMessage.id }
- val hasCurrentSources =
- searchAsWritten ||
- enabledDatasets.isNotEmpty() ||
- notesStore.state.value.collections.isNotEmpty()
+ val messages = messageStore[sessionId].orEmpty()
+ val path = conversationPath(messages, userMessage)
+ val pathIds = path.toSet()
val turn =
withContext(Dispatchers.IO) {
if (
- hasCurrentSources ||
- history.any {
+ messages.any {
+ it.id in pathIds &&
it.author == MessageAuthor.Assistant &&
- parseGroundedAssistantText(it.text).sources.isNotEmpty()
- }
+ parseGroundedAssistantText(it.text).sources.isNotEmpty()
+ }
) {
chatRepository.startFollowup(
sessionId,
- history.map { it.id } + userMessage.id,
+ path,
userMessage.text,
)
} else null
@@ -645,13 +638,12 @@ internal class ChatStoreActions(
try {
followup
?.let { turn ->
- val resolution =
+ val locators =
withContext(Dispatchers.IO) {
- if (searchAsWritten) turn.searchAsWritten()
- else turn.searchWithHistory(knowledgeHits)
+ turn.searchWithHistory(knowledgeHits)
}
if (!isActive() || stopRequested) return@launch
- resolution.referencedPassages.mapNotNull { reloadPassage(it) }
+ locators.mapNotNull { reloadPassage(it) }
}
.orEmpty()
} catch (error: kotlinx.coroutines.CancellationException) {
@@ -954,21 +946,15 @@ internal class ChatStoreActions(
return true
}
- private suspend fun reloadPassage(reference: IncludedPassage): GroundedExcerpt? {
- return try {
- when (val locator = reference.locator) {
+ private suspend fun reloadPassage(locator: PassageLocator): GroundedExcerpt? =
+ try {
+ when (locator) {
is PassageLocator.EnsuPack ->
knowledgeProvider.reload(locator, state.value.knowledge.enabledReadyDatasets)
is PassageLocator.LocalNote -> notesStore.reload(locator)
}
} catch (error: kotlinx.coroutines.CancellationException) {
throw error
- } catch (error: ConversationException.Cancelled) {
- throw error
- } catch (error: ConversationException.Stale) {
- throw error
- } catch (error: DbException) {
- throw error
} catch (error: Exception) {
logRepository.log(
LogLevel.Warning,
@@ -979,7 +965,6 @@ internal class ChatStoreActions(
)
null
}
- }
private suspend fun generateTextConversation(
sessionId: String,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
index fb48753507..b8a9e4b0d4 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatView.kt
@@ -66,7 +66,6 @@ fun ChatView(
onRemoveAttachment: (Attachment) -> Unit,
onEditMessage: (ChatMessage) -> Unit,
onRetryMessage: (ChatMessage) -> Unit,
- onSearchMessage: ((ChatMessage) -> Unit)?,
onCancelEdit: () -> Unit,
onBranchChange: (String, Int) -> Unit,
onOpenAttachment: (Attachment) -> Unit,
@@ -237,7 +236,6 @@ fun ChatView(
branchSelections = chatState.branchSelections,
onEditMessage = onEditMessage,
onRetryMessage = onRetryMessage,
- onSearchMessage = onSearchMessage,
onBranchChange = onBranchChange,
onOpenAttachment = onOpenAttachment,
onStartDownload = onStartDownload,
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt
index ca4970c808..46278812be 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt
@@ -273,10 +273,7 @@ class NotesStore(
} catch (error: CancellationException) {
throw error
} catch (_: NotesException.RebuildRequired) {
- enqueue(record.id, rebuild = true, immediate = true)
- update(record.id) {
- it.copy(status = NotesStatus.Pending, indexAvailable = false)
- }
+ requestRebuild(record.id)
} catch (error: Exception) {
fail(record.id, error)
}
@@ -292,10 +289,7 @@ class NotesStore(
val hit = provider.reload(locator) ?: return@withContext null
verify(selectMixedGroundingCandidates(emptyList(), listOf(hit), 1u)).singleOrNull()
} catch (_: NotesException.RebuildRequired) {
- enqueue(locator.collectionId, rebuild = true, immediate = true)
- update(locator.collectionId) {
- it.copy(status = NotesStatus.Pending, indexAvailable = false)
- }
+ requestRebuild(locator.collectionId)
null
}
}
@@ -365,6 +359,11 @@ class NotesStore(
}
}
+ private fun requestRebuild(id: String) {
+ enqueue(id, rebuild = true, immediate = true)
+ update(id) { it.copy(status = NotesStatus.Pending, indexAvailable = false) }
+ }
+
private fun enqueue(
id: String,
rebuild: Boolean = false,
@@ -544,13 +543,7 @@ class NotesStore(
}
is NotesException.RebuildRequired -> {
if (snapshot == null) {
- enqueue(id, rebuild = true, immediate = true)
- update(id) {
- it.copy(
- status = NotesStatus.Pending,
- indexAvailable = false,
- )
- }
+ requestRebuild(id)
} else fail(id, failure)
}
is Exception -> fail(id, failure)
diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swift
index 09c0fc5ac2..e9488c39af 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift
@@ -302,10 +302,6 @@ struct ChatView: View {
onRetry: { message in
viewModel.retryAssistantResponse(message)
},
- onSearch: viewModel.canSearchSources
- ? { message in
- viewModel.retryAssistantResponse(message, searchAsWritten: true)
- } : nil,
onBranchChange: { message, delta in
viewModel.changeBranch(for: message, delta: delta)
},
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index 1f51be1a64..cfae99ff4d 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -980,11 +980,7 @@ final class ChatViewModel: ObservableObject {
startModelDownload(userInitiated: true)
}
- var canSearchSources: Bool {
- !knowledgeStore.enabledReadyDatasets.isEmpty || !notesStore.collections.isEmpty
- }
-
- func retryAssistantResponse(_ message: RenderedChatMessage, searchAsWritten: Bool = false) {
+ func retryAssistantResponse(_ message: RenderedChatMessage) {
let priorGeneration = generationTask
let priorSummary = sessionSummaryTask
guard message.role == .assistant else { return }
@@ -1021,7 +1017,7 @@ final class ChatViewModel: ObservableObject {
_ = await priorSummary?.result
guard let self else { return }
await provider.resetContext()
- startGeneration(for: userNode, searchAsWritten: searchAsWritten)
+ startGeneration(for: userNode)
}
}
@@ -1064,7 +1060,7 @@ final class ChatViewModel: ObservableObject {
return session.id
}
- private func startGeneration(for userNode: MessageNode, searchAsWritten: Bool = false) {
+ private func startGeneration(for userNode: MessageNode) {
guard !isChatUnsupported else {
showUnsupportedDeviceDialog = true
return
@@ -1112,27 +1108,21 @@ final class ChatViewModel: ObservableObject {
defer { followup?.cancel() }
do {
if prompt.imageFiles.isEmpty {
- let history = Array(
- buildSelectedPath(
- for: userNode.sessionId,
- childrenMap: childrenByParent(sessionId: userNode.sessionId)
- ).prefix { $0.id != userNode.id })
- let path = history.map { $0.id.uuidString } + [userNode.id.uuidString]
- let assistantTexts = history.filter { $0.role == .assistant }.map { $0.text }
- let hasCurrentSources =
- searchAsWritten || !enabledDatasets.isEmpty
- || !notesStore.collections.isEmpty
+ let path = buildConversationPath(for: userNode)
+ let pathIds = Set(path)
+ let assistantTexts = (messageStore[userNode.sessionId] ?? [])
+ .filter { pathIds.contains($0.id.uuidString) && $0.role == .assistant }
+ .map { $0.text }
followup = try await Task.detached {
[chatDb] () throws -> ConversationFollowup? in
guard
- hasCurrentSources
- || assistantTexts.contains(where: {
- !parseGroundedAssistantText(storedText: $0).sources.isEmpty
- })
+ assistantTexts.contains(where: {
+ !parseGroundedAssistantText(storedText: $0).sources.isEmpty
+ })
else { return nil }
return try chatDb.startConversationFollowup(
sessionUuid: userNode.sessionId.uuidString, path: path,
- question: userNode.text, expectedUserText: userNode.text
+ question: userNode.text
)
}.value
try Task.checkCancellation()
@@ -1183,15 +1173,16 @@ final class ChatViewModel: ObservableObject {
do {
if let turn = followup {
let searched = knowledgeHits
- let resolution = try await Task.detached {
- try searchAsWritten
- ? turn.searchAsWritten() : turn.searchWithHistory(searched: searched)
+ let locators = try await Task.detached {
+ try turn.searchWithHistory(searched: searched)
}.value
try Task.checkCancellation()
guard activeGenerationId == generationId else { return }
- for reference in resolution.referencedPassages {
+ for locator in locators {
do {
- if let hit = try await reloadPassage(reference) { reloaded.append(hit) }
+ if let hit = try await reloadPassage(locator) {
+ reloaded.append(hit)
+ }
} catch {
if isCancellation(error) { throw error }
logger.warning("Source reload failed", details: "\(error)")
@@ -1518,13 +1509,13 @@ final class ChatViewModel: ObservableObject {
}
}
- private func reloadPassage(_ reference: IncludedPassage) async throws -> GroundedExcerpt? {
- switch reference.locator {
+ private func reloadPassage(_ locator: PassageLocator) async throws -> GroundedExcerpt? {
+ switch locator {
case .ensuPack:
return try await knowledgeProvider.reload(
- reference.locator, datasets: knowledgeStore.enabledReadyDatasets)
+ locator, datasets: knowledgeStore.enabledReadyDatasets)
case .localNote:
- return try await notesStore.reload(reference.locator)
+ return try await notesStore.reload(locator)
}
}
diff --git a/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift b/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift
index ae32450803..25908d5ab5 100644
--- a/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift
+++ b/apple/apps/ensu/Ensu/Chat/MessageList/MessageListComponents.swift
@@ -246,7 +246,6 @@ struct AssistantMessageBubbleView: View {
let isLastMessage: Bool
let onCopy: () -> Void
let onRetry: () -> Void
- let onSearch: (() -> Void)?
let onBranchChange: (Int) -> Void
let onOpenAttachment: (ChatAttachment) -> Void
let showsMetadata: Bool
@@ -291,12 +290,6 @@ struct AssistantMessageBubbleView: View {
hapticMedium()
onRetry()
}
- if let onSearch {
- Button("Search as written") {
- hapticMedium()
- onSearch()
- }
- }
}
}
diff --git a/apple/apps/ensu/Ensu/Chat/MessageListView.swift b/apple/apps/ensu/Ensu/Chat/MessageListView.swift
index 72b243ae7f..7ea5e43077 100644
--- a/apple/apps/ensu/Ensu/Chat/MessageListView.swift
+++ b/apple/apps/ensu/Ensu/Chat/MessageListView.swift
@@ -14,7 +14,6 @@ struct MessageListView: View {
let onEdit: (RenderedChatMessage) -> Void
let onCopy: (RenderedChatMessage) -> Void
let onRetry: (RenderedChatMessage) -> Void
- let onSearch: ((RenderedChatMessage) -> Void)?
let onBranchChange: (RenderedChatMessage, Int) -> Void
let onDismissKeyboard: () -> Void
@@ -205,7 +204,6 @@ struct MessageListView: View {
isLastMessage: message.id == messages.last?.id,
onCopy: { onCopy(message) },
onRetry: { onRetry(message) },
- onSearch: onSearch.map { search in { search(message) } },
onBranchChange: { delta in onBranchChange(message, delta) },
onOpenAttachment: openAttachment,
showsMetadata: true
diff --git a/apple/apps/ensu/Ensu/Services/NotesStore.swift b/apple/apps/ensu/Ensu/Services/NotesStore.swift
index 2b0c3dcf63..23546ed3e1 100644
--- a/apple/apps/ensu/Ensu/Services/NotesStore.swift
+++ b/apple/apps/ensu/Ensu/Services/NotesStore.swift
@@ -188,11 +188,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {
do {
hits += try await provider.search(collection.id, query: query)
} catch NotesError.RebuildRequired {
- enqueue(collection.id, rebuild: true, due: .distantPast)
- update(collection.id) {
- $0.status = .pending
- $0.indexAvailable = false
- }
+ requestRebuild(collection.id)
} catch is CancellationError {
throw CancellationError()
} catch {
@@ -212,11 +208,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {
selectMixedGroundingCandidates(packHits: [], notesHits: [hit], notesLimit: 1)
).first
} catch NotesError.RebuildRequired {
- enqueue(collectionId, rebuild: true, due: .distantPast)
- update(collectionId) {
- $0.status = .pending
- $0.indexAvailable = false
- }
+ requestRebuild(collectionId)
return nil
}
}
@@ -271,6 +263,14 @@ final class NotesStore: ObservableObject, ModelMaintenance {
}
}
+ private func requestRebuild(_ id: String) {
+ enqueue(id, rebuild: true, due: .distantPast)
+ update(id) {
+ $0.status = .pending
+ $0.indexAvailable = false
+ }
+ }
+
private func enqueue(
_ id: String, forced: Set<String> = [], rebuild: Bool = false, due: Date? = nil
) {
@@ -409,11 +409,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {
}
} catch NotesError.RebuildRequired {
if snapshot == nil {
- enqueue(id, rebuild: true, due: .distantPast)
- update(id) {
- $0.status = .pending
- $0.indexAvailable = false
- }
+ requestRebuild(id)
} else {
fail(id, NotesError.RebuildRequired)
}
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index e98c5ba014..c97d1165bb 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2536,7 +2536,6 @@ dependencies = [
"self_cell",
"serde",
"serde_json",
- "sha2 0.11.0",
"tempfile",
"thiserror 2.0.18",
"tokio",
diff --git a/rust/apps/ensu/src-tauri/src/commands/chat_db.rs b/rust/apps/ensu/src-tauri/src/commands/chat_db.rs
index 4a916fc14d..90a5ec2b22 100644
--- a/rust/apps/ensu/src-tauri/src/commands/chat_db.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/chat_db.rs
@@ -172,8 +172,6 @@ pub struct ChatMessageDto {
created_at: i64,
attachments: Vec<ChatAttachmentDto>,
sources: Vec<GroundedSourceDto>,
- #[serde(skip_serializing_if = "Option::is_none")]
- evidence_saved: Option<bool>,
}
impl From<db::Message> for ChatMessageDto {
@@ -191,7 +189,6 @@ impl From<db::Message> for ChatMessageDto {
}
};
Self {
- evidence_saved: None,
message_uuid: message.uuid.to_string(),
session_uuid: message.session_uuid.to_string(),
parent_message_uuid: message.parent_message_uuid.map(|value| value.to_string()),
@@ -388,26 +385,20 @@ pub async fn chat_db_insert_message(
let snapshot = prepared.as_ref().map(|(snapshot, _)| snapshot);
let message =
db.insert_message_guarded(session_uuid, sender, &text, parent, attachments, snapshot)?;
- let evidence_saved =
- prepared
- .filter(|(_, passages)| !passages.is_empty())
- .map(|(snapshot, passages)| {
- match db.save_answer_evidence(&snapshot, message.uuid, passages) {
- Ok(saved) => saved,
- Err(error) => {
- logging::log(
- "ChatDb",
- format!(
- "Reply saved but source reference persistence failed: {error}"
- ),
- );
- false
- }
- }
- });
- let mut result = ChatMessageDto::from(message);
- result.evidence_saved = evidence_saved;
- Ok(result)
+ if let Some((snapshot, passages)) = prepared.filter(|(_, passages)| !passages.is_empty()) {
+ match db.save_answer_evidence(&snapshot, message.uuid, passages) {
+ Ok(true) => {}
+ Ok(false) => logging::log(
+ "ChatDb",
+ "Reply saved, but its source references could not be retained",
+ ),
+ Err(error) => logging::log(
+ "ChatDb",
+ format!("Reply saved but source reference persistence failed: {error}"),
+ ),
+ }
+ }
+ Ok(ChatMessageDto::from(message))
})
.await
}
diff --git a/rust/apps/ensu/src-tauri/src/commands/conversation.rs b/rust/apps/ensu/src-tauri/src/commands/conversation.rs
index 0917d9fcad..f935772c61 100644
--- a/rust/apps/ensu/src-tauri/src/commands/conversation.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/conversation.rs
@@ -23,7 +23,7 @@ struct Slot {
token: String,
snapshot: ConversationSnapshot,
db: Arc<ChatDb<SqliteBackend>>,
- grounded: Option<ente_ensu::retrieval::GroundedPromptContext>,
+ passages: Vec<IncludedPassage>,
model_epoch: u64,
cancel_epoch: u64,
}
@@ -80,14 +80,7 @@ impl State {
}
slots
.remove(window)
- .map(|slot| {
- (
- slot.snapshot,
- slot.grounded
- .map(|context| context.included_passages)
- .unwrap_or_default(),
- )
- })
+ .map(|slot| (slot.snapshot, slot.passages))
.ok_or_else(|| error("Preparation has expired"))
}
}
@@ -253,9 +246,6 @@ pub async fn conversation_prepare(
));
}
let model_epoch = llm_state.model_state_epoch();
- if let Some(candidates) = &input.grounding_candidates {
- candidates.validate().map_err(|e| error(e.to_string()))?;
- }
state
.slots
.lock()
@@ -268,19 +258,21 @@ pub async fn conversation_prepare(
.conversation_snapshot(input.session_uuid, &input.path)
.map_err(ApiError::from)?;
if let Some(token) = &input.resolution_token {
- app.state::<super::followup::State>().validate(
+ let selection = app.state::<super::followup::State>().take_selection(
work_window.label(),
token,
&db,
&snapshot,
- input.grounding_candidates.as_ref(),
)?;
+ input.history_query = Some(selection.question);
+ input.grounding_candidates = Some(selection.candidates);
}
let candidates = match input.grounding_candidates.take() {
Some(candidates) => candidates,
None => conversation::GroundingCandidates::new(vec![], vec![], conversation::MAX_GROUNDING_BYTES)
.map_err(preparation_error)?,
};
+ candidates.validate().map_err(|e| error(e.to_string()))?;
let turn = conversation::TurnInput {
session: input.session_uuid,
messages: snapshot.messages().to_vec(),
@@ -312,18 +304,21 @@ pub async fn conversation_prepare(
));
let result = prepared.preparation.run(&mut effects).map_err(preparation_error)?;
effects.check_current().map_err(preparation_error)?;
- let grounded = prepared.grounding.context;
+ let mut grounded = prepared.grounding.context;
+ let passages = grounded.as_mut()
+ .map(|context| std::mem::take(&mut context.included_passages))
+ .unwrap_or_default();
Ok((
Response {
preparation_token: token.clone(),
messages: result.messages,
- grounded_context: grounded.clone().map(Into::into),
+ grounded_context: grounded.map(Into::into),
},
Slot {
token,
snapshot,
db,
- grounded,
+ passages,
model_epoch,
cancel_epoch: expected_epoch,
},
diff --git a/rust/apps/ensu/src-tauri/src/commands/followup.rs b/rust/apps/ensu/src-tauri/src/commands/followup.rs
index 444cafc6a7..4e6967c6e7 100644
--- a/rust/apps/ensu/src-tauri/src/commands/followup.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/followup.rs
@@ -3,10 +3,10 @@ use ente_ensu::{
conversation as core,
db::{ChatDb, SqliteBackend, chat::ConversationSnapshot},
};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
use std::collections::HashMap;
use std::sync::{Arc, Mutex, atomic::Ordering};
-use tauri::{AppHandle, Manager, State as TauriState, WebviewWindow};
+use tauri::{AppHandle, Manager, State as TauriState, WebviewWindow, async_runtime};
use uuid::Uuid;
#[derive(Default)]
@@ -15,9 +15,13 @@ pub struct State {
}
struct Guard {
token: String,
- fingerprint: String,
db: Arc<ChatDb<SqliteBackend>>,
- candidates: Option<String>,
+ selection: Option<Selection>,
+}
+pub(crate) struct Selection {
+ fingerprint: String,
+ pub question: String,
+ pub candidates: core::GroundingCandidates,
}
fn error(message: impl Into<String>) -> ApiError {
@@ -25,36 +29,35 @@ fn error(message: impl Into<String>) -> ApiError {
}
impl State {
- pub(crate) fn validate(
+ pub(crate) fn take_selection(
&self,
window: &str,
token: &str,
db: &Arc<ChatDb<SqliteBackend>>,
snapshot: &ConversationSnapshot,
- candidates: Option<&core::GroundingCandidates>,
- ) -> Result<(), ApiError> {
- let guards = self
+ ) -> Result<Selection, ApiError> {
+ let fingerprint = core::fingerprint(snapshot.messages());
+ let mut guards = self
.guards
.lock()
.map_err(|_| error("Follow-up state is unavailable"))?;
- if guards.get(window).is_some_and(|guard| {
- guard.token == token
- && guard.candidates.as_ref().is_some_and(|expected| {
- candidates
- .and_then(|c| serde_json::to_string(c).ok())
+ guards
+ .get_mut(window)
+ .filter(|guard| {
+ guard.token == token
+ && Arc::ptr_eq(&guard.db, db)
+ && guard
+ .selection
.as_ref()
- == Some(expected)
- })
- && Arc::ptr_eq(&guard.db, db)
- && guard.fingerprint == core::fingerprint(snapshot.messages())
- }) {
- Ok(())
- } else {
- Err(ApiError::new(
- "stale",
- "Conversation changed after selecting sources; retry the reply",
- ))
- }
+ .is_some_and(|selection| selection.fingerprint == fingerprint)
+ })
+ .and_then(|guard| guard.selection.take())
+ .ok_or_else(|| {
+ ApiError::new(
+ "stale",
+ "Source selection expired or conversation changed; retry the reply",
+ )
+ })
}
}
@@ -69,13 +72,6 @@ pub struct Request {
cancellation_epoch: u64,
candidates: Option<core::GroundingCandidates>,
}
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct Response {
- token: String,
- candidates: core::GroundingCandidates,
-}
-
#[tauri::command]
pub async fn conversation_resolve_followup(
app: AppHandle,
@@ -83,7 +79,7 @@ pub async fn conversation_resolve_followup(
state: TauriState<'_, State>,
llm_state: TauriState<'_, super::llm::State>,
input: Request,
-) -> Result<Response, ApiError> {
+) -> Result<String, ApiError> {
if Uuid::parse_str(&input.token).is_err()
|| input.path.is_empty()
|| input.path.len() > core::MAX_HISTORY_MESSAGES
@@ -94,23 +90,25 @@ pub async fn conversation_resolve_followup(
));
}
let db = app.state::<ChatDbState>().database()?;
- let snapshot = db
- .conversation_snapshot(input.session_uuid, &input.path)
- .map_err(ApiError::from)?;
- let question = snapshot
- .messages()
- .last()
- .ok_or_else(|| error("Missing current question"))?;
- if question.sender != ente_ensu::db::Sender::SelfUser {
- return Err(error("Follow-up must end at a user question"));
- }
+ let window = window.label().to_owned();
+ let epoch = llm_state.retrieval_epoch();
+ let expected = input.cancellation_epoch;
+ let check = move || {
+ if epoch.load(Ordering::Relaxed) == expected {
+ Ok(())
+ } else {
+ Err(ApiError::new("cancelled", "Follow-up cancelled"))
+ }
+ };
+ check()?;
{
let mut guards = state
.guards
.lock()
.map_err(|_| error("Follow-up state is unavailable"))?;
+ check()?;
if guards
- .get(window.label())
+ .get(&window)
.is_some_and(|guard| guard.token == input.token)
{
return Err(error(
@@ -118,40 +116,53 @@ pub async fn conversation_resolve_followup(
));
}
guards.insert(
- window.label().to_owned(),
+ window.clone(),
Guard {
token: input.token.clone(),
- fingerprint: core::fingerprint(snapshot.messages()),
db: db.clone(),
- candidates: None,
+ selection: None,
},
);
}
- let epoch = llm_state.retrieval_epoch();
- let expected = input.cancellation_epoch;
- let check = || {
- if epoch.load(Ordering::Relaxed) == expected {
- Ok(())
- } else {
- Err(ApiError::new("cancelled", "Follow-up cancelled"))
+ let work_db = db.clone();
+ let check_work = check.clone();
+ let (snapshot, mut selection, references) = async_runtime::spawn_blocking(move || {
+ check_work()?;
+ let snapshot = work_db
+ .conversation_snapshot(input.session_uuid, &input.path)
+ .map_err(ApiError::from)?;
+ let question = snapshot
+ .messages()
+ .last()
+ .ok_or_else(|| error("Missing current question"))?;
+ if question.sender != ente_ensu::db::Sender::SelfUser {
+ return Err(error("Follow-up must end at a user question"));
}
- };
- check()?;
- let evidence: Vec<_> = snapshot.evidence().into_iter().cloned().collect();
- let (searched, max_utf8_bytes) = input
- .candidates
- .map_or((Vec::new(), core::MAX_GROUNDING_BYTES), |fresh| {
- (fresh.searched, fresh.max_utf8_bytes)
- });
- let mut candidates = core::GroundingCandidates::new(Vec::new(), searched, max_utf8_bytes)
- .map_err(|cause| error(cause.to_string()))?;
- let references = core::direct_followup_references(
- snapshot.messages(),
- &evidence,
- &input.question,
- &candidates.searched,
- )
- .unwrap_or_default();
+ check_work()?;
+ let evidence = snapshot.evidence();
+ let (searched, max_utf8_bytes) = input
+ .candidates
+ .map_or((Vec::new(), core::MAX_GROUNDING_BYTES), |fresh| {
+ (fresh.searched, fresh.max_utf8_bytes)
+ });
+ let candidates = core::GroundingCandidates::new(Vec::new(), searched, max_utf8_bytes)
+ .map_err(|cause| error(cause.to_string()))?;
+ let references = core::direct_followup_references(
+ snapshot.messages(),
+ &evidence,
+ &input.question,
+ &candidates.searched,
+ );
+ let selection = Selection {
+ fingerprint: core::fingerprint(snapshot.messages()),
+ question: input.question,
+ candidates,
+ };
+ check_work()?;
+ Ok((snapshot, selection, references))
+ })
+ .await
+ .map_err(|_| error("Source selection task failed"))??;
for reference in references {
check()?;
match super::knowledge::reload_for_followup(
@@ -162,7 +173,7 @@ pub async fn conversation_resolve_followup(
)
.await
{
- Ok(Some(passage)) => candidates.referenced.push(passage),
+ Ok(Some(passage)) => selection.candidates.referenced.push(passage),
Ok(None) => {}
Err(error) if error.name == Some("cancelled") => return Err(error),
Err(error) => crate::logging::log(
@@ -171,42 +182,43 @@ pub async fn conversation_resolve_followup(
),
}
}
- candidates
- .validate()
- .map_err(|cause| error(cause.to_string()))?;
- check()?;
- if !db
- .conversation_snapshot_matches(&snapshot)
- .map_err(ApiError::from)?
- || !Arc::ptr_eq(&db, &app.state::<ChatDbState>().database()?)
- {
- return Err(ApiError::new(
- "stale",
- "Conversation changed while selecting sources; retry the reply",
- ));
- }
- let mut guards = state
- .guards
- .lock()
- .map_err(|_| error("Follow-up state is unavailable"))?;
- let guard = guards
- .get_mut(window.label())
- .filter(|guard| guard.token == input.token)
- .ok_or_else(|| ApiError::new("stale", "Source question was superseded"))?;
- guard.candidates =
- Some(serde_json::to_string(&candidates).map_err(|cause| error(cause.to_string()))?);
- Ok(Response {
- token: input.token,
- candidates,
+ async_runtime::spawn_blocking(move || {
+ check()?;
+ selection
+ .candidates
+ .validate()
+ .map_err(|cause| error(cause.to_string()))?;
+ if !db
+ .conversation_snapshot_matches(&snapshot)
+ .map_err(ApiError::from)?
+ || !Arc::ptr_eq(&db, &app.state::<ChatDbState>().database()?)
+ {
+ return Err(ApiError::new(
+ "stale",
+ "Conversation changed while selecting sources; retry the reply",
+ ));
+ }
+ let state = app.state::<State>();
+ let mut guards = state
+ .guards
+ .lock()
+ .map_err(|_| error("Follow-up state is unavailable"))?;
+ check()?;
+ let guard = guards
+ .get_mut(&window)
+ .filter(|guard| guard.token == input.token)
+ .ok_or_else(|| ApiError::new("stale", "Source question was superseded"))?;
+ guard.selection = Some(selection);
+ Ok(input.token)
})
+ .await
+ .map_err(|_| error("Source selection validation task failed"))?
}
#[cfg(test)]
mod tests {
use super::*;
- fn candidates() -> core::GroundingCandidates {
- core::GroundingCandidates::new(vec![], vec![], 6000).unwrap()
- }
+
fn fixture() -> (Arc<ChatDb<SqliteBackend>>, ConversationSnapshot) {
let db = Arc::new(ChatDb::open_in_memory(vec![4; 32]).unwrap());
let session = db.create_session("Synthetic source question").unwrap();
@@ -225,22 +237,20 @@ mod tests {
(db, snapshot)
}
#[test]
- fn preparation_requires_matching_completed_source_selection() {
+ fn preparation_requires_matching_source_selection_and_consumes_it_once() {
let (db, snapshot) = fixture();
let state = State::default();
- let valid = candidates();
state.guards.lock().unwrap().insert(
"window".into(),
Guard {
token: "token".into(),
- fingerprint: core::fingerprint(snapshot.messages()),
db: db.clone(),
- candidates: None,
+ selection: None,
},
);
assert!(
state
- .validate("window", "token", &db, &snapshot, Some(&valid))
+ .take_selection("window", "token", &db, &snapshot)
.is_err()
);
state
@@ -249,36 +259,28 @@ mod tests {
.unwrap()
.get_mut("window")
.unwrap()
- .candidates = Some(serde_json::to_string(&valid).unwrap());
- assert!(
- state
- .validate("window", "token", &db, &snapshot, Some(&valid))
- .is_ok()
- );
- let mut changed = candidates();
- changed.max_utf8_bytes = 3000;
- let (other_db, _) = fixture();
- for (token, db, candidates) in [
- ("token", &db, Some(&changed)),
- ("token", &db, None),
- ("another-token", &db, Some(&valid)),
- ("token", &other_db, Some(&valid)),
+ .selection = Some(Selection {
+ fingerprint: core::fingerprint(snapshot.messages()),
+ question: "Where does she track it?".into(),
+ candidates: core::GroundingCandidates::new(vec![], vec![], 6000).unwrap(),
+ });
+ let (other_db, other_snapshot) = fixture();
+ for (window, token, db, snapshot) in [
+ ("another-window", "token", &db, &snapshot),
+ ("window", "another-token", &db, &snapshot),
+ ("window", "token", &other_db, &snapshot),
+ ("window", "token", &db, &other_snapshot),
] {
- assert!(
- state
- .validate("window", token, db, &snapshot, candidates)
- .is_err()
- );
+ assert!(state.take_selection(window, token, db, snapshot).is_err());
}
- let question = snapshot.messages()[0].uuid;
- db.update_message_text(question, "Actually, use another source")
- .unwrap();
- let edited = db
- .conversation_snapshot(snapshot.session_uuid(), &[question])
+ let selection = state
+ .take_selection("window", "token", &db, &snapshot)
.unwrap();
+ assert_eq!(selection.question, "Where does she track it?");
+ assert_eq!(selection.candidates.max_utf8_bytes, 6000);
assert!(
state
- .validate("window", "token", &db, &edited, Some(&valid))
+ .take_selection("window", "token", &db, &snapshot)
.is_err()
);
}
diff --git a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
index a1778ee741..cbacb1a35b 100644
--- a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
@@ -662,14 +662,14 @@ pub async fn knowledge_retrieve(
check_cancelled()?;
let candidates = ente_ensu::conversation::GroundingCandidates::new(vec![], excerpts, context_budget as usize)
- .map_err(|error| ApiError::new("followup", error.to_string()))?;
+ .map_err(|error| ApiError::new("conversation", error.to_string()))?;
candidates.pack(context_budget as usize)
.map(|context| context.map(|context| {
let mut result = GroundedPromptContextDto::from(context);
result.candidates = Some(candidates);
result
}))
- .map_err(|error| ApiError::new("followup", error.to_string()))
+ .map_err(|error| ApiError::new("conversation", error.to_string()))
})
.await
.map_err(|_| ApiError::new("llm_thread", "Knowledge retrieval task failed"))?
diff --git a/rust/apps/ensu/src-tauri/src/commands/mod.rs b/rust/apps/ensu/src-tauri/src/commands/mod.rs
index f365f4e84b..af4b64ca95 100644
--- a/rust/apps/ensu/src-tauri/src/commands/mod.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/mod.rs
@@ -4,6 +4,7 @@ mod common;
pub(crate) mod config;
pub(crate) mod conversation;
pub(crate) mod crypto;
+pub(crate) mod followup;
pub(crate) mod knowledge;
pub(crate) mod llm;
pub(crate) mod notes;
@@ -22,5 +23,3 @@ pub fn cleanup_for_exit(app: &AppHandle) {
chat_db::clear_for_exit(app);
logging::log("App", "cleanup_for_exit complete");
}
-
-pub mod followup;
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rs
index cfc74f8b83..57fc7bfb5a 100644
--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs
@@ -169,15 +169,11 @@ impl IndexCache {
}
}
- fn search(
- &mut self,
- collection_id: &str,
- query: &[f32],
- ) -> Option<Result<Vec<ente_ensu::notes::NotesSearchHit>, NotesError>> {
+ fn get(&mut self, collection_id: &str) -> Option<&NotesCollectionIndex> {
self.clock = self.clock.wrapping_add(1);
let entry = self.entries.get_mut(collection_id)?;
entry.last_used = self.clock;
- Some(entry.index.search(query))
+ Some(&entry.index)
}
fn insert(&mut self, collection_id: String, index: NotesCollectionIndex) {
@@ -681,21 +677,7 @@ impl RetrievalHandle {
collection_id: &str,
query: &[f32],
) -> Result<Vec<ente_ensu::notes::NotesSearchHit>, NotesError> {
- if let Some(result) = self
- .cached_indexes
- .lock()
- .unwrap_or_else(PoisonError::into_inner)
- .search(collection_id, query)
- {
- return result;
- }
- let index = NotesCollectionIndex::open(&self.index_root, collection_id.to_string())?;
- let hits = index.search(query)?;
- self.cached_indexes
- .lock()
- .unwrap_or_else(PoisonError::into_inner)
- .insert(collection_id.to_string(), index);
- Ok(hits)
+ self.with_index(collection_id, |index| index.search(query))
}
pub(crate) fn reload_passage(
@@ -705,25 +687,29 @@ impl RetrievalHandle {
let ente_ensu::retrieval::PassageLocator::LocalNote { collection_id, .. } = locator else {
return Ok(None);
};
+ self.with_index(collection_id, |index| index.reload_passage(locator))
+ }
+
+ fn with_index<T>(
+ &self,
+ collection_id: &str,
+ read: impl FnOnce(&NotesCollectionIndex) -> Result<T, NotesError>,
+ ) -> Result<T, NotesError> {
+ if let Some(index) = self
+ .cached_indexes
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .get(collection_id)
{
- let mut cache = self
- .cached_indexes
- .lock()
- .unwrap_or_else(PoisonError::into_inner);
- cache.clock = cache.clock.wrapping_add(1);
- let clock = cache.clock;
- if let Some(entry) = cache.entries.get_mut(collection_id) {
- entry.last_used = clock;
- return entry.index.reload_passage(locator);
- }
+ return read(index);
}
- let index = NotesCollectionIndex::open(&self.index_root, collection_id.clone())?;
- let hit = index.reload_passage(locator)?;
+ let index = NotesCollectionIndex::open(&self.index_root, collection_id.to_owned())?;
+ let result = read(&index)?;
self.cached_indexes
.lock()
.unwrap_or_else(PoisonError::into_inner)
- .insert(collection_id.clone(), index);
- Ok(hit)
+ .insert(collection_id.to_owned(), index);
+ Ok(result)
}
pub(crate) fn verify_source_reference(&self, reference: &NoteSourceReference) -> bool {
diff --git a/rust/bindings/uniffi/ensu/src/conversation.rs b/rust/bindings/uniffi/ensu/src/conversation.rs
index f7f0914464..b121306a78 100644
--- a/rust/bindings/uniffi/ensu/src/conversation.rs
+++ b/rust/bindings/uniffi/ensu/src/conversation.rs
@@ -264,12 +264,24 @@ impl ConversationPreparation {
)
.map_err(error)?;
work.saved = true;
- if let Some(grounded) = &work.grounded {
- let _ = self.db.save_answer_evidence(
+ if let Some(grounded) = work.grounded.take()
+ && !grounded.included_passages.is_empty()
+ {
+ match self.db.save_answer_evidence(
&work.snapshot,
answer.uuid,
- grounded.included_passages.clone(),
- );
+ grounded.included_passages,
+ ) {
+ Ok(true) => {}
+ Ok(false) => ente_uniffi_log::warn!(
+ target: "Conversation",
+ "Reply saved, but its source references could not be retained"
+ ),
+ Err(error) => ente_uniffi_log::warn!(
+ target: "Conversation",
+ "Reply saved but source reference persistence failed: {error}"
+ ),
+ }
}
Ok(to_message(answer))
}
diff --git a/rust/bindings/uniffi/ensu/src/followup.rs b/rust/bindings/uniffi/ensu/src/followup.rs
index ae64860e1a..17edc75f0c 100644
--- a/rust/bindings/uniffi/ensu/src/followup.rs
+++ b/rust/bindings/uniffi/ensu/src/followup.rs
@@ -1,7 +1,7 @@
use crate::{
conversation::{ConversationError, error},
db::EnsuDb,
- retrieval::{GroundedExcerpt, IncludedPassage},
+ retrieval::{GroundedExcerpt, PassageLocator},
};
use ente_ensu::{conversation as core, db, retrieval};
use std::sync::{
@@ -10,11 +10,6 @@ use std::sync::{
};
use uuid::Uuid;
-#[derive(Clone, uniffi::Record)]
-pub struct ConversationFollowupResult {
- pub referenced_passages: Vec<IncludedPassage>,
-}
-
#[derive(uniffi::Object)]
pub struct ConversationFollowup {
db: Arc<db::ChatDb<db::SqliteBackend>>,
@@ -26,7 +21,19 @@ pub struct ConversationFollowup {
struct SourceSelection {
references: Vec<retrieval::IncludedPassage>,
- searched: Option<Vec<retrieval::GroundedExcerpt>>,
+ searched: Vec<retrieval::GroundedExcerpt>,
+}
+
+fn search_excerpts(
+ hits: impl IntoIterator<Item = GroundedExcerpt>,
+) -> Result<Vec<retrieval::GroundedExcerpt>, ConversationError> {
+ core::GroundingCandidates::new(
+ Vec::new(),
+ hits.into_iter().map(Into::into).collect(),
+ core::MAX_GROUNDING_BYTES,
+ )
+ .map(|candidates| candidates.searched)
+ .map_err(error)
}
#[uniffi::export]
@@ -36,7 +43,6 @@ impl EnsuDb {
session_uuid: String,
path: Vec<String>,
question: String,
- expected_user_text: String,
) -> Result<Arc<ConversationFollowup>, ConversationError> {
if path.is_empty()
|| path.len() > core::MAX_HISTORY_MESSAGES
@@ -58,7 +64,7 @@ impl EnsuDb {
if !snapshot
.messages()
.last()
- .is_some_and(|m| m.sender == db::Sender::SelfUser && m.text == expected_user_text)
+ .is_some_and(|m| m.sender == db::Sender::SelfUser && m.text == question)
{
return Err(ConversationError::Stale);
}
@@ -104,69 +110,13 @@ impl ConversationFollowup {
let selected = stored
.as_ref()
.ok_or_else(|| error("Source selection is not complete"))?;
- if let Some(expected) = &selected.searched {
- let actual = core::GroundingCandidates::new(
- Vec::new(),
- searched.iter().cloned().map(Into::into).collect(),
- core::MAX_GROUNDING_BYTES,
- )
- .map_err(error)?
- .searched;
- if actual != *expected {
- return Err(error(
- "Source search changed after selecting earlier passages; retry the reply",
- ));
- }
+ if search_excerpts(searched.iter().cloned())? != selected.searched {
+ return Err(error(
+ "Source search changed after selecting earlier passages; retry the reply",
+ ));
}
Ok((self.question.clone(), selected.references.clone()))
}
-
- fn direct(
- &self,
- searched: Option<Vec<GroundedExcerpt>>,
- ) -> Result<ConversationFollowupResult, ConversationError> {
- self.check()?;
- let mut stored = self.selection.lock().map_err(error)?;
- if stored.is_some() {
- return Err(error("Source selection was already attempted"));
- }
- let searched = searched
- .map(|hits| {
- core::GroundingCandidates::new(
- Vec::new(),
- hits.into_iter().map(Into::into).collect(),
- core::MAX_GROUNDING_BYTES,
- )
- .map(|candidates| candidates.searched)
- .map_err(error)
- })
- .transpose()?;
- let references = if let Some(searched) = &searched {
- let evidence = self
- .snapshot
- .evidence()
- .into_iter()
- .cloned()
- .collect::<Vec<_>>();
- core::direct_followup_references(
- self.snapshot.messages(),
- &evidence,
- &self.question,
- searched,
- )
- .unwrap_or_default()
- } else {
- Vec::new()
- };
- let result = ConversationFollowupResult {
- referenced_passages: references.iter().cloned().map(Into::into).collect(),
- };
- *stored = Some(SourceSelection {
- references,
- searched,
- });
- Ok(result)
- }
}
#[uniffi::export]
@@ -175,145 +125,31 @@ impl ConversationFollowup {
self.cancelled.store(true, Ordering::Release);
}
- pub fn search_as_written(&self) -> Result<ConversationFollowupResult, ConversationError> {
- self.direct(None)
- }
-
pub fn search_with_history(
&self,
searched: Vec<GroundedExcerpt>,
- ) -> Result<ConversationFollowupResult, ConversationError> {
- self.direct(Some(searched))
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn source_selection_is_bound_to_search_history_and_cancellation() {
- let db = EnsuDb {
- inner: Arc::new(db::ChatDb::open_in_memory(vec![7; 32]).unwrap()),
- };
- let session = db.inner.create_session("Direct follow-up").unwrap();
- let user = db
- .inner
- .insert_message(session.uuid, "self", "What does Cedar cover?", None, vec![])
- .unwrap();
- let source = retrieval::GroundedExcerpt {
- locator: retrieval::PassageLocator::LocalNote {
- collection_id: Uuid::from_u128(10).to_string(),
- document_id: "cedar.md".into(),
- indexed_revision: "a".repeat(64),
- shard_sha256: "b".repeat(64),
- chunk_index: 0,
- },
- score: 0.9,
- text: "Cedar requires 30 days.".into(),
- source: retrieval::GroundedSource::LocalNote {
- reference: ente_ensu::notes::NoteSourceReference {
- collection_id: Uuid::from_u128(10).to_string(),
- collection_label: None,
- document_id: "cedar.md".into(),
- indexed_revision: "a".repeat(64),
- title: "Cedar".into(),
- section: None,
- },
- },
- };
- let grounded =
- retrieval::build_grounded_prompt_context(std::slice::from_ref(&source), 6000)
- .unwrap()
- .unwrap();
- let before = db
- .inner
- .conversation_snapshot(session.uuid, &[user.uuid])
- .unwrap();
- let text =
- retrieval::finalize_grounded_assistant_text("Cedar covers returns.", &grounded.sources)
- .unwrap();
- let answer = db
- .inner
- .insert_message(session.uuid, "other", &text, Some(user.uuid), vec![])
- .unwrap();
- assert!(
- db.inner
- .save_answer_evidence(&before, answer.uuid, grounded.included_passages.clone())
- .unwrap()
- );
- let current = db
- .inner
- .insert_message(
- session.uuid,
- "self",
- "And the deadline?",
- Some(answer.uuid),
- vec![],
- )
- .unwrap();
- let snapshot = db
- .inner
- .conversation_snapshot(session.uuid, &[user.uuid, answer.uuid, current.uuid])
- .unwrap();
- let start = || {
- db.start_conversation_followup(
- session.uuid.to_string(),
- vec![
- user.uuid.to_string(),
- answer.uuid.to_string(),
- current.uuid.to_string(),
- ],
- current.text.clone(),
- current.text.clone(),
- )
- .unwrap()
- };
- let turn = start();
- assert!(turn.prepare(&db.inner, &snapshot, &[]).is_err());
- let result = turn.search_with_history(vec![]).unwrap();
- assert_eq!(
- turn.prepare(&db.inner, &snapshot, &[]).unwrap().1,
- grounded.included_passages
- );
- assert_eq!(result.referenced_passages.len(), 1);
- assert!(
- turn.prepare(&db.inner, &snapshot, &[source.into()])
- .is_err()
- );
- assert!(turn.search_with_history(vec![]).is_err());
- let direct = start();
- assert!(
- direct
- .search_as_written()
- .unwrap()
- .referenced_passages
- .is_empty()
- );
- assert!(
- direct
- .prepare(&db.inner, &snapshot, &[])
- .unwrap()
- .1
- .is_empty()
+ ) -> Result<Vec<PassageLocator>, ConversationError> {
+ self.check()?;
+ let mut stored = self.selection.lock().map_err(error)?;
+ if stored.is_some() {
+ return Err(error("Source selection was already attempted"));
+ }
+ let searched = search_excerpts(searched)?;
+ let evidence = self.snapshot.evidence();
+ let references = core::direct_followup_references(
+ self.snapshot.messages(),
+ &evidence,
+ &self.question,
+ &searched,
);
- let other = Arc::new(db::ChatDb::open_in_memory(vec![7; 32]).unwrap());
- assert!(matches!(
- turn.prepare(&other, &snapshot, &[]),
- Err(ConversationError::Stale)
- ));
- let cancelled = start();
- cancelled.cancel();
- assert!(matches!(
- cancelled.search_with_history(vec![]),
- Err(ConversationError::Cancelled)
- ));
- db.inner
- .update_message_text(user.uuid, "Edited original")
- .unwrap();
- assert!(matches!(
- turn.prepare(&db.inner, &snapshot, &[]),
- Err(ConversationError::Stale)
- ));
+ let locators = references
+ .iter()
+ .map(|reference| reference.locator.clone().into())
+ .collect();
+ *stored = Some(SourceSelection {
+ references,
+ searched,
+ });
+ Ok(locators)
}
}
diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rs
index d8eda806c7..e737ac970e 100644
--- a/rust/bindings/uniffi/ensu/src/retrieval.rs
+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs
@@ -101,60 +101,6 @@ impl From<PassageLocator> for core::PassageLocator {
}
}
-#[derive(Debug, Clone, uniffi::Record)]
-pub struct PassageSpan {
- pub start_utf8: u32,
- pub end_utf8: u32,
- pub text_sha256: String,
-}
-
-impl From<core::PassageSpan> for PassageSpan {
- fn from(value: core::PassageSpan) -> Self {
- Self {
- start_utf8: value.start_utf8,
- end_utf8: value.end_utf8,
- text_sha256: value.text_sha256,
- }
- }
-}
-
-impl From<PassageSpan> for core::PassageSpan {
- fn from(value: PassageSpan) -> Self {
- Self {
- start_utf8: value.start_utf8,
- end_utf8: value.end_utf8,
- text_sha256: value.text_sha256,
- }
- }
-}
-
-#[derive(Debug, Clone, uniffi::Record)]
-pub struct IncludedPassage {
- pub locator: PassageLocator,
- pub cleaning_version: u32,
- pub spans: Vec<PassageSpan>,
-}
-
-impl From<core::IncludedPassage> for IncludedPassage {
- fn from(value: core::IncludedPassage) -> Self {
- Self {
- locator: value.locator.into(),
- cleaning_version: value.cleaning_version,
- spans: value.spans.into_iter().map(Into::into).collect(),
- }
- }
-}
-
-impl From<IncludedPassage> for core::IncludedPassage {
- fn from(value: IncludedPassage) -> Self {
- Self {
- locator: value.locator.into(),
- cleaning_version: value.cleaning_version,
- spans: value.spans.into_iter().map(Into::into).collect(),
- }
- }
-}
-
#[derive(Debug, Clone, uniffi::Record)]
pub struct RetrievalHit {
pub locator: PassageLocator,
diff --git a/rust/bindings/uniffi/log/src/lib.rs b/rust/bindings/uniffi/log/src/lib.rs
index 4313ac670a..e4829774d4 100644
--- a/rust/bindings/uniffi/log/src/lib.rs
+++ b/rust/bindings/uniffi/log/src/lib.rs
@@ -1,3 +1,5 @@
+pub use log::warn;
+
#[macro_export]
macro_rules! setup {
() => {
diff --git a/rust/crates/ensu-crypto/src/lib.rs b/rust/crates/ensu-crypto/src/lib.rs
index 87bbb4fbbc..75ae9bd599 100644
--- a/rust/crates/ensu-crypto/src/lib.rs
+++ b/rust/crates/ensu-crypto/src/lib.rs
@@ -34,7 +34,15 @@ pub struct EncryptedChatPayload {
}
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
- Sha256::digest(bytes).into()
+ sha256_parts(&[bytes])
+}
+
+pub fn sha256_parts(parts: &[&[u8]]) -> [u8; 32] {
+ let mut hash = Sha256::new();
+ for bytes in parts {
+ hash.update(bytes);
+ }
+ hash.finalize().into()
}
pub fn generate_chat_key() -> String {
diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.toml
index da2d21135b..087b289d1c 100644
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@ -14,7 +14,6 @@ rusqlite = { workspace = true, default-features = true, optional = true }
self_cell = "1.0.0"
serde.workspace = true
serde_json.workspace = true
-sha2.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "time"] }
transcribe-rs = { git = "https://github.com/cjpais/transcribe-rs.git", rev = "efc66111d519a434e28d51d94740008a7ce29360", default-features = false, features = ["onnx", "vad-silero"], optional = true }
diff --git a/rust/crates/ensu/src/conversation.rs b/rust/crates/ensu/src/conversation.rs
index 570dedda7b..24d4ebbb80 100644
--- a/rust/crates/ensu/src/conversation.rs
+++ b/rust/crates/ensu/src/conversation.rs
@@ -15,8 +15,6 @@ use lookup::{HistoryLookup, lookup_history, requested_history};
pub use prepare::{Effects, Preparation, PreparationResult, PrepareError};
pub use state::AnswerEvidence;
pub(crate) use state::{ConversationEnvelope, valid_answer_passages};
-#[cfg(test)]
-pub(crate) use state::{MAX_ANSWER_PASSAGES, MAX_EVIDENCE_ANSWERS, MAX_EVIDENCE_BYTES};
pub use turn::{PreparedTurn, TurnInput, prepare_turn};
#[cfg(test)]
diff --git a/rust/crates/ensu/src/conversation/followup.rs b/rust/crates/ensu/src/conversation/followup.rs
index 37edf4dcde..b98e304c6f 100644
--- a/rust/crates/ensu/src/conversation/followup.rs
+++ b/rust/crates/ensu/src/conversation/followup.rs
@@ -2,37 +2,70 @@ use super::{AnswerEvidence, valid_answer_passages};
use crate::db::Message;
use crate::retrieval::IncludedPassage;
+fn refers_back(question: &str) -> bool {
+ question.split(|c: char| !c.is_alphanumeric()).any(|word| {
+ matches!(
+ word.to_lowercase().as_str(),
+ "it" | "its"
+ | "this"
+ | "that"
+ | "these"
+ | "those"
+ | "my"
+ | "mine"
+ | "our"
+ | "ours"
+ | "they"
+ | "them"
+ | "their"
+ | "theirs"
+ | "she"
+ | "her"
+ | "hers"
+ | "he"
+ | "him"
+ | "his"
+ | "former"
+ | "latter"
+ | "above"
+ | "previous"
+ | "earlier"
+ )
+ })
+}
+
pub fn direct_followup_references(
history: &[Message],
evidence: &[AnswerEvidence],
question: &str,
searched: &[crate::retrieval::GroundedExcerpt],
-) -> Option<Vec<IncludedPassage>> {
+) -> Vec<IncludedPassage> {
let Some((message, entry)) = history.iter().rev().skip(1).find_map(|message| {
evidence
.iter()
.find(|entry| entry.assistant_message_uuid == message.uuid && entry.matches(history))
.map(|entry| (message, entry))
}) else {
- return Some(Vec::new());
+ return Vec::new();
};
if !valid_answer_passages(&entry.passages) {
- return None;
+ return Vec::new();
}
let prior = crate::retrieval::parse_grounded_assistant_text(&message.text).sources;
let named = super::grounding::named_source_keys(
question,
prior.iter().chain(searched.iter().map(|hit| &hit.source)),
);
- if !named.is_empty()
+ if !refers_back(question)
+ && !named.is_empty()
&& named.iter().all(|key| {
searched
.iter()
.any(|hit| super::grounding::source_key(&hit.source) == *key)
})
{
- Some(Vec::new())
+ Vec::new()
} else {
- Some(entry.passages.clone())
+ entry.passages.clone()
}
}
diff --git a/rust/crates/ensu/src/conversation/grounding.rs b/rust/crates/ensu/src/conversation/grounding.rs
index 3568fdfba8..11695c5c69 100644
--- a/rust/crates/ensu/src/conversation/grounding.rs
+++ b/rust/crates/ensu/src/conversation/grounding.rs
@@ -51,7 +51,8 @@ pub(super) fn named_source_keys<'a>(
retrieval::GroundedSource::LocalNote { reference } => &reference.title,
retrieval::GroundedSource::EnsuPack { citation } => &citation.title,
};
- if !words(title).is_empty() {
+ let mut label_words = words(title);
+ if !label_words.is_empty() {
let title = tokens(title);
for (start, window) in query_tokens.windows(title.len()).enumerate() {
if window == title {
@@ -59,16 +60,13 @@ pub(super) fn named_source_keys<'a>(
}
}
}
- let label = match source {
- retrieval::GroundedSource::LocalNote { reference } => {
- format!("{} {}", reference.title, reference.document_id)
- }
- retrieval::GroundedSource::EnsuPack { citation } => citation.title.clone(),
- };
+ if let retrieval::GroundedSource::LocalNote { reference } = source {
+ label_words.extend(words(&reference.document_id));
+ }
documents
.entry(source_key(source))
.or_default()
- .extend(words(&label));
+ .extend(label_words);
}
let query_words = words(query);
documents
@@ -95,7 +93,7 @@ pub(super) fn named_source_keys<'a>(
}
impl GroundingCandidates {
- pub fn protect_named_sources(&mut self, query: &str) -> Result<(), PrepareError> {
+ pub(super) fn protect_named_sources(&mut self, query: &str) -> Result<(), PrepareError> {
self.validate()?;
if query.len() > super::lookup::LOOKUP_MAX_QUERY_BYTES {
return Ok(());
@@ -107,34 +105,86 @@ impl GroundingCandidates {
.map(|r| &r.source)
.chain(self.searched.iter().map(|r| &r.source)),
);
+ let fresh_index = if !self.referenced.is_empty() && named.is_empty() {
+ self.searched.iter().position(|hit| {
+ !hit.text.trim().is_empty()
+ && !self
+ .referenced
+ .iter()
+ .any(|r| r.reference.locator == hit.locator)
+ })
+ } else {
+ None
+ };
+ if let Some(index) = fresh_index {
+ self.searched[..=index].rotate_right(1);
+ }
let mut required: std::collections::BTreeSet<_> = self
.referenced
.iter()
.map(|r| source_key(&r.source))
.collect();
- for hit in &self.searched {
+ let mut promoted = Vec::new();
+ for (index, hit) in self.searched.iter().enumerate() {
+ let fresh = fresh_index.is_some() && index == 0;
let id = source_key(&hit.source);
- if !named.contains(&id) || !required.insert(id) {
+ if !fresh && (!named.contains(&id) || required.contains(&id)) {
continue;
}
- let Some(reference) =
- retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), usize::MAX)
+ let budgets = if fresh {
+ [self.max_utf8_bytes / 2, self.max_utf8_bytes]
+ } else {
+ [usize::MAX; 2]
+ };
+ let Some(reference) = budgets.into_iter().find_map(|budget| {
+ retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), budget)
.ok()
- .flatten()
- .and_then(|context| context.included_passages.into_iter().next())
- else {
+ .flatten()?
+ .included_passages
+ .into_iter()
+ .next()
+ }) else {
continue;
};
let Some(passages) = reference.verified_spans(&hit.text) else {
continue;
};
- self.referenced.push(ReferencedPassage {
+ required.insert(id);
+ promoted.push(ReferencedPassage {
reference,
source: hit.source.clone(),
passages,
});
+ }
+
+ let (mut named_references, incidental): (Vec<_>, Vec<_>) =
+ std::mem::take(&mut self.referenced)
+ .into_iter()
+ .partition(|r| named.contains(&source_key(&r.source)));
+ named_references.extend(promoted);
+ let mut seen = std::collections::BTreeSet::new();
+ let (distinct, additional): (Vec<_>, Vec<_>) = named_references
+ .into_iter()
+ .partition(|r| seen.insert(source_key(&r.source)));
+ let mut packs = 0;
+ let mut notes = 0;
+ for reference in distinct.into_iter().chain(additional).chain(incidental) {
+ let (count, limit) = match &reference.source {
+ retrieval::GroundedSource::EnsuPack { .. } => {
+ (&mut packs, retrieval::MAX_PACK_HITS)
+ }
+ retrieval::GroundedSource::LocalNote { .. } => {
+ (&mut notes, retrieval::MAX_NOTES_GROUNDING_HITS)
+ }
+ };
+ if *count == limit {
+ continue;
+ }
+ self.referenced.push(reference);
if self.validate().is_err() {
self.referenced.pop();
+ } else {
+ *count += 1;
}
}
Ok(())
diff --git a/rust/crates/ensu/src/conversation/grounding_tests.rs b/rust/crates/ensu/src/conversation/grounding_tests.rs
index 1f7276582b..5624796852 100644
--- a/rust/crates/ensu/src/conversation/grounding_tests.rs
+++ b/rust/crates/ensu/src/conversation/grounding_tests.rs
@@ -28,6 +28,305 @@ fn hit(text: &str, id: &str) -> GroundedExcerpt {
fn measure(messages: &[ChatMessage]) -> Result<usize, PrepareError> {
Ok(messages.iter().map(|m| m.content.chars().count() + 8).sum())
}
+
+fn pack_hit(text: &str, title: &str, row: u64) -> GroundedExcerpt {
+ GroundedExcerpt {
+ locator: PassageLocator::EnsuPack {
+ dataset_id: "simplewiki".into(),
+ revision_sha256: "a".repeat(64),
+ row,
+ },
+ source: GroundedSource::EnsuPack {
+ citation: retrieval::SourceCitation {
+ dataset_id: "simplewiki".into(),
+ dataset_label: "Wikipedia".into(),
+ credit: "Wikipedia".into(),
+ title: title.into(),
+ source_url: format!("https://example.com/{title}"),
+ license_label: "CC BY-SA 4.0".into(),
+ license_url: "https://creativecommons.org/licenses/by-sa/4.0/".into(),
+ },
+ },
+ score: 0.9,
+ text: text.into(),
+ }
+}
+
+fn followup_history(prior: &[GroundedExcerpt]) -> (Vec<Message>, Vec<AnswerEvidence>) {
+ let old = retrieval::build_grounded_prompt_context(prior, MAX_GROUNDING_BYTES)
+ .unwrap()
+ .unwrap();
+ let mut history: Vec<_> = (1..=3)
+ .map(|i| Message {
+ uuid: Uuid::from_u128(i),
+ session_uuid: Uuid::from_u128(100),
+ parent_message_uuid: (i > 1).then(|| Uuid::from_u128(i - 1)),
+ sender: if i == 2 {
+ Sender::Other
+ } else {
+ Sender::SelfUser
+ },
+ text: "Explain the sources.".into(),
+ attachments: vec![],
+ created_at: i as i64,
+ })
+ .collect();
+ history[1].text =
+ retrieval::finalize_grounded_assistant_text("See the cited sources.", &old.sources)
+ .unwrap();
+ let evidence = vec![AnswerEvidence {
+ assistant_message_uuid: history[1].uuid,
+ prefix_fingerprint: fingerprint(&history[..2]),
+ passages: old.included_passages,
+ }];
+ (history, evidence)
+}
+
+fn fit_followup(
+ prior: &[GroundedExcerpt],
+ searched: Vec<GroundedExcerpt>,
+ query: &str,
+ input_budget: usize,
+) -> GroundedPromptContext {
+ let (history, evidence) = followup_history(prior);
+ let reloaded = direct_followup_references(&history, &evidence, query, &searched)
+ .into_iter()
+ .map(|reference| {
+ let hit = prior
+ .iter()
+ .find(|hit| hit.locator == reference.locator)
+ .unwrap();
+ ReferencedPassage {
+ passages: reference.verified_spans(&hit.text).unwrap(),
+ source: hit.source.clone(),
+ reference,
+ }
+ })
+ .collect();
+ let mut candidates = GroundingCandidates::new(reloaded, searched, MAX_GROUNDING_BYTES).unwrap();
+ candidates.protect_named_sources(query).unwrap();
+ let context = fit_grounding(&candidates, "System", query, input_budget, measure)
+ .unwrap()
+ .context
+ .unwrap();
+ assert!(context.text.len() <= MAX_GROUNDING_BYTES);
+ assert!(
+ measure(&answer_messages(
+ &grounded_system("System", Some(&context)),
+ None,
+ &[],
+ query
+ ))
+ .unwrap()
+ <= input_budget
+ );
+ context
+}
+
+#[test]
+fn followup_fitting_keeps_implicit_antecedents_and_allows_named_switches() {
+ let cedar = hit("Cedar allows 30 days.", "cedar.md");
+ let juniper = hit("Juniper allows 21 days.", "juniper.md");
+ let old = retrieval::build_grounded_prompt_context(std::slice::from_ref(&cedar), 6000)
+ .unwrap()
+ .unwrap();
+ for query in [
+ "Is Juniper longer than that?",
+ "Compare its deadline with Juniper.",
+ "Is Juniper's return period longer than ours?",
+ "Is Juniper's return period longer than mine?",
+ "Compare Juniper's return period with our policy.",
+ "Compare Juniper's return period with my policy.",
+ "Does she get more time than Juniper?",
+ "Compare her deadline with Juniper.",
+ "Is Juniper's deadline longer than hers?",
+ "Does he get more time than Juniper?",
+ "Does Juniper get more time than him?",
+ "Compare his deadline with Juniper.",
+ ] {
+ let context = fit_followup(
+ std::slice::from_ref(&cedar),
+ vec![juniper.clone()],
+ query,
+ 7000,
+ );
+ assert!(context.text.contains(&cedar.text), "{query}");
+ assert!(context.text.contains(&juniper.text), "{query}");
+ assert!(
+ context
+ .included_passages
+ .contains(&old.included_passages[0])
+ );
+ }
+ let updated = hit("Cedar now requires seven days.", "cedar.md");
+ for (query, fresh) in [
+ ("What is Juniper's return period?", &juniper),
+ ("What is within Juniper's coverage?", &juniper),
+ ("What is Cedar's current deadline?", &updated),
+ ] {
+ let context = fit_followup(
+ std::slice::from_ref(&cedar),
+ vec![fresh.clone()],
+ query,
+ 7000,
+ );
+ assert_eq!(context.sources, vec![fresh.source.clone()], "{query}");
+ assert!(context.text.contains(&fresh.text), "{query}");
+ }
+}
+
+#[test]
+fn followup_fitting_prioritizes_named_sources_at_source_and_byte_limits() {
+ let mut prior: Vec<_> = ["pine", "birch", "oak", "maple", "cedar"]
+ .iter()
+ .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))
+ .collect();
+ let atlas = hit(&"Atlas allows 21 days. ".repeat(40), "atlas.md");
+ let note_count = prior.len();
+ prior.extend([
+ pack_hit("Mars is the fourth planet.", "Mars", 0),
+ pack_hit("Venus is the second planet.", "Venus", 1),
+ ]);
+ let byte_limited = [
+ prior[note_count - 1].clone(),
+ hit(&"Unrelated detail. ".repeat(400), "pine.md"),
+ ];
+ for prior in [&prior[..note_count], &prior[..], &byte_limited] {
+ let context = fit_followup(prior, vec![atlas.clone()], "Compare Cedar and Atlas", 7000);
+ assert!(context.text.contains("cedar allows 30 days."));
+ assert!(context.text.contains(atlas.text.trim()));
+ assert!(context.sources.len() <= retrieval::MAX_GROUNDING_HITS);
+ }
+
+ let mars = pack_hit("Mars is the fourth planet.", "Mars", 0);
+ let venus = pack_hit("Venus is the second planet.", "Venus", 1);
+ let jupiter = pack_hit("Jupiter is the fifth planet.", "Jupiter", 2);
+ let another_mars_chunk = pack_hit("Mars also has two moons.", "Mars", 3);
+ for prior in [[venus, mars.clone()], [mars.clone(), another_mars_chunk]] {
+ let context = fit_followup(
+ &prior,
+ vec![jupiter.clone()],
+ "Compare Mars and Jupiter",
+ 7000,
+ );
+ assert!(context.text.contains(&mars.text));
+ assert!(context.text.contains(&jupiter.text));
+ assert_eq!(context.sources.len(), 2);
+ }
+}
+
+#[test]
+fn unnamed_questions_keep_fresh_results_and_available_historical_passages() {
+ let notes: Vec<_> = ["cedar", "pine", "birch", "oak", "maple"]
+ .iter()
+ .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))
+ .collect();
+ let biology = hit(
+ "Photosynthesis converts light into energy.",
+ "biology-chapter-seven.md",
+ );
+ let mut another_chunk = notes[0].clone();
+ if let PassageLocator::LocalNote { chunk_index, .. } = &mut another_chunk.locator {
+ *chunk_index += 1;
+ }
+ another_chunk.text = "International shipping has an extended deadline.".into();
+ let old = retrieval::build_grounded_prompt_context(&notes, MAX_GROUNDING_BYTES)
+ .unwrap()
+ .unwrap();
+ for (query, fresh) in [
+ ("How does photosynthesis work?", &biology),
+ ("And the deadline?", &biology),
+ ("And the deadline?", &another_chunk),
+ ] {
+ let context = fit_followup(&notes, vec![fresh.clone()], query, 7000);
+ assert!(context.text.contains(&fresh.text), "{query}");
+ assert!(context.text.contains(&notes[0].text), "{query}");
+ assert!(
+ context
+ .included_passages
+ .contains(&old.included_passages[0])
+ );
+ assert!(context.sources.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);
+ }
+
+ let packs = [
+ pack_hit("Mars is the fourth planet.", "Mars", 0),
+ pack_hit("Venus is the second planet.", "Venus", 1),
+ ];
+ let fresh = pack_hit("Plants use sunlight to produce sugar.", "Botany", 2);
+ let context = fit_followup(
+ &packs,
+ vec![fresh.clone()],
+ "How does photosynthesis work?",
+ 7000,
+ );
+ assert!(context.text.contains(&fresh.text));
+ assert!(context.text.contains(&packs[0].text));
+ assert_eq!(context.sources.len(), retrieval::MAX_PACK_HITS);
+
+ let context = fit_followup(&notes, vec![], "And the deadline?", 7000);
+ assert_eq!(context.included_passages, old.included_passages);
+}
+
+#[test]
+fn fresh_followup_results_survive_byte_and_token_repacking() {
+ let prefix = hit("Price: 100 rupees.", "cedar.md");
+ let mut full = prefix.clone();
+ full.text.push_str(" Warranty duration: 24 months.");
+ let context = fit_followup(
+ &[prefix],
+ vec![full.clone()],
+ "And what's its warranty duration?",
+ 7000,
+ );
+ assert!(context.text.contains(&full.text));
+ assert_eq!(context.sources, vec![full.source]);
+ assert_eq!(context.included_passages.len(), 1);
+ assert_eq!(
+ context.included_passages[0]
+ .verified_spans(&full.text)
+ .unwrap(),
+ vec![full.text]
+ );
+
+ let old = hit(&"Cedar allows 30 days. ".repeat(280), "cedar.md");
+ let fresh = hit(
+ &format!(
+ "Plants use light. {}",
+ "Photosynthesis details. ".repeat(300)
+ ),
+ "biology-chapter-seven.md",
+ );
+ for input_budget in [7000, 1600] {
+ let context = fit_followup(
+ std::slice::from_ref(&old),
+ vec![old.clone(), fresh.clone()],
+ "How does photosynthesis work?",
+ input_budget,
+ );
+ assert!(context.text.contains("Plants use light."));
+ let reference = context
+ .included_passages
+ .iter()
+ .find(|reference| reference.locator == fresh.locator)
+ .unwrap();
+ let spans = reference.verified_spans(&fresh.text).unwrap();
+ assert!(context.text.contains(&spans[0]));
+ }
+
+ let mut long_label = hit("Plants use light.", "biology.md");
+ if let GroundedSource::LocalNote { reference } = &mut long_label.source {
+ reference.title = "Biology ".repeat(400);
+ }
+ let context = fit_followup(
+ &[old],
+ vec![long_label.clone()],
+ "How does photosynthesis work?",
+ 7000,
+ );
+ assert!(context.text.contains(&long_label.text));
+}
+
#[test]
fn measured_repack_keeps_text_citations_and_exact_spans_together() {
for (raw, prefix) in [
@@ -44,19 +343,15 @@ fn measured_repack_keeps_text_citations_and_exact_spans_together() {
),
] {
let candidates = GroundingCandidates::new(vec![], vec![hit(&raw, "one.md")], 6000).unwrap();
- let original = serde_json::to_string(&candidates).unwrap();
let fit = fit_grounding(&candidates, "System", "Question", 4096, measure).unwrap();
assert!(fit.repacked);
assert!(fit.evidence_tokens <= 1024);
let context = fit.context.unwrap();
- assert_eq!(context.sources.len(), context.included_passages.len());
+ assert_eq!(context.sources, vec![candidates.searched[0].source.clone()]);
assert_eq!(context.included_passages.len(), 1);
- for reference in &context.included_passages {
- let spans = reference.verified_spans(&raw).unwrap();
- assert!(context.text.contains(&spans[0]));
- assert!(spans[0].starts_with(prefix));
- }
- assert_eq!(serde_json::to_string(&candidates).unwrap(), original);
+ let spans = context.included_passages[0].verified_spans(&raw).unwrap();
+ assert!(context.text.contains(&spans[0]));
+ assert!(spans[0].starts_with(prefix));
}
}
#[test]
@@ -82,11 +377,6 @@ fn requested_old_spans_raise_soft_allowance_and_are_never_cut() {
let fit = fit_grounding(&candidates, "System", "Question", 3000, measure).unwrap();
assert!(fit.repacked && fit.evidence_tokens > 750);
assert_eq!(fit.context.unwrap().included_passages[0], reference);
- assert!(
- fit_grounding(&candidates, "System", "Question", 700, measure)
- .unwrap()
- .repacked
- );
}
#[test]
fn mandatory_input_and_insufficient_evidence_budget_are_explicit() {
@@ -134,7 +424,6 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {
candidates
.protect_named_sources("Is the Juniper period longer than Cedar?")
.unwrap();
- assert_eq!(candidates.referenced.len(), 2);
let fit = fit_grounding(
&candidates,
"System",
@@ -157,42 +446,10 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {
vec![hit.text.clone()]
);
}
- assert!(
- fit_grounding(
- &candidates,
- "System",
- "Compare Cedar and Juniper.",
- 900,
- measure
- )
- .unwrap()
- .repacked
- );
}
#[test]
fn ordinary_questions_and_shared_label_words_do_not_promote_every_source() {
- let mut partial = GroundingCandidates::new(
- vec![],
- vec![hit(" ", "cedar.md"), hit("21 days", "juniper.md")],
- 6000,
- )
- .unwrap();
- partial
- .protect_named_sources("Compare Cedar and Juniper")
- .unwrap();
- let context = fit_grounding(
- &partial,
- "System",
- "Compare Cedar and Juniper",
- 4096,
- measure,
- )
- .unwrap()
- .context
- .unwrap();
- assert_eq!(context.sources.len(), 1);
- assert!(context.text.contains("21 days"));
let base = GroundingCandidates::new(
vec![],
vec![
@@ -210,34 +467,6 @@ fn ordinary_questions_and_shared_label_words_do_not_promote_every_source() {
}
}
-#[test]
-fn an_old_reference_and_new_named_document_are_both_protected() {
- let old = hit("Cedar 30 days.", "cedar.md");
- let packed = retrieval::build_grounded_prompt_context(std::slice::from_ref(&old), 6000)
- .unwrap()
- .unwrap();
- let reference = packed.included_passages[0].clone();
- let required = ReferencedPassage {
- passages: reference.verified_spans(&old.text).unwrap(),
- source: old.source,
- reference: reference.clone(),
- };
- let juniper = hit("Juniper 21 days.", "juniper.md");
- let mut second_chunk = juniper.clone();
- if let PassageLocator::LocalNote { chunk_index, .. } = &mut second_chunk.locator {
- *chunk_index = 1;
- }
- let mut c =
- GroundingCandidates::new(vec![required], vec![juniper, second_chunk], 6000).unwrap();
- c.protect_named_sources("Is Juniper longer than that?")
- .unwrap();
- assert_eq!(c.referenced.len(), 2);
- assert_eq!(c.referenced[0].reference, reference);
- c.protect_named_sources("Compare Cedar and Juniper.")
- .unwrap();
- assert_eq!(c.referenced.len(), 2);
-}
-
fn prior_history() -> Vec<Message> {
vec![Message {
uuid: Uuid::from_u128(1),
@@ -326,23 +555,7 @@ fn source_system_and_required_history_are_measured_in_final_roles() {
#[test]
fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switches() {
- let mut pack = hit("Mars is the fourth planet.", "unused.md");
- pack.locator = PassageLocator::EnsuPack {
- dataset_id: "wiki".into(),
- revision_sha256: "a".repeat(64),
- row: 0,
- };
- pack.source = GroundedSource::EnsuPack {
- citation: crate::retrieval::SourceCitation {
- dataset_id: "wiki".into(),
- dataset_label: "Wiki".into(),
- credit: "Wiki".into(),
- title: "Mars".into(),
- source_url: "https://example.com/mars".into(),
- license_label: "CC BY-SA 4.0".into(),
- license_url: "https://creativecommons.org/licenses/by-sa/4.0/".into(),
- },
- };
+ let pack = pack_hit("Mars is the fourth planet.", "Mars", 0);
let mut note = hit("Check-in code COPPER-29.", "mars-expedition.md");
if let GroundedSource::LocalNote { reference } = &mut note.source {
reference.title = "Mars expedition checklist".into();
@@ -352,37 +565,11 @@ fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switc
c.protect_named_sources(query).unwrap();
assert_eq!(c.referenced.len(), 2);
assert_eq!(c.pack(6000).unwrap().unwrap().sources.len(), 2);
- assert!(c.pack(100).is_err());
- let context = retrieval::build_grounded_prompt_context(&[pack], 6000)
- .unwrap()
- .unwrap();
- let mut history: Vec<_> = (1..=3)
- .map(|i| Message {
- uuid: Uuid::from_u128(i),
- session_uuid: Uuid::from_u128(100),
- parent_message_uuid: (i > 1).then(|| Uuid::from_u128(i - 1)),
- sender: if i == 2 {
- Sender::Other
- } else {
- Sender::SelfUser
- },
- text: query.into(),
- attachments: vec![],
- created_at: i as i64,
- })
- .collect();
- history[1].text =
- retrieval::finalize_grounded_assistant_text("Fourth planet", &context.sources).unwrap();
- let evidence = vec![AnswerEvidence {
- assistant_message_uuid: history[1].uuid,
- prefix_fingerprint: fingerprint(&history[..2]),
- passages: context.included_passages.clone(),
- }];
+ let (history, evidence) = followup_history(&[pack]);
assert_eq!(
- direct_followup_references(&history, &evidence, query, std::slice::from_ref(&note))
- .unwrap(),
- context.included_passages
+ direct_followup_references(&history, &evidence, query, std::slice::from_ref(&note)),
+ evidence[0].passages
);
assert!(
direct_followup_references(
@@ -391,90 +578,29 @@ fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switc
"What is the check-in code in the Mars expedition checklist?",
&[note]
)
- .unwrap()
.is_empty()
);
}
#[test]
-fn direct_followups_reuse_saved_spans_but_named_search_can_change_sources() {
- let prior: Vec<_> = ["cedar", "pine", "birch", "oak", "maple"]
- .iter()
- .map(|name| hit("Archived policy fact.", &format!("{name}.md")))
- .collect();
- let context = retrieval::build_grounded_prompt_context(&prior, 6000)
- .unwrap()
- .unwrap();
- let mut history: Vec<_> = (1..=5)
- .map(|i| Message {
+fn followups_use_last_available_evidence_until_its_history_changes() {
+ let (mut history, evidence) = followup_history(&[hit("Archived policy fact.", "cedar.md")]);
+ for i in 4..=5 {
+ history.push(Message {
uuid: Uuid::from_u128(i),
- session_uuid: Uuid::from_u128(100),
- parent_message_uuid: (i > 1).then(|| Uuid::from_u128(i - 1)),
- sender: if i % 2 == 0 {
+ parent_message_uuid: Some(Uuid::from_u128(i - 1)),
+ sender: if i == 4 {
Sender::Other
} else {
Sender::SelfUser
},
- text: "Please clarify the subject.".into(),
- attachments: vec![],
- created_at: i as i64,
- })
- .collect();
- history[1].text =
- retrieval::finalize_grounded_assistant_text("Saved facts", &context.sources).unwrap();
- let evidence = vec![AnswerEvidence {
- assistant_message_uuid: history[1].uuid,
- prefix_fingerprint: fingerprint(&history[..2]),
- passages: context.included_passages.clone(),
- }];
- let atlas = hit("Warranty lasts two years.", "atlas.md");
- assert_eq!(
- direct_followup_references(&history, &evidence, "And the deadline?", &[]).unwrap(),
- context.included_passages
- );
- assert!(
- direct_followup_references(
- &history,
- &evidence,
- "How long is the Atlas warranty?",
- std::slice::from_ref(&atlas)
- )
- .unwrap()
- .is_empty()
- );
- let mut candidates = GroundingCandidates::new(vec![], vec![atlas.clone()], 6000).unwrap();
- candidates
- .protect_named_sources("How long is the Atlas warranty?")
- .unwrap();
- assert!(
- candidates
- .pack(6000)
- .unwrap()
- .unwrap()
- .text
- .contains(&atlas.text)
- );
- let mut updated = prior[0].clone();
- updated.text = "Cedar now requires seven days.".into();
- assert!(
- direct_followup_references(
- &history,
- &evidence,
- "What is Cedar's current deadline?",
- &[updated]
- )
- .unwrap()
- .is_empty()
- );
+ ..history[2].clone()
+ });
+ }
assert_eq!(
- direct_followup_references(&history, &evidence, "Compare Cedar and Atlas", &[atlas])
- .unwrap(),
- context.included_passages
+ direct_followup_references(&history, &evidence, "And the deadline?", &[]),
+ evidence[0].passages
);
history[0].text = "Edited history".into();
- assert!(
- direct_followup_references(&history, &evidence, "And the deadline?", &[])
- .unwrap()
- .is_empty()
- );
+ assert!(direct_followup_references(&history, &evidence, "And the deadline?", &[]).is_empty());
}
diff --git a/rust/crates/ensu/src/conversation/state.rs b/rust/crates/ensu/src/conversation/state.rs
index 105bc2a8a0..5873549475 100644
--- a/rust/crates/ensu/src/conversation/state.rs
+++ b/rust/crates/ensu/src/conversation/state.rs
@@ -19,7 +19,7 @@ pub struct AnswerEvidence {
}
impl AnswerEvidence {
- pub fn matches(&self, history: &[Message]) -> bool {
+ pub(crate) fn matches(&self, history: &[Message]) -> bool {
history
.iter()
.position(|m| m.uuid == self.assistant_message_uuid)
@@ -30,14 +30,13 @@ impl AnswerEvidence {
}
fn valid(&self) -> bool {
- self.prefix_fingerprint.len() == 64
- && !self.passages.is_empty()
- && valid_answer_passages(&self.passages)
+ self.prefix_fingerprint.len() == 64 && valid_answer_passages(&self.passages)
}
}
pub(crate) fn valid_answer_passages(passages: &[IncludedPassage]) -> bool {
- passages.len() <= MAX_ANSWER_PASSAGES
+ !passages.is_empty()
+ && passages.len() <= MAX_ANSWER_PASSAGES
&& passages.iter().all(IncludedPassage::valid_metadata)
&& serde_json::to_vec(passages).is_ok_and(|bytes| bytes.len() <= MAX_EVIDENCE_BYTES)
}
@@ -117,7 +116,6 @@ impl ConversationEnvelope {
if history.len() < 2
|| answer.sender != Sender::Other
|| answer.session_uuid != self.session_uuid
- || passages.is_empty()
|| !valid_answer_passages(&passages)
|| super::validate_path(history).is_err()
{
diff --git a/rust/crates/ensu/src/conversation/tests.rs b/rust/crates/ensu/src/conversation/tests.rs
index d327029078..743b8bbd85 100644
--- a/rust/crates/ensu/src/conversation/tests.rs
+++ b/rust/crates/ensu/src/conversation/tests.rs
@@ -1,3 +1,4 @@
+use super::state::{MAX_ANSWER_PASSAGES, MAX_EVIDENCE_ANSWERS, MAX_EVIDENCE_BYTES};
use super::*;
fn message(index: u128, sender: Sender, text: &str) -> Message {
@@ -76,6 +77,69 @@ fn saved_memory_preserves_uuid_strings() {
);
}
+fn evidence_passages() -> Vec<crate::retrieval::IncludedPassage> {
+ vec![crate::retrieval::passage_fixture().0]
+}
+
+#[test]
+fn invalid_summary_and_evidence_are_discarded_independently() {
+ let history = vec![
+ message(1, Sender::SelfUser, "Question"),
+ message(2, Sender::Other, "Answer"),
+ ];
+ let session = history[0].session_uuid;
+ let memory = ConversationState::new(session, &history, "Memory".into()).unwrap();
+ let mut state = ConversationEnvelope::empty(session);
+ state.summary = Some(memory.summary.clone());
+ assert!(state.add_evidence(&history, evidence_passages()));
+ let original = serde_json::to_value(&state).unwrap();
+ let decode = |value: &serde_json::Value| {
+ ConversationEnvelope::decode(&serde_json::to_vec(value).unwrap(), session)
+ };
+ let mut value = original.clone();
+ value["summary"] = serde_json::json!({"bad": true});
+ let decoded = decode(&value).unwrap();
+ assert!(decoded.summary.is_none());
+ assert_eq!(decoded.evidence, state.evidence);
+
+ value = original;
+ value["evidence"][0]["passages"][0]["spans"][0]["textSha256"] = "bad".into();
+ let decoded = decode(&value).unwrap();
+ assert!(decoded.evidence.is_empty());
+ assert_eq!(decoded.summary_state().unwrap(), memory);
+ value["format_version"] = 1.into();
+ assert!(decode(&value).is_none());
+ assert!(
+ ConversationEnvelope::decode(&serde_json::to_vec(&state).unwrap(), Uuid::nil()).is_none()
+ );
+}
+
+#[test]
+fn evidence_retention_obeys_answer_and_byte_limits() {
+ for passage_count in [1, MAX_ANSWER_PASSAGES] {
+ let passages = vec![evidence_passages().remove(0); passage_count];
+ let mut state = ConversationEnvelope::empty(Uuid::from_u128(100));
+ let mut history = Vec::new();
+ for index in 1..=10 {
+ history.push(message(index * 2 - 1, Sender::SelfUser, "Question"));
+ history.push(message(index * 2, Sender::Other, "Answer"));
+ assert!(state.add_evidence(&history, passages.clone()));
+ }
+ assert_eq!(
+ state.evidence.last().unwrap().assistant_message_uuid,
+ Uuid::from_u128(20)
+ );
+ assert_ne!(state.evidence[0].assistant_message_uuid, Uuid::from_u128(2));
+ assert!(serde_json::to_vec(&state.evidence).unwrap().len() <= MAX_EVIDENCE_BYTES);
+ if passage_count == 1 {
+ assert_eq!(state.evidence.len(), MAX_EVIDENCE_ANSWERS);
+ } else {
+ assert!(state.evidence.len() < MAX_EVIDENCE_ANSWERS);
+ }
+ assert!(!state.add_evidence(&history, vec![passages[0].clone(); MAX_ANSWER_PASSAGES + 1]));
+ }
+}
+
#[test]
fn coverage_tracks_original_text_and_branch_but_not_recent_tail() {
let mut history = vec![
diff --git a/rust/crates/ensu/src/db/chat/conversation.rs b/rust/crates/ensu/src/db/chat/conversation.rs
index 59ed836bed..9d94b50b89 100644
--- a/rust/crates/ensu/src/db/chat/conversation.rs
+++ b/rust/crates/ensu/src/db/chat/conversation.rs
@@ -24,11 +24,12 @@ impl ConversationSnapshot {
pub fn state(&self) -> Option<&ConversationState> {
self.state.as_ref()
}
- pub fn evidence(&self) -> Vec<&AnswerEvidence> {
+ pub fn evidence(&self) -> Vec<AnswerEvidence> {
self.derived
.evidence
.iter()
.filter(|entry| entry.matches(&self.messages))
+ .cloned()
.collect()
}
pub fn session_uuid(&self) -> Uuid {
@@ -165,12 +166,12 @@ impl<B: Backend> ChatDb<B> {
answer: Uuid,
passages: Vec<crate::retrieval::IncludedPassage>,
) -> Result<bool> {
- if !conversation::valid_answer_passages(&passages) || passages.is_empty() {
+ if !conversation::valid_answer_passages(&passages) {
return Ok(false);
}
let mut path: Vec<_> = snapshot.messages.iter().map(|m| m.uuid).collect();
path.push(answer);
- let current = self.conversation_snapshot(snapshot.session_uuid, &path)?;
+ let mut current = self.conversation_snapshot(snapshot.session_uuid, &path)?;
if current.envelope != snapshot.envelope
|| current.rows[..snapshot.rows.len()] != snapshot.rows
{
@@ -188,11 +189,10 @@ impl<B: Backend> ChatDb<B> {
}) {
return Ok(false);
}
- let mut derived = current.derived.clone();
- if !derived.add_evidence(&current.messages, passages) {
+ if !current.derived.add_evidence(&current.messages, passages) {
return Ok(false);
}
- let envelope = self.encode_conversation(&derived)?;
+ let envelope = self.encode_conversation(&current.derived)?;
self.replace_conversation_envelope(&current, envelope)
}
@@ -419,11 +419,7 @@ mod tests {
}
fn passages() -> Vec<crate::retrieval::IncludedPassage> {
- let fixtures: serde_json::Value = serde_json::from_str(include_str!(
- "../../../tests/fixtures/passage-references-v1.json"
- ))
- .unwrap();
- vec![serde_json::from_value(fixtures[0]["reference"].clone()).unwrap()]
+ vec![crate::retrieval::passage_fixture().0]
}
fn sourced_answer(
@@ -742,133 +738,6 @@ mod tests {
assert!(db.get_session(session).unwrap().is_none());
}
- #[test]
- fn malformed_components_do_not_destroy_valid_siblings() {
- let (db, session, messages) = seeded();
- let memory = state(session, &messages, "Memory");
- let mut captured = snapshot(&db, session, &messages);
- assert!(
- db.replace_conversation_state(&mut captured, Some(memory.clone()))
- .unwrap()
- );
- let answer = sourced_answer(&db, &captured);
- assert!(
- db.save_answer_evidence(&captured, answer.uuid, passages())
- .unwrap()
- );
- let mut path = messages;
- path.push(answer);
- let restored = snapshot(&db, session, &path);
- let mut value = serde_json::to_value(&restored.derived).unwrap();
- assert!(
- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), Uuid::new_v4())
- .is_none()
- );
- value["format_version"] = 1.into();
- assert!(
- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), session).is_none()
- );
- value = serde_json::to_value(&restored.derived).unwrap();
- value["summary"] = serde_json::json!({"bad": true});
- let decoded =
- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), session).unwrap();
- assert!(decoded.summary.is_none());
- assert_eq!(decoded.evidence.len(), 1);
- value = serde_json::to_value(&restored.derived).unwrap();
- value["evidence"][0]["passages"][0]["spans"][0]["textSha256"] = "bad".into();
- let decoded =
- ConversationEnvelope::decode(&serde_json::to_vec(&value).unwrap(), session).unwrap();
- assert!(decoded.evidence.is_empty());
- assert_eq!(decoded.summary_state().unwrap(), memory);
- }
-
- #[test]
- fn evidence_retention_is_bounded_and_keeps_newest_answers() {
- let (db, session, mut messages) = seeded();
- let mut first_answer = None;
- for index in 0..10 {
- let captured = snapshot(&db, session, &messages);
- let answer = sourced_answer(&db, &captured);
- first_answer.get_or_insert(answer.uuid);
- assert!(
- db.save_answer_evidence(&captured, answer.uuid, passages())
- .unwrap()
- );
- messages.push(answer);
- let current = snapshot(&db, session, &messages);
- assert!(current.derived.evidence.len() <= conversation::MAX_EVIDENCE_ANSWERS);
- assert!(serde_json::to_vec(&current.derived).unwrap().len() <= MAX_STATE_BYTES);
- if index == 9 {
- assert_eq!(current.evidence().len(), 8);
- assert!(
- !current
- .evidence()
- .iter()
- .any(|entry| Some(entry.assistant_message_uuid) == first_answer)
- );
- } else {
- let question = db
- .insert_message(
- session,
- "self",
- "Next",
- Some(messages.last().unwrap().uuid),
- vec![],
- )
- .unwrap();
- messages.push(question);
- }
- }
- }
-
- #[test]
- fn evidence_byte_limit_evicts_older_records_and_rejects_oversized_inputs() {
- let (db, session, mut messages) = seeded();
- let mut large = Vec::new();
- for chunk in 0..conversation::MAX_ANSWER_PASSAGES {
- let mut reference = passages().remove(0);
- if let crate::retrieval::PassageLocator::LocalNote { chunk_index, .. } =
- &mut reference.locator
- {
- *chunk_index = chunk as u64;
- }
- large.push(reference);
- }
- assert!(!conversation::valid_answer_passages(&vec![
- large[0].clone();
- 8
- ]));
- let mut future = large.clone();
- future[0].cleaning_version = 999;
- assert!(!conversation::valid_answer_passages(&future));
- for _ in 0..6 {
- let captured = snapshot(&db, session, &messages);
- let answer = sourced_answer(&db, &captured);
- assert!(
- db.save_answer_evidence(&captured, answer.uuid, large.clone())
- .unwrap()
- );
- messages.push(answer);
- let question = db
- .insert_message(
- session,
- "self",
- "Next",
- Some(messages.last().unwrap().uuid),
- vec![],
- )
- .unwrap();
- messages.push(question);
- }
- let current = snapshot(&db, session, &messages);
- assert!(current.evidence().len() < 6);
- assert!(!current.evidence().is_empty());
- assert!(
- serde_json::to_vec(&current.derived.evidence).unwrap().len()
- <= conversation::MAX_EVIDENCE_BYTES
- );
- }
-
#[test]
fn encrypted_memory_survives_titles_appends_and_recent_edits() {
let (db, session, messages) = seeded();
diff --git a/rust/crates/ensu/src/notes/index.rs b/rust/crates/ensu/src/notes/index.rs
index a95496dac3..46468f2695 100644
--- a/rust/crates/ensu/src/notes/index.rs
+++ b/rust/crates/ensu/src/notes/index.rs
@@ -6,6 +6,7 @@ use super::manifest::{
};
use super::shard::{NotesShard, load_and_validate_shard_metadata, load_and_validate_vectors};
use super::{NotesCollectionReadPin, NotesError, pinned_revision_key, validate_collection_id};
+use crate::retrieval::PassageLocator;
const RELEVANCE_THRESHOLD: f32 = 0.50;
const MAX_HITS_PER_COLLECTION: usize = 5;
@@ -13,7 +14,7 @@ const MAX_HITS_PER_DOCUMENT: usize = 3;
#[derive(Debug, Clone, PartialEq)]
pub struct NotesSearchHit {
- pub locator: crate::retrieval::PassageLocator,
+ pub locator: PassageLocator,
pub collection_id: String,
pub document_id: String,
pub revision: String,
@@ -166,25 +167,28 @@ impl NotesCollectionIndex {
}))
}
- fn passage_locator(
- &self,
- document: &IndexedDocument,
- chunk_index: usize,
- ) -> crate::retrieval::PassageLocator {
- crate::retrieval::PassageLocator::LocalNote {
- collection_id: self.collection_id.clone(),
- document_id: document.document_id.clone(),
- indexed_revision: document.revision.clone(),
- shard_sha256: document.shard_sha256.clone(),
- chunk_index: chunk_index as u64,
- }
+ fn load_shard(&self, document: &IndexedDocument) -> Result<NotesShard, NotesError> {
+ let directory = shard_directory(
+ self.read_pin.collection_directory(),
+ &document.document_id,
+ &document.revision,
+ );
+ load_and_validate_shard_metadata(
+ &directory,
+ &self.collection_id,
+ &document.document_id,
+ &document.revision,
+ document.chunk_count,
+ &document.shard_sha256,
+ None,
+ )
}
pub fn reload_passage(
&self,
- locator: &crate::retrieval::PassageLocator,
+ locator: &PassageLocator,
) -> Result<Option<NotesSearchHit>, NotesError> {
- let crate::retrieval::PassageLocator::LocalNote {
+ let PassageLocator::LocalNote {
collection_id,
document_id,
indexed_revision,
@@ -210,20 +214,7 @@ impl NotesCollectionIndex {
if chunk_index >= document.chunk_count {
return Ok(None);
}
- let directory = shard_directory(
- self.read_pin.collection_directory(),
- document_id,
- indexed_revision,
- );
- let shard = load_and_validate_shard_metadata(
- &directory,
- collection_id,
- document_id,
- indexed_revision,
- document.chunk_count,
- shard_sha256,
- None,
- )?;
+ let shard = self.load_shard(document)?;
let chunk = &shard.chunks[chunk_index];
Ok(Some(NotesSearchHit {
locator: locator.clone(),
@@ -295,28 +286,19 @@ impl NotesCollectionIndex {
let document = &self.documents[candidate.document_index];
let shard = match loaded_shards.entry(candidate.document_index) {
Entry::Occupied(entry) => entry.into_mut(),
- Entry::Vacant(entry) => {
- let directory = shard_directory(
- self.read_pin.collection_directory(),
- &document.document_id,
- &document.revision,
- );
- entry.insert(load_and_validate_shard_metadata(
- &directory,
- &self.collection_id,
- &document.document_id,
- &document.revision,
- document.chunk_count,
- &document.shard_sha256,
- None,
- )?)
- }
+ Entry::Vacant(entry) => entry.insert(self.load_shard(document)?),
};
let chunk = shard.chunks.get(candidate.chunk_index).ok_or_else(|| {
NotesError::InvalidIndex("selected shard chunk is missing".to_string())
})?;
hits.push(NotesSearchHit {
- locator: self.passage_locator(document, candidate.chunk_index),
+ locator: PassageLocator::LocalNote {
+ collection_id: self.collection_id.clone(),
+ document_id: document.document_id.clone(),
+ indexed_revision: document.revision.clone(),
+ shard_sha256: document.shard_sha256.clone(),
+ chunk_index: candidate.chunk_index as u64,
+ },
collection_id: self.collection_id.clone(),
document_id: document.document_id.clone(),
revision: document.revision.clone(),
@@ -419,7 +401,6 @@ mod tests {
#[test]
fn passage_reload_checks_revision_chunk_and_shard_integrity() {
- use crate::retrieval::PassageLocator;
let temp = tempfile::tempdir().unwrap();
let doc = document("trip.md", 2);
let mut writer = NotesIndexWriter::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
diff --git a/rust/crates/ensu/src/retrieval.rs b/rust/crates/ensu/src/retrieval.rs
index 720131fad6..6fd0445f9f 100644
--- a/rust/crates/ensu/src/retrieval.rs
+++ b/rust/crates/ensu/src/retrieval.rs
@@ -61,3 +61,6 @@ pub use prompt::{KnowledgePromptContext, KnowledgePromptHit, build_knowledge_pro
pub(crate) use grounded::{MAX_GROUNDING_HITS, MAX_PACK_HITS, build_followup_prompt_context};
pub(crate) use passage::clean_passage_text;
pub use passage::{IncludedPassage, PassageLocator, PassageSpan};
+
+#[cfg(test)]
+pub(crate) use passage::tests::passage_fixture;
diff --git a/rust/crates/ensu/src/retrieval/grounded.rs b/rust/crates/ensu/src/retrieval/grounded.rs
index a14343d2c7..c5e608eb54 100644
--- a/rust/crates/ensu/src/retrieval/grounded.rs
+++ b/rust/crates/ensu/src/retrieval/grounded.rs
@@ -265,6 +265,8 @@ pub(crate) fn build_followup_prompt_context(
}
let mut text = format!("{BEGIN_CONTEXT_SENTINEL}\n{CONTEXT_WARNING}");
let footer = format!("\n{END_CONTEXT_SENTINEL}");
+ let mut sections = Vec::new();
+ let mut used_bytes = text.len() + footer.len();
let mut sources = Vec::new();
let mut included_passages = Vec::new();
let mut packs = 0;
@@ -295,25 +297,48 @@ pub(crate) fn build_followup_prompt_context(
}
let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);
let passages = item.passages.join("\n[omitted source text]\n");
- if text
- .len()
+ if used_bytes
.saturating_add(prefix.len())
.saturating_add(passages.len())
- .saturating_add(footer.len())
> max_utf8_bytes
{
return Err(invalid());
}
- text.push_str(&prefix);
- text.push_str(&passages);
+ used_bytes += prefix.len() + passages.len();
+ sections.push(format!("{prefix}{passages}"));
sources.push(item.source.clone());
included_passages.push(item.reference.clone());
}
for item in searched {
- if included_passages
+ if let Some(index) = included_passages
.iter()
- .any(|reference| reference.locator == item.locator)
+ .position(|reference| reference.locator == item.locator)
{
+ let reference = &included_passages[index];
+ if !item.score.is_finite()
+ || !item.locator.matches_source(&item.source)
+ || reference.verified_spans(&item.text).is_none()
+ {
+ continue;
+ }
+ // Fresh search may add text beyond the saved spans, but every old
+ // span must still fit. Keep one source entry for the entire chunk.
+ let prefix = format!("\n\n# {}\n", grounded_label(&sources[index])?);
+ let available = max_utf8_bytes - used_bytes + sections[index].len() - prefix.len();
+ let cleaned = super::clean_passage_text(&item.text);
+ let passage = truncate_utf8(&cleaned, available).trim_end();
+ if reference
+ .spans
+ .iter()
+ .any(|span| span.end_utf8 as usize > passage.len())
+ {
+ continue;
+ }
+ let expanded = super::IncludedPassage::prefix(item.locator.clone(), passage)?;
+ used_bytes -= sections[index].len();
+ sections[index] = format!("{prefix}{passage}");
+ used_bytes += sections[index].len();
+ included_passages[index] = expanded;
continue;
}
let is_pack = matches!(&item.source, GroundedSource::EnsuPack { .. });
@@ -321,8 +346,7 @@ pub(crate) fn build_followup_prompt_context(
continue;
}
let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);
- let Some(available) = max_utf8_bytes.checked_sub(text.len() + prefix.len() + footer.len())
- else {
+ let Some(available) = max_utf8_bytes.checked_sub(used_bytes + prefix.len()) else {
continue;
};
if !item.score.is_finite() || !item.locator.matches_source(&item.source) {
@@ -333,8 +357,8 @@ pub(crate) fn build_followup_prompt_context(
if passage.is_empty() {
continue;
}
- text.push_str(&prefix);
- text.push_str(passage);
+ used_bytes += prefix.len() + passage.len();
+ sections.push(format!("{prefix}{passage}"));
sources.push(item.source.clone());
included_passages.push(super::IncludedPassage::prefix(
item.locator.clone(),
@@ -346,6 +370,9 @@ pub(crate) fn build_followup_prompt_context(
notes += 1;
}
}
+ for section in sections {
+ text.push_str(&section);
+ }
text.push_str(&footer);
Ok(Some(GroundedPromptContext {
text,
@@ -399,11 +426,13 @@ mod tests {
}
#[test]
- fn mismatched_source_and_locator_are_rejected() {
- let candidates = select_mixed_grounding_candidates(&[], &[note("one.md", 0.9)], 5).unwrap();
- let mut wrong = candidates;
+ fn mismatched_passage_sources_and_text_are_rejected() {
+ let mut wrong = select_mixed_grounding_candidates(&[], &[note("one.md", 0.9)], 5).unwrap();
wrong[0].locator = note("other.md", 0.9).locator;
assert!(build_grounded_prompt_context(&wrong, 6000).is_err());
+ let mut old = previous();
+ old.passages[0] = "Cafe 🙂".into();
+ assert!(build_followup_prompt_context(&[old], &[], 6000).is_err());
}
#[test]
@@ -474,12 +503,7 @@ mod tests {
}
fn previous() -> ReferencedPassage {
- let fixtures: serde_json::Value = serde_json::from_str(include_str!(
- "../../tests/fixtures/passage-references-v1.json"
- ))
- .unwrap();
- let reference: super::super::IncludedPassage =
- serde_json::from_value(fixtures[0]["reference"].clone()).unwrap();
+ let (reference, raw) = super::super::passage_fixture();
let source = GroundedSource::LocalNote {
reference: NoteSourceReference {
collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),
@@ -491,9 +515,7 @@ mod tests {
},
};
ReferencedPassage {
- passages: reference
- .verified_spans(fixtures[0]["raw"].as_str().unwrap())
- .unwrap(),
+ passages: reference.verified_spans(raw).unwrap(),
reference,
source,
}
@@ -541,12 +563,54 @@ mod tests {
assert!(packed.text.len() <= budget);
assert_eq!(packed.included_passages[0], old.reference);
}
+
+ let raw = super::super::passage_fixture().1;
+ let matching = GroundedExcerpt {
+ locator: old.reference.locator.clone(),
+ source: old.source.clone(),
+ score: 1.0,
+ text: raw.into(),
+ };
+ let expanded = build_followup_prompt_context(
+ std::slice::from_ref(&old),
+ std::slice::from_ref(&matching),
+ 6000,
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(expanded.sources, vec![old.source.clone()]);
+ assert_eq!(expanded.included_passages.len(), 1);
+ assert!(
+ expanded
+ .text
+ .contains(&super::super::clean_passage_text(raw))
+ );
+ assert_eq!(
+ expanded.included_passages[0].verified_spans(raw).unwrap(),
+ vec![super::super::clean_passage_text(raw)]
+ );
+
+ let mut prefix = old.clone();
+ prefix.reference.spans.truncate(1);
+ prefix.passages.truncate(1);
+ let required_prefix =
+ build_followup_prompt_context(std::slice::from_ref(&prefix), &[], 6000)
+ .unwrap()
+ .unwrap();
+ for budget in required_prefix.text.len()..=expanded.text.len() {
+ let packed = build_followup_prompt_context(
+ std::slice::from_ref(&prefix),
+ std::slice::from_ref(&matching),
+ budget,
+ )
+ .unwrap()
+ .unwrap();
+ assert!(packed.text.len() <= budget);
+ assert_eq!(packed.included_passages.len(), 1);
+ let spans = packed.included_passages[0].verified_spans(raw).unwrap();
+ assert!(spans[0].starts_with(&prefix.passages[0]));
+ assert!(packed.text.contains(&spans[0]));
+ }
assert!(build_followup_prompt_context(&[old], &[], required.text.len() - 1).is_err());
}
- #[test]
- fn referenced_text_must_match_the_saved_digest() {
- let mut old = previous();
- old.passages[0] = "Changed text".into();
- assert!(build_followup_prompt_context(&[old], &[], 6000).is_err());
- }
}
diff --git a/rust/crates/ensu/src/retrieval/index.rs b/rust/crates/ensu/src/retrieval/index.rs
index e29b096fdf..8cecc68fc9 100644
--- a/rust/crates/ensu/src/retrieval/index.rs
+++ b/rust/crates/ensu/src/retrieval/index.rs
@@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BinaryHeap};
use std::fs::{self, File};
use std::io::Read;
use std::path::Path;
+use std::sync::OnceLock;
use memmap2::{Mmap, MmapOptions};
use serde::Deserialize;
@@ -56,7 +57,8 @@ struct MetadataRow {
pub struct RetrievalIndex {
dataset_identity: String,
stable_id: String,
- revision_sha256: String,
+ revision_sha256: OnceLock<String>,
+ manifest_bytes: Vec<u8>,
count: usize,
dim: usize,
scale: f32,
@@ -165,6 +167,7 @@ impl RetrievalIndex {
.collect::<Vec<_>>();
validate_offsets(&offsets, metadata_len)?;
+ // Native callers prevent file swaps or deletion while mapped.
#[expect(
unsafe_code,
reason = "File-backed memory mapping requires an unsafe call"
@@ -178,7 +181,8 @@ impl RetrievalIndex {
Ok(Self {
stable_id: expected_pack.stable_id.clone(),
- revision_sha256: text_revision_digest(&manifest_bytes, &metadata, &offsets_raw),
+ revision_sha256: OnceLock::new(),
+ manifest_bytes,
dataset_identity: manifest.dataset,
count,
dim,
@@ -196,6 +200,17 @@ impl RetrievalIndex {
&self.dataset_identity
}
+ fn revision_sha256(&self) -> &str {
+ self.revision_sha256.get_or_init(|| {
+ let offsets = self
+ .offsets
+ .iter()
+ .flat_map(|offset| offset.to_le_bytes())
+ .collect::<Vec<_>>();
+ text_revision_digest(&self.manifest_bytes, &self.metadata, &offsets)
+ })
+ }
+
pub fn search(
&self,
query: &[f32],
@@ -271,13 +286,13 @@ impl RetrievalIndex {
else {
return Ok(None);
};
- if dataset_id != &self.stable_id || revision_sha256 != &self.revision_sha256 {
- return Ok(None);
- }
let Ok(row) = usize::try_from(*row) else {
return Ok(None);
};
- if row >= self.count {
+ if dataset_id != &self.stable_id
+ || row >= self.count
+ || revision_sha256 != self.revision_sha256()
+ {
return Ok(None);
}
Ok(self.load_hits(&[RankedRow { score: 0.0, row }])?.pop())
@@ -337,7 +352,7 @@ impl RetrievalIndex {
hits[rank] = Some(RetrievalHit {
locator: super::PassageLocator::EnsuPack {
dataset_id: self.stable_id.clone(),
- revision_sha256: self.revision_sha256.clone(),
+ revision_sha256: self.revision_sha256().to_owned(),
row: selected.row as u64,
},
score: selected.score,
@@ -542,18 +557,18 @@ fn python_quote(value: &str) -> String {
}
fn text_revision_digest(manifest: &[u8], metadata: &[u8], offsets: &[u8]) -> String {
- use sha2::{Digest, Sha256};
- let mut revision = Sha256::new();
- revision.update(b"ensu-pack-text-revision-v1\0");
- for bytes in [manifest, metadata, offsets] {
- revision.update((bytes.len() as u64).to_le_bytes());
- revision.update(bytes);
- }
- revision
- .finalize()
- .iter()
- .map(|byte| format!("{byte:02x}"))
- .collect()
+ ente_ensu_crypto::sha256_parts(&[
+ b"ensu-pack-text-revision-v1\0",
+ &(manifest.len() as u64).to_le_bytes(),
+ manifest,
+ &(metadata.len() as u64).to_le_bytes(),
+ metadata,
+ &(offsets.len() as u64).to_le_bytes(),
+ offsets,
+ ])
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
}
fn sanitize_prompt_text(value: &str) -> String {
@@ -665,22 +680,25 @@ pub(super) mod tests {
}
#[test]
- fn passage_reload_survives_reopen_and_rejects_changed_text_artifacts() {
+ fn passage_revisions_are_lazy_and_reloads_reject_changed_artifacts() {
use crate::retrieval::PassageLocator;
let pack = synthetic_pack("simplewiki-test");
let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();
+ assert!(index.revision_sha256.get().is_none());
let mut query = vec![0.0; 512];
+ assert!(index.search(&query, 3, 1.0).unwrap().is_empty());
+ assert!(index.revision_sha256.get().is_none());
query[0] = 1.0;
let hits = index.search(&query, 3, 0.0).unwrap();
assert_ne!(hits[0].locator, hits[1].locator);
+ let revision = text_revision_digest(
+ &fs::read(pack.revision.join(KNOWLEDGE_MANIFEST_FILE)).unwrap(),
+ &fs::read(pack.revision.join(KNOWLEDGE_META_FILE)).unwrap(),
+ &fs::read(pack.revision.join(KNOWLEDGE_OFFSETS_FILE)).unwrap(),
+ );
+ assert_eq!(index.revision_sha256.get(), Some(&revision));
drop(index);
let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();
- for hit in &hits {
- let reloaded = index.reload_passage(&hit.locator).unwrap().unwrap();
- assert_eq!(reloaded.locator, hit.locator);
- assert_eq!(reloaded.text, hit.text);
- assert_eq!(reloaded.source_url, hit.source_url);
- }
let mut wrong = hits[0].locator.clone();
if let PassageLocator::EnsuPack { row, .. } = &mut wrong {
*row = u64::MAX;
@@ -691,6 +709,14 @@ pub(super) mod tests {
*dataset_id = "wikibooks".to_string();
}
assert!(index.reload_passage(&wrong).unwrap().is_none());
+ assert!(index.revision_sha256.get().is_none());
+ for hit in &hits {
+ let reloaded = index.reload_passage(&hit.locator).unwrap().unwrap();
+ assert_eq!(reloaded.locator, hit.locator);
+ assert_eq!(reloaded.text, hit.text);
+ assert_eq!(reloaded.source_url, hit.source_url);
+ }
+ assert_eq!(index.revision_sha256.get(), Some(&revision));
drop(index);
let metadata_path = pack.revision.join(KNOWLEDGE_META_FILE);
diff --git a/rust/crates/ensu/src/retrieval/passage.rs b/rust/crates/ensu/src/retrieval/passage.rs
index 4d66d3b9a8..f957cf2e97 100644
--- a/rust/crates/ensu/src/retrieval/passage.rs
+++ b/rust/crates/ensu/src/retrieval/passage.rs
@@ -1,5 +1,4 @@
use serde::{Deserialize, Serialize};
-use sha2::{Digest, Sha256};
use super::{BEGIN_CONTEXT_SENTINEL, END_CONTEXT_SENTINEL, GroundedSource, RetrievalError};
@@ -129,35 +128,22 @@ impl IncludedPassage {
}
pub fn verified_spans(&self, reloaded_text: &str) -> Option<Vec<String>> {
- if self.cleaning_version != PASSAGE_CLEANING_VERSION
- || self.spans.is_empty()
- || self.spans.len() > MAX_PASSAGE_SPANS
- || reloaded_text.len() > MAX_PASSAGE_UTF8_BYTES
- {
+ if !self.valid_metadata() || reloaded_text.len() > MAX_PASSAGE_UTF8_BYTES {
return None;
}
let cleaned = clean_passage_text(reloaded_text);
- let mut end = 0;
- let mut texts = Vec::with_capacity(self.spans.len());
- for span in &self.spans {
- let start = usize::try_from(span.start_utf8).ok()?;
- let next_end = usize::try_from(span.end_utf8).ok()?;
- if start < end || start >= next_end {
- return None;
- }
- let text = cleaned.get(start..next_end)?;
- if digest(text.as_bytes()) != span.text_sha256 {
- return None;
- }
- texts.push(text.to_owned());
- end = next_end;
- }
- Some(texts)
+ self.spans
+ .iter()
+ .map(|span| {
+ let text = cleaned.get(span.start_utf8 as usize..span.end_utf8 as usize)?;
+ (digest(text.as_bytes()) == span.text_sha256).then(|| text.to_owned())
+ })
+ .collect()
}
}
pub(super) fn digest(bytes: &[u8]) -> String {
- Sha256::digest(bytes)
+ ente_ensu_crypto::sha256(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
@@ -194,66 +180,95 @@ pub(crate) fn clean_passage_text(value: &str) -> String {
}
#[cfg(test)]
-mod tests {
+pub(super) mod tests {
use super::*;
- #[derive(Deserialize)]
- struct Fixture {
- name: String,
- raw: String,
- cleaned: String,
- reference: IncludedPassage,
- texts: Vec<String>,
- }
-
- #[test]
- fn shared_passage_reference_fixtures() {
- let fixtures: Vec<Fixture> = serde_json::from_str(include_str!(
- "../../tests/fixtures/passage-references-v1.json"
- ))
+ pub(crate) fn passage_fixture() -> (IncludedPassage, &'static str) {
+ let reference = serde_json::from_value(serde_json::json!({
+ "locator": {
+ "type": "localNote",
+ "collectionId": "123e4567-e89b-12d3-a456-426614174000",
+ "documentId": "trip.md",
+ "indexedRevision": "a".repeat(64),
+ "shardSha256": "b".repeat(64),
+ "chunkIndex": 2
+ },
+ "cleaningVersion": 1,
+ "spans": [
+ {
+ "startUtf8": 0,
+ "endUtf8": 9,
+ "textSha256": "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"
+ },
+ {
+ "startUtf8": 20,
+ "endUtf8": 23,
+ "textSha256": "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"
+ }
+ ]
+ }))
.unwrap();
- for fixture in fixtures {
- assert_eq!(
- clean_passage_text(&fixture.raw),
- fixture.cleaned,
- "{}",
- fixture.name
- );
- assert_eq!(
- fixture.reference.verified_spans(&fixture.raw),
- Some(fixture.texts),
- "{}",
- fixture.name
- );
- }
+ (reference, " \r\nCafé🙂\r\nबीच\tend \0\r\n")
}
#[test]
- fn refuses_changed_text_invalid_offsets_and_unknown_versions() {
- let fixtures: Vec<Fixture> = serde_json::from_str(include_str!(
- "../../tests/fixtures/passage-references-v1.json"
- ))
- .unwrap();
- let fixture = &fixtures[0];
+ fn passage_references_verify_cleaned_text_and_reject_invalid_spans() {
+ let (fixture, raw) = passage_fixture();
+ for (raw, cleaned, spans, texts) in [
+ (
+ raw,
+ "Café🙂\nबीच\tend",
+ fixture.spans.clone(),
+ vec!["Café🙂", "end"],
+ ),
+ (
+ "\r\n----- BEGIN KNOWLEDGE CONTEXT -----\r\nx\r\n",
+ "[source] ----- BEGIN KNOWLEDGE CONTEXT -----\nx",
+ vec![PassageSpan {
+ start_utf8: 0,
+ end_utf8: 43,
+ text_sha256: "73c093471df3c5f2f0e96999d8d5ebc745a803f55d3f36c4523c502c2e055e02"
+ .into(),
+ }],
+ vec!["[source] ----- BEGIN KNOWLEDGE CONTEXT ----"],
+ ),
+ (
+ " a\u{1}b\tc\rd ",
+ "a b\tc\nd",
+ vec![PassageSpan {
+ start_utf8: 0,
+ end_utf8: 7,
+ text_sha256: "1205bffe81cb4550cdf063c07e75e290c2ebaf60d77780a338e18eb99d26bd4c"
+ .into(),
+ }],
+ vec!["a b\tc\nd"],
+ ),
+ ] {
+ let reference = IncludedPassage {
+ spans,
+ ..fixture.clone()
+ };
+ assert_eq!(clean_passage_text(raw), cleaned);
+ assert_eq!(reference.verified_spans(raw).unwrap(), texts);
+ }
assert!(
fixture
- .reference
- .verified_spans(&fixture.raw.replace("Café", "Cafe"))
+ .verified_spans(&raw.replace("Café", "Cafe "))
.is_none()
);
for (start, end) in [(0, 4), (4, 9), (9, 0), (0, u32::MAX)] {
- let mut reference = fixture.reference.clone();
+ let mut reference = fixture.clone();
reference.spans[0].start_utf8 = start;
reference.spans[0].end_utf8 = end;
- assert!(reference.verified_spans(&fixture.raw).is_none());
+ assert!(reference.verified_spans(raw).is_none());
}
- let mut reference = fixture.reference.clone();
+ let mut reference = fixture.clone();
reference.cleaning_version = 2;
- assert!(reference.verified_spans(&fixture.raw).is_none());
- reference = fixture.reference.clone();
+ assert!(reference.verified_spans(raw).is_none());
+ reference = fixture;
reference.spans[1] = reference.spans[0].clone();
- assert!(reference.verified_spans(&fixture.raw).is_none());
+ assert!(reference.verified_spans(raw).is_none());
reference.spans = vec![reference.spans[0].clone(); 9];
- assert!(reference.verified_spans(&fixture.raw).is_none());
+ assert!(reference.verified_spans(raw).is_none());
}
}
diff --git a/rust/crates/ensu/tests/fixtures/passage-references-v1.json b/rust/crates/ensu/tests/fixtures/passage-references-v1.json
deleted file mode 100644
index ad039dbc94..0000000000
--- a/rust/crates/ensu/tests/fixtures/passage-references-v1.json
+++ /dev/null
@@ -1,86 +0,0 @@
-[
- {
- "name": "multilingual-disjoint",
- "raw": " \r\nCafé🙂\r\nबीच\tend \u0000\r\n",
- "cleaned": "Café🙂\nबीच\tend",
- "reference": {
- "locator": {
- "type": "localNote",
- "collectionId": "123e4567-e89b-12d3-a456-426614174000",
- "documentId": "trip.md",
- "indexedRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
- "shardSha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
- "chunkIndex": 2
- },
- "cleaningVersion": 1,
- "spans": [
- {
- "startUtf8": 0,
- "endUtf8": 9,
- "textSha256": "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"
- },
- {
- "startUtf8": 20,
- "endUtf8": 23,
- "textSha256": "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"
- }
- ]
- },
- "texts": [
- "Café🙂",
- "end"
- ]
- },
- {
- "name": "sentinel",
- "raw": "\r\n----- BEGIN KNOWLEDGE CONTEXT -----\r\nx\r\n",
- "cleaned": "[source] ----- BEGIN KNOWLEDGE CONTEXT -----\nx",
- "reference": {
- "locator": {
- "type": "localNote",
- "collectionId": "123e4567-e89b-12d3-a456-426614174000",
- "documentId": "trip.md",
- "indexedRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
- "shardSha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
- "chunkIndex": 2
- },
- "cleaningVersion": 1,
- "spans": [
- {
- "startUtf8": 0,
- "endUtf8": 43,
- "textSha256": "73c093471df3c5f2f0e96999d8d5ebc745a803f55d3f36c4523c502c2e055e02"
- }
- ]
- },
- "texts": [
- "[source] ----- BEGIN KNOWLEDGE CONTEXT ----"
- ]
- },
- {
- "name": "controls",
- "raw": " a\u0001b\tc\rd ",
- "cleaned": "a b\tc\nd",
- "reference": {
- "locator": {
- "type": "localNote",
- "collectionId": "123e4567-e89b-12d3-a456-426614174000",
- "documentId": "trip.md",
- "indexedRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
- "shardSha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
- "chunkIndex": 2
- },
- "cleaningVersion": 1,
- "spans": [
- {
- "startUtf8": 0,
- "endUtf8": 7,
- "textSha256": "1205bffe81cb4550cdf063c07e75e290c2ebaf60d77780a338e18eb99d26bd4c"
- }
- ]
- },
- "texts": [
- "a b\tc\nd"
- ]
- }
-]
diff --git a/web/apps/ensu/src/components/chat/ChatMessageList.tsx b/web/apps/ensu/src/components/chat/ChatMessageList.tsx
index 508c6580c0..3009a31270 100644
--- a/web/apps/ensu/src/components/chat/ChatMessageList.tsx
+++ b/web/apps/ensu/src/components/chat/ChatMessageList.tsx
@@ -18,7 +18,6 @@ import {
Copy01Icon,
Edit01Icon,
RepeatIcon,
- Search01Icon,
} from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import {
@@ -103,8 +102,7 @@ export interface ChatMessageListProps {
) => void;
onEditMessage: (message: ChatMessage) => void;
onCopyMessage: (text: string) => void;
- onRetryMessage: (message: ChatMessage, searchAsWritten?: boolean) => void;
- canSearchSources: boolean;
+ onRetryMessage: (message: ChatMessage) => void;
onPrevBranch: (switcher: BranchSwitcher) => void;
onNextBranch: (switcher: BranchSwitcher) => void;
onRequestPreview: (attachment: ChatAttachment, sessionUuid: string) => void;
@@ -419,8 +417,7 @@ interface MessageRowProps {
) => void;
onEditMessage: (message: ChatMessage) => void;
onCopyMessage: (text: string) => void;
- onRetryMessage: (message: ChatMessage, searchAsWritten?: boolean) => void;
- canSearchSources: boolean;
+ onRetryMessage: (message: ChatMessage) => void;
onPrevBranch: (switcher: BranchSwitcher) => void;
onNextBranch: (switcher: BranchSwitcher) => void;
attachmentPreviews: Record<string, string>;
@@ -454,7 +451,6 @@ const MessageRow = memo(
onEditMessage,
onCopyMessage,
onRetryMessage,
- canSearchSources,
onPrevBranch,
onNextBranch,
attachmentPreviews,
@@ -780,24 +776,6 @@ const MessageRow = memo(
</IconButton>
</>
)}
- {!isStreaming &&
- !isSelf &&
- (!isSynthetic || isLastMessage) &&
- canSearchSources && (
- <IconButton
- aria-label="Search as written"
- title="Search as written"
- sx={actionButtonSx}
- onClick={() =>
- onRetryMessage(message, true)
- }
- >
- <HugeiconsIcon
- icon={Search01Icon}
- {...actionIconProps}
- />
- </IconButton>
- )}
</Stack>
{isStreaming || isSynthetic ? null : (
@@ -959,7 +937,6 @@ export const ChatMessageList = memo(
onEditMessage,
onCopyMessage,
onRetryMessage,
- canSearchSources,
onPrevBranch,
onNextBranch,
onRequestPreview,
@@ -1026,7 +1003,6 @@ export const ChatMessageList = memo(
onEditMessage={onEditMessage}
onCopyMessage={onCopyMessage}
onRetryMessage={onRetryMessage}
- canSearchSources={canSearchSources}
onPrevBranch={onPrevBranch}
onNextBranch={onNextBranch}
attachmentPreviews={attachmentPreviews}
@@ -1053,7 +1029,6 @@ export const ChatMessageList = memo(
assistantMarkdownSx,
attachmentPreviews,
branchSwitchers,
- canSearchSources,
dialogCloseButtonSx,
dialogCloseIconProps,
dialogTitleSx,
diff --git a/web/apps/ensu/src/pages/chat.tsx b/web/apps/ensu/src/pages/chat.tsx
index add8c1c578..cdbf17cfa4 100644
--- a/web/apps/ensu/src/pages/chat.tsx
+++ b/web/apps/ensu/src/pages/chat.tsx
@@ -54,7 +54,6 @@ import {
prepareDesktopConversation,
resolveDesktopSourceFollowup,
type PreparedReply,
- type ResolvedSourceFollowup,
} from "@/services/llm/conversation";
import { stripHiddenPartsText } from "@/services/llm/history-text";
import {
@@ -2850,12 +2849,6 @@ const Page: React.FC = () => {
activeKnowledgeSources,
));
- if (assistantMessage.evidenceSaved === false) {
- log.warn(
- "Reply saved, but its source references could not be retained",
- );
- }
-
await updateBranchSelectionState(
parentMessageUuid,
assistantMessage.messageUuid,
@@ -2940,7 +2933,6 @@ const Page: React.FC = () => {
sessionUuid,
imagePaths,
mediaMarker,
- searchAsWritten = false,
}: {
promptText: string;
parentMessage: ChatMessage;
@@ -2950,7 +2942,6 @@ const Page: React.FC = () => {
sessionUuid?: string;
imagePaths?: string[];
mediaMarker?: string;
- searchAsWritten?: boolean;
}): Promise<void> => {
const parentMessageUuid = parentMessage.messageUuid;
const activeSessionId =
@@ -3106,12 +3097,7 @@ const Page: React.FC = () => {
historyPath,
stopAtMessageUuid,
);
- const selectedPath = selectedMessages.map(
- (message) => message.messageUuid,
- );
- if (selectedPath.at(-1) !== parentMessageUuid)
- selectedPath.push(parentMessageUuid);
- let followup: ResolvedSourceFollowup | undefined;
+ let resolutionToken: string | undefined;
const remainingKnowledgeBytes = useConversationMemory
? 6000
: Math.max(
@@ -3160,12 +3146,16 @@ const Page: React.FC = () => {
if (
useConversationMemory &&
- !searchAsWritten &&
selectedMessages.some((message) => message.sources?.length)
) {
+ const selectedPath = selectedMessages.map(
+ (message) => message.messageUuid,
+ );
+ if (selectedPath.at(-1) !== parentMessageUuid)
+ selectedPath.push(parentMessageUuid);
setConversationStatus("Finding sources");
try {
- followup = await provider.withKnowledgeRetrieval(
+ resolutionToken = await provider.withKnowledgeRetrieval(
(cancellationEpoch) =>
resolveDesktopSourceFollowup({
sessionUuid: activeSessionId,
@@ -3178,11 +3168,6 @@ const Page: React.FC = () => {
isActiveGeneration,
);
if (!isActiveGeneration()) return;
- knowledgeContext = {
- text: "",
- sources: [],
- candidates: followup.candidates,
- };
} catch (error) {
const { name } = tauriCommandError(error);
if (
@@ -3265,10 +3250,14 @@ const Page: React.FC = () => {
),
system: normalSystemPrompt,
current: promptText,
- historyQuery: knowledgeQuery,
+ historyQuery: resolutionToken
+ ? undefined
+ : knowledgeQuery,
maxTokens,
- groundingCandidates: knowledgeContext?.candidates,
- resolutionToken: followup?.token,
+ groundingCandidates: resolutionToken
+ ? undefined
+ : knowledgeContext?.candidates,
+ resolutionToken,
},
provider,
{
@@ -3417,12 +3406,6 @@ const Page: React.FC = () => {
));
if (!isActiveGeneration()) return;
- if (assistantMessage.evidenceSaved === false) {
- log.warn(
- "Reply saved, but its source references could not be retained",
- );
- }
-
void updateBranchSelectionState(
parentMessageUuid,
assistantMessage.messageUuid,
@@ -3506,7 +3489,7 @@ const Page: React.FC = () => {
);
const handleRetryMessage = useCallback(
- async (message: ChatMessage, searchAsWritten = false) => {
+ async (message: ChatMessage) => {
if (message.sender !== "assistant") return;
if (!chatKey || !currentSessionId) return;
if (isDownloading) {
@@ -3549,7 +3532,6 @@ const Page: React.FC = () => {
stopAtMessageUuid: parentUuid,
resetContext: true,
sessionUuid: currentSessionId,
- searchAsWritten,
});
},
[
@@ -4724,19 +4706,6 @@ const Page: React.FC = () => {
onEditMessage={handleEditMessage}
onCopyMessage={handleCopyMessage}
onRetryMessage={handleRetryMessage}
- canSearchSources={
- isTauriRuntime &&
- (notesCollections.length > 0 ||
- knowledgePacks.some(
- (pack) =>
- enabledKnowledgePackIds.has(
- pack.stableId,
- ) &&
- (pack.status === "ready" ||
- pack.status ===
- "updateAvailable"),
- ))
- }
onPrevBranch={handlePrevBranch}
onNextBranch={handleNextBranch}
onRequestPreview={loadAttachmentPreview}
diff --git a/web/apps/ensu/src/services/chat/store.ts b/web/apps/ensu/src/services/chat/store.ts
index a5ee07c269..4b7ae133a2 100644
--- a/web/apps/ensu/src/services/chat/store.ts
+++ b/web/apps/ensu/src/services/chat/store.ts
@@ -116,7 +116,6 @@ export interface ChatMessage {
createdAt: number;
attachments?: ChatAttachment[];
sources?: GroundedSource[];
- evidenceSaved?: boolean;
isSynthetic?: boolean;
}
@@ -144,7 +143,6 @@ interface NativeMessage {
createdAt: number;
attachments?: NativeAttachment[];
sources?: GroundedSource[];
- evidenceSaved?: boolean;
}
const nowMicros = () => Date.now() * 1000;
@@ -430,7 +428,6 @@ const listMessagesNative = async (
size: attachment.size,
})),
sources: message.sources,
- evidenceSaved: message.evidenceSaved,
}));
};
@@ -503,7 +500,6 @@ const addMessageNative = async (
size: attachment.size,
})),
sources: message.sources,
- evidenceSaved: message.evidenceSaved,
};
};
diff --git a/web/apps/ensu/src/services/llm/conversation.ts b/web/apps/ensu/src/services/llm/conversation.ts
index 932c5e38b1..567ceb7c5a 100644
--- a/web/apps/ensu/src/services/llm/conversation.ts
+++ b/web/apps/ensu/src/services/llm/conversation.ts
@@ -86,11 +86,6 @@ export const prepareDesktopConversation = async (
}
};
-export interface ResolvedSourceFollowup {
- token: string;
- candidates: unknown;
-}
-
export const resolveDesktopSourceFollowup = (input: {
sessionUuid: string;
path: string[];
@@ -98,7 +93,7 @@ export const resolveDesktopSourceFollowup = (input: {
enabledStableIds: string[];
cancellationEpoch: number;
candidates?: unknown;
-}): Promise<ResolvedSourceFollowup> =>
- invoke<ResolvedSourceFollowup>("conversation_resolve_followup", {
+}): Promise<string> =>
+ invoke<string>("conversation_resolve_followup", {
input: { ...input, token: crypto.randomUUID() },
});