💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Refine follow-up grounding and simplify tests
setalp2e8fcc66ee91
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex 0ac6a4ff22..46cacb5af9 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -559,7 +559,7 @@ internal class ChatStoreActions(} else null}if (!isActive() || stopRequested) {- turn?.cancel()+ closeFollowup(turn)return@launch}followup = turn@@ -575,14 +575,8 @@ internal class ChatStoreActions(return@launch} catch (error: Throwable) {if (!isActive() || stopRequested) return@launch- logRepository.log(- LogLevel.Warning,- "Unable to prepare source search",- details = error.message,- tag = "Chat",- throwable = error,- )- if (showFollowupError(error, userMessage.id)) return@launch+ if (reportFollowupError(error, userMessage.id, "Unable to prepare source search"))+ return@launch}val retrievalQuery = userMessage.text.trim()val knowledgeHits =@@ -652,17 +646,15 @@ internal class ChatStoreActions(return@launch} catch (error: Throwable) {if (!isActive() || stopRequested) return@launch- logRepository.log(- LogLevel.Warning,- "Unable to resolve earlier sources",- details = error.message,- tag = "Chat",- throwable = error,+ if (+ reportFollowupError(+ error,+ userMessage.id,+ "Unable to resolve earlier sources",+ ))- if (showFollowupError(error, userMessage.id)) return@launch- activeFollowup = null- followup?.cancel()- followup?.destroy()+ return@launch+ closeFollowup(followup)followup = nullemptyList()}@@ -919,15 +911,26 @@ internal class ChatStoreActions(}generationJob = activeJobactiveJob.invokeOnCompletion {- if (activeFollowup === followup) activeFollowup = null- followup?.cancel()- followup?.destroy()+ closeFollowup(followup)notesScope.close()scope.launch { settleGenerationIfActive(generationToken, sessionId) }}}- private fun showFollowupError(error: Throwable, messageId: String): Boolean {+ private fun closeFollowup(followup: ConversationFollowup?) {+ if (activeFollowup === followup) activeFollowup = null+ followup?.cancel()+ followup?.destroy()+ }++ private fun reportFollowupError(error: Throwable, messageId: String, detail: String): Boolean {+ logRepository.log(+ LogLevel.Warning,+ detail,+ details = error.message,+ tag = "Chat",+ throwable = error,+ )val message =when (error) {is ConversationException.Stale -> "Conversation changed. Retry the message."@@ -951,7 +954,7 @@ internal class ChatStoreActions(when (locator) {is PassageLocator.EnsuPack ->knowledgeProvider.reload(locator, state.value.knowledge.enabledReadyDatasets)- is PassageLocator.LocalNote -> notesStore.reload(locator)+ is PassageLocator.LocalNote -> notesStore.reload(locator.locator)}} catch (error: kotlinx.coroutines.CancellationException) {throw errordiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.ktindex e314addb01..ff957e3313 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.kt@@ -7,13 +7,13 @@ import android.provider.DocumentsContractimport android.util.AtomicFileimport androidx.core.content.FileProviderimport io.ente.ensu.bindings.LlmContext+import io.ente.ensu.bindings.NotePassageLocatorimport io.ente.ensu.bindings.NoteSourceReferenceimport io.ente.ensu.bindings.NotesCancellationimport io.ente.ensu.bindings.NotesCollectionimport io.ente.ensu.bindings.NotesExceptionimport io.ente.ensu.bindings.NotesIndexOptionsimport io.ente.ensu.bindings.NotesProgressCallback-import io.ente.ensu.bindings.PassageLocatorimport io.ente.ensu.bindings.notesContentRevisionimport java.io.ByteArrayOutputStreamimport java.io.File@@ -132,7 +132,7 @@ class NotesProvider(private val context: Context) {handle(id).use { it.search(query) }}- internal suspend fun reload(locator: PassageLocator.LocalNote) = withAccess {+ internal suspend fun reload(locator: NotePassageLocator) = withAccess {registration(locator.collectionId)handle(locator.collectionId).use { it.reloadPassage(locator) }}diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.ktindex 46278812be..1af9823723 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt@@ -6,6 +6,7 @@ import android.net.Uriimport androidx.compose.runtime.staticCompositionLocalOfimport io.ente.ensu.bindings.GroundedExcerptimport io.ente.ensu.bindings.GroundedSource+import io.ente.ensu.bindings.NotePassageLocatorimport io.ente.ensu.bindings.NoteSourceReferenceimport io.ente.ensu.bindings.NotesCancellationimport io.ente.ensu.bindings.NotesException@@ -14,7 +15,6 @@ import io.ente.ensu.bindings.NotesIndexOptionsimport io.ente.ensu.bindings.NotesProgressimport io.ente.ensu.bindings.NotesProgressCallbackimport io.ente.ensu.bindings.NotesSummary-import io.ente.ensu.bindings.PassageLocatorimport io.ente.ensu.bindings.notesLimitsimport io.ente.ensu.bindings.selectMixedGroundingCandidatesimport io.ente.ensu.bindings.withNotesCollectionLabel@@ -281,7 +281,7 @@ class NotesStore(hits}- suspend fun reload(locator: PassageLocator.LocalNote): GroundedExcerpt? =+ suspend fun reload(locator: NotePassageLocator): GroundedExcerpt? =withContext(Dispatchers.Main.immediate) {if (_state.value.collections.none { it.id == locator.collectionId && it.eligible })return@withContext nulldiff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex cfae99ff4d..a92d4a64be 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -1172,20 +1172,14 @@ final class ChatViewModel: ObservableObject {do {if let turn = followup {- let searched = knowledgeHits- let locators = try await Task.detached {+ let locators = try await Task.detached { [searched = knowledgeHits] intry turn.searchWithHistory(searched: searched)}.valuetry Task.checkCancellation()guard activeGenerationId == generationId else { return }for locator in locators {- do {- if let hit = try await reloadPassage(locator) {- reloaded.append(hit)- }- } catch {- if isCancellation(error) { throw error }- logger.warning("Source reload failed", details: "\(error)")+ if let hit = try await reloadPassage(locator) {+ reloaded.append(hit)}}}@@ -1510,12 +1504,18 @@ final class ChatViewModel: ObservableObject {}private func reloadPassage(_ locator: PassageLocator) async throws -> GroundedExcerpt? {- switch locator {- case .ensuPack:- return try await knowledgeProvider.reload(- locator, datasets: knowledgeStore.enabledReadyDatasets)- case .localNote:- return try await notesStore.reload(locator)+ do {+ switch locator {+ case .ensuPack:+ return try await knowledgeProvider.reload(+ locator, datasets: knowledgeStore.enabledReadyDatasets)+ case .localNote(let note):+ return try await notesStore.reload(note)+ }+ } catch {+ if isCancellation(error) { throw error }+ logger.warning("Source reload failed", details: "\(error)")+ return nil}}diff --git a/apple/apps/ensu/Ensu/Services/NotesProvider.swift b/apple/apps/ensu/Ensu/Services/NotesProvider.swiftindex d21040809e..7efeba5af5 100644--- a/apple/apps/ensu/Ensu/Services/NotesProvider.swift+++ b/apple/apps/ensu/Ensu/Services/NotesProvider.swift@@ -162,13 +162,12 @@ actor NotesProvider {}}- func reload(_ locator: PassageLocator) async throws -> NotesHit? {- guard case .localNote(let collectionId, _, _, _, _) = locator else { return nil }- return try await diskGate.withLock {- _ = try self.registration(collectionId)+ func reload(_ locator: NotePassageLocator) async throws -> NotesHit? {+ try await diskGate.withLock {+ _ = try self.registration(locator.collectionId)let root = self.rootreturn try await Task.detached {- try Self.handle(root, collectionId).reloadPassage(locator: locator)+ try Self.handle(root, locator.collectionId).reloadPassage(locator: locator)}.value}}diff --git a/apple/apps/ensu/Ensu/Services/NotesStore.swift b/apple/apps/ensu/Ensu/Services/NotesStore.swiftindex 23546ed3e1..a95ebd4c47 100644--- a/apple/apps/ensu/Ensu/Services/NotesStore.swift+++ b/apple/apps/ensu/Ensu/Services/NotesStore.swift@@ -198,9 +198,8 @@ final class NotesStore: ObservableObject, ModelMaintenance {return hits}- func reload(_ locator: PassageLocator) async throws -> GroundedExcerpt? {- guard case .localNote(let collectionId, _, _, _, _) = locator,- collections.contains(where: { $0.id == collectionId && $0.eligible })+ func reload(_ locator: NotePassageLocator) async throws -> GroundedExcerpt? {+ guard collections.contains(where: { $0.id == locator.collectionId && $0.eligible })else { return nil }do {guard let hit = try await provider.reload(locator) else { return nil }@@ -208,7 +207,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {selectMixedGroundingCandidates(packHits: [], notesHits: [hit], notesLimit: 1)).first} catch NotesError.RebuildRequired {- requestRebuild(collectionId)+ requestRebuild(locator.collectionId)return nil}}diff --git a/rust/apps/ensu/src-tauri/src/commands/followup.rs b/rust/apps/ensu/src-tauri/src/commands/followup.rsindex 4e6967c6e7..15b3473521 100644--- a/rust/apps/ensu/src-tauri/src/commands/followup.rs+++ b/rust/apps/ensu/src-tauri/src/commands/followup.rs@@ -236,6 +236,7 @@ mod tests {.unwrap();(db, snapshot)}+#[test]fn preparation_requires_matching_source_selection_and_consumes_it_once() {let (db, snapshot) = fixture();@@ -245,39 +246,34 @@ mod tests {Guard {token: "token".into(),db: db.clone(),- selection: None,+ selection: Some(Selection {+ fingerprint: core::fingerprint(snapshot.messages()),+ question: "Where does she track it?".into(),+ candidates: core::GroundingCandidates::new(+ vec![],+ vec![],+ core::MAX_GROUNDING_BYTES,+ )+ .unwrap(),+ }),},);- assert!(- state- .take_selection("window", "token", &db, &snapshot)- .is_err()- );- state- .guards- .lock()- .unwrap()- .get_mut("window")- .unwrap()- .selection = Some(Selection {- fingerprint: core::fingerprint(snapshot.messages()),- question: "Where does she track it?".into(),- candidates: core::GroundingCandidates::new(vec![], vec![], 6000).unwrap(),- });let (other_db, other_snapshot) = fixture();- for (window, token, db, snapshot) in [- ("another-window", "token", &db, &snapshot),- ("window", "another-token", &db, &snapshot),- ("window", "token", &other_db, &snapshot),- ("window", "token", &db, &other_snapshot),+ for (case, window, token, db, snapshot) in [+ ("window", "another-window", "token", &db, &snapshot),+ ("token", "window", "another-token", &db, &snapshot),+ ("database", "window", "token", &other_db, &snapshot),+ ("history", "window", "token", &db, &other_snapshot),] {- assert!(state.take_selection(window, token, db, snapshot).is_err());+ assert!(+ state.take_selection(window, token, db, snapshot).is_err(),+ "{case}"+ );}let selection = state.take_selection("window", "token", &db, &snapshot).unwrap();assert_eq!(selection.question, "Where does she track it?");- assert_eq!(selection.candidates.max_utf8_bytes, 6000);assert!(state.take_selection("window", "token", &db, &snapshot)diff --git a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs b/rust/apps/ensu/src-tauri/src/commands/knowledge.rsindex cbacb1a35b..e62a7e25b1 100644--- a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs+++ b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs@@ -268,6 +268,37 @@ fn select_verified_mixed_grounding(Ok(verified)}+fn select_grounding(+ app: &AppHandle,+ pack_hits: &[retrieval::KnowledgePromptHit],+ note_hits: &[ente_ensu::notes::NotesSearchHit],+ notes: Option<&super::notes::RetrievalHandle>,+ mut check_cancelled: impl FnMut() -> Result<(), ApiError>,+) -> Result<Vec<retrieval::GroundedExcerpt>, ApiError> {+ let mut excerpts = select_verified_mixed_grounding(pack_hits, note_hits, |reference| {+ check_cancelled()?;+ let Some(notes) = notes else { return Ok(false) };+ let valid = notes.verify_source_reference(reference);+ if !valid {+ super::notes::mark_reference_stale(+ app,+ &reference.collection_id,+ reference.document_id.clone(),+ );+ }+ Ok(valid)+ })?;+ for excerpt in &mut excerpts {+ if let (Some(notes), retrieval::GroundedSource::LocalNote { reference }) =+ (notes, &mut excerpt.source)+ {+ reference.collection_label = notes.collection_label(&reference.collection_id);+ }+ }+ check_cancelled()?;+ Ok(excerpts)+}+fn reconcile_and_open(store: &AssetStore,indexes: &Mutex<HashMap<String, OpenIndex>>,@@ -637,29 +668,7 @@ pub async fn knowledge_retrieve(check_cancelled()?;let context_budget = max_context_utf8_bytes.min(embedding.max_context_utf8_bytes);- let mut excerpts = select_verified_mixed_grounding(&pack_hits, ¬e_hits, |reference| {- check_cancelled()?;- let Some(notes) = ¬es else {- return Ok(false);- };- let verified = notes.verify_source_reference(reference);- if !verified {- crate::commands::notes::mark_reference_stale(- &app,- &reference.collection_id,- reference.document_id.clone(),- );- }- Ok(verified)- })?;- for excerpt in &mut excerpts {- if let (Some(notes), retrieval::GroundedSource::LocalNote { reference }) =- (¬es, &mut excerpt.source)- {- reference.collection_label = notes.collection_label(&reference.collection_id);- }- }- check_cancelled()?;+ let excerpts = select_grounding(&app, &pack_hits, ¬e_hits, notes.as_ref(), check_cancelled)?;let candidates = ente_ensu::conversation::GroundingCandidates::new(vec![], excerpts, context_budget as usize).map_err(|error| ApiError::new("conversation", error.to_string()))?;@@ -727,12 +736,13 @@ pub(crate) async fn reload_for_followup(hit,});}- retrieval::PassageLocator::LocalNote { collection_id, .. } => {+ retrieval::PassageLocator::LocalNote(locator) => {+ let collection_id = &locator.collection_id;if !collection_ids.contains(collection_id) {return Ok(None);}let Some(notes) = ¬es else { return Ok(None) };- match notes.reload_passage(&reference.locator) {+ match notes.reload_passage(locator) {Ok(Some(hit)) => note_hits.push(hit),Ok(None) => return Ok(None),Err(error) => {@@ -747,31 +757,14 @@ pub(crate) async fn reload_for_followup(}}check_cancelled()?;- let mut selected = select_verified_mixed_grounding(&packs, ¬e_hits, |source| {- check_cancelled()?;- let valid = notes- .as_ref()- .is_some_and(|notes| notes.verify_source_reference(source));- if !valid {- crate::commands::notes::mark_reference_stale(- &app,- &source.collection_id,- source.document_id.clone(),- );- }- Ok(valid)- })?;- let Some(mut excerpt) = selected.pop() else {+ let Some(excerpt) =+ select_grounding(&app, &packs, ¬e_hits, notes.as_ref(), check_cancelled)?.pop()+ else {return Ok(None);};let Some(passages) = reference.verified_spans(&excerpt.text) else {return Ok(None);};- if let (Some(notes), retrieval::GroundedSource::LocalNote { reference }) =- (¬es, &mut excerpt.source)- {- reference.collection_label = notes.collection_label(&reference.collection_id);- }check_cancelled()?;Ok(Some(retrieval::ReferencedPassage {reference,@@ -811,7 +804,7 @@ mod tests {fn note(index: usize) -> NotesSearchHit {NotesSearchHit {- locator: retrieval::PassageLocator::LocalNote {+ locator: ente_ensu::notes::NotePassageLocator {collection_id: COLLECTION_ID.to_owned(),document_id: format!("note-{index}.md"),indexed_revision: "a".repeat(64),diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rsindex 57fc7bfb5a..5985860ab9 100644--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs@@ -682,12 +682,11 @@ impl RetrievalHandle {pub(crate) fn reload_passage(&self,- locator: &ente_ensu::retrieval::PassageLocator,+ locator: &ente_ensu::notes::NotePassageLocator,) -> Result<Option<ente_ensu::notes::NotesSearchHit>, NotesError> {- let ente_ensu::retrieval::PassageLocator::LocalNote { collection_id, .. } = locator else {- return Ok(None);- };- self.with_index(collection_id, |index| index.reload_passage(locator))+ self.with_index(&locator.collection_id, |index| {+ index.reload_passage(locator)+ })}fn with_index<T>(diff --git a/rust/bindings/uniffi/ensu/src/notes.rs b/rust/bindings/uniffi/ensu/src/notes.rsindex cec73c9013..fdaa40d34e 100644--- a/rust/bindings/uniffi/ensu/src/notes.rs+++ b/rust/bindings/uniffi/ensu/src/notes.rs@@ -177,9 +177,42 @@ pub struct NotesOutcome {pub unchecked_document_ids: Vec<String>,}+#[derive(Clone, Debug, uniffi::Record)]+pub struct NotePassageLocator {+ pub collection_id: String,+ pub document_id: String,+ pub indexed_revision: String,+ pub shard_sha256: String,+ pub chunk_index: u64,+}++impl From<core::NotePassageLocator> for NotePassageLocator {+ fn from(value: core::NotePassageLocator) -> Self {+ Self {+ collection_id: value.collection_id,+ document_id: value.document_id,+ indexed_revision: value.indexed_revision,+ shard_sha256: value.shard_sha256,+ chunk_index: value.chunk_index,+ }+ }+}++impl From<NotePassageLocator> for core::NotePassageLocator {+ fn from(value: NotePassageLocator) -> Self {+ Self {+ collection_id: value.collection_id,+ document_id: value.document_id,+ indexed_revision: value.indexed_revision,+ shard_sha256: value.shard_sha256,+ chunk_index: value.chunk_index,+ }+ }+}+#[derive(Clone, Debug, uniffi::Record)]pub struct NotesHit {- pub locator: crate::retrieval::PassageLocator,+ pub locator: NotePassageLocator,pub collection_id: String,pub document_id: String,pub revision: String,@@ -351,7 +384,7 @@ impl NotesCollection {pub fn reload_passage(&self,- locator: crate::retrieval::PassageLocator,+ locator: NotePassageLocator,) -> Result<Option<NotesHit>, NotesError> {core::NotesCollectionIndex::open(&self.root, self.id.clone()).and_then(|index| index.reload_passage(&locator.into()))diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rsindex e737ac970e..1c26e6ce55 100644--- a/rust/bindings/uniffi/ensu/src/retrieval.rs+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs@@ -35,11 +35,7 @@ pub enum PassageLocator {row: u64,},LocalNote {- collection_id: String,- document_id: String,- indexed_revision: String,- shard_sha256: String,- chunk_index: u64,+ locator: crate::notes::NotePassageLocator,},}@@ -55,18 +51,8 @@ impl From<core::PassageLocator> for PassageLocator {revision_sha256,row,},- core::PassageLocator::LocalNote {- collection_id,- document_id,- indexed_revision,- shard_sha256,- chunk_index,- } => Self::LocalNote {- collection_id,- document_id,- indexed_revision,- shard_sha256,- chunk_index,+ core::PassageLocator::LocalNote(locator) => Self::LocalNote {+ locator: locator.into(),},}}@@ -84,19 +70,7 @@ impl From<PassageLocator> for core::PassageLocator {revision_sha256,row,},- PassageLocator::LocalNote {- collection_id,- document_id,- indexed_revision,- shard_sha256,- chunk_index,- } => Self::LocalNote {- collection_id,- document_id,- indexed_revision,- shard_sha256,- chunk_index,- },+ PassageLocator::LocalNote { locator } => Self::LocalNote(locator.into()),}}}diff --git a/rust/crates/ensu/src/conversation/followup.rs b/rust/crates/ensu/src/conversation/followup.rsindex b98e304c6f..df85e11cd8 100644--- a/rust/crates/ensu/src/conversation/followup.rs+++ b/rust/crates/ensu/src/conversation/followup.rs@@ -2,36 +2,153 @@ use super::{AnswerEvidence, valid_answer_passages};use crate::db::Message;use crate::retrieval::IncludedPassage;-fn refers_back(question: &str) -> bool {- question.split(|c: char| !c.is_alphanumeric()).any(|word| {- matches!(- word.to_lowercase().as_str(),- "it" | "its"- | "this"- | "that"- | "these"- | "those"- | "my"- | "mine"- | "our"- | "ours"- | "they"- | "them"- | "their"- | "theirs"- | "she"- | "her"- | "hers"- | "he"- | "him"- | "his"- | "former"- | "latter"- | "above"- | "previous"- | "earlier"- )- })+pub(super) fn refers_back(question: &str) -> bool {+ let mut words = question+ .split(|c: char| !c.is_alphanumeric())+ .filter(|word| !word.is_empty());+ while let Some(word) = words.next() {+ let word = word.to_lowercase();+ // A time expression is not a source antecedent. Keep checking the rest+ // of the question: "this year, what does it cover?" still refers back.+ if matches!(+ word.as_str(),+ "this" | "that" | "these" | "those" | "previous" | "earlier"+ ) && starts_temporal_phrase(words.clone())+ {+ continue;+ }+ if [+ "it", "its", "this", "that", "these", "those", "mine", "ours", "they", "them", "their",+ "theirs", "she", "her", "hers", "he", "him", "his", "former", "latter", "above",+ "previous", "earlier",+ ]+ .contains(&word.as_str())+ {+ return true;+ }+ }+ false+}++fn starts_temporal_phrase<'a>(words: impl Iterator<Item = &'a str>) -> bool {+ // Bound lookahead, including phrases like "earlier in the previous fiscal+ // year" and "these past few months", without scanning unrelated clauses.+ for word in words.take(8) {+ let word = word.to_lowercase();+ if [+ "in",+ "the",+ "this",+ "that",+ "these",+ "those",+ "past",+ "last",+ "next",+ "previous",+ "earlier",+ "early",+ "late",+ "coming",+ "current",+ "calendar",+ "fiscal",+ "financial",+ "academic",+ "school",+ "few",+ "several",+ "one",+ "two",+ "three",+ "four",+ "first",+ "second",+ "third",+ "fourth",+ ]+ .contains(&word.as_str())+ || word.bytes().all(|byte| byte.is_ascii_digit())+ {+ continue;+ }+ // Bare "second" can be an ordinal ("that second policy"), and "may"+ // can be a verb ("that may differ"). Neither establishes a time phrase.+ return matches!(word.as_str(), "seconds" | "centuries" | "millennia")+ || [+ "minute",+ "hour",+ "day",+ "week",+ "fortnight",+ "month",+ "quarter",+ "year",+ "decade",+ "century",+ "millennium",+ "morning",+ "afternoon",+ "evening",+ "night",+ "weekend",+ "today",+ "tonight",+ "yesterday",+ "tomorrow",+ "spring",+ "summer",+ "autumn",+ "fall",+ "winter",+ "monday",+ "tuesday",+ "wednesday",+ "thursday",+ "friday",+ "saturday",+ "sunday",+ "january",+ "february",+ "march",+ "april",+ "june",+ "july",+ "august",+ "september",+ "october",+ "november",+ "december",+ ]+ .contains(&word.strip_suffix('s').unwrap_or(&word));+ }+ false+}++fn has_ownership_hint(question: &str) -> bool {+ question+ .split(|c: char| !c.is_alphanumeric())+ .any(|word| matches!(word.to_lowercase().as_str(), "my" | "our"))+}++pub(super) fn compares_owned_source(question: &str) -> bool {+ has_ownership_hint(question)+ && question.split(|c: char| !c.is_alphanumeric()).any(|word| {+ matches!(+ word.to_lowercase().as_str(),+ "compare"+ | "compared"+ | "comparing"+ | "comparison"+ | "versus"+ | "vs"+ | "differ"+ | "differs"+ | "difference"+ | "differences"+ | "than"+ )+ })}pub fn direct_followup_references(@@ -56,13 +173,15 @@ pub fn direct_followup_references(question,prior.iter().chain(searched.iter().map(|hit| &hit.source)),);- if !refers_back(question)+ // Ownership can still refer to an earlier source ("compare Juniper with my+ // policy"). Keep it eligible for reload; fitting determines whether the+ // query is a comparison that needs historical priority.+ let may_refer_back = refers_back(question) || has_ownership_hint(question);+ if !may_refer_back&& !named.is_empty()- && named.iter().all(|key| {- searched- .iter()- .any(|hit| super::grounding::source_key(&hit.source) == *key)- })+ && named+ .iter()+ .all(|key| searched.iter().any(|hit| hit.source.document_key() == *key)){Vec::new()} else {diff --git a/rust/crates/ensu/src/conversation/grounding.rs b/rust/crates/ensu/src/conversation/grounding.rsindex 11695c5c69..6e3c17a2c5 100644--- a/rust/crates/ensu/src/conversation/grounding.rs+++ b/rust/crates/ensu/src/conversation/grounding.rs@@ -1,5 +1,6 @@use super::*;use crate::retrieval::{self, GroundedExcerpt, GroundedPromptContext, ReferencedPassage};+use std::collections::{BTreeMap, BTreeSet};pub const MAX_GROUNDING_BYTES: usize = 6000;pub const MAX_CANDIDATE_BYTES: usize = 64 * 1024;@@ -12,24 +13,10 @@ pub struct GroundingCandidates {pub max_utf8_bytes: usize,}-pub(super) fn source_key(source: &retrieval::GroundedSource) -> (u8, String, String) {- match source {- retrieval::GroundedSource::LocalNote { reference } => (- 0,- reference.collection_id.clone(),- reference.document_id.clone(),- ),- retrieval::GroundedSource::EnsuPack { citation } => {- (1, citation.dataset_id.clone(), citation.source_url.clone())- }- }-}-pub(super) fn named_source_keys<'a>(query: &str,sources: impl Iterator<Item = &'a retrieval::GroundedSource>,-) -> std::collections::BTreeSet<(u8, String, String)> {- use std::collections::{BTreeMap, BTreeSet};+) -> BTreeSet<(u8, &'a str, &'a str)> {let words = |text: &str| -> BTreeSet<String> {text.split(|c: char| !c.is_alphanumeric()).filter(|s| s.chars().count() >= 3)@@ -56,7 +43,7 @@ pub(super) fn named_source_keys<'a>(let title = tokens(title);for (start, window) in query_tokens.windows(title.len()).enumerate() {if window == title {- title_matches.push((source_key(source), start, start + title.len()));+ title_matches.push((source.document_key(), start, start + title.len()));}}}@@ -64,7 +51,7 @@ pub(super) fn named_source_keys<'a>(label_words.extend(words(&reference.document_id));}documents- .entry(source_key(source))+ .entry(source.document_key()).or_default().extend(label_words);}@@ -88,88 +75,302 @@ pub(super) fn named_source_keys<'a>(== 1})})- .map(|(key, _)| key.clone())+ .map(|(key, _)| *key).collect()}+enum PassageSlot {+ Historical {+ minimum: Box<ReferencedPassage>,+ current: Option<GroundedExcerpt>,+ },+ Current(GroundedExcerpt),+}++impl PassageSlot {+ fn locator(&self) -> &retrieval::PassageLocator {+ match self {+ Self::Historical { minimum, .. } => &minimum.reference.locator,+ Self::Current(current) => ¤t.locator,+ }+ }++ fn source(&self) -> &retrieval::GroundedSource {+ match self {+ Self::Historical { minimum, .. } => &minimum.source,+ Self::Current(current) => ¤t.source,+ }+ }++ fn minimum(&self) -> Option<&ReferencedPassage> {+ match self {+ Self::Historical { minimum, .. } => Some(minimum),+ Self::Current(_) => None,+ }+ }++ fn current(&self) -> Option<&GroundedExcerpt> {+ match self {+ Self::Historical { current, .. } => current.as_ref(),+ Self::Current(current) => Some(current),+ }+ }++ fn minimum_coverage(&self) -> Result<ReferencedPassage, PrepareError> {+ let current = match self {+ Self::Historical {+ minimum,+ current: None,+ } => return Ok(minimum.as_ref().clone()),+ Self::Historical {+ current: Some(current),+ ..+ }+ | Self::Current(current) => current,+ };+ let text = retrieval::clean_passage_text(¤t.text);+ let end = if let Some(old) = self.minimum() {+ let end = old+ .reference+ .spans+ .last()+ .ok_or_else(|| {+ PrepareError::Backend("Historical source has no verified spans".into())+ })?+ .end_utf8 as usize;+ if end >= old.passages.join("\n[omitted source text]\n").len() {+ return Ok(old.clone());+ }+ end+ } else {+ text.chars()+ .next()+ .ok_or_else(|| PrepareError::Backend("Current source has no usable text".into()))?+ .len_utf8()+ };+ let passage = text[..end].to_owned();+ let reference = retrieval::IncludedPassage::prefix(current.locator.clone(), &passage)+ .map_err(retrieval_error)?;+ Ok(ReferencedPassage {+ reference,+ source: self.source().clone(),+ passages: vec![passage],+ })+ }+}++pub(super) struct GroundingPlan {+ slots: Vec<PassageSlot>,+ incidental: Vec<ReferencedPassage>,+ searched: Vec<GroundedExcerpt>,+ max_utf8_bytes: usize,+}++impl GroundingPlan {+ fn freeze(+ &self,+ context: Option<&GroundedPromptContext>,+ ) -> Result<Vec<ReferencedPassage>, PrepareError> {+ let Some(context) = context else {+ return Ok(Vec::new());+ };+ context+ .included_passages+ .iter()+ .zip(&context.sources)+ .map(|(reference, source)| {+ let passages = self+ .searched+ .iter()+ .filter(|hit| {+ hit.locator == reference.locator && hit.source.same_document(source)+ })+ .find_map(|hit| reference.verified_spans(&hit.text))+ .or_else(|| {+ self.slots+ .iter()+ .filter_map(|slot| slot.minimum())+ .chain(&self.incidental)+ .find(|old| {+ old.reference == *reference && old.source.same_document(source)+ })+ .map(|old| old.passages.clone())+ })+ .ok_or_else(|| {+ PrepareError::Backend("Fitted source coverage could not be verified".into())+ })?;+ Ok(ReferencedPassage {+ reference: reference.clone(),+ source: source.clone(),+ passages,+ })+ })+ .collect()+ }+}++struct SourceGroup<'a> {+ key: (u8, &'a str, &'a str),+ slots: Vec<PassageSlot>,+}++fn source_group<'a, 'b>(+ groups: &'a mut Vec<SourceGroup<'b>>,+ source: &'b retrieval::GroundedSource,+) -> &'a mut SourceGroup<'b> {+ let key = source.document_key();+ let index = groups+ .iter()+ .position(|group| group.key == key)+ .unwrap_or_else(|| {+ groups.push(SourceGroup {+ key,+ slots: Vec::new(),+ });+ groups.len() - 1+ });+ &mut groups[index]+}++fn retrieval_error(error: retrieval::RetrievalError) -> PrepareError {+ PrepareError::Backend(error.to_string())+}+impl GroundingCandidates {- pub(super) fn protect_named_sources(&mut self, query: &str) -> Result<(), PrepareError> {+ pub(super) fn select_sources(+ &self,+ query: Option<&str>,+ ) -> Result<GroundingPlan, PrepareError> {self.validate()?;- if query.len() > super::lookup::LOOKUP_MAX_QUERY_BYTES {- return Ok(());- }+ let query = query.filter(|query| query.len() <= super::lookup::LOOKUP_MAX_QUERY_BYTES);let named = named_source_keys(- query,+ query.unwrap_or_default(),self.referenced.iter().map(|r| &r.source).chain(self.searched.iter().map(|r| &r.source)),);- let fresh_index = if !self.referenced.is_empty() && named.is_empty() {- self.searched.iter().position(|hit| {- !hit.text.trim().is_empty()- && !self- .referenced- .iter()- .any(|r| r.reference.locator == hit.locator)- })- } else {- None- };- if let Some(index) = fresh_index {- self.searched[..=index].rotate_right(1);- }- let mut required: std::collections::BTreeSet<_> = self+ let historical: BTreeSet<_> = self.referenced.iter()- .map(|r| source_key(&r.source))+ .map(|old| old.source.document_key()).collect();- let mut promoted = Vec::new();- for (index, hit) in self.searched.iter().enumerate() {- let fresh = fresh_index.is_some() && index == 0;- let id = source_key(&hit.source);- if !fresh && (!named.contains(&id) || required.contains(&id)) {- continue;+ let compares_owned_source = query.is_some_and(super::followup::compares_owned_source)+ && self+ .searched+ .iter()+ .any(|hit| named.contains(&hit.source.document_key()));+ let refers_back = !historical.is_empty()+ && (query.is_some_and(super::followup::refers_back)+ || compares_owned_source+ || self+ .searched+ .iter()+ .all(|hit| retrieval::clean_passage_text(&hit.text).is_empty()));+ let mut searched = self.searched.clone();+ searched.sort_by_key(|hit| {+ self.referenced+ .iter()+ .any(|old| old.reference.locator == hit.locator)+ });+ let mut groups: Vec<SourceGroup> = Vec::new();+ for old in &self.referenced {+ let group = source_group(&mut groups, &old.source);+ if !group+ .slots+ .iter()+ .any(|slot| slot.locator() == &old.reference.locator)+ {+ group.slots.push(PassageSlot::Historical {+ minimum: Box::new(old.clone()),+ current: None,+ });}- let budgets = if fresh {- [self.max_utf8_bytes / 2, self.max_utf8_bytes]- } else {- [usize::MAX; 2]- };- let Some(reference) = budgets.into_iter().find_map(|budget| {- retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), budget)- .ok()- .flatten()?- .included_passages- .into_iter()- .next()- }) else {- continue;- };- let Some(passages) = reference.verified_spans(&hit.text) else {+ }+ for hit in &searched {+ if let Some(slot) = groups+ .iter_mut()+ .flat_map(|group| &mut group.slots)+ .find(|slot| slot.locator() == &hit.locator)+ {+ if let PassageSlot::Historical { minimum, current } = slot+ && current.is_none()+ && minimum.source.same_document(&hit.source)+ && minimum.reference.verified_spans(&hit.text).is_some()+ {+ *current = Some(hit.clone());+ }continue;- };- required.insert(id);- promoted.push(ReferencedPassage {- reference,- source: hit.source.clone(),- passages,+ }+ let key = hit.source.document_key();+ let selected = query.is_some()+ && (named.contains(&key)+ || (!historical.is_empty()+ && (named.is_empty() || groups.iter().any(|group| group.key == key))));+ if selected && !retrieval::clean_passage_text(&hit.text).is_empty() {+ source_group(&mut groups, &hit.source)+ .slots+ .push(PassageSlot::Current(hit.clone()));+ }+ }+ if query.is_some() {+ for group in &mut groups {+ group.slots.sort_by_key(|slot| {+ searched+ .iter()+ .position(|hit| &hit.locator == slot.locator())+ .unwrap_or(usize::MAX)+ });+ }+ let antecedent = self+ .referenced+ .iter()+ .find(|old| named.contains(&old.source.document_key()))+ .or_else(|| self.referenced.first())+ .map(|old| old.source.document_key());+ groups.sort_by_key(|group| {+ let current = searched+ .iter()+ .position(|hit| hit.source.document_key() == group.key);+ if named.contains(&group.key) {+ (0, 0)+ } else if refers_back && antecedent.as_ref() == Some(&group.key) {+ (1, 0)+ } else if refers_back && historical.contains(&group.key) {+ current.map_or((3, 0), |index| (2, index))+ } else if named.is_empty()+ && let Some(index) = current+ {+ (4, index)+ } else {+ (5, 0)+ }});}-- let (mut named_references, incidental): (Vec<_>, Vec<_>) =- std::mem::take(&mut self.referenced)- .into_iter()- .partition(|r| named.contains(&source_key(&r.source)));- named_references.extend(promoted);- let mut seen = std::collections::BTreeSet::new();- let (distinct, additional): (Vec<_>, Vec<_>) = named_references- .into_iter()- .partition(|r| seen.insert(source_key(&r.source)));+ let mut prioritized = Vec::new();+ for (group_index, group) in groups.into_iter().enumerate() {+ let primary = query.is_none()+ || named.contains(&group.key)+ || (refers_back && historical.contains(&group.key));+ for (slot_index, slot) in group.slots.into_iter().enumerate() {+ let primary =+ primary || (!refers_back && named.is_empty() && slot.current().is_some());+ prioritized.push(((!primary, slot_index > 0, group_index), slot));+ }+ }+ prioritized.sort_by_key(|(priority, _)| *priority);+ searched.sort_by_key(|hit| {+ prioritized+ .iter()+ .position(|(_, slot)| slot.locator() == &hit.locator)+ .unwrap_or(usize::MAX)+ });let mut packs = 0;let mut notes = 0;- for reference in distinct.into_iter().chain(additional).chain(incidental) {- let (count, limit) = match &reference.source {+ let mut slots = Vec::new();+ let mut incidental = Vec::new();+ for ((is_incidental, _, _), slot) in prioritized {+ let (count, limit) = match slot.source() {retrieval::GroundedSource::EnsuPack { .. } => {(&mut packs, retrieval::MAX_PACK_HITS)}@@ -180,18 +381,23 @@ impl GroundingCandidates {if *count == limit {continue;}- self.referenced.push(reference);- if self.validate().is_err() {- self.referenced.pop();- } else {- *count += 1;+ *count += 1;+ if !is_incidental {+ slots.push(slot);+ } else if let PassageSlot::Historical { minimum, .. } = slot {+ incidental.push(*minimum);}}- Ok(())+ Ok(GroundingPlan {+ slots,+ incidental,+ searched,+ max_utf8_bytes: self.max_utf8_bytes,+ })}pub fn new(- mut referenced: Vec<ReferencedPassage>,+ referenced: Vec<ReferencedPassage>,mut searched: Vec<GroundedExcerpt>,max_utf8_bytes: usize,) -> Result<Self, PrepareError> {@@ -203,16 +409,6 @@ impl GroundingCandidates {}hit.text.truncate(end);}- referenced.truncate(retrieval::MAX_GROUNDING_HITS);- while referenced- .iter()- .flat_map(|r| &r.passages)- .map(String::len)- .sum::<usize>()- > MAX_GROUNDING_BYTES- {- referenced.pop();- }let result = Self {referenced,searched,@@ -241,9 +437,7 @@ impl GroundingCandidates {.referenced.iter().flat_map(|r| &r.passages)- .map(String::len)- .sum::<usize>()- > MAX_GROUNDING_BYTES+ .any(|passage| passage.len() > MAX_GROUNDING_BYTES)|| serde_json::to_vec(self).map_err(|e| PrepareError::Backend(e.to_string()))?.len()@@ -253,6 +447,12 @@ impl GroundingCandidates {"Source candidates exceed the supported preparation limits".into(),));}+ retrieval::build_followup_prompt_context(&self.referenced, &[], usize::MAX)+ .map_err(retrieval_error)?;+ for hit in &self.searched {+ retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), usize::MAX)+ .map_err(retrieval_error)?;+ }Ok(())}@@ -262,7 +462,7 @@ impl GroundingCandidates {&self.searched,budget.min(self.max_utf8_bytes),)- .map_err(|e| PrepareError::Backend(e.to_string()))+ .map_err(retrieval_error)}}@@ -281,7 +481,7 @@ pub(super) fn grounded_system(system: &str, context: Option<&GroundedPromptConte}pub(super) fn fit_grounding_with_history(- candidates: &GroundingCandidates,+ candidates: &GroundingPlan,history: Option<&HistoryLookup>,system: &str,current: &str,@@ -308,13 +508,12 @@ pub(super) fn fit_grounding_with_history(}fn fit_grounding(- candidates: &GroundingCandidates,+ candidates: &GroundingPlan,system: &str,current: &str,input_budget: usize,mut measure: impl FnMut(&[ChatMessage]) -> Result<usize, PrepareError>,) -> Result<FittedGrounding, PrepareError> {- candidates.validate()?;let base = measure(&answer_messages(system, None, &[], current))?;let room = input_budget.checked_sub(base)@@ -327,71 +526,130 @@ fn fit_grounding(current,))};- let original_references = candidates.referenced.len();- let mut candidates = candidates.clone();- let (initial, required, required_tokens) = loop {- let packed = candidates.pack(candidates.max_utf8_bytes);- let required = retrieval::build_followup_prompt_context(- &candidates.referenced,- &[],- candidates.max_utf8_bytes,- );- if let (Ok(initial), Ok(required)) = (packed, required) {- let tokens = count_context(required.as_ref())?.saturating_sub(base);- if tokens <= room {- break (initial, required, tokens);- }+ let mut selected = candidates+ .slots+ .iter()+ .map(PassageSlot::minimum_coverage)+ .collect::<Result<Vec<_>, _>>()?;+ let preferred_hits = |count: usize| -> Vec<GroundedExcerpt> {+ candidates.slots[..count]+ .iter()+ .filter_map(|slot| slot.current().cloned())+ .collect()+ };+ let preferred = retrieval::build_followup_prompt_context(+ &selected,+ &preferred_hits(selected.len()),+ usize::MAX,+ )+ .map_err(retrieval_error)?;+ let preferred_tokens = count_context(preferred.as_ref())?.saturating_sub(base);+ let allowance = 1500.min(input_budget / 4).max(preferred_tokens).min(room);+ let ceiling = base.saturating_add(allowance).min(input_budget);++ let (minimum, minimum_count) = loop {+ let minimum = retrieval::build_followup_prompt_context(&selected, &[], usize::MAX)+ .map_err(retrieval_error)?;+ let count = count_context(minimum.as_ref())?;+ if minimum+ .as_ref()+ .is_none_or(|context| context.text.len() <= candidates.max_utf8_bytes)+ && count <= ceiling+ {+ break (minimum, count);}- if candidates.referenced.pop().is_none() {- break (None, None, 0);+ if selected.pop().is_none() {+ return Err(PrepareError::CurrentInputTooLarge);}};- let allowance = 1500.min(input_budget / 4).max(required_tokens).min(room);- let initial_count = count_context(initial.as_ref())?;- let initial_tokens = initial_count.saturating_sub(base);- if initial_count <= input_budget && initial_tokens <= allowance {- return Ok(FittedGrounding {- context: initial,- repacked: candidates.referenced.len() != original_references,- evidence_tokens: initial_tokens,- history_included: false,- });- }- let required_bytes = required.as_ref().map_or(0, |c| c.text.len());- let initial_bytes = initial.as_ref().map_or(0, |c| c.text.len());- let extra_tokens = initial_tokens.saturating_sub(required_tokens).max(1);- let extra_allowance = allowance.saturating_sub(required_tokens);- let smaller = required_bytes- + initial_bytes- .saturating_sub(required_bytes)- .saturating_mul(extra_allowance)- .saturating_mul(4)- / extra_tokens- / 5;- let packed = candidates- .pack(smaller.min(initial_bytes.saturating_sub(1)))- .ok()- .flatten();- if let Some(packed) = packed {- let final_count = count_context(Some(&packed))?;- let final_tokens = final_count.saturating_sub(base);- if final_count <= input_budget && final_tokens <= allowance {- return Ok(FittedGrounding {- context: Some(packed),- repacked: true,- evidence_tokens: final_tokens,- history_included: false,- });+ let demoted = selected.len() != candidates.slots.len();+ let (preferred, preferred_count, reduced) = fit_coverage(+ &selected,+ &preferred_hits(selected.len()),+ candidates.max_utf8_bytes,+ minimum,+ minimum_count,+ ceiling,+ &mut count_context,+ )?;+ let (context, count, optional_reduced) = if reduced {+ (preferred, preferred_count, false)+ } else {+ let mut fixed = candidates.freeze(preferred.as_ref())?;+ let mut context = preferred;+ let mut count = preferred_count;+ for old in &candidates.incidental {+ fixed.push(old.clone());+ let extended = retrieval::build_followup_prompt_context(&fixed, &[], usize::MAX)+ .map_err(retrieval_error)?;+ if extended+ .as_ref()+ .is_some_and(|context| context.text.len() > candidates.max_utf8_bytes)+ {+ fixed.pop();+ continue;+ }+ let extended_count = count_context(extended.as_ref())?;+ if extended_count > ceiling {+ fixed.pop();+ continue;+ }+ context = extended;+ count = extended_count;}- }+ fit_coverage(+ &fixed,+ &candidates.searched,+ candidates.max_utf8_bytes,+ context,+ count,+ ceiling,+ &mut count_context,+ )?+ };Ok(FittedGrounding {- context: required,- repacked: true,- evidence_tokens: required_tokens,+ context,+ repacked: demoted || reduced || optional_reduced,+ evidence_tokens: count.saturating_sub(base),history_included: false,})}+fn fit_coverage(+ selected: &[ReferencedPassage],+ searched: &[GroundedExcerpt],+ max_bytes: usize,+ minimum: Option<GroundedPromptContext>,+ minimum_count: usize,+ ceiling: usize,+ count_context: &mut impl FnMut(Option<&GroundedPromptContext>) -> Result<usize, PrepareError>,+) -> Result<(Option<GroundedPromptContext>, usize, bool), PrepareError> {+ let full = retrieval::build_followup_prompt_context(selected, searched, max_bytes)+ .map_err(retrieval_error)?;+ let full_count = count_context(full.as_ref())?;+ if full_count <= ceiling {+ return Ok((full, full_count, false));+ }+ let mut lower = minimum.as_ref().map_or(0, |context| context.text.len());+ let mut upper = full.as_ref().map_or(0, |context| context.text.len());+ let mut best = minimum;+ let mut best_count = minimum_count;+ while lower + 1 < upper {+ let budget = lower + (upper - lower) / 2;+ let packed = retrieval::build_followup_prompt_context(selected, searched, budget)+ .map_err(retrieval_error)?;+ let count = count_context(packed.as_ref())?;+ if count <= ceiling {+ lower = budget;+ best = packed;+ best_count = count;+ } else {+ upper = budget;+ }+ }+ Ok((best, best_count, true))+}+#[cfg(test)]#[path = "grounding_tests.rs"]mod tests;diff --git a/rust/crates/ensu/src/conversation/grounding_tests.rs b/rust/crates/ensu/src/conversation/grounding_tests.rsindex 5624796852..87f055671d 100644--- a/rust/crates/ensu/src/conversation/grounding_tests.rs+++ b/rust/crates/ensu/src/conversation/grounding_tests.rs@@ -1,16 +1,16 @@use super::*;-use crate::notes::NoteSourceReference;+use crate::notes::{NotePassageLocator, NoteSourceReference};use crate::retrieval::{GroundedSource, PassageLocator};fn hit(text: &str, id: &str) -> GroundedExcerpt {GroundedExcerpt {- locator: PassageLocator::LocalNote {+ locator: PassageLocator::LocalNote(NotePassageLocator {collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),document_id: id.into(),indexed_revision: "a".repeat(64),shard_sha256: "b".repeat(64),chunk_index: 0,- },+ }),source: GroundedSource::LocalNote {reference: NoteSourceReference {collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),@@ -88,7 +88,8 @@ fn fit_followup(query: &str,input_budget: usize,) -> GroundedPromptContext {- let (history, evidence) = followup_history(prior);+ let (mut history, evidence) = followup_history(prior);+ history.last_mut().unwrap().text = query.into();let reloaded = direct_followup_references(&history, &evidence, query, &searched).into_iter().map(|reference| {@@ -103,12 +104,17 @@ fn fit_followup(}}).collect();- let mut candidates = GroundingCandidates::new(reloaded, searched, MAX_GROUNDING_BYTES).unwrap();- candidates.protect_named_sources(query).unwrap();- let context = fit_grounding(&candidates, "System", query, input_budget, measure)- .unwrap()- .context- .unwrap();+ let candidates = GroundingCandidates::new(reloaded, searched, MAX_GROUNDING_BYTES).unwrap();+ let context = fit_grounding(+ &candidates.select_sources(Some(query)).unwrap(),+ "System",+ query,+ input_budget,+ measure,+ )+ .unwrap()+ .context+ .unwrap();assert!(context.text.len() <= MAX_GROUNDING_BYTES);assert!(measure(&answer_messages(@@ -123,46 +129,66 @@ fn fit_followup(context}-#[test]-fn followup_fitting_keeps_implicit_antecedents_and_allows_named_switches() {- let cedar = hit("Cedar allows 30 days.", "cedar.md");- let juniper = hit("Juniper allows 21 days.", "juniper.md");- let old = retrieval::build_grounded_prompt_context(std::slice::from_ref(&cedar), 6000)- .unwrap()+fn notes_at_limit() -> Vec<GroundedExcerpt> {+ ["cedar", "pine", "birch", "oak", "maple"]+ .iter()+ .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))+ .collect()+}++fn assert_full_passage(context: &GroundedPromptContext, hit: &GroundedExcerpt) {+ let index = context+ .included_passages+ .iter()+ .position(|reference| reference.locator == hit.locator).unwrap();- for query in [- "Is Juniper longer than that?",- "Compare its deadline with Juniper.",- "Is Juniper's return period longer than ours?",- "Is Juniper's return period longer than mine?",- "Compare Juniper's return period with our policy.",- "Compare Juniper's return period with my policy.",- "Does she get more time than Juniper?",- "Compare her deadline with Juniper.",- "Is Juniper's deadline longer than hers?",- "Does he get more time than Juniper?",- "Does Juniper get more time than him?",- "Compare his deadline with Juniper.",+ assert_eq!(context.sources[index], hit.source);+ assert_eq!(+ context.included_passages[index]+ .verified_spans(&hit.text)+ .unwrap(),+ vec![hit.text.trim()]+ );+ assert!(context.text.contains(hit.text.trim()));+}++#[test]+fn comparisons_retain_historical_passages_under_pressure() {+ let cedar = hit("Cedar permits 30 days.", "cedar.md");+ let juniper = hit(+ &format!("{}Juniper permits 21 days.", "Juniper details. ".repeat(30)),+ "juniper.md",+ );+ for (query, budget) in [+ ("Compare its deadline with Juniper.", 7000),+ ("Compare Juniper's return period with my policy.", 700),+ ("Compare Cedar and Juniper.", 700),] {let context = fit_followup(std::slice::from_ref(&cedar),vec![juniper.clone()],query,- 7000,- );- assert!(context.text.contains(&cedar.text), "{query}");- assert!(context.text.contains(&juniper.text), "{query}");- assert!(- context- .included_passages- .contains(&old.included_passages[0])+ budget,);+ assert_full_passage(&context, &cedar);+ if budget == 7000 {+ assert_full_passage(&context, &juniper);+ }}- let updated = hit("Cedar now requires seven days.", "cedar.md");+}++#[test]+fn named_questions_switch_to_fresh_sources() {+ let cedar = hit("Cedar allows 30 days.", "cedar.md");for (query, fresh) in [- ("What is Juniper's return period?", &juniper),- ("What is within Juniper's coverage?", &juniper),- ("What is Cedar's current deadline?", &updated),+ (+ "What is Juniper's return period?",+ hit("Juniper allows 21 days.", "juniper.md"),+ ),+ (+ "What is Cedar's current deadline?",+ hit("Cedar now requires seven days.", "cedar.md"),+ ),] {let context = fit_followup(std::slice::from_ref(&cedar),@@ -170,104 +196,200 @@ fn followup_fitting_keeps_implicit_antecedents_and_allows_named_switches() {query,7000,);- assert_eq!(context.sources, vec![fresh.source.clone()], "{query}");- assert!(context.text.contains(&fresh.text), "{query}");+ assert_eq!(context.sources.len(), 1);+ assert_full_passage(&context, &fresh);}}#[test]-fn followup_fitting_prioritizes_named_sources_at_source_and_byte_limits() {- let mut prior: Vec<_> = ["pine", "birch", "oak", "maple", "cedar"]- .iter()- .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))- .collect();+fn named_sources_take_priority_at_source_and_byte_limits() {+ let mut notes = notes_at_limit();+ notes.rotate_left(1);+ let cedar = notes.last().unwrap().clone();let atlas = hit(&"Atlas allows 21 days. ".repeat(40), "atlas.md");- let note_count = prior.len();- prior.extend([- pack_hit("Mars is the fourth planet.", "Mars", 0),- pack_hit("Venus is the second planet.", "Venus", 1),- ]);- let byte_limited = [- prior[note_count - 1].clone(),- hit(&"Unrelated detail. ".repeat(400), "pine.md"),- ];- for prior in [&prior[..note_count], &prior[..], &byte_limited] {- let context = fit_followup(prior, vec![atlas.clone()], "Compare Cedar and Atlas", 7000);- assert!(context.text.contains("cedar allows 30 days."));- assert!(context.text.contains(atlas.text.trim()));- assert!(context.sources.len() <= retrieval::MAX_GROUNDING_HITS);+ for prior in [+ notes,+ vec![+ cedar.clone(),+ hit(&"Unrelated detail. ".repeat(400), "pine.md"),+ ],+ ] {+ let context = fit_followup(&prior, vec![atlas.clone()], "Compare Cedar and Atlas", 7000);+ assert_full_passage(&context, &cedar);+ assert_full_passage(&context, &atlas);+ assert!(context.sources.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);}+}- let mars = pack_hit("Mars is the fourth planet.", "Mars", 0);- let venus = pack_hit("Venus is the second planet.", "Venus", 1);- let jupiter = pack_hit("Jupiter is the fifth planet.", "Jupiter", 2);- let another_mars_chunk = pack_hit("Mars also has two moons.", "Mars", 3);- for prior in [[venus, mars.clone()], [mars.clone(), another_mars_chunk]] {+#[test]+fn personal_questions_keep_fresh_results_at_the_historical_note_limit() {+ let ledger = hit("Holiday spending: 1200 rupees.", "ledger-2026.md");+ for query in [+ "How much did I spend on my holiday?",+ "Compare my holiday costs across years.",+ ] {+ let context = fit_followup(¬es_at_limit(), vec![ledger.clone()], query, 700);+ assert_full_passage(&context, &ledger);+ }+}++#[test]+fn temporal_phrases_are_distinct_from_source_references() {+ for (question, expected) in [+ (+ "How much did I spend on my holiday these past few months?",+ false,+ ),+ ("What did I spend on holiday in those 2 weeks?", false),+ ("This year, compare its deadline with Juniper.", true),+ ("Compare that second policy with Juniper.", true),+ ] {+ assert_eq!(+ super::super::followup::refers_back(question),+ expected,+ "{question}"+ );+ }+}++#[test]+fn temporal_questions_keep_fresh_results_under_source_and_byte_limits() {+ let ledger = hit("Holiday spending: 1200 rupees.", "ledger-2026.md");+ for (prior, fresh, budget) in [+ (notes_at_limit(), ledger.clone(), 7000),+ (+ vec![hit(&"Cedar allows 30 days. ".repeat(320), "cedar.md")],+ ledger.clone(),+ 7000,+ ),+ (+ vec![+ pack_hit("Mars is the fourth planet.", "Mars", 0),+ pack_hit("Venus is the second planet.", "Venus", 1),+ ],+ pack_hit(&ledger.text, "Ledger", 2),+ 700,+ ),+ ] {let context = fit_followup(&prior,- vec![jupiter.clone()],- "Compare Mars and Jupiter",- 7000,+ vec![fresh.clone()],+ "How much did I spend on my holiday this year?",+ budget,);- assert!(context.text.contains(&mars.text));- assert!(context.text.contains(&jupiter.text));- assert_eq!(context.sources.len(), 2);+ assert_full_passage(&context, &fresh);}}#[test]-fn unnamed_questions_keep_fresh_results_and_available_historical_passages() {- let notes: Vec<_> = ["cedar", "pine", "birch", "oak", "maple"]- .iter()- .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))- .collect();+fn named_pack_comparisons_retain_articles_cited_through_multiple_sections() {+ let prior: Vec<_> = [+ ("History", "Mars has been observed since antiquity."),+ ("Geography", "Mars has two polar ice caps."),+ ]+ .into_iter()+ .enumerate()+ .map(|(row, (section, text))| {+ let mut hit = pack_hit(text, &format!("Mars — {section}"), row as u64);+ if let GroundedSource::EnsuPack { citation } = &mut hit.source {+ citation.source_url = format!("https://simple.wikipedia.org/wiki/Mars#{section}");+ }+ hit+ })+ .collect();+ let jupiter = pack_hit("Jupiter is the fifth planet.", "Jupiter", 2);+ let context = fit_followup(+ &prior,+ vec![jupiter.clone()],+ "Compare Mars and Jupiter",+ 700,+ );+ assert_full_passage(&context, &prior[0]);+ assert_full_passage(&context, &jupiter);+ assert_eq!(context.sources.len(), 2);++ let context = fit_followup(&prior, vec![], "Tell me more about Mars", 7000);+ assert_eq!(+ context.sources,+ prior+ .iter()+ .map(|hit| hit.source.clone())+ .collect::<Vec<_>>()+ );+}++#[test]+fn unnamed_questions_keep_available_historical_passages() {+ let notes = notes_at_limit();let biology = hit("Photosynthesis converts light into energy.","biology-chapter-seven.md",);- let mut another_chunk = notes[0].clone();- if let PassageLocator::LocalNote { chunk_index, .. } = &mut another_chunk.locator {- *chunk_index += 1;- }- another_chunk.text = "International shipping has an extended deadline.".into();let old = retrieval::build_grounded_prompt_context(¬es, MAX_GROUNDING_BYTES).unwrap().unwrap();- for (query, fresh) in [- ("How does photosynthesis work?", &biology),- ("And the deadline?", &biology),- ("And the deadline?", &another_chunk),- ] {- let context = fit_followup(¬es, vec![fresh.clone()], query, 7000);- assert!(context.text.contains(&fresh.text), "{query}");- assert!(context.text.contains(¬es[0].text), "{query}");- assert!(- context- .included_passages- .contains(&old.included_passages[0])- );- assert!(context.sources.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);- }-- let packs = [- pack_hit("Mars is the fourth planet.", "Mars", 0),- pack_hit("Venus is the second planet.", "Venus", 1),- ];- let fresh = pack_hit("Plants use sunlight to produce sugar.", "Botany", 2);let context = fit_followup(- &packs,- vec![fresh.clone()],+ ¬es,+ vec![biology.clone()],"How does photosynthesis work?",7000,);- assert!(context.text.contains(&fresh.text));- assert!(context.text.contains(&packs[0].text));- assert_eq!(context.sources.len(), retrieval::MAX_PACK_HITS);+ assert_full_passage(&context, &biology);+ assert!(+ context+ .included_passages+ .contains(&old.included_passages[0])+ );+ assert_eq!(context.sources.len(), retrieval::MAX_NOTES_GROUNDING_HITS);let context = fit_followup(¬es, vec![], "And the deadline?", 7000);assert_eq!(context.included_passages, old.included_passages);}+#[test]+fn followups_keep_new_chunks_of_retained_sources_at_hit_limits() {+ let notes: Vec<_> = (0..3)+ .map(|index| {+ let mut note = hit(&format!("Cedar background section {index}."), "cedar.md");+ if let PassageLocator::LocalNote(locator) = &mut note.locator {+ locator.chunk_index = index;+ }+ note+ })+ .chain([+ hit("Pine background.", "pine.md"),+ hit("Birch background.", "birch.md"),+ ])+ .collect();+ let mut fresh = notes[0].clone();+ if let PassageLocator::LocalNote(locator) = &mut fresh.locator {+ locator.chunk_index = 3;+ }+ fresh.text = "Cedar warranty duration: 24 months.".into();+ let context = fit_followup(+ ¬es,+ vec![notes[0].clone(), fresh.clone()],+ "And in Cedar, how long is its warranty?",+ 700,+ );+ assert_full_passage(&context, &fresh);+ assert_full_passage(&context, ¬es[0]);+ assert!(context.included_passages.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);++ let prior = [+ pack_hit("Mars is the fourth planet.", "Mars", 0),+ pack_hit("Venus is the second planet.", "Venus", 1),+ ];+ let fresh = [+ pack_hit("Its radius is 3389.5 kilometers.", "Mars", 2),+ pack_hit("Its radius is 6051.8 kilometers.", "Venus", 3),+ ];+ let context = fit_followup(&prior, fresh.to_vec(), "Which of them is larger?", 700);+ for hit in &fresh {+ assert_full_passage(&context, hit);+ }+}+#[test]fn fresh_followup_results_survive_byte_and_token_repacking() {let prefix = hit("Price: 100 rupees.", "cedar.md");@@ -279,15 +401,8 @@ fn fresh_followup_results_survive_byte_and_token_repacking() {"And what's its warranty duration?",7000,);- assert!(context.text.contains(&full.text));- assert_eq!(context.sources, vec![full.source]);+ assert_full_passage(&context, &full);assert_eq!(context.included_passages.len(), 1);- assert_eq!(- context.included_passages[0]- .verified_spans(&full.text)- .unwrap(),- vec![full.text]- );let old = hit(&"Cedar allows 30 days. ".repeat(280), "cedar.md");let fresh = hit(@@ -304,27 +419,15 @@ fn fresh_followup_results_survive_byte_and_token_repacking() {"How does photosynthesis work?",input_budget,);- assert!(context.text.contains("Plants use light."));let reference = context.included_passages.iter().find(|reference| reference.locator == fresh.locator).unwrap();let spans = reference.verified_spans(&fresh.text).unwrap();+ assert!(spans[0].starts_with("Plants use light."));assert!(context.text.contains(&spans[0]));}-- let mut long_label = hit("Plants use light.", "biology.md");- if let GroundedSource::LocalNote { reference } = &mut long_label.source {- reference.title = "Biology ".repeat(400);- }- let context = fit_followup(- &[old],- vec![long_label.clone()],- "How does photosynthesis work?",- 7000,- );- assert!(context.text.contains(&long_label.text));}#[test]@@ -343,7 +446,14 @@ fn measured_repack_keeps_text_citations_and_exact_spans_together() {),] {let candidates = GroundingCandidates::new(vec![], vec![hit(&raw, "one.md")], 6000).unwrap();- let fit = fit_grounding(&candidates, "System", "Question", 4096, measure).unwrap();+ let fit = fit_grounding(+ &candidates.select_sources(None).unwrap(),+ "System",+ "Question",+ 4096,+ measure,+ )+ .unwrap();assert!(fit.repacked);assert!(fit.evidence_tokens <= 1024);let context = fit.context.unwrap();@@ -374,7 +484,14 @@ fn requested_old_spans_raise_soft_allowance_and_are_never_cut() {6000,).unwrap();- let fit = fit_grounding(&candidates, "System", "Question", 3000, measure).unwrap();+ let fit = fit_grounding(+ &candidates.select_sources(None).unwrap(),+ "System",+ "Question",+ 3000,+ measure,+ )+ .unwrap();assert!(fit.repacked && fit.evidence_tokens > 750);assert_eq!(fit.context.unwrap().included_passages[0], reference);}@@ -383,14 +500,26 @@ fn mandatory_input_and_insufficient_evidence_budget_are_explicit() {let candidates =GroundingCandidates::new(vec![], vec![hit(&"word ".repeat(1000), "one.md")], 6000).unwrap();assert!(matches!(- fit_grounding(&candidates, "System", &"x".repeat(1000), 100, measure),+ fit_grounding(+ &candidates.select_sources(None).unwrap(),+ "System",+ &"x".repeat(1000),+ 100,+ measure+ ),Err(PrepareError::CurrentInputTooLarge)));assert!(- fit_grounding(&candidates, "System", "Question", 60, measure)- .unwrap()- .context- .is_none()+ fit_grounding(+ &candidates.select_sources(None).unwrap(),+ "System",+ "Question",+ 60,+ measure+ )+ .unwrap()+ .context+ .is_none());}#[test]@@ -411,7 +540,7 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {&format!("{}Juniper permits 21 days.", "Juniper details. ".repeat(25)),"juniper-policy.md",);- let mut candidates = GroundingCandidates::new(+ let candidates = GroundingCandidates::new(vec![],vec![cedar.clone(),@@ -421,49 +550,14 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {6000,).unwrap();- candidates- .protect_named_sources("Is the Juniper period longer than Cedar?")+ let plan = candidates+ .select_sources(Some("Is the Juniper period longer than Cedar?")).unwrap();- let fit = fit_grounding(- &candidates,- "System",- "Compare Cedar and Juniper.",- 2500,- measure,- )- .unwrap();+ let fit = fit_grounding(&plan, "System", "Compare Cedar and Juniper.", 2500, measure).unwrap();assert!(fit.repacked);let context = fit.context.unwrap();for hit in [&cedar, &juniper] {- assert!(context.text.contains(&hit.text));- let reference = context- .included_passages- .iter()- .find(|r| r.locator == hit.locator)- .unwrap();- assert_eq!(- reference.verified_spans(&hit.text).unwrap(),- vec![hit.text.clone()]- );- }-}--#[test]-fn ordinary_questions_and_shared_label_words_do_not_promote_every_source() {- let base = GroundingCandidates::new(- vec![],- vec![- hit("30 days", "cedar-policy.md"),- hit("21 days", "juniper-policy.md"),- ],- 6000,- )- .unwrap();- for query in ["Compare the policies in my notes", "Explain photography"] {- let mut c = base.clone();- c.protect_named_sources(query).unwrap();- assert!(c.referenced.is_empty());- assert_eq!(c.pack(6000).unwrap(), base.pack(6000).unwrap());+ assert_full_passage(&context, hit);}}@@ -496,7 +590,7 @@ fn explicit_history_displaces_optional_search_without_losing_original_text() {).unwrap();let fit = fit_grounding_with_history(- &candidates,+ &candidates.select_sources(None).unwrap(),Some(&required),"System",query,@@ -508,7 +602,7 @@ fn explicit_history_displaces_optional_search_without_losing_original_text() {assert_eq!(fit.evidence_tokens, 0);assert!(fit_grounding_with_history(- &candidates,+ &candidates.select_sources(None).unwrap(),Some(&required),"System",query,@@ -531,8 +625,13 @@ fn source_system_and_required_history_are_measured_in_final_roles() {6000,).unwrap();- let fitted =- fit_grounding_with_history(&c, Some(&history), "System", query, 4096, |messages| {+ let fitted = fit_grounding_with_history(+ &c.select_sources(None).unwrap(),+ Some(&history),+ "System",+ query,+ 4096,+ |messages| {if messages[0].content.contains("BEGIN KNOWLEDGE CONTEXT") {assert_eq!(messages[0].role, "system");assert!(messages.last().unwrap().content.contains("ZX-82Q"));@@ -545,43 +644,15 @@ fn source_system_and_required_history_are_measured_in_final_roles() {.contains("BEGIN KNOWLEDGE CONTEXT"));measure(messages)- })- .unwrap();+ },+ )+ .unwrap();assert!(grounded_system("System", fitted.context.as_ref()).contains("The new rule"));let final_text = history.prepend_required(query);assert!(final_text.contains("ZX-82Q"));assert!(final_text.ends_with(query));}-#[test]-fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switches() {- let pack = pack_hit("Mars is the fourth planet.", "Mars", 0);- let mut note = hit("Check-in code COPPER-29.", "mars-expedition.md");- if let GroundedSource::LocalNote { reference } = &mut note.source {- reference.title = "Mars expedition checklist".into();- }- let query = "Using the Mars encyclopedia entry and the Mars expedition checklist, give its position from the Sun and our expedition check-in code. Answer briefly.";- let mut c = GroundingCandidates::new(vec![], vec![pack.clone(), note.clone()], 6000).unwrap();- c.protect_named_sources(query).unwrap();- assert_eq!(c.referenced.len(), 2);- assert_eq!(c.pack(6000).unwrap().unwrap().sources.len(), 2);-- let (history, evidence) = followup_history(&[pack]);- assert_eq!(- direct_followup_references(&history, &evidence, query, std::slice::from_ref(¬e)),- evidence[0].passages- );- assert!(- direct_followup_references(- &history,- &evidence,- "What is the check-in code in the Mars expedition checklist?",- &[note]- )- .is_empty()- );-}-#[test]fn followups_use_last_available_evidence_until_its_history_changes() {let (mut history, evidence) = followup_history(&[hit("Archived policy fact.", "cedar.md")]);diff --git a/rust/crates/ensu/src/conversation/state.rs b/rust/crates/ensu/src/conversation/state.rsindex 5873549475..e94d405ccc 100644--- a/rust/crates/ensu/src/conversation/state.rs+++ b/rust/crates/ensu/src/conversation/state.rs@@ -6,9 +6,9 @@ use crate::db::{Message, Sender};use crate::retrieval::IncludedPassage;pub const ENVELOPE_VERSION: u32 = 2;-pub const MAX_EVIDENCE_ANSWERS: usize = 8;-pub const MAX_EVIDENCE_BYTES: usize = 12 * 1024;-pub(crate) use crate::retrieval::MAX_GROUNDING_HITS as MAX_ANSWER_PASSAGES;+pub(super) const MAX_EVIDENCE_ANSWERS: usize = 8;+pub(super) const MAX_EVIDENCE_BYTES: usize = 12 * 1024;+pub(super) use crate::retrieval::MAX_GROUNDING_HITS as MAX_ANSWER_PASSAGES;#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct AnswerEvidence {@@ -28,10 +28,6 @@ impl AnswerEvidence {&& fingerprint(&history[..=index]) == self.prefix_fingerprint})}-- fn valid(&self) -> bool {- self.prefix_fingerprint.len() == 64 && valid_answer_passages(&self.passages)- }}pub(crate) fn valid_answer_passages(passages: &[IncludedPassage]) -> bool {@@ -76,16 +72,17 @@ impl ConversationEnvelope {.cloned().and_then(|value| serde_json::from_value::<Summary>(value).ok()).filter(|summary| validate_summary_text(&summary.text).is_ok());- let mut evidence = Vec::new();+ let mut evidence: Vec<AnswerEvidence> = Vec::new();if let Some(entries) = value.get("evidence").and_then(|value| value.as_array())&& entries.len() <= MAX_EVIDENCE_ANSWERS{for value in entries {if let Ok(entry) = serde_json::from_value::<AnswerEvidence>(value.clone())- && entry.valid()- && !evidence.iter().any(|old: &AnswerEvidence| {- old.assistant_message_uuid == entry.assistant_message_uuid- })+ && entry.prefix_fingerprint.len() == 64+ && valid_answer_passages(&entry.passages)+ && !evidence+ .iter()+ .any(|old| old.assistant_message_uuid == entry.assistant_message_uuid){evidence.push(entry);}diff --git a/rust/crates/ensu/src/conversation/tests.rs b/rust/crates/ensu/src/conversation/tests.rsindex 743b8bbd85..34d8deccec 100644--- a/rust/crates/ensu/src/conversation/tests.rs+++ b/rust/crates/ensu/src/conversation/tests.rs@@ -68,6 +68,7 @@ fn saved_memory_preserves_uuid_strings() {let mut expected = serde_json::from_str::<serde_json::Value>(encoded).unwrap();expected["evidence"] = serde_json::json!([]);assert_eq!(serde_json::to_value(&restored).unwrap(), expected);+ assert!(ConversationEnvelope::decode(encoded.as_bytes(), Uuid::nil()).is_none());let invalid = encoded.replace("00000000-0000-0000-0000-000000000002", "invalid");assert!(ConversationEnvelope::decode(invalid.as_bytes(), session)@@ -77,10 +78,6 @@ fn saved_memory_preserves_uuid_strings() {);}-fn evidence_passages() -> Vec<crate::retrieval::IncludedPassage> {- vec![crate::retrieval::passage_fixture().0]-}-#[test]fn invalid_summary_and_evidence_are_discarded_independently() {let history = vec![@@ -91,7 +88,7 @@ fn invalid_summary_and_evidence_are_discarded_independently() {let memory = ConversationState::new(session, &history, "Memory".into()).unwrap();let mut state = ConversationEnvelope::empty(session);state.summary = Some(memory.summary.clone());- assert!(state.add_evidence(&history, evidence_passages()));+ assert!(state.add_evidence(&history, vec![crate::retrieval::passage_fixture().0]));let original = serde_json::to_value(&state).unwrap();let decode = |value: &serde_json::Value| {ConversationEnvelope::decode(&serde_json::to_vec(value).unwrap(), session)@@ -107,17 +104,12 @@ fn invalid_summary_and_evidence_are_discarded_independently() {let decoded = decode(&value).unwrap();assert!(decoded.evidence.is_empty());assert_eq!(decoded.summary_state().unwrap(), memory);- value["format_version"] = 1.into();- assert!(decode(&value).is_none());- assert!(- ConversationEnvelope::decode(&serde_json::to_vec(&state).unwrap(), Uuid::nil()).is_none()- );}#[test]fn evidence_retention_obeys_answer_and_byte_limits() {for passage_count in [1, MAX_ANSWER_PASSAGES] {- let passages = vec![evidence_passages().remove(0); passage_count];+ let passages = vec![crate::retrieval::passage_fixture().0; passage_count];let mut state = ConversationEnvelope::empty(Uuid::from_u128(100));let mut history = Vec::new();for index in 1..=10 {diff --git a/rust/crates/ensu/src/conversation/turn.rs b/rust/crates/ensu/src/conversation/turn.rsindex 9a91c3da39..6bc4c60522 100644--- a/rust/crates/ensu/src/conversation/turn.rs+++ b/rust/crates/ensu/src/conversation/turn.rs@@ -44,11 +44,11 @@ pub fn prepare_turn(.transpose()?.flatten();effects.check_cancelled()?;- input- .candidates- .protect_named_sources(input.history_query.as_deref().unwrap_or(&input.current))?;+ let grounding_plan = input.candidates.select_sources(Some(+ input.history_query.as_deref().unwrap_or(&input.current),+ ))?;let grounding = fit_grounding_with_history(- &input.candidates,+ &grounding_plan,required.as_ref(),&input.system,&input.current,diff --git a/rust/crates/ensu/src/db/chat/conversation.rs b/rust/crates/ensu/src/db/chat/conversation.rsindex 9d94b50b89..26e3940685 100644--- a/rust/crates/ensu/src/db/chat/conversation.rs+++ b/rust/crates/ensu/src/db/chat/conversation.rs@@ -599,12 +599,10 @@ mod tests {.unwrap();let captured = snapshot(&db, session, std::slice::from_ref(&question));let answer = sourced_answer(&db, &captured);- let before = db.get_session(session).unwrap();assert!(db.save_answer_evidence(&captured, answer.uuid, passages()).unwrap());- assert_eq!(db.get_session(session).unwrap(), before);let full = vec![question.clone(), answer];let mut restored = snapshot(&db, session, &full);assert!(restored.state().is_none());@@ -653,17 +651,11 @@ mod tests {);messages.push(answer);let mut current = snapshot(&db, session, &messages);- assert!(- db.replace_conversation_state(&mut current, Some(state(session, &messages, "Memory")))- .unwrap()- );- assert_eq!(snapshot(&db, session, &messages).evidence().len(), 1);- assert!(db.replace_conversation_state(&mut current, None).unwrap());- assert_eq!(snapshot(&db, session, &messages).evidence().len(), 1);- assert!(- db.replace_conversation_state(&mut current, Some(state(session, &messages, "Memory")))- .unwrap()- );+ let memory = state(session, &messages, "Memory");+ for next in [Some(memory.clone()), None, Some(memory)] {+ assert!(db.replace_conversation_state(&mut current, next).unwrap());+ assert_eq!(snapshot(&db, session, &messages).evidence().len(), 1);+ }db.update_message_text(messages[2].uuid, "Edited question").unwrap();let current = snapshot(&db, session, &messages);@@ -677,12 +669,10 @@ mod tests {let captured = snapshot(&db, session, &messages);let answer = sourced_answer(&db, &captured);let mut other_writer = snapshot(&db, session, &messages);+ let memory = state(session, &messages, "Other writer");assert!(- db.replace_conversation_state(- &mut other_writer,- Some(state(session, &messages, "Other writer"))- )- .unwrap()+ db.replace_conversation_state(&mut other_writer, Some(memory.clone()))+ .unwrap());assert!(!db.save_answer_evidence(&captured, answer.uuid, passages())@@ -695,14 +685,7 @@ mod tests {.is_err());assert_eq!(db.get_message(answer.uuid).unwrap().unwrap(), answer);- assert_eq!(- snapshot(&db, session, &messages)- .state()- .unwrap()- .summary- .text,- "Other writer"- );+ assert_eq!(snapshot(&db, session, &messages).state(), Some(&memory));}#[test]@@ -735,7 +718,6 @@ mod tests {db.save_answer_evidence(&captured, answer.uuid, passages()).is_err());- assert!(db.get_session(session).unwrap().is_none());}#[test]@@ -842,6 +824,12 @@ mod tests {).unwrap());+ let mut encoded = ConversationEnvelope::empty(session);+ encoded.summary = captured.state().map(|state| state.summary.clone());+ encoded.format_version = 1;+ assert!(+ ConversationEnvelope::decode(&serde_json::to_vec(&encoded).unwrap(), session).is_none()+ );for text in [" \n".to_owned(), "x".repeat(MAX_STATE_BYTES)] {let mut invalid = state(session, &messages, "Valid memory");invalid.summary.text = text;diff --git a/rust/crates/ensu/src/notes.rs b/rust/crates/ensu/src/notes.rsindex ef7cd03d35..4c38439639 100644--- a/rust/crates/ensu/src/notes.rs+++ b/rust/crates/ensu/src/notes.rs@@ -47,7 +47,9 @@ pub enum NotesError {pub use document::{NotesSourceDocument, PreparedNotesChunk, PreparedNotesDocument, prepare_notes_document,};-pub use index::{NotesCollectionIndex, NotesCollectionIndexSummary, NotesSearchHit};+pub use index::{+ NotePassageLocator, NotesCollectionIndex, NotesCollectionIndexSummary, NotesSearchHit,+};pub use indexing::{NotesDocumentLoad, NotesIndexInput, NotesIndexOutcome, NotesIndexProgress, NotesIndexingError,NotesRevisionStatus, index_notes_collection,diff --git a/rust/crates/ensu/src/notes/index.rs b/rust/crates/ensu/src/notes/index.rsindex 46468f2695..d76b0137b2 100644--- a/rust/crates/ensu/src/notes/index.rs+++ b/rust/crates/ensu/src/notes/index.rs@@ -1,20 +1,31 @@use std::collections::{BTreeMap, btree_map::Entry};use std::path::Path;+use serde::{Deserialize, Serialize};+use super::manifest::{collection_directory, document_storage_id, load_published_manifest, shard_directory,};use super::shard::{NotesShard, load_and_validate_shard_metadata, load_and_validate_vectors};use super::{NotesCollectionReadPin, NotesError, pinned_revision_key, validate_collection_id};-use crate::retrieval::PassageLocator;const RELEVANCE_THRESHOLD: f32 = 0.50;const MAX_HITS_PER_COLLECTION: usize = 5;const MAX_HITS_PER_DOCUMENT: usize = 3;+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct NotePassageLocator {+ pub collection_id: String,+ pub document_id: String,+ pub indexed_revision: String,+ pub shard_sha256: String,+ pub chunk_index: u64,+}+#[derive(Debug, Clone, PartialEq)]pub struct NotesSearchHit {- pub locator: PassageLocator,+ pub locator: NotePassageLocator,pub collection_id: String,pub document_id: String,pub revision: String,@@ -186,18 +197,15 @@ impl NotesCollectionIndex {pub fn reload_passage(&self,- locator: &PassageLocator,+ locator: &NotePassageLocator,) -> Result<Option<NotesSearchHit>, NotesError> {- let PassageLocator::LocalNote {+ let NotePassageLocator {collection_id,document_id,indexed_revision,shard_sha256,chunk_index,- } = locator- else {- return Ok(None);- };+ } = locator;if collection_id != &self.collection_id {return Ok(None);}@@ -215,17 +223,8 @@ impl NotesCollectionIndex {return Ok(None);}let shard = self.load_shard(document)?;- let chunk = &shard.chunks[chunk_index];- Ok(Some(NotesSearchHit {- locator: locator.clone(),- collection_id: collection_id.clone(),- document_id: document_id.clone(),- revision: indexed_revision.clone(),- score: 0.0,- title: shard.title,- section: chunk.section.clone(),- text: chunk.text.clone(),- }))+ self.passage_hit(document, &shard, chunk_index, 0.0)+ .map(Some)}pub fn search(&self, query: &[f32]) -> Result<Vec<NotesSearchHit>, NotesError> {@@ -288,28 +287,38 @@ impl NotesCollectionIndex {Entry::Occupied(entry) => entry.into_mut(),Entry::Vacant(entry) => entry.insert(self.load_shard(document)?),};- let chunk = shard.chunks.get(candidate.chunk_index).ok_or_else(|| {- NotesError::InvalidIndex("selected shard chunk is missing".to_string())- })?;- hits.push(NotesSearchHit {- locator: PassageLocator::LocalNote {- collection_id: self.collection_id.clone(),- document_id: document.document_id.clone(),- indexed_revision: document.revision.clone(),- shard_sha256: document.shard_sha256.clone(),- chunk_index: candidate.chunk_index as u64,- },- collection_id: self.collection_id.clone(),- document_id: document.document_id.clone(),- revision: document.revision.clone(),- score: candidate.score,- title: shard.title.clone(),- section: chunk.section.clone(),- text: chunk.text.clone(),- });+ hits.push(self.passage_hit(document, shard, candidate.chunk_index, candidate.score)?);}Ok(hits)}++ fn passage_hit(+ &self,+ document: &IndexedDocument,+ shard: &NotesShard,+ chunk_index: usize,+ score: f32,+ ) -> Result<NotesSearchHit, NotesError> {+ let chunk = shard.chunks.get(chunk_index).ok_or_else(|| {+ NotesError::InvalidIndex("selected shard chunk is missing".to_string())+ })?;+ Ok(NotesSearchHit {+ locator: NotePassageLocator {+ collection_id: self.collection_id.clone(),+ document_id: document.document_id.clone(),+ indexed_revision: document.revision.clone(),+ shard_sha256: document.shard_sha256.clone(),+ chunk_index: chunk_index as u64,+ },+ collection_id: self.collection_id.clone(),+ document_id: document.document_id.clone(),+ revision: document.revision.clone(),+ score,+ title: shard.title.clone(),+ section: chunk.section.clone(),+ text: chunk.text.clone(),+ })+ }}fn retain_best(@@ -414,28 +423,19 @@ mod tests {assert_ne!(hits[0].locator, hits[1].locator);drop(index);let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();- for hit in &hits {+ for (hit, chunk) in hits.iter().zip(&doc.chunks) {let loaded = index.reload_passage(&hit.locator).unwrap().unwrap();- assert_eq!(loaded.text, hit.text);+ assert_eq!(loaded.text, chunk.text);assert_eq!(loaded.locator, hit.locator);}for field in ["revision", "shard", "chunk", "collection", "document"] {let mut wrong = hits[0].locator.clone();- if let PassageLocator::LocalNote {- indexed_revision,- shard_sha256,- chunk_index,- collection_id,- document_id,- } = &mut wrong- {- match field {- "revision" => *indexed_revision = "c".repeat(64),- "shard" => *shard_sha256 = "c".repeat(64),- "chunk" => *chunk_index = u64::MAX,- "collection" => *collection_id = "other".to_string(),- _ => *document_id = "other.md".to_string(),- }+ match field {+ "revision" => wrong.indexed_revision = "c".repeat(64),+ "shard" => wrong.shard_sha256 = "c".repeat(64),+ "chunk" => wrong.chunk_index = u64::MAX,+ "collection" => wrong.collection_id = "other".to_string(),+ _ => wrong.document_id = "other.md".to_string(),}assert!(index.reload_passage(&wrong).unwrap().is_none(), "{field}");}diff --git a/rust/crates/ensu/src/retrieval/grounded.rs b/rust/crates/ensu/src/retrieval/grounded.rsindex c5e608eb54..88eef5f634 100644--- a/rust/crates/ensu/src/retrieval/grounded.rs+++ b/rust/crates/ensu/src/retrieval/grounded.rs@@ -21,6 +21,25 @@ pub enum GroundedSource {LocalNote { reference: NoteSourceReference },}+impl GroundedSource {+ pub(crate) fn document_key(&self) -> (u8, &str, &str) {+ match self {+ Self::LocalNote { reference } => (0, &reference.collection_id, &reference.document_id),+ Self::EnsuPack { citation } => {+ let article_url = citation+ .source_url+ .split_once('#')+ .map_or(citation.source_url.as_str(), |(article, _)| article);+ (1, &citation.dataset_id, article_url)+ }+ }+ }++ pub(crate) fn same_document(&self, other: &Self) -> bool {+ self.document_key() == other.document_key()+ }+}+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]pub struct GroundedExcerpt {pub locator: super::PassageLocator,@@ -123,7 +142,7 @@ pub fn select_mixed_grounding_candidates(.validate().map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;selected.push(GroundedExcerpt {- locator: hit.locator.clone(),+ locator: super::PassageLocator::LocalNote(hit.locator.clone()),score: hit.score,source: GroundedSource::LocalNote { reference },text: hit.text.clone(),@@ -156,15 +175,20 @@ pub fn build_grounded_prompt_context(let footer = format!("\n{END_CONTEXT_SENTINEL}");let mut text = header;let mut sources = Vec::new();- let mut included_passages = Vec::new();+ let mut included_passages: Vec<super::IncludedPassage> = Vec::new();for excerpt in excerpts {+ if included_passages+ .iter()+ .any(|reference| reference.locator == excerpt.locator)+ {+ continue;+ }if !excerpt.locator.matches_source(&excerpt.source) {return Err(RetrievalError::InvalidInput("passage locator does not match its source".to_string(),));}- let label = grounded_label(&excerpt.source)?;- let prefix = format!("\n\n# {label}\n");+ let prefix = grounded_prefix(&excerpt.source)?;let reserved = text.len().checked_add(prefix.len())@@ -225,21 +249,22 @@ fn source_sort_key(source: &GroundedSource) -> (u8, &str, &str, &str) {}}-fn grounded_label(source: &GroundedSource) -> Result<String, RetrievalError> {- match source {- GroundedSource::EnsuPack { citation } => Ok(format!(+fn grounded_prefix(source: &GroundedSource) -> Result<String, RetrievalError> {+ let label = match source {+ GroundedSource::EnsuPack { citation } => format!("{} (Ensu Pack · {})",normalize_required(&citation.title, "Pack title")?,normalize_required(&citation.dataset_label, "Pack label")?- )),+ ),GroundedSource::LocalNote { reference } => {reference.validate().map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;let title = display_title(&reference.title, reference.section.as_deref())?;- Ok(format!("{title} (Your Notes · {})", reference.document_id))+ format!("{title} (Your Notes · {})", reference.document_id)}- }+ };+ Ok(format!("\n\n# {label}\n"))}#[derive(Debug, Clone, Serialize, Deserialize)]@@ -268,13 +293,10 @@ pub(crate) fn build_followup_prompt_context(let mut sections = Vec::new();let mut used_bytes = text.len() + footer.len();let mut sources = Vec::new();- let mut included_passages = Vec::new();+ let mut included_passages: Vec<super::IncludedPassage> = Vec::new();let mut packs = 0;let mut notes = 0;for item in referenced {- if included_passages.contains(&item.reference) {- continue;- }if !item.reference.valid_metadata()|| !item.reference.locator.matches_source(&item.source)|| item.reference.spans.len() != item.passages.len()@@ -288,6 +310,17 @@ pub(crate) fn build_followup_prompt_context(return Err(invalid());}}+ if let Some(index) = included_passages+ .iter()+ .position(|reference| reference.locator == item.reference.locator)+ {+ if included_passages[index] != item.reference+ || !item.source.same_document(&sources[index])+ {+ return Err(invalid());+ }+ continue;+ }match &item.source {GroundedSource::EnsuPack { .. } => packs += 1,GroundedSource::LocalNote { .. } => notes += 1,@@ -295,7 +328,7 @@ pub(crate) fn build_followup_prompt_context(if packs > MAX_PACK_HITS || notes > MAX_NOTES_GROUNDING_HITS {return Err(invalid());}- let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);+ let prefix = grounded_prefix(&item.source)?;let passages = item.passages.join("\n[omitted source text]\n");if used_bytes.saturating_add(prefix.len())@@ -309,7 +342,14 @@ pub(crate) fn build_followup_prompt_context(sources.push(item.source.clone());included_passages.push(item.reference.clone());}- for item in searched {+ let mut ordered: Vec<_> = searched.iter().collect();+ ordered.sort_by_key(|item| {+ referenced+ .iter()+ .position(|old| old.reference.locator == item.locator)+ .unwrap_or(usize::MAX)+ });+ for item in ordered {if let Some(index) = included_passages.iter().position(|reference| reference.locator == item.locator)@@ -317,13 +357,12 @@ pub(crate) fn build_followup_prompt_context(let reference = &included_passages[index];if !item.score.is_finite()|| !item.locator.matches_source(&item.source)+ || !item.source.same_document(&sources[index])|| reference.verified_spans(&item.text).is_none(){continue;}- // Fresh search may add text beyond the saved spans, but every old- // span must still fit. Keep one source entry for the entire chunk.- let prefix = format!("\n\n# {}\n", grounded_label(&sources[index])?);+ let prefix = grounded_prefix(&sources[index])?;let available = max_utf8_bytes - used_bytes + sections[index].len() - prefix.len();let cleaned = super::clean_passage_text(&item.text);let passage = truncate_utf8(&cleaned, available).trim_end();@@ -345,7 +384,7 @@ pub(crate) fn build_followup_prompt_context(if (is_pack && packs >= MAX_PACK_HITS) || (!is_pack && notes >= MAX_NOTES_GROUNDING_HITS) {continue;}- let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);+ let prefix = grounded_prefix(&item.source)?;let Some(available) = max_utf8_bytes.checked_sub(used_bytes + prefix.len()) else {continue;};@@ -408,7 +447,7 @@ mod tests {fn note(document_id: &str, score: f32) -> NotesSearchHit {NotesSearchHit {- locator: crate::retrieval::PassageLocator::LocalNote {+ locator: crate::notes::NotePassageLocator {collection_id: COLLECTION.to_owned(),document_id: document_id.to_owned(),indexed_revision: "a".repeat(64),@@ -428,7 +467,7 @@ mod tests {#[test]fn mismatched_passage_sources_and_text_are_rejected() {let mut wrong = select_mixed_grounding_candidates(&[], &[note("one.md", 0.9)], 5).unwrap();- wrong[0].locator = note("other.md", 0.9).locator;+ wrong[0].locator = super::super::PassageLocator::LocalNote(note("other.md", 0.9).locator);assert!(build_grounded_prompt_context(&wrong, 6000).is_err());let mut old = previous();old.passages[0] = "Cafe 🙂".into();@@ -503,7 +542,7 @@ mod tests {}fn previous() -> ReferencedPassage {- let (reference, raw) = super::super::passage_fixture();+ let (reference, _) = super::super::passage_fixture();let source = GroundedSource::LocalNote {reference: NoteSourceReference {collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),@@ -515,7 +554,7 @@ mod tests {},};ReferencedPassage {- passages: reference.verified_spans(raw).unwrap(),+ passages: vec!["Café🙂".into(), "end".into()],reference,source,}@@ -530,8 +569,8 @@ mod tests {text: "UNSUPPLIED source remainder".into(),};let mut fresh = duplicate.clone();- if let super::super::PassageLocator::LocalNote { chunk_index, .. } = &mut fresh.locator {- *chunk_index += 1;+ if let super::super::PassageLocator::LocalNote(locator) = &mut fresh.locator {+ locator.chunk_index += 1;}fresh.text = "Fresh café🙂 context".into();let full = build_followup_prompt_context(@@ -548,10 +587,9 @@ mod tests {let required = build_followup_prompt_context(std::slice::from_ref(&old), &[], 6000).unwrap().unwrap();- for budget in [- required.text.len(),- required.text.len() + 8,- full.text.len() - 1,+ for (case, budget) in [+ ("required only", required.text.len()),+ ("truncated optional passage", full.text.len() - 1),] {let packed = build_followup_prompt_context(std::slice::from_ref(&old),@@ -560,8 +598,8 @@ mod tests {).unwrap().unwrap();- assert!(packed.text.len() <= budget);- assert_eq!(packed.included_passages[0], old.reference);+ assert!(packed.text.len() <= budget, "{case}");+ assert_eq!(packed.included_passages[0], old.reference, "{case}");}let raw = super::super::passage_fixture().1;@@ -580,14 +618,10 @@ mod tests {.unwrap();assert_eq!(expanded.sources, vec![old.source.clone()]);assert_eq!(expanded.included_passages.len(), 1);- assert!(- expanded- .text- .contains(&super::super::clean_passage_text(raw))- );+ assert!(expanded.text.contains("Café🙂\nबीच\tend"));assert_eq!(expanded.included_passages[0].verified_spans(raw).unwrap(),- vec![super::super::clean_passage_text(raw)]+ ["Café🙂\nबीच\tend"]);let mut prefix = old.clone();@@ -597,7 +631,10 @@ mod tests {build_followup_prompt_context(std::slice::from_ref(&prefix), &[], 6000).unwrap().unwrap();- for budget in required_prefix.text.len()..=expanded.text.len() {+ for (case, budget) in [+ ("required prefix only", required_prefix.text.len()),+ ("truncated expansion", expanded.text.len() - 1),+ ] {let packed = build_followup_prompt_context(std::slice::from_ref(&prefix),std::slice::from_ref(&matching),@@ -605,11 +642,11 @@ mod tests {).unwrap().unwrap();- assert!(packed.text.len() <= budget);- assert_eq!(packed.included_passages.len(), 1);+ assert!(packed.text.len() <= budget, "{case}");+ assert_eq!(packed.included_passages.len(), 1, "{case}");let spans = packed.included_passages[0].verified_spans(raw).unwrap();- assert!(spans[0].starts_with(&prefix.passages[0]));- assert!(packed.text.contains(&spans[0]));+ assert!(spans[0].starts_with(&prefix.passages[0]), "{case}");+ assert!(packed.text.contains(&spans[0]), "{case}");}assert!(build_followup_prompt_context(&[old], &[], required.text.len() - 1).is_err());}diff --git a/rust/crates/ensu/src/retrieval/index.rs b/rust/crates/ensu/src/retrieval/index.rsindex 8cecc68fc9..c420a3f4c1 100644--- a/rust/crates/ensu/src/retrieval/index.rs+++ b/rust/crates/ensu/src/retrieval/index.rs@@ -680,43 +680,36 @@ pub(super) mod tests {}#[test]- fn passage_revisions_are_lazy_and_reloads_reject_changed_artifacts() {+ fn passage_reload_preserves_sources_and_rejects_changed_artifacts() {use crate::retrieval::PassageLocator;let pack = synthetic_pack("simplewiki-test");let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();- assert!(index.revision_sha256.get().is_none());let mut query = vec![0.0; 512];- assert!(index.search(&query, 3, 1.0).unwrap().is_empty());- assert!(index.revision_sha256.get().is_none());query[0] = 1.0;- let hits = index.search(&query, 3, 0.0).unwrap();+ let hits = index.search(&query, 2, 0.0).unwrap();assert_ne!(hits[0].locator, hits[1].locator);- let revision = text_revision_digest(- &fs::read(pack.revision.join(KNOWLEDGE_MANIFEST_FILE)).unwrap(),- &fs::read(pack.revision.join(KNOWLEDGE_META_FILE)).unwrap(),- &fs::read(pack.revision.join(KNOWLEDGE_OFFSETS_FILE)).unwrap(),- );- assert_eq!(index.revision_sha256.get(), Some(&revision));drop(index);let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();- let mut wrong = hits[0].locator.clone();- if let PassageLocator::EnsuPack { row, .. } = &mut wrong {- *row = u64::MAX;- }- assert!(index.reload_passage(&wrong).unwrap().is_none());- wrong = hits[0].locator.clone();- if let PassageLocator::EnsuPack { dataset_id, .. } = &mut wrong {- *dataset_id = "wikibooks".to_string();+ for field in ["row", "dataset"] {+ let mut wrong = hits[0].locator.clone();+ let PassageLocator::EnsuPack {+ dataset_id, row, ..+ } = &mut wrong+ else {+ panic!("expected a pack locator")+ };+ match field {+ "row" => *row = u64::MAX,+ _ => *dataset_id = "wikibooks".to_string(),+ }+ assert!(index.reload_passage(&wrong).unwrap().is_none(), "{field}");}- assert!(index.reload_passage(&wrong).unwrap().is_none());- assert!(index.revision_sha256.get().is_none());- for hit in &hits {+ for (hit, text) in hits.iter().zip(["first passage", "second passage"]) {let reloaded = index.reload_passage(&hit.locator).unwrap().unwrap();assert_eq!(reloaded.locator, hit.locator);- assert_eq!(reloaded.text, hit.text);+ assert_eq!(reloaded.text, text);assert_eq!(reloaded.source_url, hit.source_url);}- assert_eq!(index.revision_sha256.get(), Some(&revision));drop(index);let metadata_path = pack.revision.join(KNOWLEDGE_META_FILE);diff --git a/rust/crates/ensu/src/retrieval/passage.rs b/rust/crates/ensu/src/retrieval/passage.rsindex f957cf2e97..162dc74a4d 100644--- a/rust/crates/ensu/src/retrieval/passage.rs+++ b/rust/crates/ensu/src/retrieval/passage.rs@@ -1,5 +1,7 @@use serde::{Deserialize, Serialize};+use crate::notes::NotePassageLocator;+use super::{BEGIN_CONTEXT_SENTINEL, END_CONTEXT_SENTINEL, GroundedSource, RetrievalError};#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]@@ -14,13 +16,7 @@ pub enum PassageLocator {revision_sha256: String,row: u64,},- LocalNote {- collection_id: String,- document_id: String,- indexed_revision: String,- shard_sha256: String,- chunk_index: u64,- },+ LocalNote(NotePassageLocator),}impl PassageLocator {@@ -29,18 +25,10 @@ impl PassageLocator {(Self::EnsuPack { dataset_id, .. }, GroundedSource::EnsuPack { citation }) => {*dataset_id == citation.dataset_id}- (- Self::LocalNote {- collection_id,- document_id,- indexed_revision,- ..- },- GroundedSource::LocalNote { reference },- ) => {- *collection_id == reference.collection_id- && *document_id == reference.document_id- && *indexed_revision == reference.indexed_revision+ (Self::LocalNote(locator), GroundedSource::LocalNote { reference }) => {+ locator.collection_id == reference.collection_id+ && locator.document_id == reference.document_id+ && locator.indexed_revision == reference.indexed_revision}_ => false,}@@ -87,32 +75,26 @@ impl IncludedPassage {revision_sha256,..} => !dataset_id.is_empty() && dataset_id.len() <= 256 && sha(revision_sha256),- PassageLocator::LocalNote {- collection_id,- document_id,- indexed_revision,- shard_sha256,- ..- } => {- uuid::Uuid::parse_str(collection_id).is_ok()- && crate::notes::validate_document_id(document_id).is_ok()- && sha(indexed_revision)- && sha(shard_sha256)+ PassageLocator::LocalNote(locator) => {+ uuid::Uuid::parse_str(&locator.collection_id).is_ok()+ && crate::notes::validate_document_id(&locator.document_id).is_ok()+ && sha(&locator.indexed_revision)+ && sha(&locator.shard_sha256)}};- let mut previous_end = 0;valid_locator&& self.spans.iter().all(|span| {- let valid = span.start_utf8 >= previous_end- && span.start_utf8 < span.end_utf8+ span.start_utf8 < span.end_utf8&& span.end_utf8 as usize <= MAX_PASSAGE_UTF8_BYTES- && sha(&span.text_sha256);- previous_end = span.end_utf8;- valid+ && sha(&span.text_sha256)})+ && self+ .spans+ .windows(2)+ .all(|pair| pair[0].end_utf8 <= pair[1].start_utf8)}- pub(super) fn prefix(locator: PassageLocator, text: &str) -> Result<Self, RetrievalError> {+ pub(crate) fn prefix(locator: PassageLocator, text: &str) -> Result<Self, RetrievalError> {let end_utf8 = u32::try_from(text.len()).map_err(|_| {RetrievalError::InvalidInput("included passage is too large".to_string())})?;@@ -184,91 +166,71 @@ pub(super) mod tests {use super::*;pub(crate) fn passage_fixture() -> (IncludedPassage, &'static str) {- let reference = serde_json::from_value(serde_json::json!({- "locator": {- "type": "localNote",- "collectionId": "123e4567-e89b-12d3-a456-426614174000",- "documentId": "trip.md",- "indexedRevision": "a".repeat(64),- "shardSha256": "b".repeat(64),- "chunkIndex": 2- },- "cleaningVersion": 1,- "spans": [- {- "startUtf8": 0,- "endUtf8": 9,- "textSha256": "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"+ let reference = IncludedPassage {+ locator: PassageLocator::LocalNote(NotePassageLocator {+ collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),+ document_id: "trip.md".into(),+ indexed_revision: "a".repeat(64),+ shard_sha256: "b".repeat(64),+ chunk_index: 2,+ }),+ cleaning_version: PASSAGE_CLEANING_VERSION,+ spans: vec![+ PassageSpan {+ start_utf8: 0,+ end_utf8: 9,+ text_sha256: "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"+ .into(),},- {- "startUtf8": 20,- "endUtf8": 23,- "textSha256": "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"- }- ]- }))- .unwrap();+ PassageSpan {+ start_utf8: 20,+ end_utf8: 23,+ text_sha256: "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"+ .into(),+ },+ ],+ };(reference, " \r\nCafé🙂\r\nबीच\tend \0\r\n")}#[test]fn passage_references_verify_cleaned_text_and_reject_invalid_spans() {let (fixture, raw) = passage_fixture();- for (raw, cleaned, spans, texts) in [- (- raw,- "Café🙂\nबीच\tend",- fixture.spans.clone(),- vec!["Café🙂", "end"],- ),- (- "\r\n----- BEGIN KNOWLEDGE CONTEXT -----\r\nx\r\n",- "[source] ----- BEGIN KNOWLEDGE CONTEXT -----\nx",- vec![PassageSpan {- start_utf8: 0,- end_utf8: 43,- text_sha256: "73c093471df3c5f2f0e96999d8d5ebc745a803f55d3f36c4523c502c2e055e02"- .into(),- }],- vec!["[source] ----- BEGIN KNOWLEDGE CONTEXT ----"],- ),- (- " a\u{1}b\tc\rd ",- "a b\tc\nd",- vec![PassageSpan {- start_utf8: 0,- end_utf8: 7,- text_sha256: "1205bffe81cb4550cdf063c07e75e290c2ebaf60d77780a338e18eb99d26bd4c"- .into(),- }],- vec!["a b\tc\nd"],- ),- ] {- let reference = IncludedPassage {- spans,- ..fixture.clone()- };- assert_eq!(clean_passage_text(raw), cleaned);- assert_eq!(reference.verified_spans(raw).unwrap(), texts);- }+ assert_eq!(clean_passage_text(raw), "Café🙂\nबीच\tend");+ assert_eq!(fixture.verified_spans(raw).unwrap(), ["Café🙂", "end"]);assert!(fixture.verified_spans(&raw.replace("Café", "Cafe ")).is_none());- for (start, end) in [(0, 4), (4, 9), (9, 0), (0, u32::MAX)] {+ for (case, start, end) in [+ ("end splits UTF-8", 0, 4),+ ("start splits UTF-8", 4, 9),+ ("reversed range", 9, 0),+ ("oversized range", 0, u32::MAX),+ ] {let mut reference = fixture.clone();reference.spans[0].start_utf8 = start;reference.spans[0].end_utf8 = end;- assert!(reference.verified_spans(raw).is_none());+ assert!(reference.verified_spans(raw).is_none(), "{case}");}let mut reference = fixture.clone();reference.cleaning_version = 2;- assert!(reference.verified_spans(raw).is_none());+ assert!(reference.verified_spans(raw).is_none(), "cleaning version");reference = fixture;reference.spans[1] = reference.spans[0].clone();- assert!(reference.verified_spans(raw).is_none());- reference.spans = vec![reference.spans[0].clone(); 9];- assert!(reference.verified_spans(raw).is_none());+ assert!(reference.verified_spans(raw).is_none(), "overlapping spans");+ reference.spans = (0..9)+ .map(|start_utf8| PassageSpan {+ start_utf8,+ end_utf8: start_utf8 + 1,+ text_sha256: "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb"+ .into(),+ })+ .collect();+ assert!(+ reference.verified_spans("aaaaaaaaa").is_none(),+ "span limit"+ );}}diff --git a/web/apps/ensu/src/pages/chat.tsx b/web/apps/ensu/src/pages/chat.tsxindex cdbf17cfa4..97a62b9bca 100644--- a/web/apps/ensu/src/pages/chat.tsx+++ b/web/apps/ensu/src/pages/chat.tsx@@ -3093,10 +3093,9 @@ const Page: React.FC = () => {.text.replaceAll(MEDIA_MARKER, "").replace(/\[\d+ image attachments? provided\]/gi, "").trim();- const selectedMessages = slicePathUntil(- historyPath,- stopAtMessageUuid,- );+ const conversationPath = useConversationMemory+ ? buildConversationPath(allMessages, parentMessage)+ : [];let resolutionToken: string | undefined;const remainingKnowledgeBytes = useConversationMemory? 6000@@ -3146,20 +3145,15 @@ const Page: React.FC = () => {if (useConversationMemory &&- selectedMessages.some((message) => message.sources?.length)+ historyPath.some((message) => message.sources?.length)) {- const selectedPath = selectedMessages.map(- (message) => message.messageUuid,- );- if (selectedPath.at(-1) !== parentMessageUuid)- selectedPath.push(parentMessageUuid);setConversationStatus("Finding sources");try {resolutionToken = await provider.withKnowledgeRetrieval((cancellationEpoch) =>resolveDesktopSourceFollowup({sessionUuid: activeSessionId,- path: selectedPath,+ path: conversationPath,question: knowledgeQuery,enabledStableIds: enabledReadyPackIds,cancellationEpoch,@@ -3244,10 +3238,7 @@ const Page: React.FC = () => {const prepared = await prepareDesktopConversation({sessionUuid: activeSessionId,- path: buildConversationPath(- allMessages,- parentMessage,- ),+ path: conversationPath,system: normalSystemPrompt,current: promptText,historyQuery: resolutionToken@@ -3484,7 +3475,6 @@ const Page: React.FC = () => {enabledKnowledgePackIds,knowledgePacks,loadEnabledKnowledgeCatalogOnce,- slicePathUntil,],);