Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

[ensu] Refine follow-up grounding and simplify tests

setalp2e8fcc66ee91
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index 0ac6a4ff22..46cacb5af9 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -559,7 +559,7 @@ internal class ChatStoreActions(
} else null
}
if (!isActive() || stopRequested) {
- turn?.cancel()
+ closeFollowup(turn)
return@launch
}
followup = turn
@@ -575,14 +575,8 @@ internal class ChatStoreActions(
return@launch
} catch (error: Throwable) {
if (!isActive() || stopRequested) return@launch
- logRepository.log(
- LogLevel.Warning,
- "Unable to prepare source search",
- details = error.message,
- tag = "Chat",
- throwable = error,
- )
- if (showFollowupError(error, userMessage.id)) return@launch
+ if (reportFollowupError(error, userMessage.id, "Unable to prepare source search"))
+ return@launch
}
val retrievalQuery = userMessage.text.trim()
val knowledgeHits =
@@ -652,17 +646,15 @@ internal class ChatStoreActions(
return@launch
} catch (error: Throwable) {
if (!isActive() || stopRequested) return@launch
- logRepository.log(
- LogLevel.Warning,
- "Unable to resolve earlier sources",
- details = error.message,
- tag = "Chat",
- throwable = error,
+ if (
+ reportFollowupError(
+ error,
+ userMessage.id,
+ "Unable to resolve earlier sources",
+ )
)
- if (showFollowupError(error, userMessage.id)) return@launch
- activeFollowup = null
- followup?.cancel()
- followup?.destroy()
+ return@launch
+ closeFollowup(followup)
followup = null
emptyList()
}
@@ -919,15 +911,26 @@ internal class ChatStoreActions(
}
generationJob = activeJob
activeJob.invokeOnCompletion {
- if (activeFollowup === followup) activeFollowup = null
- followup?.cancel()
- followup?.destroy()
+ closeFollowup(followup)
notesScope.close()
scope.launch { settleGenerationIfActive(generationToken, sessionId) }
}
}
- private fun showFollowupError(error: Throwable, messageId: String): Boolean {
+ private fun closeFollowup(followup: ConversationFollowup?) {
+ if (activeFollowup === followup) activeFollowup = null
+ followup?.cancel()
+ followup?.destroy()
+ }
+
+ private fun reportFollowupError(error: Throwable, messageId: String, detail: String): Boolean {
+ logRepository.log(
+ LogLevel.Warning,
+ detail,
+ details = error.message,
+ tag = "Chat",
+ throwable = error,
+ )
val message =
when (error) {
is ConversationException.Stale -> "Conversation changed. Retry the message."
@@ -951,7 +954,7 @@ internal class ChatStoreActions(
when (locator) {
is PassageLocator.EnsuPack ->
knowledgeProvider.reload(locator, state.value.knowledge.enabledReadyDatasets)
- is PassageLocator.LocalNote -> notesStore.reload(locator)
+ is PassageLocator.LocalNote -> notesStore.reload(locator.locator)
}
} catch (error: kotlinx.coroutines.CancellationException) {
throw error
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.kt
index e314addb01..ff957e3313 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesProvider.kt
@@ -7,13 +7,13 @@ import android.provider.DocumentsContract
import android.util.AtomicFile
import androidx.core.content.FileProvider
import io.ente.ensu.bindings.LlmContext
+import io.ente.ensu.bindings.NotePassageLocator
import io.ente.ensu.bindings.NoteSourceReference
import io.ente.ensu.bindings.NotesCancellation
import io.ente.ensu.bindings.NotesCollection
import io.ente.ensu.bindings.NotesException
import io.ente.ensu.bindings.NotesIndexOptions
import io.ente.ensu.bindings.NotesProgressCallback
-import io.ente.ensu.bindings.PassageLocator
import io.ente.ensu.bindings.notesContentRevision
import java.io.ByteArrayOutputStream
import java.io.File
@@ -132,7 +132,7 @@ class NotesProvider(private val context: Context) {
handle(id).use { it.search(query) }
}
- internal suspend fun reload(locator: PassageLocator.LocalNote) = withAccess {
+ internal suspend fun reload(locator: NotePassageLocator) = withAccess {
registration(locator.collectionId)
handle(locator.collectionId).use { it.reloadPassage(locator) }
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt
index 46278812be..1af9823723 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/notes/NotesStore.kt
@@ -6,6 +6,7 @@ import android.net.Uri
import androidx.compose.runtime.staticCompositionLocalOf
import io.ente.ensu.bindings.GroundedExcerpt
import io.ente.ensu.bindings.GroundedSource
+import io.ente.ensu.bindings.NotePassageLocator
import io.ente.ensu.bindings.NoteSourceReference
import io.ente.ensu.bindings.NotesCancellation
import io.ente.ensu.bindings.NotesException
@@ -14,7 +15,6 @@ import io.ente.ensu.bindings.NotesIndexOptions
import io.ente.ensu.bindings.NotesProgress
import io.ente.ensu.bindings.NotesProgressCallback
import io.ente.ensu.bindings.NotesSummary
-import io.ente.ensu.bindings.PassageLocator
import io.ente.ensu.bindings.notesLimits
import io.ente.ensu.bindings.selectMixedGroundingCandidates
import io.ente.ensu.bindings.withNotesCollectionLabel
@@ -281,7 +281,7 @@ class NotesStore(
hits
}
- suspend fun reload(locator: PassageLocator.LocalNote): GroundedExcerpt? =
+ suspend fun reload(locator: NotePassageLocator): GroundedExcerpt? =
withContext(Dispatchers.Main.immediate) {
if (_state.value.collections.none { it.id == locator.collectionId && it.eligible })
return@withContext null
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index cfae99ff4d..a92d4a64be 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -1172,20 +1172,14 @@ final class ChatViewModel: ObservableObject {
do {
if let turn = followup {
- let searched = knowledgeHits
- let locators = try await Task.detached {
+ let locators = try await Task.detached { [searched = knowledgeHits] in
try turn.searchWithHistory(searched: searched)
}.value
try Task.checkCancellation()
guard activeGenerationId == generationId else { return }
for locator in locators {
- do {
- if let hit = try await reloadPassage(locator) {
- reloaded.append(hit)
- }
- } catch {
- if isCancellation(error) { throw error }
- logger.warning("Source reload failed", details: "\(error)")
+ if let hit = try await reloadPassage(locator) {
+ reloaded.append(hit)
}
}
}
@@ -1510,12 +1504,18 @@ final class ChatViewModel: ObservableObject {
}
private func reloadPassage(_ locator: PassageLocator) async throws -> GroundedExcerpt? {
- switch locator {
- case .ensuPack:
- return try await knowledgeProvider.reload(
- locator, datasets: knowledgeStore.enabledReadyDatasets)
- case .localNote:
- return try await notesStore.reload(locator)
+ do {
+ switch locator {
+ case .ensuPack:
+ return try await knowledgeProvider.reload(
+ locator, datasets: knowledgeStore.enabledReadyDatasets)
+ case .localNote(let note):
+ return try await notesStore.reload(note)
+ }
+ } catch {
+ if isCancellation(error) { throw error }
+ logger.warning("Source reload failed", details: "\(error)")
+ return nil
}
}
diff --git a/apple/apps/ensu/Ensu/Services/NotesProvider.swift b/apple/apps/ensu/Ensu/Services/NotesProvider.swift
index d21040809e..7efeba5af5 100644
--- a/apple/apps/ensu/Ensu/Services/NotesProvider.swift
+++ b/apple/apps/ensu/Ensu/Services/NotesProvider.swift
@@ -162,13 +162,12 @@ actor NotesProvider {
}
}
- func reload(_ locator: PassageLocator) async throws -> NotesHit? {
- guard case .localNote(let collectionId, _, _, _, _) = locator else { return nil }
- return try await diskGate.withLock {
- _ = try self.registration(collectionId)
+ func reload(_ locator: NotePassageLocator) async throws -> NotesHit? {
+ try await diskGate.withLock {
+ _ = try self.registration(locator.collectionId)
let root = self.root
return try await Task.detached {
- try Self.handle(root, collectionId).reloadPassage(locator: locator)
+ try Self.handle(root, locator.collectionId).reloadPassage(locator: locator)
}.value
}
}
diff --git a/apple/apps/ensu/Ensu/Services/NotesStore.swift b/apple/apps/ensu/Ensu/Services/NotesStore.swift
index 23546ed3e1..a95ebd4c47 100644
--- a/apple/apps/ensu/Ensu/Services/NotesStore.swift
+++ b/apple/apps/ensu/Ensu/Services/NotesStore.swift
@@ -198,9 +198,8 @@ final class NotesStore: ObservableObject, ModelMaintenance {
return hits
}
- func reload(_ locator: PassageLocator) async throws -> GroundedExcerpt? {
- guard case .localNote(let collectionId, _, _, _, _) = locator,
- collections.contains(where: { $0.id == collectionId && $0.eligible })
+ func reload(_ locator: NotePassageLocator) async throws -> GroundedExcerpt? {
+ guard collections.contains(where: { $0.id == locator.collectionId && $0.eligible })
else { return nil }
do {
guard let hit = try await provider.reload(locator) else { return nil }
@@ -208,7 +207,7 @@ final class NotesStore: ObservableObject, ModelMaintenance {
selectMixedGroundingCandidates(packHits: [], notesHits: [hit], notesLimit: 1)
).first
} catch NotesError.RebuildRequired {
- requestRebuild(collectionId)
+ requestRebuild(locator.collectionId)
return nil
}
}
diff --git a/rust/apps/ensu/src-tauri/src/commands/followup.rs b/rust/apps/ensu/src-tauri/src/commands/followup.rs
index 4e6967c6e7..15b3473521 100644
--- a/rust/apps/ensu/src-tauri/src/commands/followup.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/followup.rs
@@ -236,6 +236,7 @@ mod tests {
.unwrap();
(db, snapshot)
}
+
#[test]
fn preparation_requires_matching_source_selection_and_consumes_it_once() {
let (db, snapshot) = fixture();
@@ -245,39 +246,34 @@ mod tests {
Guard {
token: "token".into(),
db: db.clone(),
- selection: None,
+ selection: Some(Selection {
+ fingerprint: core::fingerprint(snapshot.messages()),
+ question: "Where does she track it?".into(),
+ candidates: core::GroundingCandidates::new(
+ vec![],
+ vec![],
+ core::MAX_GROUNDING_BYTES,
+ )
+ .unwrap(),
+ }),
},
);
- assert!(
- state
- .take_selection("window", "token", &db, &snapshot)
- .is_err()
- );
- state
- .guards
- .lock()
- .unwrap()
- .get_mut("window")
- .unwrap()
- .selection = Some(Selection {
- fingerprint: core::fingerprint(snapshot.messages()),
- question: "Where does she track it?".into(),
- candidates: core::GroundingCandidates::new(vec![], vec![], 6000).unwrap(),
- });
let (other_db, other_snapshot) = fixture();
- for (window, token, db, snapshot) in [
- ("another-window", "token", &db, &snapshot),
- ("window", "another-token", &db, &snapshot),
- ("window", "token", &other_db, &snapshot),
- ("window", "token", &db, &other_snapshot),
+ for (case, window, token, db, snapshot) in [
+ ("window", "another-window", "token", &db, &snapshot),
+ ("token", "window", "another-token", &db, &snapshot),
+ ("database", "window", "token", &other_db, &snapshot),
+ ("history", "window", "token", &db, &other_snapshot),
] {
- assert!(state.take_selection(window, token, db, snapshot).is_err());
+ assert!(
+ state.take_selection(window, token, db, snapshot).is_err(),
+ "{case}"
+ );
}
let selection = state
.take_selection("window", "token", &db, &snapshot)
.unwrap();
assert_eq!(selection.question, "Where does she track it?");
- assert_eq!(selection.candidates.max_utf8_bytes, 6000);
assert!(
state
.take_selection("window", "token", &db, &snapshot)
diff --git a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
index cbacb1a35b..e62a7e25b1 100644
--- a/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/knowledge.rs
@@ -268,6 +268,37 @@ fn select_verified_mixed_grounding(
Ok(verified)
}
+fn select_grounding(
+ app: &AppHandle,
+ pack_hits: &[retrieval::KnowledgePromptHit],
+ note_hits: &[ente_ensu::notes::NotesSearchHit],
+ notes: Option<&super::notes::RetrievalHandle>,
+ mut check_cancelled: impl FnMut() -> Result<(), ApiError>,
+) -> Result<Vec<retrieval::GroundedExcerpt>, ApiError> {
+ let mut excerpts = select_verified_mixed_grounding(pack_hits, note_hits, |reference| {
+ check_cancelled()?;
+ let Some(notes) = notes else { return Ok(false) };
+ let valid = notes.verify_source_reference(reference);
+ if !valid {
+ super::notes::mark_reference_stale(
+ app,
+ &reference.collection_id,
+ reference.document_id.clone(),
+ );
+ }
+ Ok(valid)
+ })?;
+ for excerpt in &mut excerpts {
+ if let (Some(notes), retrieval::GroundedSource::LocalNote { reference }) =
+ (notes, &mut excerpt.source)
+ {
+ reference.collection_label = notes.collection_label(&reference.collection_id);
+ }
+ }
+ check_cancelled()?;
+ Ok(excerpts)
+}
+
fn reconcile_and_open(
store: &AssetStore,
indexes: &Mutex<HashMap<String, OpenIndex>>,
@@ -637,29 +668,7 @@ pub async fn knowledge_retrieve(
check_cancelled()?;
let context_budget = max_context_utf8_bytes.min(embedding.max_context_utf8_bytes);
- let mut excerpts = select_verified_mixed_grounding(&pack_hits, &note_hits, |reference| {
- check_cancelled()?;
- let Some(notes) = &notes else {
- return Ok(false);
- };
- let verified = notes.verify_source_reference(reference);
- if !verified {
- crate::commands::notes::mark_reference_stale(
- &app,
- &reference.collection_id,
- reference.document_id.clone(),
- );
- }
- Ok(verified)
- })?;
- for excerpt in &mut excerpts {
- if let (Some(notes), retrieval::GroundedSource::LocalNote { reference }) =
- (&notes, &mut excerpt.source)
- {
- reference.collection_label = notes.collection_label(&reference.collection_id);
- }
- }
- check_cancelled()?;
+ let excerpts = select_grounding(&app, &pack_hits, &note_hits, notes.as_ref(), check_cancelled)?;
let candidates = ente_ensu::conversation::GroundingCandidates::new(vec![], excerpts, context_budget as usize)
.map_err(|error| ApiError::new("conversation", error.to_string()))?;
@@ -727,12 +736,13 @@ pub(crate) async fn reload_for_followup(
hit,
});
}
- retrieval::PassageLocator::LocalNote { collection_id, .. } => {
+ retrieval::PassageLocator::LocalNote(locator) => {
+ let collection_id = &locator.collection_id;
if !collection_ids.contains(collection_id) {
return Ok(None);
}
let Some(notes) = &notes else { return Ok(None) };
- match notes.reload_passage(&reference.locator) {
+ match notes.reload_passage(locator) {
Ok(Some(hit)) => note_hits.push(hit),
Ok(None) => return Ok(None),
Err(error) => {
@@ -747,31 +757,14 @@ pub(crate) async fn reload_for_followup(
}
}
check_cancelled()?;
- let mut selected = select_verified_mixed_grounding(&packs, &note_hits, |source| {
- check_cancelled()?;
- let valid = notes
- .as_ref()
- .is_some_and(|notes| notes.verify_source_reference(source));
- if !valid {
- crate::commands::notes::mark_reference_stale(
- &app,
- &source.collection_id,
- source.document_id.clone(),
- );
- }
- Ok(valid)
- })?;
- let Some(mut excerpt) = selected.pop() else {
+ let Some(excerpt) =
+ select_grounding(&app, &packs, &note_hits, notes.as_ref(), check_cancelled)?.pop()
+ else {
return Ok(None);
};
let Some(passages) = reference.verified_spans(&excerpt.text) else {
return Ok(None);
};
- if let (Some(notes), retrieval::GroundedSource::LocalNote { reference }) =
- (&notes, &mut excerpt.source)
- {
- reference.collection_label = notes.collection_label(&reference.collection_id);
- }
check_cancelled()?;
Ok(Some(retrieval::ReferencedPassage {
reference,
@@ -811,7 +804,7 @@ mod tests {
fn note(index: usize) -> NotesSearchHit {
NotesSearchHit {
- locator: retrieval::PassageLocator::LocalNote {
+ locator: ente_ensu::notes::NotePassageLocator {
collection_id: COLLECTION_ID.to_owned(),
document_id: format!("note-{index}.md"),
indexed_revision: "a".repeat(64),
diff --git a/rust/apps/ensu/src-tauri/src/commands/notes.rs b/rust/apps/ensu/src-tauri/src/commands/notes.rs
index 57fc7bfb5a..5985860ab9 100644
--- a/rust/apps/ensu/src-tauri/src/commands/notes.rs
+++ b/rust/apps/ensu/src-tauri/src/commands/notes.rs
@@ -682,12 +682,11 @@ impl RetrievalHandle {
pub(crate) fn reload_passage(
&self,
- locator: &ente_ensu::retrieval::PassageLocator,
+ locator: &ente_ensu::notes::NotePassageLocator,
) -> Result<Option<ente_ensu::notes::NotesSearchHit>, NotesError> {
- let ente_ensu::retrieval::PassageLocator::LocalNote { collection_id, .. } = locator else {
- return Ok(None);
- };
- self.with_index(collection_id, |index| index.reload_passage(locator))
+ self.with_index(&locator.collection_id, |index| {
+ index.reload_passage(locator)
+ })
}
fn with_index<T>(
diff --git a/rust/bindings/uniffi/ensu/src/notes.rs b/rust/bindings/uniffi/ensu/src/notes.rs
index cec73c9013..fdaa40d34e 100644
--- a/rust/bindings/uniffi/ensu/src/notes.rs
+++ b/rust/bindings/uniffi/ensu/src/notes.rs
@@ -177,9 +177,42 @@ pub struct NotesOutcome {
pub unchecked_document_ids: Vec<String>,
}
+#[derive(Clone, Debug, uniffi::Record)]
+pub struct NotePassageLocator {
+ pub collection_id: String,
+ pub document_id: String,
+ pub indexed_revision: String,
+ pub shard_sha256: String,
+ pub chunk_index: u64,
+}
+
+impl From<core::NotePassageLocator> for NotePassageLocator {
+ fn from(value: core::NotePassageLocator) -> Self {
+ Self {
+ collection_id: value.collection_id,
+ document_id: value.document_id,
+ indexed_revision: value.indexed_revision,
+ shard_sha256: value.shard_sha256,
+ chunk_index: value.chunk_index,
+ }
+ }
+}
+
+impl From<NotePassageLocator> for core::NotePassageLocator {
+ fn from(value: NotePassageLocator) -> Self {
+ Self {
+ collection_id: value.collection_id,
+ document_id: value.document_id,
+ indexed_revision: value.indexed_revision,
+ shard_sha256: value.shard_sha256,
+ chunk_index: value.chunk_index,
+ }
+ }
+}
+
#[derive(Clone, Debug, uniffi::Record)]
pub struct NotesHit {
- pub locator: crate::retrieval::PassageLocator,
+ pub locator: NotePassageLocator,
pub collection_id: String,
pub document_id: String,
pub revision: String,
@@ -351,7 +384,7 @@ impl NotesCollection {
pub fn reload_passage(
&self,
- locator: crate::retrieval::PassageLocator,
+ locator: NotePassageLocator,
) -> Result<Option<NotesHit>, NotesError> {
core::NotesCollectionIndex::open(&self.root, self.id.clone())
.and_then(|index| index.reload_passage(&locator.into()))
diff --git a/rust/bindings/uniffi/ensu/src/retrieval.rs b/rust/bindings/uniffi/ensu/src/retrieval.rs
index e737ac970e..1c26e6ce55 100644
--- a/rust/bindings/uniffi/ensu/src/retrieval.rs
+++ b/rust/bindings/uniffi/ensu/src/retrieval.rs
@@ -35,11 +35,7 @@ pub enum PassageLocator {
row: u64,
},
LocalNote {
- collection_id: String,
- document_id: String,
- indexed_revision: String,
- shard_sha256: String,
- chunk_index: u64,
+ locator: crate::notes::NotePassageLocator,
},
}
@@ -55,18 +51,8 @@ impl From<core::PassageLocator> for PassageLocator {
revision_sha256,
row,
},
- core::PassageLocator::LocalNote {
- collection_id,
- document_id,
- indexed_revision,
- shard_sha256,
- chunk_index,
- } => Self::LocalNote {
- collection_id,
- document_id,
- indexed_revision,
- shard_sha256,
- chunk_index,
+ core::PassageLocator::LocalNote(locator) => Self::LocalNote {
+ locator: locator.into(),
},
}
}
@@ -84,19 +70,7 @@ impl From<PassageLocator> for core::PassageLocator {
revision_sha256,
row,
},
- PassageLocator::LocalNote {
- collection_id,
- document_id,
- indexed_revision,
- shard_sha256,
- chunk_index,
- } => Self::LocalNote {
- collection_id,
- document_id,
- indexed_revision,
- shard_sha256,
- chunk_index,
- },
+ PassageLocator::LocalNote { locator } => Self::LocalNote(locator.into()),
}
}
}
diff --git a/rust/crates/ensu/src/conversation/followup.rs b/rust/crates/ensu/src/conversation/followup.rs
index b98e304c6f..df85e11cd8 100644
--- a/rust/crates/ensu/src/conversation/followup.rs
+++ b/rust/crates/ensu/src/conversation/followup.rs
@@ -2,36 +2,153 @@ use super::{AnswerEvidence, valid_answer_passages};
use crate::db::Message;
use crate::retrieval::IncludedPassage;
-fn refers_back(question: &str) -> bool {
- question.split(|c: char| !c.is_alphanumeric()).any(|word| {
- matches!(
- word.to_lowercase().as_str(),
- "it" | "its"
- | "this"
- | "that"
- | "these"
- | "those"
- | "my"
- | "mine"
- | "our"
- | "ours"
- | "they"
- | "them"
- | "their"
- | "theirs"
- | "she"
- | "her"
- | "hers"
- | "he"
- | "him"
- | "his"
- | "former"
- | "latter"
- | "above"
- | "previous"
- | "earlier"
- )
- })
+pub(super) fn refers_back(question: &str) -> bool {
+ let mut words = question
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|word| !word.is_empty());
+ while let Some(word) = words.next() {
+ let word = word.to_lowercase();
+ // A time expression is not a source antecedent. Keep checking the rest
+ // of the question: "this year, what does it cover?" still refers back.
+ if matches!(
+ word.as_str(),
+ "this" | "that" | "these" | "those" | "previous" | "earlier"
+ ) && starts_temporal_phrase(words.clone())
+ {
+ continue;
+ }
+ if [
+ "it", "its", "this", "that", "these", "those", "mine", "ours", "they", "them", "their",
+ "theirs", "she", "her", "hers", "he", "him", "his", "former", "latter", "above",
+ "previous", "earlier",
+ ]
+ .contains(&word.as_str())
+ {
+ return true;
+ }
+ }
+ false
+}
+
+fn starts_temporal_phrase<'a>(words: impl Iterator<Item = &'a str>) -> bool {
+ // Bound lookahead, including phrases like "earlier in the previous fiscal
+ // year" and "these past few months", without scanning unrelated clauses.
+ for word in words.take(8) {
+ let word = word.to_lowercase();
+ if [
+ "in",
+ "the",
+ "this",
+ "that",
+ "these",
+ "those",
+ "past",
+ "last",
+ "next",
+ "previous",
+ "earlier",
+ "early",
+ "late",
+ "coming",
+ "current",
+ "calendar",
+ "fiscal",
+ "financial",
+ "academic",
+ "school",
+ "few",
+ "several",
+ "one",
+ "two",
+ "three",
+ "four",
+ "first",
+ "second",
+ "third",
+ "fourth",
+ ]
+ .contains(&word.as_str())
+ || word.bytes().all(|byte| byte.is_ascii_digit())
+ {
+ continue;
+ }
+ // Bare "second" can be an ordinal ("that second policy"), and "may"
+ // can be a verb ("that may differ"). Neither establishes a time phrase.
+ return matches!(word.as_str(), "seconds" | "centuries" | "millennia")
+ || [
+ "minute",
+ "hour",
+ "day",
+ "week",
+ "fortnight",
+ "month",
+ "quarter",
+ "year",
+ "decade",
+ "century",
+ "millennium",
+ "morning",
+ "afternoon",
+ "evening",
+ "night",
+ "weekend",
+ "today",
+ "tonight",
+ "yesterday",
+ "tomorrow",
+ "spring",
+ "summer",
+ "autumn",
+ "fall",
+ "winter",
+ "monday",
+ "tuesday",
+ "wednesday",
+ "thursday",
+ "friday",
+ "saturday",
+ "sunday",
+ "january",
+ "february",
+ "march",
+ "april",
+ "june",
+ "july",
+ "august",
+ "september",
+ "october",
+ "november",
+ "december",
+ ]
+ .contains(&word.strip_suffix('s').unwrap_or(&word));
+ }
+ false
+}
+
+fn has_ownership_hint(question: &str) -> bool {
+ question
+ .split(|c: char| !c.is_alphanumeric())
+ .any(|word| matches!(word.to_lowercase().as_str(), "my" | "our"))
+}
+
+pub(super) fn compares_owned_source(question: &str) -> bool {
+ has_ownership_hint(question)
+ && question.split(|c: char| !c.is_alphanumeric()).any(|word| {
+ matches!(
+ word.to_lowercase().as_str(),
+ "compare"
+ | "compared"
+ | "comparing"
+ | "comparison"
+ | "versus"
+ | "vs"
+ | "differ"
+ | "differs"
+ | "difference"
+ | "differences"
+ | "than"
+ )
+ })
}
pub fn direct_followup_references(
@@ -56,13 +173,15 @@ pub fn direct_followup_references(
question,
prior.iter().chain(searched.iter().map(|hit| &hit.source)),
);
- if !refers_back(question)
+ // Ownership can still refer to an earlier source ("compare Juniper with my
+ // policy"). Keep it eligible for reload; fitting determines whether the
+ // query is a comparison that needs historical priority.
+ let may_refer_back = refers_back(question) || has_ownership_hint(question);
+ if !may_refer_back
&& !named.is_empty()
- && named.iter().all(|key| {
- searched
- .iter()
- .any(|hit| super::grounding::source_key(&hit.source) == *key)
- })
+ && named
+ .iter()
+ .all(|key| searched.iter().any(|hit| hit.source.document_key() == *key))
{
Vec::new()
} else {
diff --git a/rust/crates/ensu/src/conversation/grounding.rs b/rust/crates/ensu/src/conversation/grounding.rs
index 11695c5c69..6e3c17a2c5 100644
--- a/rust/crates/ensu/src/conversation/grounding.rs
+++ b/rust/crates/ensu/src/conversation/grounding.rs
@@ -1,5 +1,6 @@
use super::*;
use crate::retrieval::{self, GroundedExcerpt, GroundedPromptContext, ReferencedPassage};
+use std::collections::{BTreeMap, BTreeSet};
pub const MAX_GROUNDING_BYTES: usize = 6000;
pub const MAX_CANDIDATE_BYTES: usize = 64 * 1024;
@@ -12,24 +13,10 @@ pub struct GroundingCandidates {
pub max_utf8_bytes: usize,
}
-pub(super) fn source_key(source: &retrieval::GroundedSource) -> (u8, String, String) {
- match source {
- retrieval::GroundedSource::LocalNote { reference } => (
- 0,
- reference.collection_id.clone(),
- reference.document_id.clone(),
- ),
- retrieval::GroundedSource::EnsuPack { citation } => {
- (1, citation.dataset_id.clone(), citation.source_url.clone())
- }
- }
-}
-
pub(super) fn named_source_keys<'a>(
query: &str,
sources: impl Iterator<Item = &'a retrieval::GroundedSource>,
-) -> std::collections::BTreeSet<(u8, String, String)> {
- use std::collections::{BTreeMap, BTreeSet};
+) -> BTreeSet<(u8, &'a str, &'a str)> {
let words = |text: &str| -> BTreeSet<String> {
text.split(|c: char| !c.is_alphanumeric())
.filter(|s| s.chars().count() >= 3)
@@ -56,7 +43,7 @@ pub(super) fn named_source_keys<'a>(
let title = tokens(title);
for (start, window) in query_tokens.windows(title.len()).enumerate() {
if window == title {
- title_matches.push((source_key(source), start, start + title.len()));
+ title_matches.push((source.document_key(), start, start + title.len()));
}
}
}
@@ -64,7 +51,7 @@ pub(super) fn named_source_keys<'a>(
label_words.extend(words(&reference.document_id));
}
documents
- .entry(source_key(source))
+ .entry(source.document_key())
.or_default()
.extend(label_words);
}
@@ -88,88 +75,302 @@ pub(super) fn named_source_keys<'a>(
== 1
})
})
- .map(|(key, _)| key.clone())
+ .map(|(key, _)| *key)
.collect()
}
+enum PassageSlot {
+ Historical {
+ minimum: Box<ReferencedPassage>,
+ current: Option<GroundedExcerpt>,
+ },
+ Current(GroundedExcerpt),
+}
+
+impl PassageSlot {
+ fn locator(&self) -> &retrieval::PassageLocator {
+ match self {
+ Self::Historical { minimum, .. } => &minimum.reference.locator,
+ Self::Current(current) => &current.locator,
+ }
+ }
+
+ fn source(&self) -> &retrieval::GroundedSource {
+ match self {
+ Self::Historical { minimum, .. } => &minimum.source,
+ Self::Current(current) => &current.source,
+ }
+ }
+
+ fn minimum(&self) -> Option<&ReferencedPassage> {
+ match self {
+ Self::Historical { minimum, .. } => Some(minimum),
+ Self::Current(_) => None,
+ }
+ }
+
+ fn current(&self) -> Option<&GroundedExcerpt> {
+ match self {
+ Self::Historical { current, .. } => current.as_ref(),
+ Self::Current(current) => Some(current),
+ }
+ }
+
+ fn minimum_coverage(&self) -> Result<ReferencedPassage, PrepareError> {
+ let current = match self {
+ Self::Historical {
+ minimum,
+ current: None,
+ } => return Ok(minimum.as_ref().clone()),
+ Self::Historical {
+ current: Some(current),
+ ..
+ }
+ | Self::Current(current) => current,
+ };
+ let text = retrieval::clean_passage_text(&current.text);
+ let end = if let Some(old) = self.minimum() {
+ let end = old
+ .reference
+ .spans
+ .last()
+ .ok_or_else(|| {
+ PrepareError::Backend("Historical source has no verified spans".into())
+ })?
+ .end_utf8 as usize;
+ if end >= old.passages.join("\n[omitted source text]\n").len() {
+ return Ok(old.clone());
+ }
+ end
+ } else {
+ text.chars()
+ .next()
+ .ok_or_else(|| PrepareError::Backend("Current source has no usable text".into()))?
+ .len_utf8()
+ };
+ let passage = text[..end].to_owned();
+ let reference = retrieval::IncludedPassage::prefix(current.locator.clone(), &passage)
+ .map_err(retrieval_error)?;
+ Ok(ReferencedPassage {
+ reference,
+ source: self.source().clone(),
+ passages: vec![passage],
+ })
+ }
+}
+
+pub(super) struct GroundingPlan {
+ slots: Vec<PassageSlot>,
+ incidental: Vec<ReferencedPassage>,
+ searched: Vec<GroundedExcerpt>,
+ max_utf8_bytes: usize,
+}
+
+impl GroundingPlan {
+ fn freeze(
+ &self,
+ context: Option<&GroundedPromptContext>,
+ ) -> Result<Vec<ReferencedPassage>, PrepareError> {
+ let Some(context) = context else {
+ return Ok(Vec::new());
+ };
+ context
+ .included_passages
+ .iter()
+ .zip(&context.sources)
+ .map(|(reference, source)| {
+ let passages = self
+ .searched
+ .iter()
+ .filter(|hit| {
+ hit.locator == reference.locator && hit.source.same_document(source)
+ })
+ .find_map(|hit| reference.verified_spans(&hit.text))
+ .or_else(|| {
+ self.slots
+ .iter()
+ .filter_map(|slot| slot.minimum())
+ .chain(&self.incidental)
+ .find(|old| {
+ old.reference == *reference && old.source.same_document(source)
+ })
+ .map(|old| old.passages.clone())
+ })
+ .ok_or_else(|| {
+ PrepareError::Backend("Fitted source coverage could not be verified".into())
+ })?;
+ Ok(ReferencedPassage {
+ reference: reference.clone(),
+ source: source.clone(),
+ passages,
+ })
+ })
+ .collect()
+ }
+}
+
+struct SourceGroup<'a> {
+ key: (u8, &'a str, &'a str),
+ slots: Vec<PassageSlot>,
+}
+
+fn source_group<'a, 'b>(
+ groups: &'a mut Vec<SourceGroup<'b>>,
+ source: &'b retrieval::GroundedSource,
+) -> &'a mut SourceGroup<'b> {
+ let key = source.document_key();
+ let index = groups
+ .iter()
+ .position(|group| group.key == key)
+ .unwrap_or_else(|| {
+ groups.push(SourceGroup {
+ key,
+ slots: Vec::new(),
+ });
+ groups.len() - 1
+ });
+ &mut groups[index]
+}
+
+fn retrieval_error(error: retrieval::RetrievalError) -> PrepareError {
+ PrepareError::Backend(error.to_string())
+}
+
impl GroundingCandidates {
- pub(super) fn protect_named_sources(&mut self, query: &str) -> Result<(), PrepareError> {
+ pub(super) fn select_sources(
+ &self,
+ query: Option<&str>,
+ ) -> Result<GroundingPlan, PrepareError> {
self.validate()?;
- if query.len() > super::lookup::LOOKUP_MAX_QUERY_BYTES {
- return Ok(());
- }
+ let query = query.filter(|query| query.len() <= super::lookup::LOOKUP_MAX_QUERY_BYTES);
let named = named_source_keys(
- query,
+ query.unwrap_or_default(),
self.referenced
.iter()
.map(|r| &r.source)
.chain(self.searched.iter().map(|r| &r.source)),
);
- let fresh_index = if !self.referenced.is_empty() && named.is_empty() {
- self.searched.iter().position(|hit| {
- !hit.text.trim().is_empty()
- && !self
- .referenced
- .iter()
- .any(|r| r.reference.locator == hit.locator)
- })
- } else {
- None
- };
- if let Some(index) = fresh_index {
- self.searched[..=index].rotate_right(1);
- }
- let mut required: std::collections::BTreeSet<_> = self
+ let historical: BTreeSet<_> = self
.referenced
.iter()
- .map(|r| source_key(&r.source))
+ .map(|old| old.source.document_key())
.collect();
- let mut promoted = Vec::new();
- for (index, hit) in self.searched.iter().enumerate() {
- let fresh = fresh_index.is_some() && index == 0;
- let id = source_key(&hit.source);
- if !fresh && (!named.contains(&id) || required.contains(&id)) {
- continue;
+ let compares_owned_source = query.is_some_and(super::followup::compares_owned_source)
+ && self
+ .searched
+ .iter()
+ .any(|hit| named.contains(&hit.source.document_key()));
+ let refers_back = !historical.is_empty()
+ && (query.is_some_and(super::followup::refers_back)
+ || compares_owned_source
+ || self
+ .searched
+ .iter()
+ .all(|hit| retrieval::clean_passage_text(&hit.text).is_empty()));
+ let mut searched = self.searched.clone();
+ searched.sort_by_key(|hit| {
+ self.referenced
+ .iter()
+ .any(|old| old.reference.locator == hit.locator)
+ });
+ let mut groups: Vec<SourceGroup> = Vec::new();
+ for old in &self.referenced {
+ let group = source_group(&mut groups, &old.source);
+ if !group
+ .slots
+ .iter()
+ .any(|slot| slot.locator() == &old.reference.locator)
+ {
+ group.slots.push(PassageSlot::Historical {
+ minimum: Box::new(old.clone()),
+ current: None,
+ });
}
- let budgets = if fresh {
- [self.max_utf8_bytes / 2, self.max_utf8_bytes]
- } else {
- [usize::MAX; 2]
- };
- let Some(reference) = budgets.into_iter().find_map(|budget| {
- retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), budget)
- .ok()
- .flatten()?
- .included_passages
- .into_iter()
- .next()
- }) else {
- continue;
- };
- let Some(passages) = reference.verified_spans(&hit.text) else {
+ }
+ for hit in &searched {
+ if let Some(slot) = groups
+ .iter_mut()
+ .flat_map(|group| &mut group.slots)
+ .find(|slot| slot.locator() == &hit.locator)
+ {
+ if let PassageSlot::Historical { minimum, current } = slot
+ && current.is_none()
+ && minimum.source.same_document(&hit.source)
+ && minimum.reference.verified_spans(&hit.text).is_some()
+ {
+ *current = Some(hit.clone());
+ }
continue;
- };
- required.insert(id);
- promoted.push(ReferencedPassage {
- reference,
- source: hit.source.clone(),
- passages,
+ }
+ let key = hit.source.document_key();
+ let selected = query.is_some()
+ && (named.contains(&key)
+ || (!historical.is_empty()
+ && (named.is_empty() || groups.iter().any(|group| group.key == key))));
+ if selected && !retrieval::clean_passage_text(&hit.text).is_empty() {
+ source_group(&mut groups, &hit.source)
+ .slots
+ .push(PassageSlot::Current(hit.clone()));
+ }
+ }
+ if query.is_some() {
+ for group in &mut groups {
+ group.slots.sort_by_key(|slot| {
+ searched
+ .iter()
+ .position(|hit| &hit.locator == slot.locator())
+ .unwrap_or(usize::MAX)
+ });
+ }
+ let antecedent = self
+ .referenced
+ .iter()
+ .find(|old| named.contains(&old.source.document_key()))
+ .or_else(|| self.referenced.first())
+ .map(|old| old.source.document_key());
+ groups.sort_by_key(|group| {
+ let current = searched
+ .iter()
+ .position(|hit| hit.source.document_key() == group.key);
+ if named.contains(&group.key) {
+ (0, 0)
+ } else if refers_back && antecedent.as_ref() == Some(&group.key) {
+ (1, 0)
+ } else if refers_back && historical.contains(&group.key) {
+ current.map_or((3, 0), |index| (2, index))
+ } else if named.is_empty()
+ && let Some(index) = current
+ {
+ (4, index)
+ } else {
+ (5, 0)
+ }
});
}
-
- let (mut named_references, incidental): (Vec<_>, Vec<_>) =
- std::mem::take(&mut self.referenced)
- .into_iter()
- .partition(|r| named.contains(&source_key(&r.source)));
- named_references.extend(promoted);
- let mut seen = std::collections::BTreeSet::new();
- let (distinct, additional): (Vec<_>, Vec<_>) = named_references
- .into_iter()
- .partition(|r| seen.insert(source_key(&r.source)));
+ let mut prioritized = Vec::new();
+ for (group_index, group) in groups.into_iter().enumerate() {
+ let primary = query.is_none()
+ || named.contains(&group.key)
+ || (refers_back && historical.contains(&group.key));
+ for (slot_index, slot) in group.slots.into_iter().enumerate() {
+ let primary =
+ primary || (!refers_back && named.is_empty() && slot.current().is_some());
+ prioritized.push(((!primary, slot_index > 0, group_index), slot));
+ }
+ }
+ prioritized.sort_by_key(|(priority, _)| *priority);
+ searched.sort_by_key(|hit| {
+ prioritized
+ .iter()
+ .position(|(_, slot)| slot.locator() == &hit.locator)
+ .unwrap_or(usize::MAX)
+ });
let mut packs = 0;
let mut notes = 0;
- for reference in distinct.into_iter().chain(additional).chain(incidental) {
- let (count, limit) = match &reference.source {
+ let mut slots = Vec::new();
+ let mut incidental = Vec::new();
+ for ((is_incidental, _, _), slot) in prioritized {
+ let (count, limit) = match slot.source() {
retrieval::GroundedSource::EnsuPack { .. } => {
(&mut packs, retrieval::MAX_PACK_HITS)
}
@@ -180,18 +381,23 @@ impl GroundingCandidates {
if *count == limit {
continue;
}
- self.referenced.push(reference);
- if self.validate().is_err() {
- self.referenced.pop();
- } else {
- *count += 1;
+ *count += 1;
+ if !is_incidental {
+ slots.push(slot);
+ } else if let PassageSlot::Historical { minimum, .. } = slot {
+ incidental.push(*minimum);
}
}
- Ok(())
+ Ok(GroundingPlan {
+ slots,
+ incidental,
+ searched,
+ max_utf8_bytes: self.max_utf8_bytes,
+ })
}
pub fn new(
- mut referenced: Vec<ReferencedPassage>,
+ referenced: Vec<ReferencedPassage>,
mut searched: Vec<GroundedExcerpt>,
max_utf8_bytes: usize,
) -> Result<Self, PrepareError> {
@@ -203,16 +409,6 @@ impl GroundingCandidates {
}
hit.text.truncate(end);
}
- referenced.truncate(retrieval::MAX_GROUNDING_HITS);
- while referenced
- .iter()
- .flat_map(|r| &r.passages)
- .map(String::len)
- .sum::<usize>()
- > MAX_GROUNDING_BYTES
- {
- referenced.pop();
- }
let result = Self {
referenced,
searched,
@@ -241,9 +437,7 @@ impl GroundingCandidates {
.referenced
.iter()
.flat_map(|r| &r.passages)
- .map(String::len)
- .sum::<usize>()
- > MAX_GROUNDING_BYTES
+ .any(|passage| passage.len() > MAX_GROUNDING_BYTES)
|| serde_json::to_vec(self)
.map_err(|e| PrepareError::Backend(e.to_string()))?
.len()
@@ -253,6 +447,12 @@ impl GroundingCandidates {
"Source candidates exceed the supported preparation limits".into(),
));
}
+ retrieval::build_followup_prompt_context(&self.referenced, &[], usize::MAX)
+ .map_err(retrieval_error)?;
+ for hit in &self.searched {
+ retrieval::build_grounded_prompt_context(std::slice::from_ref(hit), usize::MAX)
+ .map_err(retrieval_error)?;
+ }
Ok(())
}
@@ -262,7 +462,7 @@ impl GroundingCandidates {
&self.searched,
budget.min(self.max_utf8_bytes),
)
- .map_err(|e| PrepareError::Backend(e.to_string()))
+ .map_err(retrieval_error)
}
}
@@ -281,7 +481,7 @@ pub(super) fn grounded_system(system: &str, context: Option<&GroundedPromptConte
}
pub(super) fn fit_grounding_with_history(
- candidates: &GroundingCandidates,
+ candidates: &GroundingPlan,
history: Option<&HistoryLookup>,
system: &str,
current: &str,
@@ -308,13 +508,12 @@ pub(super) fn fit_grounding_with_history(
}
fn fit_grounding(
- candidates: &GroundingCandidates,
+ candidates: &GroundingPlan,
system: &str,
current: &str,
input_budget: usize,
mut measure: impl FnMut(&[ChatMessage]) -> Result<usize, PrepareError>,
) -> Result<FittedGrounding, PrepareError> {
- candidates.validate()?;
let base = measure(&answer_messages(system, None, &[], current))?;
let room = input_budget
.checked_sub(base)
@@ -327,71 +526,130 @@ fn fit_grounding(
current,
))
};
- let original_references = candidates.referenced.len();
- let mut candidates = candidates.clone();
- let (initial, required, required_tokens) = loop {
- let packed = candidates.pack(candidates.max_utf8_bytes);
- let required = retrieval::build_followup_prompt_context(
- &candidates.referenced,
- &[],
- candidates.max_utf8_bytes,
- );
- if let (Ok(initial), Ok(required)) = (packed, required) {
- let tokens = count_context(required.as_ref())?.saturating_sub(base);
- if tokens <= room {
- break (initial, required, tokens);
- }
+ let mut selected = candidates
+ .slots
+ .iter()
+ .map(PassageSlot::minimum_coverage)
+ .collect::<Result<Vec<_>, _>>()?;
+ let preferred_hits = |count: usize| -> Vec<GroundedExcerpt> {
+ candidates.slots[..count]
+ .iter()
+ .filter_map(|slot| slot.current().cloned())
+ .collect()
+ };
+ let preferred = retrieval::build_followup_prompt_context(
+ &selected,
+ &preferred_hits(selected.len()),
+ usize::MAX,
+ )
+ .map_err(retrieval_error)?;
+ let preferred_tokens = count_context(preferred.as_ref())?.saturating_sub(base);
+ let allowance = 1500.min(input_budget / 4).max(preferred_tokens).min(room);
+ let ceiling = base.saturating_add(allowance).min(input_budget);
+
+ let (minimum, minimum_count) = loop {
+ let minimum = retrieval::build_followup_prompt_context(&selected, &[], usize::MAX)
+ .map_err(retrieval_error)?;
+ let count = count_context(minimum.as_ref())?;
+ if minimum
+ .as_ref()
+ .is_none_or(|context| context.text.len() <= candidates.max_utf8_bytes)
+ && count <= ceiling
+ {
+ break (minimum, count);
}
- if candidates.referenced.pop().is_none() {
- break (None, None, 0);
+ if selected.pop().is_none() {
+ return Err(PrepareError::CurrentInputTooLarge);
}
};
- let allowance = 1500.min(input_budget / 4).max(required_tokens).min(room);
- let initial_count = count_context(initial.as_ref())?;
- let initial_tokens = initial_count.saturating_sub(base);
- if initial_count <= input_budget && initial_tokens <= allowance {
- return Ok(FittedGrounding {
- context: initial,
- repacked: candidates.referenced.len() != original_references,
- evidence_tokens: initial_tokens,
- history_included: false,
- });
- }
- let required_bytes = required.as_ref().map_or(0, |c| c.text.len());
- let initial_bytes = initial.as_ref().map_or(0, |c| c.text.len());
- let extra_tokens = initial_tokens.saturating_sub(required_tokens).max(1);
- let extra_allowance = allowance.saturating_sub(required_tokens);
- let smaller = required_bytes
- + initial_bytes
- .saturating_sub(required_bytes)
- .saturating_mul(extra_allowance)
- .saturating_mul(4)
- / extra_tokens
- / 5;
- let packed = candidates
- .pack(smaller.min(initial_bytes.saturating_sub(1)))
- .ok()
- .flatten();
- if let Some(packed) = packed {
- let final_count = count_context(Some(&packed))?;
- let final_tokens = final_count.saturating_sub(base);
- if final_count <= input_budget && final_tokens <= allowance {
- return Ok(FittedGrounding {
- context: Some(packed),
- repacked: true,
- evidence_tokens: final_tokens,
- history_included: false,
- });
+ let demoted = selected.len() != candidates.slots.len();
+ let (preferred, preferred_count, reduced) = fit_coverage(
+ &selected,
+ &preferred_hits(selected.len()),
+ candidates.max_utf8_bytes,
+ minimum,
+ minimum_count,
+ ceiling,
+ &mut count_context,
+ )?;
+ let (context, count, optional_reduced) = if reduced {
+ (preferred, preferred_count, false)
+ } else {
+ let mut fixed = candidates.freeze(preferred.as_ref())?;
+ let mut context = preferred;
+ let mut count = preferred_count;
+ for old in &candidates.incidental {
+ fixed.push(old.clone());
+ let extended = retrieval::build_followup_prompt_context(&fixed, &[], usize::MAX)
+ .map_err(retrieval_error)?;
+ if extended
+ .as_ref()
+ .is_some_and(|context| context.text.len() > candidates.max_utf8_bytes)
+ {
+ fixed.pop();
+ continue;
+ }
+ let extended_count = count_context(extended.as_ref())?;
+ if extended_count > ceiling {
+ fixed.pop();
+ continue;
+ }
+ context = extended;
+ count = extended_count;
}
- }
+ fit_coverage(
+ &fixed,
+ &candidates.searched,
+ candidates.max_utf8_bytes,
+ context,
+ count,
+ ceiling,
+ &mut count_context,
+ )?
+ };
Ok(FittedGrounding {
- context: required,
- repacked: true,
- evidence_tokens: required_tokens,
+ context,
+ repacked: demoted || reduced || optional_reduced,
+ evidence_tokens: count.saturating_sub(base),
history_included: false,
})
}
+fn fit_coverage(
+ selected: &[ReferencedPassage],
+ searched: &[GroundedExcerpt],
+ max_bytes: usize,
+ minimum: Option<GroundedPromptContext>,
+ minimum_count: usize,
+ ceiling: usize,
+ count_context: &mut impl FnMut(Option<&GroundedPromptContext>) -> Result<usize, PrepareError>,
+) -> Result<(Option<GroundedPromptContext>, usize, bool), PrepareError> {
+ let full = retrieval::build_followup_prompt_context(selected, searched, max_bytes)
+ .map_err(retrieval_error)?;
+ let full_count = count_context(full.as_ref())?;
+ if full_count <= ceiling {
+ return Ok((full, full_count, false));
+ }
+ let mut lower = minimum.as_ref().map_or(0, |context| context.text.len());
+ let mut upper = full.as_ref().map_or(0, |context| context.text.len());
+ let mut best = minimum;
+ let mut best_count = minimum_count;
+ while lower + 1 < upper {
+ let budget = lower + (upper - lower) / 2;
+ let packed = retrieval::build_followup_prompt_context(selected, searched, budget)
+ .map_err(retrieval_error)?;
+ let count = count_context(packed.as_ref())?;
+ if count <= ceiling {
+ lower = budget;
+ best = packed;
+ best_count = count;
+ } else {
+ upper = budget;
+ }
+ }
+ Ok((best, best_count, true))
+}
+
#[cfg(test)]
#[path = "grounding_tests.rs"]
mod tests;
diff --git a/rust/crates/ensu/src/conversation/grounding_tests.rs b/rust/crates/ensu/src/conversation/grounding_tests.rs
index 5624796852..87f055671d 100644
--- a/rust/crates/ensu/src/conversation/grounding_tests.rs
+++ b/rust/crates/ensu/src/conversation/grounding_tests.rs
@@ -1,16 +1,16 @@
use super::*;
-use crate::notes::NoteSourceReference;
+use crate::notes::{NotePassageLocator, NoteSourceReference};
use crate::retrieval::{GroundedSource, PassageLocator};
fn hit(text: &str, id: &str) -> GroundedExcerpt {
GroundedExcerpt {
- locator: PassageLocator::LocalNote {
+ locator: PassageLocator::LocalNote(NotePassageLocator {
collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),
document_id: id.into(),
indexed_revision: "a".repeat(64),
shard_sha256: "b".repeat(64),
chunk_index: 0,
- },
+ }),
source: GroundedSource::LocalNote {
reference: NoteSourceReference {
collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),
@@ -88,7 +88,8 @@ fn fit_followup(
query: &str,
input_budget: usize,
) -> GroundedPromptContext {
- let (history, evidence) = followup_history(prior);
+ let (mut history, evidence) = followup_history(prior);
+ history.last_mut().unwrap().text = query.into();
let reloaded = direct_followup_references(&history, &evidence, query, &searched)
.into_iter()
.map(|reference| {
@@ -103,12 +104,17 @@ fn fit_followup(
}
})
.collect();
- let mut candidates = GroundingCandidates::new(reloaded, searched, MAX_GROUNDING_BYTES).unwrap();
- candidates.protect_named_sources(query).unwrap();
- let context = fit_grounding(&candidates, "System", query, input_budget, measure)
- .unwrap()
- .context
- .unwrap();
+ let candidates = GroundingCandidates::new(reloaded, searched, MAX_GROUNDING_BYTES).unwrap();
+ let context = fit_grounding(
+ &candidates.select_sources(Some(query)).unwrap(),
+ "System",
+ query,
+ input_budget,
+ measure,
+ )
+ .unwrap()
+ .context
+ .unwrap();
assert!(context.text.len() <= MAX_GROUNDING_BYTES);
assert!(
measure(&answer_messages(
@@ -123,46 +129,66 @@ fn fit_followup(
context
}
-#[test]
-fn followup_fitting_keeps_implicit_antecedents_and_allows_named_switches() {
- let cedar = hit("Cedar allows 30 days.", "cedar.md");
- let juniper = hit("Juniper allows 21 days.", "juniper.md");
- let old = retrieval::build_grounded_prompt_context(std::slice::from_ref(&cedar), 6000)
- .unwrap()
+fn notes_at_limit() -> Vec<GroundedExcerpt> {
+ ["cedar", "pine", "birch", "oak", "maple"]
+ .iter()
+ .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))
+ .collect()
+}
+
+fn assert_full_passage(context: &GroundedPromptContext, hit: &GroundedExcerpt) {
+ let index = context
+ .included_passages
+ .iter()
+ .position(|reference| reference.locator == hit.locator)
.unwrap();
- for query in [
- "Is Juniper longer than that?",
- "Compare its deadline with Juniper.",
- "Is Juniper's return period longer than ours?",
- "Is Juniper's return period longer than mine?",
- "Compare Juniper's return period with our policy.",
- "Compare Juniper's return period with my policy.",
- "Does she get more time than Juniper?",
- "Compare her deadline with Juniper.",
- "Is Juniper's deadline longer than hers?",
- "Does he get more time than Juniper?",
- "Does Juniper get more time than him?",
- "Compare his deadline with Juniper.",
+ assert_eq!(context.sources[index], hit.source);
+ assert_eq!(
+ context.included_passages[index]
+ .verified_spans(&hit.text)
+ .unwrap(),
+ vec![hit.text.trim()]
+ );
+ assert!(context.text.contains(hit.text.trim()));
+}
+
+#[test]
+fn comparisons_retain_historical_passages_under_pressure() {
+ let cedar = hit("Cedar permits 30 days.", "cedar.md");
+ let juniper = hit(
+ &format!("{}Juniper permits 21 days.", "Juniper details. ".repeat(30)),
+ "juniper.md",
+ );
+ for (query, budget) in [
+ ("Compare its deadline with Juniper.", 7000),
+ ("Compare Juniper's return period with my policy.", 700),
+ ("Compare Cedar and Juniper.", 700),
] {
let context = fit_followup(
std::slice::from_ref(&cedar),
vec![juniper.clone()],
query,
- 7000,
- );
- assert!(context.text.contains(&cedar.text), "{query}");
- assert!(context.text.contains(&juniper.text), "{query}");
- assert!(
- context
- .included_passages
- .contains(&old.included_passages[0])
+ budget,
);
+ assert_full_passage(&context, &cedar);
+ if budget == 7000 {
+ assert_full_passage(&context, &juniper);
+ }
}
- let updated = hit("Cedar now requires seven days.", "cedar.md");
+}
+
+#[test]
+fn named_questions_switch_to_fresh_sources() {
+ let cedar = hit("Cedar allows 30 days.", "cedar.md");
for (query, fresh) in [
- ("What is Juniper's return period?", &juniper),
- ("What is within Juniper's coverage?", &juniper),
- ("What is Cedar's current deadline?", &updated),
+ (
+ "What is Juniper's return period?",
+ hit("Juniper allows 21 days.", "juniper.md"),
+ ),
+ (
+ "What is Cedar's current deadline?",
+ hit("Cedar now requires seven days.", "cedar.md"),
+ ),
] {
let context = fit_followup(
std::slice::from_ref(&cedar),
@@ -170,104 +196,200 @@ fn followup_fitting_keeps_implicit_antecedents_and_allows_named_switches() {
query,
7000,
);
- assert_eq!(context.sources, vec![fresh.source.clone()], "{query}");
- assert!(context.text.contains(&fresh.text), "{query}");
+ assert_eq!(context.sources.len(), 1);
+ assert_full_passage(&context, &fresh);
}
}
#[test]
-fn followup_fitting_prioritizes_named_sources_at_source_and_byte_limits() {
- let mut prior: Vec<_> = ["pine", "birch", "oak", "maple", "cedar"]
- .iter()
- .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))
- .collect();
+fn named_sources_take_priority_at_source_and_byte_limits() {
+ let mut notes = notes_at_limit();
+ notes.rotate_left(1);
+ let cedar = notes.last().unwrap().clone();
let atlas = hit(&"Atlas allows 21 days. ".repeat(40), "atlas.md");
- let note_count = prior.len();
- prior.extend([
- pack_hit("Mars is the fourth planet.", "Mars", 0),
- pack_hit("Venus is the second planet.", "Venus", 1),
- ]);
- let byte_limited = [
- prior[note_count - 1].clone(),
- hit(&"Unrelated detail. ".repeat(400), "pine.md"),
- ];
- for prior in [&prior[..note_count], &prior[..], &byte_limited] {
- let context = fit_followup(prior, vec![atlas.clone()], "Compare Cedar and Atlas", 7000);
- assert!(context.text.contains("cedar allows 30 days."));
- assert!(context.text.contains(atlas.text.trim()));
- assert!(context.sources.len() <= retrieval::MAX_GROUNDING_HITS);
+ for prior in [
+ notes,
+ vec![
+ cedar.clone(),
+ hit(&"Unrelated detail. ".repeat(400), "pine.md"),
+ ],
+ ] {
+ let context = fit_followup(&prior, vec![atlas.clone()], "Compare Cedar and Atlas", 7000);
+ assert_full_passage(&context, &cedar);
+ assert_full_passage(&context, &atlas);
+ assert!(context.sources.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);
}
+}
- let mars = pack_hit("Mars is the fourth planet.", "Mars", 0);
- let venus = pack_hit("Venus is the second planet.", "Venus", 1);
- let jupiter = pack_hit("Jupiter is the fifth planet.", "Jupiter", 2);
- let another_mars_chunk = pack_hit("Mars also has two moons.", "Mars", 3);
- for prior in [[venus, mars.clone()], [mars.clone(), another_mars_chunk]] {
+#[test]
+fn personal_questions_keep_fresh_results_at_the_historical_note_limit() {
+ let ledger = hit("Holiday spending: 1200 rupees.", "ledger-2026.md");
+ for query in [
+ "How much did I spend on my holiday?",
+ "Compare my holiday costs across years.",
+ ] {
+ let context = fit_followup(&notes_at_limit(), vec![ledger.clone()], query, 700);
+ assert_full_passage(&context, &ledger);
+ }
+}
+
+#[test]
+fn temporal_phrases_are_distinct_from_source_references() {
+ for (question, expected) in [
+ (
+ "How much did I spend on my holiday these past few months?",
+ false,
+ ),
+ ("What did I spend on holiday in those 2 weeks?", false),
+ ("This year, compare its deadline with Juniper.", true),
+ ("Compare that second policy with Juniper.", true),
+ ] {
+ assert_eq!(
+ super::super::followup::refers_back(question),
+ expected,
+ "{question}"
+ );
+ }
+}
+
+#[test]
+fn temporal_questions_keep_fresh_results_under_source_and_byte_limits() {
+ let ledger = hit("Holiday spending: 1200 rupees.", "ledger-2026.md");
+ for (prior, fresh, budget) in [
+ (notes_at_limit(), ledger.clone(), 7000),
+ (
+ vec![hit(&"Cedar allows 30 days. ".repeat(320), "cedar.md")],
+ ledger.clone(),
+ 7000,
+ ),
+ (
+ vec![
+ pack_hit("Mars is the fourth planet.", "Mars", 0),
+ pack_hit("Venus is the second planet.", "Venus", 1),
+ ],
+ pack_hit(&ledger.text, "Ledger", 2),
+ 700,
+ ),
+ ] {
let context = fit_followup(
&prior,
- vec![jupiter.clone()],
- "Compare Mars and Jupiter",
- 7000,
+ vec![fresh.clone()],
+ "How much did I spend on my holiday this year?",
+ budget,
);
- assert!(context.text.contains(&mars.text));
- assert!(context.text.contains(&jupiter.text));
- assert_eq!(context.sources.len(), 2);
+ assert_full_passage(&context, &fresh);
}
}
#[test]
-fn unnamed_questions_keep_fresh_results_and_available_historical_passages() {
- let notes: Vec<_> = ["cedar", "pine", "birch", "oak", "maple"]
- .iter()
- .map(|name| hit(&format!("{name} allows 30 days."), &format!("{name}.md")))
- .collect();
+fn named_pack_comparisons_retain_articles_cited_through_multiple_sections() {
+ let prior: Vec<_> = [
+ ("History", "Mars has been observed since antiquity."),
+ ("Geography", "Mars has two polar ice caps."),
+ ]
+ .into_iter()
+ .enumerate()
+ .map(|(row, (section, text))| {
+ let mut hit = pack_hit(text, &format!("Mars — {section}"), row as u64);
+ if let GroundedSource::EnsuPack { citation } = &mut hit.source {
+ citation.source_url = format!("https://simple.wikipedia.org/wiki/Mars#{section}");
+ }
+ hit
+ })
+ .collect();
+ let jupiter = pack_hit("Jupiter is the fifth planet.", "Jupiter", 2);
+ let context = fit_followup(
+ &prior,
+ vec![jupiter.clone()],
+ "Compare Mars and Jupiter",
+ 700,
+ );
+ assert_full_passage(&context, &prior[0]);
+ assert_full_passage(&context, &jupiter);
+ assert_eq!(context.sources.len(), 2);
+
+ let context = fit_followup(&prior, vec![], "Tell me more about Mars", 7000);
+ assert_eq!(
+ context.sources,
+ prior
+ .iter()
+ .map(|hit| hit.source.clone())
+ .collect::<Vec<_>>()
+ );
+}
+
+#[test]
+fn unnamed_questions_keep_available_historical_passages() {
+ let notes = notes_at_limit();
let biology = hit(
"Photosynthesis converts light into energy.",
"biology-chapter-seven.md",
);
- let mut another_chunk = notes[0].clone();
- if let PassageLocator::LocalNote { chunk_index, .. } = &mut another_chunk.locator {
- *chunk_index += 1;
- }
- another_chunk.text = "International shipping has an extended deadline.".into();
let old = retrieval::build_grounded_prompt_context(&notes, MAX_GROUNDING_BYTES)
.unwrap()
.unwrap();
- for (query, fresh) in [
- ("How does photosynthesis work?", &biology),
- ("And the deadline?", &biology),
- ("And the deadline?", &another_chunk),
- ] {
- let context = fit_followup(&notes, vec![fresh.clone()], query, 7000);
- assert!(context.text.contains(&fresh.text), "{query}");
- assert!(context.text.contains(&notes[0].text), "{query}");
- assert!(
- context
- .included_passages
- .contains(&old.included_passages[0])
- );
- assert!(context.sources.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);
- }
-
- let packs = [
- pack_hit("Mars is the fourth planet.", "Mars", 0),
- pack_hit("Venus is the second planet.", "Venus", 1),
- ];
- let fresh = pack_hit("Plants use sunlight to produce sugar.", "Botany", 2);
let context = fit_followup(
- &packs,
- vec![fresh.clone()],
+ &notes,
+ vec![biology.clone()],
"How does photosynthesis work?",
7000,
);
- assert!(context.text.contains(&fresh.text));
- assert!(context.text.contains(&packs[0].text));
- assert_eq!(context.sources.len(), retrieval::MAX_PACK_HITS);
+ assert_full_passage(&context, &biology);
+ assert!(
+ context
+ .included_passages
+ .contains(&old.included_passages[0])
+ );
+ assert_eq!(context.sources.len(), retrieval::MAX_NOTES_GROUNDING_HITS);
let context = fit_followup(&notes, vec![], "And the deadline?", 7000);
assert_eq!(context.included_passages, old.included_passages);
}
+#[test]
+fn followups_keep_new_chunks_of_retained_sources_at_hit_limits() {
+ let notes: Vec<_> = (0..3)
+ .map(|index| {
+ let mut note = hit(&format!("Cedar background section {index}."), "cedar.md");
+ if let PassageLocator::LocalNote(locator) = &mut note.locator {
+ locator.chunk_index = index;
+ }
+ note
+ })
+ .chain([
+ hit("Pine background.", "pine.md"),
+ hit("Birch background.", "birch.md"),
+ ])
+ .collect();
+ let mut fresh = notes[0].clone();
+ if let PassageLocator::LocalNote(locator) = &mut fresh.locator {
+ locator.chunk_index = 3;
+ }
+ fresh.text = "Cedar warranty duration: 24 months.".into();
+ let context = fit_followup(
+ &notes,
+ vec![notes[0].clone(), fresh.clone()],
+ "And in Cedar, how long is its warranty?",
+ 700,
+ );
+ assert_full_passage(&context, &fresh);
+ assert_full_passage(&context, &notes[0]);
+ assert!(context.included_passages.len() <= retrieval::MAX_NOTES_GROUNDING_HITS);
+
+ let prior = [
+ pack_hit("Mars is the fourth planet.", "Mars", 0),
+ pack_hit("Venus is the second planet.", "Venus", 1),
+ ];
+ let fresh = [
+ pack_hit("Its radius is 3389.5 kilometers.", "Mars", 2),
+ pack_hit("Its radius is 6051.8 kilometers.", "Venus", 3),
+ ];
+ let context = fit_followup(&prior, fresh.to_vec(), "Which of them is larger?", 700);
+ for hit in &fresh {
+ assert_full_passage(&context, hit);
+ }
+}
+
#[test]
fn fresh_followup_results_survive_byte_and_token_repacking() {
let prefix = hit("Price: 100 rupees.", "cedar.md");
@@ -279,15 +401,8 @@ fn fresh_followup_results_survive_byte_and_token_repacking() {
"And what's its warranty duration?",
7000,
);
- assert!(context.text.contains(&full.text));
- assert_eq!(context.sources, vec![full.source]);
+ assert_full_passage(&context, &full);
assert_eq!(context.included_passages.len(), 1);
- assert_eq!(
- context.included_passages[0]
- .verified_spans(&full.text)
- .unwrap(),
- vec![full.text]
- );
let old = hit(&"Cedar allows 30 days. ".repeat(280), "cedar.md");
let fresh = hit(
@@ -304,27 +419,15 @@ fn fresh_followup_results_survive_byte_and_token_repacking() {
"How does photosynthesis work?",
input_budget,
);
- assert!(context.text.contains("Plants use light."));
let reference = context
.included_passages
.iter()
.find(|reference| reference.locator == fresh.locator)
.unwrap();
let spans = reference.verified_spans(&fresh.text).unwrap();
+ assert!(spans[0].starts_with("Plants use light."));
assert!(context.text.contains(&spans[0]));
}
-
- let mut long_label = hit("Plants use light.", "biology.md");
- if let GroundedSource::LocalNote { reference } = &mut long_label.source {
- reference.title = "Biology ".repeat(400);
- }
- let context = fit_followup(
- &[old],
- vec![long_label.clone()],
- "How does photosynthesis work?",
- 7000,
- );
- assert!(context.text.contains(&long_label.text));
}
#[test]
@@ -343,7 +446,14 @@ fn measured_repack_keeps_text_citations_and_exact_spans_together() {
),
] {
let candidates = GroundingCandidates::new(vec![], vec![hit(&raw, "one.md")], 6000).unwrap();
- let fit = fit_grounding(&candidates, "System", "Question", 4096, measure).unwrap();
+ let fit = fit_grounding(
+ &candidates.select_sources(None).unwrap(),
+ "System",
+ "Question",
+ 4096,
+ measure,
+ )
+ .unwrap();
assert!(fit.repacked);
assert!(fit.evidence_tokens <= 1024);
let context = fit.context.unwrap();
@@ -374,7 +484,14 @@ fn requested_old_spans_raise_soft_allowance_and_are_never_cut() {
6000,
)
.unwrap();
- let fit = fit_grounding(&candidates, "System", "Question", 3000, measure).unwrap();
+ let fit = fit_grounding(
+ &candidates.select_sources(None).unwrap(),
+ "System",
+ "Question",
+ 3000,
+ measure,
+ )
+ .unwrap();
assert!(fit.repacked && fit.evidence_tokens > 750);
assert_eq!(fit.context.unwrap().included_passages[0], reference);
}
@@ -383,14 +500,26 @@ fn mandatory_input_and_insufficient_evidence_budget_are_explicit() {
let candidates =
GroundingCandidates::new(vec![], vec![hit(&"word ".repeat(1000), "one.md")], 6000).unwrap();
assert!(matches!(
- fit_grounding(&candidates, "System", &"x".repeat(1000), 100, measure),
+ fit_grounding(
+ &candidates.select_sources(None).unwrap(),
+ "System",
+ &"x".repeat(1000),
+ 100,
+ measure
+ ),
Err(PrepareError::CurrentInputTooLarge)
));
assert!(
- fit_grounding(&candidates, "System", "Question", 60, measure)
- .unwrap()
- .context
- .is_none()
+ fit_grounding(
+ &candidates.select_sources(None).unwrap(),
+ "System",
+ "Question",
+ 60,
+ measure
+ )
+ .unwrap()
+ .context
+ .is_none()
);
}
#[test]
@@ -411,7 +540,7 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {
&format!("{}Juniper permits 21 days.", "Juniper details. ".repeat(25)),
"juniper-policy.md",
);
- let mut candidates = GroundingCandidates::new(
+ let candidates = GroundingCandidates::new(
vec![],
vec![
cedar.clone(),
@@ -421,49 +550,14 @@ fn named_comparison_keeps_both_complete_passages_before_optional_hits() {
6000,
)
.unwrap();
- candidates
- .protect_named_sources("Is the Juniper period longer than Cedar?")
+ let plan = candidates
+ .select_sources(Some("Is the Juniper period longer than Cedar?"))
.unwrap();
- let fit = fit_grounding(
- &candidates,
- "System",
- "Compare Cedar and Juniper.",
- 2500,
- measure,
- )
- .unwrap();
+ let fit = fit_grounding(&plan, "System", "Compare Cedar and Juniper.", 2500, measure).unwrap();
assert!(fit.repacked);
let context = fit.context.unwrap();
for hit in [&cedar, &juniper] {
- assert!(context.text.contains(&hit.text));
- let reference = context
- .included_passages
- .iter()
- .find(|r| r.locator == hit.locator)
- .unwrap();
- assert_eq!(
- reference.verified_spans(&hit.text).unwrap(),
- vec![hit.text.clone()]
- );
- }
-}
-
-#[test]
-fn ordinary_questions_and_shared_label_words_do_not_promote_every_source() {
- let base = GroundingCandidates::new(
- vec![],
- vec![
- hit("30 days", "cedar-policy.md"),
- hit("21 days", "juniper-policy.md"),
- ],
- 6000,
- )
- .unwrap();
- for query in ["Compare the policies in my notes", "Explain photography"] {
- let mut c = base.clone();
- c.protect_named_sources(query).unwrap();
- assert!(c.referenced.is_empty());
- assert_eq!(c.pack(6000).unwrap(), base.pack(6000).unwrap());
+ assert_full_passage(&context, hit);
}
}
@@ -496,7 +590,7 @@ fn explicit_history_displaces_optional_search_without_losing_original_text() {
)
.unwrap();
let fit = fit_grounding_with_history(
- &candidates,
+ &candidates.select_sources(None).unwrap(),
Some(&required),
"System",
query,
@@ -508,7 +602,7 @@ fn explicit_history_displaces_optional_search_without_losing_original_text() {
assert_eq!(fit.evidence_tokens, 0);
assert!(
fit_grounding_with_history(
- &candidates,
+ &candidates.select_sources(None).unwrap(),
Some(&required),
"System",
query,
@@ -531,8 +625,13 @@ fn source_system_and_required_history_are_measured_in_final_roles() {
6000,
)
.unwrap();
- let fitted =
- fit_grounding_with_history(&c, Some(&history), "System", query, 4096, |messages| {
+ let fitted = fit_grounding_with_history(
+ &c.select_sources(None).unwrap(),
+ Some(&history),
+ "System",
+ query,
+ 4096,
+ |messages| {
if messages[0].content.contains("BEGIN KNOWLEDGE CONTEXT") {
assert_eq!(messages[0].role, "system");
assert!(messages.last().unwrap().content.contains("ZX-82Q"));
@@ -545,43 +644,15 @@ fn source_system_and_required_history_are_measured_in_final_roles() {
.contains("BEGIN KNOWLEDGE CONTEXT")
);
measure(messages)
- })
- .unwrap();
+ },
+ )
+ .unwrap();
assert!(grounded_system("System", fitted.context.as_ref()).contains("The new rule"));
let final_text = history.prepend_required(query);
assert!(final_text.contains("ZX-82Q"));
assert!(final_text.ends_with(query));
}
-#[test]
-fn overlapping_pack_and_note_titles_preserve_both_sources_without_blocking_switches() {
- let pack = pack_hit("Mars is the fourth planet.", "Mars", 0);
- let mut note = hit("Check-in code COPPER-29.", "mars-expedition.md");
- if let GroundedSource::LocalNote { reference } = &mut note.source {
- reference.title = "Mars expedition checklist".into();
- }
- let query = "Using the Mars encyclopedia entry and the Mars expedition checklist, give its position from the Sun and our expedition check-in code. Answer briefly.";
- let mut c = GroundingCandidates::new(vec![], vec![pack.clone(), note.clone()], 6000).unwrap();
- c.protect_named_sources(query).unwrap();
- assert_eq!(c.referenced.len(), 2);
- assert_eq!(c.pack(6000).unwrap().unwrap().sources.len(), 2);
-
- let (history, evidence) = followup_history(&[pack]);
- assert_eq!(
- direct_followup_references(&history, &evidence, query, std::slice::from_ref(&note)),
- evidence[0].passages
- );
- assert!(
- direct_followup_references(
- &history,
- &evidence,
- "What is the check-in code in the Mars expedition checklist?",
- &[note]
- )
- .is_empty()
- );
-}
-
#[test]
fn followups_use_last_available_evidence_until_its_history_changes() {
let (mut history, evidence) = followup_history(&[hit("Archived policy fact.", "cedar.md")]);
diff --git a/rust/crates/ensu/src/conversation/state.rs b/rust/crates/ensu/src/conversation/state.rs
index 5873549475..e94d405ccc 100644
--- a/rust/crates/ensu/src/conversation/state.rs
+++ b/rust/crates/ensu/src/conversation/state.rs
@@ -6,9 +6,9 @@ use crate::db::{Message, Sender};
use crate::retrieval::IncludedPassage;
pub const ENVELOPE_VERSION: u32 = 2;
-pub const MAX_EVIDENCE_ANSWERS: usize = 8;
-pub const MAX_EVIDENCE_BYTES: usize = 12 * 1024;
-pub(crate) use crate::retrieval::MAX_GROUNDING_HITS as MAX_ANSWER_PASSAGES;
+pub(super) const MAX_EVIDENCE_ANSWERS: usize = 8;
+pub(super) const MAX_EVIDENCE_BYTES: usize = 12 * 1024;
+pub(super) use crate::retrieval::MAX_GROUNDING_HITS as MAX_ANSWER_PASSAGES;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AnswerEvidence {
@@ -28,10 +28,6 @@ impl AnswerEvidence {
&& fingerprint(&history[..=index]) == self.prefix_fingerprint
})
}
-
- fn valid(&self) -> bool {
- self.prefix_fingerprint.len() == 64 && valid_answer_passages(&self.passages)
- }
}
pub(crate) fn valid_answer_passages(passages: &[IncludedPassage]) -> bool {
@@ -76,16 +72,17 @@ impl ConversationEnvelope {
.cloned()
.and_then(|value| serde_json::from_value::<Summary>(value).ok())
.filter(|summary| validate_summary_text(&summary.text).is_ok());
- let mut evidence = Vec::new();
+ let mut evidence: Vec<AnswerEvidence> = Vec::new();
if let Some(entries) = value.get("evidence").and_then(|value| value.as_array())
&& entries.len() <= MAX_EVIDENCE_ANSWERS
{
for value in entries {
if let Ok(entry) = serde_json::from_value::<AnswerEvidence>(value.clone())
- && entry.valid()
- && !evidence.iter().any(|old: &AnswerEvidence| {
- old.assistant_message_uuid == entry.assistant_message_uuid
- })
+ && entry.prefix_fingerprint.len() == 64
+ && valid_answer_passages(&entry.passages)
+ && !evidence
+ .iter()
+ .any(|old| old.assistant_message_uuid == entry.assistant_message_uuid)
{
evidence.push(entry);
}
diff --git a/rust/crates/ensu/src/conversation/tests.rs b/rust/crates/ensu/src/conversation/tests.rs
index 743b8bbd85..34d8deccec 100644
--- a/rust/crates/ensu/src/conversation/tests.rs
+++ b/rust/crates/ensu/src/conversation/tests.rs
@@ -68,6 +68,7 @@ fn saved_memory_preserves_uuid_strings() {
let mut expected = serde_json::from_str::<serde_json::Value>(encoded).unwrap();
expected["evidence"] = serde_json::json!([]);
assert_eq!(serde_json::to_value(&restored).unwrap(), expected);
+ assert!(ConversationEnvelope::decode(encoded.as_bytes(), Uuid::nil()).is_none());
let invalid = encoded.replace("00000000-0000-0000-0000-000000000002", "invalid");
assert!(
ConversationEnvelope::decode(invalid.as_bytes(), session)
@@ -77,10 +78,6 @@ fn saved_memory_preserves_uuid_strings() {
);
}
-fn evidence_passages() -> Vec<crate::retrieval::IncludedPassage> {
- vec![crate::retrieval::passage_fixture().0]
-}
-
#[test]
fn invalid_summary_and_evidence_are_discarded_independently() {
let history = vec![
@@ -91,7 +88,7 @@ fn invalid_summary_and_evidence_are_discarded_independently() {
let memory = ConversationState::new(session, &history, "Memory".into()).unwrap();
let mut state = ConversationEnvelope::empty(session);
state.summary = Some(memory.summary.clone());
- assert!(state.add_evidence(&history, evidence_passages()));
+ assert!(state.add_evidence(&history, vec![crate::retrieval::passage_fixture().0]));
let original = serde_json::to_value(&state).unwrap();
let decode = |value: &serde_json::Value| {
ConversationEnvelope::decode(&serde_json::to_vec(value).unwrap(), session)
@@ -107,17 +104,12 @@ fn invalid_summary_and_evidence_are_discarded_independently() {
let decoded = decode(&value).unwrap();
assert!(decoded.evidence.is_empty());
assert_eq!(decoded.summary_state().unwrap(), memory);
- value["format_version"] = 1.into();
- assert!(decode(&value).is_none());
- assert!(
- ConversationEnvelope::decode(&serde_json::to_vec(&state).unwrap(), Uuid::nil()).is_none()
- );
}
#[test]
fn evidence_retention_obeys_answer_and_byte_limits() {
for passage_count in [1, MAX_ANSWER_PASSAGES] {
- let passages = vec![evidence_passages().remove(0); passage_count];
+ let passages = vec![crate::retrieval::passage_fixture().0; passage_count];
let mut state = ConversationEnvelope::empty(Uuid::from_u128(100));
let mut history = Vec::new();
for index in 1..=10 {
diff --git a/rust/crates/ensu/src/conversation/turn.rs b/rust/crates/ensu/src/conversation/turn.rs
index 9a91c3da39..6bc4c60522 100644
--- a/rust/crates/ensu/src/conversation/turn.rs
+++ b/rust/crates/ensu/src/conversation/turn.rs
@@ -44,11 +44,11 @@ pub fn prepare_turn(
.transpose()?
.flatten();
effects.check_cancelled()?;
- input
- .candidates
- .protect_named_sources(input.history_query.as_deref().unwrap_or(&input.current))?;
+ let grounding_plan = input.candidates.select_sources(Some(
+ input.history_query.as_deref().unwrap_or(&input.current),
+ ))?;
let grounding = fit_grounding_with_history(
- &input.candidates,
+ &grounding_plan,
required.as_ref(),
&input.system,
&input.current,
diff --git a/rust/crates/ensu/src/db/chat/conversation.rs b/rust/crates/ensu/src/db/chat/conversation.rs
index 9d94b50b89..26e3940685 100644
--- a/rust/crates/ensu/src/db/chat/conversation.rs
+++ b/rust/crates/ensu/src/db/chat/conversation.rs
@@ -599,12 +599,10 @@ mod tests {
.unwrap();
let captured = snapshot(&db, session, std::slice::from_ref(&question));
let answer = sourced_answer(&db, &captured);
- let before = db.get_session(session).unwrap();
assert!(
db.save_answer_evidence(&captured, answer.uuid, passages())
.unwrap()
);
- assert_eq!(db.get_session(session).unwrap(), before);
let full = vec![question.clone(), answer];
let mut restored = snapshot(&db, session, &full);
assert!(restored.state().is_none());
@@ -653,17 +651,11 @@ mod tests {
);
messages.push(answer);
let mut current = snapshot(&db, session, &messages);
- assert!(
- db.replace_conversation_state(&mut current, Some(state(session, &messages, "Memory")))
- .unwrap()
- );
- assert_eq!(snapshot(&db, session, &messages).evidence().len(), 1);
- assert!(db.replace_conversation_state(&mut current, None).unwrap());
- assert_eq!(snapshot(&db, session, &messages).evidence().len(), 1);
- assert!(
- db.replace_conversation_state(&mut current, Some(state(session, &messages, "Memory")))
- .unwrap()
- );
+ let memory = state(session, &messages, "Memory");
+ for next in [Some(memory.clone()), None, Some(memory)] {
+ assert!(db.replace_conversation_state(&mut current, next).unwrap());
+ assert_eq!(snapshot(&db, session, &messages).evidence().len(), 1);
+ }
db.update_message_text(messages[2].uuid, "Edited question")
.unwrap();
let current = snapshot(&db, session, &messages);
@@ -677,12 +669,10 @@ mod tests {
let captured = snapshot(&db, session, &messages);
let answer = sourced_answer(&db, &captured);
let mut other_writer = snapshot(&db, session, &messages);
+ let memory = state(session, &messages, "Other writer");
assert!(
- db.replace_conversation_state(
- &mut other_writer,
- Some(state(session, &messages, "Other writer"))
- )
- .unwrap()
+ db.replace_conversation_state(&mut other_writer, Some(memory.clone()))
+ .unwrap()
);
assert!(
!db.save_answer_evidence(&captured, answer.uuid, passages())
@@ -695,14 +685,7 @@ mod tests {
.is_err()
);
assert_eq!(db.get_message(answer.uuid).unwrap().unwrap(), answer);
- assert_eq!(
- snapshot(&db, session, &messages)
- .state()
- .unwrap()
- .summary
- .text,
- "Other writer"
- );
+ assert_eq!(snapshot(&db, session, &messages).state(), Some(&memory));
}
#[test]
@@ -735,7 +718,6 @@ mod tests {
db.save_answer_evidence(&captured, answer.uuid, passages())
.is_err()
);
- assert!(db.get_session(session).unwrap().is_none());
}
#[test]
@@ -842,6 +824,12 @@ mod tests {
)
.unwrap()
);
+ let mut encoded = ConversationEnvelope::empty(session);
+ encoded.summary = captured.state().map(|state| state.summary.clone());
+ encoded.format_version = 1;
+ assert!(
+ ConversationEnvelope::decode(&serde_json::to_vec(&encoded).unwrap(), session).is_none()
+ );
for text in [" \n".to_owned(), "x".repeat(MAX_STATE_BYTES)] {
let mut invalid = state(session, &messages, "Valid memory");
invalid.summary.text = text;
diff --git a/rust/crates/ensu/src/notes.rs b/rust/crates/ensu/src/notes.rs
index ef7cd03d35..4c38439639 100644
--- a/rust/crates/ensu/src/notes.rs
+++ b/rust/crates/ensu/src/notes.rs
@@ -47,7 +47,9 @@ pub enum NotesError {
pub use document::{
NotesSourceDocument, PreparedNotesChunk, PreparedNotesDocument, prepare_notes_document,
};
-pub use index::{NotesCollectionIndex, NotesCollectionIndexSummary, NotesSearchHit};
+pub use index::{
+ NotePassageLocator, NotesCollectionIndex, NotesCollectionIndexSummary, NotesSearchHit,
+};
pub use indexing::{
NotesDocumentLoad, NotesIndexInput, NotesIndexOutcome, NotesIndexProgress, NotesIndexingError,
NotesRevisionStatus, index_notes_collection,
diff --git a/rust/crates/ensu/src/notes/index.rs b/rust/crates/ensu/src/notes/index.rs
index 46468f2695..d76b0137b2 100644
--- a/rust/crates/ensu/src/notes/index.rs
+++ b/rust/crates/ensu/src/notes/index.rs
@@ -1,20 +1,31 @@
use std::collections::{BTreeMap, btree_map::Entry};
use std::path::Path;
+use serde::{Deserialize, Serialize};
+
use super::manifest::{
collection_directory, document_storage_id, load_published_manifest, shard_directory,
};
use super::shard::{NotesShard, load_and_validate_shard_metadata, load_and_validate_vectors};
use super::{NotesCollectionReadPin, NotesError, pinned_revision_key, validate_collection_id};
-use crate::retrieval::PassageLocator;
const RELEVANCE_THRESHOLD: f32 = 0.50;
const MAX_HITS_PER_COLLECTION: usize = 5;
const MAX_HITS_PER_DOCUMENT: usize = 3;
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct NotePassageLocator {
+ pub collection_id: String,
+ pub document_id: String,
+ pub indexed_revision: String,
+ pub shard_sha256: String,
+ pub chunk_index: u64,
+}
+
#[derive(Debug, Clone, PartialEq)]
pub struct NotesSearchHit {
- pub locator: PassageLocator,
+ pub locator: NotePassageLocator,
pub collection_id: String,
pub document_id: String,
pub revision: String,
@@ -186,18 +197,15 @@ impl NotesCollectionIndex {
pub fn reload_passage(
&self,
- locator: &PassageLocator,
+ locator: &NotePassageLocator,
) -> Result<Option<NotesSearchHit>, NotesError> {
- let PassageLocator::LocalNote {
+ let NotePassageLocator {
collection_id,
document_id,
indexed_revision,
shard_sha256,
chunk_index,
- } = locator
- else {
- return Ok(None);
- };
+ } = locator;
if collection_id != &self.collection_id {
return Ok(None);
}
@@ -215,17 +223,8 @@ impl NotesCollectionIndex {
return Ok(None);
}
let shard = self.load_shard(document)?;
- let chunk = &shard.chunks[chunk_index];
- Ok(Some(NotesSearchHit {
- locator: locator.clone(),
- collection_id: collection_id.clone(),
- document_id: document_id.clone(),
- revision: indexed_revision.clone(),
- score: 0.0,
- title: shard.title,
- section: chunk.section.clone(),
- text: chunk.text.clone(),
- }))
+ self.passage_hit(document, &shard, chunk_index, 0.0)
+ .map(Some)
}
pub fn search(&self, query: &[f32]) -> Result<Vec<NotesSearchHit>, NotesError> {
@@ -288,28 +287,38 @@ impl NotesCollectionIndex {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => entry.insert(self.load_shard(document)?),
};
- let chunk = shard.chunks.get(candidate.chunk_index).ok_or_else(|| {
- NotesError::InvalidIndex("selected shard chunk is missing".to_string())
- })?;
- hits.push(NotesSearchHit {
- locator: PassageLocator::LocalNote {
- collection_id: self.collection_id.clone(),
- document_id: document.document_id.clone(),
- indexed_revision: document.revision.clone(),
- shard_sha256: document.shard_sha256.clone(),
- chunk_index: candidate.chunk_index as u64,
- },
- collection_id: self.collection_id.clone(),
- document_id: document.document_id.clone(),
- revision: document.revision.clone(),
- score: candidate.score,
- title: shard.title.clone(),
- section: chunk.section.clone(),
- text: chunk.text.clone(),
- });
+ hits.push(self.passage_hit(document, shard, candidate.chunk_index, candidate.score)?);
}
Ok(hits)
}
+
+ fn passage_hit(
+ &self,
+ document: &IndexedDocument,
+ shard: &NotesShard,
+ chunk_index: usize,
+ score: f32,
+ ) -> Result<NotesSearchHit, NotesError> {
+ let chunk = shard.chunks.get(chunk_index).ok_or_else(|| {
+ NotesError::InvalidIndex("selected shard chunk is missing".to_string())
+ })?;
+ Ok(NotesSearchHit {
+ locator: NotePassageLocator {
+ collection_id: self.collection_id.clone(),
+ document_id: document.document_id.clone(),
+ indexed_revision: document.revision.clone(),
+ shard_sha256: document.shard_sha256.clone(),
+ chunk_index: chunk_index as u64,
+ },
+ collection_id: self.collection_id.clone(),
+ document_id: document.document_id.clone(),
+ revision: document.revision.clone(),
+ score,
+ title: shard.title.clone(),
+ section: chunk.section.clone(),
+ text: chunk.text.clone(),
+ })
+ }
}
fn retain_best(
@@ -414,28 +423,19 @@ mod tests {
assert_ne!(hits[0].locator, hits[1].locator);
drop(index);
let index = NotesCollectionIndex::open(temp.path(), COLLECTION_ID.to_string()).unwrap();
- for hit in &hits {
+ for (hit, chunk) in hits.iter().zip(&doc.chunks) {
let loaded = index.reload_passage(&hit.locator).unwrap().unwrap();
- assert_eq!(loaded.text, hit.text);
+ assert_eq!(loaded.text, chunk.text);
assert_eq!(loaded.locator, hit.locator);
}
for field in ["revision", "shard", "chunk", "collection", "document"] {
let mut wrong = hits[0].locator.clone();
- if let PassageLocator::LocalNote {
- indexed_revision,
- shard_sha256,
- chunk_index,
- collection_id,
- document_id,
- } = &mut wrong
- {
- match field {
- "revision" => *indexed_revision = "c".repeat(64),
- "shard" => *shard_sha256 = "c".repeat(64),
- "chunk" => *chunk_index = u64::MAX,
- "collection" => *collection_id = "other".to_string(),
- _ => *document_id = "other.md".to_string(),
- }
+ match field {
+ "revision" => wrong.indexed_revision = "c".repeat(64),
+ "shard" => wrong.shard_sha256 = "c".repeat(64),
+ "chunk" => wrong.chunk_index = u64::MAX,
+ "collection" => wrong.collection_id = "other".to_string(),
+ _ => wrong.document_id = "other.md".to_string(),
}
assert!(index.reload_passage(&wrong).unwrap().is_none(), "{field}");
}
diff --git a/rust/crates/ensu/src/retrieval/grounded.rs b/rust/crates/ensu/src/retrieval/grounded.rs
index c5e608eb54..88eef5f634 100644
--- a/rust/crates/ensu/src/retrieval/grounded.rs
+++ b/rust/crates/ensu/src/retrieval/grounded.rs
@@ -21,6 +21,25 @@ pub enum GroundedSource {
LocalNote { reference: NoteSourceReference },
}
+impl GroundedSource {
+ pub(crate) fn document_key(&self) -> (u8, &str, &str) {
+ match self {
+ Self::LocalNote { reference } => (0, &reference.collection_id, &reference.document_id),
+ Self::EnsuPack { citation } => {
+ let article_url = citation
+ .source_url
+ .split_once('#')
+ .map_or(citation.source_url.as_str(), |(article, _)| article);
+ (1, &citation.dataset_id, article_url)
+ }
+ }
+ }
+
+ pub(crate) fn same_document(&self, other: &Self) -> bool {
+ self.document_key() == other.document_key()
+ }
+}
+
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct GroundedExcerpt {
pub locator: super::PassageLocator,
@@ -123,7 +142,7 @@ pub fn select_mixed_grounding_candidates(
.validate()
.map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;
selected.push(GroundedExcerpt {
- locator: hit.locator.clone(),
+ locator: super::PassageLocator::LocalNote(hit.locator.clone()),
score: hit.score,
source: GroundedSource::LocalNote { reference },
text: hit.text.clone(),
@@ -156,15 +175,20 @@ pub fn build_grounded_prompt_context(
let footer = format!("\n{END_CONTEXT_SENTINEL}");
let mut text = header;
let mut sources = Vec::new();
- let mut included_passages = Vec::new();
+ let mut included_passages: Vec<super::IncludedPassage> = Vec::new();
for excerpt in excerpts {
+ if included_passages
+ .iter()
+ .any(|reference| reference.locator == excerpt.locator)
+ {
+ continue;
+ }
if !excerpt.locator.matches_source(&excerpt.source) {
return Err(RetrievalError::InvalidInput(
"passage locator does not match its source".to_string(),
));
}
- let label = grounded_label(&excerpt.source)?;
- let prefix = format!("\n\n# {label}\n");
+ let prefix = grounded_prefix(&excerpt.source)?;
let reserved = text
.len()
.checked_add(prefix.len())
@@ -225,21 +249,22 @@ fn source_sort_key(source: &GroundedSource) -> (u8, &str, &str, &str) {
}
}
-fn grounded_label(source: &GroundedSource) -> Result<String, RetrievalError> {
- match source {
- GroundedSource::EnsuPack { citation } => Ok(format!(
+fn grounded_prefix(source: &GroundedSource) -> Result<String, RetrievalError> {
+ let label = match source {
+ GroundedSource::EnsuPack { citation } => format!(
"{} (Ensu Pack · {})",
normalize_required(&citation.title, "Pack title")?,
normalize_required(&citation.dataset_label, "Pack label")?
- )),
+ ),
GroundedSource::LocalNote { reference } => {
reference
.validate()
.map_err(|error| RetrievalError::InvalidInput(error.to_string()))?;
let title = display_title(&reference.title, reference.section.as_deref())?;
- Ok(format!("{title} (Your Notes · {})", reference.document_id))
+ format!("{title} (Your Notes · {})", reference.document_id)
}
- }
+ };
+ Ok(format!("\n\n# {label}\n"))
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -268,13 +293,10 @@ pub(crate) fn build_followup_prompt_context(
let mut sections = Vec::new();
let mut used_bytes = text.len() + footer.len();
let mut sources = Vec::new();
- let mut included_passages = Vec::new();
+ let mut included_passages: Vec<super::IncludedPassage> = Vec::new();
let mut packs = 0;
let mut notes = 0;
for item in referenced {
- if included_passages.contains(&item.reference) {
- continue;
- }
if !item.reference.valid_metadata()
|| !item.reference.locator.matches_source(&item.source)
|| item.reference.spans.len() != item.passages.len()
@@ -288,6 +310,17 @@ pub(crate) fn build_followup_prompt_context(
return Err(invalid());
}
}
+ if let Some(index) = included_passages
+ .iter()
+ .position(|reference| reference.locator == item.reference.locator)
+ {
+ if included_passages[index] != item.reference
+ || !item.source.same_document(&sources[index])
+ {
+ return Err(invalid());
+ }
+ continue;
+ }
match &item.source {
GroundedSource::EnsuPack { .. } => packs += 1,
GroundedSource::LocalNote { .. } => notes += 1,
@@ -295,7 +328,7 @@ pub(crate) fn build_followup_prompt_context(
if packs > MAX_PACK_HITS || notes > MAX_NOTES_GROUNDING_HITS {
return Err(invalid());
}
- let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);
+ let prefix = grounded_prefix(&item.source)?;
let passages = item.passages.join("\n[omitted source text]\n");
if used_bytes
.saturating_add(prefix.len())
@@ -309,7 +342,14 @@ pub(crate) fn build_followup_prompt_context(
sources.push(item.source.clone());
included_passages.push(item.reference.clone());
}
- for item in searched {
+ let mut ordered: Vec<_> = searched.iter().collect();
+ ordered.sort_by_key(|item| {
+ referenced
+ .iter()
+ .position(|old| old.reference.locator == item.locator)
+ .unwrap_or(usize::MAX)
+ });
+ for item in ordered {
if let Some(index) = included_passages
.iter()
.position(|reference| reference.locator == item.locator)
@@ -317,13 +357,12 @@ pub(crate) fn build_followup_prompt_context(
let reference = &included_passages[index];
if !item.score.is_finite()
|| !item.locator.matches_source(&item.source)
+ || !item.source.same_document(&sources[index])
|| reference.verified_spans(&item.text).is_none()
{
continue;
}
- // Fresh search may add text beyond the saved spans, but every old
- // span must still fit. Keep one source entry for the entire chunk.
- let prefix = format!("\n\n# {}\n", grounded_label(&sources[index])?);
+ let prefix = grounded_prefix(&sources[index])?;
let available = max_utf8_bytes - used_bytes + sections[index].len() - prefix.len();
let cleaned = super::clean_passage_text(&item.text);
let passage = truncate_utf8(&cleaned, available).trim_end();
@@ -345,7 +384,7 @@ pub(crate) fn build_followup_prompt_context(
if (is_pack && packs >= MAX_PACK_HITS) || (!is_pack && notes >= MAX_NOTES_GROUNDING_HITS) {
continue;
}
- let prefix = format!("\n\n# {}\n", grounded_label(&item.source)?);
+ let prefix = grounded_prefix(&item.source)?;
let Some(available) = max_utf8_bytes.checked_sub(used_bytes + prefix.len()) else {
continue;
};
@@ -408,7 +447,7 @@ mod tests {
fn note(document_id: &str, score: f32) -> NotesSearchHit {
NotesSearchHit {
- locator: crate::retrieval::PassageLocator::LocalNote {
+ locator: crate::notes::NotePassageLocator {
collection_id: COLLECTION.to_owned(),
document_id: document_id.to_owned(),
indexed_revision: "a".repeat(64),
@@ -428,7 +467,7 @@ mod tests {
#[test]
fn mismatched_passage_sources_and_text_are_rejected() {
let mut wrong = select_mixed_grounding_candidates(&[], &[note("one.md", 0.9)], 5).unwrap();
- wrong[0].locator = note("other.md", 0.9).locator;
+ wrong[0].locator = super::super::PassageLocator::LocalNote(note("other.md", 0.9).locator);
assert!(build_grounded_prompt_context(&wrong, 6000).is_err());
let mut old = previous();
old.passages[0] = "Cafe 🙂".into();
@@ -503,7 +542,7 @@ mod tests {
}
fn previous() -> ReferencedPassage {
- let (reference, raw) = super::super::passage_fixture();
+ let (reference, _) = super::super::passage_fixture();
let source = GroundedSource::LocalNote {
reference: NoteSourceReference {
collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),
@@ -515,7 +554,7 @@ mod tests {
},
};
ReferencedPassage {
- passages: reference.verified_spans(raw).unwrap(),
+ passages: vec!["Café🙂".into(), "end".into()],
reference,
source,
}
@@ -530,8 +569,8 @@ mod tests {
text: "UNSUPPLIED source remainder".into(),
};
let mut fresh = duplicate.clone();
- if let super::super::PassageLocator::LocalNote { chunk_index, .. } = &mut fresh.locator {
- *chunk_index += 1;
+ if let super::super::PassageLocator::LocalNote(locator) = &mut fresh.locator {
+ locator.chunk_index += 1;
}
fresh.text = "Fresh café🙂 context".into();
let full = build_followup_prompt_context(
@@ -548,10 +587,9 @@ mod tests {
let required = build_followup_prompt_context(std::slice::from_ref(&old), &[], 6000)
.unwrap()
.unwrap();
- for budget in [
- required.text.len(),
- required.text.len() + 8,
- full.text.len() - 1,
+ for (case, budget) in [
+ ("required only", required.text.len()),
+ ("truncated optional passage", full.text.len() - 1),
] {
let packed = build_followup_prompt_context(
std::slice::from_ref(&old),
@@ -560,8 +598,8 @@ mod tests {
)
.unwrap()
.unwrap();
- assert!(packed.text.len() <= budget);
- assert_eq!(packed.included_passages[0], old.reference);
+ assert!(packed.text.len() <= budget, "{case}");
+ assert_eq!(packed.included_passages[0], old.reference, "{case}");
}
let raw = super::super::passage_fixture().1;
@@ -580,14 +618,10 @@ mod tests {
.unwrap();
assert_eq!(expanded.sources, vec![old.source.clone()]);
assert_eq!(expanded.included_passages.len(), 1);
- assert!(
- expanded
- .text
- .contains(&super::super::clean_passage_text(raw))
- );
+ assert!(expanded.text.contains("Café🙂\nबीच\tend"));
assert_eq!(
expanded.included_passages[0].verified_spans(raw).unwrap(),
- vec![super::super::clean_passage_text(raw)]
+ ["Café🙂\nबीच\tend"]
);
let mut prefix = old.clone();
@@ -597,7 +631,10 @@ mod tests {
build_followup_prompt_context(std::slice::from_ref(&prefix), &[], 6000)
.unwrap()
.unwrap();
- for budget in required_prefix.text.len()..=expanded.text.len() {
+ for (case, budget) in [
+ ("required prefix only", required_prefix.text.len()),
+ ("truncated expansion", expanded.text.len() - 1),
+ ] {
let packed = build_followup_prompt_context(
std::slice::from_ref(&prefix),
std::slice::from_ref(&matching),
@@ -605,11 +642,11 @@ mod tests {
)
.unwrap()
.unwrap();
- assert!(packed.text.len() <= budget);
- assert_eq!(packed.included_passages.len(), 1);
+ assert!(packed.text.len() <= budget, "{case}");
+ assert_eq!(packed.included_passages.len(), 1, "{case}");
let spans = packed.included_passages[0].verified_spans(raw).unwrap();
- assert!(spans[0].starts_with(&prefix.passages[0]));
- assert!(packed.text.contains(&spans[0]));
+ assert!(spans[0].starts_with(&prefix.passages[0]), "{case}");
+ assert!(packed.text.contains(&spans[0]), "{case}");
}
assert!(build_followup_prompt_context(&[old], &[], required.text.len() - 1).is_err());
}
diff --git a/rust/crates/ensu/src/retrieval/index.rs b/rust/crates/ensu/src/retrieval/index.rs
index 8cecc68fc9..c420a3f4c1 100644
--- a/rust/crates/ensu/src/retrieval/index.rs
+++ b/rust/crates/ensu/src/retrieval/index.rs
@@ -680,43 +680,36 @@ pub(super) mod tests {
}
#[test]
- fn passage_revisions_are_lazy_and_reloads_reject_changed_artifacts() {
+ fn passage_reload_preserves_sources_and_rejects_changed_artifacts() {
use crate::retrieval::PassageLocator;
let pack = synthetic_pack("simplewiki-test");
let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();
- assert!(index.revision_sha256.get().is_none());
let mut query = vec![0.0; 512];
- assert!(index.search(&query, 3, 1.0).unwrap().is_empty());
- assert!(index.revision_sha256.get().is_none());
query[0] = 1.0;
- let hits = index.search(&query, 3, 0.0).unwrap();
+ let hits = index.search(&query, 2, 0.0).unwrap();
assert_ne!(hits[0].locator, hits[1].locator);
- let revision = text_revision_digest(
- &fs::read(pack.revision.join(KNOWLEDGE_MANIFEST_FILE)).unwrap(),
- &fs::read(pack.revision.join(KNOWLEDGE_META_FILE)).unwrap(),
- &fs::read(pack.revision.join(KNOWLEDGE_OFFSETS_FILE)).unwrap(),
- );
- assert_eq!(index.revision_sha256.get(), Some(&revision));
drop(index);
let index = RetrievalIndex::open(&pack.revision, &pack.expected).unwrap();
- let mut wrong = hits[0].locator.clone();
- if let PassageLocator::EnsuPack { row, .. } = &mut wrong {
- *row = u64::MAX;
- }
- assert!(index.reload_passage(&wrong).unwrap().is_none());
- wrong = hits[0].locator.clone();
- if let PassageLocator::EnsuPack { dataset_id, .. } = &mut wrong {
- *dataset_id = "wikibooks".to_string();
+ for field in ["row", "dataset"] {
+ let mut wrong = hits[0].locator.clone();
+ let PassageLocator::EnsuPack {
+ dataset_id, row, ..
+ } = &mut wrong
+ else {
+ panic!("expected a pack locator")
+ };
+ match field {
+ "row" => *row = u64::MAX,
+ _ => *dataset_id = "wikibooks".to_string(),
+ }
+ assert!(index.reload_passage(&wrong).unwrap().is_none(), "{field}");
}
- assert!(index.reload_passage(&wrong).unwrap().is_none());
- assert!(index.revision_sha256.get().is_none());
- for hit in &hits {
+ for (hit, text) in hits.iter().zip(["first passage", "second passage"]) {
let reloaded = index.reload_passage(&hit.locator).unwrap().unwrap();
assert_eq!(reloaded.locator, hit.locator);
- assert_eq!(reloaded.text, hit.text);
+ assert_eq!(reloaded.text, text);
assert_eq!(reloaded.source_url, hit.source_url);
}
- assert_eq!(index.revision_sha256.get(), Some(&revision));
drop(index);
let metadata_path = pack.revision.join(KNOWLEDGE_META_FILE);
diff --git a/rust/crates/ensu/src/retrieval/passage.rs b/rust/crates/ensu/src/retrieval/passage.rs
index f957cf2e97..162dc74a4d 100644
--- a/rust/crates/ensu/src/retrieval/passage.rs
+++ b/rust/crates/ensu/src/retrieval/passage.rs
@@ -1,5 +1,7 @@
use serde::{Deserialize, Serialize};
+use crate::notes::NotePassageLocator;
+
use super::{BEGIN_CONTEXT_SENTINEL, END_CONTEXT_SENTINEL, GroundedSource, RetrievalError};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -14,13 +16,7 @@ pub enum PassageLocator {
revision_sha256: String,
row: u64,
},
- LocalNote {
- collection_id: String,
- document_id: String,
- indexed_revision: String,
- shard_sha256: String,
- chunk_index: u64,
- },
+ LocalNote(NotePassageLocator),
}
impl PassageLocator {
@@ -29,18 +25,10 @@ impl PassageLocator {
(Self::EnsuPack { dataset_id, .. }, GroundedSource::EnsuPack { citation }) => {
*dataset_id == citation.dataset_id
}
- (
- Self::LocalNote {
- collection_id,
- document_id,
- indexed_revision,
- ..
- },
- GroundedSource::LocalNote { reference },
- ) => {
- *collection_id == reference.collection_id
- && *document_id == reference.document_id
- && *indexed_revision == reference.indexed_revision
+ (Self::LocalNote(locator), GroundedSource::LocalNote { reference }) => {
+ locator.collection_id == reference.collection_id
+ && locator.document_id == reference.document_id
+ && locator.indexed_revision == reference.indexed_revision
}
_ => false,
}
@@ -87,32 +75,26 @@ impl IncludedPassage {
revision_sha256,
..
} => !dataset_id.is_empty() && dataset_id.len() <= 256 && sha(revision_sha256),
- PassageLocator::LocalNote {
- collection_id,
- document_id,
- indexed_revision,
- shard_sha256,
- ..
- } => {
- uuid::Uuid::parse_str(collection_id).is_ok()
- && crate::notes::validate_document_id(document_id).is_ok()
- && sha(indexed_revision)
- && sha(shard_sha256)
+ PassageLocator::LocalNote(locator) => {
+ uuid::Uuid::parse_str(&locator.collection_id).is_ok()
+ && crate::notes::validate_document_id(&locator.document_id).is_ok()
+ && sha(&locator.indexed_revision)
+ && sha(&locator.shard_sha256)
}
};
- let mut previous_end = 0;
valid_locator
&& self.spans.iter().all(|span| {
- let valid = span.start_utf8 >= previous_end
- && span.start_utf8 < span.end_utf8
+ span.start_utf8 < span.end_utf8
&& span.end_utf8 as usize <= MAX_PASSAGE_UTF8_BYTES
- && sha(&span.text_sha256);
- previous_end = span.end_utf8;
- valid
+ && sha(&span.text_sha256)
})
+ && self
+ .spans
+ .windows(2)
+ .all(|pair| pair[0].end_utf8 <= pair[1].start_utf8)
}
- pub(super) fn prefix(locator: PassageLocator, text: &str) -> Result<Self, RetrievalError> {
+ pub(crate) fn prefix(locator: PassageLocator, text: &str) -> Result<Self, RetrievalError> {
let end_utf8 = u32::try_from(text.len()).map_err(|_| {
RetrievalError::InvalidInput("included passage is too large".to_string())
})?;
@@ -184,91 +166,71 @@ pub(super) mod tests {
use super::*;
pub(crate) fn passage_fixture() -> (IncludedPassage, &'static str) {
- let reference = serde_json::from_value(serde_json::json!({
- "locator": {
- "type": "localNote",
- "collectionId": "123e4567-e89b-12d3-a456-426614174000",
- "documentId": "trip.md",
- "indexedRevision": "a".repeat(64),
- "shardSha256": "b".repeat(64),
- "chunkIndex": 2
- },
- "cleaningVersion": 1,
- "spans": [
- {
- "startUtf8": 0,
- "endUtf8": 9,
- "textSha256": "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"
+ let reference = IncludedPassage {
+ locator: PassageLocator::LocalNote(NotePassageLocator {
+ collection_id: "123e4567-e89b-12d3-a456-426614174000".into(),
+ document_id: "trip.md".into(),
+ indexed_revision: "a".repeat(64),
+ shard_sha256: "b".repeat(64),
+ chunk_index: 2,
+ }),
+ cleaning_version: PASSAGE_CLEANING_VERSION,
+ spans: vec![
+ PassageSpan {
+ start_utf8: 0,
+ end_utf8: 9,
+ text_sha256: "894473efc373309a7c6d18cbe8504c8bb91e09cba4e1ab147ff97af12d481702"
+ .into(),
},
- {
- "startUtf8": 20,
- "endUtf8": 23,
- "textSha256": "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"
- }
- ]
- }))
- .unwrap();
+ PassageSpan {
+ start_utf8: 20,
+ end_utf8: 23,
+ text_sha256: "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8"
+ .into(),
+ },
+ ],
+ };
(reference, " \r\nCafé🙂\r\nबीच\tend \0\r\n")
}
#[test]
fn passage_references_verify_cleaned_text_and_reject_invalid_spans() {
let (fixture, raw) = passage_fixture();
- for (raw, cleaned, spans, texts) in [
- (
- raw,
- "Café🙂\nबीच\tend",
- fixture.spans.clone(),
- vec!["Café🙂", "end"],
- ),
- (
- "\r\n----- BEGIN KNOWLEDGE CONTEXT -----\r\nx\r\n",
- "[source] ----- BEGIN KNOWLEDGE CONTEXT -----\nx",
- vec![PassageSpan {
- start_utf8: 0,
- end_utf8: 43,
- text_sha256: "73c093471df3c5f2f0e96999d8d5ebc745a803f55d3f36c4523c502c2e055e02"
- .into(),
- }],
- vec!["[source] ----- BEGIN KNOWLEDGE CONTEXT ----"],
- ),
- (
- " a\u{1}b\tc\rd ",
- "a b\tc\nd",
- vec![PassageSpan {
- start_utf8: 0,
- end_utf8: 7,
- text_sha256: "1205bffe81cb4550cdf063c07e75e290c2ebaf60d77780a338e18eb99d26bd4c"
- .into(),
- }],
- vec!["a b\tc\nd"],
- ),
- ] {
- let reference = IncludedPassage {
- spans,
- ..fixture.clone()
- };
- assert_eq!(clean_passage_text(raw), cleaned);
- assert_eq!(reference.verified_spans(raw).unwrap(), texts);
- }
+ assert_eq!(clean_passage_text(raw), "Café🙂\nबीच\tend");
+ assert_eq!(fixture.verified_spans(raw).unwrap(), ["Café🙂", "end"]);
assert!(
fixture
.verified_spans(&raw.replace("Café", "Cafe "))
.is_none()
);
- for (start, end) in [(0, 4), (4, 9), (9, 0), (0, u32::MAX)] {
+ for (case, start, end) in [
+ ("end splits UTF-8", 0, 4),
+ ("start splits UTF-8", 4, 9),
+ ("reversed range", 9, 0),
+ ("oversized range", 0, u32::MAX),
+ ] {
let mut reference = fixture.clone();
reference.spans[0].start_utf8 = start;
reference.spans[0].end_utf8 = end;
- assert!(reference.verified_spans(raw).is_none());
+ assert!(reference.verified_spans(raw).is_none(), "{case}");
}
let mut reference = fixture.clone();
reference.cleaning_version = 2;
- assert!(reference.verified_spans(raw).is_none());
+ assert!(reference.verified_spans(raw).is_none(), "cleaning version");
reference = fixture;
reference.spans[1] = reference.spans[0].clone();
- assert!(reference.verified_spans(raw).is_none());
- reference.spans = vec![reference.spans[0].clone(); 9];
- assert!(reference.verified_spans(raw).is_none());
+ assert!(reference.verified_spans(raw).is_none(), "overlapping spans");
+ reference.spans = (0..9)
+ .map(|start_utf8| PassageSpan {
+ start_utf8,
+ end_utf8: start_utf8 + 1,
+ text_sha256: "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb"
+ .into(),
+ })
+ .collect();
+ assert!(
+ reference.verified_spans("aaaaaaaaa").is_none(),
+ "span limit"
+ );
}
}
diff --git a/web/apps/ensu/src/pages/chat.tsx b/web/apps/ensu/src/pages/chat.tsx
index cdbf17cfa4..97a62b9bca 100644
--- a/web/apps/ensu/src/pages/chat.tsx
+++ b/web/apps/ensu/src/pages/chat.tsx
@@ -3093,10 +3093,9 @@ const Page: React.FC = () => {
.text.replaceAll(MEDIA_MARKER, "")
.replace(/\[\d+ image attachments? provided\]/gi, "")
.trim();
- const selectedMessages = slicePathUntil(
- historyPath,
- stopAtMessageUuid,
- );
+ const conversationPath = useConversationMemory
+ ? buildConversationPath(allMessages, parentMessage)
+ : [];
let resolutionToken: string | undefined;
const remainingKnowledgeBytes = useConversationMemory
? 6000
@@ -3146,20 +3145,15 @@ const Page: React.FC = () => {
if (
useConversationMemory &&
- selectedMessages.some((message) => message.sources?.length)
+ historyPath.some((message) => message.sources?.length)
) {
- const selectedPath = selectedMessages.map(
- (message) => message.messageUuid,
- );
- if (selectedPath.at(-1) !== parentMessageUuid)
- selectedPath.push(parentMessageUuid);
setConversationStatus("Finding sources");
try {
resolutionToken = await provider.withKnowledgeRetrieval(
(cancellationEpoch) =>
resolveDesktopSourceFollowup({
sessionUuid: activeSessionId,
- path: selectedPath,
+ path: conversationPath,
question: knowledgeQuery,
enabledStableIds: enabledReadyPackIds,
cancellationEpoch,
@@ -3244,10 +3238,7 @@ const Page: React.FC = () => {
const prepared = await prepareDesktopConversation(
{
sessionUuid: activeSessionId,
- path: buildConversationPath(
- allMessages,
- parentMessage,
- ),
+ path: conversationPath,
system: normalSystemPrompt,
current: promptText,
historyQuery: resolutionToken
@@ -3484,7 +3475,6 @@ const Page: React.FC = () => {
enabledKnowledgePackIds,
knowledgePacks,
loadEnabledKnowledgeCatalogOnce,
- slicePathUntil,
],
);