Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(evals): add retrieval decomposition instruments

AkitaOnRails7a2c603d7dee
diff --git a/Cargo.lock b/Cargo.lock
index bfe669d4..dd7656af 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -143,6 +143,7 @@ dependencies = [
"serde_json",
"sha2",
"tempfile",
+ "tokenizers",
"tokio",
"tracing",
"tracing-subscriber",
diff --git a/evals/Cargo.toml b/evals/Cargo.toml
index fa284e8a..bf005d10 100644
--- a/evals/Cargo.toml
+++ b/evals/Cargo.toml
@@ -20,6 +20,7 @@ clap.workspace = true
reqwest.workspace = true
sha2.workspace = true
tempfile.workspace = true
+tokenizers.workspace = true
jiff.workspace = true
schemars.workspace = true
secrecy.workspace = true
diff --git a/evals/README.md b/evals/README.md
index 47319c8f..d1843c85 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -288,6 +288,49 @@ cargo run -p ai-memory-eval -- retrieval --sample 3 \
--qa --qa-provider gemini --qa-model gemini-2.5-flash
```
+### `retrieval` decomposition instruments (M1)
+
+These flags split a hit@k number into evidence the index never saw and
+ranking on evidence it did. All are opt-in; the defaults reproduce the
+published baselines.
+
+- `--reachability` reads every evidence session back after the query and
+ reports, per slice, how many answer-bearing turns (the dataset's
+ `has_answer` flag) survive into the session page, the observation bodies,
+ and the first 512 tokens the local embedder reads. A turn counts as
+ reachable when at least half of its distinctive tokens (content words that
+ occur in at most 3 sessions of the haystack) are on that surface. The
+ module docs in `src/retrieval/reachability.rs` give the exact definition.
+ It needs the local model's tokenizer, which it fetches into
+ `evals/models/` if missing.
+- `--engine grep` ranks each question's raw sessions by query-term overlap
+ in-process, with no server. It follows agentmemory's grep adapter exactly
+ and matches its archived per-question results.
+- `--ingest whole-session` writes each raw session as one page through
+ `POST /admin/write-page` and then queries as usual: ranking on full
+ evidence. `--ingest captured-session` writes one page per session holding
+ only the text capture keeps (user turns up to 16 KiB, assistant excerpts up
+ to 2 KB).
+- `--date-prefix turn|session-start|none` moves the session date off every
+ turn (`turn` is the default).
+- `--stratify N` scores the first N questions of each type, so a cheap run
+ covers every slice (`--sample` takes a prefix, which is all one type).
+- `--dataset-variant s-cleaned` runs on the cleaned LongMemEval-S
+ re-release (pinned by sha256) that other systems report on.
+
+`--candidate-ingest` and `--candidate-date-prefix` put either knob into an
+A/B run. The A/B delta table also counts per-question flips (`+gained/-lost`),
+which shows whether a net delta is many changes cancelling out or a few in
+one direction.
+
+```bash
+cargo run --release -p ai-memory-eval -- retrieval --engine grep
+cargo run --release -p ai-memory-eval -- retrieval --reachability \
+ --ingest hooks --candidate-ingest whole-session
+cargo run --release -p ai-memory-eval -- retrieval --stratify 5 \
+ --date-prefix turn --candidate-date-prefix session-start
+```
+
Honest-numbers notes:
- capture is production-shaped: excerpts are bounded at the 2 KB
diff --git a/evals/src/retrieval/dataset.rs b/evals/src/retrieval/dataset.rs
index 27166494..6e99b7b8 100644
--- a/evals/src/retrieval/dataset.rs
+++ b/evals/src/retrieval/dataset.rs
@@ -3,6 +3,12 @@
//! The file is NOT committed (278 MB); it lives gitignored under
//! `evals/datasets/` and is fetched on demand from HuggingFace with a
//! pinned sha256. Dataset: `xiaowu0162/longmemeval` (MIT license).
+//!
+//! Two pinned variants load: the original `longmemeval_s` (every published
+//! ai-memory baseline) and the later `longmemeval_s_cleaned` re-release,
+//! which keeps the same 500 questions and evidence labels but replaces
+//! distractor sessions that also held the answer. Other systems publish on
+//! the cleaned file, so a comparison has to say which one it ran.
use std::path::Path;
@@ -19,11 +25,52 @@ pub const LONGMEMEVAL_S_URL: &str =
pub const LONGMEMEVAL_S_SHA256: &str =
"08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894";
+/// One pinned dataset file the harness accepts.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct DatasetVariant {
+ /// Name recorded in reports.
+ pub name: &'static str,
+ /// Default location under the repo root.
+ pub default_path: &'static str,
+ /// Download URL (revision-pinned where upstream allows it).
+ pub url: &'static str,
+ /// Expected sha256 of the file.
+ pub sha256: &'static str,
+}
+
+/// The original S file every published ai-memory baseline uses.
+pub const VARIANT_S: DatasetVariant = DatasetVariant {
+ name: "longmemeval_s",
+ default_path: "evals/datasets/longmemeval_s.json",
+ url: LONGMEMEVAL_S_URL,
+ sha256: LONGMEMEVAL_S_SHA256,
+};
+
+/// The cleaned S re-release (`xiaowu0162/longmemeval-cleaned`, revision
+/// `98d7416c`), the input agentmemory's 2026-10-09 baseline reports on.
+pub const VARIANT_S_CLEANED: DatasetVariant = DatasetVariant {
+ name: "longmemeval_s_cleaned",
+ default_path: "evals/datasets/longmemeval_s_cleaned.json",
+ url: "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_s_cleaned.json",
+ sha256: "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
+};
+
/// One chat turn inside a haystack session.
#[derive(Debug, Clone, Deserialize)]
pub struct Turn {
pub role: String,
pub content: String,
+ /// The dataset's evidence flag: this turn carries (part of) the answer.
+ /// Absent on most turns; only the reachability report reads it.
+ #[serde(default)]
+ pub has_answer: Option<bool>,
+}
+
+impl Turn {
+ /// True when the dataset marks this turn as answer-bearing.
+ pub fn is_evidence(&self) -> bool {
+ self.has_answer == Some(true)
+ }
}
/// One benchmark question with its private haystack of chat sessions.
@@ -63,8 +110,8 @@ impl Question {
}
}
-/// Load the dataset, verifying the pinned sha256 first.
-pub fn load(path: &Path) -> Result<Vec<Question>> {
+/// Load the dataset, verifying `variant`'s pinned sha256 first.
+pub fn load(path: &Path, variant: &DatasetVariant) -> Result<Vec<Question>> {
let bytes = std::fs::read(path).with_context(|| {
format!(
"reading dataset at {} (use --fetch to download)",
@@ -72,11 +119,13 @@ pub fn load(path: &Path) -> Result<Vec<Question>> {
)
})?;
let digest = hex::encode(Sha256::digest(&bytes));
- if digest != LONGMEMEVAL_S_SHA256 {
+ if digest != variant.sha256 {
bail!(
- "dataset sha256 mismatch at {}: got {digest}, pinned {LONGMEMEVAL_S_SHA256}. \
+ "dataset sha256 mismatch at {}: got {digest}, pinned {} for {}. \
Refusing to benchmark against an unpinned file.",
- path.display()
+ path.display(),
+ variant.sha256,
+ variant.name
);
}
let questions: Vec<Question> =
@@ -99,19 +148,19 @@ pub fn load(path: &Path) -> Result<Vec<Question>> {
/// Download the dataset to `path` (atomically: tmp + rename) and verify
/// the pinned sha256 before the rename lands it.
-pub async fn fetch(path: &Path) -> Result<()> {
+pub async fn fetch(path: &Path, variant: &DatasetVariant) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
- tracing::info!(
- url = LONGMEMEVAL_S_URL,
- "downloading LongMemEval-S (278 MB)"
- );
- let resp = reqwest::get(LONGMEMEVAL_S_URL).await?.error_for_status()?;
+ tracing::info!(url = variant.url, "downloading {} (~278 MB)", variant.name);
+ let resp = reqwest::get(variant.url).await?.error_for_status()?;
let bytes = resp.bytes().await?;
let digest = hex::encode(Sha256::digest(&bytes));
- if digest != LONGMEMEVAL_S_SHA256 {
- bail!("downloaded dataset sha256 {digest} does not match pin {LONGMEMEVAL_S_SHA256}");
+ if digest != variant.sha256 {
+ bail!(
+ "downloaded dataset sha256 {digest} does not match pin {}",
+ variant.sha256
+ );
}
let tmp = path.with_extension("part");
std::fs::write(&tmp, &bytes)?;
@@ -119,6 +168,22 @@ pub async fn fetch(path: &Path) -> Result<()> {
Ok(())
}
+/// First `per_type` questions of each question type, types in sorted order
+/// (the same deterministic selection as agentmemory's `stratifySample`).
+/// A prefix sample (`--sample`) only covers the first type, because the file
+/// is grouped by type; this one covers every slice.
+pub fn stratify(questions: Vec<Question>, per_type: usize) -> Vec<Question> {
+ let mut buckets: std::collections::BTreeMap<String, Vec<Question>> =
+ std::collections::BTreeMap::new();
+ for q in questions {
+ let bucket = buckets.entry(q.question_type.clone()).or_default();
+ if bucket.len() < per_type {
+ bucket.push(q);
+ }
+ }
+ buckets.into_values().flatten().collect()
+}
+
mod hex {
/// Minimal lowercase hex, avoiding a dependency for one call site.
pub fn encode(bytes: impl AsRef<[u8]>) -> String {
@@ -175,7 +240,35 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("longmemeval_s.json");
std::fs::write(&p, serde_json::to_vec(&sample_json()).unwrap()).unwrap();
- let err = load(&p).unwrap_err().to_string();
+ let err = load(&p, &VARIANT_S).unwrap_err().to_string();
assert!(err.contains("sha256 mismatch"), "{err}");
+ let err = load(&p, &VARIANT_S_CLEANED).unwrap_err().to_string();
+ assert!(err.contains("longmemeval_s_cleaned"), "{err}");
+ }
+
+ #[test]
+ fn the_evidence_flag_is_read_and_defaults_to_absent() {
+ let qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
+ let turns = &qs[0].haystack_sessions[0];
+ assert!(turns[0].is_evidence());
+ assert_eq!(turns[1].has_answer, None);
+ assert!(!turns[1].is_evidence());
+ }
+
+ #[test]
+ fn stratify_takes_a_prefix_of_every_type_in_sorted_order() {
+ let mut qs: Vec<Question> = Vec::new();
+ let base: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
+ for (i, ty) in ["b-type", "a-type", "b-type", "b-type", "a-type"]
+ .iter()
+ .enumerate()
+ {
+ let mut q = base[0].clone();
+ q.question_id = format!("q{i}");
+ q.question_type = (*ty).into();
+ qs.push(q);
+ }
+ let picked: Vec<String> = stratify(qs, 2).into_iter().map(|q| q.question_id).collect();
+ assert_eq!(picked, ["q1", "q4", "q0", "q2"]);
}
}
diff --git a/evals/src/retrieval/grep.rs b/evals/src/retrieval/grep.rs
new file mode 100644
index 00000000..21a8269e
--- /dev/null
+++ b/evals/src/retrieval/grep.rs
@@ -0,0 +1,166 @@
+//! `--engine grep`: a whole-session term-overlap baseline with no server.
+//!
+//! It reproduces agentmemory's `eval/runner/adapters/grep.ts` so the two
+//! harnesses can be calibrated against each other on the same questions:
+//!
+//! - a session's document is `[Source session date: <date>]` followed by
+//! every turn as `[<role>] <content>`, joined by blank lines (their
+//! `sessionText` over `loadLongMemEval`'s flattening);
+//! - query terms are the lowercased `[a-z0-9_]+` runs longer than two
+//! characters, duplicates kept;
+//! - a session scores one point per query term that occurs anywhere in its
+//! lowercased document (substring match, so `boot` matches `boots`);
+//! - sessions with no matching term are dropped, and ties keep haystack
+//! order (a stable sort, as in JavaScript);
+//! - a repeated session id keeps its first position (their loader
+//! coalesces repeats into the first occurrence).
+//!
+//! It measures what plain lexical overlap over the full raw transcript can
+//! reach, before any capture, truncation, or ranking choice of ours.
+
+use std::collections::HashSet;
+
+use super::dataset::{Question, Turn};
+use super::ingest::stored_session_uuid;
+
+/// Query terms, agentmemory-compatible: lowercase, split on anything that
+/// is not `[a-z0-9_]`, keep tokens longer than two characters. Duplicates
+/// are kept, so a repeated query word counts twice.
+pub fn tokenize(s: &str) -> Vec<String> {
+ s.to_lowercase()
+ .split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'))
+ .filter(|t| t.len() > 2)
+ .map(str::to_owned)
+ .collect()
+}
+
+/// One session rendered as a single document (see the module docs).
+pub fn session_text(date: &str, turns: &[Turn]) -> String {
+ let body = turns
+ .iter()
+ .map(|t| format!("[{}] {}", t.role, t.content))
+ .collect::<Vec<_>>()
+ .join("\n\n");
+ format!("[Source session date: {date}]\n{body}")
+}
+
+/// Rank `(id, lowercased document)` pairs for `query`, best first.
+pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str)>) -> Vec<T> {
+ let terms = tokenize(query);
+ let mut scored: Vec<(T, usize)> = docs
+ .into_iter()
+ .filter_map(|(id, body)| {
+ let hits = terms.iter().filter(|t| body.contains(t.as_str())).count();
+ (hits > 0).then_some((id, hits))
+ })
+ .collect();
+ // `sort_by_key` is stable: equal scores keep haystack order.
+ scored.sort_by_key(|(_, hits)| std::cmp::Reverse(*hits));
+ scored.into_iter().map(|(id, _)| id).collect()
+}
+
+/// Rank one question's haystack sessions, returning stored session uuids.
+pub fn rank_question(q: &Question) -> Vec<uuid::Uuid> {
+ let mut seen = HashSet::new();
+ let docs: Vec<(uuid::Uuid, String)> = q
+ .haystack_session_ids
+ .iter()
+ .zip(&q.haystack_dates)
+ .zip(&q.haystack_sessions)
+ .filter(|((id, _), _)| seen.insert(id.as_str()))
+ .map(|((id, date), turns)| {
+ (
+ stored_session_uuid(id),
+ session_text(date, turns).to_lowercase(),
+ )
+ })
+ .collect();
+ rank(
+ &q.question,
+ docs.iter().map(|(id, body)| (*id, body.as_str())),
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn turn(role: &str, content: &str) -> Turn {
+ Turn {
+ role: role.into(),
+ content: content.into(),
+ has_answer: None,
+ }
+ }
+
+ #[test]
+ fn tokenize_matches_the_reference_adapter() {
+ // `[^a-z0-9_]+` split after lowercasing, then length > 2.
+ assert_eq!(
+ tokenize("What's my dog's NAME, in 2023? a_b x"),
+ ["what", "dog", "name", "2023", "a_b"]
+ );
+ // Duplicates are kept: the reference does not dedupe query terms.
+ assert_eq!(tokenize("boot boot"), ["boot", "boot"]);
+ // Non-ASCII letters split words, as `[^a-z0-9_]` does.
+ assert_eq!(tokenize("café latte"), ["caf", "latte"]);
+ }
+
+ #[test]
+ fn session_text_flattens_roles_under_a_date_line() {
+ let t = session_text("2023/05/20", &[turn("user", "hi"), turn("assistant", "yo")]);
+ assert_eq!(
+ t,
+ "[Source session date: 2023/05/20]\n[user] hi\n\n[assistant] yo"
+ );
+ }
+
+ #[test]
+ fn rank_counts_term_occurrence_drops_zero_and_keeps_tie_order() {
+ let docs = [
+ (1, "nothing relevant"),
+ (2, "my boots are leather"),
+ (3, "leather boots and leather care"),
+ (4, "boots only"),
+ ];
+ // query terms: leather, boots → doc 2 and 3 score 2, doc 4 scores 1.
+ let ranked = rank("leather boots", docs);
+ assert_eq!(ranked, [2, 3, 4]);
+ }
+
+ #[test]
+ fn substring_matching_credits_inflections() {
+ assert_eq!(rank("boot", [(1, "two boots")]), [1]);
+ }
+
+ #[test]
+ fn repeated_query_terms_weigh_double() {
+ let docs = [(1, "alpha"), (2, "beta")];
+ assert_eq!(rank("beta beta alpha", docs), [2, 1]);
+ }
+
+ #[test]
+ fn repeated_session_ids_keep_their_first_position() {
+ let q: Question = serde_json::from_value(serde_json::json!({
+ "question_id": "q",
+ "question_type": "multi-session",
+ "question": "quokka",
+ "answer": "a",
+ "question_date": "d",
+ "haystack_dates": ["d1", "d2", "d3"],
+ "haystack_session_ids": ["s_a", "s_b", "s_a"],
+ "haystack_sessions": [
+ [{"role": "user", "content": "quokka"}],
+ [{"role": "user", "content": "quokka"}],
+ [{"role": "user", "content": "quokka"}]
+ ],
+ "answer_session_ids": ["s_b"]
+ }))
+ .unwrap();
+ let ranked = rank_question(&q);
+ assert_eq!(
+ ranked,
+ [stored_session_uuid("s_a"), stored_session_uuid("s_b")]
+ );
+ }
+}
diff --git a/evals/src/retrieval/ingest.rs b/evals/src/retrieval/ingest.rs
index ae1b2c89..9bf55120 100644
--- a/evals/src/retrieval/ingest.rs
+++ b/evals/src/retrieval/ingest.rs
@@ -12,8 +12,17 @@
//! installs see wall-clock time; replayed histories must carry their
//! own), mirroring what the official LongMemEval harness exposes to
//! retrievers as session timestamps.
+//!
+//! Two ablation knobs change that shape for the M1 decomposition (defaults
+//! keep every published baseline reproducible):
+//!
+//! - [`DatePrefix`] moves or drops the per-turn date prefix, which takes
+//! about 40 of the 80 characters a zero-LLM session page keeps per turn;
+//! - [`IngestMode`] can skip the hooks and write each haystack session as
+//! one wiki page through `POST /admin/write-page` (the real
+//! `Wiki::write_page` path), which isolates ranking from capture.
-use std::time::Duration;
+use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use serde_json::json;
@@ -32,6 +41,71 @@ const MAX_BATCH: usize = 256;
/// client to do the capping.
const EXCERPT_MAX_BYTES: usize = 2_000;
+/// Where the replay puts each session's original date.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
+pub enum DatePrefix {
+ /// `[session date: …]` before every turn (the published baselines).
+ Turn,
+ /// Once per session, as the `session-start` event's title; turns carry
+ /// no prefix.
+ SessionStart,
+ /// Nowhere.
+ None,
+}
+
+impl DatePrefix {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Turn => "turn",
+ Self::SessionStart => "session-start",
+ Self::None => "none",
+ }
+ }
+}
+
+/// How a question's haystack reaches the server.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
+pub enum IngestMode {
+ /// Replay through `/hook/batch` (the shipped capture path).
+ Hooks,
+ /// One page per session holding the full raw transcript: the
+ /// ranking-only upper bound, comparable to whole-session harnesses.
+ WholeSession,
+ /// One page per session holding exactly the text capture keeps in
+ /// observation bodies (user turns up to 16 KiB, assistant excerpts up
+ /// to 2 KB, date prefix as configured): what an observation-evidence
+ /// stream could rank on, aggregated per session.
+ CapturedSession,
+}
+
+impl IngestMode {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Hooks => "hooks",
+ Self::WholeSession => "whole-session",
+ Self::CapturedSession => "captured-session",
+ }
+ }
+}
+
+/// Server-side user-prompt body cap (`USER_PROMPT_EXCERPT_MAX_BYTES`,
+/// equal to the 16 KiB observation backstop).
+const USER_PROMPT_MAX_BYTES: usize = 16 * 1024;
+
+/// Timing and volume for one question's ingest.
+#[derive(Debug, Clone, Default)]
+pub struct IngestStats {
+ /// Hook events posted, or pages written.
+ pub items: usize,
+ /// Wall clock for the whole question.
+ pub elapsed: Duration,
+ /// Round trip of every HTTP request (batch POST or page write).
+ pub request_ms: Vec<u128>,
+ /// Pages whose body exceeded the 16 KiB observation backstop (page
+ /// modes only; the page write path has no such cap, see the report).
+ pub docs_over_16kib: usize,
+}
+
/// Project name for one benchmark question (its private haystack).
pub fn project_for(q: &Question) -> String {
format!("lme-{}", q.question_id)
@@ -49,16 +123,64 @@ pub fn stored_session_uuid(raw: &str) -> uuid::Uuid {
/// Truncate on a char boundary so the marker stays valid UTF-8.
fn cap_excerpt(s: &str) -> &str {
- if s.len() <= EXCERPT_MAX_BYTES {
+ cap_bytes(s, EXCERPT_MAX_BYTES)
+}
+
+fn cap_bytes(s: &str, max: usize) -> &str {
+ if s.len() <= max {
return s;
}
- let mut end = EXCERPT_MAX_BYTES;
+ let mut end = max;
while !s.is_char_boundary(end) {
end -= 1;
}
&s[..end]
}
+/// Turn text as the hook replay sends it under `prefix`.
+fn prefixed(prefix: DatePrefix, date: &str, text: &str) -> String {
+ match prefix {
+ DatePrefix::Turn => format!("[session date: {date}] {text}"),
+ DatePrefix::SessionStart | DatePrefix::None => text.to_string(),
+ }
+}
+
+/// One haystack session as a page body for the page ingest modes. Every
+/// turn renders as `[<role>] <text>`, joined by blank lines, under a
+/// `[Source session date: …]` line (the same document the grep baseline
+/// scores). `CapturedSession` caps each turn the way capture does and
+/// places the date exactly where the hook replay under `prefix` would.
+pub fn session_document(
+ mode: IngestMode,
+ prefix: DatePrefix,
+ date: &str,
+ turns: &[super::dataset::Turn],
+) -> String {
+ let rendered: Vec<String> = turns
+ .iter()
+ .map(|t| match mode {
+ IngestMode::CapturedSession => {
+ let text = match t.role.as_str() {
+ "assistant" => prefixed(prefix, date, cap_excerpt(&t.content)),
+ _ => prefixed(prefix, date, &t.content),
+ };
+ // The server caps the whole prompt field, prefix included.
+ format!("[{}] {}", t.role, cap_bytes(&text, USER_PROMPT_MAX_BYTES))
+ }
+ _ => format!("[{}] {}", t.role, t.content),
+ })
+ .collect();
+ // Captured text only carries a session-level date when the replay puts
+ // one on `session-start`; per-turn prefixes are already in the turns.
+ let header = !(mode == IngestMode::CapturedSession
+ && matches!(prefix, DatePrefix::Turn | DatePrefix::None));
+ if header {
+ format!("[Source session date: {date}]\n{}", rendered.join("\n\n"))
+ } else {
+ rendered.join("\n\n")
+ }
+}
+
#[derive(serde::Serialize)]
struct BatchItem {
url: String,
@@ -66,7 +188,7 @@ struct BatchItem {
}
/// Build the full event replay for one question's haystack.
-fn build_items(q: &Question) -> Vec<BatchItem> {
+fn build_items(q: &Question, prefix: DatePrefix) -> Vec<BatchItem> {
let project = project_for(q);
let cwd = format!("/lme/{project}");
let mut items = Vec::new();
@@ -92,11 +214,13 @@ fn build_items(q: &Question) -> Vec<BatchItem> {
for (idx, session) in q.haystack_sessions.iter().enumerate() {
let sid = &q.haystack_session_ids[idx];
let date = &q.haystack_dates[idx];
- push(
- "session-start",
- sid,
- json!({ "hook_event_name": "SessionStart", "source": "startup" }),
- );
+ let mut start = json!({ "hook_event_name": "SessionStart", "source": "startup" });
+ if prefix == DatePrefix::SessionStart {
+ // The session-start title lands in the page's raw-observation
+ // list, so the date stays indexed once per session.
+ start["title"] = json!(format!("session date: {date}"));
+ }
+ push("session-start", sid, start);
for turn in session {
match turn.role.as_str() {
"user" => push(
@@ -104,7 +228,7 @@ fn build_items(q: &Question) -> Vec<BatchItem> {
sid,
json!({
"hook_event_name": "UserPromptSubmit",
- "prompt": format!("[session date: {date}] {}", turn.content),
+ "prompt": prefixed(prefix, date, &turn.content),
}),
),
"assistant" => push(
@@ -114,7 +238,7 @@ fn build_items(q: &Question) -> Vec<BatchItem> {
"hook_event_name": "Stop",
"_ai_memory_assistant": {
"version": 1,
- "excerpt": format!("[session date: {date}] {}", cap_excerpt(&turn.content)),
+ "excerpt": prefixed(prefix, date, cap_excerpt(&turn.content)),
},
}),
),
@@ -139,8 +263,11 @@ pub async fn ingest_question(
client: &reqwest::Client,
base_url: &str,
q: &Question,
-) -> Result<usize> {
- let items = build_items(q);
+ prefix: DatePrefix,
+) -> Result<IngestStats> {
+ let started = Instant::now();
+ let mut request_ms = Vec::new();
+ let items = build_items(q, prefix);
let total = items.len();
let mut pending: Vec<&BatchItem> = items.iter().collect();
let mut attempts = 0usize;
@@ -156,6 +283,7 @@ pub async fn ingest_question(
}
let mut next = Vec::new();
for chunk in pending.chunks(MAX_BATCH) {
+ let sent = Instant::now();
let resp = client
.post(format!("{base_url}/hook/batch"))
.bearer_auth(EVAL_AUTH_TOKEN)
@@ -166,6 +294,7 @@ pub async fn ingest_question(
.error_for_status()
.context("hook batch rejected")?;
let ack: serde_json::Value = resp.json().await?;
+ request_ms.push(sent.elapsed().as_millis());
let accepted: Vec<usize> = match ack.get("accepted_indices") {
Some(serde_json::Value::Array(idx)) => idx
.iter()
@@ -188,7 +317,73 @@ pub async fn ingest_question(
}
pending = next;
}
- Ok(total)
+ Ok(IngestStats {
+ items: total,
+ elapsed: started.elapsed(),
+ request_ms,
+ docs_over_16kib: 0,
+ })
+}
+
+/// Page title for a session written by the page ingest modes. Neutral on
+/// purpose: it carries the date (as the hook pages' titles do) and nothing
+/// that could favour the evidence session.
+fn session_page_title(date: &str) -> String {
+ format!("Chat session {date}")
+}
+
+/// Write each haystack session of `q` as one page under
+/// `sessions/<uuid>.md` through `POST /admin/write-page`, sequentially
+/// (questions still run concurrently). A repeated session id is written
+/// once, at its first occurrence.
+pub async fn write_session_pages(
+ client: &reqwest::Client,
+ base_url: &str,
+ q: &Question,
+ mode: IngestMode,
+ prefix: DatePrefix,
+) -> Result<IngestStats> {
+ let started = Instant::now();
+ let project = project_for(q);
+ let mut seen = std::collections::HashSet::new();
+ let mut stats = IngestStats::default();
+ for ((sid, date), turns) in q
+ .haystack_session_ids
+ .iter()
+ .zip(&q.haystack_dates)
+ .zip(&q.haystack_sessions)
+ {
+ if !seen.insert(sid.as_str()) {
+ continue;
+ }
+ let body = session_document(mode, prefix, date, turns);
+ if body.len() > USER_PROMPT_MAX_BYTES {
+ stats.docs_over_16kib += 1;
+ }
+ let request = json!({
+ "workspace": EVAL_WORKSPACE,
+ "project": project,
+ "path": format!("sessions/{}.md", stored_session_uuid(sid)),
+ "title": session_page_title(date),
+ "body": body,
+ // Hook-synthesised session pages are episodic too.
+ "tier": "episodic",
+ });
+ let sent = Instant::now();
+ client
+ .post(format!("{base_url}/admin/write-page"))
+ .bearer_auth(EVAL_AUTH_TOKEN)
+ .json(&request)
+ .send()
+ .await
+ .context("posting write-page")?
+ .error_for_status()
+ .with_context(|| format!("write-page rejected for {sid}"))?;
+ stats.request_ms.push(sent.elapsed().as_millis());
+ stats.items += 1;
+ }
+ stats.elapsed = started.elapsed();
+ Ok(stats)
}
#[cfg(test)]
@@ -216,7 +411,7 @@ mod tests {
#[test]
fn replay_matches_the_production_hook_cadence() {
- let items = build_items(&q());
+ let items = build_items(&q(), DatePrefix::Turn);
let events: Vec<&str> = items
.iter()
.map(|i| {
@@ -246,7 +441,7 @@ mod tests {
#[test]
fn session_dates_are_visible_to_the_retriever() {
- let items = build_items(&q());
+ let items = build_items(&q(), DatePrefix::Turn);
let prompt = items[1].body["prompt"].as_str().unwrap();
assert!(prompt.starts_with("[session date: 2023/05/20"));
}
@@ -277,9 +472,88 @@ mod tests {
question.haystack_sessions[0].push(Turn {
role: "system".into(),
content: "x".into(),
+ has_answer: None,
});
- let items = build_items(&question);
+ let items = build_items(&question, DatePrefix::Turn);
// still start, prompt, stop, end — the system turn produced nothing
assert_eq!(items.len(), 4);
}
+
+ #[test]
+ fn session_start_prefix_moves_the_date_off_the_turns() {
+ let items = build_items(&q(), DatePrefix::SessionStart);
+ assert_eq!(
+ items[0].body["title"].as_str(),
+ Some("session date: 2023/05/20 (Sat) 02:21")
+ );
+ assert_eq!(items[1].body["prompt"].as_str(), Some("hello"));
+ assert_eq!(
+ items[2].body["_ai_memory_assistant"]["excerpt"].as_str(),
+ Some("hi")
+ );
+ }
+
+ #[test]
+ fn no_date_prefix_leaves_turns_and_session_start_bare() {
+ let items = build_items(&q(), DatePrefix::None);
+ assert!(items[0].body.get("title").is_none());
+ assert_eq!(items[1].body["prompt"].as_str(), Some("hello"));
+ }
+
+ #[test]
+ fn whole_session_documents_keep_every_byte_of_every_turn() {
+ let long = "y".repeat(EXCERPT_MAX_BYTES * 3);
+ let turns = vec![
+ Turn {
+ role: "user".into(),
+ content: "hello".into(),
+ has_answer: None,
+ },
+ Turn {
+ role: "assistant".into(),
+ content: long.clone(),
+ has_answer: None,
+ },
+ ];
+ let doc = session_document(IngestMode::WholeSession, DatePrefix::Turn, "D", &turns);
+ assert_eq!(
+ doc,
+ format!("[Source session date: D]\n[user] hello\n\n[assistant] {long}")
+ );
+ }
+
+ #[test]
+ fn captured_session_documents_apply_the_capture_caps_and_prefix() {
+ let turns = vec![
+ Turn {
+ role: "user".into(),
+ content: "u".repeat(USER_PROMPT_MAX_BYTES + 10),
+ has_answer: None,
+ },
+ Turn {
+ role: "assistant".into(),
+ content: "a".repeat(EXCERPT_MAX_BYTES + 10),
+ has_answer: None,
+ },
+ ];
+ let doc = session_document(IngestMode::CapturedSession, DatePrefix::Turn, "D", &turns);
+ let parts: Vec<&str> = doc.split("\n\n").collect();
+ let user = parts[0].strip_prefix("[user] ").unwrap();
+ assert!(user.starts_with("[session date: D] u"));
+ assert_eq!(user.len(), USER_PROMPT_MAX_BYTES);
+ let assistant = parts[1]
+ .strip_prefix("[assistant] [session date: D] ")
+ .unwrap();
+ assert_eq!(assistant.len(), EXCERPT_MAX_BYTES);
+
+ let bare = session_document(IngestMode::CapturedSession, DatePrefix::None, "D", &turns);
+ assert!(bare.starts_with("[user] uuu"));
+ let once = session_document(
+ IngestMode::CapturedSession,
+ DatePrefix::SessionStart,
+ "D",
+ &turns,
+ );
+ assert!(once.starts_with("[Source session date: D]\n[user] uuu"));
+ }
}
diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rs
index 0a4cac74..c3f6e552 100644
--- a/evals/src/retrieval/mod.rs
+++ b/evals/src/retrieval/mod.rs
@@ -15,11 +15,26 @@
//! (hit@k / recall@k), `memory_query` latency (p50/p95), and the context
//! tokens a result would cost the agent (chars/4 over returned snippets).
//! Without a candidate config the single-config path is unchanged.
+//!
+//! ## M1: decomposing the retrieval gap
+//!
+//! Four opt-in instruments split a hit@k number into capture loss and
+//! ranking loss (defaults reproduce the published baselines):
+//!
+//! - `--reachability` checks whether each answer-bearing turn survives
+//! into the page, the observations, and the embedder's 512-token window;
+//! - `--engine grep` scores a whole-session term-overlap baseline with no
+//! server, calibrated against agentmemory's grep adapter;
+//! - `--ingest whole-session|captured-session` writes one page per session
+//! through the real page write path instead of replaying hooks;
+//! - `--date-prefix turn|session-start|none` moves the per-turn date.
mod dataset;
+mod grep;
mod ingest;
mod qa;
mod query;
+mod reachability;
mod report;
mod score;
@@ -30,17 +45,24 @@ use std::sync::Arc;
use anyhow::{Context, Result, bail};
use jiff::Timestamp;
-use dataset::Question;
-use ingest::{project_for, stored_session_uuid};
+use dataset::{DatasetVariant, Question};
+use ingest::{DatePrefix, IngestMode, project_for, stored_session_uuid};
use server::{EvalEmbeddings, EvalServer, LaunchConfig};
mod server;
#[derive(clap::Args, Debug)]
pub struct RetrievalArgs {
- /// Dataset file (LongMemEval-S JSON).
- #[arg(long, default_value = "evals/datasets/longmemeval_s.json")]
- dataset: PathBuf,
+ /// Dataset file (LongMemEval-S JSON). Defaults to the selected
+ /// variant's path under `evals/datasets/`.
+ #[arg(long)]
+ dataset: Option<PathBuf>,
+
+ /// Which pinned LongMemEval-S file to use: `s` (the original, every
+ /// published ai-memory baseline) or `s-cleaned` (the re-release other
+ /// systems publish on).
+ #[arg(long, value_enum, default_value_t = Variant::S)]
+ dataset_variant: Variant,
/// Download the dataset first if the file is missing.
#[arg(long)]
@@ -56,6 +78,23 @@ pub struct RetrievalArgs {
#[arg(long)]
sample: Option<usize>,
+ /// Score only the first N scored (non-abstention) questions of each
+ /// question type: a deterministic sample that covers every slice.
+ /// Applied after `--sample`.
+ #[arg(long)]
+ stratify: Option<usize>,
+
+ /// `server` (the default) drives a real `ai-memory serve`; `grep`
+ /// scores the whole-session term-overlap baseline with no server.
+ #[arg(long, value_enum, default_value_t = Engine::Server)]
+ engine: Engine,
+
+ /// After ingest, report how much answer-bearing evidence reaches the
+ /// page, the observations, and the local embedder's 512-token window.
+ /// Needs the local model's tokenizer (fetched into `evals/models/`).
+ #[arg(long)]
+ reachability: bool,
+
/// Cutoffs for hit@k / recall@k.
#[arg(long, value_delimiter = ',', default_values_t = vec![1, 3, 5, 10])]
ks: Vec<usize>,
@@ -99,6 +138,17 @@ pub struct RetrievalArgs {
#[arg(long = "query-arg", value_name = "KEY=JSON")]
query_arg: Vec<String>,
+ /// How the haystack reaches the baseline server: `hooks` (the shipped
+ /// capture path), `whole-session` (one page per raw session), or
+ /// `captured-session` (one page per session holding the captured text).
+ #[arg(long, value_enum, default_value_t = IngestMode::Hooks)]
+ ingest: IngestMode,
+
+ /// Where the baseline replay puts each session's date: `turn` (every
+ /// turn, the published baselines), `session-start`, or `none`.
+ #[arg(long, value_enum, default_value_t = DatePrefix::Turn)]
+ date_prefix: DatePrefix,
+
// ---- candidate config knobs (A/B mode) ----
/// Enable A/B mode with a candidate config. Implied by any
/// `--candidate-*` knob. With no other candidate knob the candidate
@@ -124,6 +174,14 @@ pub struct RetrievalArgs {
#[arg(long = "candidate-query-arg", value_name = "KEY=JSON")]
candidate_query_arg: Vec<String>,
+ /// Candidate ingest mode; defaults to the baseline's `--ingest`.
+ #[arg(long, value_enum)]
+ candidate_ingest: Option<IngestMode>,
+
+ /// Candidate date placement; defaults to the baseline's `--date-prefix`.
+ #[arg(long, value_enum)]
+ candidate_date_prefix: Option<DatePrefix>,
+
// ---- QA-accuracy mode (R2b, opt-in, live LLM) ----
/// Enable end-to-end QA-accuracy grading (real LLM API calls). OFF by
/// default. Per scored question the harness synthesizes an answer from
@@ -195,9 +253,34 @@ impl RetrievalArgs {
|| self.candidate_reranker
|| !self.candidate_server_env.is_empty()
|| !self.candidate_query_arg.is_empty()
+ || self.candidate_ingest.is_some()
+ || self.candidate_date_prefix.is_some()
+ }
+}
+
+/// `--dataset-variant`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
+enum Variant {
+ S,
+ SCleaned,
+}
+
+impl Variant {
+ fn pinned(self) -> &'static DatasetVariant {
+ match self {
+ Self::S => &dataset::VARIANT_S,
+ Self::SCleaned => &dataset::VARIANT_S_CLEANED,
+ }
}
}
+/// `--engine`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
+enum Engine {
+ Server,
+ Grep,
+}
+
/// One resolved server + query configuration.
struct Knobs {
/// Short label for reports (`baseline` / `candidate`).
@@ -209,17 +292,22 @@ struct Knobs {
reranker: bool,
env: Vec<(String, String)>,
query_args: serde_json::Map<String, serde_json::Value>,
+ ingest: IngestMode,
+ date_prefix: DatePrefix,
/// Human-readable knob summary for the report.
summary: String,
}
impl Knobs {
+ #[allow(clippy::too_many_arguments)]
fn parse(
side: &'static str,
embeddings: &str,
reranker: bool,
server_env: &[String],
query_arg: &[String],
+ ingest: IngestMode,
+ date_prefix: DatePrefix,
) -> Result<Self> {
let (embeddings_kind, mode, models_root) = match embeddings {
"none" => (EvalEmbeddings::None, "zero-llm", None),
@@ -243,6 +331,14 @@ impl Knobs {
for (k, v) in &query_args {
summary.push_str(&format!(" arg:{k}={v}"));
}
+ // Only non-default ablations are named, so a default run's summary
+ // reads exactly as the published baselines do.
+ if ingest != IngestMode::Hooks {
+ summary.push_str(&format!(" ingest={}", ingest.as_str()));
+ }
+ if date_prefix != DatePrefix::Turn {
+ summary.push_str(&format!(" date_prefix={}", date_prefix.as_str()));
+ }
Ok(Self {
side,
@@ -252,6 +348,8 @@ impl Knobs {
reranker,
env,
query_args,
+ ingest,
+ date_prefix,
summary,
})
}
@@ -311,10 +409,15 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
// Captured at launch: a long run must not stamp its report with a
// commit that landed while it was in flight.
let commit = report::commit_sha();
- if args.fetch && !args.dataset.exists() {
- dataset::fetch(&args.dataset).await?;
+ let variant = args.dataset_variant.pinned();
+ let dataset_path = args
+ .dataset
+ .clone()
+ .unwrap_or_else(|| PathBuf::from(variant.default_path));
+ if args.fetch && !dataset_path.exists() {
+ dataset::fetch(&dataset_path, variant).await?;
}
- let mut questions = dataset::load(&args.dataset)?;
+ let mut questions = dataset::load(&dataset_path, variant)?;
let total_loaded = questions.len();
if let Some(n) = args.sample {
questions.truncate(n);
@@ -325,6 +428,9 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
.cloned()
.collect();
questions.retain(|q| !q.is_abstention());
+ if let Some(per_type) = args.stratify {
+ questions = dataset::stratify(questions, per_type);
+ }
tracing::info!(
loaded = total_loaded,
scored = questions.len(),
@@ -332,6 +438,30 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
"dataset ready"
);
+ let stamp: String = Timestamp::now()
+ .to_string()
+ .chars()
+ .map(|c| if c == ':' || c == '.' { '-' } else { c })
+ .collect();
+ let run_dir = args.out.join(format!("{stamp}-retrieval"));
+
+ if args.engine == Engine::Grep {
+ if args.ab_mode() || args.reachability || args.qa {
+ bail!("--engine grep runs alone: no candidate, --reachability, or --qa");
+ }
+ let report = run_grep(&args, variant, &questions, abstention.len(), &commit);
+ std::fs::create_dir_all(&run_dir)?;
+ std::fs::write(
+ run_dir.join("report.json"),
+ serde_json::to_vec_pretty(&report)?,
+ )?;
+ let md = report::to_markdown(&report);
+ std::fs::write(run_dir.join("report.md"), &md)?;
+ println!("{md}");
+ println!("run dir: {}", run_dir.display());
+ return Ok(());
+ }
+
if !args.server_bin.exists() {
bail!(
"server binary {} not found — run `cargo build --release -p ai-memory-cli` first",
@@ -345,6 +475,8 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
args.reranker,
&args.server_env,
&args.query_arg,
+ args.ingest,
+ args.date_prefix,
)?;
let candidate = if args.ab_mode() {
let embeddings = args
@@ -357,6 +489,8 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
args.candidate_reranker,
&args.candidate_server_env,
&args.candidate_query_arg,
+ args.candidate_ingest.unwrap_or(args.ingest),
+ args.candidate_date_prefix.unwrap_or(args.date_prefix),
)?)
} else {
None
@@ -367,7 +501,7 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
|| candidate
.as_ref()
.is_some_and(|c| c.embeddings == EvalEmbeddings::Local);
- if needs_local {
+ if needs_local || args.reachability {
let root = PathBuf::from("evals/models");
if !ai_memory_llm::model_present(&root) {
tracing::info!("fetching the local embedding model into evals/models (~87 MB)");
@@ -396,24 +530,28 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
None
};
+ // The embedder-window check reads the model's own tokenizer.
+ let embed_window = if args.reachability {
+ Some(Arc::new(reachability::EmbedWindow::load(
+ std::path::Path::new("evals/models"),
+ )?))
+ } else {
+ None
+ };
+
let abstention_n = abstention.len();
- let baseline_report = run_config(
- &args,
- &baseline,
- &questions,
+ let run = RunContext {
+ args: &args,
+ variant,
+ questions: &questions,
abstention_n,
- &commit,
- qa_engine.clone(),
- )
- .await?;
+ commit: &commit,
+ qa: qa_engine.clone(),
+ embed_window,
+ };
+ let baseline_report = run_config(&run, &baseline).await?;
// Timestamped run dir shared by single-config and A/B outputs.
- let stamp: String = Timestamp::now()
- .to_string()
- .chars()
- .map(|c| if c == ':' || c == '.' { '-' } else { c })
- .collect();
- let run_dir = args.out.join(format!("{stamp}-retrieval"));
std::fs::create_dir_all(&run_dir)?;
match candidate {
@@ -427,15 +565,7 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
println!("{md}");
}
Some(candidate) => {
- let candidate_report = run_config(
- &args,
- &candidate,
- &questions,
- abstention_n,
- &commit,
- qa_engine.clone(),
- )
- .await?;
+ let candidate_report = run_config(&run, &candidate).await?;
let ab = report::AbReport::new(baseline_report, candidate_report);
std::fs::write(run_dir.join("report.json"), serde_json::to_vec_pretty(&ab)?)?;
let md = report::ab_to_markdown(&ab);
@@ -447,18 +577,30 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
Ok(())
}
+/// Everything a config run shares with its A/B twin.
+struct RunContext<'a> {
+ args: &'a RetrievalArgs,
+ variant: &'static DatasetVariant,
+ questions: &'a [Question],
+ abstention_n: usize,
+ commit: &'a str,
+ qa: Option<Arc<qa::QaEngine>>,
+ embed_window: Option<Arc<reachability::EmbedWindow>>,
+}
+
+/// What one question's task hands back for aggregation.
+struct QuestionRun {
+ score: score::QuestionScore,
+ ingest: ingest::IngestStats,
+ reach: Option<reachability::QuestionReach>,
+}
+
/// Launch one server config, replay + query + score the whole question
/// set through it, and build its [`report::Report`]. Each config gets a
/// fresh server (and fresh data dir) so ingestion never bleeds between
/// baseline and candidate.
-async fn run_config(
- args: &RetrievalArgs,
- knobs: &Knobs,
- questions: &[Question],
- abstention_n: usize,
- commit: &str,
- qa: Option<Arc<qa::QaEngine>>,
-) -> Result<report::Report> {
+async fn run_config(run: &RunContext<'_>, knobs: &Knobs) -> Result<report::Report> {
+ let args = run.args;
let server = EvalServer::launch(&args.server_bin, args.keep_data_dir, &knobs.launch_config())
.await
.with_context(|| format!("launching {} server", knobs.side))?;
@@ -478,23 +620,33 @@ async fn run_config(
let base_url = Arc::new(server.base_url.clone());
let client = Arc::new(client);
let query_args = Arc::new(knobs.query_args.clone());
+ let (ingest_mode, date_prefix) = (knobs.ingest, knobs.date_prefix);
+ let reach_on = args.reachability;
+ let started = std::time::Instant::now();
let semaphore = Arc::new(tokio::sync::Semaphore::new(args.concurrency.max(1)));
let mut tasks = tokio::task::JoinSet::new();
- for q in questions.iter().cloned() {
+ for q in run.questions.iter().cloned() {
let permit = semaphore.clone().acquire_owned().await?;
- let (client, base_url, ks, query_args, qa) = (
+ let (client, base_url, ks, query_args, qa, embed_window) = (
client.clone(),
base_url.clone(),
ks.clone(),
query_args.clone(),
- qa.clone(),
+ run.qa.clone(),
+ run.embed_window.clone(),
);
tasks.spawn(async move {
let _permit = permit;
- let events = ingest::ingest_question(&client, &base_url, &q)
- .await
- .with_context(|| format!("ingesting {}", q.question_id))?;
+ let ingest_stats = match ingest_mode {
+ IngestMode::Hooks => {
+ ingest::ingest_question(&client, &base_url, &q, date_prefix).await
+ }
+ mode => {
+ ingest::write_session_pages(&client, &base_url, &q, mode, date_prefix).await
+ }
+ }
+ .with_context(|| format!("ingesting {}", q.question_id))?;
let outcome = query::memory_query(
&client,
&base_url,
@@ -520,6 +672,26 @@ async fn run_config(
outcome.latency.as_millis(),
outcome.context_tokens,
);
+ score.ingest_ms = ingest_stats.elapsed.as_millis();
+ score.raw_hits = outcome.raw_hits;
+ // Read back only after the query, so the measurement never
+ // changes what the query saw.
+ let reach = if reach_on {
+ Some(
+ reachability::measure_question(
+ &client,
+ &base_url,
+ &q,
+ ingest_mode,
+ date_prefix,
+ embed_window.as_deref(),
+ )
+ .await
+ .with_context(|| format!("reachability for {}", q.question_id))?,
+ )
+ } else {
+ None
+ };
// QA-accuracy (R2b): answer + grade. A QA failure for one
// question is logged and left ungraded — it must not waste the
// whole retrieval run (retrieval numbers are already collected).
@@ -538,7 +710,7 @@ async fn run_config(
}
tracing::info!(
question = q.question_id,
- events,
+ items = ingest_stats.items,
retrieved = outcome.retrieved.len(),
latency_ms = outcome.latency.as_millis() as u64,
context_tokens = outcome.context_tokens,
@@ -546,7 +718,11 @@ async fn run_config(
qa_correct = score.qa.as_ref().map(|o| o.correct),
"scored"
);
- anyhow::Ok(score)
+ anyhow::Ok(QuestionRun {
+ score,
+ ingest: ingest_stats,
+ reach,
+ })
});
}
@@ -554,10 +730,16 @@ async fn run_config(
// the other 499, but a partial run must never masquerade as a
// publishable number either — any failure aborts before reporting.
let mut scores = Vec::new();
+ let mut ingests = Vec::new();
+ let mut reaches = Vec::new();
let mut failures = Vec::new();
while let Some(res) = tasks.join_next().await {
match res.context("task panicked")? {
- Ok(score) => scores.push(score),
+ Ok(done) => {
+ scores.push(done.score);
+ ingests.push(done.ingest);
+ reaches.extend(done.reach);
+ }
Err(e) => failures.push(format!("{e:#}")),
}
}
@@ -572,26 +754,95 @@ async fn run_config(
failures.len() + scores.len()
);
}
+ let wall = started.elapsed();
scores.sort_by(|a, b| a.question_id.cmp(&b.question_id));
+ reaches.sort_by(|a, b| a.question_id.cmp(&b.question_id));
let slices = score::aggregate(&scores, &args.ks);
Ok(report::Report {
generated_at: Timestamp::now().to_string(),
- commit: commit.to_string(),
+ commit: run.commit.to_string(),
hardware: report::hardware(),
- dataset: "longmemeval_s",
- dataset_sha256: dataset::LONGMEMEVAL_S_SHA256,
+ dataset: run.variant.name,
+ dataset_sha256: run.variant.sha256,
mode: knobs.mode,
config: knobs.summary.clone(),
questions_scored: scores.len(),
- abstention_excluded: abstention_n,
+ abstention_excluded: run.abstention_n,
ks: args.ks.clone(),
- qa: qa.as_ref().map(|e| e.report_meta()),
+ qa: run.qa.as_ref().map(|e| e.report_meta()),
+ ingest: Some(report::IngestSummary::new(
+ ingest_mode,
+ date_prefix,
+ args.concurrency,
+ &ingests,
+ wall,
+ )),
+ reachability: (!reaches.is_empty()).then(|| report::ReachabilityReport {
+ slices: reachability::aggregate(&reaches),
+ per_question: reaches,
+ }),
slices,
per_question: scores,
})
}
+/// `--engine grep`: rank every question's raw haystack in-process.
+fn run_grep(
+ args: &RetrievalArgs,
+ variant: &'static DatasetVariant,
+ questions: &[Question],
+ abstention_n: usize,
+ commit: &str,
+) -> report::Report {
+ let scores: Vec<score::QuestionScore> = questions
+ .iter()
+ .map(|q| {
+ let started = std::time::Instant::now();
+ let ranked = grep::rank_question(q);
+ let latency = started.elapsed().as_millis();
+ let retrieved: Vec<query::Retrieved> = ranked
+ .into_iter()
+ .map(|id| query::Retrieved {
+ session_uuid: Some(id),
+ })
+ .collect();
+ let evidence: HashSet<uuid::Uuid> = q
+ .answer_session_ids
+ .iter()
+ .map(|s| stored_session_uuid(s))
+ .collect();
+ score::score_question(
+ &q.question_id,
+ &q.question_type,
+ &evidence,
+ &retrieved,
+ &args.ks,
+ latency,
+ 0,
+ )
+ })
+ .collect();
+ let slices = score::aggregate(&scores, &args.ks);
+ report::Report {
+ generated_at: Timestamp::now().to_string(),
+ commit: commit.to_string(),
+ hardware: report::hardware(),
+ dataset: variant.name,
+ dataset_sha256: variant.sha256,
+ mode: "grep",
+ config: "engine=grep (whole raw session, term overlap, no server)".into(),
+ questions_scored: scores.len(),
+ abstention_excluded: abstention_n,
+ ks: args.ks.clone(),
+ qa: None,
+ ingest: None,
+ reachability: None,
+ slices,
+ per_question: scores,
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -642,6 +893,8 @@ mod tests {
true,
&["AI_MEMORY_LLM_PROVIDER=openai".into()],
&["include_expired=true".into()],
+ IngestMode::Hooks,
+ DatePrefix::Turn,
)
.unwrap();
assert!(k.summary.contains("embeddings=local"));
@@ -649,5 +902,21 @@ mod tests {
assert!(k.summary.contains("env:AI_MEMORY_LLM_PROVIDER=openai"));
assert!(k.summary.contains("arg:include_expired=true"));
assert_eq!(k.mode, "local-embeddings");
+ // Default ablations stay out of the summary (baseline reports
+ // read as before); non-default ones are named.
+ assert!(!k.summary.contains("ingest="));
+ assert!(!k.summary.contains("date_prefix="));
+ let k = Knobs::parse(
+ "candidate",
+ "none",
+ false,
+ &[],
+ &[],
+ IngestMode::WholeSession,
+ DatePrefix::SessionStart,
+ )
+ .unwrap();
+ assert!(k.summary.contains("ingest=whole-session"));
+ assert!(k.summary.contains("date_prefix=session-start"));
}
}
diff --git a/evals/src/retrieval/qa.rs b/evals/src/retrieval/qa.rs
index ce533173..1e42a2c3 100644
--- a/evals/src/retrieval/qa.rs
+++ b/evals/src/retrieval/qa.rs
@@ -484,6 +484,7 @@ mod tests {
answer: None,
latency: std::time::Duration::ZERO,
context_tokens: 0,
+ raw_hits: 0,
};
let qa = engine
.evaluate("What is the user's dog's name?", "Rex", &outcome)
diff --git a/evals/src/retrieval/query.rs b/evals/src/retrieval/query.rs
index d0d82c88..89c7358a 100644
--- a/evals/src/retrieval/query.rs
+++ b/evals/src/retrieval/query.rs
@@ -58,6 +58,10 @@ pub struct QueryOutcome {
/// puts in front of the agent, so this is the context cost of the
/// retrieval, not of the underlying full pages.
pub context_tokens: usize,
+ /// Raw observation hits in the response. The server only falls back to
+ /// them when page search returns nothing, so a non-zero count marks a
+ /// question answered from observations.
+ pub raw_hits: usize,
}
#[derive(Debug, Deserialize)]
@@ -120,16 +124,54 @@ pub async fn memory_query(
arguments.insert("workspace".into(), json!(workspace));
arguments.insert("project".into(), json!(project));
arguments.insert("limit".into(), json!(limit));
+ let started = Instant::now();
+ let text = mcp_tool_text(
+ client,
+ base_url,
+ "memory_query",
+ serde_json::Value::Object(arguments),
+ )
+ .await?;
+ let latency = started.elapsed();
+ let parsed: QueryResponse = serde_json::from_str(&text)
+ .with_context(|| format!("parsing memory_query JSON: {text}"))?;
+ let raw_hits = parsed.raw_hits.len();
+ // A blank/whitespace `answer` is treated as absent so the QA path falls
+ // back to harness synthesis rather than grading an empty string.
+ let answer = parsed
+ .answer
+ .as_deref()
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .map(str::to_owned);
+ let (retrieved, contexts, context_tokens) = flatten(parsed);
+ Ok(QueryOutcome {
+ retrieved,
+ contexts,
+ answer,
+ latency,
+ context_tokens,
+ raw_hits,
+ })
+}
+
+/// Call one MCP tool over Streamable HTTP and return its first text
+/// content block (every ai-memory tool answers with one JSON text block).
+pub async fn mcp_tool_text(
+ client: &reqwest::Client,
+ base_url: &str,
+ tool: &str,
+ arguments: serde_json::Value,
+) -> Result<String> {
let request = json!({
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {
- "name": "memory_query",
+ "name": tool,
"arguments": arguments,
}
});
- let started = Instant::now();
let resp = client
.post(format!("{base_url}/mcp"))
.bearer_auth(EVAL_AUTH_TOKEN)
@@ -178,33 +220,16 @@ pub async fn memory_query(
.and_then(|v| v.as_bool())
.unwrap_or(false)
{
- bail!("memory_query tool error: {result}");
+ bail!("{tool} tool error: {result}");
}
- let text = result
+ result
.get("content")
.and_then(|c| c.as_array())
.and_then(|c| c.first())
.and_then(|c| c.get("text"))
.and_then(|t| t.as_str())
- .ok_or_else(|| anyhow::anyhow!("memory_query returned no text content: {result}"))?;
- let parsed: QueryResponse =
- serde_json::from_str(text).with_context(|| format!("parsing memory_query JSON: {text}"))?;
- // A blank/whitespace `answer` is treated as absent so the QA path falls
- // back to harness synthesis rather than grading an empty string.
- let answer = parsed
- .answer
- .as_deref()
- .map(str::trim)
- .filter(|s| !s.is_empty())
- .map(str::to_owned);
- let (retrieved, contexts, context_tokens) = flatten(parsed);
- Ok(QueryOutcome {
- retrieved,
- contexts,
- answer,
- latency: started.elapsed(),
- context_tokens,
- })
+ .map(str::to_owned)
+ .ok_or_else(|| anyhow::anyhow!("{tool} returned no text content: {result}"))
}
/// Flatten pages + raw hits into ranked session attributions and their
diff --git a/evals/src/retrieval/reachability.rs b/evals/src/retrieval/reachability.rs
new file mode 100644
index 00000000..1f059e3d
--- /dev/null
+++ b/evals/src/retrieval/reachability.rs
@@ -0,0 +1,840 @@
+//! `--reachability`: does the evidence reach the index at all?
+//!
+//! After a question's haystack is ingested, every answer-bearing turn (the
+//! dataset's `has_answer` flag) is checked against three surfaces of its
+//! own session, read back through the public MCP tools:
+//!
+//! 1. **page**: the session page's title plus body (`memory_read_page`),
+//! which is what page FTS indexes;
+//! 2. **observations**: every observation title and body of the session
+//! (`memory_read_session_observations`, full 16 KiB bodies), which is
+//! what `observations_fts` indexes (hook ingest only);
+//! 3. **embed512**: the prefix of the page body the local embedder actually
+//! reads: the body cut to 6 144 bytes (`truncate_for_embedding` with
+//! `MAX_TOKENS * 4`), then to 512 WordPiece tokens including `[CLS]` and
+//! `[SEP]`, using the model's own tokenizer.
+//!
+//! ## Distinctive answer content
+//!
+//! A turn's *content tokens* are its lowercased alphanumeric runs of three
+//! or more characters, plus any shorter run containing a digit, minus a
+//! fixed English stopword list. Its *distinctive* tokens are the content
+//! tokens that occur in at most [`DF_MAX`] distinct sessions of the
+//! question's haystack: words a retriever could use to single this session
+//! out, as opposed to words every chat contains. A turn with no such token
+//! falls back to all its content tokens (counted in the report).
+//!
+//! A turn is **reachable** through a surface when at least
+//! [`REACH_THRESHOLD`] of its distinctive tokens occur in that surface's
+//! token set. Matching is exact token equality, a little stricter than
+//! FTS5's tokenizer; a word cut by a title's `…` does not count.
+//!
+//! A second, answer-anchored check uses the gold answer: the answer's
+//! content tokens that literally occur in the turn must *all* occur in the
+//! surface. Many answers are derived (counts, durations, dates) and never
+//! appear verbatim, so this covers fewer turns but needs no threshold.
+//!
+//! The question-level ceiling is the share of questions with at least one
+//! reachable evidence turn: a ranker cannot put a session in the top k on
+//! evidence the index never saw.
+
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::path::Path;
+
+use anyhow::{Context, Result, bail};
+use serde::Serialize;
+use serde_json::json;
+
+use super::dataset::Question;
+use super::ingest::stored_session_uuid;
+use super::ingest::{DatePrefix, EVAL_WORKSPACE, IngestMode, project_for, session_document};
+use super::query::mcp_tool_text;
+
+/// A token is distinctive when it occurs in at most this many sessions of
+/// the question's haystack (~50 sessions; multi-session evidence spans up
+/// to a handful, so 3 keeps shared topic words of 2-3 evidence sessions).
+pub const DF_MAX: usize = 3;
+
+/// Share of a turn's distinctive tokens a surface must contain.
+pub const REACH_THRESHOLD: f64 = 0.5;
+
+/// Local embedder input bounds, mirrored from `ai-memory-llm`:
+/// `truncate_for_embedding(text, MAX_TOKENS * 4)` keeps
+/// `min(MAX_TOKENS * 4 * 3, 8000)` bytes, then the tokenizer truncates to
+/// `MAX_TOKENS`.
+const EMBED_MAX_TOKENS: usize = 512;
+const EMBED_PRE_TRUNCATE_BYTES: usize = 6_144;
+
+/// Function words that carry no retrieval signal on their own.
+const STOPWORDS: &[&str] = &[
+ "the",
+ "and",
+ "for",
+ "are",
+ "but",
+ "not",
+ "you",
+ "your",
+ "yours",
+ "all",
+ "any",
+ "can",
+ "had",
+ "her",
+ "hers",
+ "was",
+ "one",
+ "our",
+ "ours",
+ "out",
+ "has",
+ "have",
+ "him",
+ "his",
+ "how",
+ "its",
+ "may",
+ "she",
+ "they",
+ "them",
+ "their",
+ "theirs",
+ "then",
+ "than",
+ "that",
+ "this",
+ "these",
+ "those",
+ "there",
+ "here",
+ "what",
+ "when",
+ "where",
+ "which",
+ "who",
+ "whom",
+ "why",
+ "with",
+ "would",
+ "could",
+ "should",
+ "will",
+ "shall",
+ "from",
+ "into",
+ "onto",
+ "upon",
+ "about",
+ "above",
+ "below",
+ "after",
+ "before",
+ "again",
+ "also",
+ "just",
+ "only",
+ "very",
+ "some",
+ "such",
+ "more",
+ "most",
+ "much",
+ "many",
+ "each",
+ "other",
+ "own",
+ "same",
+ "too",
+ "off",
+ "over",
+ "under",
+ "were",
+ "been",
+ "being",
+ "does",
+ "did",
+ "doing",
+ "done",
+ "get",
+ "got",
+ "let",
+ "like",
+ "make",
+ "made",
+ "well",
+ "yes",
+ "yeah",
+ "okay",
+ "sure",
+ "thanks",
+ "thank",
+ "please",
+ "know",
+ "think",
+ "want",
+ "need",
+ "really",
+ "because",
+ "while",
+ "though",
+ "even",
+ "still",
+ "both",
+ "either",
+ "neither",
+ "nor",
+ "via",
+ "per",
+ "etc",
+ "don",
+ "doesn",
+ "didn",
+ "can",
+ "won",
+ "isn",
+ "aren",
+ "wasn",
+ "weren",
+ "myself",
+ "yourself",
+ "itself",
+ "himself",
+ "herself",
+ "ourselves",
+ "themselves",
+ "whose",
+ "something",
+ "anything",
+ "everything",
+ "nothing",
+ "way",
+ "lot",
+ "thing",
+ "things",
+];
+
+/// Content tokens of `text` (see the module docs).
+pub fn content_tokens(text: &str) -> HashSet<String> {
+ text.to_lowercase()
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|t| {
+ !t.is_empty()
+ && (t.chars().count() >= 3 || t.chars().any(|c| c.is_ascii_digit()))
+ && !STOPWORDS.contains(t)
+ })
+ .map(str::to_owned)
+ .collect()
+}
+
+/// One answer-bearing turn and the tokens that identify it.
+#[derive(Debug, Clone)]
+pub struct EvidenceTurn {
+ /// Dataset id of the turn's session.
+ pub session_id: String,
+ /// Distinctive tokens (or all content tokens when `fallback`).
+ pub distinctive: HashSet<String>,
+ /// No token was rare enough; `distinctive` holds every content token.
+ pub fallback: bool,
+ /// Gold-answer content tokens that occur in this turn.
+ pub answer: HashSet<String>,
+}
+
+/// Every `has_answer` turn of `q` with its distinctive and answer tokens.
+pub fn evidence_turns(q: &Question) -> Vec<EvidenceTurn> {
+ // Document frequency over distinct sessions (repeated ids count once).
+ let mut seen = HashSet::new();
+ let mut df: HashMap<String, usize> = HashMap::new();
+ for (sid, turns) in q.haystack_session_ids.iter().zip(&q.haystack_sessions) {
+ if !seen.insert(sid.as_str()) {
+ continue;
+ }
+ let mut session_tokens = HashSet::new();
+ for t in turns {
+ session_tokens.extend(content_tokens(&t.content));
+ }
+ for tok in session_tokens {
+ *df.entry(tok).or_default() += 1;
+ }
+ }
+ let answer_tokens = content_tokens(&q.gold_answer_text());
+
+ let mut out = Vec::new();
+ let mut emitted = HashSet::new();
+ for (sid, turns) in q.haystack_session_ids.iter().zip(&q.haystack_sessions) {
+ // A repeated session's turns are the same evidence, counted once.
+ if !emitted.insert(sid.as_str()) {
+ continue;
+ }
+ for t in turns.iter().filter(|t| t.is_evidence()) {
+ let tokens = content_tokens(&t.content);
+ let distinctive: HashSet<String> = tokens
+ .iter()
+ .filter(|tok| df.get(*tok).copied().unwrap_or(0) <= DF_MAX)
+ .cloned()
+ .collect();
+ let fallback = distinctive.is_empty();
+ let answer = answer_tokens.intersection(&tokens).cloned().collect();
+ out.push(EvidenceTurn {
+ session_id: sid.clone(),
+ distinctive: if fallback { tokens } else { distinctive },
+ fallback,
+ answer,
+ });
+ }
+ }
+ out
+}
+
+/// Fraction of `target` present in `surface`; `None` for an empty target.
+pub fn coverage(target: &HashSet<String>, surface: &HashSet<String>) -> Option<f64> {
+ if target.is_empty() {
+ return None;
+ }
+ let found = target.iter().filter(|t| surface.contains(*t)).count();
+ Some(found as f64 / target.len() as f64)
+}
+
+/// Whether every token of `target` is in `surface`; `None` when empty.
+pub fn contains_all(target: &HashSet<String>, surface: &HashSet<String>) -> Option<bool> {
+ (!target.is_empty()).then(|| target.iter().all(|t| surface.contains(t)))
+}
+
+/// The prefix of `body` the local embedder reads.
+pub struct EmbedWindow {
+ tokenizer: tokenizers::Tokenizer,
+}
+
+impl EmbedWindow {
+ /// Load the local model's tokenizer from `models_root`, configured the
+ /// way `LocalEmbedder` configures it (no padding, 512-token cap).
+ pub fn load(models_root: &Path) -> Result<Self> {
+ let path = models_root.join("all-MiniLM-L6-v2/tokenizer.json");
+ let mut tokenizer = tokenizers::Tokenizer::from_file(&path)
+ .map_err(|e| anyhow::anyhow!("loading tokenizer {}: {e}", path.display()))?;
+ tokenizer.with_padding(None);
+ tokenizer
+ .with_truncation(Some(tokenizers::TruncationParams {
+ max_length: EMBED_MAX_TOKENS,
+ ..Default::default()
+ }))
+ .map_err(|e| anyhow::anyhow!("configuring truncation: {e}"))?;
+ Ok(Self { tokenizer })
+ }
+
+ /// The slice of `body` that survives both embedder truncations.
+ pub fn window<'a>(&self, body: &'a str) -> Result<&'a str> {
+ let pre = pre_truncate(body);
+ let enc = self
+ .tokenizer
+ .encode(pre, true)
+ .map_err(|e| anyhow::anyhow!("tokenizing: {e}"))?;
+ let last = enc
+ .get_offsets()
+ .iter()
+ .zip(enc.get_special_tokens_mask())
+ .filter(|(_, special)| **special == 0)
+ .map(|((_, end), _)| *end)
+ .max()
+ .unwrap_or(0);
+ Ok(&pre[..last.min(pre.len())])
+ }
+}
+
+/// The byte cut `truncate_for_embedding` applies before tokenizing: bodies
+/// over the budget keep their first `budget - len("…")` bytes (the ellipsis
+/// it appends lies past the 512-token window, so it is not modelled).
+fn pre_truncate(body: &str) -> &str {
+ if body.len() <= EMBED_PRE_TRUNCATE_BYTES {
+ return body;
+ }
+ let mut end = EMBED_PRE_TRUNCATE_BYTES - "…".len();
+ while !body.is_char_boundary(end) {
+ end -= 1;
+ }
+ &body[..end]
+}
+
+/// Reachability of one evidence turn through each surface.
+#[derive(Debug, Clone, Serialize)]
+pub struct TurnReach {
+ pub page: f64,
+ /// `None` when the ingest mode stores no observations.
+ pub observations: Option<f64>,
+ /// `None` without an embed window (no tokenizer loaded).
+ pub embed512: Option<f64>,
+ pub answer_page: Option<bool>,
+ pub answer_observations: Option<bool>,
+ pub answer_embed512: Option<bool>,
+ pub fallback: bool,
+}
+
+/// Reachability for one question.
+#[derive(Debug, Clone, Serialize)]
+pub struct QuestionReach {
+ pub question_id: String,
+ pub question_type: String,
+ /// Distinct sessions holding at least one evidence turn.
+ pub evidence_sessions: usize,
+ /// Of those, how many had a session page when read back.
+ pub pages_present: usize,
+ /// Page ingest modes: pages whose stored body is shorter than the
+ /// document sent (any server-side truncation).
+ pub pages_truncated: usize,
+ pub turns: Vec<TurnReach>,
+}
+
+/// Text of one session's surfaces.
+#[derive(Default)]
+struct Surfaces {
+ page: HashSet<String>,
+ page_present: bool,
+ page_truncated: bool,
+ observations: Option<HashSet<String>>,
+ embed512: Option<HashSet<String>>,
+}
+
+/// Read back every evidence session of `q` and score its evidence turns.
+pub async fn measure_question(
+ client: &reqwest::Client,
+ base_url: &str,
+ q: &Question,
+ mode: IngestMode,
+ prefix: DatePrefix,
+ embed: Option<&EmbedWindow>,
+) -> Result<QuestionReach> {
+ let turns = evidence_turns(q);
+ let project = project_for(q);
+ let mut surfaces: BTreeMap<String, Surfaces> = BTreeMap::new();
+ for t in &turns {
+ if surfaces.contains_key(&t.session_id) {
+ continue;
+ }
+ let idx = q
+ .haystack_session_ids
+ .iter()
+ .position(|s| *s == t.session_id)
+ .context("evidence session not in haystack")?;
+ let uuid = stored_session_uuid(&t.session_id);
+ let mut s = Surfaces::default();
+ if let Some((title, body)) = read_page(client, base_url, &project, uuid).await? {
+ s.page_present = true;
+ s.page = content_tokens(&format!("{title}\n{body}"));
+ if let Some(embed) = embed {
+ s.embed512 = Some(content_tokens(embed.window(&body)?));
+ }
+ if mode != IngestMode::Hooks {
+ let sent = session_document(
+ mode,
+ prefix,
+ &q.haystack_dates[idx],
+ &q.haystack_sessions[idx],
+ );
+ s.page_truncated = body.len() < sent.len();
+ }
+ } else if embed.is_some() {
+ s.embed512 = Some(HashSet::new());
+ }
+ if mode == IngestMode::Hooks {
+ let text = read_observations(client, base_url, &project, uuid).await?;
+ s.observations = Some(content_tokens(&text));
+ }
+ surfaces.insert(t.session_id.clone(), s);
+ }
+
+ let reaches = turns
+ .iter()
+ .map(|t| {
+ let s = &surfaces[&t.session_id];
+ TurnReach {
+ page: coverage(&t.distinctive, &s.page).unwrap_or(0.0),
+ observations: s
+ .observations
+ .as_ref()
+ .map(|o| coverage(&t.distinctive, o).unwrap_or(0.0)),
+ embed512: s
+ .embed512
+ .as_ref()
+ .map(|e| coverage(&t.distinctive, e).unwrap_or(0.0)),
+ answer_page: contains_all(&t.answer, &s.page),
+ answer_observations: s
+ .observations
+ .as_ref()
+ .and_then(|o| contains_all(&t.answer, o)),
+ answer_embed512: s.embed512.as_ref().and_then(|e| contains_all(&t.answer, e)),
+ fallback: t.fallback,
+ }
+ })
+ .collect();
+ Ok(QuestionReach {
+ question_id: q.question_id.clone(),
+ question_type: q.question_type.clone(),
+ evidence_sessions: surfaces.len(),
+ pages_present: surfaces.values().filter(|s| s.page_present).count(),
+ pages_truncated: surfaces.values().filter(|s| s.page_truncated).count(),
+ turns: reaches,
+ })
+}
+
+/// `(title, body)` of `sessions/<uuid>.md`, or `None` when it does not exist.
+async fn read_page(
+ client: &reqwest::Client,
+ base_url: &str,
+ project: &str,
+ uuid: uuid::Uuid,
+) -> Result<Option<(String, String)>> {
+ let args = json!({
+ "workspace": EVAL_WORKSPACE,
+ "project": project,
+ "path": format!("sessions/{uuid}.md"),
+ });
+ let text = match mcp_tool_text(client, base_url, "memory_read_page", args).await {
+ Ok(text) => text,
+ Err(e) if format!("{e:#}").to_lowercase().contains("not found") => return Ok(None),
+ Err(e) => return Err(e),
+ };
+ let v: serde_json::Value = serde_json::from_str(&text).context("parsing memory_read_page")?;
+ let field = |k: &str| v.get(k).and_then(|x| x.as_str()).unwrap_or("").to_string();
+ Ok(Some((field("title"), field("body"))))
+}
+
+/// Every observation title and body of one session, concatenated.
+async fn read_observations(
+ client: &reqwest::Client,
+ base_url: &str,
+ project: &str,
+ uuid: uuid::Uuid,
+) -> Result<String> {
+ const PAGE: usize = 200;
+ let mut out = String::new();
+ let mut offset = 0usize;
+ loop {
+ let args = json!({
+ "workspace": EVAL_WORKSPACE,
+ "project": project,
+ "session_id": uuid.to_string(),
+ "limit": PAGE,
+ "offset": offset,
+ "body_max_chars": 16_384,
+ });
+ let text =
+ mcp_tool_text(client, base_url, "memory_read_session_observations", args).await?;
+ let v: serde_json::Value =
+ serde_json::from_str(&text).context("parsing memory_read_session_observations")?;
+ let Some(obs) = v.get("observations").and_then(|o| o.as_array()) else {
+ bail!("memory_read_session_observations returned no observations array: {text}");
+ };
+ for o in obs {
+ for k in ["title", "body"] {
+ if let Some(s) = o.get(k).and_then(|x| x.as_str()) {
+ out.push_str(s);
+ out.push('\n');
+ }
+ }
+ }
+ if obs.len() < PAGE {
+ return Ok(out);
+ }
+ offset += PAGE;
+ }
+}
+
+/// Aggregated reachability for one slice.
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct ReachSlice {
+ pub questions: usize,
+ pub evidence_turns: usize,
+ pub fallback_turns: usize,
+ pub evidence_sessions: usize,
+ pub pages_present: usize,
+ pub pages_truncated: usize,
+ /// Share of evidence turns reachable (coverage ≥ threshold).
+ pub turn_reach_page: f64,
+ pub turn_reach_observations: Option<f64>,
+ pub turn_reach_embed512: Option<f64>,
+ /// Mean distinctive-token coverage.
+ pub mean_cov_page: f64,
+ pub mean_cov_observations: Option<f64>,
+ pub mean_cov_embed512: Option<f64>,
+ /// Answer-anchored: turns whose literal answer tokens all survive.
+ pub answer_turns: usize,
+ pub answer_page: Option<f64>,
+ pub answer_observations: Option<f64>,
+ pub answer_embed512: Option<f64>,
+ /// Share of questions with at least one reachable evidence turn.
+ pub question_ceiling_page: f64,
+ pub question_ceiling_observations: Option<f64>,
+ pub question_ceiling_embed512: Option<f64>,
+}
+
+fn mean(values: &[f64]) -> Option<f64> {
+ (!values.is_empty()).then(|| values.iter().sum::<f64>() / values.len() as f64)
+}
+
+fn share(flags: impl Iterator<Item = bool>) -> Option<f64> {
+ let (mut yes, mut n) = (0usize, 0usize);
+ for f in flags {
+ n += 1;
+ yes += usize::from(f);
+ }
+ (n > 0).then(|| yes as f64 / n as f64)
+}
+
+/// Aggregate one group of questions.
+fn slice(group: &[&QuestionReach]) -> ReachSlice {
+ let turns: Vec<&TurnReach> = group.iter().flat_map(|q| &q.turns).collect();
+ let page_cov: Vec<f64> = turns.iter().map(|t| t.page).collect();
+ let obs_cov: Vec<f64> = turns.iter().filter_map(|t| t.observations).collect();
+ let emb_cov: Vec<f64> = turns.iter().filter_map(|t| t.embed512).collect();
+ let reach = |c: &f64| *c >= REACH_THRESHOLD;
+ let q_any = |pick: fn(&TurnReach) -> Option<f64>| {
+ let has_surface = group
+ .iter()
+ .any(|q| q.turns.iter().any(|t| pick(t).is_some()));
+ has_surface
+ .then(|| {
+ share(group.iter().map(|q| {
+ q.turns
+ .iter()
+ .any(|t| pick(t).is_some_and(|c| c >= REACH_THRESHOLD))
+ }))
+ })
+ .flatten()
+ };
+ ReachSlice {
+ questions: group.len(),
+ evidence_turns: turns.len(),
+ fallback_turns: turns.iter().filter(|t| t.fallback).count(),
+ evidence_sessions: group.iter().map(|q| q.evidence_sessions).sum(),
+ pages_present: group.iter().map(|q| q.pages_present).sum(),
+ pages_truncated: group.iter().map(|q| q.pages_truncated).sum(),
+ turn_reach_page: share(page_cov.iter().map(reach)).unwrap_or(0.0),
+ turn_reach_observations: share(obs_cov.iter().map(reach)),
+ turn_reach_embed512: share(emb_cov.iter().map(reach)),
+ mean_cov_page: mean(&page_cov).unwrap_or(0.0),
+ mean_cov_observations: mean(&obs_cov),
+ mean_cov_embed512: mean(&emb_cov),
+ answer_turns: turns.iter().filter(|t| t.answer_page.is_some()).count(),
+ answer_page: share(turns.iter().filter_map(|t| t.answer_page)),
+ answer_observations: share(turns.iter().filter_map(|t| t.answer_observations)),
+ answer_embed512: share(turns.iter().filter_map(|t| t.answer_embed512)),
+ question_ceiling_page: q_any(|t| Some(t.page)).unwrap_or(0.0),
+ question_ceiling_observations: q_any(|t| t.observations),
+ question_ceiling_embed512: q_any(|t| t.embed512),
+ }
+}
+
+/// Per question type plus `overall`, as the retrieval slices are keyed.
+pub fn aggregate(per_question: &[QuestionReach]) -> BTreeMap<String, ReachSlice> {
+ let mut groups: BTreeMap<String, Vec<&QuestionReach>> = BTreeMap::new();
+ for q in per_question {
+ groups.entry(q.question_type.clone()).or_default().push(q);
+ groups.entry("overall".into()).or_default().push(q);
+ }
+ groups.into_iter().map(|(k, g)| (k, slice(&g))).collect()
+}
+
+/// The reachability section appended to a run's markdown report.
+pub fn to_markdown(slices: &BTreeMap<String, ReachSlice>) -> String {
+ let opt = |v: Option<f64>| v.map_or("n/a".to_string(), |v| format!("{v:.3}"));
+ let mut md = format!(
+ "\n### Evidence reachability\n\n\
+ Distinctive tokens: content tokens in ≤{DF_MAX} haystack sessions; a turn is \
+ reachable at ≥{REACH_THRESHOLD} coverage. Ceiling = share of questions with ≥1 \
+ reachable evidence turn.\n\n\
+ | slice | q | ev. turns | pages present | truncated | turn reach page | obs | embed512 | \
+ answer turns | answer page | obs | embed512 | ceiling page | obs | embed512 |\n\
+ |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n"
+ );
+ for (name, s) in slices {
+ md.push_str(&format!(
+ "| {name} | {} | {} | {}/{} | {} | {:.3} | {} | {} | {} | {} | {} | {} | {:.3} | {} | {} |\n",
+ s.questions,
+ s.evidence_turns,
+ s.pages_present,
+ s.evidence_sessions,
+ s.pages_truncated,
+ s.turn_reach_page,
+ opt(s.turn_reach_observations),
+ opt(s.turn_reach_embed512),
+ s.answer_turns,
+ opt(s.answer_page),
+ opt(s.answer_observations),
+ opt(s.answer_embed512),
+ s.question_ceiling_page,
+ opt(s.question_ceiling_observations),
+ opt(s.question_ceiling_embed512),
+ ));
+ }
+ md
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn set(words: &[&str]) -> HashSet<String> {
+ words.iter().map(|w| (*w).to_string()).collect()
+ }
+
+ #[test]
+ fn content_tokens_drop_stopwords_and_short_words_but_keep_numbers() {
+ let t = content_tokens("I bought THE boots for $45 in 2 days, it's ok.");
+ assert_eq!(t, set(&["bought", "boots", "45", "2", "days"]));
+ }
+
+ fn question(sessions: serde_json::Value, ids: &[&str], answer: &str) -> Question {
+ let n = ids.len();
+ serde_json::from_value(json!({
+ "question_id": "q",
+ "question_type": "single-session-user",
+ "question": "?",
+ "answer": answer,
+ "question_date": "d",
+ "haystack_dates": vec!["d"; n],
+ "haystack_session_ids": ids,
+ "haystack_sessions": sessions,
+ "answer_session_ids": [ids[0]]
+ }))
+ .unwrap()
+ }
+
+ #[test]
+ fn distinctive_tokens_are_rare_across_the_haystack() {
+ // "weather" occurs in all five sessions, "quokka" only in the first.
+ let mut sessions = vec![json!([
+ {"role": "user", "content": "weather talk about my quokka", "has_answer": true}
+ ])];
+ for _ in 0..4 {
+ sessions.push(json!([{"role": "user", "content": "weather again"}]));
+ }
+ let q = question(json!(sessions), &["a", "b", "c", "d", "e"], "quokka");
+ let turns = evidence_turns(&q);
+ assert_eq!(turns.len(), 1);
+ assert_eq!(turns[0].distinctive, set(&["talk", "quokka"]));
+ assert!(!turns[0].fallback);
+ assert_eq!(turns[0].answer, set(&["quokka"]));
+ }
+
+ #[test]
+ fn a_turn_without_rare_tokens_falls_back_to_all_content_tokens() {
+ let sessions: Vec<_> = (0..5)
+ .map(|i| json!([{"role": "user", "content": "common words", "has_answer": i == 0}]))
+ .collect();
+ let q = question(json!(sessions), &["a", "b", "c", "d", "e"], "x");
+ let turns = evidence_turns(&q);
+ assert_eq!(turns.len(), 1);
+ assert!(turns[0].fallback);
+ assert_eq!(turns[0].distinctive, set(&["common", "words"]));
+ assert!(turns[0].answer.is_empty());
+ }
+
+ #[test]
+ fn repeated_sessions_count_once_for_frequency_and_evidence() {
+ let s = json!([{"role": "user", "content": "rare zebra", "has_answer": true}]);
+ let q = question(
+ json!([s.clone(), s.clone(), s.clone(), s]),
+ &["a", "a", "a", "a"],
+ "z",
+ );
+ let turns = evidence_turns(&q);
+ // One distinct session: df = 1, one evidence turn.
+ assert_eq!(turns.len(), 1);
+ assert_eq!(turns[0].distinctive, set(&["rare", "zebra"]));
+ }
+
+ #[test]
+ fn pre_truncation_matches_the_embedder_byte_budget() {
+ let short = "a".repeat(EMBED_PRE_TRUNCATE_BYTES);
+ assert_eq!(pre_truncate(&short).len(), EMBED_PRE_TRUNCATE_BYTES);
+ let long = "é".repeat(EMBED_PRE_TRUNCATE_BYTES);
+ let cut = pre_truncate(&long);
+ assert!(cut.len() <= EMBED_PRE_TRUNCATE_BYTES - "…".len());
+ assert!(cut.len() >= EMBED_PRE_TRUNCATE_BYTES - "…".len() - 1);
+ }
+
+ /// Runs only where the model has been fetched (`evals/models/`).
+ #[test]
+ fn the_embed_window_stops_at_512_tokens() {
+ let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("models");
+ let Ok(window) = EmbedWindow::load(&root) else {
+ return;
+ };
+ let words: Vec<String> = (0..2000).map(|i| format!("w{i}")).collect();
+ let body = words.join(" ");
+ let kept = window.window(&body).unwrap();
+ assert!(kept.len() < body.len());
+ assert!(body.starts_with(kept));
+ let short = "a short body";
+ assert_eq!(window.window(short).unwrap(), short);
+ }
+
+ #[test]
+ fn coverage_and_answer_checks() {
+ let target = set(&["quokka", "leather", "boots", "january"]);
+ let surface = set(&["quokka", "boots", "other"]);
+ assert_eq!(coverage(&target, &surface), Some(0.5));
+ assert_eq!(coverage(&HashSet::new(), &surface), None);
+ assert_eq!(
+ contains_all(&set(&["quokka", "boots"]), &surface),
+ Some(true)
+ );
+ assert_eq!(
+ contains_all(&set(&["quokka", "january"]), &surface),
+ Some(false)
+ );
+ assert_eq!(contains_all(&HashSet::new(), &surface), None);
+ }
+
+ #[test]
+ fn a_title_cut_word_does_not_count_as_reached() {
+ // The page title keeps "kitche…", which must not match "kitchen".
+ let page = content_tokens("[session date: 2023/05/30] I got new kitche…");
+ assert!(!page.contains("kitchen"));
+ assert!(page.contains("kitche"));
+ }
+
+ fn reach(page: f64, obs: Option<f64>) -> TurnReach {
+ TurnReach {
+ page,
+ observations: obs,
+ embed512: None,
+ answer_page: Some(page >= 1.0),
+ answer_observations: None,
+ answer_embed512: None,
+ fallback: false,
+ }
+ }
+
+ #[test]
+ fn aggregation_reports_turn_shares_and_the_question_ceiling() {
+ let q1 = QuestionReach {
+ question_id: "q1".into(),
+ question_type: "a".into(),
+ evidence_sessions: 1,
+ pages_present: 1,
+ pages_truncated: 0,
+ turns: vec![reach(0.2, Some(1.0)), reach(1.0, Some(1.0))],
+ };
+ let q2 = QuestionReach {
+ question_id: "q2".into(),
+ question_type: "b".into(),
+ evidence_sessions: 2,
+ pages_present: 1,
+ pages_truncated: 0,
+ turns: vec![reach(0.0, Some(0.6))],
+ };
+ let agg = aggregate(&[q1, q2]);
+ let all = &agg["overall"];
+ assert_eq!(all.evidence_turns, 3);
+ assert!((all.turn_reach_page - 1.0 / 3.0).abs() < 1e-9);
+ assert_eq!(all.turn_reach_observations, Some(1.0));
+ assert_eq!(all.turn_reach_embed512, None);
+ // q1 has a page-reachable turn, q2 does not.
+ assert_eq!(all.question_ceiling_page, 0.5);
+ assert_eq!(all.question_ceiling_observations, Some(1.0));
+ assert_eq!(all.question_ceiling_embed512, None);
+ assert_eq!(all.pages_present, 2);
+ assert_eq!(all.evidence_sessions, 3);
+ assert_eq!(agg["a"].questions, 1);
+ }
+}
diff --git a/evals/src/retrieval/report.rs b/evals/src/retrieval/report.rs
index 00632e59..2fff878b 100644
--- a/evals/src/retrieval/report.rs
+++ b/evals/src/retrieval/report.rs
@@ -13,7 +13,9 @@ use std::collections::BTreeMap;
use serde::Serialize;
-use super::score::{QuestionScore, SliceMetrics};
+use super::ingest::{DatePrefix, IngestMode, IngestStats};
+use super::reachability::{QuestionReach, ReachSlice};
+use super::score::{QuestionScore, SliceMetrics, percentile_u128};
#[derive(Debug, Serialize)]
pub struct Report {
@@ -33,10 +35,116 @@ pub struct Report {
/// graded. Names only — never keys. `None` on the default zero-LLM path.
#[serde(skip_serializing_if = "Option::is_none")]
pub qa: Option<QaReportMeta>,
+ /// Ingest volume and timing (server engine only).
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ingest: Option<IngestSummary>,
+ /// Evidence reachability (`--reachability` only).
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub reachability: Option<ReachabilityReport>,
pub slices: BTreeMap<String, SliceMetrics>,
pub per_question: Vec<QuestionScore>,
}
+/// `--reachability` results: per-slice aggregates plus every question.
+#[derive(Debug, Serialize)]
+pub struct ReachabilityReport {
+ pub slices: BTreeMap<String, ReachSlice>,
+ pub per_question: Vec<QuestionReach>,
+}
+
+/// Ingest throughput for one config run.
+#[derive(Debug, Clone, Serialize)]
+pub struct IngestSummary {
+ pub mode: &'static str,
+ pub date_prefix: &'static str,
+ /// Questions ingested concurrently.
+ pub concurrency: usize,
+ /// Hook events posted, or pages written.
+ pub items: usize,
+ /// HTTP requests (batch POSTs or page writes).
+ pub requests: usize,
+ /// Items per second within one question's sequential ingest stream.
+ pub items_per_sec_per_stream: f64,
+ pub question_ingest_p50_ms: u128,
+ pub question_ingest_p95_ms: u128,
+ pub request_p50_ms: u128,
+ pub request_p95_ms: u128,
+ /// Whole config run (ingest + query + read-back), wall clock.
+ pub run_wall_ms: u128,
+ /// Page modes: documents larger than the 16 KiB observation backstop.
+ pub docs_over_16kib: usize,
+}
+
+impl IngestSummary {
+ pub fn new(
+ mode: IngestMode,
+ prefix: DatePrefix,
+ concurrency: usize,
+ stats: &[IngestStats],
+ wall: std::time::Duration,
+ ) -> Self {
+ let items: usize = stats.iter().map(|s| s.items).sum();
+ let stream_secs: f64 = stats.iter().map(|s| s.elapsed.as_secs_f64()).sum();
+ let per_question: Vec<u128> = stats.iter().map(|s| s.elapsed.as_millis()).collect();
+ let requests: Vec<u128> = stats
+ .iter()
+ .flat_map(|s| s.request_ms.iter().copied())
+ .collect();
+ Self {
+ mode: mode.as_str(),
+ date_prefix: prefix.as_str(),
+ concurrency,
+ items,
+ requests: requests.len(),
+ items_per_sec_per_stream: if stream_secs > 0.0 {
+ items as f64 / stream_secs
+ } else {
+ 0.0
+ },
+ question_ingest_p50_ms: percentile_u128(&per_question, 0.50),
+ question_ingest_p95_ms: percentile_u128(&per_question, 0.95),
+ request_p50_ms: percentile_u128(&requests, 0.50),
+ request_p95_ms: percentile_u128(&requests, 0.95),
+ run_wall_ms: wall.as_millis(),
+ docs_over_16kib: stats.iter().map(|s| s.docs_over_16kib).sum(),
+ }
+ }
+}
+
+/// Ingest and reachability sections for one config, empty when absent.
+fn extras_md(r: &Report) -> String {
+ let mut md = String::new();
+ if let Some(i) = &r.ingest {
+ md.push_str(&format!(
+ "\n### Ingest\n\n- mode: {} (date prefix: {}), concurrency {}\n\
+ - {} items in {} requests; {:.0} items/s per question stream\n\
+ - per question p50/p95: {}/{} ms; per request p50/p95: {}/{} ms\n\
+ - run wall clock: {:.1} s; documents over 16 KiB: {}\n",
+ i.mode,
+ i.date_prefix,
+ i.concurrency,
+ i.items,
+ i.requests,
+ i.items_per_sec_per_stream,
+ i.question_ingest_p50_ms,
+ i.question_ingest_p95_ms,
+ i.request_p50_ms,
+ i.request_p95_ms,
+ i.run_wall_ms as f64 / 1000.0,
+ i.docs_over_16kib,
+ ));
+ let raw = r.per_question.iter().filter(|q| q.raw_hits > 0).count();
+ md.push_str(&format!(
+ "- questions answered from the raw-observation fallback: {raw}/{}\n",
+ r.per_question.len()
+ ));
+ }
+ if let Some(reach) = &r.reachability {
+ md.push_str(&super::reachability::to_markdown(&reach.slices));
+ }
+ md
+}
+
/// QA-accuracy provenance for a run (R2b). Provider + model names only, so a
/// published report never carries a key.
#[derive(Debug, Clone, Serialize)]
@@ -179,6 +287,7 @@ pub fn to_markdown(r: &Report) -> String {
md.push_str(&format!("- mode: {}\n- config: {}\n\n", r.mode, r.config));
md.push_str(&slices_table_md(r));
md.push_str(&qa_table_md(r));
+ md.push_str(&extras_md(r));
md.push_str(METRIC_NOTES);
md
}
@@ -202,6 +311,10 @@ pub struct AbDelta {
pub latency_p95_ms: i128,
pub context_tokens_mean: f64,
pub context_tokens_median: f64,
+ /// Per-question hit@k changes keyed by k: `(gained, lost)`, questions
+ /// the candidate hits and the baseline misses, and the reverse. A net
+ /// delta built from many flips both ways is noisier than its size.
+ pub hit_flips: BTreeMap<usize, (usize, usize)>,
/// candidate − baseline QA accuracy over the `overall` slice (R2b).
/// `None` unless both runs graded the overall slice.
#[serde(skip_serializing_if = "Option::is_none")]
@@ -235,7 +348,28 @@ fn compute_delta(baseline: &Report, candidate: &Report) -> AbDelta {
hit_at.insert(*k, c.hit_at[k] - b.hit_at[k]);
recall_at.insert(*k, c.recall_at[k] - b.recall_at[k]);
}
+ let base_by_id: BTreeMap<&str, &QuestionScore> = baseline
+ .per_question
+ .iter()
+ .map(|q| (q.question_id.as_str(), q))
+ .collect();
+ let mut hit_flips = BTreeMap::new();
+ for k in &candidate.ks {
+ let (mut gained, mut lost) = (0, 0);
+ for c in &candidate.per_question {
+ let Some(b) = base_by_id.get(c.question_id.as_str()) else {
+ continue;
+ };
+ match (b.hit_at.get(k) > Some(&0.0), c.hit_at.get(k) > Some(&0.0)) {
+ (false, true) => gained += 1,
+ (true, false) => lost += 1,
+ _ => {}
+ }
+ }
+ hit_flips.insert(*k, (gained, lost));
+ }
AbDelta {
+ hit_flips,
hit_at,
recall_at,
latency_p50_ms: c.latency_p50_ms as i128 - b.latency_p50_ms as i128,
@@ -268,9 +402,11 @@ pub fn ab_to_markdown(r: &AbReport) -> String {
md.push_str("## Baseline\n\n");
md.push_str(&slices_table_md(&r.baseline));
md.push_str(&qa_table_md(&r.baseline));
+ md.push_str(&extras_md(&r.baseline));
md.push_str("\n## Candidate\n\n");
md.push_str(&slices_table_md(&r.candidate));
md.push_str(&qa_table_md(&r.candidate));
+ md.push_str(&extras_md(&r.candidate));
// Delta table over the `overall` slice.
let b = overall(&r.baseline);
@@ -278,8 +414,9 @@ pub fn ab_to_markdown(r: &AbReport) -> String {
md.push_str("\n## Delta (overall, candidate − baseline)\n\n");
md.push_str("| metric | baseline | candidate | delta |\n|---|---|---|---|\n");
for k in &r.candidate.ks {
+ let (gained, lost) = r.delta.hit_flips.get(k).copied().unwrap_or((0, 0));
md.push_str(&format!(
- "| hit@{k} | {:.3} | {:.3} | {:+.3} |\n",
+ "| hit@{k} | {:.3} | {:.3} | {:+.3} (+{gained}/-{lost}) |\n",
b.hit_at[k], c.hit_at[k], r.delta.hit_at[k]
));
}
@@ -347,6 +484,8 @@ mod tests {
abstention_excluded: 1,
ks: vec![5],
qa: None,
+ ingest: None,
+ reachability: None,
slices,
per_question: vec![],
}
@@ -378,7 +517,10 @@ mod tests {
assert_eq!(ab.delta.context_tokens_mean, 0.0);
let md = ab_to_markdown(&ab);
assert!(md.contains("A/B"));
- assert!(md.contains("| hit@5 | 0.500 | 0.500 | +0.000 |"), "{md}");
+ assert!(
+ md.contains("| hit@5 | 0.500 | 0.500 | +0.000 (+0/-0) |"),
+ "{md}"
+ );
}
#[test]
@@ -421,7 +563,77 @@ mod tests {
let ab = AbReport::new(baseline, candidate);
assert!((ab.delta.hit_at[&5] - 0.2).abs() < 1e-9);
let md = ab_to_markdown(&ab);
- assert!(md.contains("| hit@5 | 0.400 | 0.600 | +0.200 |"), "{md}");
+ assert!(md.contains("| hit@5 | 0.400 | 0.600 | +0.200"), "{md}");
assert!(md.contains("candidate: embeddings=local"));
}
+
+ fn question(id: &str, hit5: f64) -> QuestionScore {
+ QuestionScore {
+ question_id: id.into(),
+ question_type: "t".into(),
+ hit_at: BTreeMap::from([(5, hit5)]),
+ recall_at: BTreeMap::from([(5, hit5)]),
+ retrieved_sessions: 1,
+ latency_ms: 0,
+ context_tokens: 0,
+ ingest_ms: 0,
+ raw_hits: 0,
+ qa: None,
+ }
+ }
+
+ #[test]
+ fn ab_delta_counts_paired_flips_both_ways() {
+ let mut baseline = report("b", 0.5, 0.5);
+ baseline.per_question = vec![
+ question("q1", 1.0),
+ question("q2", 0.0),
+ question("q3", 1.0),
+ ];
+ let mut candidate = report("c", 0.5, 0.5);
+ candidate.per_question = vec![
+ question("q1", 0.0),
+ question("q2", 1.0),
+ question("q3", 1.0),
+ ];
+ let ab = AbReport::new(baseline, candidate);
+ assert_eq!(ab.delta.hit_flips[&5], (1, 1));
+ assert!(ab_to_markdown(&ab).contains("(+1/-1)"));
+ }
+
+ #[test]
+ fn ingest_summary_aggregates_streams_and_requests() {
+ let stats = vec![
+ IngestStats {
+ items: 10,
+ elapsed: std::time::Duration::from_millis(500),
+ request_ms: vec![5, 7],
+ docs_over_16kib: 1,
+ },
+ IngestStats {
+ items: 30,
+ elapsed: std::time::Duration::from_millis(1500),
+ request_ms: vec![9],
+ docs_over_16kib: 0,
+ },
+ ];
+ let s = IngestSummary::new(
+ IngestMode::WholeSession,
+ DatePrefix::Turn,
+ 8,
+ &stats,
+ std::time::Duration::from_secs(3),
+ );
+ assert_eq!(s.items, 40);
+ assert_eq!(s.requests, 3);
+ assert!((s.items_per_sec_per_stream - 20.0).abs() < 1e-9);
+ assert_eq!(s.request_p95_ms, 9);
+ assert_eq!(s.docs_over_16kib, 1);
+ assert_eq!(s.mode, "whole-session");
+ let mut r = report("c", 0.5, 0.5);
+ r.ingest = Some(s);
+ let md = to_markdown(&r);
+ assert!(md.contains("### Ingest"));
+ assert!(md.contains("\n- 40 items in 3 requests"), "{md}");
+ }
}
diff --git a/evals/src/retrieval/score.rs b/evals/src/retrieval/score.rs
index b9c3a3cf..505a38a6 100644
--- a/evals/src/retrieval/score.rs
+++ b/evals/src/retrieval/score.rs
@@ -34,6 +34,13 @@ pub struct QuestionScore {
/// Estimated context tokens the result would cost the agent (chars/4
/// over returned hit titles + snippets).
pub context_tokens: usize,
+ /// Wall clock of this question's ingest, milliseconds (0 for `grep`).
+ #[serde(default)]
+ pub ingest_ms: u128,
+ /// Raw observation hits the server returned (it falls back to them only
+ /// when page search finds nothing).
+ #[serde(default)]
+ pub raw_hits: usize,
/// End-to-end QA outcome (R2b, opt-in): a candidate answer graded
/// against the gold answer. `None` on the default zero-LLM path and for
/// any question whose QA step failed (logged, left ungraded).
@@ -88,6 +95,8 @@ pub fn score_question(
retrieved_sessions: ranked_sessions.len(),
latency_ms,
context_tokens,
+ ingest_ms: 0,
+ raw_hits: 0,
// R2a scoring is answer-agnostic; the QA path attaches its outcome
// afterward, keeping this function pure and deterministic.
qa: None,
@@ -157,7 +166,7 @@ impl QaSliceMetrics {
/// Nearest-rank percentile (`p` in 0.0..=1.0) over a copy-sorted slice.
/// Empty input yields 0.
-fn percentile_u128(values: &[u128], p: f64) -> u128 {
+pub(crate) fn percentile_u128(values: &[u128], p: f64) -> u128 {
if values.is_empty() {
return 0;
}