Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(evals): add retrieval decomposition instruments
AkitaOnRails7a2c603d7dee
diff --git a/Cargo.lock b/Cargo.lockindex bfe669d4..dd7656af 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -143,6 +143,7 @@ dependencies = ["serde_json","sha2","tempfile",+ "tokenizers","tokio","tracing","tracing-subscriber",diff --git a/evals/Cargo.toml b/evals/Cargo.tomlindex fa284e8a..bf005d10 100644--- a/evals/Cargo.toml+++ b/evals/Cargo.toml@@ -20,6 +20,7 @@ clap.workspace = truereqwest.workspace = truesha2.workspace = truetempfile.workspace = true+tokenizers.workspace = truejiff.workspace = trueschemars.workspace = truesecrecy.workspace = truediff --git a/evals/README.md b/evals/README.mdindex 47319c8f..d1843c85 100644--- a/evals/README.md+++ b/evals/README.md@@ -288,6 +288,49 @@ cargo run -p ai-memory-eval -- retrieval --sample 3 \--qa --qa-provider gemini --qa-model gemini-2.5-flash```+### `retrieval` decomposition instruments (M1)++These flags split a hit@k number into evidence the index never saw and+ranking on evidence it did. All are opt-in; the defaults reproduce the+published baselines.++- `--reachability` reads every evidence session back after the query and+ reports, per slice, how many answer-bearing turns (the dataset's+ `has_answer` flag) survive into the session page, the observation bodies,+ and the first 512 tokens the local embedder reads. A turn counts as+ reachable when at least half of its distinctive tokens (content words that+ occur in at most 3 sessions of the haystack) are on that surface. The+ module docs in `src/retrieval/reachability.rs` give the exact definition.+ It needs the local model's tokenizer, which it fetches into+ `evals/models/` if missing.+- `--engine grep` ranks each question's raw sessions by query-term overlap+ in-process, with no server. It follows agentmemory's grep adapter exactly+ and matches its archived per-question results.+- `--ingest whole-session` writes each raw session as one page through+ `POST /admin/write-page` and then queries as usual: ranking on full+ evidence. `--ingest captured-session` writes one page per session holding+ only the text capture keeps (user turns up to 16 KiB, assistant excerpts up+ to 2 KB).+- `--date-prefix turn|session-start|none` moves the session date off every+ turn (`turn` is the default).+- `--stratify N` scores the first N questions of each type, so a cheap run+ covers every slice (`--sample` takes a prefix, which is all one type).+- `--dataset-variant s-cleaned` runs on the cleaned LongMemEval-S+ re-release (pinned by sha256) that other systems report on.++`--candidate-ingest` and `--candidate-date-prefix` put either knob into an+A/B run. The A/B delta table also counts per-question flips (`+gained/-lost`),+which shows whether a net delta is many changes cancelling out or a few in+one direction.++```bash+cargo run --release -p ai-memory-eval -- retrieval --engine grep+cargo run --release -p ai-memory-eval -- retrieval --reachability \+ --ingest hooks --candidate-ingest whole-session+cargo run --release -p ai-memory-eval -- retrieval --stratify 5 \+ --date-prefix turn --candidate-date-prefix session-start+```+Honest-numbers notes:- capture is production-shaped: excerpts are bounded at the 2 KBdiff --git a/evals/src/retrieval/dataset.rs b/evals/src/retrieval/dataset.rsindex 27166494..6e99b7b8 100644--- a/evals/src/retrieval/dataset.rs+++ b/evals/src/retrieval/dataset.rs@@ -3,6 +3,12 @@//! The file is NOT committed (278 MB); it lives gitignored under//! `evals/datasets/` and is fetched on demand from HuggingFace with a//! pinned sha256. Dataset: `xiaowu0162/longmemeval` (MIT license).+//!+//! Two pinned variants load: the original `longmemeval_s` (every published+//! ai-memory baseline) and the later `longmemeval_s_cleaned` re-release,+//! which keeps the same 500 questions and evidence labels but replaces+//! distractor sessions that also held the answer. Other systems publish on+//! the cleaned file, so a comparison has to say which one it ran.use std::path::Path;@@ -19,11 +25,52 @@ pub const LONGMEMEVAL_S_URL: &str =pub const LONGMEMEVAL_S_SHA256: &str ="08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894";+/// One pinned dataset file the harness accepts.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct DatasetVariant {+ /// Name recorded in reports.+ pub name: &'static str,+ /// Default location under the repo root.+ pub default_path: &'static str,+ /// Download URL (revision-pinned where upstream allows it).+ pub url: &'static str,+ /// Expected sha256 of the file.+ pub sha256: &'static str,+}++/// The original S file every published ai-memory baseline uses.+pub const VARIANT_S: DatasetVariant = DatasetVariant {+ name: "longmemeval_s",+ default_path: "evals/datasets/longmemeval_s.json",+ url: LONGMEMEVAL_S_URL,+ sha256: LONGMEMEVAL_S_SHA256,+};++/// The cleaned S re-release (`xiaowu0162/longmemeval-cleaned`, revision+/// `98d7416c`), the input agentmemory's 2026-10-09 baseline reports on.+pub const VARIANT_S_CLEANED: DatasetVariant = DatasetVariant {+ name: "longmemeval_s_cleaned",+ default_path: "evals/datasets/longmemeval_s_cleaned.json",+ url: "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_s_cleaned.json",+ sha256: "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",+};+/// One chat turn inside a haystack session.#[derive(Debug, Clone, Deserialize)]pub struct Turn {pub role: String,pub content: String,+ /// The dataset's evidence flag: this turn carries (part of) the answer.+ /// Absent on most turns; only the reachability report reads it.+ #[serde(default)]+ pub has_answer: Option<bool>,+}++impl Turn {+ /// True when the dataset marks this turn as answer-bearing.+ pub fn is_evidence(&self) -> bool {+ self.has_answer == Some(true)+ }}/// One benchmark question with its private haystack of chat sessions.@@ -63,8 +110,8 @@ impl Question {}}-/// Load the dataset, verifying the pinned sha256 first.-pub fn load(path: &Path) -> Result<Vec<Question>> {+/// Load the dataset, verifying `variant`'s pinned sha256 first.+pub fn load(path: &Path, variant: &DatasetVariant) -> Result<Vec<Question>> {let bytes = std::fs::read(path).with_context(|| {format!("reading dataset at {} (use --fetch to download)",@@ -72,11 +119,13 @@ pub fn load(path: &Path) -> Result<Vec<Question>> {)})?;let digest = hex::encode(Sha256::digest(&bytes));- if digest != LONGMEMEVAL_S_SHA256 {+ if digest != variant.sha256 {bail!(- "dataset sha256 mismatch at {}: got {digest}, pinned {LONGMEMEVAL_S_SHA256}. \+ "dataset sha256 mismatch at {}: got {digest}, pinned {} for {}. \Refusing to benchmark against an unpinned file.",- path.display()+ path.display(),+ variant.sha256,+ variant.name);}let questions: Vec<Question> =@@ -99,19 +148,19 @@ pub fn load(path: &Path) -> Result<Vec<Question>> {/// Download the dataset to `path` (atomically: tmp + rename) and verify/// the pinned sha256 before the rename lands it.-pub async fn fetch(path: &Path) -> Result<()> {+pub async fn fetch(path: &Path, variant: &DatasetVariant) -> Result<()> {if let Some(parent) = path.parent() {std::fs::create_dir_all(parent)?;}- tracing::info!(- url = LONGMEMEVAL_S_URL,- "downloading LongMemEval-S (278 MB)"- );- let resp = reqwest::get(LONGMEMEVAL_S_URL).await?.error_for_status()?;+ tracing::info!(url = variant.url, "downloading {} (~278 MB)", variant.name);+ let resp = reqwest::get(variant.url).await?.error_for_status()?;let bytes = resp.bytes().await?;let digest = hex::encode(Sha256::digest(&bytes));- if digest != LONGMEMEVAL_S_SHA256 {- bail!("downloaded dataset sha256 {digest} does not match pin {LONGMEMEVAL_S_SHA256}");+ if digest != variant.sha256 {+ bail!(+ "downloaded dataset sha256 {digest} does not match pin {}",+ variant.sha256+ );}let tmp = path.with_extension("part");std::fs::write(&tmp, &bytes)?;@@ -119,6 +168,22 @@ pub async fn fetch(path: &Path) -> Result<()> {Ok(())}+/// First `per_type` questions of each question type, types in sorted order+/// (the same deterministic selection as agentmemory's `stratifySample`).+/// A prefix sample (`--sample`) only covers the first type, because the file+/// is grouped by type; this one covers every slice.+pub fn stratify(questions: Vec<Question>, per_type: usize) -> Vec<Question> {+ let mut buckets: std::collections::BTreeMap<String, Vec<Question>> =+ std::collections::BTreeMap::new();+ for q in questions {+ let bucket = buckets.entry(q.question_type.clone()).or_default();+ if bucket.len() < per_type {+ bucket.push(q);+ }+ }+ buckets.into_values().flatten().collect()+}+mod hex {/// Minimal lowercase hex, avoiding a dependency for one call site.pub fn encode(bytes: impl AsRef<[u8]>) -> String {@@ -175,7 +240,35 @@ mod tests {let dir = tempfile::tempdir().unwrap();let p = dir.path().join("longmemeval_s.json");std::fs::write(&p, serde_json::to_vec(&sample_json()).unwrap()).unwrap();- let err = load(&p).unwrap_err().to_string();+ let err = load(&p, &VARIANT_S).unwrap_err().to_string();assert!(err.contains("sha256 mismatch"), "{err}");+ let err = load(&p, &VARIANT_S_CLEANED).unwrap_err().to_string();+ assert!(err.contains("longmemeval_s_cleaned"), "{err}");+ }++ #[test]+ fn the_evidence_flag_is_read_and_defaults_to_absent() {+ let qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();+ let turns = &qs[0].haystack_sessions[0];+ assert!(turns[0].is_evidence());+ assert_eq!(turns[1].has_answer, None);+ assert!(!turns[1].is_evidence());+ }++ #[test]+ fn stratify_takes_a_prefix_of_every_type_in_sorted_order() {+ let mut qs: Vec<Question> = Vec::new();+ let base: Vec<Question> = serde_json::from_value(sample_json()).unwrap();+ for (i, ty) in ["b-type", "a-type", "b-type", "b-type", "a-type"]+ .iter()+ .enumerate()+ {+ let mut q = base[0].clone();+ q.question_id = format!("q{i}");+ q.question_type = (*ty).into();+ qs.push(q);+ }+ let picked: Vec<String> = stratify(qs, 2).into_iter().map(|q| q.question_id).collect();+ assert_eq!(picked, ["q1", "q4", "q0", "q2"]);}}diff --git a/evals/src/retrieval/grep.rs b/evals/src/retrieval/grep.rsnew file mode 100644index 00000000..21a8269e--- /dev/null+++ b/evals/src/retrieval/grep.rs@@ -0,0 +1,166 @@+//! `--engine grep`: a whole-session term-overlap baseline with no server.+//!+//! It reproduces agentmemory's `eval/runner/adapters/grep.ts` so the two+//! harnesses can be calibrated against each other on the same questions:+//!+//! - a session's document is `[Source session date: <date>]` followed by+//! every turn as `[<role>] <content>`, joined by blank lines (their+//! `sessionText` over `loadLongMemEval`'s flattening);+//! - query terms are the lowercased `[a-z0-9_]+` runs longer than two+//! characters, duplicates kept;+//! - a session scores one point per query term that occurs anywhere in its+//! lowercased document (substring match, so `boot` matches `boots`);+//! - sessions with no matching term are dropped, and ties keep haystack+//! order (a stable sort, as in JavaScript);+//! - a repeated session id keeps its first position (their loader+//! coalesces repeats into the first occurrence).+//!+//! It measures what plain lexical overlap over the full raw transcript can+//! reach, before any capture, truncation, or ranking choice of ours.++use std::collections::HashSet;++use super::dataset::{Question, Turn};+use super::ingest::stored_session_uuid;++/// Query terms, agentmemory-compatible: lowercase, split on anything that+/// is not `[a-z0-9_]`, keep tokens longer than two characters. Duplicates+/// are kept, so a repeated query word counts twice.+pub fn tokenize(s: &str) -> Vec<String> {+ s.to_lowercase()+ .split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'))+ .filter(|t| t.len() > 2)+ .map(str::to_owned)+ .collect()+}++/// One session rendered as a single document (see the module docs).+pub fn session_text(date: &str, turns: &[Turn]) -> String {+ let body = turns+ .iter()+ .map(|t| format!("[{}] {}", t.role, t.content))+ .collect::<Vec<_>>()+ .join("\n\n");+ format!("[Source session date: {date}]\n{body}")+}++/// Rank `(id, lowercased document)` pairs for `query`, best first.+pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str)>) -> Vec<T> {+ let terms = tokenize(query);+ let mut scored: Vec<(T, usize)> = docs+ .into_iter()+ .filter_map(|(id, body)| {+ let hits = terms.iter().filter(|t| body.contains(t.as_str())).count();+ (hits > 0).then_some((id, hits))+ })+ .collect();+ // `sort_by_key` is stable: equal scores keep haystack order.+ scored.sort_by_key(|(_, hits)| std::cmp::Reverse(*hits));+ scored.into_iter().map(|(id, _)| id).collect()+}++/// Rank one question's haystack sessions, returning stored session uuids.+pub fn rank_question(q: &Question) -> Vec<uuid::Uuid> {+ let mut seen = HashSet::new();+ let docs: Vec<(uuid::Uuid, String)> = q+ .haystack_session_ids+ .iter()+ .zip(&q.haystack_dates)+ .zip(&q.haystack_sessions)+ .filter(|((id, _), _)| seen.insert(id.as_str()))+ .map(|((id, date), turns)| {+ (+ stored_session_uuid(id),+ session_text(date, turns).to_lowercase(),+ )+ })+ .collect();+ rank(+ &q.question,+ docs.iter().map(|(id, body)| (*id, body.as_str())),+ )+}++#[cfg(test)]+mod tests {+ use super::*;++ fn turn(role: &str, content: &str) -> Turn {+ Turn {+ role: role.into(),+ content: content.into(),+ has_answer: None,+ }+ }++ #[test]+ fn tokenize_matches_the_reference_adapter() {+ // `[^a-z0-9_]+` split after lowercasing, then length > 2.+ assert_eq!(+ tokenize("What's my dog's NAME, in 2023? a_b x"),+ ["what", "dog", "name", "2023", "a_b"]+ );+ // Duplicates are kept: the reference does not dedupe query terms.+ assert_eq!(tokenize("boot boot"), ["boot", "boot"]);+ // Non-ASCII letters split words, as `[^a-z0-9_]` does.+ assert_eq!(tokenize("café latte"), ["caf", "latte"]);+ }++ #[test]+ fn session_text_flattens_roles_under_a_date_line() {+ let t = session_text("2023/05/20", &[turn("user", "hi"), turn("assistant", "yo")]);+ assert_eq!(+ t,+ "[Source session date: 2023/05/20]\n[user] hi\n\n[assistant] yo"+ );+ }++ #[test]+ fn rank_counts_term_occurrence_drops_zero_and_keeps_tie_order() {+ let docs = [+ (1, "nothing relevant"),+ (2, "my boots are leather"),+ (3, "leather boots and leather care"),+ (4, "boots only"),+ ];+ // query terms: leather, boots → doc 2 and 3 score 2, doc 4 scores 1.+ let ranked = rank("leather boots", docs);+ assert_eq!(ranked, [2, 3, 4]);+ }++ #[test]+ fn substring_matching_credits_inflections() {+ assert_eq!(rank("boot", [(1, "two boots")]), [1]);+ }++ #[test]+ fn repeated_query_terms_weigh_double() {+ let docs = [(1, "alpha"), (2, "beta")];+ assert_eq!(rank("beta beta alpha", docs), [2, 1]);+ }++ #[test]+ fn repeated_session_ids_keep_their_first_position() {+ let q: Question = serde_json::from_value(serde_json::json!({+ "question_id": "q",+ "question_type": "multi-session",+ "question": "quokka",+ "answer": "a",+ "question_date": "d",+ "haystack_dates": ["d1", "d2", "d3"],+ "haystack_session_ids": ["s_a", "s_b", "s_a"],+ "haystack_sessions": [+ [{"role": "user", "content": "quokka"}],+ [{"role": "user", "content": "quokka"}],+ [{"role": "user", "content": "quokka"}]+ ],+ "answer_session_ids": ["s_b"]+ }))+ .unwrap();+ let ranked = rank_question(&q);+ assert_eq!(+ ranked,+ [stored_session_uuid("s_a"), stored_session_uuid("s_b")]+ );+ }+}diff --git a/evals/src/retrieval/ingest.rs b/evals/src/retrieval/ingest.rsindex ae1b2c89..9bf55120 100644--- a/evals/src/retrieval/ingest.rs+++ b/evals/src/retrieval/ingest.rs@@ -12,8 +12,17 @@//! installs see wall-clock time; replayed histories must carry their//! own), mirroring what the official LongMemEval harness exposes to//! retrievers as session timestamps.+//!+//! Two ablation knobs change that shape for the M1 decomposition (defaults+//! keep every published baseline reproducible):+//!+//! - [`DatePrefix`] moves or drops the per-turn date prefix, which takes+//! about 40 of the 80 characters a zero-LLM session page keeps per turn;+//! - [`IngestMode`] can skip the hooks and write each haystack session as+//! one wiki page through `POST /admin/write-page` (the real+//! `Wiki::write_page` path), which isolates ranking from capture.-use std::time::Duration;+use std::time::{Duration, Instant};use anyhow::{Context, Result, bail};use serde_json::json;@@ -32,6 +41,71 @@ const MAX_BATCH: usize = 256;/// client to do the capping.const EXCERPT_MAX_BYTES: usize = 2_000;+/// Where the replay puts each session's original date.+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]+pub enum DatePrefix {+ /// `[session date: …]` before every turn (the published baselines).+ Turn,+ /// Once per session, as the `session-start` event's title; turns carry+ /// no prefix.+ SessionStart,+ /// Nowhere.+ None,+}++impl DatePrefix {+ pub fn as_str(self) -> &'static str {+ match self {+ Self::Turn => "turn",+ Self::SessionStart => "session-start",+ Self::None => "none",+ }+ }+}++/// How a question's haystack reaches the server.+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]+pub enum IngestMode {+ /// Replay through `/hook/batch` (the shipped capture path).+ Hooks,+ /// One page per session holding the full raw transcript: the+ /// ranking-only upper bound, comparable to whole-session harnesses.+ WholeSession,+ /// One page per session holding exactly the text capture keeps in+ /// observation bodies (user turns up to 16 KiB, assistant excerpts up+ /// to 2 KB, date prefix as configured): what an observation-evidence+ /// stream could rank on, aggregated per session.+ CapturedSession,+}++impl IngestMode {+ pub fn as_str(self) -> &'static str {+ match self {+ Self::Hooks => "hooks",+ Self::WholeSession => "whole-session",+ Self::CapturedSession => "captured-session",+ }+ }+}++/// Server-side user-prompt body cap (`USER_PROMPT_EXCERPT_MAX_BYTES`,+/// equal to the 16 KiB observation backstop).+const USER_PROMPT_MAX_BYTES: usize = 16 * 1024;++/// Timing and volume for one question's ingest.+#[derive(Debug, Clone, Default)]+pub struct IngestStats {+ /// Hook events posted, or pages written.+ pub items: usize,+ /// Wall clock for the whole question.+ pub elapsed: Duration,+ /// Round trip of every HTTP request (batch POST or page write).+ pub request_ms: Vec<u128>,+ /// Pages whose body exceeded the 16 KiB observation backstop (page+ /// modes only; the page write path has no such cap, see the report).+ pub docs_over_16kib: usize,+}+/// Project name for one benchmark question (its private haystack).pub fn project_for(q: &Question) -> String {format!("lme-{}", q.question_id)@@ -49,16 +123,64 @@ pub fn stored_session_uuid(raw: &str) -> uuid::Uuid {/// Truncate on a char boundary so the marker stays valid UTF-8.fn cap_excerpt(s: &str) -> &str {- if s.len() <= EXCERPT_MAX_BYTES {+ cap_bytes(s, EXCERPT_MAX_BYTES)+}++fn cap_bytes(s: &str, max: usize) -> &str {+ if s.len() <= max {return s;}- let mut end = EXCERPT_MAX_BYTES;+ let mut end = max;while !s.is_char_boundary(end) {end -= 1;}&s[..end]}+/// Turn text as the hook replay sends it under `prefix`.+fn prefixed(prefix: DatePrefix, date: &str, text: &str) -> String {+ match prefix {+ DatePrefix::Turn => format!("[session date: {date}] {text}"),+ DatePrefix::SessionStart | DatePrefix::None => text.to_string(),+ }+}++/// One haystack session as a page body for the page ingest modes. Every+/// turn renders as `[<role>] <text>`, joined by blank lines, under a+/// `[Source session date: …]` line (the same document the grep baseline+/// scores). `CapturedSession` caps each turn the way capture does and+/// places the date exactly where the hook replay under `prefix` would.+pub fn session_document(+ mode: IngestMode,+ prefix: DatePrefix,+ date: &str,+ turns: &[super::dataset::Turn],+) -> String {+ let rendered: Vec<String> = turns+ .iter()+ .map(|t| match mode {+ IngestMode::CapturedSession => {+ let text = match t.role.as_str() {+ "assistant" => prefixed(prefix, date, cap_excerpt(&t.content)),+ _ => prefixed(prefix, date, &t.content),+ };+ // The server caps the whole prompt field, prefix included.+ format!("[{}] {}", t.role, cap_bytes(&text, USER_PROMPT_MAX_BYTES))+ }+ _ => format!("[{}] {}", t.role, t.content),+ })+ .collect();+ // Captured text only carries a session-level date when the replay puts+ // one on `session-start`; per-turn prefixes are already in the turns.+ let header = !(mode == IngestMode::CapturedSession+ && matches!(prefix, DatePrefix::Turn | DatePrefix::None));+ if header {+ format!("[Source session date: {date}]\n{}", rendered.join("\n\n"))+ } else {+ rendered.join("\n\n")+ }+}+#[derive(serde::Serialize)]struct BatchItem {url: String,@@ -66,7 +188,7 @@ struct BatchItem {}/// Build the full event replay for one question's haystack.-fn build_items(q: &Question) -> Vec<BatchItem> {+fn build_items(q: &Question, prefix: DatePrefix) -> Vec<BatchItem> {let project = project_for(q);let cwd = format!("/lme/{project}");let mut items = Vec::new();@@ -92,11 +214,13 @@ fn build_items(q: &Question) -> Vec<BatchItem> {for (idx, session) in q.haystack_sessions.iter().enumerate() {let sid = &q.haystack_session_ids[idx];let date = &q.haystack_dates[idx];- push(- "session-start",- sid,- json!({ "hook_event_name": "SessionStart", "source": "startup" }),- );+ let mut start = json!({ "hook_event_name": "SessionStart", "source": "startup" });+ if prefix == DatePrefix::SessionStart {+ // The session-start title lands in the page's raw-observation+ // list, so the date stays indexed once per session.+ start["title"] = json!(format!("session date: {date}"));+ }+ push("session-start", sid, start);for turn in session {match turn.role.as_str() {"user" => push(@@ -104,7 +228,7 @@ fn build_items(q: &Question) -> Vec<BatchItem> {sid,json!({"hook_event_name": "UserPromptSubmit",- "prompt": format!("[session date: {date}] {}", turn.content),+ "prompt": prefixed(prefix, date, &turn.content),}),),"assistant" => push(@@ -114,7 +238,7 @@ fn build_items(q: &Question) -> Vec<BatchItem> {"hook_event_name": "Stop","_ai_memory_assistant": {"version": 1,- "excerpt": format!("[session date: {date}] {}", cap_excerpt(&turn.content)),+ "excerpt": prefixed(prefix, date, cap_excerpt(&turn.content)),},}),),@@ -139,8 +263,11 @@ pub async fn ingest_question(client: &reqwest::Client,base_url: &str,q: &Question,-) -> Result<usize> {- let items = build_items(q);+ prefix: DatePrefix,+) -> Result<IngestStats> {+ let started = Instant::now();+ let mut request_ms = Vec::new();+ let items = build_items(q, prefix);let total = items.len();let mut pending: Vec<&BatchItem> = items.iter().collect();let mut attempts = 0usize;@@ -156,6 +283,7 @@ pub async fn ingest_question(}let mut next = Vec::new();for chunk in pending.chunks(MAX_BATCH) {+ let sent = Instant::now();let resp = client.post(format!("{base_url}/hook/batch")).bearer_auth(EVAL_AUTH_TOKEN)@@ -166,6 +294,7 @@ pub async fn ingest_question(.error_for_status().context("hook batch rejected")?;let ack: serde_json::Value = resp.json().await?;+ request_ms.push(sent.elapsed().as_millis());let accepted: Vec<usize> = match ack.get("accepted_indices") {Some(serde_json::Value::Array(idx)) => idx.iter()@@ -188,7 +317,73 @@ pub async fn ingest_question(}pending = next;}- Ok(total)+ Ok(IngestStats {+ items: total,+ elapsed: started.elapsed(),+ request_ms,+ docs_over_16kib: 0,+ })+}++/// Page title for a session written by the page ingest modes. Neutral on+/// purpose: it carries the date (as the hook pages' titles do) and nothing+/// that could favour the evidence session.+fn session_page_title(date: &str) -> String {+ format!("Chat session {date}")+}++/// Write each haystack session of `q` as one page under+/// `sessions/<uuid>.md` through `POST /admin/write-page`, sequentially+/// (questions still run concurrently). A repeated session id is written+/// once, at its first occurrence.+pub async fn write_session_pages(+ client: &reqwest::Client,+ base_url: &str,+ q: &Question,+ mode: IngestMode,+ prefix: DatePrefix,+) -> Result<IngestStats> {+ let started = Instant::now();+ let project = project_for(q);+ let mut seen = std::collections::HashSet::new();+ let mut stats = IngestStats::default();+ for ((sid, date), turns) in q+ .haystack_session_ids+ .iter()+ .zip(&q.haystack_dates)+ .zip(&q.haystack_sessions)+ {+ if !seen.insert(sid.as_str()) {+ continue;+ }+ let body = session_document(mode, prefix, date, turns);+ if body.len() > USER_PROMPT_MAX_BYTES {+ stats.docs_over_16kib += 1;+ }+ let request = json!({+ "workspace": EVAL_WORKSPACE,+ "project": project,+ "path": format!("sessions/{}.md", stored_session_uuid(sid)),+ "title": session_page_title(date),+ "body": body,+ // Hook-synthesised session pages are episodic too.+ "tier": "episodic",+ });+ let sent = Instant::now();+ client+ .post(format!("{base_url}/admin/write-page"))+ .bearer_auth(EVAL_AUTH_TOKEN)+ .json(&request)+ .send()+ .await+ .context("posting write-page")?+ .error_for_status()+ .with_context(|| format!("write-page rejected for {sid}"))?;+ stats.request_ms.push(sent.elapsed().as_millis());+ stats.items += 1;+ }+ stats.elapsed = started.elapsed();+ Ok(stats)}#[cfg(test)]@@ -216,7 +411,7 @@ mod tests {#[test]fn replay_matches_the_production_hook_cadence() {- let items = build_items(&q());+ let items = build_items(&q(), DatePrefix::Turn);let events: Vec<&str> = items.iter().map(|i| {@@ -246,7 +441,7 @@ mod tests {#[test]fn session_dates_are_visible_to_the_retriever() {- let items = build_items(&q());+ let items = build_items(&q(), DatePrefix::Turn);let prompt = items[1].body["prompt"].as_str().unwrap();assert!(prompt.starts_with("[session date: 2023/05/20"));}@@ -277,9 +472,88 @@ mod tests {question.haystack_sessions[0].push(Turn {role: "system".into(),content: "x".into(),+ has_answer: None,});- let items = build_items(&question);+ let items = build_items(&question, DatePrefix::Turn);// still start, prompt, stop, end — the system turn produced nothingassert_eq!(items.len(), 4);}++ #[test]+ fn session_start_prefix_moves_the_date_off_the_turns() {+ let items = build_items(&q(), DatePrefix::SessionStart);+ assert_eq!(+ items[0].body["title"].as_str(),+ Some("session date: 2023/05/20 (Sat) 02:21")+ );+ assert_eq!(items[1].body["prompt"].as_str(), Some("hello"));+ assert_eq!(+ items[2].body["_ai_memory_assistant"]["excerpt"].as_str(),+ Some("hi")+ );+ }++ #[test]+ fn no_date_prefix_leaves_turns_and_session_start_bare() {+ let items = build_items(&q(), DatePrefix::None);+ assert!(items[0].body.get("title").is_none());+ assert_eq!(items[1].body["prompt"].as_str(), Some("hello"));+ }++ #[test]+ fn whole_session_documents_keep_every_byte_of_every_turn() {+ let long = "y".repeat(EXCERPT_MAX_BYTES * 3);+ let turns = vec![+ Turn {+ role: "user".into(),+ content: "hello".into(),+ has_answer: None,+ },+ Turn {+ role: "assistant".into(),+ content: long.clone(),+ has_answer: None,+ },+ ];+ let doc = session_document(IngestMode::WholeSession, DatePrefix::Turn, "D", &turns);+ assert_eq!(+ doc,+ format!("[Source session date: D]\n[user] hello\n\n[assistant] {long}")+ );+ }++ #[test]+ fn captured_session_documents_apply_the_capture_caps_and_prefix() {+ let turns = vec![+ Turn {+ role: "user".into(),+ content: "u".repeat(USER_PROMPT_MAX_BYTES + 10),+ has_answer: None,+ },+ Turn {+ role: "assistant".into(),+ content: "a".repeat(EXCERPT_MAX_BYTES + 10),+ has_answer: None,+ },+ ];+ let doc = session_document(IngestMode::CapturedSession, DatePrefix::Turn, "D", &turns);+ let parts: Vec<&str> = doc.split("\n\n").collect();+ let user = parts[0].strip_prefix("[user] ").unwrap();+ assert!(user.starts_with("[session date: D] u"));+ assert_eq!(user.len(), USER_PROMPT_MAX_BYTES);+ let assistant = parts[1]+ .strip_prefix("[assistant] [session date: D] ")+ .unwrap();+ assert_eq!(assistant.len(), EXCERPT_MAX_BYTES);++ let bare = session_document(IngestMode::CapturedSession, DatePrefix::None, "D", &turns);+ assert!(bare.starts_with("[user] uuu"));+ let once = session_document(+ IngestMode::CapturedSession,+ DatePrefix::SessionStart,+ "D",+ &turns,+ );+ assert!(once.starts_with("[Source session date: D]\n[user] uuu"));+ }}diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rsindex 0a4cac74..c3f6e552 100644--- a/evals/src/retrieval/mod.rs+++ b/evals/src/retrieval/mod.rs@@ -15,11 +15,26 @@//! (hit@k / recall@k), `memory_query` latency (p50/p95), and the context//! tokens a result would cost the agent (chars/4 over returned snippets).//! Without a candidate config the single-config path is unchanged.+//!+//! ## M1: decomposing the retrieval gap+//!+//! Four opt-in instruments split a hit@k number into capture loss and+//! ranking loss (defaults reproduce the published baselines):+//!+//! - `--reachability` checks whether each answer-bearing turn survives+//! into the page, the observations, and the embedder's 512-token window;+//! - `--engine grep` scores a whole-session term-overlap baseline with no+//! server, calibrated against agentmemory's grep adapter;+//! - `--ingest whole-session|captured-session` writes one page per session+//! through the real page write path instead of replaying hooks;+//! - `--date-prefix turn|session-start|none` moves the per-turn date.mod dataset;+mod grep;mod ingest;mod qa;mod query;+mod reachability;mod report;mod score;@@ -30,17 +45,24 @@ use std::sync::Arc;use anyhow::{Context, Result, bail};use jiff::Timestamp;-use dataset::Question;-use ingest::{project_for, stored_session_uuid};+use dataset::{DatasetVariant, Question};+use ingest::{DatePrefix, IngestMode, project_for, stored_session_uuid};use server::{EvalEmbeddings, EvalServer, LaunchConfig};mod server;#[derive(clap::Args, Debug)]pub struct RetrievalArgs {- /// Dataset file (LongMemEval-S JSON).- #[arg(long, default_value = "evals/datasets/longmemeval_s.json")]- dataset: PathBuf,+ /// Dataset file (LongMemEval-S JSON). Defaults to the selected+ /// variant's path under `evals/datasets/`.+ #[arg(long)]+ dataset: Option<PathBuf>,++ /// Which pinned LongMemEval-S file to use: `s` (the original, every+ /// published ai-memory baseline) or `s-cleaned` (the re-release other+ /// systems publish on).+ #[arg(long, value_enum, default_value_t = Variant::S)]+ dataset_variant: Variant,/// Download the dataset first if the file is missing.#[arg(long)]@@ -56,6 +78,23 @@ pub struct RetrievalArgs {#[arg(long)]sample: Option<usize>,+ /// Score only the first N scored (non-abstention) questions of each+ /// question type: a deterministic sample that covers every slice.+ /// Applied after `--sample`.+ #[arg(long)]+ stratify: Option<usize>,++ /// `server` (the default) drives a real `ai-memory serve`; `grep`+ /// scores the whole-session term-overlap baseline with no server.+ #[arg(long, value_enum, default_value_t = Engine::Server)]+ engine: Engine,++ /// After ingest, report how much answer-bearing evidence reaches the+ /// page, the observations, and the local embedder's 512-token window.+ /// Needs the local model's tokenizer (fetched into `evals/models/`).+ #[arg(long)]+ reachability: bool,+/// Cutoffs for hit@k / recall@k.#[arg(long, value_delimiter = ',', default_values_t = vec![1, 3, 5, 10])]ks: Vec<usize>,@@ -99,6 +138,17 @@ pub struct RetrievalArgs {#[arg(long = "query-arg", value_name = "KEY=JSON")]query_arg: Vec<String>,+ /// How the haystack reaches the baseline server: `hooks` (the shipped+ /// capture path), `whole-session` (one page per raw session), or+ /// `captured-session` (one page per session holding the captured text).+ #[arg(long, value_enum, default_value_t = IngestMode::Hooks)]+ ingest: IngestMode,++ /// Where the baseline replay puts each session's date: `turn` (every+ /// turn, the published baselines), `session-start`, or `none`.+ #[arg(long, value_enum, default_value_t = DatePrefix::Turn)]+ date_prefix: DatePrefix,+// ---- candidate config knobs (A/B mode) ----/// Enable A/B mode with a candidate config. Implied by any/// `--candidate-*` knob. With no other candidate knob the candidate@@ -124,6 +174,14 @@ pub struct RetrievalArgs {#[arg(long = "candidate-query-arg", value_name = "KEY=JSON")]candidate_query_arg: Vec<String>,+ /// Candidate ingest mode; defaults to the baseline's `--ingest`.+ #[arg(long, value_enum)]+ candidate_ingest: Option<IngestMode>,++ /// Candidate date placement; defaults to the baseline's `--date-prefix`.+ #[arg(long, value_enum)]+ candidate_date_prefix: Option<DatePrefix>,+// ---- QA-accuracy mode (R2b, opt-in, live LLM) ----/// Enable end-to-end QA-accuracy grading (real LLM API calls). OFF by/// default. Per scored question the harness synthesizes an answer from@@ -195,9 +253,34 @@ impl RetrievalArgs {|| self.candidate_reranker|| !self.candidate_server_env.is_empty()|| !self.candidate_query_arg.is_empty()+ || self.candidate_ingest.is_some()+ || self.candidate_date_prefix.is_some()+ }+}++/// `--dataset-variant`.+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]+enum Variant {+ S,+ SCleaned,+}++impl Variant {+ fn pinned(self) -> &'static DatasetVariant {+ match self {+ Self::S => &dataset::VARIANT_S,+ Self::SCleaned => &dataset::VARIANT_S_CLEANED,+ }}}+/// `--engine`.+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]+enum Engine {+ Server,+ Grep,+}+/// One resolved server + query configuration.struct Knobs {/// Short label for reports (`baseline` / `candidate`).@@ -209,17 +292,22 @@ struct Knobs {reranker: bool,env: Vec<(String, String)>,query_args: serde_json::Map<String, serde_json::Value>,+ ingest: IngestMode,+ date_prefix: DatePrefix,/// Human-readable knob summary for the report.summary: String,}impl Knobs {+ #[allow(clippy::too_many_arguments)]fn parse(side: &'static str,embeddings: &str,reranker: bool,server_env: &[String],query_arg: &[String],+ ingest: IngestMode,+ date_prefix: DatePrefix,) -> Result<Self> {let (embeddings_kind, mode, models_root) = match embeddings {"none" => (EvalEmbeddings::None, "zero-llm", None),@@ -243,6 +331,14 @@ impl Knobs {for (k, v) in &query_args {summary.push_str(&format!(" arg:{k}={v}"));}+ // Only non-default ablations are named, so a default run's summary+ // reads exactly as the published baselines do.+ if ingest != IngestMode::Hooks {+ summary.push_str(&format!(" ingest={}", ingest.as_str()));+ }+ if date_prefix != DatePrefix::Turn {+ summary.push_str(&format!(" date_prefix={}", date_prefix.as_str()));+ }Ok(Self {side,@@ -252,6 +348,8 @@ impl Knobs {reranker,env,query_args,+ ingest,+ date_prefix,summary,})}@@ -311,10 +409,15 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {// Captured at launch: a long run must not stamp its report with a// commit that landed while it was in flight.let commit = report::commit_sha();- if args.fetch && !args.dataset.exists() {- dataset::fetch(&args.dataset).await?;+ let variant = args.dataset_variant.pinned();+ let dataset_path = args+ .dataset+ .clone()+ .unwrap_or_else(|| PathBuf::from(variant.default_path));+ if args.fetch && !dataset_path.exists() {+ dataset::fetch(&dataset_path, variant).await?;}- let mut questions = dataset::load(&args.dataset)?;+ let mut questions = dataset::load(&dataset_path, variant)?;let total_loaded = questions.len();if let Some(n) = args.sample {questions.truncate(n);@@ -325,6 +428,9 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {.cloned().collect();questions.retain(|q| !q.is_abstention());+ if let Some(per_type) = args.stratify {+ questions = dataset::stratify(questions, per_type);+ }tracing::info!(loaded = total_loaded,scored = questions.len(),@@ -332,6 +438,30 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {"dataset ready");+ let stamp: String = Timestamp::now()+ .to_string()+ .chars()+ .map(|c| if c == ':' || c == '.' { '-' } else { c })+ .collect();+ let run_dir = args.out.join(format!("{stamp}-retrieval"));++ if args.engine == Engine::Grep {+ if args.ab_mode() || args.reachability || args.qa {+ bail!("--engine grep runs alone: no candidate, --reachability, or --qa");+ }+ let report = run_grep(&args, variant, &questions, abstention.len(), &commit);+ std::fs::create_dir_all(&run_dir)?;+ std::fs::write(+ run_dir.join("report.json"),+ serde_json::to_vec_pretty(&report)?,+ )?;+ let md = report::to_markdown(&report);+ std::fs::write(run_dir.join("report.md"), &md)?;+ println!("{md}");+ println!("run dir: {}", run_dir.display());+ return Ok(());+ }+if !args.server_bin.exists() {bail!("server binary {} not found — run `cargo build --release -p ai-memory-cli` first",@@ -345,6 +475,8 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {args.reranker,&args.server_env,&args.query_arg,+ args.ingest,+ args.date_prefix,)?;let candidate = if args.ab_mode() {let embeddings = args@@ -357,6 +489,8 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {args.candidate_reranker,&args.candidate_server_env,&args.candidate_query_arg,+ args.candidate_ingest.unwrap_or(args.ingest),+ args.candidate_date_prefix.unwrap_or(args.date_prefix),)?)} else {None@@ -367,7 +501,7 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {|| candidate.as_ref().is_some_and(|c| c.embeddings == EvalEmbeddings::Local);- if needs_local {+ if needs_local || args.reachability {let root = PathBuf::from("evals/models");if !ai_memory_llm::model_present(&root) {tracing::info!("fetching the local embedding model into evals/models (~87 MB)");@@ -396,24 +530,28 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {None};+ // The embedder-window check reads the model's own tokenizer.+ let embed_window = if args.reachability {+ Some(Arc::new(reachability::EmbedWindow::load(+ std::path::Path::new("evals/models"),+ )?))+ } else {+ None+ };+let abstention_n = abstention.len();- let baseline_report = run_config(- &args,- &baseline,- &questions,+ let run = RunContext {+ args: &args,+ variant,+ questions: &questions,abstention_n,- &commit,- qa_engine.clone(),- )- .await?;+ commit: &commit,+ qa: qa_engine.clone(),+ embed_window,+ };+ let baseline_report = run_config(&run, &baseline).await?;// Timestamped run dir shared by single-config and A/B outputs.- let stamp: String = Timestamp::now()- .to_string()- .chars()- .map(|c| if c == ':' || c == '.' { '-' } else { c })- .collect();- let run_dir = args.out.join(format!("{stamp}-retrieval"));std::fs::create_dir_all(&run_dir)?;match candidate {@@ -427,15 +565,7 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {println!("{md}");}Some(candidate) => {- let candidate_report = run_config(- &args,- &candidate,- &questions,- abstention_n,- &commit,- qa_engine.clone(),- )- .await?;+ let candidate_report = run_config(&run, &candidate).await?;let ab = report::AbReport::new(baseline_report, candidate_report);std::fs::write(run_dir.join("report.json"), serde_json::to_vec_pretty(&ab)?)?;let md = report::ab_to_markdown(&ab);@@ -447,18 +577,30 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {Ok(())}+/// Everything a config run shares with its A/B twin.+struct RunContext<'a> {+ args: &'a RetrievalArgs,+ variant: &'static DatasetVariant,+ questions: &'a [Question],+ abstention_n: usize,+ commit: &'a str,+ qa: Option<Arc<qa::QaEngine>>,+ embed_window: Option<Arc<reachability::EmbedWindow>>,+}++/// What one question's task hands back for aggregation.+struct QuestionRun {+ score: score::QuestionScore,+ ingest: ingest::IngestStats,+ reach: Option<reachability::QuestionReach>,+}+/// Launch one server config, replay + query + score the whole question/// set through it, and build its [`report::Report`]. Each config gets a/// fresh server (and fresh data dir) so ingestion never bleeds between/// baseline and candidate.-async fn run_config(- args: &RetrievalArgs,- knobs: &Knobs,- questions: &[Question],- abstention_n: usize,- commit: &str,- qa: Option<Arc<qa::QaEngine>>,-) -> Result<report::Report> {+async fn run_config(run: &RunContext<'_>, knobs: &Knobs) -> Result<report::Report> {+ let args = run.args;let server = EvalServer::launch(&args.server_bin, args.keep_data_dir, &knobs.launch_config()).await.with_context(|| format!("launching {} server", knobs.side))?;@@ -478,23 +620,33 @@ async fn run_config(let base_url = Arc::new(server.base_url.clone());let client = Arc::new(client);let query_args = Arc::new(knobs.query_args.clone());+ let (ingest_mode, date_prefix) = (knobs.ingest, knobs.date_prefix);+ let reach_on = args.reachability;+ let started = std::time::Instant::now();let semaphore = Arc::new(tokio::sync::Semaphore::new(args.concurrency.max(1)));let mut tasks = tokio::task::JoinSet::new();- for q in questions.iter().cloned() {+ for q in run.questions.iter().cloned() {let permit = semaphore.clone().acquire_owned().await?;- let (client, base_url, ks, query_args, qa) = (+ let (client, base_url, ks, query_args, qa, embed_window) = (client.clone(),base_url.clone(),ks.clone(),query_args.clone(),- qa.clone(),+ run.qa.clone(),+ run.embed_window.clone(),);tasks.spawn(async move {let _permit = permit;- let events = ingest::ingest_question(&client, &base_url, &q)- .await- .with_context(|| format!("ingesting {}", q.question_id))?;+ let ingest_stats = match ingest_mode {+ IngestMode::Hooks => {+ ingest::ingest_question(&client, &base_url, &q, date_prefix).await+ }+ mode => {+ ingest::write_session_pages(&client, &base_url, &q, mode, date_prefix).await+ }+ }+ .with_context(|| format!("ingesting {}", q.question_id))?;let outcome = query::memory_query(&client,&base_url,@@ -520,6 +672,26 @@ async fn run_config(outcome.latency.as_millis(),outcome.context_tokens,);+ score.ingest_ms = ingest_stats.elapsed.as_millis();+ score.raw_hits = outcome.raw_hits;+ // Read back only after the query, so the measurement never+ // changes what the query saw.+ let reach = if reach_on {+ Some(+ reachability::measure_question(+ &client,+ &base_url,+ &q,+ ingest_mode,+ date_prefix,+ embed_window.as_deref(),+ )+ .await+ .with_context(|| format!("reachability for {}", q.question_id))?,+ )+ } else {+ None+ };// QA-accuracy (R2b): answer + grade. A QA failure for one// question is logged and left ungraded — it must not waste the// whole retrieval run (retrieval numbers are already collected).@@ -538,7 +710,7 @@ async fn run_config(}tracing::info!(question = q.question_id,- events,+ items = ingest_stats.items,retrieved = outcome.retrieved.len(),latency_ms = outcome.latency.as_millis() as u64,context_tokens = outcome.context_tokens,@@ -546,7 +718,11 @@ async fn run_config(qa_correct = score.qa.as_ref().map(|o| o.correct),"scored");- anyhow::Ok(score)+ anyhow::Ok(QuestionRun {+ score,+ ingest: ingest_stats,+ reach,+ })});}@@ -554,10 +730,16 @@ async fn run_config(// the other 499, but a partial run must never masquerade as a// publishable number either — any failure aborts before reporting.let mut scores = Vec::new();+ let mut ingests = Vec::new();+ let mut reaches = Vec::new();let mut failures = Vec::new();while let Some(res) = tasks.join_next().await {match res.context("task panicked")? {- Ok(score) => scores.push(score),+ Ok(done) => {+ scores.push(done.score);+ ingests.push(done.ingest);+ reaches.extend(done.reach);+ }Err(e) => failures.push(format!("{e:#}")),}}@@ -572,26 +754,95 @@ async fn run_config(failures.len() + scores.len());}+ let wall = started.elapsed();scores.sort_by(|a, b| a.question_id.cmp(&b.question_id));+ reaches.sort_by(|a, b| a.question_id.cmp(&b.question_id));let slices = score::aggregate(&scores, &args.ks);Ok(report::Report {generated_at: Timestamp::now().to_string(),- commit: commit.to_string(),+ commit: run.commit.to_string(),hardware: report::hardware(),- dataset: "longmemeval_s",- dataset_sha256: dataset::LONGMEMEVAL_S_SHA256,+ dataset: run.variant.name,+ dataset_sha256: run.variant.sha256,mode: knobs.mode,config: knobs.summary.clone(),questions_scored: scores.len(),- abstention_excluded: abstention_n,+ abstention_excluded: run.abstention_n,ks: args.ks.clone(),- qa: qa.as_ref().map(|e| e.report_meta()),+ qa: run.qa.as_ref().map(|e| e.report_meta()),+ ingest: Some(report::IngestSummary::new(+ ingest_mode,+ date_prefix,+ args.concurrency,+ &ingests,+ wall,+ )),+ reachability: (!reaches.is_empty()).then(|| report::ReachabilityReport {+ slices: reachability::aggregate(&reaches),+ per_question: reaches,+ }),slices,per_question: scores,})}+/// `--engine grep`: rank every question's raw haystack in-process.+fn run_grep(+ args: &RetrievalArgs,+ variant: &'static DatasetVariant,+ questions: &[Question],+ abstention_n: usize,+ commit: &str,+) -> report::Report {+ let scores: Vec<score::QuestionScore> = questions+ .iter()+ .map(|q| {+ let started = std::time::Instant::now();+ let ranked = grep::rank_question(q);+ let latency = started.elapsed().as_millis();+ let retrieved: Vec<query::Retrieved> = ranked+ .into_iter()+ .map(|id| query::Retrieved {+ session_uuid: Some(id),+ })+ .collect();+ let evidence: HashSet<uuid::Uuid> = q+ .answer_session_ids+ .iter()+ .map(|s| stored_session_uuid(s))+ .collect();+ score::score_question(+ &q.question_id,+ &q.question_type,+ &evidence,+ &retrieved,+ &args.ks,+ latency,+ 0,+ )+ })+ .collect();+ let slices = score::aggregate(&scores, &args.ks);+ report::Report {+ generated_at: Timestamp::now().to_string(),+ commit: commit.to_string(),+ hardware: report::hardware(),+ dataset: variant.name,+ dataset_sha256: variant.sha256,+ mode: "grep",+ config: "engine=grep (whole raw session, term overlap, no server)".into(),+ questions_scored: scores.len(),+ abstention_excluded: abstention_n,+ ks: args.ks.clone(),+ qa: None,+ ingest: None,+ reachability: None,+ slices,+ per_question: scores,+ }+}+#[cfg(test)]mod tests {use super::*;@@ -642,6 +893,8 @@ mod tests {true,&["AI_MEMORY_LLM_PROVIDER=openai".into()],&["include_expired=true".into()],+ IngestMode::Hooks,+ DatePrefix::Turn,).unwrap();assert!(k.summary.contains("embeddings=local"));@@ -649,5 +902,21 @@ mod tests {assert!(k.summary.contains("env:AI_MEMORY_LLM_PROVIDER=openai"));assert!(k.summary.contains("arg:include_expired=true"));assert_eq!(k.mode, "local-embeddings");+ // Default ablations stay out of the summary (baseline reports+ // read as before); non-default ones are named.+ assert!(!k.summary.contains("ingest="));+ assert!(!k.summary.contains("date_prefix="));+ let k = Knobs::parse(+ "candidate",+ "none",+ false,+ &[],+ &[],+ IngestMode::WholeSession,+ DatePrefix::SessionStart,+ )+ .unwrap();+ assert!(k.summary.contains("ingest=whole-session"));+ assert!(k.summary.contains("date_prefix=session-start"));}}diff --git a/evals/src/retrieval/qa.rs b/evals/src/retrieval/qa.rsindex ce533173..1e42a2c3 100644--- a/evals/src/retrieval/qa.rs+++ b/evals/src/retrieval/qa.rs@@ -484,6 +484,7 @@ mod tests {answer: None,latency: std::time::Duration::ZERO,context_tokens: 0,+ raw_hits: 0,};let qa = engine.evaluate("What is the user's dog's name?", "Rex", &outcome)diff --git a/evals/src/retrieval/query.rs b/evals/src/retrieval/query.rsindex d0d82c88..89c7358a 100644--- a/evals/src/retrieval/query.rs+++ b/evals/src/retrieval/query.rs@@ -58,6 +58,10 @@ pub struct QueryOutcome {/// puts in front of the agent, so this is the context cost of the/// retrieval, not of the underlying full pages.pub context_tokens: usize,+ /// Raw observation hits in the response. The server only falls back to+ /// them when page search returns nothing, so a non-zero count marks a+ /// question answered from observations.+ pub raw_hits: usize,}#[derive(Debug, Deserialize)]@@ -120,16 +124,54 @@ pub async fn memory_query(arguments.insert("workspace".into(), json!(workspace));arguments.insert("project".into(), json!(project));arguments.insert("limit".into(), json!(limit));+ let started = Instant::now();+ let text = mcp_tool_text(+ client,+ base_url,+ "memory_query",+ serde_json::Value::Object(arguments),+ )+ .await?;+ let latency = started.elapsed();+ let parsed: QueryResponse = serde_json::from_str(&text)+ .with_context(|| format!("parsing memory_query JSON: {text}"))?;+ let raw_hits = parsed.raw_hits.len();+ // A blank/whitespace `answer` is treated as absent so the QA path falls+ // back to harness synthesis rather than grading an empty string.+ let answer = parsed+ .answer+ .as_deref()+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .map(str::to_owned);+ let (retrieved, contexts, context_tokens) = flatten(parsed);+ Ok(QueryOutcome {+ retrieved,+ contexts,+ answer,+ latency,+ context_tokens,+ raw_hits,+ })+}++/// Call one MCP tool over Streamable HTTP and return its first text+/// content block (every ai-memory tool answers with one JSON text block).+pub async fn mcp_tool_text(+ client: &reqwest::Client,+ base_url: &str,+ tool: &str,+ arguments: serde_json::Value,+) -> Result<String> {let request = json!({"jsonrpc": "2.0","id": 1,"method": "tools/call","params": {- "name": "memory_query",+ "name": tool,"arguments": arguments,}});- let started = Instant::now();let resp = client.post(format!("{base_url}/mcp")).bearer_auth(EVAL_AUTH_TOKEN)@@ -178,33 +220,16 @@ pub async fn memory_query(.and_then(|v| v.as_bool()).unwrap_or(false){- bail!("memory_query tool error: {result}");+ bail!("{tool} tool error: {result}");}- let text = result+ result.get("content").and_then(|c| c.as_array()).and_then(|c| c.first()).and_then(|c| c.get("text")).and_then(|t| t.as_str())- .ok_or_else(|| anyhow::anyhow!("memory_query returned no text content: {result}"))?;- let parsed: QueryResponse =- serde_json::from_str(text).with_context(|| format!("parsing memory_query JSON: {text}"))?;- // A blank/whitespace `answer` is treated as absent so the QA path falls- // back to harness synthesis rather than grading an empty string.- let answer = parsed- .answer- .as_deref()- .map(str::trim)- .filter(|s| !s.is_empty())- .map(str::to_owned);- let (retrieved, contexts, context_tokens) = flatten(parsed);- Ok(QueryOutcome {- retrieved,- contexts,- answer,- latency: started.elapsed(),- context_tokens,- })+ .map(str::to_owned)+ .ok_or_else(|| anyhow::anyhow!("{tool} returned no text content: {result}"))}/// Flatten pages + raw hits into ranked session attributions and theirdiff --git a/evals/src/retrieval/reachability.rs b/evals/src/retrieval/reachability.rsnew file mode 100644index 00000000..1f059e3d--- /dev/null+++ b/evals/src/retrieval/reachability.rs@@ -0,0 +1,840 @@+//! `--reachability`: does the evidence reach the index at all?+//!+//! After a question's haystack is ingested, every answer-bearing turn (the+//! dataset's `has_answer` flag) is checked against three surfaces of its+//! own session, read back through the public MCP tools:+//!+//! 1. **page**: the session page's title plus body (`memory_read_page`),+//! which is what page FTS indexes;+//! 2. **observations**: every observation title and body of the session+//! (`memory_read_session_observations`, full 16 KiB bodies), which is+//! what `observations_fts` indexes (hook ingest only);+//! 3. **embed512**: the prefix of the page body the local embedder actually+//! reads: the body cut to 6 144 bytes (`truncate_for_embedding` with+//! `MAX_TOKENS * 4`), then to 512 WordPiece tokens including `[CLS]` and+//! `[SEP]`, using the model's own tokenizer.+//!+//! ## Distinctive answer content+//!+//! A turn's *content tokens* are its lowercased alphanumeric runs of three+//! or more characters, plus any shorter run containing a digit, minus a+//! fixed English stopword list. Its *distinctive* tokens are the content+//! tokens that occur in at most [`DF_MAX`] distinct sessions of the+//! question's haystack: words a retriever could use to single this session+//! out, as opposed to words every chat contains. A turn with no such token+//! falls back to all its content tokens (counted in the report).+//!+//! A turn is **reachable** through a surface when at least+//! [`REACH_THRESHOLD`] of its distinctive tokens occur in that surface's+//! token set. Matching is exact token equality, a little stricter than+//! FTS5's tokenizer; a word cut by a title's `…` does not count.+//!+//! A second, answer-anchored check uses the gold answer: the answer's+//! content tokens that literally occur in the turn must *all* occur in the+//! surface. Many answers are derived (counts, durations, dates) and never+//! appear verbatim, so this covers fewer turns but needs no threshold.+//!+//! The question-level ceiling is the share of questions with at least one+//! reachable evidence turn: a ranker cannot put a session in the top k on+//! evidence the index never saw.++use std::collections::{BTreeMap, HashMap, HashSet};+use std::path::Path;++use anyhow::{Context, Result, bail};+use serde::Serialize;+use serde_json::json;++use super::dataset::Question;+use super::ingest::stored_session_uuid;+use super::ingest::{DatePrefix, EVAL_WORKSPACE, IngestMode, project_for, session_document};+use super::query::mcp_tool_text;++/// A token is distinctive when it occurs in at most this many sessions of+/// the question's haystack (~50 sessions; multi-session evidence spans up+/// to a handful, so 3 keeps shared topic words of 2-3 evidence sessions).+pub const DF_MAX: usize = 3;++/// Share of a turn's distinctive tokens a surface must contain.+pub const REACH_THRESHOLD: f64 = 0.5;++/// Local embedder input bounds, mirrored from `ai-memory-llm`:+/// `truncate_for_embedding(text, MAX_TOKENS * 4)` keeps+/// `min(MAX_TOKENS * 4 * 3, 8000)` bytes, then the tokenizer truncates to+/// `MAX_TOKENS`.+const EMBED_MAX_TOKENS: usize = 512;+const EMBED_PRE_TRUNCATE_BYTES: usize = 6_144;++/// Function words that carry no retrieval signal on their own.+const STOPWORDS: &[&str] = &[+ "the",+ "and",+ "for",+ "are",+ "but",+ "not",+ "you",+ "your",+ "yours",+ "all",+ "any",+ "can",+ "had",+ "her",+ "hers",+ "was",+ "one",+ "our",+ "ours",+ "out",+ "has",+ "have",+ "him",+ "his",+ "how",+ "its",+ "may",+ "she",+ "they",+ "them",+ "their",+ "theirs",+ "then",+ "than",+ "that",+ "this",+ "these",+ "those",+ "there",+ "here",+ "what",+ "when",+ "where",+ "which",+ "who",+ "whom",+ "why",+ "with",+ "would",+ "could",+ "should",+ "will",+ "shall",+ "from",+ "into",+ "onto",+ "upon",+ "about",+ "above",+ "below",+ "after",+ "before",+ "again",+ "also",+ "just",+ "only",+ "very",+ "some",+ "such",+ "more",+ "most",+ "much",+ "many",+ "each",+ "other",+ "own",+ "same",+ "too",+ "off",+ "over",+ "under",+ "were",+ "been",+ "being",+ "does",+ "did",+ "doing",+ "done",+ "get",+ "got",+ "let",+ "like",+ "make",+ "made",+ "well",+ "yes",+ "yeah",+ "okay",+ "sure",+ "thanks",+ "thank",+ "please",+ "know",+ "think",+ "want",+ "need",+ "really",+ "because",+ "while",+ "though",+ "even",+ "still",+ "both",+ "either",+ "neither",+ "nor",+ "via",+ "per",+ "etc",+ "don",+ "doesn",+ "didn",+ "can",+ "won",+ "isn",+ "aren",+ "wasn",+ "weren",+ "myself",+ "yourself",+ "itself",+ "himself",+ "herself",+ "ourselves",+ "themselves",+ "whose",+ "something",+ "anything",+ "everything",+ "nothing",+ "way",+ "lot",+ "thing",+ "things",+];++/// Content tokens of `text` (see the module docs).+pub fn content_tokens(text: &str) -> HashSet<String> {+ text.to_lowercase()+ .split(|c: char| !c.is_alphanumeric())+ .filter(|t| {+ !t.is_empty()+ && (t.chars().count() >= 3 || t.chars().any(|c| c.is_ascii_digit()))+ && !STOPWORDS.contains(t)+ })+ .map(str::to_owned)+ .collect()+}++/// One answer-bearing turn and the tokens that identify it.+#[derive(Debug, Clone)]+pub struct EvidenceTurn {+ /// Dataset id of the turn's session.+ pub session_id: String,+ /// Distinctive tokens (or all content tokens when `fallback`).+ pub distinctive: HashSet<String>,+ /// No token was rare enough; `distinctive` holds every content token.+ pub fallback: bool,+ /// Gold-answer content tokens that occur in this turn.+ pub answer: HashSet<String>,+}++/// Every `has_answer` turn of `q` with its distinctive and answer tokens.+pub fn evidence_turns(q: &Question) -> Vec<EvidenceTurn> {+ // Document frequency over distinct sessions (repeated ids count once).+ let mut seen = HashSet::new();+ let mut df: HashMap<String, usize> = HashMap::new();+ for (sid, turns) in q.haystack_session_ids.iter().zip(&q.haystack_sessions) {+ if !seen.insert(sid.as_str()) {+ continue;+ }+ let mut session_tokens = HashSet::new();+ for t in turns {+ session_tokens.extend(content_tokens(&t.content));+ }+ for tok in session_tokens {+ *df.entry(tok).or_default() += 1;+ }+ }+ let answer_tokens = content_tokens(&q.gold_answer_text());++ let mut out = Vec::new();+ let mut emitted = HashSet::new();+ for (sid, turns) in q.haystack_session_ids.iter().zip(&q.haystack_sessions) {+ // A repeated session's turns are the same evidence, counted once.+ if !emitted.insert(sid.as_str()) {+ continue;+ }+ for t in turns.iter().filter(|t| t.is_evidence()) {+ let tokens = content_tokens(&t.content);+ let distinctive: HashSet<String> = tokens+ .iter()+ .filter(|tok| df.get(*tok).copied().unwrap_or(0) <= DF_MAX)+ .cloned()+ .collect();+ let fallback = distinctive.is_empty();+ let answer = answer_tokens.intersection(&tokens).cloned().collect();+ out.push(EvidenceTurn {+ session_id: sid.clone(),+ distinctive: if fallback { tokens } else { distinctive },+ fallback,+ answer,+ });+ }+ }+ out+}++/// Fraction of `target` present in `surface`; `None` for an empty target.+pub fn coverage(target: &HashSet<String>, surface: &HashSet<String>) -> Option<f64> {+ if target.is_empty() {+ return None;+ }+ let found = target.iter().filter(|t| surface.contains(*t)).count();+ Some(found as f64 / target.len() as f64)+}++/// Whether every token of `target` is in `surface`; `None` when empty.+pub fn contains_all(target: &HashSet<String>, surface: &HashSet<String>) -> Option<bool> {+ (!target.is_empty()).then(|| target.iter().all(|t| surface.contains(t)))+}++/// The prefix of `body` the local embedder reads.+pub struct EmbedWindow {+ tokenizer: tokenizers::Tokenizer,+}++impl EmbedWindow {+ /// Load the local model's tokenizer from `models_root`, configured the+ /// way `LocalEmbedder` configures it (no padding, 512-token cap).+ pub fn load(models_root: &Path) -> Result<Self> {+ let path = models_root.join("all-MiniLM-L6-v2/tokenizer.json");+ let mut tokenizer = tokenizers::Tokenizer::from_file(&path)+ .map_err(|e| anyhow::anyhow!("loading tokenizer {}: {e}", path.display()))?;+ tokenizer.with_padding(None);+ tokenizer+ .with_truncation(Some(tokenizers::TruncationParams {+ max_length: EMBED_MAX_TOKENS,+ ..Default::default()+ }))+ .map_err(|e| anyhow::anyhow!("configuring truncation: {e}"))?;+ Ok(Self { tokenizer })+ }++ /// The slice of `body` that survives both embedder truncations.+ pub fn window<'a>(&self, body: &'a str) -> Result<&'a str> {+ let pre = pre_truncate(body);+ let enc = self+ .tokenizer+ .encode(pre, true)+ .map_err(|e| anyhow::anyhow!("tokenizing: {e}"))?;+ let last = enc+ .get_offsets()+ .iter()+ .zip(enc.get_special_tokens_mask())+ .filter(|(_, special)| **special == 0)+ .map(|((_, end), _)| *end)+ .max()+ .unwrap_or(0);+ Ok(&pre[..last.min(pre.len())])+ }+}++/// The byte cut `truncate_for_embedding` applies before tokenizing: bodies+/// over the budget keep their first `budget - len("…")` bytes (the ellipsis+/// it appends lies past the 512-token window, so it is not modelled).+fn pre_truncate(body: &str) -> &str {+ if body.len() <= EMBED_PRE_TRUNCATE_BYTES {+ return body;+ }+ let mut end = EMBED_PRE_TRUNCATE_BYTES - "…".len();+ while !body.is_char_boundary(end) {+ end -= 1;+ }+ &body[..end]+}++/// Reachability of one evidence turn through each surface.+#[derive(Debug, Clone, Serialize)]+pub struct TurnReach {+ pub page: f64,+ /// `None` when the ingest mode stores no observations.+ pub observations: Option<f64>,+ /// `None` without an embed window (no tokenizer loaded).+ pub embed512: Option<f64>,+ pub answer_page: Option<bool>,+ pub answer_observations: Option<bool>,+ pub answer_embed512: Option<bool>,+ pub fallback: bool,+}++/// Reachability for one question.+#[derive(Debug, Clone, Serialize)]+pub struct QuestionReach {+ pub question_id: String,+ pub question_type: String,+ /// Distinct sessions holding at least one evidence turn.+ pub evidence_sessions: usize,+ /// Of those, how many had a session page when read back.+ pub pages_present: usize,+ /// Page ingest modes: pages whose stored body is shorter than the+ /// document sent (any server-side truncation).+ pub pages_truncated: usize,+ pub turns: Vec<TurnReach>,+}++/// Text of one session's surfaces.+#[derive(Default)]+struct Surfaces {+ page: HashSet<String>,+ page_present: bool,+ page_truncated: bool,+ observations: Option<HashSet<String>>,+ embed512: Option<HashSet<String>>,+}++/// Read back every evidence session of `q` and score its evidence turns.+pub async fn measure_question(+ client: &reqwest::Client,+ base_url: &str,+ q: &Question,+ mode: IngestMode,+ prefix: DatePrefix,+ embed: Option<&EmbedWindow>,+) -> Result<QuestionReach> {+ let turns = evidence_turns(q);+ let project = project_for(q);+ let mut surfaces: BTreeMap<String, Surfaces> = BTreeMap::new();+ for t in &turns {+ if surfaces.contains_key(&t.session_id) {+ continue;+ }+ let idx = q+ .haystack_session_ids+ .iter()+ .position(|s| *s == t.session_id)+ .context("evidence session not in haystack")?;+ let uuid = stored_session_uuid(&t.session_id);+ let mut s = Surfaces::default();+ if let Some((title, body)) = read_page(client, base_url, &project, uuid).await? {+ s.page_present = true;+ s.page = content_tokens(&format!("{title}\n{body}"));+ if let Some(embed) = embed {+ s.embed512 = Some(content_tokens(embed.window(&body)?));+ }+ if mode != IngestMode::Hooks {+ let sent = session_document(+ mode,+ prefix,+ &q.haystack_dates[idx],+ &q.haystack_sessions[idx],+ );+ s.page_truncated = body.len() < sent.len();+ }+ } else if embed.is_some() {+ s.embed512 = Some(HashSet::new());+ }+ if mode == IngestMode::Hooks {+ let text = read_observations(client, base_url, &project, uuid).await?;+ s.observations = Some(content_tokens(&text));+ }+ surfaces.insert(t.session_id.clone(), s);+ }++ let reaches = turns+ .iter()+ .map(|t| {+ let s = &surfaces[&t.session_id];+ TurnReach {+ page: coverage(&t.distinctive, &s.page).unwrap_or(0.0),+ observations: s+ .observations+ .as_ref()+ .map(|o| coverage(&t.distinctive, o).unwrap_or(0.0)),+ embed512: s+ .embed512+ .as_ref()+ .map(|e| coverage(&t.distinctive, e).unwrap_or(0.0)),+ answer_page: contains_all(&t.answer, &s.page),+ answer_observations: s+ .observations+ .as_ref()+ .and_then(|o| contains_all(&t.answer, o)),+ answer_embed512: s.embed512.as_ref().and_then(|e| contains_all(&t.answer, e)),+ fallback: t.fallback,+ }+ })+ .collect();+ Ok(QuestionReach {+ question_id: q.question_id.clone(),+ question_type: q.question_type.clone(),+ evidence_sessions: surfaces.len(),+ pages_present: surfaces.values().filter(|s| s.page_present).count(),+ pages_truncated: surfaces.values().filter(|s| s.page_truncated).count(),+ turns: reaches,+ })+}++/// `(title, body)` of `sessions/<uuid>.md`, or `None` when it does not exist.+async fn read_page(+ client: &reqwest::Client,+ base_url: &str,+ project: &str,+ uuid: uuid::Uuid,+) -> Result<Option<(String, String)>> {+ let args = json!({+ "workspace": EVAL_WORKSPACE,+ "project": project,+ "path": format!("sessions/{uuid}.md"),+ });+ let text = match mcp_tool_text(client, base_url, "memory_read_page", args).await {+ Ok(text) => text,+ Err(e) if format!("{e:#}").to_lowercase().contains("not found") => return Ok(None),+ Err(e) => return Err(e),+ };+ let v: serde_json::Value = serde_json::from_str(&text).context("parsing memory_read_page")?;+ let field = |k: &str| v.get(k).and_then(|x| x.as_str()).unwrap_or("").to_string();+ Ok(Some((field("title"), field("body"))))+}++/// Every observation title and body of one session, concatenated.+async fn read_observations(+ client: &reqwest::Client,+ base_url: &str,+ project: &str,+ uuid: uuid::Uuid,+) -> Result<String> {+ const PAGE: usize = 200;+ let mut out = String::new();+ let mut offset = 0usize;+ loop {+ let args = json!({+ "workspace": EVAL_WORKSPACE,+ "project": project,+ "session_id": uuid.to_string(),+ "limit": PAGE,+ "offset": offset,+ "body_max_chars": 16_384,+ });+ let text =+ mcp_tool_text(client, base_url, "memory_read_session_observations", args).await?;+ let v: serde_json::Value =+ serde_json::from_str(&text).context("parsing memory_read_session_observations")?;+ let Some(obs) = v.get("observations").and_then(|o| o.as_array()) else {+ bail!("memory_read_session_observations returned no observations array: {text}");+ };+ for o in obs {+ for k in ["title", "body"] {+ if let Some(s) = o.get(k).and_then(|x| x.as_str()) {+ out.push_str(s);+ out.push('\n');+ }+ }+ }+ if obs.len() < PAGE {+ return Ok(out);+ }+ offset += PAGE;+ }+}++/// Aggregated reachability for one slice.+#[derive(Debug, Clone, Default, Serialize)]+pub struct ReachSlice {+ pub questions: usize,+ pub evidence_turns: usize,+ pub fallback_turns: usize,+ pub evidence_sessions: usize,+ pub pages_present: usize,+ pub pages_truncated: usize,+ /// Share of evidence turns reachable (coverage ≥ threshold).+ pub turn_reach_page: f64,+ pub turn_reach_observations: Option<f64>,+ pub turn_reach_embed512: Option<f64>,+ /// Mean distinctive-token coverage.+ pub mean_cov_page: f64,+ pub mean_cov_observations: Option<f64>,+ pub mean_cov_embed512: Option<f64>,+ /// Answer-anchored: turns whose literal answer tokens all survive.+ pub answer_turns: usize,+ pub answer_page: Option<f64>,+ pub answer_observations: Option<f64>,+ pub answer_embed512: Option<f64>,+ /// Share of questions with at least one reachable evidence turn.+ pub question_ceiling_page: f64,+ pub question_ceiling_observations: Option<f64>,+ pub question_ceiling_embed512: Option<f64>,+}++fn mean(values: &[f64]) -> Option<f64> {+ (!values.is_empty()).then(|| values.iter().sum::<f64>() / values.len() as f64)+}++fn share(flags: impl Iterator<Item = bool>) -> Option<f64> {+ let (mut yes, mut n) = (0usize, 0usize);+ for f in flags {+ n += 1;+ yes += usize::from(f);+ }+ (n > 0).then(|| yes as f64 / n as f64)+}++/// Aggregate one group of questions.+fn slice(group: &[&QuestionReach]) -> ReachSlice {+ let turns: Vec<&TurnReach> = group.iter().flat_map(|q| &q.turns).collect();+ let page_cov: Vec<f64> = turns.iter().map(|t| t.page).collect();+ let obs_cov: Vec<f64> = turns.iter().filter_map(|t| t.observations).collect();+ let emb_cov: Vec<f64> = turns.iter().filter_map(|t| t.embed512).collect();+ let reach = |c: &f64| *c >= REACH_THRESHOLD;+ let q_any = |pick: fn(&TurnReach) -> Option<f64>| {+ let has_surface = group+ .iter()+ .any(|q| q.turns.iter().any(|t| pick(t).is_some()));+ has_surface+ .then(|| {+ share(group.iter().map(|q| {+ q.turns+ .iter()+ .any(|t| pick(t).is_some_and(|c| c >= REACH_THRESHOLD))+ }))+ })+ .flatten()+ };+ ReachSlice {+ questions: group.len(),+ evidence_turns: turns.len(),+ fallback_turns: turns.iter().filter(|t| t.fallback).count(),+ evidence_sessions: group.iter().map(|q| q.evidence_sessions).sum(),+ pages_present: group.iter().map(|q| q.pages_present).sum(),+ pages_truncated: group.iter().map(|q| q.pages_truncated).sum(),+ turn_reach_page: share(page_cov.iter().map(reach)).unwrap_or(0.0),+ turn_reach_observations: share(obs_cov.iter().map(reach)),+ turn_reach_embed512: share(emb_cov.iter().map(reach)),+ mean_cov_page: mean(&page_cov).unwrap_or(0.0),+ mean_cov_observations: mean(&obs_cov),+ mean_cov_embed512: mean(&emb_cov),+ answer_turns: turns.iter().filter(|t| t.answer_page.is_some()).count(),+ answer_page: share(turns.iter().filter_map(|t| t.answer_page)),+ answer_observations: share(turns.iter().filter_map(|t| t.answer_observations)),+ answer_embed512: share(turns.iter().filter_map(|t| t.answer_embed512)),+ question_ceiling_page: q_any(|t| Some(t.page)).unwrap_or(0.0),+ question_ceiling_observations: q_any(|t| t.observations),+ question_ceiling_embed512: q_any(|t| t.embed512),+ }+}++/// Per question type plus `overall`, as the retrieval slices are keyed.+pub fn aggregate(per_question: &[QuestionReach]) -> BTreeMap<String, ReachSlice> {+ let mut groups: BTreeMap<String, Vec<&QuestionReach>> = BTreeMap::new();+ for q in per_question {+ groups.entry(q.question_type.clone()).or_default().push(q);+ groups.entry("overall".into()).or_default().push(q);+ }+ groups.into_iter().map(|(k, g)| (k, slice(&g))).collect()+}++/// The reachability section appended to a run's markdown report.+pub fn to_markdown(slices: &BTreeMap<String, ReachSlice>) -> String {+ let opt = |v: Option<f64>| v.map_or("n/a".to_string(), |v| format!("{v:.3}"));+ let mut md = format!(+ "\n### Evidence reachability\n\n\+ Distinctive tokens: content tokens in ≤{DF_MAX} haystack sessions; a turn is \+ reachable at ≥{REACH_THRESHOLD} coverage. Ceiling = share of questions with ≥1 \+ reachable evidence turn.\n\n\+ | slice | q | ev. turns | pages present | truncated | turn reach page | obs | embed512 | \+ answer turns | answer page | obs | embed512 | ceiling page | obs | embed512 |\n\+ |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n"+ );+ for (name, s) in slices {+ md.push_str(&format!(+ "| {name} | {} | {} | {}/{} | {} | {:.3} | {} | {} | {} | {} | {} | {} | {:.3} | {} | {} |\n",+ s.questions,+ s.evidence_turns,+ s.pages_present,+ s.evidence_sessions,+ s.pages_truncated,+ s.turn_reach_page,+ opt(s.turn_reach_observations),+ opt(s.turn_reach_embed512),+ s.answer_turns,+ opt(s.answer_page),+ opt(s.answer_observations),+ opt(s.answer_embed512),+ s.question_ceiling_page,+ opt(s.question_ceiling_observations),+ opt(s.question_ceiling_embed512),+ ));+ }+ md+}++#[cfg(test)]+mod tests {+ use super::*;++ fn set(words: &[&str]) -> HashSet<String> {+ words.iter().map(|w| (*w).to_string()).collect()+ }++ #[test]+ fn content_tokens_drop_stopwords_and_short_words_but_keep_numbers() {+ let t = content_tokens("I bought THE boots for $45 in 2 days, it's ok.");+ assert_eq!(t, set(&["bought", "boots", "45", "2", "days"]));+ }++ fn question(sessions: serde_json::Value, ids: &[&str], answer: &str) -> Question {+ let n = ids.len();+ serde_json::from_value(json!({+ "question_id": "q",+ "question_type": "single-session-user",+ "question": "?",+ "answer": answer,+ "question_date": "d",+ "haystack_dates": vec!["d"; n],+ "haystack_session_ids": ids,+ "haystack_sessions": sessions,+ "answer_session_ids": [ids[0]]+ }))+ .unwrap()+ }++ #[test]+ fn distinctive_tokens_are_rare_across_the_haystack() {+ // "weather" occurs in all five sessions, "quokka" only in the first.+ let mut sessions = vec![json!([+ {"role": "user", "content": "weather talk about my quokka", "has_answer": true}+ ])];+ for _ in 0..4 {+ sessions.push(json!([{"role": "user", "content": "weather again"}]));+ }+ let q = question(json!(sessions), &["a", "b", "c", "d", "e"], "quokka");+ let turns = evidence_turns(&q);+ assert_eq!(turns.len(), 1);+ assert_eq!(turns[0].distinctive, set(&["talk", "quokka"]));+ assert!(!turns[0].fallback);+ assert_eq!(turns[0].answer, set(&["quokka"]));+ }++ #[test]+ fn a_turn_without_rare_tokens_falls_back_to_all_content_tokens() {+ let sessions: Vec<_> = (0..5)+ .map(|i| json!([{"role": "user", "content": "common words", "has_answer": i == 0}]))+ .collect();+ let q = question(json!(sessions), &["a", "b", "c", "d", "e"], "x");+ let turns = evidence_turns(&q);+ assert_eq!(turns.len(), 1);+ assert!(turns[0].fallback);+ assert_eq!(turns[0].distinctive, set(&["common", "words"]));+ assert!(turns[0].answer.is_empty());+ }++ #[test]+ fn repeated_sessions_count_once_for_frequency_and_evidence() {+ let s = json!([{"role": "user", "content": "rare zebra", "has_answer": true}]);+ let q = question(+ json!([s.clone(), s.clone(), s.clone(), s]),+ &["a", "a", "a", "a"],+ "z",+ );+ let turns = evidence_turns(&q);+ // One distinct session: df = 1, one evidence turn.+ assert_eq!(turns.len(), 1);+ assert_eq!(turns[0].distinctive, set(&["rare", "zebra"]));+ }++ #[test]+ fn pre_truncation_matches_the_embedder_byte_budget() {+ let short = "a".repeat(EMBED_PRE_TRUNCATE_BYTES);+ assert_eq!(pre_truncate(&short).len(), EMBED_PRE_TRUNCATE_BYTES);+ let long = "é".repeat(EMBED_PRE_TRUNCATE_BYTES);+ let cut = pre_truncate(&long);+ assert!(cut.len() <= EMBED_PRE_TRUNCATE_BYTES - "…".len());+ assert!(cut.len() >= EMBED_PRE_TRUNCATE_BYTES - "…".len() - 1);+ }++ /// Runs only where the model has been fetched (`evals/models/`).+ #[test]+ fn the_embed_window_stops_at_512_tokens() {+ let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("models");+ let Ok(window) = EmbedWindow::load(&root) else {+ return;+ };+ let words: Vec<String> = (0..2000).map(|i| format!("w{i}")).collect();+ let body = words.join(" ");+ let kept = window.window(&body).unwrap();+ assert!(kept.len() < body.len());+ assert!(body.starts_with(kept));+ let short = "a short body";+ assert_eq!(window.window(short).unwrap(), short);+ }++ #[test]+ fn coverage_and_answer_checks() {+ let target = set(&["quokka", "leather", "boots", "january"]);+ let surface = set(&["quokka", "boots", "other"]);+ assert_eq!(coverage(&target, &surface), Some(0.5));+ assert_eq!(coverage(&HashSet::new(), &surface), None);+ assert_eq!(+ contains_all(&set(&["quokka", "boots"]), &surface),+ Some(true)+ );+ assert_eq!(+ contains_all(&set(&["quokka", "january"]), &surface),+ Some(false)+ );+ assert_eq!(contains_all(&HashSet::new(), &surface), None);+ }++ #[test]+ fn a_title_cut_word_does_not_count_as_reached() {+ // The page title keeps "kitche…", which must not match "kitchen".+ let page = content_tokens("[session date: 2023/05/30] I got new kitche…");+ assert!(!page.contains("kitchen"));+ assert!(page.contains("kitche"));+ }++ fn reach(page: f64, obs: Option<f64>) -> TurnReach {+ TurnReach {+ page,+ observations: obs,+ embed512: None,+ answer_page: Some(page >= 1.0),+ answer_observations: None,+ answer_embed512: None,+ fallback: false,+ }+ }++ #[test]+ fn aggregation_reports_turn_shares_and_the_question_ceiling() {+ let q1 = QuestionReach {+ question_id: "q1".into(),+ question_type: "a".into(),+ evidence_sessions: 1,+ pages_present: 1,+ pages_truncated: 0,+ turns: vec![reach(0.2, Some(1.0)), reach(1.0, Some(1.0))],+ };+ let q2 = QuestionReach {+ question_id: "q2".into(),+ question_type: "b".into(),+ evidence_sessions: 2,+ pages_present: 1,+ pages_truncated: 0,+ turns: vec![reach(0.0, Some(0.6))],+ };+ let agg = aggregate(&[q1, q2]);+ let all = &agg["overall"];+ assert_eq!(all.evidence_turns, 3);+ assert!((all.turn_reach_page - 1.0 / 3.0).abs() < 1e-9);+ assert_eq!(all.turn_reach_observations, Some(1.0));+ assert_eq!(all.turn_reach_embed512, None);+ // q1 has a page-reachable turn, q2 does not.+ assert_eq!(all.question_ceiling_page, 0.5);+ assert_eq!(all.question_ceiling_observations, Some(1.0));+ assert_eq!(all.question_ceiling_embed512, None);+ assert_eq!(all.pages_present, 2);+ assert_eq!(all.evidence_sessions, 3);+ assert_eq!(agg["a"].questions, 1);+ }+}diff --git a/evals/src/retrieval/report.rs b/evals/src/retrieval/report.rsindex 00632e59..2fff878b 100644--- a/evals/src/retrieval/report.rs+++ b/evals/src/retrieval/report.rs@@ -13,7 +13,9 @@ use std::collections::BTreeMap;use serde::Serialize;-use super::score::{QuestionScore, SliceMetrics};+use super::ingest::{DatePrefix, IngestMode, IngestStats};+use super::reachability::{QuestionReach, ReachSlice};+use super::score::{QuestionScore, SliceMetrics, percentile_u128};#[derive(Debug, Serialize)]pub struct Report {@@ -33,10 +35,116 @@ pub struct Report {/// graded. Names only — never keys. `None` on the default zero-LLM path.#[serde(skip_serializing_if = "Option::is_none")]pub qa: Option<QaReportMeta>,+ /// Ingest volume and timing (server engine only).+ #[serde(skip_serializing_if = "Option::is_none")]+ pub ingest: Option<IngestSummary>,+ /// Evidence reachability (`--reachability` only).+ #[serde(skip_serializing_if = "Option::is_none")]+ pub reachability: Option<ReachabilityReport>,pub slices: BTreeMap<String, SliceMetrics>,pub per_question: Vec<QuestionScore>,}+/// `--reachability` results: per-slice aggregates plus every question.+#[derive(Debug, Serialize)]+pub struct ReachabilityReport {+ pub slices: BTreeMap<String, ReachSlice>,+ pub per_question: Vec<QuestionReach>,+}++/// Ingest throughput for one config run.+#[derive(Debug, Clone, Serialize)]+pub struct IngestSummary {+ pub mode: &'static str,+ pub date_prefix: &'static str,+ /// Questions ingested concurrently.+ pub concurrency: usize,+ /// Hook events posted, or pages written.+ pub items: usize,+ /// HTTP requests (batch POSTs or page writes).+ pub requests: usize,+ /// Items per second within one question's sequential ingest stream.+ pub items_per_sec_per_stream: f64,+ pub question_ingest_p50_ms: u128,+ pub question_ingest_p95_ms: u128,+ pub request_p50_ms: u128,+ pub request_p95_ms: u128,+ /// Whole config run (ingest + query + read-back), wall clock.+ pub run_wall_ms: u128,+ /// Page modes: documents larger than the 16 KiB observation backstop.+ pub docs_over_16kib: usize,+}++impl IngestSummary {+ pub fn new(+ mode: IngestMode,+ prefix: DatePrefix,+ concurrency: usize,+ stats: &[IngestStats],+ wall: std::time::Duration,+ ) -> Self {+ let items: usize = stats.iter().map(|s| s.items).sum();+ let stream_secs: f64 = stats.iter().map(|s| s.elapsed.as_secs_f64()).sum();+ let per_question: Vec<u128> = stats.iter().map(|s| s.elapsed.as_millis()).collect();+ let requests: Vec<u128> = stats+ .iter()+ .flat_map(|s| s.request_ms.iter().copied())+ .collect();+ Self {+ mode: mode.as_str(),+ date_prefix: prefix.as_str(),+ concurrency,+ items,+ requests: requests.len(),+ items_per_sec_per_stream: if stream_secs > 0.0 {+ items as f64 / stream_secs+ } else {+ 0.0+ },+ question_ingest_p50_ms: percentile_u128(&per_question, 0.50),+ question_ingest_p95_ms: percentile_u128(&per_question, 0.95),+ request_p50_ms: percentile_u128(&requests, 0.50),+ request_p95_ms: percentile_u128(&requests, 0.95),+ run_wall_ms: wall.as_millis(),+ docs_over_16kib: stats.iter().map(|s| s.docs_over_16kib).sum(),+ }+ }+}++/// Ingest and reachability sections for one config, empty when absent.+fn extras_md(r: &Report) -> String {+ let mut md = String::new();+ if let Some(i) = &r.ingest {+ md.push_str(&format!(+ "\n### Ingest\n\n- mode: {} (date prefix: {}), concurrency {}\n\+ - {} items in {} requests; {:.0} items/s per question stream\n\+ - per question p50/p95: {}/{} ms; per request p50/p95: {}/{} ms\n\+ - run wall clock: {:.1} s; documents over 16 KiB: {}\n",+ i.mode,+ i.date_prefix,+ i.concurrency,+ i.items,+ i.requests,+ i.items_per_sec_per_stream,+ i.question_ingest_p50_ms,+ i.question_ingest_p95_ms,+ i.request_p50_ms,+ i.request_p95_ms,+ i.run_wall_ms as f64 / 1000.0,+ i.docs_over_16kib,+ ));+ let raw = r.per_question.iter().filter(|q| q.raw_hits > 0).count();+ md.push_str(&format!(+ "- questions answered from the raw-observation fallback: {raw}/{}\n",+ r.per_question.len()+ ));+ }+ if let Some(reach) = &r.reachability {+ md.push_str(&super::reachability::to_markdown(&reach.slices));+ }+ md+}+/// QA-accuracy provenance for a run (R2b). Provider + model names only, so a/// published report never carries a key.#[derive(Debug, Clone, Serialize)]@@ -179,6 +287,7 @@ pub fn to_markdown(r: &Report) -> String {md.push_str(&format!("- mode: {}\n- config: {}\n\n", r.mode, r.config));md.push_str(&slices_table_md(r));md.push_str(&qa_table_md(r));+ md.push_str(&extras_md(r));md.push_str(METRIC_NOTES);md}@@ -202,6 +311,10 @@ pub struct AbDelta {pub latency_p95_ms: i128,pub context_tokens_mean: f64,pub context_tokens_median: f64,+ /// Per-question hit@k changes keyed by k: `(gained, lost)`, questions+ /// the candidate hits and the baseline misses, and the reverse. A net+ /// delta built from many flips both ways is noisier than its size.+ pub hit_flips: BTreeMap<usize, (usize, usize)>,/// candidate − baseline QA accuracy over the `overall` slice (R2b)./// `None` unless both runs graded the overall slice.#[serde(skip_serializing_if = "Option::is_none")]@@ -235,7 +348,28 @@ fn compute_delta(baseline: &Report, candidate: &Report) -> AbDelta {hit_at.insert(*k, c.hit_at[k] - b.hit_at[k]);recall_at.insert(*k, c.recall_at[k] - b.recall_at[k]);}+ let base_by_id: BTreeMap<&str, &QuestionScore> = baseline+ .per_question+ .iter()+ .map(|q| (q.question_id.as_str(), q))+ .collect();+ let mut hit_flips = BTreeMap::new();+ for k in &candidate.ks {+ let (mut gained, mut lost) = (0, 0);+ for c in &candidate.per_question {+ let Some(b) = base_by_id.get(c.question_id.as_str()) else {+ continue;+ };+ match (b.hit_at.get(k) > Some(&0.0), c.hit_at.get(k) > Some(&0.0)) {+ (false, true) => gained += 1,+ (true, false) => lost += 1,+ _ => {}+ }+ }+ hit_flips.insert(*k, (gained, lost));+ }AbDelta {+ hit_flips,hit_at,recall_at,latency_p50_ms: c.latency_p50_ms as i128 - b.latency_p50_ms as i128,@@ -268,9 +402,11 @@ pub fn ab_to_markdown(r: &AbReport) -> String {md.push_str("## Baseline\n\n");md.push_str(&slices_table_md(&r.baseline));md.push_str(&qa_table_md(&r.baseline));+ md.push_str(&extras_md(&r.baseline));md.push_str("\n## Candidate\n\n");md.push_str(&slices_table_md(&r.candidate));md.push_str(&qa_table_md(&r.candidate));+ md.push_str(&extras_md(&r.candidate));// Delta table over the `overall` slice.let b = overall(&r.baseline);@@ -278,8 +414,9 @@ pub fn ab_to_markdown(r: &AbReport) -> String {md.push_str("\n## Delta (overall, candidate − baseline)\n\n");md.push_str("| metric | baseline | candidate | delta |\n|---|---|---|---|\n");for k in &r.candidate.ks {+ let (gained, lost) = r.delta.hit_flips.get(k).copied().unwrap_or((0, 0));md.push_str(&format!(- "| hit@{k} | {:.3} | {:.3} | {:+.3} |\n",+ "| hit@{k} | {:.3} | {:.3} | {:+.3} (+{gained}/-{lost}) |\n",b.hit_at[k], c.hit_at[k], r.delta.hit_at[k]));}@@ -347,6 +484,8 @@ mod tests {abstention_excluded: 1,ks: vec![5],qa: None,+ ingest: None,+ reachability: None,slices,per_question: vec![],}@@ -378,7 +517,10 @@ mod tests {assert_eq!(ab.delta.context_tokens_mean, 0.0);let md = ab_to_markdown(&ab);assert!(md.contains("A/B"));- assert!(md.contains("| hit@5 | 0.500 | 0.500 | +0.000 |"), "{md}");+ assert!(+ md.contains("| hit@5 | 0.500 | 0.500 | +0.000 (+0/-0) |"),+ "{md}"+ );}#[test]@@ -421,7 +563,77 @@ mod tests {let ab = AbReport::new(baseline, candidate);assert!((ab.delta.hit_at[&5] - 0.2).abs() < 1e-9);let md = ab_to_markdown(&ab);- assert!(md.contains("| hit@5 | 0.400 | 0.600 | +0.200 |"), "{md}");+ assert!(md.contains("| hit@5 | 0.400 | 0.600 | +0.200"), "{md}");assert!(md.contains("candidate: embeddings=local"));}++ fn question(id: &str, hit5: f64) -> QuestionScore {+ QuestionScore {+ question_id: id.into(),+ question_type: "t".into(),+ hit_at: BTreeMap::from([(5, hit5)]),+ recall_at: BTreeMap::from([(5, hit5)]),+ retrieved_sessions: 1,+ latency_ms: 0,+ context_tokens: 0,+ ingest_ms: 0,+ raw_hits: 0,+ qa: None,+ }+ }++ #[test]+ fn ab_delta_counts_paired_flips_both_ways() {+ let mut baseline = report("b", 0.5, 0.5);+ baseline.per_question = vec![+ question("q1", 1.0),+ question("q2", 0.0),+ question("q3", 1.0),+ ];+ let mut candidate = report("c", 0.5, 0.5);+ candidate.per_question = vec![+ question("q1", 0.0),+ question("q2", 1.0),+ question("q3", 1.0),+ ];+ let ab = AbReport::new(baseline, candidate);+ assert_eq!(ab.delta.hit_flips[&5], (1, 1));+ assert!(ab_to_markdown(&ab).contains("(+1/-1)"));+ }++ #[test]+ fn ingest_summary_aggregates_streams_and_requests() {+ let stats = vec![+ IngestStats {+ items: 10,+ elapsed: std::time::Duration::from_millis(500),+ request_ms: vec![5, 7],+ docs_over_16kib: 1,+ },+ IngestStats {+ items: 30,+ elapsed: std::time::Duration::from_millis(1500),+ request_ms: vec![9],+ docs_over_16kib: 0,+ },+ ];+ let s = IngestSummary::new(+ IngestMode::WholeSession,+ DatePrefix::Turn,+ 8,+ &stats,+ std::time::Duration::from_secs(3),+ );+ assert_eq!(s.items, 40);+ assert_eq!(s.requests, 3);+ assert!((s.items_per_sec_per_stream - 20.0).abs() < 1e-9);+ assert_eq!(s.request_p95_ms, 9);+ assert_eq!(s.docs_over_16kib, 1);+ assert_eq!(s.mode, "whole-session");+ let mut r = report("c", 0.5, 0.5);+ r.ingest = Some(s);+ let md = to_markdown(&r);+ assert!(md.contains("### Ingest"));+ assert!(md.contains("\n- 40 items in 3 requests"), "{md}");+ }}diff --git a/evals/src/retrieval/score.rs b/evals/src/retrieval/score.rsindex b9c3a3cf..505a38a6 100644--- a/evals/src/retrieval/score.rs+++ b/evals/src/retrieval/score.rs@@ -34,6 +34,13 @@ pub struct QuestionScore {/// Estimated context tokens the result would cost the agent (chars/4/// over returned hit titles + snippets).pub context_tokens: usize,+ /// Wall clock of this question's ingest, milliseconds (0 for `grep`).+ #[serde(default)]+ pub ingest_ms: u128,+ /// Raw observation hits the server returned (it falls back to them only+ /// when page search finds nothing).+ #[serde(default)]+ pub raw_hits: usize,/// End-to-end QA outcome (R2b, opt-in): a candidate answer graded/// against the gold answer. `None` on the default zero-LLM path and for/// any question whose QA step failed (logged, left ungraded).@@ -88,6 +95,8 @@ pub fn score_question(retrieved_sessions: ranked_sessions.len(),latency_ms,context_tokens,+ ingest_ms: 0,+ raw_hits: 0,// R2a scoring is answer-agnostic; the QA path attaches its outcome// afterward, keeping this function pure and deterministic.qa: None,@@ -157,7 +166,7 @@ impl QaSliceMetrics {/// Nearest-rank percentile (`p` in 0.0..=1.0) over a copy-sorted slice./// Empty input yields 0.-fn percentile_u128(values: &[u128], p: f64) -> u128 {+pub(crate) fn percentile_u128(values: &[u128], p: f64) -> u128 {if values.is_empty() {return 0;}