Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(evals): LongMemEval retrieval benchmark through the real stack
AkitaOnRails496e419ff60e
diff --git a/.gitignore b/.gitignoreindex 3e1578bc..e6f2c031 100644--- a/.gitignore+++ b/.gitignore@@ -43,3 +43,4 @@ evals/runs/# ai-memory atomic-write backups (apply_atomic in apply_shared.rs)*.bak-*+evals/datasets/diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bcefd508..ec47a354 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- `memory_query` no longer fails with `fts5: syntax error` when the query is+ natural language containing parentheses, apostrophe-quoted phrases, or a+ stray `OR`/`AND` (e.g. *"my visit to the Museum of Modern Art (MoMA) and+ the 'Ancient Civilizations' exhibit"*). Prepared FTS5 queries are now+ validated against the engine's own parser and degrade to an always-valid+ quoted bag of words when the preserved operator form does not parse;+ deliberate well-formed operator queries are preserved as before. Found by+ the new LongMemEval retrieval harness on its first full run.++### Added+- Added a LongMemEval retrieval benchmark to the evaluation harness+ (`cargo run -p ai-memory-eval -- retrieval`). It drives a real+ `ai-memory serve` subprocess end to end: haystack chat histories replay+ through `POST /hook/batch` at the production hook cadence, questions run+ through MCP `memory_query`, and results are scored session-level+ (`hit@k` / `recall@k` per question category, abstention questions+ reported separately). The 278 MB dataset is fetched on demand with a+ pinned sha256; baselines are published under `docs/benchmarks/`. The+ existing LLM A/B harness moved to the `ab` subcommand.+## [1.39.0] - 2026-09-01### Addeddiff --git a/Cargo.lock b/Cargo.lockindex b175e491..e2134155 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -124,9 +124,12 @@ dependencies = ["anyhow","clap","jiff",+ "reqwest 0.12.28","secrecy","serde","serde_json",+ "sha2",+ "tempfile","tokio","tracing","tracing-subscriber",diff --git a/evals/Cargo.toml b/evals/Cargo.tomlindex 490f866f..6ec9bada 100644--- a/evals/Cargo.toml+++ b/evals/Cargo.toml@@ -17,6 +17,9 @@ ai-memory-consolidate.workspace = trueanyhow.workspace = trueclap.workspace = true+reqwest.workspace = true+sha2.workspace = true+tempfile.workspace = truejiff.workspace = truesecrecy.workspace = trueserde.workspace = truediff --git a/evals/README.md b/evals/README.mdindex 17400039..e0199b5a 100644--- a/evals/README.md+++ b/evals/README.md@@ -1,8 +1,13 @@-# `evals/` — live A/B harness+# `evals/` — evaluation harnesses-A small Rust binary that runs the EXACT consolidation prompt-ai-memory uses in production against two LLM providers side by side,-and saves both outputs to disk for human comparison.+One Rust binary (`ai-memory-eval`) with two subcommands:++- **`ab`** — runs the EXACT consolidation prompt ai-memory uses in+ production against two LLM providers side by side, and saves both+ outputs to disk for human comparison.+- **`retrieval`** — the LongMemEval retrieval benchmark, driven end to+ end through a real `ai-memory serve` subprocess. Published baselines+ live in `docs/benchmarks/`.**This is not part of the shipped binary.** It's a workspace memberpurely so it shares deps + builds with the rest. `cargo build` from@@ -53,7 +58,7 @@ judge faithfulness, scoping, hallucination, etc.# string for the candidate (Ollama doesn't validate).export OPENROUTER_API_KEY="sk-or-v1-..."-cargo run -p ai-memory-eval -- \+cargo run -p ai-memory-eval -- ab \--baseline-provider openai-compat \--baseline-base-url https://openrouter.ai/api/v1 \--baseline-model moonshotai/kimi-k2.6 \@@ -67,7 +72,7 @@ cargo run -p ai-memory-eval -- \### Two Ollama models against each other```bash-cargo run -p ai-memory-eval -- \+cargo run -p ai-memory-eval -- ab \--baseline-provider openai-compat \--baseline-base-url http://192.168.0.90:11434/v1 \--baseline-model qwen3:32b \@@ -84,7 +89,7 @@ Run `ai-memory auth login openai-oauth` first, then point the eval harness atthe same token file:```bash-cargo run -p ai-memory-eval -- \+cargo run -p ai-memory-eval -- ab \--baseline-provider openai-oauth \--baseline-token-file ~/.local/share/ai-memory/auth.json \--baseline-model gpt-5.5 \@@ -100,7 +105,7 @@ Run `ai-memory auth login copilot` first, then point the eval harness at thesame auth file:```bash-cargo run -p ai-memory-eval -- \+cargo run -p ai-memory-eval -- ab \--baseline-provider copilot \--baseline-token-file ~/.local/share/ai-memory/auth.json \--baseline-model gpt-5.5 \@@ -164,3 +169,52 @@ for the next human who opens the file.## Cleanup`evals/runs/` is in `.gitignore`. Drop it whenever it gets large.++## `retrieval` — LongMemEval benchmark++Measures the real retrieval stack against+[LongMemEval](https://huggingface.co/datasets/xiaowu0162/longmemeval)+(v1, S variant, MIT license, 500 questions over ~50-session chat+haystacks). Nothing is mocked:++1. a real `ai-memory serve` subprocess starts on a fresh temp data dir+ (zero-LLM: no consolidation LLM, no embedder, no reranker — fully+ deterministic and offline);+2. each question's haystack replays through `POST /hook/batch` at the+ production hook cadence (`session-start`, `user-prompt-submit` per+ user turn, `stop` with the opt-in assistant excerpt per assistant+ turn, `session-end`), one project per question;+3. the question runs through MCP `tools/call memory_query` with+ explicit workspace/project scoping;+4. results are scored session-level: `hit@k` (any evidence session in+ the top k — the "Recall@k" most systems publish) and `recall@k`+ (fraction of evidence sessions found). The 30 abstention questions+ are excluded and reported separately.++The dataset (278 MB) is NOT committed; it downloads to+`evals/datasets/` (gitignored) with a pinned sha256 that fails loudly+on upstream drift.++```bash+cargo build --release -p ai-memory-cli # the server under test+cargo run --release -p ai-memory-eval -- retrieval --fetch # full 500+cargo run -p ai-memory-eval -- retrieval --sample 10 # smoke+```++Output: a per-category table on stdout plus+`evals/runs/<timestamp>-retrieval/report.{json,md}` with full+provenance (commit, dataset sha, hardware, per-question scores).+When publishing a baseline, copy the markdown into `docs/benchmarks/`.++Honest-numbers notes:++- capture is production-shaped: excerpts are bounded at the 2 KB+ privacy boundary, so evidence buried deep inside one long turn is+ genuinely out of reach — that is the system being measured, not a+ harness artifact;+- each session's original date is prepended to its turn text, since a+ replayed history must carry its own timestamps (the official+ harness exposes the same information to retrievers);+- competitors' published numbers (e.g. agentmemory 0.967 R@5,+ mcp-memory-service 0.804 R@5) are answer-evidence Recall@5 on this+ same v1 dataset, comparable to our `hit@5`.diff --git a/evals/src/ab.rs b/evals/src/ab.rsnew file mode 100644index 00000000..a7960404--- /dev/null+++ b/evals/src/ab.rs@@ -0,0 +1,605 @@+//! Live A/B harness for the ai-memory consolidation prompt.+//!+//! ## What it does+//!+//! For each fixture under `evals/fixtures/*.json` (each one a small,+//! synthetic session log), the runner:+//!+//! 1. Calls [`ai_memory_consolidate::build_batch_request`] to build+//! the EXACT ChatRequest the production consolidator would send.+//! 2. Sends that request to two providers concurrently — a+//! *baseline* and a *candidate* — via+//! [`ai_memory_llm::complete_structured`], which is the same+//! parse-extract-and-validate path the live system uses.+//! 3. Saves the deserialised `ConsolidatedBatch`, the raw JSON, a+//! flattened markdown rendering, and a `meta.json` (timing,+//! update count, parse status) per fixture per provider under+//! `evals/runs/<timestamp>/{baseline,candidate}/`.+//! 4. Prints a side-by-side summary table to stdout.+//!+//! Quality judgement is left to the human reading the outputs — the+//! harness only reports the OBJECTIVE deltas (latency, update+//! counts, schema-parse success). Anything subtler (faithfulness,+//! hallucination, structure) is for you to eyeball.+//!+//! ## Running it+//!+//! See `evals/README.md` for canonical invocations. The short+//! version:+//!+//! ```bash+//! cargo run -p ai-memory-eval -- ab \+//! --baseline-provider openai-compat \+//! --baseline-base-url https://openrouter.ai/api/v1 \+//! --baseline-model moonshotai/kimi-k2.6 \+//! --baseline-api-key-env OPENROUTER_API_KEY \+//! --candidate-provider openai-compat \+//! --candidate-base-url http://192.168.0.90:11434/v1 \+//! --candidate-model qwen3:32b \+//! --candidate-api-key ollama-local+//! ```++use std::io::Write as _;+use std::path::{Path, PathBuf};+use std::sync::Arc;+use std::time::{Duration, Instant};++use anyhow::{Context, Result, bail};+use jiff::Timestamp;+use secrecy::SecretString;+use serde::{Deserialize, Serialize};++use ai_memory_consolidate::{ConsolidatedBatch, build_batch_request};+use ai_memory_core::{Observation, ObservationKind, ProjectId, SessionId, WorkspaceId};+use ai_memory_llm::{AuthRequirement, LlmProvider, ProviderAuth, ProviderChoice, ProviderConfig};++#[derive(clap::Args, Debug)]+pub struct AbArgs {+ /// Fixture directory (each `*.json` is one synthetic session log).+ #[arg(long, default_value = "evals/fixtures")]+ fixtures: PathBuf,++ /// Output root. A timestamped subdir gets created here per run.+ #[arg(long, default_value = "evals/runs")]+ out: PathBuf,++ /// Baseline provider (`anthropic` / `openai` / `openai-compat` / `openai-oauth` / `copilot`).+ #[arg(long)]+ baseline_provider: String,++ /// Baseline base URL (omit for native Anthropic/OpenAI).+ #[arg(long)]+ baseline_base_url: Option<String>,++ /// Baseline model id.+ #[arg(long)]+ baseline_model: String,++ /// Baseline API key (raw value). Prefer `--baseline-api-key-env`.+ #[arg(long)]+ baseline_api_key: Option<String>,++ /// Env var to read the baseline API key from.+ #[arg(long)]+ baseline_api_key_env: Option<String>,++ /// Baseline OpenAI OAuth token file for `--baseline-provider openai-oauth`.+ #[arg(long)]+ baseline_token_file: Option<PathBuf>,++ /// Candidate provider.+ #[arg(long)]+ candidate_provider: String,++ /// Candidate base URL.+ #[arg(long)]+ candidate_base_url: Option<String>,++ /// Candidate model id.+ #[arg(long)]+ candidate_model: String,++ /// Candidate API key (raw value).+ #[arg(long)]+ candidate_api_key: Option<String>,++ /// Env var to read the candidate API key from.+ #[arg(long)]+ candidate_api_key_env: Option<String>,++ /// Candidate OpenAI OAuth token file for `--candidate-provider openai-oauth`.+ #[arg(long)]+ candidate_token_file: Option<PathBuf>,+}++/// JSON shape of an eval fixture file. We only care about+/// observations the consolidation prompt actually consumes; the+/// other Observation fields get populated with dummies at runtime.+#[derive(Debug, Deserialize)]+struct Fixture {+ name: String,+ /// Free-text describing the scenario; not consumed by the runner+ /// but useful when reading fixture files by hand.+ #[allow(dead_code)]+ #[serde(default)]+ description: String,+ observations: Vec<FixtureObservation>,+}++#[derive(Debug, Deserialize)]+struct FixtureObservation {+ kind: String,+ title: String,+ #[serde(default)]+ body: String,+}++/// One result of one provider on one fixture.+#[derive(Debug, Serialize)]+struct CaseRun {+ provider: String,+ fixture: String,+ elapsed_ms: u128,+ parsed_ok: bool,+ error: Option<String>,+ update_count: Option<usize>,+}++pub async fn run(args: AbArgs) -> Result<()> {+ let baseline = resolve_provider("baseline", &args)?;+ let candidate = resolve_provider("candidate", &args)?;++ let baseline_provider =+ make_provider(baseline.clone()).context("building baseline provider")?;+ let candidate_provider =+ make_provider(candidate.clone()).context("building candidate provider")?;++ let fixtures = load_fixtures(&args.fixtures)+ .with_context(|| format!("loading fixtures from {}", args.fixtures.display()))?;+ if fixtures.is_empty() {+ bail!("no *.json fixtures found in {}", args.fixtures.display());+ }++ // Timestamped run dir; filesystem-safe (colons + dots → dashes).+ let stamp: String = Timestamp::now()+ .to_string()+ .chars()+ .map(|c| if c == ':' || c == '.' { '-' } else { c })+ .collect();+ let run_dir = args.out.join(&stamp);+ std::fs::create_dir_all(run_dir.join("baseline"))?;+ std::fs::create_dir_all(run_dir.join("candidate"))?;++ println!();+ println!("ai-memory eval — {stamp}");+ println!(+ " fixtures : {} ({})",+ fixtures.len(),+ args.fixtures.display()+ );+ println!(+ " baseline : {} {} @ {}",+ baseline.provider_name(),+ baseline.model,+ baseline.base_url.as_deref().unwrap_or("(default)")+ );+ println!(+ " candidate: {} {} @ {}",+ candidate.provider_name(),+ candidate.model,+ candidate.base_url.as_deref().unwrap_or("(default)")+ );+ println!(" out : {}", run_dir.display());+ println!();++ let mut results: Vec<(CaseRun, CaseRun)> = Vec::with_capacity(fixtures.len());+ for (path, fx) in fixtures {+ println!("→ {} ({} observations)", fx.name, fx.observations.len());+ let observations = synthesise_observations(&fx);+ let session_id = SessionId::new();++ // Run both providers concurrently per fixture so a slow one+ // doesn't double the wall-clock.+ let req_a = build_batch_request(session_id, &observations);+ let req_b = build_batch_request(session_id, &observations);+ let (a_res, b_res) = tokio::join!(+ run_one(baseline_provider.clone(), "baseline", req_a),+ run_one(candidate_provider.clone(), "candidate", req_b),+ );++ let a = persist_case("baseline", &fx.name, &path, &run_dir, a_res)?;+ let b = persist_case("candidate", &fx.name, &path, &run_dir, b_res)?;++ println!(+ " baseline : {:>5} ms parsed={} updates={}",+ a.elapsed_ms,+ a.parsed_ok,+ a.update_count.map_or("-".into(), |n| n.to_string())+ );+ println!(+ " candidate: {:>5} ms parsed={} updates={}",+ b.elapsed_ms,+ b.parsed_ok,+ b.update_count.map_or("-".into(), |n| n.to_string())+ );+ // Force-flush per fixture so progress is visible when stdout+ // is piped (e.g. via `tee` or backgrounded runs). Without+ // this, full-buffering swallows every println! until exit.+ std::io::stdout().flush().ok();+ results.push((a, b));+ }++ // Tail summary so the user can grok the run at a glance.+ println!();+ println!("=== summary ===");+ let n = results.len();+ let baseline_total: u128 = results.iter().map(|(a, _)| a.elapsed_ms).sum();+ let candidate_total: u128 = results.iter().map(|(_, b)| b.elapsed_ms).sum();+ let baseline_ok = results.iter().filter(|(a, _)| a.parsed_ok).count();+ let candidate_ok = results.iter().filter(|(_, b)| b.parsed_ok).count();+ println!(+ "baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms",+ baseline_total / n.max(1) as u128+ );+ println!(+ "candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms",+ candidate_total / n.max(1) as u128+ );+ println!();+ println!("outputs under: {}", run_dir.display());+ println!(+ "compare with: diff -ru {}/baseline {}/candidate",+ run_dir.display(),+ run_dir.display()+ );+ Ok(())+}++/// Resolve `ProviderConfig` for one side from CLI args + env.+fn resolve_provider(side: &str, args: &AbArgs) -> Result<ResolvedConfig> {+ let (provider_str, base_url, model, api_key_inline, api_key_env, token_file) = match side {+ "baseline" => (+ args.baseline_provider.as_str(),+ args.baseline_base_url.clone(),+ args.baseline_model.clone(),+ args.baseline_api_key.clone(),+ args.baseline_api_key_env.clone(),+ args.baseline_token_file.clone(),+ ),+ "candidate" => (+ args.candidate_provider.as_str(),+ args.candidate_base_url.clone(),+ args.candidate_model.clone(),+ args.candidate_api_key.clone(),+ args.candidate_api_key_env.clone(),+ args.candidate_token_file.clone(),+ ),+ _ => bail!("unknown side {side}"),+ };++ let provider = match provider_str {+ "anthropic" => ProviderChoice::Anthropic,+ "openai" => ProviderChoice::OpenAi,+ "openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,+ "openai-oauth" | "openai_oauth" => ProviderChoice::OpenAiOAuth,+ "copilot" | "github-copilot" | "github_copilot" => ProviderChoice::Copilot,+ other => {+ bail!(+ "{side}: provider {other} not one of anthropic|openai|openai-compat|openai-oauth|copilot"+ )+ }+ };++ let api_key = match (api_key_inline, api_key_env) {+ (Some(s), _) if !s.is_empty() => Some(SecretString::from(s)),+ (_, Some(env)) => match std::env::var(&env) {+ Ok(v) if !v.is_empty() => Some(SecretString::from(v)),+ _ => bail!("{side}: env var {env} is not set or empty"),+ },+ _ => None,+ };++ let auth = match provider.auth_requirement() {+ AuthRequirement::RequiredApiKey { env_var } => {+ ProviderAuth::required_api_key_from_env(env_var, api_key)+ }+ AuthRequirement::OptionalApiKey { env_var } => {+ ProviderAuth::optional_api_key_from_env(env_var, api_key)+ }+ AuthRequirement::OpenAiOAuthToken => {+ ProviderAuth::openai_oauth_token_file(token_file.ok_or_else(|| {+ anyhow::anyhow!("{side}: --{side}-token-file is required for openai-oauth")+ })?)+ }+ AuthRequirement::CopilotToken => ProviderAuth::copilot(+ token_file.ok_or_else(|| {+ anyhow::anyhow!("{side}: --{side}-token-file is required for copilot")+ })?,+ None,+ None,+ base_url.clone(),+ ),+ AuthRequirement::AnthropicOAuthToken => ProviderAuth::anthropic_oauth_token(api_key),+ };++ Ok(ResolvedConfig {+ provider,+ provider_str: provider_str.to_string(),+ model,+ base_url,+ auth,+ })+}++#[derive(Clone)]+struct ResolvedConfig {+ provider: ProviderChoice,+ provider_str: String,+ model: String,+ base_url: Option<String>,+ auth: ProviderAuth,+}++impl ResolvedConfig {+ fn provider_name(&self) -> &str {+ self.provider_str.as_str()+ }+}++impl From<ResolvedConfig> for ProviderConfig {+ fn from(r: ResolvedConfig) -> Self {+ Self {+ provider: r.provider,+ model: r.model,+ auth: r.auth,+ base_url: r.base_url,+ // Match the product default so provider comparisons exercise the+ // same schema-constrained path operators receive.+ compat_strict: true,+ request_timeout_secs: ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS,+ }+ }+}++fn make_provider(r: ResolvedConfig) -> Result<Arc<dyn LlmProvider>> {+ ai_memory_llm::build_provider(ProviderConfig::from(r))+ .map_err(anyhow::Error::from)+ .context("building provider")+}++fn load_fixtures(dir: &Path) -> Result<Vec<(PathBuf, Fixture)>> {+ let mut paths: Vec<_> = std::fs::read_dir(dir)?+ .filter_map(|e| e.ok().map(|e| e.path()))+ .filter(|p| p.extension().is_some_and(|e| e == "json"))+ .collect();+ paths.sort();+ let mut out = Vec::with_capacity(paths.len());+ for p in paths {+ let bytes = std::fs::read(&p).with_context(|| format!("reading {}", p.display()))?;+ let fx: Fixture =+ serde_json::from_slice(&bytes).with_context(|| format!("parsing {}", p.display()))?;+ out.push((p, fx));+ }+ Ok(out)+}++fn synthesise_observations(fx: &Fixture) -> Vec<Observation> {+ let workspace_id = WorkspaceId::new();+ let project_id = ProjectId::new();+ let session_id = SessionId::new();+ fx.observations+ .iter()+ .map(|f| Observation {+ id: ai_memory_core::ObservationId::new(),+ session_id,+ workspace_id,+ project_id,+ kind: f+ .kind+ .parse::<ObservationKind>()+ .unwrap_or(ObservationKind::Other),+ extension: None,+ source_event: None,+ title: f.title.clone(),+ body: f.body.clone(),+ importance: 5,+ created_at: Timestamp::now(),+ })+ .collect()+}++struct ProviderResult {+ elapsed: Duration,+ /// Raw text the model emitted — always saved, even when parsing+ /// fails. This is the load-bearing artifact for debugging+ /// schema mismatches: you can see exactly what the model said.+ raw_text: String,+ /// Parsed-from-text JSON, if the response was valid JSON at all.+ raw_json: Option<serde_json::Value>,+ /// Schema-validated batch, if the JSON also satisfied the schema.+ batch: Option<ConsolidatedBatch>,+ error: Option<String>,+}++async fn run_one(+ provider: Arc<dyn LlmProvider>,+ side: &str,+ request: ai_memory_llm::ChatRequest,+) -> ProviderResult {+ let start = Instant::now();+ // Call `complete` directly — bypassing `complete_structured` —+ // so we can capture the raw text the model emitted, even if it+ // doesn't deserialise. The cost is that we re-implement the+ // "extract first balanced {…}" fallback locally, but knowing+ // exactly what the model said is what makes the harness useful.+ let chat_res = provider.complete(request).await;+ let elapsed = start.elapsed();+ let raw_text = match &chat_res {+ Ok(r) => r.text.clone(),+ Err(e) => format!("(provider error before any response: {e})"),+ };+ let (raw_json, batch, error) = match chat_res {+ Ok(_) => parse_response(&raw_text),+ Err(e) => (None, None, Some(e.to_string())),+ };+ if let Some(err) = &error {+ tracing::warn!(side, error = %err, "model output did not satisfy schema");+ } else {+ tracing::info!(+ side,+ ms = elapsed.as_millis() as u64,+ updates = batch.as_ref().map_or(0, |b| b.updates.len()),+ "parsed ok"+ );+ }+ ProviderResult {+ elapsed,+ raw_text,+ raw_json,+ batch,+ error,+ }+}++/// Parse a model response into (json, batch, error).+///+/// 1. Try the whole response as JSON.+/// 2. If that fails, search for the first balanced `{…}` block.+/// 3. If we have JSON, try to deserialise as `ConsolidatedBatch`.+fn parse_response(+ text: &str,+) -> (+ Option<serde_json::Value>,+ Option<ConsolidatedBatch>,+ Option<String>,+) {+ let json = serde_json::from_str::<serde_json::Value>(text.trim())+ .ok()+ .or_else(|| {+ extract_first_balanced_object(text).and_then(|s| serde_json::from_str(&s).ok())+ });+ let Some(json) = json else {+ return (None, None, Some("response is not valid JSON".into()));+ };+ match serde_json::from_value::<ConsolidatedBatch>(json.clone()) {+ Ok(batch) => (Some(json), Some(batch), None),+ Err(e) => (Some(json), None, Some(format!("schema mismatch: {e}"))),+ }+}++/// Best-effort `{…}` extractor — ignores braces inside strings.+fn extract_first_balanced_object(text: &str) -> Option<String> {+ let bytes = text.as_bytes();+ let start = bytes.iter().position(|&b| b == b'{')?;+ let mut depth = 0i32;+ let mut in_string = false;+ let mut escape = false;+ for (i, &b) in bytes.iter().enumerate().skip(start) {+ if in_string {+ if escape {+ escape = false;+ } else if b == b'\\' {+ escape = true;+ } else if b == b'"' {+ in_string = false;+ }+ continue;+ }+ if b == b'"' {+ in_string = true;+ continue;+ }+ if b == b'{' {+ depth += 1;+ } else if b == b'}' {+ depth -= 1;+ if depth == 0 {+ return Some(text[start..=i].to_string());+ }+ }+ }+ None+}++fn persist_case(+ side: &str,+ fixture_name: &str,+ fixture_path: &Path,+ run_dir: &Path,+ res: ProviderResult,+) -> Result<CaseRun> {+ let safe_name = fixture_path+ .file_stem()+ .and_then(|s| s.to_str())+ .unwrap_or(fixture_name);+ let dir = run_dir.join(side);+ std::fs::create_dir_all(&dir)?;++ // 1. Raw model text — always saved, even on parse failure.+ // This is the load-bearing debug artifact: a side-by-side+ // diff of these files tells you exactly what each model said.+ let raw_path = dir.join(format!("{safe_name}.raw.txt"));+ std::fs::write(&raw_path, &res.raw_text)?;++ // 2. Parsed JSON (if response was valid JSON at all).+ if let Some(json) = &res.raw_json {+ let json_path = dir.join(format!("{safe_name}.json"));+ std::fs::write(&json_path, serde_json::to_string_pretty(json)?)?;+ }++ // 3. Flattened markdown rendering for easy eyeballing.+ let md_path = dir.join(format!("{safe_name}.md"));+ let md = render_markdown(side, fixture_name, &res);+ std::fs::write(&md_path, md)?;++ // 4. Per-case meta.+ let case = CaseRun {+ provider: side.to_string(),+ fixture: fixture_name.to_string(),+ elapsed_ms: res.elapsed.as_millis(),+ parsed_ok: res.batch.is_some(),+ error: res.error.clone(),+ update_count: res.batch.as_ref().map(|b| b.updates.len()),+ };+ let meta_path = dir.join(format!("{safe_name}.meta.json"));+ std::fs::write(&meta_path, serde_json::to_string_pretty(&case)?)?;+ Ok(case)+}++fn render_markdown(side: &str, fixture_name: &str, res: &ProviderResult) -> String {+ let mut out = String::new();+ out.push_str(&format!("# {side} — {fixture_name}\n\n"));+ out.push_str(&format!("- elapsed: {} ms\n", res.elapsed.as_millis()));+ if let Some(err) = &res.error {+ out.push_str(&format!("- error: `{err}`\n"));+ }+ // Always include the raw response so the human reviewer can+ // judge structure/style even when schema validation failed.+ if !res.raw_text.is_empty() {+ out.push_str("\n## Raw response\n\n```\n");+ out.push_str(&res.raw_text);+ out.push_str("\n```\n");+ }+ let Some(batch) = &res.batch else {+ return out;+ };+ out.push_str(&format!("\n- updates: {}\n", batch.updates.len()));+ if !batch.rationale.trim().is_empty() {+ out.push_str("\n## Rationale\n\n");+ out.push_str(batch.rationale.trim());+ out.push('\n');+ }+ for (i, u) in batch.updates.iter().enumerate() {+ out.push_str(&format!("\n## Update {} — `{}`\n\n", i + 1, u.path));+ out.push_str(&format!("**Title:** {} \n", u.title));+ out.push_str(&format!("**Kind:** `{}` \n", u.kind.as_str()));+ out.push_str(&format!("**Tier:** `{}` \n", u.tier.as_str()));+ if !u.tags.is_empty() {+ out.push_str(&format!("**Tags:** {} \n", u.tags.join(", ")));+ }+ out.push('\n');+ out.push_str(u.body_markdown.trim());+ out.push('\n');+ }+ out+}diff --git a/evals/src/main.rs b/evals/src/main.rsindex 5c00b4f7..ca38b7cb 100644--- a/evals/src/main.rs+++ b/evals/src/main.rs@@ -1,153 +1,37 @@-//! Live A/B harness for the ai-memory consolidation prompt.+//! `ai-memory-eval` — offline evaluation harnesses for ai-memory.//!-//! ## What it does+//! Two subcommands://!-//! For each fixture under `evals/fixtures/*.json` (each one a small,-//! synthetic session log), the runner:-//!-//! 1. Calls [`ai_memory_consolidate::build_batch_request`] to build-//! the EXACT ChatRequest the production consolidator would send.-//! 2. Sends that request to two providers concurrently — a-//! *baseline* and a *candidate* — via-//! [`ai_memory_llm::complete_structured`], which is the same-//! parse-extract-and-validate path the live system uses.-//! 3. Saves the deserialised `ConsolidatedBatch`, the raw JSON, a-//! flattened markdown rendering, and a `meta.json` (timing,-//! update count, parse status) per fixture per provider under-//! `evals/runs/<timestamp>/{baseline,candidate}/`.-//! 4. Prints a side-by-side summary table to stdout.-//!-//! Quality judgement is left to the human reading the outputs — the-//! harness only reports the OBJECTIVE deltas (latency, update-//! counts, schema-parse success). Anything subtler (faithfulness,-//! hallucination, structure) is for you to eyeball.-//!-//! ## Running it-//!-//! See `evals/README.md` for canonical invocations. The short-//! version:-//!-//! ```bash-//! cargo run -p ai-memory-eval -- \-//! --baseline-provider openai-compat \-//! --baseline-base-url https://openrouter.ai/api/v1 \-//! --baseline-model moonshotai/kimi-k2.6 \-//! --baseline-api-key-env OPENROUTER_API_KEY \-//! --candidate-provider openai-compat \-//! --candidate-base-url http://192.168.0.90:11434/v1 \-//! --candidate-model qwen3:32b \-//! --candidate-api-key ollama-local-//! ```+//! - `ab` — live A/B comparison of two LLM providers on the production+//! consolidation prompt (see [`ab`] and `evals/README.md`).+//! - `retrieval` — LongMemEval retrieval benchmark driven end-to-end+//! through a real `ai-memory serve` subprocess: hook-shaped ingestion,+//! real search stack, R@k / hit@k per question category (see+//! [`retrieval`] and `docs/benchmarks/`).-use std::io::Write as _;-use std::path::{Path, PathBuf};-use std::sync::Arc;-use std::time::{Duration, Instant};+mod ab;+mod retrieval;-use anyhow::{Context, Result, bail};-use clap::Parser;-use jiff::Timestamp;-use secrecy::SecretString;-use serde::{Deserialize, Serialize};--use ai_memory_consolidate::{ConsolidatedBatch, build_batch_request};-use ai_memory_core::{Observation, ObservationKind, ProjectId, SessionId, WorkspaceId};-use ai_memory_llm::{AuthRequirement, LlmProvider, ProviderAuth, ProviderChoice, ProviderConfig};+use anyhow::Result;+use clap::{Parser, Subcommand};#[derive(Parser, Debug)]#[command(name = "ai-memory-eval",- about = "Live A/B harness comparing two LLM providers on the ai-memory consolidation prompt"+ about = "Evaluation harnesses for ai-memory (LLM A/B + retrieval benchmark)")]-struct Args {- /// Fixture directory (each `*.json` is one synthetic session log).- #[arg(long, default_value = "evals/fixtures")]- fixtures: PathBuf,-- /// Output root. A timestamped subdir gets created here per run.- #[arg(long, default_value = "evals/runs")]- out: PathBuf,-- /// Baseline provider (`anthropic` / `openai` / `openai-compat` / `openai-oauth` / `copilot`).- #[arg(long)]- baseline_provider: String,-- /// Baseline base URL (omit for native Anthropic/OpenAI).- #[arg(long)]- baseline_base_url: Option<String>,-- /// Baseline model id.- #[arg(long)]- baseline_model: String,-- /// Baseline API key (raw value). Prefer `--baseline-api-key-env`.- #[arg(long)]- baseline_api_key: Option<String>,-- /// Env var to read the baseline API key from.- #[arg(long)]- baseline_api_key_env: Option<String>,-- /// Baseline OpenAI OAuth token file for `--baseline-provider openai-oauth`.- #[arg(long)]- baseline_token_file: Option<PathBuf>,-- /// Candidate provider.- #[arg(long)]- candidate_provider: String,-- /// Candidate base URL.- #[arg(long)]- candidate_base_url: Option<String>,-- /// Candidate model id.- #[arg(long)]- candidate_model: String,-- /// Candidate API key (raw value).- #[arg(long)]- candidate_api_key: Option<String>,-- /// Env var to read the candidate API key from.- #[arg(long)]- candidate_api_key_env: Option<String>,-- /// Candidate OpenAI OAuth token file for `--candidate-provider openai-oauth`.- #[arg(long)]- candidate_token_file: Option<PathBuf>,-}--/// JSON shape of an eval fixture file. We only care about-/// observations the consolidation prompt actually consumes; the-/// other Observation fields get populated with dummies at runtime.-#[derive(Debug, Deserialize)]-struct Fixture {- name: String,- /// Free-text describing the scenario; not consumed by the runner- /// but useful when reading fixture files by hand.- #[allow(dead_code)]- #[serde(default)]- description: String,- observations: Vec<FixtureObservation>,-}--#[derive(Debug, Deserialize)]-struct FixtureObservation {- kind: String,- title: String,- #[serde(default)]- body: String,+struct Cli {+ #[command(subcommand)]+ command: Command,}-/// One result of one provider on one fixture.-#[derive(Debug, Serialize)]-struct CaseRun {- provider: String,- fixture: String,- elapsed_ms: u128,- parsed_ok: bool,- error: Option<String>,- update_count: Option<usize>,+#[derive(Subcommand, Debug)]+enum Command {+ /// Live A/B harness comparing two LLM providers on the ai-memory+ /// consolidation prompt.+ Ab(Box<ab::AbArgs>),+ /// LongMemEval retrieval benchmark through the real server stack.+ Retrieval(retrieval::RetrievalArgs),}#[tokio::main]@@ -159,462 +43,8 @@ async fn main() -> Result<()> {).init();- let args = Args::parse();-- let baseline = resolve_provider("baseline", &args)?;- let candidate = resolve_provider("candidate", &args)?;-- let baseline_provider =- make_provider(baseline.clone()).context("building baseline provider")?;- let candidate_provider =- make_provider(candidate.clone()).context("building candidate provider")?;-- let fixtures = load_fixtures(&args.fixtures)- .with_context(|| format!("loading fixtures from {}", args.fixtures.display()))?;- if fixtures.is_empty() {- bail!("no *.json fixtures found in {}", args.fixtures.display());- }-- // Timestamped run dir; filesystem-safe (colons + dots → dashes).- let stamp: String = Timestamp::now()- .to_string()- .chars()- .map(|c| if c == ':' || c == '.' { '-' } else { c })- .collect();- let run_dir = args.out.join(&stamp);- std::fs::create_dir_all(run_dir.join("baseline"))?;- std::fs::create_dir_all(run_dir.join("candidate"))?;-- println!();- println!("ai-memory eval — {stamp}");- println!(- " fixtures : {} ({})",- fixtures.len(),- args.fixtures.display()- );- println!(- " baseline : {} {} @ {}",- baseline.provider_name(),- baseline.model,- baseline.base_url.as_deref().unwrap_or("(default)")- );- println!(- " candidate: {} {} @ {}",- candidate.provider_name(),- candidate.model,- candidate.base_url.as_deref().unwrap_or("(default)")- );- println!(" out : {}", run_dir.display());- println!();-- let mut results: Vec<(CaseRun, CaseRun)> = Vec::with_capacity(fixtures.len());- for (path, fx) in fixtures {- println!("→ {} ({} observations)", fx.name, fx.observations.len());- let observations = synthesise_observations(&fx);- let session_id = SessionId::new();-- // Run both providers concurrently per fixture so a slow one- // doesn't double the wall-clock.- let req_a = build_batch_request(session_id, &observations);- let req_b = build_batch_request(session_id, &observations);- let (a_res, b_res) = tokio::join!(- run_one(baseline_provider.clone(), "baseline", req_a),- run_one(candidate_provider.clone(), "candidate", req_b),- );-- let a = persist_case("baseline", &fx.name, &path, &run_dir, a_res)?;- let b = persist_case("candidate", &fx.name, &path, &run_dir, b_res)?;-- println!(- " baseline : {:>5} ms parsed={} updates={}",- a.elapsed_ms,- a.parsed_ok,- a.update_count.map_or("-".into(), |n| n.to_string())- );- println!(- " candidate: {:>5} ms parsed={} updates={}",- b.elapsed_ms,- b.parsed_ok,- b.update_count.map_or("-".into(), |n| n.to_string())- );- // Force-flush per fixture so progress is visible when stdout- // is piped (e.g. via `tee` or backgrounded runs). Without- // this, full-buffering swallows every println! until exit.- std::io::stdout().flush().ok();- results.push((a, b));- }-- // Tail summary so the user can grok the run at a glance.- println!();- println!("=== summary ===");- let n = results.len();- let baseline_total: u128 = results.iter().map(|(a, _)| a.elapsed_ms).sum();- let candidate_total: u128 = results.iter().map(|(_, b)| b.elapsed_ms).sum();- let baseline_ok = results.iter().filter(|(a, _)| a.parsed_ok).count();- let candidate_ok = results.iter().filter(|(_, b)| b.parsed_ok).count();- println!(- "baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms",- baseline_total / n.max(1) as u128- );- println!(- "candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms",- candidate_total / n.max(1) as u128- );- println!();- println!("outputs under: {}", run_dir.display());- println!(- "compare with: diff -ru {}/baseline {}/candidate",- run_dir.display(),- run_dir.display()- );- Ok(())-}--/// Resolve `ProviderConfig` for one side from CLI args + env.-fn resolve_provider(side: &str, args: &Args) -> Result<ResolvedConfig> {- let (provider_str, base_url, model, api_key_inline, api_key_env, token_file) = match side {- "baseline" => (- args.baseline_provider.as_str(),- args.baseline_base_url.clone(),- args.baseline_model.clone(),- args.baseline_api_key.clone(),- args.baseline_api_key_env.clone(),- args.baseline_token_file.clone(),- ),- "candidate" => (- args.candidate_provider.as_str(),- args.candidate_base_url.clone(),- args.candidate_model.clone(),- args.candidate_api_key.clone(),- args.candidate_api_key_env.clone(),- args.candidate_token_file.clone(),- ),- _ => bail!("unknown side {side}"),- };-- let provider = match provider_str {- "anthropic" => ProviderChoice::Anthropic,- "openai" => ProviderChoice::OpenAi,- "openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,- "openai-oauth" | "openai_oauth" => ProviderChoice::OpenAiOAuth,- "copilot" | "github-copilot" | "github_copilot" => ProviderChoice::Copilot,- other => {- bail!(- "{side}: provider {other} not one of anthropic|openai|openai-compat|openai-oauth|copilot"- )- }- };-- let api_key = match (api_key_inline, api_key_env) {- (Some(s), _) if !s.is_empty() => Some(SecretString::from(s)),- (_, Some(env)) => match std::env::var(&env) {- Ok(v) if !v.is_empty() => Some(SecretString::from(v)),- _ => bail!("{side}: env var {env} is not set or empty"),- },- _ => None,- };-- let auth = match provider.auth_requirement() {- AuthRequirement::RequiredApiKey { env_var } => {- ProviderAuth::required_api_key_from_env(env_var, api_key)- }- AuthRequirement::OptionalApiKey { env_var } => {- ProviderAuth::optional_api_key_from_env(env_var, api_key)- }- AuthRequirement::OpenAiOAuthToken => {- ProviderAuth::openai_oauth_token_file(token_file.ok_or_else(|| {- anyhow::anyhow!("{side}: --{side}-token-file is required for openai-oauth")- })?)- }- AuthRequirement::CopilotToken => ProviderAuth::copilot(- token_file.ok_or_else(|| {- anyhow::anyhow!("{side}: --{side}-token-file is required for copilot")- })?,- None,- None,- base_url.clone(),- ),- AuthRequirement::AnthropicOAuthToken => ProviderAuth::anthropic_oauth_token(api_key),- };-- Ok(ResolvedConfig {- provider,- provider_str: provider_str.to_string(),- model,- base_url,- auth,- })-}--#[derive(Clone)]-struct ResolvedConfig {- provider: ProviderChoice,- provider_str: String,- model: String,- base_url: Option<String>,- auth: ProviderAuth,-}--impl ResolvedConfig {- fn provider_name(&self) -> &str {- self.provider_str.as_str()- }-}--impl From<ResolvedConfig> for ProviderConfig {- fn from(r: ResolvedConfig) -> Self {- Self {- provider: r.provider,- model: r.model,- auth: r.auth,- base_url: r.base_url,- // Match the product default so provider comparisons exercise the- // same schema-constrained path operators receive.- compat_strict: true,- request_timeout_secs: ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS,- }- }-}--fn make_provider(r: ResolvedConfig) -> Result<Arc<dyn LlmProvider>> {- ai_memory_llm::build_provider(ProviderConfig::from(r))- .map_err(anyhow::Error::from)- .context("building provider")-}--fn load_fixtures(dir: &Path) -> Result<Vec<(PathBuf, Fixture)>> {- let mut paths: Vec<_> = std::fs::read_dir(dir)?- .filter_map(|e| e.ok().map(|e| e.path()))- .filter(|p| p.extension().is_some_and(|e| e == "json"))- .collect();- paths.sort();- let mut out = Vec::with_capacity(paths.len());- for p in paths {- let bytes = std::fs::read(&p).with_context(|| format!("reading {}", p.display()))?;- let fx: Fixture =- serde_json::from_slice(&bytes).with_context(|| format!("parsing {}", p.display()))?;- out.push((p, fx));- }- Ok(out)-}--fn synthesise_observations(fx: &Fixture) -> Vec<Observation> {- let workspace_id = WorkspaceId::new();- let project_id = ProjectId::new();- let session_id = SessionId::new();- fx.observations- .iter()- .map(|f| Observation {- id: ai_memory_core::ObservationId::new(),- session_id,- workspace_id,- project_id,- kind: f- .kind- .parse::<ObservationKind>()- .unwrap_or(ObservationKind::Other),- extension: None,- source_event: None,- title: f.title.clone(),- body: f.body.clone(),- importance: 5,- created_at: Timestamp::now(),- })- .collect()-}--struct ProviderResult {- elapsed: Duration,- /// Raw text the model emitted — always saved, even when parsing- /// fails. This is the load-bearing artifact for debugging- /// schema mismatches: you can see exactly what the model said.- raw_text: String,- /// Parsed-from-text JSON, if the response was valid JSON at all.- raw_json: Option<serde_json::Value>,- /// Schema-validated batch, if the JSON also satisfied the schema.- batch: Option<ConsolidatedBatch>,- error: Option<String>,-}--async fn run_one(- provider: Arc<dyn LlmProvider>,- side: &str,- request: ai_memory_llm::ChatRequest,-) -> ProviderResult {- let start = Instant::now();- // Call `complete` directly — bypassing `complete_structured` —- // so we can capture the raw text the model emitted, even if it- // doesn't deserialise. The cost is that we re-implement the- // "extract first balanced {…}" fallback locally, but knowing- // exactly what the model said is what makes the harness useful.- let chat_res = provider.complete(request).await;- let elapsed = start.elapsed();- let raw_text = match &chat_res {- Ok(r) => r.text.clone(),- Err(e) => format!("(provider error before any response: {e})"),- };- let (raw_json, batch, error) = match chat_res {- Ok(_) => parse_response(&raw_text),- Err(e) => (None, None, Some(e.to_string())),- };- if let Some(err) = &error {- tracing::warn!(side, error = %err, "model output did not satisfy schema");- } else {- tracing::info!(- side,- ms = elapsed.as_millis() as u64,- updates = batch.as_ref().map_or(0, |b| b.updates.len()),- "parsed ok"- );- }- ProviderResult {- elapsed,- raw_text,- raw_json,- batch,- error,- }-}--/// Parse a model response into (json, batch, error).-///-/// 1. Try the whole response as JSON.-/// 2. If that fails, search for the first balanced `{…}` block.-/// 3. If we have JSON, try to deserialise as `ConsolidatedBatch`.-fn parse_response(- text: &str,-) -> (- Option<serde_json::Value>,- Option<ConsolidatedBatch>,- Option<String>,-) {- let json = serde_json::from_str::<serde_json::Value>(text.trim())- .ok()- .or_else(|| {- extract_first_balanced_object(text).and_then(|s| serde_json::from_str(&s).ok())- });- let Some(json) = json else {- return (None, None, Some("response is not valid JSON".into()));- };- match serde_json::from_value::<ConsolidatedBatch>(json.clone()) {- Ok(batch) => (Some(json), Some(batch), None),- Err(e) => (Some(json), None, Some(format!("schema mismatch: {e}"))),- }-}--/// Best-effort `{…}` extractor — ignores braces inside strings.-fn extract_first_balanced_object(text: &str) -> Option<String> {- let bytes = text.as_bytes();- let start = bytes.iter().position(|&b| b == b'{')?;- let mut depth = 0i32;- let mut in_string = false;- let mut escape = false;- for (i, &b) in bytes.iter().enumerate().skip(start) {- if in_string {- if escape {- escape = false;- } else if b == b'\\' {- escape = true;- } else if b == b'"' {- in_string = false;- }- continue;- }- if b == b'"' {- in_string = true;- continue;- }- if b == b'{' {- depth += 1;- } else if b == b'}' {- depth -= 1;- if depth == 0 {- return Some(text[start..=i].to_string());- }- }- }- None-}--fn persist_case(- side: &str,- fixture_name: &str,- fixture_path: &Path,- run_dir: &Path,- res: ProviderResult,-) -> Result<CaseRun> {- let safe_name = fixture_path- .file_stem()- .and_then(|s| s.to_str())- .unwrap_or(fixture_name);- let dir = run_dir.join(side);- std::fs::create_dir_all(&dir)?;-- // 1. Raw model text — always saved, even on parse failure.- // This is the load-bearing debug artifact: a side-by-side- // diff of these files tells you exactly what each model said.- let raw_path = dir.join(format!("{safe_name}.raw.txt"));- std::fs::write(&raw_path, &res.raw_text)?;-- // 2. Parsed JSON (if response was valid JSON at all).- if let Some(json) = &res.raw_json {- let json_path = dir.join(format!("{safe_name}.json"));- std::fs::write(&json_path, serde_json::to_string_pretty(json)?)?;- }-- // 3. Flattened markdown rendering for easy eyeballing.- let md_path = dir.join(format!("{safe_name}.md"));- let md = render_markdown(side, fixture_name, &res);- std::fs::write(&md_path, md)?;-- // 4. Per-case meta.- let case = CaseRun {- provider: side.to_string(),- fixture: fixture_name.to_string(),- elapsed_ms: res.elapsed.as_millis(),- parsed_ok: res.batch.is_some(),- error: res.error.clone(),- update_count: res.batch.as_ref().map(|b| b.updates.len()),- };- let meta_path = dir.join(format!("{safe_name}.meta.json"));- std::fs::write(&meta_path, serde_json::to_string_pretty(&case)?)?;- Ok(case)-}--fn render_markdown(side: &str, fixture_name: &str, res: &ProviderResult) -> String {- let mut out = String::new();- out.push_str(&format!("# {side} — {fixture_name}\n\n"));- out.push_str(&format!("- elapsed: {} ms\n", res.elapsed.as_millis()));- if let Some(err) = &res.error {- out.push_str(&format!("- error: `{err}`\n"));- }- // Always include the raw response so the human reviewer can- // judge structure/style even when schema validation failed.- if !res.raw_text.is_empty() {- out.push_str("\n## Raw response\n\n```\n");- out.push_str(&res.raw_text);- out.push_str("\n```\n");- }- let Some(batch) = &res.batch else {- return out;- };- out.push_str(&format!("\n- updates: {}\n", batch.updates.len()));- if !batch.rationale.trim().is_empty() {- out.push_str("\n## Rationale\n\n");- out.push_str(batch.rationale.trim());- out.push('\n');- }- for (i, u) in batch.updates.iter().enumerate() {- out.push_str(&format!("\n## Update {} — `{}`\n\n", i + 1, u.path));- out.push_str(&format!("**Title:** {} \n", u.title));- out.push_str(&format!("**Kind:** `{}` \n", u.kind.as_str()));- out.push_str(&format!("**Tier:** `{}` \n", u.tier.as_str()));- if !u.tags.is_empty() {- out.push_str(&format!("**Tags:** {} \n", u.tags.join(", ")));- }- out.push('\n');- out.push_str(u.body_markdown.trim());- out.push('\n');+ match Cli::parse().command {+ Command::Ab(args) => ab::run(*args).await,+ Command::Retrieval(args) => retrieval::run(args).await,}- out}diff --git a/evals/src/retrieval/dataset.rs b/evals/src/retrieval/dataset.rsnew file mode 100644index 00000000..c8153ee7--- /dev/null+++ b/evals/src/retrieval/dataset.rs@@ -0,0 +1,162 @@+//! LongMemEval (v1, S variant) dataset loading.+//!+//! The file is NOT committed (278 MB); it lives gitignored under+//! `evals/datasets/` and is fetched on demand from HuggingFace with a+//! pinned sha256. Dataset: `xiaowu0162/longmemeval` (MIT license).++use std::path::Path;++use anyhow::{Context, Result, bail};+use serde::Deserialize;+use sha2::{Digest, Sha256};++/// Canonical download URL for the S variant.+pub const LONGMEMEVAL_S_URL: &str =+ "https://huggingface.co/datasets/xiaowu0162/longmemeval/resolve/main/longmemeval_s";++/// sha256 of `longmemeval_s` as pinned on 2026-09-01. A drifted upstream+/// file fails loudly instead of silently changing published numbers.+pub const LONGMEMEVAL_S_SHA256: &str =+ "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894";++/// One chat turn inside a haystack session.+#[derive(Debug, Clone, Deserialize)]+pub struct Turn {+ pub role: String,+ pub content: String,+}++/// One benchmark question with its private haystack of chat sessions.+#[derive(Debug, Clone, Deserialize)]+pub struct Question {+ pub question_id: String,+ pub question_type: String,+ pub question: String,+ #[allow(dead_code)]+ pub answer: serde_json::Value,+ #[allow(dead_code)]+ pub question_date: String,+ pub haystack_dates: Vec<String>,+ pub haystack_session_ids: Vec<String>,+ pub haystack_sessions: Vec<Vec<Turn>>,+ /// Dataset session ids of the sessions holding the evidence.+ pub answer_session_ids: Vec<String>,+}++impl Question {+ /// Abstention questions (`*_abs`) deliberately have a false premise;+ /// no session contains an answer, so they are excluded from recall+ /// metrics and reported separately.+ pub fn is_abstention(&self) -> bool {+ self.question_id.ends_with("_abs")+ }+}++/// Load the dataset, verifying the pinned sha256 first.+pub fn load(path: &Path) -> Result<Vec<Question>> {+ let bytes = std::fs::read(path).with_context(|| {+ format!(+ "reading dataset at {} (use --fetch to download)",+ path.display()+ )+ })?;+ let digest = hex::encode(Sha256::digest(&bytes));+ if digest != LONGMEMEVAL_S_SHA256 {+ bail!(+ "dataset sha256 mismatch at {}: got {digest}, pinned {LONGMEMEVAL_S_SHA256}. \+ Refusing to benchmark against an unpinned file.",+ path.display()+ );+ }+ let questions: Vec<Question> =+ serde_json::from_slice(&bytes).context("parsing longmemeval_s JSON")?;+ for q in &questions {+ if q.haystack_sessions.len() != q.haystack_session_ids.len()+ || q.haystack_sessions.len() != q.haystack_dates.len()+ {+ bail!(+ "question {}: sessions/ids/dates length mismatch ({}/{}/{})",+ q.question_id,+ q.haystack_sessions.len(),+ q.haystack_session_ids.len(),+ q.haystack_dates.len()+ );+ }+ }+ Ok(questions)+}++/// Download the dataset to `path` (atomically: tmp + rename) and verify+/// the pinned sha256 before the rename lands it.+pub async fn fetch(path: &Path) -> Result<()> {+ if let Some(parent) = path.parent() {+ std::fs::create_dir_all(parent)?;+ }+ tracing::info!(+ url = LONGMEMEVAL_S_URL,+ "downloading LongMemEval-S (278 MB)"+ );+ let resp = reqwest::get(LONGMEMEVAL_S_URL).await?.error_for_status()?;+ let bytes = resp.bytes().await?;+ let digest = hex::encode(Sha256::digest(&bytes));+ if digest != LONGMEMEVAL_S_SHA256 {+ bail!("downloaded dataset sha256 {digest} does not match pin {LONGMEMEVAL_S_SHA256}");+ }+ let tmp = path.with_extension("part");+ std::fs::write(&tmp, &bytes)?;+ std::fs::rename(&tmp, path)?;+ Ok(())+}++mod hex {+ /// Minimal lowercase hex, avoiding a dependency for one call site.+ pub fn encode(bytes: impl AsRef<[u8]>) -> String {+ bytes.as_ref().iter().map(|b| format!("{b:02x}")).collect()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn sample_json() -> serde_json::Value {+ serde_json::json!([{+ "question_id": "q1",+ "question_type": "single-session-user",+ "question": "what degree?",+ "answer": "Business",+ "question_date": "2023/05/30 (Tue) 23:40",+ "haystack_dates": ["2023/05/20 (Sat) 02:21"],+ "haystack_session_ids": ["sharegpt_abc_0"],+ "haystack_sessions": [[+ {"role": "user", "content": "I studied Business", "has_answer": true},+ {"role": "assistant", "content": "Nice."}+ ]],+ "answer_session_ids": ["sharegpt_abc_0"]+ }])+ }++ #[test]+ fn parses_the_dataset_shape() {+ let qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();+ assert_eq!(qs.len(), 1);+ assert_eq!(qs[0].haystack_sessions[0][0].role, "user");+ assert!(!qs[0].is_abstention());+ }++ #[test]+ fn abstention_is_detected_from_the_id_suffix() {+ let mut qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();+ qs[0].question_id = "q1_abs".into();+ assert!(qs[0].is_abstention());+ }++ #[test]+ fn a_tampered_dataset_fails_the_pin() {+ let dir = tempfile::tempdir().unwrap();+ let p = dir.path().join("longmemeval_s.json");+ std::fs::write(&p, serde_json::to_vec(&sample_json()).unwrap()).unwrap();+ let err = load(&p).unwrap_err().to_string();+ assert!(err.contains("sha256 mismatch"), "{err}");+ }+}diff --git a/evals/src/retrieval/ingest.rs b/evals/src/retrieval/ingest.rsnew file mode 100644index 00000000..ae1b2c89--- /dev/null+++ b/evals/src/retrieval/ingest.rs@@ -0,0 +1,285 @@+//! Replay LongMemEval haystack sessions through the real hook-ingest+//! surface (`POST /hook/batch`), at the same cadence the lifecycle hooks+//! produce in production: `session-start`, then per chat round a+//! `user-prompt-submit` (user turn) and a `stop` carrying the opt-in+//! assistant excerpt (assistant turn), then `session-end`.+//!+//! Fidelity notes, deliberately inherited from the production capture:+//!+//! - excerpts are bounded (~2 KB) at the privacy boundary, so evidence+//! buried deeper in a single long turn is genuinely out of reach;+//! - each session's original date is prepended to turn text (production+//! installs see wall-clock time; replayed histories must carry their+//! own), mirroring what the official LongMemEval harness exposes to+//! retrievers as session timestamps.++use std::time::Duration;++use anyhow::{Context, Result, bail};+use serde_json::json;++use super::dataset::Question;+use super::server::EVAL_AUTH_TOKEN;++/// Everything in one benchmark run lives in this workspace.+pub const EVAL_WORKSPACE: &str = "longmemeval";++/// Server-side cap on batch items per request.+const MAX_BATCH: usize = 256;++/// Client-side mirror of the capture excerpt cap; anything longer is+/// truncated by the server anyway, and the assistant marker requires the+/// client to do the capping.+const EXCERPT_MAX_BYTES: usize = 2_000;++/// Project name for one benchmark question (its private haystack).+pub fn project_for(q: &Question) -> String {+ format!("lme-{}", q.question_id)+}++/// The store hashes non-UUID session ids to UUIDv5(NAMESPACE_OID, raw)+/// (`resolve_native_session_id`); the scorer recomputes the same mapping+/// to attribute `sessions/<uuid>.md` page hits back to dataset sessions.+pub fn stored_session_uuid(raw: &str) -> uuid::Uuid {+ match uuid::Uuid::parse_str(raw) {+ Ok(u) => u,+ Err(_) => uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, raw.as_bytes()),+ }+}++/// Truncate on a char boundary so the marker stays valid UTF-8.+fn cap_excerpt(s: &str) -> &str {+ if s.len() <= EXCERPT_MAX_BYTES {+ return s;+ }+ let mut end = EXCERPT_MAX_BYTES;+ while !s.is_char_boundary(end) {+ end -= 1;+ }+ &s[..end]+}++#[derive(serde::Serialize)]+struct BatchItem {+ url: String,+ body: serde_json::Value,+}++/// Build the full event replay for one question's haystack.+fn build_items(q: &Question) -> Vec<BatchItem> {+ let project = project_for(q);+ let cwd = format!("/lme/{project}");+ let mut items = Vec::new();+ let mut push = |event: &str, session_id: &str, extra: serde_json::Value| {+ let mut url = format!(+ "/hook?event={event}&agent=claude-code&workspace={EVAL_WORKSPACE}&project={project}&session_id={session_id}"+ );+ if event == "stop" {+ url.push_str("&capture_assistant=1");+ }+ let mut body = json!({+ "session_id": session_id,+ "cwd": cwd,+ });+ if let serde_json::Value::Object(extra) = extra {+ body.as_object_mut()+ .expect("body is an object")+ .extend(extra);+ }+ items.push(BatchItem { url, body });+ };++ for (idx, session) in q.haystack_sessions.iter().enumerate() {+ let sid = &q.haystack_session_ids[idx];+ let date = &q.haystack_dates[idx];+ push(+ "session-start",+ sid,+ json!({ "hook_event_name": "SessionStart", "source": "startup" }),+ );+ for turn in session {+ match turn.role.as_str() {+ "user" => push(+ "user-prompt-submit",+ sid,+ json!({+ "hook_event_name": "UserPromptSubmit",+ "prompt": format!("[session date: {date}] {}", turn.content),+ }),+ ),+ "assistant" => push(+ "stop",+ sid,+ json!({+ "hook_event_name": "Stop",+ "_ai_memory_assistant": {+ "version": 1,+ "excerpt": format!("[session date: {date}] {}", cap_excerpt(&turn.content)),+ },+ }),+ ),+ other => {+ tracing::warn!(role = other, "unknown role in dataset; skipping turn");+ }+ }+ }+ push(+ "session-end",+ sid,+ json!({ "hook_event_name": "SessionEnd", "reason": "exit" }),+ );+ }+ items+}++/// Replay one question's haystack. Retries items the server's per-source+/// rate limiting skipped, until everything is accepted — silently missing+/// haystack data would corrupt the benchmark.+pub async fn ingest_question(+ client: &reqwest::Client,+ base_url: &str,+ q: &Question,+) -> Result<usize> {+ let items = build_items(q);+ let total = items.len();+ let mut pending: Vec<&BatchItem> = items.iter().collect();+ let mut attempts = 0usize;+ while !pending.is_empty() {+ attempts += 1;+ if attempts > 60 {+ bail!(+ "question {}: {} hook items still unaccepted after {} passes",+ q.question_id,+ pending.len(),+ attempts - 1+ );+ }+ let mut next = Vec::new();+ for chunk in pending.chunks(MAX_BATCH) {+ let resp = client+ .post(format!("{base_url}/hook/batch"))+ .bearer_auth(EVAL_AUTH_TOKEN)+ .json(&chunk)+ .send()+ .await+ .context("posting hook batch")?+ .error_for_status()+ .context("hook batch rejected")?;+ let ack: serde_json::Value = resp.json().await?;+ let accepted: Vec<usize> = match ack.get("accepted_indices") {+ Some(serde_json::Value::Array(idx)) => idx+ .iter()+ .filter_map(|v| v.as_u64().map(|v| v as usize))+ .collect(),+ _ => {+ let prefix = ack.get("accepted").and_then(|v| v.as_u64()).unwrap_or(0) as usize;+ (0..prefix).collect()+ }+ };+ let accepted_set: std::collections::HashSet<usize> = accepted.into_iter().collect();+ for (i, item) in chunk.iter().enumerate() {+ if !accepted_set.contains(&i) {+ next.push(*item);+ }+ }+ }+ if !next.is_empty() {+ tokio::time::sleep(Duration::from_millis(500)).await;+ }+ pending = next;+ }+ Ok(total)+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::retrieval::dataset::Turn;++ fn q() -> Question {+ serde_json::from_value(serde_json::json!({+ "question_id": "qX",+ "question_type": "single-session-user",+ "question": "?",+ "answer": "a",+ "question_date": "d",+ "haystack_dates": ["2023/05/20 (Sat) 02:21"],+ "haystack_session_ids": ["sharegpt_abc_0"],+ "haystack_sessions": [[+ {"role": "user", "content": "hello"},+ {"role": "assistant", "content": "hi"}+ ]],+ "answer_session_ids": ["sharegpt_abc_0"]+ }))+ .unwrap()+ }++ #[test]+ fn replay_matches_the_production_hook_cadence() {+ let items = build_items(&q());+ let events: Vec<&str> = items+ .iter()+ .map(|i| {+ i.url+ .split("event=")+ .nth(1)+ .unwrap()+ .split('&')+ .next()+ .unwrap()+ })+ .collect();+ assert_eq!(+ events,+ ["session-start", "user-prompt-submit", "stop", "session-end"]+ );+ // stop carries the opt-in marker and the capture flag+ let stop = &items[2];+ assert!(stop.url.contains("capture_assistant=1"));+ assert_eq!(stop.body["_ai_memory_assistant"]["version"], 1);+ // every item scopes explicitly+ for item in &items {+ assert!(item.url.contains("workspace=longmemeval"));+ assert!(item.url.contains("project=lme-qX"));+ }+ }++ #[test]+ fn session_dates_are_visible_to_the_retriever() {+ let items = build_items(&q());+ let prompt = items[1].body["prompt"].as_str().unwrap();+ assert!(prompt.starts_with("[session date: 2023/05/20"));+ }++ #[test]+ fn excerpts_are_capped_on_char_boundaries() {+ let long = format!("{}é", "x".repeat(EXCERPT_MAX_BYTES - 1));+ let capped = cap_excerpt(&long);+ assert!(capped.len() <= EXCERPT_MAX_BYTES);+ assert!(std::str::from_utf8(capped.as_bytes()).is_ok());+ }++ #[test]+ fn non_uuid_session_ids_map_to_the_stores_v5_hash() {+ // Mirrors resolve_native_session_id in the hooks router.+ let mapped = stored_session_uuid("sharegpt_abc_0");+ assert_eq!(+ mapped,+ uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, b"sharegpt_abc_0")+ );+ let real = uuid::Uuid::now_v7();+ assert_eq!(stored_session_uuid(&real.to_string()), real);+ }++ #[test]+ fn unknown_roles_are_skipped_not_mislabelled() {+ let mut question = q();+ question.haystack_sessions[0].push(Turn {+ role: "system".into(),+ content: "x".into(),+ });+ let items = build_items(&question);+ // still start, prompt, stop, end — the system turn produced nothing+ assert_eq!(items.len(), 4);+ }+}diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rsnew file mode 100644index 00000000..6fa095d4--- /dev/null+++ b/evals/src/retrieval/mod.rs@@ -0,0 +1,196 @@+//! LongMemEval retrieval benchmark, end to end through the real stack.+//!+//! Per question: replay its haystack sessions through `POST /hook/batch`+//! (production hook cadence), query with `memory_query` over MCP, and+//! score session-level hit@k / recall@k against the dataset's evidence+//! labels. See `docs/benchmarks/` for published baselines and+//! `evals/README.md` for how to run.++mod dataset;+mod ingest;+mod query;+mod report;+mod score;++use std::collections::HashSet;+use std::path::PathBuf;+use std::sync::Arc;++use anyhow::{Context, Result, bail};+use jiff::Timestamp;++use dataset::Question;+use ingest::{project_for, stored_session_uuid};+use server::EvalServer;++mod server;++#[derive(clap::Args, Debug)]+pub struct RetrievalArgs {+ /// Dataset file (LongMemEval-S JSON).+ #[arg(long, default_value = "evals/datasets/longmemeval_s.json")]+ dataset: PathBuf,++ /// Download the dataset first if the file is missing.+ #[arg(long)]+ fetch: bool,++ /// `ai-memory` server binary to benchmark. Build it first:+ /// `cargo build --release -p ai-memory-cli`.+ #[arg(long, default_value = "target/release/ai-memory")]+ server_bin: PathBuf,++ /// Score only the first N questions (deterministic prefix) — smoke+ /// runs and iteration. Omit for the full 500.+ #[arg(long)]+ sample: Option<usize>,++ /// Cutoffs for hit@k / recall@k.+ #[arg(long, value_delimiter = ',', default_values_t = vec![1, 3, 5, 10])]+ ks: Vec<usize>,++ /// Questions ingested+queried concurrently.+ #[arg(long, default_value_t = 8)]+ concurrency: usize,++ /// Output root; a timestamped run dir is created under it.+ #[arg(long, default_value = "evals/runs")]+ out: PathBuf,++ /// Keep the server's temp data dir for post-mortem inspection.+ #[arg(long)]+ keep_data_dir: bool,+}++pub async fn run(args: RetrievalArgs) -> Result<()> {+ if args.fetch && !args.dataset.exists() {+ dataset::fetch(&args.dataset).await?;+ }+ let mut questions = dataset::load(&args.dataset)?;+ let total_loaded = questions.len();+ if let Some(n) = args.sample {+ questions.truncate(n);+ }+ let abstention: Vec<Question> = questions+ .iter()+ .filter(|q| q.is_abstention())+ .cloned()+ .collect();+ questions.retain(|q| !q.is_abstention());+ tracing::info!(+ loaded = total_loaded,+ scored = questions.len(),+ abstention_excluded = abstention.len(),+ "dataset ready"+ );++ if !args.server_bin.exists() {+ bail!(+ "server binary {} not found — run `cargo build --release -p ai-memory-cli` first",+ args.server_bin.display()+ );+ }+ let server = EvalServer::launch(&args.server_bin, args.keep_data_dir).await?;+ tracing::info!(url = server.base_url, data_dir = %server.data_dir_path.display(), "eval server up");++ let client = reqwest::Client::builder()+ .timeout(std::time::Duration::from_secs(120))+ .build()?;+ let max_k = args.ks.iter().copied().max().unwrap_or(10);+ let ks = Arc::new(args.ks.clone());+ let base_url = Arc::new(server.base_url.clone());+ let client = Arc::new(client);++ let semaphore = Arc::new(tokio::sync::Semaphore::new(args.concurrency.max(1)));+ let mut tasks = tokio::task::JoinSet::new();+ for q in questions.clone() {+ let permit = semaphore.clone().acquire_owned().await?;+ let (client, base_url, ks) = (client.clone(), base_url.clone(), ks.clone());+ tasks.spawn(async move {+ let _permit = permit;+ let events = ingest::ingest_question(&client, &base_url, &q)+ .await+ .with_context(|| format!("ingesting {}", q.question_id))?;+ let retrieved = query::memory_query(+ &client,+ &base_url,+ ingest::EVAL_WORKSPACE,+ &project_for(&q),+ &q.question,+ max_k,+ )+ .await+ .with_context(|| format!("querying {}", q.question_id))?;+ let evidence: HashSet<uuid::Uuid> = q+ .answer_session_ids+ .iter()+ .map(|s| stored_session_uuid(s))+ .collect();+ let score =+ score::score_question(&q.question_id, &q.question_type, &evidence, &retrieved, &ks);+ tracing::info!(+ question = q.question_id,+ events,+ retrieved = retrieved.len(),+ hit_at_max = score.hit_at.values().next_back().copied().unwrap_or(0.0),+ "scored"+ );+ anyhow::Ok(score)+ });+ }++ // Collect everything before judging: one bad question must not waste+ // the other 499, but a partial run must never masquerade as a+ // publishable number either — any failure aborts before reporting.+ let mut scores = Vec::new();+ let mut failures = Vec::new();+ while let Some(res) = tasks.join_next().await {+ match res.context("task panicked")? {+ Ok(score) => scores.push(score),+ Err(e) => failures.push(format!("{e:#}")),+ }+ }+ if !failures.is_empty() {+ for f in &failures {+ tracing::error!("{f}");+ }+ bail!(+ "{} of {} questions failed; no report written",+ failures.len(),+ failures.len() + scores.len()+ );+ }+ scores.sort_by(|a, b| a.question_id.cmp(&b.question_id));++ let slices = score::aggregate(&scores, &args.ks);+ let stamp: String = Timestamp::now()+ .to_string()+ .chars()+ .map(|c| if c == ':' || c == '.' { '-' } else { c })+ .collect();+ let run_dir = args.out.join(format!("{stamp}-retrieval"));+ std::fs::create_dir_all(&run_dir)?;++ let rep = report::Report {+ generated_at: Timestamp::now().to_string(),+ commit: report::commit_sha(),+ hardware: report::hardware(),+ dataset: "longmemeval_s",+ dataset_sha256: dataset::LONGMEMEVAL_S_SHA256,+ mode: "zero-llm",+ questions_scored: scores.len(),+ abstention_excluded: abstention.len(),+ ks: args.ks.clone(),+ slices,+ per_question: scores,+ };+ std::fs::write(+ run_dir.join("report.json"),+ serde_json::to_vec_pretty(&rep)?,+ )?;+ let md = report::to_markdown(&rep);+ std::fs::write(run_dir.join("report.md"), &md)?;+ println!("{md}");+ println!("run dir: {}", run_dir.display());+ Ok(())+}diff --git a/evals/src/retrieval/query.rs b/evals/src/retrieval/query.rsnew file mode 100644index 00000000..7588f29a--- /dev/null+++ b/evals/src/retrieval/query.rs@@ -0,0 +1,177 @@+//! Query the eval server through the real MCP surface — the same+//! Streamable-HTTP `tools/call memory_query` an agent uses. Explicit+//! `workspace` + `project` args scope each call to one question's+//! haystack (the documented pattern for static MCP clients).++use anyhow::{Context, Result, bail};+use serde::Deserialize;+use serde_json::json;++use super::server::EVAL_AUTH_TOKEN;++/// One retrieval hit after flattening pages + raw observations, in+/// rank order (index 0 = best).+#[derive(Debug, Clone)]+pub struct Retrieved {+ /// `sessions/<uuid>.md` page or raw observation → owning session+ /// uuid; other pages have no session provenance.+ pub session_uuid: Option<uuid::Uuid>,+}++#[derive(Debug, Deserialize)]+struct QueryResponse {+ #[serde(default)]+ hits: Vec<PageHitLite>,+ #[serde(default)]+ raw_hits: Vec<RawHitLite>,+}++#[derive(Debug, Deserialize)]+struct PageHitLite {+ path: String,+}++#[derive(Debug, Deserialize)]+struct RawHitLite {+ session_id: uuid::Uuid,+}++/// Call `memory_query` and flatten the response into ranked session+/// attributions: compiled-page hits first (they are the primary answer+/// surface), then raw observation hits, preserving each list's order.+pub async fn memory_query(+ client: &reqwest::Client,+ base_url: &str,+ workspace: &str,+ project: &str,+ query: &str,+ limit: usize,+) -> Result<Vec<Retrieved>> {+ let request = json!({+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/call",+ "params": {+ "name": "memory_query",+ "arguments": {+ "query": query,+ "workspace": workspace,+ "project": project,+ "limit": limit,+ }+ }+ });+ let resp = client+ .post(format!("{base_url}/mcp"))+ .bearer_auth(EVAL_AUTH_TOKEN)+ .header("accept", "application/json, text/event-stream")+ .json(&request)+ .send()+ .await+ .context("posting MCP tools/call")?;+ let status = resp.status();+ let content_type = resp+ .headers()+ .get("content-type")+ .and_then(|v| v.to_str().ok())+ .unwrap_or("")+ .to_string();+ let body = resp.text().await?;+ if !status.is_success() {+ bail!("MCP call failed ({status}): {body}");+ }++ let rpc: serde_json::Value = if content_type.starts_with("text/event-stream") {+ // Streamable HTTP may frame the response as SSE; the JSON-RPC+ // response is the last `data:` payload carrying our id.+ let mut last = None;+ for line in body.lines() {+ if let Some(data) = line.strip_prefix("data:")+ && let Ok(v) = serde_json::from_str::<serde_json::Value>(data.trim())+ && v.get("id").is_some()+ {+ last = Some(v);+ }+ }+ last.ok_or_else(|| anyhow::anyhow!("no JSON-RPC response in SSE stream: {body}"))?+ } else {+ serde_json::from_str(&body).with_context(|| format!("parsing MCP response: {body}"))?+ };++ if let Some(err) = rpc.get("error") {+ bail!("MCP error: {err}");+ }+ let result = rpc+ .get("result")+ .ok_or_else(|| anyhow::anyhow!("MCP response missing result: {rpc}"))?;+ if result+ .get("isError")+ .and_then(|v| v.as_bool())+ .unwrap_or(false)+ {+ bail!("memory_query tool error: {result}");+ }+ let text = result+ .get("content")+ .and_then(|c| c.as_array())+ .and_then(|c| c.first())+ .and_then(|c| c.get("text"))+ .and_then(|t| t.as_str())+ .ok_or_else(|| anyhow::anyhow!("memory_query returned no text content: {result}"))?;+ let parsed: QueryResponse =+ serde_json::from_str(text).with_context(|| format!("parsing memory_query JSON: {text}"))?;+ Ok(flatten(parsed))+}++fn flatten(resp: QueryResponse) -> Vec<Retrieved> {+ let mut out = Vec::new();+ for hit in resp.hits {+ out.push(Retrieved {+ session_uuid: session_uuid_from_path(&hit.path),+ });+ }+ for hit in resp.raw_hits {+ out.push(Retrieved {+ session_uuid: Some(hit.session_id),+ });+ }+ out+}++/// `sessions/<uuid>.md` → the owning session; anything else → None.+fn session_uuid_from_path(path: &str) -> Option<uuid::Uuid> {+ let stem = path.strip_prefix("sessions/")?.strip_suffix(".md")?;+ uuid::Uuid::parse_str(stem).ok()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn session_pages_attribute_and_other_pages_do_not() {+ let sid = uuid::Uuid::now_v7();+ assert_eq!(+ session_uuid_from_path(&format!("sessions/{sid}.md")),+ Some(sid)+ );+ assert_eq!(session_uuid_from_path("gotchas/build.md"), None);+ assert_eq!(session_uuid_from_path("sessions/not-a-uuid.md"), None);+ }++ #[test]+ fn pages_rank_before_raw_hits_and_order_is_preserved() {+ let a = uuid::Uuid::now_v7();+ let b = uuid::Uuid::now_v7();+ let resp = QueryResponse {+ hits: vec![PageHitLite {+ path: format!("sessions/{a}.md"),+ }],+ raw_hits: vec![RawHitLite { session_id: b }],+ };+ let flat = flatten(resp);+ assert_eq!(flat.len(), 2);+ assert_eq!(flat[0].session_uuid, Some(a));+ assert_eq!(flat[1].session_uuid, Some(b));+ }+}diff --git a/evals/src/retrieval/report.rs b/evals/src/retrieval/report.rsnew file mode 100644index 00000000..0308bb4b--- /dev/null+++ b/evals/src/retrieval/report.rs@@ -0,0 +1,131 @@+//! Render benchmark results as JSON (machine) + markdown (human), with+//! enough provenance (dataset sha, commit, hardware, config) that a+//! published number can be audited later.++use std::collections::BTreeMap;++use serde::Serialize;++use super::score::{QuestionScore, SliceMetrics};++#[derive(Debug, Serialize)]+pub struct Report {+ pub generated_at: String,+ pub commit: String,+ pub hardware: String,+ pub dataset: &'static str,+ pub dataset_sha256: &'static str,+ pub mode: &'static str,+ pub questions_scored: usize,+ pub abstention_excluded: usize,+ pub ks: Vec<usize>,+ pub slices: BTreeMap<String, SliceMetrics>,+ pub per_question: Vec<QuestionScore>,+}++pub fn commit_sha() -> String {+ std::process::Command::new("git")+ .args(["rev-parse", "HEAD"])+ .output()+ .ok()+ .filter(|o| o.status.success())+ .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())+ .unwrap_or_else(|| "unknown".into())+}++pub fn hardware() -> String {+ let model = std::fs::read_to_string("/proc/cpuinfo")+ .ok()+ .and_then(|s| {+ s.lines()+ .find(|l| l.starts_with("model name"))+ .map(|l| l.split(':').nth(1).unwrap_or("").trim().to_string())+ })+ .unwrap_or_else(|| "unknown cpu".into());+ let threads = std::thread::available_parallelism()+ .map(|n| n.get())+ .unwrap_or(0);+ format!("{model} ({threads} threads)")+}++pub fn to_markdown(r: &Report) -> String {+ let mut md = String::new();+ md.push_str(&format!(+ "# LongMemEval-S retrieval — {}\n\n\+ - commit: `{}`\n- dataset: `{}` (sha256 `{}…`)\n- mode: {}\n- hardware: {}\n\+ - questions scored: {} ({} abstention questions excluded)\n\n",+ r.generated_at,+ r.commit,+ r.dataset,+ &r.dataset_sha256[..16],+ r.mode,+ r.hardware,+ r.questions_scored,+ r.abstention_excluded,+ ));+ md.push_str("| slice | n |");+ for k in &r.ks {+ md.push_str(&format!(" hit@{k} |"));+ }+ for k in &r.ks {+ md.push_str(&format!(" recall@{k} |"));+ }+ md.push_str("\n|---|---|");+ for _ in r.ks.iter().chain(r.ks.iter()) {+ md.push_str("---|");+ }+ md.push('\n');+ for (name, m) in &r.slices {+ md.push_str(&format!("| {name} | {} |", m.questions));+ for k in &r.ks {+ md.push_str(&format!(" {:.3} |", m.hit_at[k]));+ }+ for k in &r.ks {+ md.push_str(&format!(" {:.3} |", m.recall_at[k]));+ }+ md.push('\n');+ }+ md.push_str(+ "\nNotes: hit@k = any evidence session in top k (the \"Recall@k\" most \+ systems publish); recall@k = fraction of evidence sessions found. \+ Session attribution: `sessions/<id>.md` pages and raw observation \+ hits; unattributable pages never score. Capture is production-shaped: \+ excerpts bounded at the 2 KB privacy boundary.\n",+ );+ md+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn markdown_carries_provenance_and_all_slices() {+ let mut slices = BTreeMap::new();+ slices.insert(+ "overall".to_string(),+ SliceMetrics {+ questions: 2,+ hit_at: BTreeMap::from([(5, 0.5)]),+ recall_at: BTreeMap::from([(5, 0.25)]),+ },+ );+ let r = Report {+ generated_at: "2026-09-01".into(),+ commit: "abc".into(),+ hardware: "cpu".into(),+ dataset: "longmemeval_s",+ dataset_sha256: crate::retrieval::dataset::LONGMEMEVAL_S_SHA256,+ mode: "zero-llm",+ questions_scored: 2,+ abstention_excluded: 1,+ ks: vec![5],+ slices,+ per_question: vec![],+ };+ let md = to_markdown(&r);+ assert!(md.contains("commit: `abc`"));+ assert!(md.contains("| overall | 2 | 0.500 | 0.250 |"));+ assert!(md.contains("abstention questions excluded"));+ }+}diff --git a/evals/src/retrieval/score.rs b/evals/src/retrieval/score.rsnew file mode 100644index 00000000..beef382f--- /dev/null+++ b/evals/src/retrieval/score.rs@@ -0,0 +1,246 @@+//! Session-level retrieval scoring, matching how published LongMemEval+//! retrieval numbers are reported: for each question, did the evidence+//! sessions surface in the top k retrieved results?+//!+//! Two metrics per k:+//!+//! - `hit@k` — 1.0 when ANY evidence session appears in the top k+//! (the "Recall@k" most memory systems publish);+//! - `recall@k` — the fraction of that question's evidence sessions+//! found in the top k (stricter on multi-session questions).+//!+//! Abstention questions have no evidence sessions and are excluded+//! from both (reported separately as a count).++use std::collections::{BTreeMap, HashSet};++use serde::Serialize;++use super::query::Retrieved;++/// Per-question outcome, ready for aggregation.+#[derive(Debug, Clone, Serialize)]+pub struct QuestionScore {+ pub question_id: String,+ pub question_type: String,+ /// hit@k and recall@k keyed by k.+ pub hit_at: BTreeMap<usize, f64>,+ pub recall_at: BTreeMap<usize, f64>,+ /// Deduped session attributions actually retrieved (for forensics).+ pub retrieved_sessions: usize,+}++/// Score one question's retrieval run.+///+/// `evidence` holds the stored (uuid-mapped) ids of the answer sessions;+/// `retrieved` is the ranked flattened result list. Duplicate session+/// attributions collapse to their best (first) rank — retrieving three+/// chunks of one session fills one slot, not three.+pub fn score_question(+ question_id: &str,+ question_type: &str,+ evidence: &HashSet<uuid::Uuid>,+ retrieved: &[Retrieved],+ ks: &[usize],+) -> QuestionScore {+ let mut seen = HashSet::new();+ let mut ranked_sessions = Vec::new();+ for r in retrieved {+ if let Some(sid) = r.session_uuid+ && seen.insert(sid)+ {+ ranked_sessions.push(sid);+ }+ }+ let mut hit_at = BTreeMap::new();+ let mut recall_at = BTreeMap::new();+ for &k in ks {+ let top: HashSet<&uuid::Uuid> = ranked_sessions.iter().take(k).collect();+ let found = evidence.iter().filter(|e| top.contains(e)).count();+ hit_at.insert(k, if found > 0 { 1.0 } else { 0.0 });+ recall_at.insert(+ k,+ if evidence.is_empty() {+ 0.0+ } else {+ found as f64 / evidence.len() as f64+ },+ );+ }+ QuestionScore {+ question_id: question_id.to_string(),+ question_type: question_type.to_string(),+ hit_at,+ recall_at,+ retrieved_sessions: ranked_sessions.len(),+ }+}++/// Aggregated metrics for one slice (a category, or overall).+#[derive(Debug, Clone, Serialize)]+pub struct SliceMetrics {+ pub questions: usize,+ pub hit_at: BTreeMap<usize, f64>,+ pub recall_at: BTreeMap<usize, f64>,+}++/// Macro-average scores per category plus an `overall` slice.+pub fn aggregate(scores: &[QuestionScore], ks: &[usize]) -> BTreeMap<String, SliceMetrics> {+ let mut slices: BTreeMap<String, Vec<&QuestionScore>> = BTreeMap::new();+ for s in scores {+ slices.entry(s.question_type.clone()).or_default().push(s);+ slices.entry("overall".to_string()).or_default().push(s);+ }+ slices+ .into_iter()+ .map(|(name, group)| {+ let n = group.len() as f64;+ let mut hit_at = BTreeMap::new();+ let mut recall_at = BTreeMap::new();+ for &k in ks {+ hit_at.insert(k, group.iter().map(|s| s.hit_at[&k]).sum::<f64>() / n);+ recall_at.insert(k, group.iter().map(|s| s.recall_at[&k]).sum::<f64>() / n);+ }+ (+ name,+ SliceMetrics {+ questions: group.len(),+ hit_at,+ recall_at,+ },+ )+ })+ .collect()+}++#[cfg(test)]+mod tests {+ use super::*;++ fn retrieved(ids: &[Option<uuid::Uuid>]) -> Vec<Retrieved> {+ ids.iter()+ .map(|session_uuid| Retrieved {+ session_uuid: *session_uuid,+ })+ .collect()+ }++ #[test]+ fn a_top_one_hit_scores_across_all_ks() {+ let e = uuid::Uuid::now_v7();+ let s = score_question(+ "q",+ "single-session-user",+ &HashSet::from([e]),+ &retrieved(&[Some(e)]),+ &[1, 5],+ );+ assert_eq!(s.hit_at[&1], 1.0);+ assert_eq!(s.hit_at[&5], 1.0);+ assert_eq!(s.recall_at[&1], 1.0);+ }++ #[test]+ fn evidence_below_the_cutoff_does_not_count() {+ let e = uuid::Uuid::now_v7();+ let others: Vec<Option<uuid::Uuid>> = (0..3).map(|_| Some(uuid::Uuid::now_v7())).collect();+ let mut all = others.clone();+ all.push(Some(e)); // rank 4 (0-based 3)+ let s = score_question(+ "q",+ "multi-session",+ &HashSet::from([e]),+ &retrieved(&all),+ &[1, 3, 5],+ );+ assert_eq!(s.hit_at[&1], 0.0);+ assert_eq!(s.hit_at[&3], 0.0);+ assert_eq!(s.hit_at[&5], 1.0);+ }++ #[test]+ fn duplicate_session_chunks_fill_one_slot_not_three() {+ let e = uuid::Uuid::now_v7();+ let noise = uuid::Uuid::now_v7();+ // three chunks of the same noise session ahead of the evidence+ let s = score_question(+ "q",+ "multi-session",+ &HashSet::from([e]),+ &retrieved(&[Some(noise), Some(noise), Some(noise), Some(e)]),+ &[2],+ );+ // dedup: noise occupies rank 0, evidence rank 1 → hit@2+ assert_eq!(s.hit_at[&2], 1.0);+ }++ #[test]+ fn partial_multi_session_recall_is_fractional() {+ let e1 = uuid::Uuid::now_v7();+ let e2 = uuid::Uuid::now_v7();+ let s = score_question(+ "q",+ "multi-session",+ &HashSet::from([e1, e2]),+ &retrieved(&[Some(e1)]),+ &[5],+ );+ assert_eq!(s.recall_at[&5], 0.5);+ assert_eq!(s.hit_at[&5], 1.0);+ }++ #[test]+ fn unattributed_pages_never_score() {+ let e = uuid::Uuid::now_v7();+ let s = score_question(+ "q",+ "single-session-user",+ &HashSet::from([e]),+ &retrieved(&[None, None]),+ &[5],+ );+ assert_eq!(s.hit_at[&5], 0.0);+ assert_eq!(s.retrieved_sessions, 0);+ }++ /// Control: a scorer that credited ANY retrieval would ace a shuffled+ /// result list; the real scorer must not.+ #[test]+ fn random_sessions_score_zero() {+ let e = uuid::Uuid::now_v7();+ let random: Vec<Option<uuid::Uuid>> = (0..10).map(|_| Some(uuid::Uuid::now_v7())).collect();+ let s = score_question(+ "q",+ "knowledge-update",+ &HashSet::from([e]),+ &retrieved(&random),+ &[1, 3, 5, 10],+ );+ for k in [1, 3, 5, 10] {+ assert_eq!(s.hit_at[&k], 0.0, "hit@{k}");+ assert_eq!(s.recall_at[&k], 0.0, "recall@{k}");+ }+ }++ #[test]+ fn aggregation_macro_averages_per_category_and_overall() {+ let e = uuid::Uuid::now_v7();+ let hit = score_question(+ "q1",+ "multi-session",+ &HashSet::from([e]),+ &retrieved(&[Some(e)]),+ &[1],+ );+ let miss = score_question(+ "q2",+ "multi-session",+ &HashSet::from([uuid::Uuid::now_v7()]),+ &retrieved(&[]),+ &[1],+ );+ let agg = aggregate(&[hit, miss], &[1]);+ assert_eq!(agg["multi-session"].hit_at[&1], 0.5);+ assert_eq!(agg["overall"].questions, 2);+ }+}diff --git a/evals/src/retrieval/server.rs b/evals/src/retrieval/server.rsnew file mode 100644index 00000000..1127ffb9--- /dev/null+++ b/evals/src/retrieval/server.rs@@ -0,0 +1,121 @@+//! Spawn and supervise a real `ai-memory serve` subprocess for the+//! benchmark. The whole point of the harness is to measure the shipped+//! stack, so nothing here reaches into crates directly: the server is+//! the same binary a user runs, configured for deterministic zero-LLM+//! operation (no consolidation LLM, no embedder, no reranker).++use std::net::TcpListener;+use std::path::PathBuf;+use std::process::{Child, Command, Stdio};+use std::time::{Duration, Instant};++use anyhow::{Context, Result, bail};++/// Fixed bearer for the eval server (ephemeral data dir, loopback bind).+pub const EVAL_AUTH_TOKEN: &str = "ai-memory-eval-token";++pub struct EvalServer {+ child: Child,+ pub base_url: String,+ /// Kept for the lifetime of the server unless --keep-data-dir moved it.+ _data_dir: Option<tempfile::TempDir>,+ pub data_dir_path: PathBuf,+}++impl EvalServer {+ /// Launch `server_bin` on a free loopback port with a fresh data dir.+ pub async fn launch(server_bin: &PathBuf, keep_data_dir: bool) -> Result<Self> {+ let tmp = tempfile::Builder::new()+ .prefix("ai-memory-eval-")+ .tempdir()+ .context("creating eval data dir")?;+ let data_dir_path = tmp.path().to_path_buf();++ // Reserve a free port; the tiny close-to-bind window is fine for a+ // local benchmark run.+ let port = {+ let l = TcpListener::bind("127.0.0.1:0")?;+ l.local_addr()?.port()+ };+ let bind = format!("127.0.0.1:{port}");++ let child = Command::new(server_bin)+ .args([+ "serve",+ "--transport",+ "http",+ "--bind",+ &bind,+ "--data-dir",+ ])+ .arg(&data_dir_path)+ .env("AI_MEMORY_AUTH_TOKEN", EVAL_AUTH_TOKEN)+ .env("AI_MEMORY_CAPTURE_ASSISTANT", "true")+ // Zero-LLM determinism: nothing may call out.+ .env_remove("AI_MEMORY_LLM_PROVIDER")+ .env_remove("AI_MEMORY_EMBEDDING_PROVIDER")+ .env_remove("AI_MEMORY_RERANKER")+ .env_remove("OPENAI_API_KEY")+ .env_remove("ANTHROPIC_API_KEY")+ .env_remove("LLM_API_KEY")+ .stdout(Stdio::null())+ .stderr(Stdio::piped())+ .spawn()+ .with_context(|| format!("spawning {} serve", server_bin.display()))?;++ let base_url = format!("http://{bind}");+ let (guard, data_dir_path) = if keep_data_dir {+ // Persist the tempdir so it survives the run for inspection.+ (None, tmp.keep())+ } else {+ (Some(tmp), data_dir_path)+ };+ let mut server = Self {+ child,+ base_url,+ _data_dir: guard,+ data_dir_path,+ };+ server.wait_ready().await?;+ Ok(server)+ }++ /// Poll until the HTTP surface answers (any status counts as up).+ async fn wait_ready(&mut self) -> Result<()> {+ let client = reqwest::Client::new();+ let deadline = Instant::now() + Duration::from_secs(120);+ loop {+ if let Some(status) = self.child.try_wait()? {+ let mut err = String::new();+ if let Some(mut stderr) = self.child.stderr.take() {+ use std::io::Read as _;+ let _ = stderr.read_to_string(&mut err);+ }+ bail!("eval server exited during startup ({status}): {err}");+ }+ if client+ .get(format!("{}/", self.base_url))+ .timeout(Duration::from_secs(1))+ .send()+ .await+ .is_ok()+ {+ return Ok(());+ }+ if Instant::now() > deadline {+ bail!(+ "eval server did not answer on {} within 120s",+ self.base_url+ );+ }+ tokio::time::sleep(Duration::from_millis(200)).await;+ }+ }+}++impl Drop for EvalServer {+ fn drop(&mut self) {+ let _ = self.child.kill();+ let _ = self.child.wait();+ }+}diff --git a/evals/tests/smoke.rs b/evals/tests/smoke.rsnew file mode 100644index 00000000..c9183b3e--- /dev/null+++ b/evals/tests/smoke.rs@@ -0,0 +1,62 @@+//! On-demand end-to-end smoke for the retrieval harness (mirrors the+//! `writer_throughput` pattern: `#[ignore]`d, run deliberately).+//!+//! Requirements, both intentionally not provisioned by the test:+//! - `target/release/ai-memory` (`cargo build --release -p ai-memory-cli`)+//! - `evals/datasets/longmemeval_s.json` (run once with `--fetch`)+//!+//! ```bash+//! cargo test -p ai-memory-eval --test smoke -- --ignored+//! ```++use std::path::PathBuf;+use std::process::Command;++#[test]+#[ignore = "needs the release server binary and the downloaded dataset"]+fn the_harness_scores_a_tiny_sample_end_to_end() {+ let repo_root: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");+ let server_bin = repo_root.join("target/release/ai-memory");+ assert!(+ server_bin.exists(),+ "build the server first: cargo build --release -p ai-memory-cli"+ );+ let dataset = repo_root.join("evals/datasets/longmemeval_s.json");+ assert!(+ dataset.exists(),+ "fetch the dataset first: cargo run -p ai-memory-eval -- retrieval --fetch --sample 1"+ );++ let out = tempfile::tempdir().unwrap();+ let status = Command::new(env!("CARGO_BIN_EXE_ai-memory-eval"))+ .current_dir(&repo_root)+ .args(["retrieval", "--sample", "2", "--concurrency", "2"])+ .arg("--server-bin")+ .arg(&server_bin)+ .arg("--out")+ .arg(out.path())+ .status()+ .unwrap();+ assert!(status.success(), "harness exited non-zero");++ // Exactly one run dir with a structurally complete report.+ let run_dir = std::fs::read_dir(out.path())+ .unwrap()+ .map(|e| e.unwrap().path())+ .find(|p| p.is_dir())+ .expect("run dir created");+ let report: serde_json::Value =+ serde_json::from_slice(&std::fs::read(run_dir.join("report.json")).unwrap()).unwrap();+ assert_eq!(report["questions_scored"], 2);+ assert_eq!(report["mode"], "zero-llm");+ assert!(report["slices"]["overall"]["hit_at"]["5"].is_number());+ // Provenance that makes a published number auditable.+ assert!(report["commit"].as_str().unwrap().len() >= 7);+ assert!(+ report["dataset_sha256"]+ .as_str()+ .unwrap()+ .starts_with("08d8dad4")+ );+ assert!(run_dir.join("report.md").exists());+}