Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(evals): R2 QA-accuracy mode (LLM-as-judge) for end-to-end answer quality

AkitaOnRails7a8bed5a2b77
diff --git a/Cargo.lock b/Cargo.lock
index 3f1bf5dd..6d78842a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -130,6 +130,7 @@ dependencies = [
"clap",
"jiff",
"reqwest 0.12.28",
+ "schemars",
"secrecy",
"serde",
"serde_json",
diff --git a/evals/Cargo.toml b/evals/Cargo.toml
index 6ec9bada..fa284e8a 100644
--- a/evals/Cargo.toml
+++ b/evals/Cargo.toml
@@ -21,6 +21,7 @@ reqwest.workspace = true
sha2.workspace = true
tempfile.workspace = true
jiff.workspace = true
+schemars.workspace = true
secrecy.workspace = true
serde.workspace = true
serde_json.workspace = true
diff --git a/evals/README.md b/evals/README.md
index 3f946de6..47319c8f 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -252,6 +252,42 @@ Output: `evals/runs/<timestamp>-retrieval/report.{json,md}` — the JSON is
triples and an `overall` delta table. Publish full-dataset A/B numbers in
`docs/benchmarks/retrieval-ab-r2.md`.
+### `retrieval` QA-accuracy mode (R2b, opt-in, live LLM)
+
+R2a scores *retrieval*. `--qa` adds an opt-in **end-to-end answer-quality**
+metric so the `answer`/`reasoning` MCP features can be proven later. It is
+**OFF by default and makes REAL LLM API calls**, gated behind a key. Per
+scored question, with QA on, the harness:
+
+1. gets a candidate answer — the server's own `answer` field when a future
+ `answer=true` feature populates it, otherwise a **synthesis** LLM call
+ over the retrieved snippets (the exact context an agent would receive);
+2. **grades** that candidate against the dataset's gold answer with an
+ LLM-as-judge (`{correct, reason}`), LongMemEval-style;
+3. reports **QA accuracy** (fraction correct) per slice — plus the
+ answer-step latency (p50/p95) and answer-token estimate — alongside the
+ R2a triple, in both the single-config and A/B reports. Provider+model
+ names are recorded; **keys never are**.
+
+Flags: `--qa`, `--qa-provider <name>`, `--qa-model <model>` (required to
+enable), `--qa-base-url`, `--qa-api-key`/`--qa-api-key-env` (defaults to the
+provider's canonical env var, e.g. `GEMINI_API_KEY`), `--qa-token-file`
+(oauth/codex/copilot). The grader defaults to the answerer and is overridable
+per field: `--qa-grader-provider`/`-model`/`-base-url`/`-api-key`/
+`-api-key-env`/`-token-file`.
+
+**Gating:** if `--qa` is set but the provider/key cannot be resolved, the run
+prints a SKIP line and continues with QA off (retrieval metrics unaffected),
+exiting success — so a keyless CI stays green. The default non-QA path stays
+zero-LLM and deterministic.
+
+```bash
+# Keep keys in the process env only for the run; never echo a key.
+set -a; source ~/.config/zsh/secrets; set +a # or export GEMINI_API_KEY=...
+cargo run -p ai-memory-eval -- retrieval --sample 3 \
+ --qa --qa-provider gemini --qa-model gemini-2.5-flash
+```
+
Honest-numbers notes:
- capture is production-shaped: excerpts are bounded at the 2 KB
diff --git a/evals/src/retrieval/dataset.rs b/evals/src/retrieval/dataset.rs
index c8153ee7..27166494 100644
--- a/evals/src/retrieval/dataset.rs
+++ b/evals/src/retrieval/dataset.rs
@@ -32,7 +32,8 @@ pub struct Question {
pub question_id: String,
pub question_type: String,
pub question: String,
- #[allow(dead_code)]
+ /// Gold answer. Usually a JSON string; occasionally a richer value.
+ /// Read via [`Question::gold_answer_text`] for the QA judge (R2b).
pub answer: serde_json::Value,
#[allow(dead_code)]
pub question_date: String,
@@ -50,6 +51,16 @@ impl Question {
pub fn is_abstention(&self) -> bool {
self.question_id.ends_with("_abs")
}
+
+ /// The gold answer as text for the LLM-as-judge (R2b QA mode). A JSON
+ /// string is unwrapped to its contents; any other shape is rendered as
+ /// compact JSON so the judge still sees the reference value.
+ pub fn gold_answer_text(&self) -> String {
+ match &self.answer {
+ serde_json::Value::String(s) => s.clone(),
+ other => other.to_string(),
+ }
+ }
}
/// Load the dataset, verifying the pinned sha256 first.
@@ -144,6 +155,14 @@ mod tests {
assert!(!qs[0].is_abstention());
}
+ #[test]
+ fn gold_answer_unwraps_a_json_string_and_renders_other_shapes() {
+ let mut qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
+ assert_eq!(qs[0].gold_answer_text(), "Business");
+ qs[0].answer = serde_json::json!(["a", "b"]);
+ assert_eq!(qs[0].gold_answer_text(), "[\"a\",\"b\"]");
+ }
+
#[test]
fn abstention_is_detected_from_the_id_suffix() {
let mut qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rs
index 088715b2..0a4cac74 100644
--- a/evals/src/retrieval/mod.rs
+++ b/evals/src/retrieval/mod.rs
@@ -18,6 +18,7 @@
mod dataset;
mod ingest;
+mod qa;
mod query;
mod report;
mod score;
@@ -122,6 +123,68 @@ pub struct RetrievalArgs {
/// repeatable.
#[arg(long = "candidate-query-arg", value_name = "KEY=JSON")]
candidate_query_arg: Vec<String>,
+
+ // ---- QA-accuracy mode (R2b, opt-in, live LLM) ----
+ /// Enable end-to-end QA-accuracy grading (real LLM API calls). OFF by
+ /// default. Per scored question the harness synthesizes an answer from
+ /// the retrieved snippets (or uses a server-provided `answer` field when
+ /// present) and grades it against the gold answer with an LLM judge.
+ /// Requires `--qa-provider` + `--qa-model` and a resolvable key; if the
+ /// key/provider cannot be resolved the run SKIPS QA (retrieval metrics
+ /// are unaffected) instead of failing.
+ #[arg(long)]
+ qa: bool,
+
+ /// QA answerer/judge provider
+ /// (`anthropic|openai|openai-compat|openai-oauth|codex|copilot|gemini`).
+ #[arg(long)]
+ qa_provider: Option<String>,
+
+ /// QA answerer model id.
+ #[arg(long)]
+ qa_model: Option<String>,
+
+ /// QA answerer base URL (omit for native vendors).
+ #[arg(long)]
+ qa_base_url: Option<String>,
+
+ /// QA answerer API key (raw value). Prefer `--qa-api-key-env` or the
+ /// provider's default env var.
+ #[arg(long)]
+ qa_api_key: Option<String>,
+
+ /// Env var to read the QA answerer API key from (defaults to the
+ /// provider's canonical env var, e.g. `GEMINI_API_KEY`).
+ #[arg(long)]
+ qa_api_key_env: Option<String>,
+
+ /// Auth file for a QA answerer using `openai-oauth`/`codex`/`copilot`.
+ #[arg(long)]
+ qa_token_file: Option<PathBuf>,
+
+ /// QA grader provider; defaults to `--qa-provider`.
+ #[arg(long)]
+ qa_grader_provider: Option<String>,
+
+ /// QA grader model; defaults to `--qa-model`.
+ #[arg(long)]
+ qa_grader_model: Option<String>,
+
+ /// QA grader base URL; defaults to `--qa-base-url`.
+ #[arg(long)]
+ qa_grader_base_url: Option<String>,
+
+ /// QA grader API key (raw); defaults to `--qa-api-key`.
+ #[arg(long)]
+ qa_grader_api_key: Option<String>,
+
+ /// Env var for the QA grader API key; defaults to `--qa-api-key-env`.
+ #[arg(long)]
+ qa_grader_api_key_env: Option<String>,
+
+ /// Auth file for the QA grader; defaults to `--qa-token-file`.
+ #[arg(long)]
+ qa_grader_token_file: Option<PathBuf>,
}
impl RetrievalArgs {
@@ -312,8 +375,37 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
}
}
+ // QA-accuracy mode (R2b): opt-in, live LLM. Resolve the answerer +
+ // grader once and share them across both configs. If QA is requested but
+ // a provider/key can't be resolved, SKIP cleanly (retrieval metrics are
+ // unaffected) rather than failing the run — mirroring live-LLM tests.
+ let qa_engine = if args.qa {
+ match qa::QaEngine::resolve(&args) {
+ Ok(engine) => {
+ println!("QA-accuracy mode ON (live LLM): {}", engine.summary());
+ tracing::info!(qa = engine.summary(), "QA-accuracy mode enabled");
+ Some(Arc::new(engine))
+ }
+ Err(e) => {
+ println!("QA-accuracy mode requested but SKIPPED: {e:#}");
+ tracing::warn!(error = %format!("{e:#}"), "QA-accuracy mode skipped");
+ None
+ }
+ }
+ } else {
+ None
+ };
+
let abstention_n = abstention.len();
- let baseline_report = run_config(&args, &baseline, &questions, abstention_n, &commit).await?;
+ let baseline_report = run_config(
+ &args,
+ &baseline,
+ &questions,
+ abstention_n,
+ &commit,
+ qa_engine.clone(),
+ )
+ .await?;
// Timestamped run dir shared by single-config and A/B outputs.
let stamp: String = Timestamp::now()
@@ -335,8 +427,15 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
println!("{md}");
}
Some(candidate) => {
- let candidate_report =
- run_config(&args, &candidate, &questions, abstention_n, &commit).await?;
+ let candidate_report = run_config(
+ &args,
+ &candidate,
+ &questions,
+ abstention_n,
+ &commit,
+ qa_engine.clone(),
+ )
+ .await?;
let ab = report::AbReport::new(baseline_report, candidate_report);
std::fs::write(run_dir.join("report.json"), serde_json::to_vec_pretty(&ab)?)?;
let md = report::ab_to_markdown(&ab);
@@ -358,6 +457,7 @@ async fn run_config(
questions: &[Question],
abstention_n: usize,
commit: &str,
+ qa: Option<Arc<qa::QaEngine>>,
) -> Result<report::Report> {
let server = EvalServer::launch(&args.server_bin, args.keep_data_dir, &knobs.launch_config())
.await
@@ -383,11 +483,12 @@ async fn run_config(
let mut tasks = tokio::task::JoinSet::new();
for q in questions.iter().cloned() {
let permit = semaphore.clone().acquire_owned().await?;
- let (client, base_url, ks, query_args) = (
+ let (client, base_url, ks, query_args, qa) = (
client.clone(),
base_url.clone(),
ks.clone(),
query_args.clone(),
+ qa.clone(),
);
tasks.spawn(async move {
let _permit = permit;
@@ -410,7 +511,7 @@ async fn run_config(
.iter()
.map(|s| stored_session_uuid(s))
.collect();
- let score = score::score_question(
+ let mut score = score::score_question(
&q.question_id,
&q.question_type,
&evidence,
@@ -419,6 +520,22 @@ async fn run_config(
outcome.latency.as_millis(),
outcome.context_tokens,
);
+ // QA-accuracy (R2b): answer + grade. A QA failure for one
+ // question is logged and left ungraded — it must not waste the
+ // whole retrieval run (retrieval numbers are already collected).
+ if let Some(engine) = qa.as_ref() {
+ match engine
+ .evaluate(&q.question, &q.gold_answer_text(), &outcome)
+ .await
+ {
+ Ok(qa_outcome) => score.qa = Some(qa_outcome),
+ Err(e) => tracing::warn!(
+ question = q.question_id,
+ error = %format!("{e:#}"),
+ "QA grading failed; question left ungraded"
+ ),
+ }
+ }
tracing::info!(
question = q.question_id,
events,
@@ -426,6 +543,7 @@ async fn run_config(
latency_ms = outcome.latency.as_millis() as u64,
context_tokens = outcome.context_tokens,
hit_at_max = score.hit_at.values().next_back().copied().unwrap_or(0.0),
+ qa_correct = score.qa.as_ref().map(|o| o.correct),
"scored"
);
anyhow::Ok(score)
@@ -468,6 +586,7 @@ async fn run_config(
questions_scored: scores.len(),
abstention_excluded: abstention_n,
ks: args.ks.clone(),
+ qa: qa.as_ref().map(|e| e.report_meta()),
slices,
per_question: scores,
})
diff --git a/evals/src/retrieval/qa.rs b/evals/src/retrieval/qa.rs
new file mode 100644
index 00000000..3b575df0
--- /dev/null
+++ b/evals/src/retrieval/qa.rs
@@ -0,0 +1,491 @@
+//! Opt-in, live-LLM QA-accuracy mode for the retrieval harness (R2b).
+//!
+//! R2a scores *retrieval* (did the evidence session surface in the top k).
+//! This module measures the next hop: end-to-end **answer quality**. Per
+//! scored question, with QA enabled, it
+//!
+//! 1. obtains a candidate answer — either the server's own `answer` field
+//! (future `answer=true` feature; used verbatim when present) or, by
+//! default, a harness synthesis call over the retrieved snippets (the
+//! exact context an agent would receive);
+//! 2. grades that candidate against the dataset's GOLD answer with an
+//! LLM-as-judge, following the LongMemEval convention (a boolean verdict
+//! plus a one-line reason);
+//! 3. records the verdict, the answer source, and the answer-path latency +
+//! token estimate, which [`super::score`] aggregates into QA accuracy
+//! alongside the R2a triple.
+//!
+//! This makes REAL provider API calls and is gated behind a key. It is OFF
+//! by default; when it is requested but a provider/key cannot be resolved,
+//! the caller SKIPS QA (retrieval metrics are unaffected) rather than
+//! failing the run. Provider construction and structured output reuse the
+//! same `ai-memory-llm` path the product uses; keys are read from env by the
+//! provider auth boundary and never logged, printed, or written to reports.
+
+use std::sync::Arc;
+use std::time::Instant;
+
+use anyhow::{Result, bail};
+use schemars::JsonSchema;
+use secrecy::SecretString;
+use serde::{Deserialize, Serialize};
+
+use ai_memory_llm::{
+ AuthRequirement, ChatMessage, ChatRequest, LlmProvider, ProviderAuth, ProviderChoice,
+ ProviderConfig, Role, complete_structured,
+};
+
+use super::RetrievalArgs;
+use super::query::{HitContext, QueryOutcome};
+use super::report::QaReportMeta;
+
+/// chars/4 heuristic divisor, matching `query.rs`'s context-token estimate.
+const CHARS_PER_TOKEN: usize = 4;
+/// Cap on synthesized-answer output tokens (LongMemEval answers are short).
+const ANSWER_MAX_TOKENS: u32 = 512;
+/// Cap on grader output tokens (a verdict plus one sentence).
+const GRADE_MAX_TOKENS: u32 = 256;
+
+/// Per-question QA outcome: a candidate answer graded against the gold
+/// answer. Serialized into the per-question report; carries no key material.
+#[derive(Debug, Clone, Serialize)]
+pub struct QaOutcome {
+ /// LLM-judge verdict: did the candidate correctly answer the question?
+ pub correct: bool,
+ /// Where the candidate came from: `synthesized` (harness LLM over the
+ /// retrieved snippets) or `memory_query` (a server-provided `answer`).
+ pub answer_source: &'static str,
+ /// The candidate answer that was graded (kept for forensics).
+ pub answer: String,
+ /// One-line judge rationale.
+ pub grade_reason: String,
+ /// Wall-clock of the answer step, milliseconds. 0 when the answer came
+ /// straight from the `memory_query` result (no synthesis call was made).
+ pub answer_latency_ms: u128,
+ /// chars/4 estimate over the synthesis prompt + answer — the answer-path
+ /// analogue of the retrieval context-token estimate. 0 for a
+ /// server-provided answer.
+ pub answer_tokens: usize,
+}
+
+/// Answer synthesized from retrieved snippets (structured output).
+#[derive(Debug, Clone, Deserialize, JsonSchema)]
+struct SynthesizedAnswer {
+ /// The answer to the question, grounded strictly in the snippets, or
+ /// "I don't know." when the snippets do not contain it.
+ answer: String,
+}
+
+/// LLM-as-judge verdict (structured output), LongMemEval-style.
+#[derive(Debug, Clone, Deserialize, JsonSchema)]
+struct Grade {
+ /// True when the candidate answer is factually correct for the question,
+ /// judged against the gold answer.
+ correct: bool,
+ /// One-sentence justification for the verdict.
+ reason: String,
+}
+
+/// A resolved QA engine: the answerer + grader providers and their names,
+/// built once and shared (cloned `Arc`) across concurrent question tasks.
+pub struct QaEngine {
+ answerer: Arc<dyn LlmProvider>,
+ grader: Arc<dyn LlmProvider>,
+ answer_provider: String,
+ answer_model: String,
+ grader_provider: String,
+ grader_model: String,
+}
+
+impl QaEngine {
+ /// Resolve both providers from `--qa-*` args + env.
+ ///
+ /// Returns `Err` (which the caller turns into a clean SKIP) when QA is
+ /// requested but a provider is missing, an API key/token cannot be
+ /// resolved, or provider construction fails. The default env var for a
+ /// provider's key is used when the operator does not name one, so
+ /// `--qa --qa-provider gemini --qa-model <m>` works with `GEMINI_API_KEY`
+ /// already in the environment.
+ pub fn resolve(args: &RetrievalArgs) -> Result<Self> {
+ let Some(answer_provider_str) = args.qa_provider.as_deref() else {
+ bail!("QA mode needs --qa-provider (e.g. anthropic|gemini|openai|openai-compat)");
+ };
+ let Some(answer_model) = args.qa_model.as_deref() else {
+ bail!("QA mode needs --qa-model");
+ };
+ let answerer = build_llm_provider(
+ "answerer",
+ answer_provider_str,
+ answer_model,
+ args.qa_base_url.as_deref(),
+ args.qa_api_key.as_deref(),
+ args.qa_api_key_env.as_deref(),
+ args.qa_token_file.as_deref(),
+ )?;
+
+ // The grader defaults to the answerer's provider/model/key/URL (the
+ // common LongMemEval setup: one judge for everything), each field
+ // overridable independently.
+ let grader_provider_str = args
+ .qa_grader_provider
+ .as_deref()
+ .unwrap_or(answer_provider_str);
+ let grader_model = args.qa_grader_model.as_deref().unwrap_or(answer_model);
+ let grader = build_llm_provider(
+ "grader",
+ grader_provider_str,
+ grader_model,
+ args.qa_grader_base_url.or_ref(&args.qa_base_url),
+ args.qa_grader_api_key.or_ref(&args.qa_api_key),
+ args.qa_grader_api_key_env.or_ref(&args.qa_api_key_env),
+ args.qa_grader_token_file
+ .as_deref()
+ .or(args.qa_token_file.as_deref()),
+ )?;
+
+ Ok(Self {
+ answer_provider: answerer.name().to_string(),
+ answer_model: answerer.model().to_string(),
+ grader_provider: grader.name().to_string(),
+ grader_model: grader.model().to_string(),
+ answerer,
+ grader,
+ })
+ }
+
+ /// Report provenance (names only — never keys).
+ pub fn report_meta(&self) -> QaReportMeta {
+ QaReportMeta {
+ answer_provider: self.answer_provider.clone(),
+ answer_model: self.answer_model.clone(),
+ grader_provider: self.grader_provider.clone(),
+ grader_model: self.grader_model.clone(),
+ }
+ }
+
+ /// Short one-line description for the run header (names only).
+ pub fn summary(&self) -> String {
+ format!(
+ "answerer={} {} | grader={} {}",
+ self.answer_provider, self.answer_model, self.grader_provider, self.grader_model
+ )
+ }
+
+ /// Produce and grade a candidate answer for one question.
+ pub async fn evaluate(
+ &self,
+ question: &str,
+ gold_answer: &str,
+ outcome: &QueryOutcome,
+ ) -> Result<QaOutcome> {
+ let (answer, answer_source, answer_latency_ms, answer_tokens) = match &outcome.answer {
+ // Future `answer=true` feature: the server already produced the
+ // answer, so grade it directly (no synthesis call).
+ Some(server_answer) => (server_answer.clone(), "memory_query", 0u128, 0usize),
+ None => {
+ let synth = self.synthesize(question, &outcome.contexts).await?;
+ (synth.answer, "synthesized", synth.latency_ms, synth.tokens)
+ }
+ };
+ let grade = self.grade(question, gold_answer, &answer).await?;
+ Ok(QaOutcome {
+ correct: grade.correct,
+ answer_source,
+ answer,
+ grade_reason: grade.reason,
+ answer_latency_ms,
+ answer_tokens,
+ })
+ }
+
+ /// Synthesize an answer from the retrieved snippets (the context an agent
+ /// would actually get), measuring latency and a chars/4 token estimate.
+ async fn synthesize(&self, question: &str, contexts: &[HitContext]) -> Result<SynthResult> {
+ let user = synthesis_user_prompt(question, contexts);
+ let prompt_chars = SYNTHESIS_SYSTEM.chars().count() + user.chars().count();
+ let request = ChatRequest {
+ system: Some(SYNTHESIS_SYSTEM.to_string()),
+ messages: vec![ChatMessage {
+ role: Role::User,
+ content: user,
+ }],
+ max_tokens: ANSWER_MAX_TOKENS,
+ temperature: Some(0.0),
+ };
+ let started = Instant::now();
+ let out: SynthesizedAnswer = complete_structured(self.answerer.as_ref(), request).await?;
+ let latency_ms = started.elapsed().as_millis();
+ let tokens = (prompt_chars + out.answer.chars().count()).div_ceil(CHARS_PER_TOKEN);
+ Ok(SynthResult {
+ answer: out.answer,
+ latency_ms,
+ tokens,
+ })
+ }
+
+ /// Grade a candidate answer against the gold answer (LLM-as-judge).
+ async fn grade(&self, question: &str, gold: &str, candidate: &str) -> Result<Grade> {
+ let request = ChatRequest {
+ system: Some(GRADE_SYSTEM.to_string()),
+ messages: vec![ChatMessage {
+ role: Role::User,
+ content: grade_user_prompt(question, gold, candidate),
+ }],
+ max_tokens: GRADE_MAX_TOKENS,
+ temperature: Some(0.0),
+ };
+ let grade: Grade = complete_structured(self.grader.as_ref(), request).await?;
+ Ok(grade)
+ }
+}
+
+/// Internal synthesis result carrier.
+struct SynthResult {
+ answer: String,
+ latency_ms: u128,
+ tokens: usize,
+}
+
+/// Convenience: borrow an inner `Option<String>` as `Option<&str>`, falling
+/// back to a default, so grader overrides read cleanly.
+trait OrRef {
+ fn or_ref<'a>(&'a self, fallback: &'a Option<String>) -> Option<&'a str>;
+}
+impl OrRef for Option<String> {
+ fn or_ref<'a>(&'a self, fallback: &'a Option<String>) -> Option<&'a str> {
+ self.as_deref().or(fallback.as_deref())
+ }
+}
+
+const SYNTHESIS_SYSTEM: &str = "You answer a user's question using ONLY the memory snippets \
+ retrieved for them. Base your answer strictly on those snippets — do not use outside \
+ knowledge or guess. Be concise and factual. If the snippets do not contain the answer, \
+ reply exactly: I don't know.";
+
+const GRADE_SYSTEM: &str = "You are a strict grader for a long-term-memory QA benchmark. Given a \
+ question, the gold (reference) answer, and a candidate answer, decide whether the candidate \
+ is CORRECT. Judge factual correctness, not wording: a paraphrase or extra detail is fine \
+ when it matches the gold answer's facts. Mark it incorrect when it contradicts the gold \
+ answer, omits the key fact the question asks for, or declines to answer. Return a boolean \
+ `correct` and a one-sentence `reason`.";
+
+/// Build the synthesis user prompt: the question plus ranked snippets.
+fn synthesis_user_prompt(question: &str, contexts: &[HitContext]) -> String {
+ let mut prompt = format!("Question:\n{question}\n\n");
+ if contexts.is_empty() {
+ prompt.push_str("Retrieved memory snippets: (none)\n");
+ } else {
+ prompt.push_str("Retrieved memory snippets (ranked, best first):\n");
+ for (i, c) in contexts.iter().enumerate() {
+ prompt.push_str(&format!("[{}] {}\n{}\n\n", i + 1, c.title, c.snippet));
+ }
+ }
+ prompt.push_str("\nAnswer the question using only the snippets above.");
+ prompt
+}
+
+/// Build the grader user prompt.
+fn grade_user_prompt(question: &str, gold: &str, candidate: &str) -> String {
+ format!(
+ "Question:\n{question}\n\nGold answer:\n{gold}\n\nCandidate answer:\n{candidate}\n\n\
+ Is the candidate answer correct?"
+ )
+}
+
+/// Resolve one provider from explicit fields (mirrors the `ab` harness's
+/// resolution, reused here for the QA answerer + grader). Fails closed when a
+/// required key/token is absent so the caller can SKIP cleanly.
+fn build_llm_provider(
+ role: &str,
+ provider_str: &str,
+ model: &str,
+ base_url: Option<&str>,
+ api_key_inline: Option<&str>,
+ api_key_env: Option<&str>,
+ token_file: Option<&std::path::Path>,
+) -> Result<Arc<dyn LlmProvider>> {
+ let provider = match provider_str {
+ "anthropic" => ProviderChoice::Anthropic,
+ "openai" => ProviderChoice::OpenAi,
+ "openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,
+ "openai-oauth" | "openai_oauth" => ProviderChoice::OpenAiOAuth,
+ "codex" => ProviderChoice::Codex,
+ "copilot" | "github-copilot" | "github_copilot" => ProviderChoice::Copilot,
+ "gemini" | "google" => ProviderChoice::Gemini,
+ other => bail!(
+ "{role}: provider `{other}` not one of \
+ anthropic|openai|openai-compat|openai-oauth|codex|copilot|gemini"
+ ),
+ };
+
+ // The env var to read the key from: operator override, else the
+ // provider's own canonical env var.
+ let requirement = provider.auth_requirement();
+ let default_env = match &requirement {
+ AuthRequirement::RequiredApiKey { env_var }
+ | AuthRequirement::OptionalApiKey { env_var } => Some(*env_var),
+ _ => None,
+ };
+ let eff_env = api_key_env.or(default_env);
+
+ let api_key = match api_key_inline {
+ Some(s) if !s.is_empty() => Some(SecretString::from(s.to_string())),
+ _ => eff_env.and_then(|env| match std::env::var(env) {
+ Ok(v) if !v.is_empty() => Some(SecretString::from(v)),
+ _ => None,
+ }),
+ };
+
+ let auth = match requirement {
+ AuthRequirement::RequiredApiKey { env_var } => {
+ if api_key.is_none() {
+ bail!(
+ "{role}: provider `{provider_str}` requires an API key; \
+ set {} (or pass --qa-api-key)",
+ eff_env.unwrap_or(env_var)
+ );
+ }
+ ProviderAuth::required_api_key_from_env(env_var, api_key)
+ }
+ AuthRequirement::OptionalApiKey { env_var } => {
+ ProviderAuth::optional_api_key_from_env(env_var, api_key)
+ }
+ AuthRequirement::OpenAiOAuthToken => {
+ let tf = token_file.ok_or_else(|| {
+ anyhow::anyhow!("{role}: --qa-token-file required for openai-oauth")
+ })?;
+ ProviderAuth::openai_oauth_token_file(tf.to_path_buf())
+ }
+ AuthRequirement::CodexAuthFile => {
+ let tf = token_file
+ .ok_or_else(|| anyhow::anyhow!("{role}: --qa-token-file required for codex"))?;
+ ProviderAuth::codex(tf.to_path_buf(), "codex")
+ }
+ AuthRequirement::CopilotToken => {
+ let tf = token_file
+ .ok_or_else(|| anyhow::anyhow!("{role}: --qa-token-file required for copilot"))?;
+ ProviderAuth::copilot(tf.to_path_buf(), None, None, base_url.map(str::to_string))
+ }
+ AuthRequirement::AnthropicOAuthToken => ProviderAuth::anthropic_oauth_token(api_key),
+ };
+
+ let config = ProviderConfig {
+ provider,
+ model: model.to_string(),
+ auth,
+ base_url: base_url.map(str::to_string),
+ compat_strict: true,
+ request_timeout_secs: ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS,
+ reasoning_effort: None,
+ extra_headers: ai_memory_llm::ExtraHeaders::default(),
+ };
+ ai_memory_llm::build_provider(config)
+ .map_err(anyhow::Error::from)
+ .map_err(|e| e.context(format!("{role}: building provider `{provider_str}`")))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn ctx(title: &str, snippet: &str) -> HitContext {
+ HitContext {
+ title: title.into(),
+ snippet: snippet.into(),
+ }
+ }
+
+ #[test]
+ fn synthesis_prompt_numbers_snippets_and_carries_the_question() {
+ let p = synthesis_user_prompt(
+ "What car?",
+ &[ctx("Car", "A blue Civic"), ctx("More", "2019")],
+ );
+ assert!(p.contains("Question:\nWhat car?"));
+ assert!(p.contains("[1] Car\nA blue Civic"));
+ assert!(p.contains("[2] More\n2019"));
+ assert!(p.contains("only the snippets"));
+ }
+
+ #[test]
+ fn synthesis_prompt_handles_no_snippets() {
+ let p = synthesis_user_prompt("Anything?", &[]);
+ assert!(p.contains("(none)"), "{p}");
+ }
+
+ #[test]
+ fn grade_prompt_lays_out_all_three_fields() {
+ let p = grade_user_prompt("Q?", "gold-x", "cand-y");
+ assert!(p.contains("Question:\nQ?"));
+ assert!(p.contains("Gold answer:\ngold-x"));
+ assert!(p.contains("Candidate answer:\ncand-y"));
+ }
+
+ #[test]
+ fn unknown_provider_is_rejected() {
+ let err = build_llm_provider("answerer", "nope", "m", None, None, None, None)
+ .map(|_| ())
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("not one of"), "{err}");
+ }
+
+ #[test]
+ fn required_key_provider_without_a_key_fails_closed() {
+ // Point at an env var that is (almost certainly) unset, and pass no
+ // inline key: a required-key provider must refuse so the caller skips.
+ let err = build_llm_provider(
+ "answerer",
+ "gemini",
+ "gemini-2.5-flash",
+ None,
+ None,
+ Some("AI_MEMORY_EVAL_QA_DEFINITELY_UNSET_KEY"),
+ None,
+ )
+ .map(|_| ())
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("requires an API key"), "{err}");
+ }
+
+ /// Live end-to-end QA smoke over a hand-made retrieval outcome (no eval
+ /// server needed). `#[ignore]` so CI without keys stays green; even when
+ /// run explicitly it skips cleanly when no key is present.
+ #[tokio::test]
+ #[ignore = "live LLM: requires GEMINI_API_KEY or ANTHROPIC_API_KEY"]
+ async fn live_synthesize_and_grade_smoke() {
+ let (provider, model, key_env) = if std::env::var("GEMINI_API_KEY").is_ok() {
+ ("gemini", "gemini-2.5-flash", "GEMINI_API_KEY")
+ } else if std::env::var("ANTHROPIC_API_KEY").is_ok() {
+ ("anthropic", "claude-3-5-haiku-latest", "ANTHROPIC_API_KEY")
+ } else {
+ eprintln!("skipping: no GEMINI_API_KEY / ANTHROPIC_API_KEY in env");
+ return;
+ };
+ let answerer =
+ build_llm_provider("answerer", provider, model, None, None, Some(key_env), None)
+ .expect("provider builds with a key present");
+ let engine = QaEngine {
+ answer_provider: answerer.name().to_string(),
+ answer_model: answerer.model().to_string(),
+ grader_provider: answerer.name().to_string(),
+ grader_model: answerer.model().to_string(),
+ grader: answerer.clone(),
+ answerer,
+ };
+ let outcome = QueryOutcome {
+ retrieved: vec![],
+ contexts: vec![ctx("Pet", "The user has a dog named Rex.")],
+ answer: None,
+ latency: std::time::Duration::ZERO,
+ context_tokens: 0,
+ };
+ let qa = engine
+ .evaluate("What is the user's dog's name?", "Rex", &outcome)
+ .await
+ .expect("QA evaluate succeeds with a live provider");
+ assert_eq!(qa.answer_source, "synthesized");
+ assert!(qa.correct, "expected the judge to accept: {qa:?}");
+ }
+}
diff --git a/evals/src/retrieval/query.rs b/evals/src/retrieval/query.rs
index fd73e457..d0d82c88 100644
--- a/evals/src/retrieval/query.rs
+++ b/evals/src/retrieval/query.rs
@@ -20,6 +20,17 @@ pub struct Retrieved {
pub session_uuid: Option<uuid::Uuid>,
}
+/// The title + snippet text of one retrieval hit, in rank order. This is
+/// exactly what `memory_query` puts in front of the agent, and the context
+/// the R2b QA answer-synthesizer reads to produce a candidate answer.
+#[derive(Debug, Clone)]
+pub struct HitContext {
+ /// Hit title (page title or raw observation title).
+ pub title: String,
+ /// Hit snippet (the bounded excerpt the agent would read).
+ pub snippet: String,
+}
+
/// Outcome of one `memory_query` call: the ranked hits plus the two
/// R2 observability signals — how long the round trip took and how much
/// context the agent would have to read to consume the result.
@@ -27,6 +38,16 @@ pub struct Retrieved {
pub struct QueryOutcome {
/// Flattened ranked session attributions (index 0 = best).
pub retrieved: Vec<Retrieved>,
+ /// Ranked hit contexts (title + snippet), same order as `retrieved`.
+ /// The QA answer-synthesizer (R2b) reads these; the R2a scoring path
+ /// ignores them, so the default zero-LLM run is unaffected.
+ pub contexts: Vec<HitContext>,
+ /// A server-provided answer, when a (future) `answer=true` feature
+ /// populates a top-level `answer` field on the `memory_query` result.
+ /// `None` today; the QA path falls back to harness synthesis when it is
+ /// absent or blank. Detecting it now future-proofs R2b for the B4
+ /// `answer`/`reasoning` MCP features without a harness change.
+ pub answer: Option<String>,
/// Wall-clock of the `memory_query` MCP round trip (request send →
/// response parsed).
pub latency: Duration,
@@ -45,6 +66,11 @@ struct QueryResponse {
hits: Vec<PageHitLite>,
#[serde(default)]
raw_hits: Vec<RawHitLite>,
+ /// Optional server-synthesized answer (future `answer=true` feature).
+ /// Unknown fields are ignored by serde, so this is forward-compatible:
+ /// today's server simply never sets it.
+ #[serde(default)]
+ answer: Option<String>,
}
#[derive(Debug, Deserialize)]
@@ -163,32 +189,52 @@ pub async fn memory_query(
.ok_or_else(|| anyhow::anyhow!("memory_query returned no text content: {result}"))?;
let parsed: QueryResponse =
serde_json::from_str(text).with_context(|| format!("parsing memory_query JSON: {text}"))?;
- let (retrieved, context_tokens) = flatten(parsed);
+ // A blank/whitespace `answer` is treated as absent so the QA path falls
+ // back to harness synthesis rather than grading an empty string.
+ let answer = parsed
+ .answer
+ .as_deref()
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .map(str::to_owned);
+ let (retrieved, contexts, context_tokens) = flatten(parsed);
Ok(QueryOutcome {
retrieved,
+ contexts,
+ answer,
latency: started.elapsed(),
context_tokens,
})
}
-/// Flatten pages + raw hits into ranked session attributions, and sum the
-/// context-token estimate over every hit's `title` + `snippet`.
-fn flatten(resp: QueryResponse) -> (Vec<Retrieved>, usize) {
+/// Flatten pages + raw hits into ranked session attributions and their
+/// title+snippet contexts, and sum the context-token estimate over every
+/// hit's `title` + `snippet`.
+fn flatten(resp: QueryResponse) -> (Vec<Retrieved>, Vec<HitContext>, usize) {
let mut out = Vec::new();
+ let mut contexts = Vec::new();
let mut chars = 0usize;
for hit in resp.hits {
chars += hit.title.chars().count() + hit.snippet.chars().count();
out.push(Retrieved {
session_uuid: session_uuid_from_path(&hit.path),
});
+ contexts.push(HitContext {
+ title: hit.title,
+ snippet: hit.snippet,
+ });
}
for hit in resp.raw_hits {
chars += hit.title.chars().count() + hit.snippet.chars().count();
out.push(Retrieved {
session_uuid: Some(hit.session_id),
});
+ contexts.push(HitContext {
+ title: hit.title,
+ snippet: hit.snippet,
+ });
}
- (out, estimate_tokens(chars))
+ (out, contexts, estimate_tokens(chars))
}
/// `sessions/<uuid>.md` → the owning session; anything else → None.
@@ -227,11 +273,14 @@ mod tests {
title: String::new(),
snippet: String::new(),
}],
+ answer: None,
};
- let (flat, _) = flatten(resp);
+ let (flat, contexts, _) = flatten(resp);
assert_eq!(flat.len(), 2);
assert_eq!(flat[0].session_uuid, Some(a));
assert_eq!(flat[1].session_uuid, Some(b));
+ // Contexts track the flattened hits one-for-one, in the same order.
+ assert_eq!(contexts.len(), 2);
}
#[test]
@@ -251,9 +300,14 @@ mod tests {
title: "ijkl".into(),
snippet: "mnop".into(),
}],
+ answer: None,
};
- let (_, tokens) = flatten(resp);
+ let (_, contexts, tokens) = flatten(resp);
assert_eq!(tokens, 4);
+ // Snippets are preserved verbatim for the QA synthesizer.
+ assert_eq!(contexts[0].title, "abcd");
+ assert_eq!(contexts[0].snippet, "efgh");
+ assert_eq!(contexts[1].snippet, "mnop");
}
#[test]
diff --git a/evals/src/retrieval/report.rs b/evals/src/retrieval/report.rs
index 390de7fd..00632e59 100644
--- a/evals/src/retrieval/report.rs
+++ b/evals/src/retrieval/report.rs
@@ -29,10 +29,28 @@ pub struct Report {
pub questions_scored: usize,
pub abstention_excluded: usize,
pub ks: Vec<usize>,
+ /// QA-accuracy provenance (R2b): which provider/model answered and which
+ /// graded. Names only — never keys. `None` on the default zero-LLM path.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub qa: Option<QaReportMeta>,
pub slices: BTreeMap<String, SliceMetrics>,
pub per_question: Vec<QuestionScore>,
}
+/// QA-accuracy provenance for a run (R2b). Provider + model names only, so a
+/// published report never carries a key.
+#[derive(Debug, Clone, Serialize)]
+pub struct QaReportMeta {
+ /// Provider that synthesized/answered (e.g. `gemini`).
+ pub answer_provider: String,
+ /// Model that answered (e.g. `gemini-2.5-flash`).
+ pub answer_model: String,
+ /// Provider that graded (LLM-as-judge).
+ pub grader_provider: String,
+ /// Model that graded.
+ pub grader_model: String,
+}
+
pub fn commit_sha() -> String {
std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
@@ -104,6 +122,45 @@ fn slices_table_md(r: &Report) -> String {
md
}
+/// The per-slice QA-accuracy table (R2b). Only slices with graded questions
+/// contribute rows; returns an empty string when the run had no QA at all.
+fn qa_table_md(r: &Report) -> String {
+ let Some(meta) = &r.qa else {
+ return String::new();
+ };
+ let mut md = String::new();
+ md.push_str(&format!(
+ "\n### QA accuracy (live LLM)\n\n- answered by: {} `{}`\n- graded by: {} `{}`\n\n",
+ meta.answer_provider, meta.answer_model, meta.grader_provider, meta.grader_model,
+ ));
+ md.push_str(
+ "| slice | graded | correct | accuracy | ans p50 ms | ans p95 ms | ans tok mean |\n",
+ );
+ md.push_str("|---|---|---|---|---|---|---|\n");
+ for (name, m) in &r.slices {
+ if let Some(qa) = &m.qa {
+ md.push_str(&format!(
+ "| {name} | {} | {} | {:.3} | {} | {} | {:.1} |\n",
+ qa.graded,
+ qa.correct,
+ qa.accuracy,
+ qa.answer_latency_p50_ms,
+ qa.answer_latency_p95_ms,
+ qa.answer_tokens_mean,
+ ));
+ }
+ }
+ md
+}
+
+/// `overall`-slice QA accuracy, when the run graded any question.
+fn overall_qa_accuracy(r: &Report) -> Option<f64> {
+ r.slices
+ .get("overall")
+ .and_then(|m| m.qa.as_ref())
+ .map(|q| q.accuracy)
+}
+
const METRIC_NOTES: &str = "\nNotes: hit@k = any evidence session in top k (the \"Recall@k\" most \
systems publish); recall@k = fraction of evidence sessions found. \
Latency is the `memory_query` MCP round trip (send→parse); ctx tok is \
@@ -121,6 +178,7 @@ pub fn to_markdown(r: &Report) -> String {
md.push_str(&provenance_md(r));
md.push_str(&format!("- mode: {}\n- config: {}\n\n", r.mode, r.config));
md.push_str(&slices_table_md(r));
+ md.push_str(&qa_table_md(r));
md.push_str(METRIC_NOTES);
md
}
@@ -144,6 +202,10 @@ pub struct AbDelta {
pub latency_p95_ms: i128,
pub context_tokens_mean: f64,
pub context_tokens_median: f64,
+ /// candidate − baseline QA accuracy over the `overall` slice (R2b).
+ /// `None` unless both runs graded the overall slice.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub qa_accuracy: Option<f64>,
}
impl AbReport {
@@ -180,6 +242,13 @@ fn compute_delta(baseline: &Report, candidate: &Report) -> AbDelta {
latency_p95_ms: c.latency_p95_ms as i128 - b.latency_p95_ms as i128,
context_tokens_mean: c.context_tokens_mean - b.context_tokens_mean,
context_tokens_median: c.context_tokens_median - b.context_tokens_median,
+ qa_accuracy: match (
+ overall_qa_accuracy(baseline),
+ overall_qa_accuracy(candidate),
+ ) {
+ (Some(b), Some(c)) => Some(c - b),
+ _ => None,
+ },
}
}
@@ -198,8 +267,10 @@ pub fn ab_to_markdown(r: &AbReport) -> String {
md.push_str("## Baseline\n\n");
md.push_str(&slices_table_md(&r.baseline));
+ md.push_str(&qa_table_md(&r.baseline));
md.push_str("\n## Candidate\n\n");
md.push_str(&slices_table_md(&r.candidate));
+ md.push_str(&qa_table_md(&r.candidate));
// Delta table over the `overall` slice.
let b = overall(&r.baseline);
@@ -234,6 +305,12 @@ pub fn ab_to_markdown(r: &AbReport) -> String {
"| ctx tok median | {:.1} | {:.1} | {:+.1} |\n",
b.context_tokens_median, c.context_tokens_median, r.delta.context_tokens_median
));
+ if let (Some(bq), Some(cq), Some(dq)) = (b.qa.as_ref(), c.qa.as_ref(), r.delta.qa_accuracy) {
+ md.push_str(&format!(
+ "| qa accuracy | {:.3} | {:.3} | {:+.3} |\n",
+ bq.accuracy, cq.accuracy, dq
+ ));
+ }
md.push_str(METRIC_NOTES);
md
}
@@ -251,6 +328,7 @@ mod tests {
latency_p95_ms: 34,
context_tokens_mean: 100.0,
context_tokens_median: 90.0,
+ qa: None,
}
}
@@ -268,6 +346,7 @@ mod tests {
questions_scored: 2,
abstention_excluded: 1,
ks: vec![5],
+ qa: None,
slices,
per_question: vec![],
}
@@ -302,6 +381,39 @@ mod tests {
assert!(md.contains("| hit@5 | 0.500 | 0.500 | +0.000 |"), "{md}");
}
+ #[test]
+ fn qa_table_renders_only_when_qa_ran_and_carries_no_key() {
+ use super::super::score::QaSliceMetrics;
+ let mut r = report("embeddings=none reranker=off", 0.5, 0.25);
+ // No QA → no QA section.
+ assert!(!to_markdown(&r).contains("QA accuracy"));
+ // Attach QA.
+ r.qa = Some(QaReportMeta {
+ answer_provider: "gemini".into(),
+ answer_model: "gemini-2.5-flash".into(),
+ grader_provider: "gemini".into(),
+ grader_model: "gemini-2.5-flash".into(),
+ });
+ r.slices.get_mut("overall").unwrap().qa = Some(QaSliceMetrics {
+ graded: 2,
+ correct: 1,
+ accuracy: 0.5,
+ answer_latency_p50_ms: 800,
+ answer_latency_p95_ms: 1200,
+ answer_tokens_mean: 42.0,
+ });
+ let md = to_markdown(&r);
+ assert!(md.contains("QA accuracy (live LLM)"), "{md}");
+ assert!(
+ md.contains("answered by: gemini `gemini-2.5-flash`"),
+ "{md}"
+ );
+ assert!(
+ md.contains("| overall | 2 | 1 | 0.500 | 800 | 1200 | 42.0 |"),
+ "{md}"
+ );
+ }
+
#[test]
fn ab_delta_reports_a_real_difference() {
let baseline = report("embeddings=none", 0.4, 0.20);
diff --git a/evals/src/retrieval/score.rs b/evals/src/retrieval/score.rs
index f305c9b0..b9c3a3cf 100644
--- a/evals/src/retrieval/score.rs
+++ b/evals/src/retrieval/score.rs
@@ -16,6 +16,7 @@ use std::collections::{BTreeMap, HashSet};
use serde::Serialize;
+use super::qa::QaOutcome;
use super::query::Retrieved;
/// Per-question outcome, ready for aggregation.
@@ -33,6 +34,11 @@ pub struct QuestionScore {
/// Estimated context tokens the result would cost the agent (chars/4
/// over returned hit titles + snippets).
pub context_tokens: usize,
+ /// End-to-end QA outcome (R2b, opt-in): a candidate answer graded
+ /// against the gold answer. `None` on the default zero-LLM path and for
+ /// any question whose QA step failed (logged, left ungraded).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub qa: Option<QaOutcome>,
}
/// Score one question's retrieval run.
@@ -82,6 +88,9 @@ pub fn score_question(
retrieved_sessions: ranked_sessions.len(),
latency_ms,
context_tokens,
+ // R2a scoring is answer-agnostic; the QA path attaches its outcome
+ // afterward, keeping this function pure and deterministic.
+ qa: None,
}
}
@@ -99,6 +108,51 @@ pub struct SliceMetrics {
/// Context-token estimate central tendency over the slice.
pub context_tokens_mean: f64,
pub context_tokens_median: f64,
+ /// End-to-end QA metrics over the slice (R2b). `None` unless QA mode ran
+ /// and at least one question in the slice was graded.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub qa: Option<QaSliceMetrics>,
+}
+
+/// Aggregated QA-accuracy metrics for one slice (R2b): fraction of graded
+/// answers the LLM judge marked correct, plus the answer-path latency and
+/// token estimates so the accuracy/latency/context triple stays meaningful
+/// for the synthesized-answer path, not only for retrieval.
+#[derive(Debug, Clone, Serialize)]
+pub struct QaSliceMetrics {
+ /// Questions in the slice that produced a graded answer.
+ pub graded: usize,
+ /// Of the graded, how many the judge marked correct.
+ pub correct: usize,
+ /// `correct / graded` (0.0 when nothing was graded).
+ pub accuracy: f64,
+ /// Answer-step latency percentiles over graded questions, milliseconds.
+ pub answer_latency_p50_ms: u128,
+ pub answer_latency_p95_ms: u128,
+ /// Mean answer-token estimate (chars/4 over synthesis prompt + answer).
+ pub answer_tokens_mean: f64,
+}
+
+impl QaSliceMetrics {
+ /// Aggregate the graded QA outcomes in one slice, or `None` when the
+ /// slice has no graded questions.
+ fn from_group(group: &[&QuestionScore]) -> Option<Self> {
+ let graded: Vec<&QaOutcome> = group.iter().filter_map(|s| s.qa.as_ref()).collect();
+ if graded.is_empty() {
+ return None;
+ }
+ let correct = graded.iter().filter(|o| o.correct).count();
+ let latencies: Vec<u128> = graded.iter().map(|o| o.answer_latency_ms).collect();
+ let tokens_sum: usize = graded.iter().map(|o| o.answer_tokens).sum();
+ Some(Self {
+ graded: graded.len(),
+ correct,
+ accuracy: correct as f64 / graded.len() as f64,
+ answer_latency_p50_ms: percentile_u128(&latencies, 0.50),
+ answer_latency_p95_ms: percentile_u128(&latencies, 0.95),
+ answer_tokens_mean: tokens_sum as f64 / graded.len() as f64,
+ })
+ }
}
/// Nearest-rank percentile (`p` in 0.0..=1.0) over a copy-sorted slice.
@@ -161,6 +215,7 @@ pub fn aggregate(scores: &[QuestionScore], ks: &[usize]) -> BTreeMap<String, Sli
latency_p95_ms: percentile_u128(&latencies, 0.95),
context_tokens_mean: tokens_sum as f64 / n,
context_tokens_median: median_f64(&tokens),
+ qa: QaSliceMetrics::from_group(&group),
},
)
})
@@ -320,6 +375,77 @@ mod tests {
assert_eq!(agg["overall"].latency_p95_ms, 30);
}
+ fn qa(correct: bool, latency_ms: u128, tokens: usize) -> QaOutcome {
+ QaOutcome {
+ correct,
+ answer_source: "synthesized",
+ answer: "a".into(),
+ grade_reason: "r".into(),
+ answer_latency_ms: latency_ms,
+ answer_tokens: tokens,
+ }
+ }
+
+ #[test]
+ fn qa_aggregates_only_over_graded_questions() {
+ let e = uuid::Uuid::now_v7();
+ let mut graded = score_question(
+ "q1",
+ "single-session-user",
+ &HashSet::from([e]),
+ &retrieved(&[Some(e)]),
+ &[1],
+ 10,
+ 100,
+ );
+ graded.qa = Some(qa(true, 800, 40));
+ let mut also_graded = score_question(
+ "q2",
+ "single-session-user",
+ &HashSet::from([uuid::Uuid::now_v7()]),
+ &retrieved(&[]),
+ &[1],
+ 20,
+ 50,
+ );
+ also_graded.qa = Some(qa(false, 1200, 60));
+ // A third question with QA off (None) must not count toward graded.
+ let ungraded = score_question(
+ "q3",
+ "single-session-user",
+ &HashSet::from([uuid::Uuid::now_v7()]),
+ &retrieved(&[]),
+ &[1],
+ 30,
+ 10,
+ );
+
+ let agg = aggregate(&[graded, also_graded, ungraded], &[1]);
+ let qa = agg["overall"].qa.as_ref().expect("overall has QA");
+ assert_eq!(qa.graded, 2);
+ assert_eq!(qa.correct, 1);
+ assert_eq!(qa.accuracy, 0.5);
+ assert_eq!(qa.answer_tokens_mean, 50.0);
+ assert_eq!(qa.answer_latency_p50_ms, 800);
+ assert_eq!(qa.answer_latency_p95_ms, 1200);
+ }
+
+ #[test]
+ fn qa_metrics_absent_when_no_question_was_graded() {
+ let e = uuid::Uuid::now_v7();
+ let s = score_question(
+ "q",
+ "single-session-user",
+ &HashSet::from([e]),
+ &retrieved(&[Some(e)]),
+ &[1],
+ 0,
+ 0,
+ );
+ let agg = aggregate(&[s], &[1]);
+ assert!(agg["overall"].qa.is_none());
+ }
+
#[test]
fn percentile_nearest_rank_and_empty() {
assert_eq!(percentile_u128(&[], 0.5), 0);