Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(evals): R2 QA-accuracy mode (LLM-as-judge) for end-to-end answer quality
AkitaOnRails7a8bed5a2b77
diff --git a/Cargo.lock b/Cargo.lockindex 3f1bf5dd..6d78842a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -130,6 +130,7 @@ dependencies = ["clap","jiff","reqwest 0.12.28",+ "schemars","secrecy","serde","serde_json",diff --git a/evals/Cargo.toml b/evals/Cargo.tomlindex 6ec9bada..fa284e8a 100644--- a/evals/Cargo.toml+++ b/evals/Cargo.toml@@ -21,6 +21,7 @@ reqwest.workspace = truesha2.workspace = truetempfile.workspace = truejiff.workspace = true+schemars.workspace = truesecrecy.workspace = trueserde.workspace = trueserde_json.workspace = truediff --git a/evals/README.md b/evals/README.mdindex 3f946de6..47319c8f 100644--- a/evals/README.md+++ b/evals/README.md@@ -252,6 +252,42 @@ Output: `evals/runs/<timestamp>-retrieval/report.{json,md}` — the JSON istriples and an `overall` delta table. Publish full-dataset A/B numbers in`docs/benchmarks/retrieval-ab-r2.md`.+### `retrieval` QA-accuracy mode (R2b, opt-in, live LLM)++R2a scores *retrieval*. `--qa` adds an opt-in **end-to-end answer-quality**+metric so the `answer`/`reasoning` MCP features can be proven later. It is+**OFF by default and makes REAL LLM API calls**, gated behind a key. Per+scored question, with QA on, the harness:++1. gets a candidate answer — the server's own `answer` field when a future+ `answer=true` feature populates it, otherwise a **synthesis** LLM call+ over the retrieved snippets (the exact context an agent would receive);+2. **grades** that candidate against the dataset's gold answer with an+ LLM-as-judge (`{correct, reason}`), LongMemEval-style;+3. reports **QA accuracy** (fraction correct) per slice — plus the+ answer-step latency (p50/p95) and answer-token estimate — alongside the+ R2a triple, in both the single-config and A/B reports. Provider+model+ names are recorded; **keys never are**.++Flags: `--qa`, `--qa-provider <name>`, `--qa-model <model>` (required to+enable), `--qa-base-url`, `--qa-api-key`/`--qa-api-key-env` (defaults to the+provider's canonical env var, e.g. `GEMINI_API_KEY`), `--qa-token-file`+(oauth/codex/copilot). The grader defaults to the answerer and is overridable+per field: `--qa-grader-provider`/`-model`/`-base-url`/`-api-key`/+`-api-key-env`/`-token-file`.++**Gating:** if `--qa` is set but the provider/key cannot be resolved, the run+prints a SKIP line and continues with QA off (retrieval metrics unaffected),+exiting success — so a keyless CI stays green. The default non-QA path stays+zero-LLM and deterministic.++```bash+# Keep keys in the process env only for the run; never echo a key.+set -a; source ~/.config/zsh/secrets; set +a # or export GEMINI_API_KEY=...+cargo run -p ai-memory-eval -- retrieval --sample 3 \+ --qa --qa-provider gemini --qa-model gemini-2.5-flash+```+Honest-numbers notes:- capture is production-shaped: excerpts are bounded at the 2 KBdiff --git a/evals/src/retrieval/dataset.rs b/evals/src/retrieval/dataset.rsindex c8153ee7..27166494 100644--- a/evals/src/retrieval/dataset.rs+++ b/evals/src/retrieval/dataset.rs@@ -32,7 +32,8 @@ pub struct Question {pub question_id: String,pub question_type: String,pub question: String,- #[allow(dead_code)]+ /// Gold answer. Usually a JSON string; occasionally a richer value.+ /// Read via [`Question::gold_answer_text`] for the QA judge (R2b).pub answer: serde_json::Value,#[allow(dead_code)]pub question_date: String,@@ -50,6 +51,16 @@ impl Question {pub fn is_abstention(&self) -> bool {self.question_id.ends_with("_abs")}++ /// The gold answer as text for the LLM-as-judge (R2b QA mode). A JSON+ /// string is unwrapped to its contents; any other shape is rendered as+ /// compact JSON so the judge still sees the reference value.+ pub fn gold_answer_text(&self) -> String {+ match &self.answer {+ serde_json::Value::String(s) => s.clone(),+ other => other.to_string(),+ }+ }}/// Load the dataset, verifying the pinned sha256 first.@@ -144,6 +155,14 @@ mod tests {assert!(!qs[0].is_abstention());}+ #[test]+ fn gold_answer_unwraps_a_json_string_and_renders_other_shapes() {+ let mut qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();+ assert_eq!(qs[0].gold_answer_text(), "Business");+ qs[0].answer = serde_json::json!(["a", "b"]);+ assert_eq!(qs[0].gold_answer_text(), "[\"a\",\"b\"]");+ }+#[test]fn abstention_is_detected_from_the_id_suffix() {let mut qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rsindex 088715b2..0a4cac74 100644--- a/evals/src/retrieval/mod.rs+++ b/evals/src/retrieval/mod.rs@@ -18,6 +18,7 @@mod dataset;mod ingest;+mod qa;mod query;mod report;mod score;@@ -122,6 +123,68 @@ pub struct RetrievalArgs {/// repeatable.#[arg(long = "candidate-query-arg", value_name = "KEY=JSON")]candidate_query_arg: Vec<String>,++ // ---- QA-accuracy mode (R2b, opt-in, live LLM) ----+ /// Enable end-to-end QA-accuracy grading (real LLM API calls). OFF by+ /// default. Per scored question the harness synthesizes an answer from+ /// the retrieved snippets (or uses a server-provided `answer` field when+ /// present) and grades it against the gold answer with an LLM judge.+ /// Requires `--qa-provider` + `--qa-model` and a resolvable key; if the+ /// key/provider cannot be resolved the run SKIPS QA (retrieval metrics+ /// are unaffected) instead of failing.+ #[arg(long)]+ qa: bool,++ /// QA answerer/judge provider+ /// (`anthropic|openai|openai-compat|openai-oauth|codex|copilot|gemini`).+ #[arg(long)]+ qa_provider: Option<String>,++ /// QA answerer model id.+ #[arg(long)]+ qa_model: Option<String>,++ /// QA answerer base URL (omit for native vendors).+ #[arg(long)]+ qa_base_url: Option<String>,++ /// QA answerer API key (raw value). Prefer `--qa-api-key-env` or the+ /// provider's default env var.+ #[arg(long)]+ qa_api_key: Option<String>,++ /// Env var to read the QA answerer API key from (defaults to the+ /// provider's canonical env var, e.g. `GEMINI_API_KEY`).+ #[arg(long)]+ qa_api_key_env: Option<String>,++ /// Auth file for a QA answerer using `openai-oauth`/`codex`/`copilot`.+ #[arg(long)]+ qa_token_file: Option<PathBuf>,++ /// QA grader provider; defaults to `--qa-provider`.+ #[arg(long)]+ qa_grader_provider: Option<String>,++ /// QA grader model; defaults to `--qa-model`.+ #[arg(long)]+ qa_grader_model: Option<String>,++ /// QA grader base URL; defaults to `--qa-base-url`.+ #[arg(long)]+ qa_grader_base_url: Option<String>,++ /// QA grader API key (raw); defaults to `--qa-api-key`.+ #[arg(long)]+ qa_grader_api_key: Option<String>,++ /// Env var for the QA grader API key; defaults to `--qa-api-key-env`.+ #[arg(long)]+ qa_grader_api_key_env: Option<String>,++ /// Auth file for the QA grader; defaults to `--qa-token-file`.+ #[arg(long)]+ qa_grader_token_file: Option<PathBuf>,}impl RetrievalArgs {@@ -312,8 +375,37 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {}}+ // QA-accuracy mode (R2b): opt-in, live LLM. Resolve the answerer ++ // grader once and share them across both configs. If QA is requested but+ // a provider/key can't be resolved, SKIP cleanly (retrieval metrics are+ // unaffected) rather than failing the run — mirroring live-LLM tests.+ let qa_engine = if args.qa {+ match qa::QaEngine::resolve(&args) {+ Ok(engine) => {+ println!("QA-accuracy mode ON (live LLM): {}", engine.summary());+ tracing::info!(qa = engine.summary(), "QA-accuracy mode enabled");+ Some(Arc::new(engine))+ }+ Err(e) => {+ println!("QA-accuracy mode requested but SKIPPED: {e:#}");+ tracing::warn!(error = %format!("{e:#}"), "QA-accuracy mode skipped");+ None+ }+ }+ } else {+ None+ };+let abstention_n = abstention.len();- let baseline_report = run_config(&args, &baseline, &questions, abstention_n, &commit).await?;+ let baseline_report = run_config(+ &args,+ &baseline,+ &questions,+ abstention_n,+ &commit,+ qa_engine.clone(),+ )+ .await?;// Timestamped run dir shared by single-config and A/B outputs.let stamp: String = Timestamp::now()@@ -335,8 +427,15 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {println!("{md}");}Some(candidate) => {- let candidate_report =- run_config(&args, &candidate, &questions, abstention_n, &commit).await?;+ let candidate_report = run_config(+ &args,+ &candidate,+ &questions,+ abstention_n,+ &commit,+ qa_engine.clone(),+ )+ .await?;let ab = report::AbReport::new(baseline_report, candidate_report);std::fs::write(run_dir.join("report.json"), serde_json::to_vec_pretty(&ab)?)?;let md = report::ab_to_markdown(&ab);@@ -358,6 +457,7 @@ async fn run_config(questions: &[Question],abstention_n: usize,commit: &str,+ qa: Option<Arc<qa::QaEngine>>,) -> Result<report::Report> {let server = EvalServer::launch(&args.server_bin, args.keep_data_dir, &knobs.launch_config()).await@@ -383,11 +483,12 @@ async fn run_config(let mut tasks = tokio::task::JoinSet::new();for q in questions.iter().cloned() {let permit = semaphore.clone().acquire_owned().await?;- let (client, base_url, ks, query_args) = (+ let (client, base_url, ks, query_args, qa) = (client.clone(),base_url.clone(),ks.clone(),query_args.clone(),+ qa.clone(),);tasks.spawn(async move {let _permit = permit;@@ -410,7 +511,7 @@ async fn run_config(.iter().map(|s| stored_session_uuid(s)).collect();- let score = score::score_question(+ let mut score = score::score_question(&q.question_id,&q.question_type,&evidence,@@ -419,6 +520,22 @@ async fn run_config(outcome.latency.as_millis(),outcome.context_tokens,);+ // QA-accuracy (R2b): answer + grade. A QA failure for one+ // question is logged and left ungraded — it must not waste the+ // whole retrieval run (retrieval numbers are already collected).+ if let Some(engine) = qa.as_ref() {+ match engine+ .evaluate(&q.question, &q.gold_answer_text(), &outcome)+ .await+ {+ Ok(qa_outcome) => score.qa = Some(qa_outcome),+ Err(e) => tracing::warn!(+ question = q.question_id,+ error = %format!("{e:#}"),+ "QA grading failed; question left ungraded"+ ),+ }+ }tracing::info!(question = q.question_id,events,@@ -426,6 +543,7 @@ async fn run_config(latency_ms = outcome.latency.as_millis() as u64,context_tokens = outcome.context_tokens,hit_at_max = score.hit_at.values().next_back().copied().unwrap_or(0.0),+ qa_correct = score.qa.as_ref().map(|o| o.correct),"scored");anyhow::Ok(score)@@ -468,6 +586,7 @@ async fn run_config(questions_scored: scores.len(),abstention_excluded: abstention_n,ks: args.ks.clone(),+ qa: qa.as_ref().map(|e| e.report_meta()),slices,per_question: scores,})diff --git a/evals/src/retrieval/qa.rs b/evals/src/retrieval/qa.rsnew file mode 100644index 00000000..3b575df0--- /dev/null+++ b/evals/src/retrieval/qa.rs@@ -0,0 +1,491 @@+//! Opt-in, live-LLM QA-accuracy mode for the retrieval harness (R2b).+//!+//! R2a scores *retrieval* (did the evidence session surface in the top k).+//! This module measures the next hop: end-to-end **answer quality**. Per+//! scored question, with QA enabled, it+//!+//! 1. obtains a candidate answer — either the server's own `answer` field+//! (future `answer=true` feature; used verbatim when present) or, by+//! default, a harness synthesis call over the retrieved snippets (the+//! exact context an agent would receive);+//! 2. grades that candidate against the dataset's GOLD answer with an+//! LLM-as-judge, following the LongMemEval convention (a boolean verdict+//! plus a one-line reason);+//! 3. records the verdict, the answer source, and the answer-path latency ++//! token estimate, which [`super::score`] aggregates into QA accuracy+//! alongside the R2a triple.+//!+//! This makes REAL provider API calls and is gated behind a key. It is OFF+//! by default; when it is requested but a provider/key cannot be resolved,+//! the caller SKIPS QA (retrieval metrics are unaffected) rather than+//! failing the run. Provider construction and structured output reuse the+//! same `ai-memory-llm` path the product uses; keys are read from env by the+//! provider auth boundary and never logged, printed, or written to reports.++use std::sync::Arc;+use std::time::Instant;++use anyhow::{Result, bail};+use schemars::JsonSchema;+use secrecy::SecretString;+use serde::{Deserialize, Serialize};++use ai_memory_llm::{+ AuthRequirement, ChatMessage, ChatRequest, LlmProvider, ProviderAuth, ProviderChoice,+ ProviderConfig, Role, complete_structured,+};++use super::RetrievalArgs;+use super::query::{HitContext, QueryOutcome};+use super::report::QaReportMeta;++/// chars/4 heuristic divisor, matching `query.rs`'s context-token estimate.+const CHARS_PER_TOKEN: usize = 4;+/// Cap on synthesized-answer output tokens (LongMemEval answers are short).+const ANSWER_MAX_TOKENS: u32 = 512;+/// Cap on grader output tokens (a verdict plus one sentence).+const GRADE_MAX_TOKENS: u32 = 256;++/// Per-question QA outcome: a candidate answer graded against the gold+/// answer. Serialized into the per-question report; carries no key material.+#[derive(Debug, Clone, Serialize)]+pub struct QaOutcome {+ /// LLM-judge verdict: did the candidate correctly answer the question?+ pub correct: bool,+ /// Where the candidate came from: `synthesized` (harness LLM over the+ /// retrieved snippets) or `memory_query` (a server-provided `answer`).+ pub answer_source: &'static str,+ /// The candidate answer that was graded (kept for forensics).+ pub answer: String,+ /// One-line judge rationale.+ pub grade_reason: String,+ /// Wall-clock of the answer step, milliseconds. 0 when the answer came+ /// straight from the `memory_query` result (no synthesis call was made).+ pub answer_latency_ms: u128,+ /// chars/4 estimate over the synthesis prompt + answer — the answer-path+ /// analogue of the retrieval context-token estimate. 0 for a+ /// server-provided answer.+ pub answer_tokens: usize,+}++/// Answer synthesized from retrieved snippets (structured output).+#[derive(Debug, Clone, Deserialize, JsonSchema)]+struct SynthesizedAnswer {+ /// The answer to the question, grounded strictly in the snippets, or+ /// "I don't know." when the snippets do not contain it.+ answer: String,+}++/// LLM-as-judge verdict (structured output), LongMemEval-style.+#[derive(Debug, Clone, Deserialize, JsonSchema)]+struct Grade {+ /// True when the candidate answer is factually correct for the question,+ /// judged against the gold answer.+ correct: bool,+ /// One-sentence justification for the verdict.+ reason: String,+}++/// A resolved QA engine: the answerer + grader providers and their names,+/// built once and shared (cloned `Arc`) across concurrent question tasks.+pub struct QaEngine {+ answerer: Arc<dyn LlmProvider>,+ grader: Arc<dyn LlmProvider>,+ answer_provider: String,+ answer_model: String,+ grader_provider: String,+ grader_model: String,+}++impl QaEngine {+ /// Resolve both providers from `--qa-*` args + env.+ ///+ /// Returns `Err` (which the caller turns into a clean SKIP) when QA is+ /// requested but a provider is missing, an API key/token cannot be+ /// resolved, or provider construction fails. The default env var for a+ /// provider's key is used when the operator does not name one, so+ /// `--qa --qa-provider gemini --qa-model <m>` works with `GEMINI_API_KEY`+ /// already in the environment.+ pub fn resolve(args: &RetrievalArgs) -> Result<Self> {+ let Some(answer_provider_str) = args.qa_provider.as_deref() else {+ bail!("QA mode needs --qa-provider (e.g. anthropic|gemini|openai|openai-compat)");+ };+ let Some(answer_model) = args.qa_model.as_deref() else {+ bail!("QA mode needs --qa-model");+ };+ let answerer = build_llm_provider(+ "answerer",+ answer_provider_str,+ answer_model,+ args.qa_base_url.as_deref(),+ args.qa_api_key.as_deref(),+ args.qa_api_key_env.as_deref(),+ args.qa_token_file.as_deref(),+ )?;++ // The grader defaults to the answerer's provider/model/key/URL (the+ // common LongMemEval setup: one judge for everything), each field+ // overridable independently.+ let grader_provider_str = args+ .qa_grader_provider+ .as_deref()+ .unwrap_or(answer_provider_str);+ let grader_model = args.qa_grader_model.as_deref().unwrap_or(answer_model);+ let grader = build_llm_provider(+ "grader",+ grader_provider_str,+ grader_model,+ args.qa_grader_base_url.or_ref(&args.qa_base_url),+ args.qa_grader_api_key.or_ref(&args.qa_api_key),+ args.qa_grader_api_key_env.or_ref(&args.qa_api_key_env),+ args.qa_grader_token_file+ .as_deref()+ .or(args.qa_token_file.as_deref()),+ )?;++ Ok(Self {+ answer_provider: answerer.name().to_string(),+ answer_model: answerer.model().to_string(),+ grader_provider: grader.name().to_string(),+ grader_model: grader.model().to_string(),+ answerer,+ grader,+ })+ }++ /// Report provenance (names only — never keys).+ pub fn report_meta(&self) -> QaReportMeta {+ QaReportMeta {+ answer_provider: self.answer_provider.clone(),+ answer_model: self.answer_model.clone(),+ grader_provider: self.grader_provider.clone(),+ grader_model: self.grader_model.clone(),+ }+ }++ /// Short one-line description for the run header (names only).+ pub fn summary(&self) -> String {+ format!(+ "answerer={} {} | grader={} {}",+ self.answer_provider, self.answer_model, self.grader_provider, self.grader_model+ )+ }++ /// Produce and grade a candidate answer for one question.+ pub async fn evaluate(+ &self,+ question: &str,+ gold_answer: &str,+ outcome: &QueryOutcome,+ ) -> Result<QaOutcome> {+ let (answer, answer_source, answer_latency_ms, answer_tokens) = match &outcome.answer {+ // Future `answer=true` feature: the server already produced the+ // answer, so grade it directly (no synthesis call).+ Some(server_answer) => (server_answer.clone(), "memory_query", 0u128, 0usize),+ None => {+ let synth = self.synthesize(question, &outcome.contexts).await?;+ (synth.answer, "synthesized", synth.latency_ms, synth.tokens)+ }+ };+ let grade = self.grade(question, gold_answer, &answer).await?;+ Ok(QaOutcome {+ correct: grade.correct,+ answer_source,+ answer,+ grade_reason: grade.reason,+ answer_latency_ms,+ answer_tokens,+ })+ }++ /// Synthesize an answer from the retrieved snippets (the context an agent+ /// would actually get), measuring latency and a chars/4 token estimate.+ async fn synthesize(&self, question: &str, contexts: &[HitContext]) -> Result<SynthResult> {+ let user = synthesis_user_prompt(question, contexts);+ let prompt_chars = SYNTHESIS_SYSTEM.chars().count() + user.chars().count();+ let request = ChatRequest {+ system: Some(SYNTHESIS_SYSTEM.to_string()),+ messages: vec![ChatMessage {+ role: Role::User,+ content: user,+ }],+ max_tokens: ANSWER_MAX_TOKENS,+ temperature: Some(0.0),+ };+ let started = Instant::now();+ let out: SynthesizedAnswer = complete_structured(self.answerer.as_ref(), request).await?;+ let latency_ms = started.elapsed().as_millis();+ let tokens = (prompt_chars + out.answer.chars().count()).div_ceil(CHARS_PER_TOKEN);+ Ok(SynthResult {+ answer: out.answer,+ latency_ms,+ tokens,+ })+ }++ /// Grade a candidate answer against the gold answer (LLM-as-judge).+ async fn grade(&self, question: &str, gold: &str, candidate: &str) -> Result<Grade> {+ let request = ChatRequest {+ system: Some(GRADE_SYSTEM.to_string()),+ messages: vec![ChatMessage {+ role: Role::User,+ content: grade_user_prompt(question, gold, candidate),+ }],+ max_tokens: GRADE_MAX_TOKENS,+ temperature: Some(0.0),+ };+ let grade: Grade = complete_structured(self.grader.as_ref(), request).await?;+ Ok(grade)+ }+}++/// Internal synthesis result carrier.+struct SynthResult {+ answer: String,+ latency_ms: u128,+ tokens: usize,+}++/// Convenience: borrow an inner `Option<String>` as `Option<&str>`, falling+/// back to a default, so grader overrides read cleanly.+trait OrRef {+ fn or_ref<'a>(&'a self, fallback: &'a Option<String>) -> Option<&'a str>;+}+impl OrRef for Option<String> {+ fn or_ref<'a>(&'a self, fallback: &'a Option<String>) -> Option<&'a str> {+ self.as_deref().or(fallback.as_deref())+ }+}++const SYNTHESIS_SYSTEM: &str = "You answer a user's question using ONLY the memory snippets \+ retrieved for them. Base your answer strictly on those snippets — do not use outside \+ knowledge or guess. Be concise and factual. If the snippets do not contain the answer, \+ reply exactly: I don't know.";++const GRADE_SYSTEM: &str = "You are a strict grader for a long-term-memory QA benchmark. Given a \+ question, the gold (reference) answer, and a candidate answer, decide whether the candidate \+ is CORRECT. Judge factual correctness, not wording: a paraphrase or extra detail is fine \+ when it matches the gold answer's facts. Mark it incorrect when it contradicts the gold \+ answer, omits the key fact the question asks for, or declines to answer. Return a boolean \+ `correct` and a one-sentence `reason`.";++/// Build the synthesis user prompt: the question plus ranked snippets.+fn synthesis_user_prompt(question: &str, contexts: &[HitContext]) -> String {+ let mut prompt = format!("Question:\n{question}\n\n");+ if contexts.is_empty() {+ prompt.push_str("Retrieved memory snippets: (none)\n");+ } else {+ prompt.push_str("Retrieved memory snippets (ranked, best first):\n");+ for (i, c) in contexts.iter().enumerate() {+ prompt.push_str(&format!("[{}] {}\n{}\n\n", i + 1, c.title, c.snippet));+ }+ }+ prompt.push_str("\nAnswer the question using only the snippets above.");+ prompt+}++/// Build the grader user prompt.+fn grade_user_prompt(question: &str, gold: &str, candidate: &str) -> String {+ format!(+ "Question:\n{question}\n\nGold answer:\n{gold}\n\nCandidate answer:\n{candidate}\n\n\+ Is the candidate answer correct?"+ )+}++/// Resolve one provider from explicit fields (mirrors the `ab` harness's+/// resolution, reused here for the QA answerer + grader). Fails closed when a+/// required key/token is absent so the caller can SKIP cleanly.+fn build_llm_provider(+ role: &str,+ provider_str: &str,+ model: &str,+ base_url: Option<&str>,+ api_key_inline: Option<&str>,+ api_key_env: Option<&str>,+ token_file: Option<&std::path::Path>,+) -> Result<Arc<dyn LlmProvider>> {+ let provider = match provider_str {+ "anthropic" => ProviderChoice::Anthropic,+ "openai" => ProviderChoice::OpenAi,+ "openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,+ "openai-oauth" | "openai_oauth" => ProviderChoice::OpenAiOAuth,+ "codex" => ProviderChoice::Codex,+ "copilot" | "github-copilot" | "github_copilot" => ProviderChoice::Copilot,+ "gemini" | "google" => ProviderChoice::Gemini,+ other => bail!(+ "{role}: provider `{other}` not one of \+ anthropic|openai|openai-compat|openai-oauth|codex|copilot|gemini"+ ),+ };++ // The env var to read the key from: operator override, else the+ // provider's own canonical env var.+ let requirement = provider.auth_requirement();+ let default_env = match &requirement {+ AuthRequirement::RequiredApiKey { env_var }+ | AuthRequirement::OptionalApiKey { env_var } => Some(*env_var),+ _ => None,+ };+ let eff_env = api_key_env.or(default_env);++ let api_key = match api_key_inline {+ Some(s) if !s.is_empty() => Some(SecretString::from(s.to_string())),+ _ => eff_env.and_then(|env| match std::env::var(env) {+ Ok(v) if !v.is_empty() => Some(SecretString::from(v)),+ _ => None,+ }),+ };++ let auth = match requirement {+ AuthRequirement::RequiredApiKey { env_var } => {+ if api_key.is_none() {+ bail!(+ "{role}: provider `{provider_str}` requires an API key; \+ set {} (or pass --qa-api-key)",+ eff_env.unwrap_or(env_var)+ );+ }+ ProviderAuth::required_api_key_from_env(env_var, api_key)+ }+ AuthRequirement::OptionalApiKey { env_var } => {+ ProviderAuth::optional_api_key_from_env(env_var, api_key)+ }+ AuthRequirement::OpenAiOAuthToken => {+ let tf = token_file.ok_or_else(|| {+ anyhow::anyhow!("{role}: --qa-token-file required for openai-oauth")+ })?;+ ProviderAuth::openai_oauth_token_file(tf.to_path_buf())+ }+ AuthRequirement::CodexAuthFile => {+ let tf = token_file+ .ok_or_else(|| anyhow::anyhow!("{role}: --qa-token-file required for codex"))?;+ ProviderAuth::codex(tf.to_path_buf(), "codex")+ }+ AuthRequirement::CopilotToken => {+ let tf = token_file+ .ok_or_else(|| anyhow::anyhow!("{role}: --qa-token-file required for copilot"))?;+ ProviderAuth::copilot(tf.to_path_buf(), None, None, base_url.map(str::to_string))+ }+ AuthRequirement::AnthropicOAuthToken => ProviderAuth::anthropic_oauth_token(api_key),+ };++ let config = ProviderConfig {+ provider,+ model: model.to_string(),+ auth,+ base_url: base_url.map(str::to_string),+ compat_strict: true,+ request_timeout_secs: ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS,+ reasoning_effort: None,+ extra_headers: ai_memory_llm::ExtraHeaders::default(),+ };+ ai_memory_llm::build_provider(config)+ .map_err(anyhow::Error::from)+ .map_err(|e| e.context(format!("{role}: building provider `{provider_str}`")))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn ctx(title: &str, snippet: &str) -> HitContext {+ HitContext {+ title: title.into(),+ snippet: snippet.into(),+ }+ }++ #[test]+ fn synthesis_prompt_numbers_snippets_and_carries_the_question() {+ let p = synthesis_user_prompt(+ "What car?",+ &[ctx("Car", "A blue Civic"), ctx("More", "2019")],+ );+ assert!(p.contains("Question:\nWhat car?"));+ assert!(p.contains("[1] Car\nA blue Civic"));+ assert!(p.contains("[2] More\n2019"));+ assert!(p.contains("only the snippets"));+ }++ #[test]+ fn synthesis_prompt_handles_no_snippets() {+ let p = synthesis_user_prompt("Anything?", &[]);+ assert!(p.contains("(none)"), "{p}");+ }++ #[test]+ fn grade_prompt_lays_out_all_three_fields() {+ let p = grade_user_prompt("Q?", "gold-x", "cand-y");+ assert!(p.contains("Question:\nQ?"));+ assert!(p.contains("Gold answer:\ngold-x"));+ assert!(p.contains("Candidate answer:\ncand-y"));+ }++ #[test]+ fn unknown_provider_is_rejected() {+ let err = build_llm_provider("answerer", "nope", "m", None, None, None, None)+ .map(|_| ())+ .unwrap_err()+ .to_string();+ assert!(err.contains("not one of"), "{err}");+ }++ #[test]+ fn required_key_provider_without_a_key_fails_closed() {+ // Point at an env var that is (almost certainly) unset, and pass no+ // inline key: a required-key provider must refuse so the caller skips.+ let err = build_llm_provider(+ "answerer",+ "gemini",+ "gemini-2.5-flash",+ None,+ None,+ Some("AI_MEMORY_EVAL_QA_DEFINITELY_UNSET_KEY"),+ None,+ )+ .map(|_| ())+ .unwrap_err()+ .to_string();+ assert!(err.contains("requires an API key"), "{err}");+ }++ /// Live end-to-end QA smoke over a hand-made retrieval outcome (no eval+ /// server needed). `#[ignore]` so CI without keys stays green; even when+ /// run explicitly it skips cleanly when no key is present.+ #[tokio::test]+ #[ignore = "live LLM: requires GEMINI_API_KEY or ANTHROPIC_API_KEY"]+ async fn live_synthesize_and_grade_smoke() {+ let (provider, model, key_env) = if std::env::var("GEMINI_API_KEY").is_ok() {+ ("gemini", "gemini-2.5-flash", "GEMINI_API_KEY")+ } else if std::env::var("ANTHROPIC_API_KEY").is_ok() {+ ("anthropic", "claude-3-5-haiku-latest", "ANTHROPIC_API_KEY")+ } else {+ eprintln!("skipping: no GEMINI_API_KEY / ANTHROPIC_API_KEY in env");+ return;+ };+ let answerer =+ build_llm_provider("answerer", provider, model, None, None, Some(key_env), None)+ .expect("provider builds with a key present");+ let engine = QaEngine {+ answer_provider: answerer.name().to_string(),+ answer_model: answerer.model().to_string(),+ grader_provider: answerer.name().to_string(),+ grader_model: answerer.model().to_string(),+ grader: answerer.clone(),+ answerer,+ };+ let outcome = QueryOutcome {+ retrieved: vec![],+ contexts: vec![ctx("Pet", "The user has a dog named Rex.")],+ answer: None,+ latency: std::time::Duration::ZERO,+ context_tokens: 0,+ };+ let qa = engine+ .evaluate("What is the user's dog's name?", "Rex", &outcome)+ .await+ .expect("QA evaluate succeeds with a live provider");+ assert_eq!(qa.answer_source, "synthesized");+ assert!(qa.correct, "expected the judge to accept: {qa:?}");+ }+}diff --git a/evals/src/retrieval/query.rs b/evals/src/retrieval/query.rsindex fd73e457..d0d82c88 100644--- a/evals/src/retrieval/query.rs+++ b/evals/src/retrieval/query.rs@@ -20,6 +20,17 @@ pub struct Retrieved {pub session_uuid: Option<uuid::Uuid>,}+/// The title + snippet text of one retrieval hit, in rank order. This is+/// exactly what `memory_query` puts in front of the agent, and the context+/// the R2b QA answer-synthesizer reads to produce a candidate answer.+#[derive(Debug, Clone)]+pub struct HitContext {+ /// Hit title (page title or raw observation title).+ pub title: String,+ /// Hit snippet (the bounded excerpt the agent would read).+ pub snippet: String,+}+/// Outcome of one `memory_query` call: the ranked hits plus the two/// R2 observability signals — how long the round trip took and how much/// context the agent would have to read to consume the result.@@ -27,6 +38,16 @@ pub struct Retrieved {pub struct QueryOutcome {/// Flattened ranked session attributions (index 0 = best).pub retrieved: Vec<Retrieved>,+ /// Ranked hit contexts (title + snippet), same order as `retrieved`.+ /// The QA answer-synthesizer (R2b) reads these; the R2a scoring path+ /// ignores them, so the default zero-LLM run is unaffected.+ pub contexts: Vec<HitContext>,+ /// A server-provided answer, when a (future) `answer=true` feature+ /// populates a top-level `answer` field on the `memory_query` result.+ /// `None` today; the QA path falls back to harness synthesis when it is+ /// absent or blank. Detecting it now future-proofs R2b for the B4+ /// `answer`/`reasoning` MCP features without a harness change.+ pub answer: Option<String>,/// Wall-clock of the `memory_query` MCP round trip (request send →/// response parsed).pub latency: Duration,@@ -45,6 +66,11 @@ struct QueryResponse {hits: Vec<PageHitLite>,#[serde(default)]raw_hits: Vec<RawHitLite>,+ /// Optional server-synthesized answer (future `answer=true` feature).+ /// Unknown fields are ignored by serde, so this is forward-compatible:+ /// today's server simply never sets it.+ #[serde(default)]+ answer: Option<String>,}#[derive(Debug, Deserialize)]@@ -163,32 +189,52 @@ pub async fn memory_query(.ok_or_else(|| anyhow::anyhow!("memory_query returned no text content: {result}"))?;let parsed: QueryResponse =serde_json::from_str(text).with_context(|| format!("parsing memory_query JSON: {text}"))?;- let (retrieved, context_tokens) = flatten(parsed);+ // A blank/whitespace `answer` is treated as absent so the QA path falls+ // back to harness synthesis rather than grading an empty string.+ let answer = parsed+ .answer+ .as_deref()+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .map(str::to_owned);+ let (retrieved, contexts, context_tokens) = flatten(parsed);Ok(QueryOutcome {retrieved,+ contexts,+ answer,latency: started.elapsed(),context_tokens,})}-/// Flatten pages + raw hits into ranked session attributions, and sum the-/// context-token estimate over every hit's `title` + `snippet`.-fn flatten(resp: QueryResponse) -> (Vec<Retrieved>, usize) {+/// Flatten pages + raw hits into ranked session attributions and their+/// title+snippet contexts, and sum the context-token estimate over every+/// hit's `title` + `snippet`.+fn flatten(resp: QueryResponse) -> (Vec<Retrieved>, Vec<HitContext>, usize) {let mut out = Vec::new();+ let mut contexts = Vec::new();let mut chars = 0usize;for hit in resp.hits {chars += hit.title.chars().count() + hit.snippet.chars().count();out.push(Retrieved {session_uuid: session_uuid_from_path(&hit.path),});+ contexts.push(HitContext {+ title: hit.title,+ snippet: hit.snippet,+ });}for hit in resp.raw_hits {chars += hit.title.chars().count() + hit.snippet.chars().count();out.push(Retrieved {session_uuid: Some(hit.session_id),});+ contexts.push(HitContext {+ title: hit.title,+ snippet: hit.snippet,+ });}- (out, estimate_tokens(chars))+ (out, contexts, estimate_tokens(chars))}/// `sessions/<uuid>.md` → the owning session; anything else → None.@@ -227,11 +273,14 @@ mod tests {title: String::new(),snippet: String::new(),}],+ answer: None,};- let (flat, _) = flatten(resp);+ let (flat, contexts, _) = flatten(resp);assert_eq!(flat.len(), 2);assert_eq!(flat[0].session_uuid, Some(a));assert_eq!(flat[1].session_uuid, Some(b));+ // Contexts track the flattened hits one-for-one, in the same order.+ assert_eq!(contexts.len(), 2);}#[test]@@ -251,9 +300,14 @@ mod tests {title: "ijkl".into(),snippet: "mnop".into(),}],+ answer: None,};- let (_, tokens) = flatten(resp);+ let (_, contexts, tokens) = flatten(resp);assert_eq!(tokens, 4);+ // Snippets are preserved verbatim for the QA synthesizer.+ assert_eq!(contexts[0].title, "abcd");+ assert_eq!(contexts[0].snippet, "efgh");+ assert_eq!(contexts[1].snippet, "mnop");}#[test]diff --git a/evals/src/retrieval/report.rs b/evals/src/retrieval/report.rsindex 390de7fd..00632e59 100644--- a/evals/src/retrieval/report.rs+++ b/evals/src/retrieval/report.rs@@ -29,10 +29,28 @@ pub struct Report {pub questions_scored: usize,pub abstention_excluded: usize,pub ks: Vec<usize>,+ /// QA-accuracy provenance (R2b): which provider/model answered and which+ /// graded. Names only — never keys. `None` on the default zero-LLM path.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub qa: Option<QaReportMeta>,pub slices: BTreeMap<String, SliceMetrics>,pub per_question: Vec<QuestionScore>,}+/// QA-accuracy provenance for a run (R2b). Provider + model names only, so a+/// published report never carries a key.+#[derive(Debug, Clone, Serialize)]+pub struct QaReportMeta {+ /// Provider that synthesized/answered (e.g. `gemini`).+ pub answer_provider: String,+ /// Model that answered (e.g. `gemini-2.5-flash`).+ pub answer_model: String,+ /// Provider that graded (LLM-as-judge).+ pub grader_provider: String,+ /// Model that graded.+ pub grader_model: String,+}+pub fn commit_sha() -> String {std::process::Command::new("git").args(["rev-parse", "HEAD"])@@ -104,6 +122,45 @@ fn slices_table_md(r: &Report) -> String {md}+/// The per-slice QA-accuracy table (R2b). Only slices with graded questions+/// contribute rows; returns an empty string when the run had no QA at all.+fn qa_table_md(r: &Report) -> String {+ let Some(meta) = &r.qa else {+ return String::new();+ };+ let mut md = String::new();+ md.push_str(&format!(+ "\n### QA accuracy (live LLM)\n\n- answered by: {} `{}`\n- graded by: {} `{}`\n\n",+ meta.answer_provider, meta.answer_model, meta.grader_provider, meta.grader_model,+ ));+ md.push_str(+ "| slice | graded | correct | accuracy | ans p50 ms | ans p95 ms | ans tok mean |\n",+ );+ md.push_str("|---|---|---|---|---|---|---|\n");+ for (name, m) in &r.slices {+ if let Some(qa) = &m.qa {+ md.push_str(&format!(+ "| {name} | {} | {} | {:.3} | {} | {} | {:.1} |\n",+ qa.graded,+ qa.correct,+ qa.accuracy,+ qa.answer_latency_p50_ms,+ qa.answer_latency_p95_ms,+ qa.answer_tokens_mean,+ ));+ }+ }+ md+}++/// `overall`-slice QA accuracy, when the run graded any question.+fn overall_qa_accuracy(r: &Report) -> Option<f64> {+ r.slices+ .get("overall")+ .and_then(|m| m.qa.as_ref())+ .map(|q| q.accuracy)+}+const METRIC_NOTES: &str = "\nNotes: hit@k = any evidence session in top k (the \"Recall@k\" most \systems publish); recall@k = fraction of evidence sessions found. \Latency is the `memory_query` MCP round trip (send→parse); ctx tok is \@@ -121,6 +178,7 @@ pub fn to_markdown(r: &Report) -> String {md.push_str(&provenance_md(r));md.push_str(&format!("- mode: {}\n- config: {}\n\n", r.mode, r.config));md.push_str(&slices_table_md(r));+ md.push_str(&qa_table_md(r));md.push_str(METRIC_NOTES);md}@@ -144,6 +202,10 @@ pub struct AbDelta {pub latency_p95_ms: i128,pub context_tokens_mean: f64,pub context_tokens_median: f64,+ /// candidate − baseline QA accuracy over the `overall` slice (R2b).+ /// `None` unless both runs graded the overall slice.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub qa_accuracy: Option<f64>,}impl AbReport {@@ -180,6 +242,13 @@ fn compute_delta(baseline: &Report, candidate: &Report) -> AbDelta {latency_p95_ms: c.latency_p95_ms as i128 - b.latency_p95_ms as i128,context_tokens_mean: c.context_tokens_mean - b.context_tokens_mean,context_tokens_median: c.context_tokens_median - b.context_tokens_median,+ qa_accuracy: match (+ overall_qa_accuracy(baseline),+ overall_qa_accuracy(candidate),+ ) {+ (Some(b), Some(c)) => Some(c - b),+ _ => None,+ },}}@@ -198,8 +267,10 @@ pub fn ab_to_markdown(r: &AbReport) -> String {md.push_str("## Baseline\n\n");md.push_str(&slices_table_md(&r.baseline));+ md.push_str(&qa_table_md(&r.baseline));md.push_str("\n## Candidate\n\n");md.push_str(&slices_table_md(&r.candidate));+ md.push_str(&qa_table_md(&r.candidate));// Delta table over the `overall` slice.let b = overall(&r.baseline);@@ -234,6 +305,12 @@ pub fn ab_to_markdown(r: &AbReport) -> String {"| ctx tok median | {:.1} | {:.1} | {:+.1} |\n",b.context_tokens_median, c.context_tokens_median, r.delta.context_tokens_median));+ if let (Some(bq), Some(cq), Some(dq)) = (b.qa.as_ref(), c.qa.as_ref(), r.delta.qa_accuracy) {+ md.push_str(&format!(+ "| qa accuracy | {:.3} | {:.3} | {:+.3} |\n",+ bq.accuracy, cq.accuracy, dq+ ));+ }md.push_str(METRIC_NOTES);md}@@ -251,6 +328,7 @@ mod tests {latency_p95_ms: 34,context_tokens_mean: 100.0,context_tokens_median: 90.0,+ qa: None,}}@@ -268,6 +346,7 @@ mod tests {questions_scored: 2,abstention_excluded: 1,ks: vec![5],+ qa: None,slices,per_question: vec![],}@@ -302,6 +381,39 @@ mod tests {assert!(md.contains("| hit@5 | 0.500 | 0.500 | +0.000 |"), "{md}");}+ #[test]+ fn qa_table_renders_only_when_qa_ran_and_carries_no_key() {+ use super::super::score::QaSliceMetrics;+ let mut r = report("embeddings=none reranker=off", 0.5, 0.25);+ // No QA → no QA section.+ assert!(!to_markdown(&r).contains("QA accuracy"));+ // Attach QA.+ r.qa = Some(QaReportMeta {+ answer_provider: "gemini".into(),+ answer_model: "gemini-2.5-flash".into(),+ grader_provider: "gemini".into(),+ grader_model: "gemini-2.5-flash".into(),+ });+ r.slices.get_mut("overall").unwrap().qa = Some(QaSliceMetrics {+ graded: 2,+ correct: 1,+ accuracy: 0.5,+ answer_latency_p50_ms: 800,+ answer_latency_p95_ms: 1200,+ answer_tokens_mean: 42.0,+ });+ let md = to_markdown(&r);+ assert!(md.contains("QA accuracy (live LLM)"), "{md}");+ assert!(+ md.contains("answered by: gemini `gemini-2.5-flash`"),+ "{md}"+ );+ assert!(+ md.contains("| overall | 2 | 1 | 0.500 | 800 | 1200 | 42.0 |"),+ "{md}"+ );+ }+#[test]fn ab_delta_reports_a_real_difference() {let baseline = report("embeddings=none", 0.4, 0.20);diff --git a/evals/src/retrieval/score.rs b/evals/src/retrieval/score.rsindex f305c9b0..b9c3a3cf 100644--- a/evals/src/retrieval/score.rs+++ b/evals/src/retrieval/score.rs@@ -16,6 +16,7 @@ use std::collections::{BTreeMap, HashSet};use serde::Serialize;+use super::qa::QaOutcome;use super::query::Retrieved;/// Per-question outcome, ready for aggregation.@@ -33,6 +34,11 @@ pub struct QuestionScore {/// Estimated context tokens the result would cost the agent (chars/4/// over returned hit titles + snippets).pub context_tokens: usize,+ /// End-to-end QA outcome (R2b, opt-in): a candidate answer graded+ /// against the gold answer. `None` on the default zero-LLM path and for+ /// any question whose QA step failed (logged, left ungraded).+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub qa: Option<QaOutcome>,}/// Score one question's retrieval run.@@ -82,6 +88,9 @@ pub fn score_question(retrieved_sessions: ranked_sessions.len(),latency_ms,context_tokens,+ // R2a scoring is answer-agnostic; the QA path attaches its outcome+ // afterward, keeping this function pure and deterministic.+ qa: None,}}@@ -99,6 +108,51 @@ pub struct SliceMetrics {/// Context-token estimate central tendency over the slice.pub context_tokens_mean: f64,pub context_tokens_median: f64,+ /// End-to-end QA metrics over the slice (R2b). `None` unless QA mode ran+ /// and at least one question in the slice was graded.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub qa: Option<QaSliceMetrics>,+}++/// Aggregated QA-accuracy metrics for one slice (R2b): fraction of graded+/// answers the LLM judge marked correct, plus the answer-path latency and+/// token estimates so the accuracy/latency/context triple stays meaningful+/// for the synthesized-answer path, not only for retrieval.+#[derive(Debug, Clone, Serialize)]+pub struct QaSliceMetrics {+ /// Questions in the slice that produced a graded answer.+ pub graded: usize,+ /// Of the graded, how many the judge marked correct.+ pub correct: usize,+ /// `correct / graded` (0.0 when nothing was graded).+ pub accuracy: f64,+ /// Answer-step latency percentiles over graded questions, milliseconds.+ pub answer_latency_p50_ms: u128,+ pub answer_latency_p95_ms: u128,+ /// Mean answer-token estimate (chars/4 over synthesis prompt + answer).+ pub answer_tokens_mean: f64,+}++impl QaSliceMetrics {+ /// Aggregate the graded QA outcomes in one slice, or `None` when the+ /// slice has no graded questions.+ fn from_group(group: &[&QuestionScore]) -> Option<Self> {+ let graded: Vec<&QaOutcome> = group.iter().filter_map(|s| s.qa.as_ref()).collect();+ if graded.is_empty() {+ return None;+ }+ let correct = graded.iter().filter(|o| o.correct).count();+ let latencies: Vec<u128> = graded.iter().map(|o| o.answer_latency_ms).collect();+ let tokens_sum: usize = graded.iter().map(|o| o.answer_tokens).sum();+ Some(Self {+ graded: graded.len(),+ correct,+ accuracy: correct as f64 / graded.len() as f64,+ answer_latency_p50_ms: percentile_u128(&latencies, 0.50),+ answer_latency_p95_ms: percentile_u128(&latencies, 0.95),+ answer_tokens_mean: tokens_sum as f64 / graded.len() as f64,+ })+ }}/// Nearest-rank percentile (`p` in 0.0..=1.0) over a copy-sorted slice.@@ -161,6 +215,7 @@ pub fn aggregate(scores: &[QuestionScore], ks: &[usize]) -> BTreeMap<String, Slilatency_p95_ms: percentile_u128(&latencies, 0.95),context_tokens_mean: tokens_sum as f64 / n,context_tokens_median: median_f64(&tokens),+ qa: QaSliceMetrics::from_group(&group),},)})@@ -320,6 +375,77 @@ mod tests {assert_eq!(agg["overall"].latency_p95_ms, 30);}+ fn qa(correct: bool, latency_ms: u128, tokens: usize) -> QaOutcome {+ QaOutcome {+ correct,+ answer_source: "synthesized",+ answer: "a".into(),+ grade_reason: "r".into(),+ answer_latency_ms: latency_ms,+ answer_tokens: tokens,+ }+ }++ #[test]+ fn qa_aggregates_only_over_graded_questions() {+ let e = uuid::Uuid::now_v7();+ let mut graded = score_question(+ "q1",+ "single-session-user",+ &HashSet::from([e]),+ &retrieved(&[Some(e)]),+ &[1],+ 10,+ 100,+ );+ graded.qa = Some(qa(true, 800, 40));+ let mut also_graded = score_question(+ "q2",+ "single-session-user",+ &HashSet::from([uuid::Uuid::now_v7()]),+ &retrieved(&[]),+ &[1],+ 20,+ 50,+ );+ also_graded.qa = Some(qa(false, 1200, 60));+ // A third question with QA off (None) must not count toward graded.+ let ungraded = score_question(+ "q3",+ "single-session-user",+ &HashSet::from([uuid::Uuid::now_v7()]),+ &retrieved(&[]),+ &[1],+ 30,+ 10,+ );++ let agg = aggregate(&[graded, also_graded, ungraded], &[1]);+ let qa = agg["overall"].qa.as_ref().expect("overall has QA");+ assert_eq!(qa.graded, 2);+ assert_eq!(qa.correct, 1);+ assert_eq!(qa.accuracy, 0.5);+ assert_eq!(qa.answer_tokens_mean, 50.0);+ assert_eq!(qa.answer_latency_p50_ms, 800);+ assert_eq!(qa.answer_latency_p95_ms, 1200);+ }++ #[test]+ fn qa_metrics_absent_when_no_question_was_graded() {+ let e = uuid::Uuid::now_v7();+ let s = score_question(+ "q",+ "single-session-user",+ &HashSet::from([e]),+ &retrieved(&[Some(e)]),+ &[1],+ 0,+ 0,+ );+ let agg = aggregate(&[s], &[1]);+ assert!(agg["overall"].qa.is_none());+ }+#[test]fn percentile_nearest_rank_and_empty() {assert_eq!(percentile_u128(&[], 0.5), 0);