Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(evals): add strict recall-all@k scoring

Rohit Ghumarec82082482abb
diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md
index 35d17045..ff3a6844 100644
--- a/docs/benchmarks/README.md
+++ b/docs/benchmarks/README.md
@@ -120,6 +120,11 @@ baseline-vs-baseline determinism check is clean (all accuracy/token deltas `0.00
## Reading the numbers
+- **Evidence coverage.** `hit@k` credits any evidence session, `recall@k`
+ measures the fraction found, and `recall-all@k` requires every evidence
+ session in the top k distinct retrieved sessions. Each metric is averaged
+ per question; abstention questions are excluded. Historical reports that
+ omit `recall-all@k` do not establish complete-evidence recall.
- **mode: local embeddings** is the 2.0 default: the in-process
all-MiniLM embedder (no API key, no egress) fused with FTS5 + entity +
graph. **mode: zero-llm** is the deterministic floor (`embedding_provider
diff --git a/evals/README.md b/evals/README.md
index e9cbad28..b03fb3e4 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -281,9 +281,10 @@ haystacks). Nothing is mocked:
3. the question runs through MCP `tools/call memory_query` with
explicit workspace/project scoping;
4. results are scored session-level: `hit@k` (any evidence session in
- the top k — the "Recall@k" most systems publish) and `recall@k`
- (fraction of evidence sessions found). The 30 abstention questions
- are excluded and reported separately.
+ the top k), `recall@k` (fraction of evidence sessions found), and
+ `recall-all@k` (all evidence sessions found). Duplicate session hits
+ count once. The 30 abstention questions are excluded and reported
+ separately.
The dataset (278 MB) is NOT committed; it downloads to
`evals/datasets/` (gitignored) with a pinned sha256 that fails loudly
@@ -306,7 +307,7 @@ When publishing a baseline, copy the markdown into `docs/benchmarks/`.
Every run — single-config or A/B — reports three things per slice, not
just accuracy:
-- **accuracy** — `hit@k` / `recall@k` (unchanged);
+- **accuracy** — `hit@k` / `recall@k` / `recall-all@k`;
- **latency** — `memory_query` MCP round-trip **p50 / p95** in ms;
- **context tokens** — the context an agent would ingest from the result,
estimated as **chars / 4** over each returned hit's `title` + `snippet`
@@ -347,6 +348,12 @@ Output: `evals/runs/<timestamp>-retrieval/report.{json,md}` — the JSON is
triples and an `overall` delta table. Publish full-dataset A/B numbers in
`docs/benchmarks/retrieval-ab-r2.md`.
+Strict `recall-all@k` scores each question as 1 only when every evidence
+session appears in the top k distinct retrieved sessions, otherwise 0.
+Slice scores average those per-question values. Finding one of two required
+sessions gives `hit@k = 1`, `recall@k = 0.5`, and `recall-all@k = 0`.
+JSON reports use `recall_all_at` for per-question, slice and A/B delta values.
+
### `retrieval` QA-accuracy mode (R2b, opt-in, live LLM)
R2a scores *retrieval*. `--qa` adds an opt-in **end-to-end answer-quality**
diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rs
index aaba597a..4ab0c41b 100644
--- a/evals/src/retrieval/mod.rs
+++ b/evals/src/retrieval/mod.rs
@@ -2,8 +2,8 @@
//!
//! Per question: replay its haystack sessions through `POST /hook/batch`
//! (production hook cadence), query with `memory_query` over MCP, and
-//! score session-level hit@k / recall@k against the dataset's evidence
-//! labels. See `docs/benchmarks/` for published baselines and
+//! score session-level hit@k / recall@k / recall-all@k against the dataset's
+//! evidence labels. See `docs/benchmarks/` for published baselines and
//! `evals/README.md` for how to run.
//!
//! ## R2: A/B + the accuracy/latency/context triple
@@ -12,8 +12,9 @@
//! the same question set runs through TWO server configs — a baseline and
//! a candidate — and the report gains a baseline→candidate delta. Every
//! run, single or A/B, now reports a triple per slice: retrieval accuracy
-//! (hit@k / recall@k), `memory_query` latency (p50/p95), and the context
-//! tokens a result would cost the agent (chars/4 over returned snippets).
+//! (hit@k / recall@k / recall-all@k), `memory_query` latency (p50/p95), and
+//! the context tokens a result would cost the agent (chars/4 over returned
+//! snippets).
//! Without a candidate config the single-config path is unchanged.
//!
//! ## Decomposing the retrieval gap
@@ -101,7 +102,7 @@ pub struct RetrievalArgs {
#[arg(long)]
reachability: bool,
- /// Cutoffs for hit@k / recall@k.
+ /// Cutoffs for hit@k / recall@k / recall-all@k.
#[arg(long, value_delimiter = ',', default_values_t = vec![1, 3, 5, 10])]
ks: Vec<usize>,
diff --git a/evals/src/retrieval/report.rs b/evals/src/retrieval/report.rs
index 2fff878b..d64d446d 100644
--- a/evals/src/retrieval/report.rs
+++ b/evals/src/retrieval/report.rs
@@ -208,9 +208,12 @@ fn slices_table_md(r: &Report) -> String {
for k in &r.ks {
md.push_str(&format!(" recall@{k} |"));
}
+ for k in &r.ks {
+ md.push_str(&format!(" recall-all@{k} |"));
+ }
md.push_str(" p50 ms | p95 ms | ctx tok mean | ctx tok med |\n");
md.push_str("|---|---|");
- for _ in r.ks.iter().chain(r.ks.iter()) {
+ for _ in r.ks.iter().chain(r.ks.iter()).chain(r.ks.iter()) {
md.push_str("---|");
}
md.push_str("---|---|---|---|\n");
@@ -222,6 +225,9 @@ fn slices_table_md(r: &Report) -> String {
for k in &r.ks {
md.push_str(&format!(" {:.3} |", m.recall_at[k]));
}
+ for k in &r.ks {
+ md.push_str(&format!(" {:.3} |", m.recall_all_at[k]));
+ }
md.push_str(&format!(
" {} | {} | {:.1} | {:.1} |\n",
m.latency_p50_ms, m.latency_p95_ms, m.context_tokens_mean, m.context_tokens_median,
@@ -270,7 +276,8 @@ fn overall_qa_accuracy(r: &Report) -> Option<f64> {
}
const METRIC_NOTES: &str = "\nNotes: hit@k = any evidence session in top k (the \"Recall@k\" most \
- systems publish); recall@k = fraction of evidence sessions found. \
+ systems publish); recall@k = fraction of evidence sessions found; \
+ recall-all@k = all evidence sessions found in top k. \
Latency is the `memory_query` MCP round trip (send→parse); ctx tok is \
the estimated context an agent ingests from the result = chars/4 over \
returned hit titles + snippets. Session attribution: `sessions/<id>.md` \
@@ -307,6 +314,7 @@ pub struct AbReport {
pub struct AbDelta {
pub hit_at: BTreeMap<usize, f64>,
pub recall_at: BTreeMap<usize, f64>,
+ pub recall_all_at: BTreeMap<usize, f64>,
pub latency_p50_ms: i128,
pub latency_p95_ms: i128,
pub context_tokens_mean: f64,
@@ -344,9 +352,11 @@ fn compute_delta(baseline: &Report, candidate: &Report) -> AbDelta {
let c = overall(candidate);
let mut hit_at = BTreeMap::new();
let mut recall_at = BTreeMap::new();
+ let mut recall_all_at = BTreeMap::new();
for k in &candidate.ks {
hit_at.insert(*k, c.hit_at[k] - b.hit_at[k]);
recall_at.insert(*k, c.recall_at[k] - b.recall_at[k]);
+ recall_all_at.insert(*k, c.recall_all_at[k] - b.recall_all_at[k]);
}
let base_by_id: BTreeMap<&str, &QuestionScore> = baseline
.per_question
@@ -372,6 +382,7 @@ fn compute_delta(baseline: &Report, candidate: &Report) -> AbDelta {
hit_flips,
hit_at,
recall_at,
+ recall_all_at,
latency_p50_ms: c.latency_p50_ms as i128 - b.latency_p50_ms as i128,
latency_p95_ms: c.latency_p95_ms as i128 - b.latency_p95_ms as i128,
context_tokens_mean: c.context_tokens_mean - b.context_tokens_mean,
@@ -426,6 +437,12 @@ pub fn ab_to_markdown(r: &AbReport) -> String {
b.recall_at[k], c.recall_at[k], r.delta.recall_at[k]
));
}
+ for k in &r.candidate.ks {
+ md.push_str(&format!(
+ "| recall-all@{k} | {:.3} | {:.3} | {:+.3} |\n",
+ b.recall_all_at[k], c.recall_all_at[k], r.delta.recall_all_at[k]
+ ));
+ }
md.push_str(&format!(
"| latency p50 ms | {} | {} | {:+} |\n",
b.latency_p50_ms, c.latency_p50_ms, r.delta.latency_p50_ms
@@ -456,11 +473,12 @@ pub fn ab_to_markdown(r: &AbReport) -> String {
mod tests {
use super::*;
- fn slice(questions: usize, hit5: f64, recall5: f64) -> SliceMetrics {
+ fn slice(questions: usize, hit5: f64, recall5: f64, recall_all5: f64) -> SliceMetrics {
SliceMetrics {
questions,
hit_at: BTreeMap::from([(5, hit5)]),
recall_at: BTreeMap::from([(5, recall5)]),
+ recall_all_at: BTreeMap::from([(5, recall_all5)]),
latency_p50_ms: 12,
latency_p95_ms: 34,
context_tokens_mean: 100.0,
@@ -469,9 +487,9 @@ mod tests {
}
}
- fn report(config: &str, hit5: f64, recall5: f64) -> Report {
+ fn report(config: &str, hit5: f64, recall5: f64, recall_all5: f64) -> Report {
let mut slices = BTreeMap::new();
- slices.insert("overall".to_string(), slice(2, hit5, recall5));
+ slices.insert("overall".to_string(), slice(2, hit5, recall5, recall_all5));
Report {
generated_at: "2026-09-01".into(),
commit: "abc".into(),
@@ -493,13 +511,14 @@ mod tests {
#[test]
fn markdown_carries_provenance_and_the_triple() {
- let r = report("embeddings=none reranker=off", 0.5, 0.25);
+ let r = report("embeddings=none reranker=off", 0.5, 0.25, 0.0);
let md = to_markdown(&r);
assert!(md.contains("commit: `abc`"));
assert!(md.contains("config: embeddings=none reranker=off"));
- // triple row: hit@5 recall@5 p50 p95 ctx-mean ctx-median
+ assert!(md.contains(" hit@5 | recall@5 | recall-all@5 |"));
+ // Accuracy, latency and context-token columns remain aligned.
assert!(
- md.contains("| overall | 2 | 0.500 | 0.250 | 12 | 34 | 100.0 | 90.0 |"),
+ md.contains("| overall | 2 | 0.500 | 0.250 | 0.000 | 12 | 34 | 100.0 | 90.0 |"),
"{md}"
);
assert!(md.contains("abstention questions excluded"));
@@ -508,11 +527,12 @@ mod tests {
#[test]
fn ab_delta_is_candidate_minus_baseline_and_zero_for_identical_runs() {
- let baseline = report("embeddings=none", 0.5, 0.25);
- let candidate = report("embeddings=none", 0.5, 0.25);
+ let baseline = report("embeddings=none", 0.5, 0.25, 0.0);
+ let candidate = report("embeddings=none", 0.5, 0.25, 0.0);
let ab = AbReport::new(baseline, candidate);
assert_eq!(ab.delta.hit_at[&5], 0.0);
assert_eq!(ab.delta.recall_at[&5], 0.0);
+ assert_eq!(ab.delta.recall_all_at[&5], 0.0);
assert_eq!(ab.delta.latency_p50_ms, 0);
assert_eq!(ab.delta.context_tokens_mean, 0.0);
let md = ab_to_markdown(&ab);
@@ -521,12 +541,13 @@ mod tests {
md.contains("| hit@5 | 0.500 | 0.500 | +0.000 (+0/-0) |"),
"{md}"
);
+ assert!(md.contains("| recall-all@5 | 0.000 | 0.000 | +0.000 |"));
}
#[test]
fn qa_table_renders_only_when_qa_ran_and_carries_no_key() {
use super::super::score::QaSliceMetrics;
- let mut r = report("embeddings=none reranker=off", 0.5, 0.25);
+ let mut r = report("embeddings=none reranker=off", 0.5, 0.25, 0.0);
// No QA → no QA section.
assert!(!to_markdown(&r).contains("QA accuracy"));
// Attach QA.
@@ -558,13 +579,32 @@ mod tests {
#[test]
fn ab_delta_reports_a_real_difference() {
- let baseline = report("embeddings=none", 0.4, 0.20);
- let candidate = report("embeddings=local", 0.6, 0.30);
+ let baseline = report("embeddings=none", 0.4, 0.20, 0.1);
+ let candidate = report("embeddings=local", 0.6, 0.30, 0.2);
let ab = AbReport::new(baseline, candidate);
assert!((ab.delta.hit_at[&5] - 0.2).abs() < 1e-9);
+ assert!((ab.delta.recall_all_at[&5] - 0.1).abs() < 1e-9);
let md = ab_to_markdown(&ab);
assert!(md.contains("| hit@5 | 0.400 | 0.600 | +0.200"), "{md}");
assert!(md.contains("candidate: embeddings=local"));
+ assert!(md.contains("| recall-all@5 | 0.100 | 0.200 | +0.100 |"));
+ let json = serde_json::to_value(&ab).unwrap();
+ assert_eq!(
+ json["baseline"]["slices"]["overall"]["recall_all_at"]["5"],
+ 0.1
+ );
+ assert_eq!(
+ json["candidate"]["slices"]["overall"]["recall_all_at"]["5"],
+ 0.2
+ );
+ assert_eq!(json["delta"]["recall_all_at"]["5"], 0.1);
+
+ let reversed = AbReport::new(ab.candidate, ab.baseline);
+ assert!(ab_to_markdown(&reversed).contains("| recall-all@5 | 0.200 | 0.100 | -0.100 |"));
+ assert_eq!(
+ serde_json::to_value(&reversed).unwrap()["delta"]["recall_all_at"]["5"],
+ -0.1
+ );
}
fn question(id: &str, hit5: f64) -> QuestionScore {
@@ -573,6 +613,7 @@ mod tests {
question_type: "t".into(),
hit_at: BTreeMap::from([(5, hit5)]),
recall_at: BTreeMap::from([(5, hit5)]),
+ recall_all_at: BTreeMap::from([(5, hit5)]),
retrieved_sessions: 1,
latency_ms: 0,
context_tokens: 0,
@@ -584,13 +625,13 @@ mod tests {
#[test]
fn ab_delta_counts_paired_flips_both_ways() {
- let mut baseline = report("b", 0.5, 0.5);
+ let mut baseline = report("b", 0.5, 0.5, 0.5);
baseline.per_question = vec![
question("q1", 1.0),
question("q2", 0.0),
question("q3", 1.0),
];
- let mut candidate = report("c", 0.5, 0.5);
+ let mut candidate = report("c", 0.5, 0.5, 0.5);
candidate.per_question = vec![
question("q1", 0.0),
question("q2", 1.0),
@@ -601,6 +642,18 @@ mod tests {
assert!(ab_to_markdown(&ab).contains("(+1/-1)"));
}
+ #[test]
+ fn json_carries_per_question_and_slice_recall_all() {
+ let mut r = report("c", 0.5, 0.5, 0.5);
+ r.per_question = vec![question("q1", 1.0), question("q2", 0.0)];
+ r.slices = super::super::score::aggregate(&r.per_question, &r.ks);
+ let json = serde_json::to_value(&r).unwrap();
+ assert_eq!(json["per_question"][0]["recall_all_at"]["5"], 1.0);
+ assert_eq!(json["per_question"][1]["recall_all_at"]["5"], 0.0);
+ assert_eq!(json["slices"]["t"]["recall_all_at"]["5"], 0.5);
+ assert_eq!(json["slices"]["overall"]["recall_all_at"]["5"], 0.5);
+ }
+
#[test]
fn ingest_summary_aggregates_streams_and_requests() {
let stats = vec![
@@ -630,7 +683,7 @@ mod tests {
assert_eq!(s.request_p95_ms, 9);
assert_eq!(s.docs_over_16kib, 1);
assert_eq!(s.mode, "whole-session");
- let mut r = report("c", 0.5, 0.5);
+ let mut r = report("c", 0.5, 0.5, 0.5);
r.ingest = Some(s);
let md = to_markdown(&r);
assert!(md.contains("### Ingest"));
diff --git a/evals/src/retrieval/score.rs b/evals/src/retrieval/score.rs
index 505a38a6..1d9273a6 100644
--- a/evals/src/retrieval/score.rs
+++ b/evals/src/retrieval/score.rs
@@ -2,15 +2,17 @@
//! retrieval numbers are reported: for each question, did the evidence
//! sessions surface in the top k retrieved results?
//!
-//! Two metrics per k:
+//! Three metrics per k:
//!
//! - `hit@k` — 1.0 when ANY evidence session appears in the top k
//! (the "Recall@k" most memory systems publish);
//! - `recall@k` — the fraction of that question's evidence sessions
-//! found in the top k (stricter on multi-session questions).
+//! found in the top k;
+//! - `recall-all@k` — 1.0 only when ALL evidence sessions appear in the
+//! top k (stricter on multi-session questions).
//!
//! Abstention questions have no evidence sessions and are excluded
-//! from both (reported separately as a count).
+//! from all three (reported separately as a count).
use std::collections::{BTreeMap, HashSet};
@@ -24,9 +26,10 @@ use super::query::Retrieved;
pub struct QuestionScore {
pub question_id: String,
pub question_type: String,
- /// hit@k and recall@k keyed by k.
+ /// Retrieval metrics keyed by k.
pub hit_at: BTreeMap<usize, f64>,
pub recall_at: BTreeMap<usize, f64>,
+ pub recall_all_at: BTreeMap<usize, f64>,
/// Deduped session attributions actually retrieved (for forensics).
pub retrieved_sessions: usize,
/// `memory_query` round-trip latency for this question, milliseconds.
@@ -74,6 +77,7 @@ pub fn score_question(
}
let mut hit_at = BTreeMap::new();
let mut recall_at = BTreeMap::new();
+ let mut recall_all_at = BTreeMap::new();
for &k in ks {
let top: HashSet<&uuid::Uuid> = ranked_sessions.iter().take(k).collect();
let found = evidence.iter().filter(|e| top.contains(e)).count();
@@ -86,12 +90,21 @@ pub fn score_question(
found as f64 / evidence.len() as f64
},
);
+ recall_all_at.insert(
+ k,
+ if !evidence.is_empty() && found == evidence.len() {
+ 1.0
+ } else {
+ 0.0
+ },
+ );
}
QuestionScore {
question_id: question_id.to_string(),
question_type: question_type.to_string(),
hit_at,
recall_at,
+ recall_all_at,
retrieved_sessions: ranked_sessions.len(),
latency_ms,
context_tokens,
@@ -110,6 +123,7 @@ pub struct SliceMetrics {
pub questions: usize,
pub hit_at: BTreeMap<usize, f64>,
pub recall_at: BTreeMap<usize, f64>,
+ pub recall_all_at: BTreeMap<usize, f64>,
/// Latency percentiles over the slice's per-question `memory_query`
/// round trips, milliseconds.
pub latency_p50_ms: u128,
@@ -207,9 +221,14 @@ pub fn aggregate(scores: &[QuestionScore], ks: &[usize]) -> BTreeMap<String, Sli
let n = group.len() as f64;
let mut hit_at = BTreeMap::new();
let mut recall_at = BTreeMap::new();
+ let mut recall_all_at = BTreeMap::new();
for &k in ks {
hit_at.insert(k, group.iter().map(|s| s.hit_at[&k]).sum::<f64>() / n);
recall_at.insert(k, group.iter().map(|s| s.recall_at[&k]).sum::<f64>() / n);
+ recall_all_at.insert(
+ k,
+ group.iter().map(|s| s.recall_all_at[&k]).sum::<f64>() / n,
+ );
}
let latencies: Vec<u128> = group.iter().map(|s| s.latency_ms).collect();
let tokens: Vec<usize> = group.iter().map(|s| s.context_tokens).collect();
@@ -220,6 +239,7 @@ pub fn aggregate(scores: &[QuestionScore], ks: &[usize]) -> BTreeMap<String, Sli
questions: group.len(),
hit_at,
recall_at,
+ recall_all_at,
latency_p50_ms: percentile_u128(&latencies, 0.50),
latency_p95_ms: percentile_u128(&latencies, 0.95),
context_tokens_mean: tokens_sum as f64 / n,
@@ -258,6 +278,8 @@ mod tests {
assert_eq!(s.hit_at[&1], 1.0);
assert_eq!(s.hit_at[&5], 1.0);
assert_eq!(s.recall_at[&1], 1.0);
+ assert_eq!(s.recall_all_at[&1], 1.0);
+ assert_eq!(s.recall_all_at[&5], 1.0);
}
#[test]
@@ -278,24 +300,31 @@ mod tests {
assert_eq!(s.hit_at[&1], 0.0);
assert_eq!(s.hit_at[&3], 0.0);
assert_eq!(s.hit_at[&5], 1.0);
+ assert_eq!(s.recall_all_at[&3], 0.0);
+ assert_eq!(s.recall_all_at[&5], 1.0);
}
#[test]
fn duplicate_session_chunks_fill_one_slot_not_three() {
- let e = uuid::Uuid::now_v7();
+ let e1 = uuid::Uuid::now_v7();
+ let e2 = uuid::Uuid::now_v7();
let noise = uuid::Uuid::now_v7();
- // three chunks of the same noise session ahead of the evidence
let s = score_question(
"q",
"multi-session",
- &HashSet::from([e]),
- &retrieved(&[Some(noise), Some(noise), Some(noise), Some(e)]),
- &[2],
+ &HashSet::from([e1, e2]),
+ &retrieved(&[None, Some(noise), Some(noise), Some(e1), Some(e1), Some(e2)]),
+ &[0, 2, 3],
0,
0,
);
- // dedup: noise occupies rank 0, evidence rank 1 → hit@2
+ // Unattributed pages and duplicate chunks consume no session slots.
assert_eq!(s.hit_at[&2], 1.0);
+ assert_eq!(s.recall_at[&2], 0.5);
+ assert_eq!(s.recall_all_at[&0], 0.0);
+ assert_eq!(s.recall_all_at[&2], 0.0);
+ assert_eq!(s.recall_all_at[&3], 1.0);
+ assert_eq!(s.retrieved_sessions, 3);
}
#[test]
@@ -313,6 +342,7 @@ mod tests {
);
assert_eq!(s.recall_at[&5], 0.5);
assert_eq!(s.hit_at[&5], 1.0);
+ assert_eq!(s.recall_all_at[&5], 0.0);
}
#[test]
@@ -328,6 +358,7 @@ mod tests {
0,
);
assert_eq!(s.hit_at[&5], 0.0);
+ assert_eq!(s.recall_all_at[&5], 0.0);
assert_eq!(s.retrieved_sessions, 0);
}
@@ -349,6 +380,25 @@ mod tests {
for k in [1, 3, 5, 10] {
assert_eq!(s.hit_at[&k], 0.0, "hit@{k}");
assert_eq!(s.recall_at[&k], 0.0, "recall@{k}");
+ assert_eq!(s.recall_all_at[&k], 0.0, "recall-all@{k}");
+ }
+ }
+
+ #[test]
+ fn empty_evidence_does_not_score_as_complete() {
+ let s = score_question(
+ "q",
+ "abstention",
+ &HashSet::new(),
+ &retrieved(&[Some(uuid::Uuid::now_v7())]),
+ &[0, 5],
+ 0,
+ 0,
+ );
+ for k in [0, 5] {
+ assert_eq!(s.hit_at[&k], 0.0);
+ assert_eq!(s.recall_at[&k], 0.0);
+ assert_eq!(s.recall_all_at[&k], 0.0);
}
}
@@ -364,17 +414,20 @@ mod tests {
10,
100,
);
- let miss = score_question(
+ let partial = score_question(
"q2",
"multi-session",
- &HashSet::from([uuid::Uuid::now_v7()]),
- &retrieved(&[]),
+ &HashSet::from([e, uuid::Uuid::now_v7(), uuid::Uuid::now_v7()]),
+ &retrieved(&[Some(e)]),
&[1],
30,
200,
);
- let agg = aggregate(&[hit, miss], &[1]);
- assert_eq!(agg["multi-session"].hit_at[&1], 0.5);
+ let agg = aggregate(&[hit, partial], &[1]);
+ assert_eq!(agg["multi-session"].hit_at[&1], 1.0);
+ assert_eq!(agg["multi-session"].recall_at[&1], 2.0 / 3.0);
+ assert_eq!(agg["multi-session"].recall_all_at[&1], 0.5);
+ assert_eq!(agg["overall"].recall_all_at[&1], 0.5);
assert_eq!(agg["overall"].questions, 2);
// Latency + context-tokens triple is aggregated alongside accuracy.
assert_eq!(agg["overall"].context_tokens_mean, 150.0);
diff --git a/evals/tests/smoke.rs b/evals/tests/smoke.rs
index e1068b6c..eb7cccca 100644
--- a/evals/tests/smoke.rs
+++ b/evals/tests/smoke.rs
@@ -50,6 +50,8 @@ fn the_harness_scores_a_tiny_sample_end_to_end() {
assert_eq!(report["questions_scored"], 2);
assert_eq!(report["mode"], "zero-llm");
assert!(report["slices"]["overall"]["hit_at"]["5"].is_number());
+ assert!(report["slices"]["overall"]["recall_all_at"]["5"].is_number());
+ assert!(report["per_question"][0]["recall_all_at"]["5"].is_number());
// The R2 triple lands next to accuracy.
assert!(report["slices"]["overall"]["latency_p50_ms"].is_number());
assert!(report["slices"]["overall"]["context_tokens_mean"].is_number());
@@ -117,6 +119,7 @@ fn the_ab_path_reports_a_zero_delta_for_identical_configs() {
for k in ["1", "3", "5", "10"] {
assert_eq!(delta["hit_at"][k], 0.0, "hit@{k} delta");
assert_eq!(delta["recall_at"][k], 0.0, "recall@{k} delta");
+ assert_eq!(delta["recall_all_at"][k], 0.0, "recall-all@{k} delta");
}
assert_eq!(delta["context_tokens_mean"], 0.0);
assert_eq!(delta["context_tokens_median"], 0.0);