Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
//! Session-level retrieval scoring, matching how published LongMemEval
//! retrieval numbers are reported: for each question, did the evidence
//! sessions surface in the top k retrieved results?
//!
//! Three metrics per k:
//!
//! - `hit@k` — 1.0 when ANY evidence session appears in the top k
//! (the "Recall@k" most memory systems publish);
//! - `recall@k` — the fraction of that question's evidence sessions
//! found in the top k;
//! - `recall-all@k` — 1.0 only when ALL evidence sessions appear in the
//! top k (stricter on multi-session questions).
//!
//! Abstention questions have no evidence sessions and are excluded
//! from all three (reported separately as a count).
use std::collections::{BTreeMap, HashSet};
use serde::Serialize;
use super::qa::QaOutcome;
use super::query::Retrieved;
/// Per-question outcome, ready for aggregation.
#[derive(Debug, Clone, Serialize)]
pub struct QuestionScore {
pub question_id: String,
pub question_type: String,
/// Retrieval metrics keyed by k.
pub hit_at: BTreeMap<usize, f64>,
pub recall_at: BTreeMap<usize, f64>,
pub recall_all_at: BTreeMap<usize, f64>,
/// Deduped session attributions actually retrieved (for forensics).
pub retrieved_sessions: usize,
/// `memory_query` round-trip latency for this question, milliseconds.
pub latency_ms: u128,
/// Estimated context tokens the result would cost the agent (chars/4
/// over returned hit titles + snippets).
pub context_tokens: usize,
/// Wall clock of this question's ingest, milliseconds (0 for `grep`).
#[serde(default)]
pub ingest_ms: u128,
/// Raw observation hits the server returned (it falls back to them only
/// when page search finds nothing).
#[serde(default)]
pub raw_hits: usize,
/// End-to-end QA outcome (R2b, opt-in): a candidate answer graded
/// against the gold answer. `None` on the default zero-LLM path and for
/// any question whose QA step failed (logged, left ungraded).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub qa: Option<QaOutcome>,
}
/// Score one question's retrieval run.
///
/// `evidence` holds the stored (uuid-mapped) ids of the answer sessions;
/// `retrieved` is the ranked flattened result list. Duplicate session
/// attributions collapse to their best (first) rank — retrieving three
/// chunks of one session fills one slot, not three.
pub fn score_question(
question_id: &str,
question_type: &str,
evidence: &HashSet<uuid::Uuid>,
retrieved: &[Retrieved],
ks: &[usize],
latency_ms: u128,
context_tokens: usize,
) -> QuestionScore {
let mut seen = HashSet::new();
let mut ranked_sessions = Vec::new();
for r in retrieved {
if let Some(sid) = r.session_uuid
&& seen.insert(sid)
{
ranked_sessions.push(sid);
}
}
let mut hit_at = BTreeMap::new();
let mut recall_at = BTreeMap::new();
let mut recall_all_at = BTreeMap::new();
for &k in ks {
let top: HashSet<&uuid::Uuid> = ranked_sessions.iter().take(k).collect();
let found = evidence.iter().filter(|e| top.contains(e)).count();
hit_at.insert(k, if found > 0 { 1.0 } else { 0.0 });
recall_at.insert(
k,
if evidence.is_empty() {
0.0
} else {
found as f64 / evidence.len() as f64
},
);
recall_all_at.insert(
k,
if !evidence.is_empty() && found == evidence.len() {
1.0
} else {
0.0
},
);
}
QuestionScore {
question_id: question_id.to_string(),
question_type: question_type.to_string(),
hit_at,
recall_at,
recall_all_at,
retrieved_sessions: ranked_sessions.len(),
latency_ms,
context_tokens,
ingest_ms: 0,
raw_hits: 0,
// R2a scoring is answer-agnostic; the QA path attaches its outcome
// afterward, keeping this function pure and deterministic.
qa: None,
}
}
/// Aggregated metrics for one slice (a category, or overall): the R2
/// accuracy + latency + context-tokens triple.
#[derive(Debug, Clone, Serialize)]
pub struct SliceMetrics {
pub questions: usize,
pub hit_at: BTreeMap<usize, f64>,
pub recall_at: BTreeMap<usize, f64>,
pub recall_all_at: BTreeMap<usize, f64>,
/// Latency percentiles over the slice's per-question `memory_query`
/// round trips, milliseconds.
pub latency_p50_ms: u128,
pub latency_p95_ms: u128,
/// Context-token estimate central tendency over the slice.
pub context_tokens_mean: f64,
pub context_tokens_median: f64,
/// End-to-end QA metrics over the slice (R2b). `None` unless QA mode ran
/// and at least one question in the slice was graded.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub qa: Option<QaSliceMetrics>,
}
/// Aggregated QA-accuracy metrics for one slice (R2b): fraction of graded
/// answers the LLM judge marked correct, plus the answer-path latency and
/// token estimates so the accuracy/latency/context triple stays meaningful
/// for the synthesized-answer path, not only for retrieval.
#[derive(Debug, Clone, Serialize)]
pub struct QaSliceMetrics {
/// Questions in the slice that produced a graded answer.
pub graded: usize,
/// Of the graded, how many the judge marked correct.
pub correct: usize,
/// `correct / graded` (0.0 when nothing was graded).
pub accuracy: f64,
/// Answer-step latency percentiles over graded questions, milliseconds.
pub answer_latency_p50_ms: u128,
pub answer_latency_p95_ms: u128,
/// Mean answer-token estimate (chars/4 over synthesis prompt + answer).
pub answer_tokens_mean: f64,
}
impl QaSliceMetrics {
/// Aggregate the graded QA outcomes in one slice, or `None` when the
/// slice has no graded questions.
fn from_group(group: &[&QuestionScore]) -> Option<Self> {
let graded: Vec<&QaOutcome> = group.iter().filter_map(|s| s.qa.as_ref()).collect();
if graded.is_empty() {
return None;
}
let correct = graded.iter().filter(|o| o.correct).count();
let latencies: Vec<u128> = graded.iter().map(|o| o.answer_latency_ms).collect();
let tokens_sum: usize = graded.iter().map(|o| o.answer_tokens).sum();
Some(Self {
graded: graded.len(),
correct,
accuracy: correct as f64 / graded.len() as f64,
answer_latency_p50_ms: percentile_u128(&latencies, 0.50),
answer_latency_p95_ms: percentile_u128(&latencies, 0.95),
answer_tokens_mean: tokens_sum as f64 / graded.len() as f64,
})
}
}
/// Nearest-rank percentile (`p` in 0.0..=1.0) over a copy-sorted slice.
/// Empty input yields 0.
pub(crate) fn percentile_u128(values: &[u128], p: f64) -> u128 {
if values.is_empty() {
return 0;
}
let mut sorted = values.to_vec();
sorted.sort_unstable();
// Nearest-rank: rank = ceil(p * n), clamped to [1, n], 1-based.
let n = sorted.len();
let rank = (p * n as f64).ceil().max(1.0) as usize;
sorted[rank.min(n) - 1]
}
/// Median of an already-collected list of counts (mean of the two middle
/// elements for an even count). Empty input yields 0.
fn median_f64(values: &[usize]) -> f64 {
if values.is_empty() {
return 0.0;
}
let mut sorted = values.to_vec();
sorted.sort_unstable();
let n = sorted.len();
if n % 2 == 1 {
sorted[n / 2] as f64
} else {
(sorted[n / 2 - 1] + sorted[n / 2]) as f64 / 2.0
}
}
/// Macro-average scores per category plus an `overall` slice.
pub fn aggregate(scores: &[QuestionScore], ks: &[usize]) -> BTreeMap<String, SliceMetrics> {
let mut slices: BTreeMap<String, Vec<&QuestionScore>> = BTreeMap::new();
for s in scores {
slices.entry(s.question_type.clone()).or_default().push(s);
slices.entry("overall".to_string()).or_default().push(s);
}
slices
.into_iter()
.map(|(name, group)| {
let n = group.len() as f64;
let mut hit_at = BTreeMap::new();
let mut recall_at = BTreeMap::new();
let mut recall_all_at = BTreeMap::new();
for &k in ks {
hit_at.insert(k, group.iter().map(|s| s.hit_at[&k]).sum::<f64>() / n);
recall_at.insert(k, group.iter().map(|s| s.recall_at[&k]).sum::<f64>() / n);
recall_all_at.insert(
k,
group.iter().map(|s| s.recall_all_at[&k]).sum::<f64>() / n,
);
}
let latencies: Vec<u128> = group.iter().map(|s| s.latency_ms).collect();
let tokens: Vec<usize> = group.iter().map(|s| s.context_tokens).collect();
let tokens_sum: usize = tokens.iter().sum();
(
name,
SliceMetrics {
questions: group.len(),
hit_at,
recall_at,
recall_all_at,
latency_p50_ms: percentile_u128(&latencies, 0.50),
latency_p95_ms: percentile_u128(&latencies, 0.95),
context_tokens_mean: tokens_sum as f64 / n,
context_tokens_median: median_f64(&tokens),
qa: QaSliceMetrics::from_group(&group),
},
)
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn retrieved(ids: &[Option<uuid::Uuid>]) -> Vec<Retrieved> {
ids.iter()
.map(|session_uuid| Retrieved {
session_uuid: *session_uuid,
})
.collect()
}
#[test]
fn a_top_one_hit_scores_across_all_ks() {
let e = uuid::Uuid::now_v7();
let s = score_question(
"q",
"single-session-user",
&HashSet::from([e]),
&retrieved(&[Some(e)]),
&[1, 5],
0,
0,
);
assert_eq!(s.hit_at[&1], 1.0);
assert_eq!(s.hit_at[&5], 1.0);
assert_eq!(s.recall_at[&1], 1.0);
assert_eq!(s.recall_all_at[&1], 1.0);
assert_eq!(s.recall_all_at[&5], 1.0);
}
#[test]
fn evidence_below_the_cutoff_does_not_count() {
let e = uuid::Uuid::now_v7();
let others: Vec<Option<uuid::Uuid>> = (0..3).map(|_| Some(uuid::Uuid::now_v7())).collect();
let mut all = others.clone();
all.push(Some(e)); // rank 4 (0-based 3)
let s = score_question(
"q",
"multi-session",
&HashSet::from([e]),
&retrieved(&all),
&[1, 3, 5],
0,
0,
);
assert_eq!(s.hit_at[&1], 0.0);
assert_eq!(s.hit_at[&3], 0.0);
assert_eq!(s.hit_at[&5], 1.0);
assert_eq!(s.recall_all_at[&3], 0.0);
assert_eq!(s.recall_all_at[&5], 1.0);
}
#[test]
fn duplicate_session_chunks_fill_one_slot_not_three() {
let e1 = uuid::Uuid::now_v7();
let e2 = uuid::Uuid::now_v7();
let noise = uuid::Uuid::now_v7();
let s = score_question(
"q",
"multi-session",
&HashSet::from([e1, e2]),
&retrieved(&[None, Some(noise), Some(noise), Some(e1), Some(e1), Some(e2)]),
&[0, 2, 3],
0,
0,
);
// Unattributed pages and duplicate chunks consume no session slots.
assert_eq!(s.hit_at[&2], 1.0);
assert_eq!(s.recall_at[&2], 0.5);
assert_eq!(s.recall_all_at[&0], 0.0);
assert_eq!(s.recall_all_at[&2], 0.0);
assert_eq!(s.recall_all_at[&3], 1.0);
assert_eq!(s.retrieved_sessions, 3);
}
#[test]
fn partial_multi_session_recall_is_fractional() {
let e1 = uuid::Uuid::now_v7();
let e2 = uuid::Uuid::now_v7();
let s = score_question(
"q",
"multi-session",
&HashSet::from([e1, e2]),
&retrieved(&[Some(e1)]),
&[5],
0,
0,
);
assert_eq!(s.recall_at[&5], 0.5);
assert_eq!(s.hit_at[&5], 1.0);
assert_eq!(s.recall_all_at[&5], 0.0);
}
#[test]
fn unattributed_pages_never_score() {
let e = uuid::Uuid::now_v7();
let s = score_question(
"q",
"single-session-user",
&HashSet::from([e]),
&retrieved(&[None, None]),
&[5],
0,
0,
);
assert_eq!(s.hit_at[&5], 0.0);
assert_eq!(s.recall_all_at[&5], 0.0);
assert_eq!(s.retrieved_sessions, 0);
}
/// Control: a scorer that credited ANY retrieval would ace a shuffled
/// result list; the real scorer must not.
#[test]
fn random_sessions_score_zero() {
let e = uuid::Uuid::now_v7();
let random: Vec<Option<uuid::Uuid>> = (0..10).map(|_| Some(uuid::Uuid::now_v7())).collect();
let s = score_question(
"q",
"knowledge-update",
&HashSet::from([e]),
&retrieved(&random),
&[1, 3, 5, 10],
0,
0,
);
for k in [1, 3, 5, 10] {
assert_eq!(s.hit_at[&k], 0.0, "hit@{k}");
assert_eq!(s.recall_at[&k], 0.0, "recall@{k}");
assert_eq!(s.recall_all_at[&k], 0.0, "recall-all@{k}");
}
}
#[test]
fn empty_evidence_does_not_score_as_complete() {
let s = score_question(
"q",
"abstention",
&HashSet::new(),
&retrieved(&[Some(uuid::Uuid::now_v7())]),
&[0, 5],
0,
0,
);
for k in [0, 5] {
assert_eq!(s.hit_at[&k], 0.0);
assert_eq!(s.recall_at[&k], 0.0);
assert_eq!(s.recall_all_at[&k], 0.0);
}
}
#[test]
fn aggregation_macro_averages_per_category_and_overall() {
let e = uuid::Uuid::now_v7();
let hit = score_question(
"q1",
"multi-session",
&HashSet::from([e]),
&retrieved(&[Some(e)]),
&[1],
10,
100,
);
let partial = score_question(
"q2",
"multi-session",
&HashSet::from([e, uuid::Uuid::now_v7(), uuid::Uuid::now_v7()]),
&retrieved(&[Some(e)]),
&[1],
30,
200,
);
let agg = aggregate(&[hit, partial], &[1]);
assert_eq!(agg["multi-session"].hit_at[&1], 1.0);
assert_eq!(agg["multi-session"].recall_at[&1], 2.0 / 3.0);
assert_eq!(agg["multi-session"].recall_all_at[&1], 0.5);
assert_eq!(agg["overall"].recall_all_at[&1], 0.5);
assert_eq!(agg["overall"].questions, 2);
// Latency + context-tokens triple is aggregated alongside accuracy.
assert_eq!(agg["overall"].context_tokens_mean, 150.0);
assert_eq!(agg["overall"].context_tokens_median, 150.0);
// Nearest-rank p50 over [10, 30] → rank ceil(0.5*2)=1 → 10.
assert_eq!(agg["overall"].latency_p50_ms, 10);
assert_eq!(agg["overall"].latency_p95_ms, 30);
}
fn qa(correct: bool, latency_ms: u128, tokens: usize) -> QaOutcome {
QaOutcome {
correct,
answerability: crate::retrieval::qa::Answerability::Answerable,
answer_source: "synthesized",
answer: "a".into(),
grade_reason: "r".into(),
answer_latency_ms: latency_ms,
answer_tokens: tokens,
}
}
#[test]
fn qa_aggregates_only_over_graded_questions() {
let e = uuid::Uuid::now_v7();
let mut graded = score_question(
"q1",
"single-session-user",
&HashSet::from([e]),
&retrieved(&[Some(e)]),
&[1],
10,
100,
);
graded.qa = Some(qa(true, 800, 40));
let mut also_graded = score_question(
"q2",
"single-session-user",
&HashSet::from([uuid::Uuid::now_v7()]),
&retrieved(&[]),
&[1],
20,
50,
);
also_graded.qa = Some(qa(false, 1200, 60));
// A third question with QA off (None) must not count toward graded.
let ungraded = score_question(
"q3",
"single-session-user",
&HashSet::from([uuid::Uuid::now_v7()]),
&retrieved(&[]),
&[1],
30,
10,
);
let agg = aggregate(&[graded, also_graded, ungraded], &[1]);
let qa = agg["overall"].qa.as_ref().expect("overall has QA");
assert_eq!(qa.graded, 2);
assert_eq!(qa.correct, 1);
assert_eq!(qa.accuracy, 0.5);
assert_eq!(qa.answer_tokens_mean, 50.0);
assert_eq!(qa.answer_latency_p50_ms, 800);
assert_eq!(qa.answer_latency_p95_ms, 1200);
}
#[test]
fn qa_metrics_absent_when_no_question_was_graded() {
let e = uuid::Uuid::now_v7();
let s = score_question(
"q",
"single-session-user",
&HashSet::from([e]),
&retrieved(&[Some(e)]),
&[1],
0,
0,
);
let agg = aggregate(&[s], &[1]);
assert!(agg["overall"].qa.is_none());
}
#[test]
fn percentile_nearest_rank_and_empty() {
assert_eq!(percentile_u128(&[], 0.5), 0);
assert_eq!(percentile_u128(&[42], 0.95), 42);
let v: Vec<u128> = (1..=100).collect();
assert_eq!(percentile_u128(&v, 0.50), 50);
assert_eq!(percentile_u128(&v, 0.95), 95);
assert_eq!(percentile_u128(&v, 1.0), 100);
}
#[test]
fn median_even_and_odd() {
assert_eq!(median_f64(&[]), 0.0);
assert_eq!(median_f64(&[7]), 7.0);
assert_eq!(median_f64(&[1, 2, 3]), 2.0);
assert_eq!(median_f64(&[1, 2, 3, 10]), 2.5);
}
}