Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(evals): bound LongMemEval abstention context

AkitaOnRails3906be1a53d0
diff --git a/evals/README.md b/evals/README.md
index dccc4e1b..2d0e51b0 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -55,6 +55,12 @@ cargo run -p ai-memory-eval -- retrieval --fetch --sample 1
cargo run -p ai-memory-eval -- abstention --longmemeval-abs
```
+Those cases use the evaluator's existing whole-session grep baseline, not the
+production server: each question receives at most the top 10 term-overlap
+sessions, with normalized overlap scores and each rendered session clipped to
+4,000 characters. This keeps optional QA inputs bounded and makes the signal's
+provenance explicit; it does not claim to measure production retrieval.
+
Optional `--qa` uses the exact `--qa-*` provider/auth construction shared with
the retrieval evaluator. Synthesis returns a structured `answerability` enum
(`answerable`, `insufficient`, `conflicting`, or `partial`); the evaluator never
diff --git a/evals/src/abstention.rs b/evals/src/abstention.rs
index 2d2461a3..40925684 100644
--- a/evals/src/abstention.rs
+++ b/evals/src/abstention.rs
@@ -8,12 +8,16 @@ use jiff::Timestamp;
use serde::{Deserialize, Serialize};
use crate::retrieval::dataset::{self, DatasetVariant};
+use crate::retrieval::grep;
+use crate::retrieval::ingest::stored_session_uuid;
use crate::retrieval::qa::{Answerability, QaArgs, QaEngine};
use crate::retrieval::query::HitContext;
use crate::retrieval::report::QaReportMeta;
const FIXTURE_VERSION: u32 = 1;
const DEFAULT_THRESHOLDS: [f64; 11] = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0];
+const LONGMEMEVAL_CONTEXTS: usize = 10;
+const LONGMEMEVAL_SNIPPET_CHARS: usize = 4_000;
#[derive(clap::Args, Debug)]
pub struct AbstentionArgs {
@@ -396,19 +400,42 @@ fn load_longmemeval_abstentions() -> Result<Vec<Case>> {
.into_iter()
.filter(|question| question.is_abstention())
.map(|question| {
- let evidence = question
- .haystack_sessions
+ let mut seen = HashSet::new();
+ let sessions = question
+ .haystack_session_ids
+ .iter()
+ .zip(&question.haystack_dates)
+ .zip(&question.haystack_sessions)
+ .filter(|((session_id, _), _)| seen.insert(session_id.as_str()))
+ .map(|((session_id, date), turns)| {
+ (
+ stored_session_uuid(&question, session_id),
+ format!("sessions/{session_id}.md"),
+ grep::session_text(date, turns),
+ )
+ })
+ .collect::<Vec<_>>();
+ let normalized = sessions
+ .iter()
+ .map(|(id, _, text)| (*id, text.to_lowercase()))
+ .collect::<Vec<_>>();
+ let scores = grep::rank_with_scores(
+ &question.question,
+ normalized.iter().map(|(id, text)| (*id, text.as_str())),
+ );
+ let top_score = scores.first().map(|(_, score)| *score).unwrap_or(1) as f64;
+ let evidence = scores
.into_iter()
- .zip(question.haystack_session_ids)
- .map(|(turns, session_id)| Evidence {
- path: format!("sessions/{session_id}.md"),
- snippet: turns
- .into_iter()
- .map(|turn| turn.content)
- .collect::<Vec<_>>()
- .join("\n"),
- score: 0.0,
- belief_strength: None,
+ .take(LONGMEMEVAL_CONTEXTS)
+ .filter_map(|(id, score)| {
+ let (_, path, text) =
+ sessions.iter().find(|(candidate, _, _)| *candidate == id)?;
+ Some(Evidence {
+ path: path.clone(),
+ snippet: text.chars().take(LONGMEMEVAL_SNIPPET_CHARS).collect(),
+ score: score as f64 / top_score,
+ belief_strength: None,
+ })
})
.collect();
Case {
diff --git a/evals/src/retrieval/grep.rs b/evals/src/retrieval/grep.rs
index 6154f0ac..a16874af 100644
--- a/evals/src/retrieval/grep.rs
+++ b/evals/src/retrieval/grep.rs
@@ -45,7 +45,10 @@ pub fn session_text(date: &str, turns: &[Turn]) -> String {
}
/// Rank `(id, lowercased document)` pairs for `query`, best first.
-pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str)>) -> Vec<T> {
+pub fn rank_with_scores<'a, T: Copy>(
+ query: &str,
+ docs: impl IntoIterator<Item = (T, &'a str)>,
+) -> Vec<(T, usize)> {
let terms = tokenize(query);
let mut scored: Vec<(T, usize)> = docs
.into_iter()
@@ -56,7 +59,15 @@ pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str
.collect();
// `sort_by_key` is stable: equal scores keep haystack order.
scored.sort_by_key(|(_, hits)| std::cmp::Reverse(*hits));
- scored.into_iter().map(|(id, _)| id).collect()
+ scored
+}
+
+/// Rank documents and discard their raw overlap scores.
+pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str)>) -> Vec<T> {
+ rank_with_scores(query, docs)
+ .into_iter()
+ .map(|(id, _)| id)
+ .collect()
}
/// Rank one question's haystack sessions, returning stored session uuids.
@@ -128,6 +139,14 @@ mod tests {
assert_eq!(ranked, [2, 3, 4]);
}
+ #[test]
+ fn ranked_scores_preserve_the_overlap_signal() {
+ assert_eq!(
+ rank_with_scores("leather boots", [(1, "boots"), (2, "leather boots")]),
+ [(2, 2), (1, 1)]
+ );
+ }
+
#[test]
fn substring_matching_credits_inflections() {
assert_eq!(rank("boot", [(1, "two boots")]), [1]);
diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rs
index fd9080c4..878f3b13 100644
--- a/evals/src/retrieval/mod.rs
+++ b/evals/src/retrieval/mod.rs
@@ -31,8 +31,8 @@
//! - `--date-prefix turn|session-start|none` moves the per-turn date.
pub(crate) mod dataset;
-mod grep;
-mod ingest;
+pub(crate) mod grep;
+pub(crate) mod ingest;
pub(crate) mod qa;
pub(crate) mod query;
mod reachability;