Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(evals): bound LongMemEval abstention context
AkitaOnRails3906be1a53d0
diff --git a/evals/README.md b/evals/README.mdindex dccc4e1b..2d0e51b0 100644--- a/evals/README.md+++ b/evals/README.md@@ -55,6 +55,12 @@ cargo run -p ai-memory-eval -- retrieval --fetch --sample 1cargo run -p ai-memory-eval -- abstention --longmemeval-abs```+Those cases use the evaluator's existing whole-session grep baseline, not the+production server: each question receives at most the top 10 term-overlap+sessions, with normalized overlap scores and each rendered session clipped to+4,000 characters. This keeps optional QA inputs bounded and makes the signal's+provenance explicit; it does not claim to measure production retrieval.+Optional `--qa` uses the exact `--qa-*` provider/auth construction shared withthe retrieval evaluator. Synthesis returns a structured `answerability` enum(`answerable`, `insufficient`, `conflicting`, or `partial`); the evaluator neverdiff --git a/evals/src/abstention.rs b/evals/src/abstention.rsindex 2d2461a3..40925684 100644--- a/evals/src/abstention.rs+++ b/evals/src/abstention.rs@@ -8,12 +8,16 @@ use jiff::Timestamp;use serde::{Deserialize, Serialize};use crate::retrieval::dataset::{self, DatasetVariant};+use crate::retrieval::grep;+use crate::retrieval::ingest::stored_session_uuid;use crate::retrieval::qa::{Answerability, QaArgs, QaEngine};use crate::retrieval::query::HitContext;use crate::retrieval::report::QaReportMeta;const FIXTURE_VERSION: u32 = 1;const DEFAULT_THRESHOLDS: [f64; 11] = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0];+const LONGMEMEVAL_CONTEXTS: usize = 10;+const LONGMEMEVAL_SNIPPET_CHARS: usize = 4_000;#[derive(clap::Args, Debug)]pub struct AbstentionArgs {@@ -396,19 +400,42 @@ fn load_longmemeval_abstentions() -> Result<Vec<Case>> {.into_iter().filter(|question| question.is_abstention()).map(|question| {- let evidence = question- .haystack_sessions+ let mut seen = HashSet::new();+ let sessions = question+ .haystack_session_ids+ .iter()+ .zip(&question.haystack_dates)+ .zip(&question.haystack_sessions)+ .filter(|((session_id, _), _)| seen.insert(session_id.as_str()))+ .map(|((session_id, date), turns)| {+ (+ stored_session_uuid(&question, session_id),+ format!("sessions/{session_id}.md"),+ grep::session_text(date, turns),+ )+ })+ .collect::<Vec<_>>();+ let normalized = sessions+ .iter()+ .map(|(id, _, text)| (*id, text.to_lowercase()))+ .collect::<Vec<_>>();+ let scores = grep::rank_with_scores(+ &question.question,+ normalized.iter().map(|(id, text)| (*id, text.as_str())),+ );+ let top_score = scores.first().map(|(_, score)| *score).unwrap_or(1) as f64;+ let evidence = scores.into_iter()- .zip(question.haystack_session_ids)- .map(|(turns, session_id)| Evidence {- path: format!("sessions/{session_id}.md"),- snippet: turns- .into_iter()- .map(|turn| turn.content)- .collect::<Vec<_>>()- .join("\n"),- score: 0.0,- belief_strength: None,+ .take(LONGMEMEVAL_CONTEXTS)+ .filter_map(|(id, score)| {+ let (_, path, text) =+ sessions.iter().find(|(candidate, _, _)| *candidate == id)?;+ Some(Evidence {+ path: path.clone(),+ snippet: text.chars().take(LONGMEMEVAL_SNIPPET_CHARS).collect(),+ score: score as f64 / top_score,+ belief_strength: None,+ })}).collect();Case {diff --git a/evals/src/retrieval/grep.rs b/evals/src/retrieval/grep.rsindex 6154f0ac..a16874af 100644--- a/evals/src/retrieval/grep.rs+++ b/evals/src/retrieval/grep.rs@@ -45,7 +45,10 @@ pub fn session_text(date: &str, turns: &[Turn]) -> String {}/// Rank `(id, lowercased document)` pairs for `query`, best first.-pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str)>) -> Vec<T> {+pub fn rank_with_scores<'a, T: Copy>(+ query: &str,+ docs: impl IntoIterator<Item = (T, &'a str)>,+) -> Vec<(T, usize)> {let terms = tokenize(query);let mut scored: Vec<(T, usize)> = docs.into_iter()@@ -56,7 +59,15 @@ pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str.collect();// `sort_by_key` is stable: equal scores keep haystack order.scored.sort_by_key(|(_, hits)| std::cmp::Reverse(*hits));- scored.into_iter().map(|(id, _)| id).collect()+ scored+}++/// Rank documents and discard their raw overlap scores.+pub fn rank<'a, T: Copy>(query: &str, docs: impl IntoIterator<Item = (T, &'a str)>) -> Vec<T> {+ rank_with_scores(query, docs)+ .into_iter()+ .map(|(id, _)| id)+ .collect()}/// Rank one question's haystack sessions, returning stored session uuids.@@ -128,6 +139,14 @@ mod tests {assert_eq!(ranked, [2, 3, 4]);}+ #[test]+ fn ranked_scores_preserve_the_overlap_signal() {+ assert_eq!(+ rank_with_scores("leather boots", [(1, "boots"), (2, "leather boots")]),+ [(2, 2), (1, 1)]+ );+ }+#[test]fn substring_matching_credits_inflections() {assert_eq!(rank("boot", [(1, "two boots")]), [1]);diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rsindex fd9080c4..878f3b13 100644--- a/evals/src/retrieval/mod.rs+++ b/evals/src/retrieval/mod.rs@@ -31,8 +31,8 @@//! - `--date-prefix turn|session-start|none` moves the per-turn date.pub(crate) mod dataset;-mod grep;-mod ingest;+pub(crate) mod grep;+pub(crate) mod ingest;pub(crate) mod qa;pub(crate) mod query;mod reachability;