Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
//! Pinned retrieval datasets in the LongMemEval question/session format.
//!
//! The LongMemEval file is NOT committed (278 MB); it lives gitignored under
//! `evals/datasets/` and is fetched on demand from HuggingFace with a
//! pinned sha256. Dataset: `xiaowu0162/longmemeval` (MIT license).
//!
//! Two pinned variants load: the original `longmemeval_s` (every published
//! ai-memory baseline) and the later `longmemeval_s_cleaned` re-release,
//! which keeps the same 500 questions and evidence labels but replaces
//! distractor sessions that also held the answer. Other systems publish on
//! the cleaned file, so a comparison has to say which one it ran.
//! The small synthetic `coding-v1` corpus is committed for offline smoke runs.
use std::path::Path;
use anyhow::{Context, Result, bail};
use serde::Deserialize;
use sha2::{Digest, Sha256};
/// Canonical download URL for the S variant.
pub const LONGMEMEVAL_S_URL: &str =
"https://huggingface.co/datasets/xiaowu0162/longmemeval/resolve/main/longmemeval_s";
/// sha256 of `longmemeval_s` as pinned on 2026-09-01. A drifted upstream
/// file fails loudly instead of silently changing published numbers.
pub const LONGMEMEVAL_S_SHA256: &str =
"08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894";
/// One pinned dataset file the harness accepts.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DatasetVariant {
/// Name recorded in reports.
pub name: &'static str,
/// Default location under the repo root.
pub default_path: &'static str,
/// Download URL, or `None` for a fixture committed with the evaluator.
pub url: Option<&'static str>,
/// Expected sha256 of the file.
pub sha256: &'static str,
}
/// The original S file every published ai-memory baseline uses.
pub const VARIANT_S: DatasetVariant = DatasetVariant {
name: "longmemeval_s",
default_path: "evals/datasets/longmemeval_s.json",
url: Some(LONGMEMEVAL_S_URL),
sha256: LONGMEMEVAL_S_SHA256,
};
/// The cleaned S re-release (`xiaowu0162/longmemeval-cleaned`, revision
/// `98d7416c`), the input agentmemory's 2026-10-09 baseline reports on.
pub const VARIANT_S_CLEANED: DatasetVariant = DatasetVariant {
name: "longmemeval_s_cleaned",
default_path: "evals/datasets/longmemeval_s_cleaned.json",
url: Some(
"https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_s_cleaned.json",
),
sha256: "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
};
/// Small synthetic coding-session evidence corpus, bundled for offline checks.
pub const VARIANT_CODING_V1: DatasetVariant = DatasetVariant {
name: "coding_v1",
default_path: "evals/fixtures/retrieval/coding-v1.json",
url: None,
sha256: "af3382663a3e4e969c8bcfa744b3a794de86cb5932393a88bea16f014193d467",
};
/// One chat turn inside a haystack session.
#[derive(Debug, Clone, Deserialize)]
pub struct Turn {
pub role: String,
pub content: String,
/// The dataset's evidence flag: this turn carries (part of) the answer.
/// Absent on most turns; only the reachability report reads it.
#[serde(default)]
pub has_answer: Option<bool>,
}
impl Turn {
/// True when the dataset marks this turn as answer-bearing.
pub fn is_evidence(&self) -> bool {
self.has_answer == Some(true)
}
}
/// One benchmark question with its private haystack of chat sessions.
#[derive(Debug, Clone, Deserialize)]
pub struct Question {
pub question_id: String,
pub question_type: String,
pub question: String,
/// Gold answer. Usually a JSON string; occasionally a richer value.
/// Read via [`Question::gold_answer_text`] for the QA judge (R2b).
pub answer: serde_json::Value,
#[allow(dead_code)]
pub question_date: String,
pub haystack_dates: Vec<String>,
pub haystack_session_ids: Vec<String>,
pub haystack_sessions: Vec<Vec<Turn>>,
/// Dataset session ids of the sessions holding the evidence.
pub answer_session_ids: Vec<String>,
}
impl Question {
/// Abstention questions (`*_abs`) deliberately have a false premise;
/// no session contains an answer, so they are excluded from recall
/// metrics and reported separately.
pub fn is_abstention(&self) -> bool {
self.question_id.ends_with("_abs")
}
/// The gold answer as text for the LLM-as-judge (R2b QA mode). A JSON
/// string is unwrapped to its contents; any other shape is rendered as
/// compact JSON so the judge still sees the reference value.
pub fn gold_answer_text(&self) -> String {
match &self.answer {
serde_json::Value::String(s) => s.clone(),
other => other.to_string(),
}
}
}
/// Load the dataset, verifying `variant`'s pinned sha256 first.
pub fn load(path: &Path, variant: &DatasetVariant) -> Result<Vec<Question>> {
let bytes = std::fs::read(path).with_context(|| {
let recovery = if variant.url.is_some() {
"use --fetch to download"
} else {
"restore the bundled fixture from the repository"
};
format!("reading dataset at {} ({recovery})", path.display())
})?;
let digest = hex::encode(Sha256::digest(&bytes));
if digest != variant.sha256 {
bail!(
"dataset sha256 mismatch at {}: got {digest}, pinned {} for {}. \
Refusing to benchmark against an unpinned file.",
path.display(),
variant.sha256,
variant.name
);
}
let questions: Vec<Question> =
serde_json::from_slice(&bytes).context("parsing retrieval dataset JSON")?;
for q in &questions {
if q.haystack_sessions.len() != q.haystack_session_ids.len()
|| q.haystack_sessions.len() != q.haystack_dates.len()
{
bail!(
"question {}: sessions/ids/dates length mismatch ({}/{}/{})",
q.question_id,
q.haystack_sessions.len(),
q.haystack_session_ids.len(),
q.haystack_dates.len()
);
}
}
Ok(questions)
}
/// Download the dataset to `path` (atomically: tmp + rename) and verify
/// the pinned sha256 before the rename lands it.
pub async fn fetch(path: &Path, variant: &DatasetVariant) -> Result<()> {
let url = variant.url.with_context(|| {
format!(
"{} is bundled with the repository; restore {} instead of downloading it",
variant.name, variant.default_path
)
})?;
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
tracing::info!(url, "downloading {}", variant.name);
let resp = reqwest::get(url).await?.error_for_status()?;
let bytes = resp.bytes().await?;
let digest = hex::encode(Sha256::digest(&bytes));
if digest != variant.sha256 {
bail!(
"downloaded dataset sha256 {digest} does not match pin {}",
variant.sha256
);
}
let tmp = path.with_extension("part");
std::fs::write(&tmp, &bytes)?;
std::fs::rename(&tmp, path)?;
Ok(())
}
/// First `per_type` questions of each question type, types in sorted order
/// (the same deterministic selection as agentmemory's `stratifySample`).
/// A prefix sample (`--sample`) only covers the first type, because the file
/// is grouped by type; this one covers every slice.
pub fn stratify(questions: Vec<Question>, per_type: usize) -> Vec<Question> {
let mut buckets: std::collections::BTreeMap<String, Vec<Question>> =
std::collections::BTreeMap::new();
for q in questions {
let bucket = buckets.entry(q.question_type.clone()).or_default();
if bucket.len() < per_type {
bucket.push(q);
}
}
buckets.into_values().flatten().collect()
}
mod hex {
/// Minimal lowercase hex, avoiding a dependency for one call site.
pub fn encode(bytes: impl AsRef<[u8]>) -> String {
bytes.as_ref().iter().map(|b| format!("{b:02x}")).collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn coding_corpus_pin_and_evidence_labels_are_consistent() {
use std::collections::HashSet;
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/retrieval/coding-v1.json");
let questions = load(&path, &VARIANT_CODING_V1).unwrap();
let raw: serde_json::Value =
serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
assert_eq!(questions.len(), 4);
let mut question_ids = HashSet::new();
for (q, labels) in questions.iter().zip(raw.as_array().unwrap()) {
assert!(question_ids.insert(&q.question_id));
assert!(!q.is_abstention());
assert!(!q.gold_answer_text().is_empty());
let ids: HashSet<_> = q.haystack_session_ids.iter().collect();
assert_eq!(ids.len(), q.haystack_session_ids.len());
assert!(
ids.len() > 10,
"recall@10 must not include every session by construction"
);
let evidence: HashSet<_> = q.answer_session_ids.iter().collect();
assert!(!evidence.is_empty());
let incorrect: HashSet<_> = labels["incorrect_session_ids"]
.as_array()
.unwrap()
.iter()
.map(|v| v.as_str().unwrap())
.collect();
for (id, turns) in q.haystack_session_ids.iter().zip(&q.haystack_sessions) {
let expected = evidence.contains(id);
assert_eq!(
turns.iter().any(Turn::is_evidence),
expected,
"{}: {id}",
q.question_id
);
assert_eq!(
incorrect.contains(id.as_str()),
!expected,
"alternatives must partition the haystack"
);
}
assert!(evidence.iter().all(|id| ids.contains(id)));
assert_eq!(incorrect.len() + evidence.len(), ids.len());
}
}
#[test]
fn coding_corrections_follow_their_stale_alternatives() {
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/retrieval/coding-v1.json");
let questions = load(&path, &VARIANT_CODING_V1).unwrap();
for (id, old, current) in [
(
"coding-correction",
"coding-correction-stale",
"coding-correction-current",
),
(
"coding-long-session",
"coding-long-session-wrong-header",
"coding-long-session-investigation",
),
] {
let q = questions.iter().find(|q| q.question_id == id).unwrap();
let old_index = q
.haystack_session_ids
.iter()
.position(|sid| sid == old)
.unwrap();
let current_index = q
.haystack_session_ids
.iter()
.position(|sid| sid == current)
.unwrap();
assert!(q.haystack_dates[old_index] < q.haystack_dates[current_index]);
assert!(!q.answer_session_ids.iter().any(|sid| sid == old));
assert!(q.answer_session_ids.iter().any(|sid| sid == current));
}
}
#[test]
fn missing_bundled_corpus_points_to_repository_restore() {
let tmp = tempfile::tempdir().unwrap();
let err = load(&tmp.path().join("missing.json"), &VARIANT_CODING_V1).unwrap_err();
assert!(err.to_string().contains("restore the bundled fixture"));
assert!(!err.to_string().contains("--fetch"));
}
#[tokio::test]
async fn bundled_corpus_fetch_fails_before_creating_a_file() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("missing/coding.json");
let err = fetch(&path, &VARIANT_CODING_V1).await.unwrap_err();
assert!(err.to_string().contains("bundled with the repository"));
assert!(!path.parent().unwrap().exists());
}
fn sample_json() -> serde_json::Value {
serde_json::json!([{
"question_id": "q1",
"question_type": "single-session-user",
"question": "what degree?",
"answer": "Business",
"question_date": "2023/05/30 (Tue) 23:40",
"haystack_dates": ["2023/05/20 (Sat) 02:21"],
"haystack_session_ids": ["sharegpt_abc_0"],
"haystack_sessions": [[
{"role": "user", "content": "I studied Business", "has_answer": true},
{"role": "assistant", "content": "Nice."}
]],
"answer_session_ids": ["sharegpt_abc_0"]
}])
}
#[test]
fn parses_the_dataset_shape() {
let qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
assert_eq!(qs.len(), 1);
assert_eq!(qs[0].haystack_sessions[0][0].role, "user");
assert!(!qs[0].is_abstention());
}
#[test]
fn gold_answer_unwraps_a_json_string_and_renders_other_shapes() {
let mut qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
assert_eq!(qs[0].gold_answer_text(), "Business");
qs[0].answer = serde_json::json!(["a", "b"]);
assert_eq!(qs[0].gold_answer_text(), "[\"a\",\"b\"]");
}
#[test]
fn abstention_is_detected_from_the_id_suffix() {
let mut qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
qs[0].question_id = "q1_abs".into();
assert!(qs[0].is_abstention());
}
#[test]
fn a_tampered_dataset_fails_the_pin() {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("longmemeval_s.json");
std::fs::write(&p, serde_json::to_vec(&sample_json()).unwrap()).unwrap();
let err = load(&p, &VARIANT_S).unwrap_err().to_string();
assert!(err.contains("sha256 mismatch"), "{err}");
let err = load(&p, &VARIANT_S_CLEANED).unwrap_err().to_string();
assert!(err.contains("longmemeval_s_cleaned"), "{err}");
}
#[test]
fn the_evidence_flag_is_read_and_defaults_to_absent() {
let qs: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
let turns = &qs[0].haystack_sessions[0];
assert!(turns[0].is_evidence());
assert_eq!(turns[1].has_answer, None);
assert!(!turns[1].is_evidence());
}
#[test]
fn stratify_takes_a_prefix_of_every_type_in_sorted_order() {
let mut qs: Vec<Question> = Vec::new();
let base: Vec<Question> = serde_json::from_value(sample_json()).unwrap();
for (i, ty) in ["b-type", "a-type", "b-type", "b-type", "a-type"]
.iter()
.enumerate()
{
let mut q = base[0].clone();
q.question_id = format!("q{i}");
q.question_type = (*ty).into();
qs.push(q);
}
let picked: Vec<String> = stratify(qs, 2).into_iter().map(|q| q.question_id).collect();
assert_eq!(picked, ["q1", "q4", "q0", "q2"]);
}
}