Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(evals): select retrieval questions by id
AkitaOnRailse063916fd891
diff --git a/evals/README.md b/evals/README.mdindex e91c8c4d..1e478eca 100644--- a/evals/README.md+++ b/evals/README.md@@ -199,6 +199,7 @@ on upstream drift.cargo build --release -p ai-memory-cli # the server under testcargo run --release -p ai-memory-eval -- retrieval --fetch # full 500cargo run -p ai-memory-eval -- retrieval --sample 10 # smoke+cargo run -p ai-memory-eval -- retrieval --question-id 07741c44,07741c45 # chosen questions```Output: a per-slice table on stdout plusdiff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rsindex d6b6d985..b0b6f74c 100644--- a/evals/src/retrieval/mod.rs+++ b/evals/src/retrieval/mod.rs@@ -84,6 +84,12 @@ pub struct RetrievalArgs {#[arg(long)]stratify: Option<usize>,+ /// Score only these question ids (comma-separated or repeated), in+ /// dataset order. Applied before `--sample` and `--stratify`: a+ /// targeted re-run of a few questions without a full pass.+ #[arg(long = "question-id", value_delimiter = ',')]+ question_ids: Vec<String>,+/// `server` (the default) drives a real `ai-memory serve`; `grep`/// scores the whole-session term-overlap baseline with no server.#[arg(long, value_enum, default_value_t = Engine::Server)]@@ -425,6 +431,17 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {}let mut questions = dataset::load(&dataset_path, variant)?;let total_loaded = questions.len();+ if !args.question_ids.is_empty() {+ let unknown: Vec<&String> = args+ .question_ids+ .iter()+ .filter(|id| !questions.iter().any(|q| &q.question_id == *id))+ .collect();+ if !unknown.is_empty() {+ bail!("--question-id not in the dataset: {unknown:?}");+ }+ questions.retain(|q| args.question_ids.contains(&q.question_id));+ }if let Some(n) = args.sample {questions.truncate(n);}