Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(evals): select retrieval questions by id

AkitaOnRailse063916fd891
diff --git a/evals/README.md b/evals/README.md
index e91c8c4d..1e478eca 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -199,6 +199,7 @@ on upstream drift.
cargo build --release -p ai-memory-cli # the server under test
cargo run --release -p ai-memory-eval -- retrieval --fetch # full 500
cargo run -p ai-memory-eval -- retrieval --sample 10 # smoke
+cargo run -p ai-memory-eval -- retrieval --question-id 07741c44,07741c45 # chosen questions
```
Output: a per-slice table on stdout plus
diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rs
index d6b6d985..b0b6f74c 100644
--- a/evals/src/retrieval/mod.rs
+++ b/evals/src/retrieval/mod.rs
@@ -84,6 +84,12 @@ pub struct RetrievalArgs {
#[arg(long)]
stratify: Option<usize>,
+ /// Score only these question ids (comma-separated or repeated), in
+ /// dataset order. Applied before `--sample` and `--stratify`: a
+ /// targeted re-run of a few questions without a full pass.
+ #[arg(long = "question-id", value_delimiter = ',')]
+ question_ids: Vec<String>,
+
/// `server` (the default) drives a real `ai-memory serve`; `grep`
/// scores the whole-session term-overlap baseline with no server.
#[arg(long, value_enum, default_value_t = Engine::Server)]
@@ -425,6 +431,17 @@ pub async fn run(args: RetrievalArgs) -> Result<()> {
}
let mut questions = dataset::load(&dataset_path, variant)?;
let total_loaded = questions.len();
+ if !args.question_ids.is_empty() {
+ let unknown: Vec<&String> = args
+ .question_ids
+ .iter()
+ .filter(|id| !questions.iter().any(|q| &q.question_id == *id))
+ .collect();
+ if !unknown.is_empty() {
+ bail!("--question-id not in the dataset: {unknown:?}");
+ }
+ questions.retain(|q| args.question_ids.contains(&q.question_id));
+ }
if let Some(n) = args.sample {
questions.truncate(n);
}