Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

test(evals): add bundled offline coding-session retrieval corpus

Mihael Zamin Sousa7e9ea65b1521
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 091f8a96..d90c50f4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,11 @@ version in [`docs/changelog/`](docs/changelog/):
### Added
+- Added the bundled `coding-v1` retrieval-evaluation corpus for offline checks
+ of decision rationale, resolved-error causes, correction evidence, and late
+ session details. It reuses the existing scorer and A/B reports; it measures
+ evidence retrieval, not semantic answer correctness or handoff behavior (#1686).
+
- Added `[handoff] deliver_on_session_start = false` to `.ai-memory.toml`. A
project that sets it no longer gets its handoff, or the continuity notice, at
session start; the handoff is still created when the session ends and stays
diff --git a/evals/README.md b/evals/README.md
index b03fb3e4..d832c0a6 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -302,6 +302,50 @@ Output: a per-slice table on stdout plus
provenance (commit, dataset sha, hardware, per-question scores).
When publishing a baseline, copy the markdown into `docs/benchmarks/`.
+### Bundled coding-session retrieval corpus
+
+`--dataset-variant coding-v1` selects four synthetic questions in
+`evals/fixtures/retrieval/coding-v1.json`. The file is committed and pinned by
+SHA-256, so a zero-LLM run needs neither a dataset download nor a provider.
+Use the existing server or grep engine and A/B reporting:
+
+```bash
+cargo build -p ai-memory-cli -p ai-memory-eval
+cargo run -p ai-memory-eval -- retrieval --dataset-variant coding-v1 \
+ --server-bin target/debug/ai-memory --candidate --concurrency 1
+cargo run -p ai-memory-eval -- retrieval --dataset-variant coding-v1 --engine grep
+```
+
+| Case | Expected evidence | Incorrect alternative |
+|---|---|---|
+| `coding-architecture` | The write-pool failure and the later actor/read-pool decision | Unrelated implementation sessions |
+| `coding-resolved-error` | The observed database error and the retry-path fix | Unrelated operational sessions |
+| `coding-correction` | The later 45-second timeout correction | The earlier 15-second proposal |
+| `coding-long-session` | The exact checksum header at the end of a 41-turn investigation | The unsuccessful initial header |
+
+Each question has twelve sessions, including distractors, and one or two
+expected evidence sessions. `answer_session_ids` drives the existing scorer;
+`has_answer` marks the relevant turns. `incorrect_session_ids` is corpus
+metadata checked by the fixture test, not an extra scoring rule. It partitions
+the remaining sessions and makes stale alternatives explicit for review.
+The gold answer is available to the optional QA mode, which still needs a
+separately configured provider.
+
+These checks measure evidence retrieval: hit@k, recall@k, recall-all@k,
+context-token estimates, and query latency. Retrieving the correction does not
+prove that an agent follows it, and a stale hit is not a measured answer error.
+The corpus does not exercise `as_of`, typed supersession, handoff claiming,
+actual harness switching, private/shared authorization, or concurrent writes.
+The existing integration suites cover those contracts. Every replay still uses
+the evaluator's Claude Code hook shape and one project per question.
+
+Run identical baseline/candidate settings to check accuracy and context-token
+parity; wall-clock latency can vary. Treat the four cases as regression probes,
+not representative quality estimates or numbers comparable to LongMemEval.
+If a fixture changes, update its pin and record the new corpus version beside
+any reported results. `--fetch` cannot download a missing bundled fixture;
+restore it from the repository instead.
+
### The triple: accuracy + latency + context-tokens
Every run — single-config or A/B — reports three things per slice, not
diff --git a/evals/fixtures/retrieval/coding-v1.json b/evals/fixtures/retrieval/coding-v1.json
new file mode 100644
index 00000000..59b01b17
--- /dev/null
+++ b/evals/fixtures/retrieval/coding-v1.json
@@ -0,0 +1,668 @@
+[
+ {
+ "question_id": "coding-architecture",
+ "question_type": "coding-multi-session",
+ "question": "Why did the atlas worker choose a single-writer actor instead of a SQLite write pool, and how were reads handled?",
+ "answer": "The write pool caused SQLITE_BUSY under contention. A single-writer actor serializes writes and audit events; readers use a separate read-only pool.",
+ "question_date": "2026/10/24 (Sat) 12:00",
+ "haystack_dates": [
+ "2026/10/01 09:00",
+ "2026/10/02 09:00",
+ "2026/10/03 09:00",
+ "2026/10/04 09:00",
+ "2026/10/05 09:00",
+ "2026/10/06 09:00",
+ "2026/10/07 09:00",
+ "2026/10/08 09:00",
+ "2026/10/09 09:00",
+ "2026/10/10 09:00",
+ "2026/10/11 09:00",
+ "2026/10/12 09:00"
+ ],
+ "haystack_session_ids": [
+ "coding-architecture-logging",
+ "coding-architecture-testing",
+ "coding-architecture-docs",
+ "coding-architecture-problem",
+ "coding-architecture-assets",
+ "coding-architecture-release",
+ "coding-architecture-metrics",
+ "coding-architecture-config",
+ "coding-architecture-oauth",
+ "coding-architecture-decision",
+ "coding-architecture-backup",
+ "coding-architecture-format"
+ ],
+ "haystack_sessions": [
+ [
+ {
+ "role": "user",
+ "content": "Logging keeps request identifiers and removes authorization headers. The log rotation retention is seven days."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The test suite uses temporary databases. Integration tests clean up their directories after each run."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The documentation navigation needs a link to the migration guide. Correct the heading anchor before publishing."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Atlas worker experiment: a SQLite write pool caused SQLITE_BUSY during concurrent writes. Retries hid the ordering of audit events.",
+ "has_answer": true
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The web asset cache uses content hashes. The stylesheet is generated during development and committed for releases."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The release checklist requires platform builds and verified archive checksums. Version tags follow the changelog."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The metrics endpoint exports bounded counters. Avoid paths and user identifiers in label values."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Configuration loads once at startup. Environment overrides apply before validation, then typed settings are passed down."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The OAuth login uses a browser callback and stores refresh credentials locally. It does not change retrieval ranking."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Atlas worker decision: use a single-writer actor to serialize writes and audit events. Reads keep a separate read-only pool.",
+ "has_answer": true
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "An online SQLite backup produces a consistent database snapshot. A separate archive includes the wiki files."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Run the formatter before committing. Keep code comments focused on the reason for each guard."
+ }
+ ]
+ ],
+ "answer_session_ids": [
+ "coding-architecture-problem",
+ "coding-architecture-decision"
+ ],
+ "incorrect_session_ids": [
+ "coding-architecture-logging",
+ "coding-architecture-testing",
+ "coding-architecture-docs",
+ "coding-architecture-assets",
+ "coding-architecture-release",
+ "coding-architecture-metrics",
+ "coding-architecture-config",
+ "coding-architecture-oauth",
+ "coding-architecture-backup",
+ "coding-architecture-format"
+ ]
+ },
+ {
+ "question_id": "coding-resolved-error",
+ "question_type": "coding-error-cause",
+ "question": "What caused the beacon deployment duplicate fence_epochs_pkey error, and which change resolved it?",
+ "answer": "The retry path reused the prior deployment epoch. Allocate a fresh epoch before retrying the deployment.",
+ "question_date": "2026/10/24 (Sat) 12:00",
+ "haystack_dates": [
+ "2026/10/01 09:00",
+ "2026/10/02 09:00",
+ "2026/10/03 09:00",
+ "2026/10/04 09:00",
+ "2026/10/05 09:00",
+ "2026/10/06 09:00",
+ "2026/10/07 09:00",
+ "2026/10/08 09:00",
+ "2026/10/09 09:00",
+ "2026/10/10 09:00",
+ "2026/10/11 09:00",
+ "2026/10/12 09:00"
+ ],
+ "haystack_session_ids": [
+ "coding-resolved-error-logging",
+ "coding-resolved-error-testing",
+ "coding-resolved-error-symptom",
+ "coding-resolved-error-docs",
+ "coding-resolved-error-assets",
+ "coding-resolved-error-release",
+ "coding-resolved-error-metrics",
+ "coding-resolved-error-config",
+ "coding-resolved-error-oauth",
+ "coding-resolved-error-backup",
+ "coding-resolved-error-resolution",
+ "coding-resolved-error-format"
+ ],
+ "haystack_sessions": [
+ [
+ {
+ "role": "user",
+ "content": "Logging keeps request identifiers and removes authorization headers. The log rotation retention is seven days."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The test suite uses temporary databases. Integration tests clean up their directories after each run."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Beacon deployment failed with duplicate key value violates unique constraint fence_epochs_pkey. The normal first deployment succeeds.",
+ "has_answer": true
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The documentation navigation needs a link to the migration guide. Correct the heading anchor before publishing."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The web asset cache uses content hashes. The stylesheet is generated during development and committed for releases."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The release checklist requires platform builds and verified archive checksums. Version tags follow the changelog."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The metrics endpoint exports bounded counters. Avoid paths and user identifiers in label values."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Configuration loads once at startup. Environment overrides apply before validation, then typed settings are passed down."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The OAuth login uses a browser callback and stores refresh credentials locally. It does not change retrieval ranking."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "An online SQLite backup produces a consistent database snapshot. A separate archive includes the wiki files."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The beacon deployment retry path reused the prior deployment epoch. Allocate a fresh epoch before retrying; the fence_epochs_pkey regression now passes.",
+ "has_answer": true
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Run the formatter before committing. Keep code comments focused on the reason for each guard."
+ }
+ ]
+ ],
+ "answer_session_ids": [
+ "coding-resolved-error-symptom",
+ "coding-resolved-error-resolution"
+ ],
+ "incorrect_session_ids": [
+ "coding-resolved-error-logging",
+ "coding-resolved-error-testing",
+ "coding-resolved-error-docs",
+ "coding-resolved-error-assets",
+ "coding-resolved-error-release",
+ "coding-resolved-error-metrics",
+ "coding-resolved-error-config",
+ "coding-resolved-error-oauth",
+ "coding-resolved-error-backup",
+ "coding-resolved-error-format"
+ ]
+ },
+ {
+ "question_id": "coding-correction",
+ "question_type": "coding-later-correction",
+ "question": "What is the current comet request timeout after the correction?",
+ "answer": "The current comet request timeout is 45 seconds; the earlier 15-second value was corrected.",
+ "question_date": "2026/10/24 (Sat) 12:00",
+ "haystack_dates": [
+ "2026/10/01 09:00",
+ "2026/10/02 09:00",
+ "2026/10/03 09:00",
+ "2026/10/04 09:00",
+ "2026/10/05 09:00",
+ "2026/10/06 09:00",
+ "2026/10/07 09:00",
+ "2026/10/08 09:00",
+ "2026/10/09 09:00",
+ "2026/10/10 09:00",
+ "2026/10/11 09:00",
+ "2026/10/12 09:00"
+ ],
+ "haystack_session_ids": [
+ "coding-correction-logging",
+ "coding-correction-stale",
+ "coding-correction-testing",
+ "coding-correction-docs",
+ "coding-correction-assets",
+ "coding-correction-release",
+ "coding-correction-metrics",
+ "coding-correction-config",
+ "coding-correction-oauth",
+ "coding-correction-backup",
+ "coding-correction-format",
+ "coding-correction-current"
+ ],
+ "haystack_sessions": [
+ [
+ {
+ "role": "user",
+ "content": "Logging keeps request identifiers and removes authorization headers. The log rotation retention is seven days."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Comet request timeout decision: configure 15 seconds for every request. This is the initial proposal."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The test suite uses temporary databases. Integration tests clean up their directories after each run."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The documentation navigation needs a link to the migration guide. Correct the heading anchor before publishing."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The web asset cache uses content hashes. The stylesheet is generated during development and committed for releases."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The release checklist requires platform builds and verified archive checksums. Version tags follow the changelog."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The metrics endpoint exports bounded counters. Avoid paths and user identifiers in label values."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Configuration loads once at startup. Environment overrides apply before validation, then typed settings are passed down."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The OAuth login uses a browser callback and stores refresh credentials locally. It does not change retrieval ranking."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "An online SQLite backup produces a consistent database snapshot. A separate archive includes the wiki files."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Run the formatter before committing. Keep code comments focused on the reason for each guard."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Correction to the earlier comet request timeout decision: 15 seconds is too short for exports. The approved current timeout is 45 seconds. Use 45 seconds, not the initial 15-second proposal.",
+ "has_answer": true
+ }
+ ]
+ ],
+ "answer_session_ids": [
+ "coding-correction-current"
+ ],
+ "incorrect_session_ids": [
+ "coding-correction-logging",
+ "coding-correction-stale",
+ "coding-correction-testing",
+ "coding-correction-docs",
+ "coding-correction-assets",
+ "coding-correction-release",
+ "coding-correction-metrics",
+ "coding-correction-config",
+ "coding-correction-oauth",
+ "coding-correction-backup",
+ "coding-correction-format"
+ ]
+ },
+ {
+ "question_id": "coding-long-session",
+ "question_type": "coding-late-detail",
+ "question": "Which exact header fixed the delta upload checksum mismatch?",
+ "answer": "Use the X-Checksum-SHA256 header for the delta upload.",
+ "question_date": "2026/10/24 (Sat) 12:00",
+ "haystack_dates": [
+ "2026/10/01 09:00",
+ "2026/10/02 09:00",
+ "2026/10/03 09:00",
+ "2026/10/04 09:00",
+ "2026/10/05 09:00",
+ "2026/10/06 09:00",
+ "2026/10/07 09:00",
+ "2026/10/08 09:00",
+ "2026/10/09 09:00",
+ "2026/10/10 09:00",
+ "2026/10/11 09:00",
+ "2026/10/12 09:00"
+ ],
+ "haystack_session_ids": [
+ "coding-long-session-logging",
+ "coding-long-session-testing",
+ "coding-long-session-docs",
+ "coding-long-session-assets",
+ "coding-long-session-wrong-header",
+ "coding-long-session-release",
+ "coding-long-session-metrics",
+ "coding-long-session-config",
+ "coding-long-session-oauth",
+ "coding-long-session-backup",
+ "coding-long-session-investigation",
+ "coding-long-session-format"
+ ],
+ "haystack_sessions": [
+ [
+ {
+ "role": "user",
+ "content": "Logging keeps request identifiers and removes authorization headers. The log rotation retention is seven days."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The test suite uses temporary databases. Integration tests clean up their directories after each run."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The documentation navigation needs a link to the migration guide. Correct the heading anchor before publishing."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The web asset cache uses content hashes. The stylesheet is generated during development and committed for releases."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The initial delta upload experiment sent X-Checksum. The endpoint still reported a checksum mismatch; this was not the fix."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The release checklist requires platform builds and verified archive checksums. Version tags follow the changelog."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The metrics endpoint exports bounded counters. Avoid paths and user identifiers in label values."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Configuration loads once at startup. Environment overrides apply before validation, then typed settings are passed down."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "The OAuth login uses a browser callback and stores refresh credentials locally. It does not change retrieval ranking."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "An online SQLite backup produces a consistent database snapshot. A separate archive includes the wiki files."
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 1: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 2: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 3: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 4: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 5: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 6: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 7: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 8: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 9: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 10: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 11: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 12: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 13: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 14: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 15: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 16: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 17: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 18: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 19: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 20: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 21: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 22: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 23: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 24: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 25: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 26: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 27: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 28: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 29: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 30: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 31: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 32: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 33: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 34: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 35: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 36: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 37: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 38: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 39: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Delta upload investigation step 40: checked retry timing and trace output; no checksum fix yet."
+ },
+ {
+ "role": "user",
+ "content": "Resolved the delta upload checksum mismatch: send the digest in the exact header X-Checksum-SHA256. The old X-Checksum header is ignored by the upload endpoint.",
+ "has_answer": true
+ }
+ ],
+ [
+ {
+ "role": "user",
+ "content": "Run the formatter before committing. Keep code comments focused on the reason for each guard."
+ }
+ ]
+ ],
+ "answer_session_ids": [
+ "coding-long-session-investigation"
+ ],
+ "incorrect_session_ids": [
+ "coding-long-session-logging",
+ "coding-long-session-testing",
+ "coding-long-session-docs",
+ "coding-long-session-assets",
+ "coding-long-session-release",
+ "coding-long-session-metrics",
+ "coding-long-session-config",
+ "coding-long-session-oauth",
+ "coding-long-session-backup",
+ "coding-long-session-wrong-header",
+ "coding-long-session-format"
+ ]
+ }
+]
diff --git a/evals/src/main.rs b/evals/src/main.rs
index 0c45a3ac..dae05cc1 100644
--- a/evals/src/main.rs
+++ b/evals/src/main.rs
@@ -7,7 +7,7 @@
//! [`fidelity`] and `evals/README.md`).
//! - `projection` — the zero-LLM half of that scoring: which fixture facts
//! and literal tokens the observation projection lets into the prompt.
-//! - `retrieval` — LongMemEval retrieval benchmark driven end-to-end
+//! - `retrieval` — pinned retrieval benchmarks driven end-to-end
//! through a real `ai-memory serve` subprocess: hook-shaped ingestion,
//! real search stack, R@k / hit@k per question category (see
//! [`retrieval`] and `docs/benchmarks/`).
@@ -39,7 +39,7 @@ enum Command {
/// Zero-LLM: which facts the consolidation prompt's observation
/// projection keeps, per fixture.
Projection(projection::ProjectionArgs),
- /// LongMemEval retrieval benchmark through the real server stack.
+ /// Pinned retrieval benchmarks through the real server stack.
Retrieval(Box<retrieval::RetrievalArgs>),
}
diff --git a/evals/src/retrieval/dataset.rs b/evals/src/retrieval/dataset.rs
index 6e99b7b8..5bec0aa3 100644
--- a/evals/src/retrieval/dataset.rs
+++ b/evals/src/retrieval/dataset.rs
@@ -1,6 +1,6 @@
-//! LongMemEval (v1, S variant) dataset loading.
+//! Pinned retrieval datasets in the LongMemEval question/session format.
//!
-//! The file is NOT committed (278 MB); it lives gitignored under
+//! The LongMemEval file is NOT committed (278 MB); it lives gitignored under
//! `evals/datasets/` and is fetched on demand from HuggingFace with a
//! pinned sha256. Dataset: `xiaowu0162/longmemeval` (MIT license).
//!
@@ -9,6 +9,7 @@
//! which keeps the same 500 questions and evidence labels but replaces
//! distractor sessions that also held the answer. Other systems publish on
//! the cleaned file, so a comparison has to say which one it ran.
+//! The small synthetic `coding-v1` corpus is committed for offline smoke runs.
use std::path::Path;
@@ -32,8 +33,8 @@ pub struct DatasetVariant {
pub name: &'static str,
/// Default location under the repo root.
pub default_path: &'static str,
- /// Download URL (revision-pinned where upstream allows it).
- pub url: &'static str,
+ /// Download URL, or `None` for a fixture committed with the evaluator.
+ pub url: Option<&'static str>,
/// Expected sha256 of the file.
pub sha256: &'static str,
}
@@ -42,7 +43,7 @@ pub struct DatasetVariant {
pub const VARIANT_S: DatasetVariant = DatasetVariant {
name: "longmemeval_s",
default_path: "evals/datasets/longmemeval_s.json",
- url: LONGMEMEVAL_S_URL,
+ url: Some(LONGMEMEVAL_S_URL),
sha256: LONGMEMEVAL_S_SHA256,
};
@@ -51,10 +52,20 @@ pub const VARIANT_S: DatasetVariant = DatasetVariant {
pub const VARIANT_S_CLEANED: DatasetVariant = DatasetVariant {
name: "longmemeval_s_cleaned",
default_path: "evals/datasets/longmemeval_s_cleaned.json",
- url: "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_s_cleaned.json",
+ url: Some(
+ "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/98d7416c24c778c2fee6e6f3006e7a073259d48f/longmemeval_s_cleaned.json",
+ ),
sha256: "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
};
+/// Small synthetic coding-session evidence corpus, bundled for offline checks.
+pub const VARIANT_CODING_V1: DatasetVariant = DatasetVariant {
+ name: "coding_v1",
+ default_path: "evals/fixtures/retrieval/coding-v1.json",
+ url: None,
+ sha256: "af3382663a3e4e969c8bcfa744b3a794de86cb5932393a88bea16f014193d467",
+};
+
/// One chat turn inside a haystack session.
#[derive(Debug, Clone, Deserialize)]
pub struct Turn {
@@ -113,10 +124,12 @@ impl Question {
/// Load the dataset, verifying `variant`'s pinned sha256 first.
pub fn load(path: &Path, variant: &DatasetVariant) -> Result<Vec<Question>> {
let bytes = std::fs::read(path).with_context(|| {
- format!(
- "reading dataset at {} (use --fetch to download)",
- path.display()
- )
+ let recovery = if variant.url.is_some() {
+ "use --fetch to download"
+ } else {
+ "restore the bundled fixture from the repository"
+ };
+ format!("reading dataset at {} ({recovery})", path.display())
})?;
let digest = hex::encode(Sha256::digest(&bytes));
if digest != variant.sha256 {
@@ -129,7 +142,7 @@ pub fn load(path: &Path, variant: &DatasetVariant) -> Result<Vec<Question>> {
);
}
let questions: Vec<Question> =
- serde_json::from_slice(&bytes).context("parsing longmemeval_s JSON")?;
+ serde_json::from_slice(&bytes).context("parsing retrieval dataset JSON")?;
for q in &questions {
if q.haystack_sessions.len() != q.haystack_session_ids.len()
|| q.haystack_sessions.len() != q.haystack_dates.len()
@@ -149,11 +162,17 @@ pub fn load(path: &Path, variant: &DatasetVariant) -> Result<Vec<Question>> {
/// Download the dataset to `path` (atomically: tmp + rename) and verify
/// the pinned sha256 before the rename lands it.
pub async fn fetch(path: &Path, variant: &DatasetVariant) -> Result<()> {
+ let url = variant.url.with_context(|| {
+ format!(
+ "{} is bundled with the repository; restore {} instead of downloading it",
+ variant.name, variant.default_path
+ )
+ })?;
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
- tracing::info!(url = variant.url, "downloading {} (~278 MB)", variant.name);
- let resp = reqwest::get(variant.url).await?.error_for_status()?;
+ tracing::info!(url, "downloading {}", variant.name);
+ let resp = reqwest::get(url).await?.error_for_status()?;
let bytes = resp.bytes().await?;
let digest = hex::encode(Sha256::digest(&bytes));
if digest != variant.sha256 {
@@ -195,6 +214,102 @@ mod hex {
mod tests {
use super::*;
+ #[test]
+ fn coding_corpus_pin_and_evidence_labels_are_consistent() {
+ use std::collections::HashSet;
+ let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/retrieval/coding-v1.json");
+ let questions = load(&path, &VARIANT_CODING_V1).unwrap();
+ let raw: serde_json::Value =
+ serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
+ assert_eq!(questions.len(), 4);
+ let mut question_ids = HashSet::new();
+ for (q, labels) in questions.iter().zip(raw.as_array().unwrap()) {
+ assert!(question_ids.insert(&q.question_id));
+ assert!(!q.is_abstention());
+ assert!(!q.gold_answer_text().is_empty());
+ let ids: HashSet<_> = q.haystack_session_ids.iter().collect();
+ assert_eq!(ids.len(), q.haystack_session_ids.len());
+ assert!(
+ ids.len() > 10,
+ "recall@10 must not include every session by construction"
+ );
+ let evidence: HashSet<_> = q.answer_session_ids.iter().collect();
+ assert!(!evidence.is_empty());
+ let incorrect: HashSet<_> = labels["incorrect_session_ids"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|v| v.as_str().unwrap())
+ .collect();
+ for (id, turns) in q.haystack_session_ids.iter().zip(&q.haystack_sessions) {
+ let expected = evidence.contains(id);
+ assert_eq!(
+ turns.iter().any(Turn::is_evidence),
+ expected,
+ "{}: {id}",
+ q.question_id
+ );
+ assert_eq!(
+ incorrect.contains(id.as_str()),
+ !expected,
+ "alternatives must partition the haystack"
+ );
+ }
+ assert!(evidence.iter().all(|id| ids.contains(id)));
+ assert_eq!(incorrect.len() + evidence.len(), ids.len());
+ }
+ }
+
+ #[test]
+ fn coding_corrections_follow_their_stale_alternatives() {
+ let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/retrieval/coding-v1.json");
+ let questions = load(&path, &VARIANT_CODING_V1).unwrap();
+ for (id, old, current) in [
+ (
+ "coding-correction",
+ "coding-correction-stale",
+ "coding-correction-current",
+ ),
+ (
+ "coding-long-session",
+ "coding-long-session-wrong-header",
+ "coding-long-session-investigation",
+ ),
+ ] {
+ let q = questions.iter().find(|q| q.question_id == id).unwrap();
+ let old_index = q
+ .haystack_session_ids
+ .iter()
+ .position(|sid| sid == old)
+ .unwrap();
+ let current_index = q
+ .haystack_session_ids
+ .iter()
+ .position(|sid| sid == current)
+ .unwrap();
+ assert!(q.haystack_dates[old_index] < q.haystack_dates[current_index]);
+ assert!(!q.answer_session_ids.iter().any(|sid| sid == old));
+ assert!(q.answer_session_ids.iter().any(|sid| sid == current));
+ }
+ }
+
+ #[test]
+ fn missing_bundled_corpus_points_to_repository_restore() {
+ let tmp = tempfile::tempdir().unwrap();
+ let err = load(&tmp.path().join("missing.json"), &VARIANT_CODING_V1).unwrap_err();
+ assert!(err.to_string().contains("restore the bundled fixture"));
+ assert!(!err.to_string().contains("--fetch"));
+ }
+
+ #[tokio::test]
+ async fn bundled_corpus_fetch_fails_before_creating_a_file() {
+ let tmp = tempfile::tempdir().unwrap();
+ let path = tmp.path().join("missing/coding.json");
+ let err = fetch(&path, &VARIANT_CODING_V1).await.unwrap_err();
+ assert!(err.to_string().contains("bundled with the repository"));
+ assert!(!path.parent().unwrap().exists());
+ }
+
fn sample_json() -> serde_json::Value {
serde_json::json!([{
"question_id": "q1",
diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rs
index 4ab0c41b..94804e35 100644
--- a/evals/src/retrieval/mod.rs
+++ b/evals/src/retrieval/mod.rs
@@ -1,4 +1,4 @@
-//! LongMemEval retrieval benchmark, end to end through the real stack.
+//! Pinned retrieval benchmarks, end to end through the real stack.
//!
//! Per question: replay its haystack sessions through `POST /hook/batch`
//! (production hook cadence), query with `memory_query` over MCP, and
@@ -54,14 +54,13 @@ mod server;
#[derive(clap::Args, Debug)]
pub struct RetrievalArgs {
- /// Dataset file (LongMemEval-S JSON). Defaults to the selected
- /// variant's path under `evals/datasets/`.
+ /// Dataset file in LongMemEval JSON format. Defaults to the selected
+ /// variant's pinned path.
#[arg(long)]
dataset: Option<PathBuf>,
- /// Which pinned LongMemEval-S file to use: `s` (the original, every
- /// published ai-memory baseline) or `s-cleaned` (the re-release other
- /// systems publish on).
+ /// Pinned dataset: `s` or `s-cleaned` for LongMemEval, or the bundled
+ /// offline coding-session corpus `coding-v1`.
#[arg(long, value_enum, default_value_t = Variant::S)]
dataset_variant: Variant,
@@ -75,7 +74,7 @@ pub struct RetrievalArgs {
server_bin: PathBuf,
/// Score only the first N questions (deterministic prefix) — smoke
- /// runs and iteration. Omit for the full 500.
+ /// runs and iteration. Omit for the entire selected dataset.
#[arg(long)]
sample: Option<usize>,
@@ -276,6 +275,7 @@ impl RetrievalArgs {
enum Variant {
S,
SCleaned,
+ CodingV1,
}
impl Variant {
@@ -283,6 +283,7 @@ impl Variant {
match self {
Self::S => &dataset::VARIANT_S,
Self::SCleaned => &dataset::VARIANT_S_CLEANED,
+ Self::CodingV1 => &dataset::VARIANT_CODING_V1,
}
}
}
diff --git a/evals/src/retrieval/report.rs b/evals/src/retrieval/report.rs
index d64d446d..a91a97ba 100644
--- a/evals/src/retrieval/report.rs
+++ b/evals/src/retrieval/report.rs
@@ -287,8 +287,8 @@ const METRIC_NOTES: &str = "\nNotes: hit@k = any evidence session in top k (the
pub fn to_markdown(r: &Report) -> String {
let mut md = String::new();
md.push_str(&format!(
- "# LongMemEval-S retrieval — {}\n\n",
- r.generated_at
+ "# {} retrieval — {}\n\n",
+ r.dataset, r.generated_at
));
md.push_str(&provenance_md(r));
md.push_str(&format!("- mode: {}\n- config: {}\n\n", r.mode, r.config));
@@ -400,8 +400,8 @@ fn compute_delta(baseline: &Report, candidate: &Report) -> AbDelta {
pub fn ab_to_markdown(r: &AbReport) -> String {
let mut md = String::new();
md.push_str(&format!(
- "# LongMemEval-S retrieval A/B — {}\n\n",
- r.candidate.generated_at
+ "# {} retrieval A/B — {}\n\n",
+ r.candidate.dataset, r.candidate.generated_at
));
// Provenance is shared (same dataset/commit/hardware/question set).
md.push_str(&provenance_md(&r.candidate));
@@ -509,10 +509,24 @@ mod tests {
}
}
+ #[test]
+ fn coding_reports_name_the_selected_dataset() {
+ let mut baseline = report("embeddings=none", 1.0, 1.0, 1.0);
+ baseline.dataset = "coding_v1";
+ let mut candidate = report("embeddings=none", 1.0, 1.0, 1.0);
+ candidate.dataset = "coding_v1";
+ assert!(to_markdown(&baseline).starts_with("# coding_v1 retrieval"));
+ assert!(
+ ab_to_markdown(&AbReport::new(baseline, candidate))
+ .starts_with("# coding_v1 retrieval A/B")
+ );
+ }
+
#[test]
fn markdown_carries_provenance_and_the_triple() {
let r = report("embeddings=none reranker=off", 0.5, 0.25, 0.0);
let md = to_markdown(&r);
+ assert!(md.starts_with("# longmemeval_s retrieval"));
assert!(md.contains("commit: `abc`"));
assert!(md.contains("config: embeddings=none reranker=off"));
assert!(md.contains(" hit@5 | recall@5 | recall-all@5 |"));
diff --git a/evals/tests/smoke.rs b/evals/tests/smoke.rs
index eb7cccca..67f17080 100644
--- a/evals/tests/smoke.rs
+++ b/evals/tests/smoke.rs
@@ -125,3 +125,55 @@ fn the_ab_path_reports_a_zero_delta_for_identical_configs() {
assert_eq!(delta["context_tokens_median"], 0.0);
assert!(run_dir.join("report.md").exists());
}
+
+/// The bundled corpus exercises the same server and scorer without downloading
+/// LongMemEval. Identical configurations must agree on evidence and context.
+#[test]
+#[ignore = "needs the release server binary; uses the bundled offline corpus"]
+fn coding_corpus_replays_offline_with_identical_ab_results() {
+ let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
+ let server_bin = repo_root.join("target/release/ai-memory");
+ assert!(server_bin.exists(), "build the release server first");
+ let out = tempfile::tempdir().unwrap();
+ let status = Command::new(env!("CARGO_BIN_EXE_ai-memory-eval"))
+ .current_dir(&repo_root)
+ .args([
+ "retrieval",
+ "--dataset-variant",
+ "coding-v1",
+ "--candidate",
+ "--concurrency",
+ "1",
+ ])
+ .arg("--server-bin")
+ .arg(&server_bin)
+ .arg("--out")
+ .arg(out.path())
+ .status()
+ .unwrap();
+ assert!(status.success());
+ let run_dir = std::fs::read_dir(out.path())
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .find(|path| path.is_dir())
+ .unwrap();
+ let report: serde_json::Value =
+ serde_json::from_slice(&std::fs::read(run_dir.join("report.json")).unwrap()).unwrap();
+ assert_eq!(report["baseline"]["dataset"], "coding_v1");
+ for question in report["baseline"]["per_question"].as_array().unwrap() {
+ assert_eq!(
+ question["recall_at"]["10"], 1.0,
+ "missing expected evidence: {}",
+ question["question_id"]
+ );
+ }
+ assert_eq!(report["baseline"]["questions_scored"], 4);
+ assert_eq!(report["candidate"]["questions_scored"], 4);
+ for metric in ["hit_at", "recall_at", "recall_all_at"] {
+ for k in ["1", "3", "5", "10"] {
+ assert_eq!(report["delta"][metric][k], 0.0, "{metric}@{k}");
+ }
+ }
+ assert_eq!(report["delta"]["context_tokens_mean"], 0.0);
+ assert_eq!(report["delta"]["context_tokens_median"], 0.0);
+}