Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

evals: live A/B harness comparing two LLM providers on the consolidation prompt

AkitaOnRailsdf7671dd56f4
diff --git a/.gitignore b/.gitignore
index c58e87e3..53627586 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,4 @@ perf.data*
# Sandbox marker (not project content).
.ai-jail
.claude/
+evals/runs/
diff --git a/Cargo.lock b/Cargo.lock
index 756eba27..80be0c91 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -99,6 +99,25 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "ai-memory-eval"
+version = "0.1.0"
+dependencies = [
+ "ai-memory-consolidate",
+ "ai-memory-core",
+ "ai-memory-llm",
+ "anyhow",
+ "clap",
+ "jiff",
+ "secrecy",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+ "uuid",
+]
+
[[package]]
name = "ai-memory-hooks"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 2e8d646f..cc12e413 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,9 @@ members = [
"crates/ai-memory-llm",
"crates/ai-memory-consolidate",
"crates/ai-memory-cli",
+ # Live A/B harness — not part of the shipped binary, but in
+ # the workspace so it shares deps + builds with the rest.
+ "evals",
]
[workspace.package]
diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rs
index de2b2250..fb4910ce 100644
--- a/crates/ai-memory-consolidate/src/consolidator.rs
+++ b/crates/ai-memory-consolidate/src/consolidator.rs
@@ -306,7 +306,11 @@ const fn tier_as_str(t: Tier) -> &'static str {
}
}
-fn build_batch_request(session_id: SessionId, observations: &[Observation]) -> ChatRequest {
+/// Build the exact ChatRequest the consolidator sends for batch
+/// multi-page consolidation. Exposed so off-tree A/B harnesses
+/// (e.g. `evals/`) can exercise the same workload against
+/// alternative providers without duplicating the prompt.
+pub fn build_batch_request(session_id: SessionId, observations: &[Observation]) -> ChatRequest {
let mut buf = String::new();
buf.push_str(
"You are compiling a Karpathy-style multi-page wiki update. Given the \
@@ -351,7 +355,9 @@ fn build_batch_request(session_id: SessionId, observations: &[Observation]) -> C
}
}
-const BATCH_SYSTEM_PROMPT: &str = "You are the maintainer of a Karpathy-style LLM wiki \
+/// System prompt for batch consolidation. Public for the same
+/// reason as [`build_batch_request`].
+pub const BATCH_SYSTEM_PROMPT: &str = "You are the maintainer of a Karpathy-style LLM wiki \
for a software engineer. Your job is to compile *durable* knowledge -- not just \
restate the session log. Produce a ConsolidatedBatch with 1-5 page updates. Favour \
extracting concept / decision / gotcha pages alongside the session summary when the \
diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rs
index 0aeb9c45..82d050c0 100644
--- a/crates/ai-memory-consolidate/src/lib.rs
+++ b/crates/ai-memory-consolidate/src/lib.rs
@@ -15,7 +15,9 @@ pub mod sweep;
pub mod types;
pub use bootstrap::{Bootstrap, BootstrapConfig, BootstrapError, BootstrapOutcome};
-pub use consolidator::{Consolidator, ConsolidatorError, ConsolidatorResult};
+pub use consolidator::{
+ BATCH_SYSTEM_PROMPT, Consolidator, ConsolidatorError, ConsolidatorResult, build_batch_request,
+};
pub use lint::{LintError, LintFinding, LintReport, run_lint};
pub use sweep::{EvictedPage, SweepError, SweepReport, run_sweep};
pub use types::{
diff --git a/evals/Cargo.toml b/evals/Cargo.toml
new file mode 100644
index 00000000..80af7b43
--- /dev/null
+++ b/evals/Cargo.toml
@@ -0,0 +1,27 @@
+[package]
+name = "ai-memory-eval"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.95"
+license = "MIT OR Apache-2.0"
+publish = false
+
+[[bin]]
+name = "ai-memory-eval"
+path = "src/main.rs"
+
+[dependencies]
+ai-memory-core.workspace = true
+ai-memory-llm.workspace = true
+ai-memory-consolidate.workspace = true
+
+anyhow.workspace = true
+clap.workspace = true
+jiff.workspace = true
+secrecy.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+uuid.workspace = true
diff --git a/evals/README.md b/evals/README.md
new file mode 100644
index 00000000..643bf3cb
--- /dev/null
+++ b/evals/README.md
@@ -0,0 +1,134 @@
+# `evals/` — live A/B harness
+
+A small Rust binary that runs the EXACT consolidation prompt
+ai-memory uses in production against two LLM providers side by side,
+and saves both outputs to disk for human comparison.
+
+**This is not part of the shipped binary.** It's a workspace member
+purely so it shares deps + builds with the rest. `cargo build` from
+the repo root will compile it, but it's never bundled into the
+docker image and never run by CI.
+
+## When to reach for this
+
+After switching providers, models, or major prompt edits. Concretely:
+
+- Replacing OpenRouter/Kimi with local Ollama (the case that
+ motivated this harness — see commit log).
+- Trying a new Ollama model (`qwen3:32b` → `qwen3-coder:30b`).
+- Tuning the `BATCH_SYSTEM_PROMPT` itself — does the rewrite
+ preserve quality across providers?
+
+The fixtures are deliberately small synthetic session logs that
+exercise the prompt's hard cases (durable rule extraction,
+multi-topic separation, "say nothing" sessions, decision/gotcha
+distinction).
+
+## What it does
+
+For each `*.json` under `evals/fixtures/`:
+
+1. Builds the request via
+ [`ai_memory_consolidate::build_batch_request`] — same code path
+ the live consolidator uses.
+2. Sends it to a **baseline** and a **candidate** provider, *in
+ parallel*.
+3. Runs the result through
+ [`ai_memory_llm::complete_structured`] — same JSON-schema
+ validation the live system applies. Schema-parse failure is
+ recorded (not fatal).
+4. Persists to `evals/runs/<timestamp>/{baseline,candidate}/<fixture>.{json,md,meta.json}`.
+
+The runner prints latency + parse status per fixture and a tail
+summary. **Quality is for you to read** — open the markdown files
+in `runs/<timestamp>/baseline/` and `…/candidate/` side by side and
+judge faithfulness, scoping, hallucination, etc.
+
+## Running it
+
+### Ollama qwen3:32b vs OpenRouter Kimi (the canonical comparison)
+
+```bash
+# OPENROUTER_API_KEY in env; LLM_API_KEY can be any non-empty
+# string for the candidate (Ollama doesn't validate).
+export OPENROUTER_API_KEY="sk-or-v1-..."
+
+cargo run -p ai-memory-eval -- \
+ --baseline-provider openai-compat \
+ --baseline-base-url https://openrouter.ai/api/v1 \
+ --baseline-model moonshotai/kimi-k2.6 \
+ --baseline-api-key-env OPENROUTER_API_KEY \
+ --candidate-provider openai-compat \
+ --candidate-base-url http://192.168.0.90:11434/v1 \
+ --candidate-model qwen3:32b \
+ --candidate-api-key ollama-local
+```
+
+### Two Ollama models against each other
+
+```bash
+cargo run -p ai-memory-eval -- \
+ --baseline-provider openai-compat \
+ --baseline-base-url http://192.168.0.90:11434/v1 \
+ --baseline-model qwen3:32b \
+ --baseline-api-key ollama-local \
+ --candidate-provider openai-compat \
+ --candidate-base-url http://192.168.0.90:11434/v1 \
+ --candidate-model qwen3-coder:30b \
+ --candidate-api-key ollama-local
+```
+
+### Reading the output
+
+```
+evals/runs/2026-05-22T18-30-00Z/
+├── baseline/
+│ ├── 01-rust-bug-fix.json ← raw structured output
+│ ├── 01-rust-bug-fix.md ← flat markdown rendering, easy to read
+│ └── 01-rust-bug-fix.meta.json ← {elapsed_ms, parsed_ok, update_count}
+└── candidate/
+ └── …
+```
+
+Eyeball the `.md` files in pairs. The runner also prints a
+`diff -ru baseline candidate` command you can run.
+
+## Adding fixtures
+
+Each fixture is a JSON file:
+
+```json
+{
+ "name": "human-readable",
+ "description": "what this case is meant to surface",
+ "observations": [
+ {"kind": "session-start", "title": "...", "body": "..."},
+ {"kind": "user-prompt", "title": "user prompt", "body": "..."},
+ {"kind": "pre-tool-use", "title": "Edit", "body": "..."},
+ ...
+ ]
+}
+```
+
+`kind` values: `session-start`, `user-prompt`, `pre-tool-use`,
+`post-tool-use`, `pre-compact`, `notification`, `stop`,
+`session-end`, `other` (see `ObservationKind` in `ai-memory-core`).
+
+The `description` field isn't read by the runner — it's a comment
+for the next human who opens the file.
+
+## What this harness does NOT do
+
+- **Score quality automatically.** Pure side-by-side. If you want
+ metrics, the next layer up would be keyword recall (`must_mention`
+ per fixture) or an LLM-as-judge pass — both deliberately out of
+ scope here.
+- **Test the embedding pipeline.** This only exercises the
+ consolidation LLM. Embedding A/B would be a parallel harness
+ (probe queries + expected target pages, measure recall@5/MRR).
+- **Persist via the real wiki layer.** No SQLite, no markdown
+ writes, no git. Pure prompt → response.
+
+## Cleanup
+
+`evals/runs/` is in `.gitignore`. Drop it whenever it gets large.
diff --git a/evals/fixtures/01-rust-bug-fix.json b/evals/fixtures/01-rust-bug-fix.json
new file mode 100644
index 00000000..3fd9487a
--- /dev/null
+++ b/evals/fixtures/01-rust-bug-fix.json
@@ -0,0 +1,17 @@
+{
+ "name": "rust-bug-fix",
+ "description": "Realistic short session: user fixes a Rust async bug, decides on a pattern, hits one gotcha. Tests whether the model extracts both the decision and the gotcha as separate pages, not just a session log.",
+ "observations": [
+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory, agent=claude-code"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "the writer actor is panicking when the bounded mpsc channel fills up — fix it without unbounding the channel"},
+ {"kind": "pre-tool-use", "title": "Read", "body": "crates/ai-memory-store/src/writer.rs"},
+ {"kind": "post-tool-use", "title": "Read result", "body": "found send() being called from an async context — current code uses try_send and unwraps the Err on full"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "yes, replace the try_send unwrap with awaiting send and let backpressure propagate; do not silently drop writes"},
+ {"kind": "pre-tool-use", "title": "Edit", "body": "writer.rs: changed try_send().unwrap() to send().await?"},
+ {"kind": "post-tool-use", "title": "cargo test", "body": "1 test failed: writer_panics_on_full — needs updating since panic behavior changed; replaced with assertion on backpressure delay"},
+ {"kind": "pre-tool-use", "title": "Edit", "body": "writer.rs tests: replaced the panic-expectation test with a timing-based backpressure test"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "also note that I want fire-and-forget hook ingress to remain bounded; never block the HTTP handler on the writer"},
+ {"kind": "post-tool-use", "title": "cargo test", "body": "all green"},
+ {"kind": "session-end", "title": "session end", "body": "fixed; committed as ec1a4f0"}
+ ]
+}
diff --git a/evals/fixtures/02-architecture-decision.json b/evals/fixtures/02-architecture-decision.json
new file mode 100644
index 00000000..f66a78d0
--- /dev/null
+++ b/evals/fixtures/02-architecture-decision.json
@@ -0,0 +1,15 @@
+{
+ "name": "architecture-decision",
+ "description": "User makes a deliberate architectural choice (single-writer SQLite vs. write-pool). Tests whether the model produces a decisions/ ADR-style page distinct from the session log.",
+ "observations": [
+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "I'm seeing 'database is locked' under concurrent writes in the integration tests. should I move to a connection pool with retry, or a single-writer actor?"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "actually let me think out loud — a pool means rusqlite serialises writes anyway via BEGIN IMMEDIATE retries, and we'd still see SQLITE_BUSY under contention. With a single-writer actor we get FIFO ordering for free, no BUSY, and an obvious place to add audit log."},
+ {"kind": "user-prompt", "title": "user prompt", "body": "going with single-writer actor; pool the readers separately."},
+ {"kind": "pre-tool-use", "title": "Write", "body": "crates/ai-memory-store/src/writer.rs: new WriterActor owning one Connection, accepts WriteCmd via tokio::sync::mpsc"},
+ {"kind": "pre-tool-use", "title": "Write", "body": "crates/ai-memory-store/src/reader.rs: ReaderPool of 4 read-only Connections"},
+ {"kind": "post-tool-use", "title": "cargo test", "body": "concurrent_writes test green; no SQLITE_BUSY under 100 parallel writers"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "the audit log column was the clincher — we couldn't do it cleanly with a pool"},
+ {"kind": "session-end", "title": "session end", "body": "architecture changed; ADR worth recording"}
+ ]
+}
diff --git a/evals/fixtures/03-gotcha-with-rule.json b/evals/fixtures/03-gotcha-with-rule.json
new file mode 100644
index 00000000..f80302f2
--- /dev/null
+++ b/evals/fixtures/03-gotcha-with-rule.json
@@ -0,0 +1,14 @@
+{
+ "name": "gotcha-with-rule",
+ "description": "Session that produces both a gotcha (test ordering) AND a durable rule ('never mock the DB'). Tests whether the model classifies the rule with kind=rule and writes it as a standalone instruction (not just a note).",
+ "observations": [
+ {"kind": "user-prompt", "title": "user prompt", "body": "the test suite passes locally but fails in CI — different ordering?"},
+ {"kind": "pre-tool-use", "title": "cargo test --workspace", "body": "test results in CI: integration test 'multi_writer_serialisation' fails — relies on the writer being a singleton across tests, but parallel runners start two"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "ah, parallel test runners each spin up their own ai-memory instance; the global locking assumption doesn't hold cross-process"},
+ {"kind": "pre-tool-use", "title": "Edit", "body": "tests now use #[serial_test::serial] and a per-test temp data dir"},
+ {"kind": "post-tool-use", "title": "cargo test --workspace", "body": "all green, including CI"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "ok, two things to remember: (1) integration tests against the store must NEVER mock the database — last quarter we ate a 3-day mock/prod divergence bug; (2) any test touching the writer needs the per-test temp data dir helper, or document why not"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "the 'never mock the database' is a hard project rule. write it down somewhere durable."},
+ {"kind": "session-end", "title": "session end", "body": "two distinct lessons: per-test data dir, never mock DB"}
+ ]
+}
diff --git a/evals/fixtures/04-low-signal-session.json b/evals/fixtures/04-low-signal-session.json
new file mode 100644
index 00000000..78c36772
--- /dev/null
+++ b/evals/fixtures/04-low-signal-session.json
@@ -0,0 +1,13 @@
+{
+ "name": "low-signal-session",
+ "description": "A short, mundane session with no durable lessons — just minor edits. Tests whether the model RESISTS the urge to manufacture concept/decision pages and just returns the session log. (The system prompt explicitly says to favour returning only the session page when there's no reusable insight.)",
+ "observations": [
+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "fix the typo in README — should be 'Karpathy' not 'Karparthy'"},
+ {"kind": "pre-tool-use", "title": "Edit", "body": "README.md: Karparthy → Karpathy"},
+ {"kind": "post-tool-use", "title": "git diff", "body": "1 line changed"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "commit it"},
+ {"kind": "pre-tool-use", "title": "git commit", "body": "docs: fix Karpathy spelling"},
+ {"kind": "session-end", "title": "session end", "body": "trivial"}
+ ]
+}
diff --git a/evals/fixtures/05-multi-topic-session.json b/evals/fixtures/05-multi-topic-session.json
new file mode 100644
index 00000000..7ff4c0dc
--- /dev/null
+++ b/evals/fixtures/05-multi-topic-session.json
@@ -0,0 +1,16 @@
+{
+ "name": "multi-topic-session",
+ "description": "Mixed session covering two unrelated topics. Tests whether the model produces SEPARATE pages per topic rather than mashing them together. The two topics share no semantic overlap; if the model emits one mixed concept page it has failed at scoping.",
+ "observations": [
+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "the docker image is 380 MB which is way too big — can we strip debug symbols and use a distroless base?"},
+ {"kind": "pre-tool-use", "title": "Edit", "body": "Dockerfile: stage 1 rust:slim, stage 2 gcr.io/distroless/cc-debian12, strip symbols via CARGO_PROFILE_RELEASE_STRIP=symbols"},
+ {"kind": "post-tool-use", "title": "docker build", "body": "image dropped from 380 MB to 67 MB"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "great. unrelated — I noticed `cargo audit` is failing in CI on a transitive dep `xml-rs` with RUSTSEC-2024-xxxx. can we work around?"},
+ {"kind": "pre-tool-use", "title": "Read", "body": "Cargo.lock — xml-rs is pulled in by git2 → libgit2-sys → vendored-libgit2 transitively"},
+ {"kind": "user-prompt", "title": "user prompt", "body": "git2 is load-bearing for the wiki versioning so we can't drop it. add an ignore entry in audit.toml with a TODO+date to revisit"},
+ {"kind": "pre-tool-use", "title": "Write", "body": "audit.toml: ignore RUSTSEC-2024-xxxx with note 'transitive via libgit2-sys; revisit 2026-08'"},
+ {"kind": "post-tool-use", "title": "cargo audit", "body": "passes"},
+ {"kind": "session-end", "title": "session end", "body": "two distinct deliverables: docker slim + audit workaround"}
+ ]
+}
diff --git a/evals/src/main.rs b/evals/src/main.rs
new file mode 100644
index 00000000..c618772f
--- /dev/null
+++ b/evals/src/main.rs
@@ -0,0 +1,482 @@
+//! Live A/B harness for the ai-memory consolidation prompt.
+//!
+//! ## What it does
+//!
+//! For each fixture under `evals/fixtures/*.json` (each one a small,
+//! synthetic session log), the runner:
+//!
+//! 1. Calls [`ai_memory_consolidate::build_batch_request`] to build
+//! the EXACT ChatRequest the production consolidator would send.
+//! 2. Sends that request to two providers concurrently — a
+//! *baseline* and a *candidate* — via
+//! [`ai_memory_llm::complete_structured`], which is the same
+//! parse-extract-and-validate path the live system uses.
+//! 3. Saves the deserialised `ConsolidatedBatch`, the raw JSON, a
+//! flattened markdown rendering, and a `meta.json` (timing,
+//! update count, parse status) per fixture per provider under
+//! `evals/runs/<timestamp>/{baseline,candidate}/`.
+//! 4. Prints a side-by-side summary table to stdout.
+//!
+//! Quality judgement is left to the human reading the outputs — the
+//! harness only reports the OBJECTIVE deltas (latency, update
+//! counts, schema-parse success). Anything subtler (faithfulness,
+//! hallucination, structure) is for you to eyeball.
+//!
+//! ## Running it
+//!
+//! See `evals/README.md` for canonical invocations. The short
+//! version:
+//!
+//! ```bash
+//! cargo run -p ai-memory-eval -- \
+//! --baseline-provider openai-compat \
+//! --baseline-base-url https://openrouter.ai/api/v1 \
+//! --baseline-model moonshotai/kimi-k2.6 \
+//! --baseline-api-key-env OPENROUTER_API_KEY \
+//! --candidate-provider openai-compat \
+//! --candidate-base-url http://192.168.0.90:11434/v1 \
+//! --candidate-model qwen3:32b \
+//! --candidate-api-key ollama-local
+//! ```
+
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use anyhow::{Context, Result, bail};
+use clap::Parser;
+use jiff::Timestamp;
+use secrecy::SecretString;
+use serde::{Deserialize, Serialize};
+
+use ai_memory_consolidate::{ConsolidatedBatch, build_batch_request};
+use ai_memory_core::{Observation, ObservationKind, ProjectId, SessionId, WorkspaceId};
+use ai_memory_llm::{
+ LlmProvider, ProviderChoice, ProviderConfig, complete_structured,
+};
+
+#[derive(Parser, Debug)]
+#[command(
+ name = "ai-memory-eval",
+ about = "Live A/B harness comparing two LLM providers on the ai-memory consolidation prompt"
+)]
+struct Args {
+ /// Fixture directory (each `*.json` is one synthetic session log).
+ #[arg(long, default_value = "evals/fixtures")]
+ fixtures: PathBuf,
+
+ /// Output root. A timestamped subdir gets created here per run.
+ #[arg(long, default_value = "evals/runs")]
+ out: PathBuf,
+
+ /// Baseline provider (`anthropic` / `openai` / `openai-compat`).
+ #[arg(long)]
+ baseline_provider: String,
+
+ /// Baseline base URL (omit for native Anthropic/OpenAI).
+ #[arg(long)]
+ baseline_base_url: Option<String>,
+
+ /// Baseline model id.
+ #[arg(long)]
+ baseline_model: String,
+
+ /// Baseline API key (raw value). Prefer `--baseline-api-key-env`.
+ #[arg(long)]
+ baseline_api_key: Option<String>,
+
+ /// Env var to read the baseline API key from.
+ #[arg(long)]
+ baseline_api_key_env: Option<String>,
+
+ /// Candidate provider.
+ #[arg(long)]
+ candidate_provider: String,
+
+ /// Candidate base URL.
+ #[arg(long)]
+ candidate_base_url: Option<String>,
+
+ /// Candidate model id.
+ #[arg(long)]
+ candidate_model: String,
+
+ /// Candidate API key (raw value).
+ #[arg(long)]
+ candidate_api_key: Option<String>,
+
+ /// Env var to read the candidate API key from.
+ #[arg(long)]
+ candidate_api_key_env: Option<String>,
+}
+
+/// JSON shape of an eval fixture file. We only care about
+/// observations the consolidation prompt actually consumes; the
+/// other Observation fields get populated with dummies at runtime.
+#[derive(Debug, Deserialize)]
+struct Fixture {
+ name: String,
+ /// Free-text describing the scenario; not consumed by the runner
+ /// but useful when reading fixture files by hand.
+ #[allow(dead_code)]
+ #[serde(default)]
+ description: String,
+ observations: Vec<FixtureObservation>,
+}
+
+#[derive(Debug, Deserialize)]
+struct FixtureObservation {
+ kind: String,
+ title: String,
+ #[serde(default)]
+ body: String,
+}
+
+/// One result of one provider on one fixture.
+#[derive(Debug, Serialize)]
+struct CaseRun {
+ provider: String,
+ fixture: String,
+ elapsed_ms: u128,
+ parsed_ok: bool,
+ error: Option<String>,
+ update_count: Option<usize>,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ tracing_subscriber::fmt()
+ .with_env_filter(
+ tracing_subscriber::EnvFilter::try_from_default_env()
+ .unwrap_or_else(|_| "ai_memory_eval=info,warn".into()),
+ )
+ .init();
+
+ let args = Args::parse();
+
+ let baseline = resolve_provider("baseline", &args)?;
+ let candidate = resolve_provider("candidate", &args)?;
+
+ let baseline_provider = make_provider(baseline.clone())
+ .context("building baseline provider")?;
+ let candidate_provider = make_provider(candidate.clone())
+ .context("building candidate provider")?;
+
+ let fixtures = load_fixtures(&args.fixtures)
+ .with_context(|| format!("loading fixtures from {}", args.fixtures.display()))?;
+ if fixtures.is_empty() {
+ bail!("no *.json fixtures found in {}", args.fixtures.display());
+ }
+
+ // Timestamped run dir; filesystem-safe (colons + dots → dashes).
+ let stamp: String = Timestamp::now()
+ .to_string()
+ .chars()
+ .map(|c| if c == ':' || c == '.' { '-' } else { c })
+ .collect();
+ let run_dir = args.out.join(&stamp);
+ std::fs::create_dir_all(run_dir.join("baseline"))?;
+ std::fs::create_dir_all(run_dir.join("candidate"))?;
+
+ println!();
+ println!("ai-memory eval — {stamp}");
+ println!(" fixtures : {} ({})", fixtures.len(), args.fixtures.display());
+ println!(" baseline : {} {} @ {}", baseline.provider_name(), baseline.model, baseline.base_url.as_deref().unwrap_or("(default)"));
+ println!(" candidate: {} {} @ {}", candidate.provider_name(), candidate.model, candidate.base_url.as_deref().unwrap_or("(default)"));
+ println!(" out : {}", run_dir.display());
+ println!();
+
+ let mut results: Vec<(CaseRun, CaseRun)> = Vec::with_capacity(fixtures.len());
+ for (path, fx) in fixtures {
+ println!("→ {} ({} observations)", fx.name, fx.observations.len());
+ let observations = synthesise_observations(&fx);
+ let session_id = SessionId::new();
+
+ // Run both providers concurrently per fixture so a slow one
+ // doesn't double the wall-clock.
+ let req_a = build_batch_request(session_id, &observations);
+ let req_b = build_batch_request(session_id, &observations);
+ let (a_res, b_res) = tokio::join!(
+ run_one(baseline_provider.clone(), "baseline", req_a),
+ run_one(candidate_provider.clone(), "candidate", req_b),
+ );
+
+ let a = persist_case("baseline", &fx.name, &path, &run_dir, a_res)?;
+ let b = persist_case("candidate", &fx.name, &path, &run_dir, b_res)?;
+
+ println!(
+ " baseline : {:>5} ms parsed={} updates={}",
+ a.elapsed_ms,
+ a.parsed_ok,
+ a.update_count.map_or("-".into(), |n| n.to_string())
+ );
+ println!(
+ " candidate: {:>5} ms parsed={} updates={}",
+ b.elapsed_ms,
+ b.parsed_ok,
+ b.update_count.map_or("-".into(), |n| n.to_string())
+ );
+ results.push((a, b));
+ }
+
+ // Tail summary so the user can grok the run at a glance.
+ println!();
+ println!("=== summary ===");
+ let n = results.len();
+ let baseline_total: u128 = results.iter().map(|(a, _)| a.elapsed_ms).sum();
+ let candidate_total: u128 = results.iter().map(|(_, b)| b.elapsed_ms).sum();
+ let baseline_ok = results.iter().filter(|(a, _)| a.parsed_ok).count();
+ let candidate_ok = results.iter().filter(|(_, b)| b.parsed_ok).count();
+ println!("baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms", baseline_total / n.max(1) as u128);
+ println!("candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms", candidate_total / n.max(1) as u128);
+ println!();
+ println!("outputs under: {}", run_dir.display());
+ println!("compare with: diff -ru {}/baseline {}/candidate", run_dir.display(), run_dir.display());
+ Ok(())
+}
+
+/// Resolve `ProviderConfig` for one side from CLI args + env.
+fn resolve_provider(side: &str, args: &Args) -> Result<ResolvedConfig> {
+ let (provider_str, base_url, model, api_key_inline, api_key_env) = match side {
+ "baseline" => (
+ args.baseline_provider.as_str(),
+ args.baseline_base_url.clone(),
+ args.baseline_model.clone(),
+ args.baseline_api_key.clone(),
+ args.baseline_api_key_env.clone(),
+ ),
+ "candidate" => (
+ args.candidate_provider.as_str(),
+ args.candidate_base_url.clone(),
+ args.candidate_model.clone(),
+ args.candidate_api_key.clone(),
+ args.candidate_api_key_env.clone(),
+ ),
+ _ => bail!("unknown side {side}"),
+ };
+
+ let provider = match provider_str {
+ "anthropic" => ProviderChoice::Anthropic,
+ "openai" => ProviderChoice::OpenAi,
+ "openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,
+ other => bail!(
+ "{side}: provider {other} not one of anthropic|openai|openai-compat"
+ ),
+ };
+
+ let api_key = match (api_key_inline, api_key_env) {
+ (Some(s), _) if !s.is_empty() => Some(SecretString::from(s)),
+ (_, Some(env)) => match std::env::var(&env) {
+ Ok(v) if !v.is_empty() => Some(SecretString::from(v)),
+ _ => bail!("{side}: env var {env} is not set or empty"),
+ },
+ _ => None,
+ };
+
+ Ok(ResolvedConfig {
+ provider,
+ provider_str: provider_str.to_string(),
+ model,
+ base_url,
+ api_key,
+ })
+}
+
+struct ResolvedConfig {
+ provider: ProviderChoice,
+ provider_str: String,
+ model: String,
+ base_url: Option<String>,
+ api_key: Option<SecretString>,
+}
+
+impl ResolvedConfig {
+ fn provider_name(&self) -> &str {
+ self.provider_str.as_str()
+ }
+}
+
+impl Clone for ResolvedConfig {
+ fn clone(&self) -> Self {
+ Self {
+ provider: self.provider,
+ provider_str: self.provider_str.clone(),
+ model: self.model.clone(),
+ base_url: self.base_url.clone(),
+ // SecretString is not Clone — re-wrap if needed.
+ api_key: self.api_key.as_ref().map(|s| {
+ use secrecy::ExposeSecret;
+ SecretString::from(s.expose_secret().to_string())
+ }),
+ }
+ }
+}
+
+impl From<ResolvedConfig> for ProviderConfig {
+ fn from(r: ResolvedConfig) -> Self {
+ Self {
+ provider: r.provider,
+ model: r.model,
+ api_key: r.api_key,
+ base_url: r.base_url,
+ }
+ }
+}
+
+fn make_provider(r: ResolvedConfig) -> Result<Arc<dyn LlmProvider>> {
+ ai_memory_llm::build_provider(ProviderConfig::from(r))
+ .map_err(anyhow::Error::from)
+ .context("building provider")
+}
+
+fn load_fixtures(dir: &Path) -> Result<Vec<(PathBuf, Fixture)>> {
+ let mut paths: Vec<_> = std::fs::read_dir(dir)?
+ .filter_map(|e| e.ok().map(|e| e.path()))
+ .filter(|p| p.extension().is_some_and(|e| e == "json"))
+ .collect();
+ paths.sort();
+ let mut out = Vec::with_capacity(paths.len());
+ for p in paths {
+ let bytes = std::fs::read(&p)
+ .with_context(|| format!("reading {}", p.display()))?;
+ let fx: Fixture = serde_json::from_slice(&bytes)
+ .with_context(|| format!("parsing {}", p.display()))?;
+ out.push((p, fx));
+ }
+ Ok(out)
+}
+
+fn synthesise_observations(fx: &Fixture) -> Vec<Observation> {
+ let workspace_id = WorkspaceId::new();
+ let project_id = ProjectId::new();
+ let session_id = SessionId::new();
+ fx.observations
+ .iter()
+ .map(|f| Observation {
+ id: ai_memory_core::ObservationId::new(),
+ session_id,
+ workspace_id,
+ project_id,
+ kind: f
+ .kind
+ .parse::<ObservationKind>()
+ .unwrap_or(ObservationKind::Other),
+ title: f.title.clone(),
+ body: f.body.clone(),
+ importance: 5,
+ created_at: Timestamp::now(),
+ })
+ .collect()
+}
+
+struct ProviderResult {
+ elapsed: Duration,
+ batch: Option<ConsolidatedBatch>,
+ raw: Option<serde_json::Value>,
+ error: Option<String>,
+}
+
+async fn run_one(
+ provider: Arc<dyn LlmProvider>,
+ side: &str,
+ request: ai_memory_llm::ChatRequest,
+) -> ProviderResult {
+ let start = Instant::now();
+ // Try the schema-validated path first; fall back to the raw
+ // structured call so we can still inspect malformed output.
+ let batch_res: Result<ConsolidatedBatch, _> =
+ complete_structured(&*provider, request.clone()).await;
+ let elapsed = start.elapsed();
+ match batch_res {
+ Ok(batch) => {
+ let raw = serde_json::to_value(&batch).ok();
+ tracing::info!(side, ms = elapsed.as_millis() as u64, "parsed ok");
+ ProviderResult {
+ elapsed,
+ batch: Some(batch),
+ raw,
+ error: None,
+ }
+ }
+ Err(e) => {
+ tracing::warn!(side, error = %e, "schema parse failed");
+ ProviderResult {
+ elapsed,
+ batch: None,
+ raw: None,
+ error: Some(e.to_string()),
+ }
+ }
+ }
+}
+
+fn persist_case(
+ side: &str,
+ fixture_name: &str,
+ fixture_path: &Path,
+ run_dir: &Path,
+ res: ProviderResult,
+) -> Result<CaseRun> {
+ let safe_name = fixture_path
+ .file_stem()
+ .and_then(|s| s.to_str())
+ .unwrap_or(fixture_name);
+ let dir = run_dir.join(side);
+ std::fs::create_dir_all(&dir)?;
+
+ // 1. Raw JSON.
+ let json_path = dir.join(format!("{safe_name}.json"));
+ if let Some(raw) = &res.raw {
+ std::fs::write(&json_path, serde_json::to_string_pretty(raw)?)?;
+ }
+
+ // 2. Flattened markdown rendering for easy eyeballing / diff.
+ let md_path = dir.join(format!("{safe_name}.md"));
+ let md = render_markdown(side, fixture_name, &res);
+ std::fs::write(&md_path, md)?;
+
+ // 3. Per-case meta.
+ let case = CaseRun {
+ provider: side.to_string(),
+ fixture: fixture_name.to_string(),
+ elapsed_ms: res.elapsed.as_millis(),
+ parsed_ok: res.batch.is_some(),
+ error: res.error.clone(),
+ update_count: res.batch.as_ref().map(|b| b.updates.len()),
+ };
+ let meta_path = dir.join(format!("{safe_name}.meta.json"));
+ std::fs::write(&meta_path, serde_json::to_string_pretty(&case)?)?;
+ Ok(case)
+}
+
+fn render_markdown(side: &str, fixture_name: &str, res: &ProviderResult) -> String {
+ let mut out = String::new();
+ out.push_str(&format!("# {side} — {fixture_name}\n\n"));
+ out.push_str(&format!("- elapsed: {} ms\n", res.elapsed.as_millis()));
+ if let Some(err) = &res.error {
+ out.push_str(&format!("- error: `{err}`\n\n"));
+ }
+ let Some(batch) = &res.batch else {
+ out.push_str("\n_(no batch — see error above; .json file absent)_\n");
+ return out;
+ };
+ out.push_str(&format!("- updates: {}\n\n", batch.updates.len()));
+ if !batch.rationale.trim().is_empty() {
+ out.push_str("## Rationale\n\n");
+ out.push_str(batch.rationale.trim());
+ out.push_str("\n\n");
+ }
+ for (i, u) in batch.updates.iter().enumerate() {
+ out.push_str(&format!("## Update {} — `{}`\n\n", i + 1, u.path));
+ out.push_str(&format!("**Title:** {} \n", u.title));
+ out.push_str(&format!("**Kind:** `{}` \n", u.kind.as_str()));
+ out.push_str(&format!("**Tier:** `{}` \n", u.tier));
+ if !u.tags.is_empty() {
+ out.push_str(&format!("**Tags:** {} \n", u.tags.join(", ")));
+ }
+ out.push('\n');
+ out.push_str(u.body_markdown.trim());
+ out.push_str("\n\n");
+ }
+ out
+}