Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
evals: live A/B harness comparing two LLM providers on the consolidation prompt
AkitaOnRailsdf7671dd56f4
diff --git a/.gitignore b/.gitignoreindex c58e87e3..53627586 100644--- a/.gitignore+++ b/.gitignore@@ -29,3 +29,4 @@ perf.data*# Sandbox marker (not project content)..ai-jail.claude/+evals/runs/diff --git a/Cargo.lock b/Cargo.lockindex 756eba27..80be0c91 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -99,6 +99,25 @@ dependencies = ["uuid",]+[[package]]+name = "ai-memory-eval"+version = "0.1.0"+dependencies = [+ "ai-memory-consolidate",+ "ai-memory-core",+ "ai-memory-llm",+ "anyhow",+ "clap",+ "jiff",+ "secrecy",+ "serde",+ "serde_json",+ "tokio",+ "tracing",+ "tracing-subscriber",+ "uuid",+]+[[package]]name = "ai-memory-hooks"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex 2e8d646f..cc12e413 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -9,6 +9,9 @@ members = ["crates/ai-memory-llm","crates/ai-memory-consolidate","crates/ai-memory-cli",+ # Live A/B harness — not part of the shipped binary, but in+ # the workspace so it shares deps + builds with the rest.+ "evals",][workspace.package]diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rsindex de2b2250..fb4910ce 100644--- a/crates/ai-memory-consolidate/src/consolidator.rs+++ b/crates/ai-memory-consolidate/src/consolidator.rs@@ -306,7 +306,11 @@ const fn tier_as_str(t: Tier) -> &'static str {}}-fn build_batch_request(session_id: SessionId, observations: &[Observation]) -> ChatRequest {+/// Build the exact ChatRequest the consolidator sends for batch+/// multi-page consolidation. Exposed so off-tree A/B harnesses+/// (e.g. `evals/`) can exercise the same workload against+/// alternative providers without duplicating the prompt.+pub fn build_batch_request(session_id: SessionId, observations: &[Observation]) -> ChatRequest {let mut buf = String::new();buf.push_str("You are compiling a Karpathy-style multi-page wiki update. Given the \@@ -351,7 +355,9 @@ fn build_batch_request(session_id: SessionId, observations: &[Observation]) -> C}}-const BATCH_SYSTEM_PROMPT: &str = "You are the maintainer of a Karpathy-style LLM wiki \+/// System prompt for batch consolidation. Public for the same+/// reason as [`build_batch_request`].+pub const BATCH_SYSTEM_PROMPT: &str = "You are the maintainer of a Karpathy-style LLM wiki \for a software engineer. Your job is to compile *durable* knowledge -- not just \restate the session log. Produce a ConsolidatedBatch with 1-5 page updates. Favour \extracting concept / decision / gotcha pages alongside the session summary when the \diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rsindex 0aeb9c45..82d050c0 100644--- a/crates/ai-memory-consolidate/src/lib.rs+++ b/crates/ai-memory-consolidate/src/lib.rs@@ -15,7 +15,9 @@ pub mod sweep;pub mod types;pub use bootstrap::{Bootstrap, BootstrapConfig, BootstrapError, BootstrapOutcome};-pub use consolidator::{Consolidator, ConsolidatorError, ConsolidatorResult};+pub use consolidator::{+ BATCH_SYSTEM_PROMPT, Consolidator, ConsolidatorError, ConsolidatorResult, build_batch_request,+};pub use lint::{LintError, LintFinding, LintReport, run_lint};pub use sweep::{EvictedPage, SweepError, SweepReport, run_sweep};pub use types::{diff --git a/evals/Cargo.toml b/evals/Cargo.tomlnew file mode 100644index 00000000..80af7b43--- /dev/null+++ b/evals/Cargo.toml@@ -0,0 +1,27 @@+[package]+name = "ai-memory-eval"+version = "0.1.0"+edition = "2024"+rust-version = "1.95"+license = "MIT OR Apache-2.0"+publish = false++[[bin]]+name = "ai-memory-eval"+path = "src/main.rs"++[dependencies]+ai-memory-core.workspace = true+ai-memory-llm.workspace = true+ai-memory-consolidate.workspace = true++anyhow.workspace = true+clap.workspace = true+jiff.workspace = true+secrecy.workspace = true+serde.workspace = true+serde_json.workspace = true+tokio.workspace = true+tracing.workspace = true+tracing-subscriber.workspace = true+uuid.workspace = truediff --git a/evals/README.md b/evals/README.mdnew file mode 100644index 00000000..643bf3cb--- /dev/null+++ b/evals/README.md@@ -0,0 +1,134 @@+# `evals/` — live A/B harness++A small Rust binary that runs the EXACT consolidation prompt+ai-memory uses in production against two LLM providers side by side,+and saves both outputs to disk for human comparison.++**This is not part of the shipped binary.** It's a workspace member+purely so it shares deps + builds with the rest. `cargo build` from+the repo root will compile it, but it's never bundled into the+docker image and never run by CI.++## When to reach for this++After switching providers, models, or major prompt edits. Concretely:++- Replacing OpenRouter/Kimi with local Ollama (the case that+ motivated this harness — see commit log).+- Trying a new Ollama model (`qwen3:32b` → `qwen3-coder:30b`).+- Tuning the `BATCH_SYSTEM_PROMPT` itself — does the rewrite+ preserve quality across providers?++The fixtures are deliberately small synthetic session logs that+exercise the prompt's hard cases (durable rule extraction,+multi-topic separation, "say nothing" sessions, decision/gotcha+distinction).++## What it does++For each `*.json` under `evals/fixtures/`:++1. Builds the request via+ [`ai_memory_consolidate::build_batch_request`] — same code path+ the live consolidator uses.+2. Sends it to a **baseline** and a **candidate** provider, *in+ parallel*.+3. Runs the result through+ [`ai_memory_llm::complete_structured`] — same JSON-schema+ validation the live system applies. Schema-parse failure is+ recorded (not fatal).+4. Persists to `evals/runs/<timestamp>/{baseline,candidate}/<fixture>.{json,md,meta.json}`.++The runner prints latency + parse status per fixture and a tail+summary. **Quality is for you to read** — open the markdown files+in `runs/<timestamp>/baseline/` and `…/candidate/` side by side and+judge faithfulness, scoping, hallucination, etc.++## Running it++### Ollama qwen3:32b vs OpenRouter Kimi (the canonical comparison)++```bash+# OPENROUTER_API_KEY in env; LLM_API_KEY can be any non-empty+# string for the candidate (Ollama doesn't validate).+export OPENROUTER_API_KEY="sk-or-v1-..."++cargo run -p ai-memory-eval -- \+ --baseline-provider openai-compat \+ --baseline-base-url https://openrouter.ai/api/v1 \+ --baseline-model moonshotai/kimi-k2.6 \+ --baseline-api-key-env OPENROUTER_API_KEY \+ --candidate-provider openai-compat \+ --candidate-base-url http://192.168.0.90:11434/v1 \+ --candidate-model qwen3:32b \+ --candidate-api-key ollama-local+```++### Two Ollama models against each other++```bash+cargo run -p ai-memory-eval -- \+ --baseline-provider openai-compat \+ --baseline-base-url http://192.168.0.90:11434/v1 \+ --baseline-model qwen3:32b \+ --baseline-api-key ollama-local \+ --candidate-provider openai-compat \+ --candidate-base-url http://192.168.0.90:11434/v1 \+ --candidate-model qwen3-coder:30b \+ --candidate-api-key ollama-local+```++### Reading the output++```+evals/runs/2026-05-22T18-30-00Z/+├── baseline/+│ ├── 01-rust-bug-fix.json ← raw structured output+│ ├── 01-rust-bug-fix.md ← flat markdown rendering, easy to read+│ └── 01-rust-bug-fix.meta.json ← {elapsed_ms, parsed_ok, update_count}+└── candidate/+ └── …+```++Eyeball the `.md` files in pairs. The runner also prints a+`diff -ru baseline candidate` command you can run.++## Adding fixtures++Each fixture is a JSON file:++```json+{+ "name": "human-readable",+ "description": "what this case is meant to surface",+ "observations": [+ {"kind": "session-start", "title": "...", "body": "..."},+ {"kind": "user-prompt", "title": "user prompt", "body": "..."},+ {"kind": "pre-tool-use", "title": "Edit", "body": "..."},+ ...+ ]+}+```++`kind` values: `session-start`, `user-prompt`, `pre-tool-use`,+`post-tool-use`, `pre-compact`, `notification`, `stop`,+`session-end`, `other` (see `ObservationKind` in `ai-memory-core`).++The `description` field isn't read by the runner — it's a comment+for the next human who opens the file.++## What this harness does NOT do++- **Score quality automatically.** Pure side-by-side. If you want+ metrics, the next layer up would be keyword recall (`must_mention`+ per fixture) or an LLM-as-judge pass — both deliberately out of+ scope here.+- **Test the embedding pipeline.** This only exercises the+ consolidation LLM. Embedding A/B would be a parallel harness+ (probe queries + expected target pages, measure recall@5/MRR).+- **Persist via the real wiki layer.** No SQLite, no markdown+ writes, no git. Pure prompt → response.++## Cleanup++`evals/runs/` is in `.gitignore`. Drop it whenever it gets large.diff --git a/evals/fixtures/01-rust-bug-fix.json b/evals/fixtures/01-rust-bug-fix.jsonnew file mode 100644index 00000000..3fd9487a--- /dev/null+++ b/evals/fixtures/01-rust-bug-fix.json@@ -0,0 +1,17 @@+{+ "name": "rust-bug-fix",+ "description": "Realistic short session: user fixes a Rust async bug, decides on a pattern, hits one gotcha. Tests whether the model extracts both the decision and the gotcha as separate pages, not just a session log.",+ "observations": [+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory, agent=claude-code"},+ {"kind": "user-prompt", "title": "user prompt", "body": "the writer actor is panicking when the bounded mpsc channel fills up — fix it without unbounding the channel"},+ {"kind": "pre-tool-use", "title": "Read", "body": "crates/ai-memory-store/src/writer.rs"},+ {"kind": "post-tool-use", "title": "Read result", "body": "found send() being called from an async context — current code uses try_send and unwraps the Err on full"},+ {"kind": "user-prompt", "title": "user prompt", "body": "yes, replace the try_send unwrap with awaiting send and let backpressure propagate; do not silently drop writes"},+ {"kind": "pre-tool-use", "title": "Edit", "body": "writer.rs: changed try_send().unwrap() to send().await?"},+ {"kind": "post-tool-use", "title": "cargo test", "body": "1 test failed: writer_panics_on_full — needs updating since panic behavior changed; replaced with assertion on backpressure delay"},+ {"kind": "pre-tool-use", "title": "Edit", "body": "writer.rs tests: replaced the panic-expectation test with a timing-based backpressure test"},+ {"kind": "user-prompt", "title": "user prompt", "body": "also note that I want fire-and-forget hook ingress to remain bounded; never block the HTTP handler on the writer"},+ {"kind": "post-tool-use", "title": "cargo test", "body": "all green"},+ {"kind": "session-end", "title": "session end", "body": "fixed; committed as ec1a4f0"}+ ]+}diff --git a/evals/fixtures/02-architecture-decision.json b/evals/fixtures/02-architecture-decision.jsonnew file mode 100644index 00000000..f66a78d0--- /dev/null+++ b/evals/fixtures/02-architecture-decision.json@@ -0,0 +1,15 @@+{+ "name": "architecture-decision",+ "description": "User makes a deliberate architectural choice (single-writer SQLite vs. write-pool). Tests whether the model produces a decisions/ ADR-style page distinct from the session log.",+ "observations": [+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory"},+ {"kind": "user-prompt", "title": "user prompt", "body": "I'm seeing 'database is locked' under concurrent writes in the integration tests. should I move to a connection pool with retry, or a single-writer actor?"},+ {"kind": "user-prompt", "title": "user prompt", "body": "actually let me think out loud — a pool means rusqlite serialises writes anyway via BEGIN IMMEDIATE retries, and we'd still see SQLITE_BUSY under contention. With a single-writer actor we get FIFO ordering for free, no BUSY, and an obvious place to add audit log."},+ {"kind": "user-prompt", "title": "user prompt", "body": "going with single-writer actor; pool the readers separately."},+ {"kind": "pre-tool-use", "title": "Write", "body": "crates/ai-memory-store/src/writer.rs: new WriterActor owning one Connection, accepts WriteCmd via tokio::sync::mpsc"},+ {"kind": "pre-tool-use", "title": "Write", "body": "crates/ai-memory-store/src/reader.rs: ReaderPool of 4 read-only Connections"},+ {"kind": "post-tool-use", "title": "cargo test", "body": "concurrent_writes test green; no SQLITE_BUSY under 100 parallel writers"},+ {"kind": "user-prompt", "title": "user prompt", "body": "the audit log column was the clincher — we couldn't do it cleanly with a pool"},+ {"kind": "session-end", "title": "session end", "body": "architecture changed; ADR worth recording"}+ ]+}diff --git a/evals/fixtures/03-gotcha-with-rule.json b/evals/fixtures/03-gotcha-with-rule.jsonnew file mode 100644index 00000000..f80302f2--- /dev/null+++ b/evals/fixtures/03-gotcha-with-rule.json@@ -0,0 +1,14 @@+{+ "name": "gotcha-with-rule",+ "description": "Session that produces both a gotcha (test ordering) AND a durable rule ('never mock the DB'). Tests whether the model classifies the rule with kind=rule and writes it as a standalone instruction (not just a note).",+ "observations": [+ {"kind": "user-prompt", "title": "user prompt", "body": "the test suite passes locally but fails in CI — different ordering?"},+ {"kind": "pre-tool-use", "title": "cargo test --workspace", "body": "test results in CI: integration test 'multi_writer_serialisation' fails — relies on the writer being a singleton across tests, but parallel runners start two"},+ {"kind": "user-prompt", "title": "user prompt", "body": "ah, parallel test runners each spin up their own ai-memory instance; the global locking assumption doesn't hold cross-process"},+ {"kind": "pre-tool-use", "title": "Edit", "body": "tests now use #[serial_test::serial] and a per-test temp data dir"},+ {"kind": "post-tool-use", "title": "cargo test --workspace", "body": "all green, including CI"},+ {"kind": "user-prompt", "title": "user prompt", "body": "ok, two things to remember: (1) integration tests against the store must NEVER mock the database — last quarter we ate a 3-day mock/prod divergence bug; (2) any test touching the writer needs the per-test temp data dir helper, or document why not"},+ {"kind": "user-prompt", "title": "user prompt", "body": "the 'never mock the database' is a hard project rule. write it down somewhere durable."},+ {"kind": "session-end", "title": "session end", "body": "two distinct lessons: per-test data dir, never mock DB"}+ ]+}diff --git a/evals/fixtures/04-low-signal-session.json b/evals/fixtures/04-low-signal-session.jsonnew file mode 100644index 00000000..78c36772--- /dev/null+++ b/evals/fixtures/04-low-signal-session.json@@ -0,0 +1,13 @@+{+ "name": "low-signal-session",+ "description": "A short, mundane session with no durable lessons — just minor edits. Tests whether the model RESISTS the urge to manufacture concept/decision pages and just returns the session log. (The system prompt explicitly says to favour returning only the session page when there's no reusable insight.)",+ "observations": [+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory"},+ {"kind": "user-prompt", "title": "user prompt", "body": "fix the typo in README — should be 'Karpathy' not 'Karparthy'"},+ {"kind": "pre-tool-use", "title": "Edit", "body": "README.md: Karparthy → Karpathy"},+ {"kind": "post-tool-use", "title": "git diff", "body": "1 line changed"},+ {"kind": "user-prompt", "title": "user prompt", "body": "commit it"},+ {"kind": "pre-tool-use", "title": "git commit", "body": "docs: fix Karpathy spelling"},+ {"kind": "session-end", "title": "session end", "body": "trivial"}+ ]+}diff --git a/evals/fixtures/05-multi-topic-session.json b/evals/fixtures/05-multi-topic-session.jsonnew file mode 100644index 00000000..7ff4c0dc--- /dev/null+++ b/evals/fixtures/05-multi-topic-session.json@@ -0,0 +1,16 @@+{+ "name": "multi-topic-session",+ "description": "Mixed session covering two unrelated topics. Tests whether the model produces SEPARATE pages per topic rather than mashing them together. The two topics share no semantic overlap; if the model emits one mixed concept page it has failed at scoping.",+ "observations": [+ {"kind": "session-start", "title": "session start", "body": "cwd=/mnt/data/Projects/ai-memory"},+ {"kind": "user-prompt", "title": "user prompt", "body": "the docker image is 380 MB which is way too big — can we strip debug symbols and use a distroless base?"},+ {"kind": "pre-tool-use", "title": "Edit", "body": "Dockerfile: stage 1 rust:slim, stage 2 gcr.io/distroless/cc-debian12, strip symbols via CARGO_PROFILE_RELEASE_STRIP=symbols"},+ {"kind": "post-tool-use", "title": "docker build", "body": "image dropped from 380 MB to 67 MB"},+ {"kind": "user-prompt", "title": "user prompt", "body": "great. unrelated — I noticed `cargo audit` is failing in CI on a transitive dep `xml-rs` with RUSTSEC-2024-xxxx. can we work around?"},+ {"kind": "pre-tool-use", "title": "Read", "body": "Cargo.lock — xml-rs is pulled in by git2 → libgit2-sys → vendored-libgit2 transitively"},+ {"kind": "user-prompt", "title": "user prompt", "body": "git2 is load-bearing for the wiki versioning so we can't drop it. add an ignore entry in audit.toml with a TODO+date to revisit"},+ {"kind": "pre-tool-use", "title": "Write", "body": "audit.toml: ignore RUSTSEC-2024-xxxx with note 'transitive via libgit2-sys; revisit 2026-08'"},+ {"kind": "post-tool-use", "title": "cargo audit", "body": "passes"},+ {"kind": "session-end", "title": "session end", "body": "two distinct deliverables: docker slim + audit workaround"}+ ]+}diff --git a/evals/src/main.rs b/evals/src/main.rsnew file mode 100644index 00000000..c618772f--- /dev/null+++ b/evals/src/main.rs@@ -0,0 +1,482 @@+//! Live A/B harness for the ai-memory consolidation prompt.+//!+//! ## What it does+//!+//! For each fixture under `evals/fixtures/*.json` (each one a small,+//! synthetic session log), the runner:+//!+//! 1. Calls [`ai_memory_consolidate::build_batch_request`] to build+//! the EXACT ChatRequest the production consolidator would send.+//! 2. Sends that request to two providers concurrently — a+//! *baseline* and a *candidate* — via+//! [`ai_memory_llm::complete_structured`], which is the same+//! parse-extract-and-validate path the live system uses.+//! 3. Saves the deserialised `ConsolidatedBatch`, the raw JSON, a+//! flattened markdown rendering, and a `meta.json` (timing,+//! update count, parse status) per fixture per provider under+//! `evals/runs/<timestamp>/{baseline,candidate}/`.+//! 4. Prints a side-by-side summary table to stdout.+//!+//! Quality judgement is left to the human reading the outputs — the+//! harness only reports the OBJECTIVE deltas (latency, update+//! counts, schema-parse success). Anything subtler (faithfulness,+//! hallucination, structure) is for you to eyeball.+//!+//! ## Running it+//!+//! See `evals/README.md` for canonical invocations. The short+//! version:+//!+//! ```bash+//! cargo run -p ai-memory-eval -- \+//! --baseline-provider openai-compat \+//! --baseline-base-url https://openrouter.ai/api/v1 \+//! --baseline-model moonshotai/kimi-k2.6 \+//! --baseline-api-key-env OPENROUTER_API_KEY \+//! --candidate-provider openai-compat \+//! --candidate-base-url http://192.168.0.90:11434/v1 \+//! --candidate-model qwen3:32b \+//! --candidate-api-key ollama-local+//! ```++use std::path::{Path, PathBuf};+use std::sync::Arc;+use std::time::{Duration, Instant};++use anyhow::{Context, Result, bail};+use clap::Parser;+use jiff::Timestamp;+use secrecy::SecretString;+use serde::{Deserialize, Serialize};++use ai_memory_consolidate::{ConsolidatedBatch, build_batch_request};+use ai_memory_core::{Observation, ObservationKind, ProjectId, SessionId, WorkspaceId};+use ai_memory_llm::{+ LlmProvider, ProviderChoice, ProviderConfig, complete_structured,+};++#[derive(Parser, Debug)]+#[command(+ name = "ai-memory-eval",+ about = "Live A/B harness comparing two LLM providers on the ai-memory consolidation prompt"+)]+struct Args {+ /// Fixture directory (each `*.json` is one synthetic session log).+ #[arg(long, default_value = "evals/fixtures")]+ fixtures: PathBuf,++ /// Output root. A timestamped subdir gets created here per run.+ #[arg(long, default_value = "evals/runs")]+ out: PathBuf,++ /// Baseline provider (`anthropic` / `openai` / `openai-compat`).+ #[arg(long)]+ baseline_provider: String,++ /// Baseline base URL (omit for native Anthropic/OpenAI).+ #[arg(long)]+ baseline_base_url: Option<String>,++ /// Baseline model id.+ #[arg(long)]+ baseline_model: String,++ /// Baseline API key (raw value). Prefer `--baseline-api-key-env`.+ #[arg(long)]+ baseline_api_key: Option<String>,++ /// Env var to read the baseline API key from.+ #[arg(long)]+ baseline_api_key_env: Option<String>,++ /// Candidate provider.+ #[arg(long)]+ candidate_provider: String,++ /// Candidate base URL.+ #[arg(long)]+ candidate_base_url: Option<String>,++ /// Candidate model id.+ #[arg(long)]+ candidate_model: String,++ /// Candidate API key (raw value).+ #[arg(long)]+ candidate_api_key: Option<String>,++ /// Env var to read the candidate API key from.+ #[arg(long)]+ candidate_api_key_env: Option<String>,+}++/// JSON shape of an eval fixture file. We only care about+/// observations the consolidation prompt actually consumes; the+/// other Observation fields get populated with dummies at runtime.+#[derive(Debug, Deserialize)]+struct Fixture {+ name: String,+ /// Free-text describing the scenario; not consumed by the runner+ /// but useful when reading fixture files by hand.+ #[allow(dead_code)]+ #[serde(default)]+ description: String,+ observations: Vec<FixtureObservation>,+}++#[derive(Debug, Deserialize)]+struct FixtureObservation {+ kind: String,+ title: String,+ #[serde(default)]+ body: String,+}++/// One result of one provider on one fixture.+#[derive(Debug, Serialize)]+struct CaseRun {+ provider: String,+ fixture: String,+ elapsed_ms: u128,+ parsed_ok: bool,+ error: Option<String>,+ update_count: Option<usize>,+}++#[tokio::main]+async fn main() -> Result<()> {+ tracing_subscriber::fmt()+ .with_env_filter(+ tracing_subscriber::EnvFilter::try_from_default_env()+ .unwrap_or_else(|_| "ai_memory_eval=info,warn".into()),+ )+ .init();++ let args = Args::parse();++ let baseline = resolve_provider("baseline", &args)?;+ let candidate = resolve_provider("candidate", &args)?;++ let baseline_provider = make_provider(baseline.clone())+ .context("building baseline provider")?;+ let candidate_provider = make_provider(candidate.clone())+ .context("building candidate provider")?;++ let fixtures = load_fixtures(&args.fixtures)+ .with_context(|| format!("loading fixtures from {}", args.fixtures.display()))?;+ if fixtures.is_empty() {+ bail!("no *.json fixtures found in {}", args.fixtures.display());+ }++ // Timestamped run dir; filesystem-safe (colons + dots → dashes).+ let stamp: String = Timestamp::now()+ .to_string()+ .chars()+ .map(|c| if c == ':' || c == '.' { '-' } else { c })+ .collect();+ let run_dir = args.out.join(&stamp);+ std::fs::create_dir_all(run_dir.join("baseline"))?;+ std::fs::create_dir_all(run_dir.join("candidate"))?;++ println!();+ println!("ai-memory eval — {stamp}");+ println!(" fixtures : {} ({})", fixtures.len(), args.fixtures.display());+ println!(" baseline : {} {} @ {}", baseline.provider_name(), baseline.model, baseline.base_url.as_deref().unwrap_or("(default)"));+ println!(" candidate: {} {} @ {}", candidate.provider_name(), candidate.model, candidate.base_url.as_deref().unwrap_or("(default)"));+ println!(" out : {}", run_dir.display());+ println!();++ let mut results: Vec<(CaseRun, CaseRun)> = Vec::with_capacity(fixtures.len());+ for (path, fx) in fixtures {+ println!("→ {} ({} observations)", fx.name, fx.observations.len());+ let observations = synthesise_observations(&fx);+ let session_id = SessionId::new();++ // Run both providers concurrently per fixture so a slow one+ // doesn't double the wall-clock.+ let req_a = build_batch_request(session_id, &observations);+ let req_b = build_batch_request(session_id, &observations);+ let (a_res, b_res) = tokio::join!(+ run_one(baseline_provider.clone(), "baseline", req_a),+ run_one(candidate_provider.clone(), "candidate", req_b),+ );++ let a = persist_case("baseline", &fx.name, &path, &run_dir, a_res)?;+ let b = persist_case("candidate", &fx.name, &path, &run_dir, b_res)?;++ println!(+ " baseline : {:>5} ms parsed={} updates={}",+ a.elapsed_ms,+ a.parsed_ok,+ a.update_count.map_or("-".into(), |n| n.to_string())+ );+ println!(+ " candidate: {:>5} ms parsed={} updates={}",+ b.elapsed_ms,+ b.parsed_ok,+ b.update_count.map_or("-".into(), |n| n.to_string())+ );+ results.push((a, b));+ }++ // Tail summary so the user can grok the run at a glance.+ println!();+ println!("=== summary ===");+ let n = results.len();+ let baseline_total: u128 = results.iter().map(|(a, _)| a.elapsed_ms).sum();+ let candidate_total: u128 = results.iter().map(|(_, b)| b.elapsed_ms).sum();+ let baseline_ok = results.iter().filter(|(a, _)| a.parsed_ok).count();+ let candidate_ok = results.iter().filter(|(_, b)| b.parsed_ok).count();+ println!("baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms", baseline_total / n.max(1) as u128);+ println!("candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms", candidate_total / n.max(1) as u128);+ println!();+ println!("outputs under: {}", run_dir.display());+ println!("compare with: diff -ru {}/baseline {}/candidate", run_dir.display(), run_dir.display());+ Ok(())+}++/// Resolve `ProviderConfig` for one side from CLI args + env.+fn resolve_provider(side: &str, args: &Args) -> Result<ResolvedConfig> {+ let (provider_str, base_url, model, api_key_inline, api_key_env) = match side {+ "baseline" => (+ args.baseline_provider.as_str(),+ args.baseline_base_url.clone(),+ args.baseline_model.clone(),+ args.baseline_api_key.clone(),+ args.baseline_api_key_env.clone(),+ ),+ "candidate" => (+ args.candidate_provider.as_str(),+ args.candidate_base_url.clone(),+ args.candidate_model.clone(),+ args.candidate_api_key.clone(),+ args.candidate_api_key_env.clone(),+ ),+ _ => bail!("unknown side {side}"),+ };++ let provider = match provider_str {+ "anthropic" => ProviderChoice::Anthropic,+ "openai" => ProviderChoice::OpenAi,+ "openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,+ other => bail!(+ "{side}: provider {other} not one of anthropic|openai|openai-compat"+ ),+ };++ let api_key = match (api_key_inline, api_key_env) {+ (Some(s), _) if !s.is_empty() => Some(SecretString::from(s)),+ (_, Some(env)) => match std::env::var(&env) {+ Ok(v) if !v.is_empty() => Some(SecretString::from(v)),+ _ => bail!("{side}: env var {env} is not set or empty"),+ },+ _ => None,+ };++ Ok(ResolvedConfig {+ provider,+ provider_str: provider_str.to_string(),+ model,+ base_url,+ api_key,+ })+}++struct ResolvedConfig {+ provider: ProviderChoice,+ provider_str: String,+ model: String,+ base_url: Option<String>,+ api_key: Option<SecretString>,+}++impl ResolvedConfig {+ fn provider_name(&self) -> &str {+ self.provider_str.as_str()+ }+}++impl Clone for ResolvedConfig {+ fn clone(&self) -> Self {+ Self {+ provider: self.provider,+ provider_str: self.provider_str.clone(),+ model: self.model.clone(),+ base_url: self.base_url.clone(),+ // SecretString is not Clone — re-wrap if needed.+ api_key: self.api_key.as_ref().map(|s| {+ use secrecy::ExposeSecret;+ SecretString::from(s.expose_secret().to_string())+ }),+ }+ }+}++impl From<ResolvedConfig> for ProviderConfig {+ fn from(r: ResolvedConfig) -> Self {+ Self {+ provider: r.provider,+ model: r.model,+ api_key: r.api_key,+ base_url: r.base_url,+ }+ }+}++fn make_provider(r: ResolvedConfig) -> Result<Arc<dyn LlmProvider>> {+ ai_memory_llm::build_provider(ProviderConfig::from(r))+ .map_err(anyhow::Error::from)+ .context("building provider")+}++fn load_fixtures(dir: &Path) -> Result<Vec<(PathBuf, Fixture)>> {+ let mut paths: Vec<_> = std::fs::read_dir(dir)?+ .filter_map(|e| e.ok().map(|e| e.path()))+ .filter(|p| p.extension().is_some_and(|e| e == "json"))+ .collect();+ paths.sort();+ let mut out = Vec::with_capacity(paths.len());+ for p in paths {+ let bytes = std::fs::read(&p)+ .with_context(|| format!("reading {}", p.display()))?;+ let fx: Fixture = serde_json::from_slice(&bytes)+ .with_context(|| format!("parsing {}", p.display()))?;+ out.push((p, fx));+ }+ Ok(out)+}++fn synthesise_observations(fx: &Fixture) -> Vec<Observation> {+ let workspace_id = WorkspaceId::new();+ let project_id = ProjectId::new();+ let session_id = SessionId::new();+ fx.observations+ .iter()+ .map(|f| Observation {+ id: ai_memory_core::ObservationId::new(),+ session_id,+ workspace_id,+ project_id,+ kind: f+ .kind+ .parse::<ObservationKind>()+ .unwrap_or(ObservationKind::Other),+ title: f.title.clone(),+ body: f.body.clone(),+ importance: 5,+ created_at: Timestamp::now(),+ })+ .collect()+}++struct ProviderResult {+ elapsed: Duration,+ batch: Option<ConsolidatedBatch>,+ raw: Option<serde_json::Value>,+ error: Option<String>,+}++async fn run_one(+ provider: Arc<dyn LlmProvider>,+ side: &str,+ request: ai_memory_llm::ChatRequest,+) -> ProviderResult {+ let start = Instant::now();+ // Try the schema-validated path first; fall back to the raw+ // structured call so we can still inspect malformed output.+ let batch_res: Result<ConsolidatedBatch, _> =+ complete_structured(&*provider, request.clone()).await;+ let elapsed = start.elapsed();+ match batch_res {+ Ok(batch) => {+ let raw = serde_json::to_value(&batch).ok();+ tracing::info!(side, ms = elapsed.as_millis() as u64, "parsed ok");+ ProviderResult {+ elapsed,+ batch: Some(batch),+ raw,+ error: None,+ }+ }+ Err(e) => {+ tracing::warn!(side, error = %e, "schema parse failed");+ ProviderResult {+ elapsed,+ batch: None,+ raw: None,+ error: Some(e.to_string()),+ }+ }+ }+}++fn persist_case(+ side: &str,+ fixture_name: &str,+ fixture_path: &Path,+ run_dir: &Path,+ res: ProviderResult,+) -> Result<CaseRun> {+ let safe_name = fixture_path+ .file_stem()+ .and_then(|s| s.to_str())+ .unwrap_or(fixture_name);+ let dir = run_dir.join(side);+ std::fs::create_dir_all(&dir)?;++ // 1. Raw JSON.+ let json_path = dir.join(format!("{safe_name}.json"));+ if let Some(raw) = &res.raw {+ std::fs::write(&json_path, serde_json::to_string_pretty(raw)?)?;+ }++ // 2. Flattened markdown rendering for easy eyeballing / diff.+ let md_path = dir.join(format!("{safe_name}.md"));+ let md = render_markdown(side, fixture_name, &res);+ std::fs::write(&md_path, md)?;++ // 3. Per-case meta.+ let case = CaseRun {+ provider: side.to_string(),+ fixture: fixture_name.to_string(),+ elapsed_ms: res.elapsed.as_millis(),+ parsed_ok: res.batch.is_some(),+ error: res.error.clone(),+ update_count: res.batch.as_ref().map(|b| b.updates.len()),+ };+ let meta_path = dir.join(format!("{safe_name}.meta.json"));+ std::fs::write(&meta_path, serde_json::to_string_pretty(&case)?)?;+ Ok(case)+}++fn render_markdown(side: &str, fixture_name: &str, res: &ProviderResult) -> String {+ let mut out = String::new();+ out.push_str(&format!("# {side} — {fixture_name}\n\n"));+ out.push_str(&format!("- elapsed: {} ms\n", res.elapsed.as_millis()));+ if let Some(err) = &res.error {+ out.push_str(&format!("- error: `{err}`\n\n"));+ }+ let Some(batch) = &res.batch else {+ out.push_str("\n_(no batch — see error above; .json file absent)_\n");+ return out;+ };+ out.push_str(&format!("- updates: {}\n\n", batch.updates.len()));+ if !batch.rationale.trim().is_empty() {+ out.push_str("## Rationale\n\n");+ out.push_str(batch.rationale.trim());+ out.push_str("\n\n");+ }+ for (i, u) in batch.updates.iter().enumerate() {+ out.push_str(&format!("## Update {} — `{}`\n\n", i + 1, u.path));+ out.push_str(&format!("**Title:** {} \n", u.title));+ out.push_str(&format!("**Kind:** `{}` \n", u.kind.as_str()));+ out.push_str(&format!("**Tier:** `{}` \n", u.tier));+ if !u.tags.is_empty() {+ out.push_str(&format!("**Tags:** {} \n", u.tags.join(", ")));+ }+ out.push('\n');+ out.push_str(u.body_markdown.trim());+ out.push_str("\n\n");+ }+ out+}