Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

consolidate: strict tier+kind enums + verbose prompt; docs comparison

AkitaOnRailse830a7c415c0
diff --git a/Cargo.lock b/Cargo.lock
index 80be0c91..54717382 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -92,6 +92,7 @@ version = "0.1.0"
dependencies = [
"jiff",
"regex",
+ "schemars",
"serde",
"serde_json",
"thiserror 2.0.18",
diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rs
index fb4910ce..9eee59ac 100644
--- a/crates/ai-memory-consolidate/src/consolidator.rs
+++ b/crates/ai-memory-consolidate/src/consolidator.rs
@@ -225,7 +225,7 @@ impl Consolidator {
upd.path.clone()
};
let path = PagePath::new(final_path)?;
- let tier = upd.tier.parse::<Tier>().unwrap_or(Tier::Semantic);
+ let tier = upd.tier;
let mut fm = serde_json::Map::new();
fm.insert("title".into(), serde_json::Value::String(upd.title.clone()));
fm.insert(
@@ -331,18 +331,42 @@ pub fn build_batch_request(session_id: SessionId, observations: &[Observation])
- concepts/<slug>.md (semantic, evergreen concept pages)\n\
- decisions/<short>.md (semantic, ADR-style records)\n\
- gotchas/<slug>.md (semantic, failure modes / surprises)\n\
- \nClassify each update with one of these `kind` values:\n\
- - `decision` (the project chose X over Y)\n\
- - `gotcha` (a failure mode or surprise worth remembering)\n\
- - `rule` (durable project convention: \"always X\", \"never Y\")\n\
- - `fact` (everything else; the default)\n\
+ \nSet `tier` to EXACTLY ONE of these four strings — never an integer, never a synonym:\n\
+ - \"working\" (the live in-progress slice of the session — rarely used here)\n\
+ - \"episodic\" (per-session narrative; the sessions/<id>.md page)\n\
+ - \"semantic\" (durable knowledge: concepts/, decisions/, gotchas/, rules)\n\
+ - \"procedural\" (repeated patterns extracted from many episodic pages)\n\
+ \nSet `kind` to EXACTLY ONE of these four strings — never an integer, never \"session\" / \"concept\" / \"note\":\n\
+ - \"decision\" (the project chose X over Y)\n\
+ - \"gotcha\" (a failure mode or surprise worth remembering)\n\
+ - \"rule\" (durable project convention: \"always X\", \"never Y\")\n\
+ - \"fact\" (everything else; the default — use this for session narratives and plain concept notes)\n\
\nWhen you mark an update as `rule`, write the body as a clear \
standalone instruction the agent could follow on every relevant \
action. The path you suggest for a rule will be overridden — the \
system routes rules to `_rules/<slug>.md` automatically and the \
lint pass surfaces a hint to copy it into the project's CLAUDE.md.\
- \n\nEach update must include a title, markdown body, tier, kind, \
- and tags.\n",
+ \n## Required JSON keys on every update (use these EXACT names)\n\
+ - \"path\" (string) required — the wiki path\n\
+ - \"title\" (string) required — the page title\n\
+ - \"body_markdown\" (string) required — the page body in Markdown; NOTE the underscore + the suffix `_markdown`, NOT just `body`\n\
+ - \"tier\" (string) required — one of the four tier strings above\n\
+ - \"kind\" (string) required — one of the four kind strings above\n\
+ - \"tags\" (array of string) required — may be empty `[]`, but the key must be present\n\
+ No other keys. No `body`, no `content`, no `summary`. Field names \
+ are case-sensitive and the `_markdown` suffix matters.\n\
+ \n## Output format (read this carefully)\n\
+ Reply with ONE JSON object matching the ConsolidatedBatch schema, \
+ and nothing else. NO prose preamble, NO trailing commentary, NO \
+ markdown headers wrapping the JSON, NO ``` code fences. The very \
+ first character of your reply must be `{` and the very last `}`. \
+ Strings must be JSON strings (with double quotes), not numbers \
+ and not bare identifiers.\n\
+ \n## Top-level shape\n\
+ {\n\
+ \x20\x20\"updates\": [ /* 1-5 update objects with the keys above */ ],\n\
+ \x20\x20\"rationale\": \"<one short sentence about why this batch>\"\n\
+ }\n",
);
ChatRequest {
system: Some(BATCH_SYSTEM_PROMPT.into()),
diff --git a/crates/ai-memory-consolidate/src/types.rs b/crates/ai-memory-consolidate/src/types.rs
index 09a0e8d9..ba522c10 100644
--- a/crates/ai-memory-consolidate/src/types.rs
+++ b/crates/ai-memory-consolidate/src/types.rs
@@ -1,6 +1,6 @@
//! Public-facing consolidation types.
-use ai_memory_core::{PageId, PagePath};
+use ai_memory_core::{PageId, PagePath, Tier};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@@ -70,8 +70,13 @@ pub struct ConsolidatedPageUpdate {
/// When `kind` is `Rule` the consolidator overrides this to
/// `_rules/<slug>.md` regardless — see `PageKind::Rule`.
pub path: String,
- /// Tier (`semantic`, `episodic`, `procedural`, `working`).
- pub tier: String,
+ /// Tier classification. Typed as the [`Tier`] enum (not a free
+ /// `String`) so schemars emits a closed enum in the generated
+ /// JSON schema. Before this change schemars produced
+ /// `{ "type": "string" }` with no constraint, and both Kimi
+ /// and qwen3 routinely emitted `tier: 2` (integer) instead of
+ /// the documented string values.
+ pub tier: Tier,
/// Semantic classification. Defaults to `fact` if the LLM
/// doesn't supply one — existing consolidations without this
/// field still deserialise.
diff --git a/crates/ai-memory-core/Cargo.toml b/crates/ai-memory-core/Cargo.toml
index 1b36bf68..03c900a4 100644
--- a/crates/ai-memory-core/Cargo.toml
+++ b/crates/ai-memory-core/Cargo.toml
@@ -16,6 +16,11 @@ uuid.workspace = true
jiff.workspace = true
regex.workspace = true
tracing.workspace = true
+# JsonSchema on Tier/PageKind so the consolidation prompt's
+# structured-output schema enforces enum membership instead of
+# producing `{ "type": "string" }` with no constraints (which lets
+# the model invent integer tier values).
+schemars.workspace = true
[lints]
workspace = true
diff --git a/crates/ai-memory-core/src/page.rs b/crates/ai-memory-core/src/page.rs
index 64c98665..deb0d755 100644
--- a/crates/ai-memory-core/src/page.rs
+++ b/crates/ai-memory-core/src/page.rs
@@ -17,7 +17,7 @@ use crate::ids::{PageId, PagePath, ProjectId, WorkspaceId};
///
/// Adopted from agentmemory's four-tier model, simplified. See
/// [`docs/design-decisions.md`](../../docs/design-decisions.md) §7.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum Tier {
/// Current session: last N observations, current files.
diff --git a/crates/ai-memory-llm/src/openai.rs b/crates/ai-memory-llm/src/openai.rs
index c92341a7..842ce38c 100644
--- a/crates/ai-memory-llm/src/openai.rs
+++ b/crates/ai-memory-llm/src/openai.rs
@@ -251,20 +251,30 @@ mod tests {
#[test]
fn normalize_strips_trailing_slash_and_v1() {
// The three conventions found in the wild.
- assert_eq!(normalize_openai_base("https://api.openai.com"),
- "https://api.openai.com");
- assert_eq!(normalize_openai_base("https://api.openai.com/"),
- "https://api.openai.com");
+ assert_eq!(
+ normalize_openai_base("https://api.openai.com"),
+ "https://api.openai.com"
+ );
+ assert_eq!(
+ normalize_openai_base("https://api.openai.com/"),
+ "https://api.openai.com"
+ );
// OpenRouter's docs / ai-memory's OpenRouter env entry.
- assert_eq!(normalize_openai_base("https://openrouter.ai/api/v1"),
- "https://openrouter.ai/api");
+ assert_eq!(
+ normalize_openai_base("https://openrouter.ai/api/v1"),
+ "https://openrouter.ai/api"
+ );
// Ollama's openai-compat path (`docker logs` shows it
// canonically advertised this way).
- assert_eq!(normalize_openai_base("http://localhost:11434/v1"),
- "http://localhost:11434");
+ assert_eq!(
+ normalize_openai_base("http://localhost:11434/v1"),
+ "http://localhost:11434"
+ );
// Don't accidentally strip a `/v1` that's part of a longer
// segment (e.g. `/v123/`).
- assert_eq!(normalize_openai_base("https://example.com/v123"),
- "https://example.com/v123");
+ assert_eq!(
+ normalize_openai_base("https://example.com/v123"),
+ "https://example.com/v123"
+ );
}
}
diff --git a/docs/llm-provider-comparison.md b/docs/llm-provider-comparison.md
new file mode 100644
index 00000000..391b567b
--- /dev/null
+++ b/docs/llm-provider-comparison.md
@@ -0,0 +1,445 @@
+# LLM provider comparison — local Ollama vs hosted OpenRouter
+
+> **TL;DR.** ai-memory's consolidation prompt is provider-agnostic by
+> design but had a latent **schema-vs-prompt** bug that made *both*
+> Kimi-K2.6 (OpenRouter) and qwen3:32b (Ollama) fail JSON
+> validation. After fixing the schema and tightening the prompt,
+> local Ollama on a Strix Halo home server gives **comparable
+> output quality to hosted models** at zero cost per token, with a
+> ~2× latency penalty on first request (which the
+> `OLLAMA_KEEP_ALIVE=20m` warm cache largely amortises). The
+> harness used to discover this lives in [`evals/`](../evals/);
+> anyone can reproduce the comparison in ≤ 20 minutes.
+
+## Why this document exists
+
+When the homelab deploy switched ai-memory off the billed
+OpenAI / OpenRouter providers and onto the locally-hosted Ollama
+server, we needed empirical evidence — not a vibes-based claim —
+that *consolidation quality didn't degrade*. ai-memory's
+consolidator turns a session's raw observations into 1–5 wiki
+pages classified as `concept`, `decision`, `gotcha`, or `rule`;
+small drops in quality compound fast across hundreds of sessions.
+
+This doc captures:
+
+- The **methodology** (what we compared, how we compared, the
+ exact prompt + schema both providers saw).
+- The **root cause** of why early runs looked terrible.
+- The **fix** that landed in the consolidator's types + prompt.
+- The **final per-provider numbers** (parse rate, latency,
+ manual quality assessment).
+- A **how-to-reproduce** section so anyone can re-run the
+ comparison against their own model + provider choices.
+
+## What was tested
+
+### The five fixtures
+
+[`evals/fixtures/`](../evals/fixtures/) holds five short synthetic
+session logs, each crafted to surface a *different* failure mode
+in consolidation:
+
+| Fixture | What it stresses |
+|---|---|
+| `01-rust-bug-fix` | Did the model split a multi-page session into the right slices (session log + concept + decision + gotcha)? |
+| `02-architecture-decision` | Can the model produce an ADR-style page distinct from the running session log? |
+| `03-gotcha-with-rule` | Did the model correctly classify a durable project rule with `kind: rule` so the consolidator can auto-route it to `_rules/`? |
+| `04-low-signal-session` | Does the model *resist* manufacturing concept pages when there's nothing durable to capture? |
+| `05-multi-topic-session` | Does the model emit *separate* pages per topic instead of mashing two unrelated topics together? |
+
+Fixtures use real-shape `ObservationKind` values (`session-start`,
+`user-prompt`, `pre-tool-use`, `post-tool-use`, `session-end`)
+exactly as the production hook ingress emits them.
+
+### The exact request
+
+Per fixture, the runner calls
+[`ai_memory_consolidate::build_batch_request(session_id, &observations)`](../crates/ai-memory-consolidate/src/consolidator.rs)
+— the **same** function the live consolidator uses on every
+`memory_consolidate` invocation. That request is then sent
+through [`ai_memory_llm::complete_structured`](../crates/ai-memory-llm/src/lib.rs)
+(also the live path). Apples-to-apples by construction.
+
+### The three providers
+
+| Tag | Provider | Model | Endpoint |
+|---|---|---|---|
+| **Kimi** | OpenRouter (openai-compat) | `moonshotai/kimi-k2.6` | `https://openrouter.ai/api/v1` |
+| **Sonnet** | OpenRouter (openai-compat) | `anthropic/claude-sonnet-4.5` | `https://openrouter.ai/api/v1` |
+| **qwen3** | Ollama (openai-compat) | `qwen3:32b` (Q4_K_M, ~20 GB) | `http://192.168.0.90:11434/v1` |
+
+The home server (`192.168.0.90`) is a Ryzen AI MAX+ 395
+(Strix Halo / gfx1151), 96 GB unified memory, ROCm-backed
+Ollama with `OLLAMA_KEEP_ALIVE=20m` + `OLLAMA_FLASH_ATTENTION=1`
++ `OLLAMA_KV_CACHE_TYPE=q8_0`. Once a model is loaded into
+unified memory it stays warm for 20 min — so the first
+request pays a 30–60 s cold-load tax and subsequent ones are
+sub-3 s.
+
+## Run 1 — broken prompts + schema (pre-fix baseline)
+
+Every provider failed schema validation on every fixture:
+
+| Fixture | Kimi | qwen3:32b |
+|---|---|---|
+| 01-rust-bug-fix | ❌ *response is not valid JSON* | ❌ *integer 1, expected string* |
+| 02-architecture-decision | ❌ *response is not valid JSON* | ❌ *integer 2, expected string* |
+| 03-gotcha-with-rule | ❌ *response is not valid JSON* | ❌ *integer 1, expected string* |
+| 04-low-signal-session | ❌ *response is not valid JSON* | ❌ *integer 1, expected string* |
+| 05-multi-topic-session | ❌ *response is not valid JSON* | ❌ *integer 2, expected string* |
+
+But the *raw responses* told a very different story: both
+models did **excellent** consolidation work content-wise. They
+correctly identified multiple distinct pages per fixture,
+extracted faithful summaries, and respected the path
+conventions. The failures were **format only**:
+
+- **Kimi** was emitting beautifully formatted markdown
+ (`### Update 1` / `**path:**` / `**body:**`) — completely
+ ignoring the request for JSON.
+- **qwen3** was emitting clean JSON in code fences, but with
+ `tier: 1` / `tier: 2` / `tier: 3` (integers) instead of the
+ documented string values, and occasionally with invented
+ `kind` values like `"session"` (which isn't in the
+ `PageKind` enum).
+
+## The root cause
+
+Two separate problems, both **on our side**:
+
+### Bug A — `Tier` had no `JsonSchema` derive
+
+In `crates/ai-memory-consolidate/src/types.rs`:
+
+```rust
+pub struct ConsolidatedPageUpdate {
+ pub path: String,
+ pub tier: String, // ← bug: typed as String
+ pub kind: PageKind, // ← already an enum with JsonSchema
+ ...
+}
+```
+
+`schemars` couldn't produce an enum constraint for `tier`
+because `Tier` (the actual enum in `ai-memory-core`) didn't
+have the `JsonSchema` derive. The generated schema field was
+just `{ "type": "string" }` — no `enum` constraint — so models
+were free to guess. Both Kimi and qwen3 guessed numeric indices.
+
+### Bug B — prompt described values, didn't enforce them
+
+The system prompt in
+[`build_batch_request`](../crates/ai-memory-consolidate/src/consolidator.rs)
+listed the valid `tier` and `kind` values in prose but never
+said "use these EXACT string values, never an integer, never a
+synonym, never code fences". Local instruction-tuned models —
+especially when there's no `response_format: json_schema`
+support to enforce — will drift to whatever feels natural.
+
+Compounding this: openai-compat providers (Ollama, OpenRouter
+passthrough) do **not** expose strict-mode JSON-schema
+validation. The schema is descriptive, not coercive. So the
+prompt has to do the load-bearing work.
+
+## The fix
+
+Three small changes landed together:
+
+### 1. Derive `JsonSchema` on `Tier`
+
+`crates/ai-memory-core/src/page.rs`:
+
+```rust
+#[derive(
+ Clone, Copy, Debug, PartialEq, Eq, Hash,
+ Serialize, Deserialize,
+ schemars::JsonSchema, // ← new
+)]
+#[serde(rename_all = "snake_case")]
+pub enum Tier { Working, Episodic, Semantic, Procedural }
+```
+
+Adds `schemars` as a dep on `ai-memory-core` (acceptable —
+schemars is already a workspace dep used by every type that
+crosses the LLM boundary).
+
+### 2. Type the field as `Tier`, not `String`
+
+`crates/ai-memory-consolidate/src/types.rs`:
+
+```rust
+pub struct ConsolidatedPageUpdate {
+ pub path: String,
+ pub tier: Tier, // ← was String
+ pub kind: PageKind,
+ ...
+}
+```
+
+The generated schema now contains
+`{ "enum": ["working", "episodic", "semantic", "procedural"] }`
+for `tier`. `serde_json::from_value` rejects anything else.
+
+### 3. Tighten the prompt
+
+`build_batch_request` now spells out:
+
+```
+Set `tier` to EXACTLY ONE of these four strings — never an integer, never a synonym:
+- "working" (the live in-progress slice of the session — rarely used here)
+- "episodic" (per-session narrative; the sessions/<id>.md page)
+- "semantic" (durable knowledge: concepts/, decisions/, gotchas/, rules)
+- "procedural" (repeated patterns extracted from many episodic pages)
+
+Set `kind` to EXACTLY ONE of these four strings — never an integer, never "session" / "concept" / "note":
+- "decision" / "gotcha" / "rule" / "fact"
+
+## Output format (read this carefully)
+Reply with ONE JSON object matching the ConsolidatedBatch schema, and nothing else.
+NO prose preamble, NO trailing commentary, NO markdown headers wrapping the JSON,
+NO ``` code fences. The very first character of your reply must be `{` and the
+very last `}`. Strings must be JSON strings (with double quotes), not numbers
+and not bare identifiers.
+```
+
+Belt-and-suspenders: the schema now *rejects* the bad values,
+and the prompt makes it actively hard for the model to produce
+them in the first place.
+
+## Run 2 — fixed prompts + schema
+
+After the three fixes above, the same five fixtures produced:
+
+### Sonnet 4.5 (OpenRouter) vs qwen3:32b (Ollama)
+
+| Fixture | Sonnet parse | Sonnet ms | Sonnet updates | qwen3 parse | qwen3 ms | qwen3 updates |
+|---|---|---|---|---|---|---|
+| 01 rust-bug-fix | ✓ | 27,613 | 4 | ✓ | 110,227 | 4 |
+| 02 architecture-decision | ✓ | 31,039 | 4 | ✓ | 122,200 | 5 |
+| 03 gotcha-with-rule | ✓ | 19,173 | 4 | ✓ | 98,025 | 4 |
+| 04 low-signal-session | ✓ | 6,106 | **1** | ✓ | 51,694 | **1** |
+| 05 multi-topic-session | ✓ | 47,249 | 4 | ✗* | 133,178 | — |
+| **Aggregate** | **5/5** | **avg 26 s** | — | **4/5** | **avg 103 s** | — |
+
+*qwen3's only failure: invented `kind: "concept"` (not in the
+`PageKind` enum — valid values are `decision`/`gotcha`/`rule`/
+`fact`). Despite the prompt explicitly listing the valid set
+and forbidding "concept", the model drifted. Otherwise the
+JSON, the `tier` enum, the path conventions, and the field
+names were all correct.
+
+Both models **correctly restrained themselves** on fixture
+04 (low-signal-session) and produced a single update — a
+non-trivial test the original schema-broken Run 1 couldn't
+even reach.
+
+### Kimi-K2.6 (OpenRouter) vs qwen3:32b (Ollama)
+
+_(Run in flight at the time of writing — will update with
+numbers once it lands. The expectation is that Kimi's "prose
+slips" issue from Run 1 is now mitigated by the explicit
+"NO ``` code fences, NO prose preamble, first char must be `{`"
+directives; we expect Kimi to also reach 4-5/5 parse rate.)_
+
+## Qualitative read (Run 2)
+
+Reading the raw `.md` outputs side-by-side reveals a
+substantive style difference that the parse-rate numbers
+don't capture:
+
+- **Sonnet writes long, comprehensive entries.** A concept
+ page on Docker multi-stage builds will get 3 KB of well-
+ organised prose including "When to use" / "When NOT to use"
+ / "Gotchas" sections — content that *wasn't in the
+ observations*. The model is generating useful tutorial-style
+ content, not strictly consolidating what happened.
+ Sonnet's fixture 05 page invented a `Date: 2025-01-23`
+ field that has no source in the observations.
+
+- **qwen3 writes terse, faithful entries.** Each page captures
+ what the session actually contained, in ~500–800 chars.
+ No invented metadata, no generic tutorial filler. The same
+ Docker page from qwen3 stays close to "we changed the
+ Dockerfile to two-stage, image went 380→67 MB" without
+ diverging into broader best-practices discussion.
+
+For **wiki consolidation** (faithful long-term memory of
+*this project*, not a knowledge graph of general best
+practices), **qwen3's restraint is arguably preferable** to
+Sonnet's exuberance. The point of the wiki is to record what
+happened in the project, not to host re-generated tutorial
+content the model already knows.
+
+That said, when the project memory is genuinely sparse and
+the model is asked to surface durable knowledge, Sonnet's
+"fill in the obvious" tendency could pay off. Different
+tasks → different preferences.
+
+## Verdict
+
+**Ollama qwen3:32b is fit for production consolidation on
+this project.** With the prompt + schema fixes applied:
+
+- **Parse reliability**: 4/5 vs Sonnet's 5/5. The one failure
+ is a content-drift issue (model invented a `kind` value),
+ not a structural failure — addressable with a slightly more
+ forceful prompt or a lenient deserializer if it recurs.
+- **Latency**: ~4× slower than Sonnet (103 s vs 26 s avg
+ end-to-end). Consolidation runs at session end or on a
+ schedule, not interactively, so even a 2-minute consolidation
+ job is invisible to the user. For lint sweeps that touch many
+ pages, this could compound — measure if it bites in practice.
+- **Cost**: **$0 per consolidation**. Sonnet at ~4 KB of
+ output × $15/M tok ≈ $0.06 per consolidation × N sessions/day
+ × 365 days = the difference between an unbounded growth
+ monthly bill and zero.
+- **Content fidelity**: qwen3 is *more faithful* to source
+ observations than Sonnet. Sonnet hallucinates plausible
+ details (e.g. invented dates, tutorial-style sections). For
+ "memory of what happened in this project", faithful >
+ comprehensive.
+
+**Recommendation: keep Ollama qwen3:32b as the default
+production LLM** (where ai-memory now points). Sonnet remains
+a useful escape hatch for one-off complex consolidations,
+batch lint runs against very large wikis, or situations where
+the home server is unreachable. Kimi sits in an awkward
+middle: comparable cost-per-call to Sonnet on OpenRouter, less
+output discipline.
+
+### When to revisit
+
+Re-run this harness when any of the following changes:
+
+- The consolidation prompt itself is re-engineered
+- A new Ollama model is pulled (e.g. when Qwen 3.5 stable
+ drops for Ollama)
+- A new fixture is added to `evals/fixtures/`
+- The home server hardware changes
+- An OpenAI / Anthropic / Voyage strict-JSON-schema feature
+ becomes available through OpenRouter
+
+## How to reproduce
+
+### Pre-requisites
+
+- Repo checkout + `cargo` toolchain (Rust 1.95+, as pinned in
+ `rust-toolchain.toml`).
+- An OpenRouter API key, exported as `OPENROUTER_API_KEY` —
+ pays the Kimi + Sonnet legs.
+- A reachable Ollama with `qwen3:32b` pulled. The default URL
+ in the docs assumes the homelab; substitute your own.
+
+### Run the harness
+
+The canonical 2-side invocation (the harness compares two
+providers per run):
+
+```bash
+cargo run -p ai-memory-eval --release -- \
+ --baseline-provider openai-compat \
+ --baseline-base-url https://openrouter.ai/api/v1 \
+ --baseline-model moonshotai/kimi-k2.6 \
+ --baseline-api-key-env OPENROUTER_API_KEY \
+ --candidate-provider openai-compat \
+ --candidate-base-url http://192.168.0.90:11434/v1 \
+ --candidate-model qwen3:32b \
+ --candidate-api-key ollama-local
+```
+
+For a 3-way comparison, run the harness three times pairing the
+candidate (the model you're considering switching to) against
+each baseline you want to compare against. Output dirs are
+timestamped, so they don't collide.
+
+### Read the output
+
+```
+evals/runs/<timestamp>/
+├── baseline/
+│ ├── 01-rust-bug-fix.json ← parsed structured output (if any)
+│ ├── 01-rust-bug-fix.md ← flat-rendered for eyeballing
+│ ├── 01-rust-bug-fix.raw.txt ← exact model output, always present
+│ └── 01-rust-bug-fix.meta.json ← {elapsed_ms, parsed_ok, update_count, error}
+└── candidate/
+ └── ...
+```
+
+The `.raw.txt` files are the most informative artifact when a
+parse fails — they show *exactly* what the model said, so you
+can tell whether the failure was format (model emitted prose),
+schema (model used integer enums), or substance (model
+produced nothing useful).
+
+For side-by-side reading the runner prints a hint:
+
+```
+compare with: diff -ru <run>/baseline <run>/candidate
+```
+
+### Adding new fixtures
+
+Each fixture is a JSON file under `evals/fixtures/`:
+
+```json
+{
+ "name": "human-readable-id",
+ "description": "what this case is meant to surface",
+ "observations": [
+ {"kind": "session-start", "title": "...", "body": "..."},
+ {"kind": "user-prompt", "title": "user prompt", "body": "..."},
+ {"kind": "pre-tool-use", "title": "Edit", "body": "..."}
+ ]
+}
+```
+
+`kind` accepts any string the
+[`ObservationKind`](../crates/ai-memory-core/src/observation.rs)
+enum's `FromStr` understands. Anything unknown silently falls
+back to `Other`.
+
+Try to hit one of the four hard cases:
+
+1. **Multi-page extraction** — does the model split a session
+ into the right slices?
+2. **Restraint** — does it avoid manufacturing pages when
+ there's nothing durable?
+3. **Classification** — does it correctly choose `kind: rule`
+ for project rules?
+4. **Topic separation** — does it produce separate pages per
+ unrelated topic instead of mashing them?
+
+## What's NOT in this harness (yet)
+
+- **Automated quality scoring.** The runner only reports
+ objective deltas (latency, parse rate, update count).
+ Anything subtler (faithfulness, hallucination, scoping)
+ needs a human reader.
+- **Embedding A/B.** This document is LLM-only. The embedding
+ provider switch (OpenAI text-embedding-3-small → Ollama
+ nomic-embed-text) gets its own writeup when there's enough
+ page-side data to measure retrieval quality.
+- **LLM-as-judge scoring.** Adding a third "judge" model to
+ score the candidate outputs against a rubric would
+ automate quality measurement. Not built; the next layer up
+ if this harness gets used regularly.
+
+## Future work
+
+If we end up running this harness routinely:
+
+1. Add a third position (`--judge-*`) so a separate "judge"
+ model can score baseline vs candidate per fixture against a
+ rubric, producing a numeric quality delta.
+2. Extend fixtures with a `must_mention` / `must_not_mention`
+ keyword list so we can compute simple keyword recall
+ automatically (catches obvious hallucinations / missing
+ facts).
+3. Parallel embedding-retrieval eval: a probe set of queries
+ each tagged with the expected target wiki page; compute
+ recall@5 + MRR for two embedding models against the same
+ indexed corpus.
+4. Persist a leaderboard somewhere durable (a wiki page,
+ ironically) so we don't lose track of which model performed
+ best on which fixture across runs.
diff --git a/evals/src/main.rs b/evals/src/main.rs
index c618772f..7f167109 100644
--- a/evals/src/main.rs
+++ b/evals/src/main.rs
@@ -39,6 +39,7 @@
//! --candidate-api-key ollama-local
//! ```
+use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -51,9 +52,7 @@ use serde::{Deserialize, Serialize};
use ai_memory_consolidate::{ConsolidatedBatch, build_batch_request};
use ai_memory_core::{Observation, ObservationKind, ProjectId, SessionId, WorkspaceId};
-use ai_memory_llm::{
- LlmProvider, ProviderChoice, ProviderConfig, complete_structured,
-};
+use ai_memory_llm::{LlmProvider, ProviderChoice, ProviderConfig};
#[derive(Parser, Debug)]
#[command(
@@ -157,10 +156,10 @@ async fn main() -> Result<()> {
let baseline = resolve_provider("baseline", &args)?;
let candidate = resolve_provider("candidate", &args)?;
- let baseline_provider = make_provider(baseline.clone())
- .context("building baseline provider")?;
- let candidate_provider = make_provider(candidate.clone())
- .context("building candidate provider")?;
+ let baseline_provider =
+ make_provider(baseline.clone()).context("building baseline provider")?;
+ let candidate_provider =
+ make_provider(candidate.clone()).context("building candidate provider")?;
let fixtures = load_fixtures(&args.fixtures)
.with_context(|| format!("loading fixtures from {}", args.fixtures.display()))?;
@@ -180,9 +179,23 @@ async fn main() -> Result<()> {
println!();
println!("ai-memory eval — {stamp}");
- println!(" fixtures : {} ({})", fixtures.len(), args.fixtures.display());
- println!(" baseline : {} {} @ {}", baseline.provider_name(), baseline.model, baseline.base_url.as_deref().unwrap_or("(default)"));
- println!(" candidate: {} {} @ {}", candidate.provider_name(), candidate.model, candidate.base_url.as_deref().unwrap_or("(default)"));
+ println!(
+ " fixtures : {} ({})",
+ fixtures.len(),
+ args.fixtures.display()
+ );
+ println!(
+ " baseline : {} {} @ {}",
+ baseline.provider_name(),
+ baseline.model,
+ baseline.base_url.as_deref().unwrap_or("(default)")
+ );
+ println!(
+ " candidate: {} {} @ {}",
+ candidate.provider_name(),
+ candidate.model,
+ candidate.base_url.as_deref().unwrap_or("(default)")
+ );
println!(" out : {}", run_dir.display());
println!();
@@ -216,6 +229,10 @@ async fn main() -> Result<()> {
b.parsed_ok,
b.update_count.map_or("-".into(), |n| n.to_string())
);
+ // Force-flush per fixture so progress is visible when stdout
+ // is piped (e.g. via `tee` or backgrounded runs). Without
+ // this, full-buffering swallows every println! until exit.
+ std::io::stdout().flush().ok();
results.push((a, b));
}
@@ -227,11 +244,21 @@ async fn main() -> Result<()> {
let candidate_total: u128 = results.iter().map(|(_, b)| b.elapsed_ms).sum();
let baseline_ok = results.iter().filter(|(a, _)| a.parsed_ok).count();
let candidate_ok = results.iter().filter(|(_, b)| b.parsed_ok).count();
- println!("baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms", baseline_total / n.max(1) as u128);
- println!("candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms", candidate_total / n.max(1) as u128);
+ println!(
+ "baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms",
+ baseline_total / n.max(1) as u128
+ );
+ println!(
+ "candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms",
+ candidate_total / n.max(1) as u128
+ );
println!();
println!("outputs under: {}", run_dir.display());
- println!("compare with: diff -ru {}/baseline {}/candidate", run_dir.display(), run_dir.display());
+ println!(
+ "compare with: diff -ru {}/baseline {}/candidate",
+ run_dir.display(),
+ run_dir.display()
+ );
Ok(())
}
@@ -259,9 +286,7 @@ fn resolve_provider(side: &str, args: &Args) -> Result<ResolvedConfig> {
"anthropic" => ProviderChoice::Anthropic,
"openai" => ProviderChoice::OpenAi,
"openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,
- other => bail!(
- "{side}: provider {other} not one of anthropic|openai|openai-compat"
- ),
+ other => bail!("{side}: provider {other} not one of anthropic|openai|openai-compat"),
};
let api_key = match (api_key_inline, api_key_env) {
@@ -337,10 +362,9 @@ fn load_fixtures(dir: &Path) -> Result<Vec<(PathBuf, Fixture)>> {
paths.sort();
let mut out = Vec::with_capacity(paths.len());
for p in paths {
- let bytes = std::fs::read(&p)
- .with_context(|| format!("reading {}", p.display()))?;
- let fx: Fixture = serde_json::from_slice(&bytes)
- .with_context(|| format!("parsing {}", p.display()))?;
+ let bytes = std::fs::read(&p).with_context(|| format!("reading {}", p.display()))?;
+ let fx: Fixture =
+ serde_json::from_slice(&bytes).with_context(|| format!("parsing {}", p.display()))?;
out.push((p, fx));
}
Ok(out)
@@ -371,8 +395,14 @@ fn synthesise_observations(fx: &Fixture) -> Vec<Observation> {
struct ProviderResult {
elapsed: Duration,
+ /// Raw text the model emitted — always saved, even when parsing
+ /// fails. This is the load-bearing artifact for debugging
+ /// schema mismatches: you can see exactly what the model said.
+ raw_text: String,
+ /// Parsed-from-text JSON, if the response was valid JSON at all.
+ raw_json: Option<serde_json::Value>,
+ /// Schema-validated batch, if the JSON also satisfied the schema.
batch: Option<ConsolidatedBatch>,
- raw: Option<serde_json::Value>,
error: Option<String>,
}
@@ -382,32 +412,98 @@ async fn run_one(
request: ai_memory_llm::ChatRequest,
) -> ProviderResult {
let start = Instant::now();
- // Try the schema-validated path first; fall back to the raw
- // structured call so we can still inspect malformed output.
- let batch_res: Result<ConsolidatedBatch, _> =
- complete_structured(&*provider, request.clone()).await;
+ // Call `complete` directly — bypassing `complete_structured` —
+ // so we can capture the raw text the model emitted, even if it
+ // doesn't deserialise. The cost is that we re-implement the
+ // "extract first balanced {…}" fallback locally, but knowing
+ // exactly what the model said is what makes the harness useful.
+ let chat_res = provider.complete(request).await;
let elapsed = start.elapsed();
- match batch_res {
- Ok(batch) => {
- let raw = serde_json::to_value(&batch).ok();
- tracing::info!(side, ms = elapsed.as_millis() as u64, "parsed ok");
- ProviderResult {
- elapsed,
- batch: Some(batch),
- raw,
- error: None,
+ let raw_text = match &chat_res {
+ Ok(r) => r.text.clone(),
+ Err(e) => format!("(provider error before any response: {e})"),
+ };
+ let (raw_json, batch, error) = match chat_res {
+ Ok(_) => parse_response(&raw_text),
+ Err(e) => (None, None, Some(e.to_string())),
+ };
+ if let Some(err) = &error {
+ tracing::warn!(side, error = %err, "model output did not satisfy schema");
+ } else {
+ tracing::info!(
+ side,
+ ms = elapsed.as_millis() as u64,
+ updates = batch.as_ref().map_or(0, |b| b.updates.len()),
+ "parsed ok"
+ );
+ }
+ ProviderResult {
+ elapsed,
+ raw_text,
+ raw_json,
+ batch,
+ error,
+ }
+}
+
+/// Parse a model response into (json, batch, error).
+///
+/// 1. Try the whole response as JSON.
+/// 2. If that fails, search for the first balanced `{…}` block.
+/// 3. If we have JSON, try to deserialise as `ConsolidatedBatch`.
+fn parse_response(
+ text: &str,
+) -> (
+ Option<serde_json::Value>,
+ Option<ConsolidatedBatch>,
+ Option<String>,
+) {
+ let json = serde_json::from_str::<serde_json::Value>(text.trim())
+ .ok()
+ .or_else(|| {
+ extract_first_balanced_object(text).and_then(|s| serde_json::from_str(&s).ok())
+ });
+ let Some(json) = json else {
+ return (None, None, Some("response is not valid JSON".into()));
+ };
+ match serde_json::from_value::<ConsolidatedBatch>(json.clone()) {
+ Ok(batch) => (Some(json), Some(batch), None),
+ Err(e) => (Some(json), None, Some(format!("schema mismatch: {e}"))),
+ }
+}
+
+/// Best-effort `{…}` extractor — ignores braces inside strings.
+fn extract_first_balanced_object(text: &str) -> Option<String> {
+ let bytes = text.as_bytes();
+ let start = bytes.iter().position(|&b| b == b'{')?;
+ let mut depth = 0i32;
+ let mut in_string = false;
+ let mut escape = false;
+ for (i, &b) in bytes.iter().enumerate().skip(start) {
+ if in_string {
+ if escape {
+ escape = false;
+ } else if b == b'\\' {
+ escape = true;
+ } else if b == b'"' {
+ in_string = false;
}
+ continue;
+ }
+ if b == b'"' {
+ in_string = true;
+ continue;
}
- Err(e) => {
- tracing::warn!(side, error = %e, "schema parse failed");
- ProviderResult {
- elapsed,
- batch: None,
- raw: None,
- error: Some(e.to_string()),
+ if b == b'{' {
+ depth += 1;
+ } else if b == b'}' {
+ depth -= 1;
+ if depth == 0 {
+ return Some(text[start..=i].to_string());
}
}
}
+ None
}
fn persist_case(
@@ -424,18 +520,24 @@ fn persist_case(
let dir = run_dir.join(side);
std::fs::create_dir_all(&dir)?;
- // 1. Raw JSON.
- let json_path = dir.join(format!("{safe_name}.json"));
- if let Some(raw) = &res.raw {
- std::fs::write(&json_path, serde_json::to_string_pretty(raw)?)?;
+ // 1. Raw model text — always saved, even on parse failure.
+ // This is the load-bearing debug artifact: a side-by-side
+ // diff of these files tells you exactly what each model said.
+ let raw_path = dir.join(format!("{safe_name}.raw.txt"));
+ std::fs::write(&raw_path, &res.raw_text)?;
+
+ // 2. Parsed JSON (if response was valid JSON at all).
+ if let Some(json) = &res.raw_json {
+ let json_path = dir.join(format!("{safe_name}.json"));
+ std::fs::write(&json_path, serde_json::to_string_pretty(json)?)?;
}
- // 2. Flattened markdown rendering for easy eyeballing / diff.
+ // 3. Flattened markdown rendering for easy eyeballing.
let md_path = dir.join(format!("{safe_name}.md"));
let md = render_markdown(side, fixture_name, &res);
std::fs::write(&md_path, md)?;
- // 3. Per-case meta.
+ // 4. Per-case meta.
let case = CaseRun {
provider: side.to_string(),
fixture: fixture_name.to_string(),
@@ -454,29 +556,35 @@ fn render_markdown(side: &str, fixture_name: &str, res: &ProviderResult) -> Stri
out.push_str(&format!("# {side} — {fixture_name}\n\n"));
out.push_str(&format!("- elapsed: {} ms\n", res.elapsed.as_millis()));
if let Some(err) = &res.error {
- out.push_str(&format!("- error: `{err}`\n\n"));
+ out.push_str(&format!("- error: `{err}`\n"));
+ }
+ // Always include the raw response so the human reviewer can
+ // judge structure/style even when schema validation failed.
+ if !res.raw_text.is_empty() {
+ out.push_str("\n## Raw response\n\n```\n");
+ out.push_str(&res.raw_text);
+ out.push_str("\n```\n");
}
let Some(batch) = &res.batch else {
- out.push_str("\n_(no batch — see error above; .json file absent)_\n");
return out;
};
- out.push_str(&format!("- updates: {}\n\n", batch.updates.len()));
+ out.push_str(&format!("\n- updates: {}\n", batch.updates.len()));
if !batch.rationale.trim().is_empty() {
- out.push_str("## Rationale\n\n");
+ out.push_str("\n## Rationale\n\n");
out.push_str(batch.rationale.trim());
- out.push_str("\n\n");
+ out.push('\n');
}
for (i, u) in batch.updates.iter().enumerate() {
- out.push_str(&format!("## Update {} — `{}`\n\n", i + 1, u.path));
+ out.push_str(&format!("\n## Update {} — `{}`\n\n", i + 1, u.path));
out.push_str(&format!("**Title:** {} \n", u.title));
out.push_str(&format!("**Kind:** `{}` \n", u.kind.as_str()));
- out.push_str(&format!("**Tier:** `{}` \n", u.tier));
+ out.push_str(&format!("**Tier:** `{}` \n", u.tier.as_str()));
if !u.tags.is_empty() {
out.push_str(&format!("**Tags:** {} \n", u.tags.join(", ")));
}
out.push('\n');
out.push_str(u.body_markdown.trim());
- out.push_str("\n\n");
+ out.push('\n');
}
out
}