Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
consolidate: strict tier+kind enums + verbose prompt; docs comparison
AkitaOnRailse830a7c415c0
diff --git a/Cargo.lock b/Cargo.lockindex 80be0c91..54717382 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -92,6 +92,7 @@ version = "0.1.0"dependencies = ["jiff","regex",+ "schemars","serde","serde_json","thiserror 2.0.18",diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rsindex fb4910ce..9eee59ac 100644--- a/crates/ai-memory-consolidate/src/consolidator.rs+++ b/crates/ai-memory-consolidate/src/consolidator.rs@@ -225,7 +225,7 @@ impl Consolidator {upd.path.clone()};let path = PagePath::new(final_path)?;- let tier = upd.tier.parse::<Tier>().unwrap_or(Tier::Semantic);+ let tier = upd.tier;let mut fm = serde_json::Map::new();fm.insert("title".into(), serde_json::Value::String(upd.title.clone()));fm.insert(@@ -331,18 +331,42 @@ pub fn build_batch_request(session_id: SessionId, observations: &[Observation])- concepts/<slug>.md (semantic, evergreen concept pages)\n\- decisions/<short>.md (semantic, ADR-style records)\n\- gotchas/<slug>.md (semantic, failure modes / surprises)\n\- \nClassify each update with one of these `kind` values:\n\- - `decision` (the project chose X over Y)\n\- - `gotcha` (a failure mode or surprise worth remembering)\n\- - `rule` (durable project convention: \"always X\", \"never Y\")\n\- - `fact` (everything else; the default)\n\+ \nSet `tier` to EXACTLY ONE of these four strings — never an integer, never a synonym:\n\+ - \"working\" (the live in-progress slice of the session — rarely used here)\n\+ - \"episodic\" (per-session narrative; the sessions/<id>.md page)\n\+ - \"semantic\" (durable knowledge: concepts/, decisions/, gotchas/, rules)\n\+ - \"procedural\" (repeated patterns extracted from many episodic pages)\n\+ \nSet `kind` to EXACTLY ONE of these four strings — never an integer, never \"session\" / \"concept\" / \"note\":\n\+ - \"decision\" (the project chose X over Y)\n\+ - \"gotcha\" (a failure mode or surprise worth remembering)\n\+ - \"rule\" (durable project convention: \"always X\", \"never Y\")\n\+ - \"fact\" (everything else; the default — use this for session narratives and plain concept notes)\n\\nWhen you mark an update as `rule`, write the body as a clear \standalone instruction the agent could follow on every relevant \action. The path you suggest for a rule will be overridden — the \system routes rules to `_rules/<slug>.md` automatically and the \lint pass surfaces a hint to copy it into the project's CLAUDE.md.\- \n\nEach update must include a title, markdown body, tier, kind, \- and tags.\n",+ \n## Required JSON keys on every update (use these EXACT names)\n\+ - \"path\" (string) required — the wiki path\n\+ - \"title\" (string) required — the page title\n\+ - \"body_markdown\" (string) required — the page body in Markdown; NOTE the underscore + the suffix `_markdown`, NOT just `body`\n\+ - \"tier\" (string) required — one of the four tier strings above\n\+ - \"kind\" (string) required — one of the four kind strings above\n\+ - \"tags\" (array of string) required — may be empty `[]`, but the key must be present\n\+ No other keys. No `body`, no `content`, no `summary`. Field names \+ are case-sensitive and the `_markdown` suffix matters.\n\+ \n## Output format (read this carefully)\n\+ Reply with ONE JSON object matching the ConsolidatedBatch schema, \+ and nothing else. NO prose preamble, NO trailing commentary, NO \+ markdown headers wrapping the JSON, NO ``` code fences. The very \+ first character of your reply must be `{` and the very last `}`. \+ Strings must be JSON strings (with double quotes), not numbers \+ and not bare identifiers.\n\+ \n## Top-level shape\n\+ {\n\+ \x20\x20\"updates\": [ /* 1-5 update objects with the keys above */ ],\n\+ \x20\x20\"rationale\": \"<one short sentence about why this batch>\"\n\+ }\n",);ChatRequest {system: Some(BATCH_SYSTEM_PROMPT.into()),diff --git a/crates/ai-memory-consolidate/src/types.rs b/crates/ai-memory-consolidate/src/types.rsindex 09a0e8d9..ba522c10 100644--- a/crates/ai-memory-consolidate/src/types.rs+++ b/crates/ai-memory-consolidate/src/types.rs@@ -1,6 +1,6 @@//! Public-facing consolidation types.-use ai_memory_core::{PageId, PagePath};+use ai_memory_core::{PageId, PagePath, Tier};use schemars::JsonSchema;use serde::{Deserialize, Serialize};@@ -70,8 +70,13 @@ pub struct ConsolidatedPageUpdate {/// When `kind` is `Rule` the consolidator overrides this to/// `_rules/<slug>.md` regardless — see `PageKind::Rule`.pub path: String,- /// Tier (`semantic`, `episodic`, `procedural`, `working`).- pub tier: String,+ /// Tier classification. Typed as the [`Tier`] enum (not a free+ /// `String`) so schemars emits a closed enum in the generated+ /// JSON schema. Before this change schemars produced+ /// `{ "type": "string" }` with no constraint, and both Kimi+ /// and qwen3 routinely emitted `tier: 2` (integer) instead of+ /// the documented string values.+ pub tier: Tier,/// Semantic classification. Defaults to `fact` if the LLM/// doesn't supply one — existing consolidations without this/// field still deserialise.diff --git a/crates/ai-memory-core/Cargo.toml b/crates/ai-memory-core/Cargo.tomlindex 1b36bf68..03c900a4 100644--- a/crates/ai-memory-core/Cargo.toml+++ b/crates/ai-memory-core/Cargo.toml@@ -16,6 +16,11 @@ uuid.workspace = truejiff.workspace = trueregex.workspace = truetracing.workspace = true+# JsonSchema on Tier/PageKind so the consolidation prompt's+# structured-output schema enforces enum membership instead of+# producing `{ "type": "string" }` with no constraints (which lets+# the model invent integer tier values).+schemars.workspace = true[lints]workspace = truediff --git a/crates/ai-memory-core/src/page.rs b/crates/ai-memory-core/src/page.rsindex 64c98665..deb0d755 100644--- a/crates/ai-memory-core/src/page.rs+++ b/crates/ai-memory-core/src/page.rs@@ -17,7 +17,7 @@ use crate::ids::{PageId, PagePath, ProjectId, WorkspaceId};////// Adopted from agentmemory's four-tier model, simplified. See/// [`docs/design-decisions.md`](../../docs/design-decisions.md) §7.-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, schemars::JsonSchema)]#[serde(rename_all = "snake_case")]pub enum Tier {/// Current session: last N observations, current files.diff --git a/crates/ai-memory-llm/src/openai.rs b/crates/ai-memory-llm/src/openai.rsindex c92341a7..842ce38c 100644--- a/crates/ai-memory-llm/src/openai.rs+++ b/crates/ai-memory-llm/src/openai.rs@@ -251,20 +251,30 @@ mod tests {#[test]fn normalize_strips_trailing_slash_and_v1() {// The three conventions found in the wild.- assert_eq!(normalize_openai_base("https://api.openai.com"),- "https://api.openai.com");- assert_eq!(normalize_openai_base("https://api.openai.com/"),- "https://api.openai.com");+ assert_eq!(+ normalize_openai_base("https://api.openai.com"),+ "https://api.openai.com"+ );+ assert_eq!(+ normalize_openai_base("https://api.openai.com/"),+ "https://api.openai.com"+ );// OpenRouter's docs / ai-memory's OpenRouter env entry.- assert_eq!(normalize_openai_base("https://openrouter.ai/api/v1"),- "https://openrouter.ai/api");+ assert_eq!(+ normalize_openai_base("https://openrouter.ai/api/v1"),+ "https://openrouter.ai/api"+ );// Ollama's openai-compat path (`docker logs` shows it// canonically advertised this way).- assert_eq!(normalize_openai_base("http://localhost:11434/v1"),- "http://localhost:11434");+ assert_eq!(+ normalize_openai_base("http://localhost:11434/v1"),+ "http://localhost:11434"+ );// Don't accidentally strip a `/v1` that's part of a longer// segment (e.g. `/v123/`).- assert_eq!(normalize_openai_base("https://example.com/v123"),- "https://example.com/v123");+ assert_eq!(+ normalize_openai_base("https://example.com/v123"),+ "https://example.com/v123"+ );}}diff --git a/docs/llm-provider-comparison.md b/docs/llm-provider-comparison.mdnew file mode 100644index 00000000..391b567b--- /dev/null+++ b/docs/llm-provider-comparison.md@@ -0,0 +1,445 @@+# LLM provider comparison — local Ollama vs hosted OpenRouter++> **TL;DR.** ai-memory's consolidation prompt is provider-agnostic by+> design but had a latent **schema-vs-prompt** bug that made *both*+> Kimi-K2.6 (OpenRouter) and qwen3:32b (Ollama) fail JSON+> validation. After fixing the schema and tightening the prompt,+> local Ollama on a Strix Halo home server gives **comparable+> output quality to hosted models** at zero cost per token, with a+> ~2× latency penalty on first request (which the+> `OLLAMA_KEEP_ALIVE=20m` warm cache largely amortises). The+> harness used to discover this lives in [`evals/`](../evals/);+> anyone can reproduce the comparison in ≤ 20 minutes.++## Why this document exists++When the homelab deploy switched ai-memory off the billed+OpenAI / OpenRouter providers and onto the locally-hosted Ollama+server, we needed empirical evidence — not a vibes-based claim —+that *consolidation quality didn't degrade*. ai-memory's+consolidator turns a session's raw observations into 1–5 wiki+pages classified as `concept`, `decision`, `gotcha`, or `rule`;+small drops in quality compound fast across hundreds of sessions.++This doc captures:++- The **methodology** (what we compared, how we compared, the+ exact prompt + schema both providers saw).+- The **root cause** of why early runs looked terrible.+- The **fix** that landed in the consolidator's types + prompt.+- The **final per-provider numbers** (parse rate, latency,+ manual quality assessment).+- A **how-to-reproduce** section so anyone can re-run the+ comparison against their own model + provider choices.++## What was tested++### The five fixtures++[`evals/fixtures/`](../evals/fixtures/) holds five short synthetic+session logs, each crafted to surface a *different* failure mode+in consolidation:++| Fixture | What it stresses |+|---|---|+| `01-rust-bug-fix` | Did the model split a multi-page session into the right slices (session log + concept + decision + gotcha)? |+| `02-architecture-decision` | Can the model produce an ADR-style page distinct from the running session log? |+| `03-gotcha-with-rule` | Did the model correctly classify a durable project rule with `kind: rule` so the consolidator can auto-route it to `_rules/`? |+| `04-low-signal-session` | Does the model *resist* manufacturing concept pages when there's nothing durable to capture? |+| `05-multi-topic-session` | Does the model emit *separate* pages per topic instead of mashing two unrelated topics together? |++Fixtures use real-shape `ObservationKind` values (`session-start`,+`user-prompt`, `pre-tool-use`, `post-tool-use`, `session-end`)+exactly as the production hook ingress emits them.++### The exact request++Per fixture, the runner calls+[`ai_memory_consolidate::build_batch_request(session_id, &observations)`](../crates/ai-memory-consolidate/src/consolidator.rs)+— the **same** function the live consolidator uses on every+`memory_consolidate` invocation. That request is then sent+through [`ai_memory_llm::complete_structured`](../crates/ai-memory-llm/src/lib.rs)+(also the live path). Apples-to-apples by construction.++### The three providers++| Tag | Provider | Model | Endpoint |+|---|---|---|---|+| **Kimi** | OpenRouter (openai-compat) | `moonshotai/kimi-k2.6` | `https://openrouter.ai/api/v1` |+| **Sonnet** | OpenRouter (openai-compat) | `anthropic/claude-sonnet-4.5` | `https://openrouter.ai/api/v1` |+| **qwen3** | Ollama (openai-compat) | `qwen3:32b` (Q4_K_M, ~20 GB) | `http://192.168.0.90:11434/v1` |++The home server (`192.168.0.90`) is a Ryzen AI MAX+ 395+(Strix Halo / gfx1151), 96 GB unified memory, ROCm-backed+Ollama with `OLLAMA_KEEP_ALIVE=20m` + `OLLAMA_FLASH_ATTENTION=1`++ `OLLAMA_KV_CACHE_TYPE=q8_0`. Once a model is loaded into+unified memory it stays warm for 20 min — so the first+request pays a 30–60 s cold-load tax and subsequent ones are+sub-3 s.++## Run 1 — broken prompts + schema (pre-fix baseline)++Every provider failed schema validation on every fixture:++| Fixture | Kimi | qwen3:32b |+|---|---|---|+| 01-rust-bug-fix | ❌ *response is not valid JSON* | ❌ *integer 1, expected string* |+| 02-architecture-decision | ❌ *response is not valid JSON* | ❌ *integer 2, expected string* |+| 03-gotcha-with-rule | ❌ *response is not valid JSON* | ❌ *integer 1, expected string* |+| 04-low-signal-session | ❌ *response is not valid JSON* | ❌ *integer 1, expected string* |+| 05-multi-topic-session | ❌ *response is not valid JSON* | ❌ *integer 2, expected string* |++But the *raw responses* told a very different story: both+models did **excellent** consolidation work content-wise. They+correctly identified multiple distinct pages per fixture,+extracted faithful summaries, and respected the path+conventions. The failures were **format only**:++- **Kimi** was emitting beautifully formatted markdown+ (`### Update 1` / `**path:**` / `**body:**`) — completely+ ignoring the request for JSON.+- **qwen3** was emitting clean JSON in code fences, but with+ `tier: 1` / `tier: 2` / `tier: 3` (integers) instead of the+ documented string values, and occasionally with invented+ `kind` values like `"session"` (which isn't in the+ `PageKind` enum).++## The root cause++Two separate problems, both **on our side**:++### Bug A — `Tier` had no `JsonSchema` derive++In `crates/ai-memory-consolidate/src/types.rs`:++```rust+pub struct ConsolidatedPageUpdate {+ pub path: String,+ pub tier: String, // ← bug: typed as String+ pub kind: PageKind, // ← already an enum with JsonSchema+ ...+}+```++`schemars` couldn't produce an enum constraint for `tier`+because `Tier` (the actual enum in `ai-memory-core`) didn't+have the `JsonSchema` derive. The generated schema field was+just `{ "type": "string" }` — no `enum` constraint — so models+were free to guess. Both Kimi and qwen3 guessed numeric indices.++### Bug B — prompt described values, didn't enforce them++The system prompt in+[`build_batch_request`](../crates/ai-memory-consolidate/src/consolidator.rs)+listed the valid `tier` and `kind` values in prose but never+said "use these EXACT string values, never an integer, never a+synonym, never code fences". Local instruction-tuned models —+especially when there's no `response_format: json_schema`+support to enforce — will drift to whatever feels natural.++Compounding this: openai-compat providers (Ollama, OpenRouter+passthrough) do **not** expose strict-mode JSON-schema+validation. The schema is descriptive, not coercive. So the+prompt has to do the load-bearing work.++## The fix++Three small changes landed together:++### 1. Derive `JsonSchema` on `Tier`++`crates/ai-memory-core/src/page.rs`:++```rust+#[derive(+ Clone, Copy, Debug, PartialEq, Eq, Hash,+ Serialize, Deserialize,+ schemars::JsonSchema, // ← new+)]+#[serde(rename_all = "snake_case")]+pub enum Tier { Working, Episodic, Semantic, Procedural }+```++Adds `schemars` as a dep on `ai-memory-core` (acceptable —+schemars is already a workspace dep used by every type that+crosses the LLM boundary).++### 2. Type the field as `Tier`, not `String`++`crates/ai-memory-consolidate/src/types.rs`:++```rust+pub struct ConsolidatedPageUpdate {+ pub path: String,+ pub tier: Tier, // ← was String+ pub kind: PageKind,+ ...+}+```++The generated schema now contains+`{ "enum": ["working", "episodic", "semantic", "procedural"] }`+for `tier`. `serde_json::from_value` rejects anything else.++### 3. Tighten the prompt++`build_batch_request` now spells out:++```+Set `tier` to EXACTLY ONE of these four strings — never an integer, never a synonym:+- "working" (the live in-progress slice of the session — rarely used here)+- "episodic" (per-session narrative; the sessions/<id>.md page)+- "semantic" (durable knowledge: concepts/, decisions/, gotchas/, rules)+- "procedural" (repeated patterns extracted from many episodic pages)++Set `kind` to EXACTLY ONE of these four strings — never an integer, never "session" / "concept" / "note":+- "decision" / "gotcha" / "rule" / "fact"++## Output format (read this carefully)+Reply with ONE JSON object matching the ConsolidatedBatch schema, and nothing else.+NO prose preamble, NO trailing commentary, NO markdown headers wrapping the JSON,+NO ``` code fences. The very first character of your reply must be `{` and the+very last `}`. Strings must be JSON strings (with double quotes), not numbers+and not bare identifiers.+```++Belt-and-suspenders: the schema now *rejects* the bad values,+and the prompt makes it actively hard for the model to produce+them in the first place.++## Run 2 — fixed prompts + schema++After the three fixes above, the same five fixtures produced:++### Sonnet 4.5 (OpenRouter) vs qwen3:32b (Ollama)++| Fixture | Sonnet parse | Sonnet ms | Sonnet updates | qwen3 parse | qwen3 ms | qwen3 updates |+|---|---|---|---|---|---|---|+| 01 rust-bug-fix | ✓ | 27,613 | 4 | ✓ | 110,227 | 4 |+| 02 architecture-decision | ✓ | 31,039 | 4 | ✓ | 122,200 | 5 |+| 03 gotcha-with-rule | ✓ | 19,173 | 4 | ✓ | 98,025 | 4 |+| 04 low-signal-session | ✓ | 6,106 | **1** | ✓ | 51,694 | **1** |+| 05 multi-topic-session | ✓ | 47,249 | 4 | ✗* | 133,178 | — |+| **Aggregate** | **5/5** | **avg 26 s** | — | **4/5** | **avg 103 s** | — |++*qwen3's only failure: invented `kind: "concept"` (not in the+`PageKind` enum — valid values are `decision`/`gotcha`/`rule`/+`fact`). Despite the prompt explicitly listing the valid set+and forbidding "concept", the model drifted. Otherwise the+JSON, the `tier` enum, the path conventions, and the field+names were all correct.++Both models **correctly restrained themselves** on fixture+04 (low-signal-session) and produced a single update — a+non-trivial test the original schema-broken Run 1 couldn't+even reach.++### Kimi-K2.6 (OpenRouter) vs qwen3:32b (Ollama)++_(Run in flight at the time of writing — will update with+numbers once it lands. The expectation is that Kimi's "prose+slips" issue from Run 1 is now mitigated by the explicit+"NO ``` code fences, NO prose preamble, first char must be `{`"+directives; we expect Kimi to also reach 4-5/5 parse rate.)_++## Qualitative read (Run 2)++Reading the raw `.md` outputs side-by-side reveals a+substantive style difference that the parse-rate numbers+don't capture:++- **Sonnet writes long, comprehensive entries.** A concept+ page on Docker multi-stage builds will get 3 KB of well-+ organised prose including "When to use" / "When NOT to use"+ / "Gotchas" sections — content that *wasn't in the+ observations*. The model is generating useful tutorial-style+ content, not strictly consolidating what happened.+ Sonnet's fixture 05 page invented a `Date: 2025-01-23`+ field that has no source in the observations.++- **qwen3 writes terse, faithful entries.** Each page captures+ what the session actually contained, in ~500–800 chars.+ No invented metadata, no generic tutorial filler. The same+ Docker page from qwen3 stays close to "we changed the+ Dockerfile to two-stage, image went 380→67 MB" without+ diverging into broader best-practices discussion.++For **wiki consolidation** (faithful long-term memory of+*this project*, not a knowledge graph of general best+practices), **qwen3's restraint is arguably preferable** to+Sonnet's exuberance. The point of the wiki is to record what+happened in the project, not to host re-generated tutorial+content the model already knows.++That said, when the project memory is genuinely sparse and+the model is asked to surface durable knowledge, Sonnet's+"fill in the obvious" tendency could pay off. Different+tasks → different preferences.++## Verdict++**Ollama qwen3:32b is fit for production consolidation on+this project.** With the prompt + schema fixes applied:++- **Parse reliability**: 4/5 vs Sonnet's 5/5. The one failure+ is a content-drift issue (model invented a `kind` value),+ not a structural failure — addressable with a slightly more+ forceful prompt or a lenient deserializer if it recurs.+- **Latency**: ~4× slower than Sonnet (103 s vs 26 s avg+ end-to-end). Consolidation runs at session end or on a+ schedule, not interactively, so even a 2-minute consolidation+ job is invisible to the user. For lint sweeps that touch many+ pages, this could compound — measure if it bites in practice.+- **Cost**: **$0 per consolidation**. Sonnet at ~4 KB of+ output × $15/M tok ≈ $0.06 per consolidation × N sessions/day+ × 365 days = the difference between an unbounded growth+ monthly bill and zero.+- **Content fidelity**: qwen3 is *more faithful* to source+ observations than Sonnet. Sonnet hallucinates plausible+ details (e.g. invented dates, tutorial-style sections). For+ "memory of what happened in this project", faithful >+ comprehensive.++**Recommendation: keep Ollama qwen3:32b as the default+production LLM** (where ai-memory now points). Sonnet remains+a useful escape hatch for one-off complex consolidations,+batch lint runs against very large wikis, or situations where+the home server is unreachable. Kimi sits in an awkward+middle: comparable cost-per-call to Sonnet on OpenRouter, less+output discipline.++### When to revisit++Re-run this harness when any of the following changes:++- The consolidation prompt itself is re-engineered+- A new Ollama model is pulled (e.g. when Qwen 3.5 stable+ drops for Ollama)+- A new fixture is added to `evals/fixtures/`+- The home server hardware changes+- An OpenAI / Anthropic / Voyage strict-JSON-schema feature+ becomes available through OpenRouter++## How to reproduce++### Pre-requisites++- Repo checkout + `cargo` toolchain (Rust 1.95+, as pinned in+ `rust-toolchain.toml`).+- An OpenRouter API key, exported as `OPENROUTER_API_KEY` —+ pays the Kimi + Sonnet legs.+- A reachable Ollama with `qwen3:32b` pulled. The default URL+ in the docs assumes the homelab; substitute your own.++### Run the harness++The canonical 2-side invocation (the harness compares two+providers per run):++```bash+cargo run -p ai-memory-eval --release -- \+ --baseline-provider openai-compat \+ --baseline-base-url https://openrouter.ai/api/v1 \+ --baseline-model moonshotai/kimi-k2.6 \+ --baseline-api-key-env OPENROUTER_API_KEY \+ --candidate-provider openai-compat \+ --candidate-base-url http://192.168.0.90:11434/v1 \+ --candidate-model qwen3:32b \+ --candidate-api-key ollama-local+```++For a 3-way comparison, run the harness three times pairing the+candidate (the model you're considering switching to) against+each baseline you want to compare against. Output dirs are+timestamped, so they don't collide.++### Read the output++```+evals/runs/<timestamp>/+├── baseline/+│ ├── 01-rust-bug-fix.json ← parsed structured output (if any)+│ ├── 01-rust-bug-fix.md ← flat-rendered for eyeballing+│ ├── 01-rust-bug-fix.raw.txt ← exact model output, always present+│ └── 01-rust-bug-fix.meta.json ← {elapsed_ms, parsed_ok, update_count, error}+└── candidate/+ └── ...+```++The `.raw.txt` files are the most informative artifact when a+parse fails — they show *exactly* what the model said, so you+can tell whether the failure was format (model emitted prose),+schema (model used integer enums), or substance (model+produced nothing useful).++For side-by-side reading the runner prints a hint:++```+compare with: diff -ru <run>/baseline <run>/candidate+```++### Adding new fixtures++Each fixture is a JSON file under `evals/fixtures/`:++```json+{+ "name": "human-readable-id",+ "description": "what this case is meant to surface",+ "observations": [+ {"kind": "session-start", "title": "...", "body": "..."},+ {"kind": "user-prompt", "title": "user prompt", "body": "..."},+ {"kind": "pre-tool-use", "title": "Edit", "body": "..."}+ ]+}+```++`kind` accepts any string the+[`ObservationKind`](../crates/ai-memory-core/src/observation.rs)+enum's `FromStr` understands. Anything unknown silently falls+back to `Other`.++Try to hit one of the four hard cases:++1. **Multi-page extraction** — does the model split a session+ into the right slices?+2. **Restraint** — does it avoid manufacturing pages when+ there's nothing durable?+3. **Classification** — does it correctly choose `kind: rule`+ for project rules?+4. **Topic separation** — does it produce separate pages per+ unrelated topic instead of mashing them?++## What's NOT in this harness (yet)++- **Automated quality scoring.** The runner only reports+ objective deltas (latency, parse rate, update count).+ Anything subtler (faithfulness, hallucination, scoping)+ needs a human reader.+- **Embedding A/B.** This document is LLM-only. The embedding+ provider switch (OpenAI text-embedding-3-small → Ollama+ nomic-embed-text) gets its own writeup when there's enough+ page-side data to measure retrieval quality.+- **LLM-as-judge scoring.** Adding a third "judge" model to+ score the candidate outputs against a rubric would+ automate quality measurement. Not built; the next layer up+ if this harness gets used regularly.++## Future work++If we end up running this harness routinely:++1. Add a third position (`--judge-*`) so a separate "judge"+ model can score baseline vs candidate per fixture against a+ rubric, producing a numeric quality delta.+2. Extend fixtures with a `must_mention` / `must_not_mention`+ keyword list so we can compute simple keyword recall+ automatically (catches obvious hallucinations / missing+ facts).+3. Parallel embedding-retrieval eval: a probe set of queries+ each tagged with the expected target wiki page; compute+ recall@5 + MRR for two embedding models against the same+ indexed corpus.+4. Persist a leaderboard somewhere durable (a wiki page,+ ironically) so we don't lose track of which model performed+ best on which fixture across runs.diff --git a/evals/src/main.rs b/evals/src/main.rsindex c618772f..7f167109 100644--- a/evals/src/main.rs+++ b/evals/src/main.rs@@ -39,6 +39,7 @@//! --candidate-api-key ollama-local//! ```+use std::io::Write as _;use std::path::{Path, PathBuf};use std::sync::Arc;use std::time::{Duration, Instant};@@ -51,9 +52,7 @@ use serde::{Deserialize, Serialize};use ai_memory_consolidate::{ConsolidatedBatch, build_batch_request};use ai_memory_core::{Observation, ObservationKind, ProjectId, SessionId, WorkspaceId};-use ai_memory_llm::{- LlmProvider, ProviderChoice, ProviderConfig, complete_structured,-};+use ai_memory_llm::{LlmProvider, ProviderChoice, ProviderConfig};#[derive(Parser, Debug)]#[command(@@ -157,10 +156,10 @@ async fn main() -> Result<()> {let baseline = resolve_provider("baseline", &args)?;let candidate = resolve_provider("candidate", &args)?;- let baseline_provider = make_provider(baseline.clone())- .context("building baseline provider")?;- let candidate_provider = make_provider(candidate.clone())- .context("building candidate provider")?;+ let baseline_provider =+ make_provider(baseline.clone()).context("building baseline provider")?;+ let candidate_provider =+ make_provider(candidate.clone()).context("building candidate provider")?;let fixtures = load_fixtures(&args.fixtures).with_context(|| format!("loading fixtures from {}", args.fixtures.display()))?;@@ -180,9 +179,23 @@ async fn main() -> Result<()> {println!();println!("ai-memory eval — {stamp}");- println!(" fixtures : {} ({})", fixtures.len(), args.fixtures.display());- println!(" baseline : {} {} @ {}", baseline.provider_name(), baseline.model, baseline.base_url.as_deref().unwrap_or("(default)"));- println!(" candidate: {} {} @ {}", candidate.provider_name(), candidate.model, candidate.base_url.as_deref().unwrap_or("(default)"));+ println!(+ " fixtures : {} ({})",+ fixtures.len(),+ args.fixtures.display()+ );+ println!(+ " baseline : {} {} @ {}",+ baseline.provider_name(),+ baseline.model,+ baseline.base_url.as_deref().unwrap_or("(default)")+ );+ println!(+ " candidate: {} {} @ {}",+ candidate.provider_name(),+ candidate.model,+ candidate.base_url.as_deref().unwrap_or("(default)")+ );println!(" out : {}", run_dir.display());println!();@@ -216,6 +229,10 @@ async fn main() -> Result<()> {b.parsed_ok,b.update_count.map_or("-".into(), |n| n.to_string()));+ // Force-flush per fixture so progress is visible when stdout+ // is piped (e.g. via `tee` or backgrounded runs). Without+ // this, full-buffering swallows every println! until exit.+ std::io::stdout().flush().ok();results.push((a, b));}@@ -227,11 +244,21 @@ async fn main() -> Result<()> {let candidate_total: u128 = results.iter().map(|(_, b)| b.elapsed_ms).sum();let baseline_ok = results.iter().filter(|(a, _)| a.parsed_ok).count();let candidate_ok = results.iter().filter(|(_, b)| b.parsed_ok).count();- println!("baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms", baseline_total / n.max(1) as u128);- println!("candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms", candidate_total / n.max(1) as u128);+ println!(+ "baseline : {baseline_ok}/{n} parsed ok total {baseline_total} ms avg {} ms",+ baseline_total / n.max(1) as u128+ );+ println!(+ "candidate: {candidate_ok}/{n} parsed ok total {candidate_total} ms avg {} ms",+ candidate_total / n.max(1) as u128+ );println!();println!("outputs under: {}", run_dir.display());- println!("compare with: diff -ru {}/baseline {}/candidate", run_dir.display(), run_dir.display());+ println!(+ "compare with: diff -ru {}/baseline {}/candidate",+ run_dir.display(),+ run_dir.display()+ );Ok(())}@@ -259,9 +286,7 @@ fn resolve_provider(side: &str, args: &Args) -> Result<ResolvedConfig> {"anthropic" => ProviderChoice::Anthropic,"openai" => ProviderChoice::OpenAi,"openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,- other => bail!(- "{side}: provider {other} not one of anthropic|openai|openai-compat"- ),+ other => bail!("{side}: provider {other} not one of anthropic|openai|openai-compat"),};let api_key = match (api_key_inline, api_key_env) {@@ -337,10 +362,9 @@ fn load_fixtures(dir: &Path) -> Result<Vec<(PathBuf, Fixture)>> {paths.sort();let mut out = Vec::with_capacity(paths.len());for p in paths {- let bytes = std::fs::read(&p)- .with_context(|| format!("reading {}", p.display()))?;- let fx: Fixture = serde_json::from_slice(&bytes)- .with_context(|| format!("parsing {}", p.display()))?;+ let bytes = std::fs::read(&p).with_context(|| format!("reading {}", p.display()))?;+ let fx: Fixture =+ serde_json::from_slice(&bytes).with_context(|| format!("parsing {}", p.display()))?;out.push((p, fx));}Ok(out)@@ -371,8 +395,14 @@ fn synthesise_observations(fx: &Fixture) -> Vec<Observation> {struct ProviderResult {elapsed: Duration,+ /// Raw text the model emitted — always saved, even when parsing+ /// fails. This is the load-bearing artifact for debugging+ /// schema mismatches: you can see exactly what the model said.+ raw_text: String,+ /// Parsed-from-text JSON, if the response was valid JSON at all.+ raw_json: Option<serde_json::Value>,+ /// Schema-validated batch, if the JSON also satisfied the schema.batch: Option<ConsolidatedBatch>,- raw: Option<serde_json::Value>,error: Option<String>,}@@ -382,32 +412,98 @@ async fn run_one(request: ai_memory_llm::ChatRequest,) -> ProviderResult {let start = Instant::now();- // Try the schema-validated path first; fall back to the raw- // structured call so we can still inspect malformed output.- let batch_res: Result<ConsolidatedBatch, _> =- complete_structured(&*provider, request.clone()).await;+ // Call `complete` directly — bypassing `complete_structured` —+ // so we can capture the raw text the model emitted, even if it+ // doesn't deserialise. The cost is that we re-implement the+ // "extract first balanced {…}" fallback locally, but knowing+ // exactly what the model said is what makes the harness useful.+ let chat_res = provider.complete(request).await;let elapsed = start.elapsed();- match batch_res {- Ok(batch) => {- let raw = serde_json::to_value(&batch).ok();- tracing::info!(side, ms = elapsed.as_millis() as u64, "parsed ok");- ProviderResult {- elapsed,- batch: Some(batch),- raw,- error: None,+ let raw_text = match &chat_res {+ Ok(r) => r.text.clone(),+ Err(e) => format!("(provider error before any response: {e})"),+ };+ let (raw_json, batch, error) = match chat_res {+ Ok(_) => parse_response(&raw_text),+ Err(e) => (None, None, Some(e.to_string())),+ };+ if let Some(err) = &error {+ tracing::warn!(side, error = %err, "model output did not satisfy schema");+ } else {+ tracing::info!(+ side,+ ms = elapsed.as_millis() as u64,+ updates = batch.as_ref().map_or(0, |b| b.updates.len()),+ "parsed ok"+ );+ }+ ProviderResult {+ elapsed,+ raw_text,+ raw_json,+ batch,+ error,+ }+}++/// Parse a model response into (json, batch, error).+///+/// 1. Try the whole response as JSON.+/// 2. If that fails, search for the first balanced `{…}` block.+/// 3. If we have JSON, try to deserialise as `ConsolidatedBatch`.+fn parse_response(+ text: &str,+) -> (+ Option<serde_json::Value>,+ Option<ConsolidatedBatch>,+ Option<String>,+) {+ let json = serde_json::from_str::<serde_json::Value>(text.trim())+ .ok()+ .or_else(|| {+ extract_first_balanced_object(text).and_then(|s| serde_json::from_str(&s).ok())+ });+ let Some(json) = json else {+ return (None, None, Some("response is not valid JSON".into()));+ };+ match serde_json::from_value::<ConsolidatedBatch>(json.clone()) {+ Ok(batch) => (Some(json), Some(batch), None),+ Err(e) => (Some(json), None, Some(format!("schema mismatch: {e}"))),+ }+}++/// Best-effort `{…}` extractor — ignores braces inside strings.+fn extract_first_balanced_object(text: &str) -> Option<String> {+ let bytes = text.as_bytes();+ let start = bytes.iter().position(|&b| b == b'{')?;+ let mut depth = 0i32;+ let mut in_string = false;+ let mut escape = false;+ for (i, &b) in bytes.iter().enumerate().skip(start) {+ if in_string {+ if escape {+ escape = false;+ } else if b == b'\\' {+ escape = true;+ } else if b == b'"' {+ in_string = false;}+ continue;+ }+ if b == b'"' {+ in_string = true;+ continue;}- Err(e) => {- tracing::warn!(side, error = %e, "schema parse failed");- ProviderResult {- elapsed,- batch: None,- raw: None,- error: Some(e.to_string()),+ if b == b'{' {+ depth += 1;+ } else if b == b'}' {+ depth -= 1;+ if depth == 0 {+ return Some(text[start..=i].to_string());}}}+ None}fn persist_case(@@ -424,18 +520,24 @@ fn persist_case(let dir = run_dir.join(side);std::fs::create_dir_all(&dir)?;- // 1. Raw JSON.- let json_path = dir.join(format!("{safe_name}.json"));- if let Some(raw) = &res.raw {- std::fs::write(&json_path, serde_json::to_string_pretty(raw)?)?;+ // 1. Raw model text — always saved, even on parse failure.+ // This is the load-bearing debug artifact: a side-by-side+ // diff of these files tells you exactly what each model said.+ let raw_path = dir.join(format!("{safe_name}.raw.txt"));+ std::fs::write(&raw_path, &res.raw_text)?;++ // 2. Parsed JSON (if response was valid JSON at all).+ if let Some(json) = &res.raw_json {+ let json_path = dir.join(format!("{safe_name}.json"));+ std::fs::write(&json_path, serde_json::to_string_pretty(json)?)?;}- // 2. Flattened markdown rendering for easy eyeballing / diff.+ // 3. Flattened markdown rendering for easy eyeballing.let md_path = dir.join(format!("{safe_name}.md"));let md = render_markdown(side, fixture_name, &res);std::fs::write(&md_path, md)?;- // 3. Per-case meta.+ // 4. Per-case meta.let case = CaseRun {provider: side.to_string(),fixture: fixture_name.to_string(),@@ -454,29 +556,35 @@ fn render_markdown(side: &str, fixture_name: &str, res: &ProviderResult) -> Striout.push_str(&format!("# {side} — {fixture_name}\n\n"));out.push_str(&format!("- elapsed: {} ms\n", res.elapsed.as_millis()));if let Some(err) = &res.error {- out.push_str(&format!("- error: `{err}`\n\n"));+ out.push_str(&format!("- error: `{err}`\n"));+ }+ // Always include the raw response so the human reviewer can+ // judge structure/style even when schema validation failed.+ if !res.raw_text.is_empty() {+ out.push_str("\n## Raw response\n\n```\n");+ out.push_str(&res.raw_text);+ out.push_str("\n```\n");}let Some(batch) = &res.batch else {- out.push_str("\n_(no batch — see error above; .json file absent)_\n");return out;};- out.push_str(&format!("- updates: {}\n\n", batch.updates.len()));+ out.push_str(&format!("\n- updates: {}\n", batch.updates.len()));if !batch.rationale.trim().is_empty() {- out.push_str("## Rationale\n\n");+ out.push_str("\n## Rationale\n\n");out.push_str(batch.rationale.trim());- out.push_str("\n\n");+ out.push('\n');}for (i, u) in batch.updates.iter().enumerate() {- out.push_str(&format!("## Update {} — `{}`\n\n", i + 1, u.path));+ out.push_str(&format!("\n## Update {} — `{}`\n\n", i + 1, u.path));out.push_str(&format!("**Title:** {} \n", u.title));out.push_str(&format!("**Kind:** `{}` \n", u.kind.as_str()));- out.push_str(&format!("**Tier:** `{}` \n", u.tier));+ out.push_str(&format!("**Tier:** `{}` \n", u.tier.as_str()));if !u.tags.is_empty() {out.push_str(&format!("**Tags:** {} \n", u.tags.join(", ")));}out.push('\n');out.push_str(u.body_markdown.trim());- out.push_str("\n\n");+ out.push('\n');}out}