Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(auto-improve): run the eval gate's structural checks in process
AkitaOnRailscf705ec1c8b4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e618d1ab..6481795e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -13,6 +13,24 @@ version in [`docs/changelog/`](docs/changelog/):### Added+- `[auto_improve.eval] builtin = "structure"` runs the eval gate's structural+ checks in process, so the gate needs no interpreter, no scorer script inside+ the container and no timeout to hit. It checks an H1, an upper-case+ `TODO`/`TBD` placeholder (a hard failure), a `##` section and a list of steps+ for `procedures/`, and text beyond the heading within 2,400 characters for+ `_rules/` — never a section name or a wording, so the wiki's language does+ not matter. `eval_results` records `checker_name = "builtin:structure"`.+ `builtin` and `command` are mutually exclusive; `command` is unchanged for+ project-specific scorers (#1638).++- `ai-memory llm-test --server` probes the server's own configured provider+ instead of building one from the caller's environment, through a new+ root-only `POST /admin/llm-test`. The retry hint that `status` and the health+ panel print now names it: with the server in a container or on another host+ its keys exist only there, so the old client-side hint failed with "no API+ key" or a provider `400` while the server was healthy. The probe runs one+ fixed short prompt, reports provider, model, latency and the error class, and+ scrubs the provider's message before printing it (#1639).- Added `[handoff] deliver_on_session_start = false` to `.ai-memory.toml`. Aproject that sets it no longer gets its handoff, or the continuity notice, atsession start; the handoff is still created when the session ends and staysdiff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex 07d81786..e1ed73e6 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -2957,6 +2957,7 @@ fn auto_improve_review_config_from_settings(eval: ai_memory_consolidate::AutoImproveEvalConfig {enabled: settings.eval.enabled,command: settings.eval.command.clone(),+ builtin: settings.eval.builtin.trim().to_owned(),timeout_secs: settings.eval.timeout_secs,targets: settings.eval.targets.clone(),min_delta: settings.eval.min_delta,diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rsindex 4a3775c1..784c7edf 100644--- a/crates/ai-memory-cli/src/config.rs+++ b/crates/ai-memory-cli/src/config.rs@@ -1709,6 +1709,9 @@ pub struct AutoImproveEvalSettings {pub enabled: bool,/// Executable command plus whitespace-separated args. Executed directly, not through a shell.pub command: String,+ /// Built-in scorer, run in-process. `structure` is the only one; it is+ /// mutually exclusive with `command` (#1638).+ pub builtin: String,/// Timeout per proposal eval command.pub timeout_secs: u64,/// Wiki path prefixes that require eval when enabled.@@ -1722,6 +1725,7 @@ impl Default for AutoImproveEvalSettings {Self {enabled: false,command: String::new(),+ builtin: String::new(),timeout_secs: 120,targets: ai_memory_consolidate::default_auto_improve_eval_targets(),min_delta: 0.0,@@ -2571,6 +2575,24 @@ impl Config {).map_err(|message| anyhow::anyhow!(message))?,);+ // One scorer per gate: an operator who set both meant one of them, and+ // silently ignoring the builtin (as the gate does) would leave a config+ // that reads as if it ran both (#1638).+ let eval = &config.auto_improve.eval;+ if !eval.builtin.trim().is_empty() && !eval.command.trim().is_empty() {+ anyhow::bail!(+ "auto_improve.eval.builtin and auto_improve.eval.command are mutually exclusive; \+ remove one (builtin `structure` runs in-process, a command runs your own scorer)"+ );+ }+ let builtin = eval.builtin.trim();+ if !builtin.is_empty() && builtin != "structure" {+ anyhow::bail!(+ "auto_improve.eval.builtin `{builtin}` is not a known scorer; the only one is \+ `structure`"+ );+ }+// A4 entropy filter thresholds: reject an unusable threshold at startup// rather than silently ignoring it on the first experience pass.if let Err(message) = config@@ -5849,6 +5871,45 @@ input_token_safety_margin = {margin}assert_eq!(cfg.auto_improve.pending_path, "_pending/review-bot");}+ /// One scorer per gate: setting both is a startup error, not a config+ /// where the builtin silently loses (#1638).+ #[test]+ fn an_eval_gate_rejects_both_a_builtin_and_a_command() {+ let tmp = tempfile::TempDir::new().unwrap();+ let write = |toml: &str| {+ let path = tmp.path().join("config.toml");+ std::fs::write(&path, toml).unwrap();+ Config::load(Some(&path), Some(tmp.path().to_path_buf()))+ };+ let both = write(+ "[auto_improve.eval]\nenabled = true\nbuiltin = \"structure\"\n\+ command = \"sh score.sh\"\n",+ )+ .unwrap_err()+ .to_string();+ assert!(+ both.contains("mutually exclusive"),+ "unexpected error: {both}"+ );++ let unknown = write("[auto_improve.eval]\nenabled = true\nbuiltin = \"vibes\"\n")+ .unwrap_err()+ .to_string();+ assert!(+ unknown.contains("not a known scorer"),+ "unexpected error: {unknown}"+ );++ let builtin =+ write("[auto_improve.eval]\nenabled = true\nbuiltin = \"structure\"\n").unwrap();+ assert_eq!(builtin.auto_improve.eval.builtin, "structure");+ assert!(builtin.auto_improve.eval.command.is_empty());++ let command =+ write("[auto_improve.eval]\nenabled = true\ncommand = \"sh score.sh\"\n").unwrap();+ assert!(command.auto_improve.eval.builtin.is_empty());+ }+#[test]fn profile_defaults_are_auto_and_parse_from_file_and_env() {use ai_memory_core::profile::{EffectiveProfileShare, ProfileEnabled, ProfileShare};diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rsindex 4e3bdfcd..99e564aa 100644--- a/crates/ai-memory-consolidate/src/auto_improve.rs+++ b/crates/ai-memory-consolidate/src/auto_improve.rs@@ -120,6 +120,13 @@ pub struct AutoImproveEvalConfig {pub enabled: bool,/// Executable command plus whitespace-separated args. Executed directly, not via a shell.pub command: String,+ /// Built-in scorer to run in-process instead of `command` (#1638).+ ///+ /// Only `structure` exists: an H1, no `TODO`/`TBD` placeholder, a `##`+ /// section and a list of steps for `procedures/`, and text beyond the+ /// heading within 2,400 characters for `_rules/`. It starts no process, so+ /// it needs no interpreter in the image and cannot time out.+ pub builtin: String,/// Timeout for one proposal evaluation.pub timeout_secs: u64,/// Wiki path prefixes that require eval when enabled.@@ -133,6 +140,7 @@ impl Default for AutoImproveEvalConfig {Self {enabled: false,command: String::new(),+ builtin: String::new(),timeout_secs: 120,targets: default_auto_improve_eval_targets(),min_delta: 0.0,@@ -917,11 +925,22 @@ async fn apply_eval_gate_with_before_bodies(if !eval.enabled {return Vec::new();}- if eval.command.trim().is_empty() {+ let builtin = eval.builtin.trim();+ if !builtin.is_empty() && !eval.command.trim().is_empty() {warnings.push(- "auto-improve eval gate enabled without a command; targeted proposals rejected".into(),+ "auto-improve eval gate has both a builtin scorer and a command; the command wins \+ and the builtin is ignored — set one of them"+ .into(),);}+ if eval.command.trim().is_empty() && builtin.is_empty() {+ warnings.push(+ "auto-improve eval gate enabled without a command or builtin scorer; targeted \+ proposals rejected"+ .into(),+ );+ }+ let use_builtin = eval.command.trim().is_empty() && !builtin.is_empty();let mut accepted = Vec::with_capacity(proposals.len());let mut results = Vec::new();@@ -934,11 +953,22 @@ async fn apply_eval_gate_with_before_bodies(let stdin = serde_json::to_vec(&eval_request(&proposal, before_body)).map_err(|e| EvalRunError::Error(e.to_string()));let program = eval.command.split_whitespace().next().unwrap_or("");+ let (checker_name, checker_digest) = if use_builtin {+ (+ sanitize_eval_reason(&format!("builtin:{builtin}")),+ sha256_hex(format!("builtin:{builtin}:{}", env!("CARGO_PKG_VERSION")).as_bytes()),+ )+ } else {+ (+ sanitize_eval_reason(program.rsplit(['/', '\\']).next().unwrap_or("")),+ sha256_hex(eval.command.as_bytes()),+ )+ };let mut result = AutoImproveEvalResult {eval_id: uuid::Uuid::now_v7(),target_path: proposal.path.clone(),- checker_name: sanitize_eval_reason(program.rsplit(['/', '\\']).next().unwrap_or("")),- checker_invocation_digest: sha256_hex(eval.command.as_bytes()),+ checker_name,+ checker_invocation_digest: checker_digest,proposal_sha256: stdin.as_ref().ok().map(|bytes| sha256_hex(bytes)),before_body_sha256: sha256_hex(before_body.as_bytes()),after_body_sha256: sha256_hex(proposal.body_markdown.as_bytes()),@@ -949,9 +979,18 @@ async fn apply_eval_gate_with_before_bodies(score_after: None,reason: None,};- let outcome = match stdin {- Ok(stdin) => run_eval_for_proposal(eval, stdin).await,- Err(error) => Err(error),+ let outcome = if use_builtin {+ builtin_eval_outcome(+ builtin,+ &proposal.path,+ before_body,+ &proposal.body_markdown,+ )+ } else {+ match stdin {+ Ok(stdin) => run_eval_for_proposal(eval, stdin).await,+ Err(error) => Err(error),+ }};match outcome {Ok(outcome) => {@@ -1009,6 +1048,136 @@ fn eval_targets_path(eval: &AutoImproveEvalConfig, path: &str) -> bool {})}+/// Score one proposal in process for `--builtin` (#1638).+///+/// The checks are the ones `docs/examples/auto-improve-eval` applies, ported+/// so the gate works with no interpreter and no process: an H1, no upper-case+/// `TODO`/`TBD`, a `##` section and a list of steps for a procedure, and text+/// beyond the heading within 2,400 characters for a rule. None of them match a+/// section name or a wording, so a wiki written in any language scores the+/// same way.+fn builtin_eval_outcome(+ builtin: &str,+ path: &str,+ before_body: &str,+ after_body: &str,+) -> Result<AutoImproveEvalResponse, EvalRunError> {+ if builtin != "structure" {+ return Err(EvalRunError::Error(format!(+ "unknown builtin eval scorer `{builtin}`; the only one is `structure`"+ )));+ }+ let (score_before, _) = builtin_structure_score(path, before_body);+ let (score_after, mut reasons) = builtin_structure_score(path, after_body);+ // A placeholder is a hard failure, not a 0.20 deduction: the weights alone+ // let a well-structured page carrying `TODO: complete` reach exactly 0.70+ // and pass, which is the case the check exists to stop.+ let placeholder = builtin_has_placeholder(after_body);+ let passed = !placeholder && score_after >= 0.70 && score_after >= score_before;+ if placeholder && !reasons.iter().any(|r| r.contains("TODO")) {+ reasons.push("contains TODO/TBD placeholder".to_owned());+ }+ Ok(AutoImproveEvalResponse {+ passed,+ score_before: Some(score_before),+ score_after: Some(score_after),+ reason: (!passed).then(|| {+ if reasons.is_empty() {+ "score did not improve".to_owned()+ } else {+ reasons.join("; ")+ }+ }),+ })+}++/// Whether `body` carries an upper-case `TODO`/`TBD` placeholder.+///+/// Matched as a whole word, so the Portuguese word "todo" and a `TODO` inside+/// a longer identifier are not placeholders.+fn builtin_has_placeholder(body: &str) -> bool {+ body.split(|c: char| !c.is_alphanumeric() && c != '_')+ .any(|word| word == "TODO" || word == "TBD")+}++/// (`score`, reasons) for one body, the same weights as the shipped example+/// scorers.+fn builtin_structure_score(path: &str, body: &str) -> (f64, Vec<String>) {+ // A page written on Windows arrives with CRLF endings; score it exactly as+ // its LF twin instead of counting the carriage return as content.+ let body = body.replace("\r\n", "\n").replace('\r', "\n");+ let mut score: f64 = 0.0;+ let mut reasons = Vec::new();+ let lines: Vec<&str> = body.split('\n').collect();+ let has_h1 = lines.iter().any(|line| {+ line.strip_prefix("# ")+ .is_some_and(|rest| !rest.trim().is_empty())+ });+ let has_h2 = lines.iter().any(|line| {+ line.strip_prefix("## ")+ .is_some_and(|rest| !rest.trim().is_empty())+ });+ let has_item = lines.iter().any(|line| {+ let trimmed = line.trim_start();+ let rest = trimmed+ .strip_prefix("- ")+ .or_else(|| trimmed.strip_prefix("* "))+ .or_else(|| trimmed.strip_prefix("+ "))+ .or_else(|| {+ let digits = trimmed.len()+ - trimmed+ .trim_start_matches(|c: char| c.is_ascii_digit())+ .len();+ trimmed[digits..]+ .strip_prefix(". ")+ .or_else(|| trimmed[digits..].strip_prefix(") "))+ .filter(|_| digits > 0)+ });+ rest.is_some_and(|rest| !rest.trim().is_empty())+ });+ let placeholder = builtin_has_placeholder(&body);+ let prose_beyond_heading = body+ .lines()+ .any(|line| !line.trim().is_empty() && !line.trim_start().starts_with('#'));++ if has_h1 {+ score += 0.25;+ } else {+ reasons.push("missing H1 heading".to_owned());+ }+ if placeholder {+ reasons.push("contains TODO/TBD placeholder".to_owned());+ } else {+ score += 0.20;+ }+ if path.starts_with("procedures/") {+ if has_h2 {+ score += 0.20;+ } else {+ reasons.push("procedure has no ## section".to_owned());+ }+ if has_item {+ score += 0.25;+ } else {+ reasons.push("procedure has no list of steps".to_owned());+ }+ } else if path.starts_with("_rules/") {+ if prose_beyond_heading {+ score += 0.35;+ } else {+ reasons.push("rule has no text beyond its heading".to_owned());+ }+ if body.trim_matches([' ', '\t', '\n', '\r']).chars().count() <= 2_400 {+ score += 0.10;+ } else {+ reasons.push("rule is too long for an agent instruction".to_owned());+ }+ } else {+ score += 0.45;+ }+ (score.min(1.0), reasons)+}+#[derive(Debug)]enum EvalRunError {Timeout,@@ -2564,6 +2733,7 @@ mod tests {AutoImproveEvalConfig {enabled: true,command,+ builtin: String::new(),// Windows PowerShell cold-start is slower than 2s, and these gate// tests run in parallel — timeouts here are for the eval command// itself, not for interpreter startup.@@ -2653,6 +2823,130 @@ mod tests {assert!(warnings.is_empty());}+ /// The built-in scorer needs no process, no interpreter and no timeout:+ /// the gate answers in-process for the same structural checks the shipped+ /// example scorers apply (#1638).+ #[tokio::test]+ async fn builtin_structure_scores_in_process_without_a_command() {+ let cfg = AutoImproveEvalConfig {+ enabled: true,+ builtin: "structure".into(),+ min_delta: 0.0,+ ..AutoImproveEvalConfig::default()+ };+ let mut good = proposal("procedures/release.md", "procedure", 0.9);+ good.body_markdown = "# Release\n\n## Steps\n- Run the tests.\n- Publish.\n".into();+ let mut bad = proposal("procedures/release.md", "procedure", 0.9);+ bad.body_markdown = "# Release\n\n## Steps\nRun the tests and publish.\n".into();+ let mut proposals = vec![good, bad];+ let mut rejected = Vec::new();+ let mut warnings = Vec::new();+ let results = apply_eval_gate_with_before_bodies(+ &cfg,+ &mut proposals,+ &mut rejected,+ &mut warnings,+ &BTreeMap::new(),+ )+ .await;++ assert!(warnings.is_empty(), "{warnings:?}");+ assert_eq!(proposals.len(), 1, "only the page with steps survives");+ assert_eq!(rejected.len(), 1);+ assert_eq!(rejected[0].reason, "eval_gate_failed");+ assert_eq!(results.len(), 2);+ assert_eq!(results[0].checker_name, "builtin:structure");+ assert_eq!(results[0].status, EvalStatus::Success);+ assert_eq!(results[1].status, EvalStatus::Rejected);+ let digest = &results[0].checker_invocation_digest;+ assert_eq!(+ results[1].checker_invocation_digest, *digest,+ "the invocation digest covers the builtin name, not a command string"+ );+ }++ /// The same structural checks, in any language: a Portuguese or Japanese+ /// page is not rejected for its wording, and a `todo` word in prose is not+ /// a placeholder.+ #[test]+ fn builtin_structure_is_language_neutral_and_ignores_lowercase_todo() {+ let portuguese = "# Procedimento\n\n## Passos\n- Rodar os testes.\n- Publicar.\n";+ let japanese = "# リリース手順\n\n## 手順\n1. テストを実行する\n2) デプロイする\n";+ let rule = "# Regra\n\nSempre rode a suíte completa antes de publicar; o todo da equipe \nrevisa depois.\n";+ for (path, body) in [+ ("procedures/publicacao.md", portuguese),+ ("procedures/release.md", japanese),+ ("_rules/testes.md", rule),+ ] {+ let (score, reasons) = builtin_structure_score(path, body);+ assert!(score >= 0.70, "{path}: {score} {reasons:?}");+ }+ // A structurally complete page carrying a placeholder still fails: the+ // weights alone would let it reach exactly 0.70.+ let outcome = builtin_eval_outcome(+ "structure",+ "procedures/x.md",+ "",+ "# P\n\n## S\n- TODO: complete\n",+ )+ .unwrap();+ assert!(!outcome.passed, "{outcome:?}");+ assert!(+ outcome+ .reason+ .as_deref()+ .unwrap_or_default()+ .contains("TODO"),+ "{outcome:?}"+ );+ }++ /// CRLF is the same page: a Windows checkout must not pass a rule with no+ /// body, or fail a procedure whose step list is intact.+ #[test]+ fn builtin_structure_scores_a_crlf_page_like_its_lf_twin() {+ let cases = [+ ("_rules/a.md", "# Regra\n\nTexto.\n"),+ ("_rules/b.md", "# Regra\n"),+ ("procedures/c.md", "# P\n\n## S\n- passo\n"),+ ("procedures/d.md", "# P\n\n## S\n- \n"),+ ];+ for (path, body) in cases {+ let lf = builtin_structure_score(path, body);+ let crlf = builtin_structure_score(path, &body.replace('\n', "\r\n"));+ assert_eq!(lf.0, crlf.0, "{path}");+ assert_eq!(lf.1, crlf.1, "{path}");+ }+ assert!(+ builtin_structure_score("_rules/b.md", "# Regra\r\n").0 < 0.70,+ "the carriage return is not body text"+ );+ }++ /// An unusable scorer name fails the proposal rather than silently+ /// accepting it.+ #[tokio::test]+ async fn an_unknown_builtin_scorer_fails_closed() {+ let cfg = AutoImproveEvalConfig {+ enabled: true,+ builtin: "vibes".into(),+ ..AutoImproveEvalConfig::default()+ };+ let mut proposals = vec![proposal("procedures/x.md", "procedure", 0.9)];+ let mut rejected = Vec::new();+ let mut warnings = Vec::new();+ apply_eval_gate_with_before_bodies(+ &cfg,+ &mut proposals,+ &mut rejected,+ &mut warnings,+ &BTreeMap::new(),+ )+ .await;+ assert!(proposals.is_empty());+ assert_eq!(rejected[0].reason, "eval_gate_error");+ }+#[tokio::test]async fn passing_eval_allows_proposal_and_records_evidence() {let (_eval_dir, script) = write_eval_script(diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex e826d0f6..387e9407 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -12348,6 +12348,7 @@ mod tests {state.auto_improve_review_config.eval = ai_memory_consolidate::AutoImproveEvalConfig {enabled: true,command: String::new(),+ builtin: String::new(),timeout_secs: 1,targets: vec!["notes".into()],min_delta: 0.0,diff --git a/docs/auto-improve-eval-gates.md b/docs/auto-improve-eval-gates.mdindex e2da5c85..d24e3638 100644--- a/docs/auto-improve-eval-gates.md+++ b/docs/auto-improve-eval-gates.md@@ -19,6 +19,38 @@ The command is split on whitespace and executed directly, not through a shell.Use a wrapper script if you need quoting, environment setup, or multiplecommands.+### The built-in scorer++Writing, mounting and maintaining a scorer is optional: `builtin = "structure"`+runs the checks in process, so the gate needs no interpreter, no script inside+the container, and no timeout.++```toml+[auto_improve.eval]+enabled = true+builtin = "structure"+targets = ["_rules", "procedures"]+min_delta = 0.0+```++It checks, never matching a section name or a wording (so a wiki in any+language scores the same):++- an H1 heading;+- no upper-case `TODO`/`TBD` placeholder — a hard failure, not a deduction; the+ Portuguese word "todo" in prose is not a placeholder;+- for `procedures/`, a `##` section and a list of steps;+- for `_rules/`, text beyond the heading, within 2,400 characters.++A proposal passes at `score_after >= 0.70` and no lower than `score_before`.+`eval_results` records `checker_name = "builtin:structure"` and an invocation+digest over the scorer name and the binary version, instead of a command+string.++`builtin` and `command` are mutually exclusive: setting both is a startup+error rather than one being silently ignored. `command` keeps working+unchanged for project-specific scorers.+The eval policy is owned by the server process. `ai-memory auto-improve` anddirect HTTP requests cannot replace the configured command, targets, timeout,or score threshold. When a CLI talks to a remote server, configure