Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(auto-improve): run the eval gate's structural checks in process

AkitaOnRailscf705ec1c8b4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e618d1ab..6481795e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,24 @@ version in [`docs/changelog/`](docs/changelog/):
### Added
+- `[auto_improve.eval] builtin = "structure"` runs the eval gate's structural
+ checks in process, so the gate needs no interpreter, no scorer script inside
+ the container and no timeout to hit. It checks an H1, an upper-case
+ `TODO`/`TBD` placeholder (a hard failure), a `##` section and a list of steps
+ for `procedures/`, and text beyond the heading within 2,400 characters for
+ `_rules/` — never a section name or a wording, so the wiki's language does
+ not matter. `eval_results` records `checker_name = "builtin:structure"`.
+ `builtin` and `command` are mutually exclusive; `command` is unchanged for
+ project-specific scorers (#1638).
+
+- `ai-memory llm-test --server` probes the server's own configured provider
+ instead of building one from the caller's environment, through a new
+ root-only `POST /admin/llm-test`. The retry hint that `status` and the health
+ panel print now names it: with the server in a container or on another host
+ its keys exist only there, so the old client-side hint failed with "no API
+ key" or a provider `400` while the server was healthy. The probe runs one
+ fixed short prompt, reports provider, model, latency and the error class, and
+ scrubs the provider's message before printing it (#1639).
- Added `[handoff] deliver_on_session_start = false` to `.ai-memory.toml`. A
project that sets it no longer gets its handoff, or the continuity notice, at
session start; the handoff is still created when the session ends and stays
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index 07d81786..e1ed73e6 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -2957,6 +2957,7 @@ fn auto_improve_review_config_from_settings(
eval: ai_memory_consolidate::AutoImproveEvalConfig {
enabled: settings.eval.enabled,
command: settings.eval.command.clone(),
+ builtin: settings.eval.builtin.trim().to_owned(),
timeout_secs: settings.eval.timeout_secs,
targets: settings.eval.targets.clone(),
min_delta: settings.eval.min_delta,
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index 4a3775c1..784c7edf 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -1709,6 +1709,9 @@ pub struct AutoImproveEvalSettings {
pub enabled: bool,
/// Executable command plus whitespace-separated args. Executed directly, not through a shell.
pub command: String,
+ /// Built-in scorer, run in-process. `structure` is the only one; it is
+ /// mutually exclusive with `command` (#1638).
+ pub builtin: String,
/// Timeout per proposal eval command.
pub timeout_secs: u64,
/// Wiki path prefixes that require eval when enabled.
@@ -1722,6 +1725,7 @@ impl Default for AutoImproveEvalSettings {
Self {
enabled: false,
command: String::new(),
+ builtin: String::new(),
timeout_secs: 120,
targets: ai_memory_consolidate::default_auto_improve_eval_targets(),
min_delta: 0.0,
@@ -2571,6 +2575,24 @@ impl Config {
)
.map_err(|message| anyhow::anyhow!(message))?,
);
+ // One scorer per gate: an operator who set both meant one of them, and
+ // silently ignoring the builtin (as the gate does) would leave a config
+ // that reads as if it ran both (#1638).
+ let eval = &config.auto_improve.eval;
+ if !eval.builtin.trim().is_empty() && !eval.command.trim().is_empty() {
+ anyhow::bail!(
+ "auto_improve.eval.builtin and auto_improve.eval.command are mutually exclusive; \
+ remove one (builtin `structure` runs in-process, a command runs your own scorer)"
+ );
+ }
+ let builtin = eval.builtin.trim();
+ if !builtin.is_empty() && builtin != "structure" {
+ anyhow::bail!(
+ "auto_improve.eval.builtin `{builtin}` is not a known scorer; the only one is \
+ `structure`"
+ );
+ }
+
// A4 entropy filter thresholds: reject an unusable threshold at startup
// rather than silently ignoring it on the first experience pass.
if let Err(message) = config
@@ -5849,6 +5871,45 @@ input_token_safety_margin = {margin}
assert_eq!(cfg.auto_improve.pending_path, "_pending/review-bot");
}
+ /// One scorer per gate: setting both is a startup error, not a config
+ /// where the builtin silently loses (#1638).
+ #[test]
+ fn an_eval_gate_rejects_both_a_builtin_and_a_command() {
+ let tmp = tempfile::TempDir::new().unwrap();
+ let write = |toml: &str| {
+ let path = tmp.path().join("config.toml");
+ std::fs::write(&path, toml).unwrap();
+ Config::load(Some(&path), Some(tmp.path().to_path_buf()))
+ };
+ let both = write(
+ "[auto_improve.eval]\nenabled = true\nbuiltin = \"structure\"\n\
+ command = \"sh score.sh\"\n",
+ )
+ .unwrap_err()
+ .to_string();
+ assert!(
+ both.contains("mutually exclusive"),
+ "unexpected error: {both}"
+ );
+
+ let unknown = write("[auto_improve.eval]\nenabled = true\nbuiltin = \"vibes\"\n")
+ .unwrap_err()
+ .to_string();
+ assert!(
+ unknown.contains("not a known scorer"),
+ "unexpected error: {unknown}"
+ );
+
+ let builtin =
+ write("[auto_improve.eval]\nenabled = true\nbuiltin = \"structure\"\n").unwrap();
+ assert_eq!(builtin.auto_improve.eval.builtin, "structure");
+ assert!(builtin.auto_improve.eval.command.is_empty());
+
+ let command =
+ write("[auto_improve.eval]\nenabled = true\ncommand = \"sh score.sh\"\n").unwrap();
+ assert!(command.auto_improve.eval.builtin.is_empty());
+ }
+
#[test]
fn profile_defaults_are_auto_and_parse_from_file_and_env() {
use ai_memory_core::profile::{EffectiveProfileShare, ProfileEnabled, ProfileShare};
diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rs
index 4e3bdfcd..99e564aa 100644
--- a/crates/ai-memory-consolidate/src/auto_improve.rs
+++ b/crates/ai-memory-consolidate/src/auto_improve.rs
@@ -120,6 +120,13 @@ pub struct AutoImproveEvalConfig {
pub enabled: bool,
/// Executable command plus whitespace-separated args. Executed directly, not via a shell.
pub command: String,
+ /// Built-in scorer to run in-process instead of `command` (#1638).
+ ///
+ /// Only `structure` exists: an H1, no `TODO`/`TBD` placeholder, a `##`
+ /// section and a list of steps for `procedures/`, and text beyond the
+ /// heading within 2,400 characters for `_rules/`. It starts no process, so
+ /// it needs no interpreter in the image and cannot time out.
+ pub builtin: String,
/// Timeout for one proposal evaluation.
pub timeout_secs: u64,
/// Wiki path prefixes that require eval when enabled.
@@ -133,6 +140,7 @@ impl Default for AutoImproveEvalConfig {
Self {
enabled: false,
command: String::new(),
+ builtin: String::new(),
timeout_secs: 120,
targets: default_auto_improve_eval_targets(),
min_delta: 0.0,
@@ -917,11 +925,22 @@ async fn apply_eval_gate_with_before_bodies(
if !eval.enabled {
return Vec::new();
}
- if eval.command.trim().is_empty() {
+ let builtin = eval.builtin.trim();
+ if !builtin.is_empty() && !eval.command.trim().is_empty() {
warnings.push(
- "auto-improve eval gate enabled without a command; targeted proposals rejected".into(),
+ "auto-improve eval gate has both a builtin scorer and a command; the command wins \
+ and the builtin is ignored — set one of them"
+ .into(),
);
}
+ if eval.command.trim().is_empty() && builtin.is_empty() {
+ warnings.push(
+ "auto-improve eval gate enabled without a command or builtin scorer; targeted \
+ proposals rejected"
+ .into(),
+ );
+ }
+ let use_builtin = eval.command.trim().is_empty() && !builtin.is_empty();
let mut accepted = Vec::with_capacity(proposals.len());
let mut results = Vec::new();
@@ -934,11 +953,22 @@ async fn apply_eval_gate_with_before_bodies(
let stdin = serde_json::to_vec(&eval_request(&proposal, before_body))
.map_err(|e| EvalRunError::Error(e.to_string()));
let program = eval.command.split_whitespace().next().unwrap_or("");
+ let (checker_name, checker_digest) = if use_builtin {
+ (
+ sanitize_eval_reason(&format!("builtin:{builtin}")),
+ sha256_hex(format!("builtin:{builtin}:{}", env!("CARGO_PKG_VERSION")).as_bytes()),
+ )
+ } else {
+ (
+ sanitize_eval_reason(program.rsplit(['/', '\\']).next().unwrap_or("")),
+ sha256_hex(eval.command.as_bytes()),
+ )
+ };
let mut result = AutoImproveEvalResult {
eval_id: uuid::Uuid::now_v7(),
target_path: proposal.path.clone(),
- checker_name: sanitize_eval_reason(program.rsplit(['/', '\\']).next().unwrap_or("")),
- checker_invocation_digest: sha256_hex(eval.command.as_bytes()),
+ checker_name,
+ checker_invocation_digest: checker_digest,
proposal_sha256: stdin.as_ref().ok().map(|bytes| sha256_hex(bytes)),
before_body_sha256: sha256_hex(before_body.as_bytes()),
after_body_sha256: sha256_hex(proposal.body_markdown.as_bytes()),
@@ -949,9 +979,18 @@ async fn apply_eval_gate_with_before_bodies(
score_after: None,
reason: None,
};
- let outcome = match stdin {
- Ok(stdin) => run_eval_for_proposal(eval, stdin).await,
- Err(error) => Err(error),
+ let outcome = if use_builtin {
+ builtin_eval_outcome(
+ builtin,
+ &proposal.path,
+ before_body,
+ &proposal.body_markdown,
+ )
+ } else {
+ match stdin {
+ Ok(stdin) => run_eval_for_proposal(eval, stdin).await,
+ Err(error) => Err(error),
+ }
};
match outcome {
Ok(outcome) => {
@@ -1009,6 +1048,136 @@ fn eval_targets_path(eval: &AutoImproveEvalConfig, path: &str) -> bool {
})
}
+/// Score one proposal in process for `--builtin` (#1638).
+///
+/// The checks are the ones `docs/examples/auto-improve-eval` applies, ported
+/// so the gate works with no interpreter and no process: an H1, no upper-case
+/// `TODO`/`TBD`, a `##` section and a list of steps for a procedure, and text
+/// beyond the heading within 2,400 characters for a rule. None of them match a
+/// section name or a wording, so a wiki written in any language scores the
+/// same way.
+fn builtin_eval_outcome(
+ builtin: &str,
+ path: &str,
+ before_body: &str,
+ after_body: &str,
+) -> Result<AutoImproveEvalResponse, EvalRunError> {
+ if builtin != "structure" {
+ return Err(EvalRunError::Error(format!(
+ "unknown builtin eval scorer `{builtin}`; the only one is `structure`"
+ )));
+ }
+ let (score_before, _) = builtin_structure_score(path, before_body);
+ let (score_after, mut reasons) = builtin_structure_score(path, after_body);
+ // A placeholder is a hard failure, not a 0.20 deduction: the weights alone
+ // let a well-structured page carrying `TODO: complete` reach exactly 0.70
+ // and pass, which is the case the check exists to stop.
+ let placeholder = builtin_has_placeholder(after_body);
+ let passed = !placeholder && score_after >= 0.70 && score_after >= score_before;
+ if placeholder && !reasons.iter().any(|r| r.contains("TODO")) {
+ reasons.push("contains TODO/TBD placeholder".to_owned());
+ }
+ Ok(AutoImproveEvalResponse {
+ passed,
+ score_before: Some(score_before),
+ score_after: Some(score_after),
+ reason: (!passed).then(|| {
+ if reasons.is_empty() {
+ "score did not improve".to_owned()
+ } else {
+ reasons.join("; ")
+ }
+ }),
+ })
+}
+
+/// Whether `body` carries an upper-case `TODO`/`TBD` placeholder.
+///
+/// Matched as a whole word, so the Portuguese word "todo" and a `TODO` inside
+/// a longer identifier are not placeholders.
+fn builtin_has_placeholder(body: &str) -> bool {
+ body.split(|c: char| !c.is_alphanumeric() && c != '_')
+ .any(|word| word == "TODO" || word == "TBD")
+}
+
+/// (`score`, reasons) for one body, the same weights as the shipped example
+/// scorers.
+fn builtin_structure_score(path: &str, body: &str) -> (f64, Vec<String>) {
+ // A page written on Windows arrives with CRLF endings; score it exactly as
+ // its LF twin instead of counting the carriage return as content.
+ let body = body.replace("\r\n", "\n").replace('\r', "\n");
+ let mut score: f64 = 0.0;
+ let mut reasons = Vec::new();
+ let lines: Vec<&str> = body.split('\n').collect();
+ let has_h1 = lines.iter().any(|line| {
+ line.strip_prefix("# ")
+ .is_some_and(|rest| !rest.trim().is_empty())
+ });
+ let has_h2 = lines.iter().any(|line| {
+ line.strip_prefix("## ")
+ .is_some_and(|rest| !rest.trim().is_empty())
+ });
+ let has_item = lines.iter().any(|line| {
+ let trimmed = line.trim_start();
+ let rest = trimmed
+ .strip_prefix("- ")
+ .or_else(|| trimmed.strip_prefix("* "))
+ .or_else(|| trimmed.strip_prefix("+ "))
+ .or_else(|| {
+ let digits = trimmed.len()
+ - trimmed
+ .trim_start_matches(|c: char| c.is_ascii_digit())
+ .len();
+ trimmed[digits..]
+ .strip_prefix(". ")
+ .or_else(|| trimmed[digits..].strip_prefix(") "))
+ .filter(|_| digits > 0)
+ });
+ rest.is_some_and(|rest| !rest.trim().is_empty())
+ });
+ let placeholder = builtin_has_placeholder(&body);
+ let prose_beyond_heading = body
+ .lines()
+ .any(|line| !line.trim().is_empty() && !line.trim_start().starts_with('#'));
+
+ if has_h1 {
+ score += 0.25;
+ } else {
+ reasons.push("missing H1 heading".to_owned());
+ }
+ if placeholder {
+ reasons.push("contains TODO/TBD placeholder".to_owned());
+ } else {
+ score += 0.20;
+ }
+ if path.starts_with("procedures/") {
+ if has_h2 {
+ score += 0.20;
+ } else {
+ reasons.push("procedure has no ## section".to_owned());
+ }
+ if has_item {
+ score += 0.25;
+ } else {
+ reasons.push("procedure has no list of steps".to_owned());
+ }
+ } else if path.starts_with("_rules/") {
+ if prose_beyond_heading {
+ score += 0.35;
+ } else {
+ reasons.push("rule has no text beyond its heading".to_owned());
+ }
+ if body.trim_matches([' ', '\t', '\n', '\r']).chars().count() <= 2_400 {
+ score += 0.10;
+ } else {
+ reasons.push("rule is too long for an agent instruction".to_owned());
+ }
+ } else {
+ score += 0.45;
+ }
+ (score.min(1.0), reasons)
+}
+
#[derive(Debug)]
enum EvalRunError {
Timeout,
@@ -2564,6 +2733,7 @@ mod tests {
AutoImproveEvalConfig {
enabled: true,
command,
+ builtin: String::new(),
// Windows PowerShell cold-start is slower than 2s, and these gate
// tests run in parallel — timeouts here are for the eval command
// itself, not for interpreter startup.
@@ -2653,6 +2823,130 @@ mod tests {
assert!(warnings.is_empty());
}
+ /// The built-in scorer needs no process, no interpreter and no timeout:
+ /// the gate answers in-process for the same structural checks the shipped
+ /// example scorers apply (#1638).
+ #[tokio::test]
+ async fn builtin_structure_scores_in_process_without_a_command() {
+ let cfg = AutoImproveEvalConfig {
+ enabled: true,
+ builtin: "structure".into(),
+ min_delta: 0.0,
+ ..AutoImproveEvalConfig::default()
+ };
+ let mut good = proposal("procedures/release.md", "procedure", 0.9);
+ good.body_markdown = "# Release\n\n## Steps\n- Run the tests.\n- Publish.\n".into();
+ let mut bad = proposal("procedures/release.md", "procedure", 0.9);
+ bad.body_markdown = "# Release\n\n## Steps\nRun the tests and publish.\n".into();
+ let mut proposals = vec![good, bad];
+ let mut rejected = Vec::new();
+ let mut warnings = Vec::new();
+ let results = apply_eval_gate_with_before_bodies(
+ &cfg,
+ &mut proposals,
+ &mut rejected,
+ &mut warnings,
+ &BTreeMap::new(),
+ )
+ .await;
+
+ assert!(warnings.is_empty(), "{warnings:?}");
+ assert_eq!(proposals.len(), 1, "only the page with steps survives");
+ assert_eq!(rejected.len(), 1);
+ assert_eq!(rejected[0].reason, "eval_gate_failed");
+ assert_eq!(results.len(), 2);
+ assert_eq!(results[0].checker_name, "builtin:structure");
+ assert_eq!(results[0].status, EvalStatus::Success);
+ assert_eq!(results[1].status, EvalStatus::Rejected);
+ let digest = &results[0].checker_invocation_digest;
+ assert_eq!(
+ results[1].checker_invocation_digest, *digest,
+ "the invocation digest covers the builtin name, not a command string"
+ );
+ }
+
+ /// The same structural checks, in any language: a Portuguese or Japanese
+ /// page is not rejected for its wording, and a `todo` word in prose is not
+ /// a placeholder.
+ #[test]
+ fn builtin_structure_is_language_neutral_and_ignores_lowercase_todo() {
+ let portuguese = "# Procedimento\n\n## Passos\n- Rodar os testes.\n- Publicar.\n";
+ let japanese = "# リリース手順\n\n## 手順\n1. テストを実行する\n2) デプロイする\n";
+ let rule = "# Regra\n\nSempre rode a suíte completa antes de publicar; o todo da equipe \nrevisa depois.\n";
+ for (path, body) in [
+ ("procedures/publicacao.md", portuguese),
+ ("procedures/release.md", japanese),
+ ("_rules/testes.md", rule),
+ ] {
+ let (score, reasons) = builtin_structure_score(path, body);
+ assert!(score >= 0.70, "{path}: {score} {reasons:?}");
+ }
+ // A structurally complete page carrying a placeholder still fails: the
+ // weights alone would let it reach exactly 0.70.
+ let outcome = builtin_eval_outcome(
+ "structure",
+ "procedures/x.md",
+ "",
+ "# P\n\n## S\n- TODO: complete\n",
+ )
+ .unwrap();
+ assert!(!outcome.passed, "{outcome:?}");
+ assert!(
+ outcome
+ .reason
+ .as_deref()
+ .unwrap_or_default()
+ .contains("TODO"),
+ "{outcome:?}"
+ );
+ }
+
+ /// CRLF is the same page: a Windows checkout must not pass a rule with no
+ /// body, or fail a procedure whose step list is intact.
+ #[test]
+ fn builtin_structure_scores_a_crlf_page_like_its_lf_twin() {
+ let cases = [
+ ("_rules/a.md", "# Regra\n\nTexto.\n"),
+ ("_rules/b.md", "# Regra\n"),
+ ("procedures/c.md", "# P\n\n## S\n- passo\n"),
+ ("procedures/d.md", "# P\n\n## S\n- \n"),
+ ];
+ for (path, body) in cases {
+ let lf = builtin_structure_score(path, body);
+ let crlf = builtin_structure_score(path, &body.replace('\n', "\r\n"));
+ assert_eq!(lf.0, crlf.0, "{path}");
+ assert_eq!(lf.1, crlf.1, "{path}");
+ }
+ assert!(
+ builtin_structure_score("_rules/b.md", "# Regra\r\n").0 < 0.70,
+ "the carriage return is not body text"
+ );
+ }
+
+ /// An unusable scorer name fails the proposal rather than silently
+ /// accepting it.
+ #[tokio::test]
+ async fn an_unknown_builtin_scorer_fails_closed() {
+ let cfg = AutoImproveEvalConfig {
+ enabled: true,
+ builtin: "vibes".into(),
+ ..AutoImproveEvalConfig::default()
+ };
+ let mut proposals = vec![proposal("procedures/x.md", "procedure", 0.9)];
+ let mut rejected = Vec::new();
+ let mut warnings = Vec::new();
+ apply_eval_gate_with_before_bodies(
+ &cfg,
+ &mut proposals,
+ &mut rejected,
+ &mut warnings,
+ &BTreeMap::new(),
+ )
+ .await;
+ assert!(proposals.is_empty());
+ assert_eq!(rejected[0].reason, "eval_gate_error");
+ }
+
#[tokio::test]
async fn passing_eval_allows_proposal_and_records_evidence() {
let (_eval_dir, script) = write_eval_script(
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index e826d0f6..387e9407 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -12348,6 +12348,7 @@ mod tests {
state.auto_improve_review_config.eval = ai_memory_consolidate::AutoImproveEvalConfig {
enabled: true,
command: String::new(),
+ builtin: String::new(),
timeout_secs: 1,
targets: vec!["notes".into()],
min_delta: 0.0,
diff --git a/docs/auto-improve-eval-gates.md b/docs/auto-improve-eval-gates.md
index e2da5c85..d24e3638 100644
--- a/docs/auto-improve-eval-gates.md
+++ b/docs/auto-improve-eval-gates.md
@@ -19,6 +19,38 @@ The command is split on whitespace and executed directly, not through a shell.
Use a wrapper script if you need quoting, environment setup, or multiple
commands.
+### The built-in scorer
+
+Writing, mounting and maintaining a scorer is optional: `builtin = "structure"`
+runs the checks in process, so the gate needs no interpreter, no script inside
+the container, and no timeout.
+
+```toml
+[auto_improve.eval]
+enabled = true
+builtin = "structure"
+targets = ["_rules", "procedures"]
+min_delta = 0.0
+```
+
+It checks, never matching a section name or a wording (so a wiki in any
+language scores the same):
+
+- an H1 heading;
+- no upper-case `TODO`/`TBD` placeholder — a hard failure, not a deduction; the
+ Portuguese word "todo" in prose is not a placeholder;
+- for `procedures/`, a `##` section and a list of steps;
+- for `_rules/`, text beyond the heading, within 2,400 characters.
+
+A proposal passes at `score_after >= 0.70` and no lower than `score_before`.
+`eval_results` records `checker_name = "builtin:structure"` and an invocation
+digest over the scorer name and the binary version, instead of a command
+string.
+
+`builtin` and `command` are mutually exclusive: setting both is a startup
+error rather than one being silently ignored. `command` keeps working
+unchanged for project-specific scorers.
+
The eval policy is owned by the server process. `ai-memory auto-improve` and
direct HTTP requests cannot replace the configured command, targets, timeout,
or score threshold. When a CLI talks to a remote server, configure