Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(auto-improve): accept the operation spellings models actually emit
AkitaOnRailse7dcb8da32a8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f4f23382..ec0c599e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- Auto-improve no longer rejects every proposal from a model that spells the+ single supported operation differently (#458). `operation` was a free-form+ string validated by exact match against `create_or_update`, while the+ schema carried no constraint and the neighbouring field description — "path+ that would be created or updated" — actively invited `"create"`. Two local+ models took the invitation for 6/6 candidates, so with+ `require_approval = false` the learning loop completed having silently done+ nothing, and the rejection buffer then fed `unsupported_operation` rows back+ into later reviewer prompts as noise. The schema now advertises a+ single-value enum, so providers with constrained decoding cannot emit+ anything else, and validation normalises the unambiguous spellings+ (`create`, `update`, `upsert`, `create/update`, …) for providers without it.+ Operations the pipeline genuinely cannot perform — `delete`, `rename`,+ `move` — still fail with the same reason as before.+### Added- The `gemini` LLM provider now respects the `LLM_BASE_URL` configuration value.Previously, the base URL was hardcoded to `generativelanguage.googleapis.com`.diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rsindex d95aef2d..c1e7b387 100644--- a/crates/ai-memory-consolidate/src/auto_improve.rs+++ b/crates/ai-memory-consolidate/src/auto_improve.rs@@ -261,7 +261,15 @@ impl<'de> Deserialize<'de> for AutoImproveEvidence {#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]pub struct AutoImproveProposal {/// Currently only `create_or_update` is supported.+ ///+ /// Advertised to the model as a single-value enum so providers with+ /// constrained decoding cannot generate anything else. The field stays a+ /// `String` rather than becoming a Rust enum on purpose: a provider+ /// *without* constrained decoding that emits `"create"` should be+ /// normalised by [`normalize_operation`], not fail to deserialise and take+ /// the whole proposal with it.#[serde(default = "default_operation")]+ #[schemars(extend("enum" = ["create_or_update"]))]pub operation: String,/// Relative wiki path that would be created or updated.#[serde(default)]@@ -315,8 +323,33 @@ pub struct AutoImprovePatchEdit {pub context: Option<String>,}+/// The one operation this pipeline performs.+pub const CANONICAL_OPERATION: &str = "create_or_update";+fn default_operation() -> String {- "create_or_update".into()+ CANONICAL_OPERATION.into()+}++/// Map the ways a model spells the single supported operation onto its+/// canonical form.+///+/// Deliberately narrow: only spellings that unambiguously mean "write this+/// page" are folded in. Anything else is left untouched so it still fails+/// validation — the point is to stop losing work to a wording difference,+/// not to accept an operation the pipeline cannot perform.+fn normalize_operation(raw: &str) -> String {+ let squashed: String = raw+ .trim()+ .to_ascii_lowercase()+ .chars()+ .filter(|c| !matches!(c, ' ' | '_' | '-' | '/'))+ .collect();+ match squashed.as_str() {+ "createorupdate" | "create" | "update" | "upsert" | "createupdate" | "write" => {+ CANONICAL_OPERATION.to_string()+ }+ _ => raw.to_string(),+ }}fn default_edit_mode() -> String {@@ -1511,7 +1544,15 @@ fn validate_proposal(cfg: &AutoImproveReviewConfig,existing_index: &ExistingPageIndex,) -> Result<(), String> {- if proposal.operation != "create_or_update" {+ // Normalise before comparing. The field description next door says the+ // path "would be created or updated", which invites exactly the values+ // rejected here, and models take the invitation: two different local+ // models emitted `"create"` for 6/6 candidates, so every run finished+ // with zero validated proposals and the learning loop silently did+ // nothing (#458). Rejecting a wording difference is not a safety+ // property — there is only one operation.+ proposal.operation = normalize_operation(&proposal.operation);+ if proposal.operation != CANONICAL_OPERATION {return Err("unsupported_operation".into());}if proposal.confidence < cfg.min_confidence {@@ -2933,6 +2974,39 @@ mod tests {}}+ /// The end-to-end shape of #458: a model emits `"operation": "create"`+ /// and every candidate is rejected `unsupported_operation`, so with+ /// `require_approval = false` the loop finishes having done nothing.+ /// Reproduced there with two local models, 6/6 candidates.+ #[test]+ fn a_proposal_saying_create_is_validated_not_rejected() {+ let cfg = AutoImproveReviewConfig::default();+ let index = ExistingPageIndex::default();++ for spelling in ["create", "update", "create/update", "CREATE"] {+ let mut candidate = proposal("notes/thing.md", "note", 0.91);+ candidate.operation = spelling.into();+ let outcome = validate_proposal(&mut candidate, &cfg, &index);+ assert!(+ outcome.is_ok(),+ "{spelling:?} must validate, got {outcome:?}"+ );+ assert_eq!(+ candidate.operation, CANONICAL_OPERATION,+ "validation should also canonicalise the stored value"+ );+ }++ // An operation the pipeline cannot perform still fails, and fails+ // with the same reason as before.+ let mut deleting = proposal("notes/thing.md", "note", 0.91);+ deleting.operation = "delete".into();+ assert_eq!(+ validate_proposal(&mut deleting, &cfg, &index),+ Err("unsupported_operation".into())+ );+ }+#[test]fn patch_to_missing_or_non_context_target_rejects() {let raw = AutoImproveLlmResponse {@@ -3273,3 +3347,72 @@ mod tests {assert!(rejected.iter().any(|r| r.reason == "patch_extra_h1"));}}++#[cfg(test)]+mod operation_normalization_tests {+ use super::*;++ /// #458: two local models emitted `"create"` for every candidate, so the+ /// loop rejected 6/6 as `unsupported_operation` and silently did nothing.+ #[test]+ fn the_spellings_models_actually_emit_are_accepted() {+ for raw in [+ "create_or_update",+ "create",+ "update",+ "upsert",+ "create/update",+ "create or update",+ "CREATE",+ " Create_Or_Update ",+ "write",+ ] {+ assert_eq!(+ normalize_operation(raw),+ CANONICAL_OPERATION,+ "{raw:?} means write-this-page and must normalise"+ );+ }+ }++ /// The normaliser must not become a rubber stamp: an operation this+ /// pipeline genuinely cannot perform still has to fail validation.+ #[test]+ fn operations_we_cannot_perform_are_left_to_fail() {+ for raw in ["delete", "rename", "move", "archive", "drop", "merge", ""] {+ assert_ne!(+ normalize_operation(raw),+ CANONICAL_OPERATION,+ "{raw:?} is not a write and must keep failing validation"+ );+ }+ }++ /// The schema must advertise the constraint, so a provider with+ /// constrained decoding cannot generate a bad value in the first place.+ #[test]+ fn schema_constrains_operation_to_the_canonical_value() {+ let schema = schemars::schema_for!(AutoImproveProposal);+ let value = serde_json::to_value(&schema).expect("schema serialises");+ let op = value+ .pointer("/properties/operation")+ .expect("operation property present");+ let variants = op+ .get("enum")+ .and_then(|e| e.as_array())+ .expect("operation carries an enum constraint");+ assert_eq!(variants, &vec![serde_json::json!(CANONICAL_OPERATION)]);+ }++ /// A `String` field, not a Rust enum: a provider without constrained+ /// decoding that emits `"create"` must still deserialise, so the+ /// normaliser gets a chance to fix it rather than the whole proposal+ /// failing to parse.+ #[test]+ fn a_non_canonical_operation_still_deserialises() {+ let parsed: AutoImproveProposal =+ serde_json::from_value(serde_json::json!({ "operation": "create" }))+ .expect("must not fail to parse; normalisation happens in validation");+ assert_eq!(parsed.operation, "create");+ }+}