Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(llm): narrow openai-compat strict fallback to parse-shape failures
AkitaOnRailsdb7f57ee44dd
diff --git a/crates/ai-memory-llm/src/openai_compat.rs b/crates/ai-memory-llm/src/openai_compat.rsindex 1be06797..69d0fd0b 100644--- a/crates/ai-memory-llm/src/openai_compat.rs+++ b/crates/ai-memory-llm/src/openai_compat.rs@@ -127,9 +127,25 @@ impl LlmProvider for OpenAiCompatProvider {// response_format. The dialect stays Compat, so the api.openai.com// token/temperature quirks do NOT leak into the local engine.//- // If the strict raw call fails (engine that rejects the schema,- // truncated JSON, etc.) we fall through to the tolerant parser below- // instead of propagating that raw-call error.+ // Note: `enforce_strict_object_schemas` is invoked here because the+ // inner provider's own normalisation is gated on `RequestDialect::+ // Official` (see openai.rs::complete_structured_raw). The wrapper+ // owns the Compat-side rewrite so a refactor of the inner cannot+ // silently drop it.+ //+ // Fallback path:+ // Fallback only on *parse-shape* failures — the engine returned a+ // response but it wasn't a valid JSON object (the case the tolerant+ // path was designed for). On `LlmError::Provider` (HTTP 4xx/5xx,+ // auth, rate-limit) and transport-level failures, propagate the+ // error: a `complete()` retry inside the fallback would hit the+ // same wall and double the latency / token spend for no gain.+ //+ // The fallback itself is a SECOND HTTP call to the model — the+ // strict raw call returns parsed JSON or an error, not raw text,+ // so we can't reuse the response body. Operators on engines that+ // routinely fall back (e.g. reasoning models with `<think>` in+ // `content`) should keep `strict=false` to avoid the double call.if self.strict {let mut strict_schema = schema.clone();enforce_strict_object_schemas(&mut strict_schema);@@ -142,8 +158,14 @@ impl LlmProvider for OpenAiCompatProvider {Ok(_) => {debug!("compat strict: non-object response, falling back to tolerant parser");}+ Err(err) if is_parse_shape_error(&err) => {+ debug!(error = %err, "compat strict parse-shape mismatch, falling back to tolerant parser");+ }Err(err) => {- debug!(error = %err, "compat strict failed, falling back to tolerant parser");+ // Transport / 5xx / auth / rate-limit: a tolerant retry+ // would hit the same failure. Propagate so the caller+ // sees the real cause and can back off appropriately.+ return Err(err);}}}@@ -183,6 +205,14 @@ impl LlmProvider for OpenAiCompatProvider {}}+/// True for errors where a *retry without structured-output coercion*+/// has a real chance of succeeding — i.e. the upstream returned a+/// response, but it wasn't in the shape we wanted. Transport / HTTP /+/// auth errors are excluded because they would just reproduce.+fn is_parse_shape_error(err: &LlmError) -> bool {+ matches!(err, LlmError::UnexpectedShape(_) | LlmError::Serde(_))+}+/// Find the first balanced `{...}` object in a string, skipping/// braces that appear inside JSON string literals.///@@ -240,6 +270,36 @@ mod tests {assert!(p.strict);}+ /// The strict path's fallback policy lives in `is_parse_shape_error`:+ /// fall back ONLY when the upstream returned a response in the wrong+ /// shape (the tolerant prose-JSON parser has a real chance). Propagate+ /// transport / HTTP-status / auth errors — a tolerant retry would just+ /// hit the same wall and double cost. Regression guard for the audit+ /// finding that the original strict-fallback caught every `Err(_)`.+ #[test]+ fn is_parse_shape_error_classifies_correctly() {+ // Shape failures → fall back.+ assert!(is_parse_shape_error(&LlmError::UnexpectedShape(+ "no JSON object".into()+ )));+ assert!(is_parse_shape_error(&LlmError::Serde(+ "trailing comma".into()+ )));+ // Transport / HTTP / auth → propagate.+ assert!(!is_parse_shape_error(&LlmError::Provider {+ status: 500,+ body: "engine boom".into()+ }));+ assert!(!is_parse_shape_error(&LlmError::Provider {+ status: 401,+ body: "unauthorized".into()+ }));+ assert!(!is_parse_shape_error(&LlmError::Provider {+ status: 429,+ body: "rate limited".into()+ }));+ }+#[test]fn first_json_object_finds_balanced_object() {assert_eq!(first_json_object("noise {\"k\":1} more"), Some("{\"k\":1}"));