Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
test(llm): wiremock integration tests for openai-compat strict path
AkitaOnRailsfa1cd810703d
diff --git a/Cargo.lock b/Cargo.lockindex 8b034a8d..130d9c94 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -165,6 +165,7 @@ dependencies = ["tokio","tracing","uuid",+ "wiremock",][[package]]@@ -375,6 +376,16 @@ dependencies = ["nom",]+[[package]]+name = "assert-json-diff"+version = "2.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"+dependencies = [+ "serde",+ "serde_json",+]+[[package]]name = "async-trait"version = "0.1.89"@@ -735,6 +746,24 @@ dependencies = ["syn",]+[[package]]+name = "deadpool"+version = "0.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b"+dependencies = [+ "deadpool-runtime",+ "lazy_static",+ "num_cpus",+ "tokio",+]++[[package]]+name = "deadpool-runtime"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"+[[package]]name = "deranged"version = "0.5.8"@@ -1074,6 +1103,25 @@ dependencies = ["url",]+[[package]]+name = "h2"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"+dependencies = [+ "atomic-waker",+ "bytes",+ "fnv",+ "futures-core",+ "futures-sink",+ "http",+ "indexmap",+ "slab",+ "tokio",+ "tokio-util",+ "tracing",+]+[[package]]name = "hashbrown"version = "0.14.5"@@ -1113,6 +1161,12 @@ version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"+[[package]]+name = "hermit-abi"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"+[[package]]name = "http"version = "1.4.0"@@ -1183,6 +1237,7 @@ dependencies = ["bytes","futures-channel","futures-core",+ "h2","http","http-body","httparse",@@ -1766,6 +1821,16 @@ dependencies = ["autocfg",]+[[package]]+name = "num_cpus"+version = "1.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"+dependencies = [+ "hermit-abi",+ "libc",+]+[[package]]name = "once_cell"version = "1.21.4"@@ -3712,6 +3777,29 @@ dependencies = ["memchr",]+[[package]]+name = "wiremock"+version = "0.6.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031"+dependencies = [+ "assert-json-diff",+ "base64",+ "deadpool",+ "futures",+ "http",+ "http-body-util",+ "hyper",+ "hyper-util",+ "log",+ "once_cell",+ "regex",+ "serde",+ "serde_json",+ "tokio",+ "url",+]+[[package]]name = "wit-bindgen"version = "0.51.0"diff --git a/crates/ai-memory-llm/Cargo.toml b/crates/ai-memory-llm/Cargo.tomlindex 942a79a6..8b6d8496 100644--- a/crates/ai-memory-llm/Cargo.toml+++ b/crates/ai-memory-llm/Cargo.toml@@ -28,6 +28,9 @@ jiff.workspace = true[dev-dependencies]tokio = { workspace = true }tempfile.workspace = true+# In-process HTTP mock for the openai-compat strict-fallback tests.+# Scoped to dev-dependencies so production binaries don't pull it in.+wiremock = "0.6"[lints]workspace = truediff --git a/crates/ai-memory-llm/tests/openai_compat_strict.rs b/crates/ai-memory-llm/tests/openai_compat_strict.rsnew file mode 100644index 00000000..f061a01a--- /dev/null+++ b/crates/ai-memory-llm/tests/openai_compat_strict.rs@@ -0,0 +1,265 @@+//! Integration tests for `OpenAiCompatProvider` strict mode, driving the+//! provider against a real in-process HTTP mock (wiremock).+//!+//! The strict path is the load-bearing addition from PR #70: when+//! `compat_strict=true`, the provider sends `response_format=json_schema`+//! first; on a *parse-shape* failure (HTTP 200 with bad JSON, or 200+//! with a non-object) it falls back to the tolerant prose-JSON parser+//! the default mode uses. On transport / HTTP-status / auth failures+//! it propagates without retry. These tests assert those four outcomes+//! end-to-end against synthesised upstream responses.+//!+//! Why not unit-test in `openai_compat.rs`: the strict path delegates+//! to `OpenAiProvider::complete_structured_raw`, which speaks real+//! HTTP. The classification helper (`is_parse_shape_error`) is unit-+//! tested alongside its definition; here we cover what `is_parse_shape_error`+//! actually changes about end-to-end behaviour.++use ai_memory_llm::types::ChatRequest;+use ai_memory_llm::{LlmProvider, OpenAiCompatProvider};+use serde_json::json;+use std::sync::Arc;+use std::sync::atomic::{AtomicUsize, Ordering};+use wiremock::matchers::{method, path};+use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};++fn strict_provider(base_url: String) -> OpenAiCompatProvider {+ OpenAiCompatProvider::new(base_url, None, "mistral-nemo")+ .expect("provider builds")+ .with_strict(true)+}++fn tiny_request() -> ChatRequest {+ ChatRequest::user_prompt("emit JSON: { \"ok\": true }")+}++fn tiny_schema() -> serde_json::Value {+ json!({+ "type": "object",+ "properties": { "ok": { "type": "boolean" } },+ "required": ["ok"],+ })+}++fn body_with_content(content: &str) -> serde_json::Value {+ json!({+ "id": "id",+ "object": "chat.completion",+ "created": 0,+ "model": "mistral-nemo",+ "choices": [{+ "index": 0,+ "message": { "role": "assistant", "content": content },+ "finish_reason": "stop",+ }],+ "usage": { "prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2 },+ })+}++/// Counts every request hitting the mock server so a test can assert+/// "only one call was made" (strict succeeded) vs "two calls were made"+/// (strict raw, then tolerant fallback). The response itself is cloned+/// from a `(status, body)` pair on every call — `ResponseTemplate` doesn't+/// expose a clone, so we rebuild it each request.+#[derive(Clone)]+struct CountingResponder {+ count: Arc<AtomicUsize>,+ status: u16,+ body: Arc<serde_json::Value>,+}++impl CountingResponder {+ fn json(status: u16, body: serde_json::Value) -> Self {+ Self {+ count: Arc::new(AtomicUsize::new(0)),+ status,+ body: Arc::new(body),+ }+ }+ fn raw(status: u16, body: &str) -> Self {+ Self {+ count: Arc::new(AtomicUsize::new(0)),+ status,+ body: Arc::new(serde_json::Value::String(body.to_string())),+ }+ }+ fn count(&self) -> usize {+ self.count.load(Ordering::SeqCst)+ }+}++impl Respond for CountingResponder {+ fn respond(&self, _req: &Request) -> ResponseTemplate {+ self.count.fetch_add(1, Ordering::SeqCst);+ match self.body.as_ref() {+ serde_json::Value::String(s) => ResponseTemplate::new(self.status).set_body_string(s),+ other => ResponseTemplate::new(self.status).set_body_json(other.clone()),+ }+ }+}++/// Strict path SUCCEEDS: model honoured `response_format` and returned+/// clean JSON. Provider must surface the parsed object directly with+/// exactly ONE upstream HTTP call (the tolerant fallback is NOT invoked).+#[tokio::test]+async fn strict_success_returns_directly_and_makes_one_call() {+ let server = MockServer::start().await;+ let responder = CountingResponder::json(200, body_with_content(r#"{"ok":true}"#));+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(responder.clone())+ .mount(&server)+ .await;++ let provider = strict_provider(server.uri());+ let value = provider+ .complete_structured_raw(tiny_request(), tiny_schema())+ .await+ .expect("strict success path");+ assert_eq!(value, json!({"ok": true}));+ assert_eq!(+ responder.count(),+ 1,+ "strict success must hit upstream exactly once (no fallback retry)"+ );+}++/// Strict path returns 200 with a NON-OBJECT body (e.g. plain prose).+/// The Compat wrapper detects the non-object response and falls through+/// to the tolerant path, which makes a SECOND HTTP call with no+/// `response_format` and extracts the first balanced `{…}` from the+/// resulting prose. End result: success after two HTTP hops.+#[tokio::test]+async fn strict_non_object_falls_back_to_tolerant_parser() {+ let server = MockServer::start().await;+ // First call (strict): returns prose with a JSON object embedded.+ // Because `OpenAiProvider::complete_structured_raw` calls+ // `serde_json::from_str` on `content` directly, prose that wraps a+ // JSON object will not parse — but the inner provider's parse-fail+ // path surfaces as `LlmError::Serde`, which `is_parse_shape_error`+ // catches and the wrapper retries via `inner.complete(...)`.+ // Both calls return the same prose body. The strict raw parse fails+ // because the model's `content` isn't itself valid JSON (just prose+ // wrapping a JSON object), surfaced as `LlmError::Serde`. The+ // wrapper's `is_parse_shape_error` catches it and the tolerant path+ // makes a second call whose response runs through `first_json_object`+ // to extract the embedded `{"ok":true}`.+ let counter = Arc::new(AtomicUsize::new(0));+ let counter_clone = counter.clone();+ let prose_body = body_with_content("Sure! Here is the JSON you asked for: {\"ok\":true}");+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(move |_req: &Request| {+ counter_clone.fetch_add(1, Ordering::SeqCst);+ ResponseTemplate::new(200).set_body_json(prose_body.clone())+ })+ .mount(&server)+ .await;++ let provider = strict_provider(server.uri());+ let value = provider+ .complete_structured_raw(tiny_request(), tiny_schema())+ .await+ .expect("strict fallback to tolerant must recover the embedded object");+ assert_eq!(value, json!({"ok": true}));+ assert_eq!(+ counter.load(Ordering::SeqCst),+ 2,+ "fallback path must issue exactly two upstream calls (strict, then tolerant)"+ );+}++/// Strict path receives an HTTP 5xx (upstream outage). A tolerant+/// retry would hit the same outage and double the wall-clock /+/// token spend. Audit fix: provider must PROPAGATE the+/// `LlmError::Provider` instead of retrying, so the caller sees the+/// real cause and can back off.+#[tokio::test]+async fn strict_upstream_5xx_propagates_without_fallback_retry() {+ let server = MockServer::start().await;+ let responder = CountingResponder::raw(500, "boom");+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(responder.clone())+ .mount(&server)+ .await;++ let provider = strict_provider(server.uri());+ let err = provider+ .complete_structured_raw(tiny_request(), tiny_schema())+ .await+ .expect_err("5xx must surface");+ assert!(+ err.to_string().contains("500"),+ "error must carry the upstream status, got: {err}"+ );+ assert_eq!(+ responder.count(),+ 1,+ "5xx must NOT trigger the tolerant retry — that would double cost",+ );+}++/// Same propagation guarantee for HTTP 401 (auth) — the tolerant retry+/// would just hit the same 401. Distinct test so a future refactor+/// that special-cases auth errors fails loudly.+#[tokio::test]+async fn strict_upstream_401_propagates_without_fallback_retry() {+ let server = MockServer::start().await;+ let responder = CountingResponder::raw(401, "missing or invalid api key");+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(responder.clone())+ .mount(&server)+ .await;++ let provider = strict_provider(server.uri());+ let err = provider+ .complete_structured_raw(tiny_request(), tiny_schema())+ .await+ .expect_err("401 must surface");+ assert!(+ err.to_string().contains("401"),+ "error must carry the auth status, got: {err}"+ );+ assert_eq!(+ responder.count(),+ 1,+ "401 must NOT trigger the tolerant retry"+ );+}++/// Reasoning models (DeepSeek / Qwen3 / MiniMax M2.7) embed+/// `<think>…</think>` inside `content` before the JSON. Strict-mode+/// users on those models should keep the flag OFF; this test+/// documents that turning strict ON still WORKS — it just costs one+/// extra HTTP call. Strict-raw parses `content` directly with no+/// `<think>` strip and fails, fallback kicks in, tolerant path runs+/// `strip_reasoning_blocks` and recovers.+#[tokio::test]+async fn strict_falls_back_for_reasoning_model_with_think_in_content() {+ let server = MockServer::start().await;+ let counter = Arc::new(AtomicUsize::new(0));+ let counter_clone = counter.clone();+ let content = "<think>I should output JSON.</think>\n{\"ok\":true}";+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(move |_req: &Request| {+ counter_clone.fetch_add(1, Ordering::SeqCst);+ ResponseTemplate::new(200).set_body_json(body_with_content(content))+ })+ .mount(&server)+ .await;++ let provider = strict_provider(server.uri());+ let value = provider+ .complete_structured_raw(tiny_request(), tiny_schema())+ .await+ .expect("fallback must strip <think> and recover");+ assert_eq!(value, json!({"ok": true}));+ assert_eq!(+ counter.load(Ordering::SeqCst),+ 2,+ "reasoning content costs the double-call fallback (operator should keep strict OFF)"+ );+}