Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(llm): stop forcing the structured tool on Claude models that reject it
Vinícius DS Andrade8207d3da4dc0
diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rsindex 150d2879..4d6546e3 100644--- a/crates/ai-memory-llm/src/anthropic.rs+++ b/crates/ai-memory-llm/src/anthropic.rs@@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize};use tracing::debug;use crate::error::{LlmError, LlmResult};+use crate::openai_compat::{first_json_object, strip_reasoning_blocks};use crate::provider::LlmProvider;use crate::response::{provider_error_body, response_json_limited};use crate::types::{@@ -278,21 +279,39 @@ impl LlmProvider for AnthropicProvider {) -> LlmResult<serde_json::Value> {let body = self.build_request(&request, Some(schema));let response: AnthropicResponse = self.post(&body, operation_id).await?;+ let forced = self.forces_result_tool();+ let mut text = Vec::new();for c in response.content {- if let AnthropicContent::ToolUse { input, .. } = c {- return Ok(input);+ match c {+ AnthropicContent::ToolUse { input, .. } => return Ok(input),+ // Under `auto` the model can reply with the JSON itself+ // instead of calling the tool.+ AnthropicContent::Text { text: part } if !forced => text.push(part),+ _ => {}}}+ if let Some(value) = json_object_in(&text.join("\n")) {+ return Ok(value);+ }Err(LlmError::UnexpectedShape("anthropic response had no tool_use block".into(),))}}+/// The JSON object in a text reply, bare, fenced, or set in prose.+fn json_object_in(text: &str) -> Option<serde_json::Value> {+ let cleaned = strip_reasoning_blocks(text);+ serde_json::from_str::<serde_json::Value>(&cleaned)+ .ok()+ .filter(serde_json::Value::is_object)+ .or_else(|| serde_json::from_str(first_json_object(&cleaned)?).ok())+}+impl AnthropicProvider {/// Build the `/v1/messages` body shared by `complete` and/// `complete_structured_raw`. Passing a schema turns the call into the- /// forced-tool structured-output shape. Both paths go through here so the+ /// `result`-tool structured-output shape. Both paths go through here so the/// temperature rule below can't apply to one and silently miss the other.fn build_request<'a>(&'a self,@@ -307,14 +326,22 @@ impl AnthropicProvider {content: &m.content,}).collect();+ let forced = self.forces_result_tool();+ // Without a forced `tool_choice`, the description is what asks for+ // the call.+ let description = if forced {+ "Emit the structured result."+ } else {+ "Emit the structured result. Call this tool once instead of replying in text."+ };let (tools, tool_choice) = match schema {Some(input_schema) => (Some(vec![AnthropicTool {name: "result".into(),- description: "Emit the structured result.".into(),+ description: description.into(),input_schema,}]),- Some(AnthropicToolChoice::Tool {+ forced.then(|| AnthropicToolChoice::Tool {name: "result".into(),}),),@@ -342,6 +369,14 @@ impl AnthropicProvider {}}+ /// Whether the structured path pins `tool_choice` to the `result` tool.+ /// The models that answer a forced `tool_choice` with a 400 keep the+ /// default `auto`.+ fn forces_result_tool(&self) -> bool {+ let claude = ClaudeId::parse(&self.model);+ !claude.is_some_and(ClaudeId::rejects_forced_tool_choice)+ }+async fn post<B: Serialize, R: DeserializeOwned>(&self,body: &B,@@ -511,6 +546,25 @@ impl ClaudeId {self.meets(Some((4, 7)))}+ /// Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 400 on a forced+ /// `tool_choice` (`any` or a named tool). Haiku 5.5, Opus 5, Sonnet 5,+ /// and Fable 5 still accept one.+ fn rejects_forced_tool_choice(self) -> bool {+ match self {+ Self::Released {+ family: ClaudeFamily::Opus | ClaudeFamily::Sonnet,+ major,+ minor,+ } => (major, minor) >= (5, 5),+ Self::Released {+ family: ClaudeFamily::Fable | ClaudeFamily::Mythos,+ major,+ minor,+ } => (major, minor) >= (5, 1),+ _ => false,+ }+ }+fn supports_effort(self) -> bool {self.meets(self.family().effort_since())}diff --git a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs b/crates/ai-memory-llm/tests/suite/anthropic_provider.rsnew file mode 100644index 00000000..2b20a540--- /dev/null+++ b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs@@ -0,0 +1,119 @@+//! Wire tests for the Anthropic provider's structured output: the request it+//! sends each Claude generation and the replies it reads back.+//!+//! Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 answer a forced+//! `tool_choice` with a 400, so they get the default `auto`, and a reply that+//! carries the JSON as text instead of a tool call is read too.++use ai_memory_llm::{AnthropicProvider, ChatRequest, LlmError, LlmProvider};+use secrecy::SecretString;+use serde_json::{Value, json};+use wiremock::matchers::{method, path};+use wiremock::{Mock, MockServer, ResponseTemplate};++fn tool_use_reply() -> Value {+ json!([{+ "type": "tool_use",+ "id": "toolu_1",+ "name": "result",+ "input": {"answer": "ok"}+ }])+}++fn text_reply(text: &str) -> Value {+ json!([{"type": "text", "text": text}])+}++/// Serve `content` as the `/v1/messages` reply, run one structured call for+/// `model`, and return its outcome with the body the provider sent.+async fn structured_call(model: &str, content: Value) -> (Result<Value, LlmError>, Value) {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/messages"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "model": model,+ "content": content,+ "usage": {"input_tokens": 1, "output_tokens": 1}+ })))+ .expect(1)+ .mount(&server)+ .await;+ let provider = AnthropicProvider::new(SecretString::from("sk-ant-test"), model)+ .unwrap()+ .with_base_url(server.uri());+ let schema = json!({"type": "object", "properties": {"answer": {"type": "string"}}});+ let outcome = provider+ .complete_structured_raw(ChatRequest::user_prompt("summarize"), schema)+ .await;+ let received = server+ .received_requests()+ .await+ .expect("mock recorded requests");+ let body: Value = serde_json::from_slice(&received[0].body).unwrap();+ (outcome, body)+}++#[tokio::test]+async fn models_that_reject_a_forced_tool_choice_get_auto() {+ for model in [+ "claude-opus-5-5",+ "claude-sonnet-5-5",+ "claude-fable-5-1",+ "claude-mythos-5-1",+ ] {+ let (outcome, body) = structured_call(model, tool_use_reply()).await;+ assert!(body.get("tool_choice").is_none(), "{model}: {body}");+ assert_eq!(body["tools"][0]["name"], "result", "{model}");+ let description = body["tools"][0]["description"].as_str().unwrap();+ assert!(description.contains("Call this tool"), "{description}");+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}), "{model}");+ }+}++#[tokio::test]+async fn models_that_accept_a_forced_tool_choice_keep_it() {+ for model in [+ "claude-opus-5",+ "claude-sonnet-5",+ "claude-fable-5",+ "claude-haiku-5-5",+ "claude-sonnet-4-6",+ ] {+ let (outcome, body) = structured_call(model, tool_use_reply()).await;+ assert_eq!(+ body["tool_choice"],+ json!({"type": "tool", "name": "result"}),+ "{model}"+ );+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}), "{model}");+ }+}++#[tokio::test]+async fn an_unforced_call_reads_a_json_reply_given_as_text() {+ let fenced = text_reply("```json\n{\"answer\": \"ok\"}\n```");+ let (outcome, _) = structured_call("claude-opus-5-5", fenced).await;+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}));++ let in_prose = text_reply("Here it is: {\"answer\": \"ok\"}");+ let (outcome, _) = structured_call("claude-sonnet-5-5", in_prose).await;+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}));+}++#[tokio::test]+async fn a_reply_without_a_tool_call_or_json_is_still_an_error() {+ let prose = text_reply("I cannot produce that.");+ let (outcome, _) = structured_call("claude-opus-5-5", prose).await;+ assert!(+ matches!(outcome, Err(LlmError::UnexpectedShape(_))),+ "{outcome:?}"+ );++ // A forced call keeps its contract: text is never read as the result.+ let json_text = text_reply("{\"answer\": \"ok\"}");+ let (outcome, _) = structured_call("claude-opus-5", json_text).await;+ assert!(+ matches!(outcome, Err(LlmError::UnexpectedShape(_))),+ "{outcome:?}"+ );+}diff --git a/crates/ai-memory-llm/tests/suite/mod.rs b/crates/ai-memory-llm/tests/suite/mod.rsindex 1c69f68f..cd14007b 100644--- a/crates/ai-memory-llm/tests/suite/mod.rs+++ b/crates/ai-memory-llm/tests/suite/mod.rs@@ -2,6 +2,7 @@//! test harness (see the `integration` module in `src/lib.rs`), so they cost//! no extra binary; a new file must be declared below.+mod anthropic_provider;mod copilot_embedder;mod copilot_provider;mod cursor_provider;