Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(llm): stop forcing the structured tool on Claude models that reject it

Vinícius DS Andrade8207d3da4dc0
diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rs
index 150d2879..4d6546e3 100644
--- a/crates/ai-memory-llm/src/anthropic.rs
+++ b/crates/ai-memory-llm/src/anthropic.rs
@@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize};
use tracing::debug;
use crate::error::{LlmError, LlmResult};
+use crate::openai_compat::{first_json_object, strip_reasoning_blocks};
use crate::provider::LlmProvider;
use crate::response::{provider_error_body, response_json_limited};
use crate::types::{
@@ -278,21 +279,39 @@ impl LlmProvider for AnthropicProvider {
) -> LlmResult<serde_json::Value> {
let body = self.build_request(&request, Some(schema));
let response: AnthropicResponse = self.post(&body, operation_id).await?;
+ let forced = self.forces_result_tool();
+ let mut text = Vec::new();
for c in response.content {
- if let AnthropicContent::ToolUse { input, .. } = c {
- return Ok(input);
+ match c {
+ AnthropicContent::ToolUse { input, .. } => return Ok(input),
+ // Under `auto` the model can reply with the JSON itself
+ // instead of calling the tool.
+ AnthropicContent::Text { text: part } if !forced => text.push(part),
+ _ => {}
}
}
+ if let Some(value) = json_object_in(&text.join("\n")) {
+ return Ok(value);
+ }
Err(LlmError::UnexpectedShape(
"anthropic response had no tool_use block".into(),
))
}
}
+/// The JSON object in a text reply, bare, fenced, or set in prose.
+fn json_object_in(text: &str) -> Option<serde_json::Value> {
+ let cleaned = strip_reasoning_blocks(text);
+ serde_json::from_str::<serde_json::Value>(&cleaned)
+ .ok()
+ .filter(serde_json::Value::is_object)
+ .or_else(|| serde_json::from_str(first_json_object(&cleaned)?).ok())
+}
+
impl AnthropicProvider {
/// Build the `/v1/messages` body shared by `complete` and
/// `complete_structured_raw`. Passing a schema turns the call into the
- /// forced-tool structured-output shape. Both paths go through here so the
+ /// `result`-tool structured-output shape. Both paths go through here so the
/// temperature rule below can't apply to one and silently miss the other.
fn build_request<'a>(
&'a self,
@@ -307,14 +326,22 @@ impl AnthropicProvider {
content: &m.content,
})
.collect();
+ let forced = self.forces_result_tool();
+ // Without a forced `tool_choice`, the description is what asks for
+ // the call.
+ let description = if forced {
+ "Emit the structured result."
+ } else {
+ "Emit the structured result. Call this tool once instead of replying in text."
+ };
let (tools, tool_choice) = match schema {
Some(input_schema) => (
Some(vec![AnthropicTool {
name: "result".into(),
- description: "Emit the structured result.".into(),
+ description: description.into(),
input_schema,
}]),
- Some(AnthropicToolChoice::Tool {
+ forced.then(|| AnthropicToolChoice::Tool {
name: "result".into(),
}),
),
@@ -342,6 +369,14 @@ impl AnthropicProvider {
}
}
+ /// Whether the structured path pins `tool_choice` to the `result` tool.
+ /// The models that answer a forced `tool_choice` with a 400 keep the
+ /// default `auto`.
+ fn forces_result_tool(&self) -> bool {
+ let claude = ClaudeId::parse(&self.model);
+ !claude.is_some_and(ClaudeId::rejects_forced_tool_choice)
+ }
+
async fn post<B: Serialize, R: DeserializeOwned>(
&self,
body: &B,
@@ -511,6 +546,25 @@ impl ClaudeId {
self.meets(Some((4, 7)))
}
+ /// Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 400 on a forced
+ /// `tool_choice` (`any` or a named tool). Haiku 5.5, Opus 5, Sonnet 5,
+ /// and Fable 5 still accept one.
+ fn rejects_forced_tool_choice(self) -> bool {
+ match self {
+ Self::Released {
+ family: ClaudeFamily::Opus | ClaudeFamily::Sonnet,
+ major,
+ minor,
+ } => (major, minor) >= (5, 5),
+ Self::Released {
+ family: ClaudeFamily::Fable | ClaudeFamily::Mythos,
+ major,
+ minor,
+ } => (major, minor) >= (5, 1),
+ _ => false,
+ }
+ }
+
fn supports_effort(self) -> bool {
self.meets(self.family().effort_since())
}
diff --git a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
new file mode 100644
index 00000000..2b20a540
--- /dev/null
+++ b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
@@ -0,0 +1,119 @@
+//! Wire tests for the Anthropic provider's structured output: the request it
+//! sends each Claude generation and the replies it reads back.
+//!
+//! Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 answer a forced
+//! `tool_choice` with a 400, so they get the default `auto`, and a reply that
+//! carries the JSON as text instead of a tool call is read too.
+
+use ai_memory_llm::{AnthropicProvider, ChatRequest, LlmError, LlmProvider};
+use secrecy::SecretString;
+use serde_json::{Value, json};
+use wiremock::matchers::{method, path};
+use wiremock::{Mock, MockServer, ResponseTemplate};
+
+fn tool_use_reply() -> Value {
+ json!([{
+ "type": "tool_use",
+ "id": "toolu_1",
+ "name": "result",
+ "input": {"answer": "ok"}
+ }])
+}
+
+fn text_reply(text: &str) -> Value {
+ json!([{"type": "text", "text": text}])
+}
+
+/// Serve `content` as the `/v1/messages` reply, run one structured call for
+/// `model`, and return its outcome with the body the provider sent.
+async fn structured_call(model: &str, content: Value) -> (Result<Value, LlmError>, Value) {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/messages"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "model": model,
+ "content": content,
+ "usage": {"input_tokens": 1, "output_tokens": 1}
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let provider = AnthropicProvider::new(SecretString::from("sk-ant-test"), model)
+ .unwrap()
+ .with_base_url(server.uri());
+ let schema = json!({"type": "object", "properties": {"answer": {"type": "string"}}});
+ let outcome = provider
+ .complete_structured_raw(ChatRequest::user_prompt("summarize"), schema)
+ .await;
+ let received = server
+ .received_requests()
+ .await
+ .expect("mock recorded requests");
+ let body: Value = serde_json::from_slice(&received[0].body).unwrap();
+ (outcome, body)
+}
+
+#[tokio::test]
+async fn models_that_reject_a_forced_tool_choice_get_auto() {
+ for model in [
+ "claude-opus-5-5",
+ "claude-sonnet-5-5",
+ "claude-fable-5-1",
+ "claude-mythos-5-1",
+ ] {
+ let (outcome, body) = structured_call(model, tool_use_reply()).await;
+ assert!(body.get("tool_choice").is_none(), "{model}: {body}");
+ assert_eq!(body["tools"][0]["name"], "result", "{model}");
+ let description = body["tools"][0]["description"].as_str().unwrap();
+ assert!(description.contains("Call this tool"), "{description}");
+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}), "{model}");
+ }
+}
+
+#[tokio::test]
+async fn models_that_accept_a_forced_tool_choice_keep_it() {
+ for model in [
+ "claude-opus-5",
+ "claude-sonnet-5",
+ "claude-fable-5",
+ "claude-haiku-5-5",
+ "claude-sonnet-4-6",
+ ] {
+ let (outcome, body) = structured_call(model, tool_use_reply()).await;
+ assert_eq!(
+ body["tool_choice"],
+ json!({"type": "tool", "name": "result"}),
+ "{model}"
+ );
+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}), "{model}");
+ }
+}
+
+#[tokio::test]
+async fn an_unforced_call_reads_a_json_reply_given_as_text() {
+ let fenced = text_reply("```json\n{\"answer\": \"ok\"}\n```");
+ let (outcome, _) = structured_call("claude-opus-5-5", fenced).await;
+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}));
+
+ let in_prose = text_reply("Here it is: {\"answer\": \"ok\"}");
+ let (outcome, _) = structured_call("claude-sonnet-5-5", in_prose).await;
+ assert_eq!(outcome.unwrap(), json!({"answer": "ok"}));
+}
+
+#[tokio::test]
+async fn a_reply_without_a_tool_call_or_json_is_still_an_error() {
+ let prose = text_reply("I cannot produce that.");
+ let (outcome, _) = structured_call("claude-opus-5-5", prose).await;
+ assert!(
+ matches!(outcome, Err(LlmError::UnexpectedShape(_))),
+ "{outcome:?}"
+ );
+
+ // A forced call keeps its contract: text is never read as the result.
+ let json_text = text_reply("{\"answer\": \"ok\"}");
+ let (outcome, _) = structured_call("claude-opus-5", json_text).await;
+ assert!(
+ matches!(outcome, Err(LlmError::UnexpectedShape(_))),
+ "{outcome:?}"
+ );
+}
diff --git a/crates/ai-memory-llm/tests/suite/mod.rs b/crates/ai-memory-llm/tests/suite/mod.rs
index 1c69f68f..cd14007b 100644
--- a/crates/ai-memory-llm/tests/suite/mod.rs
+++ b/crates/ai-memory-llm/tests/suite/mod.rs
@@ -2,6 +2,7 @@
//! test harness (see the `integration` module in `src/lib.rs`), so they cost
//! no extra binary; a new file must be declared below.
+mod anthropic_provider;
mod copilot_embedder;
mod copilot_provider;
mod cursor_provider;