Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(anthropic): refuse a structured tool_use cut off at max_tokens

Rudy Mizrahi Celekli48e6333cda5d
diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rs
index fb5a0587..25c6afce 100644
--- a/crates/ai-memory-llm/src/anthropic.rs
+++ b/crates/ai-memory-llm/src/anthropic.rs
@@ -211,6 +211,11 @@ enum AnthropicToolChoice {
struct AnthropicResponse {
content: Vec<AnthropicContent>,
model: String,
+ /// Why the model stopped. `"max_tokens"` is the only value either
+ /// completion path reads, and the field is absent from the error shape,
+ /// so it defaults.
+ #[serde(default)]
+ stop_reason: Option<String>,
#[serde(default)]
usage: Option<AnthropicUsage>,
}
@@ -297,6 +302,22 @@ impl LlmProvider for AnthropicProvider {
) -> LlmResult<serde_json::Value> {
let body = self.build_request(&request, Some(schema));
let response: AnthropicResponse = self.post(&body, operation_id).await?;
+ // A `max_tokens` stop on the structured path means the model ran out
+ // of output budget part-way through the answer, so a `tool_use`
+ // block's `input` is whatever JSON had been produced by then. The
+ // batch types this feeds leave every field but `updates`
+ // `#[serde(default)]` and the Anthropic tool schema does not enforce
+ // required fields, so a short object still deserializes and the caller
+ // cannot tell the answer from a complete one. Terminal, and reported
+ // as the same class `openai.rs` uses for `finish_reason = "length"`: a
+ // retry re-truncates at the same budget. `complete` is deliberately
+ // left alone — prose cut at the budget is a usable answer there.
+ if response.stop_reason.as_deref() == Some("max_tokens") {
+ return Err(LlmError::TruncatedResponse {
+ model: self.model.clone(),
+ completion_tokens: response.usage.as_ref().map(|usage| usage.output_tokens),
+ });
+ }
let forced = self.forces_result_tool();
let mut text = Vec::new();
for c in response.content {
diff --git a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
index b741aa1c..bb764c63 100644
--- a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
+++ b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
@@ -120,3 +120,62 @@ async fn a_reply_without_a_tool_call_or_json_is_still_an_error() {
"{outcome:?}"
);
}
+
+/// A structured answer the model stopped at `max_tokens` is terminal: the
+/// `tool_use` block's `input` carries only the JSON produced before the budget
+/// ran out, and the batch types it feeds default every field but `updates`, so
+/// a short object deserializes and reads as a complete answer. The control is
+/// `complete`, where prose cut at the budget is a usable reply and must still
+/// succeed on the very same response.
+#[tokio::test]
+async fn a_structured_answer_stopped_at_max_tokens_is_terminal_truncated() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/messages"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "model": "claude-sonnet-5",
+ "stop_reason": "max_tokens",
+ "content": [
+ {"type": "text", "text": "partial prose"},
+ {
+ "type": "tool_use",
+ "id": "toolu_1",
+ "name": "result",
+ "input": {"updates": [{"path": "notes/a.md"}]}
+ }
+ ],
+ "usage": {"input_tokens": 7, "output_tokens": 16000}
+ })))
+ .expect(2)
+ .mount(&server)
+ .await;
+ let provider = AnthropicProvider::new(SecretString::from("sk-ant-test"), "claude-sonnet-5")
+ .unwrap()
+ .with_base_url(server.uri());
+
+ let schema = json!({"type": "object", "properties": {"updates": {"type": "array"}}});
+ let err = provider
+ .complete_structured_raw(ChatRequest::user_prompt("consolidate"), schema)
+ .await
+ .expect_err("a budget stop must not return a partial structured result");
+ match &err {
+ LlmError::TruncatedResponse {
+ model,
+ completion_tokens,
+ } => {
+ assert_eq!(model, "claude-sonnet-5");
+ assert_eq!(*completion_tokens, Some(16000));
+ }
+ other => panic!("expected TruncatedResponse, got {other:?}"),
+ }
+ assert_eq!(err.class(), "truncated-response");
+ assert!(!err.is_transient(), "a budget stop must not be retried");
+
+ // Control: the non-structured path reads the same `stop_reason` as a
+ // normal outcome and returns the prose it did get.
+ let response = provider
+ .complete(ChatRequest::user_prompt("summarize"))
+ .await
+ .expect("prose cut at the budget is still an answer");
+ assert_eq!(response.text, "partial prose");
+}