Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(anthropic): refuse a structured tool_use cut off at max_tokens
Rudy Mizrahi Celekli48e6333cda5d
diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rsindex fb5a0587..25c6afce 100644--- a/crates/ai-memory-llm/src/anthropic.rs+++ b/crates/ai-memory-llm/src/anthropic.rs@@ -211,6 +211,11 @@ enum AnthropicToolChoice {struct AnthropicResponse {content: Vec<AnthropicContent>,model: String,+ /// Why the model stopped. `"max_tokens"` is the only value either+ /// completion path reads, and the field is absent from the error shape,+ /// so it defaults.+ #[serde(default)]+ stop_reason: Option<String>,#[serde(default)]usage: Option<AnthropicUsage>,}@@ -297,6 +302,22 @@ impl LlmProvider for AnthropicProvider {) -> LlmResult<serde_json::Value> {let body = self.build_request(&request, Some(schema));let response: AnthropicResponse = self.post(&body, operation_id).await?;+ // A `max_tokens` stop on the structured path means the model ran out+ // of output budget part-way through the answer, so a `tool_use`+ // block's `input` is whatever JSON had been produced by then. The+ // batch types this feeds leave every field but `updates`+ // `#[serde(default)]` and the Anthropic tool schema does not enforce+ // required fields, so a short object still deserializes and the caller+ // cannot tell the answer from a complete one. Terminal, and reported+ // as the same class `openai.rs` uses for `finish_reason = "length"`: a+ // retry re-truncates at the same budget. `complete` is deliberately+ // left alone — prose cut at the budget is a usable answer there.+ if response.stop_reason.as_deref() == Some("max_tokens") {+ return Err(LlmError::TruncatedResponse {+ model: self.model.clone(),+ completion_tokens: response.usage.as_ref().map(|usage| usage.output_tokens),+ });+ }let forced = self.forces_result_tool();let mut text = Vec::new();for c in response.content {diff --git a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs b/crates/ai-memory-llm/tests/suite/anthropic_provider.rsindex b741aa1c..bb764c63 100644--- a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs+++ b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs@@ -120,3 +120,62 @@ async fn a_reply_without_a_tool_call_or_json_is_still_an_error() {"{outcome:?}");}++/// A structured answer the model stopped at `max_tokens` is terminal: the+/// `tool_use` block's `input` carries only the JSON produced before the budget+/// ran out, and the batch types it feeds default every field but `updates`, so+/// a short object deserializes and reads as a complete answer. The control is+/// `complete`, where prose cut at the budget is a usable reply and must still+/// succeed on the very same response.+#[tokio::test]+async fn a_structured_answer_stopped_at_max_tokens_is_terminal_truncated() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/messages"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "model": "claude-sonnet-5",+ "stop_reason": "max_tokens",+ "content": [+ {"type": "text", "text": "partial prose"},+ {+ "type": "tool_use",+ "id": "toolu_1",+ "name": "result",+ "input": {"updates": [{"path": "notes/a.md"}]}+ }+ ],+ "usage": {"input_tokens": 7, "output_tokens": 16000}+ })))+ .expect(2)+ .mount(&server)+ .await;+ let provider = AnthropicProvider::new(SecretString::from("sk-ant-test"), "claude-sonnet-5")+ .unwrap()+ .with_base_url(server.uri());++ let schema = json!({"type": "object", "properties": {"updates": {"type": "array"}}});+ let err = provider+ .complete_structured_raw(ChatRequest::user_prompt("consolidate"), schema)+ .await+ .expect_err("a budget stop must not return a partial structured result");+ match &err {+ LlmError::TruncatedResponse {+ model,+ completion_tokens,+ } => {+ assert_eq!(model, "claude-sonnet-5");+ assert_eq!(*completion_tokens, Some(16000));+ }+ other => panic!("expected TruncatedResponse, got {other:?}"),+ }+ assert_eq!(err.class(), "truncated-response");+ assert!(!err.is_transient(), "a budget stop must not be retried");++ // Control: the non-structured path reads the same `stop_reason` as a+ // normal outcome and returns the prose it did get.+ let response = provider+ .complete(ChatRequest::user_prompt("summarize"))+ .await+ .expect("prose cut at the budget is still an answer");+ assert_eq!(response.text, "partial prose");+}