Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: temporarily disable structured outputs for anthropic (#8491)
Sarah Wooders0cbdf452fa18
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.pyindex 777084719..2d213478f 100644--- a/letta/llm_api/anthropic_client.py+++ b/letta/llm_api/anthropic_client.py@@ -85,13 +85,15 @@ class AnthropicClient(LLMClientBase):# Structured outputs beta - only for supported models# Supported: Claude Sonnet 4.5, Opus 4.1, Opus 4.5, Haiku 4.5- supports_structured_outputs = _supports_structured_outputs(llm_config.model)-- if supports_structured_outputs:- # Always enable structured outputs beta on supported models.- # NOTE: We do NOT send `strict` on tool schemas because the current Anthropic SDK- # typed tool params reject unknown fields (e.g., `tools.0.custom.strict`).- betas.append("structured-outputs-2025-11-13")+ # DISABLED: Commenting out structured outputs to investigate TTFT latency impact+ # See PR #7495 for original implementation+ # supports_structured_outputs = _supports_structured_outputs(llm_config.model)+ #+ # if supports_structured_outputs:+ # # Always enable structured outputs beta on supported models.+ # # NOTE: We do NOT send `strict` on tool schemas because the current Anthropic SDK+ # # typed tool params reject unknown fields (e.g., `tools.0.custom.strict`).+ # betas.append("structured-outputs-2025-11-13")if betas:response = client.beta.messages.create(**request_data, betas=betas)@@ -127,10 +129,12 @@ class AnthropicClient(LLMClientBase):betas.append("context-management-2025-06-27")# Structured outputs beta - only for supported models- supports_structured_outputs = _supports_structured_outputs(llm_config.model)-- if supports_structured_outputs:- betas.append("structured-outputs-2025-11-13")+ # DISABLED: Commenting out structured outputs to investigate TTFT latency impact+ # See PR #7495 for original implementation+ # supports_structured_outputs = _supports_structured_outputs(llm_config.model)+ #+ # if supports_structured_outputs:+ # betas.append("structured-outputs-2025-11-13")if betas:response = await client.beta.messages.create(**request_data, betas=betas)@@ -174,10 +178,12 @@ class AnthropicClient(LLMClientBase):betas.append("context-management-2025-06-27")# Structured outputs beta - only for supported models- supports_structured_outputs = _supports_structured_outputs(llm_config.model)-- if supports_structured_outputs:- betas.append("structured-outputs-2025-11-13")+ # DISABLED: Commenting out structured outputs to investigate TTFT latency impact+ # See PR #7495 for original implementation+ # supports_structured_outputs = _supports_structured_outputs(llm_config.model)+ #+ # if supports_structured_outputs:+ # betas.append("structured-outputs-2025-11-13")# log failed requeststry:@@ -378,11 +384,13 @@ class AnthropicClient(LLMClientBase):}# Structured outputs via response_format- if hasattr(llm_config, "response_format") and isinstance(llm_config.response_format, JsonSchemaResponseFormat):- data["output_format"] = {- "type": "json_schema",- "schema": llm_config.response_format.json_schema["schema"],- }+ # DISABLED: Commenting out structured outputs to investigate TTFT latency impact+ # See PR #7495 for original implementation+ # if hasattr(llm_config, "response_format") and isinstance(llm_config.response_format, JsonSchemaResponseFormat):+ # data["output_format"] = {+ # "type": "json_schema",+ # "schema": llm_config.response_format.json_schema["schema"],+ # }# Tools# For an overview on tool choice:@@ -432,9 +440,11 @@ class AnthropicClient(LLMClientBase):if tools_for_request and len(tools_for_request) > 0:# TODO eventually enable parallel tool use+ # DISABLED: use_strict=False to disable structured outputs (TTFT latency impact)+ # See PR #7495 for original implementationdata["tools"] = convert_tools_to_anthropic_format(tools_for_request,- use_strict=_supports_structured_outputs(llm_config.model),+ use_strict=False, # Was: _supports_structured_outputs(llm_config.model))# Add cache control to the last tool for caching tool definitionsif len(data["tools"]) > 0:@@ -693,8 +703,10 @@ class AnthropicClient(LLMClientBase):betas.append("context-management-2025-06-27")# Structured outputs beta - only for supported models- if model and _supports_structured_outputs(model):- betas.append("structured-outputs-2025-11-13")+ # DISABLED: Commenting out structured outputs to investigate TTFT latency impact+ # See PR #7495 for original implementation+ # if model and _supports_structured_outputs(model):+ # betas.append("structured-outputs-2025-11-13")if betas:result = await client.beta.messages.count_tokens(**count_params, betas=betas)diff --git a/tests/integration_test_send_message_v2.py b/tests/integration_test_send_message_v2.pyindex 08d2785d0..e383ecc80 100644--- a/tests/integration_test_send_message_v2.py+++ b/tests/integration_test_send_message_v2.py@@ -1008,7 +1008,7 @@ async def test_conversation_streaming_raw_http([("openai/gpt-4o", "openai"),("openai/gpt-5", "openai"),- ("anthropic/claude-sonnet-4-5-20250929", "anthropic"),+ # ("anthropic/claude-sonnet-4-5-20250929", "anthropic"),],)@pytest.mark.asyncio(loop_scope="function")