Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): add Claude Sonnet 5 support (Claude 5 tokenizer aware) (#13629)

Nicholas Tindle4a2b61457e65
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py
index a18567b406..067dacaac4 100644
--- a/autogpt_platform/backend/backend/blocks/llm.py
+++ b/autogpt_platform/backend/backend/blocks/llm.py
@@ -293,9 +293,13 @@ async def _llm_call(
context_window = llm_model.context_window
if compress_prompt_to_fit:
+ # Pass the model so compaction measures in the same corrected token
+ # space as the max_tokens sizing below — or Claude-5 prompts pass
+ # compaction on raw counts and overflow at serve time.
result = await compress_context(
messages=prompt,
target_tokens=llm_model.context_window // 2,
+ model=llm_model.value,
client=None, # Truncation-only, no LLM summarization
reserve=0, # Caller handles response token budget separately
)
@@ -320,7 +324,7 @@ async def _llm_call(
)
# Calculate available tokens based on context window and input length
- estimated_input_tokens = estimate_token_count(prompt)
+ estimated_input_tokens = estimate_token_count(prompt, model=llm_model.value)
model_max_output = llm_model.max_output_tokens or int(2**15)
user_max = max_tokens or model_max_output
available_tokens = max(context_window - estimated_input_tokens, 0)
diff --git a/autogpt_platform/backend/backend/blocks/test/test_llm.py b/autogpt_platform/backend/backend/blocks/test/test_llm.py
index 10156b90e7..46a5333008 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_llm.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_llm.py
@@ -1605,3 +1605,61 @@ class TestLLMModelMissingHandler:
f"OpenRouter alias missing for {member.value} — expected slug "
f"{slug!r} to resolve back to {member.name}"
)
+
+
+class TestClaude5ModelThreading:
+ """The Claude-5 correction only works if ``llm_call`` threads the
+ selected model into BOTH token paths: compaction (``compress_context``)
+ and budget sizing (``estimate_token_count``). These pin the wiring —
+ the estimator behavior itself is covered in ``util/prompt_test.py``."""
+
+ @pytest.mark.asyncio
+ async def test_llm_call_threads_model_into_both_token_paths(self):
+ mock_response = MagicMock()
+ mock_response.output_text = "ok"
+ mock_response.output = []
+ mock_response.usage = MagicMock(input_tokens=1, output_tokens=1)
+
+ compress_result = MagicMock()
+ compress_result.error = None
+ compress_result.messages = [{"role": "user", "content": "hi"}]
+
+ with (
+ patch(
+ "backend.blocks.llm.compress_context",
+ new=AsyncMock(return_value=compress_result),
+ ) as mock_compress,
+ patch(
+ "backend.blocks.llm.estimate_token_count", return_value=100
+ ) as mock_estimate,
+ patch("openai.AsyncOpenAI") as mock_openai,
+ ):
+ mock_client = AsyncMock()
+ mock_openai.return_value = mock_client
+ mock_client.responses.create = AsyncMock(return_value=mock_response)
+
+ await llm.llm_call(
+ credentials=llm.TEST_CREDENTIALS,
+ llm_model=llm.DEFAULT_LLM_MODEL,
+ prompt=[{"role": "user", "content": "hi"}],
+ max_tokens=100,
+ compress_prompt_to_fit=True,
+ )
+
+ expected_model = llm.DEFAULT_LLM_MODEL.value
+ assert mock_compress.call_args.kwargs["model"] == expected_model
+ assert mock_estimate.call_args.kwargs["model"] == expected_model
+
+ @pytest.mark.asyncio
+ async def test_available_tokens_shrink_for_claude_5(self):
+ """Same context window, same prompt: the Claude-5 estimate must be
+ larger than the 4.6 estimate, shrinking max_tokens headroom."""
+ from backend.util.prompt import estimate_token_count
+
+ prompt = [{"role": "user", "content": "hello world " * 500}]
+ m46 = llm.LLMModel.CLAUDE_4_6_SONNET
+ m5 = llm.LLMModel.CLAUDE_5_SONNET
+ est_46 = estimate_token_count(prompt, model=m46.value)
+ est_5 = estimate_token_count(prompt, model=m5.value)
+ assert est_5 == int(est_46 * 1.5)
+ assert (m5.context_window - est_5) < (m46.context_window - est_46)
diff --git a/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py b/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py
index 55edde9e68..eac44bfb7c 100644
--- a/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py
+++ b/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py
@@ -1,6 +1,6 @@
"""Unit tests for the Anthropic rate card used in direct-mode cost computation."""
-from .anthropic_rate_card import compute_anthropic_cost_usd
+from .anthropic_rate_card import compute_anthropic_cost_usd, get_max_output_tokens
class TestComputeAnthropicCostUsd:
@@ -161,3 +161,26 @@ class TestComputeAnthropicCostUsd:
)
# fresh_input clamped to 0, cache_read = 1000 × $3 × 0.1 / M = $0.0003.
assert cost == 0.0003
+
+
+class TestSonnet5Rates:
+ """The refreshed rate card must carry claude-sonnet-5 — without it,
+ direct-Anthropic deployments cap output at the 32K fallback and bill
+ at Opus fallback rates (~7.5x)."""
+
+ def test_sonnet_5_max_output_from_rate_card(self):
+ assert get_max_output_tokens("claude-sonnet-5") == 128000
+
+ def test_sonnet_5_billed_at_litellm_rates_not_fallback(self):
+ cost = compute_anthropic_cost_usd(
+ model="claude-sonnet-5",
+ prompt_tokens=1_000_000,
+ completion_tokens=1_000_000,
+ )
+ # Intro rates in litellm today: 1M x $2 + 1M x $10 = $12 — the
+ # Opus fallback would charge $90. The catalog deliberately bills
+ # sticker ($3/$15); this asserts the vendored litellm value, so
+ # when the intro window ends (2026-08-31) and a rate refresh
+ # brings sticker rates, update this to 18.0 — that change is
+ # expected, not a mispricing.
+ assert cost == 12.0
diff --git a/autogpt_platform/backend/backend/copilot/anthropic_rates.json b/autogpt_platform/backend/backend/copilot/anthropic_rates.json
index 69848a1913..005e566fee 100644
--- a/autogpt_platform/backend/backend/copilot/anthropic_rates.json
+++ b/autogpt_platform/backend/backend/copilot/anthropic_rates.json
@@ -28,12 +28,11 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
- "supports_web_search": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_web_search": true
},
"claude-3-haiku-20240307": {
"cache_creation_input_token_cost": 3e-07,
- "cache_creation_input_token_cost_above_1hr": 5e-07,
+ "cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2.5e-07,
"litellm_provider": "anthropic",
@@ -47,12 +46,11 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 264
+ "supports_vision": true
},
"claude-3-opus-20240229": {
"cache_creation_input_token_cost": 1.875e-05,
- "cache_creation_input_token_cost_above_1hr": 3e-05,
+ "cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 1.5e-06,
"deprecation_date": "2026-05-01",
"input_cost_per_token": 1.5e-05,
@@ -67,8 +65,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 395
+ "supports_vision": true
},
"claude-4-opus-20250514": {
"cache_creation_input_token_cost": 1.875e-05,
@@ -80,6 +77,7 @@
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -93,8 +91,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-4-sonnet-20250514": {
"cache_creation_input_token_cost": 3.75e-06,
@@ -110,6 +107,7 @@
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -124,8 +122,42 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
- "supports_web_search": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_web_search": true
+ },
+ "claude-fable-5": {
+ "cache_creation_input_token_cost": 1.25e-05,
+ "cache_creation_input_token_cost_above_1hr": 2e-05,
+ "cache_read_input_token_cost": 1e-06,
+ "input_cost_per_token": 1e-05,
+ "litellm_provider": "anthropic",
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 128000,
+ "max_tokens": 128000,
+ "mode": "chat",
+ "output_cost_per_token": 5e-05,
+ "prompt_cache_min_tokens": 512,
+ "provider_specific_entry": {
+ "us": 1.1
+ },
+ "search_context_cost_per_query": {
+ "search_context_size_high": 0.01,
+ "search_context_size_low": 0.01,
+ "search_context_size_medium": 0.01
+ },
+ "supports_adaptive_thinking": true,
+ "supports_assistant_prefill": false,
+ "supports_computer_use": true,
+ "supports_function_calling": true,
+ "supports_max_reasoning_effort": true,
+ "supports_output_config": true,
+ "supports_pdf_input": true,
+ "supports_prompt_caching": true,
+ "supports_reasoning": true,
+ "supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_tool_choice": true,
+ "supports_vision": true,
+ "supports_xhigh_reasoning_effort": true
},
"claude-haiku-4-5": {
"cache_creation_input_token_cost": 1.25e-06,
@@ -138,6 +170,7 @@
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
+ "prompt_cache_min_tokens": 4096,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -159,6 +192,7 @@
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
+ "prompt_cache_min_tokens": 4096,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@@ -180,6 +214,7 @@
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -188,13 +223,13 @@
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
+ "supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-opus-4-1-20250805": {
"cache_creation_input_token_cost": 1.875e-05,
@@ -208,6 +243,7 @@
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -216,13 +252,13 @@
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
+ "supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-opus-4-20250514": {
"cache_creation_input_token_cost": 1.875e-05,
@@ -236,6 +272,7 @@
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -249,8 +286,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-opus-4-5": {
"cache_creation_input_token_cost": 6.25e-06,
@@ -263,6 +299,7 @@
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 4096,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -271,14 +308,14 @@
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-opus-4-5-20251101": {
"cache_creation_input_token_cost": 6.25e-06,
@@ -291,6 +328,7 @@
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 4096,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -299,14 +337,14 @@
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25e-06,
@@ -319,6 +357,7 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 4096,
"provider_specific_entry": {
"fast": 6.0,
"us": 1.1
@@ -328,18 +367,20 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
+ "supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_speed": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_vision": true
},
"claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
@@ -352,6 +393,7 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 4096,
"provider_specific_entry": {
"fast": 6.0,
"us": 1.1
@@ -361,18 +403,20 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
+ "supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_speed": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_vision": true
},
"claude-opus-4-7": {
"cache_creation_input_token_cost": 6.25e-06,
@@ -385,6 +429,7 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 2048,
"provider_specific_entry": {
"fast": 6.0,
"us": 1.1
@@ -394,19 +439,22 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
+ "supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_speed": true,
"supports_tool_choice": true,
"supports_vision": true,
- "supports_xhigh_reasoning_effort": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_xhigh_reasoning_effort": true
},
"claude-opus-4-7-20260416": {
"cache_creation_input_token_cost": 6.25e-06,
@@ -419,6 +467,7 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 2048,
"provider_specific_entry": {
"fast": 6.0,
"us": 1.1
@@ -428,19 +477,22 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
+ "supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_speed": true,
"supports_tool_choice": true,
"supports_vision": true,
- "supports_xhigh_reasoning_effort": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_xhigh_reasoning_effort": true
},
"claude-opus-4-8": {
"cache_creation_input_token_cost": 6.25e-06,
@@ -453,8 +505,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
+ "prompt_cache_min_tokens": 1024,
"provider_specific_entry": {
- "fast": 6.0,
+ "fast": 2.0,
"us": 1.1
},
"search_context_cost_per_query": {
@@ -462,19 +515,22 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
+ "supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
+ "supports_speed": true,
"supports_tool_choice": true,
"supports_vision": true,
- "supports_xhigh_reasoning_effort": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_xhigh_reasoning_effort": true
},
"claude-sonnet-4-20250514": {
"cache_creation_input_token_cost": 3.75e-06,
@@ -492,6 +548,7 @@
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -505,11 +562,12 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-sonnet-4-5": {
"cache_creation_input_token_cost": 3.75e-06,
+ "cache_creation_input_token_cost_above_1hr": 6e-06,
+ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost": 3e-07,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
@@ -522,6 +580,7 @@
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -530,16 +589,18 @@
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
+ "supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_vision": true
},
"claude-sonnet-4-5-20250929": {
"cache_creation_input_token_cost": 3.75e-06,
+ "cache_creation_input_token_cost_above_1hr": 6e-06,
+ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost": 3e-07,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
@@ -552,6 +613,7 @@
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@@ -560,17 +622,19 @@
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
+ "supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
- "supports_web_search": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_web_search": true
},
"claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
+ "cache_creation_input_token_cost_above_1hr": 6e-06,
+ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost": 3e-07,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
@@ -583,19 +647,21 @@
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "prompt_cache_min_tokens": 1024,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
+ "supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 159
+ "supports_vision": true
},
"claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.75e-06,
+ "cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "anthropic",
@@ -604,53 +670,60 @@
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
+ "prompt_cache_min_tokens": 1024,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
+ "supports_adaptive_thinking": true,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_native_structured_output": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
- "supports_vision": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_vision": true
},
"claude-sonnet-5": {
- "cache_creation_input_token_cost": 3.75e-06,
- "cache_creation_input_token_cost_above_1hr": 6e-06,
- "cache_read_input_token_cost": 3e-07,
- "input_cost_per_token": 3e-06,
+ "cache_creation_input_token_cost": 2.5e-06,
+ "cache_creation_input_token_cost_above_1hr": 4e-06,
+ "cache_read_input_token_cost": 2e-07,
+ "input_cost_per_token": 2e-06,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
- "output_cost_per_token": 1.5e-05,
+ "output_cost_per_token": 1e-05,
+ "prompt_cache_min_tokens": 1024,
+ "provider_specific_entry": {
+ "us": 1.1
+ },
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
+ "supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
- "supports_minimal_reasoning_effort": true,
+ "supports_output_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
+ "supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
- "supports_xhigh_reasoning_effort": true,
- "tool_use_system_prompt_tokens": 346
+ "supports_xhigh_reasoning_effort": true
}
}
}
diff --git a/autogpt_platform/backend/backend/copilot/baseline/reasoning.py b/autogpt_platform/backend/backend/copilot/baseline/reasoning.py
index 7424a7a1a6..cc9fe2de5c 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/reasoning.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/reasoning.py
@@ -39,6 +39,10 @@ from backend.copilot.response_model import (
StreamReasoningEnd,
StreamReasoningStart,
)
+from backend.data.llm_registry.llm_models import (
+ CLAUDE_5_FAMILY_PREFIXES,
+ strip_anthropic_vendor_prefix,
+)
logger = logging.getLogger(__name__)
@@ -215,6 +219,14 @@ def reasoning_extra_body(model: str, max_thinking_tokens: int) -> dict[str, Any]
"""
if not _is_reasoning_route(model) or max_thinking_tokens <= 0:
return None
+ if _is_claude_5_family(model):
+ # Claude 5 removed ``budget_tokens`` (adaptive thinking only) and
+ # OpenRouter maps ``reasoning.max_tokens`` onto it — the request
+ # would 400. Send nothing: omitting the field runs the family's
+ # documented adaptive default. Before routing copilot traffic to a
+ # 5-family model, live-probe whether OpenRouter accepts the
+ # ``reasoning.effort`` shape and wire it here.
+ return None
return {"reasoning": {"max_tokens": max_thinking_tokens}}
@@ -225,21 +237,26 @@ def reasoning_extra_body(model: str, max_thinking_tokens: int) -> dict[str, Any]
# output cap (4096 on Haiku) also 400s — so we narrow to the families
# that actually support extended thinking instead of broad-matching
# ``"claude" in model``.
+# The Claude 5 family is deliberately NOT in this list: it runs ADAPTIVE
+# thinking by default and 400s on the ``budget_tokens`` fragment (removed
+# for the 5 family only — opus-4-7/4-8 share the 5-generation tokenizer
+# but still accept ``budget_tokens`` via the ``claude-opus-4`` prefix
+# below). ``anthropic_thinking_extra_body`` also guards this explicitly,
+# so adding a 5-family slug here cannot silently reintroduce the 400.
_THINKING_CAPABLE_PREFIXES: tuple[str, ...] = (
- "claude-opus-4", # 4.0 / 4.1 / 4.5 / 4.7
+ "claude-opus-4", # 4.0 / 4.1 / 4.5 / 4.7 / 4.8
"claude-sonnet-4", # 4.0 / 4.5 / 4.6
"claude-haiku-4", # 4.5
"claude-3-7", # 3.7 sonnet — only 3.x with thinking
)
+def _is_claude_5_family(model: str) -> bool:
+ return strip_anthropic_vendor_prefix(model).startswith(CLAUDE_5_FAMILY_PREFIXES)
+
+
def _is_thinking_capable_claude(model: str) -> bool:
- lowered = model.lower()
- for prefix in ("anthropic/", "anthropic."):
- if lowered.startswith(prefix):
- lowered = lowered[len(prefix) :]
- break
- return lowered.startswith(_THINKING_CAPABLE_PREFIXES)
+ return strip_anthropic_vendor_prefix(model).startswith(_THINKING_CAPABLE_PREFIXES)
def anthropic_thinking_extra_body(
@@ -260,6 +277,12 @@ def anthropic_thinking_extra_body(
"""
if not _is_reasoning_route(model) or max_thinking_tokens <= 0:
return None
+ if _is_claude_5_family(model):
+ # Symmetric with ``reasoning_extra_body``: Claude 5 removed
+ # ``budget_tokens`` (adaptive-only thinking) and 400s on the
+ # fragment. Guarded explicitly so adding a 5-family slug to
+ # ``_THINKING_CAPABLE_PREFIXES`` can't reintroduce the 400.
+ return None
if not _is_thinking_capable_claude(model):
return None
return {
diff --git a/autogpt_platform/backend/backend/copilot/baseline/reasoning_test.py b/autogpt_platform/backend/backend/copilot/baseline/reasoning_test.py
index a413dac27f..09407f1774 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/reasoning_test.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/reasoning_test.py
@@ -6,6 +6,7 @@ Covers the typed OpenRouter delta parser, the stateful emitter, and the
parser relies on is exercised end-to-end.
"""
+import pytest
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from backend.copilot.baseline.reasoning import (
@@ -573,3 +574,53 @@ class TestBaselineReasoningEmitterRenderFlag:
assert len(events) == 2
assert isinstance(events[0], StreamReasoningStart)
assert isinstance(events[1], StreamReasoningDelta)
+
+
+class TestClaude5FamilyThinking:
+ """Claude 5 removed ``budget_tokens`` — both thinking fragments must
+ stay off (the documented adaptive default serves), while 4.x keeps
+ its budget-capped fragments."""
+
+ @pytest.mark.parametrize(
+ "family_member",
+ ["claude-sonnet-5", "claude-fable-5", "claude-mythos-5"],
+ )
+ def test_openrouter_fragment_suppressed_for_5_family(self, family_member: str):
+ assert reasoning_extra_body(f"anthropic/{family_member}", 5000) is None
+
+ @pytest.mark.parametrize(
+ "family_member",
+ ["claude-sonnet-5", "claude-fable-5", "claude-mythos-5"],
+ )
+ def test_anthropic_fragment_suppressed_for_5_family(self, family_member: str):
+ assert anthropic_thinking_extra_body(family_member, 5000) is None
+
+ @pytest.mark.parametrize("model", ["claude-opus-4-7", "claude-opus-4-8"])
+ def test_opus_4_7_and_4_8_keep_budget_fragments(self, model: str):
+ """4.7/4.8 share the 5-generation tokenizer but NOT the
+ ``budget_tokens`` removal — they must keep the thinking fragment."""
+ assert anthropic_thinking_extra_body(model, 5000) == {
+ "thinking": {"type": "enabled", "budget_tokens": 5000}
+ }
+
+ def test_explicit_guard_wins_even_when_allowlisted(self, monkeypatch):
+ """The explicit ``_is_claude_5_family`` guard must suppress the
+ fragment even if a 5-family slug is (mistakenly) added to
+ ``_THINKING_CAPABLE_PREFIXES`` — otherwise the guard is dead code
+ and this suppression only holds by allowlist omission."""
+ from backend.copilot.baseline import reasoning
+
+ monkeypatch.setattr(
+ reasoning,
+ "_THINKING_CAPABLE_PREFIXES",
+ reasoning._THINKING_CAPABLE_PREFIXES + ("claude-sonnet-5",),
+ )
+ assert anthropic_thinking_extra_body("claude-sonnet-5", 5000) is None
+
+ def test_sonnet_4_6_keeps_budget_fragments(self):
+ assert reasoning_extra_body("anthropic/claude-sonnet-4.6", 5000) == {
+ "reasoning": {"max_tokens": 5000}
+ }
+ assert anthropic_thinking_extra_body("claude-sonnet-4-6", 5000) == {
+ "thinking": {"type": "enabled", "budget_tokens": 5000}
+ }
diff --git a/autogpt_platform/backend/backend/data/llm_registry/catalog.py b/autogpt_platform/backend/backend/data/llm_registry/catalog.py
index 7ea4e0a607..fa16ef8829 100644
--- a/autogpt_platform/backend/backend/data/llm_registry/catalog.py
+++ b/autogpt_platform/backend/backend/data/llm_registry/catalog.py
@@ -185,6 +185,35 @@ def _build_catalog() -> CatalogPayload:
cache_creation_credits_per_1m=563.0,
),
),
+ CatalogModel(
+ slug="claude-sonnet-5",
+ display_name="Claude Sonnet 5",
+ provider="anthropic",
+ creator="anthropic",
+ # True window is 1M; capped at 200K like every Anthropic
+ # entry until the Claude-5 tokenizer correction factor has
+ # soaked (the family tokenizes ~30% denser than 4.x, so
+ # estimate-based compaction targets need the margin).
+ context_window=200000,
+ max_output_tokens=128000,
+ price_tier=3,
+ supports_tools=True,
+ supports_json_output=True,
+ supports_reasoning=True,
+ # Sticker price ($3/$15 per Mtok; intro pricing $2/$10 ends
+ # 2026-08-31 — pricing at sticker avoids a repricing PR).
+ # Credits at the standard 1.5x margin; cache read $0.30,
+ # cache write (5m) $3.75.
+ cost=CatalogModelCost(
+ run_credits=9,
+ input_credits_per_1m=450.0,
+ output_credits_per_1m=2250.0,
+ cache_read_credits_per_1m=45.0,
+ cache_creation_credits_per_1m=563.0,
+ provider_input_usd_per_1m=3.00,
+ provider_output_usd_per_1m=15.00,
+ ),
+ ),
# ----- Groq -----
CatalogModel(
slug="llama-3.1-8b-instant",
diff --git a/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py b/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py
index 52c4357d80..707eba0518 100644
--- a/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py
+++ b/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py
@@ -137,6 +137,20 @@ def test_kimi_k3_bills_at_authored_rates():
}
+def test_claude_sonnet_5_bills_at_authored_rates():
+ """Sonnet 5 (sticker $3/$15; intro pricing ends 2026-08-31) — flat tier
+ and per-1M projections must match the authored catalog entry."""
+ s5 = LLMModel("claude-sonnet-5")
+ assert MODEL_COST[s5] == 9
+ assert TOKEN_COST[s5].model_dump() == {
+ "input": 450.0,
+ "output": 2250.0,
+ "cache_read": 45.0,
+ "cache_creation": 563.0,
+ }
+ assert MODEL_METADATA[s5].max_output_tokens == 128000
+
+
def test_provider_usd_prices_are_all_or_nothing():
"""A half-authored provider USD price must refuse to construct — it
would silently underprice against the transport family default."""
diff --git a/autogpt_platform/backend/backend/data/llm_registry/llm_models.py b/autogpt_platform/backend/backend/data/llm_registry/llm_models.py
index d9b6c48e31..b77f1dd8d2 100644
--- a/autogpt_platform/backend/backend/data/llm_registry/llm_models.py
+++ b/autogpt_platform/backend/backend/data/llm_registry/llm_models.py
@@ -19,6 +19,45 @@ from backend.data.llm_registry.catalog_model import CatalogPayload
logger = logging.getLogger(__name__)
+# The Claude 5 model family — one definition, shared by temperature
+# stripping and thinking-fragment suppression (tokenizer estimation keys
+# off the derived CLAUDE_5_TOKENIZER_GENERATION_PREFIXES below, which
+# additionally covers 4.7/4.8). SHIPPED members only: family members
+# don't all release at once, and an unreleased slug pre-listed here would
+# skip re-verification when it actually ships. Add each member here as
+# Anthropic releases it, after checking its documented behavior matches
+# the family.
+CLAUDE_5_FAMILY_PREFIXES: tuple[str, ...] = (
+ "claude-sonnet-5",
+ "claude-fable-5",
+ "claude-mythos-5",
+)
+
+# The tokenizer generation introduced with Opus 4.7 (shared by the whole
+# 5-family): ~30% denser than the 4.x tokenizer for the same text.
+CLAUDE_5_TOKENIZER_GENERATION_PREFIXES: tuple[str, ...] = CLAUDE_5_FAMILY_PREFIXES + (
+ "claude-opus-4-7",
+ "claude-opus-4-8",
+)
+
+
+def strip_anthropic_vendor_prefix(model: str) -> str:
+ """``anthropic/claude-x``, ``openrouter/anthropic/claude-x``,
+ ``anthropic.claude-x`` and ``us.anthropic.claude-x`` → ``claude-x``.
+
+ The single normalization used by every family-prefix check, so no
+ vendor-prefixed form (slash-routed or dotted Bedrock-style, with or
+ without a region prefix) can slip past one consumer while matching
+ another.
+ """
+ lowered = model.lower().split("/")[-1]
+ marker = "anthropic."
+ idx = lowered.find(marker)
+ if idx != -1:
+ return lowered[idx + len(marker) :]
+ return lowered
+
+
# Anthropic snapshot-date suffix (claude-haiku-4-5-20251001 → -20251001).
# Shared by every slug canonicalizer so the pattern can't drift.
MODEL_DATE_SUFFIX_RE = re.compile(r"-\d{8}$")
@@ -134,6 +173,7 @@ class LLMModel(str, Enum, metaclass=LLMModelMeta):
CLAUDE_4_6_OPUS = "claude-opus-4-6"
CLAUDE_4_7_OPUS = "claude-opus-4-7"
CLAUDE_4_6_SONNET = "claude-sonnet-4-6"
+ CLAUDE_5_SONNET = "claude-sonnet-5"
# AI/ML API models
AIML_API_LLAMA3_3_70B = "meta-llama/Llama-3.3-70B-Instruct-Turbo"
# Groq models
diff --git a/autogpt_platform/backend/backend/util/llm/providers.py b/autogpt_platform/backend/backend/util/llm/providers.py
index 85ec502502..509131aa66 100644
--- a/autogpt_platform/backend/backend/util/llm/providers.py
+++ b/autogpt_platform/backend/backend/util/llm/providers.py
@@ -45,6 +45,10 @@ from openai.types.shared_params import ResponseFormatJSONObject
from pydantic import Field
from pydantic.dataclasses import dataclass
+from backend.data.llm_registry.llm_models import (
+ CLAUDE_5_FAMILY_PREFIXES,
+ strip_anthropic_vendor_prefix,
+)
from backend.util.clients import OPENROUTER_BASE_URL
from backend.util.llm.conversions import (
ToolCall,
@@ -109,19 +113,34 @@ _FLEX_SUPPORTED_PROVIDERS: set[str] = {"openai", "open_router"}
# error, so unknown future models self-heal; this list only matters
# for batch submissions, whose errors come back hours later in the
# result rows.
+# 4.7/4.8 verified live; the whole Claude 5 family per litellm
+# supports_sampling_params=false. Load-bearing for batch submissions
+# (dream), which have no retry self-heal. Membership currently coincides
+# with CLAUDE_5_TOKENIZER_GENERATION_PREFIXES (both traits shipped with
+# the 4.7 generation) but is maintained separately: temperature rejection
+# is verified per-model against the live API, not inferred from the
+# tokenizer lineage.
_ANTHROPIC_TEMPERATURE_DEPRECATED_PREFIXES = (
"claude-opus-4-7",
"claude-opus-4-8",
-)
+) + CLAUDE_5_FAMILY_PREFIXES
def _anthropic_accepts_temperature(model: str) -> bool:
- return not model.startswith(_ANTHROPIC_TEMPERATURE_DEPRECATED_PREFIXES)
+ return not strip_anthropic_vendor_prefix(model).startswith(
+ _ANTHROPIC_TEMPERATURE_DEPRECATED_PREFIXES
+ )
def _is_temperature_deprecation_error(exc: anthropic.BadRequestError) -> bool:
+ # The Claude 5 family's rejection wording is unverified (4.7/4.8 say
+ # "deprecated") — match the sibling phrasings so the sync-path
+ # self-heal fires for future families regardless of the exact word.
error_text = str(exc).lower()
- return "temperature" in error_text and "deprecated" in error_text
+ return "temperature" in error_text and any(
+ phrase in error_text
+ for phrase in ("deprecated", "not supported", "unsupported", "removed")
+ )
# ---------------------------------------------------------------------------
diff --git a/autogpt_platform/backend/backend/util/llm/providers_test.py b/autogpt_platform/backend/backend/util/llm/providers_test.py
index ebc6e149b1..5b59cfef4e 100644
--- a/autogpt_platform/backend/backend/util/llm/providers_test.py
+++ b/autogpt_platform/backend/backend/util/llm/providers_test.py
@@ -14,11 +14,14 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import anthropic
+import httpx
import pytest
from backend.util.llm.providers import (
DEFAULT_REQUEST_TIMEOUT_SECONDS,
ProviderResponse,
+ _anthropic_accepts_temperature,
+ _is_temperature_deprecation_error,
call_provider,
)
@@ -2093,3 +2096,73 @@ class TestAnthropicTemperatureDeprecation:
)
params = async_create.call_args.kwargs["requests"][0]["params"]
assert params["temperature"] == 0.2
+
+
+class TestClaude5TemperatureRejection:
+ """The Claude 5 family 400s on non-default sampling params — the strip
+ list is load-bearing for batch (dream) submissions, which have no
+ retry self-heal."""
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "claude-sonnet-5",
+ "claude-fable-5",
+ "claude-mythos-5",
+ "claude-opus-4-7",
+ "claude-opus-4-8",
+ ],
+ )
+ def test_rejecting_family_members_stripped(self, model: str):
+ assert _anthropic_accepts_temperature(model) is False
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "anthropic/claude-sonnet-5",
+ "anthropic.claude-sonnet-5",
+ "openrouter/anthropic/claude-sonnet-5",
+ "us.anthropic.claude-sonnet-5",
+ ],
+ )
+ def test_vendor_prefixed_forms_stripped(self, model: str):
+ assert _anthropic_accepts_temperature(model) is False
+
+ @pytest.mark.parametrize(
+ "model",
+ ["claude-sonnet-4-6", "claude-opus-4-6", "anthropic/claude-sonnet-4-6"],
+ )
+ def test_accepting_models_keep_temperature(self, model: str):
+ assert _anthropic_accepts_temperature(model) is True
+
+ @staticmethod
+ def _err(msg: str) -> anthropic.BadRequestError:
+ resp = httpx.Response(
+ 400,
+ request=httpx.Request("POST", "https://api.anthropic.com"),
+ json={"error": {"message": msg}},
+ )
+ return anthropic.BadRequestError(msg, response=resp, body=None)
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "`temperature` is deprecated for this model",
+ "temperature is not supported by this model",
+ "temperature is unsupported on this model",
+ "the temperature parameter was removed for this model",
+ ],
+ )
+ def test_self_heal_matches_all_rejection_phrasings(self, message: str):
+ assert _is_temperature_deprecation_error(self._err(message))
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "rate limited",
+ # Phrase words without 'temperature' must not match.
+ "this model is deprecated",
+ ],
+ )
+ def test_self_heal_ignores_unrelated_errors(self, message: str):
+ assert not _is_temperature_deprecation_error(self._err(message))
diff --git a/autogpt_platform/backend/backend/util/prompt.py b/autogpt_platform/backend/backend/util/prompt.py
index bed7e6e8c9..95c0a07860 100644
--- a/autogpt_platform/backend/backend/util/prompt.py
+++ b/autogpt_platform/backend/backend/util/prompt.py
@@ -5,8 +5,12 @@ from copy import deepcopy
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
-from tiktoken import encoding_for_model
+from tiktoken import encoding_for_model, encoding_name_for_model
+from backend.data.llm_registry.llm_models import (
+ CLAUDE_5_TOKENIZER_GENERATION_PREFIXES,
+ strip_anthropic_vendor_prefix,
+)
from backend.util import json
if TYPE_CHECKING:
@@ -216,7 +220,8 @@ def estimate_token_count(
"""
token_model = _normalize_model_for_tokenizer(model)
enc = encoding_for_model(token_model)
- return sum(_msg_tokens(m, enc) for m in messages)
+ raw = sum(_msg_tokens(m, enc) for m in messages)
+ return int(raw * _token_estimate_factor(model))
def estimate_token_count_str(
@@ -240,7 +245,7 @@ def estimate_token_count_str(
token_model = _normalize_model_for_tokenizer(model)
enc = encoding_for_model(token_model)
text = json.dumps(text) if not isinstance(text, str) else text
- return _tok_len(text, enc)
+ return int(_tok_len(text, enc) * _token_estimate_factor(model))
# ---------------------------------------------------------------------------#
@@ -309,6 +314,25 @@ class CompressResult:
messages_dropped: int = 0
+# Estimation correction for the Claude 5 family (sonnet-5/fable-5/
+# mythos-5, plus the shared 4.7/4.8 tokenizer generation): Anthropic ships
+# no local tokenizer, so estimates ride tiktoken o200k_base, which already
+# undercounts Claude 4.x by ~15-20%; the Claude 5 tokenizer additionally
+# counts ~30% MORE tokens for the same text (Anthropic migration guide).
+# 1.5 ≈ 1.175 x 1.3 — deliberately on the high side: overestimating
+# shrinks compaction targets and max_tokens headroom (safe), while
+# underestimating overflows the context window (a 400 at serve time).
+CLAUDE_5_TOKEN_FACTOR = 1.5
+
+
+def _token_estimate_factor(model: str) -> float:
+ if strip_anthropic_vendor_prefix(model).startswith(
+ CLAUDE_5_TOKENIZER_GENERATION_PREFIXES
+ ):
+ return CLAUDE_5_TOKEN_FACTOR
+ return 1.0
+
+
def _normalize_model_for_tokenizer(model: str) -> str:
"""Normalize model name for tiktoken tokenizer selection."""
if "/" in model:
@@ -317,6 +341,13 @@ def _normalize_model_for_tokenizer(model: str) -> str:
known in model.lower() for known in ["gpt", "o1", "chatgpt", "text-"]
):
return "gpt-4o"
+ try:
+ encoding_name_for_model(model)
+ except KeyError:
+ # GPT-named model newer than the pinned tiktoken's mapping table
+ # (e.g. gpt-5.6-*) — estimates must degrade to the default
+ # encoding, not crash the LLM call.
+ return "gpt-4o"
return model
@@ -765,8 +796,17 @@ async def compress_context(
enc = encoding_for_model(token_model)
msgs = deepcopy(messages)
+ # Compaction must measure in the same (corrected) space as estimation:
+ # for the Claude-5 tokenizer generation, raw tiktoken sums undercount
+ # by ~30-50%, so an uncorrected total can sit under the nominal target
+ # while exceeding the model's real budget.
+ factor = _token_estimate_factor(model)
+
+ def _scaled_msg_tokens(m: dict) -> int:
+ return int(_msg_tokens(m, enc) * factor)
+
def total_tokens() -> int:
- return sum(_msg_tokens(m, enc) for m in msgs)
+ return sum(_scaled_msg_tokens(m) for m in msgs)
original_count = total_tokens()
@@ -852,12 +892,12 @@ async def compress_context(
# Steps 3-5 repeatedly check the total size while mutating or deleting one
# message at a time. Re-summing all messages there re-tokenizes the whole
# history on each iteration, which is quadratic on long transcripts.
- counts = [_msg_tokens(m, enc) for m in msgs]
+ counts = [_scaled_msg_tokens(m) for m in msgs]
total = sum(counts)
def _recount(i: int) -> None:
nonlocal total
- new = _msg_tokens(msgs[i], enc)
+ new = _scaled_msg_tokens(msgs[i])
total += new - counts[i]
counts[i] = new
@@ -927,7 +967,7 @@ async def compress_context(
# to prevent API errors (e.g., Anthropic's "unexpected tool_use_id").
final_msgs = validate_and_remove_orphan_tool_responses(final_msgs)
- final_count = sum(_msg_tokens(m, enc) for m in final_msgs)
+ final_count = sum(_scaled_msg_tokens(m) for m in final_msgs)
error = None
if final_count + reserve > target_tokens:
error = f"Could not compress below target ({final_count + reserve} > {target_tokens})"
diff --git a/autogpt_platform/backend/backend/util/prompt_test.py b/autogpt_platform/backend/backend/util/prompt_test.py
index 1d7aeee5ff..468c578483 100644
--- a/autogpt_platform/backend/backend/util/prompt_test.py
+++ b/autogpt_platform/backend/backend/util/prompt_test.py
@@ -17,6 +17,7 @@ from backend.util.prompt import (
_truncate_tool_message_content,
compress_context,
estimate_token_count,
+ estimate_token_count_str,
get_compression_target,
get_context_window,
)
@@ -1119,3 +1120,96 @@ class TestGetCompressionTarget:
def test_small_model_returns_default(self) -> None:
# Unknown models fall back to DEFAULT_TOKEN_THRESHOLD
assert get_compression_target("some-tiny-model") == DEFAULT_TOKEN_THRESHOLD
+
+
+class TestClaude5TokenFactor:
+ """The Claude 5 family tokenizes ~30% denser than 4.x and ships no
+ local tokenizer — estimates get the conservative correction factor."""
+
+ def test_sonnet_5_estimate_is_scaled(self):
+ text = "hello world " * 200
+ base = estimate_token_count_str(text, model="claude-sonnet-4-6")
+ scaled = estimate_token_count_str(text, model="claude-sonnet-5")
+ # Literal 1.5, not CLAUDE_5_TOKEN_FACTOR: importing the constant
+ # under test would let a wrong magnitude pass unnoticed.
+ assert scaled == int(base * 1.5)
+
+ @pytest.mark.parametrize(
+ "prefixed",
+ [
+ "anthropic/claude-sonnet-5",
+ "anthropic.claude-sonnet-5",
+ "openrouter/anthropic/claude-sonnet-5",
+ "us.anthropic.claude-sonnet-5",
+ ],
+ )
+ def test_factor_applies_through_vendor_prefix(self, prefixed: str):
+ text = "hello world " * 200
+ assert estimate_token_count_str(
+ text, model=prefixed
+ ) == estimate_token_count_str(text, model="claude-sonnet-5")
+
+ def test_unknown_gpt_model_falls_back_to_default_encoding(self):
+ """A GPT-named model tiktoken doesn't know (newer than its mapping
+ table) must fall back to the default encoding, not KeyError — this
+ is the llm_call path for every catalog default."""
+ text = "hello world " * 200
+ assert estimate_token_count_str(
+ text, model="gpt-5.6-terra"
+ ) == estimate_token_count_str(text, model="gpt-4o")
+
+ @pytest.mark.parametrize("model", ["claude-sonnet-4-6", "claude-opus-4-6"])
+ def test_pre_4_7_generation_unscaled(self, model: str):
+ text = "hello world " * 200
+ assert estimate_token_count_str(text, model=model) == estimate_token_count_str(
+ text, model="gpt-4o"
+ )
+
+ @pytest.mark.parametrize("model", ["claude-opus-4-7", "claude-opus-4-8"])
+ def test_opus_4_7_generation_shares_the_new_tokenizer(self, model: str):
+ text = "hello world " * 200
+ assert estimate_token_count_str(text, model=model) == estimate_token_count_str(
+ text, model="claude-sonnet-5"
+ )
+
+
+class TestClaude5FactorOnMessagePaths:
+ """The factor must reach the paths that gate real behavior: the
+ messages-based estimator (max_tokens sizing) and compaction totals."""
+
+ def _messages(self):
+ return [
+ {"role": "user", "content": "hello world " * 300},
+ {"role": "assistant", "content": "reply text " * 300},
+ ]
+
+ def test_estimate_token_count_messages_scaled(self):
+ base = estimate_token_count(self._messages(), model="claude-sonnet-4-6")
+ scaled = estimate_token_count(self._messages(), model="claude-sonnet-5")
+ assert scaled == int(base * 1.5)
+
+ @pytest.mark.asyncio
+ async def test_compaction_measures_in_corrected_space(self):
+ """A history under the nominal target in raw tiktoken space but
+ over it in corrected space must trigger compaction for Sonnet 5
+ (and must NOT for 4.6)."""
+ msgs = self._messages()
+ raw = estimate_token_count(msgs, model="claude-sonnet-4-6")
+ target = int(raw * 1.2) # between raw (1.0x) and corrected (1.5x)
+
+ r46 = await compress_context(
+ messages=msgs,
+ model="claude-sonnet-4-6",
+ target_tokens=target,
+ reserve=0,
+ )
+ assert r46.was_compacted is False
+
+ r5 = await compress_context(
+ messages=msgs,
+ model="claude-sonnet-5",
+ target_tokens=target,
+ reserve=0,
+ )
+ assert r5.was_compacted is True
+ assert r5.original_token_count > target
diff --git a/docs/integrations/block-integrations/llm.md b/docs/integrations/block-integrations/llm.md
index f0065d1fe6..ab51a4460e 100644
--- a/docs/integrations/block-integrations/llm.md
+++ b/docs/integrations/block-integrations/llm.md
@@ -65,7 +65,7 @@ The result routes data to yes_output or no_output, enabling intelligent branchin
| condition | A plaintext English description of the condition to evaluate | str | Yes |
| yes_value | (Optional) Value to output if the condition is true. If not provided, input_value will be used. | Yes Value | No |
| no_value | (Optional) Value to output if the condition is false. If not provided, input_value will be used. | No Value | No |
-| model | The language model to use for evaluating the condition. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
+| model | The language model to use for evaluating the condition. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "claude-sonnet-5" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
### Outputs
@@ -103,7 +103,7 @@ The block sends the entire conversation history to the chosen LLM, including sys
|-------|-------------|------|----------|
| prompt | The prompt to send to the language model. | str | No |
| messages | List of messages in the conversation. | List[Any] | Yes |
-| model | The language model to use for the conversation. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
+| model | The language model to use for the conversation. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "claude-sonnet-5" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
| max_tokens | The maximum number of tokens to generate in the chat completion. | int | No |
| ollama_host | Ollama host for local models | str | No |
@@ -257,7 +257,7 @@ The block formulates a prompt based on the given focus or source data, sends it
|-------|-------------|------|----------|
| focus | The focus of the list to generate. | str | No |
| source_data | The data to generate the list from. | str | No |
-| model | The language model to use for generating the list. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
+| model | The language model to use for generating the list. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "claude-sonnet-5" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
| max_retries | Maximum number of retries for generating a valid list. | int | No |
| force_json_output | Whether to force the LLM to produce a JSON-only response. This can increase the block's reliability, but may also reduce the quality of the response because it prohibits the LLM from reasoning before providing its JSON response. | bool | No |
| max_tokens | The maximum number of tokens to generate in the chat completion. | int | No |
@@ -424,7 +424,7 @@ The block sends the input prompt to a chosen LLM, along with any system prompts
| prompt | The prompt to send to the language model. | str | Yes |
| expected_format | Expected format of the response. If provided, the response will be validated against this format. The keys should be the expected fields in the response, and the values should be the description of the field. | Dict[str, str] | Yes |
| list_result | Whether the response should be a list of objects in the expected format. | bool | No |
-| model | The language model to use for answering the prompt. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
+| model | The language model to use for answering the prompt. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "claude-sonnet-5" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
| force_json_output | Whether to force the LLM to produce a JSON-only response. This can increase the block's reliability, but may also reduce the quality of the response because it prohibits the LLM from reasoning before providing its JSON response. | bool | No |
| sys_prompt | The system prompt to provide additional context to the model. | str | No |
| conversation_history | The conversation history to provide context for the prompt. | List[Dict[str, Any]] | No |
@@ -464,7 +464,7 @@ The block sends the input prompt to a chosen LLM, processes the response, and re
| Input | Description | Type | Required |
|-------|-------------|------|----------|
| prompt | The prompt to send to the language model. You can use any of the {keys} from Prompt Values to fill in the prompt with values from the prompt values dictionary by putting them in curly braces. | str | Yes |
-| model | The language model to use for answering the prompt. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
+| model | The language model to use for answering the prompt. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "claude-sonnet-5" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
| sys_prompt | The system prompt to provide additional context to the model. | str | No |
| retry | Number of times to retry the LLM call if the response does not match the expected format. | int | No |
| prompt_values | Values used to fill in the prompt. The values can be used in the prompt by putting them in a double curly braces, e.g. {{variable_name}}. | Dict[str, str] | No |
@@ -501,7 +501,7 @@ The block splits the input text into smaller chunks, sends each chunk to an LLM
| Input | Description | Type | Required |
|-------|-------------|------|----------|
| text | The text to summarize. | str | Yes |
-| model | The language model to use for summarizing the text. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
+| model | The language model to use for summarizing the text. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "claude-sonnet-5" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
| focus | The topic to focus on in the summary | str | No |
| style | The style of the summary to generate. | "concise" \| "detailed" \| "bullet points" \| "numbered list" | No |
| max_tokens | The maximum number of tokens to generate in the chat completion. | int | No |
@@ -721,7 +721,7 @@ _Add technical explanation here._
| Input | Description | Type | Required |
|-------|-------------|------|----------|
| prompt | The prompt to send to the language model. | str | Yes |
-| model | The language model to use for answering the prompt. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
+| model | The language model to use for answering the prompt. | "o3-mini" \| "o3-2025-04-16" \| "gpt-5.2-2025-12-11" \| "gpt-5.1-2025-11-13" \| "gpt-5-2025-08-07" \| "gpt-5-mini-2025-08-07" \| "o4-mini" \| "o3-pro" \| "o1" \| "o1-mini" \| "gpt-5.6-sol" \| "gpt-5.6-terra" \| "gpt-5.6-luna" \| "gpt-5.5-2026-04-23" \| "gpt-5.5-pro" \| "gpt-5.4-2026-03-05" \| "gpt-5.4-mini-2026-03-17" \| "gpt-5.4-nano-2026-03-17" \| "gpt-5.4-pro" \| "gpt-5.3-chat-latest" \| "gpt-5.3-codex" \| "gpt-5.2-pro" \| "gpt-5.1-codex" \| "gpt-5-pro" \| "gpt-4.1-nano" \| "gpt-5-nano-2025-08-07" \| "gpt-5-chat-latest" \| "gpt-4.1-2025-04-14" \| "gpt-4.1-mini-2025-04-14" \| "gpt-4o-mini" \| "gpt-4o" \| "claude-opus-4-5-20251101" \| "claude-sonnet-4-5-20250929" \| "claude-haiku-4-5-20251001" \| "claude-opus-4-6" \| "claude-opus-4-7" \| "claude-sonnet-4-6" \| "claude-sonnet-5" \| "meta-llama/Llama-3.3-70B-Instruct-Turbo" \| "llama-3.3-70b-versatile" \| "llama-3.1-8b-instant" \| "llama3.3" \| "llama3.2" \| "llama3" \| "llama3.1:405b" \| "dolphin-mistral:latest" \| "openai/gpt-oss-120b" \| "openai/gpt-oss-20b" \| "google/gemini-2.5-pro" \| "google/gemini-3.1-pro-preview" \| "google/gemini-3-flash-preview" \| "google/gemini-2.5-flash" \| "google/gemini-2.0-flash-001" \| "google/gemini-3.1-flash-lite-preview" \| "google/gemini-2.5-flash-lite" \| "google/gemini-2.0-flash-lite-001" \| "mistralai/mistral-large-2512" \| "mistralai/mistral-medium-3.1" \| "mistralai/mistral-small-3.2-24b-instruct" \| "mistralai/codestral-2508" \| "cohere/command-a-03-2025" \| "cohere/command-a-translate-08-2025" \| "cohere/command-a-reasoning-08-2025" \| "cohere/command-a-vision-07-2025" \| "deepseek/deepseek-chat" \| "deepseek/deepseek-r1-0528" \| "perplexity/sonar" \| "perplexity/sonar-pro" \| "perplexity/sonar-reasoning-pro" \| "perplexity/sonar-deep-research" \| "nousresearch/hermes-3-llama-3.1-405b" \| "nousresearch/hermes-3-llama-3.1-70b" \| "amazon/nova-lite-v1" \| "amazon/nova-micro-v1" \| "amazon/nova-pro-v1" \| "microsoft/phi-4" \| "gryphe/mythomax-l2-13b" \| "meta-llama/llama-4-scout" \| "meta-llama/llama-4-maverick" \| "x-ai/grok-3" \| "x-ai/grok-4" \| "x-ai/grok-4-fast" \| "x-ai/grok-4.1-fast" \| "x-ai/grok-4.20" \| "x-ai/grok-4.20-multi-agent" \| "x-ai/grok-code-fast-1" \| "moonshotai/kimi-k2.5" \| "moonshotai/kimi-k2.6" \| "moonshotai/kimi-k2-thinking" \| "moonshotai/kimi-k3" \| "qwen/qwen3-235b-a22b-thinking-2507" \| "qwen/qwen3-coder" \| "z-ai/glm-4.6" \| "z-ai/glm-4.6v" \| "z-ai/glm-4.7" \| "z-ai/glm-4.7-flash" \| "z-ai/glm-5" \| "z-ai/glm-5-turbo" \| "z-ai/glm-5v-turbo" \| "Llama-4-Scout-17B-16E-Instruct-FP8" \| "Llama-4-Maverick-17B-128E-Instruct-FP8" \| "Llama-3.3-8B-Instruct" \| "Llama-3.3-70B-Instruct" \| "v0-1.5-md" \| "v0-1.5-lg" \| "v0-1.0-md" | No |
| multiple_tool_calls | Whether to allow multiple tool calls in a single response. | bool | No |
| sys_prompt | The system prompt to provide additional context to the model. | str | No |
| conversation_history | The conversation history to provide context for the prompt. | List[Dict[str, Any]] | No |