AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): teach LlmModel about OpenRouter Anthropic aliases + dry-run sim default Haiku OR-slug (#13177)
Zamil Majdy9d138e217f94
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.pyindex 14a3263dc2..5bb54e59a9 100644--- a/autogpt_platform/backend/backend/blocks/llm.py+++ b/autogpt_platform/backend/backend/blocks/llm.py@@ -6,6 +6,7 @@ import mathimport reimport secretsfrom abc import ABC+from collections.abc import Mappingfrom enum import Enum, EnumMetafrom json import JSONDecodeErrorfrom typing import Any, Iterable, List, Literal, NamedTuple, Optional, cast@@ -119,8 +120,25 @@ class LlmModelMeta(EnumMeta):class LlmModel(str, Enum, metaclass=LlmModelMeta):@classmethoddef _missing_(cls, value: object) -> "LlmModel | None":- """Handle provider-prefixed model names like 'anthropic/claude-sonnet-4-6'."""- if isinstance(value, str) and "/" in value:+ """Resolve provider-prefixed model names.++ Handles two shapes:++ 1. OpenRouter aliases for Anthropic models whose direct-API+ identifier carries a snapshot suffix that the OpenRouter slug+ drops — e.g. ``anthropic/claude-haiku-4-5`` ↔ enum value+ ``claude-haiku-4-5-20251001``. Looked up via+ ``_OPENROUTER_ALIASES`` (defined below the class so it can hold+ ``LlmModel`` members directly).+ 2. Generic provider prefix strip — e.g.+ ``anthropic/claude-sonnet-4-6`` → ``claude-sonnet-4-6``.+ """+ if not isinstance(value, str):+ return None+ alias = _OPENROUTER_ALIASES.get(value)+ if alias is not None:+ return alias+ if "/" in value:stripped = value.split("/", 1)[1]try:return cls(stripped)@@ -261,6 +279,24 @@ class LlmModel(str, Enum, metaclass=LlmModelMeta):return self.metadata.max_output_tokens+# OpenRouter exposes Anthropic models under canonical ``anthropic/<model>``+# slugs that drop the snapshot-date suffix Anthropic's own API uses+# (``claude-haiku-4-5-20251001`` → ``anthropic/claude-haiku-4-5``). The+# generic provider-prefix strip in ``_missing_`` can't reverse the date+# truncation, so map the OpenRouter slugs to ``LlmModel`` members here.+# Only models whose canonical enum value carries a ``-YYYYMMDD`` snapshot+# suffix need entries; values without a snapshot (4.6/4.7+) are already+# covered by the prefix-strip path alone. Stored as ``LlmModel`` instances+# (not strings) so a rename or snapshot rotation on the enum follows the+# alias automatically — a stale entry becomes a load-time ``AttributeError``+# rather than a silent ``_missing_`` miss at runtime.+_OPENROUTER_ALIASES: Mapping[str, LlmModel] = {+ "anthropic/claude-haiku-4-5": LlmModel.CLAUDE_4_5_HAIKU,+ "anthropic/claude-opus-4-5": LlmModel.CLAUDE_4_5_OPUS,+ "anthropic/claude-sonnet-4-5": LlmModel.CLAUDE_4_5_SONNET,+}++MODEL_METADATA = {# https://platform.openai.com/docs/modelsLlmModel.O3: ModelMetadata("openai", 200000, 100000, "O3", "OpenAI", "OpenAI", 2),diff --git a/autogpt_platform/backend/backend/blocks/test/test_llm.py b/autogpt_platform/backend/backend/blocks/test/test_llm.pyindex 81285cfe1b..5c1ca2bff3 100644--- a/autogpt_platform/backend/backend/blocks/test/test_llm.py+++ b/autogpt_platform/backend/backend/blocks/test/test_llm.py@@ -1,4 +1,5 @@import asyncio+import refrom typing import Any, castfrom unittest.mock import AsyncMock, MagicMock, patch@@ -1548,3 +1549,59 @@ class TestLLMRequestTimeout:assert (call_count["n"] == 1), f"Expected exactly 1 call (no retry on timeout), got {call_count['n']}"+++class TestLlmModelMissingHandler:+ """``LlmModel._missing_`` resolves provider-prefixed slugs back to enum+ members. Used by the dry-run simulator to send an OpenRouter slug+ (``anthropic/claude-haiku-4-5``) that OpenRouter's OpenAI-compat endpoint+ accepts, while still validating through ``OrchestratorBlock.Input.model``+ which is typed as ``LlmModel``."""++ def test_openrouter_alias_maps_to_canonical(self) -> None:+ # The OpenRouter slug for Haiku 4.5 drops the snapshot suffix that+ # Anthropic's direct API uses — pure prefix strip wouldn't find it,+ # so the alias map is required.+ assert (+ llm.LlmModel("anthropic/claude-haiku-4-5") is llm.LlmModel.CLAUDE_4_5_HAIKU+ )+ assert llm.LlmModel("anthropic/claude-opus-4-5") is llm.LlmModel.CLAUDE_4_5_OPUS+ assert (+ llm.LlmModel("anthropic/claude-sonnet-4-5")+ is llm.LlmModel.CLAUDE_4_5_SONNET+ )++ def test_prefix_strip_still_works(self) -> None:+ # 4.6/4.7 enum values already match OpenRouter's slug after prefix+ # strip — no alias-map entry needed.+ assert llm.LlmModel("anthropic/claude-opus-4-7") is llm.LlmModel.CLAUDE_4_7_OPUS+ assert llm.LlmModel("anthropic/claude-opus-4-6") is llm.LlmModel.CLAUDE_4_6_OPUS++ def test_unknown_slug_raises(self) -> None:+ # Bare unknown strings still fail loudly — the alias map is an+ # additive shortcut, not a silent catch-all.+ with pytest.raises(ValueError):+ llm.LlmModel("anthropic/claude-haiku-999")++ def test_all_claude_snapshots_reachable_via_openrouter_slug(self) -> None:+ """Every Claude enum value whose canonical form carries a trailing+ ``-YYYYMMDD`` snapshot suffix must be reachable via the OpenRouter+ ``anthropic/<base>`` slug — otherwise the dry-run simulator (and any+ other caller passing the OpenRouter form) hits ``_missing_``'s+ prefix-strip path with a string the enum doesn't recognise and+ validation fails. Catches the drift class where a new snapshot-+ dated Claude member is added without updating ``_OPENROUTER_ALIASES``.+ """+ # Capture the full ``claude-...`` base so the OpenRouter slug is+ # constructed correctly: ``claude-haiku-4-5-20251001`` →+ # ``anthropic/claude-haiku-4-5`` (NOT ``anthropic/haiku-4-5``).+ snapshot_re = re.compile(r"^(claude-.+)-\d{8}$")+ for member in llm.LlmModel:+ match = snapshot_re.match(member.value)+ if match is None:+ continue+ slug = f"anthropic/{match.group(1)}"+ assert llm.LlmModel(slug) is member, (+ f"OpenRouter alias missing for {member.value} — expected slug "+ f"{slug!r} to resolve back to {member.name}"+ )diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.pyindex 6b967e2a11..48972ec786 100644--- a/autogpt_platform/backend/backend/copilot/config.py+++ b/autogpt_platform/backend/backend/copilot/config.py@@ -7,7 +7,6 @@ from urllib.parse import urlparsefrom pydantic import AliasChoices, Field, field_validator, model_validatorfrom pydantic_settings import BaseSettings-from backend.blocks.llm import LlmModelfrom backend.util.clients import OPENROUTER_BASE_URL@@ -111,14 +110,17 @@ class ChatConfig(BaseSettings):"via env without code changes.",)simulation_model: str = Field(- default=LlmModel.CLAUDE_4_5_HAIKU.value,- description="Model for dry-run block simulation. Must be a real LlmModel "- "slug — OrchestratorBlock's input validates this against the enum, and a "- "bad value here breaks every dry-run of an agent that contains an "- "Orchestrator. Default is Claude Haiku 4.5 because the Orchestrator "- "dry-run picks tools and produces a finish message, so a substitute "- "with stronger reasoning gives a more faithful preview than Flash-Lite "- "while staying cheap.",+ default="anthropic/claude-haiku-4-5",+ description="Model for dry-run block simulation. Must be in "+ "OpenRouter ``<vendor>/<model>`` format AND resolvable through "+ "``LlmModel`` (directly or via ``LlmModel._missing_``'s "+ "``_OPENROUTER_ALIASES`` map). The LLM-simulation path hits "+ "OpenRouter's OpenAI-compat endpoint, which rejects direct-Anthropic "+ "snapshot IDs like ``claude-haiku-4-5-20251001`` with HTTP 400. "+ "OrchestratorBlock validates this against ``LlmModel``, so it must "+ "resolve there. Haiku 4.5 via OpenRouter is already used in "+ "production for ``title_model``, so it's battle-tested through "+ "this same client.",)api_key: str | None = Field(default=None, description="OpenAI API key")base_url: str | None = Field(diff --git a/autogpt_platform/backend/backend/executor/simulator.py b/autogpt_platform/backend/backend/executor/simulator.pyindex d61d4b44b4..cb36f19947 100644--- a/autogpt_platform/backend/backend/executor/simulator.py+++ b/autogpt_platform/backend/backend/executor/simulator.py@@ -39,7 +39,6 @@ from openai.types import CompletionUsagefrom backend.blocks.agent import AgentExecutorBlockfrom backend.blocks.io import AgentInputBlock, AgentOutputBlock-from backend.blocks.llm import LlmModelfrom backend.blocks.orchestrator import OrchestratorBlockfrom backend.copilot.token_tracking import persist_and_record_usagefrom backend.util.clients import get_openai_client@@ -48,13 +47,23 @@ logger = logging.getLogger(__name__)# Default simulator model — Claude Haiku 4.5 via the platform's OpenRouter-# credential. Dry-run isn't pure shape-matching for the Orchestrator block:-# it picks tools and produces a finish message, so the substitute model's-# reasoning quality affects how faithful the preview is. Haiku 4.5 keeps-# the substitute small/cheap while giving materially better tool-use-# decisions than Flash-Lite. Configurable via ChatConfig.simulation_model-# (CHAT_SIMULATION_MODEL env var) — value must be a real LlmModel slug.-_DEFAULT_SIMULATOR_MODEL = LlmModel.CLAUDE_4_5_HAIKU.value+# credential, in OpenRouter slug form (``anthropic/claude-haiku-4-5``).+# The LLM-simulation path (``_call_llm_for_simulation``) hits OpenRouter's+# OpenAI-compat endpoint at ``openrouter.ai/api/v1/chat/completions``,+# which only accepts ``<vendor>/<model>`` slugs — the direct-Anthropic+# snapshot ID ``claude-haiku-4-5-20251001`` is rejected with HTTP 400.+# For OrchestratorBlock dry-runs, this string is parsed via+# ``LlmModel._missing_`` (the ``_OPENROUTER_ALIASES`` table maps it back+# to ``LlmModel.CLAUDE_4_5_HAIKU``), and the orchestrator's Anthropic SDK+# sends the snapshot value to OpenRouter's Anthropic-compat endpoint at+# ``openrouter.ai/api/v1/messages``, which accepts both formats. Haiku+# 4.5 via OpenRouter is already battle-tested for copilot title generation+# (see ``ChatConfig.title_model``) and gives materially better tool-use+# decisions for Orchestrator dry-runs than the Flash-Lite tier while+# staying cheap. Configurable via ``ChatConfig.simulation_model``+# (``CHAT_SIMULATION_MODEL`` env var) — overrides must satisfy both+# constraints (OpenRouter-slug shape AND resolvable through ``LlmModel``).+_DEFAULT_SIMULATOR_MODEL = "anthropic/claude-haiku-4-5"# OpenRouter-specific extra_body flag that embeds the real generation cost on# the response usage object. Same shape used by the baseline copilot servicediff --git a/autogpt_platform/backend/backend/executor/simulator_test.py b/autogpt_platform/backend/backend/executor/simulator_test.pyindex 00cad55e8a..f8684f48ca 100644--- a/autogpt_platform/backend/backend/executor/simulator_test.py+++ b/autogpt_platform/backend/backend/executor/simulator_test.py@@ -570,18 +570,29 @@ def _sim_completion(*, content: str, usage: CompletionUsage) -> ChatCompletion:class TestDefaultSimulatorModel:- """Pin the default model — anyone flipping this without a cost review- trips the test, and anyone setting it to a non-enum value fails the- LlmModel parseability check (the bug class behind SECRT-2368)."""+ """Pin the default model — three guards. Anyone flipping this without+ a cost review trips the value pin. Anyone setting it to a non-enum+ value fails the ``LlmModel`` parseability check (the bug class behind+ SECRT-2368). Anyone setting it to a direct-provider snapshot ID like+ ``claude-haiku-4-5-20251001`` fails the OpenRouter-slug check —+ OpenRouter's OpenAI-compat endpoint, which the LLM-simulation path+ uses, rejects anything that isn't in ``<vendor>/<model>`` form."""- def test_default_is_haiku_4_5(self) -> None:- assert _DEFAULT_SIMULATOR_MODEL == "claude-haiku-4-5-20251001"+ def test_default_is_haiku_openrouter_slug(self) -> None:+ assert _DEFAULT_SIMULATOR_MODEL == "anthropic/claude-haiku-4-5"def test_default_parses_as_llm_model(self) -> None:- # OrchestratorBlock.Input.model is typed as LlmModel and rejects- # any default that isn't an enum member — keep this guard tight.+ # OrchestratorBlock.Input.model is typed as LlmModel; the default+ # must resolve through the enum (either directly or via the+ # OpenRouter-alias map in ``LlmModel._missing_``).assert LlmModel(_DEFAULT_SIMULATOR_MODEL) is LlmModel.CLAUDE_4_5_HAIKU+ def test_default_is_openrouter_slug(self) -> None:+ # The LLM-simulation path hits OpenRouter's OpenAI-compat endpoint,+ # which only accepts canonical ``<vendor>/<model>`` slugs. Anything+ # else returns HTTP 400 even if it's a real provider model.+ assert "/" in _DEFAULT_SIMULATOR_MODEL+class TestExtractCostUsd:"""Provider-reported USD cost via typed ``model_extra`` — mirrors