Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): unblock orchestrator dry-run end-to-end (canonical model + SDK auth) (#13180)

Zamil Majdy6a6f2d2c23af
diff --git a/autogpt_platform/backend/backend/blocks/orchestrator.py b/autogpt_platform/backend/backend/blocks/orchestrator.py
index 5979f90dde..cbb89c9600 100644
--- a/autogpt_platform/backend/backend/blocks/orchestrator.py
+++ b/autogpt_platform/backend/backend/blocks/orchestrator.py
@@ -1656,8 +1656,26 @@ class OrchestratorBlock(Block):
return
api_key = credentials.api_key.get_secret_value()
if provider == "open_router":
- # Route through OpenRouter proxy: set base URL + auth token,
- # clear API key so the SDK uses AUTH_TOKEN instead.
+ # Route through OpenRouter proxy: point ``ANTHROPIC_BASE_URL`` at
+ # OpenRouter's Anthropic-compat endpoint and set BOTH auth env
+ # vars to the OpenRouter key. OpenRouter accepts either
+ # ``x-api-key`` (set by ``ANTHROPIC_API_KEY``) or
+ # ``Authorization: Bearer`` (set by ``ANTHROPIC_AUTH_TOKEN``);
+ # whichever the Claude Code CLI happens to send on the wire is
+ # a valid OR credential.
+ #
+ # We MUST explicitly set ``ANTHROPIC_API_KEY`` here rather than
+ # omit it or set it to an empty string. The Claude Agent SDK
+ # merges ``options.env`` on top of ``os.environ``, so omitting
+ # the key lets any inherited platform ``ANTHROPIC_API_KEY``
+ # (e.g. a deployment's direct-Anthropic key) leak through to
+ # OpenRouter and 401. Setting it to ``""`` was the previous
+ # approach to "force AUTH_TOKEN usage", but the CLI's HTTP
+ # layer treats the empty string as set and emits
+ # ``x-api-key:`` (empty value), which OpenRouter rejects with
+ # ``invalid x-api-key``. Using the OR key for both vars
+ # avoids both failure modes.
+ #
# NOTE: We use the platform's global OpenRouter base URL from
# ChatConfig. Per-credential base URLs are not yet supported;
# if the user's credential targets a custom proxy, the SDK will
@@ -1670,7 +1688,7 @@ class OrchestratorBlock(Block):
sdk_env = {
"ANTHROPIC_BASE_URL": or_base,
"ANTHROPIC_AUTH_TOKEN": api_key,
- "ANTHROPIC_API_KEY": "", # force CLI to use AUTH_TOKEN
+ "ANTHROPIC_API_KEY": api_key,
}
else:
# Direct Anthropic key
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index 48972ec786..f3dabbac8c 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -7,6 +7,7 @@ from urllib.parse import urlparse
from pydantic import AliasChoices, Field, field_validator, model_validator
from pydantic_settings import BaseSettings
+from backend.blocks.llm import LlmModel
from backend.util.clients import OPENROUTER_BASE_URL
@@ -110,17 +111,17 @@ class ChatConfig(BaseSettings):
"via env without code changes.",
)
simulation_model: str = Field(
- default="anthropic/claude-haiku-4-5",
- description="Model for dry-run block simulation. Must be in "
- "OpenRouter ``<vendor>/<model>`` format AND resolvable through "
- "``LlmModel`` (directly or via ``LlmModel._missing_``'s "
- "``_OPENROUTER_ALIASES`` map). The LLM-simulation path hits "
- "OpenRouter's OpenAI-compat endpoint, which rejects direct-Anthropic "
- "snapshot IDs like ``claude-haiku-4-5-20251001`` with HTTP 400. "
- "OrchestratorBlock validates this against ``LlmModel``, so it must "
- "resolve there. Haiku 4.5 via OpenRouter is already used in "
- "production for ``title_model``, so it's battle-tested through "
- "this same client.",
+ default=LlmModel.GEMINI_2_5_FLASH_LITE.value,
+ description="Model for dry-run block simulation. Constraints: "
+ "(1) the LLM-simulation path calls OpenRouter's OpenAI-compat "
+ "endpoint with ``response_format=json_object`` and "
+ "``json.loads()`` the response — Claude wraps JSON in markdown "
+ "fences here, Gemini doesn't; (2) the orchestrator BUILT_IN "
+ "dry-run path dispatches on ``llm_model.metadata.provider`` — "
+ "a Claude default would hit ``api.anthropic.com`` with the "
+ "platform OR key and 401, while Gemini has provider=open_router "
+ "and routes correctly. Flash-Lite satisfies both and keeps "
+ "platform OR cost low.",
)
api_key: str | None = Field(default=None, description="OpenAI API key")
base_url: str | None = Field(
diff --git a/autogpt_platform/backend/backend/executor/simulator.py b/autogpt_platform/backend/backend/executor/simulator.py
index cb36f19947..c93956ada7 100644
--- a/autogpt_platform/backend/backend/executor/simulator.py
+++ b/autogpt_platform/backend/backend/executor/simulator.py
@@ -39,31 +39,40 @@ from openai.types import CompletionUsage
from backend.blocks.agent import AgentExecutorBlock
from backend.blocks.io import AgentInputBlock, AgentOutputBlock
-from backend.blocks.orchestrator import OrchestratorBlock
+from backend.blocks.llm import LlmModel
+from backend.blocks.orchestrator import ExecutionMode, OrchestratorBlock
from backend.copilot.token_tracking import persist_and_record_usage
from backend.util.clients import get_openai_client
logger = logging.getLogger(__name__)
-# Default simulator model — Claude Haiku 4.5 via the platform's OpenRouter
-# credential, in OpenRouter slug form (``anthropic/claude-haiku-4-5``).
-# The LLM-simulation path (``_call_llm_for_simulation``) hits OpenRouter's
-# OpenAI-compat endpoint at ``openrouter.ai/api/v1/chat/completions``,
-# which only accepts ``<vendor>/<model>`` slugs — the direct-Anthropic
-# snapshot ID ``claude-haiku-4-5-20251001`` is rejected with HTTP 400.
-# For OrchestratorBlock dry-runs, this string is parsed via
-# ``LlmModel._missing_`` (the ``_OPENROUTER_ALIASES`` table maps it back
-# to ``LlmModel.CLAUDE_4_5_HAIKU``), and the orchestrator's Anthropic SDK
-# sends the snapshot value to OpenRouter's Anthropic-compat endpoint at
-# ``openrouter.ai/api/v1/messages``, which accepts both formats. Haiku
-# 4.5 via OpenRouter is already battle-tested for copilot title generation
-# (see ``ChatConfig.title_model``) and gives materially better tool-use
-# decisions for Orchestrator dry-runs than the Flash-Lite tier while
-# staying cheap. Configurable via ``ChatConfig.simulation_model``
-# (``CHAT_SIMULATION_MODEL`` env var) — overrides must satisfy both
-# constraints (OpenRouter-slug shape AND resolvable through ``LlmModel``).
-_DEFAULT_SIMULATOR_MODEL = "anthropic/claude-haiku-4-5"
+# Default simulator model — Gemini 2.5 Flash-Lite via OpenRouter. The
+# choice is forced by three independent routing constraints; Flash-Lite
+# is the smallest enum member that satisfies all three:
+#
+# 1. **LLM-simulation path** (``_call_llm_for_simulation``) calls
+# OpenRouter's OpenAI-compat endpoint with
+# ``response_format=json_object`` and ``json.loads()`` the response.
+# Claude via OR's OpenAI-compat wraps JSON in markdown code fences
+# (```​``json\\n{...}\\n``​```) even when JSON mode is requested
+# — ``json.loads`` trips on the leading backtick with
+# ``Expecting value: line 1 column 1 (char 0)``. Gemini emits raw JSON.
+# 2. **Orchestrator BUILT_IN dry-run path** (``llm.llm_call``) dispatches
+# on ``llm_model.metadata.provider``, not credential type. A Claude
+# default would land in the Anthropic-Python-SDK branch against
+# ``api.anthropic.com`` with the platform OR key → 401. Gemini has
+# ``metadata.provider == "open_router"`` so dispatch routes through
+# the OpenRouter branch correctly.
+# 3. **Orchestrator EXTENDED_THINKING dry-run path** would impose
+# ``model.value.startswith("claude")`` (orchestrator.py SDK guard) —
+# Flash-Lite would fail that check, which is why we override
+# ``execution_mode`` to BUILT_IN below regardless of user choice.
+#
+# Configurable via ``ChatConfig.simulation_model``
+# (``CHAT_SIMULATION_MODEL`` env var); overrides must satisfy
+# constraints (1) and (2).
+_DEFAULT_SIMULATOR_MODEL = LlmModel.GEMINI_2_5_FLASH_LITE.value
# OpenRouter-specific extra_body flag that embeds the real generation cost on
# the response usage object. Same shape used by the baseline copilot service
@@ -411,17 +420,68 @@ def prepare_dry_run(block: Any, input_data: dict[str, Any]) -> dict[str, Any] |
# untouched so a tiny bounded cap was asymmetric anyway.
original = input_data.get("agent_mode_max_iterations", 0)
max_iters = min(original, 10) if original != 0 else 0
- sim_model = _simulator_model()
+ # Resolve the configured simulator model (typically an OpenRouter
+ # slug like "anthropic/claude-haiku-4-5") to its canonical
+ # ``LlmModel.value`` (e.g. "claude-haiku-4-5-20251001"). We have
+ # to do this BEFORE injecting into ``input["model"]`` because the
+ # block runs through ``validate_data`` → ``jsonschema.validate``
+ # before Pydantic gets the value, and the JSON Schema's ``enum``
+ # is the literal list of ``LlmModel.value``s — the
+ # ``_OPENROUTER_ALIASES`` map in ``LlmModel._missing_`` does NOT
+ # surface in the generated schema, so the OR-slug is rejected
+ # with ``"'<slug>' is not one of [...]"``. After this
+ # translation, the OrchestratorBlock receives a model identifier
+ # that both validators accept; the OR-routing still works
+ # because OpenRouter's Anthropic-compat endpoint (which the
+ # orchestrator's SDK mode hits when credentials are OPEN_ROUTER)
+ # accepts both the snapshot and the OR slug.
+ #
+ # Fall back to the default if the configured override is malformed
+ # or unmapped — an invalid ``CHAT_SIMULATION_MODEL`` env value
+ # shouldn't take out every dry-run. The default is asserted as
+ # ``LlmModel``-parseable by ``TestDefaultSimulatorModel``, so the
+ # fallback ``LlmModel(_DEFAULT_SIMULATOR_MODEL)`` cannot raise
+ # unless someone breaks that pin too.
+ configured_sim_model = _simulator_model()
+ try:
+ sim_model = LlmModel(configured_sim_model).value
+ except ValueError:
+ logger.warning(
+ "Dry-run: invalid simulation model %r; falling back to default %r",
+ configured_sim_model,
+ _DEFAULT_SIMULATOR_MODEL,
+ )
+ sim_model = LlmModel(_DEFAULT_SIMULATOR_MODEL).value
# Keep the original credentials dict in input_data so the block's
# JSON schema validation passes (validate_data strips None values,
# making the field absent and failing the "required" check).
# The actual credentials are injected via extra_exec_kwargs in
# manager.py using _dry_run_api_key.
+ #
+ # Force ``execution_mode = BUILT_IN`` for the dry-run, regardless
+ # of what the user picked. With Flash-Lite as the sim model:
+ #
+ # - BUILT_IN routes ``llm.llm_call`` to the open_router branch
+ # (dispatch on ``metadata.provider``) → OpenAI SDK against
+ # openrouter.ai with the platform OR key → works.
+ # - EXTENDED_THINKING would hit the SDK subprocess's
+ # ``model.value.startswith("claude")`` guard
+ # (orchestrator.py) and ``ValueError`` immediately — Flash-Lite
+ # is not Claude.
+ #
+ # Honouring the user's pick of EXTENDED_THINKING would force the
+ # sim model back to Claude (Haiku, etc.), which then trips the
+ # JSON-mode markdown-wrap on the LLM-simulation path for every
+ # *other* block in the same graph. BUILT_IN sidesteps every
+ # path-specific quirk by staying on the simpler OpenAI-SDK-
+ # with-tool-calling route — the actual purpose of dry-run is a
+ # smoke check, not a perfect mirror of EXTENDED_THINKING.
return {
**input_data,
"agent_mode_max_iterations": max_iters,
"model": sim_model,
+ "execution_mode": ExecutionMode.BUILT_IN.value,
"_dry_run_api_key": or_key,
}
diff --git a/autogpt_platform/backend/backend/executor/simulator_test.py b/autogpt_platform/backend/backend/executor/simulator_test.py
index f8684f48ca..8e50f671c0 100644
--- a/autogpt_platform/backend/backend/executor/simulator_test.py
+++ b/autogpt_platform/backend/backend/executor/simulator_test.py
@@ -20,6 +20,7 @@ from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from backend.blocks.llm import LlmModel
+from backend.blocks.orchestrator import ExecutionMode, OrchestratorBlock
from backend.executor.simulator import (
_DEFAULT_SIMULATOR_MODEL,
_extract_cost_usd,
@@ -171,11 +172,130 @@ class TestPrepareDryRun:
# Simulation model must parse as a real LlmModel so OrchestratorBlock's
# Pydantic input validation accepts it.
assert LlmModel(result["model"]) is not None
+ # The injected model must be a canonical LlmModel value (string equal
+ # to one of the enum's ``.value``s), not an OpenRouter alias slug —
+ # OrchestratorBlock.validate_data → jsonschema only accepts literal
+ # ``LlmModel.value``s in the schema's ``enum``, and the alias map
+ # in ``LlmModel._missing_`` does not surface in the generated
+ # JSON Schema. Anything else trips
+ # ``"'<slug>' is not one of [...]"`` at runtime.
+ canonical_values = {m.value for m in LlmModel}
+ assert result["model"] in canonical_values, (
+ f"prepare_dry_run injected non-canonical model {result['model']!r}; "
+ f"jsonschema validation will reject it"
+ )
# credentials left as-is so block schema validation passes —
# actual creds injected via extra_exec_kwargs in manager.py
assert "credentials" not in result
assert result["_dry_run_api_key"] == "sk-or-test-key"
+ def test_orchestrator_invalid_sim_model_override_falls_back_to_default(
+ self,
+ ) -> None:
+ """An invalid ``CHAT_SIMULATION_MODEL`` env value must not crash
+ ``prepare_dry_run`` — fall back to the default so dry-run keeps
+ working. Without the guard, ``LlmModel('<garbage>')`` raises
+ ``ValueError`` and aborts every Orchestrator dry-run."""
+ with (
+ patch(
+ "backend.executor.simulator._get_platform_openrouter_key",
+ return_value="sk-or-test-key",
+ ),
+ patch(
+ "backend.executor.simulator._simulator_model",
+ return_value="not-a-real-model-slug",
+ ),
+ ):
+ result = prepare_dry_run(
+ OrchestratorBlock(),
+ {
+ "prompt": "test",
+ "model": LlmModel.CLAUDE_4_7_OPUS.value,
+ "agent_mode_max_iterations": 1,
+ },
+ )
+ assert result is not None
+ # Must land on the default value, not the garbage override.
+ assert result["model"] == LlmModel(_DEFAULT_SIMULATOR_MODEL).value, (
+ "Invalid CHAT_SIMULATION_MODEL should fall back to default; "
+ f"got {result['model']!r}"
+ )
+
+ def test_orchestrator_forces_built_in_execution_mode(self) -> None:
+ """prepare_dry_run overrides ``execution_mode`` to ``BUILT_IN``
+ regardless of user choice. With ``sim_model`` defaulting to
+ Gemini Flash-Lite (provider=open_router):
+
+ - BUILT_IN routes ``llm.llm_call`` to its open_router branch
+ (OpenAI SDK against openrouter.ai with the OR key) — works.
+ - EXTENDED_THINKING would hit the SDK subprocess's
+ ``model.value.startswith("claude")`` guard and raise
+ ``ValueError`` for any non-Claude sim_model.
+
+ Honouring the user's pick would force sim_model back to Claude
+ (to satisfy EXTENDED_THINKING), which in turn breaks the
+ LLM-simulation path for every non-Orchestrator block in the
+ same graph (Claude wraps JSON-mode output in markdown fences,
+ Gemini doesn't)."""
+ block = OrchestratorBlock()
+ with patch(
+ "backend.executor.simulator._get_platform_openrouter_key",
+ return_value="sk-or-test-key",
+ ):
+ # User explicitly picked EXTENDED_THINKING — the dry-run
+ # still overrides to BUILT_IN.
+ result = prepare_dry_run(
+ block,
+ {
+ "prompt": "test",
+ "model": LlmModel.CLAUDE_4_7_OPUS.value,
+ "execution_mode": ExecutionMode.EXTENDED_THINKING.value,
+ "agent_mode_max_iterations": 1,
+ },
+ )
+ assert result is not None
+ assert result["execution_mode"] == ExecutionMode.BUILT_IN.value, (
+ f"prepare_dry_run must force BUILT_IN to keep Gemini sim_model "
+ f"off the SDK's Claude-only gate; got {result['execution_mode']!r}"
+ )
+
+ def test_orchestrator_input_passes_jsonschema_validation(self) -> None:
+ """The injected dry-run input must pass OrchestratorBlock.validate_data.
+
+ Pinning this prevents the SECRT-2368 follow-up bug class where
+ prepare_dry_run injects an OpenRouter slug that LlmModel resolves
+ via the alias map at the Pydantic layer, but jsonschema enum
+ validation (which runs *before* Pydantic) rejects.
+ """
+ block = OrchestratorBlock()
+ user_input = {
+ "prompt": "test",
+ "model": LlmModel.CLAUDE_4_7_OPUS.value,
+ "credentials": {
+ "id": "00000000-0000-0000-0000-000000000000",
+ "provider": "open_router",
+ "type": "api_key",
+ },
+ "agent_mode_max_iterations": 1,
+ "execution_mode": "built_in",
+ "multiple_tool_calls": False,
+ "max_tokens": 50,
+ "retry": 0,
+ }
+ with patch(
+ "backend.executor.simulator._get_platform_openrouter_key",
+ return_value="sk-or-test-key",
+ ):
+ dry_input = prepare_dry_run(block, user_input)
+ assert dry_input is not None
+ # Strip simulator-internal markers before validating, just like
+ # manager.py does before calling Input(**...).
+ validation_input = {k: v for k, v in dry_input.items() if not k.startswith("_")}
+ err = block.input_schema.validate_data(validation_input)
+ assert (
+ err is None
+ ), f"prepare_dry_run produced input that fails jsonschema validation: {err}"
+
def test_orchestrator_zero_stays_zero(self) -> None:
from unittest.mock import patch
@@ -570,29 +690,32 @@ def _sim_completion(*, content: str, usage: CompletionUsage) -> ChatCompletion:
class TestDefaultSimulatorModel:
- """Pin the default model — three guards. Anyone flipping this without
- a cost review trips the value pin. Anyone setting it to a non-enum
- value fails the ``LlmModel`` parseability check (the bug class behind
- SECRT-2368). Anyone setting it to a direct-provider snapshot ID like
- ``claude-haiku-4-5-20251001`` fails the OpenRouter-slug check —
- OpenRouter's OpenAI-compat endpoint, which the LLM-simulation path
- uses, rejects anything that isn't in ``<vendor>/<model>`` form."""
+ """Pin the default model. Four guards line up with the constraints
+ laid out next to ``_DEFAULT_SIMULATOR_MODEL`` in ``simulator.py``:
+ value pin, ``LlmModel`` parseability, OpenRouter slug shape, and
+ ``open_router`` provider routing (so the BUILT_IN orchestrator path
+ in ``llm.llm_call`` doesn't get routed through ``api.anthropic.com``
+ with the platform OR key)."""
- def test_default_is_haiku_openrouter_slug(self) -> None:
- assert _DEFAULT_SIMULATOR_MODEL == "anthropic/claude-haiku-4-5"
+ def test_default_is_gemini_flash_lite(self) -> None:
+ assert _DEFAULT_SIMULATOR_MODEL == "google/gemini-2.5-flash-lite"
def test_default_parses_as_llm_model(self) -> None:
- # OrchestratorBlock.Input.model is typed as LlmModel; the default
- # must resolve through the enum (either directly or via the
- # OpenRouter-alias map in ``LlmModel._missing_``).
- assert LlmModel(_DEFAULT_SIMULATOR_MODEL) is LlmModel.CLAUDE_4_5_HAIKU
+ assert LlmModel(_DEFAULT_SIMULATOR_MODEL) is LlmModel.GEMINI_2_5_FLASH_LITE
def test_default_is_openrouter_slug(self) -> None:
# The LLM-simulation path hits OpenRouter's OpenAI-compat endpoint,
- # which only accepts canonical ``<vendor>/<model>`` slugs. Anything
- # else returns HTTP 400 even if it's a real provider model.
+ # which only accepts canonical ``<vendor>/<model>`` slugs.
assert "/" in _DEFAULT_SIMULATOR_MODEL
+ def test_default_provider_is_open_router(self) -> None:
+ # ``llm.llm_call`` dispatches on ``llm_model.metadata.provider``.
+ # An ``anthropic`` provider would route OR-dry-run-credentials
+ # at ``api.anthropic.com`` → 401. Pin ``open_router`` here so
+ # a future default change that breaks this routing trips at
+ # unit-test time.
+ assert LlmModel(_DEFAULT_SIMULATOR_MODEL).metadata.provider == "open_router"
+
class TestExtractCostUsd:
"""Provider-reported USD cost via typed ``model_extra`` — mirrors