AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): auto-reroute OpenAI models through OpenRouter when configured (#13029)
Zamil Majdy4a39a08861af
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.pyindex a947edffc7..52ccda9827 100644--- a/autogpt_platform/backend/backend/blocks/llm.py+++ b/autogpt_platform/backend/backend/blocks/llm.py@@ -15,6 +15,7 @@ import openaifrom anthropic.types import ToolParamfrom groq import AsyncGroqfrom openai.types.chat import ChatCompletion as OpenAIChatCompletion+from openai.types.shared_params import ResponseFormatJSONObjectfrom pydantic import BaseModel, SecretStrfrom backend.blocks._base import (@@ -816,32 +817,29 @@ def convert_openai_tool_fmt_to_anthropic(def extract_openrouter_cost(response: OpenAIChatCompletion) -> float | None:- """Extract OpenRouter's `x-total-cost` header from an OpenAI SDK response.-- OpenRouter returns the per-request USD cost in a response header. The- OpenAI SDK exposes the raw httpx response via an undocumented `_response`- attribute. We use try/except AttributeError so that if the SDK ever drops- or renames that attribute, the warning is visible in logs rather than- silently degrading to no cost tracking.+ """Extract OpenRouter's per-request USD cost from a chat-completion response.++ OpenRouter populates a ``cost`` field on the standard ``usage`` object (a+ USD float) when the request body includes ``usage: {"include": True}``.+ The OpenAI SDK's typed ``CompletionUsage`` does not declare it, so we read+ it off ``model_extra`` (pydantic v2's typed extras container) — no+ ``getattr``. Mirrors backend/executor/simulator.py::_extract_cost_usd —+ keep the two aligned."""- try:- raw_resp = response._response # type: ignore[attr-defined]- except AttributeError:- logger.warning(- "OpenAI SDK response missing _response attribute"- " — OpenRouter cost tracking unavailable"- )+ usage = response.usage+ if usage is None:+ return None+ extras = usage.model_extra or {}+ cost = extras.get("cost")+ if cost is None:return Nonetry:- cost_header = raw_resp.headers.get("x-total-cost")- if not cost_header:- return None- cost = float(cost_header)- if not math.isfinite(cost) or cost < 0:- return None- return cost- except (ValueError, TypeError, AttributeError):+ cost_f = float(cost)+ except (TypeError, ValueError):+ return None+ if not math.isfinite(cost_f) or cost_f < 0:return None+ return cost_fdef extract_openai_reasoning(response) -> str | None:@@ -926,10 +924,10 @@ async def llm_call(provider = llm_model.metadata.providercontext_window = llm_model.context_window- # Transparent OpenRouter routing for Anthropic models: when an OpenRouter API key- # is configured, route direct-Anthropic models through OpenRouter instead. This- # gives us the x-total-cost header for free, so provider_cost is always populated- # without manual token-rate arithmetic.+ # Transparent OpenRouter routing for Anthropic models: when an OpenRouter+ # API key is configured, route direct-Anthropic models through OpenRouter+ # instead. This gives us per-request USD cost for free, so provider_cost+ # is populated without manual token-rate arithmetic.or_key = settings.secrets.open_router_api_keyor_model_id: str | None = Noneif provider == "anthropic" and or_key:@@ -1199,11 +1197,19 @@ async def llm_call("HTTP-Referer": "https://agpt.co","X-Title": "AutoGPT",},+ # Ask OpenRouter to include the per-request USD cost on the usage+ # object. Same shape used by simulator.py — keep aligned.+ extra_body={"usage": {"include": True}},model=or_model_id or llm_model.value,messages=prompt, # type: ignoremax_tokens=max_tokens,tools=tools_param, # type: ignoreparallel_tool_calls=parallel_tool_calls_param,+ response_format=(+ ResponseFormatJSONObject(type="json_object")+ if force_json_output+ else openai.omit+ ),)if not response.choices:diff --git a/autogpt_platform/backend/backend/blocks/test/test_llm.py b/autogpt_platform/backend/backend/blocks/test/test_llm.pyindex f7be1e100f..45e9be60ee 100644--- a/autogpt_platform/backend/backend/blocks/test/test_llm.py+++ b/autogpt_platform/backend/backend/blocks/test/test_llm.py@@ -1,4 +1,4 @@-from typing import cast+from typing import Any, castfrom unittest.mock import AsyncMock, MagicMock, patchimport anthropic@@ -28,8 +28,14 @@ class TestLLMStatsTracking:mock_response.output = []mock_response.usage = MagicMock(input_tokens=10, output_tokens=20)- # Test with mocked OpenAI response- with patch("openai.AsyncOpenAI") as mock_openai:+ # Test with mocked OpenAI response. Suppress the OpenRouter key so the+ # auto-reroute branch doesn't divert this test to the chat.completions+ # path under a local .env that has open_router_api_key set.+ with (+ patch("openai.AsyncOpenAI") as mock_openai,+ patch("backend.blocks.llm.settings") as mock_settings,+ ):+ mock_settings.secrets.open_router_api_key = ""mock_client = AsyncMock()mock_openai.return_value = mock_clientmock_client.responses.create = AsyncMock(return_value=mock_response)@@ -523,7 +529,11 @@ class TestLLMStatsTracking:mock_response.usage = MagicMock(input_tokens=50, output_tokens=30)return mock_response- with patch("openai.AsyncOpenAI") as mock_openai:+ with (+ patch("openai.AsyncOpenAI") as mock_openai,+ patch("backend.blocks.llm.settings") as mock_settings,+ ):+ mock_settings.secrets.open_router_api_key = ""mock_client = AsyncMock()mock_openai.return_value = mock_clientmock_client.responses.create = mock_create@@ -1227,67 +1237,72 @@ class TestLlmModelMissing:class TestExtractOpenRouterCost:- """Tests for extract_openrouter_cost — the x-total-cost header parser."""+ """Tests for extract_openrouter_cost — reads ``response.usage.model_extra['cost']``.++ OpenRouter's ``cost`` field is not a declared attribute on the OpenAI SDK's+ ``CompletionUsage``; pydantic v2 puts unknown fields in ``model_extra`` when+ the model is configured with ``extra='allow'``. Tests here exercise that+ typed-access path — no duck typing.+ """- def _mk_response(self, headers: dict | None):+ def _mk_response(self, *, cost: Any = ...):+ """Build a response with ``usage.model_extra['cost']`` set to ``cost``.++ Pass ``cost=...`` (the default) to omit the cost key entirely.+ """response = MagicMock()- if headers is None:- response._response = None+ response.usage = MagicMock()+ if cost is ...:+ response.usage.model_extra = {}else:- raw = MagicMock()- raw.headers = headers- response._response = raw+ response.usage.model_extra = {"cost": cost}return responsedef test_extracts_numeric_cost(self):- response = self._mk_response({"x-total-cost": "0.0042"})- assert llm.extract_openrouter_cost(response) == 0.0042-- def test_returns_none_when_header_missing(self):- response = self._mk_response({})- assert llm.extract_openrouter_cost(response) is None+ assert llm.extract_openrouter_cost(self._mk_response(cost=0.0042)) == 0.0042- def test_returns_none_when_header_empty_string(self):- response = self._mk_response({"x-total-cost": ""})- assert llm.extract_openrouter_cost(response) is None+ def test_extracts_string_cost(self):+ assert llm.extract_openrouter_cost(self._mk_response(cost="0.0042")) == 0.0042- def test_returns_none_when_header_non_numeric(self):- response = self._mk_response({"x-total-cost": "not-a-number"})- assert llm.extract_openrouter_cost(response) is None+ def test_returns_none_when_cost_missing(self):+ assert llm.extract_openrouter_cost(self._mk_response()) is None- def test_returns_none_when_no_response_attr(self):- response = MagicMock(spec=[]) # no _response attr- assert llm.extract_openrouter_cost(response) is None+ def test_returns_none_when_cost_is_none(self):+ assert llm.extract_openrouter_cost(self._mk_response(cost=None)) is None- def test_returns_none_when_raw_is_none(self):- response = self._mk_response(None)+ def test_returns_none_when_usage_missing(self):+ response = MagicMock()+ response.usage = Noneassert llm.extract_openrouter_cost(response) is None- def test_returns_none_when_raw_has_no_headers(self):+ def test_returns_none_when_model_extra_is_none(self):response = MagicMock()- response._response = MagicMock(spec=[]) # no headers attr+ response.usage = MagicMock()+ response.usage.model_extra = Noneassert llm.extract_openrouter_cost(response) is Nonedef test_returns_zero_for_zero_cost(self):"""Zero-cost is a valid value (free tier) and must not become None."""- response = self._mk_response({"x-total-cost": "0"})- assert llm.extract_openrouter_cost(response) == 0.0+ assert llm.extract_openrouter_cost(self._mk_response(cost=0)) == 0.0++ def test_returns_none_for_non_numeric(self):+ assert (+ llm.extract_openrouter_cost(self._mk_response(cost="not-a-number")) is None+ )def test_returns_none_for_inf(self):- response = self._mk_response({"x-total-cost": "inf"})- assert llm.extract_openrouter_cost(response) is None+ assert llm.extract_openrouter_cost(self._mk_response(cost=float("inf"))) is Nonedef test_returns_none_for_negative_inf(self):- response = self._mk_response({"x-total-cost": "-inf"})- assert llm.extract_openrouter_cost(response) is None+ assert (+ llm.extract_openrouter_cost(self._mk_response(cost=float("-inf"))) is None+ )def test_returns_none_for_nan(self):- response = self._mk_response({"x-total-cost": "nan"})- assert llm.extract_openrouter_cost(response) is None+ assert llm.extract_openrouter_cost(self._mk_response(cost=float("nan"))) is Nonedef test_returns_none_for_negative_cost(self):- response = self._mk_response({"x-total-cost": "-0.005"})- assert llm.extract_openrouter_cost(response) is None+ assert llm.extract_openrouter_cost(self._mk_response(cost=-0.005)) is Noneclass TestAnthropicCacheControl: