Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

refactor(backend/copilot): price dream inference from the LLM catalog and fix structured output on the native Anthropic path (#14957)

Nicholas Tindlea8ca21ad9d13
diff --git a/.gitignore b/.gitignore
index 0661426b73..4d03dc5576 100644
--- a/.gitignore
+++ b/.gitignore
@@ -185,6 +185,9 @@ ig*
!autogpt_platform/frontend/public/experts/**
!autogpt_platform/frontend/public/autogpt-characters/**
!autogpt_platform/backend/backend/api/features/experts/avatar_references/**
+# And the copilot price card: an ignored module is untracked, and ruff
+# (force-exclude) and black skip it too, so it would ship unlinted.
+!autogpt_platform/backend/backend/copilot/price_card*.py
.github_access_token
LICENSE.rtf
autogpt_platform/backend/settings.py
diff --git a/autogpt_platform/backend/backend/blocks/test/test_llm.py b/autogpt_platform/backend/backend/blocks/test/test_llm.py
index b2c2440515..8b7da010df 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_llm.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_llm.py
@@ -1267,7 +1267,7 @@ class TestAnthropicCacheControl:
@pytest.mark.asyncio
async def test_system_prompt_sent_as_block_with_cache_control(self):
"""The system prompt is wrapped in a structured block with cache_control ephemeral."""
- mock_resp = MagicMock()
+ mock_resp = MagicMock(stop_reason="end_turn")
mock_resp.content = [MagicMock(type="text", text="hello")]
mock_resp.usage = MagicMock(input_tokens=5, output_tokens=3)
@@ -1304,7 +1304,7 @@ class TestAnthropicCacheControl:
@pytest.mark.asyncio
async def test_last_tool_gets_cache_control(self):
"""cache_control is placed on the last tool in the Anthropic tools list."""
- mock_resp = MagicMock()
+ mock_resp = MagicMock(stop_reason="end_turn")
mock_resp.content = [MagicMock(type="text", text="ok")]
mock_resp.usage = MagicMock(input_tokens=10, output_tokens=5)
@@ -1360,7 +1360,7 @@ class TestAnthropicCacheControl:
@pytest.mark.asyncio
async def test_no_tools_no_cache_control_on_tools(self):
"""When there are no tools, the Anthropic call receives anthropic.NOT_GIVEN for tools."""
- mock_resp = MagicMock()
+ mock_resp = MagicMock(stop_reason="end_turn")
mock_resp.content = [MagicMock(type="text", text="ok")]
mock_resp.usage = MagicMock(input_tokens=5, output_tokens=2)
@@ -1401,7 +1401,7 @@ class TestAnthropicCacheControl:
Anthropic rejects empty text blocks; the guard in llm_call must ensure
the system argument is omitted entirely when no system messages are present.
"""
- mock_resp = MagicMock()
+ mock_resp = MagicMock(stop_reason="end_turn")
mock_resp.content = [MagicMock(type="text", text="ok")]
mock_resp.usage = MagicMock(input_tokens=3, output_tokens=2)
@@ -1436,7 +1436,7 @@ class TestAnthropicCacheControl:
only whitespace should NOT reach the Anthropic API (it would be rejected
as an empty text block).
"""
- mock_resp = MagicMock()
+ mock_resp = MagicMock(stop_reason="end_turn")
mock_resp.content = [MagicMock(type="text", text="ok")]
mock_resp.usage = MagicMock(input_tokens=3, output_tokens=2)
diff --git a/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py b/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py
index 4ff06b2ac1..842140c99b 100644
--- a/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py
+++ b/autogpt_platform/backend/backend/copilot/anthropic_rate_card_test.py
@@ -179,12 +179,11 @@ class TestSonnet5Rates:
prompt_tokens=1_000_000,
completion_tokens=1_000_000,
)
- # Intro rates in litellm today: 1M x $2 + 1M x $10 = $12 — the
- # Opus fallback would charge $90. The catalog deliberately bills
- # sticker ($3/$15); this asserts the vendored litellm value, so
- # when the intro window ends (2026-08-31) and a rate refresh
- # brings sticker rates, update this to 18.0 — that change is
- # expected, not a mispricing.
+ # Anthropic's list rates: 1M x $2 + 1M x $10 = $12 — the Opus
+ # fallback would charge $90. The scheduled rise to $3/$15 was
+ # cancelled, so $12 stays right; the catalog's Sonnet 5 credits
+ # are set against $3/$15, but credits are the platform's billing
+ # rate, not Anthropic's.
assert cost == 12.0
diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/dream_billing_test.py b/autogpt_platform/backend/backend/copilot/billing_e2e/dream_billing_test.py
new file mode 100644
index 0000000000..af7c0a8dc3
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/dream_billing_test.py
@@ -0,0 +1,152 @@
+"""Dream inference over real HTTP through the production usage ledger seam.
+
+The provider speaks Anthropic's wire format. Ledger writes use the existing
+billing_e2e capture, so these tests do not require a database or Redis.
+"""
+
+import pytest
+
+from backend.copilot.dream import billing, llm, orchestrator
+from backend.copilot.dream.llm import DreamLLMError
+from backend.copilot.dream.schemas import ConsolidationOutput
+from backend.copilot.transport_routing import ProviderRoutingKwargs
+from backend.data.block_cost_config import compute_token_credits
+from backend.data.execution import NodeExecutionStats
+
+from . import dream_provider as provider_fixtures
+from .ledger import PlatformLedger
+
+
[email protected](loop_scope="function")
[email protected]("prose", [False, True])
+ "model,expected_microdollars",
+ [
+ ("anthropic/claude-sonnet-5-5", 5605),
+ ("anthropic/claude-opus-5-5", 11210),
+ ("anthropic/claude-haiku-4-5", 2953),
+ ("anthropic/claude-haiku-5-5", 295),
+ ("anthropic/claude-sonnet-5", 5906),
+ ("anthropic/claude-fable-5-1", 27274),
+ ],
+)
+async def test_native_dream_http_usage_reaches_ledger(
+ model: str,
+ expected_microdollars: int,
+ prose: bool,
+ dream_provider: provider_fixtures.DreamProvider,
+ ledger: PlatformLedger,
+) -> None:
+ dream_provider.prose = prose
+ completion = await llm.structured_completion(
+ model=model,
+ messages=[{"role": "user", "content": "Consolidate facts."}],
+ response_model=ConsolidationOutput,
+ )
+ assert completion.value.facts == []
+ usage = orchestrator._phase_usage_from_completion(
+ "consolidate", completion.usage, "sync_baseline"
+ )
+ await billing.record_phase_cost(
+ user_id="dream-owner",
+ pass_id="dream-pass",
+ phase_usage=usage,
+ execution_path="sync_baseline",
+ )
+ await ledger.settle()
+ assert len(dream_provider.requests) == 1
+ sent = dream_provider.requests[0]
+ assert "facts" in sent["tools"][0]["input_schema"]["properties"]
+ assert sent["model"] == model.removeprefix("anthropic/")
+ if model.endswith(("sonnet-5-5", "opus-5-5", "fable-5-1")):
+ assert sent["tool_choice"]["type"] == "auto"
+ assert [(s.user_id, s.microdollars, s.skip_daily) for s in ledger.spend] == [
+ ("dream-owner", expected_microdollars, True)
+ ]
+ (row,) = ledger.rows_in(0)
+ assert row.cost_microdollars == expected_microdollars
+ assert (
+ row.input_tokens,
+ row.output_tokens,
+ row.cache_read_tokens,
+ row.cache_creation_tokens,
+ ) == (1001, 203, 3007, 509)
+ assert row.graph_exec_id == "dream-pass"
+ assert row.provider == "anthropic"
+ assert row.metadata is not None
+ assert row.metadata["discount_applied"] == 0.0
+
+
[email protected](loop_scope="function")
+async def test_truncated_http_response_preserves_usage_but_rejects_valid_json(
+ dream_provider: provider_fixtures.DreamProvider,
+) -> None:
+ dream_provider.stop_reason = "max_tokens"
+ with pytest.raises(DreamLLMError, match="max_tokens") as caught:
+ await llm.structured_completion(
+ model="anthropic/claude-sonnet-5-5",
+ messages=[{"role": "user", "content": "Consolidate facts."}],
+ response_model=ConsolidationOutput,
+ )
+ assert caught.value.usage is not None
+ assert caught.value.usage.input_tokens == 1001
+ assert caught.value.usage.output_tokens == 203
+ assert caught.value.usage.cache_read_tokens == 3007
+ assert caught.value.usage.cache_creation_tokens == 509
+
+
+ "cache_tokens,expected", [(1, 1), (66666, 1), (66667, 2), (1000000, 15)]
+)
+def test_sonnet_cache_credit_rate_and_rounding(
+ cache_tokens: int, expected: int
+) -> None:
+ stats = NodeExecutionStats(cache_read_token_count=cache_tokens)
+ assert compute_token_credits({"model": "claude-sonnet-5-5"}, stats) == expected
+
+
[email protected](loop_scope="function")
+ "reported_cost,expected_microdollars", [(None, 5605), (0.0042, 4200)]
+)
+async def test_openrouter_cache_buckets_and_reported_cost(
+ reported_cost: float | None,
+ expected_microdollars: int,
+ dream_provider: provider_fixtures.DreamProvider,
+ ledger: PlatformLedger,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ dream_provider.cost = reported_cost
+ routing = ProviderRoutingKwargs(
+ provider="open_router",
+ api_key="sk-test",
+ base_url=None,
+ supports_flex=True,
+ cost_log_provider="open_router",
+ )
+ monkeypatch.setattr(llm, "routing_kwargs_for_chat_transport", lambda: routing)
+ monkeypatch.setattr(billing, "routing_kwargs_for_chat_transport", lambda: routing)
+ completion = await llm.structured_completion(
+ model="anthropic/claude-sonnet-5-5",
+ messages=[{"role": "user", "content": "Consolidate facts as JSON."}],
+ response_model=ConsolidationOutput,
+ )
+ usage = orchestrator._phase_usage_from_completion(
+ "consolidate", completion.usage, "sync_baseline"
+ )
+ await billing.record_phase_cost(
+ user_id="dream-owner",
+ pass_id="dream-pass",
+ phase_usage=usage,
+ execution_path="sync_baseline",
+ )
+ await ledger.settle()
+ (row,) = ledger.rows_in(0)
+ assert row.input_tokens == 1001
+ assert row.cache_read_tokens == 3007
+ assert row.cache_creation_tokens == 509
+ assert row.cost_microdollars == expected_microdollars
+ assert [s.microdollars for s in ledger.spend] == [expected_microdollars]
+
+
+dream_provider = provider_fixtures.dream_provider
diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/dream_provider.py b/autogpt_platform/backend/backend/copilot/billing_e2e/dream_provider.py
new file mode 100644
index 0000000000..2f3ca8ccb3
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/dream_provider.py
@@ -0,0 +1,125 @@
+"""Local Anthropic and OpenRouter HTTP responses for dream billing tests."""
+
+from collections.abc import AsyncIterator
+
+import anthropic
+import openai
+import pytest
+import pytest_asyncio
+from aiohttp import web
+
+from backend.copilot.dream import billing, llm
+from backend.copilot.transport_routing import ProviderRoutingKwargs
+from backend.util.llm import providers
+
+
+class DreamProvider:
+ def __init__(self) -> None:
+ self.requests: list[dict] = []
+ self.stop_reason = "tool_use"
+ self.prose = False
+ self.cost: float | None = None
+ self.usage = {
+ "input_tokens": 1001,
+ "output_tokens": 203,
+ "cache_read_input_tokens": 3007,
+ "cache_creation_input_tokens": 509,
+ }
+
+ async def messages(self, request: web.Request) -> web.Response:
+ body = await request.json()
+ self.requests.append(body)
+ tool = body["tools"][0]
+ content = [
+ {"type": "text", "text": "Here is the result."},
+ {
+ "type": "tool_use",
+ "id": "toolu_test",
+ "name": tool["name"],
+ "input": {"facts": []},
+ },
+ ]
+ if self.prose:
+ content = [
+ {"type": "text", "text": 'Here it is:\n```json\n{"facts": []}\n```'}
+ ]
+ return web.json_response(
+ {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "model": body["model"],
+ "content": content,
+ "stop_reason": self.stop_reason,
+ "stop_sequence": None,
+ "usage": self.usage,
+ }
+ )
+
+ async def chat(self, request: web.Request) -> web.Response:
+ body = await request.json()
+ self.requests.append(body)
+ return web.json_response(
+ {
+ "id": "gen-test",
+ "object": "chat.completion",
+ "created": 1,
+ "model": body["model"],
+ "choices": [
+ {
+ "index": 0,
+ "finish_reason": "stop",
+ "message": {
+ "role": "assistant",
+ "content": '{"facts": []}',
+ },
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 4517,
+ "completion_tokens": 203,
+ "total_tokens": 4720,
+ "prompt_tokens_details": {
+ "cached_tokens": 3007,
+ "cache_write_tokens": 509,
+ },
+ "cost": self.cost,
+ },
+ }
+ )
+
+
+@pytest_asyncio.fixture(loop_scope="function")
+async def dream_provider(
+ monkeypatch: pytest.MonkeyPatch,
+) -> AsyncIterator[DreamProvider]:
+ provider = DreamProvider()
+ app = web.Application()
+ app.router.add_post("/v1/messages", provider.messages)
+ app.router.add_post("/v1/chat/completions", provider.chat)
+ runner = web.AppRunner(app)
+ await runner.setup()
+ await web.TCPSite(runner, "127.0.0.1", 0).start()
+ host, port = runner.addresses[0][:2]
+ async with anthropic.AsyncAnthropic(
+ api_key="sk-test", base_url=f"http://{host}:{port}", max_retries=0
+ ) as client, openai.AsyncOpenAI(
+ api_key="sk-test", base_url=f"http://{host}:{port}/v1", max_retries=0
+ ) as openai_client:
+ monkeypatch.setattr(providers.openai, "AsyncOpenAI", lambda **_: openai_client)
+ monkeypatch.setattr(providers.anthropic, "AsyncAnthropic", lambda **_: client)
+ routing = ProviderRoutingKwargs(
+ provider="anthropic",
+ api_key="sk-test",
+ base_url=None,
+ supports_flex=False,
+ cost_log_provider="anthropic",
+ )
+ monkeypatch.setattr(llm, "routing_kwargs_for_chat_transport", lambda: routing)
+ monkeypatch.setattr(
+ billing, "routing_kwargs_for_chat_transport", lambda: routing
+ )
+ try:
+ yield provider
+ finally:
+ await runner.cleanup()
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
index c56e6446f8..7893ca58e8 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py
@@ -53,9 +53,9 @@ from .batch_submit import (
read_lock_token,
submit_phase,
)
-from .billing import record_phase_cost
+from .billing import priced_phase_usage, record_phase_cost
+from .llm import DreamLLMError, parse_json_with_prose_fallback
from .locks import release_dream_lock
-from .model_pricing import compute_cost_usd
from .schemas import (
DreamOperations,
DreamOperationsSnapshot,
@@ -503,10 +503,15 @@ async def _handle_phase_result(
# Validate the row's content matches the phase's Pydantic schema
# BEFORE persisting — corrupted content shouldn't pollute the
- # accumulator for the next phase to read back.
+ # accumulator for the next phase to read back. Parsed the way the sync
+ # path parses it: a model the output tool couldn't be forced on may
+ # answer in text, its JSON fenced or behind prose, so what is stored
+ # (and read back by the next phase) is the JSON alone.
try:
- PHASE_RESPONSE_MODELS[phase].model_validate(json.loads(row.content))
- except (json.JSONDecodeError, ValidationError) as exc:
+ payload = parse_json_with_prose_fallback(row.content)
+ PHASE_RESPONSE_MODELS[phase].model_validate(payload)
+ except (DreamLLMError, ValidationError) as exc:
+ row.error = f"invalid output shape — {type(exc).__name__}"
await _write_phase_to_state(pass_id=pass_id, phase=phase, row=row)
await _fail_pass(
user_id=user_id,
@@ -514,11 +519,13 @@ async def _handle_phase_result(
pass_id=pass_id,
job_id=job_id,
phase_models=phase_models,
- error=f"{phase}: invalid output shape — {type(exc).__name__}",
+ error=f"{phase}: {row.error}",
)
return
- await _write_phase_to_state(pass_id=pass_id, phase=phase, row=row)
+ await _write_phase_to_state(
+ pass_id=pass_id, phase=phase, row=row.with_content(json.dumps(payload))
+ )
next_phase = NEXT_PHASE[phase]
if next_phase is not None:
@@ -995,11 +1002,10 @@ async def _log_all_phase_costs(
whatever phases landed (matches the documented "partial pass
charges for completed phases" semantic in ``dream/billing.py``).
- Cost is computed via ``dream/model_pricing.compute_cost_usd`` —
- the dream rate card — so the batch path uses the same native-
- Anthropic token convention (additive cache buckets, not subtracted)
- as the sync path. The ``execution_path="anthropic_batch"`` arg
- applies the 50% batch discount there.
+ Each phase is priced by ``billing.priced_phase_usage`` from its
+ model's catalog price card (``backend/copilot/price_card.py``), the
+ card the sync path falls back to as well: Anthropic's additive cache
+ buckets, less the batch path's half-price discount.
No-ops on per-phase failure — apply already wrote the user-facing
memory operations; a cost-log blip shouldn't take that down.
@@ -1014,8 +1020,6 @@ async def _log_all_phase_costs(
for phase, row in state.items():
try:
if row.get("error"):
- # Phase errored — don't record usage for a phase that
- # didn't complete; downstream phases never ran either.
continue
phase_model = phase_models.get(phase)
if not phase_model:
@@ -1029,14 +1033,6 @@ async def _log_all_phase_costs(
output_tokens = int(row.get("output_tokens") or 0)
cache_read_tokens = int(row.get("cache_read_tokens") or 0)
cache_creation_tokens = int(row.get("cache_creation_tokens") or 0)
- cost_usd = compute_cost_usd(
- model=phase_model,
- input_tokens=input_tokens,
- output_tokens=output_tokens,
- cache_read_tokens=cache_read_tokens,
- cache_creation_tokens=cache_creation_tokens,
- execution_path="anthropic_batch",
- )
usage = PhaseUsage(
phase=phase, # type: ignore[arg-type]
model=phase_model,
@@ -1044,12 +1040,11 @@ async def _log_all_phase_costs(
output_tokens=output_tokens,
cache_read_tokens=cache_read_tokens,
cache_creation_tokens=cache_creation_tokens,
- cost_usd=cost_usd,
)
await record_phase_cost(
user_id=user_id,
pass_id=pass_id,
- phase_usage=usage,
+ phase_usage=priced_phase_usage(usage, "anthropic_batch"),
execution_path="anthropic_batch",
)
except Exception:
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
index c837017038..d225e415ae 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py
@@ -14,7 +14,10 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from backend.copilot.dream.batch_callbacks import handle_dream_batch_result
+from backend.copilot.dream.batch_callbacks import (
+ _write_phase_to_state,
+ handle_dream_batch_result,
+)
from backend.copilot.dream.batch_submit import input_bundle_key, persist_input_bundle
from backend.copilot.dream.fetch import DreamInput
from backend.copilot.dream.job_status import read_status, write_initial_status
@@ -101,9 +104,9 @@ def _entry(
"job_id": job_id,
"phase": phase,
"phase_models": {
- "consolidate": "claude-sonnet-4-6",
- "recombine": "claude-opus-4-7",
- "sanitize": "claude-sonnet-4-6",
+ "consolidate": "claude-sonnet-5",
+ "recombine": "claude-opus-5-5",
+ "sanitize": "claude-sonnet-5",
},
"custom_ids": [custom_id],
"phase_for_custom_id": {custom_id: phase},
@@ -402,7 +405,7 @@ class TestPhaseChaining:
self, fake_redis
):
"""Phase 3 is terminal: apply runs, all three phases logged at
- anthropic_batch path (50% discount in the rate card), JobStatus
+ anthropic_batch path (half the catalog list price), JobStatus
flips to complete."""
from backend.copilot.dream.batch_callbacks import _write_phase_to_state
from backend.copilot.dream.batch_submit import persist_input_bundle
@@ -493,9 +496,20 @@ class TestPhaseChaining:
call.kwargs["phase_usage"].phase: call.kwargs["phase_usage"].model
for call in record_cost.await_args_list
}
- assert models_by_phase["consolidate"] == "claude-sonnet-4-6"
- assert models_by_phase["recombine"] == "claude-opus-4-7"
- assert models_by_phase["sanitize"] == "claude-sonnet-4-6"
+ assert models_by_phase["consolidate"] == "claude-sonnet-5"
+ assert models_by_phase["recombine"] == "claude-opus-5-5"
+ assert models_by_phase["sanitize"] == "claude-sonnet-5"
+ # ...and priced from that model's catalog card at half the list
+ # price: each ``_row`` carries 10 input + 20 output tokens.
+ costs_by_phase = {
+ call.kwargs["phase_usage"].phase: call.kwargs["phase_usage"].cost_usd
+ for call in record_cost.await_args_list
+ }
+ sonnet_5_cost = (10 * 2.0 + 20 * 10.0) / 1_000_000 / 2
+ opus_5_5_cost = (10 * 4.0 + 20 * 20.0) / 1_000_000 / 2
+ assert costs_by_phase["consolidate"] == pytest.approx(sonnet_5_cost)
+ assert costs_by_phase["recombine"] == pytest.approx(opus_5_5_cost)
+ assert costs_by_phase["sanitize"] == pytest.approx(sonnet_5_cost)
@pytest.mark.asyncio
async def test_expert_terminal_result_applies_and_releases_in_expert_scope(
@@ -782,7 +796,8 @@ class TestErrorPaths:
assert "unknown batch phase" in (final.error or "")
@pytest.mark.asyncio
- async def test_errored_row_short_circuits_to_mark_errored(self, fake_redis):
+ @pytest.mark.parametrize("error", ["content moderation", "max_tokens"])
+ async def test_errored_row_short_circuits_to_mark_errored(self, fake_redis, error):
await _persist_autopilot_bundle()
mark_errored = AsyncMock()
record_cost = AsyncMock()
@@ -798,22 +813,22 @@ class TestErrorPaths:
_row(
custom_id="p1:consolidate",
content="",
- error="content moderation",
+ error=error,
)
],
)
mark_errored.assert_awaited_once()
- assert "content moderation" in mark_errored.call_args.kwargs["error"]
- # The errored phase itself is not billed.
+ assert error in mark_errored.call_args.kwargs["error"]
record_cost.assert_not_awaited()
+ redis, _, _ = fake_redis
+ stored = json.loads(redis.hset.await_args.args[2])
+ assert stored["error"] == error
+ assert stored["input_tokens"] == 10
+ assert stored["output_tokens"] == 20
@pytest.mark.asyncio
- async def test_invalid_json_marks_errored_does_not_chain(self, fake_redis):
- """A row whose content is not parseable JSON must NOT chain to
- the next phase — the next phase's prompt would be built on
- garbage. Pydantic's default ``model_validate`` is permissive
- about unknown fields (extra="ignore"), so the failure mode the
- test pins is invalid-JSON not unknown-field."""
+ @pytest.mark.parametrize("content", ["this is not JSON", '{"facts": "invalid"}'])
+ async def test_invalid_json_marks_errored_does_not_chain(self, fake_redis, content):
await _persist_autopilot_bundle()
submit_phase = AsyncMock()
mark_errored = AsyncMock()
@@ -830,13 +845,87 @@ class TestErrorPaths:
[
_row(
custom_id="p1:consolidate",
- content="this is not JSON at all",
+ content=content,
)
],
)
submit_phase.assert_not_awaited()
mark_errored.assert_awaited_once()
assert "invalid output shape" in mark_errored.call_args.kwargs["error"]
+ record_cost.assert_not_awaited()
+ redis, _, _ = fake_redis
+ stored = json.loads(redis.hset.await_args.args[2])
+ assert "invalid output shape" in stored["error"]
+ assert stored["input_tokens"] == 10
+ assert stored["output_tokens"] == 20
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "content,error",
+ [
+ (_RECOMBINE_CONTENT, "max_tokens"),
+ ('{"proposals": "invalid"}', None),
+ ("", "provider error"),
+ ],
+ )
+ async def test_failed_phase_only_charges_earlier_completed_phase(
+ self, fake_redis, content, error
+ ):
+ await _persist_autopilot_bundle()
+ await _write_phase_to_state(
+ pass_id="p1",
+ phase="consolidate",
+ row=_row(custom_id="p1:consolidate", content=_CONSOLIDATE_CONTENT),
+ )
+ record_cost = AsyncMock()
+ with (
+ patch("backend.copilot.dream.job_status.mark_errored", AsyncMock()),
+ patch(
+ "backend.copilot.dream.batch_callbacks.record_phase_cost", record_cost
+ ),
+ ):
+ await handle_dream_batch_result(
+ _entry(phase="recombine"),
+ [_row(custom_id="p1:recombine", content=content, error=error)],
+ )
+ record_cost.assert_awaited_once()
+ usage = record_cost.await_args.kwargs["phase_usage"]
+ assert usage.phase == "consolidate"
+ assert usage.cost_usd == pytest.approx((10 * 2 + 20 * 10) / 1_000_000 / 2)
+
+ @pytest.mark.asyncio
+ async def test_text_answer_around_the_json_still_chains(self, fake_redis):
+ """A model the output tool can't be forced on (``auto``) may answer
+ in text. The JSON in it, fenced behind a line of prose, is the
+ phase result, and the next phase reads the JSON alone."""
+ await _persist_autopilot_bundle()
+ submit_phase = AsyncMock(
+ return_value=MagicMock(provider_batch_id="msgbatch_recombine")
+ )
+ mark_errored = AsyncMock()
+ with (
+ patch("backend.copilot.dream.batch_callbacks.submit_phase", submit_phase),
+ patch("backend.copilot.dream.job_status.mark_errored", mark_errored),
+ patch("backend.copilot.dream.job_status.update_status_phase", AsyncMock()),
+ patch(
+ "backend.copilot.dream.batch_callbacks._anthropic_api_key",
+ return_value="sk-ant-test",
+ ),
+ ):
+ await handle_dream_batch_result(
+ _entry(phase="consolidate"),
+ [
+ _row(
+ custom_id="p1:consolidate",
+ content='Here are the facts:\n```json\n{"facts": []}\n```',
+ )
+ ],
+ )
+ mark_errored.assert_not_awaited()
+ submit_phase.assert_awaited_once()
+ assert submit_phase.call_args.kwargs["consolidated_json"] == (
+ _CONSOLIDATE_CONTENT
+ )
@pytest.mark.asyncio
async def test_apply_crash_marks_errored_still_records_usage(self, fake_redis):
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_submit.py b/autogpt_platform/backend/backend/copilot/dream/batch_submit.py
index 4575d15c7a..d97ca8c879 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_submit.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_submit.py
@@ -3,9 +3,10 @@
Both the orchestrator (kicking off phase 1) and the dream
``batch_callbacks`` (chaining phases 2 → 3 as results arrive) need to
submit identically: build the per-phase prompt with whatever upstream
-phase outputs we have, force tool_use structured output against the
-phase's Pydantic schema, then enqueue the submission so the
-BatchExecutor polls it. This module owns that flow.
+phase outputs we have, ask for tool_use structured output against the
+phase's Pydantic schema (forced where the phase model accepts a forced
+tool), then enqueue the submission so the BatchExecutor polls it. This
+module owns that flow.
Splitting it out:
* Keeps the orchestrator focused on routing / cost accounting and
@@ -25,19 +26,25 @@ from typing import TYPE_CHECKING, Literal
from pydantic import BaseModel
from backend.copilot.graphiti.scope import MemoryScope
+from backend.copilot.model_normalize import normalize_model_for_anthropic
from backend.copilot.trial_cost_context import capture_cost_context
from backend.executor.batch_executor import (
INITIAL_POLL_DELAY_SECONDS,
PendingEntry,
enqueue_pending,
)
+from backend.util.feature_flag import Flag
from backend.util.llm.providers import (
BatchSubmissionRef,
ProviderResponse,
call_provider,
cancel_batch,
)
-from backend.util.llm.tool_use import force_tool_choice, pydantic_to_anthropic_tool
+from backend.util.llm.tool_use import (
+ is_forced_tool_choice,
+ pydantic_to_anthropic_tool,
+ structured_tool_choice,
+)
from .fetch import DreamInput
from .locks import read_dream_lock_token
@@ -47,6 +54,7 @@ from .prompts import (
build_sanitize_prompt,
)
from .schemas import ConsolidationOutput, DreamOperations, RecombinationOutput
+from .structured_output import OUTPUT_TOOL_CALL_ONCE, with_output_tool_instruction
if TYPE_CHECKING:
from backend.copilot.config import ChatConfig
@@ -56,9 +64,14 @@ logger = logging.getLogger(__name__)
DreamPhase = Literal["consolidate", "recombine", "sanitize"]
-# Tool name the model must emit for each phase. Forced ``tool_choice``
-# constrains output to exactly one tool_use block whose ``input`` matches
-# the corresponding Pydantic schema — no JSON-parse-prose failures.
+# Tool name the model must emit for each phase. A forced ``tool_choice``
+# makes the model answer with exactly one tool_use block — no prose around
+# the JSON. It does not make the ``input`` schema-valid (that takes
+# Anthropic's strict tool mode, which is not enabled), so ``batch_callbacks``
+# still validates each result against the phase's Pydantic model. A model
+# that rejects a forced tool gets ``auto``, with the tool description and
+# the last user turn asking for the call; a text answer is parsed on
+# arrival.
PHASE_TOOL_NAMES: dict[DreamPhase, str] = {
"consolidate": "emit_consolidation",
"recombine": "emit_recombination",
@@ -73,9 +86,11 @@ PHASE_RESPONSE_MODELS: dict[DreamPhase, type[BaseModel]] = {
}
-# Max output tokens per phase (mirrors the sync orchestrator's values).
+# Max output tokens per phase (mirrors the sync orchestrator's values). The
+# budget covers thinking as well as the JSON: orchestrator.py's
+# CONSOLIDATE_MAX_TOKENS says why consolidate's is not 4096.
PHASE_MAX_TOKENS: dict[DreamPhase, int] = {
- "consolidate": 4096,
+ "consolidate": 16384,
"recombine": 16384,
"sanitize": 16384,
}
@@ -103,20 +118,6 @@ PHASE_DESCRIPTIONS: dict[DreamPhase, str] = {
}
-def _to_native_anthropic_model(model: str) -> str:
- """``anthropic/claude-opus-4.7`` → ``claude-opus-4-7``.
-
- The batch path always submits to Anthropic's native Batches API
- (direct key), regardless of the chat transport, so strip the
- OpenRouter vendor prefix AND convert dots to hyphens — native
- Anthropic rejects both the prefix and the dot-separated version,
- and the dream rate card only carries the hyphenated id.
- """
- if "/" in model:
- model = model.split("/", 1)[1]
- return model.replace(".", "-")
-
-
def phase_models_for_config(config: "ChatConfig") -> dict[str, str]:
"""Per-phase model map (native-Anthropic form) for the batch path.
@@ -125,14 +126,42 @@ def phase_models_for_config(config: "ChatConfig") -> dict[str, str]:
thinking) for recombine. Built once at submit and threaded through
the batch payload so the model used to *submit* a phase is the exact
model used to *price* it — no single-model fan-out across phases.
+
+ The batch path always submits to Anthropic's native Batches API,
+ whatever the chat transport, so every model takes the native spelling
+ (``normalize_model_for_anthropic``), and a non-Anthropic model raises
+ ``ValueError`` here rather than submitting a batch that fails hours
+ later.
"""
+ standard = _native_phase_model(
+ config.fast_standard_model, "CHAT_FAST_STANDARD_MODEL"
+ )
return {
- "consolidate": _to_native_anthropic_model(config.fast_standard_model),
- "recombine": _to_native_anthropic_model(config.fast_advanced_model),
- "sanitize": _to_native_anthropic_model(config.fast_standard_model),
+ "consolidate": standard,
+ "recombine": _native_phase_model(
+ config.fast_advanced_model, "CHAT_FAST_ADVANCED_MODEL"
+ ),
+ "sanitize": standard,
}
+def _native_phase_model(model: str, setting: str) -> str:
+ """*model* in its native Anthropic spelling. The shared normalizer's
+ error suggests enabling OpenRouter, which this path never uses, so a
+ non-Anthropic model gets the two remedies that do apply."""
+ try:
+ return normalize_model_for_anthropic(model)
+ except ValueError as exc:
+ raise ValueError(
+ "The dream batch path submits every phase to Anthropic's Message "
+ f"Batches API, so its phase models must be Anthropic models; got "
+ f"{setting}={model!r}. Choose Anthropic phase models (anthropic/* "
+ "or claude-* slugs), or disable batch routing (the "
+ f"{Flag.DREAM_PASS_BATCH_ENABLED.value!r} flag) so dreams run on "
+ "the sync path."
+ ) from exc
+
+
async def submit_phase(
*,
user_id: str,
@@ -171,11 +200,18 @@ async def submit_phase(
recombined_json=recombined_json,
)
tool_name = PHASE_TOOL_NAMES[phase]
+ tool_choice = structured_tool_choice(model, tool_name)
+ description = PHASE_DESCRIPTIONS[phase]
+ if not is_forced_tool_choice(tool_choice):
+ # Nothing forces the call, so the tool description and the last
+ # user turn ask for it.
+ description = f"{description} {OUTPUT_TOOL_CALL_ONCE}"
+ messages = with_output_tool_instruction(messages, tool_name)
tools = [
pydantic_to_anthropic_tool(
PHASE_RESPONSE_MODELS[phase],
tool_name=tool_name,
- description=PHASE_DESCRIPTIONS[phase],
+ description=description,
)
]
# Anthropic's custom_id must match ``^[a-zA-Z0-9_-]{1,64}$`` —
@@ -192,7 +228,7 @@ async def submit_phase(
max_tokens=PHASE_MAX_TOKENS[phase],
temperature=PHASE_TEMPERATURES[phase],
tools=tools,
- tool_choice=force_tool_choice(tool_name),
+ tool_choice=tool_choice,
execution_mode="batch",
custom_id=custom_id,
)
diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_submit_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_submit_test.py
index 92ad7e76d3..f82558be0d 100644
--- a/autogpt_platform/backend/backend/copilot/dream/batch_submit_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/batch_submit_test.py
@@ -2,30 +2,43 @@
Covers the orphan-prevention guard (a paid provider batch must be
cancelled when the BatchExecutor enqueue fails afterwards — otherwise it
-runs to completion with no callback to consume it) and the dream-lock
+runs to completion with no callback to consume it), the dream-lock
ownership token riding on the persisted input bundle so the batch
-callback can compare-and-delete the lock hours later.
+callback can compare-and-delete the lock hours later, and what a phase
+submits: the native model spelling and a tool carrying its schema, forced
+where the model accepts a forced tool.
"""
from __future__ import annotations
from datetime import datetime, timezone
+from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from backend.copilot import usage_activation_fixtures
+from backend.copilot.config import ChatConfig
from backend.copilot.dream.batch_submit import (
INPUT_TTL_SECONDS,
+ PHASE_DESCRIPTIONS,
input_bundle_key,
persist_input_bundle,
+ phase_models_for_config,
read_input_bundle,
read_lock_token,
submit_phase,
)
from backend.copilot.dream.fetch import DreamInput
+from backend.copilot.dream.schemas import ConsolidationOutput
+from backend.copilot.dream.structured_output import OUTPUT_TOOL_CALL_ONCE
from backend.copilot.graphiti.scope import DREAM_LOCK_KEY_PREFIX, MemoryScope
from backend.util.llm.providers import BatchSubmissionRef
+from backend.util.llm.tool_use import (
+ auto_tool_choice,
+ force_tool_choice,
+ pydantic_to_anthropic_tool,
+)
def _bundle(expert_id: str | None = None) -> DreamInput:
@@ -249,4 +262,140 @@ async def test_read_lock_token_none_when_bundle_corrupted(fake_redis):
assert await read_lock_token("p3") is None
+async def test_submitted_phase_tool_carries_the_phase_schema(fake_redis):
+ """The forced tool is built Anthropic-shaped (``input_schema``). The
+ provider's tool conversion read only ``parameters``, so every phase
+ went out with an empty schema and the model had to guess the shape."""
+ create = AsyncMock(return_value=SimpleNamespace(id="msgbatch_schema"))
+ client = SimpleNamespace(
+ messages=SimpleNamespace(batches=SimpleNamespace(create=create))
+ )
+ with patch(
+ "backend.util.llm.providers.anthropic.AsyncAnthropic", return_value=client
+ ), patch(
+ "backend.copilot.dream.batch_submit.enqueue_pending",
+ AsyncMock(return_value=None),
+ ):
+ await submit_phase(
+ user_id="u1",
+ pass_id="p1",
+ job_id="j1",
+ phase="consolidate",
+ phase_models={"consolidate": "claude-sonnet-5"},
+ api_key="sk-test",
+ input_bundle=_bundle(),
+ )
+
+ params = create.call_args.kwargs["requests"][0]["params"]
+ expected = pydantic_to_anthropic_tool(
+ ConsolidationOutput, tool_name="emit_consolidation", description=""
+ )["input_schema"]
+ (tool,) = params["tools"]
+ assert tool["name"] == "emit_consolidation"
+ assert tool["input_schema"]["properties"] == expected["properties"]
+ assert tool["input_schema"]["required"] == expected.get("required", [])
+ assert tool["input_schema"]["properties"]["facts"]["items"]["properties"]
+ # A forced tool keeps its description as written.
+ assert tool["description"] == PHASE_DESCRIPTIONS["consolidate"]
+ assert params["tool_choice"] == force_tool_choice("emit_consolidation")
+ assert params["model"] == "claude-sonnet-5"
+ # A forced tool needs no prompt line asking for it.
+ assert "emit_consolidation" not in params["messages"][-1]["content"]
+
+
[email protected]("model", ["claude-opus-5-5", "claude-sonnet-5-5"])
+async def test_restricted_model_phase_offers_the_tool_under_auto(fake_redis, model):
+ """Opus and Sonnet 5.5 answer a forced ``tool_choice`` with a 400, on the Batches
+ API only hours later in the result row, so its phase goes out under
+ ``auto``, the last user turn asking for one call with the full result."""
+ create = AsyncMock(return_value=SimpleNamespace(id="msgbatch_auto"))
+ client = SimpleNamespace(
+ messages=SimpleNamespace(batches=SimpleNamespace(create=create))
+ )
+ with patch(
+ "backend.util.llm.providers.anthropic.AsyncAnthropic", return_value=client
+ ), patch(
+ "backend.copilot.dream.batch_submit.enqueue_pending",
+ AsyncMock(return_value=None),
+ ):
+ await submit_phase(
+ user_id="u1",
+ pass_id="p1",
+ job_id="j1",
+ phase="recombine",
+ phase_models={"recombine": model},
+ api_key="sk-test",
+ input_bundle=_bundle(),
+ consolidated_json='{"facts": []}',
+ )
+
+ params = create.call_args.kwargs["requests"][0]["params"]
+ assert params["model"] == model
+ assert params["tool_choice"] == auto_tool_choice()
+ (tool,) = params["tools"]
+ assert tool["name"] == "emit_recombination"
+ assert tool["description"] == (
+ f"{PHASE_DESCRIPTIONS['recombine']} {OUTPUT_TOOL_CALL_ONCE}"
+ )
+ last = params["messages"][-1]
+ assert last["role"] == "user"
+ assert "emit_recombination" in last["content"]
+ # The ``$ref``'d enum field keeps its own default and description.
+ proposal = tool["input_schema"]["properties"]["proposals"]["items"]
+ memory_kind = proposal["properties"]["memory_kind"]
+ assert "finding" in memory_kind["enum"]
+ assert memory_kind["default"] == "finding"
+ assert memory_kind["description"].startswith("Envelope kind")
+
+
+def test_phase_models_take_the_native_anthropic_spelling():
+ """The batch path submits to Anthropic directly even when chat runs on
+ OpenRouter, so the OpenRouter spellings in config are converted."""
+ config = ChatConfig.model_construct(
+ fast_standard_model="anthropic/claude-sonnet-5",
+ fast_advanced_model="anthropic/claude-opus-5.5",
+ )
+ assert phase_models_for_config(config) == {
+ "consolidate": "claude-sonnet-5",
+ "recombine": "claude-opus-5-5",
+ "sanitize": "claude-sonnet-5",
+ }
+
+
+ "standard,advanced,named",
+ [
+ (
+ "openai/gpt-4.1-mini",
+ "anthropic/claude-opus-5.5",
+ "CHAT_FAST_STANDARD_MODEL='openai/gpt-4.1-mini'",
+ ),
+ (
+ "anthropic/claude-sonnet-5",
+ "llama3.1:8b",
+ "CHAT_FAST_ADVANCED_MODEL='llama3.1:8b'",
+ ),
+ ],
+)
+def test_phase_models_refuse_a_non_anthropic_model(
+ standard: str, advanced: str, named: str
+):
+ """Refused at submit rather than failing in the batch results hours
+ later, with the remedies this path has: it always submits to Anthropic
+ directly, so enabling OpenRouter is not one of them."""
+ config = ChatConfig.model_construct(
+ fast_standard_model=standard, fast_advanced_model=advanced
+ )
+ with pytest.raises(ValueError) as exc_info:
+ phase_models_for_config(config)
+ message = str(exc_info.value)
+ assert named in message
+ assert "Choose Anthropic phase models" in message
+ assert "disable batch routing" in message
+ assert "dream-pass-batch-enabled" in message
+ assert "OpenRouter" not in message
+
+
usage_snapshot = usage_activation_fixtures.usage_snapshot
diff --git a/autogpt_platform/backend/backend/copilot/dream/billing.py b/autogpt_platform/backend/backend/copilot/dream/billing.py
index 6a1ee741e7..4e7dedf042 100644
--- a/autogpt_platform/backend/backend/copilot/dream/billing.py
+++ b/autogpt_platform/backend/backend/copilot/dream/billing.py
@@ -9,6 +9,9 @@ shared billing primitives:
Redis lock is acquired, before phase 1 runs. Refuses the pass when
the user is paywalled (``NO_TIER`` + ``ENABLE_PLATFORM_PAYMENT``) or
has already exhausted their daily/weekly cap.
+* :func:`priced_phase_usage` — prices a phase the provider did not
+ price (native Anthropic, sync or batch) from the catalog price card
+ (``backend/copilot/price_card.py``) at the path's batch discount.
* :func:`record_phase_cost` — per-phase charge. Called after each of
consolidate / recombine / sanitize completes, charging the real LLM
spend against the user's window and writing a ``PlatformCostLog``
@@ -21,8 +24,10 @@ the existing per-block / per-provider rollups in the admin dashboard
keep working without dream-specific code paths.
Failure semantics mirror the chat path: a partial pass still charges
-for the phases that completed before the error, because we already
-paid the provider for those tokens.
+for the phases that completed before the error. The failed phase is
+absorbed on both sync and batch paths, including truncated or invalid
+structured output. Its usage stays on the error for telemetry, but its
+provider spend is not written to a cost row.
"""
from __future__ import annotations
@@ -31,6 +36,7 @@ import logging
from typing import Literal
from backend.copilot.config import ChatConfig
+from backend.copilot.price_card import compute_cost_usd, price_for
from backend.copilot.rate_limit import (
RateLimitExceeded,
RateLimitUnavailable,
@@ -41,7 +47,7 @@ from backend.copilot.rate_limit import (
from backend.copilot.token_tracking import persist_and_record_usage
from backend.copilot.transport_routing import routing_kwargs_for_chat_transport
-from .routing import ExecutionPath
+from .routing import ExecutionPath, batch_discount
from .schemas import PhaseUsage
logger = logging.getLogger(__name__)
@@ -144,6 +150,28 @@ async def check_dream_budget(
return True, None
+def priced_phase_usage(usage: PhaseUsage, execution_path: ExecutionPath) -> PhaseUsage:
+ """*usage* with its cost read off the model's catalog price card.
+
+ Each token bucket at the model's list rate, less the batch discount of
+ *execution_path*. The cost stays ``None`` when the model has no catalog
+ price (``price_for`` logs that); the phase then logs its tokens without
+ a charge.
+ """
+ price = price_for(usage.model)
+ if price is None:
+ return usage
+ cost = compute_cost_usd(
+ price=price,
+ input_tokens=usage.input_tokens,
+ output_tokens=usage.output_tokens,
+ cache_read_tokens=usage.cache_read_tokens,
+ cache_creation_tokens=usage.cache_creation_tokens,
+ discount=batch_discount(execution_path),
+ )
+ return usage.model_copy(update={"cost_usd": cost})
+
+
async def record_phase_cost(
*,
user_id: str,
@@ -164,9 +192,10 @@ async def record_phase_cost(
back to its parent dream pass — the same join key the future P9
inline ``dream.operations`` SSE event will reference.
- No-ops when the phase has no cost (skipped phase / unknown rate
- card). Matches ``persist_and_record_usage``'s contract: tokens
- without cost still log but don't charge the rate-limit counter.
+ No-ops when the phase has neither tokens nor a cost (a skipped
+ phase). Matches ``persist_and_record_usage``'s contract: tokens
+ without a cost (a model with no catalog price) still log but don't
+ charge the rate-limit counter.
"""
if (
phase_usage.cost_usd is None
@@ -200,6 +229,7 @@ async def record_phase_cost(
"dream_pass_id": pass_id,
"dream_phase": phase_usage.phase,
"execution_path": execution_path,
+ "discount_applied": batch_discount(execution_path),
},
# Dream is background work — it rolls up under the user's
# weekly cap but must not eat the interactive daily budget.
diff --git a/autogpt_platform/backend/backend/copilot/dream/billing_test.py b/autogpt_platform/backend/backend/copilot/dream/billing_test.py
index 23b3e11350..c795eff1d6 100644
--- a/autogpt_platform/backend/backend/copilot/dream/billing_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/billing_test.py
@@ -1,9 +1,11 @@
"""Dream-pass billing tests — pre-flight check + per-phase cost log.
-Covers the two billing seams the orchestrator uses:
+Covers the billing seams the orchestrator uses:
* check_dream_budget — paywall, rate-limit cap, Redis brown-out
* record_phase_cost — provider tag mapping, dream metadata,
block_name, graph_exec_id correlation, no-op on empty phase
+ * priced_phase_usage — catalog list price, batch discount, unpriced
+ model left unknown
"""
from __future__ import annotations
@@ -240,6 +242,23 @@ async def test_record_phase_cost_writes_dream_metadata():
assert metadata["dream_pass_id"] == "pass-xyz"
assert metadata["dream_phase"] == "consolidate"
assert metadata["execution_path"] == "anthropic_batch"
+ assert metadata["discount_applied"] == 0.5
+
+
+async def test_record_phase_cost_records_no_discount_on_the_sync_path():
+ spy = AsyncMock()
+ with patch.object(billing_mod, "persist_and_record_usage", new=spy):
+ await billing_mod.record_phase_cost(
+ user_id="u1",
+ pass_id="pass-xyz",
+ phase_usage=PhaseUsage(
+ phase="consolidate", model="m", input_tokens=1, cost_usd=0.001
+ ),
+ execution_path="sync_baseline",
+ )
+ assert spy.await_args is not None
+ assert spy.await_args.kwargs["extra_metadata"]["discount_applied"] == 0.0
@pytest.mark.asyncio
@@ -273,3 +292,44 @@ async def test_record_phase_cost_logs_token_only_phase_even_without_cost():
execution_path="sync_baseline",
)
spy.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# priced_phase_usage
+# ---------------------------------------------------------------------------
+
+
+def _sonnet_5_usage() -> PhaseUsage:
+ return PhaseUsage(
+ phase="consolidate",
+ model="anthropic/claude-sonnet-5",
+ input_tokens=1_000_000,
+ output_tokens=100_000,
+ cache_read_tokens=200_000,
+ cache_creation_tokens=10_000,
+ )
+
+
+# Sonnet 5's catalog card: $2 in, $10 out, $0.20 cache read, $2.50 cache
+# write per Mtok, every bucket billed on its own.
+_SONNET_5_LIST_COST = 2.0 + 1.0 + 0.04 + 0.025
+
+
+def test_priced_phase_usage_charges_the_list_price_on_the_sync_path():
+ priced = billing_mod.priced_phase_usage(_sonnet_5_usage(), "sync_baseline")
+ assert priced.cost_usd == pytest.approx(_SONNET_5_LIST_COST)
+ assert priced.model_dump(exclude={"cost_usd"}) == _sonnet_5_usage().model_dump(
+ exclude={"cost_usd"}
+ )
+
+
+def test_priced_phase_usage_halves_the_list_price_on_the_batch_path():
+ priced = billing_mod.priced_phase_usage(_sonnet_5_usage(), "anthropic_batch")
+ assert priced.cost_usd == pytest.approx(_SONNET_5_LIST_COST / 2)
+
+
+def test_priced_phase_usage_leaves_an_unpriced_model_unknown():
+ """No catalog price means an unknown cost, never a zero one: the row
+ logs tokens and charges nothing."""
+ usage = PhaseUsage(phase="recombine", model="vendor/no-such-model", input_tokens=10)
+ assert billing_mod.priced_phase_usage(usage, "sync_baseline").cost_usd is None
diff --git a/autogpt_platform/backend/backend/copilot/dream/llm.py b/autogpt_platform/backend/backend/copilot/dream/llm.py
index b5f208c4c3..e3af24e1d0 100644
--- a/autogpt_platform/backend/backend/copilot/dream/llm.py
+++ b/autogpt_platform/backend/backend/copilot/dream/llm.py
@@ -23,12 +23,14 @@ pinning a provider:
subscription Anthropic, direct Anthropic, and OpenRouter all
land at the same call site without per-transport branches here.
* ``response_format={"type":"json_object"}`` is supported across
- OpenAI, OpenRouter, and Ollama (forced JSON works on every
- transport this dispatcher accepts).
- * The Anthropic batch path in the orchestrator (see
- ``plans/idempotent-launching-moth.md`` component E) layers in
- *below* this wrapper via ``call_provider(execution_mode="batch")``
- when the transport supports it; this wrapper stays sync-only.
+ OpenAI, OpenRouter, and Ollama. The native Anthropic API ignores
+ it, so on the ``anthropic`` provider the output comes from one
+ tool built from the response model instead, the way the batch
+ path gets it (``structured_output.py``): forced where the model
+ accepts that, left to the model (``auto``) where it doesn't.
+ * The Anthropic batch path (``batch_submit.py``) calls
+ ``call_provider(execution_mode="batch")`` itself, beside this
+ wrapper rather than through it; this wrapper stays sync-only.
"""
from __future__ import annotations
@@ -37,10 +39,22 @@ import json
import logging
from typing import Generic, TypeVar
+import anthropic
from pydantic import BaseModel, ValidationError
-from backend.copilot.transport_routing import routing_kwargs_for_chat_transport
-from backend.util.llm.providers import ProviderLiteral, ProviderResponse, call_provider
+from backend.copilot.transport_routing import (
+ ProviderRoutingKwargs,
+ routing_kwargs_for_chat_transport,
+)
+from backend.util.llm.providers import (
+ BatchSubmissionRef,
+ ProviderLiteral,
+ ProviderResponse,
+ call_provider,
+ is_forced_tool_choice_rejection,
+)
+
+from .structured_output import StructuredRequest, structured_payload, structured_request
logger = logging.getLogger(__name__)
@@ -95,7 +109,13 @@ async def structured_completion(
max_output_tokens: int = 4096,
timeout_seconds: float | None = None,
) -> StructuredCompletion[T]:
- """Call the LLM in JSON mode and parse into ``response_model``.
+ """Call the LLM for structured output and parse into ``response_model``.
+
+ JSON mode on every provider but the native Anthropic API, which gets
+ ``model`` in its native spelling and one tool built from
+ ``response_model``, forced where the model accepts that (see
+ ``structured_output.py``); the tool call's arguments are parsed, or
+ the message text when there is none.
Returns a ``StructuredCompletion`` carrying both the parsed value
and a ``CompletionUsage`` block so the dream orchestrator can roll
@@ -115,16 +135,76 @@ async def structured_completion(
routing = routing_kwargs_for_chat_transport()
if not routing.api_key and routing.provider != "ollama":
raise DreamLLMError(_missing_api_key_message(routing.provider))
+ try:
+ request = structured_request(routing.provider, model, response_model)
+ except ValueError as exc:
+ raise DreamLLMError(str(exc)) from exc
+
+ response = await _call_provider_sync(
+ routing,
+ request,
+ messages=messages,
+ temperature=temperature,
+ max_output_tokens=max_output_tokens,
+ timeout_seconds=timeout_seconds,
+ )
+ usage = _usage_from_provider_response(response, request.model, routing.provider)
+
+ if response.stop_reason == "max_tokens":
+ raise DreamLLMError("LLM output truncated at max_tokens", usage)
+
+ content = structured_payload(response)
+ if not content:
+ raise DreamLLMError("LLM returned empty content", usage)
+ # Everything below this point is post-billing: the provider already
+ # charged for the tokens it sent, so every failure carries the usage out.
try:
- response = await call_provider(
+ payload = parse_json_with_prose_fallback(content)
+ return StructuredCompletion(
+ value=response_model.model_validate(payload),
+ usage=usage,
+ )
+ except DreamLLMError as exc:
+ exc.usage = usage
+ raise
+ except ValidationError as exc:
+ raise DreamLLMError(
+ f"LLM JSON did not match {response_model.__name__}: {exc}", usage
+ ) from exc
+
+
+async def _call_provider_sync(
+ routing: ProviderRoutingKwargs,
+ request: StructuredRequest,
+ *,
+ messages: list[dict[str, str]],
+ temperature: float,
+ max_output_tokens: int,
+ timeout_seconds: float | None,
+) -> ProviderResponse:
+ """One sync ``call_provider`` round trip; any failure is a ``DreamLLMError``
+ with no usage, since no response came back to bill.
+
+ A model that turns the forced output tool down (Anthropic's 400
+ ``BadRequestError`` naming ``tool_choice``, and only that) is asked
+ once more with the tool left to its choice and the prompt asking for
+ the call: the self-heal for a model missing from the provider's
+ forced-tool list. A second failure is final."""
+
+ async def send(
+ attempt: StructuredRequest,
+ ) -> ProviderResponse | BatchSubmissionRef:
+ return await call_provider(
provider=routing.provider,
- model=model,
+ model=attempt.model,
api_key=routing.api_key,
- messages=messages,
+ messages=attempt.prompt(messages),
max_tokens=max_output_tokens,
temperature=temperature,
- force_json_output=True,
+ force_json_output=attempt.force_json_output,
+ tools=attempt.tools,
+ tool_choice=attempt.tool_choice,
timeout_seconds=timeout_seconds,
# ``call_provider`` only honors ``ollama_host`` when
# ``provider="ollama"``; passing it on cloud transports is
@@ -134,50 +214,54 @@ async def structured_completion(
# ``ollama.AsyncClient`` wants the raw host:port.
ollama_host=_normalize_ollama_host(routing.base_url),
)
+
+ try:
+ try:
+ response = await send(request)
+ except anthropic.BadRequestError as exc:
+ if not (
+ request.forces_output_tool and is_forced_tool_choice_rejection(exc)
+ ):
+ raise
+ logger.warning(
+ "Model %s rejected the forced output tool (%s); retrying once "
+ "with tool_choice=auto. Add it to "
+ "_ANTHROPIC_FORCED_TOOL_CHOICE_UNSUPPORTED in "
+ "backend/util/llm/providers.py.",
+ request.model,
+ exc,
+ )
+ response = await send(request.with_output_tool_optional())
except DreamLLMError:
raise
except Exception as exc:
raise DreamLLMError(f"LLM call failed: {type(exc).__name__}: {exc}") from exc
if not isinstance(response, ProviderResponse):
- # ``call_provider`` only returns a non-ProviderResponse when the
- # caller passed ``execution_mode="batch"`` (lands later). The
- # dream's sync wrapper never opts in, so anything else is a bug.
+ # ``call_provider`` returns a ``BatchSubmissionRef`` only for
+ # ``execution_mode="batch"``, which the batch path
+ # (``batch_submit.submit_phase``) asks for; this sync wrapper never
+ # does, so anything else is a bug.
raise DreamLLMError(
"structured_completion expected a sync ProviderResponse but got a "
f"{type(response).__name__} — execution_mode must stay 'sync' here."
)
-
- usage = _usage_from_provider_response(response, model)
-
- content = (response.content or "").strip()
- if not content:
- raise DreamLLMError("LLM returned empty content", usage)
-
- # Everything below this point is post-billing: the provider already
- # charged for the tokens it sent, so every failure carries the usage out.
- try:
- payload = _parse_json_with_prose_fallback(content)
- return StructuredCompletion(
- value=response_model.model_validate(payload),
- usage=usage,
- )
- except DreamLLMError as exc:
- exc.usage = usage
- raise
- except ValidationError as exc:
- raise DreamLLMError(
- f"LLM JSON did not match {response_model.__name__}: {exc}", usage
- ) from exc
+ return response
def _usage_from_provider_response(
- response: ProviderResponse, model: str
+ response: ProviderResponse, model: str, provider: ProviderLiteral
) -> CompletionUsage:
- """Convert a ``ProviderResponse`` into the dream's ``CompletionUsage``."""
+ """Normalize input to disjoint buckets; native Anthropic already excludes caches."""
+ input_tokens = response.prompt_tokens
+ if provider != "anthropic":
+ input_tokens = max(
+ 0,
+ input_tokens - response.cache_read_tokens - response.cache_creation_tokens,
+ )
return CompletionUsage(
model=model,
- input_tokens=response.prompt_tokens,
+ input_tokens=input_tokens,
output_tokens=response.completion_tokens,
cache_read_tokens=response.cache_read_tokens,
cache_creation_tokens=response.cache_creation_tokens,
@@ -185,7 +269,7 @@ def _usage_from_provider_response(
)
-def _parse_json_with_prose_fallback(content: str) -> object:
+def parse_json_with_prose_fallback(content: str) -> object:
"""Parse JSON from a model response, recovering from common preamble bugs.
Two layers of defense, in order:
diff --git a/autogpt_platform/backend/backend/copilot/dream/llm_test.py b/autogpt_platform/backend/backend/copilot/dream/llm_test.py
index 329cfd2ce7..6a5a0db3a1 100644
--- a/autogpt_platform/backend/backend/copilot/dream/llm_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/llm_test.py
@@ -9,13 +9,19 @@ the fence-stripper that prevents the regression.
from __future__ import annotations
+import logging
+from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
+import anthropic
+import httpx
import pytest
from pydantic import BaseModel
from backend.copilot.transport_routing import ProviderRoutingKwargs
+from backend.util.llm.conversions import ToolCall, ToolContentBlock
from backend.util.llm.providers import ProviderResponse
+from backend.util.llm.tool_use import auto_tool_choice, force_tool_choice
from .llm import (
DreamLLMError,
@@ -24,6 +30,11 @@ from .llm import (
_strip_json_code_fence,
structured_completion,
)
+from .structured_output import (
+ output_tool_name,
+ structured_request,
+ with_output_tool_instruction,
+)
@pytest.mark.parametrize(
@@ -230,6 +241,9 @@ class TestStructuredCompletionDelegation:
# falls back to free-form text and our prose parser does extra
# work.
assert kwargs["force_json_output"] is True
+ # The forced tool is the native Anthropic path's device only.
+ assert kwargs["tools"] is None
+ assert kwargs["tool_choice"] is None
@pytest.mark.asyncio
async def test_caller_supplied_phase_timeout_reaches_call_provider(self):
@@ -444,3 +458,432 @@ class TestStructuredCompletionDelegation:
response_model=_SampleOutput,
)
assert result.value.facts[0].content == "recovered"
+
+
+# ---------------------------------------------------------------------------
+# Native Anthropic sync path: forced tool instead of JSON mode
+# ---------------------------------------------------------------------------
+
+
+def _bad_request(message: str) -> anthropic.BadRequestError:
+ response = httpx.Response(
+ 400, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
+ )
+ return anthropic.BadRequestError(message, response=response, body=None)
+
+
+def _server_error(message: str) -> anthropic.InternalServerError:
+ response = httpx.Response(
+ 500, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
+ )
+ return anthropic.InternalServerError(message, response=response, body=None)
+
+
+# Anthropic's 400 for a forced ``tool_choice`` on a model that takes none.
+_FORCED_TOOL_REJECTION = (
+ 'tool_choice: type "tool" and "any" are not supported for this model.'
+)
+
+
+def _tool_response(arguments: str, **usage: int) -> ProviderResponse:
+ """What ``call_provider`` returns for a forced tool call: ``content``
+ is the tool NAME, the JSON is in the tool call's arguments."""
+ name = output_tool_name(_SampleOutput)
+ return ProviderResponse(
+ content=name,
+ prompt_tokens=usage.get("prompt_tokens", 1),
+ completion_tokens=usage.get("completion_tokens", 1),
+ cache_read_tokens=usage.get("cache_read_tokens", 0),
+ tool_calls=[
+ ToolContentBlock(
+ id="toolu_1",
+ type="tool_use",
+ function=ToolCall(name=name, arguments=arguments),
+ )
+ ],
+ )
+
+
+class TestAnthropicToolPath:
+ """The native Anthropic API ignores JSON mode, so structured output
+ comes from one forced tool built from the response model, read back
+ from the tool call, with the model in its native spelling."""
+
+ @pytest.mark.asyncio
+ async def test_forces_one_tool_and_sends_the_native_model(self):
+ fake = _tool_response(
+ '{"facts": [{"content": "x", "confidence": 0.9}]}',
+ prompt_tokens=12,
+ completion_tokens=4,
+ cache_read_tokens=3,
+ )
+ call_provider_mock = AsyncMock(return_value=fake)
+
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ result = await structured_completion(
+ model="anthropic/claude-sonnet-5",
+ messages=[{"role": "user", "content": "give me a fact"}],
+ response_model=_SampleOutput,
+ )
+
+ assert result.value.facts[0].content == "x"
+ kwargs = call_provider_mock.call_args.kwargs
+ assert kwargs["provider"] == "anthropic"
+ assert kwargs["model"] == "claude-sonnet-5"
+ assert kwargs["force_json_output"] is False
+ tool_name = output_tool_name(_SampleOutput)
+ assert [tool["name"] for tool in kwargs["tools"]] == [tool_name]
+ assert kwargs["tools"][0]["input_schema"]["required"] == ["facts"]
+ assert kwargs["tool_choice"] == force_tool_choice(tool_name)
+ assert kwargs["tools"][0]["description"] == (
+ "Return the _SampleOutput result: call this once, with every "
+ "field the schema requires."
+ )
+ # A forced tool needs no prompt line asking for it.
+ assert kwargs["messages"] == [{"role": "user", "content": "give me a fact"}]
+ # Usage names the model that was called, the spelling the price
+ # card resolves, with its tokens.
+ assert result.usage.model == "claude-sonnet-5"
+ assert (result.usage.input_tokens, result.usage.cache_read_tokens) == (12, 3)
+
+ @pytest.mark.asyncio
+ async def test_opus_5_5_gets_auto_and_the_prompt_asks_for_the_call(self):
+ """Opus 5.5 answers a forced ``tool_choice`` with a 400, so the tool
+ goes out under ``auto``, and both its description and the last user
+ turn ask for one call with the complete result."""
+ fake = _tool_response('{"facts": [{"content": "x", "confidence": 0.9}]}')
+ call_provider_mock = AsyncMock(return_value=fake)
+
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ result = await structured_completion(
+ model="anthropic/claude-opus-5.5",
+ messages=[
+ {"role": "system", "content": "you consolidate facts"},
+ {"role": "user", "content": "give me a fact"},
+ ],
+ response_model=_SampleOutput,
+ )
+
+ assert result.value.facts[0].content == "x"
+ kwargs = call_provider_mock.call_args.kwargs
+ tool_name = output_tool_name(_SampleOutput)
+ assert kwargs["model"] == "claude-opus-5-5"
+ assert kwargs["tool_choice"] == auto_tool_choice()
+ # The sync description asks for one complete call in either mode.
+ assert "call this once" in kwargs["tools"][0]["description"]
+ system, user = kwargs["messages"]
+ assert system == {"role": "system", "content": "you consolidate facts"}
+ assert user["role"] == "user"
+ assert user["content"].startswith("give me a fact\n\n")
+ assert tool_name in user["content"]
+
+ @pytest.mark.asyncio
+ async def test_parses_the_message_text_when_no_tool_was_called(self):
+ fake = ProviderResponse(
+ content='{"facts": [{"content": "text", "confidence": 1.0}]}',
+ prompt_tokens=1,
+ completion_tokens=1,
+ )
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch(
+ "backend.copilot.dream.llm.call_provider",
+ new=AsyncMock(return_value=fake),
+ ):
+ result = await structured_completion(
+ model="claude-opus-5.5",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+ assert result.value.facts[0].content == "text"
+ assert result.usage.model == "claude-opus-5-5"
+
+ @pytest.mark.asyncio
+ async def test_tool_arguments_off_schema_raise_with_usage(self):
+ """The tool call was billed even when its arguments don't fit."""
+ fake = _tool_response('{"facts": "not a list"}', prompt_tokens=7)
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch(
+ "backend.copilot.dream.llm.call_provider",
+ new=AsyncMock(return_value=fake),
+ ):
+ with pytest.raises(DreamLLMError, match="did not match") as exc_info:
+ await structured_completion(
+ model="anthropic/claude-sonnet-5",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+ assert exc_info.value.usage is not None
+ assert exc_info.value.usage.input_tokens == 7
+
+ @pytest.mark.asyncio
+ async def test_non_anthropic_model_fails_before_any_call(self):
+ call_provider_mock = AsyncMock()
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ with pytest.raises(DreamLLMError, match="requires an Anthropic model"):
+ await structured_completion(
+ model="openai/gpt-4.1-mini",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+ call_provider_mock.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_through_the_real_anthropic_messages_call(self):
+ """End to end through ``call_provider``: the tool reaches the
+ Messages API carrying the schema, and its tool_use block comes back
+ as the parsed value."""
+ tool_name = output_tool_name(_SampleOutput)
+ message = anthropic.types.Message(
+ id="msg-1",
+ type="message",
+ role="assistant",
+ model="claude-sonnet-5",
+ content=[
+ anthropic.types.ToolUseBlock(
+ type="tool_use",
+ id="toolu_1",
+ name=tool_name,
+ input={"facts": [{"content": "e2e", "confidence": 0.5}]},
+ )
+ ],
+ stop_reason="tool_use",
+ usage=anthropic.types.Usage(input_tokens=20, output_tokens=6),
+ )
+ create = AsyncMock(return_value=message)
+ client = SimpleNamespace(messages=SimpleNamespace(create=create))
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch(
+ "backend.util.llm.providers.anthropic.AsyncAnthropic",
+ return_value=client,
+ ):
+ result = await structured_completion(
+ model="anthropic/claude-sonnet-5",
+ messages=[
+ {"role": "system", "content": "you consolidate facts"},
+ {"role": "user", "content": "facts"},
+ ],
+ response_model=_SampleOutput,
+ )
+
+ assert result.value.facts[0].content == "e2e"
+ assert result.usage.output_tokens == 6
+ sent = create.call_args.kwargs
+ assert sent["model"] == "claude-sonnet-5"
+ assert sent["tool_choice"] == force_tool_choice(tool_name)
+ (tool,) = sent["tools"]
+ assert set(tool["input_schema"]["properties"]) == {"facts"}
+ assert tool["input_schema"]["required"] == ["facts"]
+
+ @pytest.mark.asyncio
+ async def test_a_text_answer_under_auto_still_parses(self):
+ """End to end on Opus 5.5: ``auto`` goes out, the model answers in
+ text rather than calling the tool, and the JSON in that text
+ (fenced, behind a line of prose) is still the parsed value."""
+ message = anthropic.types.Message(
+ id="msg-2",
+ type="message",
+ role="assistant",
+ model="claude-opus-5-5",
+ content=[
+ anthropic.types.TextBlock(
+ type="text",
+ text=(
+ "Here are the facts:\n```json\n"
+ '{"facts": [{"content": "prose", "confidence": 0.4}]}'
+ "\n```"
+ ),
+ )
+ ],
+ stop_reason="end_turn",
+ usage=anthropic.types.Usage(input_tokens=9, output_tokens=5),
+ )
+ create = AsyncMock(return_value=message)
+ client = SimpleNamespace(messages=SimpleNamespace(create=create))
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch(
+ "backend.util.llm.providers.anthropic.AsyncAnthropic",
+ return_value=client,
+ ):
+ result = await structured_completion(
+ model="anthropic/claude-opus-5.5",
+ messages=[
+ {"role": "system", "content": "you consolidate facts"},
+ {"role": "user", "content": "facts"},
+ ],
+ response_model=_SampleOutput,
+ )
+
+ assert result.value.facts[0].content == "prose"
+ assert result.usage.model == "claude-opus-5-5"
+ assert result.usage.output_tokens == 5
+ sent = create.call_args.kwargs
+ assert sent["model"] == "claude-opus-5-5"
+ assert sent["tool_choice"] == auto_tool_choice()
+ assert output_tool_name(_SampleOutput) in sent["messages"][-1]["content"]
+
+ @pytest.mark.asyncio
+ async def test_forced_tool_rejection_retries_once_with_auto(self, caplog):
+ """A model missing from the forced-tool list answers the forced
+ choice with a 400. The call goes once more under ``auto``, the
+ prompt asking for the tool, and the swap is logged as a warning."""
+ fake = _tool_response('{"facts": [{"content": "healed", "confidence": 0.7}]}')
+ call_provider_mock = AsyncMock(
+ side_effect=[_bad_request(_FORCED_TOOL_REJECTION), fake]
+ )
+
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch(
+ "backend.copilot.dream.llm.call_provider", call_provider_mock
+ ), caplog.at_level(
+ logging.WARNING, logger="backend.copilot.dream.llm"
+ ):
+ result = await structured_completion(
+ model="anthropic/claude-sonnet-5",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+
+ assert result.value.facts[0].content == "healed"
+ tool_name = output_tool_name(_SampleOutput)
+ first, second = call_provider_mock.call_args_list
+ assert first.kwargs["tool_choice"] == force_tool_choice(tool_name)
+ assert first.kwargs["messages"] == [{"role": "user", "content": "hi"}]
+ assert second.kwargs["tool_choice"] == auto_tool_choice()
+ assert second.kwargs["model"] == "claude-sonnet-5"
+ assert tool_name in second.kwargs["messages"][-1]["content"]
+ assert "retrying once with tool_choice=auto" in caplog.text
+
+ @pytest.mark.asyncio
+ async def test_forced_tool_rejection_is_retried_only_once(self):
+ call_provider_mock = AsyncMock(
+ side_effect=[
+ _bad_request(_FORCED_TOOL_REJECTION),
+ _bad_request(_FORCED_TOOL_REJECTION),
+ ]
+ )
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ with pytest.raises(DreamLLMError, match="tool_choice") as exc_info:
+ await structured_completion(
+ model="anthropic/claude-sonnet-5",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+ assert call_provider_mock.await_count == 2
+ # Neither call came back with a response, so nothing was billed.
+ assert exc_info.value.usage is None
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "error",
+ [_server_error(_FORCED_TOOL_REJECTION), RuntimeError(_FORCED_TOOL_REJECTION)],
+ ids=["500", "runtime-error"],
+ )
+ async def test_only_a_400_rejection_is_retried(self, error: Exception):
+ """The documented text is not enough: a 5xx or an exception of our
+ own quoting it keeps the forced tool and fails once."""
+ call_provider_mock = AsyncMock(side_effect=error)
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ with pytest.raises(DreamLLMError, match="tool_choice"):
+ await structured_completion(
+ model="anthropic/claude-sonnet-5",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+ assert call_provider_mock.await_count == 1
+
+ @pytest.mark.asyncio
+ async def test_other_bad_requests_are_not_retried(self):
+ call_provider_mock = AsyncMock(
+ side_effect=_bad_request("messages: at least one message is required")
+ )
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="sk-ant-test"),
+ ), patch("backend.copilot.dream.llm.call_provider", call_provider_mock):
+ with pytest.raises(DreamLLMError, match="at least one message"):
+ await structured_completion(
+ model="anthropic/claude-sonnet-5",
+ messages=[{"role": "user", "content": "hi"}],
+ response_model=_SampleOutput,
+ )
+ assert call_provider_mock.await_count == 1
+
+
+class TestStructuredRequest:
+ """Which structured-output device each provider and model gets."""
+
+ def test_opus_5_5_offers_the_tool_under_auto(self):
+ request = structured_request(
+ "anthropic", "anthropic/claude-opus-5.5", _SampleOutput
+ )
+ assert request.model == "claude-opus-5-5"
+ assert request.tool_choice == auto_tool_choice()
+ assert not request.forces_output_tool
+
+ def test_sonnet_5_forces_the_tool_and_leaves_the_prompt_alone(self):
+ request = structured_request(
+ "anthropic", "anthropic/claude-sonnet-5", _SampleOutput
+ )
+ assert request.tool_choice == force_tool_choice(output_tool_name(_SampleOutput))
+ messages = [{"role": "user", "content": "hi"}]
+ assert request.prompt(messages) is messages
+
+ def test_json_mode_providers_get_no_tool(self):
+ request = structured_request(
+ "open_router", "anthropic/claude-opus-5.5", _SampleOutput
+ )
+ assert request.force_json_output
+ assert request.tools is None and request.tool_choice is None
+ messages = [{"role": "user", "content": "hi"}]
+ assert request.prompt(messages) is messages
+
+ def test_instruction_gets_its_own_turn_after_a_non_user_message(self):
+ messages = [{"role": "system", "content": "sys"}]
+ prompted = with_output_tool_instruction(messages, "emit_x")
+ assert [m["role"] for m in prompted] == ["system", "user"]
+ assert "emit_x" in prompted[-1]["content"]
+ assert messages == [{"role": "system", "content": "sys"}]
+
+
[email protected]("arguments", ["{}", '{"facts": []}'])
+async def test_truncated_tool_output_fails_with_usage(arguments):
+ fake = _tool_response(arguments, prompt_tokens=12, completion_tokens=4096)
+ fake.stop_reason = "max_tokens"
+ with patch(
+ "backend.copilot.dream.llm.routing_kwargs_for_chat_transport",
+ return_value=_anthropic_routing(api_key="test"),
+ ), patch("backend.copilot.dream.llm.call_provider", AsyncMock(return_value=fake)):
+ with pytest.raises(DreamLLMError, match="max_tokens") as caught:
+ await structured_completion(
+ model="claude-sonnet-5",
+ messages=[{"role": "user", "content": "test"}],
+ response_model=_SampleOutput,
+ )
+ assert caught.value.usage.input_tokens == 12
+ assert caught.value.usage.output_tokens == 4096
diff --git a/autogpt_platform/backend/backend/copilot/dream/model_pricing.py b/autogpt_platform/backend/backend/copilot/dream/model_pricing.py
deleted file mode 100644
index 4ee0fc3021..0000000000
--- a/autogpt_platform/backend/backend/copilot/dream/model_pricing.py
+++ /dev/null
@@ -1,168 +0,0 @@
-"""Per-model USD pricing for dream-pass cost calculation.
-
-Used as a fallback when the provider doesn't include a real cost on
-``usage.cost`` (direct OpenAI / Anthropic APIs) or when computing the
-batch-discounted equivalent. Rates are USD per 1M tokens.
-
-The hard-coded table is intentionally narrow — only the models the
-dream pass actually calls today. Adding a model requires updating this
-file; we'd rather fail loud on an unknown model than silently bill at
-zero. The OpenRouter path bypasses this entirely because OpenRouter
-already returns the real spot price as ``usage.cost``.
-
-On the Anthropic batch path the 50% discount from
-``execution_path_discount`` is multiplied into the computed cost so the
-savings flow through to the user via the shared cost ledger.
-"""
-
-from __future__ import annotations
-
-import logging
-
-from pydantic.dataclasses import dataclass
-
-from .routing import ExecutionPath
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass(frozen=True)
-class ModelRate:
- """Rate card for one model.
-
- All four token classes priced separately. ``None`` for a class
- means the model doesn't support it (e.g. no prompt cache).
- """
-
- input_per_mtok: float
- output_per_mtok: float
- cache_read_per_mtok: float | None = None
- cache_write_per_mtok: float | None = None
-
-
-# Rate card. Last updated 2026-05. Source: provider pricing pages.
-# Keep alphabetized by provider, then by model.
-_RATES: dict[str, ModelRate] = {
- # ---- Anthropic ----
- # Claude Opus 4.7
- "claude-opus-4-7": ModelRate(
- input_per_mtok=15.0,
- output_per_mtok=75.0,
- cache_read_per_mtok=1.5,
- cache_write_per_mtok=18.75,
- ),
- "anthropic/claude-opus-4.7": ModelRate(
- input_per_mtok=15.0,
- output_per_mtok=75.0,
- cache_read_per_mtok=1.5,
- cache_write_per_mtok=18.75,
- ),
- # Claude Opus 4.8 — published at $5/$25 per Mtok in/out (Anthropic's
- # official price list, verified 2026-09-26), matching catalog.py's
- # claude-opus-4-8 credits (750/3750 -> /150 margin divisor = $5/$25).
- # cache_write_per_mtok uses the 5-min-TTL rate ($6.25); the 1-hour-TTL
- # rate is $10.00 but this table has no per-TTL granularity yet.
- "claude-opus-4-8": ModelRate(
- input_per_mtok=5.0,
- output_per_mtok=25.0,
- cache_read_per_mtok=0.50,
- cache_write_per_mtok=6.25,
- ),
- "anthropic/claude-opus-4.8": ModelRate(
- input_per_mtok=5.0,
- output_per_mtok=25.0,
- cache_read_per_mtok=0.50,
- cache_write_per_mtok=6.25,
- ),
- # Claude Sonnet 4.6
- "claude-sonnet-4-6": ModelRate(
- input_per_mtok=3.0,
- output_per_mtok=15.0,
- cache_read_per_mtok=0.3,
- cache_write_per_mtok=3.75,
- ),
- "anthropic/claude-sonnet-4.6": ModelRate(
- input_per_mtok=3.0,
- output_per_mtok=15.0,
- cache_read_per_mtok=0.3,
- cache_write_per_mtok=3.75,
- ),
- # Claude Haiku 4.5
- "claude-haiku-4-5-20251001": ModelRate(
- input_per_mtok=1.0,
- output_per_mtok=5.0,
- cache_read_per_mtok=0.1,
- cache_write_per_mtok=1.25,
- ),
- # ---- OpenAI ----
- "gpt-5": ModelRate(input_per_mtok=10.0, output_per_mtok=30.0),
- "gpt-4o": ModelRate(
- input_per_mtok=2.5,
- output_per_mtok=10.0,
- cache_read_per_mtok=1.25,
- ),
-}
-
-
-# Anthropic's Message Batches API offers a 50% discount on the normal
-# rate for asynchronous batch processing. We pass that savings through
-# to the user — recorded in the cost ledger so the discount is auditable.
-_BATCH_DISCOUNTS: dict[ExecutionPath, float] = {
- "sync_baseline": 0.0,
- "anthropic_batch": 0.5,
-}
-
-
-def execution_path_discount(path: ExecutionPath) -> float:
- """Discount factor in [0.0, 1.0] applied to the base rate.
-
- Returns 0.5 for batch (50% off), 0.0 for sync_baseline.
- """
- return _BATCH_DISCOUNTS.get(path, 0.0)
-
-
-def compute_cost_usd(
- *,
- model: str,
- input_tokens: int,
- output_tokens: int,
- cache_read_tokens: int = 0,
- cache_creation_tokens: int = 0,
- execution_path: ExecutionPath = "sync_baseline",
-) -> float | None:
- """Compute USD cost for a single LLM call from the rate card.
-
- Returns None when the model isn't in the rate card — caller should
- log a warning and treat the cost as unknown rather than silently
- bill at zero. When the provider already supplied a real cost on
- ``usage.cost`` the caller should prefer that value over this
- fallback; this function is the synthetic estimate when no spot
- price is available.
-
- ``execution_path`` discount is applied at the very end so the
- return value is the *final* user-facing cost, not the pre-discount
- base.
- """
- rate = _RATES.get(model) or _RATES.get(model.lower())
- if rate is None:
- logger.warning(
- "dream model_pricing: no rate card for model %r — cost unknown", model
- )
- return None
-
- input_cost = (input_tokens / 1_000_000.0) * rate.input_per_mtok
- output_cost = (output_tokens / 1_000_000.0) * rate.output_per_mtok
-
- cache_read_cost = 0.0
- if cache_read_tokens and rate.cache_read_per_mtok is not None:
- cache_read_cost = (cache_read_tokens / 1_000_000.0) * rate.cache_read_per_mtok
-
- cache_write_cost = 0.0
- if cache_creation_tokens and rate.cache_write_per_mtok is not None:
- cache_write_cost = (
- cache_creation_tokens / 1_000_000.0
- ) * rate.cache_write_per_mtok
-
- base = input_cost + output_cost + cache_read_cost + cache_write_cost
- discount = execution_path_discount(execution_path)
- return base * (1.0 - discount)
diff --git a/autogpt_platform/backend/backend/copilot/dream/model_pricing_test.py b/autogpt_platform/backend/backend/copilot/dream/model_pricing_test.py
deleted file mode 100644
index 631f2fc7bc..0000000000
--- a/autogpt_platform/backend/backend/copilot/dream/model_pricing_test.py
+++ /dev/null
@@ -1,96 +0,0 @@
-from __future__ import annotations
-
-import pytest
-
-from .model_pricing import ModelRate, compute_cost_usd, execution_path_discount
-
-
-def test_unknown_model_returns_none():
- """Unknown model → None so callers can flag the cost as unknown
- rather than silently bill at zero."""
- assert (
- compute_cost_usd(
- model="totally-fake-model-xyz", input_tokens=100, output_tokens=50
- )
- is None
- )
-
-
-def test_known_model_basic_math():
- """Sonnet 4.6: $3/M input + $15/M output. Spot check arithmetic."""
- cost = compute_cost_usd(
- model="claude-sonnet-4-6", input_tokens=1_000_000, output_tokens=1_000_000
- )
- assert cost == pytest.approx(3.0 + 15.0)
-
-
-def test_cache_tokens_priced_separately():
- """Cache reads + cache writes use their own per-Mtok rates."""
- # Sonnet 4.6 cache_read=$0.30/M, cache_write=$3.75/M
- cost = compute_cost_usd(
- model="claude-sonnet-4-6",
- input_tokens=0,
- output_tokens=0,
- cache_read_tokens=1_000_000,
- cache_creation_tokens=1_000_000,
- )
- assert cost == pytest.approx(0.30 + 3.75)
-
-
- "path,expected_discount",
- [
- ("sync_baseline", 0.0),
- ("anthropic_batch", 0.5),
- ],
-)
-def test_execution_path_discount(path, expected_discount):
- assert execution_path_discount(path) == expected_discount
-
-
-def test_batch_discount_applied_to_final_cost():
- """Batch path multiplies the rate-card cost by (1 - 0.5) = 0.5."""
- sync = compute_cost_usd(
- model="claude-sonnet-4-6",
- input_tokens=1_000_000,
- output_tokens=1_000_000,
- execution_path="sync_baseline",
- )
- batch = compute_cost_usd(
- model="claude-sonnet-4-6",
- input_tokens=1_000_000,
- output_tokens=1_000_000,
- execution_path="anthropic_batch",
- )
- assert sync == pytest.approx(18.0)
- assert batch == pytest.approx(9.0)
- assert batch == pytest.approx(sync * 0.5)
-
-
-def test_model_with_no_cache_pricing_ignores_cache_tokens():
- """gpt-5 has no cache_*_per_mtok — cache tokens contribute zero."""
- cost = compute_cost_usd(
- model="gpt-5",
- input_tokens=0,
- output_tokens=0,
- cache_read_tokens=1_000_000,
- cache_creation_tokens=1_000_000,
- )
- assert cost == pytest.approx(0.0)
-
-
-def test_modelrate_is_frozen():
- """ModelRate is frozen so a typo in calling code can't mutate the
- shared rate card by accident."""
- rate = ModelRate(input_per_mtok=1.0, output_per_mtok=2.0)
- with pytest.raises(Exception):
- rate.input_per_mtok = 99.0 # type: ignore[misc]
-
-
-def test_model_lookup_is_case_insensitive():
- """We don't want a registry miss because a caller passes the model
- name with the wrong case."""
- a = compute_cost_usd(model="claude-sonnet-4-6", input_tokens=100, output_tokens=100)
- b = compute_cost_usd(model="CLAUDE-SONNET-4-6", input_tokens=100, output_tokens=100)
- assert a is not None and b is not None
- assert a == pytest.approx(b)
diff --git a/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py b/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py
index 0f1ab8d3f0..fa80c231dd 100644
--- a/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py
+++ b/autogpt_platform/backend/backend/copilot/dream/nightly_batch.py
@@ -11,17 +11,20 @@ submitter:
* Future P2 dedup / P3 self-model refresh / P4 scenario pre-warm /
P11 threat rehearsal land as additional submitters here
-Each submitter is fast to ENQUEUE (seconds) but may take up to ~1h to
-COMPLETE when running against the Anthropic batch API. The submitters fire in sequence at cron time; their
-batch results land asynchronously via the separate
-``copilot_batch_executor`` poller service that dispatches by
-``custom_id`` to per-stage apply handlers.
-
-Until the real batch providers land (currently scaffolded with
-``NotImplementedError`` stubs in ``dream/batch/``), the dream pass
-runs end-to-end via ``execute_dream_pass`` (sync_baseline path —
-30s LLM thinking + seconds-long apply). The function shape is the
-same either way.
+Each submitter is fast to ENQUEUE (seconds). A dream pass routed to
+Anthropic's Message Batches API (``anthropic_batch``: the
+``dream-pass-batch-enabled`` flag plus a direct Anthropic key) submits
+its first phase and returns; the separate ``copilot_batch_executor``
+poller service collects each finished batch and dispatches it by
+``custom_id`` to ``batch_callbacks``, which submits the next phase and,
+after the last one, applies. Every phase batch gets its own 24-hour
+window (Anthropic's batch SLA, ``MAX_BATCH_LIFETIME_SECONDS``). There
+is no OpenAI batch path: ``call_provider`` refuses
+``execution_mode="batch"`` for OpenAI.
+
+Otherwise the dream pass runs end-to-end via ``execute_dream_pass`` on
+the sync_baseline path (30s LLM thinking + seconds-long apply). The
+function shape is the same either way.
Concurrency model (per ``dream/p0-spec.md`` §5 and the architecture
plan): LLM-thinking phases stay concurrent with user writes (they're
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
index 65d4463b11..d39092216f 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
@@ -4,10 +4,12 @@ Walks a user's recent memory window through the consolidate → recombine
→ sanitize pipeline, then applies the sanitizer's ``DreamOperations``
to Graphiti + Postgres.
-Slice 1 of P-0 deliberately bypasses Anthropic batch — every phase
-calls the OpenRouter-fronted OpenAI-compat client. The batch path
-slots in below this layer (`routing.py` returns ``"batch"``) in a
-future PR.
+Each phase here is one ``structured_completion`` call on the chat
+transport's provider. When ``routing.resolve_dream_execution_path``
+picks ``anthropic_batch``, ``_submit_dream_pass_batch`` submits the
+first phase to Anthropic's Message Batches API instead and
+``batch_callbacks`` runs the later phases and the apply step as the
+results land.
The orchestrator never raises out — every failure becomes a
``DreamPassResult`` with ``error`` set, so the admin trigger always
@@ -30,7 +32,7 @@ from backend.copilot.trial_cost_context import attributed_usage
from backend.util.feature_flag import Flag, is_feature_enabled
from .apply import apply_operations, drain_status_from_stats
-from .billing import check_dream_budget, record_phase_cost
+from .billing import check_dream_budget, priced_phase_usage, record_phase_cost
from .fetch import (
DreamInput,
EpisodeRow,
@@ -51,7 +53,6 @@ from .locks import (
DreamLockHeld,
dream_lock,
)
-from .model_pricing import compute_cost_usd, execution_path_discount
from .prompts import (
MAX_DEMOTIONS_PER_PASS,
MAX_PROPOSALS_PER_PASS,
@@ -60,7 +61,7 @@ from .prompts import (
build_recombine_prompt,
build_sanitize_prompt,
)
-from .routing import ExecutionPath, resolve_dream_execution_path
+from .routing import ExecutionPath, batch_discount, resolve_dream_execution_path
from .schemas import (
ConsolidatedFact,
ConsolidationOutput,
@@ -83,7 +84,13 @@ logger = logging.getLogger(__name__)
CONSOLIDATE_TEMP = 0.2
RECOMBINE_TEMP = 0.9
SANITIZE_TEMP = 0.0
-CONSOLIDATE_MAX_TOKENS = 4096
+# The budget covers thinking as well as the JSON, and the standard slot can
+# run a model that thinks unasked. Sonnet 5 through OpenRouter spent all of
+# 4096 on reasoning over a 31-fact, 31-episode window (a 22.5K-token prompt)
+# and answered nothing, failing every pass of that account at this phase;
+# given 16384, the same call answered in 5,912 tokens, 5,068 of them
+# reasoning. Budget a model does not spend costs nothing.
+CONSOLIDATE_MAX_TOKENS = 16384
# Recombine + sanitize emit list-heavy JSON (up to 30 writes + 20
# proposals + 10 demotions, each with uuid arrays). At 8192 the
# response truncates mid-array and the JSON-balanced-brace extractor
@@ -125,7 +132,8 @@ SANITIZE_MAX_TOKENS = 16384
# Recombine (fast_advanced_model, Opus-class — the slowest decoder) gets
# the largest share: 600s covers 16384 tokens at ~27 tok/s. Sanitize runs
# on the faster fast_standard_model: 480s covers 16384 at ~34 tok/s.
-# Consolidate's 4096-token budget fits 240s at ~17 tok/s.
+# Consolidate's 240s covers its whole 16384 at ~68 tok/s, and a thinking
+# answer the size measured above (5,912 tokens) at ~25 tok/s.
CONSOLIDATE_TIMEOUT_SECONDS = 240
RECOMBINE_TIMEOUT_SECONDS = 600
SANITIZE_TIMEOUT_SECONDS = 480
@@ -582,34 +590,26 @@ async def _run_sanitize(
def _phase_usage_from_completion(
phase: str, completion_usage: CompletionUsage, execution_path: ExecutionPath
) -> PhaseUsage:
- """Build a ``PhaseUsage`` from a raw ``CompletionUsage``, computing
- cost from the rate card when the provider didn't supply one.
-
- Provider-supplied cost (OpenRouter ``usage.cost``) wins when
- present; otherwise we fall back to ``model_pricing.compute_cost_usd``.
- Either way the execution-path discount has been applied: OpenRouter
- spot prices are already post-discount (they ARE the rate we paid),
- and the rate-card fallback explicitly multiplies the discount in.
+ """Build a ``PhaseUsage`` from a raw ``CompletionUsage``, pricing it
+ from the catalog price card when the provider didn't.
+
+ Provider-supplied cost (OpenRouter ``usage.cost``) wins when present:
+ it is what we were billed. Otherwise ``billing.priced_phase_usage``
+ prices the tokens at the model's catalog list rate, less the
+ execution path's batch discount.
"""
- cost = completion_usage.cost_usd
- if cost is None:
- cost = compute_cost_usd(
- model=completion_usage.model,
- input_tokens=completion_usage.input_tokens,
- output_tokens=completion_usage.output_tokens,
- cache_read_tokens=completion_usage.cache_read_tokens,
- cache_creation_tokens=completion_usage.cache_creation_tokens,
- execution_path=execution_path,
- )
- return PhaseUsage(
+ usage = PhaseUsage(
phase=phase, # type: ignore[arg-type]
model=completion_usage.model,
input_tokens=completion_usage.input_tokens,
output_tokens=completion_usage.output_tokens,
cache_read_tokens=completion_usage.cache_read_tokens,
cache_creation_tokens=completion_usage.cache_creation_tokens,
- cost_usd=cost,
+ cost_usd=completion_usage.cost_usd,
)
+ if usage.cost_usd is not None:
+ return usage
+ return priced_phase_usage(usage, execution_path)
def _aggregate_usage(
@@ -633,7 +633,7 @@ def _aggregate_usage(
total_cache_read_tokens=sum(p.cache_read_tokens for p in phases),
total_cache_creation_tokens=sum(p.cache_creation_tokens for p in phases),
total_cost_usd=total_cost,
- discount_applied=execution_path_discount(execution_path),
+ discount_applied=batch_discount(execution_path),
)
@@ -747,7 +747,7 @@ async def _execute_dream_pass_async(
# batch path ship dark and roll out per-cohort. When on, phase 1 submits
# via call_provider(execution_mode="batch"); the BatchExecutor polls and
# dream's batch_callbacks chain phases 2 → 3 + apply when results land
- # (~50% off the rate card, see model_pricing.execution_path_discount).
+ # (half the list price, see routing.batch_discount).
#
# ``transport_name`` short-circuits to sync_baseline for transports that
# can't honour a batch path (local backends have no batch API;
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
index 2c0eb4b8cb..a033014d58 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
@@ -18,6 +18,7 @@ from backend.copilot import usage_activation_fixtures
from backend.copilot.graphiti.scope import MemoryScope
from backend.executor.scheduler import SCHEDULER_DREAM_OPERATION_TIMEOUT_SECONDS
+from . import batch_submit
from . import orchestrator as orchestrator_mod
from .apply import INGESTION_DRAIN_TIMEOUT_SECONDS, LOCK_DRAIN_RENEWAL_SECONDS
from .fetch import DreamInput, EpisodeRow, FactRow
@@ -470,6 +471,25 @@ def test_long_output_phase_timeouts_clear_the_120s_truncation_threshold():
assert orchestrator_mod.CONSOLIDATE_TIMEOUT_SECONDS >= _TRUNCATING_TIMEOUT_SECONDS
+# A thinking standard model's consolidate answer, measured on a test bed:
+# Sonnet 5 through OpenRouter, on a 31-fact, 31-episode window, answered in
+# 5,912 output tokens, 5,068 of them reasoning. Given 4096 it spent them all
+# on reasoning, answered nothing, and the pass failed with empty_content.
+_THINKING_CONSOLIDATE_ANSWER_TOKENS = 5_912
+
+
+def test_consolidate_budget_leaves_a_thinking_model_room_to_answer():
+ """Regression pin: the consolidate budget covers a thinking model's
+ reasoning as well as its answer, and the batch path asks for the same
+ budgets as the sync one."""
+ assert orchestrator_mod.CONSOLIDATE_MAX_TOKENS > _THINKING_CONSOLIDATE_ANSWER_TOKENS
+ assert batch_submit.PHASE_MAX_TOKENS == {
+ "consolidate": orchestrator_mod.CONSOLIDATE_MAX_TOKENS,
+ "recombine": orchestrator_mod.RECOMBINE_MAX_TOKENS,
+ "sanitize": orchestrator_mod.SANITIZE_MAX_TOKENS,
+ }
+
+
def test_phase_timeouts_plus_headroom_fit_scheduler_and_lock_envelope():
"""Budget-math invariant with the drain budgeted as its OWN line item.
@@ -956,19 +976,39 @@ async def test_each_completed_phase_charges_once(mocker):
@pytest.mark.asyncio
-async def test_partial_failure_still_charges_completed_phases(mocker):
- """If recombine errors after consolidate succeeded, we must still
- bill for the consolidate tokens — we already paid the provider."""
[email protected]("failed_phase", ["consolidate", "recombine", "sanitize"])
+ "error", ["max_tokens", "invalid output shape", "provider error"]
+)
+async def test_partial_failure_still_charges_completed_phases(
+ mocker, failed_phase, error
+):
mocker.patch.object(
orchestrator_mod,
"gather_dream_input",
AsyncMock(return_value=_build_input()),
)
- consolidated = ConsolidationOutput(facts=[])
+ phases = ["consolidate", "recombine", "sanitize"]
+ completed = phases.index(failed_phase)
+ failure_usage = CompletionUsage(
+ model="claude-sonnet-5-5",
+ input_tokens=1001,
+ output_tokens=203,
+ cache_read_tokens=3007,
+ cache_creation_tokens=509,
+ cost_usd=0.0056052,
+ )
+ failure = DreamLLMError(error, usage=failure_usage)
mocker.patch.object(
orchestrator_mod,
"structured_completion",
- AsyncMock(side_effect=[_wrap(consolidated), DreamLLMError("recombine boom")]),
+ AsyncMock(
+ side_effect=[
+ _wrap(ConsolidationOutput(facts=[])),
+ _wrap(RecombinationOutput(proposals=[])),
+ ][:completed]
+ + [failure]
+ ),
)
apply_mock = mocker.patch.object(orchestrator_mod, "apply_operations", AsyncMock())
charge_spy = AsyncMock()
@@ -977,9 +1017,13 @@ async def test_partial_failure_still_charges_completed_phases(mocker):
result = await orchestrator_mod.execute_dream_pass("u")
assert result.error is not None
- assert result.error.startswith("recombine:")
- assert charge_spy.await_count == 1 # consolidate charged, recombine never ran
- assert charge_spy.await_args.kwargs["phase_usage"].phase == "consolidate"
+ assert result.error.startswith(f"{failed_phase}:")
+ assert error in result.error
+ assert charge_spy.await_count == completed
+ assert [c.kwargs["phase_usage"].phase for c in charge_spy.await_args_list] == (
+ phases[:completed]
+ )
+ assert failure.usage == failure_usage
apply_mock.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/copilot/dream/routing.py b/autogpt_platform/backend/backend/copilot/dream/routing.py
index 5adce76e4e..4a965f6feb 100644
--- a/autogpt_platform/backend/backend/copilot/dream/routing.py
+++ b/autogpt_platform/backend/backend/copilot/dream/routing.py
@@ -14,10 +14,13 @@ from typing import Literal
from backend.copilot.config import TransportName
-# The single source of truth for execution paths; ``model_pricing``,
-# ``billing`` and ``schemas`` import it from here.
+# The single source of truth for execution paths; ``billing``,
+# ``orchestrator`` and ``schemas`` import it from here.
ExecutionPath = Literal["sync_baseline", "anthropic_batch"]
+# Anthropic's Message Batches API bills half the synchronous list price.
+_ANTHROPIC_BATCH_DISCOUNT = 0.5
+
# Transports that cannot honour the batch path even when an Anthropic key
# is configured separately.
@@ -61,3 +64,13 @@ def resolve_dream_execution_path(
if batch_processing_enabled and has_anthropic_key:
return "anthropic_batch"
return "sync_baseline"
+
+
+def batch_discount(path: ExecutionPath) -> float:
+ """The share of the list price the provider takes off on *path*: half
+ on Anthropic's Message Batches API, nothing on the sync path.
+
+ Priced into the dream's cost rows and recorded next to them as
+ ``discount_applied``.
+ """
+ return _ANTHROPIC_BATCH_DISCOUNT if path == "anthropic_batch" else 0.0
diff --git a/autogpt_platform/backend/backend/copilot/dream/routing_test.py b/autogpt_platform/backend/backend/copilot/dream/routing_test.py
index c810c06122..33858313a5 100644
--- a/autogpt_platform/backend/backend/copilot/dream/routing_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/routing_test.py
@@ -4,7 +4,7 @@ from __future__ import annotations
import pytest
-from .routing import resolve_dream_execution_path
+from .routing import batch_discount, resolve_dream_execution_path
@pytest.mark.parametrize(
@@ -47,6 +47,13 @@ def test_local_and_subscription_transports_force_sync_baseline(transport_name):
)
+ "path,expected", [("sync_baseline", 0.0), ("anthropic_batch", 0.5)]
+)
+def test_batch_discount_is_half_on_the_anthropic_batch_path_only(path, expected):
+ assert batch_discount(path) == expected
+
+
@pytest.mark.parametrize("transport_name", ["openrouter", "direct_anthropic", None])
def test_batch_eligible_transports_unaffected(transport_name):
"""openrouter / direct_anthropic / ``None`` (no override) preserve
diff --git a/autogpt_platform/backend/backend/copilot/dream/schemas.py b/autogpt_platform/backend/backend/copilot/dream/schemas.py
index 30f2421ace..065c403e15 100644
--- a/autogpt_platform/backend/backend/copilot/dream/schemas.py
+++ b/autogpt_platform/backend/backend/copilot/dream/schemas.py
@@ -282,8 +282,9 @@ class PhaseUsage(BaseModel):
cache_creation_tokens: int = 0
cost_usd: float | None = None
"""``None`` when the provider didn't return a cost and the model
- isn't in ``model_pricing.py`` — caller treats as unknown rather
- than zero. ``0.0`` is legitimate (zero tokens, edge case)."""
+ has no catalog price (``backend/copilot/price_card.py``) — caller
+ treats as unknown rather than zero. ``0.0`` is legitimate (zero
+ tokens, edge case)."""
class DreamPassUsage(BaseModel):
diff --git a/autogpt_platform/backend/backend/copilot/dream/structured_output.py b/autogpt_platform/backend/backend/copilot/dream/structured_output.py
new file mode 100644
index 0000000000..237e8d3712
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/dream/structured_output.py
@@ -0,0 +1,127 @@
+"""How ``structured_completion`` asks each provider for structured output.
+
+JSON mode (``force_json_output``) constrains OpenAI, OpenRouter, Groq and
+Ollama, but the native Anthropic Messages call ignores it. There the output
+comes from one tool whose input schema is the response model, read back from
+the tool call, as on the dream batch path. The tool is forced where the model
+accepts that. Models that answer a forced ``tool_choice`` with a 400 (Opus
+5.5 among them) get ``auto`` instead, a prompt line asking for the call
+(``with_output_tool_instruction``), and their message text parsed when they
+answer in prose after all.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from backend.copilot.model_normalize import normalize_model_for_anthropic
+from backend.util.llm.providers import ProviderLiteral, ProviderResponse
+from backend.util.llm.tool_use import (
+ auto_tool_choice,
+ is_forced_tool_choice,
+ pydantic_to_anthropic_tool,
+ structured_tool_choice,
+)
+
+# Anthropic tool names take letters, digits, "_" and "-", 64 at most.
+_TOOL_NAME_UNSAFE_RE = re.compile(r"[^a-zA-Z0-9_-]")
+_TOOL_NAME_MAX_LENGTH = 64
+
+# Appended to a batch phase tool's description when the tool is not forced
+# (``tool_choice`` ``auto``): with the prompt line from
+# ``with_output_tool_instruction`` it is what gets the tool called. Forced
+# tools keep their description as written, and the sync path's description
+# asks for one complete call in either mode.
+OUTPUT_TOOL_CALL_ONCE = (
+ "Call this tool exactly once, with the complete result as its input."
+)
+
+
+class StructuredRequest(BaseModel):
+ """What ``call_provider`` is asked for beyond the prompt."""
+
+ model_config = ConfigDict(frozen=True)
+
+ model: str
+ force_json_output: bool = False
+ tools: list[dict[str, Any]] | None = None
+ tool_choice: dict[str, Any] | None = None
+
+ @property
+ def forces_output_tool(self) -> bool:
+ return is_forced_tool_choice(self.tool_choice)
+
+ def with_output_tool_optional(self) -> StructuredRequest:
+ """This request with the tool left to the model (``auto``): the
+ retry for a model that turned the forced tool down."""
+ return self.model_copy(update={"tool_choice": auto_tool_choice()})
+
+ def prompt(self, messages: list[dict[str, str]]) -> list[dict[str, str]]:
+ """*messages* as sent: when the output tool is offered but not
+ forced, with the line asking the model to call it."""
+ if not self.tools or self.forces_output_tool:
+ return messages
+ return with_output_tool_instruction(messages, self.tools[0]["name"])
+
+
+def structured_request(
+ provider: ProviderLiteral, model: str, response_model: type[BaseModel]
+) -> StructuredRequest:
+ """JSON mode, except on the native Anthropic API: there the model takes
+ its native spelling and the call carries one tool built from
+ *response_model*, forced where the model accepts a forced tool
+ (``structured_tool_choice``).
+
+ Raises ``ValueError`` for a non-Anthropic model on the ``anthropic``
+ provider.
+ """
+ if provider != "anthropic":
+ return StructuredRequest(model=model, force_json_output=True)
+ native_model = normalize_model_for_anthropic(model)
+ tool_name = output_tool_name(response_model)
+ tool = pydantic_to_anthropic_tool(
+ response_model,
+ tool_name=tool_name,
+ description=(
+ f"Return the {response_model.__name__} result: call this once, "
+ "with every field the schema requires."
+ ),
+ )
+ return StructuredRequest(
+ model=native_model,
+ tools=[tool],
+ tool_choice=structured_tool_choice(native_model, tool_name),
+ )
+
+
+def structured_payload(response: ProviderResponse) -> str:
+ """The text to parse: the output tool's arguments when the model called
+ it, the message text otherwise."""
+ if response.tool_calls:
+ return response.tool_calls[0].function.arguments
+ return (response.content or "").strip()
+
+
+def output_tool_name(response_model: type[BaseModel]) -> str:
+ safe_name = _TOOL_NAME_UNSAFE_RE.sub("_", response_model.__name__)
+ return f"emit_{safe_name}"[:_TOOL_NAME_MAX_LENGTH]
+
+
+def with_output_tool_instruction(
+ messages: list[dict[str, str]], tool_name: str
+) -> list[dict[str, str]]:
+ """*messages* with one line asking for the answer through *tool_name*,
+ for a model the tool can't be forced on. The line closes the last user
+ turn, after any "reply with only the JSON" wording, which it redirects
+ into the tool call."""
+ instruction = (
+ f"Give your answer by calling the {tool_name} tool exactly once, with "
+ "the complete result as its input, rather than writing it out as text."
+ )
+ if messages and messages[-1]["role"] == "user":
+ *earlier, last = messages
+ return [*earlier, {**last, "content": f"{last['content']}\n\n{instruction}"}]
+ return [*messages, {"role": "user", "content": instruction}]
diff --git a/autogpt_platform/backend/backend/copilot/learning/reviewer.py b/autogpt_platform/backend/backend/copilot/learning/reviewer.py
index f1e958ad40..e62794c8ac 100644
--- a/autogpt_platform/backend/backend/copilot/learning/reviewer.py
+++ b/autogpt_platform/backend/backend/copilot/learning/reviewer.py
@@ -17,7 +17,7 @@ from backend.copilot.dream.llm import (
StructuredCompletion,
structured_completion,
)
-from backend.copilot.dream.model_pricing import compute_cost_usd
+from backend.copilot.price_card import compute_cost_usd, price_for
from backend.copilot.token_tracking import persist_and_record_usage
from backend.copilot.tools.skills import ParsedSkill
from backend.copilot.transport_routing import routing_kwargs_for_chat_transport
@@ -65,9 +65,10 @@ async def record_review_cost(
were already paid for; the ledger row still records the token counts.
"""
cost = usage.cost_usd
- if cost is None:
+ price = price_for(usage.model) if cost is None else None
+ if price is not None:
cost = compute_cost_usd(
- model=usage.model,
+ price=price,
input_tokens=usage.input_tokens,
output_tokens=usage.output_tokens,
cache_read_tokens=usage.cache_read_tokens,
diff --git a/autogpt_platform/backend/backend/copilot/model_normalize.py b/autogpt_platform/backend/backend/copilot/model_normalize.py
index 03710e84ca..015fa10cb3 100644
--- a/autogpt_platform/backend/backend/copilot/model_normalize.py
+++ b/autogpt_platform/backend/backend/copilot/model_normalize.py
@@ -52,14 +52,31 @@ def normalize_model_for_transport(raw_model: str, cfg: ChatConfig | None = None)
# must never have them rewritten — both pass through unchanged.
if config.effective_transport in ("openrouter", "local"):
return raw_model
+ return normalize_model_for_anthropic(
+ raw_model, caller=f"{config.effective_transport!r} transport"
+ )
+
+
+def normalize_model_for_anthropic(
+ raw_model: str, *, caller: str = "The native Anthropic API"
+) -> str:
+ """The native Anthropic API's spelling of *raw_model*, whatever the
+ chat transport: the ``anthropic/`` prefix stripped and dots turned into
+ hyphens (``anthropic/claude-opus-4.6`` → ``claude-opus-4-6``).
+
+ The Anthropic branch of :func:`normalize_model_for_transport`, and the
+ whole rule for calls that always reach Anthropic directly — the dream
+ batch submit, which runs under the OpenRouter chat transport too, and
+ the dream sync path on the ``anthropic`` provider. *caller* names the
+ path in the error raised for a non-Anthropic model.
+ """
model = raw_model
if "/" in model:
vendor, model = model.split("/", 1)
if vendor != "anthropic":
raise ValueError(
- f"{config.effective_transport!r} transport requires an "
- f"Anthropic model, got vendor={vendor!r} from "
- f"model={raw_model!r}. Set CHAT_THINKING_STANDARD_MODEL/"
+ f"{caller} requires an Anthropic model, got vendor={vendor!r} "
+ f"from model={raw_model!r}. Set CHAT_THINKING_STANDARD_MODEL/"
f"CHAT_THINKING_ADVANCED_MODEL/CHAT_FAST_STANDARD_MODEL/"
f"CHAT_FAST_ADVANCED_MODEL to an anthropic/* slug, or "
f"enable OpenRouter."
@@ -72,8 +89,8 @@ def normalize_model_for_transport(raw_model: str, cfg: ChatConfig | None = None)
# ``gemini-pro``, ...) would fail with an opaque
# ``model_not_found`` at request time — surface it here.
raise ValueError(
- f"{config.effective_transport!r} transport requires an "
- f"Anthropic model slug, got model={raw_model!r}. Use an "
- f"``anthropic/*`` or ``claude-*`` slug, or enable OpenRouter."
+ f"{caller} requires an Anthropic model slug, got "
+ f"model={raw_model!r}. Use an ``anthropic/*`` or ``claude-*`` "
+ f"slug, or enable OpenRouter."
)
return model.replace(".", "-")
diff --git a/autogpt_platform/backend/backend/copilot/model_normalize_test.py b/autogpt_platform/backend/backend/copilot/model_normalize_test.py
index ec229f4aca..01a9cdee77 100644
--- a/autogpt_platform/backend/backend/copilot/model_normalize_test.py
+++ b/autogpt_platform/backend/backend/copilot/model_normalize_test.py
@@ -1,21 +1,37 @@
"""Unit tests for the shared model-slug normalizer."""
+from typing import Any
+
import pytest
from .config import ChatConfig
-from .model_normalize import normalize_model_for_transport
+from .model_normalize import (
+ normalize_model_for_anthropic,
+ normalize_model_for_transport,
+)
-def _make_cfg(**kwargs) -> ChatConfig:
- defaults: dict = {
+def _make_cfg(**kwargs: Any) -> ChatConfig:
+ """A config built from these settings alone: no ``.env`` file
+ (``_env_file=None``), and the models and transport flags the
+ normalizer reads pinned, so neither a developer's ``.env`` nor a
+ ``CHAT_*`` variable in the environment can pick the transport or
+ model under test. Explicit arguments outrank the environment."""
+ settings: dict[str, Any] = {
+ "_env_file": None,
+ "fast_standard_model": "anthropic/claude-sonnet-4-6",
+ "fast_advanced_model": "anthropic/claude-opus-4-7",
"thinking_standard_model": "anthropic/claude-sonnet-4-6",
"thinking_advanced_model": "anthropic/claude-opus-4-7",
+ "use_local": False,
+ "use_openrouter": False,
+ "use_claude_code_subscription": False,
# Aux key satisfies ``_validate_aux_client_for_direct_main`` —
# these tests target normalize behavior, not the aux check.
"aux_api_key": "or-aux-key",
+ **kwargs,
}
- defaults.update(kwargs)
- return ChatConfig(**defaults)
+ return ChatConfig(**settings)
class TestNormalizeModelForTransport:
@@ -87,12 +103,22 @@ class TestNormalizeModelForTransport:
with pytest.raises(ValueError, match="Anthropic model slug"):
normalize_model_for_transport("gpt-4o-mini", cfg)
+ def test_direct_anthropic_error_names_the_transport(self):
+ cfg = _make_cfg(
+ use_openrouter=False,
+ api_key=None,
+ base_url=None,
+ use_claude_code_subscription=False,
+ )
+ with pytest.raises(ValueError, match="'direct_anthropic' transport requires"):
+ normalize_model_for_transport("openai/gpt-4o-mini", cfg)
+
def test_local_transport_passes_bare_slug_unchanged(self):
# Local backends (Ollama, vLLM, …) use operator-chosen slugs
# like ``llama3.1:8b-instruct-q4_K_M`` that don't fit the
# ``vendor/model`` convention and must not be rewritten — the
# Anthropic-prefix rule applies only to cloud transports.
- cfg = ChatConfig(
+ cfg = _make_cfg(
use_local=True,
api_key="ollama",
base_url="http://ollama:11434/v1",
@@ -102,3 +128,34 @@ class TestNormalizeModelForTransport:
normalize_model_for_transport("llama3.1:8b-instruct-q4_K_M", cfg)
== "llama3.1:8b-instruct-q4_K_M"
)
+
+
+class TestNormalizeModelForAnthropic:
+ """The native-Anthropic rule on its own: no config read, so it holds
+ under the OpenRouter chat transport too (the dream batch path)."""
+
+ @pytest.mark.parametrize(
+ "raw,expected",
+ [
+ ("anthropic/claude-opus-5.5", "claude-opus-5-5"),
+ ("anthropic/claude-sonnet-5", "claude-sonnet-5"),
+ ("claude-sonnet-4.6", "claude-sonnet-4-6"),
+ ("claude-haiku-4-5-20251001", "claude-haiku-4-5-20251001"),
+ ],
+ )
+ def test_strips_prefix_and_dots(self, raw, expected):
+ assert normalize_model_for_anthropic(raw) == expected
+
+ def test_is_idempotent(self):
+ once = normalize_model_for_anthropic("anthropic/claude-opus-5.5")
+ assert normalize_model_for_anthropic(once) == once
+
+ def test_rejects_non_anthropic_vendor(self):
+ with pytest.raises(
+ ValueError, match="The native Anthropic API requires an Anthropic model"
+ ):
+ normalize_model_for_anthropic("openai/gpt-4.1-mini")
+
+ def test_rejects_bare_non_claude_slug(self):
+ with pytest.raises(ValueError, match="Anthropic model slug"):
+ normalize_model_for_anthropic("gpt-4o-mini")
diff --git a/autogpt_platform/backend/backend/copilot/price_card.py b/autogpt_platform/backend/backend/copilot/price_card.py
new file mode 100644
index 0000000000..6c56df3a1e
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/price_card.py
@@ -0,0 +1,165 @@
+"""One price card for background LLM inference, read from the LLM catalog.
+
+The catalog (``backend/data/llm_registry/catalog.py``) authors what each
+provider charges per million tokens in its ``provider_*_usd_per_1m`` cost
+fields. :func:`price_for` finds a model's card from whichever spelling a
+transport uses, and :func:`compute_cost_usd` prices one call from its
+token counts.
+
+This reads the catalog file itself (``get_catalog()``), not the in-process
+registry: the registry is filled only where ``load_catalog()`` runs (the
+REST API and the copilot executor), while background inference also runs
+in the scheduler and the batch executor.
+"""
+
+from __future__ import annotations
+
+import functools
+import logging
+import re
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from backend.data.llm_registry.catalog import get_catalog
+from backend.data.llm_registry.catalog_model import CatalogModel, CatalogModelCost
+from backend.data.llm_registry.llm_models import transport_slug_candidates
+
+logger = logging.getLogger(__name__)
+
+_TOKENS_PER_MTOK = 1_000_000
+
+# Snapshot suffixes a transport spelling drops: Anthropic's -YYYYMMDD
+# (claude-haiku-4-5-20251001) and OpenAI's -YYYY-MM-DD
+# (gpt-4.1-mini-2025-04-14). The registry's date-stripped index knows only
+# the first, which is why ``model_router.catalog_lookup`` misses gpt-4.1-mini.
+_SNAPSHOT_SUFFIX_RE = re.compile(r"-(?:\d{8}|\d{4}-\d{2}-\d{2})$")
+
+# Slugs already reported at error level in this process (see _report_unpriced).
+_reported_unpriced: set[str] = set()
+
+
+class PriceCard(BaseModel):
+ """What a model costs, in USD per million tokens, bucket by bucket.
+
+ The buckets add up: ``input`` is the uncached input alone, and cache
+ reads and cache writes are priced on top of it (Anthropic's usage
+ convention, which the dream token counts follow).
+ """
+
+ model_config = ConfigDict(frozen=True)
+
+ input_usd_per_mtok: float
+ output_usd_per_mtok: float
+ cache_read_usd_per_mtok: float
+ cache_creation_usd_per_mtok: float
+ source: Literal["catalog"] = "catalog"
+
+
+def price_for(model: str) -> PriceCard | None:
+ """The catalog price card for *model*, in any transport spelling.
+
+ Takes the forms copilot config and the transports use, ignoring case:
+ ``anthropic/claude-sonnet-5``, ``claude-opus-5.5``,
+ ``anthropic/claude-haiku-4-5`` (a dated snapshot in the catalog) and
+ bare ``gpt-4.1-mini``.
+
+ Returns ``None``, logged at error level, when the model is not in the
+ catalog or its entry lacks any of the four USD prices; a missing bucket
+ price would otherwise have to be invented or read as free. Callers then
+ treat the cost as unknown, never as zero.
+ """
+ entry = _catalog_model(model)
+ if entry is None:
+ _report_unpriced(model, "is not in the LLM catalog")
+ return None
+ card = _card_from_cost(entry.cost)
+ if card is None:
+ _report_unpriced(
+ model,
+ f"resolves to catalog model {entry.slug!r}, which lacks a provider "
+ "USD price for input, output, cache read or cache write",
+ )
+ return card
+
+
+def compute_cost_usd(
+ *,
+ price: PriceCard,
+ input_tokens: int,
+ output_tokens: int,
+ cache_read_tokens: int = 0,
+ cache_creation_tokens: int = 0,
+ discount: float = 0.0,
+) -> float:
+ """USD cost of one call: each bucket at its own rate, then the discount.
+
+ *discount* is the share of the list price taken off, e.g. ``0.5`` on
+ Anthropic's Message Batches API.
+ """
+ if not 0.0 <= discount <= 1.0:
+ raise ValueError(f"discount must be within [0, 1], got {discount}")
+ list_cost = (
+ input_tokens * price.input_usd_per_mtok
+ + output_tokens * price.output_usd_per_mtok
+ + cache_read_tokens * price.cache_read_usd_per_mtok
+ + cache_creation_tokens * price.cache_creation_usd_per_mtok
+ ) / _TOKENS_PER_MTOK
+ return list_cost * (1.0 - discount)
+
+
+def _catalog_model(model: str) -> CatalogModel | None:
+ """Every exact spelling first, then the undated ones: the order
+ ``model_router.catalog_lookup`` tries them in."""
+ exact, undated = _catalog_index()
+ candidates = transport_slug_candidates(model.lower())
+ for index in (exact, undated):
+ for candidate in candidates:
+ entry = index.get(candidate)
+ if entry is not None:
+ return entry
+ return None
+
+
+def _catalog_index() -> tuple[dict[str, CatalogModel], dict[str, CatalogModel]]:
+ """Lower-cased slug to entry, and snapshot-free slug to entry. Built once:
+ the catalog is code, so it cannot change under a running process."""
+ models = get_catalog().models
+ exact = {m.slug.lower(): m for m in models}
+ undated = {
+ stripped: m
+ for m in models
+ if (stripped := _SNAPSHOT_SUFFIX_RE.sub("", m.slug.lower())) != m.slug.lower()
+ }
+ return exact, undated
+
+
+def _card_from_cost(cost: CatalogModelCost | None) -> PriceCard | None:
+ if (
+ cost is None
+ or cost.provider_input_usd_per_1m is None
+ or cost.provider_output_usd_per_1m is None
+ or cost.provider_cache_read_usd_per_1m is None
+ or cost.provider_cache_creation_usd_per_1m is None
+ ):
+ return None
+ return PriceCard(
+ input_usd_per_mtok=cost.provider_input_usd_per_1m,
+ output_usd_per_mtok=cost.provider_output_usd_per_1m,
+ cache_read_usd_per_mtok=cost.provider_cache_read_usd_per_1m,
+ cache_creation_usd_per_mtok=cost.provider_cache_creation_usd_per_1m,
+ )
+
+
+def _report_unpriced(model: str, reason: str) -> None:
+ """Error level the first time per slug in a process (Sentry's logging
+ integration turns error records into events), a warning after that: a
+ misconfigured model is priced on every background call, and one Sentry
+ event per call would be noise."""
+ message = f"[price_card] {model!r} {reason}; its cost is unknown"
+ if model in _reported_unpriced:
+ logger.warning(message)
+ return
+ _reported_unpriced.add(model)
+ logger.error(message)
diff --git a/autogpt_platform/backend/backend/copilot/price_card_test.py b/autogpt_platform/backend/backend/copilot/price_card_test.py
new file mode 100644
index 0000000000..66d76849fc
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/price_card_test.py
@@ -0,0 +1,238 @@
+"""Guards for the background-inference price card.
+
+The copilot model defaults (the standard, advanced, title and gate models)
+and Graphiti's extraction and reranker LLMs run when nothing overrides them. Each
+must resolve to a catalog price, or a call priced from the card logs its
+tokens and charges nothing.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+from backend.copilot import price_card
+from backend.copilot.config import ChatConfig
+from backend.copilot.dream.routing import batch_discount
+from backend.copilot.graphiti.config import GraphitiConfig
+from backend.copilot.price_card import PriceCard, compute_cost_usd, price_for
+from backend.data.llm_registry import registry
+from backend.data.llm_registry.catalog_model import CatalogModel, CatalogModelCost
+
+# ChatConfig ``*_model`` fields are picked up by these role words in their
+# names, so a new tier or gate model is guarded without editing this file.
+_PRICED_ROLES = ("standard", "advanced", "title", "gate")
+# Role-named fields that are not LLMs from the catalog.
+_NOT_CATALOG_LLMS = {"gate_jev_model": "Typesafe's Jev classifier"}
+
+
+def _copilot_default_models() -> dict[str, str]:
+ return {
+ name: field.default
+ for name, field in ChatConfig.model_fields.items()
+ if name.endswith("_model")
+ and set(name.split("_")) & set(_PRICED_ROLES)
+ and name not in _NOT_CATALOG_LLMS
+ }
+
+
+def _background_default_models() -> dict[str, str]:
+ return {
+ **_copilot_default_models(),
+ **{
+ f"graphiti.{name}": GraphitiConfig.model_fields[name].default
+ for name in ("llm_model", "reranker_model")
+ },
+ }
+
+
[email protected](autouse=True)
+def fresh_unpriced_reports(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Each test sees the first-report-per-process error, whatever ran before."""
+ monkeypatch.setattr(price_card, "_reported_unpriced", set())
+
+
+def test_every_role_has_a_default_model():
+ found = _copilot_default_models()
+ for role in _PRICED_ROLES:
+ assert any(role in name.split("_") for name in found), role
+ assert all(isinstance(slug, str) and slug for slug in found.values()), found
+
+
[email protected]("field,slug", sorted(_background_default_models().items()))
+def test_background_default_model_is_priced(field: str, slug: str):
+ assert price_for(slug) is not None, f"{field}={slug!r} has no catalog price"
+
+
+ "spelling,catalog_slug",
+ [
+ ("anthropic/claude-sonnet-5", "claude-sonnet-5"),
+ ("claude-opus-5.5", "claude-opus-5-5"),
+ ("anthropic/claude-opus-5.5", "claude-opus-5-5"),
+ ("anthropic/claude-haiku-4-5", "claude-haiku-4-5-20251001"),
+ ("gpt-4.1-mini", "gpt-4.1-mini-2025-04-14"),
+ ("openai/gpt-4.1-mini", "gpt-4.1-mini-2025-04-14"),
+ ("Anthropic/Claude-Sonnet-5", "claude-sonnet-5"),
+ ],
+)
+def test_transport_spellings_resolve_to_the_catalog_entry(spelling, catalog_slug):
+ card = price_for(spelling)
+ assert card is not None
+ assert card == price_for(catalog_slug)
+
+
+def test_resolution_does_not_need_the_in_process_registry(monkeypatch):
+ """The scheduler and the batch executor never run ``load_catalog()``,
+ so their registry stays empty; pricing must not notice."""
+ monkeypatch.setattr(registry, "_dynamic_models", {})
+ monkeypatch.setattr(registry, "_date_stripped_models", {})
+ monkeypatch.setattr(registry, "_loaded", False)
+ price_card._catalog_index.cache_clear()
+ assert not registry.has_models()
+
+ for slug in _background_default_models().values():
+ assert price_for(slug) is not None, slug
+
+
+def test_sonnet_5_card_carries_the_authored_list_prices():
+ """Anthropic's list price: the scheduled rise to $3/$15 was cancelled."""
+ assert price_for("anthropic/claude-sonnet-5") == PriceCard(
+ input_usd_per_mtok=2.0,
+ output_usd_per_mtok=10.0,
+ cache_read_usd_per_mtok=0.20,
+ cache_creation_usd_per_mtok=2.50,
+ )
+
+
+def test_sonnet_5_5_card_carries_anthropics_published_prices():
+ """The default standard model: $2/$10, cache writes (5m) $2.50, and cache
+ reads at 5% of input, $0.10, not the usual 10%."""
+ assert price_for("anthropic/claude-sonnet-5-5") == PriceCard(
+ input_usd_per_mtok=2.0,
+ output_usd_per_mtok=10.0,
+ cache_read_usd_per_mtok=0.10,
+ cache_creation_usd_per_mtok=2.50,
+ )
+
+
+ "path, expected_usd",
+ [("sync_baseline", 0.10), ("anthropic_batch", 0.05)],
+)
+def test_sonnet_5_5_cache_reads_cost_the_published_rate(path, expected_usd):
+ card = price_for("claude-sonnet-5-5")
+ assert card is not None
+ cost = compute_cost_usd(
+ price=card,
+ input_tokens=0,
+ output_tokens=0,
+ cache_read_tokens=1_000_000,
+ discount=batch_discount(path),
+ )
+ assert cost == pytest.approx(expected_usd)
+
+
+def test_gpt_5_is_priced_at_openai_list_prices():
+ """The deleted dream rate table priced ``gpt-5``; the catalog card
+ keeps it priced, at OpenAI's list rates."""
+ assert price_for("gpt-5") == PriceCard(
+ input_usd_per_mtok=1.25,
+ output_usd_per_mtok=10.0,
+ cache_read_usd_per_mtok=0.125,
+ cache_creation_usd_per_mtok=0.0,
+ )
+
+
+def test_gpt_6_luna_is_priced_at_openai_list_prices():
+ assert price_for("gpt-6-luna") == PriceCard(
+ input_usd_per_mtok=0.10,
+ output_usd_per_mtok=0.50,
+ cache_read_usd_per_mtok=0.01,
+ cache_creation_usd_per_mtok=0.125,
+ )
+
+
+def test_each_bucket_is_priced_at_its_own_rate():
+ card = price_for("claude-sonnet-5")
+ assert card is not None
+ one_million = 1_000_000
+ assert compute_cost_usd(
+ price=card, input_tokens=one_million, output_tokens=0
+ ) == pytest.approx(2.0)
+ assert compute_cost_usd(
+ price=card,
+ input_tokens=one_million,
+ output_tokens=one_million,
+ cache_read_tokens=one_million,
+ cache_creation_tokens=one_million,
+ ) == pytest.approx(2.0 + 10.0 + 0.20 + 2.50)
+
+
+def test_batch_discount_halves_the_list_cost():
+ card = price_for("claude-opus-5-5")
+ assert card is not None
+ tokens = dict(
+ input_tokens=120_000,
+ output_tokens=8_000,
+ cache_read_tokens=40_000,
+ cache_creation_tokens=10_000,
+ )
+ sync_cost = compute_cost_usd(
+ price=card, discount=batch_discount("sync_baseline"), **tokens
+ )
+ batch_cost = compute_cost_usd(
+ price=card, discount=batch_discount("anthropic_batch"), **tokens
+ )
+ # $4 in, $20 out, $0.20 cache read, $5 cache write per Mtok.
+ assert sync_cost == pytest.approx(0.48 + 0.16 + 0.008 + 0.05)
+ assert batch_cost == pytest.approx(sync_cost / 2)
+
+
[email protected]("discount", [-0.1, 1.5])
+def test_discount_outside_zero_to_one_is_refused(discount):
+ card = price_for("claude-sonnet-5")
+ assert card is not None
+ with pytest.raises(ValueError, match="discount"):
+ compute_cost_usd(price=card, input_tokens=1, output_tokens=1, discount=discount)
+
+
+def test_unknown_model_is_unpriced_and_reported_once_at_error(caplog):
+ with caplog.at_level(logging.WARNING, logger=price_card.__name__):
+ assert price_for("vendor/no-such-model") is None
+ assert price_for("vendor/no-such-model") is None
+
+ levels = [r.levelno for r in caplog.records if "no-such-model" in r.message]
+ assert levels == [logging.ERROR, logging.WARNING]
+
+
+def test_catalog_entry_without_every_usd_price_is_unpriced(monkeypatch, caplog):
+ """A missing cache price would have to be invented or read as free, so
+ the card needs all four buckets."""
+ partial = CatalogModel(
+ slug="claude-partial-1",
+ display_name="Partial",
+ provider="anthropic",
+ context_window=1000,
+ cost=CatalogModelCost(
+ provider_input_usd_per_1m=1.0, provider_output_usd_per_1m=5.0
+ ),
+ )
+ monkeypatch.setattr(
+ price_card, "_catalog_index", lambda: ({partial.slug: partial}, {})
+ )
+ with caplog.at_level(logging.ERROR, logger=price_card.__name__):
+ assert price_for("claude-partial-1") is None
+ assert "claude-partial-1" in caplog.text
+
+
+ "slug,read,write",
+ [("claude-haiku-5-5", 0.01, 0.125), ("claude-fable-5-1", 0.25, 12.5)],
+)
+def test_new_anthropic_cache_prices(slug, read, write):
+ card = price_for(slug)
+ assert card is not None
+ assert card.cache_read_usd_per_mtok == read
+ assert card.cache_creation_usd_per_mtok == write
diff --git a/autogpt_platform/backend/backend/copilot/token_tracking.py b/autogpt_platform/backend/backend/copilot/token_tracking.py
index 4eb331d8af..5b525d1b91 100644
--- a/autogpt_platform/backend/backend/copilot/token_tracking.py
+++ b/autogpt_platform/backend/backend/copilot/token_tracking.py
@@ -250,7 +250,7 @@ async def persist_and_record_usage(
# Include entries where cost_usd is set even if token count is 0
# (e.g. fully-cached Anthropic responses where only cache tokens
# accumulate a charge without incrementing total_tokens).
- if user_id and (total_tokens > 0 or cost_float is not None):
+ if user_id and (not no_tokens or cost_float is not None):
# A dream pass logs under its own id as graph_exec_id; only a chat's
# usage names the chat.
chat_session_id = chat_session_id_override or (
diff --git a/autogpt_platform/backend/backend/copilot/token_tracking_test.py b/autogpt_platform/backend/backend/copilot/token_tracking_test.py
index 3cc8ccd201..96b5cce7d6 100644
--- a/autogpt_platform/backend/backend/copilot/token_tracking_test.py
+++ b/autogpt_platform/backend/backend/copilot/token_tracking_test.py
@@ -413,8 +413,14 @@ class TestPlatformCostLogging:
entry = mock_log.call_args[0][0]
assert (entry.graph_exec_id, entry.chat_session_id) == expected
+ @pytest.mark.parametrize(
+ "prompt,completion,cache_read,cache_write",
+ [(100, 50, 0, 0), (0, 0, 100, 0), (0, 0, 0, 100), (0, 0, 100, 100)],
+ )
@pytest.mark.asyncio
- async def test_logs_cost_entry_without_cost_usd(self):
+ async def test_logs_cost_entry_without_cost_usd(
+ self, prompt, completion, cache_read, cache_write
+ ):
"""When cost_usd is None, tracking_type should be 'tokens'."""
mock_log = AsyncMock()
with (
@@ -432,8 +438,10 @@ class TestPlatformCostLogging:
await persist_and_record_usage(
session=None,
user_id="user-tokens",
- prompt_tokens=100,
- completion_tokens=50,
+ prompt_tokens=prompt,
+ completion_tokens=completion,
+ cache_read_tokens=cache_read,
+ cache_creation_tokens=cache_write,
log_prefix="[Baseline]",
)
await asyncio.sleep(0)
@@ -442,10 +450,13 @@ class TestPlatformCostLogging:
assert entry.cost_microdollars is None
assert entry.tracking_type == "tokens"
assert entry.metadata["tracking_type"] == "tokens"
- assert entry.metadata["tracking_amount"] == 150
+ assert entry.metadata["tracking_amount"] == prompt + completion
assert entry.graph_exec_id is None
assert entry.block_name == "copilot:Baseline"
+ assert entry.cache_read_tokens == (cache_read or None)
+ assert entry.cache_creation_tokens == (cache_write or None)
+
@pytest.mark.asyncio
async def test_skips_cost_log_when_no_user_id(self):
"""No PlatformCostLog entry when user_id is None."""
diff --git a/autogpt_platform/backend/backend/data/llm_registry/catalog.py b/autogpt_platform/backend/backend/data/llm_registry/catalog.py
index 46c66c810f..bf52985947 100644
--- a/autogpt_platform/backend/backend/data/llm_registry/catalog.py
+++ b/autogpt_platform/backend/backend/data/llm_registry/catalog.py
@@ -103,12 +103,18 @@ def _build_catalog() -> CatalogPayload:
context_window=200000,
max_output_tokens=64000,
price_tier=2,
+ # Anthropic list price $1/$5 per Mtok; cache read 0.1x input
+ # ($0.10), cache write (5m) 1.25x input ($1.25).
cost=CatalogModelCost(
run_credits=4,
input_credits_per_1m=150.0,
output_credits_per_1m=750.0,
cache_read_credits_per_1m=15.0,
cache_creation_credits_per_1m=188.0,
+ provider_input_usd_per_1m=1.00,
+ provider_output_usd_per_1m=5.00,
+ provider_cache_read_usd_per_1m=0.10,
+ provider_cache_creation_usd_per_1m=1.25,
),
),
CatalogModel(
@@ -138,6 +144,8 @@ def _build_catalog() -> CatalogPayload:
cache_creation_credits_per_1m=18.75,
provider_input_usd_per_1m=0.10,
provider_output_usd_per_1m=0.50,
+ provider_cache_read_usd_per_1m=0.01,
+ provider_cache_creation_usd_per_1m=0.125,
),
),
CatalogModel(
@@ -186,6 +194,10 @@ def _build_catalog() -> CatalogPayload:
output_credits_per_1m=3750.0,
cache_read_credits_per_1m=75.0,
cache_creation_credits_per_1m=938.0,
+ provider_input_usd_per_1m=5.00,
+ provider_output_usd_per_1m=25.00,
+ provider_cache_read_usd_per_1m=0.50,
+ provider_cache_creation_usd_per_1m=6.25,
),
),
CatalogModel(
@@ -208,6 +220,10 @@ def _build_catalog() -> CatalogPayload:
output_credits_per_1m=3750.0,
cache_read_credits_per_1m=75.0,
cache_creation_credits_per_1m=938.0,
+ provider_input_usd_per_1m=5.00,
+ provider_output_usd_per_1m=25.00,
+ provider_cache_read_usd_per_1m=0.50,
+ provider_cache_creation_usd_per_1m=6.25,
),
),
CatalogModel(
@@ -222,6 +238,8 @@ def _build_catalog() -> CatalogPayload:
supports_tools=True,
supports_json_output=True,
supports_reasoning=True,
+ # Anthropic list price $5/$25 per Mtok; cache read $0.50,
+ # cache write (5m) $6.25.
cost=CatalogModelCost(
run_credits=14,
input_credits_per_1m=750.0,
@@ -230,6 +248,8 @@ def _build_catalog() -> CatalogPayload:
cache_creation_credits_per_1m=938.0,
provider_input_usd_per_1m=5.00,
provider_output_usd_per_1m=25.00,
+ provider_cache_read_usd_per_1m=0.50,
+ provider_cache_creation_usd_per_1m=6.25,
),
),
CatalogModel(
@@ -255,6 +275,8 @@ def _build_catalog() -> CatalogPayload:
cache_creation_credits_per_1m=750.0,
provider_input_usd_per_1m=4.00,
provider_output_usd_per_1m=20.00,
+ provider_cache_read_usd_per_1m=0.20,
+ provider_cache_creation_usd_per_1m=5.00,
),
),
CatalogModel(
@@ -287,6 +309,10 @@ def _build_catalog() -> CatalogPayload:
output_credits_per_1m=2250.0,
cache_read_credits_per_1m=45.0,
cache_creation_credits_per_1m=563.0,
+ provider_input_usd_per_1m=3.00,
+ provider_output_usd_per_1m=15.00,
+ provider_cache_read_usd_per_1m=0.30,
+ provider_cache_creation_usd_per_1m=3.75,
),
),
CatalogModel(
@@ -304,18 +330,22 @@ def _build_catalog() -> CatalogPayload:
supports_tools=True,
supports_json_output=True,
supports_reasoning=True,
- # Sticker price ($3/$15 per Mtok; intro pricing $2/$10 ends
- # 2026-08-31 — pricing at sticker avoids a repricing PR).
- # Credits at the standard 1.5x margin; cache read $0.30,
- # cache write (5m) $3.75.
+ # Anthropic list price $2/$10 per Mtok; cache read $0.20,
+ # cache write (5m) $2.50. Anthropic cancelled the scheduled
+ # rise to $3/$15, so the provider USD fields carry today's
+ # price. The credits stay as set against that $3/$15
+ # sticker: they are what the platform bills, not what the
+ # provider charges.
cost=CatalogModelCost(
run_credits=9,
input_credits_per_1m=450.0,
output_credits_per_1m=2250.0,
cache_read_credits_per_1m=45.0,
cache_creation_credits_per_1m=563.0,
- provider_input_usd_per_1m=3.00,
- provider_output_usd_per_1m=15.00,
+ provider_input_usd_per_1m=2.00,
+ provider_output_usd_per_1m=10.00,
+ provider_cache_read_usd_per_1m=0.20,
+ provider_cache_creation_usd_per_1m=2.50,
),
),
CatalogModel(
@@ -332,20 +362,21 @@ def _build_catalog() -> CatalogPayload:
supports_json_output=True,
supports_reasoning=True,
# Sticker price $2/$10 per Mtok, undercutting Sonnet 5's
- # $3/$15 sticker. Cache read $0.20/1M, cache write (5m)
- # $2.50/1M, (1h) $4.00/1M. Verified 2026-09-28 against
- # Anthropic's sonnet-5-5 docs, the vendored rate card and
- # OpenRouter's live listing — no intro-pricing end date is
- # published, unlike Sonnet 5. Credits at the standard 1.5x
- # margin.
+ # $3/$15 sticker. Cache read $0.10/1M (5% of input on Sonnet
+ # 5.5, not the usual 10%), cache write (5m) $2.50/1M, (1h)
+ # $4.00/1M, per Anthropic's sonnet-5-5 overview (checked
+ # 2026-10-12); no intro-pricing end date is published,
+ # unlike Sonnet 5. Credits at the standard 1.5x margin.
cost=CatalogModelCost(
run_credits=9,
input_credits_per_1m=300.0,
output_credits_per_1m=1500.0,
- cache_read_credits_per_1m=30.0,
+ cache_read_credits_per_1m=15.0,
cache_creation_credits_per_1m=375.0,
provider_input_usd_per_1m=2.00,
provider_output_usd_per_1m=10.00,
+ provider_cache_read_usd_per_1m=0.10,
+ provider_cache_creation_usd_per_1m=2.50,
),
),
CatalogModel(
@@ -376,6 +407,8 @@ def _build_catalog() -> CatalogPayload:
cache_creation_credits_per_1m=1875.0,
provider_input_usd_per_1m=10.00,
provider_output_usd_per_1m=50.00,
+ provider_cache_read_usd_per_1m=0.25,
+ provider_cache_creation_usd_per_1m=12.50,
),
),
# ----- Groq -----
@@ -1582,10 +1615,16 @@ def _build_catalog() -> CatalogPayload:
context_window=1047576,
max_output_tokens=32768,
price_tier=1,
+ # OpenAI list price $0.40/$1.60 per Mtok, cached input $0.10;
+ # GPT-4.1 mini bills nothing for writing the cache.
cost=CatalogModelCost(
run_credits=1,
input_credits_per_1m=60.0,
output_credits_per_1m=240.0,
+ provider_input_usd_per_1m=0.40,
+ provider_output_usd_per_1m=1.60,
+ provider_cache_read_usd_per_1m=0.10,
+ provider_cache_creation_usd_per_1m=0.0,
),
),
CatalogModel(
@@ -1600,6 +1639,10 @@ def _build_catalog() -> CatalogPayload:
run_credits=3,
input_credits_per_1m=375.0,
output_credits_per_1m=1500.0,
+ provider_input_usd_per_1m=2.50,
+ provider_output_usd_per_1m=10.00,
+ provider_cache_read_usd_per_1m=1.25,
+ provider_cache_creation_usd_per_1m=0.00,
),
),
CatalogModel(
@@ -1622,10 +1665,16 @@ def _build_catalog() -> CatalogPayload:
context_window=400000,
max_output_tokens=128000,
price_tier=1,
+ # OpenAI list price $1.25/$10 per Mtok, cached input $0.125;
+ # GPT-5 bills nothing for writing the cache.
cost=CatalogModelCost(
run_credits=2,
input_credits_per_1m=188.0,
output_credits_per_1m=1500.0,
+ provider_input_usd_per_1m=1.25,
+ provider_output_usd_per_1m=10.00,
+ provider_cache_read_usd_per_1m=0.125,
+ provider_cache_creation_usd_per_1m=0.0,
),
),
CatalogModel(
@@ -1823,6 +1872,10 @@ def _build_catalog() -> CatalogPayload:
run_credits=1,
input_credits_per_1m=15.0,
output_credits_per_1m=75.0,
+ provider_input_usd_per_1m=0.10,
+ provider_output_usd_per_1m=0.50,
+ provider_cache_read_usd_per_1m=0.01,
+ provider_cache_creation_usd_per_1m=0.125,
),
),
CatalogModel(
diff --git a/autogpt_platform/backend/backend/data/llm_registry/catalog_model.py b/autogpt_platform/backend/backend/data/llm_registry/catalog_model.py
index c9737c49d4..9329a4e0e7 100644
--- a/autogpt_platform/backend/backend/data/llm_registry/catalog_model.py
+++ b/autogpt_platform/backend/backend/data/llm_registry/catalog_model.py
@@ -65,12 +65,19 @@ class CatalogModelCost(BaseModel):
output_credits_per_1m: float | None = Field(default=None, ge=0)
cache_read_credits_per_1m: float | None = Field(default=None, ge=0)
cache_creation_credits_per_1m: float | None = Field(default=None, ge=0)
- # What the PROVIDER charges us (USD list price per 1M tokens), for
- # in-turn cost estimation — authored per model when it diverges from
- # the transport module's family default (e.g. copilot/moonshot.py).
- # Not the billing source; the credits fields above are what users pay.
+ # What the PROVIDER charges us (USD list price per 1M tokens). Read for
+ # in-turn cost estimation where it diverges from a transport module's
+ # family default (e.g. copilot/moonshot.py), and as the price source for
+ # background inference's USD spend accounting (copilot/price_card.py,
+ # e.g. the dream pass's cost rows). Still not the billing source: the
+ # credits fields above are what users pay.
provider_input_usd_per_1m: float | None = Field(default=None, ge=0)
provider_output_usd_per_1m: float | None = Field(default=None, ge=0)
+ # The provider's prompt-cache list prices (USD per 1M tokens), billed
+ # on top of the uncached input, never instead of it. Read by the
+ # background-inference price card (``backend/copilot/price_card.py``).
+ provider_cache_read_usd_per_1m: float | None = Field(default=None, ge=0)
+ provider_cache_creation_usd_per_1m: float | None = Field(default=None, ge=0)
@model_validator(mode="after")
def _provider_usd_both_or_neither(self) -> "CatalogModelCost":
@@ -86,6 +93,24 @@ class CatalogModelCost(BaseModel):
)
return self
+ @model_validator(mode="after")
+ def _provider_cache_usd_needs_base_price(self) -> "CatalogModelCost":
+ # A cache price alone would price the cache buckets of a call while
+ # its plain input and output stayed unpriced.
+ has_cache_price = (
+ self.provider_cache_read_usd_per_1m is not None
+ or self.provider_cache_creation_usd_per_1m is not None
+ )
+ if has_cache_price and (
+ self.provider_input_usd_per_1m is None
+ or self.provider_output_usd_per_1m is None
+ ):
+ raise ValueError(
+ "provider cache USD prices require provider_input_usd_per_1m "
+ "and provider_output_usd_per_1m"
+ )
+ return self
+
class CatalogModel(BaseModel):
model_config = ConfigDict(frozen=True)
diff --git a/autogpt_platform/backend/backend/data/llm_registry/catalog_model_test.py b/autogpt_platform/backend/backend/data/llm_registry/catalog_model_test.py
index 0cf5899e50..679bb63f19 100644
--- a/autogpt_platform/backend/backend/data/llm_registry/catalog_model_test.py
+++ b/autogpt_platform/backend/backend/data/llm_registry/catalog_model_test.py
@@ -54,6 +54,10 @@ def test_invalid_model_fields_rejected(field, value):
CatalogModel(**_model_kwargs(**{field: value}))
+# A valid base USD pair, so a USD cache price is only refused for its sign.
+_USD_PAIR = {"provider_input_usd_per_1m": 1.0, "provider_output_usd_per_1m": 5.0}
+
+
@pytest.mark.parametrize(
"field",
[
@@ -62,11 +66,34 @@ def test_invalid_model_fields_rejected(field, value):
"output_credits_per_1m",
"cache_read_credits_per_1m",
"cache_creation_credits_per_1m",
+ "provider_input_usd_per_1m",
+ "provider_output_usd_per_1m",
+ "provider_cache_read_usd_per_1m",
+ "provider_cache_creation_usd_per_1m",
],
)
def test_negative_costs_rejected(field):
with pytest.raises(pydantic.ValidationError):
- CatalogModelCost(**{field: -1})
+ CatalogModelCost.model_validate({**_USD_PAIR, field: -1})
+
+
+ "field", ["provider_cache_read_usd_per_1m", "provider_cache_creation_usd_per_1m"]
+)
+def test_usd_cache_price_requires_the_base_usd_pair(field):
+ with pytest.raises(pydantic.ValidationError, match="require"):
+ CatalogModelCost.model_validate({field: 0.1})
+
+
+def test_usd_cache_prices_accepted_alongside_the_base_pair():
+ cost = CatalogModelCost(
+ provider_input_usd_per_1m=1.0,
+ provider_output_usd_per_1m=5.0,
+ provider_cache_read_usd_per_1m=0.1,
+ provider_cache_creation_usd_per_1m=1.25,
+ )
+ assert cost.provider_cache_read_usd_per_1m == 0.1
+ assert cost.provider_cache_creation_usd_per_1m == 1.25
def test_routing_shape_accepts_nested_cells():
diff --git a/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py b/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py
index aed9693c21..045671da98 100644
--- a/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py
+++ b/autogpt_platform/backend/backend/data/llm_registry/catalog_test.py
@@ -179,8 +179,10 @@ def test_deepseek_chat_display_rate_tracks_openrouter():
def test_claude_sonnet_5_bills_at_authored_rates():
- """Sonnet 5 (sticker $3/$15; intro pricing ends 2026-08-31) — flat tier
- and per-1M projections must match the authored catalog entry."""
+ """Sonnet 5 bills credits set against a $3/$15 sticker (Anthropic's
+ list price stayed $2/$10 when the rise was cancelled; credits are the
+ platform's rate) — flat tier and per-1M projections must match the
+ authored catalog entry."""
s5 = LLMModel("claude-sonnet-5")
assert MODEL_COST[s5] == 9
assert TOKEN_COST[s5].model_dump() == {
@@ -199,10 +201,11 @@ def test_claude_sonnet_5_5_bills_at_authored_rates():
model, so a silent repricing here moves every standard chat."""
s55 = LLMModel("claude-sonnet-5-5")
assert MODEL_COST[s55] == 9
+ # Cache reads at 1.5x Anthropic's $0.10/1M (5% of input on Sonnet 5.5).
assert TOKEN_COST[s55].model_dump() == {
"input": 300.0,
"output": 1500.0,
- "cache_read": 30.0,
+ "cache_read": 15.0,
"cache_creation": 375.0,
}
assert MODEL_METADATA[s55].max_output_tokens == 128000
diff --git a/autogpt_platform/backend/backend/util/llm/conversions.py b/autogpt_platform/backend/backend/util/llm/conversions.py
index 065d09d1d2..c1cf156539 100644
--- a/autogpt_platform/backend/backend/util/llm/conversions.py
+++ b/autogpt_platform/backend/backend/util/llm/conversions.py
@@ -52,6 +52,12 @@ def convert_openai_tool_fmt_to_anthropic(
) -> Iterable[ToolParam] | anthropic.NotGiven:
"""Convert OpenAI tool definitions into Anthropic ``ToolParam`` shape.
+ The tool's JSON Schema goes over whole (bar OpenAI's ``strict`` flag),
+ so its root ``description``, ``additionalProperties: false`` (a response
+ model with ``extra="forbid"``) and the ``$defs`` its ``$ref``s point
+ into all survive. ``type`` is always ``object`` (Anthropic requires an
+ object schema), and ``properties`` / ``required`` default to empty.
+
Returns ``anthropic.NOT_GIVEN`` when the caller passed no tools so
the SDK omits the field from the serialized request (Anthropic
rejects an empty tools array with HTTP 400).
@@ -61,20 +67,30 @@ def convert_openai_tool_fmt_to_anthropic(
anthropic_tools: list[ToolParam] = []
for tool in openai_tools:
- # Accept both forms: {"type":"function","function":{...}} (OpenAI
- # canonical) and {"name":..., "parameters":...} (raw function def).
+ # Accept three forms: {"type":"function","function":{...}} (OpenAI
+ # canonical), {"name":..., "parameters":...} (raw function def) and
+ # {"name":..., "input_schema":...} (already Anthropic-shaped, as
+ # ``tool_use.pydantic_to_anthropic_tool`` builds it).
if "function" in tool:
function_data = tool["function"]
else:
function_data = tool
+ schema: dict[str, Any] = dict(
+ function_data.get("input_schema") or function_data.get("parameters") or {}
+ )
+ # OpenAI's ``strict`` flag sometimes rides inside ``parameters`` (the
+ # orchestrator block puts it there). It is not JSON Schema, and
+ # Anthropic's own ``strict`` is a tool-level field, so it stays behind.
+ schema.pop("strict", None)
anthropic_tool: ToolParam = {
"name": function_data["name"],
"description": function_data.get("description", ""),
"input_schema": {
+ **schema,
"type": "object",
- "properties": function_data.get("parameters", {}).get("properties", {}),
- "required": function_data.get("parameters", {}).get("required", []),
+ "properties": schema.get("properties", {}),
+ "required": schema.get("required", []),
},
}
anthropic_tools.append(anthropic_tool)
diff --git a/autogpt_platform/backend/backend/util/llm/conversions_test.py b/autogpt_platform/backend/backend/util/llm/conversions_test.py
new file mode 100644
index 0000000000..dc7f78dacd
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/llm/conversions_test.py
@@ -0,0 +1,167 @@
+"""Tests for the OpenAI → Anthropic tool conversion ``call_provider`` runs."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import anthropic
+from pydantic import BaseModel, ConfigDict
+
+from backend.util.llm.conversions import convert_openai_tool_fmt_to_anthropic
+from backend.util.llm.tool_use import pydantic_to_anthropic_tool
+
+_LOOKUP_SCHEMA = {
+ "type": "object",
+ "properties": {"query": {"type": "string"}},
+ "required": ["query"],
+}
+_LOOKUP_TOOL = {
+ "name": "lookup",
+ "description": "Look something up.",
+ "input_schema": _LOOKUP_SCHEMA,
+}
+
+
+class _Fact(BaseModel):
+ content: str
+ confidence: float
+
+
+class _Output(BaseModel):
+ facts: list[_Fact]
+ summary: str | None = None
+
+
+class _StrictOutput(BaseModel):
+ """Facts, and nothing else."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ facts: list[_Fact]
+
+
+def _input_schema(tool: dict[str, Any]) -> dict[str, object]:
+ converted = convert_openai_tool_fmt_to_anthropic([tool])
+ assert isinstance(converted, list)
+ return dict(converted[0]["input_schema"])
+
+
+def test_no_tools_leaves_the_field_out():
+ assert convert_openai_tool_fmt_to_anthropic(None) is anthropic.NOT_GIVEN
+ assert convert_openai_tool_fmt_to_anthropic([]) is anthropic.NOT_GIVEN
+
+
+def test_openai_function_tool_keeps_its_parameters():
+ converted = convert_openai_tool_fmt_to_anthropic(
+ [
+ {
+ "type": "function",
+ "function": {
+ "name": "lookup",
+ "description": "Look something up.",
+ "parameters": _LOOKUP_SCHEMA,
+ },
+ }
+ ]
+ )
+ assert converted == [_LOOKUP_TOOL]
+
+
+def test_raw_function_def_keeps_its_parameters():
+ converted = convert_openai_tool_fmt_to_anthropic(
+ [
+ {
+ "name": "lookup",
+ "description": "Look something up.",
+ "parameters": _LOOKUP_SCHEMA,
+ }
+ ]
+ )
+ assert converted == [_LOOKUP_TOOL]
+
+
+def test_anthropic_shaped_tool_keeps_its_input_schema():
+ """``pydantic_to_anthropic_tool`` builds ``input_schema``; reading only
+ ``parameters`` sent such a tool with an empty schema (the dream batch
+ path's forced tool, for one)."""
+ tool = pydantic_to_anthropic_tool(
+ _Output, tool_name="emit_output", description="Emit the output."
+ )
+ schema = tool["input_schema"]
+
+ converted = convert_openai_tool_fmt_to_anthropic([tool])
+
+ assert converted == [
+ {
+ "name": "emit_output",
+ "description": "Emit the output.",
+ "input_schema": {
+ "type": "object",
+ "properties": schema["properties"],
+ "required": ["facts"],
+ },
+ }
+ ]
+ # The nested model survives inside the properties, inlined.
+ assert set(schema["properties"]["facts"]["items"]["properties"]) == {
+ "content",
+ "confidence",
+ }
+
+
+def test_the_whole_schema_reaches_anthropic():
+ """A response model with ``extra="forbid"`` keeps its root
+ ``additionalProperties: false``, and its docstring the root
+ ``description``: the schema is forwarded, not rebuilt."""
+ tool = pydantic_to_anthropic_tool(
+ _StrictOutput, tool_name="emit_strict", description="Emit it."
+ )
+
+ schema = _input_schema(tool)
+
+ assert schema["additionalProperties"] is False
+ assert schema["description"] == "Facts, and nothing else."
+ assert schema == tool["input_schema"]
+
+
+def test_openai_parameters_keep_their_defs():
+ """Parameters taken straight from ``model_json_schema()`` point their
+ ``$ref``s into ``$defs``, so the definitions travel with them."""
+ parameters = _Output.model_json_schema()
+ assert "$defs" in parameters
+
+ schema = _input_schema(
+ {
+ "type": "function",
+ "function": {"name": "emit_output", "parameters": parameters},
+ }
+ )
+
+ assert schema["$defs"] == parameters["$defs"]
+ assert schema["properties"] == parameters["properties"]
+
+
+def test_openai_strict_flag_is_not_sent_as_schema():
+ """``strict`` inside ``parameters`` is OpenAI's tool flag, not JSON
+ Schema; the rest of the schema still goes over."""
+ schema = _input_schema(
+ {
+ "name": "lookup",
+ "parameters": {
+ **_LOOKUP_SCHEMA,
+ "additionalProperties": False,
+ "strict": True,
+ },
+ }
+ )
+
+ assert "strict" not in schema
+ assert schema["additionalProperties"] is False
+
+
+def test_empty_parameters_become_an_empty_object_schema():
+ assert _input_schema({"name": "ping", "parameters": {}}) == {
+ "type": "object",
+ "properties": {},
+ "required": [],
+ }
diff --git a/autogpt_platform/backend/backend/util/llm/providers.py b/autogpt_platform/backend/backend/util/llm/providers.py
index ee84e5948c..2b74cdb494 100644
--- a/autogpt_platform/backend/backend/util/llm/providers.py
+++ b/autogpt_platform/backend/backend/util/llm/providers.py
@@ -48,6 +48,7 @@ from pydantic.dataclasses import dataclass
from backend.data.llm_registry.llm_models import (
CLAUDE_5_FAMILY_PREFIXES,
+ MODEL_DATE_SUFFIX_RE,
strip_anthropic_vendor_prefix,
)
from backend.util.clients import OPENROUTER_BASE_URL
@@ -161,6 +162,51 @@ def _is_temperature_deprecation_error(exc: anthropic.BadRequestError) -> bool:
)
+# Anthropic's newest models reject a forced ``tool_choice`` (``{"type":
+# "tool", ...}`` or ``{"type": "any"}``) with a 400 — ``tool_choice: type
+# "tool" and "any" are not supported for this model.`` — on the Messages API
+# and on the Message Batches API, where it only comes back hours later in the
+# result rows. ``tool_use.structured_tool_choice`` sends ``auto`` to these
+# models instead. The dream pass's sync call also retries once with ``auto``
+# on that exact error (``is_forced_tool_choice_rejection``), so an unlisted
+# model self-heals there; batch submissions rely on this list alone.
+#
+# Maintained apart from the temperature list above because the two
+# restrictions cover different models: every Claude 5 model and Opus 4.7/4.8
+# reject ``temperature``, but only these reject a forced tool — Sonnet 5 and
+# Opus 5 still honour one. Hence exact native slugs rather than family
+# prefixes (``claude-opus-5`` is a prefix of ``claude-opus-5-5``), compared
+# after the vendor prefix and any ``-YYYYMMDD`` snapshot suffix are stripped.
+# Anthropic documents the restriction for Opus/Sonnet 5.5 and the Fable 5.1 /
+# Mythos 5.1 line; add a model here when it documents the same for another.
+_ANTHROPIC_FORCED_TOOL_CHOICE_UNSUPPORTED: frozenset[str] = frozenset(
+ {"claude-opus-5-5", "claude-sonnet-5-5", "claude-fable-5-1", "claude-mythos-5-1"}
+)
+
+
+def anthropic_accepts_forced_tool_choice(model: str) -> bool:
+ """False for a model that answers a forced ``tool_choice`` with a 400,
+ in any spelling: ``anthropic/claude-opus-5.5``, ``claude-opus-5-5`` or a
+ dated snapshot of either."""
+ native = strip_anthropic_vendor_prefix(model).replace(".", "-")
+ return (
+ MODEL_DATE_SUFFIX_RE.sub("", native)
+ not in _ANTHROPIC_FORCED_TOOL_CHOICE_UNSUPPORTED
+ )
+
+
+def is_forced_tool_choice_rejection(exc: BaseException) -> bool:
+ """Whether *exc* is Anthropic's 400 for a forced ``tool_choice`` on a
+ model that takes none: an ``anthropic.BadRequestError`` reading
+ ``tool_choice: type "tool" and "any" are not supported for this
+ model.`` Nothing else counts, not even another error quoting that text:
+ a 5xx or a failure of our own is not a reason to drop the forced tool."""
+ if not isinstance(exc, anthropic.BadRequestError):
+ return False
+ error_text = str(exc).lower()
+ return "tool_choice" in error_text and "not supported" in error_text
+
+
# ---------------------------------------------------------------------------
# Result + submission carriers
# ---------------------------------------------------------------------------
@@ -183,6 +229,7 @@ class ProviderResponse:
tool_calls: list[ToolContentBlock] | None = None
reasoning: str | None = None
cost_usd: float | None = None
+ stop_reason: str | None = None
# Provider-native response object for callers that need raw access.
# Kept out of repr + pydantic serialization (it's an opaque SDK object).
raw_response: Any = Field(default=None, repr=False, exclude=True)
@@ -550,7 +597,9 @@ async def _call_anthropic_messages(
an_tools = convert_openai_tool_fmt_to_anthropic(tools)
# Cache tool definitions alongside the system prompt — placing
# cache_control on the last tool caches all tool schemas as a
- # single prefix; reads cost 10% of normal input tokens.
+ # single prefix. Cache reads bill at a fraction of the input rate:
+ # 10% on most Claude models, 5% on Opus 5.5, 2.5% on Fable 5.1 (the
+ # catalog's provider cache-read prices carry each model's figure).
if isinstance(an_tools, list) and an_tools:
an_tools[-1] = {**an_tools[-1], "cache_control": {"type": "ephemeral"}}
@@ -657,6 +706,7 @@ async def _call_anthropic_messages(
or 0,
tool_calls=tool_calls,
reasoning=reasoning,
+ stop_reason=resp.stop_reason,
raw_response=resp,
)
@@ -1299,6 +1349,21 @@ class BatchResultRow:
# Kept out of repr + pydantic serialization (it's an opaque SDK object).
raw_result: Any = Field(default=None, repr=False, exclude=True)
+ def with_content(self, content: str) -> BatchResultRow:
+ """This row with *content* in place of the provider's text, every
+ other field kept: the dream callback stores the JSON it parsed out
+ of a text answer."""
+ return BatchResultRow(
+ custom_id=self.custom_id,
+ content=content,
+ input_tokens=self.input_tokens,
+ output_tokens=self.output_tokens,
+ cache_read_tokens=self.cache_read_tokens,
+ cache_creation_tokens=self.cache_creation_tokens,
+ error=self.error,
+ raw_result=self.raw_result,
+ )
+
async def poll_batch(
*, provider: ProviderLiteral, provider_batch_id: str, api_key: str
@@ -1377,6 +1442,11 @@ async def download_batch_results(
output_tokens=output_tokens,
cache_read_tokens=cache_read,
cache_creation_tokens=cache_creation,
+ error=(
+ "max_tokens"
+ if message is not None and message.stop_reason == "max_tokens"
+ else None
+ ),
raw_result=entry,
)
)
@@ -1416,28 +1486,24 @@ def _anthropic_answer_blocks(
def _anthropic_content_to_text(message: anthropic.types.Message | None) -> str:
"""Flatten an Anthropic ``message.content`` blob into a string.
- Three shapes the BatchExecutor's downstream parser handles:
- * Single ``text`` block → return its ``text``.
- * Single ``tool_use`` block → return ``json.dumps(input)``. This
- is the "structured output" path (forced ``tool_choice`` makes
- Claude emit exactly one tool_use block; the dream pass parses
- the JSON straight into a Pydantic model).
- * Multi-block / unknown → join all ``text`` blocks with newlines.
+ Two shapes the BatchExecutor's downstream parser handles:
+ * A ``tool_use`` block → ``json.dumps(input)`` of the first one. This
+ is the "structured output" path: the dream pass offers one output
+ tool, forced where the model accepts that (exactly one tool_use
+ block) and left to the model's choice (``auto``) where it doesn't,
+ in which case a short text block can come before the call. The
+ dream pass parses the JSON straight into a Pydantic model.
+ * No ``tool_use`` block → the ``text`` blocks joined with newlines
+ (a single text block comes back as is).
Returns ``""`` when the message is missing or empty.
"""
content = _anthropic_answer_blocks(message)
- if not content:
- return ""
- first = content[0]
- first_type = first.type
- if first_type == "tool_use" and len(content) == 1:
- return json_module.dumps(getattr(first, "input", {}) or {})
- if first_type == "text" and len(content) == 1:
- return getattr(first, "text", "") or ""
- # Multi-block fallback — concatenate text blocks.
- parts: list[str] = []
- for block in content:
- if getattr(block, "type", None) == "text":
- parts.append(getattr(block, "text", "") or "")
- return "\n".join(parts)
+ tool_use = next((block for block in content if block.type == "tool_use"), None)
+ if tool_use is not None:
+ return json_module.dumps(getattr(tool_use, "input", {}) or {})
+ return "\n".join(
+ getattr(block, "text", "") or ""
+ for block in content
+ if getattr(block, "type", None) == "text"
+ )
diff --git a/autogpt_platform/backend/backend/util/llm/providers_test.py b/autogpt_platform/backend/backend/util/llm/providers_test.py
index 6fd2fb6913..4aa6b09904 100644
--- a/autogpt_platform/backend/backend/util/llm/providers_test.py
+++ b/autogpt_platform/backend/backend/util/llm/providers_test.py
@@ -27,8 +27,10 @@ from backend.util.llm.providers import (
ProviderResponse,
_anthropic_accepts_temperature,
_is_temperature_deprecation_error,
+ anthropic_accepts_forced_tool_choice,
call_provider,
call_provider_openai_compat_sync,
+ is_forced_tool_choice_rejection,
request_timeout,
)
from backend.util.settings import Config, Settings
@@ -469,6 +471,7 @@ class TestAnthropicMessages:
)
assert isinstance(result, ProviderResponse)
assert result.content == expected
+ assert result.stop_reason == response.stop_reason
assert result.reasoning == "reasoning"
assert result.completion_tokens == 7
if answer.type == "tool_use":
@@ -1893,6 +1896,7 @@ class TestDownloadBatchResults:
result=SimpleNamespace(
type="succeeded",
message=SimpleNamespace(
+ stop_reason="end_turn",
content=[
SimpleNamespace(type="text", text='{"facts": []}')
],
@@ -1934,7 +1938,8 @@ class TestDownloadBatchResults:
@pytest.mark.asyncio
@pytest.mark.parametrize("thinking_first", [False, True])
- async def test_extracts_tool_use_input_as_json(self, thinking_first):
+ @pytest.mark.parametrize("stop_reason", ["tool_use", "max_tokens"])
+ async def test_extracts_tool_use_input_as_json(self, thinking_first, stop_reason):
"""Step 5 leans on this: when the dream pass uses
``tool_choice={"type":"tool","name":...}`` to force structured
output, the result is one ``tool_use`` block whose ``input``
@@ -1957,6 +1962,7 @@ class TestDownloadBatchResults:
result=SimpleNamespace(
type="succeeded",
message=SimpleNamespace(
+ stop_reason=stop_reason,
content=thinking
+ [
anthropic.types.ToolUseBlock(
@@ -1990,6 +1996,54 @@ class TestDownloadBatchResults:
api_key="x",
)
assert rows[0].content == '{"writes": [], "summary_for_user": "ok"}'
+ assert rows[0].error == ("max_tokens" if stop_reason == "max_tokens" else None)
+ assert rows[0].output_tokens == 1
+
+ @pytest.mark.asyncio
+ async def test_tool_use_after_a_text_block_is_still_the_result(self):
+ """Under ``tool_choice`` ``auto`` (a model that rejects a forced
+ tool) a short text block can come before the call. The tool input
+ is still the structured payload, not the text."""
+ results_iter = _fake_async_iter(
+ [
+ SimpleNamespace(
+ custom_id="passid-1_recombine",
+ result=SimpleNamespace(
+ type="succeeded",
+ message=SimpleNamespace(
+ stop_reason="end_turn",
+ content=[
+ anthropic.types.TextBlock(
+ type="text", text="Recording the proposals."
+ ),
+ anthropic.types.ToolUseBlock(
+ type="tool_use",
+ id="tool-1",
+ name="emit_recombination",
+ input={"proposals": []},
+ ),
+ ],
+ usage=SimpleNamespace(input_tokens=1, output_tokens=1),
+ ),
+ ),
+ )
+ ]
+ )
+ fake_client = SimpleNamespace(
+ messages=SimpleNamespace(
+ batches=SimpleNamespace(results=AsyncMock(return_value=results_iter))
+ ),
+ )
+ with patch(
+ "backend.util.llm.providers.anthropic.AsyncAnthropic",
+ return_value=fake_client,
+ ):
+ rows = await download_batch_results(
+ provider="anthropic",
+ provider_batch_id="x",
+ api_key="x",
+ )
+ assert rows[0].content == '{"proposals": []}'
@pytest.mark.asyncio
async def test_errored_row_carries_error_string_with_zero_tokens(self):
@@ -2028,6 +2082,28 @@ class TestDownloadBatchResults:
assert rows[0].output_tokens == 0
+def test_batch_result_row_with_content_keeps_every_other_field():
+ row = BatchResultRow(
+ custom_id="passid-1_consolidate",
+ content='Here are the facts: {"facts": []}',
+ input_tokens=10,
+ output_tokens=20,
+ cache_read_tokens=3,
+ cache_creation_tokens=2,
+ error="kept as is",
+ raw_result=object(),
+ )
+
+ swapped = row.with_content('{"facts": []}')
+
+ assert swapped.content == '{"facts": []}'
+ assert row.content == 'Here are the facts: {"facts": []}'
+ # Every field the row has, so a field added later can't be dropped.
+ for name in getattr(BatchResultRow, "__pydantic_fields__"):
+ if name != "content":
+ assert getattr(swapped, name) == getattr(row, name), name
+
+
def _fake_async_iter(items):
"""Build an awaitable that yields an async iterator over ``items``.
@@ -2475,3 +2551,82 @@ class TestClaude5TemperatureRejection:
)
def test_self_heal_ignores_unrelated_errors(self, message: str):
assert not _is_temperature_deprecation_error(self._err(message))
+
+
+class TestForcedToolChoiceCapability:
+ """Opus 5.5 and the Fable 5.1 / Mythos 5.1 line answer a forced
+ ``tool_choice`` with a 400. Exact slugs, not family prefixes: Sonnet 5
+ and Opus 5 still accept a forced tool."""
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "claude-opus-5-5",
+ "anthropic/claude-opus-5.5",
+ "anthropic.claude-opus-5-5",
+ "claude-opus-5-5-20261015",
+ "claude-sonnet-5-5",
+ "anthropic/claude-sonnet-5.5",
+ "anthropic.claude-sonnet-5-5",
+ "claude-sonnet-5-5-20260928",
+ "claude-fable-5-1",
+ "claude-mythos-5-1",
+ ],
+ )
+ def test_rejecting_models(self, model: str):
+ assert anthropic_accepts_forced_tool_choice(model) is False
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "claude-opus-5",
+ "anthropic/claude-opus-5",
+ "claude-sonnet-5",
+ "claude-fable-5",
+ "claude-opus-4-8",
+ "claude-haiku-4-5-20251001",
+ ],
+ )
+ def test_accepting_models(self, model: str):
+ assert anthropic_accepts_forced_tool_choice(model) is True
+
+ @staticmethod
+ def _err(msg: str) -> anthropic.BadRequestError:
+ resp = httpx.Response(
+ 400,
+ request=httpx.Request("POST", "https://api.anthropic.com"),
+ json={"error": {"message": msg}},
+ )
+ return anthropic.BadRequestError(msg, response=resp, body=None)
+
+ def test_rejection_matches_anthropics_400(self):
+ assert is_forced_tool_choice_rejection(
+ self._err(
+ 'tool_choice: type "tool" and "any" are not supported for this model.'
+ )
+ )
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "temperature is not supported by this model",
+ "tool_choice.name: Field required",
+ "rate limited",
+ ],
+ )
+ def test_rejection_ignores_other_errors(self, message: str):
+ assert not is_forced_tool_choice_rejection(self._err(message))
+
+ def test_only_a_bad_request_counts_as_the_rejection(self):
+ """The documented text is not enough on its own: a 500 or an
+ exception of our own quoting it is not Anthropic's 400."""
+ text = 'tool_choice: type "tool" and "any" are not supported for this model.'
+ server_error = anthropic.InternalServerError(
+ text,
+ response=httpx.Response(
+ 500, request=httpx.Request("POST", "https://api.anthropic.com")
+ ),
+ body=None,
+ )
+ assert not is_forced_tool_choice_rejection(server_error)
+ assert not is_forced_tool_choice_rejection(RuntimeError(text))
diff --git a/autogpt_platform/backend/backend/util/llm/tool_use.py b/autogpt_platform/backend/backend/util/llm/tool_use.py
index 4c2a2d605a..40ffaca2fe 100644
--- a/autogpt_platform/backend/backend/util/llm/tool_use.py
+++ b/autogpt_platform/backend/backend/util/llm/tool_use.py
@@ -1,16 +1,22 @@
"""Pydantic → Anthropic tool definition conversion.
-Anthropic's Messages API supports forced structured output via tool
-use: provide a tool whose ``input_schema`` matches the desired output
-shape, then set ``tool_choice={"type":"tool","name":<tool_name>}``
-and the model is constrained to call exactly that tool with arguments
-matching the schema — no preamble, no markdown, no "Looking at the
-inputs, I need to..." prose. The model literally cannot emit anything
-else.
+Anthropic's Messages API supports structured output via tool use:
+provide a tool whose ``input_schema`` matches the desired output shape,
+then set ``tool_choice={"type":"tool","name":<tool_name>}`` and the
+model answers with exactly one call to that tool — no preamble, no
+markdown, no "Looking at the inputs, I need to..." prose. Forcing the
+call does not make its arguments schema-valid: that takes Anthropic's
+strict tool mode, which these helpers don't enable, so callers still
+validate the arguments against their Pydantic model.
+
+Anthropic's newest models (Opus 5.5 among them) answer a forced
+``tool_choice`` with a 400, so ``structured_tool_choice`` picks per
+model: the forced choice where it is accepted, ``auto`` where it is not.
+With ``auto`` the model can still reply in text, so the caller asks for
+the call in the prompt and parses a text reply as the fallback.
This module turns any Pydantic ``BaseModel`` subclass into the tool
-definition Anthropic expects + provides the forced ``tool_choice``
-helper.
+definition Anthropic expects + provides the ``tool_choice`` helpers.
"""
from __future__ import annotations
@@ -19,6 +25,8 @@ from typing import Any
from pydantic import BaseModel
+from backend.util.llm.providers import anthropic_accepts_forced_tool_choice
+
def pydantic_to_anthropic_tool(
response_model: type[BaseModel],
@@ -69,6 +77,35 @@ def force_tool_choice(tool_name: str) -> dict[str, Any]:
}
+def auto_tool_choice() -> dict[str, Any]:
+ """A ``tool_choice`` that leaves calling the tool to the model, for
+ models that reject a forced one. ``disable_parallel_tool_use`` still
+ holds under ``auto``: at most one tool_use block, never a result split
+ across two calls.
+ """
+ return {"type": "auto", "disable_parallel_tool_use": True}
+
+
+def structured_tool_choice(model: str, tool_name: str) -> dict[str, Any]:
+ """The ``tool_choice`` for a structured-output call to *model*:
+ ``force_tool_choice(tool_name)`` where the model accepts a forced tool,
+ ``auto_tool_choice()`` where it answers one with a 400 (see
+ ``providers.anthropic_accepts_forced_tool_choice``).
+
+ Under ``auto`` the model may reply in text instead of calling the
+ tool, so the caller also asks for the call in the prompt and parses a
+ text reply as the fallback.
+ """
+ if anthropic_accepts_forced_tool_choice(model):
+ return force_tool_choice(tool_name)
+ return auto_tool_choice()
+
+
+def is_forced_tool_choice(tool_choice: dict[str, Any] | None) -> bool:
+ """Whether *tool_choice* forces a tool call (``tool`` or ``any``)."""
+ return tool_choice is not None and tool_choice.get("type") in ("tool", "any")
+
+
# ---------------------------------------------------------------------------
# $ref inlining
# ---------------------------------------------------------------------------
@@ -79,9 +116,11 @@ def _inline_refs(schema: dict[str, Any]) -> dict[str, Any]:
Pydantic emits ``{"$ref": "#/$defs/Foo"}`` for nested models. This
walks the schema, replaces each ``$ref`` with the referenced
- definition, and returns a fully inlined copy. Mutually recursive
- schemas would loop forever; the dream-pass schemas don't have
- that shape (validated by tests).
+ definition, and returns a fully inlined copy. Keys beside a ``$ref``
+ (a field's ``default`` and ``description``) are laid over the
+ inlined definition and win over its own. Mutually recursive schemas
+ would loop forever; the dream-pass schemas don't have that shape
+ (validated by tests).
"""
defs = schema.get("$defs", {})
return _resolve(schema, defs)
@@ -95,8 +134,16 @@ def _resolve(node: Any, defs: dict[str, Any]) -> Any:
key = ref.split("/")[-1]
target = defs.get(key)
if target is not None:
- # Resolve nested refs in the target before returning.
- return _resolve(target, defs)
+ # Resolve nested refs in the target, then keep what the
+ # referencing node says itself: pydantic puts a field's
+ # ``default`` and ``description`` beside its ``$ref``.
+ resolved = _resolve(target, defs)
+ siblings = {
+ k: _resolve(v, defs) for k, v in node.items() if k != "$ref"
+ }
+ if isinstance(resolved, dict):
+ return {**resolved, **siblings}
+ return resolved
# Unknown ref form — leave as-is rather than fabricate.
return node
return {k: _resolve(v, defs) for k, v in node.items()}
diff --git a/autogpt_platform/backend/backend/util/llm/tool_use_test.py b/autogpt_platform/backend/backend/util/llm/tool_use_test.py
index 36d2288eb3..e276d6b92d 100644
--- a/autogpt_platform/backend/backend/util/llm/tool_use_test.py
+++ b/autogpt_platform/backend/backend/util/llm/tool_use_test.py
@@ -2,14 +2,19 @@
from __future__ import annotations
+from enum import Enum
from typing import Literal
+import pytest
from pydantic import BaseModel, Field
from backend.util.llm.tool_use import (
_inline_refs,
+ auto_tool_choice,
force_tool_choice,
+ is_forced_tool_choice,
pydantic_to_anthropic_tool,
+ structured_tool_choice,
)
@@ -25,6 +30,17 @@ class _Operations(BaseModel):
summary_for_user: str = ""
+class _Kind(str, Enum):
+ """Kinds of memory."""
+
+ FACT = "fact"
+ RULE = "rule"
+
+
+class _Finding(BaseModel):
+ kind: _Kind = Field(default=_Kind.FACT, description="What the finding is.")
+
+
class TestPydanticToAnthropicTool:
def test_returns_name_description_and_input_schema(self):
tool = pydantic_to_anthropic_tool(
@@ -81,6 +97,48 @@ class TestForceToolChoice:
assert choice["disable_parallel_tool_use"] is True
+class TestStructuredToolChoice:
+ """Opus 5.5 answers a forced ``tool_choice`` with a 400, so its output
+ tool goes out under ``auto``; models that accept forcing keep it."""
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "claude-opus-5-5",
+ "anthropic/claude-opus-5.5",
+ "anthropic/claude-opus-5-5",
+ "claude-opus-5-5-20261015",
+ "claude-fable-5-1",
+ ],
+ )
+ def test_models_that_reject_forcing_get_auto(self, model: str):
+ assert structured_tool_choice(model, "emit_x") == auto_tool_choice()
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "claude-sonnet-5",
+ "anthropic/claude-sonnet-5",
+ "claude-opus-5",
+ "claude-haiku-4-5-20251001",
+ ],
+ )
+ def test_models_that_accept_forcing_keep_it(self, model: str):
+ assert structured_tool_choice(model, "emit_x") == force_tool_choice("emit_x")
+
+ def test_auto_still_allows_at_most_one_call(self):
+ assert auto_tool_choice() == {
+ "type": "auto",
+ "disable_parallel_tool_use": True,
+ }
+
+ def test_only_tool_and_any_count_as_forced(self):
+ assert is_forced_tool_choice(force_tool_choice("emit_x"))
+ assert is_forced_tool_choice({"type": "any"})
+ assert not is_forced_tool_choice(auto_tool_choice())
+ assert not is_forced_tool_choice(None)
+
+
class TestInlineRefs:
def test_passes_through_schema_without_refs(self):
schema = {"type": "object", "properties": {"a": {"type": "string"}}}
@@ -112,6 +170,44 @@ class TestInlineRefs:
== "integer"
)
+ def test_keys_beside_a_ref_survive_and_win(self):
+ """A field's ``default`` and ``description`` sit beside its ``$ref``;
+ inlining keeps them, over the definition's own description."""
+ schema = {
+ "type": "object",
+ "properties": {
+ "kind": {
+ "$ref": "#/$defs/Kind",
+ "default": "fact",
+ "description": "What the finding is.",
+ }
+ },
+ "$defs": {
+ "Kind": {
+ "type": "string",
+ "enum": ["fact", "rule"],
+ "description": "Kinds of memory.",
+ }
+ },
+ }
+ result = _inline_refs(schema)
+ assert result["properties"]["kind"] == {
+ "type": "string",
+ "enum": ["fact", "rule"],
+ "default": "fact",
+ "description": "What the finding is.",
+ }
+
+ def test_pydantic_ref_field_keeps_its_default_and_description(self):
+ """An enum field with a default and a description, as pydantic
+ emits it: the tool schema keeps both next to the inlined enum."""
+ tool = pydantic_to_anthropic_tool(_Finding, tool_name="x", description="x")
+ kind = tool["input_schema"]["properties"]["kind"]
+ assert kind["enum"] == ["fact", "rule"]
+ assert kind["default"] == "fact"
+ assert kind["description"] == "What the finding is."
+ assert "$ref" not in kind
+
def test_leaves_unknown_ref_form_alone(self):
"""A ref to an external schema (not #/$defs/...) shouldn't crash;
we leave it as-is and let Anthropic decide."""