Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(copilot): baseline cost tracking and cache token display (#12762)

Zamil Majdyb3a58389e58a
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
index 172da2d8c6..bb3906811c 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -103,6 +103,7 @@ _TRANSCRIPT_UPLOAD_TIMEOUT_S = 5
# MIME types that can be embedded as vision content blocks (OpenAI format).
_VISION_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"})
+
# Max size for embedding images directly in the user message (20 MiB raw).
_MAX_INLINE_IMAGE_BYTES = 20 * 1024 * 1024
@@ -247,6 +248,8 @@ class _BaselineStreamState:
text_started: bool = False
turn_prompt_tokens: int = 0
turn_completion_tokens: int = 0
+ turn_cache_read_tokens: int = 0
+ turn_cache_creation_tokens: int = 0
cost_usd: float | None = None
thinking_stripper: _ThinkingStripper = field(default_factory=_ThinkingStripper)
session_messages: list[ChatMessage] = field(default_factory=list)
@@ -294,6 +297,18 @@ async def _baseline_llm_caller(
if chunk.usage:
state.turn_prompt_tokens += chunk.usage.prompt_tokens or 0
state.turn_completion_tokens += chunk.usage.completion_tokens or 0
+ # Extract cache token details when available (OpenAI /
+ # OpenRouter include these in prompt_tokens_details).
+ ptd = getattr(chunk.usage, "prompt_tokens_details", None)
+ if ptd:
+ state.turn_cache_read_tokens += (
+ getattr(ptd, "cached_tokens", 0) or 0
+ )
+ # cache_creation_input_tokens is reported by some providers
+ # (e.g. Anthropic native) but not standard OpenAI streaming.
+ state.turn_cache_creation_tokens += (
+ getattr(ptd, "cache_creation_input_tokens", 0) or 0
+ )
delta = chunk.choices[0].delta if chunk.choices else None
if not delta:
@@ -1190,16 +1205,22 @@ async def stream_chat_completion_baseline(
state.turn_prompt_tokens,
state.turn_completion_tokens,
)
-
# Persist token usage to session and record for rate limiting.
- # NOTE: OpenRouter folds cached tokens into prompt_tokens, so we
- # cannot break out cache_read/cache_creation weights. Users on the
- # baseline path may be slightly over-counted vs the SDK path.
+ # When prompt_tokens_details.cached_tokens is reported, subtract
+ # them from prompt_tokens to get the uncached count so the cost
+ # breakdown stays accurate.
+ uncached_prompt = state.turn_prompt_tokens
+ if state.turn_cache_read_tokens > 0:
+ uncached_prompt = max(
+ 0, state.turn_prompt_tokens - state.turn_cache_read_tokens
+ )
await persist_and_record_usage(
session=session,
user_id=user_id,
- prompt_tokens=state.turn_prompt_tokens,
+ prompt_tokens=uncached_prompt,
completion_tokens=state.turn_completion_tokens,
+ cache_read_tokens=state.turn_cache_read_tokens,
+ cache_creation_tokens=state.turn_cache_creation_tokens,
log_prefix="[Baseline]",
cost_usd=state.cost_usd,
model=active_model,
@@ -1269,10 +1290,13 @@ async def stream_chat_completion_baseline(
# On GeneratorExit the client is already gone, so unreachable yields
# are harmless; on normal completion they reach the SSE stream.
if state.turn_prompt_tokens > 0 or state.turn_completion_tokens > 0:
+ # Report uncached prompt tokens to match what was billed — cached tokens
+ # are excluded so the frontend display is consistent with cost_usd.
+ billed_prompt = max(0, state.turn_prompt_tokens - state.turn_cache_read_tokens)
yield StreamUsage(
- prompt_tokens=state.turn_prompt_tokens,
+ prompt_tokens=billed_prompt,
completion_tokens=state.turn_completion_tokens,
- total_tokens=state.turn_prompt_tokens + state.turn_completion_tokens,
+ total_tokens=billed_prompt + state.turn_completion_tokens,
)
yield StreamFinish()
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
index 83945409e1..881018175f 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
@@ -769,3 +769,244 @@ class TestBaselineCostExtraction:
# response was never assigned so cost extraction must not raise
assert state.cost_usd is None
+
+ @pytest.mark.asyncio
+ async def test_no_cost_when_header_missing(self):
+ """cost_usd remains None when x-total-cost is absent."""
+ from backend.copilot.baseline.service import (
+ _baseline_llm_caller,
+ _BaselineStreamState,
+ )
+
+ state = _BaselineStreamState(model="anthropic/claude-sonnet-4")
+
+ mock_raw = MagicMock()
+ mock_raw.headers = {} # no x-total-cost
+ mock_stream = MagicMock()
+ mock_stream._response = mock_raw
+
+ mock_chunk = MagicMock()
+ mock_chunk.usage = MagicMock()
+ mock_chunk.usage.prompt_tokens = 1000
+ mock_chunk.usage.completion_tokens = 500
+ mock_chunk.usage.prompt_tokens_details = None
+ mock_chunk.choices = []
+
+ async def chunk_aiter():
+ yield mock_chunk
+
+ mock_stream.__aiter__ = lambda self: chunk_aiter()
+
+ mock_client = MagicMock()
+ mock_client.chat.completions.create = AsyncMock(return_value=mock_stream)
+
+ with patch(
+ "backend.copilot.baseline.service._get_openai_client",
+ return_value=mock_client,
+ ):
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "hi"}],
+ tools=[],
+ state=state,
+ )
+
+ assert state.cost_usd is None
+
+ @pytest.mark.asyncio
+ async def test_cache_tokens_extracted_from_usage_details(self):
+ """cache tokens are extracted from prompt_tokens_details.cached_tokens."""
+ from backend.copilot.baseline.service import (
+ _baseline_llm_caller,
+ _BaselineStreamState,
+ )
+
+ state = _BaselineStreamState(model="openai/gpt-4o")
+
+ mock_raw = MagicMock()
+ mock_raw.headers = {"x-total-cost": "0.01"}
+ mock_stream = MagicMock()
+ mock_stream._response = mock_raw
+
+ # Create a chunk with prompt_tokens_details
+ mock_ptd = MagicMock()
+ mock_ptd.cached_tokens = 800
+
+ mock_chunk = MagicMock()
+ mock_chunk.usage = MagicMock()
+ mock_chunk.usage.prompt_tokens = 1000
+ mock_chunk.usage.completion_tokens = 200
+ mock_chunk.usage.prompt_tokens_details = mock_ptd
+ mock_chunk.choices = []
+
+ async def chunk_aiter():
+ yield mock_chunk
+
+ mock_stream.__aiter__ = lambda self: chunk_aiter()
+
+ mock_client = MagicMock()
+ mock_client.chat.completions.create = AsyncMock(return_value=mock_stream)
+
+ with patch(
+ "backend.copilot.baseline.service._get_openai_client",
+ return_value=mock_client,
+ ):
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "hi"}],
+ tools=[],
+ state=state,
+ )
+
+ assert state.turn_cache_read_tokens == 800
+ assert state.turn_prompt_tokens == 1000
+
+ @pytest.mark.asyncio
+ async def test_cache_creation_tokens_extracted_from_usage_details(self):
+ """cache_creation_tokens are extracted from prompt_tokens_details."""
+ from backend.copilot.baseline.service import (
+ _baseline_llm_caller,
+ _BaselineStreamState,
+ )
+
+ state = _BaselineStreamState(model="openai/gpt-4o")
+
+ mock_raw = MagicMock()
+ mock_raw.headers = {"x-total-cost": "0.01"}
+ mock_stream = MagicMock()
+ mock_stream._response = mock_raw
+
+ mock_ptd = MagicMock()
+ mock_ptd.cached_tokens = 0
+ mock_ptd.cache_creation_input_tokens = 500
+
+ mock_chunk = MagicMock()
+ mock_chunk.usage = MagicMock()
+ mock_chunk.usage.prompt_tokens = 1000
+ mock_chunk.usage.completion_tokens = 200
+ mock_chunk.usage.prompt_tokens_details = mock_ptd
+ mock_chunk.choices = []
+
+ async def chunk_aiter():
+ yield mock_chunk
+
+ mock_stream.__aiter__ = lambda self: chunk_aiter()
+
+ mock_client = MagicMock()
+ mock_client.chat.completions.create = AsyncMock(return_value=mock_stream)
+
+ with patch(
+ "backend.copilot.baseline.service._get_openai_client",
+ return_value=mock_client,
+ ):
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "hi"}],
+ tools=[],
+ state=state,
+ )
+
+ assert state.turn_cache_creation_tokens == 500
+
+ @pytest.mark.asyncio
+ async def test_token_accumulators_track_across_multiple_calls(self):
+ """Token accumulators grow correctly across multiple _baseline_llm_caller calls."""
+ from backend.copilot.baseline.service import (
+ _baseline_llm_caller,
+ _BaselineStreamState,
+ )
+
+ state = _BaselineStreamState(model="anthropic/claude-sonnet-4")
+
+ def make_stream(prompt_tokens: int, completion_tokens: int):
+ mock_raw = MagicMock()
+ mock_raw.headers = {} # no x-total-cost
+ mock_stream = MagicMock()
+ mock_stream._response = mock_raw
+
+ mock_chunk = MagicMock()
+ mock_chunk.usage = MagicMock()
+ mock_chunk.usage.prompt_tokens = prompt_tokens
+ mock_chunk.usage.completion_tokens = completion_tokens
+ mock_chunk.usage.prompt_tokens_details = None
+ mock_chunk.choices = []
+
+ async def chunk_aiter():
+ yield mock_chunk
+
+ mock_stream.__aiter__ = lambda self: chunk_aiter()
+ return mock_stream
+
+ mock_client = MagicMock()
+ mock_client.chat.completions.create = AsyncMock(
+ side_effect=[
+ make_stream(1000, 200),
+ make_stream(1100, 300),
+ ]
+ )
+
+ with patch(
+ "backend.copilot.baseline.service._get_openai_client",
+ return_value=mock_client,
+ ):
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "hi"}],
+ tools=[],
+ state=state,
+ )
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "follow up"}],
+ tools=[],
+ state=state,
+ )
+
+ # No x-total-cost header and empty pricing table -- cost_usd remains None
+ assert state.cost_usd is None
+ # Accumulators hold all tokens across both turns
+ assert state.turn_prompt_tokens == 2100
+ assert state.turn_completion_tokens == 500
+
+ @pytest.mark.asyncio
+ async def test_cost_usd_remains_none_when_header_missing(self):
+ """cost_usd stays None when x-total-cost header is absent.
+
+ Token counts are still tracked; persist_and_record_usage handles
+ the None cost by falling back to tracking_type='tokens'.
+ """
+ from backend.copilot.baseline.service import (
+ _baseline_llm_caller,
+ _BaselineStreamState,
+ )
+
+ state = _BaselineStreamState(model="anthropic/claude-sonnet-4")
+
+ mock_raw = MagicMock()
+ mock_raw.headers = {} # no x-total-cost
+ mock_stream = MagicMock()
+ mock_stream._response = mock_raw
+
+ mock_chunk = MagicMock()
+ mock_chunk.usage = MagicMock()
+ mock_chunk.usage.prompt_tokens = 1000
+ mock_chunk.usage.completion_tokens = 500
+ mock_chunk.usage.prompt_tokens_details = None
+ mock_chunk.choices = []
+
+ async def chunk_aiter():
+ yield mock_chunk
+
+ mock_stream.__aiter__ = lambda self: chunk_aiter()
+
+ mock_client = MagicMock()
+ mock_client.chat.completions.create = AsyncMock(return_value=mock_stream)
+
+ with patch(
+ "backend.copilot.baseline.service._get_openai_client",
+ return_value=mock_client,
+ ):
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "hi"}],
+ tools=[],
+ state=state,
+ )
+
+ assert state.cost_usd is None
+ assert state.turn_prompt_tokens == 1000
+ assert state.turn_completion_tokens == 500
diff --git a/autogpt_platform/backend/backend/data/platform_cost.py b/autogpt_platform/backend/backend/data/platform_cost.py
index 17915e115c..ec27572058 100644
--- a/autogpt_platform/backend/backend/data/platform_cost.py
+++ b/autogpt_platform/backend/backend/data/platform_cost.py
@@ -139,6 +139,8 @@ class UserCostSummary(BaseModel):
total_cost_microdollars: int
total_input_tokens: int
total_output_tokens: int
+ total_cache_read_tokens: int = 0
+ total_cache_creation_tokens: int = 0
request_count: int
@@ -265,38 +267,41 @@ async def get_platform_cost_dashboard(
}
# Run all four aggregation queries in parallel.
- by_provider_groups, by_user_groups, total_user_groups, total_agg_groups = (
- await asyncio.gather(
- # (provider, trackingType, model) aggregation — no ORDER BY in ORM;
- # sort by total cost descending in Python after fetch.
- PrismaLog.prisma().group_by(
- by=["provider", "trackingType", "model"],
- where=where,
- sum=sum_fields,
- count=True,
- ),
- # userId aggregation — emails fetched separately below.
- PrismaLog.prisma().group_by(
- by=["userId"],
- where=where,
- sum=sum_fields,
- count=True,
- ),
- # Distinct user count: group by userId, count groups.
- PrismaLog.prisma().group_by(
- by=["userId"],
- where=where,
- count=True,
- ),
- # Total aggregate: group by provider (no limit) to sum across all
- # matching rows. Summed in Python to get grand totals.
- PrismaLog.prisma().group_by(
- by=["provider"],
- where=where,
- sum={"costMicrodollars": True},
- count=True,
- ),
- )
+ (
+ by_provider_groups,
+ by_user_groups,
+ total_user_groups,
+ total_agg_groups,
+ ) = await asyncio.gather(
+ # (provider, trackingType, model) aggregation — no ORDER BY in ORM;
+ # sort by total cost descending in Python after fetch.
+ PrismaLog.prisma().group_by(
+ by=["provider", "trackingType", "model"],
+ where=where,
+ sum=sum_fields,
+ count=True,
+ ),
+ # userId aggregation — emails fetched separately below.
+ PrismaLog.prisma().group_by(
+ by=["userId"],
+ where=where,
+ sum=sum_fields,
+ count=True,
+ ),
+ # Distinct user count: group by userId, count groups.
+ PrismaLog.prisma().group_by(
+ by=["userId"],
+ where=where,
+ count=True,
+ ),
+ # Total aggregate: group by provider (no limit) to sum across all
+ # matching rows. Summed in Python to get grand totals.
+ PrismaLog.prisma().group_by(
+ by=["provider"],
+ where=where,
+ sum={"costMicrodollars": True},
+ count=True,
+ ),
)
# Sort by_provider by total cost descending and cap at MAX_PROVIDER_ROWS.
@@ -347,6 +352,8 @@ async def get_platform_cost_dashboard(
total_cost_microdollars=_si(r, "costMicrodollars"),
total_input_tokens=_si(r, "inputTokens"),
total_output_tokens=_si(r, "outputTokens"),
+ total_cache_read_tokens=_si(r, "cacheReadTokens"),
+ total_cache_creation_tokens=_si(r, "cacheCreationTokens"),
request_count=_ca(r),
)
for r in by_user_groups
diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx
index 46920d15bc..056eef06b8 100644
--- a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx
@@ -67,7 +67,10 @@ function LogsTable({
Cost
</th>
<th scope="col" className="px-3 py-3 text-right">
- Tokens
+ In / Out
+ </th>
+ <th scope="col" className="px-3 py-3 text-right">
+ Cache (R/W)
</th>
<th scope="col" className="px-3 py-3 text-right">
Duration
@@ -105,6 +108,11 @@ function LogsTable({
? `${formatTokens(Number(log.input_tokens ?? 0))} / ${formatTokens(Number(log.output_tokens ?? 0))}`
: "-"}
</td>
+ <td className="px-3 py-2 text-right text-xs">
+ {log.cache_read_tokens || log.cache_creation_tokens
+ ? `${formatTokens(Number(log.cache_read_tokens ?? 0))} / ${formatTokens(Number(log.cache_creation_tokens ?? 0))}`
+ : "-"}
+ </td>
<td className="px-3 py-2 text-right text-xs">
{log.duration != null
? formatDuration(Number(log.duration))
@@ -120,7 +128,7 @@ function LogsTable({
{logs.length === 0 && (
<tr>
<td
- colSpan={10}
+ colSpan={11}
className="px-4 py-8 text-center text-muted-foreground"
>
No logs found
diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx
index 7c08f85e1b..c2ee70ce72 100644
--- a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx
@@ -26,6 +26,12 @@ function UserTable({ data }: Props) {
<th scope="col" className="px-4 py-3 text-right">
Output Tokens
</th>
+ <th scope="col" className="px-4 py-3 text-right">
+ Cache Read
+ </th>
+ <th scope="col" className="px-4 py-3 text-right">
+ Cache Write
+ </th>
</tr>
</thead>
<tbody>
@@ -54,12 +60,22 @@ function UserTable({ data }: Props) {
<td className="px-4 py-3 text-right">
{formatTokens(row.total_output_tokens)}
</td>
+ <td className="px-4 py-3 text-right">
+ {(row.total_cache_read_tokens ?? 0) > 0
+ ? formatTokens(row.total_cache_read_tokens ?? 0)
+ : "-"}
+ </td>
+ <td className="px-4 py-3 text-right">
+ {(row.total_cache_creation_tokens ?? 0) > 0
+ ? formatTokens(row.total_cache_creation_tokens ?? 0)
+ : "-"}
+ </td>
</tr>
))}
{data.length === 0 && (
<tr>
<td
- colSpan={5}
+ colSpan={7}
className="px-4 py-8 text-center text-muted-foreground"
>
No cost data yet
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 446b2eb079..43f14a13fd 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -15575,6 +15575,16 @@
"type": "integer",
"title": "Total Output Tokens"
},
+ "total_cache_read_tokens": {
+ "type": "integer",
+ "title": "Total Cache Read Tokens",
+ "default": 0
+ },
+ "total_cache_creation_tokens": {
+ "type": "integer",
+ "title": "Total Cache Creation Tokens",
+ "default": 0
+ },
"request_count": { "type": "integer", "title": "Request Count" }
},
"type": "object",