Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/executor): persist activity-status LLM cost to PlatformCostLog (#13009)

Zamil Majdy4ea3ce6e3ca4
diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator.py b/autogpt_platform/backend/backend/executor/activity_status_generator.py
index dd5d7ddfc4..443257434f 100644
--- a/autogpt_platform/backend/backend/executor/activity_status_generator.py
+++ b/autogpt_platform/backend/backend/executor/activity_status_generator.py
@@ -16,7 +16,13 @@ from pydantic import SecretStr
from backend.blocks import get_block
from backend.blocks.llm import AIStructuredResponseGeneratorBlock, LlmModel
from backend.data.execution import ExecutionStatus, NodeExecutionResult
-from backend.data.model import APIKeyCredentials, GraphExecutionStats
+from backend.data.model import (
+ APIKeyCredentials,
+ GraphExecutionStats,
+ NodeExecutionStats,
+)
+from backend.data.platform_cost import PlatformCostEntry, usd_to_microdollars
+from backend.executor.cost_tracking import schedule_platform_cost_log
from backend.util.feature_flag import Flag, is_feature_enabled
from backend.util.settings import Settings
from backend.util.truncate import truncate
@@ -358,6 +364,18 @@ async def generate_activity_status_for_execution(
f"Generated activity status for {graph_exec_id}: {activity_response}"
)
+ # Persist this LLM call's cost to PlatformCostLog so the admin
+ # dashboard attributes the spend per-user / per-graph_exec.
+ # Platform-side overhead (not user-billed and not user rate-limited).
+ _persist_activity_status_cost(
+ stats=structured_block.execution_stats,
+ user_id=user_id,
+ graph_exec_id=graph_exec_id,
+ graph_id=graph_id,
+ model_name=model_name,
+ db_client=db_client,
+ )
+
return activity_response
except Exception as e:
@@ -367,6 +385,71 @@ async def generate_activity_status_for_execution(
return None
+def _persist_activity_status_cost(
+ *,
+ stats: NodeExecutionStats,
+ user_id: str,
+ graph_exec_id: str,
+ graph_id: str,
+ model_name: str,
+ db_client: "DatabaseManagerAsyncClient",
+) -> None:
+ """Schedule a PlatformCostLog entry for the activity-status LLM call.
+
+ Mirrors ``backend.copilot.token_tracking._schedule_cost_log``: the platform
+ pays for this call (system OpenAI key) but the user is not billed and not
+ rate-limited, so we skip ``record_cost_usage`` and only write to
+ ``PlatformCostLog`` for admin attribution.
+
+ Cost-logging is best-effort: any failure here is swallowed so a transient
+ DB / scheduling error never strips a successful activity-status response
+ from the user.
+ """
+ try:
+ cost_usd = stats.provider_cost
+ input_tokens = stats.input_token_count or 0
+ output_tokens = stats.output_token_count or 0
+ # Skip when there is genuinely nothing to log. ``not cost_usd`` covers
+ # both ``None`` and ``0.0`` so a zero-cost zero-token call doesn't
+ # write an empty row that just dilutes dashboard averages.
+ if not cost_usd and input_tokens == 0 and output_tokens == 0:
+ return
+
+ cost_microdollars = (
+ usd_to_microdollars(cost_usd) if cost_usd is not None else None
+ )
+ if cost_usd is not None:
+ tracking_type = "cost_usd"
+ tracking_amount = float(cost_usd)
+ else:
+ tracking_type = "tokens"
+ tracking_amount = float(input_tokens + output_tokens)
+
+ schedule_platform_cost_log(
+ db_client,
+ PlatformCostEntry(
+ user_id=user_id,
+ graph_exec_id=graph_exec_id,
+ graph_id=graph_id,
+ block_name="activity_status_generator",
+ provider="openai",
+ cost_microdollars=cost_microdollars,
+ input_tokens=input_tokens or None,
+ output_tokens=output_tokens or None,
+ model=model_name,
+ tracking_type=tracking_type,
+ tracking_amount=tracking_amount,
+ metadata={"source": "activity_status_generator"},
+ ),
+ )
+ except Exception:
+ logger.exception(
+ "Failed to persist activity-status cost for graph_exec %s; "
+ "the activity status itself was returned successfully",
+ graph_exec_id,
+ )
+
+
def _build_execution_summary(
node_executions: list[NodeExecutionResult],
execution_stats: GraphExecutionStats,
diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py b/autogpt_platform/backend/backend/executor/activity_status_generator_test.py
index c3ce0b6bf0..6b6d717e11 100644
--- a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py
+++ b/autogpt_platform/backend/backend/executor/activity_status_generator_test.py
@@ -9,7 +9,7 @@ import pytest
from backend.blocks.llm import LlmModel, LLMResponse
from backend.data.execution import ExecutionStatus, NodeExecutionResult
-from backend.data.model import GraphExecutionStats
+from backend.data.model import GraphExecutionStats, NodeExecutionStats
from backend.executor.activity_status_generator import (
_build_execution_summary,
generate_activity_status_for_execution,
@@ -531,6 +531,9 @@ class TestGenerateActivityStatusForExecution:
# Mock the structured block to return our expected response
mock_instance = mock_structured_block.return_value
+ # Real NodeExecutionStats so the post-run cost-persist short-circuit
+ # has concrete numbers instead of MagicMock attributes.
+ mock_instance.execution_stats = NodeExecutionStats()
async def mock_run(*args, **kwargs):
yield "response", {
@@ -560,6 +563,211 @@ class TestGenerateActivityStatusForExecution:
mock_db_client.get_graph.assert_called_once()
mock_structured_block.assert_called_once()
+ @pytest.mark.asyncio
+ async def test_generate_status_persists_platform_cost(
+ self, mock_node_executions, mock_execution_stats, mock_blocks
+ ):
+ """Successful generation schedules a PlatformCostLog entry with the
+ right user_id / graph_exec_id / model so admin attribution works."""
+ from backend.data.model import NodeExecutionStats
+
+ mock_db_client = AsyncMock()
+ mock_db_client.get_node_executions.return_value = mock_node_executions
+
+ mock_graph_metadata = MagicMock()
+ mock_graph_metadata.name = "Test Agent"
+ mock_graph_metadata.description = "A test agent"
+ mock_db_client.get_graph_metadata.return_value = mock_graph_metadata
+
+ mock_graph = MagicMock()
+ mock_graph.links = []
+ mock_db_client.get_graph.return_value = mock_graph
+
+ with patch(
+ "backend.executor.activity_status_generator.get_block"
+ ) as mock_get_block, patch(
+ "backend.executor.activity_status_generator.Settings"
+ ) as mock_settings, patch(
+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
+ ) as mock_structured_block, patch(
+ "backend.executor.activity_status_generator.is_feature_enabled",
+ return_value=True,
+ ), patch(
+ "backend.executor.activity_status_generator.schedule_platform_cost_log"
+ ) as mock_schedule_log:
+ mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
+ mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
+
+ mock_instance = mock_structured_block.return_value
+ # Simulate the block recording its OpenRouter-style USD cost.
+ mock_instance.execution_stats = NodeExecutionStats(
+ input_token_count=120,
+ output_token_count=40,
+ provider_cost=0.0042,
+ provider_cost_type="cost_usd",
+ )
+
+ async def mock_run(*args, **kwargs):
+ yield "response", {
+ "activity_status": "Done.",
+ "correctness_score": 0.9,
+ }
+
+ mock_instance.run = mock_run
+
+ result = await generate_activity_status_for_execution(
+ graph_exec_id="test_exec",
+ graph_id="test_graph",
+ graph_version=1,
+ execution_stats=mock_execution_stats,
+ db_client=mock_db_client,
+ user_id="test_user",
+ model_name="gpt-4o-mini",
+ )
+
+ assert result is not None
+ mock_schedule_log.assert_called_once()
+ args, _ = mock_schedule_log.call_args
+ scheduled_db_client, entry = args
+ assert scheduled_db_client is mock_db_client
+ assert entry.user_id == "test_user"
+ assert entry.graph_exec_id == "test_exec"
+ assert entry.graph_id == "test_graph"
+ assert entry.block_name == "activity_status_generator"
+ assert entry.provider == "openai"
+ assert entry.model == "gpt-4o-mini"
+ assert entry.tracking_type == "cost_usd"
+ assert entry.tracking_amount == pytest.approx(0.0042)
+ # 0.0042 USD * 1_000_000 µ$/USD = 4200 µ$
+ assert entry.cost_microdollars == 4200
+ assert entry.input_tokens == 120
+ assert entry.output_tokens == 40
+ assert entry.metadata == {"source": "activity_status_generator"}
+
+ @pytest.mark.asyncio
+ async def test_generate_status_no_cost_no_log(
+ self, mock_node_executions, mock_execution_stats, mock_blocks
+ ):
+ """When the block records neither cost nor tokens, skip the log
+ write so we don't pollute the cost dashboard with empty rows."""
+ from backend.data.model import NodeExecutionStats
+
+ mock_db_client = AsyncMock()
+ mock_db_client.get_node_executions.return_value = mock_node_executions
+ mock_graph_metadata = MagicMock()
+ mock_graph_metadata.name = "Test Agent"
+ mock_graph_metadata.description = "A test agent"
+ mock_db_client.get_graph_metadata.return_value = mock_graph_metadata
+ mock_graph = MagicMock()
+ mock_graph.links = []
+ mock_db_client.get_graph.return_value = mock_graph
+
+ with patch(
+ "backend.executor.activity_status_generator.get_block"
+ ) as mock_get_block, patch(
+ "backend.executor.activity_status_generator.Settings"
+ ) as mock_settings, patch(
+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
+ ) as mock_structured_block, patch(
+ "backend.executor.activity_status_generator.is_feature_enabled",
+ return_value=True,
+ ), patch(
+ "backend.executor.activity_status_generator.schedule_platform_cost_log"
+ ) as mock_schedule_log:
+ mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
+ mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
+
+ mock_instance = mock_structured_block.return_value
+ mock_instance.execution_stats = NodeExecutionStats()
+
+ async def mock_run(*args, **kwargs):
+ yield "response", {
+ "activity_status": "Done.",
+ "correctness_score": 0.9,
+ }
+
+ mock_instance.run = mock_run
+
+ result = await generate_activity_status_for_execution(
+ graph_exec_id="test_exec",
+ graph_id="test_graph",
+ graph_version=1,
+ execution_stats=mock_execution_stats,
+ db_client=mock_db_client,
+ user_id="test_user",
+ )
+
+ assert result is not None
+ mock_schedule_log.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_generate_status_tokens_only_branch(
+ self, mock_node_executions, mock_execution_stats, mock_blocks
+ ):
+ """When the provider doesn't report a USD cost but tokens are present,
+ log the entry with tracking_type='tokens' and the summed token count
+ as tracking_amount (so admin dashboards still see request volume even
+ when cost data is missing)."""
+ from backend.data.model import NodeExecutionStats
+
+ mock_db_client = AsyncMock()
+ mock_db_client.get_node_executions.return_value = mock_node_executions
+ mock_graph_metadata = MagicMock()
+ mock_graph_metadata.name = "Test Agent"
+ mock_graph_metadata.description = "A test agent"
+ mock_db_client.get_graph_metadata.return_value = mock_graph_metadata
+ mock_graph = MagicMock()
+ mock_graph.links = []
+ mock_db_client.get_graph.return_value = mock_graph
+
+ with patch(
+ "backend.executor.activity_status_generator.get_block"
+ ) as mock_get_block, patch(
+ "backend.executor.activity_status_generator.Settings"
+ ) as mock_settings, patch(
+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
+ ) as mock_structured_block, patch(
+ "backend.executor.activity_status_generator.is_feature_enabled",
+ return_value=True,
+ ), patch(
+ "backend.executor.activity_status_generator.schedule_platform_cost_log"
+ ) as mock_schedule_log:
+ mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
+ mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
+
+ mock_instance = mock_structured_block.return_value
+ mock_instance.execution_stats = NodeExecutionStats(
+ input_token_count=200,
+ output_token_count=80,
+ provider_cost=None,
+ )
+
+ async def mock_run(*args, **kwargs):
+ yield "response", {
+ "activity_status": "Done.",
+ "correctness_score": 0.9,
+ }
+
+ mock_instance.run = mock_run
+
+ result = await generate_activity_status_for_execution(
+ graph_exec_id="test_exec",
+ graph_id="test_graph",
+ graph_version=1,
+ execution_stats=mock_execution_stats,
+ db_client=mock_db_client,
+ user_id="test_user",
+ )
+
+ assert result is not None
+ mock_schedule_log.assert_called_once()
+ _, entry = mock_schedule_log.call_args.args
+ assert entry.tracking_type == "tokens"
+ assert entry.tracking_amount == 280.0
+ assert entry.cost_microdollars is None
+ assert entry.input_tokens == 200
+ assert entry.output_tokens == 80
+
@pytest.mark.asyncio
async def test_generate_status_feature_disabled(self, mock_execution_stats):
"""Test activity status generation when feature is disabled."""
@@ -657,6 +865,7 @@ class TestGenerateActivityStatusForExecution:
# Mock the structured block to return our expected response
mock_instance = mock_structured_block.return_value
+ mock_instance.execution_stats = NodeExecutionStats()
async def mock_run(*args, **kwargs):
yield "response", {
@@ -720,6 +929,7 @@ class TestIntegration:
# Mock the structured block to return our expected response
mock_instance = mock_structured_block.return_value
+ mock_instance.execution_stats = NodeExecutionStats()
async def mock_run(*args, **kwargs):
yield "response", {
diff --git a/autogpt_platform/backend/backend/executor/cost_tracking.py b/autogpt_platform/backend/backend/executor/cost_tracking.py
index afe8ab9b10..7e6c9a5884 100644
--- a/autogpt_platform/backend/backend/executor/cost_tracking.py
+++ b/autogpt_platform/backend/backend/executor/cost_tracking.py
@@ -100,7 +100,7 @@ async def drain_pending_cost_logs(timeout: float = 5.0) -> None:
)
-def _schedule_log(
+def schedule_platform_cost_log(
db_client: "DatabaseManagerAsyncClient", entry: PlatformCostEntry
) -> None:
async def _safe_log() -> None:
@@ -263,7 +263,7 @@ async def log_system_credential_cost(
# type (USD for cost_usd, count for items/characters/per_run, etc.)
meta["provider_cost_raw"] = stats.provider_cost
- _schedule_log(
+ schedule_platform_cost_log(
db_client,
PlatformCostEntry(
user_id=node_exec.user_id,