AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/executor): persist activity-status LLM cost to PlatformCostLog (#13009)
Zamil Majdy4ea3ce6e3ca4
diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator.py b/autogpt_platform/backend/backend/executor/activity_status_generator.pyindex dd5d7ddfc4..443257434f 100644--- a/autogpt_platform/backend/backend/executor/activity_status_generator.py+++ b/autogpt_platform/backend/backend/executor/activity_status_generator.py@@ -16,7 +16,13 @@ from pydantic import SecretStrfrom backend.blocks import get_blockfrom backend.blocks.llm import AIStructuredResponseGeneratorBlock, LlmModelfrom backend.data.execution import ExecutionStatus, NodeExecutionResult-from backend.data.model import APIKeyCredentials, GraphExecutionStats+from backend.data.model import (+ APIKeyCredentials,+ GraphExecutionStats,+ NodeExecutionStats,+)+from backend.data.platform_cost import PlatformCostEntry, usd_to_microdollars+from backend.executor.cost_tracking import schedule_platform_cost_logfrom backend.util.feature_flag import Flag, is_feature_enabledfrom backend.util.settings import Settingsfrom backend.util.truncate import truncate@@ -358,6 +364,18 @@ async def generate_activity_status_for_execution(f"Generated activity status for {graph_exec_id}: {activity_response}")+ # Persist this LLM call's cost to PlatformCostLog so the admin+ # dashboard attributes the spend per-user / per-graph_exec.+ # Platform-side overhead (not user-billed and not user rate-limited).+ _persist_activity_status_cost(+ stats=structured_block.execution_stats,+ user_id=user_id,+ graph_exec_id=graph_exec_id,+ graph_id=graph_id,+ model_name=model_name,+ db_client=db_client,+ )+return activity_responseexcept Exception as e:@@ -367,6 +385,71 @@ async def generate_activity_status_for_execution(return None+def _persist_activity_status_cost(+ *,+ stats: NodeExecutionStats,+ user_id: str,+ graph_exec_id: str,+ graph_id: str,+ model_name: str,+ db_client: "DatabaseManagerAsyncClient",+) -> None:+ """Schedule a PlatformCostLog entry for the activity-status LLM call.++ Mirrors ``backend.copilot.token_tracking._schedule_cost_log``: the platform+ pays for this call (system OpenAI key) but the user is not billed and not+ rate-limited, so we skip ``record_cost_usage`` and only write to+ ``PlatformCostLog`` for admin attribution.++ Cost-logging is best-effort: any failure here is swallowed so a transient+ DB / scheduling error never strips a successful activity-status response+ from the user.+ """+ try:+ cost_usd = stats.provider_cost+ input_tokens = stats.input_token_count or 0+ output_tokens = stats.output_token_count or 0+ # Skip when there is genuinely nothing to log. ``not cost_usd`` covers+ # both ``None`` and ``0.0`` so a zero-cost zero-token call doesn't+ # write an empty row that just dilutes dashboard averages.+ if not cost_usd and input_tokens == 0 and output_tokens == 0:+ return++ cost_microdollars = (+ usd_to_microdollars(cost_usd) if cost_usd is not None else None+ )+ if cost_usd is not None:+ tracking_type = "cost_usd"+ tracking_amount = float(cost_usd)+ else:+ tracking_type = "tokens"+ tracking_amount = float(input_tokens + output_tokens)++ schedule_platform_cost_log(+ db_client,+ PlatformCostEntry(+ user_id=user_id,+ graph_exec_id=graph_exec_id,+ graph_id=graph_id,+ block_name="activity_status_generator",+ provider="openai",+ cost_microdollars=cost_microdollars,+ input_tokens=input_tokens or None,+ output_tokens=output_tokens or None,+ model=model_name,+ tracking_type=tracking_type,+ tracking_amount=tracking_amount,+ metadata={"source": "activity_status_generator"},+ ),+ )+ except Exception:+ logger.exception(+ "Failed to persist activity-status cost for graph_exec %s; "+ "the activity status itself was returned successfully",+ graph_exec_id,+ )++def _build_execution_summary(node_executions: list[NodeExecutionResult],execution_stats: GraphExecutionStats,diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py b/autogpt_platform/backend/backend/executor/activity_status_generator_test.pyindex c3ce0b6bf0..6b6d717e11 100644--- a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py+++ b/autogpt_platform/backend/backend/executor/activity_status_generator_test.py@@ -9,7 +9,7 @@ import pytestfrom backend.blocks.llm import LlmModel, LLMResponsefrom backend.data.execution import ExecutionStatus, NodeExecutionResult-from backend.data.model import GraphExecutionStats+from backend.data.model import GraphExecutionStats, NodeExecutionStatsfrom backend.executor.activity_status_generator import (_build_execution_summary,generate_activity_status_for_execution,@@ -531,6 +531,9 @@ class TestGenerateActivityStatusForExecution:# Mock the structured block to return our expected responsemock_instance = mock_structured_block.return_value+ # Real NodeExecutionStats so the post-run cost-persist short-circuit+ # has concrete numbers instead of MagicMock attributes.+ mock_instance.execution_stats = NodeExecutionStats()async def mock_run(*args, **kwargs):yield "response", {@@ -560,6 +563,211 @@ class TestGenerateActivityStatusForExecution:mock_db_client.get_graph.assert_called_once()mock_structured_block.assert_called_once()+ @pytest.mark.asyncio+ async def test_generate_status_persists_platform_cost(+ self, mock_node_executions, mock_execution_stats, mock_blocks+ ):+ """Successful generation schedules a PlatformCostLog entry with the+ right user_id / graph_exec_id / model so admin attribution works."""+ from backend.data.model import NodeExecutionStats++ mock_db_client = AsyncMock()+ mock_db_client.get_node_executions.return_value = mock_node_executions++ mock_graph_metadata = MagicMock()+ mock_graph_metadata.name = "Test Agent"+ mock_graph_metadata.description = "A test agent"+ mock_db_client.get_graph_metadata.return_value = mock_graph_metadata++ mock_graph = MagicMock()+ mock_graph.links = []+ mock_db_client.get_graph.return_value = mock_graph++ with patch(+ "backend.executor.activity_status_generator.get_block"+ ) as mock_get_block, patch(+ "backend.executor.activity_status_generator.Settings"+ ) as mock_settings, patch(+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"+ ) as mock_structured_block, patch(+ "backend.executor.activity_status_generator.is_feature_enabled",+ return_value=True,+ ), patch(+ "backend.executor.activity_status_generator.schedule_platform_cost_log"+ ) as mock_schedule_log:+ mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)+ mock_settings.return_value.secrets.openai_internal_api_key = "test_key"++ mock_instance = mock_structured_block.return_value+ # Simulate the block recording its OpenRouter-style USD cost.+ mock_instance.execution_stats = NodeExecutionStats(+ input_token_count=120,+ output_token_count=40,+ provider_cost=0.0042,+ provider_cost_type="cost_usd",+ )++ async def mock_run(*args, **kwargs):+ yield "response", {+ "activity_status": "Done.",+ "correctness_score": 0.9,+ }++ mock_instance.run = mock_run++ result = await generate_activity_status_for_execution(+ graph_exec_id="test_exec",+ graph_id="test_graph",+ graph_version=1,+ execution_stats=mock_execution_stats,+ db_client=mock_db_client,+ user_id="test_user",+ model_name="gpt-4o-mini",+ )++ assert result is not None+ mock_schedule_log.assert_called_once()+ args, _ = mock_schedule_log.call_args+ scheduled_db_client, entry = args+ assert scheduled_db_client is mock_db_client+ assert entry.user_id == "test_user"+ assert entry.graph_exec_id == "test_exec"+ assert entry.graph_id == "test_graph"+ assert entry.block_name == "activity_status_generator"+ assert entry.provider == "openai"+ assert entry.model == "gpt-4o-mini"+ assert entry.tracking_type == "cost_usd"+ assert entry.tracking_amount == pytest.approx(0.0042)+ # 0.0042 USD * 1_000_000 µ$/USD = 4200 µ$+ assert entry.cost_microdollars == 4200+ assert entry.input_tokens == 120+ assert entry.output_tokens == 40+ assert entry.metadata == {"source": "activity_status_generator"}++ @pytest.mark.asyncio+ async def test_generate_status_no_cost_no_log(+ self, mock_node_executions, mock_execution_stats, mock_blocks+ ):+ """When the block records neither cost nor tokens, skip the log+ write so we don't pollute the cost dashboard with empty rows."""+ from backend.data.model import NodeExecutionStats++ mock_db_client = AsyncMock()+ mock_db_client.get_node_executions.return_value = mock_node_executions+ mock_graph_metadata = MagicMock()+ mock_graph_metadata.name = "Test Agent"+ mock_graph_metadata.description = "A test agent"+ mock_db_client.get_graph_metadata.return_value = mock_graph_metadata+ mock_graph = MagicMock()+ mock_graph.links = []+ mock_db_client.get_graph.return_value = mock_graph++ with patch(+ "backend.executor.activity_status_generator.get_block"+ ) as mock_get_block, patch(+ "backend.executor.activity_status_generator.Settings"+ ) as mock_settings, patch(+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"+ ) as mock_structured_block, patch(+ "backend.executor.activity_status_generator.is_feature_enabled",+ return_value=True,+ ), patch(+ "backend.executor.activity_status_generator.schedule_platform_cost_log"+ ) as mock_schedule_log:+ mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)+ mock_settings.return_value.secrets.openai_internal_api_key = "test_key"++ mock_instance = mock_structured_block.return_value+ mock_instance.execution_stats = NodeExecutionStats()++ async def mock_run(*args, **kwargs):+ yield "response", {+ "activity_status": "Done.",+ "correctness_score": 0.9,+ }++ mock_instance.run = mock_run++ result = await generate_activity_status_for_execution(+ graph_exec_id="test_exec",+ graph_id="test_graph",+ graph_version=1,+ execution_stats=mock_execution_stats,+ db_client=mock_db_client,+ user_id="test_user",+ )++ assert result is not None+ mock_schedule_log.assert_not_called()++ @pytest.mark.asyncio+ async def test_generate_status_tokens_only_branch(+ self, mock_node_executions, mock_execution_stats, mock_blocks+ ):+ """When the provider doesn't report a USD cost but tokens are present,+ log the entry with tracking_type='tokens' and the summed token count+ as tracking_amount (so admin dashboards still see request volume even+ when cost data is missing)."""+ from backend.data.model import NodeExecutionStats++ mock_db_client = AsyncMock()+ mock_db_client.get_node_executions.return_value = mock_node_executions+ mock_graph_metadata = MagicMock()+ mock_graph_metadata.name = "Test Agent"+ mock_graph_metadata.description = "A test agent"+ mock_db_client.get_graph_metadata.return_value = mock_graph_metadata+ mock_graph = MagicMock()+ mock_graph.links = []+ mock_db_client.get_graph.return_value = mock_graph++ with patch(+ "backend.executor.activity_status_generator.get_block"+ ) as mock_get_block, patch(+ "backend.executor.activity_status_generator.Settings"+ ) as mock_settings, patch(+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"+ ) as mock_structured_block, patch(+ "backend.executor.activity_status_generator.is_feature_enabled",+ return_value=True,+ ), patch(+ "backend.executor.activity_status_generator.schedule_platform_cost_log"+ ) as mock_schedule_log:+ mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)+ mock_settings.return_value.secrets.openai_internal_api_key = "test_key"++ mock_instance = mock_structured_block.return_value+ mock_instance.execution_stats = NodeExecutionStats(+ input_token_count=200,+ output_token_count=80,+ provider_cost=None,+ )++ async def mock_run(*args, **kwargs):+ yield "response", {+ "activity_status": "Done.",+ "correctness_score": 0.9,+ }++ mock_instance.run = mock_run++ result = await generate_activity_status_for_execution(+ graph_exec_id="test_exec",+ graph_id="test_graph",+ graph_version=1,+ execution_stats=mock_execution_stats,+ db_client=mock_db_client,+ user_id="test_user",+ )++ assert result is not None+ mock_schedule_log.assert_called_once()+ _, entry = mock_schedule_log.call_args.args+ assert entry.tracking_type == "tokens"+ assert entry.tracking_amount == 280.0+ assert entry.cost_microdollars is None+ assert entry.input_tokens == 200+ assert entry.output_tokens == 80+@pytest.mark.asyncioasync def test_generate_status_feature_disabled(self, mock_execution_stats):"""Test activity status generation when feature is disabled."""@@ -657,6 +865,7 @@ class TestGenerateActivityStatusForExecution:# Mock the structured block to return our expected responsemock_instance = mock_structured_block.return_value+ mock_instance.execution_stats = NodeExecutionStats()async def mock_run(*args, **kwargs):yield "response", {@@ -720,6 +929,7 @@ class TestIntegration:# Mock the structured block to return our expected responsemock_instance = mock_structured_block.return_value+ mock_instance.execution_stats = NodeExecutionStats()async def mock_run(*args, **kwargs):yield "response", {diff --git a/autogpt_platform/backend/backend/executor/cost_tracking.py b/autogpt_platform/backend/backend/executor/cost_tracking.pyindex afe8ab9b10..7e6c9a5884 100644--- a/autogpt_platform/backend/backend/executor/cost_tracking.py+++ b/autogpt_platform/backend/backend/executor/cost_tracking.py@@ -100,7 +100,7 @@ async def drain_pending_cost_logs(timeout: float = 5.0) -> None:)-def _schedule_log(+def schedule_platform_cost_log(db_client: "DatabaseManagerAsyncClient", entry: PlatformCostEntry) -> None:async def _safe_log() -> None:@@ -263,7 +263,7 @@ async def log_system_credential_cost(# type (USD for cost_usd, count for items/characters/per_run, etc.)meta["provider_cost_raw"] = stats.provider_cost- _schedule_log(+ schedule_platform_cost_log(db_client,PlatformCostEntry(user_id=node_exec.user_id,