AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): bill SDK provider base costs in every process, on platform keys only (#15092)
Bentlyde2d613030b4
diff --git a/autogpt_platform/backend/backend/blocks/__init__.py b/autogpt_platform/backend/backend/blocks/__init__.pyindex f36ed98f36..155ee888d6 100644--- a/autogpt_platform/backend/backend/blocks/__init__.py+++ b/autogpt_platform/backend/backend/blocks/__init__.py@@ -116,6 +116,12 @@ def load_all_blocks() -> dict[str, type["AnyBlockSchema"]]:if is_block_auth_configured(block_cls):filtered_blocks[block_id] = block_cls+ # Price SDK blocks from their provider's `with_base_cost` here, so every+ # process that loads blocks bills them, not only the REST API process.+ from backend.sdk.cost_integration import sync_all_provider_costs++ sync_all_provider_costs(filtered_blocks.values())+return filtered_blocksdiff --git a/autogpt_platform/backend/backend/blocks/_base.py b/autogpt_platform/backend/backend/blocks/_base.pyindex 8d854398c7..9473c4ff5e 100644--- a/autogpt_platform/backend/backend/blocks/_base.py+++ b/autogpt_platform/backend/backend/blocks/_base.py@@ -205,6 +205,12 @@ class BlockCost(BaseModel):**data,)+ @property+ def is_free(self) -> bool:+ """Whether this entry can never charge. Every cost type scales+ ``cost_amount`` except TOKENS, which prices from the token rate table."""+ return self.cost_amount == 0 and self.cost_type != BlockCostType.TOKENS+class BlockInfo(BaseModel):id: strdiff --git a/autogpt_platform/backend/backend/data/block.py b/autogpt_platform/backend/backend/data/block.pyindex d7273aa059..6648fade9d 100644--- a/autogpt_platform/backend/backend/data/block.py+++ b/autogpt_platform/backend/backend/data/block.py@@ -20,11 +20,8 @@ CompletedBlockOutput = dict[str, list[Any]] # Completed stream, collected as aasync def initialize_blocks() -> None:from backend.blocks import get_blocks- from backend.sdk.cost_integration import sync_all_provider_costsfrom backend.util.retry import func_retry- sync_all_provider_costs()-@func_retryasync def sync_block_to_db(block: "AnyBlockSchema") -> None:existing_block = await AgentBlock.prisma().find_first(diff --git a/autogpt_platform/backend/backend/data/block_cost_config_test.py b/autogpt_platform/backend/backend/data/block_cost_config_test.pyindex 7570a29ae9..2a0c0ba51a 100644--- a/autogpt_platform/backend/backend/data/block_cost_config_test.py+++ b/autogpt_platform/backend/backend/data/block_cost_config_test.py@@ -1,4 +1,7 @@+from unittest.mock import patch+import pytest+from pydantic import SecretStrfrom backend.blocks._base import BlockCostTypefrom backend.blocks.ayrshare.post_to_bluesky import PostToBlueskyBlock@@ -14,6 +17,7 @@ from backend.blocks.ayrshare.post_to_threads import PostToThreadsBlockfrom backend.blocks.ayrshare.post_to_tiktok import PostToTikTokBlockfrom backend.blocks.ayrshare.post_to_x import PostToXBlockfrom backend.blocks.ayrshare.post_to_youtube import PostToYouTubeBlock+from backend.blocks.bannerbear._config import bannerbearfrom backend.blocks.bannerbear.text_overlay import BannerbearTextOverlayBlockfrom backend.blocks.code_executor import (ExecuteCodeBlock,@@ -25,7 +29,7 @@ from backend.blocks.jina.chunking import JinaChunkingBlockfrom backend.blocks.llm import AITextGeneratorBlock, LLMModelfrom backend.blocks.youtube import TranscribeYoutubeVideoBlockfrom backend.data.block_cost_config import BLOCK_COSTS-from backend.data.model import NodeExecutionStats+from backend.data.model import APIKeyCredentials, NodeExecutionStatsfrom backend.executor import utils as executor_utilsfrom backend.executor.utils import block_usage_costfrom backend.integrations.credentials_store import (@@ -35,6 +39,7 @@ from backend.integrations.credentials_store import (open_router_credentials,webshare_proxy_credentials,)+from backend.sdk.cost_integration import register_provider_costs_for_block@pytest.fixture(autouse=True)@@ -119,11 +124,28 @@ def test_jina_chunking_has_flat_cost_floor():assert cost == 1-def test_bannerbear_base_cost_is_three_credits():- # Bannerbear is registered via the SDK ProviderBuilder with base_cost=3.+def test_bannerbear_base_cost_is_three_credits_on_the_platform_key(monkeypatch):+ # Bannerbear is registered via the SDK ProviderBuilder with base_cost=3,+ # billed only when the run uses the platform's BANNERBEAR_API_KEY.+ platform_key = APIKeyCredentials(+ id="bannerbear-default",+ provider="bannerbear",+ api_key=SecretStr("platform-key"),+ title="Bannerbear API Key",+ )+ monkeypatch.setattr(bannerbear, "default_credentials", [platform_key])block = BannerbearTextOverlayBlock()- cost, _ = block_usage_cost(block, {})- assert cost == 3++ # patch.dict restores BLOCK_COSTS afterwards, including when the session's+ # block load registered no Bannerbear entry (no BANNERBEAR_API_KEY set).+ with patch.dict(BLOCK_COSTS):+ BLOCK_COSTS.pop(BannerbearTextOverlayBlock, None)+ register_provider_costs_for_block(BannerbearTextOverlayBlock)+ platform, _ = block_usage_cost(block, {"credentials": {"id": platform_key.id}})+ own_key, _ = block_usage_cost(block, {"credentials": {"id": "user-own-key"}})++ assert platform == 3+ assert own_key == 0def test_e2b_sandbox_blocks_bill_per_walltime_second():diff --git a/autogpt_platform/backend/backend/executor/billing.py b/autogpt_platform/backend/backend/executor/billing.pyindex d557252422..c4d6fb23fb 100644--- a/autogpt_platform/backend/backend/executor/billing.py+++ b/autogpt_platform/backend/backend/executor/billing.py@@ -75,12 +75,14 @@ async def clear_insufficient_funds_notifications(user_id: str) -> int:def _block_has_paid_cost_entry(block: Block, input_data: "dict[str, Any]") -> bool:- """Whether any BLOCK_COSTS entry matches this input — even if pre-flight is 0.+ """Whether any non-free BLOCK_COSTS entry matches this input — even if+ pre-flight is 0.Used to guard dynamic-cost blocks (SECOND/ITEMS/COST_USD) whosepre-flight cost is 0 but whose post-flight reconciliation will debita real amount. A user with non-positive balance must not be allowed- to start such a block.+ to start such a block. Free entries (e.g. Ollama models, Jev's BYOK+ price) never debit, so they don't count."""from backend.data.block_cost_config import BLOCK_COSTS@@ -89,7 +91,10 @@ def _block_has_paid_cost_entry(block: Block, input_data: "dict[str, Any]") -> boblock_costs = BLOCK_COSTS.get(type(block))if not block_costs:return False- return any(_is_cost_filter_match(bc.cost_filter, input_data) for bc in block_costs)+ return any(+ not bc.is_free and _is_cost_filter_match(bc.cost_filter, input_data)+ for bc in block_costs+ )def resolve_block_cost(diff --git a/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py b/autogpt_platform/backend/backend/executor/billing_reconciliation_test.pyindex b0adc987e2..d8af0bfe5a 100644--- a/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py+++ b/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py@@ -7,11 +7,13 @@ import pytestfrom backend.blocks._base import BlockCost, BlockCostTypefrom backend.blocks.jina.search import SearchTheWebBlock+from backend.blocks.typesafe.choice import JevChoiceBlockfrom backend.data.block_cost_config import BLOCK_COSTSfrom backend.data.execution import ExecutionContext, NodeExecutionEntryfrom backend.data.model import NodeExecutionStatsfrom backend.executor import utils as executor_utils-from backend.executor.billing import charge_reconciled_usage+from backend.executor.billing import charge_reconciled_usage, charge_usage+from backend.sdk.cost_integration import sync_all_provider_costs@pytest.fixture(autouse=True)@@ -120,6 +122,27 @@ def test_dynamic_cost_block_with_positive_balance_starts(tmp_block_costs_overriddb_client.spend_credits.assert_not_called()+def test_free_sdk_price_runs_with_an_empty_wallet():+ """Jev's SDK price is 0 (TypeSafe bills the user's own key), and block+ loading registers it, unfiltered, in the executor too. A free entry never+ debits, so it must not trip the dynamic-cost balance guard."""+ exec_entry = _node_exec(JevChoiceBlock().id)+ db_client = MagicMock()+ db_client.get_credits.return_value = 0 # empty wallet++ with (+ patch.dict(BLOCK_COSTS),+ patch("backend.executor.billing.get_db_client", return_value=db_client),+ patch("backend.executor.billing.get_block", return_value=JevChoiceBlock()),+ ):+ BLOCK_COSTS.pop(JevChoiceBlock, None)+ sync_all_provider_costs([JevChoiceBlock])+ total_cost, _, block_pre_flight = charge_usage(exec_entry, execution_count=0)++ assert (total_cost, block_pre_flight) == (0, 0)+ db_client.spend_credits.assert_not_called()++@pytest.mark.asyncioasync def test_run_cost_produces_zero_delta_noop(tmp_block_costs_override):tmp_block_costs_override([BlockCost(cost_amount=7, cost_type=BlockCostType.RUN)])diff --git a/autogpt_platform/backend/backend/sdk/builder.py b/autogpt_platform/backend/backend/sdk/builder.pyindex cc633844df..808d6ae280 100644--- a/autogpt_platform/backend/backend/sdk/builder.py+++ b/autogpt_platform/backend/backend/sdk/builder.py@@ -140,6 +140,11 @@ class ProviderBuilder:) -> "ProviderBuilder":"""Set base cost for all blocks using this provider.+ A non-zero cost is only charged when the run uses a credential the+ platform provides: the ``<provider>-default`` key from ``with_api_key``+ or ``with_user_password``, or a system credential for this provider in+ ``credentials_store``. Runs on a user's own credential are free.+``cost_divisor`` only applies to SECOND / ITEMS. TOKENS is billed viathe TOKEN_COST rate table (per-model pricing) and ignores the divisor.Example: ``with_base_cost(1, BlockCostType.SECOND, cost_divisor=10)``diff --git a/autogpt_platform/backend/backend/sdk/cost_integration.py b/autogpt_platform/backend/backend/sdk/cost_integration.pyindex f3be41cc52..46a53170e3 100644--- a/autogpt_platform/backend/backend/sdk/cost_integration.py+++ b/autogpt_platform/backend/backend/sdk/cost_integration.py@@ -6,10 +6,13 @@ BLOCK_COSTS configuration used by the execution system."""import logging-from typing import List, Type, TypeVar+from typing import Iterable, List, Type, TypeVarfrom backend.blocks._base import Block, BlockCostfrom backend.data.block_cost_config import BLOCK_COSTS+from backend.data.model import Credentials+from backend.integrations.credentials_store import DEFAULT_CREDENTIALS+from backend.sdk.provider import Providerfrom backend.sdk.registry import AutoRegistryBlockT = TypeVar("BlockT", bound=Type[Block])@@ -24,6 +27,11 @@ def register_provider_costs_for_block(block_class: Type[Block]) -> None:This function checks if the block uses credentials from a provider that hasbase costs defined, and automatically registers those costs for the block.+ Like the rest of BLOCK_COSTS, a priced base cost only applies when the run+ uses a platform-provided credential: each one is registered with a+ ``cost_filter`` on that credential's id, so runs with the user's own key+ stay free. Free base costs (``BlockCost.is_free``) stay unfiltered.+Args:block_class: The block class to register costs for"""@@ -52,6 +60,8 @@ def register_provider_costs_for_block(block_class: Type[Block]) -> None:logger.debug(f"Block {block_class.__name__} has no credentials fields")return+ block_costs: List[BlockCost] = []+# Get provider information from credentials fieldsfor field_name, field_info in credentials_fields.items():# Get the field schema to extract provider information@@ -63,43 +73,80 @@ def register_provider_costs_for_block(block_class: Type[Block]) -> None:continue# For each provider, check if it has base costs- block_costs: List[BlockCost] = []for provider_name in providers:provider = AutoRegistry.get_provider(provider_name)if not provider:logger.debug(f"Provider {provider_name} not found in registry")continue+ if not provider.base_costs:+ continue- # Add provider's base costs to the block- if provider.base_costs:- logger.debug(- f"Registering {len(provider.base_costs)} base costs from provider {provider_name} for block {block_class.__name__}"- )- block_costs.extend(provider.base_costs)-- # Register costs if any were found- if block_costs:- BLOCK_COSTS[block_class] = block_costslogger.debug(- f"Registered {len(block_costs)} total costs for block {block_class.__name__}"+ f"Registering {len(provider.base_costs)} base costs from provider {provider_name} for block {block_class.__name__}")+ block_costs.extend(_platform_priced_costs(provider, field_name))+ # Register costs if any were found+ if block_costs:+ BLOCK_COSTS[block_class] = block_costs+ logger.debug(+ f"Registered {len(block_costs)} total costs for block {block_class.__name__}"+ )-def sync_all_provider_costs() -> None:++def _platform_priced_costs(provider: Provider, field_name: str) -> List[BlockCost]:+ """The provider's base costs, each gated on one of its platform credentials+ in ``field_name``. Free costs charge nothing, so they stay unfiltered."""+ platform_ids = _platform_credential_ids(provider.name, provider.default_credentials)+ costs: List[BlockCost] = []+ for base_cost in provider.base_costs:+ if base_cost.is_free:+ costs.append(base_cost)+ continue+ costs.extend(+ base_cost.model_copy(+ update={+ "cost_filter": {+ **base_cost.cost_filter,+ field_name: {"id": cred_id},+ }+ }+ )+ for cred_id in platform_ids+ )+ return costs+++def _platform_credential_ids(+ provider_name: str, provider_creds: List[Credentials]+) -> List[str]:+ """IDs of the credentials the platform provides for ``provider_name``: the+ SDK ``<provider>-default`` key (set from the provider's env var) and any+ system credential in ``credentials_store`` (e.g. Exa's "Use Credits for Exa+ search")."""+ ids = [cred.id for cred in provider_creds]+ ids += [+ cred.id+ for cred in DEFAULT_CREDENTIALS+ if cred.provider == provider_name and cred.id not in ids+ ]+ return ids+++def sync_all_provider_costs(block_classes: Iterable[Type[Block]]) -> None:"""Sync all provider base costs to blocks that use them.- This should be called after all providers and blocks are registered,- typically during application startup.+ Called by ``load_all_blocks()`` once every provider and block module is+ imported, so each process that loads blocks (REST API, executor, copilot+ executor, ...) prices SDK blocks the same way."""- from backend.blocks import load_all_blocks-logger.info("Syncing provider costs to blocks...")blocks_with_costs = 0total_costs = 0- for block_id, block_class in load_all_blocks().items():+ for block_class in block_classes:initial_count = len(BLOCK_COSTS.get(block_class, []))register_provider_costs_for_block(block_class)final_count = len(BLOCK_COSTS.get(block_class, []))diff --git a/autogpt_platform/backend/backend/sdk/cost_integration_test.py b/autogpt_platform/backend/backend/sdk/cost_integration_test.pynew file mode 100644index 0000000000..de5945781a--- /dev/null+++ b/autogpt_platform/backend/backend/sdk/cost_integration_test.py@@ -0,0 +1,182 @@+"""SDK `with_base_cost` prices reach every process, and only bill platform keys.++The executor and copilot executor never run `initialize_blocks()`; they only+load blocks (`get_block` -> `load_all_blocks`), which syncs provider costs. The+in-process tests start from that state: the block's BLOCK_COSTS entry removed,+then priced by the same `sync_all_provider_costs` call `load_all_blocks` makes.+They never clear or re-run the cached `load_all_blocks()`: in the full suite,+test modules define their own Block subclasses (e.g. `TestWebhookBlock`), so a+fresh load would pick those up and fail. The subprocess test covers the loader.+"""++import subprocess+import sys+import textwrap++import pytest+from pydantic import SecretStr++from backend.blocks._base import BlockCost, BlockCostType+from backend.blocks.agent_mail._config import agent_mail+from backend.blocks.agent_mail.inbox import AgentMailListInboxesBlock+from backend.blocks.baas.bots import BaasBotJoinMeetingBlock+from backend.blocks.exa.search import ExaSearchBlock+from backend.data.block_cost_config import BLOCK_COSTS+from backend.data.model import APIKeyCredentials, NodeExecutionStats+from backend.executor.utils import block_usage_cost+from backend.integrations.credentials_store import exa_credentials+from backend.sdk.cost_integration import (+ register_provider_costs_for_block,+ sync_all_provider_costs,+)++AGENT_MAIL_PLATFORM_KEY = APIKeyCredentials(+ id="agent_mail-default",+ provider="agent_mail",+ api_key=SecretStr("platform-key"),+ title="AgentMail API Key",+)++[email protected](autouse=True)+def restore_block_costs():+ saved = dict(BLOCK_COSTS)+ yield+ BLOCK_COSTS.clear()+ BLOCK_COSTS.update(saved)+++def _credentials(cred_id: str, provider: str) -> dict:+ return {"credentials": {"id": cred_id, "provider": provider, "type": "api_key"}}+++def _block_loaded_like_the_executor(block_class: type):+ """Price the block the way the executor's `load_all_blocks()` does, in a+ process that never ran initialize_blocks(): no BLOCK_COSTS entry yet, then+ the provider costs synced for the loaded blocks."""+ BLOCK_COSTS.pop(block_class, None)+ sync_all_provider_costs([block_class])+ return block_class()+++def test_fresh_process_bills_sdk_price_like_the_executor():+ """The regression test that would have caught it: a new interpreter, like the+ executor and copilot executor processes, that loads blocks on demand and+ never runs initialize_blocks(). The pytest process can't show this, because+ the session test server already ran initialize_blocks() in it."""+ script = textwrap.dedent(+ f"""+ from backend.blocks import get_block+ from backend.data.model import NodeExecutionStats+ from backend.executor.utils import block_usage_cost++ block = get_block("{ExaSearchBlock().id}")+ stats = NodeExecutionStats(provider_cost=0.05, provider_cost_type="cost_usd")+ for cred_id in ("{exa_credentials.id}", "user-own-key"):+ credentials = {{"credentials": {{"id": cred_id, "provider": "exa"}}}}+ cost, _ = block_usage_cost(block, credentials, stats=stats)+ print("COST", cost)+ """+ )++ result = subprocess.run(+ [sys.executable, "-c", script],+ capture_output=True,+ text=True,+ timeout=600,+ check=True,+ )++ costs = [+ line.split()[1]+ for line in result.stdout.splitlines()+ if line.startswith("COST ")+ ]+ # Platform key: $0.05 * 100 credits/USD; own key: free.+ assert costs == ["5", "0"]+++def test_executor_lookup_bills_sdk_run_price_on_the_platform_key(monkeypatch):+ monkeypatch.setattr(agent_mail, "default_credentials", [AGENT_MAIL_PLATFORM_KEY])+ block = _block_loaded_like_the_executor(AgentMailListInboxesBlock)++ cost, _ = block_usage_cost(+ block, _credentials(AGENT_MAIL_PLATFORM_KEY.id, "agent_mail")+ )++ assert cost == 1+++def test_executor_lookup_bills_sdk_usd_price_on_the_platform_key():+ block = _block_loaded_like_the_executor(ExaSearchBlock)+ stats = NodeExecutionStats(provider_cost=0.05, provider_cost_type="cost_usd")++ cost, _ = block_usage_cost(+ block, _credentials(exa_credentials.id, "exa"), stats=stats+ )++ # $0.05 provider spend * 100 credits/USD.+ assert cost == 5+++def test_own_key_runs_stay_free(monkeypatch):+ monkeypatch.setattr(agent_mail, "default_credentials", [AGENT_MAIL_PLATFORM_KEY])+ mail = _block_loaded_like_the_executor(AgentMailListInboxesBlock)+ exa = _block_loaded_like_the_executor(ExaSearchBlock)+ stats = NodeExecutionStats(provider_cost=0.05, provider_cost_type="cost_usd")++ own_mail, _ = block_usage_cost(mail, _credentials("user-own-key", "agent_mail"))+ own_exa, _ = block_usage_cost(exa, _credentials("user-own-key", "exa"), stats=stats)++ assert own_mail == 0+ assert own_exa == 0+++def test_provider_without_a_platform_key_bills_nothing(monkeypatch):+ monkeypatch.setattr(agent_mail, "default_credentials", [])+ block = _block_loaded_like_the_executor(AgentMailListInboxesBlock)++ cost, _ = block_usage_cost(block, _credentials("user-own-key", "agent_mail"))++ assert cost == 0+++def test_a_zero_amount_tokens_price_still_needs_the_platform_key(monkeypatch):+ """TOKENS prices from the token rate table, not cost_amount, so a zero+ amount doesn't make it free. Only a free cost stays unfiltered."""+ monkeypatch.setattr(agent_mail, "default_credentials", [AGENT_MAIL_PLATFORM_KEY])+ monkeypatch.setattr(+ agent_mail,+ "base_costs",+ [+ BlockCost(cost_amount=0),+ BlockCost(cost_amount=0, cost_type=BlockCostType.TOKENS),+ ],+ )++ _block_loaded_like_the_executor(AgentMailListInboxesBlock)++ assert [c.cost_filter for c in BLOCK_COSTS[AgentMailListInboxesBlock]] == [+ {},+ {"credentials": {"id": AGENT_MAIL_PLATFORM_KEY.id}},+ ]+++def test_existing_block_costs_entries_are_left_alone():+ before = BLOCK_COSTS[BaasBotJoinMeetingBlock]++ register_provider_costs_for_block(BaasBotJoinMeetingBlock)++ assert BLOCK_COSTS[BaasBotJoinMeetingBlock] is before+++def test_syncing_twice_does_not_add_costs(monkeypatch):+ monkeypatch.setattr(agent_mail, "default_credentials", [AGENT_MAIL_PLATFORM_KEY])+ BLOCK_COSTS.pop(AgentMailListInboxesBlock, None)++ sync_all_provider_costs([AgentMailListInboxesBlock])+ first = list(BLOCK_COSTS[AgentMailListInboxesBlock])+ sync_all_provider_costs([AgentMailListInboxesBlock])++ assert BLOCK_COSTS[AgentMailListInboxesBlock] == first+ assert len(first) == 1diff --git a/docs/platform/block-sdk-guide.md b/docs/platform/block-sdk-guide.mdindex 32979722c2..bfe0d569bb 100644--- a/docs/platform/block-sdk-guide.md+++ b/docs/platform/block-sdk-guide.md@@ -148,7 +148,7 @@ class MyBlock(Block):The `ProviderBuilder` is available for advanced use cases (OAuth, webhooks, cost tracking). It allows you to:- **`.with_api_key()`**: Add API key authentication- **`.with_oauth()`**: Add OAuth authentication-- **`.with_base_cost()`**: Set resource costs for the block+- **`.with_base_cost()`**: Set what a run costs when it uses the platform's own key for this provider (for example the one `.with_api_key()` reads from the environment); runs on a user's own key are free- **`.with_webhook_manager()`**: Add webhook support- **`.with_user_password()`**: Add username/password auth