:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tasks): stop workflow tasks when the team is out of ai credits (#119414)
Harley Alexanderf8f3f6e5055a
diff --git a/products/tasks/backend/facade/workflow_tasks.py b/products/tasks/backend/facade/workflow_tasks.pyindex d41cd8fa26d5..a0c5af8266ba 100644--- a/products/tasks/backend/facade/workflow_tasks.py+++ b/products/tasks/backend/facade/workflow_tasks.py@@ -20,6 +20,7 @@ from products.tasks.backend.logic.services.workflow_tasks import (MAX_SELF_SERVE_WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY,WORKFLOW_TASK_RATE_CAP_PER_DAY,WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY,+ WorkflowTaskAICreditsExhausted,WorkflowTaskConnectorsInvalid,WorkflowTaskLimitExceeded,WorkflowTaskOriginKeyConflict,@@ -38,6 +39,7 @@ __all__ = ["OUTPUT_FIELD_TYPES","WORKFLOW_TASK_RATE_CAP_PER_DAY","WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY",+ "WorkflowTaskAICreditsExhausted","WorkflowTaskConnectorsInvalid","WorkflowTaskLimitExceeded","WorkflowTaskOriginKeyConflict",diff --git a/products/tasks/backend/logic/services/workflow_tasks.py b/products/tasks/backend/logic/services/workflow_tasks.pyindex b5596639d241..8f5566794104 100644--- a/products/tasks/backend/logic/services/workflow_tasks.py+++ b/products/tasks/backend/logic/services/workflow_tasks.py@@ -44,6 +44,8 @@ from products.tasks.backend.metrics import observe_workflow_task_createfrom products.tasks.backend.models import Channel, Task, TaskRunfrom products.tasks.backend.temporal.constants import WORKFLOW_RUN_IDLE_TIMEOUT_SECONDS+from ee.billing.quota_limiting import QuotaResource, is_team_over_credit_budget+logger = structlog.get_logger(__name__)ACTIVE_RUN_STATUSES = [TaskRun.Status.NOT_STARTED, TaskRun.Status.QUEUED, TaskRun.Status.IN_PROGRESS]@@ -142,6 +144,10 @@ class WorkflowTaskUsageLimited(Exception):pass+class WorkflowTaskAICreditsExhausted(Exception):+ pass++def create_workflow_task(*,team: Team,@@ -172,7 +178,7 @@ def create_workflow_task(with the whole project, `WorkflowTaskOwnerIneligible` when the owner lost access to the project,and `WorkflowTaskLimitExceeded` when the workflow already has `max_parallel_tasks`runs in flight. Also raises `WorkflowTaskUsageLimited` when the owner is over the- AI usage limit, and `WorkflowTaskRateCapped` / `WorkflowTaskTeamRateCapped` when+ AI usage limit, `WorkflowTaskAICreditsExhausted` when the team has used its AI credits, and `WorkflowTaskRateCapped` / `WorkflowTaskTeamRateCapped` whenthe workflow or its team reached the daily created-task cap. A replayed`origin_key` bypasses the gate and every cap. `rate_limits` can override eitherproduct default for this project; a missing value keeps the default.@@ -240,6 +246,12 @@ def create_workflow_task(if usage_limit_response(gate_owner, team.id) is not None:observe_workflow_task_create(reason="gate_blocked")raise WorkflowTaskUsageLimited()+ # Workflow runs bill to the AI credits bucket, which the gate above does not read. Both+ # gateways refuse an exhausted bucket, and the legacy gateway answers with a retryable 429,+ # so a run created here would retry until the idle timeout instead of failing.+ if _team_out_of_ai_credits(team):+ observe_workflow_task_create(reason="ai_credits_exhausted")+ raise WorkflowTaskAICreditsExhausted()# Resolved after the gate so a capped or blocked fire never pays for the query, and outside# the transaction below so the skills store read never happens while holding the team lock.@@ -410,6 +422,15 @@ def create_workflow_task(return _task_dto(task, created=True)+def _team_out_of_ai_credits(team: Team) -> bool:+ """False (fail open) when the bucket can't be read, so a Redis error never blocks every fire."""+ try:+ return is_team_over_credit_budget(team.api_token, QuotaResource.AI_CREDITS)+ except Exception:+ logger.warning("workflow_task_ai_credits_check_failed", team_id=team.id, exc_info=True)+ return False++def _resolve_channel(team_id: int, owner_id: int, channel_ref: str | None) -> Channel | None:"""The space the task is filed into, or None to file it in no space.diff --git a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.pyindex 515a4dbfe0e3..764197cf8fe1 100644--- a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py+++ b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py@@ -1370,10 +1370,9 @@ class TestWorkflowsMint:assert env["AI_GATEWAY_TOKEN"] == "phe_abc"mint.assert_called_once()- def test_team_out_of_ai_credits_does_not_mint(self, mint_settings):- env, mint = self._env(mint_settings, over_quota=True)- assert "AI_GATEWAY_TOKEN" not in env- mint.assert_not_called()+ def test_team_out_of_ai_credits_fails_the_run_instead_of_degrading(self, mint_settings):+ with pytest.raises(GatewayTokenUnavailableError, match="used its monthly PostHog AI credits"):+ self._env(mint_settings, over_quota=True)def test_credit_lookup_failure_does_not_mint(self, mint_settings):with patch(_CREDIT_LOOKUP, side_effect=RuntimeError("billing is down")):diff --git a/products/tasks/backend/temporal/process_task/utils.py b/products/tasks/backend/temporal/process_task/utils.pyindex db99d113b672..b3ca5cbe21a6 100644--- a/products/tasks/backend/temporal/process_task/utils.py+++ b/products/tasks/backend/temporal/process_task/utils.py@@ -1627,6 +1627,14 @@ def _record_pinned_gateway_product(run_id: str, state: dict | None, minted_produreturn True+WORKFLOWS_PRODUCT = "workflows"+WORKFLOWS_CREDITS_EXHAUSTED_REFUSAL = "ai_credits_exhausted"+WORKFLOWS_CREDITS_EXHAUSTED_DETAIL = (+ "This project has used its monthly PostHog AI credits. "+ "Raise the AI credit limit in billing settings, or wait for the next billing period."+)++def ai_gateway_env_vars(*,team_id: int | None = None,@@ -1652,7 +1660,8 @@ def ai_gateway_env_vars(Go gateway only when the token is present, so a missing token (mint failure,or a caller that cannot supply run context) degrades the run to the Pythongateway, except a `posthog_code` run, which raises: the Python gateway- refuses `posthog_code`.+ refuses `posthog_code`. A `workflows` run whose team is out of AI credits+ also raises, because the Python gateway refuses it too."""ai_product = (resolve_sandbox_ai_product(origin_product, ai_stage, internal=internal) if team_id is not None else None@@ -1703,6 +1712,13 @@ def ai_gateway_env_vars()if requires_go:raise posthog_code_token_unavailable(refusal, team_id)+ # The legacy gateway refuses an exhausted bucket with a retryable 429. A background+ # workflow run has nobody to read the refusal, so the agent retries until the idle+ # timeout and the run fails as lost. Fail it here with the real reason.+ if ai_product == WORKFLOWS_PRODUCT and refusal == WORKFLOWS_CREDITS_EXHAUSTED_REFUSAL:+ raise GatewayTokenUnavailableError(+ WORKFLOWS_CREDITS_EXHAUSTED_DETAIL, {"team_id": team_id, "reason": refusal}+ )return env_varsmint_kwargs: dict[str, Any] = {}if ai_product == POSTHOG_CODE_PRODUCT:diff --git a/products/tasks/backend/tests/test_workflow_tasks_api.py b/products/tasks/backend/tests/test_workflow_tasks_api.pyindex 9c8d508aa9f8..c4eebf597a3f 100644--- a/products/tasks/backend/tests/test_workflow_tasks_api.py+++ b/products/tasks/backend/tests/test_workflow_tasks_api.py@@ -443,14 +443,19 @@ class TestWorkflowTasksAPI(APIBaseTest):assert response.status_code == status.HTTP_201_CREATED, response.json()- @patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response")- def test_rejects_creation_when_the_owner_is_over_the_usage_limit(self, usage_limit_response_mock) -> None:- usage_limit_response_mock.return_value = object() # any non-None reply means blocked-- response = self._post()+ @parameterized.expand(+ [+ # Any non-None reply from the Desktop usage gate means blocked.+ ("owner_over_usage_limit", "usage_limit_response", object(), "usage limit"),+ ("team_out_of_ai_credits", "is_team_over_credit_budget", True, "monthly PostHog AI credits"),+ ]+ )+ def test_rejects_creation_when_usage_is_blocked(self, _name: str, gate: str, blocked: Any, detail: str) -> None:+ with patch(f"products.tasks.backend.logic.services.workflow_tasks.{gate}", return_value=blocked):+ response = self._post()assert response.status_code == status.HTTP_409_CONFLICT, response.json()- assert "usage limit" in response.json()["detail"]+ assert detail in response.json()["detail"]assert Task.objects.count() == 0def test_replaying_an_idempotency_key_returns_the_existing_task(self) -> None:diff --git a/products/workflows/backend/presentation/views/workflow_tasks.py b/products/workflows/backend/presentation/views/workflow_tasks.pyindex f952e455e972..1ba85d55ceeb 100644--- a/products/workflows/backend/presentation/views/workflow_tasks.py+++ b/products/workflows/backend/presentation/views/workflow_tasks.py@@ -14,6 +14,7 @@ from posthog.models.team.team import Teamfrom products.tasks.backend.facade.workflow_tasks import (MAX_ATTACHED_SKILLS,OUTPUT_FIELD_TYPES,+ WorkflowTaskAICreditsExhausted,WorkflowTaskConnectorsInvalid,WorkflowTaskLimitExceeded,WorkflowTaskOriginKeyConflict,@@ -297,6 +298,17 @@ class WorkflowTaskViewSet(viewsets.GenericViewSet):"Task creation resumes when the limit resets.",status.HTTP_409_CONFLICT,)+ except WorkflowTaskAICreditsExhausted:+ logger.info(+ "workflow_task_create_ai_credits_exhausted",+ team_id=team_id,+ hog_flow_id=str(hog_flow_id),+ )+ return _rejected(+ "This project has used its monthly PostHog AI credits, so no task was created. "+ "Raise the AI credit limit in billing settings, or wait for the next billing period.",+ status.HTTP_409_CONFLICT,+ )return Response(WorkflowTaskResponseSerializer({"id": result.task_id, "run_id": result.run_id}).data,