Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tasks): stop workflow tasks when the team is out of ai credits (#119414)

Harley Alexanderf8f3f6e5055a
diff --git a/products/tasks/backend/facade/workflow_tasks.py b/products/tasks/backend/facade/workflow_tasks.py
index d41cd8fa26d5..a0c5af8266ba 100644
--- a/products/tasks/backend/facade/workflow_tasks.py
+++ b/products/tasks/backend/facade/workflow_tasks.py
@@ -20,6 +20,7 @@ from products.tasks.backend.logic.services.workflow_tasks import (
MAX_SELF_SERVE_WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY,
WORKFLOW_TASK_RATE_CAP_PER_DAY,
WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY,
+ WorkflowTaskAICreditsExhausted,
WorkflowTaskConnectorsInvalid,
WorkflowTaskLimitExceeded,
WorkflowTaskOriginKeyConflict,
@@ -38,6 +39,7 @@ __all__ = [
"OUTPUT_FIELD_TYPES",
"WORKFLOW_TASK_RATE_CAP_PER_DAY",
"WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY",
+ "WorkflowTaskAICreditsExhausted",
"WorkflowTaskConnectorsInvalid",
"WorkflowTaskLimitExceeded",
"WorkflowTaskOriginKeyConflict",
diff --git a/products/tasks/backend/logic/services/workflow_tasks.py b/products/tasks/backend/logic/services/workflow_tasks.py
index b5596639d241..8f5566794104 100644
--- a/products/tasks/backend/logic/services/workflow_tasks.py
+++ b/products/tasks/backend/logic/services/workflow_tasks.py
@@ -44,6 +44,8 @@ from products.tasks.backend.metrics import observe_workflow_task_create
from products.tasks.backend.models import Channel, Task, TaskRun
from products.tasks.backend.temporal.constants import WORKFLOW_RUN_IDLE_TIMEOUT_SECONDS
+from ee.billing.quota_limiting import QuotaResource, is_team_over_credit_budget
+
logger = structlog.get_logger(__name__)
ACTIVE_RUN_STATUSES = [TaskRun.Status.NOT_STARTED, TaskRun.Status.QUEUED, TaskRun.Status.IN_PROGRESS]
@@ -142,6 +144,10 @@ class WorkflowTaskUsageLimited(Exception):
pass
+class WorkflowTaskAICreditsExhausted(Exception):
+ pass
+
+
def create_workflow_task(
*,
team: Team,
@@ -172,7 +178,7 @@ def create_workflow_task(
with the whole project, `WorkflowTaskOwnerIneligible` when the owner lost access to the project,
and `WorkflowTaskLimitExceeded` when the workflow already has `max_parallel_tasks`
runs in flight. Also raises `WorkflowTaskUsageLimited` when the owner is over the
- AI usage limit, and `WorkflowTaskRateCapped` / `WorkflowTaskTeamRateCapped` when
+ AI usage limit, `WorkflowTaskAICreditsExhausted` when the team has used its AI credits, and `WorkflowTaskRateCapped` / `WorkflowTaskTeamRateCapped` when
the workflow or its team reached the daily created-task cap. A replayed
`origin_key` bypasses the gate and every cap. `rate_limits` can override either
product default for this project; a missing value keeps the default.
@@ -240,6 +246,12 @@ def create_workflow_task(
if usage_limit_response(gate_owner, team.id) is not None:
observe_workflow_task_create(reason="gate_blocked")
raise WorkflowTaskUsageLimited()
+ # Workflow runs bill to the AI credits bucket, which the gate above does not read. Both
+ # gateways refuse an exhausted bucket, and the legacy gateway answers with a retryable 429,
+ # so a run created here would retry until the idle timeout instead of failing.
+ if _team_out_of_ai_credits(team):
+ observe_workflow_task_create(reason="ai_credits_exhausted")
+ raise WorkflowTaskAICreditsExhausted()
# Resolved after the gate so a capped or blocked fire never pays for the query, and outside
# the transaction below so the skills store read never happens while holding the team lock.
@@ -410,6 +422,15 @@ def create_workflow_task(
return _task_dto(task, created=True)
+def _team_out_of_ai_credits(team: Team) -> bool:
+ """False (fail open) when the bucket can't be read, so a Redis error never blocks every fire."""
+ try:
+ return is_team_over_credit_budget(team.api_token, QuotaResource.AI_CREDITS)
+ except Exception:
+ logger.warning("workflow_task_ai_credits_check_failed", team_id=team.id, exc_info=True)
+ return False
+
+
def _resolve_channel(team_id: int, owner_id: int, channel_ref: str | None) -> Channel | None:
"""The space the task is filed into, or None to file it in no space.
diff --git a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
index 515a4dbfe0e3..764197cf8fe1 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
@@ -1370,10 +1370,9 @@ class TestWorkflowsMint:
assert env["AI_GATEWAY_TOKEN"] == "phe_abc"
mint.assert_called_once()
- def test_team_out_of_ai_credits_does_not_mint(self, mint_settings):
- env, mint = self._env(mint_settings, over_quota=True)
- assert "AI_GATEWAY_TOKEN" not in env
- mint.assert_not_called()
+ def test_team_out_of_ai_credits_fails_the_run_instead_of_degrading(self, mint_settings):
+ with pytest.raises(GatewayTokenUnavailableError, match="used its monthly PostHog AI credits"):
+ self._env(mint_settings, over_quota=True)
def test_credit_lookup_failure_does_not_mint(self, mint_settings):
with patch(_CREDIT_LOOKUP, side_effect=RuntimeError("billing is down")):
diff --git a/products/tasks/backend/temporal/process_task/utils.py b/products/tasks/backend/temporal/process_task/utils.py
index db99d113b672..b3ca5cbe21a6 100644
--- a/products/tasks/backend/temporal/process_task/utils.py
+++ b/products/tasks/backend/temporal/process_task/utils.py
@@ -1627,6 +1627,14 @@ def _record_pinned_gateway_product(run_id: str, state: dict | None, minted_produ
return True
+WORKFLOWS_PRODUCT = "workflows"
+WORKFLOWS_CREDITS_EXHAUSTED_REFUSAL = "ai_credits_exhausted"
+WORKFLOWS_CREDITS_EXHAUSTED_DETAIL = (
+ "This project has used its monthly PostHog AI credits. "
+ "Raise the AI credit limit in billing settings, or wait for the next billing period."
+)
+
+
def ai_gateway_env_vars(
*,
team_id: int | None = None,
@@ -1652,7 +1660,8 @@ def ai_gateway_env_vars(
Go gateway only when the token is present, so a missing token (mint failure,
or a caller that cannot supply run context) degrades the run to the Python
gateway, except a `posthog_code` run, which raises: the Python gateway
- refuses `posthog_code`.
+ refuses `posthog_code`. A `workflows` run whose team is out of AI credits
+ also raises, because the Python gateway refuses it too.
"""
ai_product = (
resolve_sandbox_ai_product(origin_product, ai_stage, internal=internal) if team_id is not None else None
@@ -1703,6 +1712,13 @@ def ai_gateway_env_vars(
)
if requires_go:
raise posthog_code_token_unavailable(refusal, team_id)
+ # The legacy gateway refuses an exhausted bucket with a retryable 429. A background
+ # workflow run has nobody to read the refusal, so the agent retries until the idle
+ # timeout and the run fails as lost. Fail it here with the real reason.
+ if ai_product == WORKFLOWS_PRODUCT and refusal == WORKFLOWS_CREDITS_EXHAUSTED_REFUSAL:
+ raise GatewayTokenUnavailableError(
+ WORKFLOWS_CREDITS_EXHAUSTED_DETAIL, {"team_id": team_id, "reason": refusal}
+ )
return env_vars
mint_kwargs: dict[str, Any] = {}
if ai_product == POSTHOG_CODE_PRODUCT:
diff --git a/products/tasks/backend/tests/test_workflow_tasks_api.py b/products/tasks/backend/tests/test_workflow_tasks_api.py
index 9c8d508aa9f8..c4eebf597a3f 100644
--- a/products/tasks/backend/tests/test_workflow_tasks_api.py
+++ b/products/tasks/backend/tests/test_workflow_tasks_api.py
@@ -443,14 +443,19 @@ class TestWorkflowTasksAPI(APIBaseTest):
assert response.status_code == status.HTTP_201_CREATED, response.json()
- @patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response")
- def test_rejects_creation_when_the_owner_is_over_the_usage_limit(self, usage_limit_response_mock) -> None:
- usage_limit_response_mock.return_value = object() # any non-None reply means blocked
-
- response = self._post()
+ @parameterized.expand(
+ [
+ # Any non-None reply from the Desktop usage gate means blocked.
+ ("owner_over_usage_limit", "usage_limit_response", object(), "usage limit"),
+ ("team_out_of_ai_credits", "is_team_over_credit_budget", True, "monthly PostHog AI credits"),
+ ]
+ )
+ def test_rejects_creation_when_usage_is_blocked(self, _name: str, gate: str, blocked: Any, detail: str) -> None:
+ with patch(f"products.tasks.backend.logic.services.workflow_tasks.{gate}", return_value=blocked):
+ response = self._post()
assert response.status_code == status.HTTP_409_CONFLICT, response.json()
- assert "usage limit" in response.json()["detail"]
+ assert detail in response.json()["detail"]
assert Task.objects.count() == 0
def test_replaying_an_idempotency_key_returns_the_existing_task(self) -> None:
diff --git a/products/workflows/backend/presentation/views/workflow_tasks.py b/products/workflows/backend/presentation/views/workflow_tasks.py
index f952e455e972..1ba85d55ceeb 100644
--- a/products/workflows/backend/presentation/views/workflow_tasks.py
+++ b/products/workflows/backend/presentation/views/workflow_tasks.py
@@ -14,6 +14,7 @@ from posthog.models.team.team import Team
from products.tasks.backend.facade.workflow_tasks import (
MAX_ATTACHED_SKILLS,
OUTPUT_FIELD_TYPES,
+ WorkflowTaskAICreditsExhausted,
WorkflowTaskConnectorsInvalid,
WorkflowTaskLimitExceeded,
WorkflowTaskOriginKeyConflict,
@@ -297,6 +298,17 @@ class WorkflowTaskViewSet(viewsets.GenericViewSet):
"Task creation resumes when the limit resets.",
status.HTTP_409_CONFLICT,
)
+ except WorkflowTaskAICreditsExhausted:
+ logger.info(
+ "workflow_task_create_ai_credits_exhausted",
+ team_id=team_id,
+ hog_flow_id=str(hog_flow_id),
+ )
+ return _rejected(
+ "This project has used its monthly PostHog AI credits, so no task was created. "
+ "Raise the AI credit limit in billing settings, or wait for the next billing period.",
+ status.HTTP_409_CONFLICT,
+ )
return Response(
WorkflowTaskResponseSerializer({"id": result.task_id, "run_id": result.run_id}).data,