Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): count sandbox run starts by endpoint and outcome (#119547)

Alessandro Pogliaghi2481d63c8aa2
diff --git a/products/tasks/backend/facade/agent_run_starts.py b/products/tasks/backend/facade/agent_run_starts.py
index 67cba3a871a1..f1f363e7e0cf 100644
--- a/products/tasks/backend/facade/agent_run_starts.py
+++ b/products/tasks/backend/facade/agent_run_starts.py
@@ -9,6 +9,7 @@ records its parent in its state, which ``run_task`` writes before the workflow s
from uuid import UUID
from products.tasks.backend.facade import contracts
+from products.tasks.backend.metrics import observe_sandbox_run_start
from products.tasks.backend.models import Task, TaskRun
__all__ = [
@@ -17,6 +18,7 @@ __all__ = [
"AGENT_RUN_FANOUT_CAP",
"AGENT_STARTED_RUN_COUNT_STATE_KEY",
"capture_sandbox_run_start_refused",
+ "observe_sandbox_run_start",
"release_agent_run_slot",
"reserve_agent_run_slot",
"resolve_sandbox_run_starter",
diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py
index 3b1c99a24859..79735a2ff2cd 100644
--- a/products/tasks/backend/metrics.py
+++ b/products/tasks/backend/metrics.py
@@ -73,6 +73,12 @@ TASK_RUN_WORKFLOW_START_TOTAL = Counter(
],
)
+SANDBOX_RUN_START_TOTAL = Counter(
+ "posthog_tasks_sandbox_run_start_total",
+ "Run starts requested by task sandboxes, by endpoint and outcome",
+ labelnames=["endpoint", "outcome"],
+)
+
TASK_RUN_DISPATCH_CALLBACK_TOTAL = Counter(
"posthog_tasks_task_run_dispatch_callback_total",
"on_commit workflow-dispatch callback lifecycle: 'scheduled' when registered, 'fired' when it runs. "
@@ -532,6 +538,10 @@ def observe_task_run_created(task_run: "TaskRun") -> None:
TASK_RUN_CREATED_TOTAL.labels(**_task_run_labels(task_run)).inc()
+def observe_sandbox_run_start(*, endpoint: str, outcome: str) -> None:
+ SANDBOX_RUN_START_TOTAL.labels(endpoint=endpoint, outcome=outcome).inc()
+
+
def observe_task_run_dispatch_callback(task_run: "TaskRun | None", *, phase: Literal["scheduled", "fired"]) -> None:
TASK_RUN_DISPATCH_CALLBACK_TOTAL.labels(**_task_run_labels(task_run), phase=phase).inc()
diff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.py
index f00c1731f423..d163d59818b4 100644
--- a/products/tasks/backend/presentation/views/api.py
+++ b/products/tasks/backend/presentation/views/api.py
@@ -63,6 +63,7 @@ from posthog.security.outbound_proxy import internal_requests_session
from posthog.temporal.oauth import SANDBOX_OAUTH_APP_CLIENT_IDS, TASK_AGENT_OAUTH_APP_CLIENT_IDS
from posthog.token_bucket import (
BucketDecision,
+ BucketUnavailable,
Budget,
consume as consume_token_bucket,
refund as refund_token_bucket,
@@ -951,8 +952,11 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
except tasks_facade.WarmRunActivationUnavailable as error:
return self._warm_activation_unavailable_response(error)
finally:
- if sandbox_starter is not None and not run_started:
- tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)
+ if sandbox_starter is not None:
+ if run_started:
+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint="create", outcome="started")
+ else:
+ tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)
self._forward_signals_discussion_note(request, task, relationship, discussion_question)
response_data = TaskSerializer(task).data
if run_error:
@@ -1584,8 +1588,11 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
except ReportTaskCapExceeded as error:
return self._report_task_cap_response(error.detail)
finally:
- if sandbox_starter is not None and not run_started:
- tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)
+ if sandbox_starter is not None:
+ if run_started:
+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint="run", outcome="started")
+ else:
+ tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)
if result is None:
raise NotFound()
if result.error is not None:
@@ -1634,6 +1641,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
tasks_agent_run_starts.capture_sandbox_run_start_refused(
self.team_id, sandbox_task_id=sandbox_task_id, reason=starter.value, endpoint=endpoint
)
+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint=endpoint, outcome=starter.value)
return _sandbox_run_start_denied_response(starter)
return starter
@@ -1641,9 +1649,13 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
self, request, starter: tasks_contracts.SandboxRunStarter, *, endpoint: str
) -> Response | None:
charged: list[tuple[str, Budget]] = []
+ bucket_unavailable = False
for scope, budget in SANDBOX_AGENT_RUN_START_BUDGETS:
key = f"{scope}:{request.user.id}"
decision = consume_token_bucket(key, budget)
+ if isinstance(decision, BucketUnavailable) and not bucket_unavailable:
+ bucket_unavailable = True
+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint=endpoint, outcome="bucket_unavailable")
# Fail open when Redis cannot answer, because the fan-out cap below is the durable bound.
if isinstance(decision, BucketDecision) and not decision.allowed:
for charged_key, charged_budget in charged:
@@ -1671,6 +1683,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
tasks_agent_run_starts.capture_sandbox_run_start_refused(
self.team_id, sandbox_task_id=starter.parent_task_id, reason=code, endpoint=endpoint
)
+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint=endpoint, outcome=code)
return _agent_run_rate_limited_response(code, detail)
@staticmethod
diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py
index a400e8c1589e..2363e9bf082a 100644
--- a/products/tasks/backend/tests/test_api.py
+++ b/products/tasks/backend/tests/test_api.py
@@ -25,6 +25,7 @@ from django.utils.html import escape
import jwt
import posthoganalytics
from parameterized import parameterized
+from prometheus_client import REGISTRY
from rest_framework import status
from rest_framework.exceptions import APIException
from rest_framework.response import Response
@@ -49,7 +50,7 @@ from posthog.temporal.oauth import (
POSTHOG_AI_APP_CLIENT_ID_DEV,
POSTHOG_DESKTOP_MOBILE_APP_CLIENT_ID_US,
)
-from posthog.token_bucket import BucketDecision
+from posthog.token_bucket import BucketDecision, BucketUnavailable
from posthog.utils import absolute_uri
from products.posthog_ai.backend.models.assistant import Conversation
@@ -2406,6 +2407,10 @@ class TestTaskAPI(BaseTaskAPITest):
send_feature_flag_events=False,
)
+ def _sandbox_start_count(self, endpoint: str, outcome: str) -> float:
+ labels = {"endpoint": endpoint, "outcome": outcome}
+ return REGISTRY.get_sample_value("posthog_tasks_sandbox_run_start_total", labels) or 0.0
+
def _sandbox_start(self, client: APIClient, path: str, target: Task | None = None, target_id: str | None = None):
if path == "create":
return client.post(
@@ -2427,10 +2432,12 @@ class TestTaskAPI(BaseTaskAPITest):
parent_run = parent.create_run(environment=TaskRun.Environment.CLOUD)
client = self._sandbox_oauth_client(parent.id, internal_scope=True)
target = self.create_task(title="Existing task")
+ started_before = self._sandbox_start_count(path, "started")
response = self._sandbox_start(client, path, target)
self.assertIn(response.status_code, (status.HTTP_200_OK, status.HTTP_201_CREATED), response.json())
+ self.assertEqual(self._sandbox_start_count(path, "started"), started_before + 1)
child = Task.objects.get(id=response.json()["id"])
run = child.runs.get()
state = run.state or {}
@@ -2480,20 +2487,22 @@ class TestTaskAPI(BaseTaskAPITest):
target = parent if case.startswith("own_task") else self.create_task(title="Existing task")
task_count = Task.objects.count()
target_run_count = target.runs.count()
-
- response = self._sandbox_start(
- client, path, target, target_id=str(target.id).upper() if case == "own_task_uppercase" else None
- )
-
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
expected_code = {
"nested": "nested",
"own_task": "nested",
"own_task_uppercase": "nested",
"not_internal": None,
}.get(case, "no_actor")
+ refused_before = self._sandbox_start_count(path, expected_code) if expected_code else 0.0
+
+ response = self._sandbox_start(
+ client, path, target, target_id=str(target.id).upper() if case == "own_task_uppercase" else None
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
if expected_code is not None:
self.assertEqual(response.json()["code"], expected_code)
+ self.assertEqual(self._sandbox_start_count(path, expected_code), refused_before + 1)
self.assertEqual(Task.objects.count(), task_count)
self.assertEqual(target.runs.count(), target_run_count)
mock_workflow.assert_not_called()
@@ -2511,6 +2520,7 @@ class TestTaskAPI(BaseTaskAPITest):
client = self._sandbox_oauth_client(parent.id, internal_scope=True)
target = self.create_task(title="Existing task")
task_count = Task.objects.count()
+ limited_before = self._sandbox_start_count(path, limit)
with patch(
"products.tasks.backend.presentation.views.api.consume_token_bucket",
@@ -2522,6 +2532,7 @@ class TestTaskAPI(BaseTaskAPITest):
self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
self.assertEqual(response.json()["code"], limit)
+ self.assertEqual(self._sandbox_start_count(path, limit), limited_before + 1)
self.assertEqual(Task.objects.count(), task_count)
self.assertFalse(target.runs.exists())
mock_workflow.assert_not_called()
@@ -2529,6 +2540,27 @@ class TestTaskAPI(BaseTaskAPITest):
expected_count = 10 if limit == "agent_run_fanout_cap" else None
self.assertEqual((parent_run.state or {}).get("agent_started_run_count"), expected_count)
+ @patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)
+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
+ def test_sandbox_oauth_unavailable_buckets_fail_open_and_count_once(self, mock_workflow, _mock_internal_team):
+ parent = self.create_task()
+ parent.create_run(environment=TaskRun.Environment.CLOUD)
+ client = self._sandbox_oauth_client(parent.id, internal_scope=True)
+ target = self.create_task(title="Existing task")
+ unavailable_before = self._sandbox_start_count("run", "bucket_unavailable")
+ started_before = self._sandbox_start_count("run", "started")
+
+ with patch(
+ "products.tasks.backend.presentation.views.api.consume_token_bucket",
+ return_value=BucketUnavailable(error="redis down"),
+ ):
+ response = self._sandbox_start(client, "run", target)
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertEqual(self._sandbox_start_count("run", "bucket_unavailable"), unavailable_before + 1)
+ self.assertEqual(self._sandbox_start_count("run", "started"), started_before + 1)
+ mock_workflow.assert_called_once()
+
@patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)
def test_sandbox_oauth_failed_run_start_gives_back_its_fanout_slot(self, _mock_internal_team):
parent = self.create_task()