:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): count sandbox run starts by endpoint and outcome (#119547)
Alessandro Pogliaghi2481d63c8aa2
diff --git a/products/tasks/backend/facade/agent_run_starts.py b/products/tasks/backend/facade/agent_run_starts.pyindex 67cba3a871a1..f1f363e7e0cf 100644--- a/products/tasks/backend/facade/agent_run_starts.py+++ b/products/tasks/backend/facade/agent_run_starts.py@@ -9,6 +9,7 @@ records its parent in its state, which ``run_task`` writes before the workflow sfrom uuid import UUIDfrom products.tasks.backend.facade import contracts+from products.tasks.backend.metrics import observe_sandbox_run_startfrom products.tasks.backend.models import Task, TaskRun__all__ = [@@ -17,6 +18,7 @@ __all__ = ["AGENT_RUN_FANOUT_CAP","AGENT_STARTED_RUN_COUNT_STATE_KEY","capture_sandbox_run_start_refused",+ "observe_sandbox_run_start","release_agent_run_slot","reserve_agent_run_slot","resolve_sandbox_run_starter",diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.pyindex 3b1c99a24859..79735a2ff2cd 100644--- a/products/tasks/backend/metrics.py+++ b/products/tasks/backend/metrics.py@@ -73,6 +73,12 @@ TASK_RUN_WORKFLOW_START_TOTAL = Counter(],)+SANDBOX_RUN_START_TOTAL = Counter(+ "posthog_tasks_sandbox_run_start_total",+ "Run starts requested by task sandboxes, by endpoint and outcome",+ labelnames=["endpoint", "outcome"],+)+TASK_RUN_DISPATCH_CALLBACK_TOTAL = Counter("posthog_tasks_task_run_dispatch_callback_total","on_commit workflow-dispatch callback lifecycle: 'scheduled' when registered, 'fired' when it runs. "@@ -532,6 +538,10 @@ def observe_task_run_created(task_run: "TaskRun") -> None:TASK_RUN_CREATED_TOTAL.labels(**_task_run_labels(task_run)).inc()+def observe_sandbox_run_start(*, endpoint: str, outcome: str) -> None:+ SANDBOX_RUN_START_TOTAL.labels(endpoint=endpoint, outcome=outcome).inc()++def observe_task_run_dispatch_callback(task_run: "TaskRun | None", *, phase: Literal["scheduled", "fired"]) -> None:TASK_RUN_DISPATCH_CALLBACK_TOTAL.labels(**_task_run_labels(task_run), phase=phase).inc()diff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.pyindex f00c1731f423..d163d59818b4 100644--- a/products/tasks/backend/presentation/views/api.py+++ b/products/tasks/backend/presentation/views/api.py@@ -63,6 +63,7 @@ from posthog.security.outbound_proxy import internal_requests_sessionfrom posthog.temporal.oauth import SANDBOX_OAUTH_APP_CLIENT_IDS, TASK_AGENT_OAUTH_APP_CLIENT_IDSfrom posthog.token_bucket import (BucketDecision,+ BucketUnavailable,Budget,consume as consume_token_bucket,refund as refund_token_bucket,@@ -951,8 +952,11 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):except tasks_facade.WarmRunActivationUnavailable as error:return self._warm_activation_unavailable_response(error)finally:- if sandbox_starter is not None and not run_started:- tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)+ if sandbox_starter is not None:+ if run_started:+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint="create", outcome="started")+ else:+ tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)self._forward_signals_discussion_note(request, task, relationship, discussion_question)response_data = TaskSerializer(task).dataif run_error:@@ -1584,8 +1588,11 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):except ReportTaskCapExceeded as error:return self._report_task_cap_response(error.detail)finally:- if sandbox_starter is not None and not run_started:- tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)+ if sandbox_starter is not None:+ if run_started:+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint="run", outcome="started")+ else:+ tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)if result is None:raise NotFound()if result.error is not None:@@ -1634,6 +1641,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):tasks_agent_run_starts.capture_sandbox_run_start_refused(self.team_id, sandbox_task_id=sandbox_task_id, reason=starter.value, endpoint=endpoint)+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint=endpoint, outcome=starter.value)return _sandbox_run_start_denied_response(starter)return starter@@ -1641,9 +1649,13 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):self, request, starter: tasks_contracts.SandboxRunStarter, *, endpoint: str) -> Response | None:charged: list[tuple[str, Budget]] = []+ bucket_unavailable = Falsefor scope, budget in SANDBOX_AGENT_RUN_START_BUDGETS:key = f"{scope}:{request.user.id}"decision = consume_token_bucket(key, budget)+ if isinstance(decision, BucketUnavailable) and not bucket_unavailable:+ bucket_unavailable = True+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint=endpoint, outcome="bucket_unavailable")# Fail open when Redis cannot answer, because the fan-out cap below is the durable bound.if isinstance(decision, BucketDecision) and not decision.allowed:for charged_key, charged_budget in charged:@@ -1671,6 +1683,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):tasks_agent_run_starts.capture_sandbox_run_start_refused(self.team_id, sandbox_task_id=starter.parent_task_id, reason=code, endpoint=endpoint)+ tasks_agent_run_starts.observe_sandbox_run_start(endpoint=endpoint, outcome=code)return _agent_run_rate_limited_response(code, detail)@staticmethoddiff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.pyindex a400e8c1589e..2363e9bf082a 100644--- a/products/tasks/backend/tests/test_api.py+++ b/products/tasks/backend/tests/test_api.py@@ -25,6 +25,7 @@ from django.utils.html import escapeimport jwtimport posthoganalyticsfrom parameterized import parameterized+from prometheus_client import REGISTRYfrom rest_framework import statusfrom rest_framework.exceptions import APIExceptionfrom rest_framework.response import Response@@ -49,7 +50,7 @@ from posthog.temporal.oauth import (POSTHOG_AI_APP_CLIENT_ID_DEV,POSTHOG_DESKTOP_MOBILE_APP_CLIENT_ID_US,)-from posthog.token_bucket import BucketDecision+from posthog.token_bucket import BucketDecision, BucketUnavailablefrom posthog.utils import absolute_urifrom products.posthog_ai.backend.models.assistant import Conversation@@ -2406,6 +2407,10 @@ class TestTaskAPI(BaseTaskAPITest):send_feature_flag_events=False,)+ def _sandbox_start_count(self, endpoint: str, outcome: str) -> float:+ labels = {"endpoint": endpoint, "outcome": outcome}+ return REGISTRY.get_sample_value("posthog_tasks_sandbox_run_start_total", labels) or 0.0+def _sandbox_start(self, client: APIClient, path: str, target: Task | None = None, target_id: str | None = None):if path == "create":return client.post(@@ -2427,10 +2432,12 @@ class TestTaskAPI(BaseTaskAPITest):parent_run = parent.create_run(environment=TaskRun.Environment.CLOUD)client = self._sandbox_oauth_client(parent.id, internal_scope=True)target = self.create_task(title="Existing task")+ started_before = self._sandbox_start_count(path, "started")response = self._sandbox_start(client, path, target)self.assertIn(response.status_code, (status.HTTP_200_OK, status.HTTP_201_CREATED), response.json())+ self.assertEqual(self._sandbox_start_count(path, "started"), started_before + 1)child = Task.objects.get(id=response.json()["id"])run = child.runs.get()state = run.state or {}@@ -2480,20 +2487,22 @@ class TestTaskAPI(BaseTaskAPITest):target = parent if case.startswith("own_task") else self.create_task(title="Existing task")task_count = Task.objects.count()target_run_count = target.runs.count()-- response = self._sandbox_start(- client, path, target, target_id=str(target.id).upper() if case == "own_task_uppercase" else None- )-- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)expected_code = {"nested": "nested","own_task": "nested","own_task_uppercase": "nested","not_internal": None,}.get(case, "no_actor")+ refused_before = self._sandbox_start_count(path, expected_code) if expected_code else 0.0++ response = self._sandbox_start(+ client, path, target, target_id=str(target.id).upper() if case == "own_task_uppercase" else None+ )++ self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)if expected_code is not None:self.assertEqual(response.json()["code"], expected_code)+ self.assertEqual(self._sandbox_start_count(path, expected_code), refused_before + 1)self.assertEqual(Task.objects.count(), task_count)self.assertEqual(target.runs.count(), target_run_count)mock_workflow.assert_not_called()@@ -2511,6 +2520,7 @@ class TestTaskAPI(BaseTaskAPITest):client = self._sandbox_oauth_client(parent.id, internal_scope=True)target = self.create_task(title="Existing task")task_count = Task.objects.count()+ limited_before = self._sandbox_start_count(path, limit)with patch("products.tasks.backend.presentation.views.api.consume_token_bucket",@@ -2522,6 +2532,7 @@ class TestTaskAPI(BaseTaskAPITest):self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)self.assertEqual(response.json()["code"], limit)+ self.assertEqual(self._sandbox_start_count(path, limit), limited_before + 1)self.assertEqual(Task.objects.count(), task_count)self.assertFalse(target.runs.exists())mock_workflow.assert_not_called()@@ -2529,6 +2540,27 @@ class TestTaskAPI(BaseTaskAPITest):expected_count = 10 if limit == "agent_run_fanout_cap" else Noneself.assertEqual((parent_run.state or {}).get("agent_started_run_count"), expected_count)+ @patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")+ def test_sandbox_oauth_unavailable_buckets_fail_open_and_count_once(self, mock_workflow, _mock_internal_team):+ parent = self.create_task()+ parent.create_run(environment=TaskRun.Environment.CLOUD)+ client = self._sandbox_oauth_client(parent.id, internal_scope=True)+ target = self.create_task(title="Existing task")+ unavailable_before = self._sandbox_start_count("run", "bucket_unavailable")+ started_before = self._sandbox_start_count("run", "started")++ with patch(+ "products.tasks.backend.presentation.views.api.consume_token_bucket",+ return_value=BucketUnavailable(error="redis down"),+ ):+ response = self._sandbox_start(client, "run", target)++ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(self._sandbox_start_count("run", "bucket_unavailable"), unavailable_before + 1)+ self.assertEqual(self._sandbox_start_count("run", "started"), started_before + 1)+ mock_workflow.assert_called_once()+@patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)def test_sandbox_oauth_failed_run_start_gives_back_its_fanout_slot(self, _mock_internal_team):parent = self.create_task()