:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): count every terminal task run status in a metric (#119417)
Harley Alexanderb02515729e2c
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.pyindex ac04ffb47348..56532606cbd1 100644--- a/products/tasks/backend/facade/api.py+++ b/products/tasks/backend/facade/api.py@@ -2344,6 +2344,7 @@ def complete_idle_local_task_run(run_id: str | UUID) -> bool:run = TaskRun.objects.filter(pk=run_id).first() # nosemgrep: celery-task-team-scope-auditif run is not None:run.mark_completed(notify=False, analytics_properties={"finalized_by": "stale_local_queued_sweep"})+ _observe_claimed_terminal(run, error_type=None)return True@@ -2364,10 +2365,21 @@ def claim_and_fail_stale_run(run_id: str | UUID, error: str, error_type: str | Nrun = TaskRun.objects.filter(pk=run_id).first() # nosemgrep: celery-task-team-scope-auditif run is not None:run.mark_failed(error, error_type=error_type)+ _observe_claimed_terminal(run, error_type=error_type)resume_workflow_step_for_run(run)return True+def _observe_claimed_terminal(run: TaskRun, *, error_type: str | None) -> None:+ # The conditional update already wrote the terminal status, so mark_* sees a terminal run and+ # skips its own count. The claim winner counts the transition here instead.+ from products.tasks.backend.metrics import ( # noqa: PLC0415 — keep prometheus deps off the api import path+ observe_task_run_terminal,+ )++ observe_task_run_terminal(run, error_type=error_type, termination_reason=None)++def redispatch_task_run(run_id: str | UUID) -> str:"""Re-dispatch a QUEUED run whose create-time workflow dispatch was lost. Cross-team janitor call.@@ -3467,6 +3479,7 @@ def update_task_run(from products.tasks.backend.metrics import ( # noqa: PLC0415 — keep prometheus deps off the api import pathobserve_agent_turn_failed,observe_prewarmed_unused_if_never_activated,+ observe_task_run_terminal,observe_wizard_run_unbound,)@@ -3607,6 +3620,13 @@ def update_task_run(run.publish_stream_state_event()if new_status in _TERMINAL_TASK_RUN_STATUSES and old_status != new_status:+ # This PATCH performed the transition, so the status-update activity skips its count.+ observe_task_run_terminal(+ run,+ error_type=validated_data.get("error_type")+ or ("agent_reported" if new_status == TaskRun.Status.FAILED else None),+ termination_reason=None,+ )try:if run.environment == TaskRun.Environment.CLOUD:refresh_task_run_cost(run_id=run.id, team_id=run.team_id)diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.pyindex 79735a2ff2cd..2ff13f06ead2 100644--- a/products/tasks/backend/metrics.py+++ b/products/tasks/backend/metrics.py@@ -196,6 +196,14 @@ TASK_RUN_FAILED_TOTAL = Counter(],)+# Counted once per run on the terminal status write. TASK_RUN_FAILED_TOTAL only sees failures+# that raise, so it misses runs the workflow fails directly, such as an agent lost mid-turn.+TASK_RUN_TERMINAL_TOTAL = Counter(+ "posthog_tasks_task_run_terminal_total",+ "TaskRun terminal status transitions",+ labelnames=["origin_product", "run_environment", "status", "error_type", "termination_reason"],+)+CUSTOM_IMAGE_BUILD_TOTAL = Counter("posthog_tasks_custom_image_build_total","Custom sandbox image build lifecycle events",@@ -707,6 +715,17 @@ def observe_agent_turn_failed(task_run: "TaskRun") -> None:_WIZARD_TERMINAL_STATUSES = {"completed", "failed", "cancelled"}+def observe_task_run_terminal(task_run: "TaskRun", *, error_type: str | None, termination_reason: str | None) -> None:+ labels = _task_run_labels(task_run)+ TASK_RUN_TERMINAL_TOTAL.labels(+ origin_product=labels["origin_product"],+ run_environment=labels["run_environment"],+ status=_metric_label(task_run.status),+ error_type=_failure_metric_label(error_type),+ termination_reason=_metric_label(termination_reason),+ ).inc()++def observe_wizard_run_unbound(task_run: "TaskRun") -> None:"""Record a wizard run ending without its PR ever binding.diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.pyindex 26fe5fdc1ab5..35cec71039bc 100644--- a/products/tasks/backend/models.py+++ b/products/tasks/backend/models.py@@ -59,7 +59,11 @@ from products.tasks.backend.feature_flags import ()from products.tasks.backend.logic.model_access import resolve_model_accessfrom products.tasks.backend.logic.stream.redis_stream import publish_task_run_stream_event-from products.tasks.backend.metrics import observe_task_run_created, observe_task_run_dispatch_callback+from products.tasks.backend.metrics import (+ observe_task_run_created,+ observe_task_run_dispatch_callback,+ observe_task_run_terminal,+)from products.tasks.backend.pr_urls import read_pr_urlsfrom products.tasks.backend.redis import evaluate_dedicated_stream_flag, run_uses_dedicated_streamfrom products.tasks.backend.storage import append_jsonl_object@@ -3032,6 +3036,21 @@ class TaskRun(models.Model):return round((self.completed_at - self.created_at).total_seconds(), 1)return 0.0+ def _save_terminal_status(self, update_fields: list[str]) -> bool:+ """Save a terminal status and return whether this write moved the row out of a live status.++ Reads the status under the row lock that the run PATCH and the status activity also take,+ so a concurrent terminal write cannot make two writers each report the transition.+ """+ with transaction.atomic():+ # Match start_task_run's lock order: saving the run also bumps its task's activity.+ Task.objects.select_for_update().get(pk=self.task_id)+ stored_status = (+ TaskRun.objects.select_for_update().filter(pk=self.pk).values_list("status", flat=True).first()+ )+ self.save(update_fields=update_fields)+ return stored_status not in {self.Status.COMPLETED, self.Status.FAILED, self.Status.CANCELLED}+def mark_completed(self, *, notify: bool = True, analytics_properties: dict | None = None) -> None:"""Mark the progress as completed.@@ -3042,7 +3061,8 @@ class TaskRun(models.Model):"""self.status = self.Status.COMPLETEDself.completed_at = django_timezone.now()- self.save(update_fields=["status", "completed_at"])+ if self._save_terminal_status(["status", "completed_at"]):+ observe_task_run_terminal(self, error_type=None, termination_reason=None)self.publish_stream_state_event()self.capture_event("task_run_completed",@@ -3073,7 +3093,8 @@ class TaskRun(models.Model):self.status = self.Status.FAILEDself.error_message = errorself.completed_at = django_timezone.now()- self.save(update_fields=["status", "error_message", "completed_at"])+ if self._save_terminal_status(["status", "error_message", "completed_at"]):+ observe_task_run_terminal(self, error_type=error_type, termination_reason=None)self.publish_stream_state_event()self.capture_event("task_run_failed",diff --git a/products/tasks/backend/temporal/client.py b/products/tasks/backend/temporal/client.pyindex 92916d550604..9db34dc419b3 100644--- a/products/tasks/backend/temporal/client.py+++ b/products/tasks/backend/temporal/client.py@@ -41,7 +41,11 @@ from products.tasks.backend.logic.stream.redis_stream import (TASK_RUN_STREAM_INITIAL_GENERATION,get_task_run_stream_generation,)-from products.tasks.backend.metrics import AGENT_OTEL_TELEMETRY_STAMPED_TOTAL, observe_task_run_workflow_start+from products.tasks.backend.metrics import (+ AGENT_OTEL_TELEMETRY_STAMPED_TOTAL,+ observe_task_run_terminal,+ observe_task_run_workflow_start,+)from products.tasks.backend.models import TaskRunfrom products.tasks.backend.redis import run_uses_dedicated_streamfrom products.tasks.backend.temporal.bake_dev_stack_image.workflow import BakeDevStackImageInput@@ -125,6 +129,7 @@ def _terminalize_unstarted_task_run(run_id: str, error_message: str) -> bool:logger.warning("task_processing_start_failure_task_run_missing", extra={"run_id": run_id})return False+ observe_task_run_terminal(task_run, error_type="workflow_start_failed", termination_reason=None)task_run.publish_stream_state_event()task_run.capture_event("task_run_failed",diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.pyindex 8c8d4b083543..cbf11d719246 100644--- a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py@@ -621,6 +621,33 @@ class TestUpdateTaskRunStatusActivity:after = REGISTRY.get_sample_value("posthog_tasks_wizard_run_unbound_total", labels) or 0.0assert after == before + expected_delta+ @pytest.mark.django_db(transaction=True)+ def test_terminal_transition_is_counted_once_with_its_failure_reason(self, activity_environment, test_task_run):+ from prometheus_client import REGISTRY++ test_task_run.task.origin_product = Task.OriginProduct.WORKFLOW+ test_task_run.task.save(update_fields=["origin_product"])+ labels = {+ "origin_product": "workflow",+ "run_environment": test_task_run.environment,+ "status": "failed",+ "error_type": "unknown",+ "termination_reason": "timed_out_inactivity",+ }+ before = REGISTRY.get_sample_value("posthog_tasks_task_run_terminal_total", labels) or 0.0++ input_data = UpdateTaskRunStatusInput(+ run_id=str(test_task_run.id),+ status=TaskRun.Status.FAILED,+ error_message="The agent stopped before finishing its turn",+ timed_out_inactivity=True,+ )+ async_to_sync(activity_environment.run)(update_task_run_status, input_data)+ async_to_sync(activity_environment.run)(update_task_run_status, input_data)++ after = REGISTRY.get_sample_value("posthog_tasks_task_run_terminal_total", labels) or 0.0+ assert after == before + 1+class TestRecordRunTokenUsageMetrics:def _record(self, activity_environment, **overrides):diff --git a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.pyindex 16f3ff103ed8..ffbe1147acbd 100644--- a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py+++ b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py@@ -15,7 +15,11 @@ from products.tasks.backend.constants import TIMED_OUT_INACTIVITY_STATE_KEYfrom products.tasks.backend.error_telemetry import truncate_error_messagefrom products.tasks.backend.logic.services.gateway_usage import refresh_task_run_costfrom products.tasks.backend.logic.services.workflow_step_resume import resume_workflow_step_for_run-from products.tasks.backend.metrics import observe_prewarmed_unused_if_never_activated, observe_wizard_run_unbound+from products.tasks.backend.metrics import (+ observe_prewarmed_unused_if_never_activated,+ observe_task_run_terminal,+ observe_wizard_run_unbound,+)from products.tasks.backend.models import Task, TaskRunfrom products.tasks.backend.temporal.metrics import record_run_token_usagefrom products.tasks.backend.temporal.observability import log_with_activity_context@@ -124,6 +128,15 @@ def update_task_run_status(input: UpdateTaskRunStatusInput) -> None:if input.status == TaskRun.Status.FAILED:input = replace(input, error_type=(task_run.state or {}).get("failure_error_type") or input.error_type)+ # Counted before the fallible side effects below. A retry after one of them raises sees the+ # row already terminal and skips this block, so a later count would be lost.+ if input.status in _TERMINAL_STATUSES and old_status != input.status:+ observe_task_run_terminal(+ task_run,+ error_type=input.error_type,+ termination_reason=marker if marker in _TERMINAL_STATE_MARKERS else None,+ )+# Side effects run after commit, outside the row lock (repo convention: no side effects in atomic).if input.status in _TERMINAL_STATUSES:try:diff --git a/products/tasks/backend/tests/test_task_run_metrics.py b/products/tasks/backend/tests/test_task_run_metrics.pyindex b9e0ebf63933..a8853a8f5e0a 100644--- a/products/tasks/backend/tests/test_task_run_metrics.py+++ b/products/tasks/backend/tests/test_task_run_metrics.py@@ -4,7 +4,9 @@ from datetime import timedeltaimport pytestfrom unittest.mock import AsyncMock, MagicMock, patch+from django.db import connectionfrom django.test import TestCase+from django.test.utils import CaptureQueriesContextimport posthoganalyticsfrom parameterized import parameterized@@ -342,6 +344,89 @@ class TestTaskRunMetrics(TestCase):assert _sample_value("posthog_tasks_prewarmed_unused_total", labels) == before + 1+ @parameterized.expand(+ [+ ("mark_failed", "failed", "janitor_timeout"),+ ("mark_completed", "completed", "unknown"),+ ("dispatch_failure", "failed", "workflow_start_failed"),+ ("claim_and_fail_stale_run", "failed", "sandbox_lost"),+ ]+ )+ def test_writers_outside_the_status_activity_count_the_terminal_transition_once(+ self, writer: str, status: str, error_type: str+ ) -> None:+ from products.tasks.backend.facade import api as facade+ from products.tasks.backend.temporal import client++ run = self.task.create_run(environment=TaskRun.Environment.CLOUD)+ labels = {+ "origin_product": "user_created",+ "run_environment": "cloud",+ "status": status,+ "error_type": error_type,+ "termination_reason": "unknown",+ }+ before = _sample_value("posthog_tasks_task_run_terminal_total", labels)++ with (+ patch("products.tasks.backend.push_dispatcher.notify_task_run_failed"),+ patch.object(client, "resume_workflow_step_for_run"),+ patch.object(facade, "resume_workflow_step_for_run"),+ ):+ for _ in range(2): # a repeat write on a terminal run must not count again+ run.refresh_from_db()+ if writer == "mark_failed":+ run.mark_failed("stuck", error_type="janitor_timeout")+ elif writer == "mark_completed":+ run.mark_completed(notify=False)+ elif writer == "claim_and_fail_stale_run":+ facade.claim_and_fail_stale_run(run.id, "worker lost", error_type="sandbox_lost")+ else:+ client._terminalize_unstarted_task_run(str(run.id), "start failed")++ assert _sample_value("posthog_tasks_task_run_terminal_total", labels) == before + 1++ def test_terminal_save_locks_task_before_run(self) -> None:+ run = self.task.create_run(environment=TaskRun.Environment.CLOUD)+ run.status = TaskRun.Status.COMPLETED++ with CaptureQueriesContext(connection) as queries:+ assert run._save_terminal_status(["status"])++ locks = [query["sql"] for query in queries if "FOR UPDATE" in query["sql"]]+ assert len(locks) == 2+ assert f'FROM "{Task._meta.db_table}"' in locks[0]+ assert f'FROM "{TaskRun._meta.db_table}"' in locks[1]++ @parameterized.expand([("mark_failed",), ("mark_completed",)])+ def test_finalizer_on_a_stale_instance_does_not_count_a_run_another_writer_ended(self, writer: str) -> None:+ def terminal_total() -> float:+ return sum(+ _sample_value(+ "posthog_tasks_task_run_terminal_total",+ {+ "origin_product": "user_created",+ "run_environment": "cloud",+ "status": status,+ "error_type": error_type,+ "termination_reason": "unknown",+ },+ )+ for status, error_type in (("failed", "janitor_timeout"), ("completed", "unknown"))+ )++ run = self.task.create_run(environment=TaskRun.Environment.CLOUD)+ TaskRun.objects.filter(pk=run.pk).update(status=TaskRun.Status.CANCELLED)+ before = terminal_total()++ with patch("products.tasks.backend.push_dispatcher.notify_task_run_failed"):+ if writer == "mark_failed":+ run.mark_failed("stuck", error_type="janitor_timeout")+ else:+ run.mark_completed(notify=False)++ assert terminal_total() == before+def test_activating_run_without_prewarmed_marker_does_not_increment(self) -> None:# Warm Runs provisioned before this ships have await_user_message but no prewarmed marker;# activating them must not inflate the numerator (would push hit rate above 1).@@ -530,11 +615,13 @@ class TestTaskRunMetrics(TestCase):@parameterized.expand([- ("failed", 1.0),- ("completed", 0.0),+ ("failed", 1.0, "agent_reported"),+ ("completed", 0.0, "unknown"),])- def test_agent_turn_failure_counter_only_on_failed_transition(self, new_status: str, expected_delta: float) -> None:+ def test_agent_turn_failure_counter_only_on_failed_transition(+ self, new_status: str, expected_delta: float, terminal_error_type: str+ ) -> None:from products.tasks.backend.facade import api as facaderun = self.task.create_run(@@ -549,17 +636,27 @@ class TestTaskRunMetrics(TestCase):"task_runtime": "acp","runtime_adapter": "codex",}+ terminal_labels = {+ "origin_product": "user_created",+ "run_environment": "cloud",+ "status": new_status,+ "error_type": terminal_error_type,+ "termination_reason": "unknown",+ }before = _sample_value("posthog_tasks_agent_turn_failed_total", labels)+ terminal_before = _sample_value("posthog_tasks_task_run_terminal_total", terminal_labels)with patch.object(facade, "signal_workflow_completion"):- facade.update_task_run(- run.id,- self.task.id,- self.team.id,- validated_data={"status": new_status, "error_message": "boom"},- )+ for _ in range(2): # the repeat PATCH must not double count+ facade.update_task_run(+ run.id,+ self.task.id,+ self.team.id,+ validated_data={"status": new_status, "error_message": "boom"},+ )assert _sample_value("posthog_tasks_agent_turn_failed_total", labels) == before + expected_delta+ assert _sample_value("posthog_tasks_task_run_terminal_total", terminal_labels) == terminal_before + 1@parameterized.expand([