Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): count every terminal task run status in a metric (#119417)

Harley Alexanderb02515729e2c
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index ac04ffb47348..56532606cbd1 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -2344,6 +2344,7 @@ def complete_idle_local_task_run(run_id: str | UUID) -> bool:
run = TaskRun.objects.filter(pk=run_id).first() # nosemgrep: celery-task-team-scope-audit
if run is not None:
run.mark_completed(notify=False, analytics_properties={"finalized_by": "stale_local_queued_sweep"})
+ _observe_claimed_terminal(run, error_type=None)
return True
@@ -2364,10 +2365,21 @@ def claim_and_fail_stale_run(run_id: str | UUID, error: str, error_type: str | N
run = TaskRun.objects.filter(pk=run_id).first() # nosemgrep: celery-task-team-scope-audit
if run is not None:
run.mark_failed(error, error_type=error_type)
+ _observe_claimed_terminal(run, error_type=error_type)
resume_workflow_step_for_run(run)
return True
+def _observe_claimed_terminal(run: TaskRun, *, error_type: str | None) -> None:
+ # The conditional update already wrote the terminal status, so mark_* sees a terminal run and
+ # skips its own count. The claim winner counts the transition here instead.
+ from products.tasks.backend.metrics import ( # noqa: PLC0415 — keep prometheus deps off the api import path
+ observe_task_run_terminal,
+ )
+
+ observe_task_run_terminal(run, error_type=error_type, termination_reason=None)
+
+
def redispatch_task_run(run_id: str | UUID) -> str:
"""Re-dispatch a QUEUED run whose create-time workflow dispatch was lost. Cross-team janitor call.
@@ -3467,6 +3479,7 @@ def update_task_run(
from products.tasks.backend.metrics import ( # noqa: PLC0415 — keep prometheus deps off the api import path
observe_agent_turn_failed,
observe_prewarmed_unused_if_never_activated,
+ observe_task_run_terminal,
observe_wizard_run_unbound,
)
@@ -3607,6 +3620,13 @@ def update_task_run(
run.publish_stream_state_event()
if new_status in _TERMINAL_TASK_RUN_STATUSES and old_status != new_status:
+ # This PATCH performed the transition, so the status-update activity skips its count.
+ observe_task_run_terminal(
+ run,
+ error_type=validated_data.get("error_type")
+ or ("agent_reported" if new_status == TaskRun.Status.FAILED else None),
+ termination_reason=None,
+ )
try:
if run.environment == TaskRun.Environment.CLOUD:
refresh_task_run_cost(run_id=run.id, team_id=run.team_id)
diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py
index 79735a2ff2cd..2ff13f06ead2 100644
--- a/products/tasks/backend/metrics.py
+++ b/products/tasks/backend/metrics.py
@@ -196,6 +196,14 @@ TASK_RUN_FAILED_TOTAL = Counter(
],
)
+# Counted once per run on the terminal status write. TASK_RUN_FAILED_TOTAL only sees failures
+# that raise, so it misses runs the workflow fails directly, such as an agent lost mid-turn.
+TASK_RUN_TERMINAL_TOTAL = Counter(
+ "posthog_tasks_task_run_terminal_total",
+ "TaskRun terminal status transitions",
+ labelnames=["origin_product", "run_environment", "status", "error_type", "termination_reason"],
+)
+
CUSTOM_IMAGE_BUILD_TOTAL = Counter(
"posthog_tasks_custom_image_build_total",
"Custom sandbox image build lifecycle events",
@@ -707,6 +715,17 @@ def observe_agent_turn_failed(task_run: "TaskRun") -> None:
_WIZARD_TERMINAL_STATUSES = {"completed", "failed", "cancelled"}
+def observe_task_run_terminal(task_run: "TaskRun", *, error_type: str | None, termination_reason: str | None) -> None:
+ labels = _task_run_labels(task_run)
+ TASK_RUN_TERMINAL_TOTAL.labels(
+ origin_product=labels["origin_product"],
+ run_environment=labels["run_environment"],
+ status=_metric_label(task_run.status),
+ error_type=_failure_metric_label(error_type),
+ termination_reason=_metric_label(termination_reason),
+ ).inc()
+
+
def observe_wizard_run_unbound(task_run: "TaskRun") -> None:
"""Record a wizard run ending without its PR ever binding.
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index 26fe5fdc1ab5..35cec71039bc 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -59,7 +59,11 @@ from products.tasks.backend.feature_flags import (
)
from products.tasks.backend.logic.model_access import resolve_model_access
from products.tasks.backend.logic.stream.redis_stream import publish_task_run_stream_event
-from products.tasks.backend.metrics import observe_task_run_created, observe_task_run_dispatch_callback
+from products.tasks.backend.metrics import (
+ observe_task_run_created,
+ observe_task_run_dispatch_callback,
+ observe_task_run_terminal,
+)
from products.tasks.backend.pr_urls import read_pr_urls
from products.tasks.backend.redis import evaluate_dedicated_stream_flag, run_uses_dedicated_stream
from products.tasks.backend.storage import append_jsonl_object
@@ -3032,6 +3036,21 @@ class TaskRun(models.Model):
return round((self.completed_at - self.created_at).total_seconds(), 1)
return 0.0
+ def _save_terminal_status(self, update_fields: list[str]) -> bool:
+ """Save a terminal status and return whether this write moved the row out of a live status.
+
+ Reads the status under the row lock that the run PATCH and the status activity also take,
+ so a concurrent terminal write cannot make two writers each report the transition.
+ """
+ with transaction.atomic():
+ # Match start_task_run's lock order: saving the run also bumps its task's activity.
+ Task.objects.select_for_update().get(pk=self.task_id)
+ stored_status = (
+ TaskRun.objects.select_for_update().filter(pk=self.pk).values_list("status", flat=True).first()
+ )
+ self.save(update_fields=update_fields)
+ return stored_status not in {self.Status.COMPLETED, self.Status.FAILED, self.Status.CANCELLED}
+
def mark_completed(self, *, notify: bool = True, analytics_properties: dict | None = None) -> None:
"""Mark the progress as completed.
@@ -3042,7 +3061,8 @@ class TaskRun(models.Model):
"""
self.status = self.Status.COMPLETED
self.completed_at = django_timezone.now()
- self.save(update_fields=["status", "completed_at"])
+ if self._save_terminal_status(["status", "completed_at"]):
+ observe_task_run_terminal(self, error_type=None, termination_reason=None)
self.publish_stream_state_event()
self.capture_event(
"task_run_completed",
@@ -3073,7 +3093,8 @@ class TaskRun(models.Model):
self.status = self.Status.FAILED
self.error_message = error
self.completed_at = django_timezone.now()
- self.save(update_fields=["status", "error_message", "completed_at"])
+ if self._save_terminal_status(["status", "error_message", "completed_at"]):
+ observe_task_run_terminal(self, error_type=error_type, termination_reason=None)
self.publish_stream_state_event()
self.capture_event(
"task_run_failed",
diff --git a/products/tasks/backend/temporal/client.py b/products/tasks/backend/temporal/client.py
index 92916d550604..9db34dc419b3 100644
--- a/products/tasks/backend/temporal/client.py
+++ b/products/tasks/backend/temporal/client.py
@@ -41,7 +41,11 @@ from products.tasks.backend.logic.stream.redis_stream import (
TASK_RUN_STREAM_INITIAL_GENERATION,
get_task_run_stream_generation,
)
-from products.tasks.backend.metrics import AGENT_OTEL_TELEMETRY_STAMPED_TOTAL, observe_task_run_workflow_start
+from products.tasks.backend.metrics import (
+ AGENT_OTEL_TELEMETRY_STAMPED_TOTAL,
+ observe_task_run_terminal,
+ observe_task_run_workflow_start,
+)
from products.tasks.backend.models import TaskRun
from products.tasks.backend.redis import run_uses_dedicated_stream
from products.tasks.backend.temporal.bake_dev_stack_image.workflow import BakeDevStackImageInput
@@ -125,6 +129,7 @@ def _terminalize_unstarted_task_run(run_id: str, error_message: str) -> bool:
logger.warning("task_processing_start_failure_task_run_missing", extra={"run_id": run_id})
return False
+ observe_task_run_terminal(task_run, error_type="workflow_start_failed", termination_reason=None)
task_run.publish_stream_state_event()
task_run.capture_event(
"task_run_failed",
diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py
index 8c8d4b083543..cbf11d719246 100644
--- a/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py
+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_update_task_run_status.py
@@ -621,6 +621,33 @@ class TestUpdateTaskRunStatusActivity:
after = REGISTRY.get_sample_value("posthog_tasks_wizard_run_unbound_total", labels) or 0.0
assert after == before + expected_delta
+ @pytest.mark.django_db(transaction=True)
+ def test_terminal_transition_is_counted_once_with_its_failure_reason(self, activity_environment, test_task_run):
+ from prometheus_client import REGISTRY
+
+ test_task_run.task.origin_product = Task.OriginProduct.WORKFLOW
+ test_task_run.task.save(update_fields=["origin_product"])
+ labels = {
+ "origin_product": "workflow",
+ "run_environment": test_task_run.environment,
+ "status": "failed",
+ "error_type": "unknown",
+ "termination_reason": "timed_out_inactivity",
+ }
+ before = REGISTRY.get_sample_value("posthog_tasks_task_run_terminal_total", labels) or 0.0
+
+ input_data = UpdateTaskRunStatusInput(
+ run_id=str(test_task_run.id),
+ status=TaskRun.Status.FAILED,
+ error_message="The agent stopped before finishing its turn",
+ timed_out_inactivity=True,
+ )
+ async_to_sync(activity_environment.run)(update_task_run_status, input_data)
+ async_to_sync(activity_environment.run)(update_task_run_status, input_data)
+
+ after = REGISTRY.get_sample_value("posthog_tasks_task_run_terminal_total", labels) or 0.0
+ assert after == before + 1
+
class TestRecordRunTokenUsageMetrics:
def _record(self, activity_environment, **overrides):
diff --git a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
index 16f3ff103ed8..ffbe1147acbd 100644
--- a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
+++ b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
@@ -15,7 +15,11 @@ from products.tasks.backend.constants import TIMED_OUT_INACTIVITY_STATE_KEY
from products.tasks.backend.error_telemetry import truncate_error_message
from products.tasks.backend.logic.services.gateway_usage import refresh_task_run_cost
from products.tasks.backend.logic.services.workflow_step_resume import resume_workflow_step_for_run
-from products.tasks.backend.metrics import observe_prewarmed_unused_if_never_activated, observe_wizard_run_unbound
+from products.tasks.backend.metrics import (
+ observe_prewarmed_unused_if_never_activated,
+ observe_task_run_terminal,
+ observe_wizard_run_unbound,
+)
from products.tasks.backend.models import Task, TaskRun
from products.tasks.backend.temporal.metrics import record_run_token_usage
from products.tasks.backend.temporal.observability import log_with_activity_context
@@ -124,6 +128,15 @@ def update_task_run_status(input: UpdateTaskRunStatusInput) -> None:
if input.status == TaskRun.Status.FAILED:
input = replace(input, error_type=(task_run.state or {}).get("failure_error_type") or input.error_type)
+ # Counted before the fallible side effects below. A retry after one of them raises sees the
+ # row already terminal and skips this block, so a later count would be lost.
+ if input.status in _TERMINAL_STATUSES and old_status != input.status:
+ observe_task_run_terminal(
+ task_run,
+ error_type=input.error_type,
+ termination_reason=marker if marker in _TERMINAL_STATE_MARKERS else None,
+ )
+
# Side effects run after commit, outside the row lock (repo convention: no side effects in atomic).
if input.status in _TERMINAL_STATUSES:
try:
diff --git a/products/tasks/backend/tests/test_task_run_metrics.py b/products/tasks/backend/tests/test_task_run_metrics.py
index b9e0ebf63933..a8853a8f5e0a 100644
--- a/products/tasks/backend/tests/test_task_run_metrics.py
+++ b/products/tasks/backend/tests/test_task_run_metrics.py
@@ -4,7 +4,9 @@ from datetime import timedelta
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
+from django.db import connection
from django.test import TestCase
+from django.test.utils import CaptureQueriesContext
import posthoganalytics
from parameterized import parameterized
@@ -342,6 +344,89 @@ class TestTaskRunMetrics(TestCase):
assert _sample_value("posthog_tasks_prewarmed_unused_total", labels) == before + 1
+ @parameterized.expand(
+ [
+ ("mark_failed", "failed", "janitor_timeout"),
+ ("mark_completed", "completed", "unknown"),
+ ("dispatch_failure", "failed", "workflow_start_failed"),
+ ("claim_and_fail_stale_run", "failed", "sandbox_lost"),
+ ]
+ )
+ def test_writers_outside_the_status_activity_count_the_terminal_transition_once(
+ self, writer: str, status: str, error_type: str
+ ) -> None:
+ from products.tasks.backend.facade import api as facade
+ from products.tasks.backend.temporal import client
+
+ run = self.task.create_run(environment=TaskRun.Environment.CLOUD)
+ labels = {
+ "origin_product": "user_created",
+ "run_environment": "cloud",
+ "status": status,
+ "error_type": error_type,
+ "termination_reason": "unknown",
+ }
+ before = _sample_value("posthog_tasks_task_run_terminal_total", labels)
+
+ with (
+ patch("products.tasks.backend.push_dispatcher.notify_task_run_failed"),
+ patch.object(client, "resume_workflow_step_for_run"),
+ patch.object(facade, "resume_workflow_step_for_run"),
+ ):
+ for _ in range(2): # a repeat write on a terminal run must not count again
+ run.refresh_from_db()
+ if writer == "mark_failed":
+ run.mark_failed("stuck", error_type="janitor_timeout")
+ elif writer == "mark_completed":
+ run.mark_completed(notify=False)
+ elif writer == "claim_and_fail_stale_run":
+ facade.claim_and_fail_stale_run(run.id, "worker lost", error_type="sandbox_lost")
+ else:
+ client._terminalize_unstarted_task_run(str(run.id), "start failed")
+
+ assert _sample_value("posthog_tasks_task_run_terminal_total", labels) == before + 1
+
+ def test_terminal_save_locks_task_before_run(self) -> None:
+ run = self.task.create_run(environment=TaskRun.Environment.CLOUD)
+ run.status = TaskRun.Status.COMPLETED
+
+ with CaptureQueriesContext(connection) as queries:
+ assert run._save_terminal_status(["status"])
+
+ locks = [query["sql"] for query in queries if "FOR UPDATE" in query["sql"]]
+ assert len(locks) == 2
+ assert f'FROM "{Task._meta.db_table}"' in locks[0]
+ assert f'FROM "{TaskRun._meta.db_table}"' in locks[1]
+
+ @parameterized.expand([("mark_failed",), ("mark_completed",)])
+ def test_finalizer_on_a_stale_instance_does_not_count_a_run_another_writer_ended(self, writer: str) -> None:
+ def terminal_total() -> float:
+ return sum(
+ _sample_value(
+ "posthog_tasks_task_run_terminal_total",
+ {
+ "origin_product": "user_created",
+ "run_environment": "cloud",
+ "status": status,
+ "error_type": error_type,
+ "termination_reason": "unknown",
+ },
+ )
+ for status, error_type in (("failed", "janitor_timeout"), ("completed", "unknown"))
+ )
+
+ run = self.task.create_run(environment=TaskRun.Environment.CLOUD)
+ TaskRun.objects.filter(pk=run.pk).update(status=TaskRun.Status.CANCELLED)
+ before = terminal_total()
+
+ with patch("products.tasks.backend.push_dispatcher.notify_task_run_failed"):
+ if writer == "mark_failed":
+ run.mark_failed("stuck", error_type="janitor_timeout")
+ else:
+ run.mark_completed(notify=False)
+
+ assert terminal_total() == before
+
def test_activating_run_without_prewarmed_marker_does_not_increment(self) -> None:
# Warm Runs provisioned before this ships have await_user_message but no prewarmed marker;
# activating them must not inflate the numerator (would push hit rate above 1).
@@ -530,11 +615,13 @@ class TestTaskRunMetrics(TestCase):
@parameterized.expand(
[
- ("failed", 1.0),
- ("completed", 0.0),
+ ("failed", 1.0, "agent_reported"),
+ ("completed", 0.0, "unknown"),
]
)
- def test_agent_turn_failure_counter_only_on_failed_transition(self, new_status: str, expected_delta: float) -> None:
+ def test_agent_turn_failure_counter_only_on_failed_transition(
+ self, new_status: str, expected_delta: float, terminal_error_type: str
+ ) -> None:
from products.tasks.backend.facade import api as facade
run = self.task.create_run(
@@ -549,17 +636,27 @@ class TestTaskRunMetrics(TestCase):
"task_runtime": "acp",
"runtime_adapter": "codex",
}
+ terminal_labels = {
+ "origin_product": "user_created",
+ "run_environment": "cloud",
+ "status": new_status,
+ "error_type": terminal_error_type,
+ "termination_reason": "unknown",
+ }
before = _sample_value("posthog_tasks_agent_turn_failed_total", labels)
+ terminal_before = _sample_value("posthog_tasks_task_run_terminal_total", terminal_labels)
with patch.object(facade, "signal_workflow_completion"):
- facade.update_task_run(
- run.id,
- self.task.id,
- self.team.id,
- validated_data={"status": new_status, "error_message": "boom"},
- )
+ for _ in range(2): # the repeat PATCH must not double count
+ facade.update_task_run(
+ run.id,
+ self.task.id,
+ self.team.id,
+ validated_data={"status": new_status, "error_message": "boom"},
+ )
assert _sample_value("posthog_tasks_agent_turn_failed_total", labels) == before + expected_delta
+ assert _sample_value("posthog_tasks_task_run_terminal_total", terminal_labels) == terminal_before + 1
@parameterized.expand(
[