:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): log runs started beside a live cloud run (#120023)
Charles Viend69b0e7116aa
diff --git a/products/posthog_ai/backend/message_routing.py b/products/posthog_ai/backend/message_routing.pyindex 6e6f703b7385..8beba4cfc09d 100644--- a/products/posthog_ai/backend/message_routing.py+++ b/products/posthog_ai/backend/message_routing.py@@ -380,7 +380,7 @@ class SandboxSession(BaseSandboxService):# credits). Best-effort: a failure only over-counts the warm pool until the Run terminates,# and the key is simply absent on Runs that were never warm (the remove is then a no-op).try:- tasks_facade.update_task_run_state(run.id, remove_keys=["await_user_message"])+ tasks_facade.activate_warm_task_run(run.id)except Exception as e:logger.warning("sandbox_followup_activation_flip_failed", run_id=str(run.id), error=str(e))diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.pyindex ed0d7d6a1812..885af287db54 100644--- a/products/tasks/backend/facade/api.py+++ b/products/tasks/backend/facade/api.py@@ -2309,6 +2309,21 @@ def update_task_run_state(return TaskRun.update_state_atomic(run_id, updates=updates, remove_keys=remove_keys)+def activate_warm_task_run(run_id: str | UUID) -> None:+ warm_runs = TaskRun.objects.filter(id=run_id, state__await_user_message=True)+ task_id = warm_runs.values_list("task_id", flat=True).first()+ if task_id is None:+ return+ with transaction.atomic():+ task = Task.objects.select_for_update().get(id=task_id)+ run = warm_runs.select_for_update(of=("self",)).select_related("task").first()+ if run is None:+ return+ run.state.pop("await_user_message", None)+ run.save(update_fields=["state", "updated_at"])+ task.observe_run_started_beside_live_runs(run)++def record_run_failure_cause(run_id: str | UUID, error_type: str) -> None:try:TaskRun.update_state_atomic(run_id, updates={"failure_error_type": error_type})@@ -6782,6 +6797,7 @@ def resume_task_run_in_cloud(prior_queued_at = run.queued_atprior_state = dict(run.state or {})run.prepare_for_cloud_resume()+ task.observe_run_started_beside_live_runs(run)if restart_dispatch_enabled:snapshot = RestartSnapshot(@@ -8862,12 +8878,14 @@ def _deliver_warm_run_message(raise WarmRunActivationUnavailable("delivery_failed")with transaction.atomic():+ task = Task.objects.select_for_update().get(id=run.task_id)activated_run = eligible_runs.select_for_update(of=("self",)).first()if activated_run is None or activated_run.workflow_id != workflow_id:raise WarmRunActivationUnavailable("target_unavailable")activated_run.state.pop("await_user_message", None)activated_run.state["warm_activated"] = Trueactivated_run.save(update_fields=["state", "updated_at"])+ task.observe_run_started_beside_live_runs(activated_run)except WarmRunActivationUnavailable as error:log = logger.info if error.retry_token else logger.warninglog(diff --git a/products/tasks/backend/logic/services/workflow_dispatch.py b/products/tasks/backend/logic/services/workflow_dispatch.pyindex fb0f7c5bc512..bdf011ec4122 100644--- a/products/tasks/backend/logic/services/workflow_dispatch.py+++ b/products/tasks/backend/logic/services/workflow_dispatch.py@@ -25,7 +25,7 @@ from products.tasks.backend.metrics import (WORKFLOW_DISPATCH_READY,observe_task_run_dispatch_callback,)-from products.tasks.backend.models import TaskRun, TaskWorkflowDispatch, execute_after_commit+from products.tasks.backend.models import Task, TaskRun, TaskWorkflowDispatch, execute_after_commitfrom products.tasks.backend.temporal.process_task.workflow import PendingFollowupDISPATCH_PAYLOAD_VERSION = 1@@ -273,9 +273,24 @@ def materialize_due_scheduled_task_runs(batch_size: int) -> int:SCHEDULED_TASK_RUN_MATERIALIZATION_TOTAL.labels(outcome="materialized").inc(len(valid_run_ids))WORKFLOW_DISPATCH_CREATED_TOTAL.labels(kind=TaskWorkflowDispatch.Kind.CREATE).inc(len(dispatches))SCHEDULED_TASK_RUN_MATERIALIZATION_TOTAL.labels(outcome="invalid").inc(len(invalid_run_ids))+ _observe_runs_started_beside_live_runs(valid_run_ids)return len(valid_run_ids)+def _observe_runs_started_beside_live_runs(run_ids: list[UUID]) -> None:+ if not run_ids:+ return+ with transaction.atomic():+ tasks = {+ task.id: task+ for task in Task.objects.select_for_update()+ .filter(id__in=TaskRun.objects.filter(id__in=run_ids).values("task_id"))+ .order_by("id")+ }+ for task_run in TaskRun.objects.filter(id__in=run_ids).select_related("task"):+ tasks[task_run.task_id].observe_run_started_beside_live_runs(task_run)++def claim_dispatches(instance_id: str, batch_size: int, lease: timedelta) -> list[TaskWorkflowDispatch]:if batch_size <= 0:return []diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.pyindex 38fce6fe0775..43f8d6252171 100644--- a/products/tasks/backend/metrics.py+++ b/products/tasks/backend/metrics.py@@ -57,6 +57,20 @@ TASK_RUN_CREATED_TOTAL = Counter(],)+TASK_RUN_STARTED_BESIDE_LIVE_RUN_TOTAL = Counter(+ "posthog_tasks_task_run_started_beside_live_run_total",+ "Cloud TaskRuns started while another cloud run of the same task was still live",+ labelnames=[+ "origin_product",+ "run_environment",+ "mode",+ "run_source",+ "task_runtime",+ "runtime_adapter",+ "prewarmed",+ ],+)+TASK_RUN_WORKFLOW_START_TOTAL = Counter("posthog_tasks_task_run_workflow_start_total","TaskRun workflow start lifecycle events",@@ -545,6 +559,10 @@ def observe_task_run_created(task_run: "TaskRun") -> None:TASK_RUN_CREATED_TOTAL.labels(**_task_run_labels(task_run)).inc()+def observe_task_run_started_beside_live_run(task_run: "TaskRun") -> None:+ TASK_RUN_STARTED_BESIDE_LIVE_RUN_TOTAL.labels(**_task_run_labels(task_run)).inc()++def observe_sandbox_run_start(*, endpoint: str, outcome: str) -> None:SANDBOX_RUN_START_TOTAL.labels(endpoint=endpoint, outcome=outcome).inc()diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.pyindex f6261b1a0ca0..3d9071bfb7a8 100644--- a/products/tasks/backend/models.py+++ b/products/tasks/backend/models.py@@ -62,6 +62,7 @@ from products.tasks.backend.logic.stream.redis_stream import publish_task_run_stfrom products.tasks.backend.metrics import (observe_task_run_created,observe_task_run_dispatch_callback,+ observe_task_run_started_beside_live_run,observe_task_run_terminal,)from products.tasks.backend.pr_urls import read_pr_urls@@ -82,6 +83,23 @@ def execute_after_commit(callback: Callable[[], object]) -> None:transaction.on_commit(callback)+def log_run_started_beside_live_run(task_run: "TaskRun", live_run_ids: list[uuid.UUID]) -> None:+ observe_task_run_started_beside_live_run(task_run)+ state = task_run.state or {}+ logger.warning(+ "task_run_started_beside_live_run",+ task_id=str(task_run.task_id),+ team_id=task_run.team_id,+ run_id=str(task_run.id),+ live_run_ids=[str(run_id) for run_id in live_run_ids],+ origin_product=task_run.task.origin_product,+ mode=state.get("mode"),+ interaction_origin=state.get("interaction_origin"),+ resume_from_run_id=state.get("resume_from_run_id"),+ prewarmed=bool(state.get("prewarmed")),+ )++LogLevel = Literal["debug", "info", "warn", "error"]MCPBuiltInAgentKey = Literal["support", "scout", "workflow"]MCP_BUILT_IN_AGENT_STATE_KEY = "mcp_builtin_agent_key"@@ -807,6 +825,27 @@ class Task(Taggable, DeletedMetaFields, models.Model):state["initial_permission_mode"] = permission_modestate["ai_defaults_source"] = resolved.source+ def live_cloud_run_ids(self, exclude_run_id: uuid.UUID) -> list[uuid.UUID]:+ live_runs = (+ self.runs.filter(+ team_id=self.team_id,+ environment=TaskRun.Environment.CLOUD,+ status__in=[TaskRun.Status.QUEUED, TaskRun.Status.IN_PROGRESS],+ )+ .exclude(id=exclude_run_id)+ .values_list("id", "state")+ )+ return [+ run_id+ for run_id, state in live_runs+ if not (state or {}).get("await_user_message") and not (state or {}).get("cancel_requested_at")+ ]++ def observe_run_started_beside_live_runs(self, task_run: "TaskRun") -> None:+ live_run_ids = self.live_cloud_run_ids(exclude_run_id=task_run.id)+ if live_run_ids:+ execute_after_commit(lambda: log_run_started_beside_live_run(task_run, live_run_ids))+def create_run(self,environment: Optional["TaskRun.Environment"] = None,@@ -948,6 +987,8 @@ class Task(Taggable, DeletedMetaFields, models.Model):state=state,branch=branch,)+ if effective_environment == TaskRun.Environment.CLOUD and scheduled_at is None:+ task.observe_run_started_beside_live_runs(task_run)def emit_created_events() -> None:task_run.publish_stream_state_event()@@ -2409,6 +2450,7 @@ class TaskRun(models.Model):"sandbox_connect_token","sandbox_backend","failure_error_type",+ "cancel_requested_at",)def restore_cloud_resume_state(self, prior_state: dict[str, Any]) -> None:diff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.pyindex 93a60703bf10..5a50e9266b72 100644--- a/products/tasks/backend/presentation/views/api.py+++ b/products/tasks/backend/presentation/views/api.py@@ -3758,7 +3758,7 @@ class TaskRunViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):# (see products/tasks/backend/logic/services/warm.py) stops counting it. No-op for Runs# that were never warm; best-effort, since a failure only over-counts the pool until terminal.try:- tasks_facade.update_task_run_state(pk, remove_keys=["await_user_message"])+ tasks_facade.activate_warm_task_run(pk)except Exception:logger.warning("Failed to clear await_user_message for task run %s", pk)diff --git a/products/tasks/backend/tests/test_facade.py b/products/tasks/backend/tests/test_facade.pyindex 408fe09fd596..ef42361f31b2 100644--- a/products/tasks/backend/tests/test_facade.py+++ b/products/tasks/backend/tests/test_facade.py@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, patchfrom django.apps import appsfrom django.db import close_old_connections, transaction-from django.test import SimpleTestCase, TestCase, TransactionTestCase+from django.test import SimpleTestCase, TestCase, TransactionTestCase, override_settingsfrom django.utils import timezone as django_timezonefrom parameterized import parameterized@@ -428,6 +428,26 @@ class TestFacadeReadsAndMappers(TestCase):self.assertEqual(outcome, "invalid_origin")self.assertIsNone(resumed_run)++ @override_settings(TASKS_WORKFLOW_DISPATCH_OUTBOX_ENABLED=True)+ def test_resume_in_cloud_logs_a_restart_beside_a_live_run(self):+ task = self._make_task()+ live = TaskRun.objects.create(+ task=task, team=self.team, environment=TaskRun.Environment.CLOUD, status=TaskRun.Status.QUEUED+ )+ run = TaskRun.objects.create(+ task=task,+ team=self.team,+ environment=TaskRun.Environment.CLOUD,+ status=TaskRun.Status.COMPLETED,+ state={"pr_authorship_mode": "bot"},+ )++ with patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started:+ outcome, _, _ = facade.resume_task_run_in_cloud(run.id, task.id, self.team.id, self.user.id)++ self.assertEqual(outcome, "resumed")+ log_started.assert_called_once_with(run, [live.id])run.refresh_from_db()self.assertEqual(run.environment, TaskRun.Environment.CLOUD)diff --git a/products/tasks/backend/tests/test_models.py b/products/tasks/backend/tests/test_models.pyindex f5b0720c5f31..891f5e178c43 100644--- a/products/tasks/backend/tests/test_models.py+++ b/products/tasks/backend/tests/test_models.py@@ -2,7 +2,7 @@ import jsonimport uuidfrom contextlib import nullcontextfrom datetime import UTC, datetime, timedelta-from typing import ClassVar+from typing import Any, ClassVarfrom unittest.mock import AsyncMock, patch@@ -1120,6 +1120,61 @@ class TestTaskRun(TestCase):self.assertEqual(call_args.args[1]["status"], TaskRun.Status.QUEUED)self.assertEqual(call_args.args[1]["branch"], "main")+ @parameterized.expand(+ [+ ("no_runs", [], "cloud", False),+ (+ "terminal_cloud_runs",+ [+ (TaskRun.Status.COMPLETED, TaskRun.Environment.CLOUD, {}),+ (TaskRun.Status.FAILED, TaskRun.Environment.CLOUD, {}),+ (TaskRun.Status.CANCELLED, TaskRun.Environment.CLOUD, {}),+ ],+ "cloud",+ False,+ ),+ ("queued_cloud_run", [(TaskRun.Status.QUEUED, TaskRun.Environment.CLOUD, {})], "cloud", True),+ ("in_progress_cloud_run", [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {})], "cloud", True),+ (+ "idle_warm_run",+ [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {"await_user_message": True})],+ "cloud",+ False,+ ),+ (+ "cancelling_run",+ [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {"cancel_requested_at": "2026-01-01"})],+ "cloud",+ False,+ ),+ ("scheduled_cloud_run", [(TaskRun.Status.NOT_STARTED, TaskRun.Environment.CLOUD, {})], "cloud", False),+ ("queued_local_run", [(TaskRun.Status.QUEUED, TaskRun.Environment.LOCAL, {})], "cloud", False),+ ("new_local_run", [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {})], "local", False),+ ("new_scheduled_run", [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {})], "scheduled", False),+ ]+ )+ def test_create_run_logs_a_run_started_beside_a_live_cloud_run(+ self, _name: str, existing_runs: list[tuple[str, str, dict]], new_run: str, flagged: bool+ ) -> None:+ existing = [+ TaskRun.objects.create(task=self.task, team=self.team, status=status, environment=environment, state=state)+ for status, environment, state in existing_runs+ ]+ create_kwargs: dict[str, dict[str, Any]] = {+ "cloud": {},+ "local": {"environment": TaskRun.Environment.LOCAL},+ "scheduled": {"scheduled_at": django_timezone.now() + timedelta(days=1)},+ }++ with patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started:+ run = self.task.create_run(**create_kwargs[new_run])++ self.assertEqual(TaskRun.objects.filter(task=self.task).count(), len(existing_runs) + 1)+ if flagged:+ log_started.assert_called_once_with(run, [existing[0].id])+ else:+ log_started.assert_not_called()+def test_create_run_does_not_inject_permission_mode_by_default(self):run = self.task.create_run(mode="interactive")@@ -1284,6 +1339,7 @@ class TestTaskRun(TestCase):"pending_user_message": "Review the attachment","pending_user_artifact_ids": ["artifact-1"],"failure_error_type": "poll_timeout",+ "cancel_requested_at": requested_at.isoformat(),"startup_requested_at": requested_at.isoformat(),},)@@ -1296,6 +1352,7 @@ class TestTaskRun(TestCase):self.assertEqual(run.analytics_properties()["startup_requested_at"], run.queued_at.isoformat())self.assertNotIn("failure_error_type", run.state)+ self.assertNotIn("cancel_requested_at", run.state)self.assertNotIn("sandbox_id", run.state)self.assertNotIn("sandbox_url", run.state)self.assertNotIn("sandbox_jwt_kid", run.state)diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.pyindex 6cc6dddd3218..ffb1111b2a71 100644--- a/products/tasks/backend/tests/test_warm_facade.py+++ b/products/tasks/backend/tests/test_warm_facade.py@@ -1920,6 +1920,33 @@ class TestRunTaskWarmActivation(APIBaseTest):assert "await_user_message" not in run.stateassert run.state.get("auto_publish") is True+ @parameterized.expand([("run_task",), ("user_message",)])+ def test_first_message_to_a_warm_run_logs_a_live_sibling_once(self, activation: str) -> None:+ task, run = self._warm_run()+ live = TaskRun.objects.create(+ task=task, team=self.team, environment=TaskRun.Environment.CLOUD, status=TaskRun.Status.IN_PROGRESS+ )+ TaskRun.objects.filter(id=live.id).update(created_at=run.created_at - timedelta(minutes=1))++ with (+ patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started,+ patch(f"{FACADE}.signal_task_run_user_message", return_value=True),+ ):+ if activation == "run_task":+ facade.run_task(+ task.id,+ self.team.id,+ self.user.id,+ validated_data={"mode": "interactive", "branch": "main", "pending_user_message": "go"},+ )+ facade.activate_warm_task_run(run.id)+ facade.activate_warm_task_run(run.id)++ assert task.runs.count() == 2+ log_started.assert_called_once_with(run, [live.id])+ run.refresh_from_db()+ assert "await_user_message" not in run.state+def test_falls_back_to_description_when_no_pending_message(self):task, run = self._warm_run()task.description = "from description"diff --git a/products/tasks/backend/tests/test_workflow_dispatch.py b/products/tasks/backend/tests/test_workflow_dispatch.pyindex 799e31fdab54..98f1a8842d06 100644--- a/products/tasks/backend/tests/test_workflow_dispatch.py+++ b/products/tasks/backend/tests/test_workflow_dispatch.py@@ -605,8 +605,10 @@ class TestWorkflowDispatchPersistence(TestCase):state=state,)- self.assertEqual(materialize_due_scheduled_task_runs(100), 1)+ with patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started:+ self.assertEqual(materialize_due_scheduled_task_runs(100), 1)+ log_started.assert_called_once_with(due, [self.task_run.id])due.refresh_from_db()future.refresh_from_db()local.refresh_from_db()