Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): log runs started beside a live cloud run (#120023)

Charles Viend69b0e7116aa
diff --git a/products/posthog_ai/backend/message_routing.py b/products/posthog_ai/backend/message_routing.py
index 6e6f703b7385..8beba4cfc09d 100644
--- a/products/posthog_ai/backend/message_routing.py
+++ b/products/posthog_ai/backend/message_routing.py
@@ -380,7 +380,7 @@ class SandboxSession(BaseSandboxService):
# credits). Best-effort: a failure only over-counts the warm pool until the Run terminates,
# and the key is simply absent on Runs that were never warm (the remove is then a no-op).
try:
- tasks_facade.update_task_run_state(run.id, remove_keys=["await_user_message"])
+ tasks_facade.activate_warm_task_run(run.id)
except Exception as e:
logger.warning("sandbox_followup_activation_flip_failed", run_id=str(run.id), error=str(e))
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index ed0d7d6a1812..885af287db54 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -2309,6 +2309,21 @@ def update_task_run_state(
return TaskRun.update_state_atomic(run_id, updates=updates, remove_keys=remove_keys)
+def activate_warm_task_run(run_id: str | UUID) -> None:
+ warm_runs = TaskRun.objects.filter(id=run_id, state__await_user_message=True)
+ task_id = warm_runs.values_list("task_id", flat=True).first()
+ if task_id is None:
+ return
+ with transaction.atomic():
+ task = Task.objects.select_for_update().get(id=task_id)
+ run = warm_runs.select_for_update(of=("self",)).select_related("task").first()
+ if run is None:
+ return
+ run.state.pop("await_user_message", None)
+ run.save(update_fields=["state", "updated_at"])
+ task.observe_run_started_beside_live_runs(run)
+
+
def record_run_failure_cause(run_id: str | UUID, error_type: str) -> None:
try:
TaskRun.update_state_atomic(run_id, updates={"failure_error_type": error_type})
@@ -6782,6 +6797,7 @@ def resume_task_run_in_cloud(
prior_queued_at = run.queued_at
prior_state = dict(run.state or {})
run.prepare_for_cloud_resume()
+ task.observe_run_started_beside_live_runs(run)
if restart_dispatch_enabled:
snapshot = RestartSnapshot(
@@ -8862,12 +8878,14 @@ def _deliver_warm_run_message(
raise WarmRunActivationUnavailable("delivery_failed")
with transaction.atomic():
+ task = Task.objects.select_for_update().get(id=run.task_id)
activated_run = eligible_runs.select_for_update(of=("self",)).first()
if activated_run is None or activated_run.workflow_id != workflow_id:
raise WarmRunActivationUnavailable("target_unavailable")
activated_run.state.pop("await_user_message", None)
activated_run.state["warm_activated"] = True
activated_run.save(update_fields=["state", "updated_at"])
+ task.observe_run_started_beside_live_runs(activated_run)
except WarmRunActivationUnavailable as error:
log = logger.info if error.retry_token else logger.warning
log(
diff --git a/products/tasks/backend/logic/services/workflow_dispatch.py b/products/tasks/backend/logic/services/workflow_dispatch.py
index fb0f7c5bc512..bdf011ec4122 100644
--- a/products/tasks/backend/logic/services/workflow_dispatch.py
+++ b/products/tasks/backend/logic/services/workflow_dispatch.py
@@ -25,7 +25,7 @@ from products.tasks.backend.metrics import (
WORKFLOW_DISPATCH_READY,
observe_task_run_dispatch_callback,
)
-from products.tasks.backend.models import TaskRun, TaskWorkflowDispatch, execute_after_commit
+from products.tasks.backend.models import Task, TaskRun, TaskWorkflowDispatch, execute_after_commit
from products.tasks.backend.temporal.process_task.workflow import PendingFollowup
DISPATCH_PAYLOAD_VERSION = 1
@@ -273,9 +273,24 @@ def materialize_due_scheduled_task_runs(batch_size: int) -> int:
SCHEDULED_TASK_RUN_MATERIALIZATION_TOTAL.labels(outcome="materialized").inc(len(valid_run_ids))
WORKFLOW_DISPATCH_CREATED_TOTAL.labels(kind=TaskWorkflowDispatch.Kind.CREATE).inc(len(dispatches))
SCHEDULED_TASK_RUN_MATERIALIZATION_TOTAL.labels(outcome="invalid").inc(len(invalid_run_ids))
+ _observe_runs_started_beside_live_runs(valid_run_ids)
return len(valid_run_ids)
+def _observe_runs_started_beside_live_runs(run_ids: list[UUID]) -> None:
+ if not run_ids:
+ return
+ with transaction.atomic():
+ tasks = {
+ task.id: task
+ for task in Task.objects.select_for_update()
+ .filter(id__in=TaskRun.objects.filter(id__in=run_ids).values("task_id"))
+ .order_by("id")
+ }
+ for task_run in TaskRun.objects.filter(id__in=run_ids).select_related("task"):
+ tasks[task_run.task_id].observe_run_started_beside_live_runs(task_run)
+
+
def claim_dispatches(instance_id: str, batch_size: int, lease: timedelta) -> list[TaskWorkflowDispatch]:
if batch_size <= 0:
return []
diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py
index 38fce6fe0775..43f8d6252171 100644
--- a/products/tasks/backend/metrics.py
+++ b/products/tasks/backend/metrics.py
@@ -57,6 +57,20 @@ TASK_RUN_CREATED_TOTAL = Counter(
],
)
+TASK_RUN_STARTED_BESIDE_LIVE_RUN_TOTAL = Counter(
+ "posthog_tasks_task_run_started_beside_live_run_total",
+ "Cloud TaskRuns started while another cloud run of the same task was still live",
+ labelnames=[
+ "origin_product",
+ "run_environment",
+ "mode",
+ "run_source",
+ "task_runtime",
+ "runtime_adapter",
+ "prewarmed",
+ ],
+)
+
TASK_RUN_WORKFLOW_START_TOTAL = Counter(
"posthog_tasks_task_run_workflow_start_total",
"TaskRun workflow start lifecycle events",
@@ -545,6 +559,10 @@ def observe_task_run_created(task_run: "TaskRun") -> None:
TASK_RUN_CREATED_TOTAL.labels(**_task_run_labels(task_run)).inc()
+def observe_task_run_started_beside_live_run(task_run: "TaskRun") -> None:
+ TASK_RUN_STARTED_BESIDE_LIVE_RUN_TOTAL.labels(**_task_run_labels(task_run)).inc()
+
+
def observe_sandbox_run_start(*, endpoint: str, outcome: str) -> None:
SANDBOX_RUN_START_TOTAL.labels(endpoint=endpoint, outcome=outcome).inc()
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index f6261b1a0ca0..3d9071bfb7a8 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -62,6 +62,7 @@ from products.tasks.backend.logic.stream.redis_stream import publish_task_run_st
from products.tasks.backend.metrics import (
observe_task_run_created,
observe_task_run_dispatch_callback,
+ observe_task_run_started_beside_live_run,
observe_task_run_terminal,
)
from products.tasks.backend.pr_urls import read_pr_urls
@@ -82,6 +83,23 @@ def execute_after_commit(callback: Callable[[], object]) -> None:
transaction.on_commit(callback)
+def log_run_started_beside_live_run(task_run: "TaskRun", live_run_ids: list[uuid.UUID]) -> None:
+ observe_task_run_started_beside_live_run(task_run)
+ state = task_run.state or {}
+ logger.warning(
+ "task_run_started_beside_live_run",
+ task_id=str(task_run.task_id),
+ team_id=task_run.team_id,
+ run_id=str(task_run.id),
+ live_run_ids=[str(run_id) for run_id in live_run_ids],
+ origin_product=task_run.task.origin_product,
+ mode=state.get("mode"),
+ interaction_origin=state.get("interaction_origin"),
+ resume_from_run_id=state.get("resume_from_run_id"),
+ prewarmed=bool(state.get("prewarmed")),
+ )
+
+
LogLevel = Literal["debug", "info", "warn", "error"]
MCPBuiltInAgentKey = Literal["support", "scout", "workflow"]
MCP_BUILT_IN_AGENT_STATE_KEY = "mcp_builtin_agent_key"
@@ -807,6 +825,27 @@ class Task(Taggable, DeletedMetaFields, models.Model):
state["initial_permission_mode"] = permission_mode
state["ai_defaults_source"] = resolved.source
+ def live_cloud_run_ids(self, exclude_run_id: uuid.UUID) -> list[uuid.UUID]:
+ live_runs = (
+ self.runs.filter(
+ team_id=self.team_id,
+ environment=TaskRun.Environment.CLOUD,
+ status__in=[TaskRun.Status.QUEUED, TaskRun.Status.IN_PROGRESS],
+ )
+ .exclude(id=exclude_run_id)
+ .values_list("id", "state")
+ )
+ return [
+ run_id
+ for run_id, state in live_runs
+ if not (state or {}).get("await_user_message") and not (state or {}).get("cancel_requested_at")
+ ]
+
+ def observe_run_started_beside_live_runs(self, task_run: "TaskRun") -> None:
+ live_run_ids = self.live_cloud_run_ids(exclude_run_id=task_run.id)
+ if live_run_ids:
+ execute_after_commit(lambda: log_run_started_beside_live_run(task_run, live_run_ids))
+
def create_run(
self,
environment: Optional["TaskRun.Environment"] = None,
@@ -948,6 +987,8 @@ class Task(Taggable, DeletedMetaFields, models.Model):
state=state,
branch=branch,
)
+ if effective_environment == TaskRun.Environment.CLOUD and scheduled_at is None:
+ task.observe_run_started_beside_live_runs(task_run)
def emit_created_events() -> None:
task_run.publish_stream_state_event()
@@ -2409,6 +2450,7 @@ class TaskRun(models.Model):
"sandbox_connect_token",
"sandbox_backend",
"failure_error_type",
+ "cancel_requested_at",
)
def restore_cloud_resume_state(self, prior_state: dict[str, Any]) -> None:
diff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.py
index 93a60703bf10..5a50e9266b72 100644
--- a/products/tasks/backend/presentation/views/api.py
+++ b/products/tasks/backend/presentation/views/api.py
@@ -3758,7 +3758,7 @@ class TaskRunViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
# (see products/tasks/backend/logic/services/warm.py) stops counting it. No-op for Runs
# that were never warm; best-effort, since a failure only over-counts the pool until terminal.
try:
- tasks_facade.update_task_run_state(pk, remove_keys=["await_user_message"])
+ tasks_facade.activate_warm_task_run(pk)
except Exception:
logger.warning("Failed to clear await_user_message for task run %s", pk)
diff --git a/products/tasks/backend/tests/test_facade.py b/products/tasks/backend/tests/test_facade.py
index 408fe09fd596..ef42361f31b2 100644
--- a/products/tasks/backend/tests/test_facade.py
+++ b/products/tasks/backend/tests/test_facade.py
@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, patch
from django.apps import apps
from django.db import close_old_connections, transaction
-from django.test import SimpleTestCase, TestCase, TransactionTestCase
+from django.test import SimpleTestCase, TestCase, TransactionTestCase, override_settings
from django.utils import timezone as django_timezone
from parameterized import parameterized
@@ -428,6 +428,26 @@ class TestFacadeReadsAndMappers(TestCase):
self.assertEqual(outcome, "invalid_origin")
self.assertIsNone(resumed_run)
+
+ @override_settings(TASKS_WORKFLOW_DISPATCH_OUTBOX_ENABLED=True)
+ def test_resume_in_cloud_logs_a_restart_beside_a_live_run(self):
+ task = self._make_task()
+ live = TaskRun.objects.create(
+ task=task, team=self.team, environment=TaskRun.Environment.CLOUD, status=TaskRun.Status.QUEUED
+ )
+ run = TaskRun.objects.create(
+ task=task,
+ team=self.team,
+ environment=TaskRun.Environment.CLOUD,
+ status=TaskRun.Status.COMPLETED,
+ state={"pr_authorship_mode": "bot"},
+ )
+
+ with patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started:
+ outcome, _, _ = facade.resume_task_run_in_cloud(run.id, task.id, self.team.id, self.user.id)
+
+ self.assertEqual(outcome, "resumed")
+ log_started.assert_called_once_with(run, [live.id])
run.refresh_from_db()
self.assertEqual(run.environment, TaskRun.Environment.CLOUD)
diff --git a/products/tasks/backend/tests/test_models.py b/products/tasks/backend/tests/test_models.py
index f5b0720c5f31..891f5e178c43 100644
--- a/products/tasks/backend/tests/test_models.py
+++ b/products/tasks/backend/tests/test_models.py
@@ -2,7 +2,7 @@ import json
import uuid
from contextlib import nullcontext
from datetime import UTC, datetime, timedelta
-from typing import ClassVar
+from typing import Any, ClassVar
from unittest.mock import AsyncMock, patch
@@ -1120,6 +1120,61 @@ class TestTaskRun(TestCase):
self.assertEqual(call_args.args[1]["status"], TaskRun.Status.QUEUED)
self.assertEqual(call_args.args[1]["branch"], "main")
+ @parameterized.expand(
+ [
+ ("no_runs", [], "cloud", False),
+ (
+ "terminal_cloud_runs",
+ [
+ (TaskRun.Status.COMPLETED, TaskRun.Environment.CLOUD, {}),
+ (TaskRun.Status.FAILED, TaskRun.Environment.CLOUD, {}),
+ (TaskRun.Status.CANCELLED, TaskRun.Environment.CLOUD, {}),
+ ],
+ "cloud",
+ False,
+ ),
+ ("queued_cloud_run", [(TaskRun.Status.QUEUED, TaskRun.Environment.CLOUD, {})], "cloud", True),
+ ("in_progress_cloud_run", [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {})], "cloud", True),
+ (
+ "idle_warm_run",
+ [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {"await_user_message": True})],
+ "cloud",
+ False,
+ ),
+ (
+ "cancelling_run",
+ [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {"cancel_requested_at": "2026-01-01"})],
+ "cloud",
+ False,
+ ),
+ ("scheduled_cloud_run", [(TaskRun.Status.NOT_STARTED, TaskRun.Environment.CLOUD, {})], "cloud", False),
+ ("queued_local_run", [(TaskRun.Status.QUEUED, TaskRun.Environment.LOCAL, {})], "cloud", False),
+ ("new_local_run", [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {})], "local", False),
+ ("new_scheduled_run", [(TaskRun.Status.IN_PROGRESS, TaskRun.Environment.CLOUD, {})], "scheduled", False),
+ ]
+ )
+ def test_create_run_logs_a_run_started_beside_a_live_cloud_run(
+ self, _name: str, existing_runs: list[tuple[str, str, dict]], new_run: str, flagged: bool
+ ) -> None:
+ existing = [
+ TaskRun.objects.create(task=self.task, team=self.team, status=status, environment=environment, state=state)
+ for status, environment, state in existing_runs
+ ]
+ create_kwargs: dict[str, dict[str, Any]] = {
+ "cloud": {},
+ "local": {"environment": TaskRun.Environment.LOCAL},
+ "scheduled": {"scheduled_at": django_timezone.now() + timedelta(days=1)},
+ }
+
+ with patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started:
+ run = self.task.create_run(**create_kwargs[new_run])
+
+ self.assertEqual(TaskRun.objects.filter(task=self.task).count(), len(existing_runs) + 1)
+ if flagged:
+ log_started.assert_called_once_with(run, [existing[0].id])
+ else:
+ log_started.assert_not_called()
+
def test_create_run_does_not_inject_permission_mode_by_default(self):
run = self.task.create_run(mode="interactive")
@@ -1284,6 +1339,7 @@ class TestTaskRun(TestCase):
"pending_user_message": "Review the attachment",
"pending_user_artifact_ids": ["artifact-1"],
"failure_error_type": "poll_timeout",
+ "cancel_requested_at": requested_at.isoformat(),
"startup_requested_at": requested_at.isoformat(),
},
)
@@ -1296,6 +1352,7 @@ class TestTaskRun(TestCase):
self.assertEqual(run.analytics_properties()["startup_requested_at"], run.queued_at.isoformat())
self.assertNotIn("failure_error_type", run.state)
+ self.assertNotIn("cancel_requested_at", run.state)
self.assertNotIn("sandbox_id", run.state)
self.assertNotIn("sandbox_url", run.state)
self.assertNotIn("sandbox_jwt_kid", run.state)
diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.py
index 6cc6dddd3218..ffb1111b2a71 100644
--- a/products/tasks/backend/tests/test_warm_facade.py
+++ b/products/tasks/backend/tests/test_warm_facade.py
@@ -1920,6 +1920,33 @@ class TestRunTaskWarmActivation(APIBaseTest):
assert "await_user_message" not in run.state
assert run.state.get("auto_publish") is True
+ @parameterized.expand([("run_task",), ("user_message",)])
+ def test_first_message_to_a_warm_run_logs_a_live_sibling_once(self, activation: str) -> None:
+ task, run = self._warm_run()
+ live = TaskRun.objects.create(
+ task=task, team=self.team, environment=TaskRun.Environment.CLOUD, status=TaskRun.Status.IN_PROGRESS
+ )
+ TaskRun.objects.filter(id=live.id).update(created_at=run.created_at - timedelta(minutes=1))
+
+ with (
+ patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started,
+ patch(f"{FACADE}.signal_task_run_user_message", return_value=True),
+ ):
+ if activation == "run_task":
+ facade.run_task(
+ task.id,
+ self.team.id,
+ self.user.id,
+ validated_data={"mode": "interactive", "branch": "main", "pending_user_message": "go"},
+ )
+ facade.activate_warm_task_run(run.id)
+ facade.activate_warm_task_run(run.id)
+
+ assert task.runs.count() == 2
+ log_started.assert_called_once_with(run, [live.id])
+ run.refresh_from_db()
+ assert "await_user_message" not in run.state
+
def test_falls_back_to_description_when_no_pending_message(self):
task, run = self._warm_run()
task.description = "from description"
diff --git a/products/tasks/backend/tests/test_workflow_dispatch.py b/products/tasks/backend/tests/test_workflow_dispatch.py
index 799e31fdab54..98f1a8842d06 100644
--- a/products/tasks/backend/tests/test_workflow_dispatch.py
+++ b/products/tasks/backend/tests/test_workflow_dispatch.py
@@ -605,8 +605,10 @@ class TestWorkflowDispatchPersistence(TestCase):
state=state,
)
- self.assertEqual(materialize_due_scheduled_task_runs(100), 1)
+ with patch("products.tasks.backend.models.log_run_started_beside_live_run") as log_started:
+ self.assertEqual(materialize_due_scheduled_task_runs(100), 1)
+ log_started.assert_called_once_with(due, [self.task_run.id])
due.refresh_from_db()
future.refresh_from_db()
local.refresh_from_db()