:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): cap active sandbox sessions and throttle warm churn (#128466)
Alessandro Pogliaghi67127d63bb63
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex c7304ac855e3..bcc13b1a6f0a 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -9105,6 +9105,7 @@ export namespace Schemas {* * `subscription_usage_limit` - subscription_usage_limit* * `task_spend_limit` - task_spend_limit* * `agent_error` - agent_error+ * * `ActiveSessionCapError` - ActiveSessionCapError*/export type ErrorTypeEnum =| "agent_reported"@@ -9122,7 +9123,8 @@ export namespace Schemas {| "turn_ended_without_response"| "subscription_usage_limit"| "task_spend_limit"- | "agent_error";+ | "agent_error"+ | "ActiveSessionCapError";/*** * `active` - Active* * `paused` - Pauseddiff --git a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.ts b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.tsindex 8fd35db271a3..7c55906302de 100644--- a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.ts+++ b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.ts@@ -745,6 +745,25 @@ describe('taskTrackerSceneLogic', () => {})+ it('toasts the active-session cap refusal and keeps the draft', async () => {+ const detail =+ 'You already have 10 cloud sandboxes running. Wait for one to finish or stop one, then try again.'+ useMocks({+ post: {+ '/api/projects/:team/tasks/:id/run/': () => [400, { detail, code: 'active_session_cap' }],+ },+ })+ const toast = jest.spyOn(lemonToast, 'error')+ logic.mount()+ logic.actions.setNewTaskData({ description: 'Inspect the example chart' })+ await expectLogic(logic, () => logic.actions.submitNewTask()).toFinishAllListeners()++ expect(toast).toHaveBeenCalledWith(detail)+ expect(logic.values.isSubmittingTask).toBe(false)+ expect(logic.values.newTaskData.description).toBe('Inspect the example chart')+ toast.mockRestore()+ })+// The seeded first message wraps the on-screen context, and the wrapped non-text refs must be marked// sent under the created task's id — otherwise the run's first follow-up (sent via// `runInteractionLogic`, which prunes against the task-scoped store) re-wraps the same refs.diff --git a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.ts b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.tsindex 748d926cb60f..de72d659010a 100644--- a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.ts+++ b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.ts@@ -1303,6 +1303,9 @@ export const taskTrackerSceneLogic = kea<taskTrackerSceneLogicType>([if (error instanceof ApiError && error.code === 'warm_run_activation_unavailable') {lemonToast.error("Couldn't start this run yet. Please try again.")}+ if (error instanceof ApiError && error.code === 'active_session_cap' && error.detail) {+ lemonToast.error(error.detail)+ }if (error instanceof CloudTargetUnresolvedError) {lemonToast.error("Couldn't load your default environment. Please try again.")}diff --git a/products/tasks/backend/exceptions.py b/products/tasks/backend/exceptions.pyindex 59dedfd6bb2e..efd96adc1615 100644--- a/products/tasks/backend/exceptions.py+++ b/products/tasks/backend/exceptions.py@@ -147,6 +147,18 @@ class ComputeBillingLimitError(ProcessTaskError, ComputeBillingLimitExceeded):)+class ActiveSessionCapError(ProcessTaskError):+ def __init__(self, context: dict[str, Any], scope: str, message: str):+ self.scope = scope+ super().__init__(+ message,+ {**context, "scope": scope},+ None,+ capture=False,+ non_retryable=True,+ )++class SandboxNetworkPolicyError(ProcessTaskFatalError):passdiff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.pyindex 918cfa180a77..3494e4d49816 100644--- a/products/tasks/backend/facade/api.py+++ b/products/tasks/backend/facade/api.py@@ -235,6 +235,7 @@ _TASK_LOG_READ_EXECUTOR = ThreadPoolExecutor(max_workers=8, thread_name_prefix="TaskRunStatus = TaskRun.StatusTaskRunEnvironment = TaskRun.EnvironmentTaskOriginProduct = Task.OriginProduct+TASK_SERVER_ONLY_ORIGIN_PRODUCTS = Task.SERVER_ONLY_ORIGIN_PRODUCTSTaskRuntime = Task.RuntimeSandboxNetworkAccessLevel = SandboxEnvironment.NetworkAccessLevelSandboxSnapshotStatus = SandboxSnapshot.Status@@ -260,10 +261,12 @@ __all__ = ["SandboxNetworkAccessLevel","SandboxSnapshotStatus","TASK_RUN_SUMMARY_STATE_KEY",+ "TASK_SERVER_ONLY_ORIGIN_PRODUCTS","TaskOriginProduct","TaskRuntime","TaskRunEnvironment","TaskRunStatus",+ "ActiveSessionCapRefused","WarmRunActivationUnavailable","append_imported_task_run_log","append_task_run_log",@@ -3210,6 +3213,7 @@ TASK_RUN_FAILURE_ERROR_TYPES = ("subscription_usage_limit","task_spend_limit","agent_error",+ "ActiveSessionCapError",)# `output.pr_merged` is GitHub's word, recorded by the PR webhook (`_record_run_pr_merged`) — never@@ -8108,6 +8112,8 @@ def create_task(raise ComputeBillingLimitError({"team_id": team_id, "task_id": str(warm_task.id), "run_id": str(warm_run.id)}, reason)+ if (cap_detail := _active_session_cap_refusal(warm_task)) is not None:+ raise ActiveSessionCapRefused(cap_detail)report_reservation = (_reserve_report_warm_activation(team,@@ -8960,6 +8966,17 @@ REPORT_IMPLEMENTATION_REPOSITORY_UNRESOLVED = ()+ACTIVE_SESSION_CAP_CODE = "active_session_cap"+++class ActiveSessionCapRefused(Exception):+ code = ACTIVE_SESSION_CAP_CODE++ def __init__(self, detail: str) -> None:+ self.detail = detail+ super().__init__(detail)++class WarmRunActivationUnavailable(Exception):code = "warm_run_activation_unavailable"@@ -9564,6 +9581,27 @@ def _branches_worked_on(run: TaskRun) -> set[str]:return branches+def _active_session_cap_refusal(task: Task) -> str | None:+ from products.tasks.backend.logic.services.warm import (+ SandboxWarmer, # noqa: PLC0415 — keep warming deps off the api import path+ )+ from products.tasks.backend.metrics import (+ observe_active_session_cap_hit, # noqa: PLC0415 — keep prometheus deps off the api import path+ )++ scope = SandboxWarmer.active_session_cap_hit(+ task.origin_product,+ organization_id=task.team.organization_id,+ created_by_id=task.created_by_id,+ signal_report_id=task.signal_report_id,+ task_id=task.id,+ )+ if scope is None:+ return None+ observe_active_session_cap_hit(task.origin_product, scope)+ return SandboxWarmer.active_session_cap_detail(task.origin_product, scope, signal_report_id=task.signal_report_id)++def run_task(task_id: str | UUID,team_id: int,@@ -9611,6 +9649,12 @@ def run_task(return contracts.TaskRunResult(error=contracts.TaskValidationError(kind="detail", detail=REPORT_WARM_RUN_NOT_ACTIVATED))+ if not validated_data.get("scheduled_at"):+ cap_detail = _active_session_cap_refusal(task)+ if cap_detail is not None:+ return contracts.TaskRunResult(+ error=contracts.TaskValidationError(kind="detail", detail=cap_detail, code=ACTIVE_SESSION_CAP_CODE)+ )report_id_for_slot_check = (str(task.signal_report_id)if task.signal_report_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORTdiff --git a/products/tasks/backend/logic/services/sandbox_usage.py b/products/tasks/backend/logic/services/sandbox_usage.pyindex 29bba05a9b2f..6beb2549da44 100644--- a/products/tasks/backend/logic/services/sandbox_usage.py+++ b/products/tasks/backend/logic/services/sandbox_usage.py@@ -16,17 +16,18 @@ from dataclasses import dataclassfrom datetime import datetime, timedeltafrom decimal import ROUND_HALF_EVEN, Decimalfrom functools import wraps-from typing import ParamSpec, TypeVar+from typing import Any, ParamSpec, TypeVarfrom uuid import UUID-from django.db import transaction-from django.db.models import Case, F, Q, Value, When+from django.db import connection, transaction+from django.db.models import Case, F, Q, QuerySet, Value, Whenfrom django.utils import timezoneimport structlogfrom posthog.dataclasses import frozen+from products.tasks.backend.exceptions import ActiveSessionCapErrorfrom products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_sessionfrom products.tasks.backend.logic.services.sandbox import SandboxBase, SandboxConfig, get_sandbox_class_for_sandbox_idfrom products.tasks.backend.logic.services.sandbox_pricing import (@@ -36,6 +37,7 @@ from products.tasks.backend.logic.services.sandbox_pricing import (validate_compute_rate_cards,validate_reporting_window,)+from products.tasks.backend.metrics import observe_active_session_cap_hitfrom products.tasks.backend.models import SandboxSession, Task, TaskClientProvenance, TaskRunlogger = structlog.get_logger(__name__)@@ -108,6 +110,50 @@ def measure_task_run_cpu_attribution(run_id: str | UUID, team_id: int) -> dict[sreturn measurements+def _lock_active_sessions(organization_id: UUID | str, created_by_id: int | None) -> None:+ # Call only inside transaction.atomic(): the locks hold until commit, so a cap count and the+ # attribution it guards cannot interleave with another start by the same creator or organization.+ # Always take the creator lock before the organization lock, so two transactions cannot deadlock.+ keys = [f"tasks-active-sessions:org:{organization_id}"]+ if created_by_id is not None:+ keys.insert(0, f"tasks-active-sessions:user:{created_by_id}")+ with connection.cursor() as cursor:+ for key in keys:+ cursor.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", [key])+++def enforce_active_session_cap(+ *,+ origin_product: str,+ organization_id: UUID | str,+ created_by_id: int | None,+ run_id: str | UUID | None,+ context: dict[str, Any],+ signal_report_id: UUID | str | None = None,+ task_id: str | UUID | None = None,+) -> None:+ from products.tasks.backend.logic.services.warm import ( # noqa: PLC0415 — warm imports the temporal client, which imports this module+ SandboxWarmer,+ )++ scope = SandboxWarmer.active_session_cap_hit(+ origin_product,+ organization_id=organization_id,+ created_by_id=created_by_id,+ exclude_run_id=run_id,+ signal_report_id=signal_report_id,+ task_id=task_id,+ )+ if scope is None:+ return+ observe_active_session_cap_hit(origin_product, scope)+ raise ActiveSessionCapError(+ context,+ scope,+ SandboxWarmer.active_session_cap_detail(origin_product, scope, signal_report_id=signal_report_id),+ )++def open_sandbox_session(*,run_id: str | UUID,@@ -124,11 +170,35 @@ def open_sandbox_session(with transaction.atomic():run = (TaskRun.objects.select_for_update(of=("self",))- .select_related("task")- .only("id", "team_id", "state", "task__origin_product", "task__client_provenance")+ .select_related("task", "team")+ .only(+ "id",+ "team_id",+ "task_id",+ "state",+ "task__origin_product",+ "task__client_provenance",+ "task__created_by_id",+ "task__signal_report_id",+ "team__organization_id",+ ).get(id=run_id))state = run.state or {}+ if (+ not state.get("await_user_message")+ and not SandboxSession.objects.for_team(run.team_id).filter(sandbox_id=sandbox_id).exists()+ ):+ _lock_active_sessions(run.team.organization_id, run.task.created_by_id)+ enforce_active_session_cap(+ origin_product=run.task.origin_product,+ organization_id=run.team.organization_id,+ created_by_id=run.task.created_by_id,+ run_id=run.id,+ context={"team_id": run.team_id, "task_id": str(run.task_id), "run_id": str(run.id)},+ signal_report_id=run.task.signal_report_id,+ task_id=run.task_id,+ )created_at = sandbox_created_at or timezone.now()shape = {"team_id": run.team_id,@@ -166,6 +236,8 @@ def open_sandbox_session(),},)+ except ActiveSessionCapError:+ raiseexcept Exception:logger.exception("sandbox_usage.ledger_write_failed", helper="open_sandbox_session")if required:@@ -254,6 +326,16 @@ def record_task_run_user_activity(run_uuid = run_id if isinstance(run_id, UUID) else UUID(run_id)open_sessions = SandboxSession.objects.for_team(team_id).filter(task_run_id=run_uuid, ended_at__isnull=True)open_sessions.update(last_user_activity_at=now)+ _stamp_user_attribution(open_sessions, run_uuid, team_id, cpu_attribution, now)+++def _stamp_user_attribution(+ open_sessions: QuerySet[SandboxSession],+ run_uuid: UUID,+ team_id: int,+ cpu_attribution: dict[str, SandboxCpuAttribution] | None,+ now: datetime,+) -> None:client_provenance = (TaskRun.objects.filter(id=run_uuid, team_id=team_id).values_list("task__client_provenance", flat=True).first())@@ -283,6 +365,52 @@ def record_task_run_user_activity()+def claim_task_run_sandbox_sessions(+ run_id: str | UUID,+ team_id: int,+ cpu_attribution: dict[str, SandboxCpuAttribution] | None = None,+) -> None:+ """Attribute the run's open warm sandboxes to its user, refusing at the active-session cap.++ A run with no open unattributed session is not a claim and returns at once, so the CPU+ sample is only taken for a claim. Raises ``ActiveSessionCapError`` and leaves the sessions+ unattributed when the creator or organization already holds the max open attributed sessions.+ """+ run_uuid = run_id if isinstance(run_id, UUID) else UUID(run_id)+ open_sessions = SandboxSession.objects.for_team(team_id).filter(task_run_id=run_uuid, ended_at__isnull=True)+ if not open_sessions.filter(user_attributed_at__isnull=True).exists():+ return+ if cpu_attribution is None:+ cpu_attribution = measure_task_run_cpu_attribution(run_id, team_id)+ run = (+ TaskRun.objects.select_related("task", "team")+ .only(+ "id",+ "team_id",+ "task_id",+ "task__origin_product",+ "task__created_by_id",+ "task__signal_report_id",+ "team__organization_id",+ )+ .get(id=run_uuid, team_id=team_id)+ )+ with transaction.atomic():+ _lock_active_sessions(run.team.organization_id, run.task.created_by_id)+ if not open_sessions.filter(user_attributed_at__isnull=True).exists():+ return+ enforce_active_session_cap(+ origin_product=run.task.origin_product,+ organization_id=run.team.organization_id,+ created_by_id=run.task.created_by_id,+ run_id=run_uuid,+ context={"team_id": team_id, "task_id": str(run.task_id), "run_id": str(run_uuid)},+ signal_report_id=run.task.signal_report_id,+ task_id=run.task_id,+ )+ _stamp_user_attribution(open_sessions, run_uuid, team_id, cpu_attribution, timezone.now())++@dataclass(frozen=True)class SandboxUsageByTeam:"""Raw per-team sandbox usage over a period, as (team_id, amount) rows."""diff --git a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py b/products/tasks/backend/logic/services/tests/test_sandbox_usage.pyindex 5deceb1d9df7..ff49fabbff0b 100644--- a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py+++ b/products/tasks/backend/logic/services/tests/test_sandbox_usage.py@@ -10,10 +10,14 @@ from parameterized import parameterizedfrom posthog.models.organization import Organizationfrom posthog.models.team.team import Team+from products.signals.backend.models import SignalReport+from products.tasks.backend.exceptions import ActiveSessionCapErrorfrom products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_sessionfrom products.tasks.backend.logic.services.sandbox import Sandbox, SandboxConfigfrom products.tasks.backend.logic.services.sandbox_pricing import ComputeRateCard, ComputeRateCardConfigurationErrorfrom products.tasks.backend.logic.services.sandbox_usage import (+ SandboxCpuAttribution,+ claim_task_run_sandbox_sessions,close_sandbox_session,get_billable_sandbox_compute_usage_by_team,get_task_sandbox_usage_by_team,@@ -21,6 +25,7 @@ from products.tasks.backend.logic.services.sandbox_usage import (open_sandbox_session,record_task_run_user_activity,)+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmerfrom products.tasks.backend.models import SandboxSession, Task, TaskClientProvenance, TaskRun@@ -325,6 +330,91 @@ class TestSandboxSessionWrites(SandboxUsageBase):assert SandboxSession.objects.unscoped().get(sandbox_id="sb-claim").user_attributed_at is None+class TestActiveSessionCapReservation(SandboxUsageBase):+ def setUp(self) -> None:+ super().setUp()+ caps = patch.dict(+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=1)},+ )+ caps.start()+ self.addCleanup(caps.stop)++ def _fill_the_cap(self) -> None:+ with patch.dict(+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=10, per_org=10)},+ ):+ open_sandbox_session(run_id=self._run().id, sandbox_id="sb-busy", config=_config())++ def test_cold_open_at_the_cap_is_refused_without_a_row(self) -> None:+ self._fill_the_cap()+ run = self._run()++ with self.assertRaises(ActiveSessionCapError) as error_context:+ open_sandbox_session(run_id=run.id, sandbox_id="sb-new", config=_config())++ assert error_context.exception.context["scope"] == "org"+ assert not SandboxSession.objects.unscoped().filter(sandbox_id="sb-new").exists()++ def test_cold_open_on_a_report_task_with_a_live_sandbox_is_refused(self) -> None:+ task = Task.objects.create(+ team=self.team,+ title="",+ description="",+ origin_product=Task.OriginProduct.SIGNAL_REPORT,+ signal_report=SignalReport.objects.create(team=self.team),+ created_by=self.user,+ )+ open_sandbox_session(run_id=task.create_run(mode="interactive").id, sandbox_id="sb-first", config=_config())+ second = task.create_run(mode="interactive")++ with self.assertRaises(ActiveSessionCapError) as error_context:+ open_sandbox_session(run_id=second.id, sandbox_id="sb-second", config=_config())++ assert error_context.exception.context["scope"] == "task"+ assert not SandboxSession.objects.unscoped().filter(sandbox_id="sb-second").exists()++ def test_cold_open_below_the_cap_creates_the_attributed_row(self) -> None:+ run = self._run()++ open_sandbox_session(run_id=run.id, sandbox_id="sb-new", config=_config())++ assert SandboxSession.objects.unscoped().get(sandbox_id="sb-new").user_attributed_at is not None++ @parameterized.expand([("warm_open", True), ("reprovision_update", False)])+ def test_open_at_the_cap_is_not_refused(self, _name: str, warm: bool) -> None:+ run = self._run(state={"await_user_message": True} if warm else None)+ if not warm:+ open_sandbox_session(run_id=run.id, sandbox_id="sb-own", config=_config())+ self._fill_the_cap()++ open_sandbox_session(run_id=run.id, sandbox_id="sb-own", config=_config(cpu_cores=2.0))++ assert SandboxSession.objects.unscoped().get(sandbox_id="sb-own").cpu_cores == 2.0++ @parameterized.expand([("below_cap", False), ("at_cap", True)])+ def test_warm_claim_reserves_capacity(self, _name: str, at_cap: bool) -> None:+ run = self._run(state={"await_user_message": True})+ open_sandbox_session(run_id=run.id, sandbox_id="sb-warm", config=_config())+ if at_cap:+ self._fill_the_cap()+ measured_at = datetime(2026, 1, 2, 10, tzinfo=UTC)+ attribution = {+ "sb-warm": SandboxCpuAttribution(cpu_usage_usec=1_000, billed_cpu_usage_usec=None, measured_at=measured_at)+ }++ if at_cap:+ with self.assertRaises(ActiveSessionCapError):+ claim_task_run_sandbox_sessions(run.id, self.team.id, attribution)+ else:+ claim_task_run_sandbox_sessions(run.id, self.team.id, attribution)++ session = SandboxSession.objects.unscoped().get(sandbox_id="sb-warm")+ assert session.user_attributed_at == (None if at_cap else measured_at)+ assert session.last_user_activity_at is None++class TestSandboxUsageAggregation(SandboxUsageBase):BEGIN = datetime(2026, 1, 2, tzinfo=UTC)END = datetime(2026, 1, 3, tzinfo=UTC)diff --git a/products/tasks/backend/logic/services/tests/test_warm.py b/products/tasks/backend/logic/services/tests/test_warm.pyindex 19d57cb1c274..1676d2dc1981 100644--- a/products/tasks/backend/logic/services/tests/test_warm.py+++ b/products/tasks/backend/logic/services/tests/test_warm.py@@ -1,18 +1,48 @@+from datetime import datetime, timedelta+from posthog.test.base import APIBaseTestfrom unittest.mock import patch+from django.utils import timezone+from parameterized import parameterizedfrom rest_framework.exceptions import PermissionDenied, Throttledfrom posthog.exceptions import QuotaLimitExceeded+from posthog.models.team import Team+from products.signals.backend.models import SignalReportfrom products.tasks.backend.access import PostHogAIAccessDecisionfrom products.tasks.backend.constants import DEFAULT_SANDBOX_WORKING_DIR-from products.tasks.backend.logic.services.warm import SandboxWarmer, WarmSourceChanged-from products.tasks.backend.models import Task, TaskRun+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmer, WarmSourceChanged+from products.tasks.backend.models import SandboxSession, Task, TaskRunWARM = "products.tasks.backend.logic.services.warm"_CAPS = SandboxWarmer.ORIGIN_PRODUCT_CAPS[Task.OriginProduct.POSTHOG_AI]+_ACTIVE_CAPS = SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.POSTHOG_AI]+++def _session(+ team: Team,+ run: TaskRun,+ *,+ prewarmed: bool = False,+ user_attributed_at: datetime | None = None,+ ended_at: datetime | None = None,+) -> SandboxSession:+ return SandboxSession.objects.unscoped().create(+ team=team,+ task_run=run,+ sandbox_id=f"sb-{SandboxSession.objects.unscoped().count()}-{run.id}",+ cpu_cores=4.0,+ memory_gb=16.0,+ ttl_seconds=3600,+ ttl_expires_at=timezone.now() + timedelta(hours=1),+ prewarmed=prewarmed,+ user_attributed_at=user_attributed_at,+ ended_at=ended_at,+ origin_product=run.task.origin_product,+ )def test_code_cloud_warm_pool_caps():@@ -218,6 +248,76 @@ class TestSandboxWarmerWarm(APIBaseTest):assert task.runs.count() == 0m_workflow.assert_not_called()+ @parameterized.expand(+ [+ ("four_unclaimed", 4, False, timedelta(minutes=10), None, None, False, False),+ ("five_unclaimed", 5, False, timedelta(minutes=10), None, None, False, True),+ ("five_claimed", 5, True, timedelta(minutes=10), None, None, False, False),+ ("five_ended_over_an_hour_ago", 5, False, timedelta(minutes=70), None, None, False, False),+ (+ "five_of_another_bucket",+ 5,+ False,+ timedelta(minutes=10),+ Task.OriginProduct.USER_CREATED,+ None,+ False,+ False,+ ),+ (+ "five_of_another_client_origin",+ 5,+ False,+ timedelta(minutes=10),+ Task.OriginProduct.ERROR_TRACKING,+ Task.OriginProduct.USER_CREATED,+ False,+ True,+ ),+ ("five_by_another_user", 5, False, timedelta(minutes=10), None, None, True, False),+ ]+ )+ def test_warm_churn_throttle(+ self,+ _name: str,+ sessions: int,+ claimed: bool,+ ended_ago: timedelta,+ churned_origin: str | None,+ task_origin: str | None,+ other_user: bool,+ throttled: bool,+ ) -> None:+ churned_run = self._task(+ origin=churned_origin or Task.OriginProduct.POSTHOG_AI,+ created_by=self._create_user("[email protected]") if other_user else None,+ ).create_run(mode="interactive")+ churned_run.status = TaskRun.Status.CANCELLED+ churned_run.save(update_fields=["status"])+ now = timezone.now()+ for _ in range(sessions):+ _session(+ self.team,+ churned_run,+ prewarmed=True,+ user_attributed_at=now - ended_ago - timedelta(minutes=1) if claimed else None,+ ended_at=now - ended_ago,+ )++ task = self._task(origin=task_origin or Task.OriginProduct.POSTHOG_AI)+ with (+ patch(f"{WARM}.execute_task_processing_workflow"),+ patch(f"{WARM}.is_team_limited", return_value=False),+ self.captureOnCommitCallbacks(execute=True),+ ):+ if throttled:+ with self.assertRaises(Throttled):+ SandboxWarmer(task, user=self.user).warm()+ else:+ SandboxWarmer(task, user=self.user).warm()++ assert task.runs.count() == (0 if throttled else 1)+class TestSandboxWarmerAtCapacity(APIBaseTest):def _warm_run_on_new_task(self, *, created_by=None, status=TaskRun.Status.QUEUED, warm=True) -> TaskRun:@@ -261,3 +361,170 @@ class TestSandboxWarmerAtCapacity(APIBaseTest):# The requesting user holds no warm runs, but the org cap is full.assert SandboxWarmer.at_capacity(origin, self.team, self.user) is True++ def test_active_session_cap_hit_counts_attributed_open_sessions_only(self) -> None:+ origin = Task.OriginProduct.POSTHOG_AI+ now = timezone.now()+ run = self._warm_run_on_new_task(warm=False)++ for _ in range(_ACTIVE_CAPS.per_org):+ _session(self.team, run, prewarmed=True)+ _session(self.team, run, user_attributed_at=now, ended_at=now)+ assert (+ SandboxWarmer.active_session_cap_hit(+ origin, organization_id=self.organization.id, created_by_id=self.user.pk+ )+ is None+ )++ for _ in range(_ACTIVE_CAPS.per_user):+ _session(self.team, run, user_attributed_at=now)+ assert (+ SandboxWarmer.active_session_cap_hit(+ origin, organization_id=self.organization.id, created_by_id=self.user.pk+ )+ == "user"+ )++ other = self._create_user("[email protected]")+ other_run = self._warm_run_on_new_task(created_by=other, warm=False)+ for _ in range(_ACTIVE_CAPS.per_org - _ACTIVE_CAPS.per_user):+ _session(self.team, other_run, user_attributed_at=now)+ third = self._create_user("[email protected]")+ assert (+ SandboxWarmer.active_session_cap_hit(origin, organization_id=self.organization.id, created_by_id=third.pk)+ == "org"+ )++ @parameterized.expand(+ [+ ("modal_live", "modal", timedelta(hours=1), TaskRun.Status.IN_PROGRESS, False, False, True),+ ("modal_past_ttl", "modal", -timedelta(hours=1), TaskRun.Status.IN_PROGRESS, False, False, False),+ (+ "hogland_past_ttl_live_run",+ "hogland",+ -timedelta(hours=1),+ TaskRun.Status.IN_PROGRESS,+ False,+ False,+ True,+ ),+ (+ "hogland_past_ttl_terminal_run",+ "hogland",+ -timedelta(hours=1),+ TaskRun.Status.COMPLETED,+ False,+ False,+ False,+ ),+ (+ "hogland_past_ttl_not_started_run",+ "hogland",+ -timedelta(hours=1),+ TaskRun.Status.NOT_STARTED,+ False,+ False,+ True,+ ),+ ("own_run_excluded", "modal", timedelta(hours=1), TaskRun.Status.IN_PROGRESS, True, False, False),+ ("deleted_task", "modal", timedelta(hours=1), TaskRun.Status.IN_PROGRESS, False, True, True),+ ]+ )+ def test_active_session_cap_hit_counts_live_sessions(+ self,+ _name: str,+ backend: str,+ ttl_from_now: timedelta,+ status: str,+ exclude_own: bool,+ deleted: bool,+ counts: bool,+ ) -> None:+ run = self._warm_run_on_new_task(warm=False, status=status)+ session = _session(self.team, run, user_attributed_at=timezone.now())+ SandboxSession.objects.unscoped().filter(id=session.id).update(+ sandbox_backend=backend, ttl_expires_at=timezone.now() + ttl_from_now+ )+ if deleted:+ Task.objects.filter(id=run.task_id).update(deleted=True)++ with patch.dict(+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,+ {Task.OriginProduct.POSTHOG_AI: ActiveSessionCaps(per_user=1, per_org=1)},+ ):+ scope = SandboxWarmer.active_session_cap_hit(+ Task.OriginProduct.POSTHOG_AI,+ organization_id=self.organization.id,+ created_by_id=self.user.pk,+ exclude_run_id=run.id if exclude_own else None,+ )++ assert scope == ("user" if counts else None)++ def test_active_session_cap_hit_caps_a_report_linked_task_to_one_live_sandbox(self) -> None:+ report = SignalReport.objects.create(team=self.team)+ task, sibling = (+ Task.objects.create(+ team=self.team,+ title="",+ description="",+ origin_product=Task.OriginProduct.SIGNAL_REPORT,+ signal_report=report,+ created_by=self.user,+ )+ for _ in range(2)+ )+ run = task.create_run(mode="interactive")+ _session(self.team, run, user_attributed_at=timezone.now())++ def hit(task_id, **kwargs):+ return SandboxWarmer.active_session_cap_hit(+ Task.OriginProduct.SIGNAL_REPORT,+ organization_id=self.organization.id,+ created_by_id=self.user.pk,+ signal_report_id=report.id,+ task_id=task_id,+ **kwargs,+ )++ assert hit(task.id) == "task"+ assert hit(task.id, exclude_run_id=run.id) is None+ assert hit(sibling.id) is None+ assert SandboxWarmer.active_session_cap_detail(+ Task.OriginProduct.SIGNAL_REPORT, "task", signal_report_id=report.id+ ) == ("This report task already has a cloud sandbox running. Wait for it to finish or stop it, then try again.")++ @parameterized.expand(+ [+ ("server_only_origin", Task.OriginProduct.SLACK, False, None),+ ("client_origin_shares_bucket", Task.OriginProduct.ERROR_TRACKING, False, "user"),+ ("bare_signal_report_shares_bucket", Task.OriginProduct.SIGNAL_REPORT, False, "user"),+ ("linked_signal_report", Task.OriginProduct.SIGNAL_REPORT, True, None),+ ]+ )+ def test_active_session_cap_hit_buckets_by_provenance(+ self, _name: str, origin: str, linked_report: bool, expected: str | None+ ) -> None:+ held = Task.objects.create(+ team=self.team,+ title="",+ description="",+ origin_product=Task.OriginProduct.USER_CREATED,+ created_by=self.user,+ )+ _session(self.team, held.create_run(mode="interactive"), user_attributed_at=timezone.now())+ report = SignalReport.objects.create(team=self.team) if linked_report else None++ with patch.dict(+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=100)},+ ):+ scope = SandboxWarmer.active_session_cap_hit(+ origin,+ organization_id=self.organization.id,+ created_by_id=self.user.pk,+ signal_report_id=report.id if report else None,+ )++ assert scope == expecteddiff --git a/products/tasks/backend/logic/services/warm.py b/products/tasks/backend/logic/services/warm.pyindex ae5d152eba11..47f909a152bc 100644--- a/products/tasks/backend/logic/services/warm.py+++ b/products/tasks/backend/logic/services/warm.py@@ -14,14 +14,18 @@ quota gate before it can warm.from collections.abc import Callablefrom dataclasses import dataclass-from typing import Any+from datetime import timedelta+from typing import Any, Literalfrom uuid import UUIDfrom django.db import transaction+from django.db.models import Q, QuerySet+from django.utils import timezoneimport structlogfrom rest_framework.exceptions import PermissionDenied, Throttled+from posthog.dataclasses import frozenfrom posthog.exceptions import QuotaLimitExceededfrom posthog.models.team import Teamfrom posthog.models.user import User@@ -29,7 +33,8 @@ from posthog.models.user import Userfrom products.tasks.backend.access import get_posthog_ai_access_decisionfrom products.tasks.backend.logic.services.compute_quota import organization_deactivatedfrom products.tasks.backend.logic.services.workflow_dispatch import WorkflowDispatchOptions, enqueue_or_start_workflow-from products.tasks.backend.models import Task, TaskRun+from products.tasks.backend.metrics import observe_warm_churn_throttled+from products.tasks.backend.models import SandboxSession, Task, TaskRunfrom products.tasks.backend.temporal.client import execute_task_processing_workflowfrom products.tasks.backend.temporal.process_task.utils import parse_run_state@@ -56,6 +61,35 @@ class WarmPoolCaps:per_org: int+@frozen+class ActiveSessionCaps:+ per_user: int+ per_org: int+++ActiveSessionCapScope = Literal["user", "org", "task"]++# Client-selectable origins share one cap bucket, so picking a different origin per task gives no fresh allowance.+_CLIENT_ORIGIN_PRODUCTS: frozenset[str] = (+ frozenset(Task.OriginProduct.values)+ - Task.SERVER_ONLY_ORIGIN_PRODUCTS+ - {Task.OriginProduct.POSTHOG_AI, Task.OriginProduct.SIGNAL_REPORT}+)+_CLIENT_ORIGIN_SESSIONS = Q(origin_product__in=_CLIENT_ORIGIN_PRODUCTS) | Q(+ origin_product=Task.OriginProduct.SIGNAL_REPORT, task_run__task__signal_report_id__isnull=True+)+++def _session_bucket(origin_product: str, signal_report_id: UUID | str | None) -> Q:+ if origin_product in _CLIENT_ORIGIN_PRODUCTS:+ return _CLIENT_ORIGIN_SESSIONS+ if origin_product == Task.OriginProduct.SIGNAL_REPORT:+ if signal_report_id is None:+ return _CLIENT_ORIGIN_SESSIONS+ return Q(origin_product=origin_product, task_run__task__signal_report_id__isnull=False)+ return Q(origin_product=origin_product)++def _ai_credits_checker(team: Team, user: User) -> None:if is_team_limited(team.api_token, QuotaResource.AI_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY):raise QuotaLimitExceeded(@@ -81,6 +115,8 @@ class SandboxWarmer:# A warm Run is non-terminal and idling; these are the statuses it can hold before activation._NON_TERMINAL_STATUSES: frozenset[str] = frozenset({TaskRun.Status.QUEUED, TaskRun.Status.IN_PROGRESS})+ # A run's status is client-writable, so a live box counts until its run is terminal, not only while it runs.+ _LIVE_RUN_STATUSES: frozenset[str] = _NON_TERMINAL_STATUSES | {TaskRun.Status.NOT_STARTED}# Maps an origin product to its pre-warm quota gate. A checker returns None or raises a DRF# APIException (the in-process caller can't return an HTTP Response, so the contract is exception-@@ -101,6 +137,13 @@ class SandboxWarmer:}_DEFAULT_CAPS: WarmPoolCaps = WarmPoolCaps(per_user=2, per_org=10)+ ORIGIN_ACTIVE_SESSION_CAPS: dict[str, ActiveSessionCaps] = {+ Task.OriginProduct.POSTHOG_AI: ActiveSessionCaps(per_user=6, per_org=30),+ Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=10, per_org=100),+ }++ WARM_CHURN_LIMIT_PER_HOUR: int = 5+def __init__(self, task: "Task", *, user: User) -> None:self.task = taskself.user = user@@ -144,6 +187,112 @@ class SandboxWarmer:return Truereturn warm_runs.filter(task__team__organization_id=team.organization_id).count() >= caps.per_org+ @classmethod+ def _active_session_caps(cls, origin_product: str, signal_report_id: UUID | str | None) -> ActiveSessionCaps | None:+ if origin_product in Task.SERVER_ONLY_ORIGIN_PRODUCTS:+ return None+ # A report-linked task, from the signals batch or a person starting work on a report, is bounded+ # per report by the signals task cap and to one live sandbox per task instead; a batch fans out+ # several tasks from one creator by design. A bare signal_report origin has no report to bound+ # it, so it falls in the client bucket.+ if origin_product == Task.OriginProduct.SIGNAL_REPORT and signal_report_id is not None:+ return None+ if origin_product == Task.OriginProduct.POSTHOG_AI:+ return cls.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.POSTHOG_AI]+ return cls.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.USER_CREATED]++ @classmethod+ def _live_attributed_sessions(+ cls, origin_product: str, signal_report_id: UUID | str | None, exclude_run_id: str | UUID | None+ ) -> QuerySet[SandboxSession]:+ open_sessions = (+ SandboxSession.objects.unscoped()+ .filter(+ _session_bucket(origin_product, signal_report_id),+ ended_at__isnull=True,+ user_attributed_at__isnull=False,+ )+ .filter(+ # The provider kills a Modal box at its TTL even when the close stamp never lands, so a+ # stale row must not lock anyone out. Hogland's TTL is an idle timeout that proxied+ # requests extend without touching the row, so a hogland row counts while its run is live.+ Q(ttl_expires_at__gt=timezone.now())+ | Q(sandbox_backend="hogland", task_run__status__in=cls._LIVE_RUN_STATUSES)+ )+ )+ if exclude_run_id is not None:+ open_sessions = open_sessions.exclude(+ task_run_id=exclude_run_id if isinstance(exclude_run_id, UUID) else UUID(exclude_run_id)+ )+ return open_sessions++ @classmethod+ def active_session_cap_hit(+ cls,+ origin_product: str,+ *,+ organization_id: UUID | str,+ created_by_id: int | None,+ exclude_run_id: str | UUID | None = None,+ signal_report_id: UUID | str | None = None,+ task_id: str | UUID | None = None,+ ) -> ActiveSessionCapScope | None:+ caps = cls._active_session_caps(origin_product, signal_report_id)+ if caps is None:+ report_linked = origin_product == Task.OriginProduct.SIGNAL_REPORT and signal_report_id is not None+ if not report_linked or task_id is None:+ return None+ # The per-report task cap bounds how many report tasks exist; this bounds the runs a caller+ # can stack on one of them, since a run endpoint call on a discussion task is unlimited.+ own_sessions = cls._live_attributed_sessions(origin_product, signal_report_id, exclude_run_id)+ return "task" if own_sessions.filter(task_run__task_id=task_id).exists() else None+ open_sessions = cls._live_attributed_sessions(origin_product, signal_report_id, exclude_run_id)+ if (+ created_by_id is not None+ and open_sessions.filter(task_run__task__created_by_id=created_by_id).count() >= caps.per_user+ ):+ return "user"+ if open_sessions.filter(team__organization_id=organization_id).count() >= caps.per_org:+ return "org"+ return None++ @classmethod+ def active_session_cap_detail(+ cls, origin_product: str, scope: ActiveSessionCapScope, *, signal_report_id: UUID | str | None = None+ ) -> str:+ if scope == "task":+ return (+ "This report task already has a cloud sandbox running. "+ "Wait for it to finish or stop it, then try again."+ )+ caps = cls._active_session_caps(origin_product, signal_report_id)+ if caps is None:+ raise ValueError(f"origin_product '{origin_product}' has no active-session cap")+ if scope == "user":+ return (+ f"You already have {caps.per_user} cloud sandboxes running. "+ "Wait for one to finish or stop one, then try again."+ )+ return (+ f"Your organization already has {caps.per_org} cloud sandboxes running. "+ "Wait for one to finish or stop one, then try again."+ )++ def _warm_churn_exceeded(self) -> bool:+ return (+ SandboxSession.objects.unscoped()+ .filter(+ _session_bucket(self.task.origin_product, self.task.signal_report_id),+ task_run__task__created_by_id=self.user.pk,+ prewarmed=True,+ user_attributed_at__isnull=True,+ ended_at__isnull=False,+ ended_at__gte=timezone.now() - timedelta(hours=1),+ )+ .count()+ >= self.WARM_CHURN_LIMIT_PER_HOUR+ )+def warm(self,*,@@ -164,12 +313,13 @@ class SandboxWarmer:runs don't open PRs). The Code-app caller passes ``True`` so the activated run opens a PR.Raises ``QuotaLimitExceeded`` (402) / ``PermissionDenied`` (403) when gated, and ``Throttled``- (429) when the warm pool is full.+ (429) when the warm pool is full or the user ended too many unclaimed warm sandboxes in the last hour."""# Quota is a gateway/cache call — check it before taking the row lock, never while holding it.origin_product = self.task.origin_productteam = self.task.teamself.enforce_quota(origin_product, team, self.user)+ churned = self._warm_churn_exceeded()pool_full = self.at_capacity(origin_product, team, self.user)new_run: TaskRun@@ -216,6 +366,9 @@ class SandboxWarmer:if pool_full:raise Throttled(detail="Warm-pool capacity reached. Release an idle warm session and try again.")+ if churned:+ observe_warm_churn_throttled(origin_product)+ raise Throttled(detail="Too many unused warm sandboxes in the last hour. Try again later.")run_state: dict[str, Any] = {"await_user_message": True,diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.pyindex 46fdef08be8d..b693ebc0688e 100644--- a/products/tasks/backend/metrics.py+++ b/products/tasks/backend/metrics.py@@ -365,6 +365,18 @@ TASK_RUN_FOLLOWUP_DELIVERY_FAILED_TOTAL = Counter(labelnames=["origin_product", "retryable"],)+ACTIVE_SESSION_CAP_HITS_TOTAL = Counter(+ "posthog_tasks_active_session_cap_hits_total",+ "Run starts and warm claims refused because the user or organization already holds the max open cloud sandbox sessions",+ labelnames=["origin_product", "scope"],+)++WARM_CHURN_THROTTLED_TOTAL = Counter(+ "posthog_tasks_warm_churn_throttled_total",+ "Warm requests refused because the user ended too many unclaimed warm sandboxes in the last hour",+ labelnames=["origin_product"],+)+SANDBOX_DEADLINE_BUCKETS = [5.0, 15.0, 30.0, 60.0, 120.0, 180.0, 300.0, 600.0]SANDBOX_DEADLINE_TOTAL = Counter(@@ -808,6 +820,14 @@ def observe_followup_delivery_failed(task_run: "TaskRun", *, retryable: bool) ->).inc()+def observe_active_session_cap_hit(origin_product: str, scope: str) -> None:+ ACTIVE_SESSION_CAP_HITS_TOTAL.labels(origin_product=_metric_label(origin_product), scope=scope).inc()+++def observe_warm_churn_throttled(origin_product: str) -> None:+ WARM_CHURN_THROTTLED_TOTAL.labels(origin_product=_metric_label(origin_product)).inc()++_SANDBOX_DEADLINE_OUTCOMES = {"rotated", "snapshot_only", "snapshot_failed", "routing_lost"}_SANDBOX_DEADLINE_REASONS = {"none",diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.pyindex a7a1a78aca88..d6dbaeb87715 100644--- a/products/tasks/backend/models.py+++ b/products/tasks/backend/models.py@@ -463,6 +463,45 @@ class Task(Taggable, DeletedMetaFields, models.Model):KNOWLEDGE_WRITER = "knowledge_writer", "Knowledge Writer"CONTEXT_LAYER = "context_layer", "Context Layer"+ # Origins set only by server-side flows. The task write serializer rejects them from API callers.+ SERVER_ONLY_ORIGIN_PRODUCTS: frozenset[str] = frozenset(+ {+ OriginProduct.SPACE_SETUP,+ OriginProduct.IMAGE_BUILDER,+ OriginProduct.EXPERIMENTS,+ OriginProduct.SIGNALS_SCOUT,+ OriginProduct.SIGNALS_SCOUT_SUGGESTIONS,+ OriginProduct.SUPPORT_REPLY,+ # Only the autoresearch training loop creates these, in-process. Nothing legitimately+ # sets the origin from outside, so reserve it before anything starts depending on it.+ OriginProduct.AUTORESEARCH,+ # Routes the run's LLM traffic to the unbilled `onboarding` gateway product, so a+ # forged origin would be free model access. Only create_wizard_cloud_run sets it,+ # behind its own rate limits and daily cap.+ OriginProduct.ONBOARDING,+ OriginProduct.ACCOUNT_AUDIT,+ # Exempt from the Desktop code-access gate on run endpoints, so a forged origin+ # would bypass the waitlist. Only the signals scout-chat endpoint sets it.+ OriginProduct.SIGNALS_CHAT,+ # Attributes the task to a workflow, which the workflow_tasks endpoint proves+ # via its service JWT. A forged origin would fake that provenance.+ OriginProduct.WORKFLOW,+ # Maps to the mintable `review_hog` gateway product, so a forged origin would+ # mint an internally funded scoped token. Only ReviewHog's executor sets it.+ OriginProduct.REVIEW_HOG,+ OriginProduct.TASK_ANALYSIS,+ # Maps to the mintable `slack_app` gateway product. Only the Slack app's server flows set it.+ OriginProduct.SLACK,+ # Internal business-knowledge sandbox runs. Only that product's sandbox endpoint sets it.+ OriginProduct.BUSINESS_KNOWLEDGE,+ # Maps to the billed `posthog_ai` gateway product. Only the metrics dashboard import sets it.+ OriginProduct.METRICS_IMPORT,+ # Owns the docs repository branch it pushes to. Only the knowledge writer intake sets it.+ OriginProduct.KNOWLEDGE_WRITER,+ OriginProduct.CONTEXT_LAYER,+ }+ )+# nosemgrep: prefer-uuid7-django-pk -- TODO: migrate to uuid7 or clarify intentid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+")diff --git a/products/tasks/backend/presentation/serializers.py b/products/tasks/backend/presentation/serializers.pyindex 6a111c18dc37..1852d9aaf0b1 100644--- a/products/tasks/backend/presentation/serializers.py+++ b/products/tasks/backend/presentation/serializers.py@@ -982,42 +982,7 @@ class TaskWriteSerializer(serializers.Serializer):def validate_origin_product(self, value):"""Reject internal-only origins that are set by server-side flows, never by API callers."""- reserved_origins = {- tasks_facade.TaskOriginProduct.SPACE_SETUP,- tasks_facade.TaskOriginProduct.IMAGE_BUILDER,- tasks_facade.TaskOriginProduct.EXPERIMENTS,- tasks_facade.TaskOriginProduct.SIGNALS_SCOUT,- tasks_facade.TaskOriginProduct.SIGNALS_SCOUT_SUGGESTIONS,- tasks_facade.TaskOriginProduct.SUPPORT_REPLY,- # Only the autoresearch training loop creates these, in-process. Nothing legitimately- # sets the origin from outside, so reserve it before anything starts depending on it.- tasks_facade.TaskOriginProduct.AUTORESEARCH,- # Routes the run's LLM traffic to the unbilled `onboarding` gateway product, so a- # forged origin would be free model access. Only create_wizard_cloud_run sets it,- # behind its own rate limits and daily cap.- tasks_facade.TaskOriginProduct.ONBOARDING,- tasks_facade.TaskOriginProduct.ACCOUNT_AUDIT,- # Exempt from the Desktop code-access gate on run endpoints, so a forged origin- # would bypass the waitlist. Only the signals scout-chat endpoint sets it.- tasks_facade.TaskOriginProduct.SIGNALS_CHAT,- # Attributes the task to a workflow, which the workflow_tasks endpoint proves- # via its service JWT. A forged origin would fake that provenance.- tasks_facade.TaskOriginProduct.WORKFLOW,- # Maps to the mintable `review_hog` gateway product, so a forged origin would- # mint an internally funded scoped token. Only ReviewHog's executor sets it.- tasks_facade.TaskOriginProduct.REVIEW_HOG,- tasks_facade.TaskOriginProduct.TASK_ANALYSIS,- # Maps to the mintable `slack_app` gateway product. Only the Slack app's server flows set it.- tasks_facade.TaskOriginProduct.SLACK,- # Internal business-knowledge sandbox runs. Only that product's sandbox endpoint sets it.- tasks_facade.TaskOriginProduct.BUSINESS_KNOWLEDGE,- # Maps to the billed `posthog_ai` gateway product. Only the metrics dashboard import sets it.- tasks_facade.TaskOriginProduct.METRICS_IMPORT,- # Owns the docs repository branch it pushes to. Only the knowledge writer intake sets it.- tasks_facade.TaskOriginProduct.KNOWLEDGE_WRITER,- tasks_facade.TaskOriginProduct.CONTEXT_LAYER,- }- if value in reserved_origins:+ if value in tasks_facade.TASK_SERVER_ONLY_ORIGIN_PRODUCTS:raise serializers.ValidationError(f"origin_product '{value}' is reserved for server-created tasks")return valuediff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.pyindex 92b2a60e0663..39e2a6f802d2 100644--- a/products/tasks/backend/presentation/views/api.py+++ b/products/tasks/backend/presentation/views/api.py@@ -949,6 +949,8 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):return compute_quota_limit_response(error.reason)except ReportTaskCapExceeded as error:return self._report_task_cap_response(error.detail)+ except tasks_facade.ActiveSessionCapRefused as error:+ return Response({"detail": error.detail, "code": error.code}, status=status.HTTP_400_BAD_REQUEST)except tasks_facade.WarmRunActivationUnavailable as error:return self._warm_activation_unavailable_response(error)finally:@@ -1894,6 +1896,8 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):)if error.kind == "detail":body: dict[str, Any] = {"detail": error.detail}+ if error.code is not None:+ body["code"] = error.codeif error.missing_artifact_ids is not None:body["missing_artifact_ids"] = error.missing_artifact_idsreturn Response(body, status=status.HTTP_400_BAD_REQUEST)diff --git a/products/tasks/backend/temporal/process_task/activities/forward_pending_message.py b/products/tasks/backend/temporal/process_task/activities/forward_pending_message.pyindex d53122f42b29..c522045fb2ea 100644--- a/products/tasks/backend/temporal/process_task/activities/forward_pending_message.py+++ b/products/tasks/backend/temporal/process_task/activities/forward_pending_message.py@@ -142,10 +142,13 @@ def forward_pending_user_message(run_id: str) -> None:refresh_agent_instructions_state(task_run, actor_user, reason="warm_activation")from products.tasks.backend.logic.services.sandbox_usage import ( # noqa: PLC0415 — matches the file's deferred-import pattern+ claim_task_run_sandbox_sessions,measure_task_run_cpu_attribution,)cpu_attribution = measure_task_run_cpu_attribution(run_id, task_run.team_id)+ if state.get("await_user_message"):+ claim_task_run_sandbox_sessions(run_id, task_run.team_id, cpu_attribution)result = send_user_message(task_run,pending_message,diff --git a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.pyindex bef359ee903c..7cd605b490f3 100644--- a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py@@ -64,6 +64,7 @@ from products.tasks.backend.logic.services.sandbox import (sandbox_repo_path,workload_for_origin_product,)+from products.tasks.backend.logic.services.sandbox_usage import enforce_active_session_capfrom products.tasks.backend.models import TASK_OWNERSHIP_VERSION_STATE_KEY, SandboxSnapshot, Task, TaskRunfrom products.tasks.backend.temporal.metrics import (StepTimer,@@ -909,6 +910,17 @@ def _create_sandbox_for_repository(input: CreateSandboxForRepositoryInput) -> Crraise ComputeBillingLimitError({"team_id": ctx.team_id, "task_id": ctx.task_id, "run_id": ctx.run_id}, reason)+ # Not atomic: open_sandbox_session re-checks under the active-session locks after the+ # create, so the loser of a race between two starts destroys the box it just provisioned.+ enforce_active_session_cap(+ origin_product=task.origin_product,+ organization_id=task.team.organization_id,+ created_by_id=task.created_by_id,+ run_id=ctx.run_id,+ context={"team_id": ctx.team_id, "task_id": ctx.task_id, "run_id": ctx.run_id},+ signal_report_id=task.signal_report_id,+ task_id=task.id,+ )_emit_image_source_log(ctx, prepared)log_with_activity_context(f"Provisioning sandbox from {prepared.image_source_label} (image build may take a few minutes on first run)",diff --git a/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.py b/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.pyindex 0884c4dacf2a..bf205a024d58 100644--- a/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.py@@ -14,7 +14,7 @@ from posthog.temporal.common.heartbeat_sync import HeartbeaterSyncfrom posthog.temporal.common.utils import close_db_connectionsfrom posthog.temporal.oauth import PosthogMcpScopes-from products.tasks.backend.exceptions import CredentialUnavailableError+from products.tasks.backend.exceptions import ActiveSessionCapError, CredentialUnavailableErrorfrom products.tasks.backend.feature_flags import run_stream_presence_gatedfrom products.tasks.backend.logic.services.agent_command import (FOLLOWUP_TIMEOUT_SECONDS,@@ -358,6 +358,23 @@ def _deliver_followup(input: SendFollowupToSandboxInput) -> str | None:_fail_when_sandbox_stopped(input.run_id, state, actor_user, task_run=task_run)+ from products.tasks.backend.logic.services.sandbox_usage import ( # noqa: PLC0415 — matches forward_pending_message's deferred-import pattern+ claim_task_run_sandbox_sessions,+ record_task_run_user_activity,+ )++ try:+ claim_task_run_sandbox_sessions(input.run_id, task_run.team_id)+ except ActiveSessionCapError as error:+ _write_error_and_complete(+ input.run_id,+ error.message,+ run_uses_dedicated_stream(task_run.state),+ run_stream_presence_gated(task_run.state),+ task_run.task.origin_product,+ )+ raise+# Rebind the sandbox's MCP session to this actor before the turn. On an# actor transition this must rebind or clear the prior session; if it can't,# fail closed rather than run the turn under the previous actor's creds.@@ -434,6 +451,8 @@ def _deliver_followup(input: SendFollowupToSandboxInput) -> str | None:message_id=input.message_id,steer=input.steer,)+ if result.turn_in_flight or (result.success and not (input.steer and _is_steer_declined(result.data))):+ record_task_run_user_activity(input.run_id, task_run.team_id)logger.info("send_followup_to_sandbox_attempted",run_id=input.run_id,diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.pyindex b4af8ab6ace8..c330df3da417 100644--- a/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py@@ -18,6 +18,7 @@ from asgiref.sync import async_to_syncfrom posthog.temporal.common.posthog_client import is_expected_activity_failurefrom products.tasks.backend.exceptions import (+ ActiveSessionCapError,ComputeBillingLimitError,OrganizationExecutionError,RepositoryAccessError,@@ -27,6 +28,7 @@ from products.tasks.backend.exceptions import ()from products.tasks.backend.logic.services.docker_sandbox import DockerSandboxfrom products.tasks.backend.logic.services.sandbox import ExecutionResult, SandboxTemplate+from products.tasks.backend.logic.services.warm import SandboxWarmerfrom products.tasks.backend.models import Taskfrom products.tasks.backend.temporal.metrics import modal_sandbox_backend_label, resume_mode_labelfrom products.tasks.backend.temporal.process_task.activities import provision_sandbox as provision_sandbox_module@@ -947,12 +949,55 @@ def test_blocked_organization_creates_no_sandbox(sandbox_class.return_value.create.assert_not_called()[email protected]("warm", [False, True])+def test_active_session_cap_refuses_a_cold_start_before_create(mocker, warm: bool) -> None:+ context = _context_for_desktop_bootstrap()+ context.state = {"await_user_message": warm}+ task = mocker.Mock(+ origin_product=Task.OriginProduct.USER_CREATED, created_by_id=7, signal_report_id=None, id="task-id"+ )+ task.team.organization_id = "organization-id"+ mocker.patch.object(provision_sandbox_module, "_load_task", return_value=task)+ mocker.patch.object(provision_sandbox_module, "get_compute_quota_denial_reason", return_value=None)+ cap_hit = mocker.patch.object(SandboxWarmer, "active_session_cap_hit", return_value="user")+ mocker.patch.object(provision_sandbox_module, "_emit_image_source_log")+ mocker.patch.object(provision_sandbox_module, "_apply_modal_network_policy")+ mocker.patch.object(provision_sandbox_module, "_build_sandbox_tags", return_value={})+ create = mocker.Mock(side_effect=RuntimeError("created"))+ mocker.patch.object(+ provision_sandbox_module, "get_sandbox_class_for_run_backend", return_value=mocker.Mock(create=create)+ )++ with pytest.raises(RuntimeError if warm else ActiveSessionCapError) as error:+ async_to_sync(provision_sandbox_module._create_sandbox_for_repository)(+ CreateSandboxForRepositoryInput(context=context, prepared=_prepared_for_create())+ )++ if warm:+ cap_hit.assert_not_called()+ create.assert_called_once()+ else:+ assert isinstance(error.value, ActiveSessionCapError)+ assert error.value.non_retryable is True+ assert error.value.message == SandboxWarmer.active_session_cap_detail(Task.OriginProduct.USER_CREATED, "user")+ cap_hit.assert_called_once_with(+ Task.OriginProduct.USER_CREATED,+ organization_id="organization-id",+ created_by_id=7,+ exclude_run_id="run-id",+ signal_report_id=None,+ task_id="task-id",+ )+ create.assert_not_called()++@pytest.mark.parametrize("failing_step,destroy_fails",[("get_connect_credentials", False),("get_connect_credentials", True),("start_cpu_billing_sampler", False),+ ("persist_sandbox_connection", False),],)def test_a_failure_after_create_destroys_the_fresh_sandbox(mocker, failing_step: str, destroy_fails: bool):@@ -973,9 +1018,14 @@ def test_a_failure_after_create_destroys_the_fresh_sandbox(mocker, failing_step:sandbox.launch_dev_stack_bootstrap.return_value = Falsesandbox.start_cpu_billing_sampler.return_value = Truesandbox.execute.return_value = ExecutionResult(stdout='{"version": "1.2.3"}', stderr="", exit_code=0)- getattr(sandbox, failing_step).side_effect = SandboxRateLimitedError(+ failure: Exception = SandboxRateLimitedError("Sandbox control plane is rate limited", {"sandbox_id": "sandbox-id", "operation": "create_connect_token"})+ if failing_step == "persist_sandbox_connection":+ failure = ActiveSessionCapError({"run_id": "run-id"}, "org", "Your organization is at its cap.")+ mocker.patch.object(provision_sandbox_module, "persist_sandbox_connection", side_effect=failure)+ else:+ getattr(sandbox, failing_step).side_effect = failureif destroy_fails:sandbox.destroy.side_effect = SandboxCleanupError("Failed to destroy sandbox", {"sandbox_id": "sandbox-id"}, cause=RuntimeError("terminate failed")@@ -996,7 +1046,7 @@ def test_a_failure_after_create_destroys_the_fresh_sandbox(mocker, failing_step:task_run = mocker.patch.object(provision_sandbox_module, "TaskRun")mocker.patch.object(provision_sandbox_module, "pinned_agent_version", return_value="1.2.3")- with pytest.raises(SandboxRateLimitedError):+ with pytest.raises(type(failure)):async_to_sync(provision_sandbox_module._create_sandbox_for_repository)(CreateSandboxForRepositoryInput(context=context, prepared=_prepared_for_create()))diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.py b/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.pyindex 2888db811153..de1991a7b264 100644--- a/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.py@@ -1,18 +1,24 @@+from datetime import timedeltafrom uuid import uuid4from posthog.test.base import BaseTestfrom unittest.mock import MagicMock, patchfrom django.test import SimpleTestCase+from django.utils import timezonefrom parameterized import parameterizedfrom temporalio.testing import ActivityEnvironment+from products.tasks.backend.exceptions import ActiveSessionCapError+from products.tasks.backend.logic.services.sandbox_usage import SandboxCpuAttribution+from products.tasks.backend.logic.services.warm import SandboxWarmerfrom products.tasks.backend.logic.stream.redis_stream import (TASK_RUN_STREAM_WATCHED_TIMEOUT,get_task_run_stream_key,get_task_run_stream_watched_key,)+from products.tasks.backend.models import SandboxSession, Task, TaskRunfrom products.tasks.backend.redis import get_tasks_stream_redis_syncfrom products.tasks.backend.temporal.process_task.activities.send_followup_to_sandbox import (STEER_DECLINE_REASON_UNREPORTED,@@ -23,6 +29,8 @@ from products.tasks.backend.temporal.process_task.activities.send_followup_to_sasend_followup_to_sandbox,)+_ACTIVITY = "products.tasks.backend.temporal.process_task.activities.send_followup_to_sandbox"+class TestGetStopReason(SimpleTestCase):def test_returns_result_stop_reason_when_present(self):@@ -82,6 +90,8 @@ class TestWriteTurnComplete(BaseTest):client.delete(stream_key, watched_key)+@patch("products.tasks.backend.logic.services.sandbox_usage.record_task_run_user_activity", new=MagicMock())+@patch("products.tasks.backend.logic.services.sandbox_usage.claim_task_run_sandbox_sessions", new=MagicMock())class TestSendFollowupToSandbox(BaseTest):@patch("products.tasks.backend.temporal.process_task.activities.send_followup_to_sandbox.publish_task_run_stream_event"@@ -92,6 +102,8 @@ class TestSendFollowupToSandbox(BaseTest):self, mock_task_run_objects, mock_send_user_message, mock_publish):task_run = MagicMock()+ task_run.id = uuid4()+ task_run.team_id = self.team.idtask_run.state = {}task_run.task.created_by = Nonemock_task_run_objects.select_related.return_value.get.return_value = task_run@@ -125,7 +137,8 @@ class TestSendFollowupToSandbox(BaseTest):mock_refresh_sandbox_mcp,):task_run = MagicMock()- task_run.id = "run-123"+ task_run.id = uuid4()+ task_run.team_id = self.team.idtask_run.state = {"sandbox_id": "sandbox-123"}task_run.task.created_by_id = 42task_run.task.created_by = MagicMock(id=42)@@ -167,3 +180,94 @@ class TestSendFollowupToSandbox(BaseTest):steer=True,)mock_publish.assert_not_called()+++@patch(f"{_ACTIVITY}._refresh_sandbox_github", return_value=None)+@patch(f"{_ACTIVITY}._refresh_sandbox_mcp", return_value=None)+@patch(f"{_ACTIVITY}.create_sandbox_connection_token", return_value=None)+@patch(f"{_ACTIVITY}.publish_task_run_stream_event")+@patch("products.tasks.backend.logic.services.sandbox_usage.measure_task_run_cpu_attribution", return_value={})+@patch(f"{_ACTIVITY}.send_user_message")+class TestSendFollowupSandboxAttribution(BaseTest):+ def _run_with_session(self, *, attributed: bool) -> TaskRun:+ task = Task.objects.create(+ team=self.team,+ title="Warm",+ description="",+ origin_product=Task.OriginProduct.USER_CREATED,+ created_by=self.user,+ )+ run = TaskRun.objects.create(task=task, team=self.team, status=TaskRun.Status.IN_PROGRESS, state={})+ now = timezone.now()+ SandboxSession.objects.unscoped().create(+ team=self.team,+ task_run=run,+ sandbox_id=f"sb-{run.id}",+ cpu_cores=4.0,+ memory_gb=16.0,+ ttl_seconds=3600,+ ttl_expires_at=now + timedelta(hours=1),+ prewarmed=True,+ user_attributed_at=now if attributed else None,+ origin_product=task.origin_product,+ )+ return run++ @parameterized.expand([("warm_claim", False), ("already_attributed", True)])+ def test_delivery_stamps_user_activity(+ self, _name, attributed, mock_send, mock_measure, _publish, _token, _mcp, _github+ ) -> None:+ run = self._run_with_session(attributed=attributed)+ session = SandboxSession.objects.unscoped().get(task_run=run)+ measured_at = timezone.now() - timedelta(seconds=5)+ mock_measure.return_value = {+ session.sandbox_id: SandboxCpuAttribution(+ cpu_usage_usec=1_000, billed_cpu_usage_usec=None, measured_at=measured_at+ )+ }+ attributed_at_send = []++ def send(*args, **kwargs):+ attributed_at_send.append(SandboxSession.objects.unscoped().get(id=session.id).user_attributed_at)+ return MagicMock(success=True, turn_in_flight=False, data={"result": {}})++ mock_send.side_effect = send++ ActivityEnvironment().run(+ send_followup_to_sandbox, SendFollowupToSandboxInput(run_id=str(run.id), message="hi")+ )++ session.refresh_from_db()+ assert attributed_at_send == [session.user_attributed_at]+ assert (session.user_attributed_at == measured_at) is not attributed+ assert session.last_user_activity_at is not None+ assert mock_measure.called is not attributed++ def test_warm_claim_at_the_active_session_cap_is_refused(+ self, mock_send, _measure, _publish, _token, _mcp, _github+ ) -> None:+ busy_run = self._run_with_session(attributed=True)+ now = timezone.now()+ for i in range(SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.USER_CREATED].per_user - 1):+ SandboxSession.objects.unscoped().create(+ team=self.team,+ task_run=busy_run,+ sandbox_id=f"sb-busy-{i}",+ cpu_cores=4.0,+ memory_gb=16.0,+ ttl_seconds=3600,+ ttl_expires_at=now + timedelta(hours=1),+ user_attributed_at=now,+ origin_product=Task.OriginProduct.USER_CREATED,+ )+ run = self._run_with_session(attributed=False)++ with self.assertRaises(ActiveSessionCapError) as error_context:+ ActivityEnvironment().run(+ send_followup_to_sandbox, SendFollowupToSandboxInput(run_id=str(run.id), message="hi")+ )++ assert error_context.exception.non_retryable is True+ assert error_context.exception.context["scope"] == "user"+ mock_send.assert_not_called()+ assert SandboxSession.objects.unscoped().get(task_run=run).user_attributed_at is Nonediff --git a/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.py b/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.pyindex fa4438062626..da73375d4f0e 100644--- a/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.py+++ b/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.py@@ -18,8 +18,9 @@ from posthog.models.organization import Organizationfrom posthog.models.team.team import Teamfrom posthog.models.user import User-from products.tasks.backend.exceptions import OrganizationExecutionError+from products.tasks.backend.exceptions import ActiveSessionCapError, OrganizationExecutionErrorfrom products.tasks.backend.logic.services.sandbox_usage import SandboxCpuAttribution+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmerfrom products.tasks.backend.models import SandboxSession, Task, TaskRun_module = importlib.import_module("products.tasks.backend.temporal.process_task.activities.forward_pending_message")@@ -143,6 +144,55 @@ class TestForwardPendingUserMessage(TestCase):assert "pending_user_message" not in run.stateassert "pending_user_message_id" not in run.state+ @patch.dict(+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=10)},+ )+ @patch("products.tasks.backend.logic.services.agent_command.send_user_message")+ def test_warm_claim_at_the_active_session_cap_is_refused(self, mock_send: Mock) -> None:+ Task.objects.filter(id=self.task.id).update(origin_product=Task.OriginProduct.USER_CREATED)+ self.task.origin_product = Task.OriginProduct.USER_CREATED+ now = timezone.now()+ busy_run = self._make_run()+ SandboxSession.objects.unscoped().create(+ team=self.team,+ task_run=busy_run,+ sandbox_id="sb-busy",+ cpu_cores=4.0,+ memory_gb=16.0,+ ttl_seconds=3600,+ ttl_expires_at=now + timedelta(hours=1),+ user_attributed_at=now,+ origin_product=self.task.origin_product,+ )+ run = self._make_run(+ state={+ "await_user_message": True,+ "pending_user_message": "fix the tests",+ "sandbox_url": "https://sandbox.example.com/rpc",+ }+ )+ SandboxSession.objects.unscoped().create(+ team=self.team,+ task_run=run,+ sandbox_id="sb-warm",+ cpu_cores=4.0,+ memory_gb=16.0,+ ttl_seconds=3600,+ ttl_expires_at=now + timedelta(hours=1),+ prewarmed=True,+ origin_product=self.task.origin_product,+ )++ with self.assertRaises(ActiveSessionCapError) as error_context:+ forward_pending_user_message(str(run.id))++ assert error_context.exception.type == "ActiveSessionCapError"+ assert error_context.exception.non_retryable is True+ assert error_context.exception.context["scope"] == "user"+ mock_send.assert_not_called()+ assert SandboxSession.objects.unscoped().get(sandbox_id="sb-warm").user_attributed_at is None+@patch("products.tasks.backend.logic.services.store_skills.refresh_store_skills_state")@patch("products.tasks.backend.logic.services.sandbox_usage.measure_task_run_cpu_attribution")@patch("products.tasks.backend.logic.services.connection_token.create_sandbox_connection_token", return_value="jwt")@@ -167,8 +217,9 @@ class TestForwardPendingUserMessage(TestCase):ttl_seconds=3600,ttl_expires_at=timezone.now() + timedelta(seconds=3600),)- measured_at = timezone.now()+ measured_at = timezone.now() - timedelta(seconds=5)calls = []+ attributed_at_send = []def measure(*args):calls.append("measure")@@ -182,6 +233,7 @@ class TestForwardPendingUserMessage(TestCase):def send(*args, **kwargs):calls.append("send")+ attributed_at_send.append(SandboxSession.objects.unscoped().get(sandbox_id="sb-fwd").user_attributed_at)return _command_result(success=True, status_code=200)def refresh_store_skills(*args, **kwargs):@@ -194,7 +246,8 @@ class TestForwardPendingUserMessage(TestCase):forward_pending_user_message(str(run.id))session = SandboxSession.objects.unscoped().get(sandbox_id="sb-fwd")- assert session.user_attributed_at is not None+ assert attributed_at_send == [measured_at]+ assert session.user_attributed_at == measured_atassert session.last_user_activity_at is not Noneassert session.provider_cpu_usage_attribution_usec == 1_234_567assert session.provider_billed_cpu_usage_attribution_usec == 1_500_000diff --git a/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.py b/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.pyindex c2c66370a683..ac70d6c69aed 100644--- a/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.py+++ b/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.py@@ -47,6 +47,16 @@ def _clear_session_cache():cache.clear()[email protected](autouse=True)+def _no_sandbox_session_bookkeeping():+ """The mocked runs here have no sandbox session rows, so the claim and activity stamps are stubbed."""+ with (+ patch("products.tasks.backend.logic.services.sandbox_usage.claim_task_run_sandbox_sessions"),+ patch("products.tasks.backend.logic.services.sandbox_usage.record_task_run_user_activity"),+ ):+ yield++def _run_activity(activity_input: SendFollowupToSandboxInput) -> str | None:"""The activity opens `HeartbeaterSync`, which needs a Temporal activity context."""return ActivityEnvironment().run(send_followup_to_sandbox, activity_input)diff --git a/products/tasks/backend/temporal/process_task/tests/test_workflow.py b/products/tasks/backend/temporal/process_task/tests/test_workflow.pyindex 74dbbd484224..50a713e31b18 100644--- a/products/tasks/backend/temporal/process_task/tests/test_workflow.py+++ b/products/tasks/backend/temporal/process_task/tests/test_workflow.py@@ -1305,14 +1305,14 @@ class TestProcessTaskFollowupDispatch:class TestFollowupDeliveryFailureBookkeeping:- def _workflow(self, monkeypatch, *, mode: str):+ def _workflow(self, monkeypatch, *, mode: str, delivery_error: Exception | None = None):workflow = ProcessTaskWorkflow()workflow._context = _build_context(github_integration_id=123, state={"mode": mode})emitted: list[EmitProgressInput] = []async def fake_execute_activity(_activity, activity_input, **_kwargs):if isinstance(activity_input, SendFollowupToSandboxInput):- raise RuntimeError("delivery failed")+ raise delivery_error or RuntimeError("delivery failed")if isinstance(activity_input, EmitProgressInput):emitted.append(activity_input)return None@@ -1348,6 +1348,34 @@ class TestFollowupDeliveryFailureBookkeeping:assert workflow._completion_error_type == "followup_delivery_failed"assert emitted == []+ @pytest.mark.parametrize("mode", ["interactive", "background"])+ async def test_active_session_cap_on_delivery_fails_the_run(self, monkeypatch, mode: str) -> None:+ delivery_error = ActivityError(+ "Activity task failed",+ scheduled_event_id=10,+ started_event_id=11,+ identity="worker-1",+ activity_type="send_followup_to_sandbox",+ activity_id="activity-1",+ retry_state=RetryState.NON_RETRYABLE_FAILURE,+ )+ delivery_error.__cause__ = ApplicationError(+ "You already have 2 cloud sandboxes running.", type="ActiveSessionCapError"+ )+ workflow, emitted = self._workflow(monkeypatch, mode=mode, delivery_error=delivery_error)+ await workflow.send_followup_message("try this", [], "msg-1")+ assert any(key.endswith(":msg-1") for key in workflow._accepted_message_ids)++ result = await workflow._send_followup_to_sandbox("try this", [], message_id="msg-1")++ assert result is None+ assert workflow._task_completed is True+ assert workflow._completion_status == "failed"+ assert workflow._completion_error_type == "ActiveSessionCapError"+ assert workflow._completion_error == "You already have 2 cloud sandboxes running."+ assert emitted == []+ assert not any(key.endswith(":msg-1") for key in workflow._accepted_message_ids)+async def test_agent_shadow_result_is_reported_separately(monkeypatch):workflow_instance = ProcessTaskWorkflow()@@ -2435,18 +2463,31 @@ class TestProcessTaskWorkflowUnit:run_id="run-id", stream_generation="resumed-generation")- async def test_run_persists_activity_failure_cause_not_the_wrapper(self, monkeypatch):+ @pytest.mark.parametrize(+ "activity_type, error_type, message",+ [+ ("inject_fresh_tokens_on_resume", "SandboxNotRunningError", "Sandbox not in running state."),+ (+ "create_sandbox_for_repository",+ "ActiveSessionCapError",+ "You already have 10 cloud sandboxes running. Wait for one to finish or stop one, then try again.",+ ),+ ],+ )+ async def test_run_persists_activity_failure_cause_not_the_wrapper(+ self, monkeypatch, activity_type: str, error_type: str, message: str+ ):workflow = ProcessTaskWorkflow()error = ActivityError("Activity task failed",scheduled_event_id=10,started_event_id=11,identity="worker-1",- activity_type="inject_fresh_tokens_on_resume",+ activity_type=activity_type,activity_id="activity-1",retry_state=RetryState.MAXIMUM_ATTEMPTS_REACHED,)- error.__cause__ = ApplicationError("Sandbox not in running state.", type="SandboxNotRunningError")+ error.__cause__ = ApplicationError(message, type=error_type)update_task_run_status_mock = AsyncMock()monkeypatch.setattr(@@ -2461,12 +2502,12 @@ class TestProcessTaskWorkflowUnit:result = await workflow.run(ProcessTaskInput(run_id="run-id"))assert result.success is False- assert result.error == "Sandbox not in running state."+ assert result.error == messageupdate_task_run_status_mock.assert_awaited_with("failed",- error_message="Sandbox not in running state.",+ error_message=message,run_id="run-id",- error_type="SandboxNotRunningError",+ error_type=error_type,)async def test_run_skips_relay_when_sandbox_event_ingest_is_enabled(self, monkeypatch):diff --git a/products/tasks/backend/temporal/process_task/workflow.py b/products/tasks/backend/temporal/process_task/workflow.pyindex 41e5a07b40c8..e55dc8b5db82 100644--- a/products/tasks/backend/temporal/process_task/workflow.py+++ b/products/tasks/backend/temporal/process_task/workflow.py@@ -147,6 +147,7 @@ from .credential_refresh import CredentialRefreshExitReason, run_credential_refrfrom .slack_agent_design_relay import SlackAgentDesignRelayInput, SlackAgentDesignRelayWorkflowDEAD_SANDBOX_ERROR_TYPES = ("SandboxNotRunningError", "SandboxNotFoundError")+RUN_ENDING_FOLLOWUP_ERROR_TYPES = ("ActiveSessionCapError",)MAX_ACCEPTED_MESSAGE_IDS = 500_PATCH_ID_FAIL_RUN_ON_PRIMARY_CLONE_FAILURE = "tasks-fail-run-on-primary-clone-failure"_PATCH_ID_SHARED_STARTUP_TIMING = "tasks-shared-startup-timing"@@ -3748,12 +3749,24 @@ class ProcessTaskWorkflow(PostHogWorkflow):**error_properties,},)+ cause = e.cause if isinstance(e, temporalio.exceptions.ActivityError) else None+ run_ending = (+ isinstance(cause, temporalio.exceptions.ApplicationError)+ and cause.type in RUN_ENDING_FOLLOWUP_ERROR_TYPES+ )+ if message_id and (run_ending or self.context.mode == "interactive"):+ dedupe_key = _message_dedupe_key(message_id, actor_user_id, context)+ if dedupe_key in self._accepted_message_id_set:+ self._accepted_message_id_set.discard(dedupe_key)+ self._accepted_message_ids.remove(dedupe_key)+ if run_ending:+ assert isinstance(cause, temporalio.exceptions.ApplicationError)+ self._completion_status = "failed"+ self._completion_error = cause.message+ self._completion_error_type = cause.type+ self._task_completed = True+ return Noneif self.context.mode == "interactive":- if message_id:- dedupe_key = _message_dedupe_key(message_id, actor_user_id, context)- if dedupe_key in self._accepted_message_id_set:- self._accepted_message_id_set.discard(dedupe_key)- self._accepted_message_ids.remove(dedupe_key)if user_originated:# A user follow-up can arrive without a message_id, so the card can't hinge on# one; the generated group still gives the user a failure notice instead of adiff --git a/products/tasks/backend/tests/test_facade.py b/products/tasks/backend/tests/test_facade.pyindex 38058dacffbd..6ad69b175421 100644--- a/products/tasks/backend/tests/test_facade.py+++ b/products/tasks/backend/tests/test_facade.py@@ -37,12 +37,14 @@ from products.tasks.backend.facade.profiles import (TaskSource,get_task_profile,)+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmerfrom products.tasks.backend.models import (TASK_OWNERSHIP_VERSION_STATE_KEY,Channel,ChannelMembership,SandboxCustomImage,SandboxEnvironment,+ SandboxSession,Task,TaskRun,TaskWorkflowDispatch,@@ -1333,6 +1335,62 @@ class TestFacadeReadsAndMappers(TestCase):trigger.assert_not_called()assert not task.runs.exists()+ @parameterized.expand(+ [+ ("user_immediate", "user", False),+ ("user_scheduled", "user", True),+ ("org_immediate", "org", False),+ ]+ )+ def test_run_task_at_the_active_session_cap(self, _name: str, scope: str, scheduled: bool) -> None:+ caps = ActiveSessionCaps(per_user=2, per_org=3)+ holder = self.user+ if scope == "org":+ holder = User.objects.create_and_join(self.organization, "[email protected]", None)+ busy_run = TaskRun.objects.create(+ task=self._make_task(created_by=holder), team=self.team, status=TaskRun.Status.IN_PROGRESS+ )+ now = django_timezone.now()+ for i in range(caps.per_user if scope == "user" else caps.per_org):+ SandboxSession.objects.unscoped().create(+ team=self.team,+ task_run=busy_run,+ sandbox_id=f"sb-busy-{i}",+ cpu_cores=4.0,+ memory_gb=16.0,+ ttl_seconds=3600,+ ttl_expires_at=now + timedelta(hours=1),+ user_attributed_at=now,+ origin_product=Task.OriginProduct.USER_CREATED,+ )+ task = self._make_task()+ validated_data: dict[str, Any] = {"mode": "interactive"}+ if scheduled:+ validated_data["scheduled_at"] = now + timedelta(hours=2)++ with (+ patch.dict(SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS, {Task.OriginProduct.USER_CREATED: caps}),+ patch("products.tasks.backend.facade.api._trigger_task_processing_workflow", return_value=None) as trigger,+ ):+ result = facade.run_task(task.id, self.team.id, self.user.id, validated_data=validated_data)++ assert result is not None+ if scheduled:+ assert result.error is None+ assert task.runs.count() == 1+ else:+ assert result.error is not None+ self.assertEqual(+ result.error.detail,+ "You already have 2 cloud sandboxes running. Wait for one to finish or stop one, then try again."+ if scope == "user"+ else "Your organization already has 3 cloud sandboxes running. "+ "Wait for one to finish or stop one, then try again.",+ )+ assert result.error.code == "active_session_cap"+ trigger.assert_not_called()+ assert not task.runs.exists()+def test_run_task_resumed_from_an_import_run_marks_the_new_run(self):task = self._make_task()import_run = TaskRun.objects.create(diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.pyindex 333c60ec31cb..27f26bca4e5b 100644--- a/products/tasks/backend/tests/test_warm_facade.py+++ b/products/tasks/backend/tests/test_warm_facade.py@@ -28,11 +28,12 @@ from products.tasks.backend.logic.services.staged_artifacts import (build_task_artifact_entry,build_task_staged_artifact_cache_key,)-from products.tasks.backend.logic.services.warm import WarmResult+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmer, WarmResultfrom products.tasks.backend.models import (TASK_OWNERSHIP_VERSION_STATE_KEY,SandboxCustomImage,SandboxEnvironment,+ SandboxSession,Task,TaskRun,)@@ -1550,6 +1551,61 @@ class TestCreateTaskWarmReuse(APIBaseTest):run.refresh_from_db()assert run.state.get("await_user_message") is True+ def test_create_endpoint_refuses_a_warm_reuse_at_the_active_session_cap(self):+ _warm_task, run = self._warm_run()+ busy_task = Task.objects.create(+ team=self.team,+ title="",+ description="",+ origin_product=Task.OriginProduct.USER_CREATED,+ created_by=self.user,+ )+ busy_run = TaskRun.objects.create(task=busy_task, team=self.team, status=TaskRun.Status.IN_PROGRESS)+ now = django_timezone.now()+ SandboxSession.objects.unscoped().create(+ team=self.team,+ task_run=busy_run,+ sandbox_id="sb-busy",+ cpu_cores=4.0,+ memory_gb=16.0,+ ttl_seconds=3600,+ ttl_expires_at=now + timedelta(hours=1),+ user_attributed_at=now,+ origin_product=busy_task.origin_product,+ )++ with (+ patch.dict(+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=10)},+ ),+ patch(f"{FACADE}.signal_task_run_user_message", return_value=True) as m_signal,+ ):+ response = self.client.post(+ "/api/projects/@current/tasks/",+ {+ "description": "fix the bug",+ "repository": "posthog/posthog",+ "github_integration": self.integration.id,+ "branch": "main",+ },+ format="json",+ )++ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content+ self.assertEqual(+ response.json(),+ {+ "detail": "You already have 1 cloud sandboxes running. "+ "Wait for one to finish or stop one, then try again.",+ "code": "active_session_cap",+ },+ )+ m_signal.assert_not_called()+ run.refresh_from_db()+ assert run.state.get("await_user_message") is True+ assert Task.objects.filter(team=self.team).count() == 2+def test_create_endpoint_without_warm_hints_is_not_regated(self):# Only a create that can activate a sandbox takes the run-start gates; a plain create still# goes through, and the run endpoint gates it when execution is actually requested.diff --git a/products/tasks/frontend/generated/api.schemas.ts b/products/tasks/frontend/generated/api.schemas.tsindex d6256c416f53..c6f8eb0293c3 100644--- a/products/tasks/frontend/generated/api.schemas.ts+++ b/products/tasks/frontend/generated/api.schemas.ts@@ -3191,6 +3191,7 @@ export const RunStatusEnumApi = {* * `subscription_usage_limit` - subscription_usage_limit* * `task_spend_limit` - task_spend_limit* * `agent_error` - agent_error+ * * `ActiveSessionCapError` - ActiveSessionCapError*/export type ErrorTypeEnumApi = (typeof ErrorTypeEnumApi)[keyof typeof ErrorTypeEnumApi]@@ -3211,6 +3212,7 @@ export const ErrorTypeEnumApi = {SubscriptionUsageLimit: 'subscription_usage_limit',TaskSpendLimit: 'task_spend_limit',AgentError: 'agent_error',+ ActiveSessionCapError: 'ActiveSessionCapError',} as constexport interface PatchedTaskRunUpdateApi {@@ -3263,7 +3265,8 @@ export interface PatchedTaskRunUpdateApi {* * `turn_ended_without_response` - turn_ended_without_response* * `subscription_usage_limit` - subscription_usage_limit* * `task_spend_limit` - task_spend_limit- * * `agent_error` - agent_error */+ * * `agent_error` - agent_error+ * * `ActiveSessionCapError` - ActiveSessionCapError */error_type?: ErrorTypeEnumApi | null}diff --git a/products/tasks/frontend/generated/api.zod.ts b/products/tasks/frontend/generated/api.zod.tsindex 17c06452c83f..863f58b6abc3 100644--- a/products/tasks/frontend/generated/api.zod.ts+++ b/products/tasks/frontend/generated/api.zod.ts@@ -2063,15 +2063,16 @@ export const TasksRunsPartialUpdateBody = /* @__PURE__ */ zod.object({'subscription_usage_limit','task_spend_limit','agent_error',+ 'ActiveSessionCapError',]).describe(- '\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error'+ '\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error\n\* `ActiveSessionCapError` - ActiveSessionCapError'),zod.null(),]).optional().describe(- 'Why the run failed, set by the caller that marked it failed.\n\n\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error'+ 'Why the run failed, set by the caller that marked it failed.\n\n\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error\n\* `ActiveSessionCapError` - ActiveSessionCapError'),})diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 448bba577c83..da7e6898ef5f 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -40375,6 +40375,7 @@ export namespace Schemas {* * `subscription_usage_limit` - subscription_usage_limit* * `task_spend_limit` - task_spend_limit* * `agent_error` - agent_error+ * * `ActiveSessionCapError` - ActiveSessionCapError*/export type ErrorTypeEnum = typeof ErrorTypeEnum[keyof typeof ErrorTypeEnum];@@ -40396,6 +40397,7 @@ export namespace Schemas {SubscriptionUsageLimit: 'subscription_usage_limit',TaskSpendLimit: 'task_spend_limit',AgentError: 'agent_error',+ ActiveSessionCapError: 'ActiveSessionCapError',} as const;export interface EstimateExperimentScope {@@ -87607,7 +87609,8 @@ export namespace Schemas {* * `turn_ended_without_response` - turn_ended_without_response* * `subscription_usage_limit` - subscription_usage_limit* * `task_spend_limit` - task_spend_limit- * * `agent_error` - agent_error */+ * * `agent_error` - agent_error+ * * `ActiveSessionCapError` - ActiveSessionCapError */error_type?: ErrorTypeEnum | null;}