Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): cap active sandbox sessions and throttle warm churn (#128466)

Alessandro Pogliaghi67127d63bb63
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index c7304ac855e3..bcc13b1a6f0a 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9105,6 +9105,7 @@ export namespace Schemas {
* * `subscription_usage_limit` - subscription_usage_limit
* * `task_spend_limit` - task_spend_limit
* * `agent_error` - agent_error
+ * * `ActiveSessionCapError` - ActiveSessionCapError
*/
export type ErrorTypeEnum =
| "agent_reported"
@@ -9122,7 +9123,8 @@ export namespace Schemas {
| "turn_ended_without_response"
| "subscription_usage_limit"
| "task_spend_limit"
- | "agent_error";
+ | "agent_error"
+ | "ActiveSessionCapError";
/**
* * `active` - Active
* * `paused` - Paused
diff --git a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.ts b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.ts
index 8fd35db271a3..7c55906302de 100644
--- a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.ts
+++ b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.test.ts
@@ -745,6 +745,25 @@ describe('taskTrackerSceneLogic', () => {
}
)
+ it('toasts the active-session cap refusal and keeps the draft', async () => {
+ const detail =
+ 'You already have 10 cloud sandboxes running. Wait for one to finish or stop one, then try again.'
+ useMocks({
+ post: {
+ '/api/projects/:team/tasks/:id/run/': () => [400, { detail, code: 'active_session_cap' }],
+ },
+ })
+ const toast = jest.spyOn(lemonToast, 'error')
+ logic.mount()
+ logic.actions.setNewTaskData({ description: 'Inspect the example chart' })
+ await expectLogic(logic, () => logic.actions.submitNewTask()).toFinishAllListeners()
+
+ expect(toast).toHaveBeenCalledWith(detail)
+ expect(logic.values.isSubmittingTask).toBe(false)
+ expect(logic.values.newTaskData.description).toBe('Inspect the example chart')
+ toast.mockRestore()
+ })
+
// The seeded first message wraps the on-screen context, and the wrapped non-text refs must be marked
// sent under the created task's id — otherwise the run's first follow-up (sent via
// `runInteractionLogic`, which prunes against the task-scoped store) re-wraps the same refs.
diff --git a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.ts b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.ts
index 748d926cb60f..de72d659010a 100644
--- a/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.ts
+++ b/products/posthog_ai/frontend/scenes/TaskTracker/taskTrackerSceneLogic.ts
@@ -1303,6 +1303,9 @@ export const taskTrackerSceneLogic = kea<taskTrackerSceneLogicType>([
if (error instanceof ApiError && error.code === 'warm_run_activation_unavailable') {
lemonToast.error("Couldn't start this run yet. Please try again.")
}
+ if (error instanceof ApiError && error.code === 'active_session_cap' && error.detail) {
+ lemonToast.error(error.detail)
+ }
if (error instanceof CloudTargetUnresolvedError) {
lemonToast.error("Couldn't load your default environment. Please try again.")
}
diff --git a/products/tasks/backend/exceptions.py b/products/tasks/backend/exceptions.py
index 59dedfd6bb2e..efd96adc1615 100644
--- a/products/tasks/backend/exceptions.py
+++ b/products/tasks/backend/exceptions.py
@@ -147,6 +147,18 @@ class ComputeBillingLimitError(ProcessTaskError, ComputeBillingLimitExceeded):
)
+class ActiveSessionCapError(ProcessTaskError):
+ def __init__(self, context: dict[str, Any], scope: str, message: str):
+ self.scope = scope
+ super().__init__(
+ message,
+ {**context, "scope": scope},
+ None,
+ capture=False,
+ non_retryable=True,
+ )
+
+
class SandboxNetworkPolicyError(ProcessTaskFatalError):
pass
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index 918cfa180a77..3494e4d49816 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -235,6 +235,7 @@ _TASK_LOG_READ_EXECUTOR = ThreadPoolExecutor(max_workers=8, thread_name_prefix="
TaskRunStatus = TaskRun.Status
TaskRunEnvironment = TaskRun.Environment
TaskOriginProduct = Task.OriginProduct
+TASK_SERVER_ONLY_ORIGIN_PRODUCTS = Task.SERVER_ONLY_ORIGIN_PRODUCTS
TaskRuntime = Task.Runtime
SandboxNetworkAccessLevel = SandboxEnvironment.NetworkAccessLevel
SandboxSnapshotStatus = SandboxSnapshot.Status
@@ -260,10 +261,12 @@ __all__ = [
"SandboxNetworkAccessLevel",
"SandboxSnapshotStatus",
"TASK_RUN_SUMMARY_STATE_KEY",
+ "TASK_SERVER_ONLY_ORIGIN_PRODUCTS",
"TaskOriginProduct",
"TaskRuntime",
"TaskRunEnvironment",
"TaskRunStatus",
+ "ActiveSessionCapRefused",
"WarmRunActivationUnavailable",
"append_imported_task_run_log",
"append_task_run_log",
@@ -3210,6 +3213,7 @@ TASK_RUN_FAILURE_ERROR_TYPES = (
"subscription_usage_limit",
"task_spend_limit",
"agent_error",
+ "ActiveSessionCapError",
)
# `output.pr_merged` is GitHub's word, recorded by the PR webhook (`_record_run_pr_merged`) — never
@@ -8108,6 +8112,8 @@ def create_task(
raise ComputeBillingLimitError(
{"team_id": team_id, "task_id": str(warm_task.id), "run_id": str(warm_run.id)}, reason
)
+ if (cap_detail := _active_session_cap_refusal(warm_task)) is not None:
+ raise ActiveSessionCapRefused(cap_detail)
report_reservation = (
_reserve_report_warm_activation(
team,
@@ -8960,6 +8966,17 @@ REPORT_IMPLEMENTATION_REPOSITORY_UNRESOLVED = (
)
+ACTIVE_SESSION_CAP_CODE = "active_session_cap"
+
+
+class ActiveSessionCapRefused(Exception):
+ code = ACTIVE_SESSION_CAP_CODE
+
+ def __init__(self, detail: str) -> None:
+ self.detail = detail
+ super().__init__(detail)
+
+
class WarmRunActivationUnavailable(Exception):
code = "warm_run_activation_unavailable"
@@ -9564,6 +9581,27 @@ def _branches_worked_on(run: TaskRun) -> set[str]:
return branches
+def _active_session_cap_refusal(task: Task) -> str | None:
+ from products.tasks.backend.logic.services.warm import (
+ SandboxWarmer, # noqa: PLC0415 — keep warming deps off the api import path
+ )
+ from products.tasks.backend.metrics import (
+ observe_active_session_cap_hit, # noqa: PLC0415 — keep prometheus deps off the api import path
+ )
+
+ scope = SandboxWarmer.active_session_cap_hit(
+ task.origin_product,
+ organization_id=task.team.organization_id,
+ created_by_id=task.created_by_id,
+ signal_report_id=task.signal_report_id,
+ task_id=task.id,
+ )
+ if scope is None:
+ return None
+ observe_active_session_cap_hit(task.origin_product, scope)
+ return SandboxWarmer.active_session_cap_detail(task.origin_product, scope, signal_report_id=task.signal_report_id)
+
+
def run_task(
task_id: str | UUID,
team_id: int,
@@ -9611,6 +9649,12 @@ def run_task(
return contracts.TaskRunResult(
error=contracts.TaskValidationError(kind="detail", detail=REPORT_WARM_RUN_NOT_ACTIVATED)
)
+ if not validated_data.get("scheduled_at"):
+ cap_detail = _active_session_cap_refusal(task)
+ if cap_detail is not None:
+ return contracts.TaskRunResult(
+ error=contracts.TaskValidationError(kind="detail", detail=cap_detail, code=ACTIVE_SESSION_CAP_CODE)
+ )
report_id_for_slot_check = (
str(task.signal_report_id)
if task.signal_report_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT
diff --git a/products/tasks/backend/logic/services/sandbox_usage.py b/products/tasks/backend/logic/services/sandbox_usage.py
index 29bba05a9b2f..6beb2549da44 100644
--- a/products/tasks/backend/logic/services/sandbox_usage.py
+++ b/products/tasks/backend/logic/services/sandbox_usage.py
@@ -16,17 +16,18 @@ from dataclasses import dataclass
from datetime import datetime, timedelta
from decimal import ROUND_HALF_EVEN, Decimal
from functools import wraps
-from typing import ParamSpec, TypeVar
+from typing import Any, ParamSpec, TypeVar
from uuid import UUID
-from django.db import transaction
-from django.db.models import Case, F, Q, Value, When
+from django.db import connection, transaction
+from django.db.models import Case, F, Q, QuerySet, Value, When
from django.utils import timezone
import structlog
from posthog.dataclasses import frozen
+from products.tasks.backend.exceptions import ActiveSessionCapError
from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_session
from products.tasks.backend.logic.services.sandbox import SandboxBase, SandboxConfig, get_sandbox_class_for_sandbox_id
from products.tasks.backend.logic.services.sandbox_pricing import (
@@ -36,6 +37,7 @@ from products.tasks.backend.logic.services.sandbox_pricing import (
validate_compute_rate_cards,
validate_reporting_window,
)
+from products.tasks.backend.metrics import observe_active_session_cap_hit
from products.tasks.backend.models import SandboxSession, Task, TaskClientProvenance, TaskRun
logger = structlog.get_logger(__name__)
@@ -108,6 +110,50 @@ def measure_task_run_cpu_attribution(run_id: str | UUID, team_id: int) -> dict[s
return measurements
+def _lock_active_sessions(organization_id: UUID | str, created_by_id: int | None) -> None:
+ # Call only inside transaction.atomic(): the locks hold until commit, so a cap count and the
+ # attribution it guards cannot interleave with another start by the same creator or organization.
+ # Always take the creator lock before the organization lock, so two transactions cannot deadlock.
+ keys = [f"tasks-active-sessions:org:{organization_id}"]
+ if created_by_id is not None:
+ keys.insert(0, f"tasks-active-sessions:user:{created_by_id}")
+ with connection.cursor() as cursor:
+ for key in keys:
+ cursor.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", [key])
+
+
+def enforce_active_session_cap(
+ *,
+ origin_product: str,
+ organization_id: UUID | str,
+ created_by_id: int | None,
+ run_id: str | UUID | None,
+ context: dict[str, Any],
+ signal_report_id: UUID | str | None = None,
+ task_id: str | UUID | None = None,
+) -> None:
+ from products.tasks.backend.logic.services.warm import ( # noqa: PLC0415 — warm imports the temporal client, which imports this module
+ SandboxWarmer,
+ )
+
+ scope = SandboxWarmer.active_session_cap_hit(
+ origin_product,
+ organization_id=organization_id,
+ created_by_id=created_by_id,
+ exclude_run_id=run_id,
+ signal_report_id=signal_report_id,
+ task_id=task_id,
+ )
+ if scope is None:
+ return
+ observe_active_session_cap_hit(origin_product, scope)
+ raise ActiveSessionCapError(
+ context,
+ scope,
+ SandboxWarmer.active_session_cap_detail(origin_product, scope, signal_report_id=signal_report_id),
+ )
+
+
def open_sandbox_session(
*,
run_id: str | UUID,
@@ -124,11 +170,35 @@ def open_sandbox_session(
with transaction.atomic():
run = (
TaskRun.objects.select_for_update(of=("self",))
- .select_related("task")
- .only("id", "team_id", "state", "task__origin_product", "task__client_provenance")
+ .select_related("task", "team")
+ .only(
+ "id",
+ "team_id",
+ "task_id",
+ "state",
+ "task__origin_product",
+ "task__client_provenance",
+ "task__created_by_id",
+ "task__signal_report_id",
+ "team__organization_id",
+ )
.get(id=run_id)
)
state = run.state or {}
+ if (
+ not state.get("await_user_message")
+ and not SandboxSession.objects.for_team(run.team_id).filter(sandbox_id=sandbox_id).exists()
+ ):
+ _lock_active_sessions(run.team.organization_id, run.task.created_by_id)
+ enforce_active_session_cap(
+ origin_product=run.task.origin_product,
+ organization_id=run.team.organization_id,
+ created_by_id=run.task.created_by_id,
+ run_id=run.id,
+ context={"team_id": run.team_id, "task_id": str(run.task_id), "run_id": str(run.id)},
+ signal_report_id=run.task.signal_report_id,
+ task_id=run.task_id,
+ )
created_at = sandbox_created_at or timezone.now()
shape = {
"team_id": run.team_id,
@@ -166,6 +236,8 @@ def open_sandbox_session(
),
},
)
+ except ActiveSessionCapError:
+ raise
except Exception:
logger.exception("sandbox_usage.ledger_write_failed", helper="open_sandbox_session")
if required:
@@ -254,6 +326,16 @@ def record_task_run_user_activity(
run_uuid = run_id if isinstance(run_id, UUID) else UUID(run_id)
open_sessions = SandboxSession.objects.for_team(team_id).filter(task_run_id=run_uuid, ended_at__isnull=True)
open_sessions.update(last_user_activity_at=now)
+ _stamp_user_attribution(open_sessions, run_uuid, team_id, cpu_attribution, now)
+
+
+def _stamp_user_attribution(
+ open_sessions: QuerySet[SandboxSession],
+ run_uuid: UUID,
+ team_id: int,
+ cpu_attribution: dict[str, SandboxCpuAttribution] | None,
+ now: datetime,
+) -> None:
client_provenance = (
TaskRun.objects.filter(id=run_uuid, team_id=team_id).values_list("task__client_provenance", flat=True).first()
)
@@ -283,6 +365,52 @@ def record_task_run_user_activity(
)
+def claim_task_run_sandbox_sessions(
+ run_id: str | UUID,
+ team_id: int,
+ cpu_attribution: dict[str, SandboxCpuAttribution] | None = None,
+) -> None:
+ """Attribute the run's open warm sandboxes to its user, refusing at the active-session cap.
+
+ A run with no open unattributed session is not a claim and returns at once, so the CPU
+ sample is only taken for a claim. Raises ``ActiveSessionCapError`` and leaves the sessions
+ unattributed when the creator or organization already holds the max open attributed sessions.
+ """
+ run_uuid = run_id if isinstance(run_id, UUID) else UUID(run_id)
+ open_sessions = SandboxSession.objects.for_team(team_id).filter(task_run_id=run_uuid, ended_at__isnull=True)
+ if not open_sessions.filter(user_attributed_at__isnull=True).exists():
+ return
+ if cpu_attribution is None:
+ cpu_attribution = measure_task_run_cpu_attribution(run_id, team_id)
+ run = (
+ TaskRun.objects.select_related("task", "team")
+ .only(
+ "id",
+ "team_id",
+ "task_id",
+ "task__origin_product",
+ "task__created_by_id",
+ "task__signal_report_id",
+ "team__organization_id",
+ )
+ .get(id=run_uuid, team_id=team_id)
+ )
+ with transaction.atomic():
+ _lock_active_sessions(run.team.organization_id, run.task.created_by_id)
+ if not open_sessions.filter(user_attributed_at__isnull=True).exists():
+ return
+ enforce_active_session_cap(
+ origin_product=run.task.origin_product,
+ organization_id=run.team.organization_id,
+ created_by_id=run.task.created_by_id,
+ run_id=run_uuid,
+ context={"team_id": team_id, "task_id": str(run.task_id), "run_id": str(run_uuid)},
+ signal_report_id=run.task.signal_report_id,
+ task_id=run.task_id,
+ )
+ _stamp_user_attribution(open_sessions, run_uuid, team_id, cpu_attribution, timezone.now())
+
+
@dataclass(frozen=True)
class SandboxUsageByTeam:
"""Raw per-team sandbox usage over a period, as (team_id, amount) rows."""
diff --git a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py b/products/tasks/backend/logic/services/tests/test_sandbox_usage.py
index 5deceb1d9df7..ff49fabbff0b 100644
--- a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py
+++ b/products/tasks/backend/logic/services/tests/test_sandbox_usage.py
@@ -10,10 +10,14 @@ from parameterized import parameterized
from posthog.models.organization import Organization
from posthog.models.team.team import Team
+from products.signals.backend.models import SignalReport
+from products.tasks.backend.exceptions import ActiveSessionCapError
from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_session
from products.tasks.backend.logic.services.sandbox import Sandbox, SandboxConfig
from products.tasks.backend.logic.services.sandbox_pricing import ComputeRateCard, ComputeRateCardConfigurationError
from products.tasks.backend.logic.services.sandbox_usage import (
+ SandboxCpuAttribution,
+ claim_task_run_sandbox_sessions,
close_sandbox_session,
get_billable_sandbox_compute_usage_by_team,
get_task_sandbox_usage_by_team,
@@ -21,6 +25,7 @@ from products.tasks.backend.logic.services.sandbox_usage import (
open_sandbox_session,
record_task_run_user_activity,
)
+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmer
from products.tasks.backend.models import SandboxSession, Task, TaskClientProvenance, TaskRun
@@ -325,6 +330,91 @@ class TestSandboxSessionWrites(SandboxUsageBase):
assert SandboxSession.objects.unscoped().get(sandbox_id="sb-claim").user_attributed_at is None
+class TestActiveSessionCapReservation(SandboxUsageBase):
+ def setUp(self) -> None:
+ super().setUp()
+ caps = patch.dict(
+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,
+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=1)},
+ )
+ caps.start()
+ self.addCleanup(caps.stop)
+
+ def _fill_the_cap(self) -> None:
+ with patch.dict(
+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,
+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=10, per_org=10)},
+ ):
+ open_sandbox_session(run_id=self._run().id, sandbox_id="sb-busy", config=_config())
+
+ def test_cold_open_at_the_cap_is_refused_without_a_row(self) -> None:
+ self._fill_the_cap()
+ run = self._run()
+
+ with self.assertRaises(ActiveSessionCapError) as error_context:
+ open_sandbox_session(run_id=run.id, sandbox_id="sb-new", config=_config())
+
+ assert error_context.exception.context["scope"] == "org"
+ assert not SandboxSession.objects.unscoped().filter(sandbox_id="sb-new").exists()
+
+ def test_cold_open_on_a_report_task_with_a_live_sandbox_is_refused(self) -> None:
+ task = Task.objects.create(
+ team=self.team,
+ title="",
+ description="",
+ origin_product=Task.OriginProduct.SIGNAL_REPORT,
+ signal_report=SignalReport.objects.create(team=self.team),
+ created_by=self.user,
+ )
+ open_sandbox_session(run_id=task.create_run(mode="interactive").id, sandbox_id="sb-first", config=_config())
+ second = task.create_run(mode="interactive")
+
+ with self.assertRaises(ActiveSessionCapError) as error_context:
+ open_sandbox_session(run_id=second.id, sandbox_id="sb-second", config=_config())
+
+ assert error_context.exception.context["scope"] == "task"
+ assert not SandboxSession.objects.unscoped().filter(sandbox_id="sb-second").exists()
+
+ def test_cold_open_below_the_cap_creates_the_attributed_row(self) -> None:
+ run = self._run()
+
+ open_sandbox_session(run_id=run.id, sandbox_id="sb-new", config=_config())
+
+ assert SandboxSession.objects.unscoped().get(sandbox_id="sb-new").user_attributed_at is not None
+
+ @parameterized.expand([("warm_open", True), ("reprovision_update", False)])
+ def test_open_at_the_cap_is_not_refused(self, _name: str, warm: bool) -> None:
+ run = self._run(state={"await_user_message": True} if warm else None)
+ if not warm:
+ open_sandbox_session(run_id=run.id, sandbox_id="sb-own", config=_config())
+ self._fill_the_cap()
+
+ open_sandbox_session(run_id=run.id, sandbox_id="sb-own", config=_config(cpu_cores=2.0))
+
+ assert SandboxSession.objects.unscoped().get(sandbox_id="sb-own").cpu_cores == 2.0
+
+ @parameterized.expand([("below_cap", False), ("at_cap", True)])
+ def test_warm_claim_reserves_capacity(self, _name: str, at_cap: bool) -> None:
+ run = self._run(state={"await_user_message": True})
+ open_sandbox_session(run_id=run.id, sandbox_id="sb-warm", config=_config())
+ if at_cap:
+ self._fill_the_cap()
+ measured_at = datetime(2026, 1, 2, 10, tzinfo=UTC)
+ attribution = {
+ "sb-warm": SandboxCpuAttribution(cpu_usage_usec=1_000, billed_cpu_usage_usec=None, measured_at=measured_at)
+ }
+
+ if at_cap:
+ with self.assertRaises(ActiveSessionCapError):
+ claim_task_run_sandbox_sessions(run.id, self.team.id, attribution)
+ else:
+ claim_task_run_sandbox_sessions(run.id, self.team.id, attribution)
+
+ session = SandboxSession.objects.unscoped().get(sandbox_id="sb-warm")
+ assert session.user_attributed_at == (None if at_cap else measured_at)
+ assert session.last_user_activity_at is None
+
+
class TestSandboxUsageAggregation(SandboxUsageBase):
BEGIN = datetime(2026, 1, 2, tzinfo=UTC)
END = datetime(2026, 1, 3, tzinfo=UTC)
diff --git a/products/tasks/backend/logic/services/tests/test_warm.py b/products/tasks/backend/logic/services/tests/test_warm.py
index 19d57cb1c274..1676d2dc1981 100644
--- a/products/tasks/backend/logic/services/tests/test_warm.py
+++ b/products/tasks/backend/logic/services/tests/test_warm.py
@@ -1,18 +1,48 @@
+from datetime import datetime, timedelta
+
from posthog.test.base import APIBaseTest
from unittest.mock import patch
+from django.utils import timezone
+
from parameterized import parameterized
from rest_framework.exceptions import PermissionDenied, Throttled
from posthog.exceptions import QuotaLimitExceeded
+from posthog.models.team import Team
+from products.signals.backend.models import SignalReport
from products.tasks.backend.access import PostHogAIAccessDecision
from products.tasks.backend.constants import DEFAULT_SANDBOX_WORKING_DIR
-from products.tasks.backend.logic.services.warm import SandboxWarmer, WarmSourceChanged
-from products.tasks.backend.models import Task, TaskRun
+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmer, WarmSourceChanged
+from products.tasks.backend.models import SandboxSession, Task, TaskRun
WARM = "products.tasks.backend.logic.services.warm"
_CAPS = SandboxWarmer.ORIGIN_PRODUCT_CAPS[Task.OriginProduct.POSTHOG_AI]
+_ACTIVE_CAPS = SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.POSTHOG_AI]
+
+
+def _session(
+ team: Team,
+ run: TaskRun,
+ *,
+ prewarmed: bool = False,
+ user_attributed_at: datetime | None = None,
+ ended_at: datetime | None = None,
+) -> SandboxSession:
+ return SandboxSession.objects.unscoped().create(
+ team=team,
+ task_run=run,
+ sandbox_id=f"sb-{SandboxSession.objects.unscoped().count()}-{run.id}",
+ cpu_cores=4.0,
+ memory_gb=16.0,
+ ttl_seconds=3600,
+ ttl_expires_at=timezone.now() + timedelta(hours=1),
+ prewarmed=prewarmed,
+ user_attributed_at=user_attributed_at,
+ ended_at=ended_at,
+ origin_product=run.task.origin_product,
+ )
def test_code_cloud_warm_pool_caps():
@@ -218,6 +248,76 @@ class TestSandboxWarmerWarm(APIBaseTest):
assert task.runs.count() == 0
m_workflow.assert_not_called()
+ @parameterized.expand(
+ [
+ ("four_unclaimed", 4, False, timedelta(minutes=10), None, None, False, False),
+ ("five_unclaimed", 5, False, timedelta(minutes=10), None, None, False, True),
+ ("five_claimed", 5, True, timedelta(minutes=10), None, None, False, False),
+ ("five_ended_over_an_hour_ago", 5, False, timedelta(minutes=70), None, None, False, False),
+ (
+ "five_of_another_bucket",
+ 5,
+ False,
+ timedelta(minutes=10),
+ Task.OriginProduct.USER_CREATED,
+ None,
+ False,
+ False,
+ ),
+ (
+ "five_of_another_client_origin",
+ 5,
+ False,
+ timedelta(minutes=10),
+ Task.OriginProduct.ERROR_TRACKING,
+ Task.OriginProduct.USER_CREATED,
+ False,
+ True,
+ ),
+ ("five_by_another_user", 5, False, timedelta(minutes=10), None, None, True, False),
+ ]
+ )
+ def test_warm_churn_throttle(
+ self,
+ _name: str,
+ sessions: int,
+ claimed: bool,
+ ended_ago: timedelta,
+ churned_origin: str | None,
+ task_origin: str | None,
+ other_user: bool,
+ throttled: bool,
+ ) -> None:
+ churned_run = self._task(
+ origin=churned_origin or Task.OriginProduct.POSTHOG_AI,
+ created_by=self._create_user("[email protected]") if other_user else None,
+ ).create_run(mode="interactive")
+ churned_run.status = TaskRun.Status.CANCELLED
+ churned_run.save(update_fields=["status"])
+ now = timezone.now()
+ for _ in range(sessions):
+ _session(
+ self.team,
+ churned_run,
+ prewarmed=True,
+ user_attributed_at=now - ended_ago - timedelta(minutes=1) if claimed else None,
+ ended_at=now - ended_ago,
+ )
+
+ task = self._task(origin=task_origin or Task.OriginProduct.POSTHOG_AI)
+ with (
+ patch(f"{WARM}.execute_task_processing_workflow"),
+ patch(f"{WARM}.is_team_limited", return_value=False),
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ if throttled:
+ with self.assertRaises(Throttled):
+ SandboxWarmer(task, user=self.user).warm()
+ else:
+ SandboxWarmer(task, user=self.user).warm()
+
+ assert task.runs.count() == (0 if throttled else 1)
+
class TestSandboxWarmerAtCapacity(APIBaseTest):
def _warm_run_on_new_task(self, *, created_by=None, status=TaskRun.Status.QUEUED, warm=True) -> TaskRun:
@@ -261,3 +361,170 @@ class TestSandboxWarmerAtCapacity(APIBaseTest):
# The requesting user holds no warm runs, but the org cap is full.
assert SandboxWarmer.at_capacity(origin, self.team, self.user) is True
+
+ def test_active_session_cap_hit_counts_attributed_open_sessions_only(self) -> None:
+ origin = Task.OriginProduct.POSTHOG_AI
+ now = timezone.now()
+ run = self._warm_run_on_new_task(warm=False)
+
+ for _ in range(_ACTIVE_CAPS.per_org):
+ _session(self.team, run, prewarmed=True)
+ _session(self.team, run, user_attributed_at=now, ended_at=now)
+ assert (
+ SandboxWarmer.active_session_cap_hit(
+ origin, organization_id=self.organization.id, created_by_id=self.user.pk
+ )
+ is None
+ )
+
+ for _ in range(_ACTIVE_CAPS.per_user):
+ _session(self.team, run, user_attributed_at=now)
+ assert (
+ SandboxWarmer.active_session_cap_hit(
+ origin, organization_id=self.organization.id, created_by_id=self.user.pk
+ )
+ == "user"
+ )
+
+ other = self._create_user("[email protected]")
+ other_run = self._warm_run_on_new_task(created_by=other, warm=False)
+ for _ in range(_ACTIVE_CAPS.per_org - _ACTIVE_CAPS.per_user):
+ _session(self.team, other_run, user_attributed_at=now)
+ third = self._create_user("[email protected]")
+ assert (
+ SandboxWarmer.active_session_cap_hit(origin, organization_id=self.organization.id, created_by_id=third.pk)
+ == "org"
+ )
+
+ @parameterized.expand(
+ [
+ ("modal_live", "modal", timedelta(hours=1), TaskRun.Status.IN_PROGRESS, False, False, True),
+ ("modal_past_ttl", "modal", -timedelta(hours=1), TaskRun.Status.IN_PROGRESS, False, False, False),
+ (
+ "hogland_past_ttl_live_run",
+ "hogland",
+ -timedelta(hours=1),
+ TaskRun.Status.IN_PROGRESS,
+ False,
+ False,
+ True,
+ ),
+ (
+ "hogland_past_ttl_terminal_run",
+ "hogland",
+ -timedelta(hours=1),
+ TaskRun.Status.COMPLETED,
+ False,
+ False,
+ False,
+ ),
+ (
+ "hogland_past_ttl_not_started_run",
+ "hogland",
+ -timedelta(hours=1),
+ TaskRun.Status.NOT_STARTED,
+ False,
+ False,
+ True,
+ ),
+ ("own_run_excluded", "modal", timedelta(hours=1), TaskRun.Status.IN_PROGRESS, True, False, False),
+ ("deleted_task", "modal", timedelta(hours=1), TaskRun.Status.IN_PROGRESS, False, True, True),
+ ]
+ )
+ def test_active_session_cap_hit_counts_live_sessions(
+ self,
+ _name: str,
+ backend: str,
+ ttl_from_now: timedelta,
+ status: str,
+ exclude_own: bool,
+ deleted: bool,
+ counts: bool,
+ ) -> None:
+ run = self._warm_run_on_new_task(warm=False, status=status)
+ session = _session(self.team, run, user_attributed_at=timezone.now())
+ SandboxSession.objects.unscoped().filter(id=session.id).update(
+ sandbox_backend=backend, ttl_expires_at=timezone.now() + ttl_from_now
+ )
+ if deleted:
+ Task.objects.filter(id=run.task_id).update(deleted=True)
+
+ with patch.dict(
+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,
+ {Task.OriginProduct.POSTHOG_AI: ActiveSessionCaps(per_user=1, per_org=1)},
+ ):
+ scope = SandboxWarmer.active_session_cap_hit(
+ Task.OriginProduct.POSTHOG_AI,
+ organization_id=self.organization.id,
+ created_by_id=self.user.pk,
+ exclude_run_id=run.id if exclude_own else None,
+ )
+
+ assert scope == ("user" if counts else None)
+
+ def test_active_session_cap_hit_caps_a_report_linked_task_to_one_live_sandbox(self) -> None:
+ report = SignalReport.objects.create(team=self.team)
+ task, sibling = (
+ Task.objects.create(
+ team=self.team,
+ title="",
+ description="",
+ origin_product=Task.OriginProduct.SIGNAL_REPORT,
+ signal_report=report,
+ created_by=self.user,
+ )
+ for _ in range(2)
+ )
+ run = task.create_run(mode="interactive")
+ _session(self.team, run, user_attributed_at=timezone.now())
+
+ def hit(task_id, **kwargs):
+ return SandboxWarmer.active_session_cap_hit(
+ Task.OriginProduct.SIGNAL_REPORT,
+ organization_id=self.organization.id,
+ created_by_id=self.user.pk,
+ signal_report_id=report.id,
+ task_id=task_id,
+ **kwargs,
+ )
+
+ assert hit(task.id) == "task"
+ assert hit(task.id, exclude_run_id=run.id) is None
+ assert hit(sibling.id) is None
+ assert SandboxWarmer.active_session_cap_detail(
+ Task.OriginProduct.SIGNAL_REPORT, "task", signal_report_id=report.id
+ ) == ("This report task already has a cloud sandbox running. Wait for it to finish or stop it, then try again.")
+
+ @parameterized.expand(
+ [
+ ("server_only_origin", Task.OriginProduct.SLACK, False, None),
+ ("client_origin_shares_bucket", Task.OriginProduct.ERROR_TRACKING, False, "user"),
+ ("bare_signal_report_shares_bucket", Task.OriginProduct.SIGNAL_REPORT, False, "user"),
+ ("linked_signal_report", Task.OriginProduct.SIGNAL_REPORT, True, None),
+ ]
+ )
+ def test_active_session_cap_hit_buckets_by_provenance(
+ self, _name: str, origin: str, linked_report: bool, expected: str | None
+ ) -> None:
+ held = Task.objects.create(
+ team=self.team,
+ title="",
+ description="",
+ origin_product=Task.OriginProduct.USER_CREATED,
+ created_by=self.user,
+ )
+ _session(self.team, held.create_run(mode="interactive"), user_attributed_at=timezone.now())
+ report = SignalReport.objects.create(team=self.team) if linked_report else None
+
+ with patch.dict(
+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,
+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=100)},
+ ):
+ scope = SandboxWarmer.active_session_cap_hit(
+ origin,
+ organization_id=self.organization.id,
+ created_by_id=self.user.pk,
+ signal_report_id=report.id if report else None,
+ )
+
+ assert scope == expected
diff --git a/products/tasks/backend/logic/services/warm.py b/products/tasks/backend/logic/services/warm.py
index ae5d152eba11..47f909a152bc 100644
--- a/products/tasks/backend/logic/services/warm.py
+++ b/products/tasks/backend/logic/services/warm.py
@@ -14,14 +14,18 @@ quota gate before it can warm.
from collections.abc import Callable
from dataclasses import dataclass
-from typing import Any
+from datetime import timedelta
+from typing import Any, Literal
from uuid import UUID
from django.db import transaction
+from django.db.models import Q, QuerySet
+from django.utils import timezone
import structlog
from rest_framework.exceptions import PermissionDenied, Throttled
+from posthog.dataclasses import frozen
from posthog.exceptions import QuotaLimitExceeded
from posthog.models.team import Team
from posthog.models.user import User
@@ -29,7 +33,8 @@ from posthog.models.user import User
from products.tasks.backend.access import get_posthog_ai_access_decision
from products.tasks.backend.logic.services.compute_quota import organization_deactivated
from products.tasks.backend.logic.services.workflow_dispatch import WorkflowDispatchOptions, enqueue_or_start_workflow
-from products.tasks.backend.models import Task, TaskRun
+from products.tasks.backend.metrics import observe_warm_churn_throttled
+from products.tasks.backend.models import SandboxSession, Task, TaskRun
from products.tasks.backend.temporal.client import execute_task_processing_workflow
from products.tasks.backend.temporal.process_task.utils import parse_run_state
@@ -56,6 +61,35 @@ class WarmPoolCaps:
per_org: int
+@frozen
+class ActiveSessionCaps:
+ per_user: int
+ per_org: int
+
+
+ActiveSessionCapScope = Literal["user", "org", "task"]
+
+# Client-selectable origins share one cap bucket, so picking a different origin per task gives no fresh allowance.
+_CLIENT_ORIGIN_PRODUCTS: frozenset[str] = (
+ frozenset(Task.OriginProduct.values)
+ - Task.SERVER_ONLY_ORIGIN_PRODUCTS
+ - {Task.OriginProduct.POSTHOG_AI, Task.OriginProduct.SIGNAL_REPORT}
+)
+_CLIENT_ORIGIN_SESSIONS = Q(origin_product__in=_CLIENT_ORIGIN_PRODUCTS) | Q(
+ origin_product=Task.OriginProduct.SIGNAL_REPORT, task_run__task__signal_report_id__isnull=True
+)
+
+
+def _session_bucket(origin_product: str, signal_report_id: UUID | str | None) -> Q:
+ if origin_product in _CLIENT_ORIGIN_PRODUCTS:
+ return _CLIENT_ORIGIN_SESSIONS
+ if origin_product == Task.OriginProduct.SIGNAL_REPORT:
+ if signal_report_id is None:
+ return _CLIENT_ORIGIN_SESSIONS
+ return Q(origin_product=origin_product, task_run__task__signal_report_id__isnull=False)
+ return Q(origin_product=origin_product)
+
+
def _ai_credits_checker(team: Team, user: User) -> None:
if is_team_limited(team.api_token, QuotaResource.AI_CREDITS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY):
raise QuotaLimitExceeded(
@@ -81,6 +115,8 @@ class SandboxWarmer:
# A warm Run is non-terminal and idling; these are the statuses it can hold before activation.
_NON_TERMINAL_STATUSES: frozenset[str] = frozenset({TaskRun.Status.QUEUED, TaskRun.Status.IN_PROGRESS})
+ # A run's status is client-writable, so a live box counts until its run is terminal, not only while it runs.
+ _LIVE_RUN_STATUSES: frozenset[str] = _NON_TERMINAL_STATUSES | {TaskRun.Status.NOT_STARTED}
# Maps an origin product to its pre-warm quota gate. A checker returns None or raises a DRF
# APIException (the in-process caller can't return an HTTP Response, so the contract is exception-
@@ -101,6 +137,13 @@ class SandboxWarmer:
}
_DEFAULT_CAPS: WarmPoolCaps = WarmPoolCaps(per_user=2, per_org=10)
+ ORIGIN_ACTIVE_SESSION_CAPS: dict[str, ActiveSessionCaps] = {
+ Task.OriginProduct.POSTHOG_AI: ActiveSessionCaps(per_user=6, per_org=30),
+ Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=10, per_org=100),
+ }
+
+ WARM_CHURN_LIMIT_PER_HOUR: int = 5
+
def __init__(self, task: "Task", *, user: User) -> None:
self.task = task
self.user = user
@@ -144,6 +187,112 @@ class SandboxWarmer:
return True
return warm_runs.filter(task__team__organization_id=team.organization_id).count() >= caps.per_org
+ @classmethod
+ def _active_session_caps(cls, origin_product: str, signal_report_id: UUID | str | None) -> ActiveSessionCaps | None:
+ if origin_product in Task.SERVER_ONLY_ORIGIN_PRODUCTS:
+ return None
+ # A report-linked task, from the signals batch or a person starting work on a report, is bounded
+ # per report by the signals task cap and to one live sandbox per task instead; a batch fans out
+ # several tasks from one creator by design. A bare signal_report origin has no report to bound
+ # it, so it falls in the client bucket.
+ if origin_product == Task.OriginProduct.SIGNAL_REPORT and signal_report_id is not None:
+ return None
+ if origin_product == Task.OriginProduct.POSTHOG_AI:
+ return cls.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.POSTHOG_AI]
+ return cls.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.USER_CREATED]
+
+ @classmethod
+ def _live_attributed_sessions(
+ cls, origin_product: str, signal_report_id: UUID | str | None, exclude_run_id: str | UUID | None
+ ) -> QuerySet[SandboxSession]:
+ open_sessions = (
+ SandboxSession.objects.unscoped()
+ .filter(
+ _session_bucket(origin_product, signal_report_id),
+ ended_at__isnull=True,
+ user_attributed_at__isnull=False,
+ )
+ .filter(
+ # The provider kills a Modal box at its TTL even when the close stamp never lands, so a
+ # stale row must not lock anyone out. Hogland's TTL is an idle timeout that proxied
+ # requests extend without touching the row, so a hogland row counts while its run is live.
+ Q(ttl_expires_at__gt=timezone.now())
+ | Q(sandbox_backend="hogland", task_run__status__in=cls._LIVE_RUN_STATUSES)
+ )
+ )
+ if exclude_run_id is not None:
+ open_sessions = open_sessions.exclude(
+ task_run_id=exclude_run_id if isinstance(exclude_run_id, UUID) else UUID(exclude_run_id)
+ )
+ return open_sessions
+
+ @classmethod
+ def active_session_cap_hit(
+ cls,
+ origin_product: str,
+ *,
+ organization_id: UUID | str,
+ created_by_id: int | None,
+ exclude_run_id: str | UUID | None = None,
+ signal_report_id: UUID | str | None = None,
+ task_id: str | UUID | None = None,
+ ) -> ActiveSessionCapScope | None:
+ caps = cls._active_session_caps(origin_product, signal_report_id)
+ if caps is None:
+ report_linked = origin_product == Task.OriginProduct.SIGNAL_REPORT and signal_report_id is not None
+ if not report_linked or task_id is None:
+ return None
+ # The per-report task cap bounds how many report tasks exist; this bounds the runs a caller
+ # can stack on one of them, since a run endpoint call on a discussion task is unlimited.
+ own_sessions = cls._live_attributed_sessions(origin_product, signal_report_id, exclude_run_id)
+ return "task" if own_sessions.filter(task_run__task_id=task_id).exists() else None
+ open_sessions = cls._live_attributed_sessions(origin_product, signal_report_id, exclude_run_id)
+ if (
+ created_by_id is not None
+ and open_sessions.filter(task_run__task__created_by_id=created_by_id).count() >= caps.per_user
+ ):
+ return "user"
+ if open_sessions.filter(team__organization_id=organization_id).count() >= caps.per_org:
+ return "org"
+ return None
+
+ @classmethod
+ def active_session_cap_detail(
+ cls, origin_product: str, scope: ActiveSessionCapScope, *, signal_report_id: UUID | str | None = None
+ ) -> str:
+ if scope == "task":
+ return (
+ "This report task already has a cloud sandbox running. "
+ "Wait for it to finish or stop it, then try again."
+ )
+ caps = cls._active_session_caps(origin_product, signal_report_id)
+ if caps is None:
+ raise ValueError(f"origin_product '{origin_product}' has no active-session cap")
+ if scope == "user":
+ return (
+ f"You already have {caps.per_user} cloud sandboxes running. "
+ "Wait for one to finish or stop one, then try again."
+ )
+ return (
+ f"Your organization already has {caps.per_org} cloud sandboxes running. "
+ "Wait for one to finish or stop one, then try again."
+ )
+
+ def _warm_churn_exceeded(self) -> bool:
+ return (
+ SandboxSession.objects.unscoped()
+ .filter(
+ _session_bucket(self.task.origin_product, self.task.signal_report_id),
+ task_run__task__created_by_id=self.user.pk,
+ prewarmed=True,
+ user_attributed_at__isnull=True,
+ ended_at__isnull=False,
+ ended_at__gte=timezone.now() - timedelta(hours=1),
+ )
+ .count()
+ >= self.WARM_CHURN_LIMIT_PER_HOUR
+ )
+
def warm(
self,
*,
@@ -164,12 +313,13 @@ class SandboxWarmer:
runs don't open PRs). The Code-app caller passes ``True`` so the activated run opens a PR.
Raises ``QuotaLimitExceeded`` (402) / ``PermissionDenied`` (403) when gated, and ``Throttled``
- (429) when the warm pool is full.
+ (429) when the warm pool is full or the user ended too many unclaimed warm sandboxes in the last hour.
"""
# Quota is a gateway/cache call — check it before taking the row lock, never while holding it.
origin_product = self.task.origin_product
team = self.task.team
self.enforce_quota(origin_product, team, self.user)
+ churned = self._warm_churn_exceeded()
pool_full = self.at_capacity(origin_product, team, self.user)
new_run: TaskRun
@@ -216,6 +366,9 @@ class SandboxWarmer:
if pool_full:
raise Throttled(detail="Warm-pool capacity reached. Release an idle warm session and try again.")
+ if churned:
+ observe_warm_churn_throttled(origin_product)
+ raise Throttled(detail="Too many unused warm sandboxes in the last hour. Try again later.")
run_state: dict[str, Any] = {
"await_user_message": True,
diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py
index 46fdef08be8d..b693ebc0688e 100644
--- a/products/tasks/backend/metrics.py
+++ b/products/tasks/backend/metrics.py
@@ -365,6 +365,18 @@ TASK_RUN_FOLLOWUP_DELIVERY_FAILED_TOTAL = Counter(
labelnames=["origin_product", "retryable"],
)
+ACTIVE_SESSION_CAP_HITS_TOTAL = Counter(
+ "posthog_tasks_active_session_cap_hits_total",
+ "Run starts and warm claims refused because the user or organization already holds the max open cloud sandbox sessions",
+ labelnames=["origin_product", "scope"],
+)
+
+WARM_CHURN_THROTTLED_TOTAL = Counter(
+ "posthog_tasks_warm_churn_throttled_total",
+ "Warm requests refused because the user ended too many unclaimed warm sandboxes in the last hour",
+ labelnames=["origin_product"],
+)
+
SANDBOX_DEADLINE_BUCKETS = [5.0, 15.0, 30.0, 60.0, 120.0, 180.0, 300.0, 600.0]
SANDBOX_DEADLINE_TOTAL = Counter(
@@ -808,6 +820,14 @@ def observe_followup_delivery_failed(task_run: "TaskRun", *, retryable: bool) ->
).inc()
+def observe_active_session_cap_hit(origin_product: str, scope: str) -> None:
+ ACTIVE_SESSION_CAP_HITS_TOTAL.labels(origin_product=_metric_label(origin_product), scope=scope).inc()
+
+
+def observe_warm_churn_throttled(origin_product: str) -> None:
+ WARM_CHURN_THROTTLED_TOTAL.labels(origin_product=_metric_label(origin_product)).inc()
+
+
_SANDBOX_DEADLINE_OUTCOMES = {"rotated", "snapshot_only", "snapshot_failed", "routing_lost"}
_SANDBOX_DEADLINE_REASONS = {
"none",
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index a7a1a78aca88..d6dbaeb87715 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -463,6 +463,45 @@ class Task(Taggable, DeletedMetaFields, models.Model):
KNOWLEDGE_WRITER = "knowledge_writer", "Knowledge Writer"
CONTEXT_LAYER = "context_layer", "Context Layer"
+ # Origins set only by server-side flows. The task write serializer rejects them from API callers.
+ SERVER_ONLY_ORIGIN_PRODUCTS: frozenset[str] = frozenset(
+ {
+ OriginProduct.SPACE_SETUP,
+ OriginProduct.IMAGE_BUILDER,
+ OriginProduct.EXPERIMENTS,
+ OriginProduct.SIGNALS_SCOUT,
+ OriginProduct.SIGNALS_SCOUT_SUGGESTIONS,
+ OriginProduct.SUPPORT_REPLY,
+ # Only the autoresearch training loop creates these, in-process. Nothing legitimately
+ # sets the origin from outside, so reserve it before anything starts depending on it.
+ OriginProduct.AUTORESEARCH,
+ # Routes the run's LLM traffic to the unbilled `onboarding` gateway product, so a
+ # forged origin would be free model access. Only create_wizard_cloud_run sets it,
+ # behind its own rate limits and daily cap.
+ OriginProduct.ONBOARDING,
+ OriginProduct.ACCOUNT_AUDIT,
+ # Exempt from the Desktop code-access gate on run endpoints, so a forged origin
+ # would bypass the waitlist. Only the signals scout-chat endpoint sets it.
+ OriginProduct.SIGNALS_CHAT,
+ # Attributes the task to a workflow, which the workflow_tasks endpoint proves
+ # via its service JWT. A forged origin would fake that provenance.
+ OriginProduct.WORKFLOW,
+ # Maps to the mintable `review_hog` gateway product, so a forged origin would
+ # mint an internally funded scoped token. Only ReviewHog's executor sets it.
+ OriginProduct.REVIEW_HOG,
+ OriginProduct.TASK_ANALYSIS,
+ # Maps to the mintable `slack_app` gateway product. Only the Slack app's server flows set it.
+ OriginProduct.SLACK,
+ # Internal business-knowledge sandbox runs. Only that product's sandbox endpoint sets it.
+ OriginProduct.BUSINESS_KNOWLEDGE,
+ # Maps to the billed `posthog_ai` gateway product. Only the metrics dashboard import sets it.
+ OriginProduct.METRICS_IMPORT,
+ # Owns the docs repository branch it pushes to. Only the knowledge writer intake sets it.
+ OriginProduct.KNOWLEDGE_WRITER,
+ OriginProduct.CONTEXT_LAYER,
+ }
+ )
+
# nosemgrep: prefer-uuid7-django-pk -- TODO: migrate to uuid7 or clarify intent
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+")
diff --git a/products/tasks/backend/presentation/serializers.py b/products/tasks/backend/presentation/serializers.py
index 6a111c18dc37..1852d9aaf0b1 100644
--- a/products/tasks/backend/presentation/serializers.py
+++ b/products/tasks/backend/presentation/serializers.py
@@ -982,42 +982,7 @@ class TaskWriteSerializer(serializers.Serializer):
def validate_origin_product(self, value):
"""Reject internal-only origins that are set by server-side flows, never by API callers."""
- reserved_origins = {
- tasks_facade.TaskOriginProduct.SPACE_SETUP,
- tasks_facade.TaskOriginProduct.IMAGE_BUILDER,
- tasks_facade.TaskOriginProduct.EXPERIMENTS,
- tasks_facade.TaskOriginProduct.SIGNALS_SCOUT,
- tasks_facade.TaskOriginProduct.SIGNALS_SCOUT_SUGGESTIONS,
- tasks_facade.TaskOriginProduct.SUPPORT_REPLY,
- # Only the autoresearch training loop creates these, in-process. Nothing legitimately
- # sets the origin from outside, so reserve it before anything starts depending on it.
- tasks_facade.TaskOriginProduct.AUTORESEARCH,
- # Routes the run's LLM traffic to the unbilled `onboarding` gateway product, so a
- # forged origin would be free model access. Only create_wizard_cloud_run sets it,
- # behind its own rate limits and daily cap.
- tasks_facade.TaskOriginProduct.ONBOARDING,
- tasks_facade.TaskOriginProduct.ACCOUNT_AUDIT,
- # Exempt from the Desktop code-access gate on run endpoints, so a forged origin
- # would bypass the waitlist. Only the signals scout-chat endpoint sets it.
- tasks_facade.TaskOriginProduct.SIGNALS_CHAT,
- # Attributes the task to a workflow, which the workflow_tasks endpoint proves
- # via its service JWT. A forged origin would fake that provenance.
- tasks_facade.TaskOriginProduct.WORKFLOW,
- # Maps to the mintable `review_hog` gateway product, so a forged origin would
- # mint an internally funded scoped token. Only ReviewHog's executor sets it.
- tasks_facade.TaskOriginProduct.REVIEW_HOG,
- tasks_facade.TaskOriginProduct.TASK_ANALYSIS,
- # Maps to the mintable `slack_app` gateway product. Only the Slack app's server flows set it.
- tasks_facade.TaskOriginProduct.SLACK,
- # Internal business-knowledge sandbox runs. Only that product's sandbox endpoint sets it.
- tasks_facade.TaskOriginProduct.BUSINESS_KNOWLEDGE,
- # Maps to the billed `posthog_ai` gateway product. Only the metrics dashboard import sets it.
- tasks_facade.TaskOriginProduct.METRICS_IMPORT,
- # Owns the docs repository branch it pushes to. Only the knowledge writer intake sets it.
- tasks_facade.TaskOriginProduct.KNOWLEDGE_WRITER,
- tasks_facade.TaskOriginProduct.CONTEXT_LAYER,
- }
- if value in reserved_origins:
+ if value in tasks_facade.TASK_SERVER_ONLY_ORIGIN_PRODUCTS:
raise serializers.ValidationError(f"origin_product '{value}' is reserved for server-created tasks")
return value
diff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.py
index 92b2a60e0663..39e2a6f802d2 100644
--- a/products/tasks/backend/presentation/views/api.py
+++ b/products/tasks/backend/presentation/views/api.py
@@ -949,6 +949,8 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
return compute_quota_limit_response(error.reason)
except ReportTaskCapExceeded as error:
return self._report_task_cap_response(error.detail)
+ except tasks_facade.ActiveSessionCapRefused as error:
+ return Response({"detail": error.detail, "code": error.code}, status=status.HTTP_400_BAD_REQUEST)
except tasks_facade.WarmRunActivationUnavailable as error:
return self._warm_activation_unavailable_response(error)
finally:
@@ -1894,6 +1896,8 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
)
if error.kind == "detail":
body: dict[str, Any] = {"detail": error.detail}
+ if error.code is not None:
+ body["code"] = error.code
if error.missing_artifact_ids is not None:
body["missing_artifact_ids"] = error.missing_artifact_ids
return Response(body, status=status.HTTP_400_BAD_REQUEST)
diff --git a/products/tasks/backend/temporal/process_task/activities/forward_pending_message.py b/products/tasks/backend/temporal/process_task/activities/forward_pending_message.py
index d53122f42b29..c522045fb2ea 100644
--- a/products/tasks/backend/temporal/process_task/activities/forward_pending_message.py
+++ b/products/tasks/backend/temporal/process_task/activities/forward_pending_message.py
@@ -142,10 +142,13 @@ def forward_pending_user_message(run_id: str) -> None:
refresh_agent_instructions_state(task_run, actor_user, reason="warm_activation")
from products.tasks.backend.logic.services.sandbox_usage import ( # noqa: PLC0415 — matches the file's deferred-import pattern
+ claim_task_run_sandbox_sessions,
measure_task_run_cpu_attribution,
)
cpu_attribution = measure_task_run_cpu_attribution(run_id, task_run.team_id)
+ if state.get("await_user_message"):
+ claim_task_run_sandbox_sessions(run_id, task_run.team_id, cpu_attribution)
result = send_user_message(
task_run,
pending_message,
diff --git a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py
index bef359ee903c..7cd605b490f3 100644
--- a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py
+++ b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py
@@ -64,6 +64,7 @@ from products.tasks.backend.logic.services.sandbox import (
sandbox_repo_path,
workload_for_origin_product,
)
+from products.tasks.backend.logic.services.sandbox_usage import enforce_active_session_cap
from products.tasks.backend.models import TASK_OWNERSHIP_VERSION_STATE_KEY, SandboxSnapshot, Task, TaskRun
from products.tasks.backend.temporal.metrics import (
StepTimer,
@@ -909,6 +910,17 @@ def _create_sandbox_for_repository(input: CreateSandboxForRepositoryInput) -> Cr
raise ComputeBillingLimitError(
{"team_id": ctx.team_id, "task_id": ctx.task_id, "run_id": ctx.run_id}, reason
)
+ # Not atomic: open_sandbox_session re-checks under the active-session locks after the
+ # create, so the loser of a race between two starts destroys the box it just provisioned.
+ enforce_active_session_cap(
+ origin_product=task.origin_product,
+ organization_id=task.team.organization_id,
+ created_by_id=task.created_by_id,
+ run_id=ctx.run_id,
+ context={"team_id": ctx.team_id, "task_id": ctx.task_id, "run_id": ctx.run_id},
+ signal_report_id=task.signal_report_id,
+ task_id=task.id,
+ )
_emit_image_source_log(ctx, prepared)
log_with_activity_context(
f"Provisioning sandbox from {prepared.image_source_label} (image build may take a few minutes on first run)",
diff --git a/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.py b/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.py
index 0884c4dacf2a..bf205a024d58 100644
--- a/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.py
+++ b/products/tasks/backend/temporal/process_task/activities/send_followup_to_sandbox.py
@@ -14,7 +14,7 @@ from posthog.temporal.common.heartbeat_sync import HeartbeaterSync
from posthog.temporal.common.utils import close_db_connections
from posthog.temporal.oauth import PosthogMcpScopes
-from products.tasks.backend.exceptions import CredentialUnavailableError
+from products.tasks.backend.exceptions import ActiveSessionCapError, CredentialUnavailableError
from products.tasks.backend.feature_flags import run_stream_presence_gated
from products.tasks.backend.logic.services.agent_command import (
FOLLOWUP_TIMEOUT_SECONDS,
@@ -358,6 +358,23 @@ def _deliver_followup(input: SendFollowupToSandboxInput) -> str | None:
_fail_when_sandbox_stopped(input.run_id, state, actor_user, task_run=task_run)
+ from products.tasks.backend.logic.services.sandbox_usage import ( # noqa: PLC0415 — matches forward_pending_message's deferred-import pattern
+ claim_task_run_sandbox_sessions,
+ record_task_run_user_activity,
+ )
+
+ try:
+ claim_task_run_sandbox_sessions(input.run_id, task_run.team_id)
+ except ActiveSessionCapError as error:
+ _write_error_and_complete(
+ input.run_id,
+ error.message,
+ run_uses_dedicated_stream(task_run.state),
+ run_stream_presence_gated(task_run.state),
+ task_run.task.origin_product,
+ )
+ raise
+
# Rebind the sandbox's MCP session to this actor before the turn. On an
# actor transition this must rebind or clear the prior session; if it can't,
# fail closed rather than run the turn under the previous actor's creds.
@@ -434,6 +451,8 @@ def _deliver_followup(input: SendFollowupToSandboxInput) -> str | None:
message_id=input.message_id,
steer=input.steer,
)
+ if result.turn_in_flight or (result.success and not (input.steer and _is_steer_declined(result.data))):
+ record_task_run_user_activity(input.run_id, task_run.team_id)
logger.info(
"send_followup_to_sandbox_attempted",
run_id=input.run_id,
diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py
index b4af8ab6ace8..c330df3da417 100644
--- a/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py
+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py
@@ -18,6 +18,7 @@ from asgiref.sync import async_to_sync
from posthog.temporal.common.posthog_client import is_expected_activity_failure
from products.tasks.backend.exceptions import (
+ ActiveSessionCapError,
ComputeBillingLimitError,
OrganizationExecutionError,
RepositoryAccessError,
@@ -27,6 +28,7 @@ from products.tasks.backend.exceptions import (
)
from products.tasks.backend.logic.services.docker_sandbox import DockerSandbox
from products.tasks.backend.logic.services.sandbox import ExecutionResult, SandboxTemplate
+from products.tasks.backend.logic.services.warm import SandboxWarmer
from products.tasks.backend.models import Task
from products.tasks.backend.temporal.metrics import modal_sandbox_backend_label, resume_mode_label
from products.tasks.backend.temporal.process_task.activities import provision_sandbox as provision_sandbox_module
@@ -947,12 +949,55 @@ def test_blocked_organization_creates_no_sandbox(
sandbox_class.return_value.create.assert_not_called()
[email protected]("warm", [False, True])
+def test_active_session_cap_refuses_a_cold_start_before_create(mocker, warm: bool) -> None:
+ context = _context_for_desktop_bootstrap()
+ context.state = {"await_user_message": warm}
+ task = mocker.Mock(
+ origin_product=Task.OriginProduct.USER_CREATED, created_by_id=7, signal_report_id=None, id="task-id"
+ )
+ task.team.organization_id = "organization-id"
+ mocker.patch.object(provision_sandbox_module, "_load_task", return_value=task)
+ mocker.patch.object(provision_sandbox_module, "get_compute_quota_denial_reason", return_value=None)
+ cap_hit = mocker.patch.object(SandboxWarmer, "active_session_cap_hit", return_value="user")
+ mocker.patch.object(provision_sandbox_module, "_emit_image_source_log")
+ mocker.patch.object(provision_sandbox_module, "_apply_modal_network_policy")
+ mocker.patch.object(provision_sandbox_module, "_build_sandbox_tags", return_value={})
+ create = mocker.Mock(side_effect=RuntimeError("created"))
+ mocker.patch.object(
+ provision_sandbox_module, "get_sandbox_class_for_run_backend", return_value=mocker.Mock(create=create)
+ )
+
+ with pytest.raises(RuntimeError if warm else ActiveSessionCapError) as error:
+ async_to_sync(provision_sandbox_module._create_sandbox_for_repository)(
+ CreateSandboxForRepositoryInput(context=context, prepared=_prepared_for_create())
+ )
+
+ if warm:
+ cap_hit.assert_not_called()
+ create.assert_called_once()
+ else:
+ assert isinstance(error.value, ActiveSessionCapError)
+ assert error.value.non_retryable is True
+ assert error.value.message == SandboxWarmer.active_session_cap_detail(Task.OriginProduct.USER_CREATED, "user")
+ cap_hit.assert_called_once_with(
+ Task.OriginProduct.USER_CREATED,
+ organization_id="organization-id",
+ created_by_id=7,
+ exclude_run_id="run-id",
+ signal_report_id=None,
+ task_id="task-id",
+ )
+ create.assert_not_called()
+
+
@pytest.mark.parametrize(
"failing_step,destroy_fails",
[
("get_connect_credentials", False),
("get_connect_credentials", True),
("start_cpu_billing_sampler", False),
+ ("persist_sandbox_connection", False),
],
)
def test_a_failure_after_create_destroys_the_fresh_sandbox(mocker, failing_step: str, destroy_fails: bool):
@@ -973,9 +1018,14 @@ def test_a_failure_after_create_destroys_the_fresh_sandbox(mocker, failing_step:
sandbox.launch_dev_stack_bootstrap.return_value = False
sandbox.start_cpu_billing_sampler.return_value = True
sandbox.execute.return_value = ExecutionResult(stdout='{"version": "1.2.3"}', stderr="", exit_code=0)
- getattr(sandbox, failing_step).side_effect = SandboxRateLimitedError(
+ failure: Exception = SandboxRateLimitedError(
"Sandbox control plane is rate limited", {"sandbox_id": "sandbox-id", "operation": "create_connect_token"}
)
+ if failing_step == "persist_sandbox_connection":
+ failure = ActiveSessionCapError({"run_id": "run-id"}, "org", "Your organization is at its cap.")
+ mocker.patch.object(provision_sandbox_module, "persist_sandbox_connection", side_effect=failure)
+ else:
+ getattr(sandbox, failing_step).side_effect = failure
if destroy_fails:
sandbox.destroy.side_effect = SandboxCleanupError(
"Failed to destroy sandbox", {"sandbox_id": "sandbox-id"}, cause=RuntimeError("terminate failed")
@@ -996,7 +1046,7 @@ def test_a_failure_after_create_destroys_the_fresh_sandbox(mocker, failing_step:
task_run = mocker.patch.object(provision_sandbox_module, "TaskRun")
mocker.patch.object(provision_sandbox_module, "pinned_agent_version", return_value="1.2.3")
- with pytest.raises(SandboxRateLimitedError):
+ with pytest.raises(type(failure)):
async_to_sync(provision_sandbox_module._create_sandbox_for_repository)(
CreateSandboxForRepositoryInput(context=context, prepared=_prepared_for_create())
)
diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.py b/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.py
index 2888db811153..de1991a7b264 100644
--- a/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.py
+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_send_followup_to_sandbox.py
@@ -1,18 +1,24 @@
+from datetime import timedelta
from uuid import uuid4
from posthog.test.base import BaseTest
from unittest.mock import MagicMock, patch
from django.test import SimpleTestCase
+from django.utils import timezone
from parameterized import parameterized
from temporalio.testing import ActivityEnvironment
+from products.tasks.backend.exceptions import ActiveSessionCapError
+from products.tasks.backend.logic.services.sandbox_usage import SandboxCpuAttribution
+from products.tasks.backend.logic.services.warm import SandboxWarmer
from products.tasks.backend.logic.stream.redis_stream import (
TASK_RUN_STREAM_WATCHED_TIMEOUT,
get_task_run_stream_key,
get_task_run_stream_watched_key,
)
+from products.tasks.backend.models import SandboxSession, Task, TaskRun
from products.tasks.backend.redis import get_tasks_stream_redis_sync
from products.tasks.backend.temporal.process_task.activities.send_followup_to_sandbox import (
STEER_DECLINE_REASON_UNREPORTED,
@@ -23,6 +29,8 @@ from products.tasks.backend.temporal.process_task.activities.send_followup_to_sa
send_followup_to_sandbox,
)
+_ACTIVITY = "products.tasks.backend.temporal.process_task.activities.send_followup_to_sandbox"
+
class TestGetStopReason(SimpleTestCase):
def test_returns_result_stop_reason_when_present(self):
@@ -82,6 +90,8 @@ class TestWriteTurnComplete(BaseTest):
client.delete(stream_key, watched_key)
+@patch("products.tasks.backend.logic.services.sandbox_usage.record_task_run_user_activity", new=MagicMock())
+@patch("products.tasks.backend.logic.services.sandbox_usage.claim_task_run_sandbox_sessions", new=MagicMock())
class TestSendFollowupToSandbox(BaseTest):
@patch(
"products.tasks.backend.temporal.process_task.activities.send_followup_to_sandbox.publish_task_run_stream_event"
@@ -92,6 +102,8 @@ class TestSendFollowupToSandbox(BaseTest):
self, mock_task_run_objects, mock_send_user_message, mock_publish
):
task_run = MagicMock()
+ task_run.id = uuid4()
+ task_run.team_id = self.team.id
task_run.state = {}
task_run.task.created_by = None
mock_task_run_objects.select_related.return_value.get.return_value = task_run
@@ -125,7 +137,8 @@ class TestSendFollowupToSandbox(BaseTest):
mock_refresh_sandbox_mcp,
):
task_run = MagicMock()
- task_run.id = "run-123"
+ task_run.id = uuid4()
+ task_run.team_id = self.team.id
task_run.state = {"sandbox_id": "sandbox-123"}
task_run.task.created_by_id = 42
task_run.task.created_by = MagicMock(id=42)
@@ -167,3 +180,94 @@ class TestSendFollowupToSandbox(BaseTest):
steer=True,
)
mock_publish.assert_not_called()
+
+
+@patch(f"{_ACTIVITY}._refresh_sandbox_github", return_value=None)
+@patch(f"{_ACTIVITY}._refresh_sandbox_mcp", return_value=None)
+@patch(f"{_ACTIVITY}.create_sandbox_connection_token", return_value=None)
+@patch(f"{_ACTIVITY}.publish_task_run_stream_event")
+@patch("products.tasks.backend.logic.services.sandbox_usage.measure_task_run_cpu_attribution", return_value={})
+@patch(f"{_ACTIVITY}.send_user_message")
+class TestSendFollowupSandboxAttribution(BaseTest):
+ def _run_with_session(self, *, attributed: bool) -> TaskRun:
+ task = Task.objects.create(
+ team=self.team,
+ title="Warm",
+ description="",
+ origin_product=Task.OriginProduct.USER_CREATED,
+ created_by=self.user,
+ )
+ run = TaskRun.objects.create(task=task, team=self.team, status=TaskRun.Status.IN_PROGRESS, state={})
+ now = timezone.now()
+ SandboxSession.objects.unscoped().create(
+ team=self.team,
+ task_run=run,
+ sandbox_id=f"sb-{run.id}",
+ cpu_cores=4.0,
+ memory_gb=16.0,
+ ttl_seconds=3600,
+ ttl_expires_at=now + timedelta(hours=1),
+ prewarmed=True,
+ user_attributed_at=now if attributed else None,
+ origin_product=task.origin_product,
+ )
+ return run
+
+ @parameterized.expand([("warm_claim", False), ("already_attributed", True)])
+ def test_delivery_stamps_user_activity(
+ self, _name, attributed, mock_send, mock_measure, _publish, _token, _mcp, _github
+ ) -> None:
+ run = self._run_with_session(attributed=attributed)
+ session = SandboxSession.objects.unscoped().get(task_run=run)
+ measured_at = timezone.now() - timedelta(seconds=5)
+ mock_measure.return_value = {
+ session.sandbox_id: SandboxCpuAttribution(
+ cpu_usage_usec=1_000, billed_cpu_usage_usec=None, measured_at=measured_at
+ )
+ }
+ attributed_at_send = []
+
+ def send(*args, **kwargs):
+ attributed_at_send.append(SandboxSession.objects.unscoped().get(id=session.id).user_attributed_at)
+ return MagicMock(success=True, turn_in_flight=False, data={"result": {}})
+
+ mock_send.side_effect = send
+
+ ActivityEnvironment().run(
+ send_followup_to_sandbox, SendFollowupToSandboxInput(run_id=str(run.id), message="hi")
+ )
+
+ session.refresh_from_db()
+ assert attributed_at_send == [session.user_attributed_at]
+ assert (session.user_attributed_at == measured_at) is not attributed
+ assert session.last_user_activity_at is not None
+ assert mock_measure.called is not attributed
+
+ def test_warm_claim_at_the_active_session_cap_is_refused(
+ self, mock_send, _measure, _publish, _token, _mcp, _github
+ ) -> None:
+ busy_run = self._run_with_session(attributed=True)
+ now = timezone.now()
+ for i in range(SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS[Task.OriginProduct.USER_CREATED].per_user - 1):
+ SandboxSession.objects.unscoped().create(
+ team=self.team,
+ task_run=busy_run,
+ sandbox_id=f"sb-busy-{i}",
+ cpu_cores=4.0,
+ memory_gb=16.0,
+ ttl_seconds=3600,
+ ttl_expires_at=now + timedelta(hours=1),
+ user_attributed_at=now,
+ origin_product=Task.OriginProduct.USER_CREATED,
+ )
+ run = self._run_with_session(attributed=False)
+
+ with self.assertRaises(ActiveSessionCapError) as error_context:
+ ActivityEnvironment().run(
+ send_followup_to_sandbox, SendFollowupToSandboxInput(run_id=str(run.id), message="hi")
+ )
+
+ assert error_context.exception.non_retryable is True
+ assert error_context.exception.context["scope"] == "user"
+ mock_send.assert_not_called()
+ assert SandboxSession.objects.unscoped().get(task_run=run).user_attributed_at is None
diff --git a/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.py b/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.py
index fa4438062626..da73375d4f0e 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_forward_pending_message.py
@@ -18,8 +18,9 @@ from posthog.models.organization import Organization
from posthog.models.team.team import Team
from posthog.models.user import User
-from products.tasks.backend.exceptions import OrganizationExecutionError
+from products.tasks.backend.exceptions import ActiveSessionCapError, OrganizationExecutionError
from products.tasks.backend.logic.services.sandbox_usage import SandboxCpuAttribution
+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmer
from products.tasks.backend.models import SandboxSession, Task, TaskRun
_module = importlib.import_module("products.tasks.backend.temporal.process_task.activities.forward_pending_message")
@@ -143,6 +144,55 @@ class TestForwardPendingUserMessage(TestCase):
assert "pending_user_message" not in run.state
assert "pending_user_message_id" not in run.state
+ @patch.dict(
+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,
+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=10)},
+ )
+ @patch("products.tasks.backend.logic.services.agent_command.send_user_message")
+ def test_warm_claim_at_the_active_session_cap_is_refused(self, mock_send: Mock) -> None:
+ Task.objects.filter(id=self.task.id).update(origin_product=Task.OriginProduct.USER_CREATED)
+ self.task.origin_product = Task.OriginProduct.USER_CREATED
+ now = timezone.now()
+ busy_run = self._make_run()
+ SandboxSession.objects.unscoped().create(
+ team=self.team,
+ task_run=busy_run,
+ sandbox_id="sb-busy",
+ cpu_cores=4.0,
+ memory_gb=16.0,
+ ttl_seconds=3600,
+ ttl_expires_at=now + timedelta(hours=1),
+ user_attributed_at=now,
+ origin_product=self.task.origin_product,
+ )
+ run = self._make_run(
+ state={
+ "await_user_message": True,
+ "pending_user_message": "fix the tests",
+ "sandbox_url": "https://sandbox.example.com/rpc",
+ }
+ )
+ SandboxSession.objects.unscoped().create(
+ team=self.team,
+ task_run=run,
+ sandbox_id="sb-warm",
+ cpu_cores=4.0,
+ memory_gb=16.0,
+ ttl_seconds=3600,
+ ttl_expires_at=now + timedelta(hours=1),
+ prewarmed=True,
+ origin_product=self.task.origin_product,
+ )
+
+ with self.assertRaises(ActiveSessionCapError) as error_context:
+ forward_pending_user_message(str(run.id))
+
+ assert error_context.exception.type == "ActiveSessionCapError"
+ assert error_context.exception.non_retryable is True
+ assert error_context.exception.context["scope"] == "user"
+ mock_send.assert_not_called()
+ assert SandboxSession.objects.unscoped().get(sandbox_id="sb-warm").user_attributed_at is None
+
@patch("products.tasks.backend.logic.services.store_skills.refresh_store_skills_state")
@patch("products.tasks.backend.logic.services.sandbox_usage.measure_task_run_cpu_attribution")
@patch("products.tasks.backend.logic.services.connection_token.create_sandbox_connection_token", return_value="jwt")
@@ -167,8 +217,9 @@ class TestForwardPendingUserMessage(TestCase):
ttl_seconds=3600,
ttl_expires_at=timezone.now() + timedelta(seconds=3600),
)
- measured_at = timezone.now()
+ measured_at = timezone.now() - timedelta(seconds=5)
calls = []
+ attributed_at_send = []
def measure(*args):
calls.append("measure")
@@ -182,6 +233,7 @@ class TestForwardPendingUserMessage(TestCase):
def send(*args, **kwargs):
calls.append("send")
+ attributed_at_send.append(SandboxSession.objects.unscoped().get(sandbox_id="sb-fwd").user_attributed_at)
return _command_result(success=True, status_code=200)
def refresh_store_skills(*args, **kwargs):
@@ -194,7 +246,8 @@ class TestForwardPendingUserMessage(TestCase):
forward_pending_user_message(str(run.id))
session = SandboxSession.objects.unscoped().get(sandbox_id="sb-fwd")
- assert session.user_attributed_at is not None
+ assert attributed_at_send == [measured_at]
+ assert session.user_attributed_at == measured_at
assert session.last_user_activity_at is not None
assert session.provider_cpu_usage_attribution_usec == 1_234_567
assert session.provider_billed_cpu_usage_attribution_usec == 1_500_000
diff --git a/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.py b/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.py
index c2c66370a683..ac70d6c69aed 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_send_followup_to_sandbox.py
@@ -47,6 +47,16 @@ def _clear_session_cache():
cache.clear()
[email protected](autouse=True)
+def _no_sandbox_session_bookkeeping():
+ """The mocked runs here have no sandbox session rows, so the claim and activity stamps are stubbed."""
+ with (
+ patch("products.tasks.backend.logic.services.sandbox_usage.claim_task_run_sandbox_sessions"),
+ patch("products.tasks.backend.logic.services.sandbox_usage.record_task_run_user_activity"),
+ ):
+ yield
+
+
def _run_activity(activity_input: SendFollowupToSandboxInput) -> str | None:
"""The activity opens `HeartbeaterSync`, which needs a Temporal activity context."""
return ActivityEnvironment().run(send_followup_to_sandbox, activity_input)
diff --git a/products/tasks/backend/temporal/process_task/tests/test_workflow.py b/products/tasks/backend/temporal/process_task/tests/test_workflow.py
index 74dbbd484224..50a713e31b18 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_workflow.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_workflow.py
@@ -1305,14 +1305,14 @@ class TestProcessTaskFollowupDispatch:
class TestFollowupDeliveryFailureBookkeeping:
- def _workflow(self, monkeypatch, *, mode: str):
+ def _workflow(self, monkeypatch, *, mode: str, delivery_error: Exception | None = None):
workflow = ProcessTaskWorkflow()
workflow._context = _build_context(github_integration_id=123, state={"mode": mode})
emitted: list[EmitProgressInput] = []
async def fake_execute_activity(_activity, activity_input, **_kwargs):
if isinstance(activity_input, SendFollowupToSandboxInput):
- raise RuntimeError("delivery failed")
+ raise delivery_error or RuntimeError("delivery failed")
if isinstance(activity_input, EmitProgressInput):
emitted.append(activity_input)
return None
@@ -1348,6 +1348,34 @@ class TestFollowupDeliveryFailureBookkeeping:
assert workflow._completion_error_type == "followup_delivery_failed"
assert emitted == []
+ @pytest.mark.parametrize("mode", ["interactive", "background"])
+ async def test_active_session_cap_on_delivery_fails_the_run(self, monkeypatch, mode: str) -> None:
+ delivery_error = ActivityError(
+ "Activity task failed",
+ scheduled_event_id=10,
+ started_event_id=11,
+ identity="worker-1",
+ activity_type="send_followup_to_sandbox",
+ activity_id="activity-1",
+ retry_state=RetryState.NON_RETRYABLE_FAILURE,
+ )
+ delivery_error.__cause__ = ApplicationError(
+ "You already have 2 cloud sandboxes running.", type="ActiveSessionCapError"
+ )
+ workflow, emitted = self._workflow(monkeypatch, mode=mode, delivery_error=delivery_error)
+ await workflow.send_followup_message("try this", [], "msg-1")
+ assert any(key.endswith(":msg-1") for key in workflow._accepted_message_ids)
+
+ result = await workflow._send_followup_to_sandbox("try this", [], message_id="msg-1")
+
+ assert result is None
+ assert workflow._task_completed is True
+ assert workflow._completion_status == "failed"
+ assert workflow._completion_error_type == "ActiveSessionCapError"
+ assert workflow._completion_error == "You already have 2 cloud sandboxes running."
+ assert emitted == []
+ assert not any(key.endswith(":msg-1") for key in workflow._accepted_message_ids)
+
async def test_agent_shadow_result_is_reported_separately(monkeypatch):
workflow_instance = ProcessTaskWorkflow()
@@ -2435,18 +2463,31 @@ class TestProcessTaskWorkflowUnit:
run_id="run-id", stream_generation="resumed-generation"
)
- async def test_run_persists_activity_failure_cause_not_the_wrapper(self, monkeypatch):
+ @pytest.mark.parametrize(
+ "activity_type, error_type, message",
+ [
+ ("inject_fresh_tokens_on_resume", "SandboxNotRunningError", "Sandbox not in running state."),
+ (
+ "create_sandbox_for_repository",
+ "ActiveSessionCapError",
+ "You already have 10 cloud sandboxes running. Wait for one to finish or stop one, then try again.",
+ ),
+ ],
+ )
+ async def test_run_persists_activity_failure_cause_not_the_wrapper(
+ self, monkeypatch, activity_type: str, error_type: str, message: str
+ ):
workflow = ProcessTaskWorkflow()
error = ActivityError(
"Activity task failed",
scheduled_event_id=10,
started_event_id=11,
identity="worker-1",
- activity_type="inject_fresh_tokens_on_resume",
+ activity_type=activity_type,
activity_id="activity-1",
retry_state=RetryState.MAXIMUM_ATTEMPTS_REACHED,
)
- error.__cause__ = ApplicationError("Sandbox not in running state.", type="SandboxNotRunningError")
+ error.__cause__ = ApplicationError(message, type=error_type)
update_task_run_status_mock = AsyncMock()
monkeypatch.setattr(
@@ -2461,12 +2502,12 @@ class TestProcessTaskWorkflowUnit:
result = await workflow.run(ProcessTaskInput(run_id="run-id"))
assert result.success is False
- assert result.error == "Sandbox not in running state."
+ assert result.error == message
update_task_run_status_mock.assert_awaited_with(
"failed",
- error_message="Sandbox not in running state.",
+ error_message=message,
run_id="run-id",
- error_type="SandboxNotRunningError",
+ error_type=error_type,
)
async def test_run_skips_relay_when_sandbox_event_ingest_is_enabled(self, monkeypatch):
diff --git a/products/tasks/backend/temporal/process_task/workflow.py b/products/tasks/backend/temporal/process_task/workflow.py
index 41e5a07b40c8..e55dc8b5db82 100644
--- a/products/tasks/backend/temporal/process_task/workflow.py
+++ b/products/tasks/backend/temporal/process_task/workflow.py
@@ -147,6 +147,7 @@ from .credential_refresh import CredentialRefreshExitReason, run_credential_refr
from .slack_agent_design_relay import SlackAgentDesignRelayInput, SlackAgentDesignRelayWorkflow
DEAD_SANDBOX_ERROR_TYPES = ("SandboxNotRunningError", "SandboxNotFoundError")
+RUN_ENDING_FOLLOWUP_ERROR_TYPES = ("ActiveSessionCapError",)
MAX_ACCEPTED_MESSAGE_IDS = 500
_PATCH_ID_FAIL_RUN_ON_PRIMARY_CLONE_FAILURE = "tasks-fail-run-on-primary-clone-failure"
_PATCH_ID_SHARED_STARTUP_TIMING = "tasks-shared-startup-timing"
@@ -3748,12 +3749,24 @@ class ProcessTaskWorkflow(PostHogWorkflow):
**error_properties,
},
)
+ cause = e.cause if isinstance(e, temporalio.exceptions.ActivityError) else None
+ run_ending = (
+ isinstance(cause, temporalio.exceptions.ApplicationError)
+ and cause.type in RUN_ENDING_FOLLOWUP_ERROR_TYPES
+ )
+ if message_id and (run_ending or self.context.mode == "interactive"):
+ dedupe_key = _message_dedupe_key(message_id, actor_user_id, context)
+ if dedupe_key in self._accepted_message_id_set:
+ self._accepted_message_id_set.discard(dedupe_key)
+ self._accepted_message_ids.remove(dedupe_key)
+ if run_ending:
+ assert isinstance(cause, temporalio.exceptions.ApplicationError)
+ self._completion_status = "failed"
+ self._completion_error = cause.message
+ self._completion_error_type = cause.type
+ self._task_completed = True
+ return None
if self.context.mode == "interactive":
- if message_id:
- dedupe_key = _message_dedupe_key(message_id, actor_user_id, context)
- if dedupe_key in self._accepted_message_id_set:
- self._accepted_message_id_set.discard(dedupe_key)
- self._accepted_message_ids.remove(dedupe_key)
if user_originated:
# A user follow-up can arrive without a message_id, so the card can't hinge on
# one; the generated group still gives the user a failure notice instead of a
diff --git a/products/tasks/backend/tests/test_facade.py b/products/tasks/backend/tests/test_facade.py
index 38058dacffbd..6ad69b175421 100644
--- a/products/tasks/backend/tests/test_facade.py
+++ b/products/tasks/backend/tests/test_facade.py
@@ -37,12 +37,14 @@ from products.tasks.backend.facade.profiles import (
TaskSource,
get_task_profile,
)
+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmer
from products.tasks.backend.models import (
TASK_OWNERSHIP_VERSION_STATE_KEY,
Channel,
ChannelMembership,
SandboxCustomImage,
SandboxEnvironment,
+ SandboxSession,
Task,
TaskRun,
TaskWorkflowDispatch,
@@ -1333,6 +1335,62 @@ class TestFacadeReadsAndMappers(TestCase):
trigger.assert_not_called()
assert not task.runs.exists()
+ @parameterized.expand(
+ [
+ ("user_immediate", "user", False),
+ ("user_scheduled", "user", True),
+ ("org_immediate", "org", False),
+ ]
+ )
+ def test_run_task_at_the_active_session_cap(self, _name: str, scope: str, scheduled: bool) -> None:
+ caps = ActiveSessionCaps(per_user=2, per_org=3)
+ holder = self.user
+ if scope == "org":
+ holder = User.objects.create_and_join(self.organization, "[email protected]", None)
+ busy_run = TaskRun.objects.create(
+ task=self._make_task(created_by=holder), team=self.team, status=TaskRun.Status.IN_PROGRESS
+ )
+ now = django_timezone.now()
+ for i in range(caps.per_user if scope == "user" else caps.per_org):
+ SandboxSession.objects.unscoped().create(
+ team=self.team,
+ task_run=busy_run,
+ sandbox_id=f"sb-busy-{i}",
+ cpu_cores=4.0,
+ memory_gb=16.0,
+ ttl_seconds=3600,
+ ttl_expires_at=now + timedelta(hours=1),
+ user_attributed_at=now,
+ origin_product=Task.OriginProduct.USER_CREATED,
+ )
+ task = self._make_task()
+ validated_data: dict[str, Any] = {"mode": "interactive"}
+ if scheduled:
+ validated_data["scheduled_at"] = now + timedelta(hours=2)
+
+ with (
+ patch.dict(SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS, {Task.OriginProduct.USER_CREATED: caps}),
+ patch("products.tasks.backend.facade.api._trigger_task_processing_workflow", return_value=None) as trigger,
+ ):
+ result = facade.run_task(task.id, self.team.id, self.user.id, validated_data=validated_data)
+
+ assert result is not None
+ if scheduled:
+ assert result.error is None
+ assert task.runs.count() == 1
+ else:
+ assert result.error is not None
+ self.assertEqual(
+ result.error.detail,
+ "You already have 2 cloud sandboxes running. Wait for one to finish or stop one, then try again."
+ if scope == "user"
+ else "Your organization already has 3 cloud sandboxes running. "
+ "Wait for one to finish or stop one, then try again.",
+ )
+ assert result.error.code == "active_session_cap"
+ trigger.assert_not_called()
+ assert not task.runs.exists()
+
def test_run_task_resumed_from_an_import_run_marks_the_new_run(self):
task = self._make_task()
import_run = TaskRun.objects.create(
diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.py
index 333c60ec31cb..27f26bca4e5b 100644
--- a/products/tasks/backend/tests/test_warm_facade.py
+++ b/products/tasks/backend/tests/test_warm_facade.py
@@ -28,11 +28,12 @@ from products.tasks.backend.logic.services.staged_artifacts import (
build_task_artifact_entry,
build_task_staged_artifact_cache_key,
)
-from products.tasks.backend.logic.services.warm import WarmResult
+from products.tasks.backend.logic.services.warm import ActiveSessionCaps, SandboxWarmer, WarmResult
from products.tasks.backend.models import (
TASK_OWNERSHIP_VERSION_STATE_KEY,
SandboxCustomImage,
SandboxEnvironment,
+ SandboxSession,
Task,
TaskRun,
)
@@ -1550,6 +1551,61 @@ class TestCreateTaskWarmReuse(APIBaseTest):
run.refresh_from_db()
assert run.state.get("await_user_message") is True
+ def test_create_endpoint_refuses_a_warm_reuse_at_the_active_session_cap(self):
+ _warm_task, run = self._warm_run()
+ busy_task = Task.objects.create(
+ team=self.team,
+ title="",
+ description="",
+ origin_product=Task.OriginProduct.USER_CREATED,
+ created_by=self.user,
+ )
+ busy_run = TaskRun.objects.create(task=busy_task, team=self.team, status=TaskRun.Status.IN_PROGRESS)
+ now = django_timezone.now()
+ SandboxSession.objects.unscoped().create(
+ team=self.team,
+ task_run=busy_run,
+ sandbox_id="sb-busy",
+ cpu_cores=4.0,
+ memory_gb=16.0,
+ ttl_seconds=3600,
+ ttl_expires_at=now + timedelta(hours=1),
+ user_attributed_at=now,
+ origin_product=busy_task.origin_product,
+ )
+
+ with (
+ patch.dict(
+ SandboxWarmer.ORIGIN_ACTIVE_SESSION_CAPS,
+ {Task.OriginProduct.USER_CREATED: ActiveSessionCaps(per_user=1, per_org=10)},
+ ),
+ patch(f"{FACADE}.signal_task_run_user_message", return_value=True) as m_signal,
+ ):
+ response = self.client.post(
+ "/api/projects/@current/tasks/",
+ {
+ "description": "fix the bug",
+ "repository": "posthog/posthog",
+ "github_integration": self.integration.id,
+ "branch": "main",
+ },
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content
+ self.assertEqual(
+ response.json(),
+ {
+ "detail": "You already have 1 cloud sandboxes running. "
+ "Wait for one to finish or stop one, then try again.",
+ "code": "active_session_cap",
+ },
+ )
+ m_signal.assert_not_called()
+ run.refresh_from_db()
+ assert run.state.get("await_user_message") is True
+ assert Task.objects.filter(team=self.team).count() == 2
+
def test_create_endpoint_without_warm_hints_is_not_regated(self):
# Only a create that can activate a sandbox takes the run-start gates; a plain create still
# goes through, and the run endpoint gates it when execution is actually requested.
diff --git a/products/tasks/frontend/generated/api.schemas.ts b/products/tasks/frontend/generated/api.schemas.ts
index d6256c416f53..c6f8eb0293c3 100644
--- a/products/tasks/frontend/generated/api.schemas.ts
+++ b/products/tasks/frontend/generated/api.schemas.ts
@@ -3191,6 +3191,7 @@ export const RunStatusEnumApi = {
* * `subscription_usage_limit` - subscription_usage_limit
* * `task_spend_limit` - task_spend_limit
* * `agent_error` - agent_error
+ * * `ActiveSessionCapError` - ActiveSessionCapError
*/
export type ErrorTypeEnumApi = (typeof ErrorTypeEnumApi)[keyof typeof ErrorTypeEnumApi]
@@ -3211,6 +3212,7 @@ export const ErrorTypeEnumApi = {
SubscriptionUsageLimit: 'subscription_usage_limit',
TaskSpendLimit: 'task_spend_limit',
AgentError: 'agent_error',
+ ActiveSessionCapError: 'ActiveSessionCapError',
} as const
export interface PatchedTaskRunUpdateApi {
@@ -3263,7 +3265,8 @@ export interface PatchedTaskRunUpdateApi {
* * `turn_ended_without_response` - turn_ended_without_response
* * `subscription_usage_limit` - subscription_usage_limit
* * `task_spend_limit` - task_spend_limit
- * * `agent_error` - agent_error */
+ * * `agent_error` - agent_error
+ * * `ActiveSessionCapError` - ActiveSessionCapError */
error_type?: ErrorTypeEnumApi | null
}
diff --git a/products/tasks/frontend/generated/api.zod.ts b/products/tasks/frontend/generated/api.zod.ts
index 17c06452c83f..863f58b6abc3 100644
--- a/products/tasks/frontend/generated/api.zod.ts
+++ b/products/tasks/frontend/generated/api.zod.ts
@@ -2063,15 +2063,16 @@ export const TasksRunsPartialUpdateBody = /* @__PURE__ */ zod.object({
'subscription_usage_limit',
'task_spend_limit',
'agent_error',
+ 'ActiveSessionCapError',
])
.describe(
- '\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error'
+ '\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error\n\* `ActiveSessionCapError` - ActiveSessionCapError'
),
zod.null(),
])
.optional()
.describe(
- 'Why the run failed, set by the caller that marked it failed.\n\n\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error'
+ 'Why the run failed, set by the caller that marked it failed.\n\n\* `agent_reported` - agent_reported\n\* `agent_server_crash` - agent_server_crash\n\* `agent_declined` - agent_declined\n\* `output_parse_failed` - output_parse_failed\n\* `poll_timeout` - poll_timeout\n\* `pi_runtime_error` - pi_runtime_error\n\* `upstream_stream_terminated` - upstream_stream_terminated\n\* `upstream_connection_error` - upstream_connection_error\n\* `upstream_timeout` - upstream_timeout\n\* `upstream_provider_failure` - upstream_provider_failure\n\* `upstream_request_rejected` - upstream_request_rejected\n\* `content_block_rejection` - content_block_rejection\n\* `turn_ended_without_response` - turn_ended_without_response\n\* `subscription_usage_limit` - subscription_usage_limit\n\* `task_spend_limit` - task_spend_limit\n\* `agent_error` - agent_error\n\* `ActiveSessionCapError` - ActiveSessionCapError'
),
})
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 448bba577c83..da7e6898ef5f 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -40375,6 +40375,7 @@ export namespace Schemas {
* * `subscription_usage_limit` - subscription_usage_limit
* * `task_spend_limit` - task_spend_limit
* * `agent_error` - agent_error
+ * * `ActiveSessionCapError` - ActiveSessionCapError
*/
export type ErrorTypeEnum = typeof ErrorTypeEnum[keyof typeof ErrorTypeEnum];
@@ -40396,6 +40397,7 @@ export namespace Schemas {
SubscriptionUsageLimit: 'subscription_usage_limit',
TaskSpendLimit: 'task_spend_limit',
AgentError: 'agent_error',
+ ActiveSessionCapError: 'ActiveSessionCapError',
} as const;
export interface EstimateExperimentScope {
@@ -87607,7 +87609,8 @@ export namespace Schemas {
* * `turn_ended_without_response` - turn_ended_without_response
* * `subscription_usage_limit` - subscription_usage_limit
* * `task_spend_limit` - task_spend_limit
- * * `agent_error` - agent_error */
+ * * `agent_error` - agent_error
+ * * `ActiveSessionCapError` - ActiveSessionCapError */
error_type?: ErrorTypeEnum | null;
}