Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): let sandbox agents start task runs for the task owner (#118579)

posthog[bot]ca9c353ed100
diff --git a/docs/internal/task-run-api.md b/docs/internal/task-run-api.md
index b92391f90652..51db287a385c 100644
--- a/docs/internal/task-run-api.md
+++ b/docs/internal/task-run-api.md
@@ -3,8 +3,16 @@
`POST /api/projects/{team_id}/tasks/` creates a task without starting a run by default.
Set `start_run` to `true` to create the task and start its first background run in one request.
This mode uses the `tasks-mcp-agent-run-start` feature flag and the internal-project restriction.
-Sandbox tokens cannot start another cloud run.
-This restriction also applies to warm creation, warm activation, warm resumes, and cloud resumes.
+A sandbox token can start a background agent run through `start_run` or the task `run` endpoint, within these rules:
+
+- The token must be bound to a run of a task that the token's user created. Otherwise the response is `403` with code `no_actor`.
+- A run that a sandbox started cannot start more runs, and neither can a resume of that run. A sandbox also cannot start a run of its own task. These cases return `403` with code `nested`.
+- Every other gate runs as the token's user: the internal project, the `tasks-mcp-agent-run-start` feature flag, Desktop access, usage limits, and task visibility.
+- One parent run can start up to 10 runs. A start that fails gives its slot back. Past the cap, the response is `429` with code `agent_run_fanout_cap`.
+- Each user has hourly and daily limits on runs that sandboxes start. Past them, the response is `429` with code `agent_run_rate_limited`.
+- The new run is always a background run. Its state records `agent_parent_task_id` and `agent_parent_run_id`, and callers cannot change or remove them.
+
+Sandbox tokens still cannot start runs through warm creation, warm activation, warm resumes, cloud resumes, or the endpoints under `/tasks/{task_id}/runs/`.
Requests through a PostHog connection keep the sandbox origin and the same run restrictions.
The origin marker does not grant sandbox access.
PostHog Desktop access, runtime access, and usage limits still apply.
@@ -97,5 +105,5 @@ Other task streams include the effective summary.
- `tasks-run-create` starts a run for an existing task.
The two run-start tools require the agent-run feature flag.
-They are not available to sandbox tokens.
+Sandbox tokens can use them only within the sandbox rules above.
These operations are not idempotent. If a response is lost, inspect the task and its runs before you retry.
diff --git a/products/tasks/backend/facade/agent_run_starts.py b/products/tasks/backend/facade/agent_run_starts.py
new file mode 100644
index 000000000000..67cba3a871a1
--- /dev/null
+++ b/products/tasks/backend/facade/agent_run_starts.py
@@ -0,0 +1,116 @@
+"""
+Function facade for runs that an agent in a task sandbox starts.
+
+The request layer reads the sandbox credential. This module decides whether the run behind
+that credential may start a run, and caps how many runs one parent run may start. Each child run
+records its parent in its state, which ``run_task`` writes before the workflow starts.
+"""
+
+from uuid import UUID
+
+from products.tasks.backend.facade import contracts
+from products.tasks.backend.models import Task, TaskRun
+
+__all__ = [
+ "AGENT_PARENT_RUN_ID_STATE_KEY",
+ "AGENT_PARENT_TASK_ID_STATE_KEY",
+ "AGENT_RUN_FANOUT_CAP",
+ "AGENT_STARTED_RUN_COUNT_STATE_KEY",
+ "capture_sandbox_run_start_refused",
+ "release_agent_run_slot",
+ "reserve_agent_run_slot",
+ "resolve_sandbox_run_starter",
+]
+
+AGENT_RUN_FANOUT_CAP = 10
+AGENT_PARENT_TASK_ID_STATE_KEY = "agent_parent_task_id"
+AGENT_PARENT_RUN_ID_STATE_KEY = "agent_parent_run_id"
+AGENT_STARTED_RUN_COUNT_STATE_KEY = "agent_started_run_count"
+
+
+def _as_uuid(value: str | UUID | None) -> UUID | None:
+ if value is None or isinstance(value, UUID):
+ return value
+ try:
+ return UUID(str(value))
+ except ValueError:
+ return None
+
+
+def resolve_sandbox_run_starter(
+ team_id: int,
+ *,
+ sandbox_task_id: UUID | None,
+ token_id: int | None,
+ user_id: int | None,
+ target_task_id: str | UUID | None = None,
+) -> contracts.SandboxRunStarter | contracts.SandboxRunStartDenial:
+ """Resolve the run whose sandbox sent the request, and check that it may start a run.
+
+ The credential user must be the person who created the run's task, so every gate the caller
+ applies next runs as that person. A run that a sandbox started cannot start more runs.
+ """
+ if sandbox_task_id is None or user_id is None or token_id is None:
+ return contracts.SandboxRunStartDenial.NO_ACTOR
+ if not Task.objects.filter(id=sandbox_task_id, team_id=team_id, deleted=False, created_by_id=user_id).exists():
+ return contracts.SandboxRunStartDenial.NO_ACTOR
+ bound_run = (
+ TaskRun.objects.filter(
+ task_id=sandbox_task_id,
+ team_id=team_id,
+ state__sandbox_oauth_token_ids__contains=[str(token_id)],
+ )
+ .only("id", "state")
+ .first()
+ )
+ if bound_run is None:
+ return contracts.SandboxRunStartDenial.NO_ACTOR
+ # The bound run decides, not other runs of the task, so a person can still resume a task that
+ # once ran as a child. `run_task` carries the parent keys across a resume of a child run.
+ if _as_uuid(target_task_id) == sandbox_task_id or AGENT_PARENT_TASK_ID_STATE_KEY in (bound_run.state or {}):
+ return contracts.SandboxRunStartDenial.NESTED
+ return contracts.SandboxRunStarter(parent_task_id=sandbox_task_id, parent_run_id=bound_run.id)
+
+
+def reserve_agent_run_slot(parent_run_id: UUID) -> bool:
+ """Count one more run against the parent run's cap. Returns False when the cap is full.
+
+ The count lives on the parent run, so each new run of the parent task starts with a full cap.
+ """
+ reserved = False
+
+ def _reserve(state: dict) -> None:
+ nonlocal reserved
+ count = state.get(AGENT_STARTED_RUN_COUNT_STATE_KEY, 0)
+ if count >= AGENT_RUN_FANOUT_CAP:
+ return
+ state[AGENT_STARTED_RUN_COUNT_STATE_KEY] = count + 1
+ reserved = True
+
+ TaskRun.mutate_state_atomic(parent_run_id, _reserve)
+ return reserved
+
+
+def release_agent_run_slot(parent_run_id: UUID) -> None:
+ """Give back a slot that ``reserve_agent_run_slot`` took for a run that did not start."""
+
+ def _release(state: dict) -> None:
+ count = state.get(AGENT_STARTED_RUN_COUNT_STATE_KEY, 0)
+ if count > 0:
+ state[AGENT_STARTED_RUN_COUNT_STATE_KEY] = count - 1
+
+ TaskRun.mutate_state_atomic(parent_run_id, _release)
+
+
+def capture_sandbox_run_start_refused(
+ team_id: int, *, sandbox_task_id: UUID | None, reason: str, endpoint: str
+) -> None:
+ """Record that a task sandbox asked to start a run and was refused, on the sandbox's own task.
+
+ A refusal without a task behind the credential has no task to attach to, so it is not recorded.
+ """
+ if sandbox_task_id is None:
+ return
+ task = Task.objects.filter(id=sandbox_task_id, team_id=team_id).select_related("created_by", "team").first()
+ if task is not None:
+ task.capture_event("task_run_agent_start_refused", {"reason": reason, "endpoint": endpoint})
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index d7e1092c13ab..7cb48d92afca 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -2847,6 +2847,12 @@ _PROTECTED_RUN_STATE_KEYS = frozenset(
"include_live_context",
"analytics_query_context",
"sandbox_oauth_token_ids",
+ # Sandbox run-start lineage and fan-out count. A child sandbox that removed its parent keys
+ # would count as a root run and start more generations, and a parent that reset its count
+ # would lift its fan-out cap.
+ "agent_parent_task_id",
+ "agent_parent_run_id",
+ "agent_started_run_count",
"resume_from_run_id",
"run_source",
"pr_base_branch",
@@ -9600,6 +9606,8 @@ def run_task(
("benjamin_enabled", validated_data.get("benjamin_enabled")),
("claude_model_access", claude_model_access),
("codex_model_access", codex_model_access),
+ ("agent_parent_task_id", validated_data.get("agent_parent_task_id")),
+ ("agent_parent_run_id", validated_data.get("agent_parent_run_id")),
):
if value is not None:
extra_state = extra_state or {}
@@ -9651,6 +9659,12 @@ def run_task(
if (previous_run.state or {}).get("github_read_access") is True:
extra_state["github_read_access"] = True
+ # A resumed child run stays a child, so its sandbox still cannot start more runs. The keys
+ # are PATCH-protected, so this server-side copy is the only way they reach a successor run.
+ for lineage_key in ("agent_parent_task_id", "agent_parent_run_id"):
+ if lineage_key not in extra_state and (lineage_value := (previous_run.state or {}).get(lineage_key)):
+ extra_state[lineage_key] = lineage_value
+
if prev_state.sandbox_environment_id and sandbox_environment_id is None:
sandbox_environment_id = prev_state.sandbox_environment_id
diff --git a/products/tasks/backend/facade/contracts.py b/products/tasks/backend/facade/contracts.py
index c719fc21f5b6..1be7f0073390 100644
--- a/products/tasks/backend/facade/contracts.py
+++ b/products/tasks/backend/facade/contracts.py
@@ -1109,3 +1109,18 @@ class LivingArtifactVersionPreview:
class LivingArtifactVersionDownload:
url: str | None
error: Literal["not_found", "not_stored", "unavailable"] | None
+
+
+class SandboxRunStartDenial(StrEnum):
+ """Why a task sandbox cannot start a run."""
+
+ NO_ACTOR = "no_actor"
+ NESTED = "nested"
+
+
+@dataclass(frozen=True)
+class SandboxRunStarter:
+ """The task and run whose sandbox starts a run for the task's owner."""
+
+ parent_task_id: UUID
+ parent_run_id: UUID
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index 2fcd2d3f23ba..20a01eb6b612 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -941,6 +941,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
# under an unclobbered name too — as TaskRun.capture_event already does.
"run_environment": task_run.environment,
"is_resume": is_resume,
+ "started_by_agent_sandbox": "agent_parent_task_id" in (state or {}),
"has_pending_message": has_pending,
"startup_requested_at": startup_requested_at.isoformat(),
"startup_run_created_at": task_run.created_at.isoformat(),
@@ -2963,6 +2964,7 @@ class TaskRun(models.Model):
"run_environment": self.environment,
"startup_requested_at": self.startup_requested_at,
"mode": self.mode,
+ "started_by_agent_sandbox": "agent_parent_task_id" in (self.state or {}),
"slack_session_id": self._slack_session_id(),
**self._analytics_usage_properties(),
}
diff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.py
index f5561c4787b7..f00c1731f423 100644
--- a/products/tasks/backend/presentation/views/api.py
+++ b/products/tasks/backend/presentation/views/api.py
@@ -61,6 +61,12 @@ from posthog.renderers import ServerSentEventRenderer
from posthog.schema_migrations.upgrade import upgrade
from posthog.security.outbound_proxy import internal_requests_session
from posthog.temporal.oauth import SANDBOX_OAUTH_APP_CLIENT_IDS, TASK_AGENT_OAUTH_APP_CLIENT_IDS
+from posthog.token_bucket import (
+ BucketDecision,
+ Budget,
+ consume as consume_token_bucket,
+ refund as refund_token_bucket,
+)
from posthog.utils import absolute_uri
from products.exports.backend.facade.api import render_png_export
@@ -69,6 +75,7 @@ if TYPE_CHECKING:
from products.exports.backend.facade.api import ExportedAsset
from products.tasks.backend.facade import (
access as tasks_access,
+ agent_run_starts as tasks_agent_run_starts,
api as tasks_facade,
cancellation as tasks_cancellation,
contracts as tasks_contracts,
@@ -271,11 +278,43 @@ def _pi_cloud_runtime_disabled_response() -> Response:
def _agent_run_disabled_response() -> Response:
return Response(
- TaskRunErrorResponseSerializer({"error": "Agent-started task runs are not available for this project"}).data,
+ TaskRunErrorResponseSerializer({"error": "Agent-started task runs are not enabled for this project"}).data,
status=status.HTTP_403_FORBIDDEN,
)
+def _sandbox_run_disabled_response() -> Response:
+ return Response(
+ TaskRunErrorResponseSerializer({"error": "Task sandboxes cannot start this kind of run"}).data,
+ status=status.HTTP_403_FORBIDDEN,
+ )
+
+
+_SANDBOX_RUN_START_DENIAL_MESSAGES: dict[tasks_contracts.SandboxRunStartDenial, str] = {
+ tasks_contracts.SandboxRunStartDenial.NO_ACTOR: (
+ "This sandbox is not linked to a run of a task that its user created, so it cannot start task runs. "
+ "A person must start this run."
+ ),
+ tasks_contracts.SandboxRunStartDenial.NESTED: (
+ "This run was started from inside a task, so it cannot start more task runs. A person must start this run."
+ ),
+}
+
+
+def _sandbox_run_start_denied_response(denial: tasks_contracts.SandboxRunStartDenial) -> Response:
+ return Response(
+ TaskRunErrorResponseSerializer({"error": _SANDBOX_RUN_START_DENIAL_MESSAGES[denial], "code": denial}).data,
+ status=status.HTTP_403_FORBIDDEN,
+ )
+
+
+def _agent_run_rate_limited_response(code: str, detail: str) -> Response:
+ return Response(
+ TaskRunErrorResponseSerializer({"type": "rate_limit", "code": code, "error": detail}).data,
+ status=status.HTTP_429_TOO_MANY_REQUESTS,
+ )
+
+
TASKS_PREWARM_SANDBOX_FLAG = "tasks-prewarm-sandbox"
WARM_SANDBOX_FLAGS_BY_ORIGIN_PRODUCT: dict[str, str] = {
@@ -462,6 +501,12 @@ class SignalReportTaskWarmSustainedThrottle(_SignalReportTaskThrottle):
rate = "120/day"
+SANDBOX_AGENT_RUN_START_BUDGETS: tuple[tuple[str, Budget], ...] = (
+ ("sandbox_agent_run_start_hour", Budget(burst=30, per_hour=30)),
+ ("sandbox_agent_run_start_day", Budget(burst=100, per_hour=4)),
+)
+
+
@extend_schema(tags=["tasks"])
class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
"""
@@ -767,7 +812,9 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
403: OpenApiResponse(
response=TaskRunErrorResponseSerializer,
description=(
- "Agent-started runs are limited to the internal project and cannot start from a sandbox token. "
+ "Agent-started runs are limited to the internal project. A task sandbox can start a run only "
+ "for the person who created its task (code `no_actor`), and a run that a sandbox started "
+ "cannot start more runs (code `nested`). "
"Immediate runs also require PostHog Desktop access and an enabled Pi cloud runtime."
),
),
@@ -775,7 +822,9 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
response=TaskRunErrorResponseSerializer,
description=(
"Organization reached its PostHog Desktop usage limit, the signal report reached its "
- "task limit (code `signal_report_task_cap`), or the per-user creation rate limit fired"
+ "task limit (code `signal_report_task_cap`), or the per-user creation rate limit fired. "
+ "A task sandbox gets code `agent_run_fanout_cap` when its run started its maximum number "
+ "of runs, or code `agent_run_rate_limited` when its user starts runs too fast"
),
),
503: OpenApiResponse(
@@ -796,7 +845,13 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
origin_product = validated_data.get("origin_product", tasks_facade.TaskOriginProduct.USER_CREATED)
relationship = serializer.validated_data.get("signal_report_task_relationship")
- if start_run and (not self._agent_run_enabled(request) or is_sandbox_origin_request(request)):
+ sandbox_starter: tasks_contracts.SandboxRunStarter | None = None
+ if start_run and is_sandbox_origin_request(request):
+ sandbox_starter_or_response = self._sandbox_run_starter(request, endpoint="create")
+ if isinstance(sandbox_starter_or_response, Response):
+ return sandbox_starter_or_response
+ sandbox_starter = sandbox_starter_or_response
+ if start_run and not self._agent_run_enabled(request):
return _agent_run_disabled_response()
if (
@@ -814,8 +869,9 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
# discussion is one of them and stays code-access exempt below, matching how the warm endpoint gates.
# `origin_product` is optional on the wire; `create_task` defaults it the same way.
can_activate_warm_run = "branch" in validated_data and origin_product in WARMABLE_ORIGIN_PRODUCTS
- if can_activate_warm_run and is_sandbox_origin_request(request):
- return _agent_run_disabled_response()
+ # An agent run never activates a warm run (see `run_task`), so only a create without a run is refused.
+ if can_activate_warm_run and not start_run and is_sandbox_origin_request(request):
+ return _sandbox_run_disabled_response()
if start_run or can_activate_warm_run:
is_code_access_exempt = (
origin_product == tasks_facade.TaskOriginProduct.SIGNAL_REPORT
@@ -832,6 +888,10 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
return access_response
if limit_response := usage_limit_response(request.user, self.team_id):
return limit_response
+ if sandbox_starter is not None:
+ if limit_response := self._reserve_sandbox_agent_run(request, sandbox_starter, endpoint="create"):
+ return limit_response
+ run_data = {**run_data, **self._sandbox_run_data(sandbox_starter)}
discussion_question = validated_data.pop("signal_report_discussion_question", None)
@@ -853,6 +913,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
ReportTaskCapExceeded,
)
+ run_started = False
try:
run_error = None
if start_run:
@@ -869,6 +930,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
assert create_result.task is not None
task = create_result.task
run_error = create_result.run_error
+ run_started = not run_error
else:
task = tasks_facade.create_task(
self.team_id,
@@ -888,6 +950,9 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
return self._report_task_cap_response(error.detail)
except tasks_facade.WarmRunActivationUnavailable as error:
return self._warm_activation_unavailable_response(error)
+ finally:
+ if sandbox_starter is not None and not run_started:
+ tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)
self._forward_signals_discussion_note(request, task, relationship, discussion_question)
response_data = TaskSerializer(task).data
if run_error:
@@ -1407,7 +1472,12 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
),
403: OpenApiResponse(
response=TaskRunErrorResponseSerializer,
- description="PostHog Desktop access is required, or Pi cloud runtime is disabled",
+ description=(
+ "PostHog Desktop access is required, Pi cloud runtime is disabled, or agent-started runs are "
+ "not enabled for this project. A task sandbox can start a run only for the person who created "
+ "its task (code `no_actor`), and a run that a sandbox started cannot start more runs "
+ "(code `nested`)."
+ ),
),
404: OpenApiResponse(description="Task not found"),
503: OpenApiResponse(
@@ -1422,7 +1492,9 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
response=TaskRunErrorResponseSerializer,
description=(
"Team is over its posthog_code usage limit, or the task's signal report has "
- "reached its task limit (code `signal_report_task_cap`)"
+ "reached its task limit (code `signal_report_task_cap`). A task sandbox gets code "
+ "`agent_run_fanout_cap` when its run started its maximum number of runs, or code "
+ "`agent_run_rate_limited` when its user starts runs too fast"
),
),
},
@@ -1444,9 +1516,14 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
previous_source = tasks_facade.get_task_run_source(resume_id, pk, self.team_id)
if previous_source == RunSource.AGENT:
run_source = previous_source
- if is_sandbox_origin_request(request) or (
- run_source == RunSource.AGENT and not self._agent_run_enabled(request)
- ):
+ sandbox_starter: tasks_contracts.SandboxRunStarter | None = None
+ if is_sandbox_origin_request(request):
+ sandbox_starter_or_response = self._sandbox_run_starter(request, endpoint="run", target_task_id=pk)
+ if isinstance(sandbox_starter_or_response, Response):
+ return sandbox_starter_or_response
+ sandbox_starter = sandbox_starter_or_response
+ run_source = RunSource.AGENT
+ if run_source == RunSource.AGENT and not self._agent_run_enabled(request):
return _agent_run_disabled_response()
# Original order: 404 if the task isn't visible, then gate (always cloud) before the run.
@@ -1472,11 +1549,17 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
return access_response
if limit_response := usage_limit_response(request.user, self.team_id):
return limit_response
+ sandbox_run_data: dict = {}
+ if sandbox_starter is not None:
+ if limit_response := self._reserve_sandbox_agent_run(request, sandbox_starter, endpoint="run"):
+ return limit_response
+ sandbox_run_data = self._sandbox_run_data(sandbox_starter)
from products.signals.backend.facade.api import ( # noqa: PLC0415 — keeps the signals stack off this module's import path
ReportTaskCapExceeded,
)
+ run_started = False
try:
result = tasks_facade.run_task(
pk,
@@ -1484,6 +1567,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
self._user_id(),
validated_data={
**request.validated_data,
+ **sandbox_run_data,
"client_platform": "mobile"
if request.headers.get("X-PostHog-Client-Platform") == "mobile"
else None,
@@ -1494,10 +1578,14 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
else {}
),
)
+ run_started = result is not None and result.error is None and not result.run_error
except tasks_facade.WarmRunActivationUnavailable as error:
return self._warm_activation_unavailable_response(error)
except ReportTaskCapExceeded as error:
return self._report_task_cap_response(error.detail)
+ finally:
+ if sandbox_starter is not None and not run_started:
+ tasks_agent_run_starts.release_agent_run_slot(sandbox_starter.parent_run_id)
if result is None:
raise NotFound()
if result.error is not None:
@@ -1526,6 +1614,76 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
def _agent_run_enabled(self, request) -> bool:
return _agent_run_enabled(request, self.team)
+ def _sandbox_run_starter(
+ self, request, *, endpoint: str, target_task_id: str | None = None
+ ) -> tasks_contracts.SandboxRunStarter | Response:
+ """Resolve the run behind a sandbox credential, or the 403 that explains why it cannot start runs.
+
+ The credential user must be the creator of that run's task, so the gates that follow run as that person.
+ """
+ token = get_oauth_access_token(request) if is_sandbox_oauth_request(request) else None
+ sandbox_task_id = getattr(token, "sandbox_task_id", None)
+ starter = tasks_agent_run_starts.resolve_sandbox_run_starter(
+ self.team_id,
+ sandbox_task_id=sandbox_task_id,
+ token_id=getattr(token, "id", None),
+ user_id=self._user_id(),
+ target_task_id=target_task_id,
+ )
+ if isinstance(starter, tasks_contracts.SandboxRunStartDenial):
+ tasks_agent_run_starts.capture_sandbox_run_start_refused(
+ self.team_id, sandbox_task_id=sandbox_task_id, reason=starter.value, endpoint=endpoint
+ )
+ return _sandbox_run_start_denied_response(starter)
+ return starter
+
+ def _reserve_sandbox_agent_run(
+ self, request, starter: tasks_contracts.SandboxRunStarter, *, endpoint: str
+ ) -> Response | None:
+ charged: list[tuple[str, Budget]] = []
+ for scope, budget in SANDBOX_AGENT_RUN_START_BUDGETS:
+ key = f"{scope}:{request.user.id}"
+ decision = consume_token_bucket(key, budget)
+ # Fail open when Redis cannot answer, because the fan-out cap below is the durable bound.
+ if isinstance(decision, BucketDecision) and not decision.allowed:
+ for charged_key, charged_budget in charged:
+ refund_token_bucket(charged_key, charged_budget)
+ return self._refuse_sandbox_agent_run(
+ starter,
+ endpoint=endpoint,
+ code="agent_run_rate_limited",
+ detail="Task sandboxes started too many runs for this user. Try again later.",
+ )
+ charged.append((key, budget))
+ if not tasks_agent_run_starts.reserve_agent_run_slot(starter.parent_run_id):
+ return self._refuse_sandbox_agent_run(
+ starter,
+ endpoint=endpoint,
+ code="agent_run_fanout_cap",
+ detail=f"This run already started {tasks_agent_run_starts.AGENT_RUN_FANOUT_CAP} runs, "
+ "which is the limit. A person must start more runs.",
+ )
+ return None
+
+ def _refuse_sandbox_agent_run(
+ self, starter: tasks_contracts.SandboxRunStarter, *, endpoint: str, code: str, detail: str
+ ) -> Response:
+ tasks_agent_run_starts.capture_sandbox_run_start_refused(
+ self.team_id, sandbox_task_id=starter.parent_task_id, reason=code, endpoint=endpoint
+ )
+ return _agent_run_rate_limited_response(code, detail)
+
+ @staticmethod
+ def _sandbox_run_data(starter: tasks_contracts.SandboxRunStarter) -> dict[str, str]:
+ # A sandbox-started run is always a background agent run. An interactive run is exempt from
+ # the autonomous wall-clock cap, so a wedged child that keeps heartbeating could run forever.
+ return {
+ "run_source": RunSource.AGENT,
+ "mode": "background",
+ tasks_agent_run_starts.AGENT_PARENT_TASK_ID_STATE_KEY: str(starter.parent_task_id),
+ tasks_agent_run_starts.AGENT_PARENT_RUN_ID_STATE_KEY: str(starter.parent_run_id),
+ }
+
def _warm_enabled(self, origin_product: str) -> bool:
"""Person + org level gate for the sandbox-warming feature. Fail-closed on any error.
@@ -1587,7 +1745,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
@action(detail=False, methods=["post"], url_path="warm", required_scopes=["task:write"])
def warm(self, request, **kwargs):
if is_sandbox_origin_request(request):
- return _agent_run_disabled_response()
+ return _sandbox_run_disabled_response()
origin_product = request.validated_data["origin_product"]
if not self._warm_enabled(origin_product):
return Response(status=status.HTTP_200_OK)
@@ -1666,7 +1824,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
def warm_resume(self, request, pk=None, **kwargs):
_reject_scout_trial_run_creation(str(pk), self.team_id)
if is_sandbox_origin_request(request):
- return _agent_run_disabled_response()
+ return _sandbox_run_disabled_response()
gate = tasks_facade.task_control_runtime_and_origin(pk, self.team_id, self._user_id())
if gate is None:
raise NotFound()
@@ -2088,7 +2246,7 @@ class TaskRunViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
# Gate cloud runs before the run row is created; local runs aren't limited.
if environment == tasks_facade.TaskRunEnvironment.CLOUD:
if is_sandbox_origin_request(request):
- return _agent_run_disabled_response()
+ return _sandbox_run_disabled_response()
if access_response := self._cloud_run_access_response(task_id):
return access_response
@@ -2128,9 +2286,9 @@ class TaskRunViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
startable, run_source = tasks_facade.check_task_run_startable(pk, task_id, self.team_id)
if startable == "not_found":
raise NotFound()
- if is_sandbox_origin_request(request) or (
- run_source == RunSource.AGENT and not _agent_run_enabled(request, self.team)
- ):
+ if is_sandbox_origin_request(request):
+ return _sandbox_run_disabled_response()
+ if run_source == RunSource.AGENT and not _agent_run_enabled(request, self.team):
return _agent_run_disabled_response()
if tasks_facade.task_runtime(
@@ -4093,7 +4251,7 @@ class TaskRunViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
)
def resume_in_cloud(self, request, pk=None, **kwargs):
if is_sandbox_origin_request(request):
- return _agent_run_disabled_response()
+ return _sandbox_run_disabled_response()
task_id = self._ensure_task_accessible()
_reject_scout_trial_run_creation(task_id, self.team_id)
if tasks_facade.get_task_run_detail(pk, task_id, self.team_id) is None:
diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py
index c0277cb53dad..a400e8c1589e 100644
--- a/products/tasks/backend/tests/test_api.py
+++ b/products/tasks/backend/tests/test_api.py
@@ -49,6 +49,7 @@ from posthog.temporal.oauth import (
POSTHOG_AI_APP_CLIENT_ID_DEV,
POSTHOG_DESKTOP_MOBILE_APP_CLIENT_ID_US,
)
+from posthog.token_bucket import BucketDecision
from posthog.utils import absolute_uri
from products.posthog_ai.backend.models.assistant import Conversation
@@ -59,6 +60,7 @@ from products.tasks.backend.exceptions import SandboxNotFoundError
from products.tasks.backend.facade import (
access as tasks_access,
api as tasks_facade,
+ contracts as tasks_contracts,
)
from products.tasks.backend.facade.repo_selection import RepoSelectionResult
from products.tasks.backend.facade.run_config import TaskArtifactAdapter, TaskArtifactType
@@ -2404,40 +2406,147 @@ class TestTaskAPI(BaseTaskAPITest):
send_feature_flag_events=False,
)
+ def _sandbox_start(self, client: APIClient, path: str, target: Task | None = None, target_id: str | None = None):
+ if path == "create":
+ return client.post(
+ "/api/projects/@current/tasks/", {"description": "Child task", "start_run": True}, format="json"
+ )
+ assert target is not None
+ return client.post(
+ f"/api/projects/@current/tasks/{target_id or target.id}/run/",
+ {"run_source": "manual", "mode": "interactive"},
+ format="json",
+ )
+
+ @parameterized.expand(["create", "run"])
+ @patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)
+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
+ def test_sandbox_oauth_starts_agent_run_for_task_owner(self, path, mock_workflow, _mock_internal_team):
+ parent = self.create_task()
+ parent.create_run(extra_state={"agent_parent_task_id": str(uuid.uuid4())})
+ parent_run = parent.create_run(environment=TaskRun.Environment.CLOUD)
+ client = self._sandbox_oauth_client(parent.id, internal_scope=True)
+ target = self.create_task(title="Existing task")
+
+ response = self._sandbox_start(client, path, target)
+
+ self.assertIn(response.status_code, (status.HTTP_200_OK, status.HTTP_201_CREATED), response.json())
+ child = Task.objects.get(id=response.json()["id"])
+ run = child.runs.get()
+ state = run.state or {}
+ self.assertEqual(state["run_source"], "agent")
+ self.assertEqual(state["agent_parent_task_id"], str(parent.id))
+ self.assertEqual(state["agent_parent_run_id"], str(parent_run.id))
+ self.assertEqual(run.mode, "background")
+ parent_run.refresh_from_db()
+ self.assertEqual((parent_run.state or {})["agent_started_run_count"], 1)
+ mock_workflow.assert_called_once()
+
+ # The child is agent-started, so its own sandbox cannot start more runs.
+ child_client = self._sandbox_oauth_client(child.id, client_id=ARRAY_APP_CLIENT_ID_US, internal_scope=True)
+ response = self._sandbox_start(child_client, path, target)
+ self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+ self.assertEqual(response.json()["code"], "nested")
+
@parameterized.expand(
[
- (payload, forwarded)
- for payload in ({}, {"run_source": "manual"}, {"run_source": "agent"})
- for forwarded in (False, True)
+ (path, case)
+ for path in ("create", "run")
+ for case in (
+ "forwarded",
+ "unbound",
+ "no_bound_run",
+ "other_owner",
+ "nested",
+ "own_task",
+ "own_task_uppercase",
+ "not_internal",
+ )
+ if not (path == "create" and case.startswith("own_task"))
]
)
- @patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)
- def test_sandbox_oauth_cannot_start_agent_runs(self, run_payload, forwarded, _mock_internal_team):
- parent = self.create_task()
- if forwarded:
+ @patch("products.tasks.backend.presentation.views.api._is_internal_debug_team")
+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
+ def test_sandbox_oauth_cannot_start_agent_run(self, path, case, mock_workflow, mock_internal_team):
+ mock_internal_team.return_value = case != "not_internal"
+ parent = self.create_task(created_by=self.create_organization_user("owner") if case == "other_owner" else None)
+ if case != "no_bound_run":
+ parent.create_run(extra_state={"agent_parent_task_id": str(uuid.uuid4())} if case == "nested" else None)
+ if case == "forwarded":
client = self._oauth_client(ARRAY_APP_CLIENT_ID_DEV)
client.defaults["HTTP_X_POSTHOG_SANDBOX_ORIGIN"] = "1"
else:
- client = self._sandbox_oauth_client(parent.id, internal_scope=True)
+ client = self._sandbox_oauth_client(parent.id, bound=case != "unbound", internal_scope=True)
+ target = parent if case.startswith("own_task") else self.create_task(title="Existing task")
task_count = Task.objects.count()
+ target_run_count = target.runs.count()
- response = client.post(
- "/api/projects/@current/tasks/",
- {"description": "Child task", "start_run": True},
- format="json",
+ response = self._sandbox_start(
+ client, path, target, target_id=str(target.id).upper() if case == "own_task_uppercase" else None
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+ expected_code = {
+ "nested": "nested",
+ "own_task": "nested",
+ "own_task_uppercase": "nested",
+ "not_internal": None,
+ }.get(case, "no_actor")
+ if expected_code is not None:
+ self.assertEqual(response.json()["code"], expected_code)
self.assertEqual(Task.objects.count(), task_count)
+ self.assertEqual(target.runs.count(), target_run_count)
+ mock_workflow.assert_not_called()
- response = client.post(
- f"/api/projects/@current/tasks/{parent.id}/run/",
- run_payload,
- format="json",
+ @parameterized.expand(
+ [(path, limit) for path in ("create", "run") for limit in ("agent_run_fanout_cap", "agent_run_rate_limited")]
+ )
+ @patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)
+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
+ def test_sandbox_oauth_agent_run_limits(self, path, limit, mock_workflow, _mock_internal_team):
+ parent = self.create_task()
+ parent_run = parent.create_run(
+ extra_state={"agent_started_run_count": 10} if limit == "agent_run_fanout_cap" else None
)
+ client = self._sandbox_oauth_client(parent.id, internal_scope=True)
+ target = self.create_task(title="Existing task")
+ task_count = Task.objects.count()
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
- self.assertFalse(parent.runs.exists())
+ with patch(
+ "products.tasks.backend.presentation.views.api.consume_token_bucket",
+ return_value=BucketDecision(
+ allowed=limit != "agent_run_rate_limited", remaining=0, limit=30, retry_after=60, reset=60
+ ),
+ ):
+ response = self._sandbox_start(client, path, target)
+
+ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
+ self.assertEqual(response.json()["code"], limit)
+ self.assertEqual(Task.objects.count(), task_count)
+ self.assertFalse(target.runs.exists())
+ mock_workflow.assert_not_called()
+ parent_run.refresh_from_db()
+ expected_count = 10 if limit == "agent_run_fanout_cap" else None
+ self.assertEqual((parent_run.state or {}).get("agent_started_run_count"), expected_count)
+
+ @patch("products.tasks.backend.presentation.views.api._is_internal_debug_team", return_value=True)
+ def test_sandbox_oauth_failed_run_start_gives_back_its_fanout_slot(self, _mock_internal_team):
+ parent = self.create_task()
+ parent_run = parent.create_run(extra_state={"agent_started_run_count": 3})
+ client = self._sandbox_oauth_client(parent.id, internal_scope=True)
+ target = self.create_task(title="Existing task")
+
+ with patch(
+ "products.tasks.backend.presentation.views.api.tasks_facade.run_task",
+ return_value=tasks_contracts.TaskRunResult(
+ error=tasks_contracts.TaskValidationError(kind="detail", detail="Invalid resume_from_run_id")
+ ),
+ ):
+ response = self._sandbox_start(client, "run", target)
+
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ parent_run.refresh_from_db()
+ self.assertEqual((parent_run.state or {})["agent_started_run_count"], 3)
@parameterized.expand(
[(origin, forwarded) for origin in ("user_created", "posthog_ai") for forwarded in (False, True)]
@@ -4043,11 +4152,24 @@ class TestTaskAPI(BaseTaskAPITest):
"run_source": "manual",
"pr_base_branch": "other",
"failure_error_type": "ComputeBillingLimitError",
+ "agent_parent_task_id": "00000000-0000-4000-8000-000000000002",
}
},
True,
),
- ("remove", {"state_remove_keys": ["run_source", "pr_base_branch", "failure_error_type"]}, True),
+ (
+ "remove",
+ {
+ "state_remove_keys": [
+ "run_source",
+ "pr_base_branch",
+ "failure_error_type",
+ "agent_parent_task_id",
+ "agent_parent_run_id",
+ ]
+ },
+ True,
+ ),
(
"append",
{
@@ -4055,6 +4177,7 @@ class TestTaskAPI(BaseTaskAPITest):
"run_source": ["manual"],
"pr_base_branch": ["other"],
"failure_error_type": ["ComputeBillingLimitError"],
+ "agent_parent_task_id": ["00000000-0000-4000-8000-000000000002"],
}
},
True,
@@ -4079,8 +4202,13 @@ class TestTaskAPI(BaseTaskAPITest):
self.assertEqual(start_response.status_code, status.HTTP_200_OK)
run = task.runs.get()
self.assertEqual(run.state["pr_base_branch"], base_branch)
+ lineage = {
+ "agent_parent_task_id": "00000000-0000-4000-8000-000000000001",
+ "agent_parent_run_id": "00000000-0000-4000-8000-000000000003",
+ }
run.status = TaskRun.Status.IN_PROGRESS
- run.save(update_fields=["status"])
+ run.state = {**run.state, **lineage}
+ run.save(update_fields=["status", "state"])
client = self._sandbox_oauth_client(task.id, internal_scope=True)
response = client.patch(f"/api/projects/@current/tasks/{task.id}/runs/{run.id}/", payload, format="json")
@@ -4094,6 +4222,7 @@ class TestTaskAPI(BaseTaskAPITest):
self.assertEqual(run.state["run_source"], "agent")
self.assertEqual(run.state["pr_base_branch"], base_branch)
self.assertNotIn("failure_error_type", run.state)
+ self.assertEqual({key: run.state.get(key) for key in lineage}, lineage)
run.status = TaskRun.Status.COMPLETED
run.save(update_fields=["status"])
@@ -4105,6 +4234,7 @@ class TestTaskAPI(BaseTaskAPITest):
resumed_run = task.runs.exclude(id=run.id).get()
self.assertEqual(resumed_run.state["run_source"], "agent")
self.assertEqual(resumed_run.state["pr_base_branch"], base_branch)
+ self.assertEqual({key: resumed_run.state.get(key) for key in lineage}, lineage)
self.assertEqual(resumed_run.branch, base_branch)
self.assertEqual(mock_workflow.call_args.kwargs["posthog_mcp_scopes"], "read_only")