:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tasks): stop task-run lock convoys (#129923)
Alessandro Pogliaghieae7723edd8a
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yamlindex 1844844a287c..32e6e872aaf9 100644--- a/.semgrep/rules/security/idor-team-scoped-models.yaml+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml@@ -465,6 +465,7 @@ rules:|TaskAutomation|TaskPresence|TaskRun+ |TaskRunGatewayRequest|TaskWorkflowDispatch|TaskSearchDocument|TaskSession@@ -969,6 +970,7 @@ rules:|TaskAutomation|TaskPresence|TaskRun+ |TaskRunGatewayRequest|TaskWorkflowDispatch|TaskSession|UserTasksConfigdiff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.pyindex 4f293b658cdd..fee31fa3ebf4 100644--- a/posthog/tasks/scheduled.py+++ b/posthog/tasks/scheduled.py@@ -147,6 +147,7 @@ from products.tasks.backend.facade.tasks import (refresh_stale_sandbox_custom_images_task,settle_overdue_sandbox_compute_task,sweep_inactive_tasks_task,+ sweep_queued_gateway_usage,)from products.visual_review.backend.facade.tasks import (send_visual_review_debt_digests,@@ -299,6 +300,15 @@ def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:name="tasks run state metrics",)+ # Backstop for a gateway callback whose Celery kick was lost: the run's usage workflow only+ # starts from that kick, so nothing would drain the intake table otherwise.+ add_periodic_task_with_expiry(+ sender,+ crontab(minute="*/4"),+ sweep_queued_gateway_usage.s(),+ name="tasks queued gateway usage sweep",+ )+add_periodic_task_with_expiry(sender,crontab(minute="*/2"),diff --git a/products/tasks/backend/facade/gateway.py b/products/tasks/backend/facade/gateway.pyindex 939c8ae1aee5..815752fdd2d0 100644--- a/products/tasks/backend/facade/gateway.py+++ b/products/tasks/backend/facade/gateway.py@@ -1,14 +1,18 @@from typing import TYPE_CHECKINGfrom uuid import UUID+from celery import current_app+from posthog.llm.gateway_client import GatewayNotConfiguredError-from products.tasks.backend.logic.services.gateway_usage import record_generation_request, schedule_gateway_usage-from products.tasks.backend.models import TaskRun+from products.tasks.backend.logic.services.gateway_usage import record_generation_requestif TYPE_CHECKING:from posthog.llm.gateway_client import AIGatewayConfig+# Sent by name so the web worker never imports the task module, which drags the Temporal tree into it.+_SCHEDULE_GATEWAY_USAGE_TASK = "products.tasks.backend.tasks.tasks.schedule_task_run_gateway_usage"+def mint_private_gateway_token(*,@@ -45,10 +49,9 @@ def revoke_private_gateway_token(token: str, *, gateway_config: "AIGatewayConfigdef accept_generation_request(*, team_id: int, run_id: UUID, request_id: str) -> bool:- try:- recorded = record_generation_request(team_id=team_id, run_id=run_id, request_id=request_id)- except TaskRun.DoesNotExist:+ recorded = record_generation_request(team_id=team_id, run_id=run_id, request_id=request_id)+ if recorded is None:return Falseif recorded:- schedule_gateway_usage(team_id=team_id, run_id=run_id)+ current_app.send_task(_SCHEDULE_GATEWAY_USAGE_TASK, kwargs={"run_id": str(run_id), "team_id": team_id})return Truediff --git a/products/tasks/backend/facade/tasks.py b/products/tasks/backend/facade/tasks.pyindex 82781c3ffbad..1d2af0598471 100644--- a/products/tasks/backend/facade/tasks.py+++ b/products/tasks/backend/facade/tasks.py@@ -10,7 +10,7 @@ from products.tasks.backend.tasks.sandbox_images import (refresh_dev_stack_image_task,refresh_stale_sandbox_custom_images_task,)-from products.tasks.backend.tasks.tasks import settle_overdue_sandbox_compute_task+from products.tasks.backend.tasks.tasks import settle_overdue_sandbox_compute_task, sweep_queued_gateway_usage__all__ = ["bake_dev_stack_image_task",@@ -18,4 +18,5 @@ __all__ = ["refresh_stale_sandbox_custom_images_task","settle_overdue_sandbox_compute_task","sweep_inactive_tasks_task",+ "sweep_queued_gateway_usage",]diff --git a/products/tasks/backend/logic/services/gateway_usage.py b/products/tasks/backend/logic/services/gateway_usage.pyindex 24b825eaa932..c1bb7500b898 100644--- a/products/tasks/backend/logic/services/gateway_usage.py+++ b/products/tasks/backend/logic/services/gateway_usage.py@@ -1,6 +1,7 @@import reimport timeimport asyncio+import threadingfrom collections.abc import Iterable, Iteratorfrom contextlib import contextmanagerfrom datetime import datetime, timedelta@@ -11,6 +12,7 @@ from uuid import UUIDfrom django.conf import settingsfrom django.db import transaction+from django.db.models import Minfrom django.utils import timezoneimport structlog@@ -22,7 +24,7 @@ from posthog.llm.gateway_usage import GatewayRequestCost, fetch_gateway_costfrom products.tasks.backend.facade.contracts import TaskRunCost, TaskRunSpend, TaskRunTokenUsagefrom products.tasks.backend.facade.task_run_signals import task_run_cost_updatedfrom products.tasks.backend.logic.services.sandbox_pricing import COMPUTE_RATE_CARDS, calculate_sandbox_compute_cost-from products.tasks.backend.models import SandboxSession, TaskRun+from products.tasks.backend.models import SandboxSession, TaskRun, TaskRunGatewayRequestif TYPE_CHECKING:from temporalio.client import Client@@ -30,6 +32,9 @@ if TYPE_CHECKING:logger = structlog.get_logger(__name__)_REQUEST_ID = re.compile(r"[a-zA-Z0-9_-]{1,255}\Z")_PROCESSING_SECONDS = 10+_DRAIN_BATCH_SIZE = 200+_SCHEDULING_LOOP: asyncio.AbstractEventLoop | None = None+_SCHEDULING_LOCK = threading.Lock()@frozen@@ -47,7 +52,11 @@ class _CostSources:def _locked_run(run_id: UUID | str, team_id: int) -> TaskRun:- return TaskRun.objects.select_for_update().get(id=run_id, team_id=team_id)+ return (+ TaskRun.objects.select_for_update()+ .only("id", "team_id", "state", "environment", "task_id", "status", "created_at")+ .get(id=run_id, team_id=team_id)+ )def gateway_usage_enabled(run: TaskRun) -> bool:@@ -63,7 +72,7 @@ def _save_accounting_state(run: TaskRun) -> None:# Accounting can finish after completion; it must not emit completion signals again.# It also leaves `updated_at` alone, because recency gauges and stale-run sweeps read it.previous = TaskRun.objects.select_related("task").get(id=run.id, team_id=run.team_id)- delta = (_token_usage(run).cost_microusd or 0) - (_token_usage(previous).cost_microusd or 0)+ delta = (_recorded_cost_microusd(run) or 0) - (_recorded_cost_microusd(previous) or 0)TaskRun.objects.filter(id=run.id, team_id=run.team_id).update(state=run.state)task_run_cost_updated.send(sender=TaskRun,@@ -111,24 +120,56 @@ def _processed_gateway_request_ids(state: dict[str, Any]) -> set[str]:}-def record_generation_request(*, team_id: int, run_id: UUID, request_id: str) -> bool:- with transaction.atomic():- run = _locked_run(run_id, team_id)- if run.environment != TaskRun.Environment.CLOUD:- return False- state = dict(run.state or {})- pending = state.get("unprocessed_request_ids")- if not isinstance(pending, list):- pending = []- if request_id not in pending and request_id not in _processed_gateway_request_ids(state):- pending = [*pending, request_id]- state["unprocessed_request_ids"] = pending- state.setdefault("token_cost", {})- if state == run.state:- return True- run.state = state- _save_accounting_state(run)- return True+def record_generation_request(*, team_id: int, run_id: UUID, request_id: str) -> bool | None:+ environment = TaskRun.objects.filter(id=run_id, team_id=team_id).values_list("environment", flat=True).first()+ if environment is None:+ return None+ if environment != TaskRun.Environment.CLOUD:+ return False+ TaskRunGatewayRequest.objects.for_team(team_id, canonical=True).bulk_create(+ [TaskRunGatewayRequest(team_id=team_id, task_run_id=run_id, request_id=request_id)],+ ignore_conflicts=True,+ )+ return True+++def _drain_queued_requests(run: TaskRun, state: dict[str, Any]) -> bool:+ queued = list(+ TaskRunGatewayRequest.objects.for_team(run.team_id, canonical=True)+ .filter(task_run_id=run.id)+ .order_by("created_at", "id")+ .values_list("id", "request_id")[:_DRAIN_BATCH_SIZE]+ )+ if not queued:+ return False++ before = _pending_ids(state)+ seen = set(before) | _processed_gateway_request_ids(state)+ merged = list(before)+ for _row_id, request_id in queued:+ if request_id not in seen:+ seen.add(request_id)+ merged.append(request_id)++ state["unprocessed_request_ids"] = merged+ state.setdefault("token_cost", {})+ TaskRunGatewayRequest.objects.for_team(run.team_id, canonical=True).filter(+ id__in=[row_id for row_id, _ in queued]+ ).delete()+ return merged != before+++def _has_queued_requests(run: TaskRun) -> bool:+ return TaskRunGatewayRequest.objects.for_team(run.team_id, canonical=True).filter(task_run_id=run.id).exists()+++def _runs_with_queued_requests(team_id: int, run_ids: Iterable[UUID]) -> set[UUID]:+ return set(+ TaskRunGatewayRequest.objects.for_team(team_id, canonical=True)+ .filter(task_run_id__in=list(run_ids))+ .values_list("task_run_id", flat=True)+ .distinct()+ )def _pending_ids(state: dict[str, Any]) -> list[str]:@@ -144,7 +185,10 @@ def _pending_ids(state: dict[str, Any]) -> list[str]:def process_pending_gateway_usage(*, run_id: UUID, team_id: int, limit: int = 20) -> TaskRunCost:with transaction.atomic():run = _locked_run(run_id, team_id)- state = run.state or {}+ state = dict(run.state or {})+ if _drain_queued_requests(run, state):+ run.state = state+ _save_accounting_state(run)if not gateway_usage_enabled(run):return _cost_sources(run).as_contract()pending = _pending_ids(state)[:limit]@@ -179,8 +223,8 @@ def process_pending_gateway_usage(*, run_id: UUID, team_id: int, limit: int = 20def refresh_task_run_cost(*, run_id: UUID, team_id: int) -> TaskRunCost:with transaction.atomic():run = _locked_run(run_id, team_id)- cost = _cost_sources(run).as_contract()state = dict(run.state or {})+ cost = _cost_sources(run).as_contract()if "compute_cost" not in state or state["compute_cost"] != cost.compute_cost:state["compute_cost"] = cost.compute_costrun.state = state@@ -196,10 +240,11 @@ def get_task_cost(*, team_id: int, task_id: UUID) -> TaskRunCost:runs = list(TaskRun.objects.filter(team_id=team_id, task_id=task_id))if not runs:return TaskRunCost(token_cost=None, compute_cost=None)+ queued = _runs_with_queued_requests(team_id, [run.id for run in runs])sessions_by_run: dict[UUID, list[SandboxSession]] = {}for session in SandboxSession.objects.for_team(team_id).filter(task_run__in=runs):sessions_by_run.setdefault(session.task_run_id, []).append(session)- sources = [_cost_sources(run, sessions=sessions_by_run.get(run.id, [])) for run in runs]+ sources = [_cost_sources(run, sessions=sessions_by_run.get(run.id, []), queued=run.id in queued) for run in runs]return _CostSources(token_cost_microusd=Noneif any(s.token_cost_microusd is None for s in sources)@@ -220,24 +265,30 @@ def get_task_run_token_cost_microusd(*, run_id: UUID, team_id: int) -> int | Nonreturn _token_cost_microusd(TaskRun.objects.get(id=run_id, team_id=team_id))-def _token_usage(run: TaskRun) -> TaskRunTokenUsage:+def _token_usage(run: TaskRun, *, queued: bool | None = None) -> TaskRunTokenUsage:state = run.state or {}return TaskRunTokenUsage(- cost_microusd=sum(bucket.get("cost_microusd", 0) for bucket in _cost_buckets(state))- if isinstance(state.get("token_cost"), dict)- else None,+ cost_microusd=_recorded_cost_microusd(run),complete=gateway_usage_enabled(run)and not state.get("token_cost_incomplete", False)- and not bool(state.get("unprocessed_request_ids")),+ and not bool(state.get("unprocessed_request_ids"))+ and not (_has_queued_requests(run) if queued is None else queued),)+def _recorded_cost_microusd(run: TaskRun) -> int | None:+ state = run.state or {}+ if not isinstance(state.get("token_cost"), dict):+ return None+ return sum(bucket.get("cost_microusd", 0) for bucket in _cost_buckets(state))++def get_task_run_token_usage(*, run_id: UUID, team_id: int) -> TaskRunTokenUsage:return _token_usage(TaskRun.objects.get(id=run_id, team_id=team_id))-def _run_spend(run: TaskRun) -> TaskRunSpend:- usage = _token_usage(run)+def _run_spend(run: TaskRun, *, queued: bool | None = None) -> TaskRunSpend:+ usage = _token_usage(run, queued=queued)return TaskRunSpend(created_at=run.created_at,run_id=run.id,@@ -252,21 +303,26 @@ def _run_spend(run: TaskRun) -> TaskRunSpend:@contextmanagerdef locked_task_run_spend(*, team_id: int, run_id: UUID) -> Iterator[TaskRunSpend]:with transaction.atomic():- yield _run_spend(_locked_run(run_id, team_id))+ run = _locked_run(run_id, team_id)+ yield _run_spend(run)def get_task_runs_spend(*, team_id: int, run_ids: Iterable[UUID]) -> list[TaskRunSpend]:- return [_run_spend(run) for run in TaskRun.objects.filter(team_id=team_id, id__in=run_ids)]+ runs = list(TaskRun.objects.filter(team_id=team_id, id__in=run_ids))+ queued = _runs_with_queued_requests(team_id, [run.id for run in runs])+ return [_run_spend(run, queued=run.id in queued) for run in runs]-def _token_cost_microusd(run: TaskRun) -> int | None:- usage = _token_usage(run)+def _token_cost_microusd(run: TaskRun, *, queued: bool | None = None) -> int | None:+ usage = _token_usage(run, queued=queued)return usage.cost_microusd if usage.complete else None-def _cost_sources(run: TaskRun, *, sessions: list[SandboxSession] | None = None) -> _CostSources:+def _cost_sources(+ run: TaskRun, *, sessions: list[SandboxSession] | None = None, queued: bool | None = None+) -> _CostSources:return _CostSources(- token_cost_microusd=_token_cost_microusd(run),+ token_cost_microusd=_token_cost_microusd(run, queued=queued),compute_cost_usd=_compute_cost_source(run, sessions=sessions),)@@ -329,4 +385,34 @@ async def _schedule_gateway_usage(*, run_id: UUID, team_id: int) -> None:def schedule_gateway_usage(*, run_id: UUID, team_id: int) -> None:- async_to_sync(_schedule_gateway_usage)(run_id=run_id, team_id=team_id)+ with _SCHEDULING_LOCK:+ _scheduling_loop().run_until_complete(_schedule_gateway_usage(run_id=run_id, team_id=team_id))+++def _scheduling_loop() -> asyncio.AbstractEventLoop:+ # asgiref builds a fresh loop per sync call, so the client cache below would miss on every callback.+ global _SCHEDULING_LOOP+ if _SCHEDULING_LOOP is None or _SCHEDULING_LOOP.is_closed():+ _SCHEDULING_LOOP = asyncio.new_event_loop()+ return _SCHEDULING_LOOP+++def sweep_queued_gateway_requests(*, older_than_seconds: int = 600, limit: int = 50) -> int:+ cutoff = timezone.now() - timedelta(seconds=older_than_seconds)+ stale = list(+ TaskRunGatewayRequest.objects.unscoped()+ .values("team_id", "task_run_id")+ .annotate(oldest=Min("created_at"))+ .filter(oldest__lt=cutoff)+ .order_by("oldest")+ .values_list("team_id", "task_run_id")[:limit]+ )+ scheduled = 0+ for team_id, run_id in stale:+ try:+ schedule_gateway_usage(run_id=run_id, team_id=team_id)+ except Exception:+ logger.exception("task_gateway_usage.sweep_schedule_failed", run_id=str(run_id), team_id=team_id)+ continue+ scheduled += 1+ return scheduleddiff --git a/products/tasks/backend/migrations/0149_task_run_gateway_request.py b/products/tasks/backend/migrations/0149_task_run_gateway_request.pynew file mode 100644index 000000000000..4b8085a42881--- /dev/null+++ b/products/tasks/backend/migrations/0149_task_run_gateway_request.py@@ -0,0 +1,47 @@+import django.utils.timezone+import django.db.models.deletion+from django.db import migrations, models++import posthog.models.utils+++class Migration(migrations.Migration):+ dependencies = [+ ("posthog", "1425_workload_trust_policy_request"),+ ("tasks", "0148_task_source_indexes"),+ ]++ operations = [+ migrations.CreateModel(+ name="TaskRunGatewayRequest",+ fields=[+ (+ "id",+ models.UUIDField(+ default=posthog.models.utils.uuid7, editable=False, primary_key=True, serialize=False+ ),+ ),+ ("task_run_id", models.UUIDField()),+ ("request_id", models.CharField(max_length=255)),+ ("created_at", models.DateTimeField(default=django.utils.timezone.now)),+ (+ "team",+ models.ForeignKey(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="posthog.team",+ ),+ ),+ ],+ options={+ "db_table": "posthog_task_run_gateway_request",+ "constraints": [+ models.UniqueConstraint(+ fields=("task_run_id", "request_id"), name="task_run_gateway_request_unique"+ )+ ],+ "indexes": [models.Index(fields=["task_run_id", "created_at"], name="task_run_gw_request_idx")],+ },+ ),+ ]diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txtindex 3113eb44471c..d351f90c1851 100644--- a/products/tasks/backend/migrations/max_migration.txt+++ b/products/tasks/backend/migrations/max_migration.txt@@ -1 +1 @@-0148_task_source_indexes+0149_task_run_gateway_requestdiff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.pyindex 70fb5a745518..1042c0935ed3 100644--- a/products/tasks/backend/models.py+++ b/products/tasks/backend/models.py@@ -2382,7 +2382,8 @@ def bump_task_activity(*, team_id: int, task_id: uuid.UUID | str, at: datetime)@receiver(post_save, sender=TaskThreadMessage)def bump_task_activity_on_thread_message(sender, instance: "TaskThreadMessage", created: bool, **kwargs) -> None:if created:- bump_task_activity(team_id=instance.team_id, task_id=instance.task_id, at=instance.created_at)+ team_id, task_id, at = instance.team_id, instance.task_id, instance.created_at+ transaction.on_commit(lambda: bump_task_activity(team_id=team_id, task_id=task_id, at=at))@receiver(post_save, sender=Task)@@ -3731,6 +3732,25 @@ class TaskRun(models.Model):raise Exception("Cannot delete TaskRun. Task runs are immutable records.")+class TaskRunGatewayRequest(TeamScopedRootMixin, UUIDModel):+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False)+ task_run_id = models.UUIDField()+ request_id = models.CharField(max_length=255)+ created_at = models.DateTimeField(default=django_timezone.now)++ class Meta:+ db_table = "posthog_task_run_gateway_request"+ constraints = [+ models.UniqueConstraint(+ fields=["task_run_id", "request_id"],+ name="task_run_gateway_request_unique",+ )+ ]+ indexes = [+ models.Index(fields=["task_run_id", "created_at"], name="task_run_gw_request_idx"),+ ]++class TaskWorkflowDispatch(TeamScopedRootMixin):class Kind(models.TextChoices):CREATE = "create", "create"@@ -4507,7 +4527,8 @@ def bump_task_activity_on_run(sender, instance: TaskRun, created: bool, update_f# to bump would silently leave a live session reading as idle.if not created and update_fields is not None and not (RUN_ACTIVITY_FIELDS & set(update_fields)):return- bump_task_activity(team_id=instance.team_id, task_id=instance.task_id, at=django_timezone.now())+ team_id, task_id, at = instance.team_id, instance.task_id, django_timezone.now()+ transaction.on_commit(lambda: bump_task_activity(team_id=team_id, task_id=task_id, at=at))class TeamTasksConfig(models.Model):diff --git a/products/tasks/backend/tasks/tasks.py b/products/tasks/backend/tasks/tasks.pyindex d31b05b0fc5b..0af53b35ee79 100644--- a/products/tasks/backend/tasks/tasks.py+++ b/products/tasks/backend/tasks/tasks.py@@ -95,6 +95,24 @@ def deliver_comment_slack_dms()+@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=5)+def schedule_task_run_gateway_usage(*, run_id: str, team_id: int) -> None:+ from products.tasks.backend.logic.services.gateway_usage import ( # noqa: PLC0415 — keeps temporalio off the celery import path+ schedule_gateway_usage,+ )++ schedule_gateway_usage(run_id=UUID(run_id), team_id=team_id)+++@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=3)+def sweep_queued_gateway_usage() -> None:+ from products.tasks.backend.logic.services.gateway_usage import ( # noqa: PLC0415 — keeps temporalio off the celery import path+ sweep_queued_gateway_requests,+ )++ sweep_queued_gateway_requests()++# No retries: the wake is best-effort by design, and the parked step has its own deadline.@shared_task(ignore_result=True)def resume_workflow_step_for_run_deferred(run_id: str) -> None:diff --git a/products/tasks/backend/temporal/oauth.py b/products/tasks/backend/temporal/oauth.pyindex 14dc38f6878e..426ce6f910ce 100644--- a/products/tasks/backend/temporal/oauth.py+++ b/products/tasks/backend/temporal/oauth.py@@ -186,6 +186,22 @@ def _workflow_run_scopes(requested: PosthogMcpScopes, state: dict[str, Any] | Noreturn sorted(resolved)+def _builtin_agent_enforcement(task: Task) -> bool | None:+ if Task.mcp_agent_key_for(task.origin_product, task.stored_profile) is None:+ return None+ return is_builtin_agent_enforcement_enabled(task.team_id)+++def _monitoring_result_scope(task: Task, run_id: UUID | str | None) -> bool | None:+ if task.origin_product != Task.OriginProduct.SIGNALS_SCOUT or run_id is None:+ return None+ from products.signals.backend.facade.monitoring import ( # noqa: PLC0415 -- avoids loading Signals during Tasks startup+ monitoring_scout_can_submit,+ )++ return monitoring_scout_can_submit(team_id=task.team_id, task_id=task.id, task_run_id=run_id)++def create_oauth_access_token(task: Task,*,@@ -194,6 +210,8 @@ def create_oauth_access_token(allow_task_creator_fallback: bool = True,run_state: dict[str, Any] | None = None,run_id: UUID | str | None = None,+ builtin_agent_enforcement: bool | None = None,+ monitoring_result_scope: bool | None = None,) -> str:"""Create an OAuth access token for the task's sandbox app, scoped to the task's team.@@ -215,9 +233,9 @@ def create_oauth_access_token("application": _oauth_application_for_task(task, run_state),"sandbox_task_id": task.id,}- if Task.mcp_agent_key_for(task.origin_product, task.stored_profile) is not None and (- is_builtin_agent_enforcement_enabled(task.team_id)- ):+ if builtin_agent_enforcement is None:+ builtin_agent_enforcement = _builtin_agent_enforcement(task)+ if builtin_agent_enforcement:# This scope only removes access to the human MCP Store surface. Add it# even when a legacy task lacks trusted provenance so an old spoofed# origin fails closed instead of inheriting its creator's connections.@@ -235,13 +253,10 @@ def create_oauth_access_token(withheld.update(RESEARCH_WITHHELD_SCOPES)if withheld:token_options["withhold_scopes"] = sorted(withheld)- if task.origin_product == Task.OriginProduct.SIGNALS_SCOUT and run_id is not None:- from products.signals.backend.facade.monitoring import ( # noqa: PLC0415 -- avoids loading Signals during Tasks startup- monitoring_scout_can_submit,- )-- if monitoring_scout_can_submit(team_id=task.team_id, task_id=task.id, task_run_id=run_id):- token_options["include_monitoring_result_scope"] = True+ if monitoring_result_scope is None:+ monitoring_result_scope = _monitoring_result_scope(task, run_id)+ if monitoring_result_scope:+ token_options["include_monitoring_result_scope"] = Truereturn create_oauth_access_token_for_user(actor, task.team_id, **token_options)@@ -280,9 +295,11 @@ def create_oauth_access_token_for_run(elif scopes == "signals_assessment":# Runtime uploads are bound to this run; the separately minted MCP token stays read-only.scopes = [*resolve_scopes(scopes), "internal_run:write"]+ builtin_agent_enforcement = _builtin_agent_enforcement(task)+ monitoring_result_scope = _monitoring_result_scope(task, run_id)with transaction.atomic():locked_task = (- Task.objects.select_for_update(of=("self",))+ Task.objects.select_for_update(of=("self",), no_key=True).select_related("created_by", "team").get(id=task.id, team_id=task.team_id))@@ -328,6 +345,8 @@ def create_oauth_access_token_for_run(allow_task_creator_fallback=not is_slack_interaction_state(state),run_state=state,run_id=run_id,+ builtin_agent_enforcement=builtin_agent_enforcement,+ monitoring_result_scope=monitoring_result_scope,)if run_id is not None:run = TaskRun.objects.select_for_update().get(diff --git a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/start_agent_server.pyindex adfe25e6ec00..8c57bb744cd2 100644--- a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py+++ b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py@@ -618,18 +618,18 @@ def _record_agent_startup_attempt(ctx: TaskProcessingContext, runtime: str) -> Ndef _record_agent_startup_success(ctx: TaskProcessingContext) -> None:with transaction.atomic():- state = TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).values_list("state", flat=True).first()- attempt = (state or {}).get("agent_startup_attempt")- TaskRun.update_state_atomic(- ctx.run_id,- updates={"agent_startup_success": {**attempt, "completed_at": datetime.now(UTC).isoformat()}}- if isinstance(attempt, dict)- else {},- remove_keys=["startup_failure"],- )- TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).exclude(- status__in=[TaskRun.Status.COMPLETED, TaskRun.Status.FAILED, TaskRun.Status.CANCELLED]- ).update(error_message=None)+ run = TaskRun.objects.select_for_update().only("id", "state", "status").get(id=ctx.run_id, team_id=ctx.team_id)+ state = dict(run.state or {})+ attempt = state.get("agent_startup_attempt")+ state.pop("startup_failure", None)+ if isinstance(attempt, dict):+ state["agent_startup_success"] = {**attempt, "completed_at": datetime.now(UTC).isoformat()}+ run.state = state+ update_fields = ["state", "updated_at"]+ if not run.is_terminal:+ run.error_message = None+ update_fields.append("error_message")+ run.save(update_fields=update_fields)def _invoke_start_agent_server(diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.pyindex 3c6f6b767a52..139efb609307 100644--- a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py@@ -62,6 +62,10 @@ def task_run_state(mocker):)state = runs.return_value.values_list.return_value.firststate.return_value = None+ startup = mocker.patch(+ "products.tasks.backend.temporal.process_task.activities.start_agent_server.TaskRun.objects.select_for_update"+ )+ startup.return_value.only.return_value.get.return_value = mocker.Mock(state={}, is_terminal=False)return statediff --git a/products/tasks/backend/tests/test_gateway_usage.py b/products/tasks/backend/tests/test_gateway_usage.pyindex 80b055795879..0ea653710465 100644--- a/products/tasks/backend/tests/test_gateway_usage.py+++ b/products/tasks/backend/tests/test_gateway_usage.py@@ -24,9 +24,10 @@ from products.tasks.backend.logic.services.gateway_usage import (record_gateway_routing,record_generation_request,refresh_task_run_cost,+ sweep_queued_gateway_requests,)from products.tasks.backend.logic.services.sandbox_pricing import COMPUTE_RATE_CARDS-from products.tasks.backend.models import SandboxSession, Task, TaskRun+from products.tasks.backend.models import SandboxSession, Task, TaskRun, TaskRunGatewayRequest@override_settings(SANDBOX_AI_GATEWAY_URL="https://gateway.example.com/v1", SANDBOX_AI_GATEWAY_MINT_KEY="phs_test")@@ -73,15 +74,18 @@ class TestGatewayUsage(BaseTest):task_run_cost_updated.connect(cost_updated, sender=TaskRun)self.addCleanup(task_run_cost_updated.disconnect, cost_updated, sender=TaskRun)self._report(run, ["parent", "subagent", "other-model", "second-turn", "parent"])+ record_generation_request(team_id=self.team.id, run_id=run.id, request_id="queued")get.side_effect = [self._response("parent", "0.005"),self._response("subagent", "0.010001", provider="provider-b"),self._response("other-model", "0.000009", model="model-b"),self._response("second-turn", "0.005"),+ self._response("queued", "0.009"),]- assert self._process(run).token_cost == 2- assert observed_costs == {run.id: 20_010}- assert get.call_count == 4+ assert self._process(run).token_cost == 3+ assert observed_costs == {run.id: 29_010}+ assert not TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).exists()+ assert get.call_count == 5assert get.call_args.kwargs["timeout"].total == 15assert get.call_args.args[0].trust_env is Falserun.refresh_from_db()@@ -89,7 +93,7 @@ class TestGatewayUsage(BaseTest):"unprocessed_request_ids": [],"token_cost": {"model-a": {- "provider-a": {"cost_microusd": 10_000, "request_ids": ["parent", "second-turn"]},+ "provider-a": {"cost_microusd": 19_000, "request_ids": ["parent", "second-turn", "queued"]},"provider-b": {"cost_microusd": 10_001, "request_ids": ["subagent"]},},"model-b": {"provider-a": {"cost_microusd": 9, "request_ids": ["other-model"]}},@@ -97,7 +101,7 @@ class TestGatewayUsage(BaseTest):"compute_cost": None,}get.reset_mock()- assert self._process(run).token_cost == 2+ assert self._process(run).token_cost == 3get.assert_not_called()@patch("aiohttp.ClientSession._request")@@ -235,7 +239,7 @@ class TestGatewayUsage(BaseTest):get.reset_mock()assert get_task_run_cost(run_id=first.id, team_id=self.team.id) == TaskRunCost(token_cost=0, compute_cost=0)assert get_task_run_cost(run_id=second.id, team_id=self.team.id) == TaskRunCost(token_cost=0, compute_cost=0)- with self.assertNumQueries(3):+ with self.assertNumQueries(4):assert get_task_cost(team_id=self.team.id, task_id=first.task_id) == TaskRunCost(token_cost=1, compute_cost=1)@@ -248,6 +252,51 @@ class TestGatewayUsage(BaseTest):run.save(update_fields=["state"])assert get_task_run_cost(run_id=run.id, team_id=self.team.id).token_cost is None+ @patch("aiohttp.ClientSession._request")+ def test_queued_callbacks_keep_usage_incomplete_until_they_are_costed(self, get: Mock) -> None:+ run = self._run()+ record_generation_request(run_id=run.id, team_id=run.team_id, request_id="queued")+ get.return_value = self._response("queued", "0.009")++ assert not get_task_run_token_usage(run_id=run.id, team_id=self.team.id).complete+ assert get_task_run_cost(run_id=run.id, team_id=self.team.id).token_cost is None+ assert get_task_run_token_cost_microusd(run_id=run.id, team_id=self.team.id) is None++ assert refresh_task_run_cost(run_id=run.id, team_id=self.team.id).token_cost is None+ assert TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).exists()+ assert not get_task_run_token_usage(run_id=run.id, team_id=self.team.id).complete+ get.assert_not_called()++ assert self._process(run).token_cost == 1+ assert not TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).exists()+ assert get_task_run_token_usage(run_id=run.id, team_id=self.team.id).complete+ assert get_task_run_token_cost_microusd(run_id=run.id, team_id=self.team.id) == 9_000++ def test_sweep_reschedules_a_stale_queued_callback(self) -> None:+ run = self._run()+ record_generation_request(run_id=run.id, team_id=run.team_id, request_id="queued")+ TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).update(+ created_at=timezone.now() - timedelta(minutes=15)+ )++ with patch("products.tasks.backend.logic.services.gateway_usage.schedule_gateway_usage") as schedule:+ assert sweep_queued_gateway_requests() == 1++ schedule.assert_called_once_with(run_id=run.id, team_id=self.team.id)++ def test_sweep_groups_by_run_so_one_backlog_cannot_starve_another(self) -> None:+ stuck, waiting = self._run(), self._run()+ queued = TaskRunGatewayRequest.objects.for_team(self.team.id)+ for index in range(60):+ queued.create(team=self.team, task_run_id=stuck.id, request_id=f"stuck-{index}")+ queued.create(team=self.team, task_run_id=waiting.id, request_id="waiting")+ queued.update(created_at=timezone.now() - timedelta(minutes=15))++ with patch("products.tasks.backend.logic.services.gateway_usage.schedule_gateway_usage") as schedule:+ assert sweep_queued_gateway_requests() == 2++ assert {call.kwargs["run_id"] for call in schedule.call_args_list} == {stuck.id, waiting.id}+def test_compute_cost_uses_existing_ledger_attribution_and_resource_floors(self) -> None:run = self._run()now = timezone.now()diff --git a/products/tasks/backend/tests/test_gateway_usage_api.py b/products/tasks/backend/tests/test_gateway_usage_api.pyindex a9ca392d7b68..223551ef1592 100644--- a/products/tasks/backend/tests/test_gateway_usage_api.py+++ b/products/tasks/backend/tests/test_gateway_usage_api.py@@ -15,14 +15,14 @@ from rest_framework.test import APIClientfrom posthog.models import ProjectSecretAPIKey, Teamfrom posthog.models.utils import hash_key_value-from products.tasks.backend.models import Task, TaskRun+from products.tasks.backend.models import Task, TaskRun, TaskRunGatewayRequest@override_settings(AI_GATEWAY_INTERNAL_TOKEN="gateway-token")class TestTaskRunGatewayUsageAPI(APIBaseTest):def setUp(self) -> None:super().setUp()- self.schedule_usage = self.enterContext(patch("products.tasks.backend.facade.gateway.schedule_gateway_usage"))+ self.schedule_usage = self.enterContext(patch("products.tasks.backend.facade.gateway.current_app.send_task"))def _run(self,@@ -75,13 +75,18 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):assert response.status_code == status.HTTP_204_NO_CONTENTrun.refresh_from_db()+ queued = TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id)if environment == TaskRun.Environment.CLOUD:- assert run.state == {"unprocessed_request_ids": ["request_1"], "token_cost": {}}- self.schedule_usage.assert_called_once()+ assert list(queued.values_list("request_id", flat=True)) == ["request_1"]+ self.schedule_usage.assert_called_once_with(+ "products.tasks.backend.tasks.tasks.schedule_task_run_gateway_usage",+ kwargs={"run_id": str(run.id), "team_id": run.team_id},+ )else:- assert run.state == {}- assert run.updated_at == updated_at+ assert not queued.exists()self.schedule_usage.assert_not_called()+ assert run.state == {}+ assert run.updated_at == updated_atdef test_callback_rejects_missing_or_invalid_service_credential(self) -> None:run = self._run()@@ -98,32 +103,39 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):run = self._run()assert self._post(run).status_code == status.HTTP_401_UNAUTHORIZED- @parameterized.expand([("state_write",), ("scheduling",)])+ @parameterized.expand([("queue_write",), ("scheduling",)])def test_callback_does_not_acknowledge_a_failed_write_or_schedule(self, failure: str) -> None:run = self._run()client = APIClient()client.raise_request_exception = False- target = (- "products.tasks.backend.logic.services.gateway_usage._save_accounting_state"- if failure == "state_write"- else "products.tasks.backend.facade.gateway.schedule_gateway_usage"- )- with patch(target, side_effect=OperationalError("unavailable")):- response = client.post(++ def post() -> HttpResponse:+ return client.post(self._url(run),{},format="json",HTTP_AUTHORIZATION="Bearer gateway-token",HTTP_X_POSTHOG_GATEWAY_TEAM_ID=str(run.team_id),)++ if failure == "queue_write":+ with patch("django.db.models.QuerySet.bulk_create", side_effect=OperationalError("unavailable")):+ response = post()+ else:+ self.schedule_usage.side_effect = OperationalError("unavailable")+ try:+ response = post()+ finally:+ self.schedule_usage.side_effect = None+assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERRORrun.refresh_from_db()- assert run.state == (- {} if failure == "state_write" else {"unprocessed_request_ids": ["request_1"], "token_cost": {}}- )+ assert run.state == {}+ queued = TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id, request_id="request_1")+ assert queued.exists() == (failure == "scheduling")+assert self._post(run).status_code == status.HTTP_204_NO_CONTENT- run.refresh_from_db()- assert run.state["unprocessed_request_ids"] == ["request_1"]+ assert queued.exists()def test_callback_requires_a_valid_wallet_and_request_identifiers(self) -> None:run = self._run()@@ -151,8 +163,11 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):response = self._post(run, wallet_team_id=other_team.id)assert response.status_code == status.HTTP_204_NO_CONTENT- run.refresh_from_db()- assert run.state["unprocessed_request_ids"] == ["request_1"]+ assert (+ TaskRunGatewayRequest.objects.for_team(self.team.id)+ .filter(task_run_id=run.id, request_id="request_1")+ .exists()+ )@override_settings(SANDBOX_AI_GATEWAY_MINT_KEY="phs_callback_mint")def test_callback_rejects_a_foreign_wallet(self) -> None:@@ -170,10 +185,17 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):def test_callback_is_idempotent_before_and_after_processing_and_terminalization(self) -> None:run = self._run()+ def queued_request_ids() -> list[str]:+ return list(+ TaskRunGatewayRequest.objects.for_team(self.team.id)+ .filter(task_run_id=run.id)+ .order_by("created_at", "id")+ .values_list("request_id", flat=True)+ )+assert self._post(run).status_code == status.HTTP_204_NO_CONTENTassert self._post(run).status_code == status.HTTP_204_NO_CONTENT- run.refresh_from_db()- assert run.state["unprocessed_request_ids"] == ["request_1"]+ assert queued_request_ids() == ["request_1"]run.status = TaskRun.Status.COMPLETEDrun.state = {@@ -185,11 +207,11 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):assert self._post(run).status_code == status.HTTP_204_NO_CONTENTrun.refresh_from_db()assert run.status == TaskRun.Status.COMPLETED- assert run.state["unprocessed_request_ids"] == []assert run.state["token_cost"]["model"]["provider"]["cost_microusd"] == 4+ assert queued_request_ids() == ["request_1"]assert self._post(run, "request_2").status_code == status.HTTP_204_NO_CONTENT+ assert queued_request_ids() == ["request_1", "request_2"]run.refresh_from_db()- assert run.state["unprocessed_request_ids"] == ["request_2"]assert run.status == TaskRun.Status.COMPLETEDdef test_callback_returns_not_found_for_a_run_outside_the_path_team(self) -> None:diff --git a/products/tasks/backend/tests/test_gateway_usage_workflow.py b/products/tasks/backend/tests/test_gateway_usage_workflow.pyindex 193b825e9377..93b7eb211ee9 100644--- a/products/tasks/backend/tests/test_gateway_usage_workflow.py+++ b/products/tasks/backend/tests/test_gateway_usage_workflow.py@@ -38,17 +38,27 @@ from products.tasks.backend.temporal.gateway_usage import (@pytest.mark.parametrize("failed_connection", [False, True])async def test_scheduling_reuses_connection_after_success_and_retries_failure(failed_connection: bool) -> None:client = Mock(start_workflow=AsyncMock())+ attempts = 0++ async def connect_once() -> Mock:+ nonlocal attempts+ attempts += 1+ if failed_connection and attempts == 1:+ raise RuntimeError("unavailable")+ return client+_gateway_usage_client.cache_clear()with patch("posthog.temporal.common.client.async_connect", new_callable=AsyncMock) as connect:- connect.side_effect = [RuntimeError("unavailable"), client] if failed_connection else None- connect.return_value = client+ connect.side_effect = connect_onceif failed_connection:with pytest.raises(RuntimeError, match="unavailable"):await _schedule_gateway_usage(run_id=uuid4(), team_id=7)await asyncio.gather(*(_schedule_gateway_usage(run_id=uuid4(), team_id=7) for _ in range(2)))+ # The sync entry point keeps one loop per process, so repeated callbacks share a connection.+ await sync_to_async(schedule_gateway_usage)(run_id=uuid4(), team_id=7)await sync_to_async(schedule_gateway_usage)(run_id=uuid4(), team_id=7)- assert connect.await_count == (2 if failed_connection else 1)- assert client.start_workflow.await_count == 3+ assert connect.await_count == (3 if failed_connection else 2)+ assert client.start_workflow.await_count == 4_gateway_usage_client.cache_clear()diff --git a/products/tasks/backend/tests/test_models.py b/products/tasks/backend/tests/test_models.pyindex 49cec0a06ce4..c9ead6d11be6 100644--- a/products/tasks/backend/tests/test_models.py+++ b/products/tasks/backend/tests/test_models.py@@ -1484,7 +1484,7 @@ class TestTaskActivityClock(TestCase):def test_thread_message_moves_the_clock_without_touching_updated_at(self):task = self._task()- with team_scope(self.team.id):+ with team_scope(self.team.id), self.captureOnCommitCallbacks(execute=True):TaskThreadMessage.objects.create(team=self.team, task=task, author=self.user, content="Any news?", author_kind="human")@@ -1509,7 +1509,10 @@ class TestTaskActivityClock(TestCase):Task.objects.filter(id=task.id).update(last_activity_at=STALE_ACTIVITY_AT)run.status = TaskRun.Status.IN_PROGRESS- run.save(update_fields=update_fields)+ with self.captureOnCommitCallbacks(execute=True):+ run.save(update_fields=update_fields)+ task.refresh_from_db()+ self.assertEqual(task.last_activity_at, STALE_ACTIVITY_AT)task.refresh_from_db()assert task.last_activity_at is not None