Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tasks): stop task-run lock convoys (#129923)

Alessandro Pogliaghieae7723edd8a
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index 1844844a287c..32e6e872aaf9 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -465,6 +465,7 @@ rules:
|TaskAutomation
|TaskPresence
|TaskRun
+ |TaskRunGatewayRequest
|TaskWorkflowDispatch
|TaskSearchDocument
|TaskSession
@@ -969,6 +970,7 @@ rules:
|TaskAutomation
|TaskPresence
|TaskRun
+ |TaskRunGatewayRequest
|TaskWorkflowDispatch
|TaskSession
|UserTasksConfig
diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.py
index 4f293b658cdd..fee31fa3ebf4 100644
--- a/posthog/tasks/scheduled.py
+++ b/posthog/tasks/scheduled.py
@@ -147,6 +147,7 @@ from products.tasks.backend.facade.tasks import (
refresh_stale_sandbox_custom_images_task,
settle_overdue_sandbox_compute_task,
sweep_inactive_tasks_task,
+ sweep_queued_gateway_usage,
)
from products.visual_review.backend.facade.tasks import (
send_visual_review_debt_digests,
@@ -299,6 +300,15 @@ def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:
name="tasks run state metrics",
)
+ # Backstop for a gateway callback whose Celery kick was lost: the run's usage workflow only
+ # starts from that kick, so nothing would drain the intake table otherwise.
+ add_periodic_task_with_expiry(
+ sender,
+ crontab(minute="*/4"),
+ sweep_queued_gateway_usage.s(),
+ name="tasks queued gateway usage sweep",
+ )
+
add_periodic_task_with_expiry(
sender,
crontab(minute="*/2"),
diff --git a/products/tasks/backend/facade/gateway.py b/products/tasks/backend/facade/gateway.py
index 939c8ae1aee5..815752fdd2d0 100644
--- a/products/tasks/backend/facade/gateway.py
+++ b/products/tasks/backend/facade/gateway.py
@@ -1,14 +1,18 @@
from typing import TYPE_CHECKING
from uuid import UUID
+from celery import current_app
+
from posthog.llm.gateway_client import GatewayNotConfiguredError
-from products.tasks.backend.logic.services.gateway_usage import record_generation_request, schedule_gateway_usage
-from products.tasks.backend.models import TaskRun
+from products.tasks.backend.logic.services.gateway_usage import record_generation_request
if TYPE_CHECKING:
from posthog.llm.gateway_client import AIGatewayConfig
+# Sent by name so the web worker never imports the task module, which drags the Temporal tree into it.
+_SCHEDULE_GATEWAY_USAGE_TASK = "products.tasks.backend.tasks.tasks.schedule_task_run_gateway_usage"
+
def mint_private_gateway_token(
*,
@@ -45,10 +49,9 @@ def revoke_private_gateway_token(token: str, *, gateway_config: "AIGatewayConfig
def accept_generation_request(*, team_id: int, run_id: UUID, request_id: str) -> bool:
- try:
- recorded = record_generation_request(team_id=team_id, run_id=run_id, request_id=request_id)
- except TaskRun.DoesNotExist:
+ recorded = record_generation_request(team_id=team_id, run_id=run_id, request_id=request_id)
+ if recorded is None:
return False
if recorded:
- schedule_gateway_usage(team_id=team_id, run_id=run_id)
+ current_app.send_task(_SCHEDULE_GATEWAY_USAGE_TASK, kwargs={"run_id": str(run_id), "team_id": team_id})
return True
diff --git a/products/tasks/backend/facade/tasks.py b/products/tasks/backend/facade/tasks.py
index 82781c3ffbad..1d2af0598471 100644
--- a/products/tasks/backend/facade/tasks.py
+++ b/products/tasks/backend/facade/tasks.py
@@ -10,7 +10,7 @@ from products.tasks.backend.tasks.sandbox_images import (
refresh_dev_stack_image_task,
refresh_stale_sandbox_custom_images_task,
)
-from products.tasks.backend.tasks.tasks import settle_overdue_sandbox_compute_task
+from products.tasks.backend.tasks.tasks import settle_overdue_sandbox_compute_task, sweep_queued_gateway_usage
__all__ = [
"bake_dev_stack_image_task",
@@ -18,4 +18,5 @@ __all__ = [
"refresh_stale_sandbox_custom_images_task",
"settle_overdue_sandbox_compute_task",
"sweep_inactive_tasks_task",
+ "sweep_queued_gateway_usage",
]
diff --git a/products/tasks/backend/logic/services/gateway_usage.py b/products/tasks/backend/logic/services/gateway_usage.py
index 24b825eaa932..c1bb7500b898 100644
--- a/products/tasks/backend/logic/services/gateway_usage.py
+++ b/products/tasks/backend/logic/services/gateway_usage.py
@@ -1,6 +1,7 @@
import re
import time
import asyncio
+import threading
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
from datetime import datetime, timedelta
@@ -11,6 +12,7 @@ from uuid import UUID
from django.conf import settings
from django.db import transaction
+from django.db.models import Min
from django.utils import timezone
import structlog
@@ -22,7 +24,7 @@ from posthog.llm.gateway_usage import GatewayRequestCost, fetch_gateway_cost
from products.tasks.backend.facade.contracts import TaskRunCost, TaskRunSpend, TaskRunTokenUsage
from products.tasks.backend.facade.task_run_signals import task_run_cost_updated
from products.tasks.backend.logic.services.sandbox_pricing import COMPUTE_RATE_CARDS, calculate_sandbox_compute_cost
-from products.tasks.backend.models import SandboxSession, TaskRun
+from products.tasks.backend.models import SandboxSession, TaskRun, TaskRunGatewayRequest
if TYPE_CHECKING:
from temporalio.client import Client
@@ -30,6 +32,9 @@ if TYPE_CHECKING:
logger = structlog.get_logger(__name__)
_REQUEST_ID = re.compile(r"[a-zA-Z0-9_-]{1,255}\Z")
_PROCESSING_SECONDS = 10
+_DRAIN_BATCH_SIZE = 200
+_SCHEDULING_LOOP: asyncio.AbstractEventLoop | None = None
+_SCHEDULING_LOCK = threading.Lock()
@frozen
@@ -47,7 +52,11 @@ class _CostSources:
def _locked_run(run_id: UUID | str, team_id: int) -> TaskRun:
- return TaskRun.objects.select_for_update().get(id=run_id, team_id=team_id)
+ return (
+ TaskRun.objects.select_for_update()
+ .only("id", "team_id", "state", "environment", "task_id", "status", "created_at")
+ .get(id=run_id, team_id=team_id)
+ )
def gateway_usage_enabled(run: TaskRun) -> bool:
@@ -63,7 +72,7 @@ def _save_accounting_state(run: TaskRun) -> None:
# Accounting can finish after completion; it must not emit completion signals again.
# It also leaves `updated_at` alone, because recency gauges and stale-run sweeps read it.
previous = TaskRun.objects.select_related("task").get(id=run.id, team_id=run.team_id)
- delta = (_token_usage(run).cost_microusd or 0) - (_token_usage(previous).cost_microusd or 0)
+ delta = (_recorded_cost_microusd(run) or 0) - (_recorded_cost_microusd(previous) or 0)
TaskRun.objects.filter(id=run.id, team_id=run.team_id).update(state=run.state)
task_run_cost_updated.send(
sender=TaskRun,
@@ -111,24 +120,56 @@ def _processed_gateway_request_ids(state: dict[str, Any]) -> set[str]:
}
-def record_generation_request(*, team_id: int, run_id: UUID, request_id: str) -> bool:
- with transaction.atomic():
- run = _locked_run(run_id, team_id)
- if run.environment != TaskRun.Environment.CLOUD:
- return False
- state = dict(run.state or {})
- pending = state.get("unprocessed_request_ids")
- if not isinstance(pending, list):
- pending = []
- if request_id not in pending and request_id not in _processed_gateway_request_ids(state):
- pending = [*pending, request_id]
- state["unprocessed_request_ids"] = pending
- state.setdefault("token_cost", {})
- if state == run.state:
- return True
- run.state = state
- _save_accounting_state(run)
- return True
+def record_generation_request(*, team_id: int, run_id: UUID, request_id: str) -> bool | None:
+ environment = TaskRun.objects.filter(id=run_id, team_id=team_id).values_list("environment", flat=True).first()
+ if environment is None:
+ return None
+ if environment != TaskRun.Environment.CLOUD:
+ return False
+ TaskRunGatewayRequest.objects.for_team(team_id, canonical=True).bulk_create(
+ [TaskRunGatewayRequest(team_id=team_id, task_run_id=run_id, request_id=request_id)],
+ ignore_conflicts=True,
+ )
+ return True
+
+
+def _drain_queued_requests(run: TaskRun, state: dict[str, Any]) -> bool:
+ queued = list(
+ TaskRunGatewayRequest.objects.for_team(run.team_id, canonical=True)
+ .filter(task_run_id=run.id)
+ .order_by("created_at", "id")
+ .values_list("id", "request_id")[:_DRAIN_BATCH_SIZE]
+ )
+ if not queued:
+ return False
+
+ before = _pending_ids(state)
+ seen = set(before) | _processed_gateway_request_ids(state)
+ merged = list(before)
+ for _row_id, request_id in queued:
+ if request_id not in seen:
+ seen.add(request_id)
+ merged.append(request_id)
+
+ state["unprocessed_request_ids"] = merged
+ state.setdefault("token_cost", {})
+ TaskRunGatewayRequest.objects.for_team(run.team_id, canonical=True).filter(
+ id__in=[row_id for row_id, _ in queued]
+ ).delete()
+ return merged != before
+
+
+def _has_queued_requests(run: TaskRun) -> bool:
+ return TaskRunGatewayRequest.objects.for_team(run.team_id, canonical=True).filter(task_run_id=run.id).exists()
+
+
+def _runs_with_queued_requests(team_id: int, run_ids: Iterable[UUID]) -> set[UUID]:
+ return set(
+ TaskRunGatewayRequest.objects.for_team(team_id, canonical=True)
+ .filter(task_run_id__in=list(run_ids))
+ .values_list("task_run_id", flat=True)
+ .distinct()
+ )
def _pending_ids(state: dict[str, Any]) -> list[str]:
@@ -144,7 +185,10 @@ def _pending_ids(state: dict[str, Any]) -> list[str]:
def process_pending_gateway_usage(*, run_id: UUID, team_id: int, limit: int = 20) -> TaskRunCost:
with transaction.atomic():
run = _locked_run(run_id, team_id)
- state = run.state or {}
+ state = dict(run.state or {})
+ if _drain_queued_requests(run, state):
+ run.state = state
+ _save_accounting_state(run)
if not gateway_usage_enabled(run):
return _cost_sources(run).as_contract()
pending = _pending_ids(state)[:limit]
@@ -179,8 +223,8 @@ def process_pending_gateway_usage(*, run_id: UUID, team_id: int, limit: int = 20
def refresh_task_run_cost(*, run_id: UUID, team_id: int) -> TaskRunCost:
with transaction.atomic():
run = _locked_run(run_id, team_id)
- cost = _cost_sources(run).as_contract()
state = dict(run.state or {})
+ cost = _cost_sources(run).as_contract()
if "compute_cost" not in state or state["compute_cost"] != cost.compute_cost:
state["compute_cost"] = cost.compute_cost
run.state = state
@@ -196,10 +240,11 @@ def get_task_cost(*, team_id: int, task_id: UUID) -> TaskRunCost:
runs = list(TaskRun.objects.filter(team_id=team_id, task_id=task_id))
if not runs:
return TaskRunCost(token_cost=None, compute_cost=None)
+ queued = _runs_with_queued_requests(team_id, [run.id for run in runs])
sessions_by_run: dict[UUID, list[SandboxSession]] = {}
for session in SandboxSession.objects.for_team(team_id).filter(task_run__in=runs):
sessions_by_run.setdefault(session.task_run_id, []).append(session)
- sources = [_cost_sources(run, sessions=sessions_by_run.get(run.id, [])) for run in runs]
+ sources = [_cost_sources(run, sessions=sessions_by_run.get(run.id, []), queued=run.id in queued) for run in runs]
return _CostSources(
token_cost_microusd=None
if any(s.token_cost_microusd is None for s in sources)
@@ -220,24 +265,30 @@ def get_task_run_token_cost_microusd(*, run_id: UUID, team_id: int) -> int | Non
return _token_cost_microusd(TaskRun.objects.get(id=run_id, team_id=team_id))
-def _token_usage(run: TaskRun) -> TaskRunTokenUsage:
+def _token_usage(run: TaskRun, *, queued: bool | None = None) -> TaskRunTokenUsage:
state = run.state or {}
return TaskRunTokenUsage(
- cost_microusd=sum(bucket.get("cost_microusd", 0) for bucket in _cost_buckets(state))
- if isinstance(state.get("token_cost"), dict)
- else None,
+ cost_microusd=_recorded_cost_microusd(run),
complete=gateway_usage_enabled(run)
and not state.get("token_cost_incomplete", False)
- and not bool(state.get("unprocessed_request_ids")),
+ and not bool(state.get("unprocessed_request_ids"))
+ and not (_has_queued_requests(run) if queued is None else queued),
)
+def _recorded_cost_microusd(run: TaskRun) -> int | None:
+ state = run.state or {}
+ if not isinstance(state.get("token_cost"), dict):
+ return None
+ return sum(bucket.get("cost_microusd", 0) for bucket in _cost_buckets(state))
+
+
def get_task_run_token_usage(*, run_id: UUID, team_id: int) -> TaskRunTokenUsage:
return _token_usage(TaskRun.objects.get(id=run_id, team_id=team_id))
-def _run_spend(run: TaskRun) -> TaskRunSpend:
- usage = _token_usage(run)
+def _run_spend(run: TaskRun, *, queued: bool | None = None) -> TaskRunSpend:
+ usage = _token_usage(run, queued=queued)
return TaskRunSpend(
created_at=run.created_at,
run_id=run.id,
@@ -252,21 +303,26 @@ def _run_spend(run: TaskRun) -> TaskRunSpend:
@contextmanager
def locked_task_run_spend(*, team_id: int, run_id: UUID) -> Iterator[TaskRunSpend]:
with transaction.atomic():
- yield _run_spend(_locked_run(run_id, team_id))
+ run = _locked_run(run_id, team_id)
+ yield _run_spend(run)
def get_task_runs_spend(*, team_id: int, run_ids: Iterable[UUID]) -> list[TaskRunSpend]:
- return [_run_spend(run) for run in TaskRun.objects.filter(team_id=team_id, id__in=run_ids)]
+ runs = list(TaskRun.objects.filter(team_id=team_id, id__in=run_ids))
+ queued = _runs_with_queued_requests(team_id, [run.id for run in runs])
+ return [_run_spend(run, queued=run.id in queued) for run in runs]
-def _token_cost_microusd(run: TaskRun) -> int | None:
- usage = _token_usage(run)
+def _token_cost_microusd(run: TaskRun, *, queued: bool | None = None) -> int | None:
+ usage = _token_usage(run, queued=queued)
return usage.cost_microusd if usage.complete else None
-def _cost_sources(run: TaskRun, *, sessions: list[SandboxSession] | None = None) -> _CostSources:
+def _cost_sources(
+ run: TaskRun, *, sessions: list[SandboxSession] | None = None, queued: bool | None = None
+) -> _CostSources:
return _CostSources(
- token_cost_microusd=_token_cost_microusd(run),
+ token_cost_microusd=_token_cost_microusd(run, queued=queued),
compute_cost_usd=_compute_cost_source(run, sessions=sessions),
)
@@ -329,4 +385,34 @@ async def _schedule_gateway_usage(*, run_id: UUID, team_id: int) -> None:
def schedule_gateway_usage(*, run_id: UUID, team_id: int) -> None:
- async_to_sync(_schedule_gateway_usage)(run_id=run_id, team_id=team_id)
+ with _SCHEDULING_LOCK:
+ _scheduling_loop().run_until_complete(_schedule_gateway_usage(run_id=run_id, team_id=team_id))
+
+
+def _scheduling_loop() -> asyncio.AbstractEventLoop:
+ # asgiref builds a fresh loop per sync call, so the client cache below would miss on every callback.
+ global _SCHEDULING_LOOP
+ if _SCHEDULING_LOOP is None or _SCHEDULING_LOOP.is_closed():
+ _SCHEDULING_LOOP = asyncio.new_event_loop()
+ return _SCHEDULING_LOOP
+
+
+def sweep_queued_gateway_requests(*, older_than_seconds: int = 600, limit: int = 50) -> int:
+ cutoff = timezone.now() - timedelta(seconds=older_than_seconds)
+ stale = list(
+ TaskRunGatewayRequest.objects.unscoped()
+ .values("team_id", "task_run_id")
+ .annotate(oldest=Min("created_at"))
+ .filter(oldest__lt=cutoff)
+ .order_by("oldest")
+ .values_list("team_id", "task_run_id")[:limit]
+ )
+ scheduled = 0
+ for team_id, run_id in stale:
+ try:
+ schedule_gateway_usage(run_id=run_id, team_id=team_id)
+ except Exception:
+ logger.exception("task_gateway_usage.sweep_schedule_failed", run_id=str(run_id), team_id=team_id)
+ continue
+ scheduled += 1
+ return scheduled
diff --git a/products/tasks/backend/migrations/0149_task_run_gateway_request.py b/products/tasks/backend/migrations/0149_task_run_gateway_request.py
new file mode 100644
index 000000000000..4b8085a42881
--- /dev/null
+++ b/products/tasks/backend/migrations/0149_task_run_gateway_request.py
@@ -0,0 +1,47 @@
+import django.utils.timezone
+import django.db.models.deletion
+from django.db import migrations, models
+
+import posthog.models.utils
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "1425_workload_trust_policy_request"),
+ ("tasks", "0148_task_source_indexes"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="TaskRunGatewayRequest",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ default=posthog.models.utils.uuid7, editable=False, primary_key=True, serialize=False
+ ),
+ ),
+ ("task_run_id", models.UUIDField()),
+ ("request_id", models.CharField(max_length=255)),
+ ("created_at", models.DateTimeField(default=django.utils.timezone.now)),
+ (
+ "team",
+ models.ForeignKey(
+ db_constraint=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="+",
+ to="posthog.team",
+ ),
+ ),
+ ],
+ options={
+ "db_table": "posthog_task_run_gateway_request",
+ "constraints": [
+ models.UniqueConstraint(
+ fields=("task_run_id", "request_id"), name="task_run_gateway_request_unique"
+ )
+ ],
+ "indexes": [models.Index(fields=["task_run_id", "created_at"], name="task_run_gw_request_idx")],
+ },
+ ),
+ ]
diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txt
index 3113eb44471c..d351f90c1851 100644
--- a/products/tasks/backend/migrations/max_migration.txt
+++ b/products/tasks/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0148_task_source_indexes
+0149_task_run_gateway_request
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index 70fb5a745518..1042c0935ed3 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -2382,7 +2382,8 @@ def bump_task_activity(*, team_id: int, task_id: uuid.UUID | str, at: datetime)
@receiver(post_save, sender=TaskThreadMessage)
def bump_task_activity_on_thread_message(sender, instance: "TaskThreadMessage", created: bool, **kwargs) -> None:
if created:
- bump_task_activity(team_id=instance.team_id, task_id=instance.task_id, at=instance.created_at)
+ team_id, task_id, at = instance.team_id, instance.task_id, instance.created_at
+ transaction.on_commit(lambda: bump_task_activity(team_id=team_id, task_id=task_id, at=at))
@receiver(post_save, sender=Task)
@@ -3731,6 +3732,25 @@ class TaskRun(models.Model):
raise Exception("Cannot delete TaskRun. Task runs are immutable records.")
+class TaskRunGatewayRequest(TeamScopedRootMixin, UUIDModel):
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False)
+ task_run_id = models.UUIDField()
+ request_id = models.CharField(max_length=255)
+ created_at = models.DateTimeField(default=django_timezone.now)
+
+ class Meta:
+ db_table = "posthog_task_run_gateway_request"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["task_run_id", "request_id"],
+ name="task_run_gateway_request_unique",
+ )
+ ]
+ indexes = [
+ models.Index(fields=["task_run_id", "created_at"], name="task_run_gw_request_idx"),
+ ]
+
+
class TaskWorkflowDispatch(TeamScopedRootMixin):
class Kind(models.TextChoices):
CREATE = "create", "create"
@@ -4507,7 +4527,8 @@ def bump_task_activity_on_run(sender, instance: TaskRun, created: bool, update_f
# to bump would silently leave a live session reading as idle.
if not created and update_fields is not None and not (RUN_ACTIVITY_FIELDS & set(update_fields)):
return
- bump_task_activity(team_id=instance.team_id, task_id=instance.task_id, at=django_timezone.now())
+ team_id, task_id, at = instance.team_id, instance.task_id, django_timezone.now()
+ transaction.on_commit(lambda: bump_task_activity(team_id=team_id, task_id=task_id, at=at))
class TeamTasksConfig(models.Model):
diff --git a/products/tasks/backend/tasks/tasks.py b/products/tasks/backend/tasks/tasks.py
index d31b05b0fc5b..0af53b35ee79 100644
--- a/products/tasks/backend/tasks/tasks.py
+++ b/products/tasks/backend/tasks/tasks.py
@@ -95,6 +95,24 @@ def deliver_comment_slack_dms(
)
+@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=5)
+def schedule_task_run_gateway_usage(*, run_id: str, team_id: int) -> None:
+ from products.tasks.backend.logic.services.gateway_usage import ( # noqa: PLC0415 — keeps temporalio off the celery import path
+ schedule_gateway_usage,
+ )
+
+ schedule_gateway_usage(run_id=UUID(run_id), team_id=team_id)
+
+
+@shared_task(ignore_result=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=3)
+def sweep_queued_gateway_usage() -> None:
+ from products.tasks.backend.logic.services.gateway_usage import ( # noqa: PLC0415 — keeps temporalio off the celery import path
+ sweep_queued_gateway_requests,
+ )
+
+ sweep_queued_gateway_requests()
+
+
# No retries: the wake is best-effort by design, and the parked step has its own deadline.
@shared_task(ignore_result=True)
def resume_workflow_step_for_run_deferred(run_id: str) -> None:
diff --git a/products/tasks/backend/temporal/oauth.py b/products/tasks/backend/temporal/oauth.py
index 14dc38f6878e..426ce6f910ce 100644
--- a/products/tasks/backend/temporal/oauth.py
+++ b/products/tasks/backend/temporal/oauth.py
@@ -186,6 +186,22 @@ def _workflow_run_scopes(requested: PosthogMcpScopes, state: dict[str, Any] | No
return sorted(resolved)
+def _builtin_agent_enforcement(task: Task) -> bool | None:
+ if Task.mcp_agent_key_for(task.origin_product, task.stored_profile) is None:
+ return None
+ return is_builtin_agent_enforcement_enabled(task.team_id)
+
+
+def _monitoring_result_scope(task: Task, run_id: UUID | str | None) -> bool | None:
+ if task.origin_product != Task.OriginProduct.SIGNALS_SCOUT or run_id is None:
+ return None
+ from products.signals.backend.facade.monitoring import ( # noqa: PLC0415 -- avoids loading Signals during Tasks startup
+ monitoring_scout_can_submit,
+ )
+
+ return monitoring_scout_can_submit(team_id=task.team_id, task_id=task.id, task_run_id=run_id)
+
+
def create_oauth_access_token(
task: Task,
*,
@@ -194,6 +210,8 @@ def create_oauth_access_token(
allow_task_creator_fallback: bool = True,
run_state: dict[str, Any] | None = None,
run_id: UUID | str | None = None,
+ builtin_agent_enforcement: bool | None = None,
+ monitoring_result_scope: bool | None = None,
) -> str:
"""Create an OAuth access token for the task's sandbox app, scoped to the task's team.
@@ -215,9 +233,9 @@ def create_oauth_access_token(
"application": _oauth_application_for_task(task, run_state),
"sandbox_task_id": task.id,
}
- if Task.mcp_agent_key_for(task.origin_product, task.stored_profile) is not None and (
- is_builtin_agent_enforcement_enabled(task.team_id)
- ):
+ if builtin_agent_enforcement is None:
+ builtin_agent_enforcement = _builtin_agent_enforcement(task)
+ if builtin_agent_enforcement:
# This scope only removes access to the human MCP Store surface. Add it
# even when a legacy task lacks trusted provenance so an old spoofed
# origin fails closed instead of inheriting its creator's connections.
@@ -235,13 +253,10 @@ def create_oauth_access_token(
withheld.update(RESEARCH_WITHHELD_SCOPES)
if withheld:
token_options["withhold_scopes"] = sorted(withheld)
- if task.origin_product == Task.OriginProduct.SIGNALS_SCOUT and run_id is not None:
- from products.signals.backend.facade.monitoring import ( # noqa: PLC0415 -- avoids loading Signals during Tasks startup
- monitoring_scout_can_submit,
- )
-
- if monitoring_scout_can_submit(team_id=task.team_id, task_id=task.id, task_run_id=run_id):
- token_options["include_monitoring_result_scope"] = True
+ if monitoring_result_scope is None:
+ monitoring_result_scope = _monitoring_result_scope(task, run_id)
+ if monitoring_result_scope:
+ token_options["include_monitoring_result_scope"] = True
return create_oauth_access_token_for_user(actor, task.team_id, **token_options)
@@ -280,9 +295,11 @@ def create_oauth_access_token_for_run(
elif scopes == "signals_assessment":
# Runtime uploads are bound to this run; the separately minted MCP token stays read-only.
scopes = [*resolve_scopes(scopes), "internal_run:write"]
+ builtin_agent_enforcement = _builtin_agent_enforcement(task)
+ monitoring_result_scope = _monitoring_result_scope(task, run_id)
with transaction.atomic():
locked_task = (
- Task.objects.select_for_update(of=("self",))
+ Task.objects.select_for_update(of=("self",), no_key=True)
.select_related("created_by", "team")
.get(id=task.id, team_id=task.team_id)
)
@@ -328,6 +345,8 @@ def create_oauth_access_token_for_run(
allow_task_creator_fallback=not is_slack_interaction_state(state),
run_state=state,
run_id=run_id,
+ builtin_agent_enforcement=builtin_agent_enforcement,
+ monitoring_result_scope=monitoring_result_scope,
)
if run_id is not None:
run = TaskRun.objects.select_for_update().get(
diff --git a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py
index adfe25e6ec00..8c57bb744cd2 100644
--- a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py
+++ b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py
@@ -618,18 +618,18 @@ def _record_agent_startup_attempt(ctx: TaskProcessingContext, runtime: str) -> N
def _record_agent_startup_success(ctx: TaskProcessingContext) -> None:
with transaction.atomic():
- state = TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).values_list("state", flat=True).first()
- attempt = (state or {}).get("agent_startup_attempt")
- TaskRun.update_state_atomic(
- ctx.run_id,
- updates={"agent_startup_success": {**attempt, "completed_at": datetime.now(UTC).isoformat()}}
- if isinstance(attempt, dict)
- else {},
- remove_keys=["startup_failure"],
- )
- TaskRun.objects.filter(id=ctx.run_id, team_id=ctx.team_id).exclude(
- status__in=[TaskRun.Status.COMPLETED, TaskRun.Status.FAILED, TaskRun.Status.CANCELLED]
- ).update(error_message=None)
+ run = TaskRun.objects.select_for_update().only("id", "state", "status").get(id=ctx.run_id, team_id=ctx.team_id)
+ state = dict(run.state or {})
+ attempt = state.get("agent_startup_attempt")
+ state.pop("startup_failure", None)
+ if isinstance(attempt, dict):
+ state["agent_startup_success"] = {**attempt, "completed_at": datetime.now(UTC).isoformat()}
+ run.state = state
+ update_fields = ["state", "updated_at"]
+ if not run.is_terminal:
+ run.error_message = None
+ update_fields.append("error_message")
+ run.save(update_fields=update_fields)
def _invoke_start_agent_server(
diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py
index 3c6f6b767a52..139efb609307 100644
--- a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py
+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py
@@ -62,6 +62,10 @@ def task_run_state(mocker):
)
state = runs.return_value.values_list.return_value.first
state.return_value = None
+ startup = mocker.patch(
+ "products.tasks.backend.temporal.process_task.activities.start_agent_server.TaskRun.objects.select_for_update"
+ )
+ startup.return_value.only.return_value.get.return_value = mocker.Mock(state={}, is_terminal=False)
return state
diff --git a/products/tasks/backend/tests/test_gateway_usage.py b/products/tasks/backend/tests/test_gateway_usage.py
index 80b055795879..0ea653710465 100644
--- a/products/tasks/backend/tests/test_gateway_usage.py
+++ b/products/tasks/backend/tests/test_gateway_usage.py
@@ -24,9 +24,10 @@ from products.tasks.backend.logic.services.gateway_usage import (
record_gateway_routing,
record_generation_request,
refresh_task_run_cost,
+ sweep_queued_gateway_requests,
)
from products.tasks.backend.logic.services.sandbox_pricing import COMPUTE_RATE_CARDS
-from products.tasks.backend.models import SandboxSession, Task, TaskRun
+from products.tasks.backend.models import SandboxSession, Task, TaskRun, TaskRunGatewayRequest
@override_settings(SANDBOX_AI_GATEWAY_URL="https://gateway.example.com/v1", SANDBOX_AI_GATEWAY_MINT_KEY="phs_test")
@@ -73,15 +74,18 @@ class TestGatewayUsage(BaseTest):
task_run_cost_updated.connect(cost_updated, sender=TaskRun)
self.addCleanup(task_run_cost_updated.disconnect, cost_updated, sender=TaskRun)
self._report(run, ["parent", "subagent", "other-model", "second-turn", "parent"])
+ record_generation_request(team_id=self.team.id, run_id=run.id, request_id="queued")
get.side_effect = [
self._response("parent", "0.005"),
self._response("subagent", "0.010001", provider="provider-b"),
self._response("other-model", "0.000009", model="model-b"),
self._response("second-turn", "0.005"),
+ self._response("queued", "0.009"),
]
- assert self._process(run).token_cost == 2
- assert observed_costs == {run.id: 20_010}
- assert get.call_count == 4
+ assert self._process(run).token_cost == 3
+ assert observed_costs == {run.id: 29_010}
+ assert not TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).exists()
+ assert get.call_count == 5
assert get.call_args.kwargs["timeout"].total == 15
assert get.call_args.args[0].trust_env is False
run.refresh_from_db()
@@ -89,7 +93,7 @@ class TestGatewayUsage(BaseTest):
"unprocessed_request_ids": [],
"token_cost": {
"model-a": {
- "provider-a": {"cost_microusd": 10_000, "request_ids": ["parent", "second-turn"]},
+ "provider-a": {"cost_microusd": 19_000, "request_ids": ["parent", "second-turn", "queued"]},
"provider-b": {"cost_microusd": 10_001, "request_ids": ["subagent"]},
},
"model-b": {"provider-a": {"cost_microusd": 9, "request_ids": ["other-model"]}},
@@ -97,7 +101,7 @@ class TestGatewayUsage(BaseTest):
"compute_cost": None,
}
get.reset_mock()
- assert self._process(run).token_cost == 2
+ assert self._process(run).token_cost == 3
get.assert_not_called()
@patch("aiohttp.ClientSession._request")
@@ -235,7 +239,7 @@ class TestGatewayUsage(BaseTest):
get.reset_mock()
assert get_task_run_cost(run_id=first.id, team_id=self.team.id) == TaskRunCost(token_cost=0, compute_cost=0)
assert get_task_run_cost(run_id=second.id, team_id=self.team.id) == TaskRunCost(token_cost=0, compute_cost=0)
- with self.assertNumQueries(3):
+ with self.assertNumQueries(4):
assert get_task_cost(team_id=self.team.id, task_id=first.task_id) == TaskRunCost(
token_cost=1, compute_cost=1
)
@@ -248,6 +252,51 @@ class TestGatewayUsage(BaseTest):
run.save(update_fields=["state"])
assert get_task_run_cost(run_id=run.id, team_id=self.team.id).token_cost is None
+ @patch("aiohttp.ClientSession._request")
+ def test_queued_callbacks_keep_usage_incomplete_until_they_are_costed(self, get: Mock) -> None:
+ run = self._run()
+ record_generation_request(run_id=run.id, team_id=run.team_id, request_id="queued")
+ get.return_value = self._response("queued", "0.009")
+
+ assert not get_task_run_token_usage(run_id=run.id, team_id=self.team.id).complete
+ assert get_task_run_cost(run_id=run.id, team_id=self.team.id).token_cost is None
+ assert get_task_run_token_cost_microusd(run_id=run.id, team_id=self.team.id) is None
+
+ assert refresh_task_run_cost(run_id=run.id, team_id=self.team.id).token_cost is None
+ assert TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).exists()
+ assert not get_task_run_token_usage(run_id=run.id, team_id=self.team.id).complete
+ get.assert_not_called()
+
+ assert self._process(run).token_cost == 1
+ assert not TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).exists()
+ assert get_task_run_token_usage(run_id=run.id, team_id=self.team.id).complete
+ assert get_task_run_token_cost_microusd(run_id=run.id, team_id=self.team.id) == 9_000
+
+ def test_sweep_reschedules_a_stale_queued_callback(self) -> None:
+ run = self._run()
+ record_generation_request(run_id=run.id, team_id=run.team_id, request_id="queued")
+ TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id).update(
+ created_at=timezone.now() - timedelta(minutes=15)
+ )
+
+ with patch("products.tasks.backend.logic.services.gateway_usage.schedule_gateway_usage") as schedule:
+ assert sweep_queued_gateway_requests() == 1
+
+ schedule.assert_called_once_with(run_id=run.id, team_id=self.team.id)
+
+ def test_sweep_groups_by_run_so_one_backlog_cannot_starve_another(self) -> None:
+ stuck, waiting = self._run(), self._run()
+ queued = TaskRunGatewayRequest.objects.for_team(self.team.id)
+ for index in range(60):
+ queued.create(team=self.team, task_run_id=stuck.id, request_id=f"stuck-{index}")
+ queued.create(team=self.team, task_run_id=waiting.id, request_id="waiting")
+ queued.update(created_at=timezone.now() - timedelta(minutes=15))
+
+ with patch("products.tasks.backend.logic.services.gateway_usage.schedule_gateway_usage") as schedule:
+ assert sweep_queued_gateway_requests() == 2
+
+ assert {call.kwargs["run_id"] for call in schedule.call_args_list} == {stuck.id, waiting.id}
+
def test_compute_cost_uses_existing_ledger_attribution_and_resource_floors(self) -> None:
run = self._run()
now = timezone.now()
diff --git a/products/tasks/backend/tests/test_gateway_usage_api.py b/products/tasks/backend/tests/test_gateway_usage_api.py
index a9ca392d7b68..223551ef1592 100644
--- a/products/tasks/backend/tests/test_gateway_usage_api.py
+++ b/products/tasks/backend/tests/test_gateway_usage_api.py
@@ -15,14 +15,14 @@ from rest_framework.test import APIClient
from posthog.models import ProjectSecretAPIKey, Team
from posthog.models.utils import hash_key_value
-from products.tasks.backend.models import Task, TaskRun
+from products.tasks.backend.models import Task, TaskRun, TaskRunGatewayRequest
@override_settings(AI_GATEWAY_INTERNAL_TOKEN="gateway-token")
class TestTaskRunGatewayUsageAPI(APIBaseTest):
def setUp(self) -> None:
super().setUp()
- self.schedule_usage = self.enterContext(patch("products.tasks.backend.facade.gateway.schedule_gateway_usage"))
+ self.schedule_usage = self.enterContext(patch("products.tasks.backend.facade.gateway.current_app.send_task"))
def _run(
self,
@@ -75,13 +75,18 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):
assert response.status_code == status.HTTP_204_NO_CONTENT
run.refresh_from_db()
+ queued = TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id)
if environment == TaskRun.Environment.CLOUD:
- assert run.state == {"unprocessed_request_ids": ["request_1"], "token_cost": {}}
- self.schedule_usage.assert_called_once()
+ assert list(queued.values_list("request_id", flat=True)) == ["request_1"]
+ self.schedule_usage.assert_called_once_with(
+ "products.tasks.backend.tasks.tasks.schedule_task_run_gateway_usage",
+ kwargs={"run_id": str(run.id), "team_id": run.team_id},
+ )
else:
- assert run.state == {}
- assert run.updated_at == updated_at
+ assert not queued.exists()
self.schedule_usage.assert_not_called()
+ assert run.state == {}
+ assert run.updated_at == updated_at
def test_callback_rejects_missing_or_invalid_service_credential(self) -> None:
run = self._run()
@@ -98,32 +103,39 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):
run = self._run()
assert self._post(run).status_code == status.HTTP_401_UNAUTHORIZED
- @parameterized.expand([("state_write",), ("scheduling",)])
+ @parameterized.expand([("queue_write",), ("scheduling",)])
def test_callback_does_not_acknowledge_a_failed_write_or_schedule(self, failure: str) -> None:
run = self._run()
client = APIClient()
client.raise_request_exception = False
- target = (
- "products.tasks.backend.logic.services.gateway_usage._save_accounting_state"
- if failure == "state_write"
- else "products.tasks.backend.facade.gateway.schedule_gateway_usage"
- )
- with patch(target, side_effect=OperationalError("unavailable")):
- response = client.post(
+
+ def post() -> HttpResponse:
+ return client.post(
self._url(run),
{},
format="json",
HTTP_AUTHORIZATION="Bearer gateway-token",
HTTP_X_POSTHOG_GATEWAY_TEAM_ID=str(run.team_id),
)
+
+ if failure == "queue_write":
+ with patch("django.db.models.QuerySet.bulk_create", side_effect=OperationalError("unavailable")):
+ response = post()
+ else:
+ self.schedule_usage.side_effect = OperationalError("unavailable")
+ try:
+ response = post()
+ finally:
+ self.schedule_usage.side_effect = None
+
assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
run.refresh_from_db()
- assert run.state == (
- {} if failure == "state_write" else {"unprocessed_request_ids": ["request_1"], "token_cost": {}}
- )
+ assert run.state == {}
+ queued = TaskRunGatewayRequest.objects.for_team(self.team.id).filter(task_run_id=run.id, request_id="request_1")
+ assert queued.exists() == (failure == "scheduling")
+
assert self._post(run).status_code == status.HTTP_204_NO_CONTENT
- run.refresh_from_db()
- assert run.state["unprocessed_request_ids"] == ["request_1"]
+ assert queued.exists()
def test_callback_requires_a_valid_wallet_and_request_identifiers(self) -> None:
run = self._run()
@@ -151,8 +163,11 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):
response = self._post(run, wallet_team_id=other_team.id)
assert response.status_code == status.HTTP_204_NO_CONTENT
- run.refresh_from_db()
- assert run.state["unprocessed_request_ids"] == ["request_1"]
+ assert (
+ TaskRunGatewayRequest.objects.for_team(self.team.id)
+ .filter(task_run_id=run.id, request_id="request_1")
+ .exists()
+ )
@override_settings(SANDBOX_AI_GATEWAY_MINT_KEY="phs_callback_mint")
def test_callback_rejects_a_foreign_wallet(self) -> None:
@@ -170,10 +185,17 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):
def test_callback_is_idempotent_before_and_after_processing_and_terminalization(self) -> None:
run = self._run()
+ def queued_request_ids() -> list[str]:
+ return list(
+ TaskRunGatewayRequest.objects.for_team(self.team.id)
+ .filter(task_run_id=run.id)
+ .order_by("created_at", "id")
+ .values_list("request_id", flat=True)
+ )
+
assert self._post(run).status_code == status.HTTP_204_NO_CONTENT
assert self._post(run).status_code == status.HTTP_204_NO_CONTENT
- run.refresh_from_db()
- assert run.state["unprocessed_request_ids"] == ["request_1"]
+ assert queued_request_ids() == ["request_1"]
run.status = TaskRun.Status.COMPLETED
run.state = {
@@ -185,11 +207,11 @@ class TestTaskRunGatewayUsageAPI(APIBaseTest):
assert self._post(run).status_code == status.HTTP_204_NO_CONTENT
run.refresh_from_db()
assert run.status == TaskRun.Status.COMPLETED
- assert run.state["unprocessed_request_ids"] == []
assert run.state["token_cost"]["model"]["provider"]["cost_microusd"] == 4
+ assert queued_request_ids() == ["request_1"]
assert self._post(run, "request_2").status_code == status.HTTP_204_NO_CONTENT
+ assert queued_request_ids() == ["request_1", "request_2"]
run.refresh_from_db()
- assert run.state["unprocessed_request_ids"] == ["request_2"]
assert run.status == TaskRun.Status.COMPLETED
def test_callback_returns_not_found_for_a_run_outside_the_path_team(self) -> None:
diff --git a/products/tasks/backend/tests/test_gateway_usage_workflow.py b/products/tasks/backend/tests/test_gateway_usage_workflow.py
index 193b825e9377..93b7eb211ee9 100644
--- a/products/tasks/backend/tests/test_gateway_usage_workflow.py
+++ b/products/tasks/backend/tests/test_gateway_usage_workflow.py
@@ -38,17 +38,27 @@ from products.tasks.backend.temporal.gateway_usage import (
@pytest.mark.parametrize("failed_connection", [False, True])
async def test_scheduling_reuses_connection_after_success_and_retries_failure(failed_connection: bool) -> None:
client = Mock(start_workflow=AsyncMock())
+ attempts = 0
+
+ async def connect_once() -> Mock:
+ nonlocal attempts
+ attempts += 1
+ if failed_connection and attempts == 1:
+ raise RuntimeError("unavailable")
+ return client
+
_gateway_usage_client.cache_clear()
with patch("posthog.temporal.common.client.async_connect", new_callable=AsyncMock) as connect:
- connect.side_effect = [RuntimeError("unavailable"), client] if failed_connection else None
- connect.return_value = client
+ connect.side_effect = connect_once
if failed_connection:
with pytest.raises(RuntimeError, match="unavailable"):
await _schedule_gateway_usage(run_id=uuid4(), team_id=7)
await asyncio.gather(*(_schedule_gateway_usage(run_id=uuid4(), team_id=7) for _ in range(2)))
+ # The sync entry point keeps one loop per process, so repeated callbacks share a connection.
+ await sync_to_async(schedule_gateway_usage)(run_id=uuid4(), team_id=7)
await sync_to_async(schedule_gateway_usage)(run_id=uuid4(), team_id=7)
- assert connect.await_count == (2 if failed_connection else 1)
- assert client.start_workflow.await_count == 3
+ assert connect.await_count == (3 if failed_connection else 2)
+ assert client.start_workflow.await_count == 4
_gateway_usage_client.cache_clear()
diff --git a/products/tasks/backend/tests/test_models.py b/products/tasks/backend/tests/test_models.py
index 49cec0a06ce4..c9ead6d11be6 100644
--- a/products/tasks/backend/tests/test_models.py
+++ b/products/tasks/backend/tests/test_models.py
@@ -1484,7 +1484,7 @@ class TestTaskActivityClock(TestCase):
def test_thread_message_moves_the_clock_without_touching_updated_at(self):
task = self._task()
- with team_scope(self.team.id):
+ with team_scope(self.team.id), self.captureOnCommitCallbacks(execute=True):
TaskThreadMessage.objects.create(
team=self.team, task=task, author=self.user, content="Any news?", author_kind="human"
)
@@ -1509,7 +1509,10 @@ class TestTaskActivityClock(TestCase):
Task.objects.filter(id=task.id).update(last_activity_at=STALE_ACTIVITY_AT)
run.status = TaskRun.Status.IN_PROGRESS
- run.save(update_fields=update_fields)
+ with self.captureOnCommitCallbacks(execute=True):
+ run.save(update_fields=update_fields)
+ task.refresh_from_db()
+ self.assertEqual(task.last_activity_at, STALE_ACTIVITY_AT)
task.refresh_from_db()
assert task.last_activity_at is not None