:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): settle sandbox compute to the gateway ledger (#127448)
Alessandro Pogliaghie60de2cb1eb8
diff --git a/posthog/llm/gateway_internal_client.py b/posthog/llm/gateway_internal_client.pyindex a0590192b022..3326c87a8d46 100644--- a/posthog/llm/gateway_internal_client.py+++ b/posthog/llm/gateway_internal_client.py@@ -6,11 +6,13 @@ from django.conf import settingsimport httpxINTERNAL_API_TIMEOUT_SECONDS = 5.0+REPORTED_SPEND_TIMEOUT_SECONDS = 2.0IDEMPOTENCY_KEY_HEADER = "Idempotency-Key"INTERNAL_ACTOR_HEADER = "X-Internal-Actor"# Ledger provenance tag (meta.source): admin-originated vs the billing path's# "billing". The acting human is audited in ActivityLog, not here.ADMIN_ACTOR = "posthog-admin"+TASKS_COMPUTE_ACTOR = "tasks-compute"class AIGatewayInternalError(Exception):@@ -52,6 +54,13 @@ class CreditResult:duplicate: bool[email protected](frozen=True)+class ReportedSpendResponse:+ status_code: int+ body: dict[str, Any] | None+ text: str++def _config() -> tuple[str, str]:url = (settings.AI_GATEWAY_INTERNAL_URL or "").rstrip("/")token = settings.AI_GATEWAY_INTERNAL_TOKEN or ""@@ -142,6 +151,46 @@ def add_credit(team_id: int, amount_usd: str, reason: str, idempotency_key: str))+def report_spend(wallet_team_id: int, body: dict[str, Any]) -> ReportedSpendResponse:+ url, token = _config()+ response = httpx.post(+ f"{url}/internal/teams/{wallet_team_id}/reported-spend",+ headers=_auth_headers(token, {INTERNAL_ACTOR_HEADER: TASKS_COMPUTE_ACTOR}),+ json=body,+ timeout=REPORTED_SPEND_TIMEOUT_SECONDS,+ trust_env=False,+ )+ try:+ parsed: Any = response.json()+ except ValueError:+ parsed = None+ return ReportedSpendResponse(+ status_code=response.status_code,+ body=parsed if isinstance(parsed, dict) else None,+ text=response.text,+ )+++def get_usage_report(params: dict[str, str]) -> list[dict[str, Any]]:+ url, token = _config()+ try:+ response = httpx.get(+ f"{url}/internal/usage/report",+ headers=_auth_headers(token, {INTERNAL_ACTOR_HEADER: TASKS_COMPUTE_ACTOR}),+ params=params,+ timeout=INTERNAL_API_TIMEOUT_SECONDS,+ trust_env=False,+ )+ response.raise_for_status()+ data: Any = response.json()+ except (httpx.HTTPError, ValueError) as exc:+ raise AIGatewayInternalError(f"usage report read failed: {exc}") from exc+ rows = data.get("rows") if isinstance(data, dict) else data+ if not isinstance(rows, list):+ raise AIGatewayInternalError("usage report response had no rows")+ return [row for row in rows if isinstance(row, dict)]++def _error_detail(response: httpx.Response) -> str:try:body: Any = response.json()diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 4efaf0ac9576..f6a52dcf323d 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -145,6 +145,7 @@ SANDBOX_AI_GATEWAY_TOKEN_CAP_USD_PRODUCT_OVERRIDES: str = get_from_env("SANDBOX_AI_GATEWAY_TOKEN_CAP_USD_PRODUCT_OVERRIDES", "")SANDBOX_AI_GATEWAY_TOKEN_TTL_SECONDS: int = get_from_env("SANDBOX_AI_GATEWAY_TOKEN_TTL_SECONDS", 0, type_cast=int)+GATEWAY_COMPUTE_DEBIT_ENABLED: bool = get_from_env("GATEWAY_COMPUTE_DEBIT_ENABLED", False, type_cast=str_to_bool)SANDBOX_MCP_URL: str | None = get_from_env("SANDBOX_MCP_URL", None, optional=True)# OTLP destinations for agent-server run telemetry (PostHog Logs/APM).diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.pyindex ff5c8ab7b3e8..c29d83d8c339 100644--- a/posthog/tasks/scheduled.py+++ b/posthog/tasks/scheduled.py@@ -145,6 +145,7 @@ from products.tasks.backend.facade.tasks import (bake_dev_stack_image_task,refresh_dev_stack_image_task,refresh_stale_sandbox_custom_images_task,+ settle_overdue_sandbox_compute_task,sweep_inactive_tasks_task,)from products.visual_review.backend.facade.tasks import (@@ -527,6 +528,13 @@ def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:name="archive inactive tasks",)+ add_periodic_task_with_expiry(+ sender,+ crontab(minute="47"),+ settle_overdue_sandbox_compute_task.s(),+ name="settle overdue sandbox compute",+ )+# AWS SES account reputation → gauges for team-facing alerting (charts alerts/specs/ses.yaml)add_periodic_task_with_expiry(sender,diff --git a/products/tasks/backend/facade/gateway.py b/products/tasks/backend/facade/gateway.pyindex 1a86276f36ef..939c8ae1aee5 100644--- a/products/tasks/backend/facade/gateway.py+++ b/products/tasks/backend/facade/gateway.py@@ -16,6 +16,7 @@ def mint_private_gateway_token(user: str | None = None,expires_in_seconds: int | None = None,gateway_config: "AIGatewayConfig | None" = None,+ run_id: str | None = None,) -> str:from products.tasks.backend.temporal.process_task.ai_gateway_token import ( # noqa: PLC0415 -- breaks the Temporal aggregate's import cycle through process_task.utilsmint_scoped_token,@@ -28,6 +29,7 @@ def mint_private_gateway_token(capture_mode="none",expires_in_seconds=expires_in_seconds,gateway_config=gateway_config,+ run_id=run_id,)if token is None:raise GatewayNotConfiguredError("Scout trials require a private AI gateway credential")diff --git a/products/tasks/backend/facade/tasks.py b/products/tasks/backend/facade/tasks.pyindex f04e68060243..82781c3ffbad 100644--- a/products/tasks/backend/facade/tasks.py+++ b/products/tasks/backend/facade/tasks.py@@ -10,10 +10,12 @@ from products.tasks.backend.tasks.sandbox_images import (refresh_dev_stack_image_task,refresh_stale_sandbox_custom_images_task,)+from products.tasks.backend.tasks.tasks import settle_overdue_sandbox_compute_task__all__ = ["bake_dev_stack_image_task","refresh_dev_stack_image_task","refresh_stale_sandbox_custom_images_task",+ "settle_overdue_sandbox_compute_task","sweep_inactive_tasks_task",]diff --git a/products/tasks/backend/logic/services/compute_quota.py b/products/tasks/backend/logic/services/compute_quota.pyindex 17a141bab15b..c3554992954a 100644--- a/products/tasks/backend/logic/services/compute_quota.py+++ b/products/tasks/backend/logic/services/compute_quota.py@@ -1,4 +1,5 @@import logging+from typing import TYPE_CHECKINGfrom django.conf import settings@@ -8,6 +9,9 @@ from products.tasks.backend.facade.contracts import ComputeQuotaDenialReasonfrom products.tasks.backend.metrics import observe_compute_quota_checkfrom products.tasks.backend.models import Task, TaskClientProvenance+if TYPE_CHECKING:+ from products.tasks.backend.models import SandboxSession+COMPUTE_QUOTA_DENIAL_CODE = ComputeQuotaDenialReason.COMPUTE_QUOTA_EXHAUSTED.valueORGANIZATION_DEACTIVATED_DENIAL_CODE = ComputeQuotaDenialReason.ORGANIZATION_DEACTIVATED.value@@ -35,6 +39,14 @@ def is_billable_compute(*, origin_product: str | None, client_provenance: str |return origin_product in (Task.OriginProduct.USER_CREATED, Task.OriginProduct.SPACE_SETUP)+def is_billable_sandbox_session(session: "SandboxSession") -> bool:+ return (+ session.user_attributed_at is not None+ and session.origin_product == Task.OriginProduct.USER_CREATED+ and is_billable_compute(origin_product=session.origin_product, client_provenance=session.client_provenance)+ )++def get_compute_quota_denial_reason(task: Task) -> ComputeQuotaDenialReason | None:if organization_deactivated(task.team_id):observe_compute_quota_check("checked_blocked")diff --git a/products/tasks/backend/logic/services/compute_settlement.py b/products/tasks/backend/logic/services/compute_settlement.pynew file mode 100644index 000000000000..c34b16bdc0a7--- /dev/null+++ b/products/tasks/backend/logic/services/compute_settlement.py@@ -0,0 +1,566 @@+import time+from collections import Counter+from datetime import UTC, datetime, timedelta+from decimal import ROUND_HALF_EVEN, Decimal, InvalidOperation+from enum import StrEnum+from typing import Any+from uuid import UUID++from django.conf import settings+from django.db.models import Count, F, QuerySet, Sum+from django.utils import timezone++import httpx+import structlog++from posthog.dataclasses import frozen+from posthog.llm.gateway_internal_client import (+ AIGatewayNotConfigured,+ ReportedSpendResponse,+ get_usage_report,+ report_spend,+)+from posthog.models.project_secret_api_key import find_project_secret_api_key++from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_session+from products.tasks.backend.logic.services.run_actor import get_actor_distinct_id, get_task_run_credential_user+from products.tasks.backend.logic.services.sandbox_pricing import (+ COMPUTE_RATE_CARDS,+ SandboxComputeCost,+ calculate_sandbox_compute_cost,+)+from products.tasks.backend.logic.services.sandbox_usage import (+ close_sandbox_session,+ get_billable_sandbox_compute_usage_by_team,+)+from products.tasks.backend.metrics import (+ ComputeSettlementOutcome,+ observe_compute_settlement,+ set_hogland_sandbox_sessions_open_over_day,+)+from products.tasks.backend.models import SandboxSession, TaskRun++logger = structlog.get_logger(__name__)++MAX_SETTLE_ATTEMPTS = 20+ATTEMPTS_EXHAUSTED_ERROR = "attempts_exhausted"+ENDED_GRACE = timedelta(minutes=10)+TTL_GRACE = timedelta(hours=1)+HOGLAND_OPEN_ALERT_AGE = timedelta(days=1)+DETAIL_MAX_BYTES = 2048+_USD_QUANTUM = Decimal("0.000001")+_RETRYABLE_STATUSES = frozenset({408, 429})+_DEPLOYMENT_STATUSES = frozenset({401, 403, 404})+_ERROR_MAX_LENGTH = 200+++class SettlementOutcome(StrEnum):+ DISABLED = "disabled"+ SKIPPED = "skipped"+ SETTLED = "settled"+ DUPLICATE = "duplicate"+ SETTLED_LOCALLY = "settled_locally"+ CONFLICT = "conflict"+ RETRY = "retry"+ ATTEMPTS_EXHAUSTED = "attempts_exhausted"+ CONTRACT_ERROR = "contract_error"+ CONTRACT_ERROR_REPEAT = "contract_error_repeat"+++_CONTRACT_ERROR_OUTCOMES = frozenset({SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.CONTRACT_ERROR_REPEAT})+_SETTLED_OUTCOMES = frozenset({SettlementOutcome.SETTLED, SettlementOutcome.DUPLICATE})+ParkReason = SandboxSession.ComputeSettleParkReason+++class ComputeSettlementDeploymentError(Exception):+ pass+++def _unsettled(queryset: QuerySet[SandboxSession]) -> QuerySet[SandboxSession]:+ return queryset.filter(compute_settled_at__isnull=True, compute_settle_conflict_at__isnull=True)+++def _session_rows(session: SandboxSession) -> QuerySet[SandboxSession]:+ return SandboxSession.objects.for_team(session.team_id).filter(id=session.id)+++def _iso(value: datetime) -> str:+ return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")+++def _fixed6(value: Decimal) -> str:+ return str(value.quantize(_USD_QUANTUM, rounding=ROUND_HALF_EVEN))+++def _window_end(session: SandboxSession) -> datetime:+ assert session.ended_at is not None+ return min(session.ended_at, session.ttl_expires_at)+++def _resource(session: SandboxSession) -> str:+ if session.sandbox_backend == "hogland":+ return "hogland"+ return "sandbox-vm" if session.vm_runtime else "modal"+++def session_compute_cost(session: SandboxSession) -> SandboxComputeCost | None:+ if not COMPUTE_RATE_CARDS or session.user_attributed_at is None or session.ended_at is None:+ return None+ start = COMPUTE_RATE_CARDS[0].effective_at+ end = max(_window_end(session), start + timedelta(seconds=1))+ return calculate_sandbox_compute_cost(+ session, start, end, calculated_at=session.ended_at, rate_cards=COMPUTE_RATE_CARDS+ )+++def _ai_product(run: TaskRun) -> str:+ from products.tasks.backend.temporal.process_task.ai_gateway_token import ( # noqa: PLC0415 - the temporal package imports this module via cleanup_sandbox+ resolve_sandbox_ai_product,+ )++ state = run.state or {}+ profile = run.task.stored_profile+ return resolve_sandbox_ai_product(+ run.task.origin_product,+ state.get("ai_stage"),+ internal=run.task.internal,+ billing_product=state.get("billing_product"),+ profile_billing_product=profile.billing_product if profile else None,+ )+++def build_settlement_body(session: SandboxSession, run: TaskRun, cost: SandboxComputeCost) -> dict[str, Any]:+ assert session.user_attributed_at is not None+ window_end = _window_end(session)+ detail: dict[str, Any] = {+ "rate_cards": [+ {+ "version": item.rate_card.version,+ "seconds": _fixed6(item.billable_seconds),+ "usd": _fixed6(item.total_cost_usd),+ }+ for item in cost.line_items+ ],+ "resource": _resource(session),+ "cpu_cores": session.cpu_cores,+ "memory_gb": session.memory_gb,+ "attributed_seconds": round(max((window_end - session.user_attributed_at).total_seconds(), 0.0), 3),+ "window_start": _iso(session.user_attributed_at),+ "window_end": _iso(window_end),+ "ended_reason": session.ended_reason or "",+ "sandbox_id": session.sandbox_id,+ }+ body: dict[str, Any] = {+ "reference": f"compute:{session.id}",+ "amount_usd": _fixed6(cost.total_cost_usd),+ "charge_kind": "compute",+ "product": _ai_product(run),+ }+ actor = get_task_run_credential_user(run.task, run.state)+ if actor is not None:+ body["user"] = get_actor_distinct_id(actor)+ body.update(+ {+ "obo": str(session.team_id),+ "run": str(session.task_run_id),+ "billable": is_billable_sandbox_session(session),+ "detail": detail,+ "reason": "sandbox compute",+ }+ )+ return body+++def wallet_team_id() -> int:+ mint_key = settings.SANDBOX_AI_GATEWAY_MINT_KEY+ record = find_project_secret_api_key(mint_key) if mint_key else None+ if record is None:+ raise ComputeSettlementDeploymentError("SANDBOX_AI_GATEWAY_MINT_KEY does not resolve to a wallet team")+ return record.team_id+++_METRIC_LABELS: dict[SettlementOutcome, ComputeSettlementOutcome] = {+ SettlementOutcome.SKIPPED: "skipped",+ SettlementOutcome.SETTLED: "settled",+ SettlementOutcome.DUPLICATE: "duplicate",+ SettlementOutcome.SETTLED_LOCALLY: "settled_locally",+ SettlementOutcome.CONFLICT: "conflict",+ SettlementOutcome.RETRY: "retry",+ SettlementOutcome.ATTEMPTS_EXHAUSTED: "attempts_exhausted",+ SettlementOutcome.CONTRACT_ERROR: "contract_error",+ SettlementOutcome.CONTRACT_ERROR_REPEAT: "contract_error",+}+++def _record(outcome: SettlementOutcome) -> SettlementOutcome:+ label = _METRIC_LABELS.get(outcome)+ if label is not None:+ observe_compute_settlement(label)+ return outcome+++def _settle_locally(session: SandboxSession) -> SettlementOutcome:+ _unsettled(_session_rows(session)).update(+ compute_settled_at=timezone.now(), compute_settled_usd=Decimal(0), compute_settle_error=""+ )+ return _record(SettlementOutcome.SETTLED_LOCALLY)+++def _error_text(response: ReportedSpendResponse) -> str:+ body = response.body or {}+ message = body.get("message")+ code = body.get("code")+ if isinstance(message, str) and message:+ detail = f"{code}: {message}" if isinstance(code, str) and code else message+ else:+ detail = response.text+ return f"HTTP {response.status_code}: {detail}"[:_ERROR_MAX_LENGTH]+++def _mark_conflict(session: SandboxSession, error: str, reason: ParkReason) -> None:+ _unsettled(_session_rows(session)).update(+ compute_settle_conflict_at=timezone.now(), compute_settle_error=error, compute_settle_park_reason=reason+ )+++def _mark_malformed(session: SandboxSession) -> SettlementOutcome:+ rows = _unsettled(_session_rows(session))+ rows.update(compute_settle_attempts=F("compute_settle_attempts") + 1)+ attempts = rows.values_list("compute_settle_attempts", flat=True).first()+ if attempts is not None and attempts >= MAX_SETTLE_ATTEMPTS:+ _mark_conflict(session, ATTEMPTS_EXHAUSTED_ERROR, ParkReason.ATTEMPTS_EXHAUSTED)+ logger.error("compute_settlement.attempts_exhausted", session_id=str(session.id), attempts=attempts)+ return _record(SettlementOutcome.ATTEMPTS_EXHAUSTED)+ return _record(SettlementOutcome.RETRY)+++def _mark_contract_error(session: SandboxSession, response: ReportedSpendResponse) -> SettlementOutcome:+ error = _error_text(response)+ rows = _unsettled(_session_rows(session))+ stored = rows.values_list("compute_settle_error", flat=True).first()+ rows.update(compute_settle_error=error)+ logger.error("compute_settlement.contract_error", session_id=str(session.id), error=error)+ if stored == error:+ return _record(SettlementOutcome.CONTRACT_ERROR_REPEAT)+ return _record(SettlementOutcome.CONTRACT_ERROR)+++def _mark_settled(session: SandboxSession, response: ReportedSpendResponse) -> SettlementOutcome:+ body = response.body or {}+ entry_id = body.get("entry_id")+ try:+ amount = Decimal(str(body.get("amount_usd")))+ except InvalidOperation:+ amount = None+ if not isinstance(entry_id, str) or not entry_id or amount is None or not amount.is_finite():+ logger.warning("compute_settlement.malformed_response", session_id=str(session.id))+ return _mark_malformed(session)+ _unsettled(_session_rows(session)).update(+ compute_settled_at=timezone.now(),+ compute_settled_usd=amount,+ compute_ledger_entry_id=entry_id[:64],+ compute_settle_error="",+ )+ return _record(SettlementOutcome.DUPLICATE if body.get("duplicate") is True else SettlementOutcome.SETTLED)+++def settle_sandbox_session(session: SandboxSession) -> SettlementOutcome:+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:+ return SettlementOutcome.DISABLED+ if (+ session.ended_at is None+ or session.compute_settled_at is not None+ or session.compute_settle_conflict_at is not None+ ):+ return _record(SettlementOutcome.SKIPPED)+ if session.user_attributed_at is None:+ return _settle_locally(session)+ cost = session_compute_cost(session)+ if cost is None or _fixed6(cost.total_cost_usd) == _fixed6(Decimal(0)):+ return _settle_locally(session)++ body = session.compute_settle_body+ rows = _unsettled(_session_rows(session))+ if body:+ rows.update(compute_settle_attempted_at=timezone.now())+ else:+ run = TaskRun.objects.select_related("task", "task__created_by").get(+ id=session.task_run_id, team_id=session.team_id+ )+ built = build_settlement_body(session, run, cost)+ rows.update(compute_settle_body=built, compute_settle_attempted_at=timezone.now())+ body = _session_rows(session).values_list("compute_settle_body", flat=True).first() or built+ try:+ response = report_spend(wallet_team_id(), body)+ except AIGatewayNotConfigured as error:+ observe_compute_settlement("deployment_error")+ logger.exception("compute_settlement.not_configured", session_id=str(session.id))+ raise ComputeSettlementDeploymentError(str(error)) from error+ except httpx.HTTPError:+ logger.warning("compute_settlement.transport_error", session_id=str(session.id), exc_info=True)+ return _record(SettlementOutcome.RETRY)++ status = response.status_code+ if status == 200:+ return _mark_settled(session, response)+ if status in _DEPLOYMENT_STATUSES:+ observe_compute_settlement("deployment_error")+ logger.error("compute_settlement.deployment_error", session_id=str(session.id), status=status)+ raise ComputeSettlementDeploymentError(f"gateway refused the compute debit with HTTP {status}")+ if status >= 500 or status in _RETRYABLE_STATUSES:+ logger.warning("compute_settlement.retryable_error", session_id=str(session.id), status=status)+ return _record(SettlementOutcome.RETRY)+ if status == 400:+ return _mark_contract_error(session, response)+ if status == 409:+ logger.error("compute_settlement.conflict", session_id=str(session.id), status=status)+ _mark_conflict(session, _error_text(response), ParkReason.HTTP_409)+ else:+ logger.error("compute_settlement.unexpected_status", session_id=str(session.id), status=status)+ _mark_conflict(session, _error_text(response), ParkReason.UNEXPECTED_STATUS)+ return _record(SettlementOutcome.CONFLICT)+++def settle_task_run_compute(task_run_id: UUID | str, team_id: int) -> list[SettlementOutcome]:+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:+ return []+ run_uuid = task_run_id if isinstance(task_run_id, UUID) else UUID(task_run_id)+ sessions = _unsettled(SandboxSession.objects.for_team(team_id).filter(task_run_id=run_uuid, ended_at__isnull=False))+ return [settle_sandbox_session(session) for session in sessions]+++def _overdue_ended(now: datetime, batch_size: int, processed: set[UUID]) -> list[SandboxSession]:+ return list(+ _unsettled(SandboxSession.objects.unscoped())+ .filter(user_attributed_at__isnull=False, ended_at__lt=now - ENDED_GRACE)+ .exclude(id__in=processed)+ .order_by(F("compute_settle_attempted_at").asc(nulls_first=True), "ended_at")[:batch_size]+ )+++def _overdue_open_modal(now: datetime, batch_size: int, processed: set[UUID]) -> list[SandboxSession]:+ return list(+ _unsettled(SandboxSession.objects.unscoped())+ .filter(user_attributed_at__isnull=False, ended_at__isnull=True, ttl_expires_at__lt=now - TTL_GRACE)+ .exclude(sandbox_backend="hogland")+ .exclude(id__in=processed)+ .order_by("ttl_expires_at")[:batch_size]+ )+++def _refresh_hogland_open_gauge(now: datetime) -> None:+ count = (+ SandboxSession.objects.unscoped()+ .filter(sandbox_backend="hogland", ended_at__isnull=True, created_at__lt=now - HOGLAND_OPEN_ALERT_AGE)+ .count()+ )+ set_hogland_sandbox_sessions_open_over_day(count)+++def _park_repeated_contract_errors(+ results: list[tuple[SandboxSession, SettlementOutcome]],+) -> list[SettlementOutcome]:+ contract_ids = [session.id for session, outcome in results if outcome in _CONTRACT_ERROR_OUTCOMES]+ if not contract_ids:+ return [outcome for _, outcome in results]+ errors: dict[UUID, str] = dict(+ SandboxSession.objects.unscoped().filter(id__in=contract_ids).values_list("id", "compute_settle_error")+ )+ if not any(outcome in _SETTLED_OUTCOMES for _, outcome in results):+ shared = Counter(errors[session_id] for session_id in contract_ids if errors.get(session_id)).most_common(1)+ if shared and shared[0][1] >= 2:+ error, count = shared[0]+ observe_compute_settlement("deployment_error")+ raise ComputeSettlementDeploymentError(+ f"gateway rejected {count} compute debits with the same error: {error}"+ )+ final: list[SettlementOutcome] = []+ for session, outcome in results:+ if outcome == SettlementOutcome.CONTRACT_ERROR_REPEAT:+ _mark_conflict(session, errors.get(session.id, ""), ParkReason.HTTP_400)+ outcome = _record(SettlementOutcome.CONFLICT)+ final.append(outcome)+ return final+++def settle_overdue_sandbox_compute(+ now: datetime, batch_size: int = 50, time_budget: timedelta = timedelta(minutes=15)+) -> list[SettlementOutcome]:+ _refresh_hogland_open_gauge(now)+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:+ return []+ deadline = time.monotonic() + time_budget.total_seconds()+ results: list[tuple[SandboxSession, SettlementOutcome]] = []+ processed: set[UUID] = set()+ try:+ while True:+ batch = _overdue_ended(now, batch_size, processed)+ for session in batch:+ processed.add(session.id)+ results.append((session, _settle_safely(session)))+ if len(batch) < batch_size or time.monotonic() >= deadline:+ break+ while True:+ batch = _overdue_open_modal(now, batch_size, processed)+ for session in batch:+ processed.add(session.id)+ close_sandbox_session(session.sandbox_id, reason=SandboxSession.EndedReason.TTL_SWEEP)+ closed = SandboxSession.objects.unscoped().filter(id=session.id).first()+ if closed is not None and closed.ended_at is not None:+ results.append((closed, _settle_safely(closed)))+ if len(batch) < batch_size or time.monotonic() >= deadline:+ break+ return _park_repeated_contract_errors(results)+ except ComputeSettlementDeploymentError as error:+ logger.exception("compute_settlement.sweep_stopped", error=str(error))+ return [outcome for _, outcome in results]+++def _settle_safely(session: SandboxSession) -> SettlementOutcome:+ try:+ return settle_sandbox_session(session)+ except ComputeSettlementDeploymentError:+ raise+ except Exception:+ logger.exception("compute_settlement.settle_failed", session_id=str(session.id))+ return SettlementOutcome.SKIPPED+++def unpark_sandbox_compute(reason: str, since: datetime, until: datetime) -> int:+ reset: dict[str, Any] = {+ "compute_settle_conflict_at": None,+ "compute_settle_park_reason": "",+ "compute_settle_error": "",+ "compute_settle_attempts": 0,+ "compute_settle_attempted_at": None,+ }+ if reason == ParkReason.HTTP_400:+ reset["compute_settle_body"] = {}+ return (+ SandboxSession.objects.unscoped()+ .filter(+ compute_settled_at__isnull=True,+ compute_settle_conflict_at__gte=since,+ compute_settle_conflict_at__lte=until,+ compute_settle_park_reason=reason,+ )+ .update(**reset)+ )+++@frozen+class TeamComputeDelta:+ team_id: str+ local_usd: Decimal+ gateway_usd: Decimal+ local_rows: int++ @property+ def delta_usd(self) -> Decimal:+ return self.local_usd - self.gateway_usd+++@frozen+class ComputeReconciliation:+ writer_parity: tuple[TeamComputeDelta, ...]+ pricing_parity: tuple[TeamComputeDelta, ...]+ billing_parity: tuple[TeamComputeDelta, ...]+++def _report_column(rows: list[dict[str, Any]], column: str) -> dict[str, Decimal]:+ totals: dict[str, Decimal] = {}+ for row in rows:+ keys = row.get("keys")+ obo = str((keys if isinstance(keys, dict) else row).get("obo") or "")+ try:+ value = Decimal(str(row.get(column) or "0"))+ except InvalidOperation:+ value = Decimal(0)+ totals[obo] = totals.get(obo, Decimal(0)) + value+ return totals+++def _deltas(+ local: dict[str, Decimal], counts: dict[str, int], gateway: dict[str, Decimal]+) -> tuple[TeamComputeDelta, ...]:+ return tuple(+ TeamComputeDelta(+ team_id=team_id,+ local_usd=local.get(team_id, Decimal(0)),+ gateway_usd=gateway.get(team_id, Decimal(0)),+ local_rows=counts.get(team_id, 0),+ )+ for team_id in sorted(set(local) | set(gateway))+ )+++def reconcile_compute_settlement(since: datetime, until: datetime, wallet_team: int) -> ComputeReconciliation:+ rows = get_usage_report(+ {+ "team_id": str(wallet_team),+ "group_by": "obo,charge_kind",+ "charge_kind": "compute",+ "from": since.isoformat(),+ "to": until.isoformat(),+ }+ )+ gateway_compute = _report_column(rows, "compute_usd")++ settled_usd: dict[str, Decimal] = {}+ settled_rows: dict[str, int] = {}+ for row in (+ SandboxSession.objects.unscoped()+ .filter(compute_settled_at__gte=since, compute_settled_at__lt=until)+ .exclude(compute_ledger_entry_id="")+ .order_by()+ .values("team_id")+ .annotate(total=Sum("compute_settled_usd"), rows=Count("id"))+ ):+ key = str(row["team_id"])+ settled_usd[key] = row["total"] or Decimal(0)+ settled_rows[key] = row["rows"]++ priced_usd: dict[str, Decimal] = {}+ priced_rows: dict[str, int] = {}+ for session in (+ SandboxSession.objects.unscoped()+ .filter(user_attributed_at__isnull=False, ended_at__gte=since, ended_at__lt=until)+ .iterator()+ ):+ cost = session_compute_cost(session)+ key = str(session.team_id)+ priced_usd[key] = priced_usd.get(key, Decimal(0)) + (cost.total_cost_usd if cost else Decimal(0))+ priced_rows[key] = priced_rows.get(key, 0) + 1++ billing = get_billable_sandbox_compute_usage_by_team(since, until)+ billed_usd = {str(team_id): Decimal(credits) / 100 for team_id, credits in billing.credits}++ prorated_usd: dict[str, Decimal] = {}+ prorated_rows: dict[str, int] = {}+ calculated_at = timezone.now()+ for session in (+ SandboxSession.objects.unscoped()+ .filter(compute_settled_usd__isnull=False, user_attributed_at__lt=until, ended_at__gt=since)+ .exclude(compute_ledger_entry_id="")+ .iterator()+ ):+ if not is_billable_sandbox_session(session):+ continue+ lifetime = session_compute_cost(session)+ if lifetime is None or lifetime.total_cost_usd == 0:+ continue+ clipped = calculate_sandbox_compute_cost(+ session, since, until, calculated_at=calculated_at, rate_cards=COMPUTE_RATE_CARDS+ ).total_cost_usd+ assert session.compute_settled_usd is not None+ key = str(session.team_id)+ prorated_usd[key] = (+ prorated_usd.get(key, Decimal(0)) + session.compute_settled_usd * clipped / lifetime.total_cost_usd+ )+ prorated_rows[key] = prorated_rows.get(key, 0) + 1++ return ComputeReconciliation(+ writer_parity=_deltas(settled_usd, settled_rows, gateway_compute),+ pricing_parity=_deltas(priced_usd, priced_rows, gateway_compute),+ billing_parity=_deltas(billed_usd, prorated_rows, prorated_usd),+ )diff --git a/products/tasks/backend/logic/services/gateway_usage.py b/products/tasks/backend/logic/services/gateway_usage.pyindex 9e8bf9c46b61..24b825eaa932 100644--- a/products/tasks/backend/logic/services/gateway_usage.py+++ b/products/tasks/backend/logic/services/gateway_usage.py@@ -3,7 +3,7 @@ import timeimport asynciofrom collections.abc import Iterable, Iteratorfrom contextlib import contextmanager-from datetime import timedelta+from datetime import datetime, timedeltafrom decimal import ROUND_HALF_EVEN, Decimalfrom functools import lru_cachefrom typing import TYPE_CHECKING, Any@@ -280,19 +280,19 @@ def _compute_cost_source(run: TaskRun, *, sessions: list[SandboxSession] | Nonereturn Nonenow = timezone.now()try:- return sum(- (- calculate_sandbox_compute_cost(- session, COMPUTE_RATE_CARDS[0].effective_at, now, calculated_at=now, rate_cards=COMPUTE_RATE_CARDS- ).total_cost_usd- for session in sessions- ),- Decimal(0),- )+ return sum((_session_compute_cost(session, now) for session in sessions), Decimal(0))except ValueError:return None+def _session_compute_cost(session: SandboxSession, now: datetime) -> Decimal:+ if session.compute_settled_at is not None and session.compute_settled_usd is not None:+ return session.compute_settled_usd+ return calculate_sandbox_compute_cost(+ session, COMPUTE_RATE_CARDS[0].effective_at, now, calculated_at=now, rate_cards=COMPUTE_RATE_CARDS+ ).total_cost_usd++def _cents(value: Decimal) -> int:return int((value * 100).to_integral_value(rounding=ROUND_HALF_EVEN))diff --git a/products/tasks/backend/logic/services/sandbox_usage.py b/products/tasks/backend/logic/services/sandbox_usage.pyindex 1a8f263caf64..29bba05a9b2f 100644--- a/products/tasks/backend/logic/services/sandbox_usage.py+++ b/products/tasks/backend/logic/services/sandbox_usage.py@@ -27,7 +27,7 @@ import structlogfrom posthog.dataclasses import frozen-from products.tasks.backend.logic.services.compute_quota import is_billable_compute+from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_sessionfrom products.tasks.backend.logic.services.sandbox import SandboxBase, SandboxConfig, get_sandbox_class_for_sandbox_idfrom products.tasks.backend.logic.services.sandbox_pricing import (COMPUTE_RATE_CARDS,@@ -325,7 +325,7 @@ def get_billable_sandbox_compute_usage_by_team(usage: dict[int, list[Decimal]] = {}calculated_at = timezone.now()for session in sessions.iterator():- if not is_billable_compute(origin_product=session.origin_product, client_provenance=session.client_provenance):+ if not is_billable_sandbox_session(session):continuecost = calculate_sandbox_compute_cost(session, begin, end, calculated_at=calculated_at, rate_cards=cards)totals = usage.setdefault(session.team_id, [Decimal(0) for _ in range(3)])diff --git a/products/tasks/backend/logic/services/tests/test_compute_settlement.py b/products/tasks/backend/logic/services/tests/test_compute_settlement.pynew file mode 100644index 000000000000..905ab3d5665f--- /dev/null+++ b/products/tasks/backend/logic/services/tests/test_compute_settlement.py@@ -0,0 +1,454 @@+import json+from datetime import timedelta+from decimal import Decimal+from typing import Any++import pytest+from posthog.test.base import BaseTest+from unittest.mock import MagicMock, patch++from django.test import override_settings+from django.utils import timezone++import httpx+from parameterized import parameterized++from posthog.models.project_secret_api_key import ProjectSecretAPIKey+from posthog.models.utils import hash_key_value++from products.tasks.backend.logic.services.compute_settlement import (+ ComputeSettlementDeploymentError,+ SettlementOutcome,+ TeamComputeDelta,+ reconcile_compute_settlement,+ settle_overdue_sandbox_compute,+ settle_sandbox_session,+ settle_task_run_compute,+ unpark_sandbox_compute,+)+from products.tasks.backend.management.commands.reconcile_sandbox_compute_settlement import CENT_ROUNDING_TOLERANCE_USD+from products.tasks.backend.models import SandboxSession, Task, TaskClientProvenance, TaskRun++MINT_KEY = "phs_" + "c" * 35+GATEWAY_URL = "http://gateway.internal"+HTTPX_POST = "posthog.llm.gateway_internal_client.httpx.post"+++def _response(status: int, body: dict[str, Any] | None = None) -> httpx.Response:+ return httpx.Response(status, json=body or {}, request=httpx.Request("POST", GATEWAY_URL))+++def _ok(*, amount: str = "1.500000", entry_id: str = "entry-1", duplicate: bool = False) -> httpx.Response:+ return _response(200, {"team_id": 1, "entry_id": entry_id, "amount_usd": amount, "duplicate": duplicate})+++@override_settings(+ GATEWAY_COMPUTE_DEBIT_ENABLED=True,+ SANDBOX_AI_GATEWAY_MINT_KEY=MINT_KEY,+ AI_GATEWAY_INTERNAL_URL=GATEWAY_URL,+ AI_GATEWAY_INTERNAL_TOKEN="operator-token",+)+class TestComputeSettlement(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.now = timezone.now()+ ProjectSecretAPIKey.objects.create(+ team=self.team, label="mint", secure_value=hash_key_value(MINT_KEY), scopes=["llm_gateway:read"]+ )++ def _session(self, *, with_actor: bool = True, **overrides: Any) -> SandboxSession:+ task = Task.objects.create(+ team=self.team,+ title="t",+ description="",+ origin_product=Task.OriginProduct.USER_CREATED,+ client_provenance=TaskClientProvenance.POSTHOG_DESKTOP,+ created_by=self.user if with_actor else None,+ )+ run = TaskRun.objects.create(task=task, team=self.team, environment=TaskRun.Environment.CLOUD)+ created_at = overrides.pop("created_at", self.now - timedelta(hours=2))+ defaults: dict[str, Any] = {+ "team": self.team,+ "task_run": run,+ "sandbox_id": f"sb-{SandboxSession.objects.unscoped().count()}",+ "origin_product": Task.OriginProduct.USER_CREATED,+ "client_provenance": TaskClientProvenance.POSTHOG_DESKTOP,+ "cpu_cores": 4.0,+ "memory_gb": 16.0,+ "ttl_seconds": 6 * 60 * 60,+ "created_at": created_at,+ "ttl_expires_at": created_at + timedelta(hours=6),+ "user_attributed_at": self.now - timedelta(hours=2),+ "ended_at": self.now - timedelta(hours=1),+ "ended_reason": SandboxSession.EndedReason.CLEANUP,+ }+ defaults.update(overrides)+ return SandboxSession.objects.unscoped().create(**defaults)++ def _reload(self, session: SandboxSession) -> SandboxSession:+ return SandboxSession.objects.unscoped().get(id=session.id)++ @parameterized.expand([(False, SettlementOutcome.SETTLED), (True, SettlementOutcome.DUPLICATE)])+ def test_success_stores_the_gateway_amount_and_entry(self, duplicate: bool, expected: SettlementOutcome) -> None:+ session = self._session()+ with patch(HTTPX_POST, return_value=_ok(duplicate=duplicate)) as post:+ assert settle_sandbox_session(session) == expected++ assert post.call_args.args[0] == f"{GATEWAY_URL}/internal/teams/{self.team.id}/reported-spend"+ assert post.call_args.kwargs["headers"]["X-Internal-Actor"] == "tasks-compute"+ settled = self._reload(session)+ assert settled.compute_settled_at is not None+ assert settled.compute_settled_usd == Decimal("1.500000")+ assert settled.compute_ledger_entry_id == "entry-1"++ def test_body_shape_is_clamped_and_byte_identical_across_retries(self) -> None:+ session = self._session(+ with_actor=False,+ created_at=self.now - timedelta(hours=7),+ user_attributed_at=self.now - timedelta(hours=6, minutes=30),+ ttl_expires_at=self.now - timedelta(hours=6),+ ended_at=self.now - timedelta(hours=1),+ )+ with patch(HTTPX_POST, side_effect=[httpx.ConnectError("refused"), _response(503)]) as post:+ settle_sandbox_session(session)+ Task.objects.filter(runs__id=session.task_run_id).update(created_by=self.user)+ settle_sandbox_session(self._reload(session))++ first, second = (json.dumps(call.kwargs["json"]) for call in post.call_args_list)+ assert first == second+ body = post.call_args_list[0].kwargs["json"]+ assert "user" not in body+ assert body["reference"] == f"compute:{session.id}"+ assert body["obo"] == str(self.team.id)+ assert body["run"] == str(session.task_run_id)+ assert body["billable"] is True+ assert body["charge_kind"] == "compute"+ assert body["product"] == "posthog_code"+ assert body["amount_usd"] == "0.770400"+ detail = body["detail"]+ assert detail["window_end"] == session.ttl_expires_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")+ assert detail["attributed_seconds"] == 1800.0+ assert detail["resource"] == "modal"+ assert [card["version"] for card in detail["rate_cards"]] == ["v1"]+ assert detail["rate_cards"][0]["usd"] == "0.770400"+ assert len(json.dumps(detail).encode()) <= 2048++ def test_body_pins_the_acting_user(self) -> None:+ session = self._session()+ with patch(HTTPX_POST, return_value=_ok()) as post:+ settle_sandbox_session(session)++ assert post.call_args.kwargs["json"]["user"] == self.user.distinct_id++ @parameterized.expand(+ [+ ("conflict", _response(409, {"error": "reference reused"}), SettlementOutcome.CONFLICT, [], "http_409"),+ ("unexpected", _response(418), SettlementOutcome.CONFLICT, [], "unexpected_status"),+ (+ "validation",+ _response(400, {"error": "bad"}),+ SettlementOutcome.CONTRACT_ERROR,+ [SettlementOutcome.CONTRACT_ERROR_REPEAT],+ "",+ ),+ (+ "validation_request_id",+ [+ _response(400, {"code": "bad_request", "message": "pool_type must be prepaid", "request_id": "a1"}),+ _response(400, {"code": "bad_request", "message": "pool_type must be prepaid", "request_id": "b2"}),+ ],+ SettlementOutcome.CONTRACT_ERROR,+ [SettlementOutcome.CONTRACT_ERROR_REPEAT],+ "",+ ),+ ("unavailable", _response(503), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),+ ("timeout", _response(408), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),+ ("rate_limited", _response(429), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),+ ("transport", httpx.ConnectError("refused"), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),+ ]+ )+ def test_failure_statuses(+ self,+ _name: str,+ result: Any,+ expected: SettlementOutcome,+ again: list[SettlementOutcome],+ park_reason: str,+ ) -> None:+ session = self._session()+ mock = MagicMock(side_effect=result) if isinstance(result, Exception | list) else MagicMock(return_value=result)+ with patch(HTTPX_POST, mock):+ assert settle_sandbox_session(session) == expected+ assert settle_task_run_compute(session.task_run_id, self.team.id) == again++ row = self._reload(session)+ assert row.compute_settled_at is None+ assert (row.compute_settle_conflict_at is not None) is bool(park_reason)+ assert row.compute_settle_park_reason == park_reason+ assert row.compute_settle_attempts == 0+ assert row.compute_settle_attempted_at is not None+ assert mock.call_count == 1 + len(again)+ transient = expected == SettlementOutcome.RETRY+ first = result[0] if isinstance(result, list) else result+ assert row.compute_settle_error.startswith("" if transient else f"HTTP {first.status_code}: ")+ assert (row.compute_settle_error == "") is transient+ if isinstance(result, list):+ assert row.compute_settle_error == "HTTP 400: bad_request: pool_type must be prepaid"++ def test_retry_cap_excludes_the_row(self) -> None:+ session = self._session()+ with patch(HTTPX_POST, return_value=_response(200, {"duplicate": False})):+ outcomes = [settle_sandbox_session(self._reload(session)) for _ in range(20)]++ assert outcomes == [SettlementOutcome.RETRY] * 19 + [SettlementOutcome.ATTEMPTS_EXHAUSTED]+ row = self._reload(session)+ assert row.compute_settle_error == "attempts_exhausted"+ assert row.compute_settle_park_reason == "attempts_exhausted"+ assert row.compute_settle_conflict_at is not None+ assert row.compute_settled_at is None++ def test_unpark_clears_only_the_requested_reason_in_the_window(self) -> None:+ parked_at = self.now - timedelta(hours=1)+ frozen_body = {"reference": "compute:frozen", "amount_usd": "1.000000"}+ rejected = self._session(+ compute_settle_conflict_at=parked_at,+ compute_settle_park_reason="http_400",+ compute_settle_error="HTTP 400: bad",+ compute_settle_attempts=3,+ compute_settle_body=frozen_body,+ compute_settle_attempted_at=parked_at,+ )+ exhausted = self._session(+ compute_settle_conflict_at=parked_at,+ compute_settle_park_reason="attempts_exhausted",+ compute_settle_error="attempts_exhausted",+ compute_settle_attempts=20,+ compute_settle_body=frozen_body,+ compute_settle_attempted_at=parked_at,+ )+ reused = self._session(+ compute_settle_conflict_at=parked_at,+ compute_settle_park_reason="http_409",+ compute_settle_error="HTTP 409: reused",+ )+ older = self._session(+ compute_settle_conflict_at=self.now - timedelta(days=3),+ compute_settle_park_reason="http_400",+ compute_settle_error="HTTP 400: bad",+ )++ assert unpark_sandbox_compute("http_400", self.now - timedelta(days=1), self.now) == 1++ row = self._reload(rejected)+ assert row.compute_settle_conflict_at is None+ assert row.compute_settle_park_reason == ""+ assert row.compute_settle_error == ""+ assert row.compute_settle_attempts == 0+ assert row.compute_settle_body == {}+ assert row.compute_settle_attempted_at is None+ assert self._reload(reused).compute_settle_park_reason == "http_409"+ assert self._reload(older).compute_settle_conflict_at is not None+ assert self._reload(exhausted).compute_settle_park_reason == "attempts_exhausted"++ assert unpark_sandbox_compute("attempts_exhausted", self.now - timedelta(days=1), self.now) == 1++ row = self._reload(exhausted)+ assert row.compute_settle_conflict_at is None+ assert row.compute_settle_attempts == 0+ assert row.compute_settle_body == frozen_body+ assert row.compute_settle_attempted_at is None++ def test_deployment_error_raises_and_leaves_the_row_untouched(self) -> None:+ session = self._session()+ with patch(HTTPX_POST, return_value=_response(403)), pytest.raises(ComputeSettlementDeploymentError):+ settle_sandbox_session(session)++ row = self._reload(session)+ assert row.compute_settled_at is None+ assert row.compute_settle_conflict_at is None+ assert row.compute_settle_attempts == 0+ assert row.compute_settle_error == ""++ def test_flag_off_is_a_no_op(self) -> None:+ session = self._session()+ with override_settings(GATEWAY_COMPUTE_DEBIT_ENABLED=False), patch(HTTPX_POST) as post:+ assert settle_sandbox_session(session) == SettlementOutcome.DISABLED+ assert settle_task_run_compute(session.task_run_id, self.team.id) == []+ assert settle_overdue_sandbox_compute(self.now) == []++ post.assert_not_called()+ assert self._reload(session).compute_settled_at is None++ @parameterized.expand([("unattributed", True), ("zero_amount", False)])+ def test_settles_locally_without_a_call(self, _name: str, unattributed: bool) -> None:+ ended_at = self.now - timedelta(hours=1)+ session = self._session(user_attributed_at=None if unattributed else ended_at, ended_at=ended_at)+ with patch(HTTPX_POST) as post:+ assert settle_sandbox_session(session) == SettlementOutcome.SETTLED_LOCALLY++ post.assert_not_called()+ row = self._reload(session)+ assert row.compute_settled_at is not None+ assert row.compute_settled_usd == Decimal(0)++ def test_sweep_selects_overdue_rows_only(self) -> None:+ straggler = self._session(ended_at=self.now - timedelta(minutes=30))+ fresh = self._session(ended_at=self.now - timedelta(minutes=2))+ expired_open = self._session(+ ended_at=None,+ ended_reason=None,+ created_at=self.now - timedelta(hours=8),+ user_attributed_at=self.now - timedelta(hours=7, minutes=30),+ )+ hogland_open = self._session(+ ended_at=None,+ ended_reason=None,+ sandbox_backend="hogland",+ created_at=self.now - timedelta(hours=8),+ user_attributed_at=self.now - timedelta(hours=7, minutes=30),+ )++ with patch(HTTPX_POST, return_value=_ok()) as post:+ outcomes = settle_overdue_sandbox_compute(self.now)+ again = settle_overdue_sandbox_compute(self.now)++ assert outcomes == [SettlementOutcome.SETTLED, SettlementOutcome.SETTLED]+ assert again == []+ assert post.call_count == 2+ assert self._reload(straggler).compute_settled_at is not None+ assert self._reload(fresh).compute_settled_at is None+ closed = self._reload(expired_open)+ assert closed.ended_reason == SandboxSession.EndedReason.TTL_SWEEP+ assert closed.compute_settled_at is not None+ untouched = self._reload(hogland_open)+ assert untouched.ended_at is None+ assert untouched.compute_settled_at is None++ @parameterized.expand(+ [("settles", _ok(), SettlementOutcome.SETTLED), ("retries", _response(503), SettlementOutcome.RETRY)]+ )+ def test_sweep_drains_every_batch_once(+ self, _name: str, response: httpx.Response, expected: SettlementOutcome+ ) -> None:+ sessions = [self._session(ended_at=self.now - timedelta(minutes=20 + index)) for index in range(5)]+ with patch(HTTPX_POST, return_value=response) as post:+ assert settle_overdue_sandbox_compute(self.now, batch_size=2) == [expected] * 5++ assert post.call_count == 5+ for session in sessions:+ row = self._reload(session)+ assert (row.compute_settled_at is not None) is (expected == SettlementOutcome.SETTLED)+ assert row.compute_settle_attempts == 0++ def test_sweep_rotates_past_rows_it_already_retried(self) -> None:+ oldest = self._session(ended_at=self.now - timedelta(minutes=40))+ newer = self._session(ended_at=self.now - timedelta(minutes=30))+ with patch(HTTPX_POST, return_value=_response(503)) as post:+ first = settle_overdue_sandbox_compute(self.now, batch_size=1, time_budget=timedelta(0))+ assert [call.kwargs["json"]["reference"] for call in post.call_args_list] == [f"compute:{oldest.id}"]+ assert self._reload(oldest).compute_settle_attempted_at is not None+ post.reset_mock()+ second = settle_overdue_sandbox_compute(self.now, batch_size=1, time_budget=timedelta(0))+ assert [call.kwargs["json"]["reference"] for call in post.call_args_list] == [f"compute:{newer.id}"]++ assert first == [SettlementOutcome.RETRY]+ assert second == [SettlementOutcome.RETRY]++ def test_sweep_stops_on_a_deployment_error(self) -> None:+ self._session(ended_at=self.now - timedelta(minutes=30))+ self._session(ended_at=self.now - timedelta(minutes=40))+ with patch(HTTPX_POST, return_value=_response(401)) as post:+ assert settle_overdue_sandbox_compute(self.now) == []++ assert post.call_count == 1++ @parameterized.expand(+ [+ ("lone_repeat_parks", False, False, [SettlementOutcome.CONFLICT]),+ (+ "repeat_parks_beside_a_settled_row",+ True,+ True,+ [SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.SETTLED, SettlementOutcome.CONFLICT],+ ),+ (+ "shared_error_without_success_stops",+ False,+ True,+ [SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.CONTRACT_ERROR_REPEAT],+ ),+ ]+ )+ def test_sweep_parks_repeated_contract_errors(+ self, _name: str, with_success: bool, with_fresh_failure: bool, expected: list[SettlementOutcome]+ ) -> None:+ repeat = self._session(ended_at=self.now - timedelta(minutes=40))+ fresh = self._session(ended_at=self.now - timedelta(minutes=30)) if with_fresh_failure else None+ success = self._session(ended_at=self.now - timedelta(minutes=20)) if with_success else None+ with patch(HTTPX_POST, return_value=_response(400, {"error": "bad"})):+ assert settle_sandbox_session(repeat) == SettlementOutcome.CONTRACT_ERROR+ stored = self._reload(repeat).compute_settle_error+ assert stored.startswith("HTTP 400: ")++ def respond(_url: str, **kwargs: Any) -> httpx.Response:+ if success is not None and kwargs["json"]["reference"] == f"compute:{success.id}":+ return _ok()+ return _response(400, {"error": "bad"})++ with (+ patch(HTTPX_POST, side_effect=respond),+ patch("products.tasks.backend.logic.services.compute_settlement.logger") as logger,+ ):+ assert settle_overdue_sandbox_compute(self.now) == expected++ parks = SettlementOutcome.CONFLICT in expected+ stopped = [+ call for call in logger.exception.call_args_list if call.args[0] == "compute_settlement.sweep_stopped"+ ]+ assert bool(stopped) is not parks+ row = self._reload(repeat)+ assert row.compute_settle_error == stored+ assert (row.compute_settle_conflict_at is not None) is parks+ assert row.compute_settle_park_reason == ("http_400" if parks else "")+ if fresh is not None:+ fresh_row = self._reload(fresh)+ assert fresh_row.compute_settle_conflict_at is None+ assert fresh_row.compute_settle_error == stored+ if success is not None:+ assert self._reload(success).compute_settled_at is not None++ def test_reconciliation_compares_per_customer_team(self) -> None:+ since = self.now - timedelta(days=1)+ self._session(+ compute_settled_at=self.now - timedelta(minutes=5),+ compute_settled_usd=Decimal("1.540800"),+ compute_ledger_entry_id="entry-1",+ )+ self._session(+ created_at=since - timedelta(hours=2),+ user_attributed_at=since - timedelta(hours=1),+ ended_at=since + timedelta(hours=1),+ compute_settled_at=self.now - timedelta(minutes=5),+ compute_settled_usd=Decimal("3.081600"),+ compute_ledger_entry_id="entry-2",+ )+ rows = [+ {"keys": {"obo": str(self.team.id)}, "compute_usd": "4.622400", "billable_compute_usd": "4.622400"},+ {"keys": {"obo": "999"}, "compute_usd": "2.000000", "billable_compute_usd": "0"},+ ]+ report = httpx.Response(200, json={"rows": rows}, request=httpx.Request("GET", GATEWAY_URL))+ with patch("posthog.llm.gateway_internal_client.httpx.get", return_value=report) as get:+ result = reconcile_compute_settlement(since, self.now, self.team.id)++ assert get.call_args.kwargs["params"]["group_by"] == "obo,charge_kind"+ assert result.writer_parity == (+ TeamComputeDelta(+ team_id=str(self.team.id), local_usd=Decimal("4.622400"), gateway_usd=Decimal("4.622400"), local_rows=2+ ),+ TeamComputeDelta(team_id="999", local_usd=Decimal(0), gateway_usd=Decimal("2.000000"), local_rows=0),+ )+ assert result.pricing_parity[0].local_usd == Decimal("4.622400")+ billing = result.billing_parity[0]+ assert billing.team_id == str(self.team.id)+ assert billing.local_rows == 2+ assert abs(billing.delta_usd) <= CENT_ROUNDING_TOLERANCE_USDdiff --git a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py b/products/tasks/backend/logic/services/tests/test_sandbox_usage.pyindex bc27997e2991..5deceb1d9df7 100644--- a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py+++ b/products/tasks/backend/logic/services/tests/test_sandbox_usage.py@@ -10,6 +10,7 @@ from parameterized import parameterizedfrom posthog.models.organization import Organizationfrom posthog.models.team.team import Team+from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_sessionfrom products.tasks.backend.logic.services.sandbox import Sandbox, SandboxConfigfrom products.tasks.backend.logic.services.sandbox_pricing import ComputeRateCard, ComputeRateCardConfigurationErrorfrom products.tasks.backend.logic.services.sandbox_usage import (@@ -376,6 +377,29 @@ class TestSandboxUsageAggregation(SandboxUsageBase):assert usage.memory_mib_seconds == [(self.team.id, 176_947_200)]assert usage.credits == [(self.team.id, 6048)]+ @parameterized.expand(+ [+ ("desktop_user_created", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.USER_CREATED, True),+ ("desktop_loop", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.LOOP, True),+ ("desktop_space_setup", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.SPACE_SETUP, True),+ ("desktop_slack", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.SLACK, True),+ ("no_provenance", None, Task.OriginProduct.USER_CREATED, True),+ ("unattributed", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.USER_CREATED, False),+ ]+ )+ def test_billable_session_predicate_matches_the_billing_aggregate(+ self, _name: str, provenance: TaskClientProvenance | None, origin: str, attributed: bool+ ) -> None:+ session = self._session(+ client_provenance=provenance,+ origin_product=origin,+ user_attributed_at=datetime(2026, 1, 2, 1, tzinfo=UTC) if attributed else None,+ )++ usage = get_billable_sandbox_compute_usage_by_team(self.BEGIN, self.END, rate_cards=(self._rate(),))++ assert is_billable_sandbox_session(session) is bool(usage.credits)+def test_billable_compute_uses_session_snapshot_after_task_changes(self):session = self._session(client_provenance=TaskClientProvenance.POSTHOG_DESKTOP)Task.objects.filter(id=session.task_run.task_id).update(diff --git a/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.pynew file mode 100644index 000000000000..5148c602cf82--- /dev/null+++ b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py@@ -0,0 +1,78 @@+from datetime import datetime, timedelta+from decimal import Decimal++from django.core.management.base import BaseCommand, CommandError, CommandParser+from django.utils import timezone++from products.tasks.backend.logic.services.compute_settlement import (+ TeamComputeDelta,+ reconcile_compute_settlement,+ unpark_sandbox_compute,+ wallet_team_id,+)+from products.tasks.backend.models import SandboxSession++CENT_ROUNDING_TOLERANCE_USD = Decimal("0.005")+++class Command(BaseCommand):+ help = (+ "Report settled sandbox compute against the gateway ledger, per customer team. "+ "With --unpark, first return rows parked for that reason in the window to the settlement sweep"+ )++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--since", type=datetime.fromisoformat, default=None)+ parser.add_argument("--until", type=datetime.fromisoformat, default=None)+ parser.add_argument("--wallet-team-id", type=int, default=None)+ parser.add_argument(+ "--unpark",+ choices=[+ SandboxSession.ComputeSettleParkReason.ATTEMPTS_EXHAUSTED.value,+ SandboxSession.ComputeSettleParkReason.HTTP_400.value,+ ],+ default=None,+ help="Unpark rows parked for this reason within --since/--until so the sweep retries them",+ )++ def handle(self, *args: object, **options: object) -> None:+ until = options["until"] or timezone.now()+ if not isinstance(until, datetime):+ raise CommandError("--until must be an ISO-8601 timestamp")+ since = options["since"] or until - timedelta(days=7)+ if not isinstance(since, datetime):+ raise CommandError("--since must be an ISO-8601 timestamp")+ if timezone.is_naive(since) or timezone.is_naive(until) or until <= since:+ raise CommandError("--since and --until must be timezone-aware and --until must follow --since")+ unpark = options["unpark"]+ if isinstance(unpark, str):+ count = unpark_sandbox_compute(unpark, since, until)+ self.stdout.write(+ f"Unparked {count} sessions parked for {unpark} between {since.isoformat()} and {until.isoformat()}"+ )+ wallet = options["wallet_team_id"]+ wallet_team = wallet if isinstance(wallet, int) else wallet_team_id()++ report = reconcile_compute_settlement(since, until, wallet_team)+ self.stdout.write(f"Window {since.isoformat()} to {until.isoformat()}, wallet team {wallet_team}")+ self._section("Writer parity: settled compute_settled_usd vs gateway compute_usd", report.writer_parity)+ self._section(+ "Pricing parity: attributed sessions ended in window, priced live, vs gateway compute_usd",+ report.pricing_parity,+ )+ self._section(+ "Billing parity: billing aggregate for the window vs settled ledger amounts prorated to the window",+ report.billing_parity,+ tolerance_usd=CENT_ROUNDING_TOLERANCE_USD,+ )++ def _section(self, title: str, rows: tuple[TeamComputeDelta, ...], *, tolerance_usd: Decimal = Decimal(0)) -> None:+ self.stdout.write("")+ self.stdout.write(title)+ self.stdout.write(f"{'team':>12} {'local_usd':>14} {'gateway_usd':>14} {'delta_usd':>14} {'local_rows':>10}")+ for row in rows:+ line = (+ f"{row.team_id:>12} {row.local_usd:>14.6f} {row.gateway_usd:>14.6f} "+ f"{row.delta_usd:>14.6f} {row.local_rows:>10}"+ )+ self.stdout.write(self.style.WARNING(line) if abs(row.delta_usd) > tolerance_usd else line)diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.pyindex 43f8d6252171..46fdef08be8d 100644--- a/products/tasks/backend/metrics.py+++ b/products/tasks/backend/metrics.py@@ -474,6 +474,39 @@ def observe_compute_quota_check(outcome: ComputeQuotaOutcome) -> None:COMPUTE_QUOTA_CHECK_TOTAL.labels(outcome=outcome).inc()+ComputeSettlementOutcome = Literal[+ "settled",+ "duplicate",+ "settled_locally",+ "conflict",+ "retry",+ "attempts_exhausted",+ "contract_error",+ "deployment_error",+ "skipped",+]++COMPUTE_SETTLEMENTS_TOTAL = Counter(+ "posthog_tasks_compute_settlements_total",+ "Sandbox compute settlement attempts against the gateway ledger, by outcome",+ labelnames=["outcome"],+)++HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY = Gauge(+ "posthog_tasks_hogland_sandbox_sessions_open_over_day",+ "Hogland sandbox sessions with no end stamp that opened more than a day ago",+ multiprocess_mode="mostrecent",+)+++def observe_compute_settlement(outcome: ComputeSettlementOutcome) -> None:+ COMPUTE_SETTLEMENTS_TOTAL.labels(outcome=outcome).inc()+++def set_hogland_sandbox_sessions_open_over_day(count: int) -> None:+ HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY.set(count)++# scoped: "true" when the delivery's installation resolved to at least one team, so the# TaskRun lookup could ride the team_id index. "false" means no team links the installation,# so the lookup is skipped and the delivery matches no run.diff --git a/products/tasks/backend/migrations/0145_sandboxsession_compute_settlement.py b/products/tasks/backend/migrations/0145_sandboxsession_compute_settlement.pynew file mode 100644index 000000000000..7e48318fe41c--- /dev/null+++ b/products/tasks/backend/migrations/0145_sandboxsession_compute_settlement.py@@ -0,0 +1,116 @@+# Generated by Django 6.1.2 on 2026-11-02 12:54++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("tasks", "0144_task_options"),+ ]++ operations = [+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_ledger_entry_id",+ field=models.CharField(+ blank=True,+ default="",+ help_text="Gateway ledger entry id of the compute debit",+ max_length=64,+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settle_attempted_at",+ field=models.DateTimeField(+ blank=True,+ help_text="When the last reported-spend request was sent; the sweep tries never-attempted rows first",+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settle_attempts",+ field=models.PositiveIntegerField(+ default=0, help_text="Gateway 200 responses whose body could not be parsed so far"+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settle_body",+ field=models.JSONField(+ blank=True,+ default=dict,+ help_text="Reported-spend body frozen before the first attempt so retries replay it unchanged",+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settle_conflict_at",+ field=models.DateTimeField(+ blank=True,+ help_text="When the gateway refused the debit for good; the sweep skips the row",+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settle_error",+ field=models.CharField(+ blank=True,+ default="",+ help_text="Last gateway error text; a 400 stays so the sweep can recognise a repeat",+ max_length=200,+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settle_park_reason",+ field=models.CharField(+ blank=True,+ choices=[+ ("http_400", "HTTP 400"),+ ("http_409", "HTTP 409"),+ ("attempts_exhausted", "Attempts exhausted"),+ ("unexpected_status", "Unexpected status"),+ ],+ default="",+ help_text="Why settlement stopped retrying the row; empty while the row is not parked",+ max_length=32,+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settled_at",+ field=models.DateTimeField(+ blank=True,+ help_text="When the session's compute was settled on the gateway ledger",+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="sandboxsession",+ name="compute_settled_usd",+ field=models.DecimalField(+ blank=True,+ decimal_places=6,+ help_text="Compute amount the ledger recorded",+ max_digits=20,+ null=True,+ ),+ ),+ migrations.AlterField(+ model_name="sandboxsession",+ name="ended_reason",+ field=models.CharField(+ blank=True,+ choices=[+ ("cleanup", "Cleanup"),+ ("reaped", "Reaped"),+ ("ttl_sweep", "TTL sweep"),+ ],+ max_length=20,+ null=True,+ ),+ ),+ ]diff --git a/products/tasks/backend/migrations/0146_sandboxsession_settle_indexes.py b/products/tasks/backend/migrations/0146_sandboxsession_settle_indexes.pynew file mode 100644index 000000000000..e7b11caa27d2--- /dev/null+++ b/products/tasks/backend/migrations/0146_sandboxsession_settle_indexes.py@@ -0,0 +1,39 @@+from django.db import migrations, models++from posthog.migration_helpers import SafeAddIndexConcurrently+++class Migration(migrations.Migration):+ atomic = False++ dependencies = [+ ("tasks", "0145_sandboxsession_compute_settlement"),+ ]++ operations = [+ SafeAddIndexConcurrently(+ model_name="sandboxsession",+ index=models.Index(+ condition=models.Q(+ ("compute_settle_conflict_at__isnull", True),+ ("compute_settled_at__isnull", True),+ ("user_attributed_at__isnull", False),+ ),+ fields=["ttl_expires_at"],+ name="sandbox_session_settle_ttl_idx",+ ),+ ),+ SafeAddIndexConcurrently(+ model_name="sandboxsession",+ index=models.Index(+ models.OrderBy(models.F("compute_settle_attempted_at"), nulls_first=True),+ models.F("ended_at"),+ condition=models.Q(+ ("compute_settle_conflict_at__isnull", True),+ ("compute_settled_at__isnull", True),+ ("user_attributed_at__isnull", False),+ ),+ name="sandbox_session_settle_rt_idx",+ ),+ ),+ ]diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txtindex 4434490791b7..eada7f0b5316 100644--- a/products/tasks/backend/migrations/max_migration.txt+++ b/products/tasks/backend/migrations/max_migration.txt@@ -1 +1 @@-0144_task_options+0146_sandboxsession_settle_indexesdiff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.pyindex 5352b5e77494..a7a1a78aca88 100644--- a/products/tasks/backend/models.py+++ b/products/tasks/backend/models.py@@ -3785,6 +3785,13 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):class EndedReason(models.TextChoices):CLEANUP = "cleanup", "Cleanup"REAPED = "reaped", "Reaped"+ TTL_SWEEP = "ttl_sweep", "TTL sweep"++ class ComputeSettleParkReason(models.TextChoices):+ HTTP_400 = "http_400", "HTTP 400"+ HTTP_409 = "http_409", "HTTP 409"+ ATTEMPTS_EXHAUSTED = "attempts_exhausted", "Attempts exhausted"+ UNEXPECTED_STATUS = "unexpected_status", "Unexpected status"# db_constraint=False on the team FK: adding an FK constraint to that hot table# locks it and stalls deploys; Django still enforces the relation and on_delete at@@ -3863,6 +3870,44 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):provider_usage_measured_at = models.DateTimeField(null=True, blank=True, help_text="When provider resource usage was sampled")+ compute_settled_at = models.DateTimeField(+ null=True, blank=True, help_text="When the session's compute was settled on the gateway ledger"+ )+ compute_settled_usd = models.DecimalField(+ max_digits=20, decimal_places=6, null=True, blank=True, help_text="Compute amount the ledger recorded"+ )+ compute_ledger_entry_id = models.CharField(+ max_length=64, blank=True, default="", help_text="Gateway ledger entry id of the compute debit"+ )+ compute_settle_attempts = models.PositiveIntegerField(+ default=0, help_text="Gateway 200 responses whose body could not be parsed so far"+ )+ compute_settle_attempted_at = models.DateTimeField(+ null=True,+ blank=True,+ help_text="When the last reported-spend request was sent; the sweep tries never-attempted rows first",+ )+ compute_settle_conflict_at = models.DateTimeField(+ null=True, blank=True, help_text="When the gateway refused the debit for good; the sweep skips the row"+ )+ compute_settle_error = models.CharField(+ max_length=200,+ blank=True,+ default="",+ help_text="Last gateway error text; a 400 stays so the sweep can recognise a repeat",+ )+ compute_settle_park_reason = models.CharField(+ max_length=32,+ choices=ComputeSettleParkReason,+ blank=True,+ default="",+ help_text="Why settlement stopped retrying the row; empty while the row is not parked",+ )+ compute_settle_body = models.JSONField(+ default=dict,+ blank=True,+ help_text="Reported-spend body frozen before the first attempt so retries replay it unchanged",+ )class Meta:db_table = "posthog_task_sandbox_session"@@ -3879,6 +3924,25 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):),# For per-team/per-origin re-aggregation once pricing decides which origins bill.models.Index(fields=["team", "user_attributed_at"], name="sandbox_session_team_attr_idx"),+ models.Index(+ fields=["ttl_expires_at"],+ condition=models.Q(+ compute_settled_at__isnull=True,+ compute_settle_conflict_at__isnull=True,+ user_attributed_at__isnull=False,+ ),+ name="sandbox_session_settle_ttl_idx",+ ),+ models.Index(+ models.OrderBy(models.F("compute_settle_attempted_at"), nulls_first=True),+ models.F("ended_at"),+ condition=models.Q(+ compute_settled_at__isnull=True,+ compute_settle_conflict_at__isnull=True,+ user_attributed_at__isnull=False,+ ),+ name="sandbox_session_settle_rt_idx",+ ),]def __str__(self):diff --git a/products/tasks/backend/tasks/tasks.py b/products/tasks/backend/tasks/tasks.pyindex 756898ba9aa4..d31b05b0fc5b 100644--- a/products/tasks/backend/tasks/tasks.py+++ b/products/tasks/backend/tasks/tasks.py@@ -2,6 +2,7 @@ from datetime import datetimefrom uuid import UUIDfrom django.db import OperationalError+from django.utils import timezonefrom celery import shared_taskfrom celery.app.task import Task as CeleryTask@@ -15,6 +16,7 @@ from posthog.models.github_integration_base import GitHubIntegrationErrorfrom products.tasks.backend.facade.api import record_comment_activityfrom products.tasks.backend.logic.services.comment_slack_dm import send_comment_slack_dms+from products.tasks.backend.logic.services.compute_settlement import settle_overdue_sandbox_computefrom products.tasks.backend.logic.services.slack_pr_cards import post_pr_closed_slack_updatefrom products.tasks.backend.logic.services.workflow_step_resume import resume_workflow_step_for_run_idfrom products.tasks.backend.logic.stream.budget_steer import BudgetSteerCapture, BudgetSteerProperties@@ -103,3 +105,8 @@ def resume_workflow_step_for_run_deferred(run_id: str) -> None:@shared_task(ignore_result=True)def notify_slack_thread_pr_closed(run_id: str, pr_url: str, merged: bool = False) -> None:post_pr_closed_slack_update(run_id, pr_url, merged=merged)+++@shared_task(ignore_result=True)+def settle_overdue_sandbox_compute_task() -> None:+ settle_overdue_sandbox_compute(timezone.now())diff --git a/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.py b/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.pyindex 08b4c336b2a3..2db1576c6287 100644--- a/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.py+++ b/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.py@@ -15,6 +15,7 @@ a destroy failure does not block the clear — a stale id staying in statewould just be re-reaped (and fail Modal-side again) on the next start."""+import loggingfrom dataclasses import dataclassfrom typing import Optional@@ -22,6 +23,7 @@ from temporalio import activityfrom posthog.temporal.common.utils import asyncify+from products.tasks.backend.logic.services.compute_settlement import settle_task_run_computefrom products.tasks.backend.logic.services.sandbox import get_sandbox_class_for_sandbox_idfrom products.tasks.backend.logic.services.sandbox_usage import (close_sandbox_session,@@ -32,6 +34,8 @@ from products.tasks.backend.models import SandboxSession, TaskRunfrom products.tasks.backend.temporal.execute_sandbox.activities.sandbox_state import SANDBOX_ID_STATE_KEYfrom products.tasks.backend.temporal.observability import log_activity_execution+logger = logging.getLogger(__name__)+@dataclassclass ReapOrphanedSandboxInput:@@ -57,7 +61,7 @@ class ReapOrphanedSandboxResult:def reap_orphaned_sandbox(input: ReapOrphanedSandboxInput) -> ReapOrphanedSandboxResult:with log_activity_execution("reap_orphaned_sandbox", run_id=input.run_id):try:- task_run = TaskRun.objects.only("state").get(id=input.run_id)+ task_run = TaskRun.objects.only("state", "team_id").get(id=input.run_id)except TaskRun.DoesNotExist:return ReapOrphanedSandboxResult(reaped_sandbox_id=None, destroy_succeeded=True)@@ -91,6 +95,12 @@ def reap_orphaned_sandbox(input: ReapOrphanedSandboxInput) -> ReapOrphanedSandbobilled_cpu_usage_usec=billed_cpu_usage_usec,cpu_usage_measured_at=cpu_usage_measured_at,)+ try:+ settle_task_run_compute(input.run_id, task_run.team_id)+ except Exception:+ logger.warning(+ "reap_orphaned_sandbox_compute_settlement_failed", extra={"run_id": input.run_id}, exc_info=True+ )TaskRun.clear_sandbox_connection_state_atomic(input.run_id, sandbox_id)return ReapOrphanedSandboxResult(reaped_sandbox_id=sandbox_id, destroy_succeeded=destroy_succeeded)diff --git a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.pyindex ed822d2d37bd..14d9c1936ea7 100644--- a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py@@ -7,6 +7,7 @@ from temporalio import activityfrom posthog.temporal.common.utils import asyncifyfrom products.tasks.backend.exceptions import SandboxNotFoundError+from products.tasks.backend.logic.services.compute_settlement import settle_task_run_computefrom products.tasks.backend.logic.services.gateway_usage import refresh_task_run_costfrom products.tasks.backend.logic.services.sandbox import get_sandbox_class_for_sandbox_idfrom products.tasks.backend.logic.services.sandbox_usage import (@@ -136,6 +137,14 @@ def cleanup_sandbox_now(input: CleanupSandboxInput) -> None:)if accounting_run is not None:+ try:+ settle_task_run_compute(accounting_run.id, accounting_run.team_id)+ except Exception:+ logger.warning(+ "cleanup_sandbox_compute_settlement_failed",+ extra={"run_id": str(accounting_run.id)},+ exc_info=True,+ )try:refresh_task_run_cost(run_id=accounting_run.id, team_id=accounting_run.team_id)except Exception:diff --git a/products/tasks/backend/temporal/process_task/ai_gateway_token.py b/products/tasks/backend/temporal/process_task/ai_gateway_token.pyindex a00ea6ec2f42..285cd5bca4cb 100644--- a/products/tasks/backend/temporal/process_task/ai_gateway_token.py+++ b/products/tasks/backend/temporal/process_task/ai_gateway_token.py@@ -383,6 +383,7 @@ def mint_scoped_token(expires_in_seconds: int | None = None,gateway_config: "AIGatewayConfig | None" = None,raise_on_failure: bool = False,+ run_id: str | None = None,) -> str | None:"""Mint a `phe_` scoped token pinned to (ai_product, obo=team_id), or None on failure.@@ -410,6 +411,8 @@ def mint_scoped_token(}if user:body["user"] = user+ if run_id:+ body["run"] = run_idpin = allowed_models if allowed_models is not None else _PRODUCT_ALLOWED_MODELS.get(ai_product)if pin:body["allowed_models"] = gateway_pin(pin)diff --git a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.pyindex aa3e10ce9acc..ef902e482555 100644--- a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py+++ b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py@@ -770,6 +770,7 @@ class TestProvisioningBoundaries:runtime="acp",prior_slack_run=False,profile_billing_product=profile_billing_product,+ run_id=str(self._ctx().run_id),)def test_non_slack_origin_skips_the_prior_run_lookup(self, mint_settings):@@ -854,6 +855,7 @@ class TestProvisioningBoundaries:assert post.call_args.kwargs["json"]["capture_mode"] == "none"assert post.call_args.kwargs["json"]["obo"] == str(ctx.team_id)assert post.call_args.kwargs["json"]["user"] == ctx.distinct_id+ assert post.call_args.kwargs["json"]["run"] == ctx.run_idrecord_routing.assert_called_once_with(run_id=ctx.run_id, team_id=ctx.team_id, uses_gateway=True)else:assert out == {}@@ -1103,14 +1105,16 @@ class TestUserPinAndCapOverride:def test_mint_pins_the_acting_user(self, mint_settings):with patch("products.tasks.backend.temporal.process_task.ai_gateway_token.requests.post") as post:post.return_value = self._response({"token": "phe_abc"})- mint_scoped_token(ai_product="signals_scout", team_id=123, user="user-distinct-1")+ mint_scoped_token(ai_product="signals_scout", team_id=123, user="user-distinct-1", run_id="run-1")assert post.call_args.kwargs["json"]["user"] == "user-distinct-1"+ assert post.call_args.kwargs["json"]["run"] == "run-1"def test_mint_omits_user_when_unknown(self, mint_settings):with patch("products.tasks.backend.temporal.process_task.ai_gateway_token.requests.post") as post:post.return_value = self._response({"token": "phe_abc"})mint_scoped_token(ai_product="signals_scout", team_id=123)assert "user" not in post.call_args.kwargs["json"]+ assert "run" not in post.call_args.kwargs["json"]def test_cap_override_applies_per_team(self, mint_settings):mint_settings.SANDBOX_AI_GATEWAY_TOKEN_CAP_USD_OVERRIDES = '{"2": "10"}'diff --git a/products/tasks/backend/temporal/process_task/utils.py b/products/tasks/backend/temporal/process_task/utils.pyindex 9dcfa27476ad..a62b0b4b93fe 100644--- a/products/tasks/backend/temporal/process_task/utils.py+++ b/products/tasks/backend/temporal/process_task/utils.py@@ -1600,7 +1600,7 @@ def run_gateway_env_vars(ctx: TaskProcessingContext, task: Task) -> dict[str, stgateway_url = settings.SANDBOX_AI_GATEWAY_URLif not gateway_url:raise GatewayNotConfiguredError("Scout trials require SANDBOX_AI_GATEWAY_URL")- token = mint_private_gateway_token(team_id=ctx.team_id, user=ctx.distinct_id)+ token = mint_private_gateway_token(team_id=ctx.team_id, user=ctx.distinct_id, run_id=str(ctx.run_id))try:record_gateway_routing(run_id=ctx.run_id, team_id=ctx.team_id, uses_gateway=True)except Exception:@@ -1650,6 +1650,7 @@ def run_gateway_env_vars(ctx: TaskProcessingContext, task: Task) -> dict[str, ststate=ctx.state,model=ctx.model,runtime=ctx.task_runtime,+ run_id=str(ctx.run_id),)pinned = env_vars.pop(_PIN_KEY_ENV, None) or env_vars.get("AI_GATEWAY_PRODUCT")if not _record_pinned_gateway_product(ctx.run_id, ctx.state, pinned):@@ -1754,6 +1755,7 @@ def ai_gateway_env_vars(runtime: str | None = None,prior_slack_run: bool = False,profile_billing_product: str | None = None,+ run_id: str | None = None,) -> dict[str, str]:"""Env vars routing listed products to the Go ai-gateway, shared by everyinjection site so the both-or-nothing guard cannot drift per site. Both@@ -1861,6 +1863,8 @@ def ai_gateway_env_vars(if free_pin is not None:mint_kwargs["allowed_models"] = free_pinmint_kwargs["limit_tier"] = posthog_code_limit_tier(team_id, distinct_id)+ if run_id:+ mint_kwargs["run_id"] = run_idtoken = mint_scoped_token(ai_product=ai_product, team_id=team_id, user=distinct_id, **mint_kwargs)if token:env_vars["AI_GATEWAY_TOKEN"] = tokendiff --git a/products/tasks/backend/tests/test_gateway_usage.py b/products/tasks/backend/tests/test_gateway_usage.pyindex 887ea141cd4b..80b055795879 100644--- a/products/tasks/backend/tests/test_gateway_usage.py+++ b/products/tasks/backend/tests/test_gateway_usage.py@@ -277,6 +277,10 @@ class TestGatewayUsage(BaseTest):run.refresh_from_db()assert run.state["compute_cost"] == expectedassert set(run.state) == {"unprocessed_request_ids", "token_cost", "compute_cost"}+ SandboxSession.objects.for_team(self.team.id).filter(sandbox_id="claimed").update(+ compute_settled_at=now, compute_settled_usd=Decimal("1.234567")+ )+ assert get_task_run_cost(run_id=run.id, team_id=self.team.id).compute_cost == 123@patch("aiohttp.ClientSession._request")def test_accounting_after_completion_has_no_completion_side_effects(self, get: Mock) -> None: