Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): settle sandbox compute to the gateway ledger (#127448)

Alessandro Pogliaghie60de2cb1eb8
diff --git a/posthog/llm/gateway_internal_client.py b/posthog/llm/gateway_internal_client.py
index a0590192b022..3326c87a8d46 100644
--- a/posthog/llm/gateway_internal_client.py
+++ b/posthog/llm/gateway_internal_client.py
@@ -6,11 +6,13 @@ from django.conf import settings
import httpx
INTERNAL_API_TIMEOUT_SECONDS = 5.0
+REPORTED_SPEND_TIMEOUT_SECONDS = 2.0
IDEMPOTENCY_KEY_HEADER = "Idempotency-Key"
INTERNAL_ACTOR_HEADER = "X-Internal-Actor"
# Ledger provenance tag (meta.source): admin-originated vs the billing path's
# "billing". The acting human is audited in ActivityLog, not here.
ADMIN_ACTOR = "posthog-admin"
+TASKS_COMPUTE_ACTOR = "tasks-compute"
class AIGatewayInternalError(Exception):
@@ -52,6 +54,13 @@ class CreditResult:
duplicate: bool
[email protected](frozen=True)
+class ReportedSpendResponse:
+ status_code: int
+ body: dict[str, Any] | None
+ text: str
+
+
def _config() -> tuple[str, str]:
url = (settings.AI_GATEWAY_INTERNAL_URL or "").rstrip("/")
token = settings.AI_GATEWAY_INTERNAL_TOKEN or ""
@@ -142,6 +151,46 @@ def add_credit(team_id: int, amount_usd: str, reason: str, idempotency_key: str)
)
+def report_spend(wallet_team_id: int, body: dict[str, Any]) -> ReportedSpendResponse:
+ url, token = _config()
+ response = httpx.post(
+ f"{url}/internal/teams/{wallet_team_id}/reported-spend",
+ headers=_auth_headers(token, {INTERNAL_ACTOR_HEADER: TASKS_COMPUTE_ACTOR}),
+ json=body,
+ timeout=REPORTED_SPEND_TIMEOUT_SECONDS,
+ trust_env=False,
+ )
+ try:
+ parsed: Any = response.json()
+ except ValueError:
+ parsed = None
+ return ReportedSpendResponse(
+ status_code=response.status_code,
+ body=parsed if isinstance(parsed, dict) else None,
+ text=response.text,
+ )
+
+
+def get_usage_report(params: dict[str, str]) -> list[dict[str, Any]]:
+ url, token = _config()
+ try:
+ response = httpx.get(
+ f"{url}/internal/usage/report",
+ headers=_auth_headers(token, {INTERNAL_ACTOR_HEADER: TASKS_COMPUTE_ACTOR}),
+ params=params,
+ timeout=INTERNAL_API_TIMEOUT_SECONDS,
+ trust_env=False,
+ )
+ response.raise_for_status()
+ data: Any = response.json()
+ except (httpx.HTTPError, ValueError) as exc:
+ raise AIGatewayInternalError(f"usage report read failed: {exc}") from exc
+ rows = data.get("rows") if isinstance(data, dict) else data
+ if not isinstance(rows, list):
+ raise AIGatewayInternalError("usage report response had no rows")
+ return [row for row in rows if isinstance(row, dict)]
+
+
def _error_detail(response: httpx.Response) -> str:
try:
body: Any = response.json()
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 4efaf0ac9576..f6a52dcf323d 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -145,6 +145,7 @@ SANDBOX_AI_GATEWAY_TOKEN_CAP_USD_PRODUCT_OVERRIDES: str = get_from_env(
"SANDBOX_AI_GATEWAY_TOKEN_CAP_USD_PRODUCT_OVERRIDES", ""
)
SANDBOX_AI_GATEWAY_TOKEN_TTL_SECONDS: int = get_from_env("SANDBOX_AI_GATEWAY_TOKEN_TTL_SECONDS", 0, type_cast=int)
+GATEWAY_COMPUTE_DEBIT_ENABLED: bool = get_from_env("GATEWAY_COMPUTE_DEBIT_ENABLED", False, type_cast=str_to_bool)
SANDBOX_MCP_URL: str | None = get_from_env("SANDBOX_MCP_URL", None, optional=True)
# OTLP destinations for agent-server run telemetry (PostHog Logs/APM).
diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.py
index ff5c8ab7b3e8..c29d83d8c339 100644
--- a/posthog/tasks/scheduled.py
+++ b/posthog/tasks/scheduled.py
@@ -145,6 +145,7 @@ from products.tasks.backend.facade.tasks import (
bake_dev_stack_image_task,
refresh_dev_stack_image_task,
refresh_stale_sandbox_custom_images_task,
+ settle_overdue_sandbox_compute_task,
sweep_inactive_tasks_task,
)
from products.visual_review.backend.facade.tasks import (
@@ -527,6 +528,13 @@ def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:
name="archive inactive tasks",
)
+ add_periodic_task_with_expiry(
+ sender,
+ crontab(minute="47"),
+ settle_overdue_sandbox_compute_task.s(),
+ name="settle overdue sandbox compute",
+ )
+
# AWS SES account reputation → gauges for team-facing alerting (charts alerts/specs/ses.yaml)
add_periodic_task_with_expiry(
sender,
diff --git a/products/tasks/backend/facade/gateway.py b/products/tasks/backend/facade/gateway.py
index 1a86276f36ef..939c8ae1aee5 100644
--- a/products/tasks/backend/facade/gateway.py
+++ b/products/tasks/backend/facade/gateway.py
@@ -16,6 +16,7 @@ def mint_private_gateway_token(
user: str | None = None,
expires_in_seconds: int | None = None,
gateway_config: "AIGatewayConfig | None" = None,
+ run_id: str | None = None,
) -> str:
from products.tasks.backend.temporal.process_task.ai_gateway_token import ( # noqa: PLC0415 -- breaks the Temporal aggregate's import cycle through process_task.utils
mint_scoped_token,
@@ -28,6 +29,7 @@ def mint_private_gateway_token(
capture_mode="none",
expires_in_seconds=expires_in_seconds,
gateway_config=gateway_config,
+ run_id=run_id,
)
if token is None:
raise GatewayNotConfiguredError("Scout trials require a private AI gateway credential")
diff --git a/products/tasks/backend/facade/tasks.py b/products/tasks/backend/facade/tasks.py
index f04e68060243..82781c3ffbad 100644
--- a/products/tasks/backend/facade/tasks.py
+++ b/products/tasks/backend/facade/tasks.py
@@ -10,10 +10,12 @@ from products.tasks.backend.tasks.sandbox_images import (
refresh_dev_stack_image_task,
refresh_stale_sandbox_custom_images_task,
)
+from products.tasks.backend.tasks.tasks import settle_overdue_sandbox_compute_task
__all__ = [
"bake_dev_stack_image_task",
"refresh_dev_stack_image_task",
"refresh_stale_sandbox_custom_images_task",
+ "settle_overdue_sandbox_compute_task",
"sweep_inactive_tasks_task",
]
diff --git a/products/tasks/backend/logic/services/compute_quota.py b/products/tasks/backend/logic/services/compute_quota.py
index 17a141bab15b..c3554992954a 100644
--- a/products/tasks/backend/logic/services/compute_quota.py
+++ b/products/tasks/backend/logic/services/compute_quota.py
@@ -1,4 +1,5 @@
import logging
+from typing import TYPE_CHECKING
from django.conf import settings
@@ -8,6 +9,9 @@ from products.tasks.backend.facade.contracts import ComputeQuotaDenialReason
from products.tasks.backend.metrics import observe_compute_quota_check
from products.tasks.backend.models import Task, TaskClientProvenance
+if TYPE_CHECKING:
+ from products.tasks.backend.models import SandboxSession
+
COMPUTE_QUOTA_DENIAL_CODE = ComputeQuotaDenialReason.COMPUTE_QUOTA_EXHAUSTED.value
ORGANIZATION_DEACTIVATED_DENIAL_CODE = ComputeQuotaDenialReason.ORGANIZATION_DEACTIVATED.value
@@ -35,6 +39,14 @@ def is_billable_compute(*, origin_product: str | None, client_provenance: str |
return origin_product in (Task.OriginProduct.USER_CREATED, Task.OriginProduct.SPACE_SETUP)
+def is_billable_sandbox_session(session: "SandboxSession") -> bool:
+ return (
+ session.user_attributed_at is not None
+ and session.origin_product == Task.OriginProduct.USER_CREATED
+ and is_billable_compute(origin_product=session.origin_product, client_provenance=session.client_provenance)
+ )
+
+
def get_compute_quota_denial_reason(task: Task) -> ComputeQuotaDenialReason | None:
if organization_deactivated(task.team_id):
observe_compute_quota_check("checked_blocked")
diff --git a/products/tasks/backend/logic/services/compute_settlement.py b/products/tasks/backend/logic/services/compute_settlement.py
new file mode 100644
index 000000000000..c34b16bdc0a7
--- /dev/null
+++ b/products/tasks/backend/logic/services/compute_settlement.py
@@ -0,0 +1,566 @@
+import time
+from collections import Counter
+from datetime import UTC, datetime, timedelta
+from decimal import ROUND_HALF_EVEN, Decimal, InvalidOperation
+from enum import StrEnum
+from typing import Any
+from uuid import UUID
+
+from django.conf import settings
+from django.db.models import Count, F, QuerySet, Sum
+from django.utils import timezone
+
+import httpx
+import structlog
+
+from posthog.dataclasses import frozen
+from posthog.llm.gateway_internal_client import (
+ AIGatewayNotConfigured,
+ ReportedSpendResponse,
+ get_usage_report,
+ report_spend,
+)
+from posthog.models.project_secret_api_key import find_project_secret_api_key
+
+from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_session
+from products.tasks.backend.logic.services.run_actor import get_actor_distinct_id, get_task_run_credential_user
+from products.tasks.backend.logic.services.sandbox_pricing import (
+ COMPUTE_RATE_CARDS,
+ SandboxComputeCost,
+ calculate_sandbox_compute_cost,
+)
+from products.tasks.backend.logic.services.sandbox_usage import (
+ close_sandbox_session,
+ get_billable_sandbox_compute_usage_by_team,
+)
+from products.tasks.backend.metrics import (
+ ComputeSettlementOutcome,
+ observe_compute_settlement,
+ set_hogland_sandbox_sessions_open_over_day,
+)
+from products.tasks.backend.models import SandboxSession, TaskRun
+
+logger = structlog.get_logger(__name__)
+
+MAX_SETTLE_ATTEMPTS = 20
+ATTEMPTS_EXHAUSTED_ERROR = "attempts_exhausted"
+ENDED_GRACE = timedelta(minutes=10)
+TTL_GRACE = timedelta(hours=1)
+HOGLAND_OPEN_ALERT_AGE = timedelta(days=1)
+DETAIL_MAX_BYTES = 2048
+_USD_QUANTUM = Decimal("0.000001")
+_RETRYABLE_STATUSES = frozenset({408, 429})
+_DEPLOYMENT_STATUSES = frozenset({401, 403, 404})
+_ERROR_MAX_LENGTH = 200
+
+
+class SettlementOutcome(StrEnum):
+ DISABLED = "disabled"
+ SKIPPED = "skipped"
+ SETTLED = "settled"
+ DUPLICATE = "duplicate"
+ SETTLED_LOCALLY = "settled_locally"
+ CONFLICT = "conflict"
+ RETRY = "retry"
+ ATTEMPTS_EXHAUSTED = "attempts_exhausted"
+ CONTRACT_ERROR = "contract_error"
+ CONTRACT_ERROR_REPEAT = "contract_error_repeat"
+
+
+_CONTRACT_ERROR_OUTCOMES = frozenset({SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.CONTRACT_ERROR_REPEAT})
+_SETTLED_OUTCOMES = frozenset({SettlementOutcome.SETTLED, SettlementOutcome.DUPLICATE})
+ParkReason = SandboxSession.ComputeSettleParkReason
+
+
+class ComputeSettlementDeploymentError(Exception):
+ pass
+
+
+def _unsettled(queryset: QuerySet[SandboxSession]) -> QuerySet[SandboxSession]:
+ return queryset.filter(compute_settled_at__isnull=True, compute_settle_conflict_at__isnull=True)
+
+
+def _session_rows(session: SandboxSession) -> QuerySet[SandboxSession]:
+ return SandboxSession.objects.for_team(session.team_id).filter(id=session.id)
+
+
+def _iso(value: datetime) -> str:
+ return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def _fixed6(value: Decimal) -> str:
+ return str(value.quantize(_USD_QUANTUM, rounding=ROUND_HALF_EVEN))
+
+
+def _window_end(session: SandboxSession) -> datetime:
+ assert session.ended_at is not None
+ return min(session.ended_at, session.ttl_expires_at)
+
+
+def _resource(session: SandboxSession) -> str:
+ if session.sandbox_backend == "hogland":
+ return "hogland"
+ return "sandbox-vm" if session.vm_runtime else "modal"
+
+
+def session_compute_cost(session: SandboxSession) -> SandboxComputeCost | None:
+ if not COMPUTE_RATE_CARDS or session.user_attributed_at is None or session.ended_at is None:
+ return None
+ start = COMPUTE_RATE_CARDS[0].effective_at
+ end = max(_window_end(session), start + timedelta(seconds=1))
+ return calculate_sandbox_compute_cost(
+ session, start, end, calculated_at=session.ended_at, rate_cards=COMPUTE_RATE_CARDS
+ )
+
+
+def _ai_product(run: TaskRun) -> str:
+ from products.tasks.backend.temporal.process_task.ai_gateway_token import ( # noqa: PLC0415 - the temporal package imports this module via cleanup_sandbox
+ resolve_sandbox_ai_product,
+ )
+
+ state = run.state or {}
+ profile = run.task.stored_profile
+ return resolve_sandbox_ai_product(
+ run.task.origin_product,
+ state.get("ai_stage"),
+ internal=run.task.internal,
+ billing_product=state.get("billing_product"),
+ profile_billing_product=profile.billing_product if profile else None,
+ )
+
+
+def build_settlement_body(session: SandboxSession, run: TaskRun, cost: SandboxComputeCost) -> dict[str, Any]:
+ assert session.user_attributed_at is not None
+ window_end = _window_end(session)
+ detail: dict[str, Any] = {
+ "rate_cards": [
+ {
+ "version": item.rate_card.version,
+ "seconds": _fixed6(item.billable_seconds),
+ "usd": _fixed6(item.total_cost_usd),
+ }
+ for item in cost.line_items
+ ],
+ "resource": _resource(session),
+ "cpu_cores": session.cpu_cores,
+ "memory_gb": session.memory_gb,
+ "attributed_seconds": round(max((window_end - session.user_attributed_at).total_seconds(), 0.0), 3),
+ "window_start": _iso(session.user_attributed_at),
+ "window_end": _iso(window_end),
+ "ended_reason": session.ended_reason or "",
+ "sandbox_id": session.sandbox_id,
+ }
+ body: dict[str, Any] = {
+ "reference": f"compute:{session.id}",
+ "amount_usd": _fixed6(cost.total_cost_usd),
+ "charge_kind": "compute",
+ "product": _ai_product(run),
+ }
+ actor = get_task_run_credential_user(run.task, run.state)
+ if actor is not None:
+ body["user"] = get_actor_distinct_id(actor)
+ body.update(
+ {
+ "obo": str(session.team_id),
+ "run": str(session.task_run_id),
+ "billable": is_billable_sandbox_session(session),
+ "detail": detail,
+ "reason": "sandbox compute",
+ }
+ )
+ return body
+
+
+def wallet_team_id() -> int:
+ mint_key = settings.SANDBOX_AI_GATEWAY_MINT_KEY
+ record = find_project_secret_api_key(mint_key) if mint_key else None
+ if record is None:
+ raise ComputeSettlementDeploymentError("SANDBOX_AI_GATEWAY_MINT_KEY does not resolve to a wallet team")
+ return record.team_id
+
+
+_METRIC_LABELS: dict[SettlementOutcome, ComputeSettlementOutcome] = {
+ SettlementOutcome.SKIPPED: "skipped",
+ SettlementOutcome.SETTLED: "settled",
+ SettlementOutcome.DUPLICATE: "duplicate",
+ SettlementOutcome.SETTLED_LOCALLY: "settled_locally",
+ SettlementOutcome.CONFLICT: "conflict",
+ SettlementOutcome.RETRY: "retry",
+ SettlementOutcome.ATTEMPTS_EXHAUSTED: "attempts_exhausted",
+ SettlementOutcome.CONTRACT_ERROR: "contract_error",
+ SettlementOutcome.CONTRACT_ERROR_REPEAT: "contract_error",
+}
+
+
+def _record(outcome: SettlementOutcome) -> SettlementOutcome:
+ label = _METRIC_LABELS.get(outcome)
+ if label is not None:
+ observe_compute_settlement(label)
+ return outcome
+
+
+def _settle_locally(session: SandboxSession) -> SettlementOutcome:
+ _unsettled(_session_rows(session)).update(
+ compute_settled_at=timezone.now(), compute_settled_usd=Decimal(0), compute_settle_error=""
+ )
+ return _record(SettlementOutcome.SETTLED_LOCALLY)
+
+
+def _error_text(response: ReportedSpendResponse) -> str:
+ body = response.body or {}
+ message = body.get("message")
+ code = body.get("code")
+ if isinstance(message, str) and message:
+ detail = f"{code}: {message}" if isinstance(code, str) and code else message
+ else:
+ detail = response.text
+ return f"HTTP {response.status_code}: {detail}"[:_ERROR_MAX_LENGTH]
+
+
+def _mark_conflict(session: SandboxSession, error: str, reason: ParkReason) -> None:
+ _unsettled(_session_rows(session)).update(
+ compute_settle_conflict_at=timezone.now(), compute_settle_error=error, compute_settle_park_reason=reason
+ )
+
+
+def _mark_malformed(session: SandboxSession) -> SettlementOutcome:
+ rows = _unsettled(_session_rows(session))
+ rows.update(compute_settle_attempts=F("compute_settle_attempts") + 1)
+ attempts = rows.values_list("compute_settle_attempts", flat=True).first()
+ if attempts is not None and attempts >= MAX_SETTLE_ATTEMPTS:
+ _mark_conflict(session, ATTEMPTS_EXHAUSTED_ERROR, ParkReason.ATTEMPTS_EXHAUSTED)
+ logger.error("compute_settlement.attempts_exhausted", session_id=str(session.id), attempts=attempts)
+ return _record(SettlementOutcome.ATTEMPTS_EXHAUSTED)
+ return _record(SettlementOutcome.RETRY)
+
+
+def _mark_contract_error(session: SandboxSession, response: ReportedSpendResponse) -> SettlementOutcome:
+ error = _error_text(response)
+ rows = _unsettled(_session_rows(session))
+ stored = rows.values_list("compute_settle_error", flat=True).first()
+ rows.update(compute_settle_error=error)
+ logger.error("compute_settlement.contract_error", session_id=str(session.id), error=error)
+ if stored == error:
+ return _record(SettlementOutcome.CONTRACT_ERROR_REPEAT)
+ return _record(SettlementOutcome.CONTRACT_ERROR)
+
+
+def _mark_settled(session: SandboxSession, response: ReportedSpendResponse) -> SettlementOutcome:
+ body = response.body or {}
+ entry_id = body.get("entry_id")
+ try:
+ amount = Decimal(str(body.get("amount_usd")))
+ except InvalidOperation:
+ amount = None
+ if not isinstance(entry_id, str) or not entry_id or amount is None or not amount.is_finite():
+ logger.warning("compute_settlement.malformed_response", session_id=str(session.id))
+ return _mark_malformed(session)
+ _unsettled(_session_rows(session)).update(
+ compute_settled_at=timezone.now(),
+ compute_settled_usd=amount,
+ compute_ledger_entry_id=entry_id[:64],
+ compute_settle_error="",
+ )
+ return _record(SettlementOutcome.DUPLICATE if body.get("duplicate") is True else SettlementOutcome.SETTLED)
+
+
+def settle_sandbox_session(session: SandboxSession) -> SettlementOutcome:
+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:
+ return SettlementOutcome.DISABLED
+ if (
+ session.ended_at is None
+ or session.compute_settled_at is not None
+ or session.compute_settle_conflict_at is not None
+ ):
+ return _record(SettlementOutcome.SKIPPED)
+ if session.user_attributed_at is None:
+ return _settle_locally(session)
+ cost = session_compute_cost(session)
+ if cost is None or _fixed6(cost.total_cost_usd) == _fixed6(Decimal(0)):
+ return _settle_locally(session)
+
+ body = session.compute_settle_body
+ rows = _unsettled(_session_rows(session))
+ if body:
+ rows.update(compute_settle_attempted_at=timezone.now())
+ else:
+ run = TaskRun.objects.select_related("task", "task__created_by").get(
+ id=session.task_run_id, team_id=session.team_id
+ )
+ built = build_settlement_body(session, run, cost)
+ rows.update(compute_settle_body=built, compute_settle_attempted_at=timezone.now())
+ body = _session_rows(session).values_list("compute_settle_body", flat=True).first() or built
+ try:
+ response = report_spend(wallet_team_id(), body)
+ except AIGatewayNotConfigured as error:
+ observe_compute_settlement("deployment_error")
+ logger.exception("compute_settlement.not_configured", session_id=str(session.id))
+ raise ComputeSettlementDeploymentError(str(error)) from error
+ except httpx.HTTPError:
+ logger.warning("compute_settlement.transport_error", session_id=str(session.id), exc_info=True)
+ return _record(SettlementOutcome.RETRY)
+
+ status = response.status_code
+ if status == 200:
+ return _mark_settled(session, response)
+ if status in _DEPLOYMENT_STATUSES:
+ observe_compute_settlement("deployment_error")
+ logger.error("compute_settlement.deployment_error", session_id=str(session.id), status=status)
+ raise ComputeSettlementDeploymentError(f"gateway refused the compute debit with HTTP {status}")
+ if status >= 500 or status in _RETRYABLE_STATUSES:
+ logger.warning("compute_settlement.retryable_error", session_id=str(session.id), status=status)
+ return _record(SettlementOutcome.RETRY)
+ if status == 400:
+ return _mark_contract_error(session, response)
+ if status == 409:
+ logger.error("compute_settlement.conflict", session_id=str(session.id), status=status)
+ _mark_conflict(session, _error_text(response), ParkReason.HTTP_409)
+ else:
+ logger.error("compute_settlement.unexpected_status", session_id=str(session.id), status=status)
+ _mark_conflict(session, _error_text(response), ParkReason.UNEXPECTED_STATUS)
+ return _record(SettlementOutcome.CONFLICT)
+
+
+def settle_task_run_compute(task_run_id: UUID | str, team_id: int) -> list[SettlementOutcome]:
+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:
+ return []
+ run_uuid = task_run_id if isinstance(task_run_id, UUID) else UUID(task_run_id)
+ sessions = _unsettled(SandboxSession.objects.for_team(team_id).filter(task_run_id=run_uuid, ended_at__isnull=False))
+ return [settle_sandbox_session(session) for session in sessions]
+
+
+def _overdue_ended(now: datetime, batch_size: int, processed: set[UUID]) -> list[SandboxSession]:
+ return list(
+ _unsettled(SandboxSession.objects.unscoped())
+ .filter(user_attributed_at__isnull=False, ended_at__lt=now - ENDED_GRACE)
+ .exclude(id__in=processed)
+ .order_by(F("compute_settle_attempted_at").asc(nulls_first=True), "ended_at")[:batch_size]
+ )
+
+
+def _overdue_open_modal(now: datetime, batch_size: int, processed: set[UUID]) -> list[SandboxSession]:
+ return list(
+ _unsettled(SandboxSession.objects.unscoped())
+ .filter(user_attributed_at__isnull=False, ended_at__isnull=True, ttl_expires_at__lt=now - TTL_GRACE)
+ .exclude(sandbox_backend="hogland")
+ .exclude(id__in=processed)
+ .order_by("ttl_expires_at")[:batch_size]
+ )
+
+
+def _refresh_hogland_open_gauge(now: datetime) -> None:
+ count = (
+ SandboxSession.objects.unscoped()
+ .filter(sandbox_backend="hogland", ended_at__isnull=True, created_at__lt=now - HOGLAND_OPEN_ALERT_AGE)
+ .count()
+ )
+ set_hogland_sandbox_sessions_open_over_day(count)
+
+
+def _park_repeated_contract_errors(
+ results: list[tuple[SandboxSession, SettlementOutcome]],
+) -> list[SettlementOutcome]:
+ contract_ids = [session.id for session, outcome in results if outcome in _CONTRACT_ERROR_OUTCOMES]
+ if not contract_ids:
+ return [outcome for _, outcome in results]
+ errors: dict[UUID, str] = dict(
+ SandboxSession.objects.unscoped().filter(id__in=contract_ids).values_list("id", "compute_settle_error")
+ )
+ if not any(outcome in _SETTLED_OUTCOMES for _, outcome in results):
+ shared = Counter(errors[session_id] for session_id in contract_ids if errors.get(session_id)).most_common(1)
+ if shared and shared[0][1] >= 2:
+ error, count = shared[0]
+ observe_compute_settlement("deployment_error")
+ raise ComputeSettlementDeploymentError(
+ f"gateway rejected {count} compute debits with the same error: {error}"
+ )
+ final: list[SettlementOutcome] = []
+ for session, outcome in results:
+ if outcome == SettlementOutcome.CONTRACT_ERROR_REPEAT:
+ _mark_conflict(session, errors.get(session.id, ""), ParkReason.HTTP_400)
+ outcome = _record(SettlementOutcome.CONFLICT)
+ final.append(outcome)
+ return final
+
+
+def settle_overdue_sandbox_compute(
+ now: datetime, batch_size: int = 50, time_budget: timedelta = timedelta(minutes=15)
+) -> list[SettlementOutcome]:
+ _refresh_hogland_open_gauge(now)
+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:
+ return []
+ deadline = time.monotonic() + time_budget.total_seconds()
+ results: list[tuple[SandboxSession, SettlementOutcome]] = []
+ processed: set[UUID] = set()
+ try:
+ while True:
+ batch = _overdue_ended(now, batch_size, processed)
+ for session in batch:
+ processed.add(session.id)
+ results.append((session, _settle_safely(session)))
+ if len(batch) < batch_size or time.monotonic() >= deadline:
+ break
+ while True:
+ batch = _overdue_open_modal(now, batch_size, processed)
+ for session in batch:
+ processed.add(session.id)
+ close_sandbox_session(session.sandbox_id, reason=SandboxSession.EndedReason.TTL_SWEEP)
+ closed = SandboxSession.objects.unscoped().filter(id=session.id).first()
+ if closed is not None and closed.ended_at is not None:
+ results.append((closed, _settle_safely(closed)))
+ if len(batch) < batch_size or time.monotonic() >= deadline:
+ break
+ return _park_repeated_contract_errors(results)
+ except ComputeSettlementDeploymentError as error:
+ logger.exception("compute_settlement.sweep_stopped", error=str(error))
+ return [outcome for _, outcome in results]
+
+
+def _settle_safely(session: SandboxSession) -> SettlementOutcome:
+ try:
+ return settle_sandbox_session(session)
+ except ComputeSettlementDeploymentError:
+ raise
+ except Exception:
+ logger.exception("compute_settlement.settle_failed", session_id=str(session.id))
+ return SettlementOutcome.SKIPPED
+
+
+def unpark_sandbox_compute(reason: str, since: datetime, until: datetime) -> int:
+ reset: dict[str, Any] = {
+ "compute_settle_conflict_at": None,
+ "compute_settle_park_reason": "",
+ "compute_settle_error": "",
+ "compute_settle_attempts": 0,
+ "compute_settle_attempted_at": None,
+ }
+ if reason == ParkReason.HTTP_400:
+ reset["compute_settle_body"] = {}
+ return (
+ SandboxSession.objects.unscoped()
+ .filter(
+ compute_settled_at__isnull=True,
+ compute_settle_conflict_at__gte=since,
+ compute_settle_conflict_at__lte=until,
+ compute_settle_park_reason=reason,
+ )
+ .update(**reset)
+ )
+
+
+@frozen
+class TeamComputeDelta:
+ team_id: str
+ local_usd: Decimal
+ gateway_usd: Decimal
+ local_rows: int
+
+ @property
+ def delta_usd(self) -> Decimal:
+ return self.local_usd - self.gateway_usd
+
+
+@frozen
+class ComputeReconciliation:
+ writer_parity: tuple[TeamComputeDelta, ...]
+ pricing_parity: tuple[TeamComputeDelta, ...]
+ billing_parity: tuple[TeamComputeDelta, ...]
+
+
+def _report_column(rows: list[dict[str, Any]], column: str) -> dict[str, Decimal]:
+ totals: dict[str, Decimal] = {}
+ for row in rows:
+ keys = row.get("keys")
+ obo = str((keys if isinstance(keys, dict) else row).get("obo") or "")
+ try:
+ value = Decimal(str(row.get(column) or "0"))
+ except InvalidOperation:
+ value = Decimal(0)
+ totals[obo] = totals.get(obo, Decimal(0)) + value
+ return totals
+
+
+def _deltas(
+ local: dict[str, Decimal], counts: dict[str, int], gateway: dict[str, Decimal]
+) -> tuple[TeamComputeDelta, ...]:
+ return tuple(
+ TeamComputeDelta(
+ team_id=team_id,
+ local_usd=local.get(team_id, Decimal(0)),
+ gateway_usd=gateway.get(team_id, Decimal(0)),
+ local_rows=counts.get(team_id, 0),
+ )
+ for team_id in sorted(set(local) | set(gateway))
+ )
+
+
+def reconcile_compute_settlement(since: datetime, until: datetime, wallet_team: int) -> ComputeReconciliation:
+ rows = get_usage_report(
+ {
+ "team_id": str(wallet_team),
+ "group_by": "obo,charge_kind",
+ "charge_kind": "compute",
+ "from": since.isoformat(),
+ "to": until.isoformat(),
+ }
+ )
+ gateway_compute = _report_column(rows, "compute_usd")
+
+ settled_usd: dict[str, Decimal] = {}
+ settled_rows: dict[str, int] = {}
+ for row in (
+ SandboxSession.objects.unscoped()
+ .filter(compute_settled_at__gte=since, compute_settled_at__lt=until)
+ .exclude(compute_ledger_entry_id="")
+ .order_by()
+ .values("team_id")
+ .annotate(total=Sum("compute_settled_usd"), rows=Count("id"))
+ ):
+ key = str(row["team_id"])
+ settled_usd[key] = row["total"] or Decimal(0)
+ settled_rows[key] = row["rows"]
+
+ priced_usd: dict[str, Decimal] = {}
+ priced_rows: dict[str, int] = {}
+ for session in (
+ SandboxSession.objects.unscoped()
+ .filter(user_attributed_at__isnull=False, ended_at__gte=since, ended_at__lt=until)
+ .iterator()
+ ):
+ cost = session_compute_cost(session)
+ key = str(session.team_id)
+ priced_usd[key] = priced_usd.get(key, Decimal(0)) + (cost.total_cost_usd if cost else Decimal(0))
+ priced_rows[key] = priced_rows.get(key, 0) + 1
+
+ billing = get_billable_sandbox_compute_usage_by_team(since, until)
+ billed_usd = {str(team_id): Decimal(credits) / 100 for team_id, credits in billing.credits}
+
+ prorated_usd: dict[str, Decimal] = {}
+ prorated_rows: dict[str, int] = {}
+ calculated_at = timezone.now()
+ for session in (
+ SandboxSession.objects.unscoped()
+ .filter(compute_settled_usd__isnull=False, user_attributed_at__lt=until, ended_at__gt=since)
+ .exclude(compute_ledger_entry_id="")
+ .iterator()
+ ):
+ if not is_billable_sandbox_session(session):
+ continue
+ lifetime = session_compute_cost(session)
+ if lifetime is None or lifetime.total_cost_usd == 0:
+ continue
+ clipped = calculate_sandbox_compute_cost(
+ session, since, until, calculated_at=calculated_at, rate_cards=COMPUTE_RATE_CARDS
+ ).total_cost_usd
+ assert session.compute_settled_usd is not None
+ key = str(session.team_id)
+ prorated_usd[key] = (
+ prorated_usd.get(key, Decimal(0)) + session.compute_settled_usd * clipped / lifetime.total_cost_usd
+ )
+ prorated_rows[key] = prorated_rows.get(key, 0) + 1
+
+ return ComputeReconciliation(
+ writer_parity=_deltas(settled_usd, settled_rows, gateway_compute),
+ pricing_parity=_deltas(priced_usd, priced_rows, gateway_compute),
+ billing_parity=_deltas(billed_usd, prorated_rows, prorated_usd),
+ )
diff --git a/products/tasks/backend/logic/services/gateway_usage.py b/products/tasks/backend/logic/services/gateway_usage.py
index 9e8bf9c46b61..24b825eaa932 100644
--- a/products/tasks/backend/logic/services/gateway_usage.py
+++ b/products/tasks/backend/logic/services/gateway_usage.py
@@ -3,7 +3,7 @@ import time
import asyncio
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
-from datetime import timedelta
+from datetime import datetime, timedelta
from decimal import ROUND_HALF_EVEN, Decimal
from functools import lru_cache
from typing import TYPE_CHECKING, Any
@@ -280,19 +280,19 @@ def _compute_cost_source(run: TaskRun, *, sessions: list[SandboxSession] | None
return None
now = timezone.now()
try:
- return sum(
- (
- calculate_sandbox_compute_cost(
- session, COMPUTE_RATE_CARDS[0].effective_at, now, calculated_at=now, rate_cards=COMPUTE_RATE_CARDS
- ).total_cost_usd
- for session in sessions
- ),
- Decimal(0),
- )
+ return sum((_session_compute_cost(session, now) for session in sessions), Decimal(0))
except ValueError:
return None
+def _session_compute_cost(session: SandboxSession, now: datetime) -> Decimal:
+ if session.compute_settled_at is not None and session.compute_settled_usd is not None:
+ return session.compute_settled_usd
+ return calculate_sandbox_compute_cost(
+ session, COMPUTE_RATE_CARDS[0].effective_at, now, calculated_at=now, rate_cards=COMPUTE_RATE_CARDS
+ ).total_cost_usd
+
+
def _cents(value: Decimal) -> int:
return int((value * 100).to_integral_value(rounding=ROUND_HALF_EVEN))
diff --git a/products/tasks/backend/logic/services/sandbox_usage.py b/products/tasks/backend/logic/services/sandbox_usage.py
index 1a8f263caf64..29bba05a9b2f 100644
--- a/products/tasks/backend/logic/services/sandbox_usage.py
+++ b/products/tasks/backend/logic/services/sandbox_usage.py
@@ -27,7 +27,7 @@ import structlog
from posthog.dataclasses import frozen
-from products.tasks.backend.logic.services.compute_quota import is_billable_compute
+from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_session
from products.tasks.backend.logic.services.sandbox import SandboxBase, SandboxConfig, get_sandbox_class_for_sandbox_id
from products.tasks.backend.logic.services.sandbox_pricing import (
COMPUTE_RATE_CARDS,
@@ -325,7 +325,7 @@ def get_billable_sandbox_compute_usage_by_team(
usage: dict[int, list[Decimal]] = {}
calculated_at = timezone.now()
for session in sessions.iterator():
- if not is_billable_compute(origin_product=session.origin_product, client_provenance=session.client_provenance):
+ if not is_billable_sandbox_session(session):
continue
cost = calculate_sandbox_compute_cost(session, begin, end, calculated_at=calculated_at, rate_cards=cards)
totals = usage.setdefault(session.team_id, [Decimal(0) for _ in range(3)])
diff --git a/products/tasks/backend/logic/services/tests/test_compute_settlement.py b/products/tasks/backend/logic/services/tests/test_compute_settlement.py
new file mode 100644
index 000000000000..905ab3d5665f
--- /dev/null
+++ b/products/tasks/backend/logic/services/tests/test_compute_settlement.py
@@ -0,0 +1,454 @@
+import json
+from datetime import timedelta
+from decimal import Decimal
+from typing import Any
+
+import pytest
+from posthog.test.base import BaseTest
+from unittest.mock import MagicMock, patch
+
+from django.test import override_settings
+from django.utils import timezone
+
+import httpx
+from parameterized import parameterized
+
+from posthog.models.project_secret_api_key import ProjectSecretAPIKey
+from posthog.models.utils import hash_key_value
+
+from products.tasks.backend.logic.services.compute_settlement import (
+ ComputeSettlementDeploymentError,
+ SettlementOutcome,
+ TeamComputeDelta,
+ reconcile_compute_settlement,
+ settle_overdue_sandbox_compute,
+ settle_sandbox_session,
+ settle_task_run_compute,
+ unpark_sandbox_compute,
+)
+from products.tasks.backend.management.commands.reconcile_sandbox_compute_settlement import CENT_ROUNDING_TOLERANCE_USD
+from products.tasks.backend.models import SandboxSession, Task, TaskClientProvenance, TaskRun
+
+MINT_KEY = "phs_" + "c" * 35
+GATEWAY_URL = "http://gateway.internal"
+HTTPX_POST = "posthog.llm.gateway_internal_client.httpx.post"
+
+
+def _response(status: int, body: dict[str, Any] | None = None) -> httpx.Response:
+ return httpx.Response(status, json=body or {}, request=httpx.Request("POST", GATEWAY_URL))
+
+
+def _ok(*, amount: str = "1.500000", entry_id: str = "entry-1", duplicate: bool = False) -> httpx.Response:
+ return _response(200, {"team_id": 1, "entry_id": entry_id, "amount_usd": amount, "duplicate": duplicate})
+
+
+@override_settings(
+ GATEWAY_COMPUTE_DEBIT_ENABLED=True,
+ SANDBOX_AI_GATEWAY_MINT_KEY=MINT_KEY,
+ AI_GATEWAY_INTERNAL_URL=GATEWAY_URL,
+ AI_GATEWAY_INTERNAL_TOKEN="operator-token",
+)
+class TestComputeSettlement(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.now = timezone.now()
+ ProjectSecretAPIKey.objects.create(
+ team=self.team, label="mint", secure_value=hash_key_value(MINT_KEY), scopes=["llm_gateway:read"]
+ )
+
+ def _session(self, *, with_actor: bool = True, **overrides: Any) -> SandboxSession:
+ task = Task.objects.create(
+ team=self.team,
+ title="t",
+ description="",
+ origin_product=Task.OriginProduct.USER_CREATED,
+ client_provenance=TaskClientProvenance.POSTHOG_DESKTOP,
+ created_by=self.user if with_actor else None,
+ )
+ run = TaskRun.objects.create(task=task, team=self.team, environment=TaskRun.Environment.CLOUD)
+ created_at = overrides.pop("created_at", self.now - timedelta(hours=2))
+ defaults: dict[str, Any] = {
+ "team": self.team,
+ "task_run": run,
+ "sandbox_id": f"sb-{SandboxSession.objects.unscoped().count()}",
+ "origin_product": Task.OriginProduct.USER_CREATED,
+ "client_provenance": TaskClientProvenance.POSTHOG_DESKTOP,
+ "cpu_cores": 4.0,
+ "memory_gb": 16.0,
+ "ttl_seconds": 6 * 60 * 60,
+ "created_at": created_at,
+ "ttl_expires_at": created_at + timedelta(hours=6),
+ "user_attributed_at": self.now - timedelta(hours=2),
+ "ended_at": self.now - timedelta(hours=1),
+ "ended_reason": SandboxSession.EndedReason.CLEANUP,
+ }
+ defaults.update(overrides)
+ return SandboxSession.objects.unscoped().create(**defaults)
+
+ def _reload(self, session: SandboxSession) -> SandboxSession:
+ return SandboxSession.objects.unscoped().get(id=session.id)
+
+ @parameterized.expand([(False, SettlementOutcome.SETTLED), (True, SettlementOutcome.DUPLICATE)])
+ def test_success_stores_the_gateway_amount_and_entry(self, duplicate: bool, expected: SettlementOutcome) -> None:
+ session = self._session()
+ with patch(HTTPX_POST, return_value=_ok(duplicate=duplicate)) as post:
+ assert settle_sandbox_session(session) == expected
+
+ assert post.call_args.args[0] == f"{GATEWAY_URL}/internal/teams/{self.team.id}/reported-spend"
+ assert post.call_args.kwargs["headers"]["X-Internal-Actor"] == "tasks-compute"
+ settled = self._reload(session)
+ assert settled.compute_settled_at is not None
+ assert settled.compute_settled_usd == Decimal("1.500000")
+ assert settled.compute_ledger_entry_id == "entry-1"
+
+ def test_body_shape_is_clamped_and_byte_identical_across_retries(self) -> None:
+ session = self._session(
+ with_actor=False,
+ created_at=self.now - timedelta(hours=7),
+ user_attributed_at=self.now - timedelta(hours=6, minutes=30),
+ ttl_expires_at=self.now - timedelta(hours=6),
+ ended_at=self.now - timedelta(hours=1),
+ )
+ with patch(HTTPX_POST, side_effect=[httpx.ConnectError("refused"), _response(503)]) as post:
+ settle_sandbox_session(session)
+ Task.objects.filter(runs__id=session.task_run_id).update(created_by=self.user)
+ settle_sandbox_session(self._reload(session))
+
+ first, second = (json.dumps(call.kwargs["json"]) for call in post.call_args_list)
+ assert first == second
+ body = post.call_args_list[0].kwargs["json"]
+ assert "user" not in body
+ assert body["reference"] == f"compute:{session.id}"
+ assert body["obo"] == str(self.team.id)
+ assert body["run"] == str(session.task_run_id)
+ assert body["billable"] is True
+ assert body["charge_kind"] == "compute"
+ assert body["product"] == "posthog_code"
+ assert body["amount_usd"] == "0.770400"
+ detail = body["detail"]
+ assert detail["window_end"] == session.ttl_expires_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+ assert detail["attributed_seconds"] == 1800.0
+ assert detail["resource"] == "modal"
+ assert [card["version"] for card in detail["rate_cards"]] == ["v1"]
+ assert detail["rate_cards"][0]["usd"] == "0.770400"
+ assert len(json.dumps(detail).encode()) <= 2048
+
+ def test_body_pins_the_acting_user(self) -> None:
+ session = self._session()
+ with patch(HTTPX_POST, return_value=_ok()) as post:
+ settle_sandbox_session(session)
+
+ assert post.call_args.kwargs["json"]["user"] == self.user.distinct_id
+
+ @parameterized.expand(
+ [
+ ("conflict", _response(409, {"error": "reference reused"}), SettlementOutcome.CONFLICT, [], "http_409"),
+ ("unexpected", _response(418), SettlementOutcome.CONFLICT, [], "unexpected_status"),
+ (
+ "validation",
+ _response(400, {"error": "bad"}),
+ SettlementOutcome.CONTRACT_ERROR,
+ [SettlementOutcome.CONTRACT_ERROR_REPEAT],
+ "",
+ ),
+ (
+ "validation_request_id",
+ [
+ _response(400, {"code": "bad_request", "message": "pool_type must be prepaid", "request_id": "a1"}),
+ _response(400, {"code": "bad_request", "message": "pool_type must be prepaid", "request_id": "b2"}),
+ ],
+ SettlementOutcome.CONTRACT_ERROR,
+ [SettlementOutcome.CONTRACT_ERROR_REPEAT],
+ "",
+ ),
+ ("unavailable", _response(503), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),
+ ("timeout", _response(408), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),
+ ("rate_limited", _response(429), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),
+ ("transport", httpx.ConnectError("refused"), SettlementOutcome.RETRY, [SettlementOutcome.RETRY], ""),
+ ]
+ )
+ def test_failure_statuses(
+ self,
+ _name: str,
+ result: Any,
+ expected: SettlementOutcome,
+ again: list[SettlementOutcome],
+ park_reason: str,
+ ) -> None:
+ session = self._session()
+ mock = MagicMock(side_effect=result) if isinstance(result, Exception | list) else MagicMock(return_value=result)
+ with patch(HTTPX_POST, mock):
+ assert settle_sandbox_session(session) == expected
+ assert settle_task_run_compute(session.task_run_id, self.team.id) == again
+
+ row = self._reload(session)
+ assert row.compute_settled_at is None
+ assert (row.compute_settle_conflict_at is not None) is bool(park_reason)
+ assert row.compute_settle_park_reason == park_reason
+ assert row.compute_settle_attempts == 0
+ assert row.compute_settle_attempted_at is not None
+ assert mock.call_count == 1 + len(again)
+ transient = expected == SettlementOutcome.RETRY
+ first = result[0] if isinstance(result, list) else result
+ assert row.compute_settle_error.startswith("" if transient else f"HTTP {first.status_code}: ")
+ assert (row.compute_settle_error == "") is transient
+ if isinstance(result, list):
+ assert row.compute_settle_error == "HTTP 400: bad_request: pool_type must be prepaid"
+
+ def test_retry_cap_excludes_the_row(self) -> None:
+ session = self._session()
+ with patch(HTTPX_POST, return_value=_response(200, {"duplicate": False})):
+ outcomes = [settle_sandbox_session(self._reload(session)) for _ in range(20)]
+
+ assert outcomes == [SettlementOutcome.RETRY] * 19 + [SettlementOutcome.ATTEMPTS_EXHAUSTED]
+ row = self._reload(session)
+ assert row.compute_settle_error == "attempts_exhausted"
+ assert row.compute_settle_park_reason == "attempts_exhausted"
+ assert row.compute_settle_conflict_at is not None
+ assert row.compute_settled_at is None
+
+ def test_unpark_clears_only_the_requested_reason_in_the_window(self) -> None:
+ parked_at = self.now - timedelta(hours=1)
+ frozen_body = {"reference": "compute:frozen", "amount_usd": "1.000000"}
+ rejected = self._session(
+ compute_settle_conflict_at=parked_at,
+ compute_settle_park_reason="http_400",
+ compute_settle_error="HTTP 400: bad",
+ compute_settle_attempts=3,
+ compute_settle_body=frozen_body,
+ compute_settle_attempted_at=parked_at,
+ )
+ exhausted = self._session(
+ compute_settle_conflict_at=parked_at,
+ compute_settle_park_reason="attempts_exhausted",
+ compute_settle_error="attempts_exhausted",
+ compute_settle_attempts=20,
+ compute_settle_body=frozen_body,
+ compute_settle_attempted_at=parked_at,
+ )
+ reused = self._session(
+ compute_settle_conflict_at=parked_at,
+ compute_settle_park_reason="http_409",
+ compute_settle_error="HTTP 409: reused",
+ )
+ older = self._session(
+ compute_settle_conflict_at=self.now - timedelta(days=3),
+ compute_settle_park_reason="http_400",
+ compute_settle_error="HTTP 400: bad",
+ )
+
+ assert unpark_sandbox_compute("http_400", self.now - timedelta(days=1), self.now) == 1
+
+ row = self._reload(rejected)
+ assert row.compute_settle_conflict_at is None
+ assert row.compute_settle_park_reason == ""
+ assert row.compute_settle_error == ""
+ assert row.compute_settle_attempts == 0
+ assert row.compute_settle_body == {}
+ assert row.compute_settle_attempted_at is None
+ assert self._reload(reused).compute_settle_park_reason == "http_409"
+ assert self._reload(older).compute_settle_conflict_at is not None
+ assert self._reload(exhausted).compute_settle_park_reason == "attempts_exhausted"
+
+ assert unpark_sandbox_compute("attempts_exhausted", self.now - timedelta(days=1), self.now) == 1
+
+ row = self._reload(exhausted)
+ assert row.compute_settle_conflict_at is None
+ assert row.compute_settle_attempts == 0
+ assert row.compute_settle_body == frozen_body
+ assert row.compute_settle_attempted_at is None
+
+ def test_deployment_error_raises_and_leaves_the_row_untouched(self) -> None:
+ session = self._session()
+ with patch(HTTPX_POST, return_value=_response(403)), pytest.raises(ComputeSettlementDeploymentError):
+ settle_sandbox_session(session)
+
+ row = self._reload(session)
+ assert row.compute_settled_at is None
+ assert row.compute_settle_conflict_at is None
+ assert row.compute_settle_attempts == 0
+ assert row.compute_settle_error == ""
+
+ def test_flag_off_is_a_no_op(self) -> None:
+ session = self._session()
+ with override_settings(GATEWAY_COMPUTE_DEBIT_ENABLED=False), patch(HTTPX_POST) as post:
+ assert settle_sandbox_session(session) == SettlementOutcome.DISABLED
+ assert settle_task_run_compute(session.task_run_id, self.team.id) == []
+ assert settle_overdue_sandbox_compute(self.now) == []
+
+ post.assert_not_called()
+ assert self._reload(session).compute_settled_at is None
+
+ @parameterized.expand([("unattributed", True), ("zero_amount", False)])
+ def test_settles_locally_without_a_call(self, _name: str, unattributed: bool) -> None:
+ ended_at = self.now - timedelta(hours=1)
+ session = self._session(user_attributed_at=None if unattributed else ended_at, ended_at=ended_at)
+ with patch(HTTPX_POST) as post:
+ assert settle_sandbox_session(session) == SettlementOutcome.SETTLED_LOCALLY
+
+ post.assert_not_called()
+ row = self._reload(session)
+ assert row.compute_settled_at is not None
+ assert row.compute_settled_usd == Decimal(0)
+
+ def test_sweep_selects_overdue_rows_only(self) -> None:
+ straggler = self._session(ended_at=self.now - timedelta(minutes=30))
+ fresh = self._session(ended_at=self.now - timedelta(minutes=2))
+ expired_open = self._session(
+ ended_at=None,
+ ended_reason=None,
+ created_at=self.now - timedelta(hours=8),
+ user_attributed_at=self.now - timedelta(hours=7, minutes=30),
+ )
+ hogland_open = self._session(
+ ended_at=None,
+ ended_reason=None,
+ sandbox_backend="hogland",
+ created_at=self.now - timedelta(hours=8),
+ user_attributed_at=self.now - timedelta(hours=7, minutes=30),
+ )
+
+ with patch(HTTPX_POST, return_value=_ok()) as post:
+ outcomes = settle_overdue_sandbox_compute(self.now)
+ again = settle_overdue_sandbox_compute(self.now)
+
+ assert outcomes == [SettlementOutcome.SETTLED, SettlementOutcome.SETTLED]
+ assert again == []
+ assert post.call_count == 2
+ assert self._reload(straggler).compute_settled_at is not None
+ assert self._reload(fresh).compute_settled_at is None
+ closed = self._reload(expired_open)
+ assert closed.ended_reason == SandboxSession.EndedReason.TTL_SWEEP
+ assert closed.compute_settled_at is not None
+ untouched = self._reload(hogland_open)
+ assert untouched.ended_at is None
+ assert untouched.compute_settled_at is None
+
+ @parameterized.expand(
+ [("settles", _ok(), SettlementOutcome.SETTLED), ("retries", _response(503), SettlementOutcome.RETRY)]
+ )
+ def test_sweep_drains_every_batch_once(
+ self, _name: str, response: httpx.Response, expected: SettlementOutcome
+ ) -> None:
+ sessions = [self._session(ended_at=self.now - timedelta(minutes=20 + index)) for index in range(5)]
+ with patch(HTTPX_POST, return_value=response) as post:
+ assert settle_overdue_sandbox_compute(self.now, batch_size=2) == [expected] * 5
+
+ assert post.call_count == 5
+ for session in sessions:
+ row = self._reload(session)
+ assert (row.compute_settled_at is not None) is (expected == SettlementOutcome.SETTLED)
+ assert row.compute_settle_attempts == 0
+
+ def test_sweep_rotates_past_rows_it_already_retried(self) -> None:
+ oldest = self._session(ended_at=self.now - timedelta(minutes=40))
+ newer = self._session(ended_at=self.now - timedelta(minutes=30))
+ with patch(HTTPX_POST, return_value=_response(503)) as post:
+ first = settle_overdue_sandbox_compute(self.now, batch_size=1, time_budget=timedelta(0))
+ assert [call.kwargs["json"]["reference"] for call in post.call_args_list] == [f"compute:{oldest.id}"]
+ assert self._reload(oldest).compute_settle_attempted_at is not None
+ post.reset_mock()
+ second = settle_overdue_sandbox_compute(self.now, batch_size=1, time_budget=timedelta(0))
+ assert [call.kwargs["json"]["reference"] for call in post.call_args_list] == [f"compute:{newer.id}"]
+
+ assert first == [SettlementOutcome.RETRY]
+ assert second == [SettlementOutcome.RETRY]
+
+ def test_sweep_stops_on_a_deployment_error(self) -> None:
+ self._session(ended_at=self.now - timedelta(minutes=30))
+ self._session(ended_at=self.now - timedelta(minutes=40))
+ with patch(HTTPX_POST, return_value=_response(401)) as post:
+ assert settle_overdue_sandbox_compute(self.now) == []
+
+ assert post.call_count == 1
+
+ @parameterized.expand(
+ [
+ ("lone_repeat_parks", False, False, [SettlementOutcome.CONFLICT]),
+ (
+ "repeat_parks_beside_a_settled_row",
+ True,
+ True,
+ [SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.SETTLED, SettlementOutcome.CONFLICT],
+ ),
+ (
+ "shared_error_without_success_stops",
+ False,
+ True,
+ [SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.CONTRACT_ERROR_REPEAT],
+ ),
+ ]
+ )
+ def test_sweep_parks_repeated_contract_errors(
+ self, _name: str, with_success: bool, with_fresh_failure: bool, expected: list[SettlementOutcome]
+ ) -> None:
+ repeat = self._session(ended_at=self.now - timedelta(minutes=40))
+ fresh = self._session(ended_at=self.now - timedelta(minutes=30)) if with_fresh_failure else None
+ success = self._session(ended_at=self.now - timedelta(minutes=20)) if with_success else None
+ with patch(HTTPX_POST, return_value=_response(400, {"error": "bad"})):
+ assert settle_sandbox_session(repeat) == SettlementOutcome.CONTRACT_ERROR
+ stored = self._reload(repeat).compute_settle_error
+ assert stored.startswith("HTTP 400: ")
+
+ def respond(_url: str, **kwargs: Any) -> httpx.Response:
+ if success is not None and kwargs["json"]["reference"] == f"compute:{success.id}":
+ return _ok()
+ return _response(400, {"error": "bad"})
+
+ with (
+ patch(HTTPX_POST, side_effect=respond),
+ patch("products.tasks.backend.logic.services.compute_settlement.logger") as logger,
+ ):
+ assert settle_overdue_sandbox_compute(self.now) == expected
+
+ parks = SettlementOutcome.CONFLICT in expected
+ stopped = [
+ call for call in logger.exception.call_args_list if call.args[0] == "compute_settlement.sweep_stopped"
+ ]
+ assert bool(stopped) is not parks
+ row = self._reload(repeat)
+ assert row.compute_settle_error == stored
+ assert (row.compute_settle_conflict_at is not None) is parks
+ assert row.compute_settle_park_reason == ("http_400" if parks else "")
+ if fresh is not None:
+ fresh_row = self._reload(fresh)
+ assert fresh_row.compute_settle_conflict_at is None
+ assert fresh_row.compute_settle_error == stored
+ if success is not None:
+ assert self._reload(success).compute_settled_at is not None
+
+ def test_reconciliation_compares_per_customer_team(self) -> None:
+ since = self.now - timedelta(days=1)
+ self._session(
+ compute_settled_at=self.now - timedelta(minutes=5),
+ compute_settled_usd=Decimal("1.540800"),
+ compute_ledger_entry_id="entry-1",
+ )
+ self._session(
+ created_at=since - timedelta(hours=2),
+ user_attributed_at=since - timedelta(hours=1),
+ ended_at=since + timedelta(hours=1),
+ compute_settled_at=self.now - timedelta(minutes=5),
+ compute_settled_usd=Decimal("3.081600"),
+ compute_ledger_entry_id="entry-2",
+ )
+ rows = [
+ {"keys": {"obo": str(self.team.id)}, "compute_usd": "4.622400", "billable_compute_usd": "4.622400"},
+ {"keys": {"obo": "999"}, "compute_usd": "2.000000", "billable_compute_usd": "0"},
+ ]
+ report = httpx.Response(200, json={"rows": rows}, request=httpx.Request("GET", GATEWAY_URL))
+ with patch("posthog.llm.gateway_internal_client.httpx.get", return_value=report) as get:
+ result = reconcile_compute_settlement(since, self.now, self.team.id)
+
+ assert get.call_args.kwargs["params"]["group_by"] == "obo,charge_kind"
+ assert result.writer_parity == (
+ TeamComputeDelta(
+ team_id=str(self.team.id), local_usd=Decimal("4.622400"), gateway_usd=Decimal("4.622400"), local_rows=2
+ ),
+ TeamComputeDelta(team_id="999", local_usd=Decimal(0), gateway_usd=Decimal("2.000000"), local_rows=0),
+ )
+ assert result.pricing_parity[0].local_usd == Decimal("4.622400")
+ billing = result.billing_parity[0]
+ assert billing.team_id == str(self.team.id)
+ assert billing.local_rows == 2
+ assert abs(billing.delta_usd) <= CENT_ROUNDING_TOLERANCE_USD
diff --git a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py b/products/tasks/backend/logic/services/tests/test_sandbox_usage.py
index bc27997e2991..5deceb1d9df7 100644
--- a/products/tasks/backend/logic/services/tests/test_sandbox_usage.py
+++ b/products/tasks/backend/logic/services/tests/test_sandbox_usage.py
@@ -10,6 +10,7 @@ from parameterized import parameterized
from posthog.models.organization import Organization
from posthog.models.team.team import Team
+from products.tasks.backend.logic.services.compute_quota import is_billable_sandbox_session
from products.tasks.backend.logic.services.sandbox import Sandbox, SandboxConfig
from products.tasks.backend.logic.services.sandbox_pricing import ComputeRateCard, ComputeRateCardConfigurationError
from products.tasks.backend.logic.services.sandbox_usage import (
@@ -376,6 +377,29 @@ class TestSandboxUsageAggregation(SandboxUsageBase):
assert usage.memory_mib_seconds == [(self.team.id, 176_947_200)]
assert usage.credits == [(self.team.id, 6048)]
+ @parameterized.expand(
+ [
+ ("desktop_user_created", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.USER_CREATED, True),
+ ("desktop_loop", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.LOOP, True),
+ ("desktop_space_setup", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.SPACE_SETUP, True),
+ ("desktop_slack", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.SLACK, True),
+ ("no_provenance", None, Task.OriginProduct.USER_CREATED, True),
+ ("unattributed", TaskClientProvenance.POSTHOG_DESKTOP, Task.OriginProduct.USER_CREATED, False),
+ ]
+ )
+ def test_billable_session_predicate_matches_the_billing_aggregate(
+ self, _name: str, provenance: TaskClientProvenance | None, origin: str, attributed: bool
+ ) -> None:
+ session = self._session(
+ client_provenance=provenance,
+ origin_product=origin,
+ user_attributed_at=datetime(2026, 1, 2, 1, tzinfo=UTC) if attributed else None,
+ )
+
+ usage = get_billable_sandbox_compute_usage_by_team(self.BEGIN, self.END, rate_cards=(self._rate(),))
+
+ assert is_billable_sandbox_session(session) is bool(usage.credits)
+
def test_billable_compute_uses_session_snapshot_after_task_changes(self):
session = self._session(client_provenance=TaskClientProvenance.POSTHOG_DESKTOP)
Task.objects.filter(id=session.task_run.task_id).update(
diff --git a/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py
new file mode 100644
index 000000000000..5148c602cf82
--- /dev/null
+++ b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py
@@ -0,0 +1,78 @@
+from datetime import datetime, timedelta
+from decimal import Decimal
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from django.utils import timezone
+
+from products.tasks.backend.logic.services.compute_settlement import (
+ TeamComputeDelta,
+ reconcile_compute_settlement,
+ unpark_sandbox_compute,
+ wallet_team_id,
+)
+from products.tasks.backend.models import SandboxSession
+
+CENT_ROUNDING_TOLERANCE_USD = Decimal("0.005")
+
+
+class Command(BaseCommand):
+ help = (
+ "Report settled sandbox compute against the gateway ledger, per customer team. "
+ "With --unpark, first return rows parked for that reason in the window to the settlement sweep"
+ )
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--since", type=datetime.fromisoformat, default=None)
+ parser.add_argument("--until", type=datetime.fromisoformat, default=None)
+ parser.add_argument("--wallet-team-id", type=int, default=None)
+ parser.add_argument(
+ "--unpark",
+ choices=[
+ SandboxSession.ComputeSettleParkReason.ATTEMPTS_EXHAUSTED.value,
+ SandboxSession.ComputeSettleParkReason.HTTP_400.value,
+ ],
+ default=None,
+ help="Unpark rows parked for this reason within --since/--until so the sweep retries them",
+ )
+
+ def handle(self, *args: object, **options: object) -> None:
+ until = options["until"] or timezone.now()
+ if not isinstance(until, datetime):
+ raise CommandError("--until must be an ISO-8601 timestamp")
+ since = options["since"] or until - timedelta(days=7)
+ if not isinstance(since, datetime):
+ raise CommandError("--since must be an ISO-8601 timestamp")
+ if timezone.is_naive(since) or timezone.is_naive(until) or until <= since:
+ raise CommandError("--since and --until must be timezone-aware and --until must follow --since")
+ unpark = options["unpark"]
+ if isinstance(unpark, str):
+ count = unpark_sandbox_compute(unpark, since, until)
+ self.stdout.write(
+ f"Unparked {count} sessions parked for {unpark} between {since.isoformat()} and {until.isoformat()}"
+ )
+ wallet = options["wallet_team_id"]
+ wallet_team = wallet if isinstance(wallet, int) else wallet_team_id()
+
+ report = reconcile_compute_settlement(since, until, wallet_team)
+ self.stdout.write(f"Window {since.isoformat()} to {until.isoformat()}, wallet team {wallet_team}")
+ self._section("Writer parity: settled compute_settled_usd vs gateway compute_usd", report.writer_parity)
+ self._section(
+ "Pricing parity: attributed sessions ended in window, priced live, vs gateway compute_usd",
+ report.pricing_parity,
+ )
+ self._section(
+ "Billing parity: billing aggregate for the window vs settled ledger amounts prorated to the window",
+ report.billing_parity,
+ tolerance_usd=CENT_ROUNDING_TOLERANCE_USD,
+ )
+
+ def _section(self, title: str, rows: tuple[TeamComputeDelta, ...], *, tolerance_usd: Decimal = Decimal(0)) -> None:
+ self.stdout.write("")
+ self.stdout.write(title)
+ self.stdout.write(f"{'team':>12} {'local_usd':>14} {'gateway_usd':>14} {'delta_usd':>14} {'local_rows':>10}")
+ for row in rows:
+ line = (
+ f"{row.team_id:>12} {row.local_usd:>14.6f} {row.gateway_usd:>14.6f} "
+ f"{row.delta_usd:>14.6f} {row.local_rows:>10}"
+ )
+ self.stdout.write(self.style.WARNING(line) if abs(row.delta_usd) > tolerance_usd else line)
diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py
index 43f8d6252171..46fdef08be8d 100644
--- a/products/tasks/backend/metrics.py
+++ b/products/tasks/backend/metrics.py
@@ -474,6 +474,39 @@ def observe_compute_quota_check(outcome: ComputeQuotaOutcome) -> None:
COMPUTE_QUOTA_CHECK_TOTAL.labels(outcome=outcome).inc()
+ComputeSettlementOutcome = Literal[
+ "settled",
+ "duplicate",
+ "settled_locally",
+ "conflict",
+ "retry",
+ "attempts_exhausted",
+ "contract_error",
+ "deployment_error",
+ "skipped",
+]
+
+COMPUTE_SETTLEMENTS_TOTAL = Counter(
+ "posthog_tasks_compute_settlements_total",
+ "Sandbox compute settlement attempts against the gateway ledger, by outcome",
+ labelnames=["outcome"],
+)
+
+HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY = Gauge(
+ "posthog_tasks_hogland_sandbox_sessions_open_over_day",
+ "Hogland sandbox sessions with no end stamp that opened more than a day ago",
+ multiprocess_mode="mostrecent",
+)
+
+
+def observe_compute_settlement(outcome: ComputeSettlementOutcome) -> None:
+ COMPUTE_SETTLEMENTS_TOTAL.labels(outcome=outcome).inc()
+
+
+def set_hogland_sandbox_sessions_open_over_day(count: int) -> None:
+ HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY.set(count)
+
+
# scoped: "true" when the delivery's installation resolved to at least one team, so the
# TaskRun lookup could ride the team_id index. "false" means no team links the installation,
# so the lookup is skipped and the delivery matches no run.
diff --git a/products/tasks/backend/migrations/0145_sandboxsession_compute_settlement.py b/products/tasks/backend/migrations/0145_sandboxsession_compute_settlement.py
new file mode 100644
index 000000000000..7e48318fe41c
--- /dev/null
+++ b/products/tasks/backend/migrations/0145_sandboxsession_compute_settlement.py
@@ -0,0 +1,116 @@
+# Generated by Django 6.1.2 on 2026-11-02 12:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("tasks", "0144_task_options"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_ledger_entry_id",
+ field=models.CharField(
+ blank=True,
+ default="",
+ help_text="Gateway ledger entry id of the compute debit",
+ max_length=64,
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settle_attempted_at",
+ field=models.DateTimeField(
+ blank=True,
+ help_text="When the last reported-spend request was sent; the sweep tries never-attempted rows first",
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settle_attempts",
+ field=models.PositiveIntegerField(
+ default=0, help_text="Gateway 200 responses whose body could not be parsed so far"
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settle_body",
+ field=models.JSONField(
+ blank=True,
+ default=dict,
+ help_text="Reported-spend body frozen before the first attempt so retries replay it unchanged",
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settle_conflict_at",
+ field=models.DateTimeField(
+ blank=True,
+ help_text="When the gateway refused the debit for good; the sweep skips the row",
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settle_error",
+ field=models.CharField(
+ blank=True,
+ default="",
+ help_text="Last gateway error text; a 400 stays so the sweep can recognise a repeat",
+ max_length=200,
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settle_park_reason",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("http_400", "HTTP 400"),
+ ("http_409", "HTTP 409"),
+ ("attempts_exhausted", "Attempts exhausted"),
+ ("unexpected_status", "Unexpected status"),
+ ],
+ default="",
+ help_text="Why settlement stopped retrying the row; empty while the row is not parked",
+ max_length=32,
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settled_at",
+ field=models.DateTimeField(
+ blank=True,
+ help_text="When the session's compute was settled on the gateway ledger",
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="sandboxsession",
+ name="compute_settled_usd",
+ field=models.DecimalField(
+ blank=True,
+ decimal_places=6,
+ help_text="Compute amount the ledger recorded",
+ max_digits=20,
+ null=True,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="sandboxsession",
+ name="ended_reason",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("cleanup", "Cleanup"),
+ ("reaped", "Reaped"),
+ ("ttl_sweep", "TTL sweep"),
+ ],
+ max_length=20,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/products/tasks/backend/migrations/0146_sandboxsession_settle_indexes.py b/products/tasks/backend/migrations/0146_sandboxsession_settle_indexes.py
new file mode 100644
index 000000000000..e7b11caa27d2
--- /dev/null
+++ b/products/tasks/backend/migrations/0146_sandboxsession_settle_indexes.py
@@ -0,0 +1,39 @@
+from django.db import migrations, models
+
+from posthog.migration_helpers import SafeAddIndexConcurrently
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("tasks", "0145_sandboxsession_compute_settlement"),
+ ]
+
+ operations = [
+ SafeAddIndexConcurrently(
+ model_name="sandboxsession",
+ index=models.Index(
+ condition=models.Q(
+ ("compute_settle_conflict_at__isnull", True),
+ ("compute_settled_at__isnull", True),
+ ("user_attributed_at__isnull", False),
+ ),
+ fields=["ttl_expires_at"],
+ name="sandbox_session_settle_ttl_idx",
+ ),
+ ),
+ SafeAddIndexConcurrently(
+ model_name="sandboxsession",
+ index=models.Index(
+ models.OrderBy(models.F("compute_settle_attempted_at"), nulls_first=True),
+ models.F("ended_at"),
+ condition=models.Q(
+ ("compute_settle_conflict_at__isnull", True),
+ ("compute_settled_at__isnull", True),
+ ("user_attributed_at__isnull", False),
+ ),
+ name="sandbox_session_settle_rt_idx",
+ ),
+ ),
+ ]
diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txt
index 4434490791b7..eada7f0b5316 100644
--- a/products/tasks/backend/migrations/max_migration.txt
+++ b/products/tasks/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0144_task_options
+0146_sandboxsession_settle_indexes
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index 5352b5e77494..a7a1a78aca88 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -3785,6 +3785,13 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):
class EndedReason(models.TextChoices):
CLEANUP = "cleanup", "Cleanup"
REAPED = "reaped", "Reaped"
+ TTL_SWEEP = "ttl_sweep", "TTL sweep"
+
+ class ComputeSettleParkReason(models.TextChoices):
+ HTTP_400 = "http_400", "HTTP 400"
+ HTTP_409 = "http_409", "HTTP 409"
+ ATTEMPTS_EXHAUSTED = "attempts_exhausted", "Attempts exhausted"
+ UNEXPECTED_STATUS = "unexpected_status", "Unexpected status"
# db_constraint=False on the team FK: adding an FK constraint to that hot table
# locks it and stalls deploys; Django still enforces the relation and on_delete at
@@ -3863,6 +3870,44 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):
provider_usage_measured_at = models.DateTimeField(
null=True, blank=True, help_text="When provider resource usage was sampled"
)
+ compute_settled_at = models.DateTimeField(
+ null=True, blank=True, help_text="When the session's compute was settled on the gateway ledger"
+ )
+ compute_settled_usd = models.DecimalField(
+ max_digits=20, decimal_places=6, null=True, blank=True, help_text="Compute amount the ledger recorded"
+ )
+ compute_ledger_entry_id = models.CharField(
+ max_length=64, blank=True, default="", help_text="Gateway ledger entry id of the compute debit"
+ )
+ compute_settle_attempts = models.PositiveIntegerField(
+ default=0, help_text="Gateway 200 responses whose body could not be parsed so far"
+ )
+ compute_settle_attempted_at = models.DateTimeField(
+ null=True,
+ blank=True,
+ help_text="When the last reported-spend request was sent; the sweep tries never-attempted rows first",
+ )
+ compute_settle_conflict_at = models.DateTimeField(
+ null=True, blank=True, help_text="When the gateway refused the debit for good; the sweep skips the row"
+ )
+ compute_settle_error = models.CharField(
+ max_length=200,
+ blank=True,
+ default="",
+ help_text="Last gateway error text; a 400 stays so the sweep can recognise a repeat",
+ )
+ compute_settle_park_reason = models.CharField(
+ max_length=32,
+ choices=ComputeSettleParkReason,
+ blank=True,
+ default="",
+ help_text="Why settlement stopped retrying the row; empty while the row is not parked",
+ )
+ compute_settle_body = models.JSONField(
+ default=dict,
+ blank=True,
+ help_text="Reported-spend body frozen before the first attempt so retries replay it unchanged",
+ )
class Meta:
db_table = "posthog_task_sandbox_session"
@@ -3879,6 +3924,25 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):
),
# For per-team/per-origin re-aggregation once pricing decides which origins bill.
models.Index(fields=["team", "user_attributed_at"], name="sandbox_session_team_attr_idx"),
+ models.Index(
+ fields=["ttl_expires_at"],
+ condition=models.Q(
+ compute_settled_at__isnull=True,
+ compute_settle_conflict_at__isnull=True,
+ user_attributed_at__isnull=False,
+ ),
+ name="sandbox_session_settle_ttl_idx",
+ ),
+ models.Index(
+ models.OrderBy(models.F("compute_settle_attempted_at"), nulls_first=True),
+ models.F("ended_at"),
+ condition=models.Q(
+ compute_settled_at__isnull=True,
+ compute_settle_conflict_at__isnull=True,
+ user_attributed_at__isnull=False,
+ ),
+ name="sandbox_session_settle_rt_idx",
+ ),
]
def __str__(self):
diff --git a/products/tasks/backend/tasks/tasks.py b/products/tasks/backend/tasks/tasks.py
index 756898ba9aa4..d31b05b0fc5b 100644
--- a/products/tasks/backend/tasks/tasks.py
+++ b/products/tasks/backend/tasks/tasks.py
@@ -2,6 +2,7 @@ from datetime import datetime
from uuid import UUID
from django.db import OperationalError
+from django.utils import timezone
from celery import shared_task
from celery.app.task import Task as CeleryTask
@@ -15,6 +16,7 @@ from posthog.models.github_integration_base import GitHubIntegrationError
from products.tasks.backend.facade.api import record_comment_activity
from products.tasks.backend.logic.services.comment_slack_dm import send_comment_slack_dms
+from products.tasks.backend.logic.services.compute_settlement import settle_overdue_sandbox_compute
from products.tasks.backend.logic.services.slack_pr_cards import post_pr_closed_slack_update
from products.tasks.backend.logic.services.workflow_step_resume import resume_workflow_step_for_run_id
from products.tasks.backend.logic.stream.budget_steer import BudgetSteerCapture, BudgetSteerProperties
@@ -103,3 +105,8 @@ def resume_workflow_step_for_run_deferred(run_id: str) -> None:
@shared_task(ignore_result=True)
def notify_slack_thread_pr_closed(run_id: str, pr_url: str, merged: bool = False) -> None:
post_pr_closed_slack_update(run_id, pr_url, merged=merged)
+
+
+@shared_task(ignore_result=True)
+def settle_overdue_sandbox_compute_task() -> None:
+ settle_overdue_sandbox_compute(timezone.now())
diff --git a/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.py b/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.py
index 08b4c336b2a3..2db1576c6287 100644
--- a/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.py
+++ b/products/tasks/backend/temporal/execute_sandbox/activities/reap_orphaned_sandbox.py
@@ -15,6 +15,7 @@ a destroy failure does not block the clear — a stale id staying in state
would just be re-reaped (and fail Modal-side again) on the next start.
"""
+import logging
from dataclasses import dataclass
from typing import Optional
@@ -22,6 +23,7 @@ from temporalio import activity
from posthog.temporal.common.utils import asyncify
+from products.tasks.backend.logic.services.compute_settlement import settle_task_run_compute
from products.tasks.backend.logic.services.sandbox import get_sandbox_class_for_sandbox_id
from products.tasks.backend.logic.services.sandbox_usage import (
close_sandbox_session,
@@ -32,6 +34,8 @@ from products.tasks.backend.models import SandboxSession, TaskRun
from products.tasks.backend.temporal.execute_sandbox.activities.sandbox_state import SANDBOX_ID_STATE_KEY
from products.tasks.backend.temporal.observability import log_activity_execution
+logger = logging.getLogger(__name__)
+
@dataclass
class ReapOrphanedSandboxInput:
@@ -57,7 +61,7 @@ class ReapOrphanedSandboxResult:
def reap_orphaned_sandbox(input: ReapOrphanedSandboxInput) -> ReapOrphanedSandboxResult:
with log_activity_execution("reap_orphaned_sandbox", run_id=input.run_id):
try:
- task_run = TaskRun.objects.only("state").get(id=input.run_id)
+ task_run = TaskRun.objects.only("state", "team_id").get(id=input.run_id)
except TaskRun.DoesNotExist:
return ReapOrphanedSandboxResult(reaped_sandbox_id=None, destroy_succeeded=True)
@@ -91,6 +95,12 @@ def reap_orphaned_sandbox(input: ReapOrphanedSandboxInput) -> ReapOrphanedSandbo
billed_cpu_usage_usec=billed_cpu_usage_usec,
cpu_usage_measured_at=cpu_usage_measured_at,
)
+ try:
+ settle_task_run_compute(input.run_id, task_run.team_id)
+ except Exception:
+ logger.warning(
+ "reap_orphaned_sandbox_compute_settlement_failed", extra={"run_id": input.run_id}, exc_info=True
+ )
TaskRun.clear_sandbox_connection_state_atomic(input.run_id, sandbox_id)
return ReapOrphanedSandboxResult(reaped_sandbox_id=sandbox_id, destroy_succeeded=destroy_succeeded)
diff --git a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py
index ed822d2d37bd..14d9c1936ea7 100644
--- a/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py
+++ b/products/tasks/backend/temporal/process_task/activities/cleanup_sandbox.py
@@ -7,6 +7,7 @@ from temporalio import activity
from posthog.temporal.common.utils import asyncify
from products.tasks.backend.exceptions import SandboxNotFoundError
+from products.tasks.backend.logic.services.compute_settlement import settle_task_run_compute
from products.tasks.backend.logic.services.gateway_usage import refresh_task_run_cost
from products.tasks.backend.logic.services.sandbox import get_sandbox_class_for_sandbox_id
from products.tasks.backend.logic.services.sandbox_usage import (
@@ -136,6 +137,14 @@ def cleanup_sandbox_now(input: CleanupSandboxInput) -> None:
)
if accounting_run is not None:
+ try:
+ settle_task_run_compute(accounting_run.id, accounting_run.team_id)
+ except Exception:
+ logger.warning(
+ "cleanup_sandbox_compute_settlement_failed",
+ extra={"run_id": str(accounting_run.id)},
+ exc_info=True,
+ )
try:
refresh_task_run_cost(run_id=accounting_run.id, team_id=accounting_run.team_id)
except Exception:
diff --git a/products/tasks/backend/temporal/process_task/ai_gateway_token.py b/products/tasks/backend/temporal/process_task/ai_gateway_token.py
index a00ea6ec2f42..285cd5bca4cb 100644
--- a/products/tasks/backend/temporal/process_task/ai_gateway_token.py
+++ b/products/tasks/backend/temporal/process_task/ai_gateway_token.py
@@ -383,6 +383,7 @@ def mint_scoped_token(
expires_in_seconds: int | None = None,
gateway_config: "AIGatewayConfig | None" = None,
raise_on_failure: bool = False,
+ run_id: str | None = None,
) -> str | None:
"""Mint a `phe_` scoped token pinned to (ai_product, obo=team_id), or None on failure.
@@ -410,6 +411,8 @@ def mint_scoped_token(
}
if user:
body["user"] = user
+ if run_id:
+ body["run"] = run_id
pin = allowed_models if allowed_models is not None else _PRODUCT_ALLOWED_MODELS.get(ai_product)
if pin:
body["allowed_models"] = gateway_pin(pin)
diff --git a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
index aa3e10ce9acc..ef902e482555 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
@@ -770,6 +770,7 @@ class TestProvisioningBoundaries:
runtime="acp",
prior_slack_run=False,
profile_billing_product=profile_billing_product,
+ run_id=str(self._ctx().run_id),
)
def test_non_slack_origin_skips_the_prior_run_lookup(self, mint_settings):
@@ -854,6 +855,7 @@ class TestProvisioningBoundaries:
assert post.call_args.kwargs["json"]["capture_mode"] == "none"
assert post.call_args.kwargs["json"]["obo"] == str(ctx.team_id)
assert post.call_args.kwargs["json"]["user"] == ctx.distinct_id
+ assert post.call_args.kwargs["json"]["run"] == ctx.run_id
record_routing.assert_called_once_with(run_id=ctx.run_id, team_id=ctx.team_id, uses_gateway=True)
else:
assert out == {}
@@ -1103,14 +1105,16 @@ class TestUserPinAndCapOverride:
def test_mint_pins_the_acting_user(self, mint_settings):
with patch("products.tasks.backend.temporal.process_task.ai_gateway_token.requests.post") as post:
post.return_value = self._response({"token": "phe_abc"})
- mint_scoped_token(ai_product="signals_scout", team_id=123, user="user-distinct-1")
+ mint_scoped_token(ai_product="signals_scout", team_id=123, user="user-distinct-1", run_id="run-1")
assert post.call_args.kwargs["json"]["user"] == "user-distinct-1"
+ assert post.call_args.kwargs["json"]["run"] == "run-1"
def test_mint_omits_user_when_unknown(self, mint_settings):
with patch("products.tasks.backend.temporal.process_task.ai_gateway_token.requests.post") as post:
post.return_value = self._response({"token": "phe_abc"})
mint_scoped_token(ai_product="signals_scout", team_id=123)
assert "user" not in post.call_args.kwargs["json"]
+ assert "run" not in post.call_args.kwargs["json"]
def test_cap_override_applies_per_team(self, mint_settings):
mint_settings.SANDBOX_AI_GATEWAY_TOKEN_CAP_USD_OVERRIDES = '{"2": "10"}'
diff --git a/products/tasks/backend/temporal/process_task/utils.py b/products/tasks/backend/temporal/process_task/utils.py
index 9dcfa27476ad..a62b0b4b93fe 100644
--- a/products/tasks/backend/temporal/process_task/utils.py
+++ b/products/tasks/backend/temporal/process_task/utils.py
@@ -1600,7 +1600,7 @@ def run_gateway_env_vars(ctx: TaskProcessingContext, task: Task) -> dict[str, st
gateway_url = settings.SANDBOX_AI_GATEWAY_URL
if not gateway_url:
raise GatewayNotConfiguredError("Scout trials require SANDBOX_AI_GATEWAY_URL")
- token = mint_private_gateway_token(team_id=ctx.team_id, user=ctx.distinct_id)
+ token = mint_private_gateway_token(team_id=ctx.team_id, user=ctx.distinct_id, run_id=str(ctx.run_id))
try:
record_gateway_routing(run_id=ctx.run_id, team_id=ctx.team_id, uses_gateway=True)
except Exception:
@@ -1650,6 +1650,7 @@ def run_gateway_env_vars(ctx: TaskProcessingContext, task: Task) -> dict[str, st
state=ctx.state,
model=ctx.model,
runtime=ctx.task_runtime,
+ run_id=str(ctx.run_id),
)
pinned = env_vars.pop(_PIN_KEY_ENV, None) or env_vars.get("AI_GATEWAY_PRODUCT")
if not _record_pinned_gateway_product(ctx.run_id, ctx.state, pinned):
@@ -1754,6 +1755,7 @@ def ai_gateway_env_vars(
runtime: str | None = None,
prior_slack_run: bool = False,
profile_billing_product: str | None = None,
+ run_id: str | None = None,
) -> dict[str, str]:
"""Env vars routing listed products to the Go ai-gateway, shared by every
injection site so the both-or-nothing guard cannot drift per site. Both
@@ -1861,6 +1863,8 @@ def ai_gateway_env_vars(
if free_pin is not None:
mint_kwargs["allowed_models"] = free_pin
mint_kwargs["limit_tier"] = posthog_code_limit_tier(team_id, distinct_id)
+ if run_id:
+ mint_kwargs["run_id"] = run_id
token = mint_scoped_token(ai_product=ai_product, team_id=team_id, user=distinct_id, **mint_kwargs)
if token:
env_vars["AI_GATEWAY_TOKEN"] = token
diff --git a/products/tasks/backend/tests/test_gateway_usage.py b/products/tasks/backend/tests/test_gateway_usage.py
index 887ea141cd4b..80b055795879 100644
--- a/products/tasks/backend/tests/test_gateway_usage.py
+++ b/products/tasks/backend/tests/test_gateway_usage.py
@@ -277,6 +277,10 @@ class TestGatewayUsage(BaseTest):
run.refresh_from_db()
assert run.state["compute_cost"] == expected
assert set(run.state) == {"unprocessed_request_ids", "token_cost", "compute_cost"}
+ SandboxSession.objects.for_team(self.team.id).filter(sandbox_id="claimed").update(
+ compute_settled_at=now, compute_settled_usd=Decimal("1.234567")
+ )
+ assert get_task_run_cost(run_id=run.id, team_id=self.team.id).compute_cost == 123
@patch("aiohttp.ClientSession._request")
def test_accounting_after_completion_has_no_completion_side_effects(self, get: Mock) -> None: