Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): settle sandbox compute strictly and park wallet refusals (#130890)

Alessandro Pogliaghid3441aef9115
diff --git a/products/tasks/backend/logic/services/compute_settlement.py b/products/tasks/backend/logic/services/compute_settlement.py
index 5651fa7a0eed..dec58050d42c 100644
--- a/products/tasks/backend/logic/services/compute_settlement.py
+++ b/products/tasks/backend/logic/services/compute_settlement.py
@@ -66,6 +66,7 @@ class SettlementOutcome(StrEnum):
ATTEMPTS_EXHAUSTED = "attempts_exhausted"
CONTRACT_ERROR = "contract_error"
CONTRACT_ERROR_REPEAT = "contract_error_repeat"
+ INSUFFICIENT_CREDITS = "insufficient_credits"
_CONTRACT_ERROR_OUTCOMES = frozenset({SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.CONTRACT_ERROR_REPEAT})
@@ -167,6 +168,7 @@ def build_settlement_body(session: SandboxSession, run: TaskRun, cost: SandboxCo
"billable": is_billable_sandbox_session(session),
"detail": detail,
"reason": "sandbox compute",
+ "strict": True,
}
)
return body
@@ -190,6 +192,7 @@ _METRIC_LABELS: dict[SettlementOutcome, ComputeSettlementOutcome] = {
SettlementOutcome.ATTEMPTS_EXHAUSTED: "attempts_exhausted",
SettlementOutcome.CONTRACT_ERROR: "contract_error",
SettlementOutcome.CONTRACT_ERROR_REPEAT: "contract_error",
+ SettlementOutcome.INSUFFICIENT_CREDITS: "insufficient_credits",
}
@@ -313,6 +316,13 @@ def settle_sandbox_session(session: SandboxSession) -> SettlementOutcome:
return _record(SettlementOutcome.RETRY)
if status == 400:
return _mark_contract_error(session, response)
+ if status == 402 and (response.body or {}).get("code") == "insufficient_credits":
+ error_text = _error_text(response)
+ logger.error(
+ "compute_settlement.insufficient_credits", session_id=str(session.id), status=status, error=error_text
+ )
+ _mark_conflict(session, error_text, ParkReason.HTTP_402)
+ return _record(SettlementOutcome.INSUFFICIENT_CREDITS)
if status == 409:
logger.error("compute_settlement.conflict", session_id=str(session.id), status=status)
_mark_conflict(session, _error_text(response), ParkReason.HTTP_409)
@@ -362,8 +372,18 @@ def _push_sweep_gauges(now: datetime) -> None:
if settings.GATEWAY_COMPUTE_DEBIT_ENABLED
else 0
)
+ parked_counts = {
+ row["compute_settle_park_reason"]: row["rows"]
+ for row in SandboxSession.objects.unscoped()
+ .filter(compute_settled_at__isnull=True, compute_settle_conflict_at__isnull=False)
+ .order_by()
+ .values("compute_settle_park_reason")
+ .annotate(rows=Count("id"))
+ }
push_compute_settlement_sweep_gauges(
- hogland_open_over_day=hogland_open_over_day, unsettled_over_hour=unsettled_over_hour
+ hogland_open_over_day=hogland_open_over_day,
+ unsettled_over_hour=unsettled_over_hour,
+ parked_by_reason={reason: parked_counts.get(reason, 0) for reason in ParkReason.values},
)
diff --git a/products/tasks/backend/logic/services/tests/test_compute_settlement.py b/products/tasks/backend/logic/services/tests/test_compute_settlement.py
index ed46b32f7cf6..cbd0dcc7467c 100644
--- a/products/tasks/backend/logic/services/tests/test_compute_settlement.py
+++ b/products/tasks/backend/logic/services/tests/test_compute_settlement.py
@@ -125,6 +125,7 @@ class TestComputeSettlement(BaseTest):
assert body["charge_kind"] == "compute"
assert body["product"] == "posthog_code"
assert body["amount_usd"] == "0.770400"
+ assert body["strict"] is True
detail = body["detail"]
assert detail["window_end"] == session.ttl_expires_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
assert detail["attributed_seconds"] == 1800.0
@@ -144,6 +145,26 @@ class TestComputeSettlement(BaseTest):
[
("conflict", _response(409, {"error": "reference reused"}), SettlementOutcome.CONFLICT, [], "http_409"),
("unexpected", _response(418), SettlementOutcome.CONFLICT, [], "unexpected_status"),
+ (
+ "insufficient_credits",
+ _response(
+ 402,
+ {
+ "code": "insufficient_credits",
+ "message": "reported spend refused: team wallet has insufficient funds",
+ },
+ ),
+ SettlementOutcome.INSUFFICIENT_CREDITS,
+ [],
+ "http_402",
+ ),
+ (
+ "payment_required_other_code",
+ _response(402, {"code": "cap_exceeded", "message": "team hard spend cap exceeded"}),
+ SettlementOutcome.CONFLICT,
+ [],
+ "unexpected_status",
+ ),
(
"validation",
_response(400, {"error": "bad"}),
@@ -226,6 +247,14 @@ class TestComputeSettlement(BaseTest):
compute_settle_body=frozen_body,
compute_settle_attempted_at=parked_at,
)
+ refused = self._session(
+ compute_settle_conflict_at=parked_at,
+ compute_settle_park_reason="http_402",
+ compute_settle_error="HTTP 402: insufficient_credits",
+ compute_settle_attempts=1,
+ compute_settle_body=frozen_body,
+ compute_settle_attempted_at=parked_at,
+ )
reused = self._session(
compute_settle_conflict_at=parked_at,
compute_settle_park_reason="http_409",
@@ -249,6 +278,7 @@ class TestComputeSettlement(BaseTest):
assert self._reload(reused).compute_settle_park_reason == "http_409"
assert self._reload(older).compute_settle_conflict_at is not None
assert self._reload(exhausted).compute_settle_park_reason == "attempts_exhausted"
+ assert self._reload(refused).compute_settle_park_reason == "http_402"
assert unpark_sandbox_compute("attempts_exhausted", self.now - timedelta(days=1), self.now) == 1
@@ -258,6 +288,17 @@ class TestComputeSettlement(BaseTest):
assert row.compute_settle_body == frozen_body
assert row.compute_settle_attempted_at is None
+ assert unpark_sandbox_compute("http_402", self.now - timedelta(days=1), self.now) == 1
+
+ row = self._reload(refused)
+ assert row.compute_settle_conflict_at is None
+ assert row.compute_settle_park_reason == ""
+ assert row.compute_settle_error == ""
+ assert row.compute_settle_attempts == 0
+ assert row.compute_settle_body == frozen_body
+ assert row.compute_settle_attempted_at is None
+ assert self._reload(reused).compute_settle_park_reason == "http_409"
+
def test_deployment_error_raises_and_leaves_the_row_untouched(self) -> None:
session = self._session()
with patch(HTTPX_POST, return_value=_response(403)), pytest.raises(ComputeSettlementDeploymentError):
@@ -329,8 +370,9 @@ class TestComputeSettlement(BaseTest):
overdue_at = self.now - timedelta(minutes=90)
self._session(ended_at=overdue_at)
stuck = self._session(ended_at=overdue_at)
+ broke = self._session(ended_at=overdue_at)
self._session(ended_at=overdue_at, compute_settled_at=overdue_at, compute_settled_usd=Decimal(0))
- self._session(ended_at=overdue_at, compute_settle_conflict_at=overdue_at)
+ self._session(ended_at=overdue_at, compute_settle_conflict_at=overdue_at, compute_settle_park_reason="http_409")
self._session(ended_at=overdue_at, user_attributed_at=None)
self._session(ended_at=self.now - timedelta(minutes=30))
self._session(
@@ -343,6 +385,8 @@ class TestComputeSettlement(BaseTest):
def respond(_url: str, **kwargs: Any) -> httpx.Response:
if kwargs["json"]["reference"] == f"compute:{stuck.id}":
return _response(503)
+ if kwargs["json"]["reference"] == f"compute:{broke.id}":
+ return _response(402, {"code": "insufficient_credits", "message": "team wallet has insufficient funds"})
return _ok()
with (
@@ -357,6 +401,11 @@ class TestComputeSettlement(BaseTest):
registry = push.call_args.args[1]
assert registry.get_sample_value("posthog_tasks_hogland_sandbox_sessions_open_over_day") == 1
assert registry.get_sample_value("posthog_tasks_sandbox_sessions_unsettled_over_hour") == expected_unsettled
+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_parked", {"reason": "http_402"}) == int(
+ enabled
+ )
+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_parked", {"reason": "http_409"}) == 1
+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_parked", {"reason": "attempts_exhausted"}) == 0
def test_sweep_settles_when_the_gauge_push_fails(self) -> None:
session = self._session()
diff --git a/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py
index 5148c602cf82..b45a15445b01 100644
--- a/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py
+++ b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py
@@ -30,6 +30,7 @@ class Command(BaseCommand):
choices=[
SandboxSession.ComputeSettleParkReason.ATTEMPTS_EXHAUSTED.value,
SandboxSession.ComputeSettleParkReason.HTTP_400.value,
+ SandboxSession.ComputeSettleParkReason.HTTP_402.value,
],
default=None,
help="Unpark rows parked for this reason within --since/--until so the sweep retries them",
diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py
index 701363f0541e..e2ed2eb8d99d 100644
--- a/products/tasks/backend/metrics.py
+++ b/products/tasks/backend/metrics.py
@@ -1,3 +1,4 @@
+from collections.abc import Mapping
from typing import TYPE_CHECKING, Literal
import structlog
@@ -502,6 +503,7 @@ ComputeSettlementOutcome = Literal[
"retry",
"attempts_exhausted",
"contract_error",
+ "insufficient_credits",
"deployment_error",
"skipped",
]
@@ -517,7 +519,9 @@ def observe_compute_settlement(outcome: ComputeSettlementOutcome) -> None:
COMPUTE_SETTLEMENTS_TOTAL.labels(outcome=outcome).inc()
-def push_compute_settlement_sweep_gauges(*, hogland_open_over_day: int, unsettled_over_hour: int) -> None:
+def push_compute_settlement_sweep_gauges(
+ *, hogland_open_over_day: int, unsettled_over_hour: int, parked_by_reason: Mapping[str, int]
+) -> None:
with pushed_metrics_registry("tasks_compute_settlement") as registry:
Gauge(
"posthog_tasks_hogland_sandbox_sessions_open_over_day",
@@ -529,6 +533,14 @@ def push_compute_settlement_sweep_gauges(*, hogland_open_over_day: int, unsettle
"Attributed sandbox sessions that ended more than an hour ago and are neither settled nor parked",
registry=registry,
).set(unsettled_over_hour)
+ parked = Gauge(
+ "posthog_tasks_sandbox_sessions_parked",
+ "Sandbox sessions that are parked and not settled, by park reason",
+ labelnames=["reason"],
+ registry=registry,
+ )
+ for reason, rows in parked_by_reason.items():
+ parked.labels(reason=reason).set(rows)
# scoped: "true" when the delivery's installation resolved to at least one team, so the
diff --git a/products/tasks/backend/migrations/0150_sandboxsession_park_reason_http_402.py b/products/tasks/backend/migrations/0150_sandboxsession_park_reason_http_402.py
new file mode 100644
index 000000000000..6ff17b546ec9
--- /dev/null
+++ b/products/tasks/backend/migrations/0150_sandboxsession_park_reason_http_402.py
@@ -0,0 +1,29 @@
+# Generated by Django 6.1.3 on 2026-11-06 11:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("tasks", "0149_task_run_gateway_request"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="sandboxsession",
+ name="compute_settle_park_reason",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("http_400", "HTTP 400"),
+ ("http_402", "HTTP 402"),
+ ("http_409", "HTTP 409"),
+ ("attempts_exhausted", "Attempts exhausted"),
+ ("unexpected_status", "Unexpected status"),
+ ],
+ default="",
+ help_text="Why settlement stopped retrying the row; empty while the row is not parked",
+ max_length=32,
+ ),
+ ),
+ ]
diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txt
index d351f90c1851..cc548d29a961 100644
--- a/products/tasks/backend/migrations/max_migration.txt
+++ b/products/tasks/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0149_task_run_gateway_request
+0150_sandboxsession_park_reason_http_402
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index 2e296e2d19a1..154e6e7bee8f 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -3929,6 +3929,7 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):
class ComputeSettleParkReason(models.TextChoices):
HTTP_400 = "http_400", "HTTP 400"
+ HTTP_402 = "http_402", "HTTP 402"
HTTP_409 = "http_409", "HTTP 409"
ATTEMPTS_EXHAUSTED = "attempts_exhausted", "Attempts exhausted"
UNEXPECTED_STATUS = "unexpected_status", "Unexpected status"