:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): settle sandbox compute strictly and park wallet refusals (#130890)
Alessandro Pogliaghid3441aef9115
diff --git a/products/tasks/backend/logic/services/compute_settlement.py b/products/tasks/backend/logic/services/compute_settlement.pyindex 5651fa7a0eed..dec58050d42c 100644--- a/products/tasks/backend/logic/services/compute_settlement.py+++ b/products/tasks/backend/logic/services/compute_settlement.py@@ -66,6 +66,7 @@ class SettlementOutcome(StrEnum):ATTEMPTS_EXHAUSTED = "attempts_exhausted"CONTRACT_ERROR = "contract_error"CONTRACT_ERROR_REPEAT = "contract_error_repeat"+ INSUFFICIENT_CREDITS = "insufficient_credits"_CONTRACT_ERROR_OUTCOMES = frozenset({SettlementOutcome.CONTRACT_ERROR, SettlementOutcome.CONTRACT_ERROR_REPEAT})@@ -167,6 +168,7 @@ def build_settlement_body(session: SandboxSession, run: TaskRun, cost: SandboxCo"billable": is_billable_sandbox_session(session),"detail": detail,"reason": "sandbox compute",+ "strict": True,})return body@@ -190,6 +192,7 @@ _METRIC_LABELS: dict[SettlementOutcome, ComputeSettlementOutcome] = {SettlementOutcome.ATTEMPTS_EXHAUSTED: "attempts_exhausted",SettlementOutcome.CONTRACT_ERROR: "contract_error",SettlementOutcome.CONTRACT_ERROR_REPEAT: "contract_error",+ SettlementOutcome.INSUFFICIENT_CREDITS: "insufficient_credits",}@@ -313,6 +316,13 @@ def settle_sandbox_session(session: SandboxSession) -> SettlementOutcome:return _record(SettlementOutcome.RETRY)if status == 400:return _mark_contract_error(session, response)+ if status == 402 and (response.body or {}).get("code") == "insufficient_credits":+ error_text = _error_text(response)+ logger.error(+ "compute_settlement.insufficient_credits", session_id=str(session.id), status=status, error=error_text+ )+ _mark_conflict(session, error_text, ParkReason.HTTP_402)+ return _record(SettlementOutcome.INSUFFICIENT_CREDITS)if status == 409:logger.error("compute_settlement.conflict", session_id=str(session.id), status=status)_mark_conflict(session, _error_text(response), ParkReason.HTTP_409)@@ -362,8 +372,18 @@ def _push_sweep_gauges(now: datetime) -> None:if settings.GATEWAY_COMPUTE_DEBIT_ENABLEDelse 0)+ parked_counts = {+ row["compute_settle_park_reason"]: row["rows"]+ for row in SandboxSession.objects.unscoped()+ .filter(compute_settled_at__isnull=True, compute_settle_conflict_at__isnull=False)+ .order_by()+ .values("compute_settle_park_reason")+ .annotate(rows=Count("id"))+ }push_compute_settlement_sweep_gauges(- hogland_open_over_day=hogland_open_over_day, unsettled_over_hour=unsettled_over_hour+ hogland_open_over_day=hogland_open_over_day,+ unsettled_over_hour=unsettled_over_hour,+ parked_by_reason={reason: parked_counts.get(reason, 0) for reason in ParkReason.values},)diff --git a/products/tasks/backend/logic/services/tests/test_compute_settlement.py b/products/tasks/backend/logic/services/tests/test_compute_settlement.pyindex ed46b32f7cf6..cbd0dcc7467c 100644--- a/products/tasks/backend/logic/services/tests/test_compute_settlement.py+++ b/products/tasks/backend/logic/services/tests/test_compute_settlement.py@@ -125,6 +125,7 @@ class TestComputeSettlement(BaseTest):assert body["charge_kind"] == "compute"assert body["product"] == "posthog_code"assert body["amount_usd"] == "0.770400"+ assert body["strict"] is Truedetail = body["detail"]assert detail["window_end"] == session.ttl_expires_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")assert detail["attributed_seconds"] == 1800.0@@ -144,6 +145,26 @@ class TestComputeSettlement(BaseTest):[("conflict", _response(409, {"error": "reference reused"}), SettlementOutcome.CONFLICT, [], "http_409"),("unexpected", _response(418), SettlementOutcome.CONFLICT, [], "unexpected_status"),+ (+ "insufficient_credits",+ _response(+ 402,+ {+ "code": "insufficient_credits",+ "message": "reported spend refused: team wallet has insufficient funds",+ },+ ),+ SettlementOutcome.INSUFFICIENT_CREDITS,+ [],+ "http_402",+ ),+ (+ "payment_required_other_code",+ _response(402, {"code": "cap_exceeded", "message": "team hard spend cap exceeded"}),+ SettlementOutcome.CONFLICT,+ [],+ "unexpected_status",+ ),("validation",_response(400, {"error": "bad"}),@@ -226,6 +247,14 @@ class TestComputeSettlement(BaseTest):compute_settle_body=frozen_body,compute_settle_attempted_at=parked_at,)+ refused = self._session(+ compute_settle_conflict_at=parked_at,+ compute_settle_park_reason="http_402",+ compute_settle_error="HTTP 402: insufficient_credits",+ compute_settle_attempts=1,+ compute_settle_body=frozen_body,+ compute_settle_attempted_at=parked_at,+ )reused = self._session(compute_settle_conflict_at=parked_at,compute_settle_park_reason="http_409",@@ -249,6 +278,7 @@ class TestComputeSettlement(BaseTest):assert self._reload(reused).compute_settle_park_reason == "http_409"assert self._reload(older).compute_settle_conflict_at is not Noneassert self._reload(exhausted).compute_settle_park_reason == "attempts_exhausted"+ assert self._reload(refused).compute_settle_park_reason == "http_402"assert unpark_sandbox_compute("attempts_exhausted", self.now - timedelta(days=1), self.now) == 1@@ -258,6 +288,17 @@ class TestComputeSettlement(BaseTest):assert row.compute_settle_body == frozen_bodyassert row.compute_settle_attempted_at is None+ assert unpark_sandbox_compute("http_402", self.now - timedelta(days=1), self.now) == 1++ row = self._reload(refused)+ assert row.compute_settle_conflict_at is None+ assert row.compute_settle_park_reason == ""+ assert row.compute_settle_error == ""+ assert row.compute_settle_attempts == 0+ assert row.compute_settle_body == frozen_body+ assert row.compute_settle_attempted_at is None+ assert self._reload(reused).compute_settle_park_reason == "http_409"+def test_deployment_error_raises_and_leaves_the_row_untouched(self) -> None:session = self._session()with patch(HTTPX_POST, return_value=_response(403)), pytest.raises(ComputeSettlementDeploymentError):@@ -329,8 +370,9 @@ class TestComputeSettlement(BaseTest):overdue_at = self.now - timedelta(minutes=90)self._session(ended_at=overdue_at)stuck = self._session(ended_at=overdue_at)+ broke = self._session(ended_at=overdue_at)self._session(ended_at=overdue_at, compute_settled_at=overdue_at, compute_settled_usd=Decimal(0))- self._session(ended_at=overdue_at, compute_settle_conflict_at=overdue_at)+ self._session(ended_at=overdue_at, compute_settle_conflict_at=overdue_at, compute_settle_park_reason="http_409")self._session(ended_at=overdue_at, user_attributed_at=None)self._session(ended_at=self.now - timedelta(minutes=30))self._session(@@ -343,6 +385,8 @@ class TestComputeSettlement(BaseTest):def respond(_url: str, **kwargs: Any) -> httpx.Response:if kwargs["json"]["reference"] == f"compute:{stuck.id}":return _response(503)+ if kwargs["json"]["reference"] == f"compute:{broke.id}":+ return _response(402, {"code": "insufficient_credits", "message": "team wallet has insufficient funds"})return _ok()with (@@ -357,6 +401,11 @@ class TestComputeSettlement(BaseTest):registry = push.call_args.args[1]assert registry.get_sample_value("posthog_tasks_hogland_sandbox_sessions_open_over_day") == 1assert registry.get_sample_value("posthog_tasks_sandbox_sessions_unsettled_over_hour") == expected_unsettled+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_parked", {"reason": "http_402"}) == int(+ enabled+ )+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_parked", {"reason": "http_409"}) == 1+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_parked", {"reason": "attempts_exhausted"}) == 0def test_sweep_settles_when_the_gauge_push_fails(self) -> None:session = self._session()diff --git a/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.pyindex 5148c602cf82..b45a15445b01 100644--- a/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py+++ b/products/tasks/backend/management/commands/reconcile_sandbox_compute_settlement.py@@ -30,6 +30,7 @@ class Command(BaseCommand):choices=[SandboxSession.ComputeSettleParkReason.ATTEMPTS_EXHAUSTED.value,SandboxSession.ComputeSettleParkReason.HTTP_400.value,+ SandboxSession.ComputeSettleParkReason.HTTP_402.value,],default=None,help="Unpark rows parked for this reason within --since/--until so the sweep retries them",diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.pyindex 701363f0541e..e2ed2eb8d99d 100644--- a/products/tasks/backend/metrics.py+++ b/products/tasks/backend/metrics.py@@ -1,3 +1,4 @@+from collections.abc import Mappingfrom typing import TYPE_CHECKING, Literalimport structlog@@ -502,6 +503,7 @@ ComputeSettlementOutcome = Literal["retry","attempts_exhausted","contract_error",+ "insufficient_credits","deployment_error","skipped",]@@ -517,7 +519,9 @@ def observe_compute_settlement(outcome: ComputeSettlementOutcome) -> None:COMPUTE_SETTLEMENTS_TOTAL.labels(outcome=outcome).inc()-def push_compute_settlement_sweep_gauges(*, hogland_open_over_day: int, unsettled_over_hour: int) -> None:+def push_compute_settlement_sweep_gauges(+ *, hogland_open_over_day: int, unsettled_over_hour: int, parked_by_reason: Mapping[str, int]+) -> None:with pushed_metrics_registry("tasks_compute_settlement") as registry:Gauge("posthog_tasks_hogland_sandbox_sessions_open_over_day",@@ -529,6 +533,14 @@ def push_compute_settlement_sweep_gauges(*, hogland_open_over_day: int, unsettle"Attributed sandbox sessions that ended more than an hour ago and are neither settled nor parked",registry=registry,).set(unsettled_over_hour)+ parked = Gauge(+ "posthog_tasks_sandbox_sessions_parked",+ "Sandbox sessions that are parked and not settled, by park reason",+ labelnames=["reason"],+ registry=registry,+ )+ for reason, rows in parked_by_reason.items():+ parked.labels(reason=reason).set(rows)# scoped: "true" when the delivery's installation resolved to at least one team, so thediff --git a/products/tasks/backend/migrations/0150_sandboxsession_park_reason_http_402.py b/products/tasks/backend/migrations/0150_sandboxsession_park_reason_http_402.pynew file mode 100644index 000000000000..6ff17b546ec9--- /dev/null+++ b/products/tasks/backend/migrations/0150_sandboxsession_park_reason_http_402.py@@ -0,0 +1,29 @@+# Generated by Django 6.1.3 on 2026-11-06 11:13++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("tasks", "0149_task_run_gateway_request"),+ ]++ operations = [+ migrations.AlterField(+ model_name="sandboxsession",+ name="compute_settle_park_reason",+ field=models.CharField(+ blank=True,+ choices=[+ ("http_400", "HTTP 400"),+ ("http_402", "HTTP 402"),+ ("http_409", "HTTP 409"),+ ("attempts_exhausted", "Attempts exhausted"),+ ("unexpected_status", "Unexpected status"),+ ],+ default="",+ help_text="Why settlement stopped retrying the row; empty while the row is not parked",+ max_length=32,+ ),+ ),+ ]diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txtindex d351f90c1851..cc548d29a961 100644--- a/products/tasks/backend/migrations/max_migration.txt+++ b/products/tasks/backend/migrations/max_migration.txt@@ -1 +1 @@-0149_task_run_gateway_request+0150_sandboxsession_park_reason_http_402diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.pyindex 2e296e2d19a1..154e6e7bee8f 100644--- a/products/tasks/backend/models.py+++ b/products/tasks/backend/models.py@@ -3929,6 +3929,7 @@ class SandboxSession(TeamScopedRootMixin, UUIDModel):class ComputeSettleParkReason(models.TextChoices):HTTP_400 = "http_400", "HTTP 400"+ HTTP_402 = "http_402", "HTTP 402"HTTP_409 = "http_409", "HTTP 409"ATTEMPTS_EXHAUSTED = "attempts_exhausted", "Attempts exhausted"UNEXPECTED_STATUS = "unexpected_status", "Unexpected status"