:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): push settlement sweep gauges to the pushgateway (#129369)
Alessandro Pogliaghi080d2bcd3d3e
diff --git a/products/tasks/backend/logic/services/compute_settlement.py b/products/tasks/backend/logic/services/compute_settlement.pyindex c34b16bdc0a7..5651fa7a0eed 100644--- a/products/tasks/backend/logic/services/compute_settlement.py+++ b/products/tasks/backend/logic/services/compute_settlement.py@@ -36,7 +36,7 @@ from products.tasks.backend.logic.services.sandbox_usage import (from products.tasks.backend.metrics import (ComputeSettlementOutcome,observe_compute_settlement,- set_hogland_sandbox_sessions_open_over_day,+ push_compute_settlement_sweep_gauges,)from products.tasks.backend.models import SandboxSession, TaskRun@@ -47,6 +47,7 @@ ATTEMPTS_EXHAUSTED_ERROR = "attempts_exhausted"ENDED_GRACE = timedelta(minutes=10)TTL_GRACE = timedelta(hours=1)HOGLAND_OPEN_ALERT_AGE = timedelta(days=1)+UNSETTLED_ALERT_AGE = timedelta(hours=1)DETAIL_MAX_BYTES = 2048_USD_QUANTUM = Decimal("0.000001")_RETRYABLE_STATUSES = frozenset({408, 429})@@ -348,13 +349,22 @@ def _overdue_open_modal(now: datetime, batch_size: int, processed: set[UUID]) ->)-def _refresh_hogland_open_gauge(now: datetime) -> None:- count = (+def _push_sweep_gauges(now: datetime) -> None:+ hogland_open_over_day = (SandboxSession.objects.unscoped().filter(sandbox_backend="hogland", ended_at__isnull=True, created_at__lt=now - HOGLAND_OPEN_ALERT_AGE).count())- set_hogland_sandbox_sessions_open_over_day(count)+ unsettled_over_hour = (+ _unsettled(SandboxSession.objects.unscoped())+ .filter(user_attributed_at__isnull=False, ended_at__lt=now - UNSETTLED_ALERT_AGE)+ .count()+ if settings.GATEWAY_COMPUTE_DEBIT_ENABLED+ else 0+ )+ push_compute_settlement_sweep_gauges(+ hogland_open_over_day=hogland_open_over_day, unsettled_over_hour=unsettled_over_hour+ )def _park_repeated_contract_errors(@@ -386,34 +396,36 @@ def _park_repeated_contract_errors(def settle_overdue_sandbox_compute(now: datetime, batch_size: int = 50, time_budget: timedelta = timedelta(minutes=15)) -> list[SettlementOutcome]:- _refresh_hogland_open_gauge(now)- if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:- return []- deadline = time.monotonic() + time_budget.total_seconds()- results: list[tuple[SandboxSession, SettlementOutcome]] = []- processed: set[UUID] = set()try:- while True:- batch = _overdue_ended(now, batch_size, processed)- for session in batch:- processed.add(session.id)- results.append((session, _settle_safely(session)))- if len(batch) < batch_size or time.monotonic() >= deadline:- break- while True:- batch = _overdue_open_modal(now, batch_size, processed)- for session in batch:- processed.add(session.id)- close_sandbox_session(session.sandbox_id, reason=SandboxSession.EndedReason.TTL_SWEEP)- closed = SandboxSession.objects.unscoped().filter(id=session.id).first()- if closed is not None and closed.ended_at is not None:- results.append((closed, _settle_safely(closed)))- if len(batch) < batch_size or time.monotonic() >= deadline:- break- return _park_repeated_contract_errors(results)- except ComputeSettlementDeploymentError as error:- logger.exception("compute_settlement.sweep_stopped", error=str(error))- return [outcome for _, outcome in results]+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:+ return []+ deadline = time.monotonic() + time_budget.total_seconds()+ results: list[tuple[SandboxSession, SettlementOutcome]] = []+ processed: set[UUID] = set()+ try:+ while True:+ batch = _overdue_ended(now, batch_size, processed)+ for session in batch:+ processed.add(session.id)+ results.append((session, _settle_safely(session)))+ if len(batch) < batch_size or time.monotonic() >= deadline:+ break+ while True:+ batch = _overdue_open_modal(now, batch_size, processed)+ for session in batch:+ processed.add(session.id)+ close_sandbox_session(session.sandbox_id, reason=SandboxSession.EndedReason.TTL_SWEEP)+ closed = SandboxSession.objects.unscoped().filter(id=session.id).first()+ if closed is not None and closed.ended_at is not None:+ results.append((closed, _settle_safely(closed)))+ if len(batch) < batch_size or time.monotonic() >= deadline:+ break+ return _park_repeated_contract_errors(results)+ except ComputeSettlementDeploymentError as error:+ logger.exception("compute_settlement.sweep_stopped", error=str(error))+ return [outcome for _, outcome in results]+ finally:+ _push_sweep_gauges(now)def _settle_safely(session: SandboxSession) -> SettlementOutcome:diff --git a/products/tasks/backend/logic/services/tests/test_compute_settlement.py b/products/tasks/backend/logic/services/tests/test_compute_settlement.pyindex 905ab3d5665f..ed46b32f7cf6 100644--- a/products/tasks/backend/logic/services/tests/test_compute_settlement.py+++ b/products/tasks/backend/logic/services/tests/test_compute_settlement.py@@ -324,6 +324,52 @@ class TestComputeSettlement(BaseTest):assert untouched.ended_at is Noneassert untouched.compute_settled_at is None+ @parameterized.expand([("flag_on", True, 1), ("flag_off", False, 0)])+ def test_sweep_pushes_gauges_after_settling(self, _name: str, enabled: bool, expected_unsettled: int) -> None:+ overdue_at = self.now - timedelta(minutes=90)+ self._session(ended_at=overdue_at)+ stuck = self._session(ended_at=overdue_at)+ self._session(ended_at=overdue_at, compute_settled_at=overdue_at, compute_settled_usd=Decimal(0))+ self._session(ended_at=overdue_at, compute_settle_conflict_at=overdue_at)+ self._session(ended_at=overdue_at, user_attributed_at=None)+ self._session(ended_at=self.now - timedelta(minutes=30))+ self._session(+ ended_at=None, ended_reason=None, sandbox_backend="hogland", created_at=self.now - timedelta(days=2)+ )+ self._session(+ ended_at=None, ended_reason=None, sandbox_backend="hogland", created_at=self.now - timedelta(hours=20)+ )++ def respond(_url: str, **kwargs: Any) -> httpx.Response:+ if kwargs["json"]["reference"] == f"compute:{stuck.id}":+ return _response(503)+ return _ok()++ with (+ override_settings(GATEWAY_COMPUTE_DEBIT_ENABLED=enabled),+ patch(HTTPX_POST, side_effect=respond),+ patch("posthog.metrics.push_registry") as push,+ ):+ settle_overdue_sandbox_compute(self.now)++ push.assert_called_once()+ assert push.call_args.args[0] == "tasks_compute_settlement"+ registry = push.call_args.args[1]+ assert registry.get_sample_value("posthog_tasks_hogland_sandbox_sessions_open_over_day") == 1+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_unsettled_over_hour") == expected_unsettled++ def test_sweep_settles_when_the_gauge_push_fails(self) -> None:+ session = self._session()+ with (+ override_settings(PROM_PUSHGATEWAY_ADDRESS="http://pushgateway.internal"),+ patch(HTTPX_POST, return_value=_ok()),+ patch("posthog.metrics.push_to_gateway", side_effect=OSError("refused")) as push,+ ):+ assert settle_overdue_sandbox_compute(self.now) == [SettlementOutcome.SETTLED]++ push.assert_called_once()+ assert self._reload(session).compute_settled_at is not None+@parameterized.expand([("settles", _ok(), SettlementOutcome.SETTLED), ("retries", _response(503), SettlementOutcome.RETRY)])diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.pyindex f06d1fad68ec..701363f0541e 100644--- a/products/tasks/backend/metrics.py+++ b/products/tasks/backend/metrics.py@@ -3,6 +3,8 @@ from typing import TYPE_CHECKING, Literalimport structlogfrom prometheus_client import Counter, Gauge, Histogram+from posthog.metrics import pushed_metrics_registry+logger = structlog.get_logger(__name__)if TYPE_CHECKING:@@ -510,19 +512,23 @@ COMPUTE_SETTLEMENTS_TOTAL = Counter(labelnames=["outcome"],)-HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY = Gauge(- "posthog_tasks_hogland_sandbox_sessions_open_over_day",- "Hogland sandbox sessions with no end stamp that opened more than a day ago",- multiprocess_mode="mostrecent",-)-def observe_compute_settlement(outcome: ComputeSettlementOutcome) -> None:COMPUTE_SETTLEMENTS_TOTAL.labels(outcome=outcome).inc()-def set_hogland_sandbox_sessions_open_over_day(count: int) -> None:- HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY.set(count)+def push_compute_settlement_sweep_gauges(*, hogland_open_over_day: int, unsettled_over_hour: int) -> None:+ with pushed_metrics_registry("tasks_compute_settlement") as registry:+ Gauge(+ "posthog_tasks_hogland_sandbox_sessions_open_over_day",+ "Hogland sandbox sessions with no end stamp that opened more than a day ago",+ registry=registry,+ ).set(hogland_open_over_day)+ Gauge(+ "posthog_tasks_sandbox_sessions_unsettled_over_hour",+ "Attributed sandbox sessions that ended more than an hour ago and are neither settled nor parked",+ registry=registry,+ ).set(unsettled_over_hour)# scoped: "true" when the delivery's installation resolved to at least one team, so the