Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): push settlement sweep gauges to the pushgateway (#129369)

Alessandro Pogliaghi080d2bcd3d3e
diff --git a/products/tasks/backend/logic/services/compute_settlement.py b/products/tasks/backend/logic/services/compute_settlement.py
index c34b16bdc0a7..5651fa7a0eed 100644
--- a/products/tasks/backend/logic/services/compute_settlement.py
+++ b/products/tasks/backend/logic/services/compute_settlement.py
@@ -36,7 +36,7 @@ from products.tasks.backend.logic.services.sandbox_usage import (
from products.tasks.backend.metrics import (
ComputeSettlementOutcome,
observe_compute_settlement,
- set_hogland_sandbox_sessions_open_over_day,
+ push_compute_settlement_sweep_gauges,
)
from products.tasks.backend.models import SandboxSession, TaskRun
@@ -47,6 +47,7 @@ ATTEMPTS_EXHAUSTED_ERROR = "attempts_exhausted"
ENDED_GRACE = timedelta(minutes=10)
TTL_GRACE = timedelta(hours=1)
HOGLAND_OPEN_ALERT_AGE = timedelta(days=1)
+UNSETTLED_ALERT_AGE = timedelta(hours=1)
DETAIL_MAX_BYTES = 2048
_USD_QUANTUM = Decimal("0.000001")
_RETRYABLE_STATUSES = frozenset({408, 429})
@@ -348,13 +349,22 @@ def _overdue_open_modal(now: datetime, batch_size: int, processed: set[UUID]) ->
)
-def _refresh_hogland_open_gauge(now: datetime) -> None:
- count = (
+def _push_sweep_gauges(now: datetime) -> None:
+ hogland_open_over_day = (
SandboxSession.objects.unscoped()
.filter(sandbox_backend="hogland", ended_at__isnull=True, created_at__lt=now - HOGLAND_OPEN_ALERT_AGE)
.count()
)
- set_hogland_sandbox_sessions_open_over_day(count)
+ unsettled_over_hour = (
+ _unsettled(SandboxSession.objects.unscoped())
+ .filter(user_attributed_at__isnull=False, ended_at__lt=now - UNSETTLED_ALERT_AGE)
+ .count()
+ if settings.GATEWAY_COMPUTE_DEBIT_ENABLED
+ else 0
+ )
+ push_compute_settlement_sweep_gauges(
+ hogland_open_over_day=hogland_open_over_day, unsettled_over_hour=unsettled_over_hour
+ )
def _park_repeated_contract_errors(
@@ -386,34 +396,36 @@ def _park_repeated_contract_errors(
def settle_overdue_sandbox_compute(
now: datetime, batch_size: int = 50, time_budget: timedelta = timedelta(minutes=15)
) -> list[SettlementOutcome]:
- _refresh_hogland_open_gauge(now)
- if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:
- return []
- deadline = time.monotonic() + time_budget.total_seconds()
- results: list[tuple[SandboxSession, SettlementOutcome]] = []
- processed: set[UUID] = set()
try:
- while True:
- batch = _overdue_ended(now, batch_size, processed)
- for session in batch:
- processed.add(session.id)
- results.append((session, _settle_safely(session)))
- if len(batch) < batch_size or time.monotonic() >= deadline:
- break
- while True:
- batch = _overdue_open_modal(now, batch_size, processed)
- for session in batch:
- processed.add(session.id)
- close_sandbox_session(session.sandbox_id, reason=SandboxSession.EndedReason.TTL_SWEEP)
- closed = SandboxSession.objects.unscoped().filter(id=session.id).first()
- if closed is not None and closed.ended_at is not None:
- results.append((closed, _settle_safely(closed)))
- if len(batch) < batch_size or time.monotonic() >= deadline:
- break
- return _park_repeated_contract_errors(results)
- except ComputeSettlementDeploymentError as error:
- logger.exception("compute_settlement.sweep_stopped", error=str(error))
- return [outcome for _, outcome in results]
+ if not settings.GATEWAY_COMPUTE_DEBIT_ENABLED:
+ return []
+ deadline = time.monotonic() + time_budget.total_seconds()
+ results: list[tuple[SandboxSession, SettlementOutcome]] = []
+ processed: set[UUID] = set()
+ try:
+ while True:
+ batch = _overdue_ended(now, batch_size, processed)
+ for session in batch:
+ processed.add(session.id)
+ results.append((session, _settle_safely(session)))
+ if len(batch) < batch_size or time.monotonic() >= deadline:
+ break
+ while True:
+ batch = _overdue_open_modal(now, batch_size, processed)
+ for session in batch:
+ processed.add(session.id)
+ close_sandbox_session(session.sandbox_id, reason=SandboxSession.EndedReason.TTL_SWEEP)
+ closed = SandboxSession.objects.unscoped().filter(id=session.id).first()
+ if closed is not None and closed.ended_at is not None:
+ results.append((closed, _settle_safely(closed)))
+ if len(batch) < batch_size or time.monotonic() >= deadline:
+ break
+ return _park_repeated_contract_errors(results)
+ except ComputeSettlementDeploymentError as error:
+ logger.exception("compute_settlement.sweep_stopped", error=str(error))
+ return [outcome for _, outcome in results]
+ finally:
+ _push_sweep_gauges(now)
def _settle_safely(session: SandboxSession) -> SettlementOutcome:
diff --git a/products/tasks/backend/logic/services/tests/test_compute_settlement.py b/products/tasks/backend/logic/services/tests/test_compute_settlement.py
index 905ab3d5665f..ed46b32f7cf6 100644
--- a/products/tasks/backend/logic/services/tests/test_compute_settlement.py
+++ b/products/tasks/backend/logic/services/tests/test_compute_settlement.py
@@ -324,6 +324,52 @@ class TestComputeSettlement(BaseTest):
assert untouched.ended_at is None
assert untouched.compute_settled_at is None
+ @parameterized.expand([("flag_on", True, 1), ("flag_off", False, 0)])
+ def test_sweep_pushes_gauges_after_settling(self, _name: str, enabled: bool, expected_unsettled: int) -> None:
+ overdue_at = self.now - timedelta(minutes=90)
+ self._session(ended_at=overdue_at)
+ stuck = self._session(ended_at=overdue_at)
+ self._session(ended_at=overdue_at, compute_settled_at=overdue_at, compute_settled_usd=Decimal(0))
+ self._session(ended_at=overdue_at, compute_settle_conflict_at=overdue_at)
+ self._session(ended_at=overdue_at, user_attributed_at=None)
+ self._session(ended_at=self.now - timedelta(minutes=30))
+ self._session(
+ ended_at=None, ended_reason=None, sandbox_backend="hogland", created_at=self.now - timedelta(days=2)
+ )
+ self._session(
+ ended_at=None, ended_reason=None, sandbox_backend="hogland", created_at=self.now - timedelta(hours=20)
+ )
+
+ def respond(_url: str, **kwargs: Any) -> httpx.Response:
+ if kwargs["json"]["reference"] == f"compute:{stuck.id}":
+ return _response(503)
+ return _ok()
+
+ with (
+ override_settings(GATEWAY_COMPUTE_DEBIT_ENABLED=enabled),
+ patch(HTTPX_POST, side_effect=respond),
+ patch("posthog.metrics.push_registry") as push,
+ ):
+ settle_overdue_sandbox_compute(self.now)
+
+ push.assert_called_once()
+ assert push.call_args.args[0] == "tasks_compute_settlement"
+ registry = push.call_args.args[1]
+ assert registry.get_sample_value("posthog_tasks_hogland_sandbox_sessions_open_over_day") == 1
+ assert registry.get_sample_value("posthog_tasks_sandbox_sessions_unsettled_over_hour") == expected_unsettled
+
+ def test_sweep_settles_when_the_gauge_push_fails(self) -> None:
+ session = self._session()
+ with (
+ override_settings(PROM_PUSHGATEWAY_ADDRESS="http://pushgateway.internal"),
+ patch(HTTPX_POST, return_value=_ok()),
+ patch("posthog.metrics.push_to_gateway", side_effect=OSError("refused")) as push,
+ ):
+ assert settle_overdue_sandbox_compute(self.now) == [SettlementOutcome.SETTLED]
+
+ push.assert_called_once()
+ assert self._reload(session).compute_settled_at is not None
+
@parameterized.expand(
[("settles", _ok(), SettlementOutcome.SETTLED), ("retries", _response(503), SettlementOutcome.RETRY)]
)
diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py
index f06d1fad68ec..701363f0541e 100644
--- a/products/tasks/backend/metrics.py
+++ b/products/tasks/backend/metrics.py
@@ -3,6 +3,8 @@ from typing import TYPE_CHECKING, Literal
import structlog
from prometheus_client import Counter, Gauge, Histogram
+from posthog.metrics import pushed_metrics_registry
+
logger = structlog.get_logger(__name__)
if TYPE_CHECKING:
@@ -510,19 +512,23 @@ COMPUTE_SETTLEMENTS_TOTAL = Counter(
labelnames=["outcome"],
)
-HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY = Gauge(
- "posthog_tasks_hogland_sandbox_sessions_open_over_day",
- "Hogland sandbox sessions with no end stamp that opened more than a day ago",
- multiprocess_mode="mostrecent",
-)
-
def observe_compute_settlement(outcome: ComputeSettlementOutcome) -> None:
COMPUTE_SETTLEMENTS_TOTAL.labels(outcome=outcome).inc()
-def set_hogland_sandbox_sessions_open_over_day(count: int) -> None:
- HOGLAND_SANDBOX_SESSIONS_OPEN_OVER_DAY.set(count)
+def push_compute_settlement_sweep_gauges(*, hogland_open_over_day: int, unsettled_over_hour: int) -> None:
+ with pushed_metrics_registry("tasks_compute_settlement") as registry:
+ Gauge(
+ "posthog_tasks_hogland_sandbox_sessions_open_over_day",
+ "Hogland sandbox sessions with no end stamp that opened more than a day ago",
+ registry=registry,
+ ).set(hogland_open_over_day)
+ Gauge(
+ "posthog_tasks_sandbox_sessions_unsettled_over_hour",
+ "Attributed sandbox sessions that ended more than an hour ago and are neither settled nor parked",
+ registry=registry,
+ ).set(unsettled_over_hour)
# scoped: "true" when the delivery's installation resolved to at least one team, so the