:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): evaluate billing alerts on the platform (#113661)
Jimmy Zhudfe72f438217
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.pyindex 6cbc480138cd..3ea3ee79f83e 100644--- a/posthog/management/commands/start_temporal_worker.py+++ b/posthog/management/commands/start_temporal_worker.py@@ -179,6 +179,8 @@ from products.batch_exports.backend.facade.temporal import ()from products.billing_alerts.backend.temporal import (ACTIVITIES as BILLING_ALERTS_ACTIVITIES,+ PLATFORM_EVALUATION_ACTIVITIES as BILLING_PLATFORM_EVALUATION_ACTIVITIES,+ PLATFORM_EVALUATION_WORKFLOWS as BILLING_PLATFORM_EVALUATION_WORKFLOWS,WORKFLOWS as BILLING_ALERTS_WORKFLOWS,)from products.business_knowledge.backend.temporal import (@@ -648,10 +650,14 @@ _task_queue_specs = [),(settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,- ALERTS_PLATFORM_EVALUATION_WORKFLOWS + LOGS_SOURCE_EVALUATION_WORKFLOWS + INSIGHT_PLATFORM_EVALUATION_WORKFLOWS,+ ALERTS_PLATFORM_EVALUATION_WORKFLOWS+ + LOGS_SOURCE_EVALUATION_WORKFLOWS+ + INSIGHT_PLATFORM_EVALUATION_WORKFLOWS+ + BILLING_PLATFORM_EVALUATION_WORKFLOWS,ALERTS_PLATFORM_EVALUATION_ACTIVITIES+ LOGS_SOURCE_EVALUATION_ACTIVITIES- + INSIGHT_PLATFORM_EVALUATION_ACTIVITIES,+ + INSIGHT_PLATFORM_EVALUATION_ACTIVITIES+ + BILLING_PLATFORM_EVALUATION_ACTIVITIES,),(settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 6b07a41cd7b2..54355b4be8fc 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -421,6 +421,14 @@ if ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS <= 0:raise ImproperlyConfigured("ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS must be a positive integer, or no platform insight chart renders")+# Billing batch keys the platform starts per tick, across every team. A key is one team's due alerts+# and makes one billing service call, so this bounds the platform's load on that service. With a tick+# a minute, 60 times this must cover the keys due per hour, or hourly alerts fall behind their cadence.+ALERTS_PLATFORM_BILLING_KEYS_PER_TICK: int = get_from_env("ALERTS_PLATFORM_BILLING_KEYS_PER_TICK", 5, type_cast=int)+if ALERTS_PLATFORM_BILLING_KEYS_PER_TICK <= 0:+ raise ImproperlyConfigured(+ "ALERTS_PLATFORM_BILLING_KEYS_PER_TICK must be a positive integer, or no platform billing check starts"+ )# The platform's parallel logs batches allowed to run at once, across every team. A batch runs its# cohort queries one after another, so this bounds the parallel run's concurrent logs queries.ALERTS_PLATFORM_LOGS_MAX_INFLIGHT_BATCHES: int = get_from_env(diff --git a/products/alerts_platform/backend/delivery/message.py b/products/alerts_platform/backend/delivery/message.pyindex c5fafab895b7..0d82b5148fcf 100644--- a/products/alerts_platform/backend/delivery/message.py+++ b/products/alerts_platform/backend/delivery/message.py@@ -42,6 +42,7 @@ _SYMBOLS: Final[dict[AlertEventKind, str]] = {_SOURCE_LABELS: Final[dict[SourceKind, str]] = {SourceKind.LOGS: "Log",SourceKind.INSIGHT: "Insight",+ SourceKind.BILLING: "Billing",}# What a held row says, when cooldown or mute kept its announcement and only its incident moved.diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.pyindex 487e6fb1db32..7d4fe5a1f890 100644--- a/products/alerts_platform/backend/facade/contracts.py+++ b/products/alerts_platform/backend/facade/contracts.py@@ -29,6 +29,7 @@ if TYPE_CHECKING:class SourceKind(StrEnum):LOGS = "logs"INSIGHT = "insight"+ BILLING = "billing"@frozendiff --git a/products/alerts_platform/backend/facade/lifecycle.py b/products/alerts_platform/backend/facade/lifecycle.pyindex c44ccae18d8f..13ba709bca63 100644--- a/products/alerts_platform/backend/facade/lifecycle.py+++ b/products/alerts_platform/backend/facade/lifecycle.py@@ -105,9 +105,12 @@ class AlertPolicy:LOGS_ALERT_POLICY = AlertPolicy()-# Two constants rather than one changed default, because production logs keeps its own mute-# semantics until it migrates.-PLATFORM_LOGS_ALERT_POLICY = replace(LOGS_ALERT_POLICY, mute_gates_notification_only=True)+# What a source on the platform runs when nothing about its alerts needs a lifecycle of its own.+# It differs from `LOGS_ALERT_POLICY` because production logs keeps its own mute semantics until it+# migrates.+PLATFORM_DEFAULT_ALERT_POLICY = replace(LOGS_ALERT_POLICY, mute_gates_notification_only=True)++PLATFORM_LOGS_ALERT_POLICY = PLATFORM_DEFAULT_ALERT_POLICYBILLING_ALERT_POLICY = AlertPolicy(broken_is_terminal=False,diff --git a/products/alerts_platform/backend/temporal/sources.py b/products/alerts_platform/backend/temporal/sources.pyindex fb6e941c69c3..a938ab555dd0 100644--- a/products/alerts_platform/backend/temporal/sources.py+++ b/products/alerts_platform/backend/temporal/sources.py@@ -56,6 +56,12 @@ SOURCE_BINDINGS: dict[SourceKind, SourceBinding] = {# only start workflows that find the pool full.discovery_limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS,),+ SourceKind.BILLING: SourceBinding(+ workflow="billing-alert-platform-evaluate",+ task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,+ evaluation_timeout=dt.timedelta(minutes=2),+ discovery_limit=settings.ALERTS_PLATFORM_BILLING_KEYS_PER_TICK,+ ),}diff --git a/products/billing_alerts/backend/logic/evaluator.py b/products/billing_alerts/backend/logic/evaluator.pyindex c67c0e70d4ec..e0a1c9234480 100644--- a/products/billing_alerts/backend/logic/evaluator.py+++ b/products/billing_alerts/backend/logic/evaluator.py@@ -101,6 +101,15 @@ def _customer(billing_response: dict[str, Any]) -> dict[str, Any]:return customer+def current_period_amount(alert: BillingAlertConfiguration, billing_response: dict[str, Any]) -> Decimal | None:+ """The alert's metric for the current billing period, or None when the billing service has+ no total yet."""+ _validate_supported_metric(alert)+ amount_field = _METRIC_AMOUNT_FIELD[alert.metric]+ raw_amount = _customer(billing_response).get(amount_field)+ return None if raw_amount is None else _decimal(raw_amount, field=amount_field)++def evaluate_billing_alert(alert: BillingAlertConfiguration,*,@@ -126,7 +135,7 @@ def evaluate_billing_alert(period_end = _parse_period_boundary(billing_period.get("current_period_end")) or (default_start + timedelta(days=1))amount_field = _METRIC_AMOUNT_FIELD[alert.metric]- raw_amount = customer.get(amount_field)+ current_value = current_period_amount(alert, billing_response)payload: dict[str, Any] = {"expected_evaluation_date": expected_date.isoformat(),@@ -154,14 +163,12 @@ def evaluate_billing_alert(}return BillingAlertEvaluation(reason=reason, **values)- if raw_amount is None:+ if current_value is None:return result(reason="Billing status did not include a spend total for this billing period yet.",is_inconclusive=True,)- current_value = _decimal(raw_amount, field=amount_field)-if current_value < alert.minimum_value:return result(reason=f"Current value {current_value} is below the minimum value {alert.minimum_value}.",diff --git a/products/billing_alerts/backend/platform_source_cycle.py b/products/billing_alerts/backend/platform_source_cycle.pynew file mode 100644index 000000000000..1a2de24f6c07--- /dev/null+++ b/products/billing_alerts/backend/platform_source_cycle.py@@ -0,0 +1,293 @@+"""Evaluation of due billing alerts on the shared alerts platform.++Each check reads the organization's running spend total for the current billing period and+compares it with the alert's threshold. The platform's default lifecycle decides what the result+means, and the platform's interval schedule decides when the next check runs. A failed check needs+no retry of its own, because the next check reads the same running total.++Billing's own stack still evaluates the same alerts and is the only stack that notifies anyone.+This decides what the platform would have said and delivers nothing.++Each platform row resolves to its billing alert through `legacy_configuration_id`, because the+metric, the threshold, `enabled` and the snooze are edited on that row.+"""++from datetime import UTC, datetime, timedelta+from decimal import Decimal+from typing import Any+from uuid import UUID++from django.db.models import F++import structlog++from posthog.dataclasses import frozen++from products.alerts_platform.backend.facade.api import due_checks, slot_of+from products.alerts_platform.backend.facade.contracts import (+ PlatformAlertCheckInput,+ PlatformAlertOutcome,+ SkipReason,+ SourceBatchEvaluation,+ SourceKind,+)+from products.alerts_platform.backend.facade.lifecycle import (+ NOTIFICATION_EVENT_KINDS,+ PLATFORM_DEFAULT_ALERT_POLICY,+ AlertSnapshot,+ AlertState,+ CheckInput,+ ControlPlaneOutcome,+ NotificationAction,+ Outcome,+ decide_firing_episode,+ evaluate_alert_check,+)+from products.alerts_platform.backend.facade.payload import MAX_BATCH_PAYLOAD_BYTES, payload_bytes+from products.alerts_platform.backend.facade.platform_metrics import (+ increment_checks,+ increment_checks_deferred,+ increment_checks_skipped,+ increment_state_transition,+ record_scheduler_lag,+ safe_record,+)+from products.billing_alerts.backend.logic.evaluator import (+ BillingAlertEvaluationError,+ current_period_amount,+ fetch_billing_data,+)+from products.billing_alerts.backend.models import BillingAlertConfiguration++logger = structlog.get_logger(__name__)++SOURCE = SourceKind.BILLING.value++FETCH_FAILED = "Billing alert data fetch failed."+++@frozen+class _BillingStatus:+ """One organization's billing status, fetched once per batch, or None when the call failed."""++ response: dict[str, Any] | None+ duration_ms: int | None = None+++def evaluate_billing_batch(team_id: int, slot: str, cutoff: datetime) -> SourceBatchEvaluation:+ """Decides every due billing check in one batch key. Writes nothing."""+ checks = due_checks(team_id, SOURCE, slot, cutoff)+ if not checks:+ return SourceBatchEvaluation(outcomes=(), deliveries=())+ legacy_ids = [check.legacy_configuration_id for check in checks if check.legacy_configuration_id is not None]+ # Scoped to the batch's team and its organization, so a copy can only read an alert it belongs to.+ alerts = {+ alert.id: alert+ for alert in BillingAlertConfiguration.objects.filter(+ id__in=legacy_ids, team_id=team_id, organization_id=F("team__organization_id")+ ).select_related("organization")+ }+ statuses: dict[UUID, _BillingStatus] = {}++ # Read before any billing call, so the fetches do not count as scheduler lag.+ started_at = datetime.now(UTC)+ for check in checks:+ if check.next_check_at is not None:+ lag_ms = int((started_at - check.next_check_at).total_seconds() * 1000)+ if lag_ms > 0:+ safe_record(record_scheduler_lag, SOURCE, lag_ms)++ outcomes = tuple(+ _decide(+ check,+ alerts.get(check.legacy_configuration_id) if check.legacy_configuration_id else None,+ cutoff,+ statuses,+ )+ for check in checks+ )+ return _within_payload_bound(outcomes, team_id)+++def _within_payload_bound(outcomes: tuple[PlatformAlertOutcome, ...], team_id: int) -> SourceBatchEvaluation:+ """The outcomes that fit the platform's batch payload bound. A check left out keeps its due time,+ so a later tick evaluates it again. Billing fetches once per organization, so evaluating the+ whole batch before trimming costs no extra billing call."""+ kept: list[PlatformAlertOutcome] = []+ spent = 0+ for outcome in outcomes:+ size = payload_bytes(outcome)+ if size > MAX_BATCH_PAYLOAD_BYTES:+ # It never fits, so it would stay due forever without anyone seeing why.+ logger.error(+ "Billing alert outcome exceeds the whole batch payload budget",+ team_id=team_id,+ configuration_id=str(outcome.configuration_id),+ size_bytes=size,+ budget_bytes=MAX_BATCH_PAYLOAD_BYTES,+ )+ if spent + size > MAX_BATCH_PAYLOAD_BYTES:+ # Skip rather than stop, so one oversized outcome cannot hold back every check after it.+ continue+ spent += size+ kept.append(outcome)+ deferred = len(outcomes) - len(kept)+ if deferred:+ logger.warning(+ "Deferred billing alert checks over the batch payload byte budget",+ team_id=team_id,+ recorded=len(kept),+ deferred=deferred,+ budget_bytes=MAX_BATCH_PAYLOAD_BYTES,+ )+ safe_record(increment_checks_deferred, SOURCE, deferred)+ return SourceBatchEvaluation(outcomes=tuple(kept), deliveries=(), omitted=deferred)+++def _decide(+ check: PlatformAlertCheckInput,+ alert: BillingAlertConfiguration | None,+ now: datetime,+ statuses: dict[UUID, _BillingStatus],+) -> PlatformAlertOutcome:+ snapshot = _snapshot(check, alert)+ if alert is None:+ # The billing alert is gone or has moved to another team, so this copy can never run again.+ return _skipped(check, snapshot, now=now, disable=True)+ if not alert.enabled:+ return _skipped(check, snapshot, now=now)++ # A snooze set on the billing row after the backfill copied it lives only there. The platform+ # mutes rather than skips, so the check still tracks the total.+ muted = alert.snoozed_until is not None and alert.snoozed_until > now+ status = _billing_status(alert, statuses)+ if status.response is None:+ failed = CheckInput(threshold_breached=False, error_message=FETCH_FAILED, is_transient_error=True, muted=muted)+ return _verdict(check, snapshot, failed, now=now, skip=SkipReason.QUERY_FAILED)++ try:+ amount = current_period_amount(alert, status.response)+ except BillingAlertEvaluationError as error:+ # Billing's response, not the alert, is at fault, so it must not count toward breaking the copy:+ # a broken copy stays out of discovery after billing recovers.+ failed = CheckInput(threshold_breached=False, error_message=str(error), is_transient_error=True, muted=muted)+ return _verdict(check, snapshot, failed, now=now, skip=SkipReason.QUERY_FAILED)++ if amount is None:+ inconclusive = CheckInput(threshold_breached=False, is_inconclusive=True, muted=muted)+ return _verdict(check, snapshot, inconclusive, now=now, query_duration_ms=status.duration_ms)+ threshold = alert.threshold_value or Decimal("0")+ return _verdict(+ check,+ snapshot,+ CheckInput(threshold_breached=amount >= threshold, muted=muted),+ now=now,+ value=float(amount),+ query_duration_ms=status.duration_ms,+ )+++def _snapshot(check: PlatformAlertCheckInput, alert: BillingAlertConfiguration | None) -> AlertSnapshot:+ # A billing alert does not group, so its state is the one instance on the empty key.+ instance = check.instance()+ return AlertSnapshot(+ state=AlertState(instance.state),+ cooldown=timedelta(minutes=check.cooldown_minutes),+ last_notified_at=instance.last_notified_at,+ # The billing row's snooze, because the copy keeps the one the backfill saw after a user clears it.+ snooze_until=alert.snoozed_until if alert is not None else instance.snooze_until,+ consecutive_failures=check.consecutive_failures,+ firing_started_at=instance.firing_started_at,+ )+++def _billing_status(alert: BillingAlertConfiguration, statuses: dict[UUID, _BillingStatus]) -> _BillingStatus:+ """Fetches an organization's billing status once per batch key, however many of its alerts are due."""+ organization_id = alert.organization_id+ if organization_id not in statuses:+ try:+ response, duration_ms = fetch_billing_data(alert, alert.organization)+ statuses[organization_id] = _BillingStatus(response=response, duration_ms=duration_ms)+ except Exception as error:+ logger.warning("Platform billing data fetch failed", organization_id=str(organization_id), error=str(error))+ statuses[organization_id] = _BillingStatus(response=None)+ return statuses[organization_id]+++def _verdict(+ check: PlatformAlertCheckInput,+ snapshot: AlertSnapshot,+ check_input: CheckInput,+ *,+ now: datetime,+ value: float | None = None,+ query_duration_ms: int | None = None,+ skip: SkipReason | None = None,+) -> PlatformAlertOutcome:+ outcome = evaluate_alert_check(snapshot, check_input, now, policy=PLATFORM_DEFAULT_ALERT_POLICY)+ return _recorded(+ check,+ snapshot,+ outcome,+ now=now,+ notification=outcome.notification,+ muted_notification=outcome.muted_notification,+ notified=outcome.update_last_notified_at,+ value=value,+ error_message=outcome.error_message,+ query_duration_ms=query_duration_ms,+ skip=skip,+ disable=outcome.disable,+ )+++def _skipped(+ check: PlatformAlertCheckInput, snapshot: AlertSnapshot, *, now: datetime, disable: bool = False+) -> PlatformAlertOutcome:+ """A check that reached no verdict. Recorded all the same, so its schedule moves on."""+ unchanged = ControlPlaneOutcome(new_state=snapshot.state, consecutive_failures=snapshot.consecutive_failures)+ return _recorded(check, snapshot, unchanged, now=now, skip=SkipReason.SOURCE_RULE, disable=disable)+++def _recorded(+ check: PlatformAlertCheckInput,+ snapshot: AlertSnapshot,+ outcome: Outcome,+ *,+ now: datetime,+ notification: NotificationAction = NotificationAction.NONE,+ muted_notification: NotificationAction = NotificationAction.NONE,+ notified: bool = False,+ value: float | None = None,+ error_message: str | None = None,+ query_duration_ms: int | None = None,+ skip: SkipReason | None = None,+ disable: bool = False,+) -> PlatformAlertOutcome:+ """The one place an outcome is built. Every path goes through the firing decision, because+ an outcome without an episode clears the start of a firing the alert is still in."""+ safe_record(increment_checks, SOURCE, notification.value)+ if skip is not None:+ safe_record(increment_checks_skipped, SOURCE, skip.value)+ if snapshot.state != outcome.new_state:+ safe_record(increment_state_transition, SOURCE, snapshot.state.value, outcome.new_state.value)+ failed = skip == SkipReason.QUERY_FAILED+ return PlatformAlertOutcome.ungrouped(+ configuration_id=check.id,+ evaluation_key=f"slot:{slot_of(check.next_check_at, now)}",+ consecutive_failures=outcome.consecutive_failures,+ kind=NOTIFICATION_EVENT_KINDS[notification],+ new_state=outcome.new_state.value,+ notified=notified,+ failed=failed,+ firing_episode=decide_firing_episode(snapshot, outcome, now, policy=PLATFORM_DEFAULT_ALERT_POLICY),+ value=value,+ muted_notification="" if muted_notification == NotificationAction.NONE else muted_notification.value,+ error_message=error_message,+ query_duration_ms=query_duration_ms,+ disable=disable,+ skipped=skip is not None and not failed,+ # A batch key is one team, and so one organization, so one group keeps a team's alerts in+ # one slot and one billing call. A broken alert is never discovered again.+ schedule_group=None if outcome.new_state == AlertState.BROKEN else SOURCE,+ )diff --git a/products/billing_alerts/backend/temporal/__init__.py b/products/billing_alerts/backend/temporal/__init__.pyindex 0166dbf5747a..20198820848c 100644--- a/products/billing_alerts/backend/temporal/__init__.py+++ b/products/billing_alerts/backend/temporal/__init__.py@@ -2,6 +2,10 @@ from products.billing_alerts.backend.temporal.activities import (discover_due_billing_alerts_activity,evaluate_billing_alert_batch_activity,)+from products.billing_alerts.backend.temporal.platform_evaluate import (+ PLATFORM_EVALUATION_ACTIVITIES,+ PLATFORM_EVALUATION_WORKFLOWS,+)from products.billing_alerts.backend.temporal.workflows import (CheckBillingAlertBatchWorkflow,ScheduleDueBillingAlertChecksWorkflow,@@ -12,6 +16,13 @@ WORKFLOWS = [CheckBillingAlertBatchWorkflow,]+__all__ = [+ "ACTIVITIES",+ "PLATFORM_EVALUATION_ACTIVITIES",+ "PLATFORM_EVALUATION_WORKFLOWS",+ "WORKFLOWS",+]+ACTIVITIES = [discover_due_billing_alerts_activity,evaluate_billing_alert_batch_activity,diff --git a/products/billing_alerts/backend/temporal/platform_evaluate.py b/products/billing_alerts/backend/temporal/platform_evaluate.pynew file mode 100644index 000000000000..afe26afbed52--- /dev/null+++ b/products/billing_alerts/backend/temporal/platform_evaluate.py@@ -0,0 +1,77 @@+"""Temporal wiring for billing alert evaluation on the shared alerts platform.++The evaluation itself lives in `products/billing_alerts/backend/platform_source_cycle.py`, so it+stays a plain function a test can call without Temporal.+"""++import datetime as dt++from temporalio import activity, workflow+from temporalio.common import RetryPolicy++from posthog.temporal.common.base import PostHogWorkflow++with workflow.unsafe.imports_passed_through():+ from posthog.temporal.common.utils import close_db_connections++ from products.alerts_platform.backend.facade.contracts import (+ RECORD_OUTCOMES_ACTIVITY,+ SourceBatchEvaluation,+ SourceEvaluationInputs,+ SourceOutcomeInputs,+ )++WORKFLOW_NAME = "billing-alert-platform-evaluate"++# A batch key is one team, so a batch makes one billing service call and the budget covers a slow one.+EVALUATE_START_TO_CLOSE = dt.timedelta(seconds=60)+EVALUATE_SCHEDULE_TO_CLOSE = EVALUATE_START_TO_CLOSE + dt.timedelta(seconds=20)+RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)+RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)+# What the source needs from its binding's `evaluation_timeout`.+EVALUATION_BUDGET = EVALUATE_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE+++@close_db_connections+def evaluate_billing_alerts_activity(inputs: SourceEvaluationInputs) -> SourceBatchEvaluation:+ # Imported in the activity body, because the workflow below forces this module to evaluate+ # inside Temporal's sandbox, which a Django model import trips.+ from products.billing_alerts.backend.platform_source_cycle import evaluate_billing_batch # noqa: PLC0415++ return evaluate_billing_batch(+ inputs.batch_key.team_id, inputs.batch_key.slot, dt.datetime.fromisoformat(inputs.cutoff)+ )++[email protected](name=WORKFLOW_NAME)+class BillingAlertPlatformEvaluateWorkflow(PostHogWorkflow):+ """Evaluates one batch key and records what it decided. Delivers nothing, and writes only the+ shared platform's own rows: billing's own stack owns the billing tables."""++ inputs_cls = SourceEvaluationInputs++ @workflow.run+ async def run(self, inputs: SourceEvaluationInputs) -> int:+ # One attempt. A failed batch keeps its due time, so the next tick reaches it anyway.+ evaluation = await workflow.execute_activity(+ evaluate_billing_alerts_activity,+ inputs,+ start_to_close_timeout=EVALUATE_START_TO_CLOSE,+ schedule_to_close_timeout=EVALUATE_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ if not evaluation.outcomes:+ return 0+ return await workflow.execute_activity(+ RECORD_OUTCOMES_ACTIVITY,+ SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=evaluation.outcomes),+ start_to_close_timeout=RECORD_START_TO_CLOSE,+ schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=3),+ result_type=int,+ )+++PLATFORM_EVALUATION_WORKFLOWS = [BillingAlertPlatformEvaluateWorkflow]+PLATFORM_EVALUATION_ACTIVITIES = [evaluate_billing_alerts_activity]diff --git a/products/billing_alerts/backend/test/test_platform_source_cycle.py b/products/billing_alerts/backend/test/test_platform_source_cycle.pynew file mode 100644index 000000000000..58cd7870bed9--- /dev/null+++ b/products/billing_alerts/backend/test/test_platform_source_cycle.py@@ -0,0 +1,213 @@+from datetime import UTC, datetime, timedelta+from decimal import Decimal+from typing import Any+from uuid import UUID, uuid4++import time_machine+from posthog.test.base import BaseTest+from unittest.mock import patch++from parameterized import parameterized++from posthog.models.scoping import team_scope++from products.alerts_platform.backend.facade.api import record_outcomes, slot_of, upsert_configuration+from products.alerts_platform.backend.facade.contracts import (+ AlertEventKind,+ CheckFailure,+ GroupOutcome,+ PlatformAlertOutcome,+ PlatformAlertUpsert,+ SourceBatchEvaluation,+ SourceKind,+)+from products.alerts_platform.backend.facade.temporal import source_evaluation_timeout+from products.alerts_platform.backend.facade.testing import (+ configuration,+ count_still_due,+ create_configuration,+ set_due_at,+)+from products.billing_alerts.backend.models import BillingAlertConfiguration, BillingAlertEvaluationClaim+from products.billing_alerts.backend.platform_source_cycle import evaluate_billing_batch+from products.billing_alerts.backend.temporal.platform_evaluate import EVALUATION_BUDGET++_MODULE = "products.billing_alerts.backend.platform_source_cycle"+CUTOFF = datetime(2026, 6, 23, 12, 0, tzinfo=UTC)+DUE_AT = CUTOFF - timedelta(minutes=1)+++def _billing_response(current: int | None) -> dict[str, Any]:+ customer: dict[str, Any] = {}+ if current is not None:+ customer["current_total_amount_usd_after_discount"] = str(current)+ return {"customer": customer}+++def _decided(outcome: PlatformAlertOutcome) -> GroupOutcome | CheckFailure:+ """The one verdict an ungrouped billing check reports, or its failure."""+ return outcome.failure or outcome.groups[0]+++@time_machine.travel(CUTOFF, tick=False)+class TestPlatformBillingEvaluation(BaseTest):+ def _alert(self, **overrides: Any) -> BillingAlertConfiguration:+ fields: dict[str, Any] = {+ "organization_id": self.organization.id,+ "team_id": self.team.id,+ "created_by_id": self.user.id,+ "name": "Period spend cap",+ "metric": BillingAlertConfiguration.Metric.SPEND,+ "threshold_type": BillingAlertConfiguration.ThresholdType.ABSOLUTE_VALUE,+ "threshold_value": Decimal("100"),+ }+ fields.update(overrides)+ return BillingAlertConfiguration.objects.create(**fields)++ def _copy(self, alert: BillingAlertConfiguration | None) -> UUID:+ with team_scope(self.team.id):+ return create_configuration(+ team=self.team,+ name="Period spend cap",+ source_kind=SourceKind.BILLING.value,+ check_interval_minutes=60,+ cooldown_minutes=0,+ next_check_at=DUE_AT,+ legacy_configuration_id=alert.id if alert else uuid4(),+ ).id++ def _evaluate(self, *responses: dict[str, Any] | Exception) -> tuple[SourceBatchEvaluation, Any]:+ side_effect = [r if isinstance(r, Exception) else (r, 12) for r in responses]+ with patch(f"{_MODULE}.fetch_billing_data", side_effect=side_effect) as fetch:+ evaluation = evaluate_billing_batch(self.team.id, slot_of(DUE_AT, CUTOFF), CUTOFF)+ return evaluation, fetch++ def test_a_breach_fires_on_the_platform_and_leaves_billings_tables_alone(self) -> None:+ alert = self._alert()+ before = BillingAlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id)+ self._copy(alert)++ evaluation, _ = self._evaluate(_billing_response(150))++ (outcome,) = evaluation.outcomes+ (group,) = outcome.groups+ assert (group.kind, group.new_state, group.value, outcome.evaluation_key) == (+ AlertEventKind.FIRING,+ "firing",+ 150.0,+ f"slot:{slot_of(DUE_AT, CUTOFF)}",+ )+ assert evaluation.deliveries == ()+ assert (+ BillingAlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id)+ == before+ )+ assert not BillingAlertEvaluationClaim.objects.filter(alert=alert).exists()++ @parameterized.expand(+ [+ ("no_total_yet", _billing_response(None)),+ ("billing_service_down", ConnectionError("billing service timed out")),+ ("billing_returns_no_customer", {}),+ ("billing_returns_an_unreadable_amount", {"customer": {"current_total_amount_usd_after_discount": "n/a"}}),+ ]+ )+ def test_a_check_that_reads_no_total_leaves_a_firing_alert_firing_quietly(+ self, _name: str, response: dict[str, Any] | Exception+ ) -> None:+ configuration_id = self._copy(self._alert())+ fired, _ = self._evaluate(_billing_response(150))+ with team_scope(self.team.id):+ record_outcomes(self.team.id, fired.outcomes, CUTOFF)+ set_due_at(configuration_id, DUE_AT)++ evaluation, _ = self._evaluate(response)++ (outcome,) = evaluation.outcomes+ decided = _decided(outcome)+ assert (decided.kind, decided.new_state, outcome.consecutive_failures, outcome.disable) == (+ AlertEventKind.CHECK,+ "firing",+ 0,+ False,+ )++ def test_a_snooze_cleared_on_the_billing_alert_after_the_backfill_no_longer_holds_the_fire(self) -> None:+ alert = self._alert()+ upsert_configuration(+ PlatformAlertUpsert(+ legacy_configuration_id=alert.id,+ team_id=self.team.id,+ name=alert.name,+ enabled=True,+ source_kind=SourceKind.BILLING,+ source_config={},+ check_interval_minutes=60,+ evaluation_periods=1,+ datapoints_to_alarm=1,+ cooldown_minutes=0,+ schedule_restriction=None,+ next_check_at=DUE_AT,+ # The snooze the backfill copied, since cleared on the billing alert.+ snooze_until=CUTOFF + timedelta(hours=2),+ )+ )++ evaluation, _ = self._evaluate(_billing_response(150))++ (outcome,) = evaluation.outcomes+ (group,) = outcome.groups+ assert (group.kind, group.muted_notification) == (AlertEventKind.FIRING, "")++ def test_a_snooze_set_on_the_billing_alert_holds_the_fire(self) -> None:+ self._copy(self._alert(snoozed_until=CUTOFF + timedelta(hours=2)))++ evaluation, _ = self._evaluate(_billing_response(150))++ (outcome,) = evaluation.outcomes+ (group,) = outcome.groups+ assert (group.kind, group.muted_notification) == (AlertEventKind.CHECK, "fire")++ def test_one_billing_call_serves_every_alert_of_an_organization_in_a_batch(self) -> None:+ configuration_ids = [self._copy(self._alert(name="First")), self._copy(self._alert(name="Second"))]++ evaluation, fetch = self._evaluate(_billing_response(150))+ with team_scope(self.team.id):+ record_outcomes(self.team.id, evaluation.outcomes, CUTOFF)+ next_checks = {configuration(c).next_check_at for c in configuration_ids}++ assert fetch.call_count == 1+ assert [o.groups[0].kind for o in evaluation.outcomes] == [AlertEventKind.FIRING, AlertEventKind.FIRING]+ assert len(next_checks) == 1++ @parameterized.expand([("gone", None, True), ("disabled", {"enabled": False}, False)])+ def test_a_copy_of_an_alert_that_cannot_run_skips_without_a_billing_call(+ self, _name: str, overrides: dict[str, Any] | None, disables_the_copy: bool+ ) -> None:+ self._copy(None if overrides is None else self._alert(**overrides))++ evaluation, fetch = self._evaluate()++ fetch.assert_not_called()+ (outcome,) = evaluation.outcomes+ (group,) = outcome.groups+ assert (group.kind, outcome.disable) == (AlertEventKind.CHECK, disables_the_copy)++ def test_checks_past_the_payload_bound_are_left_due(self) -> None:+ configuration_ids = [self._copy(self._alert(name=f"Bound {n}")) for n in range(3)]++ # The first outcome alone overruns a 25-byte budget; the two after it fit.+ with (+ patch(f"{_MODULE}.payload_bytes", side_effect=[30, 10, 10]),+ patch(f"{_MODULE}.MAX_BATCH_PAYLOAD_BYTES", 25),+ ):+ evaluation, _ = self._evaluate(_billing_response(150))+ with team_scope(self.team.id):+ record_outcomes(self.team.id, evaluation.outcomes, CUTOFF)+ still_due = count_still_due(configuration_ids, at=CUTOFF)++ assert (len(evaluation.outcomes), evaluation.omitted, still_due) == (2, 1, 1)+++def test_the_binding_holds_the_whole_batch() -> None:+ assert source_evaluation_timeout(SourceKind.BILLING) > EVALUATION_BUDGETdiff --git a/tach.toml b/tach.tomlindex a292cc3f57e7..0809bb9ca260 100644--- a/tach.toml+++ b/tach.toml@@ -1189,11 +1189,14 @@ from = ["products.experiments",]-# Billing alerts registers product-owned workflows and a schedule with the shared billing worker.+# Billing alerts registers product-owned workflows and a schedule with the shared billing worker,+# and its platform evaluation with the alerts platform's evaluation worker.[[interfaces]]expose = ["backend\\.temporal\\.ACTIVITIES",+ "backend\\.temporal\\.PLATFORM_EVALUATION_ACTIVITIES",+ "backend\\.temporal\\.PLATFORM_EVALUATION_WORKFLOWS","backend\\.temporal\\.WORKFLOWS","backend\\.temporal\\.schedule\\.create_schedule_due_billing_alert_checks_schedule",]