Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): evaluate billing alerts on the platform (#113661)

Jimmy Zhudfe72f438217
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 6cbc480138cd..3ea3ee79f83e 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -179,6 +179,8 @@ from products.batch_exports.backend.facade.temporal import (
)
from products.billing_alerts.backend.temporal import (
ACTIVITIES as BILLING_ALERTS_ACTIVITIES,
+ PLATFORM_EVALUATION_ACTIVITIES as BILLING_PLATFORM_EVALUATION_ACTIVITIES,
+ PLATFORM_EVALUATION_WORKFLOWS as BILLING_PLATFORM_EVALUATION_WORKFLOWS,
WORKFLOWS as BILLING_ALERTS_WORKFLOWS,
)
from products.business_knowledge.backend.temporal import (
@@ -648,10 +650,14 @@ _task_queue_specs = [
),
(
settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,
- ALERTS_PLATFORM_EVALUATION_WORKFLOWS + LOGS_SOURCE_EVALUATION_WORKFLOWS + INSIGHT_PLATFORM_EVALUATION_WORKFLOWS,
+ ALERTS_PLATFORM_EVALUATION_WORKFLOWS
+ + LOGS_SOURCE_EVALUATION_WORKFLOWS
+ + INSIGHT_PLATFORM_EVALUATION_WORKFLOWS
+ + BILLING_PLATFORM_EVALUATION_WORKFLOWS,
ALERTS_PLATFORM_EVALUATION_ACTIVITIES
+ LOGS_SOURCE_EVALUATION_ACTIVITIES
- + INSIGHT_PLATFORM_EVALUATION_ACTIVITIES,
+ + INSIGHT_PLATFORM_EVALUATION_ACTIVITIES
+ + BILLING_PLATFORM_EVALUATION_ACTIVITIES,
),
(
settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 6b07a41cd7b2..54355b4be8fc 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -421,6 +421,14 @@ if ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS <= 0:
raise ImproperlyConfigured(
"ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS must be a positive integer, or no platform insight chart renders"
)
+# Billing batch keys the platform starts per tick, across every team. A key is one team's due alerts
+# and makes one billing service call, so this bounds the platform's load on that service. With a tick
+# a minute, 60 times this must cover the keys due per hour, or hourly alerts fall behind their cadence.
+ALERTS_PLATFORM_BILLING_KEYS_PER_TICK: int = get_from_env("ALERTS_PLATFORM_BILLING_KEYS_PER_TICK", 5, type_cast=int)
+if ALERTS_PLATFORM_BILLING_KEYS_PER_TICK <= 0:
+ raise ImproperlyConfigured(
+ "ALERTS_PLATFORM_BILLING_KEYS_PER_TICK must be a positive integer, or no platform billing check starts"
+ )
# The platform's parallel logs batches allowed to run at once, across every team. A batch runs its
# cohort queries one after another, so this bounds the parallel run's concurrent logs queries.
ALERTS_PLATFORM_LOGS_MAX_INFLIGHT_BATCHES: int = get_from_env(
diff --git a/products/alerts_platform/backend/delivery/message.py b/products/alerts_platform/backend/delivery/message.py
index c5fafab895b7..0d82b5148fcf 100644
--- a/products/alerts_platform/backend/delivery/message.py
+++ b/products/alerts_platform/backend/delivery/message.py
@@ -42,6 +42,7 @@ _SYMBOLS: Final[dict[AlertEventKind, str]] = {
_SOURCE_LABELS: Final[dict[SourceKind, str]] = {
SourceKind.LOGS: "Log",
SourceKind.INSIGHT: "Insight",
+ SourceKind.BILLING: "Billing",
}
# What a held row says, when cooldown or mute kept its announcement and only its incident moved.
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index 487e6fb1db32..7d4fe5a1f890 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -29,6 +29,7 @@ if TYPE_CHECKING:
class SourceKind(StrEnum):
LOGS = "logs"
INSIGHT = "insight"
+ BILLING = "billing"
@frozen
diff --git a/products/alerts_platform/backend/facade/lifecycle.py b/products/alerts_platform/backend/facade/lifecycle.py
index c44ccae18d8f..13ba709bca63 100644
--- a/products/alerts_platform/backend/facade/lifecycle.py
+++ b/products/alerts_platform/backend/facade/lifecycle.py
@@ -105,9 +105,12 @@ class AlertPolicy:
LOGS_ALERT_POLICY = AlertPolicy()
-# Two constants rather than one changed default, because production logs keeps its own mute
-# semantics until it migrates.
-PLATFORM_LOGS_ALERT_POLICY = replace(LOGS_ALERT_POLICY, mute_gates_notification_only=True)
+# What a source on the platform runs when nothing about its alerts needs a lifecycle of its own.
+# It differs from `LOGS_ALERT_POLICY` because production logs keeps its own mute semantics until it
+# migrates.
+PLATFORM_DEFAULT_ALERT_POLICY = replace(LOGS_ALERT_POLICY, mute_gates_notification_only=True)
+
+PLATFORM_LOGS_ALERT_POLICY = PLATFORM_DEFAULT_ALERT_POLICY
BILLING_ALERT_POLICY = AlertPolicy(
broken_is_terminal=False,
diff --git a/products/alerts_platform/backend/temporal/sources.py b/products/alerts_platform/backend/temporal/sources.py
index fb6e941c69c3..a938ab555dd0 100644
--- a/products/alerts_platform/backend/temporal/sources.py
+++ b/products/alerts_platform/backend/temporal/sources.py
@@ -56,6 +56,12 @@ SOURCE_BINDINGS: dict[SourceKind, SourceBinding] = {
# only start workflows that find the pool full.
discovery_limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS,
),
+ SourceKind.BILLING: SourceBinding(
+ workflow="billing-alert-platform-evaluate",
+ task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,
+ evaluation_timeout=dt.timedelta(minutes=2),
+ discovery_limit=settings.ALERTS_PLATFORM_BILLING_KEYS_PER_TICK,
+ ),
}
diff --git a/products/billing_alerts/backend/logic/evaluator.py b/products/billing_alerts/backend/logic/evaluator.py
index c67c0e70d4ec..e0a1c9234480 100644
--- a/products/billing_alerts/backend/logic/evaluator.py
+++ b/products/billing_alerts/backend/logic/evaluator.py
@@ -101,6 +101,15 @@ def _customer(billing_response: dict[str, Any]) -> dict[str, Any]:
return customer
+def current_period_amount(alert: BillingAlertConfiguration, billing_response: dict[str, Any]) -> Decimal | None:
+ """The alert's metric for the current billing period, or None when the billing service has
+ no total yet."""
+ _validate_supported_metric(alert)
+ amount_field = _METRIC_AMOUNT_FIELD[alert.metric]
+ raw_amount = _customer(billing_response).get(amount_field)
+ return None if raw_amount is None else _decimal(raw_amount, field=amount_field)
+
+
def evaluate_billing_alert(
alert: BillingAlertConfiguration,
*,
@@ -126,7 +135,7 @@ def evaluate_billing_alert(
period_end = _parse_period_boundary(billing_period.get("current_period_end")) or (default_start + timedelta(days=1))
amount_field = _METRIC_AMOUNT_FIELD[alert.metric]
- raw_amount = customer.get(amount_field)
+ current_value = current_period_amount(alert, billing_response)
payload: dict[str, Any] = {
"expected_evaluation_date": expected_date.isoformat(),
@@ -154,14 +163,12 @@ def evaluate_billing_alert(
}
return BillingAlertEvaluation(reason=reason, **values)
- if raw_amount is None:
+ if current_value is None:
return result(
reason="Billing status did not include a spend total for this billing period yet.",
is_inconclusive=True,
)
- current_value = _decimal(raw_amount, field=amount_field)
-
if current_value < alert.minimum_value:
return result(
reason=f"Current value {current_value} is below the minimum value {alert.minimum_value}.",
diff --git a/products/billing_alerts/backend/platform_source_cycle.py b/products/billing_alerts/backend/platform_source_cycle.py
new file mode 100644
index 000000000000..1a2de24f6c07
--- /dev/null
+++ b/products/billing_alerts/backend/platform_source_cycle.py
@@ -0,0 +1,293 @@
+"""Evaluation of due billing alerts on the shared alerts platform.
+
+Each check reads the organization's running spend total for the current billing period and
+compares it with the alert's threshold. The platform's default lifecycle decides what the result
+means, and the platform's interval schedule decides when the next check runs. A failed check needs
+no retry of its own, because the next check reads the same running total.
+
+Billing's own stack still evaluates the same alerts and is the only stack that notifies anyone.
+This decides what the platform would have said and delivers nothing.
+
+Each platform row resolves to its billing alert through `legacy_configuration_id`, because the
+metric, the threshold, `enabled` and the snooze are edited on that row.
+"""
+
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from typing import Any
+from uuid import UUID
+
+from django.db.models import F
+
+import structlog
+
+from posthog.dataclasses import frozen
+
+from products.alerts_platform.backend.facade.api import due_checks, slot_of
+from products.alerts_platform.backend.facade.contracts import (
+ PlatformAlertCheckInput,
+ PlatformAlertOutcome,
+ SkipReason,
+ SourceBatchEvaluation,
+ SourceKind,
+)
+from products.alerts_platform.backend.facade.lifecycle import (
+ NOTIFICATION_EVENT_KINDS,
+ PLATFORM_DEFAULT_ALERT_POLICY,
+ AlertSnapshot,
+ AlertState,
+ CheckInput,
+ ControlPlaneOutcome,
+ NotificationAction,
+ Outcome,
+ decide_firing_episode,
+ evaluate_alert_check,
+)
+from products.alerts_platform.backend.facade.payload import MAX_BATCH_PAYLOAD_BYTES, payload_bytes
+from products.alerts_platform.backend.facade.platform_metrics import (
+ increment_checks,
+ increment_checks_deferred,
+ increment_checks_skipped,
+ increment_state_transition,
+ record_scheduler_lag,
+ safe_record,
+)
+from products.billing_alerts.backend.logic.evaluator import (
+ BillingAlertEvaluationError,
+ current_period_amount,
+ fetch_billing_data,
+)
+from products.billing_alerts.backend.models import BillingAlertConfiguration
+
+logger = structlog.get_logger(__name__)
+
+SOURCE = SourceKind.BILLING.value
+
+FETCH_FAILED = "Billing alert data fetch failed."
+
+
+@frozen
+class _BillingStatus:
+ """One organization's billing status, fetched once per batch, or None when the call failed."""
+
+ response: dict[str, Any] | None
+ duration_ms: int | None = None
+
+
+def evaluate_billing_batch(team_id: int, slot: str, cutoff: datetime) -> SourceBatchEvaluation:
+ """Decides every due billing check in one batch key. Writes nothing."""
+ checks = due_checks(team_id, SOURCE, slot, cutoff)
+ if not checks:
+ return SourceBatchEvaluation(outcomes=(), deliveries=())
+ legacy_ids = [check.legacy_configuration_id for check in checks if check.legacy_configuration_id is not None]
+ # Scoped to the batch's team and its organization, so a copy can only read an alert it belongs to.
+ alerts = {
+ alert.id: alert
+ for alert in BillingAlertConfiguration.objects.filter(
+ id__in=legacy_ids, team_id=team_id, organization_id=F("team__organization_id")
+ ).select_related("organization")
+ }
+ statuses: dict[UUID, _BillingStatus] = {}
+
+ # Read before any billing call, so the fetches do not count as scheduler lag.
+ started_at = datetime.now(UTC)
+ for check in checks:
+ if check.next_check_at is not None:
+ lag_ms = int((started_at - check.next_check_at).total_seconds() * 1000)
+ if lag_ms > 0:
+ safe_record(record_scheduler_lag, SOURCE, lag_ms)
+
+ outcomes = tuple(
+ _decide(
+ check,
+ alerts.get(check.legacy_configuration_id) if check.legacy_configuration_id else None,
+ cutoff,
+ statuses,
+ )
+ for check in checks
+ )
+ return _within_payload_bound(outcomes, team_id)
+
+
+def _within_payload_bound(outcomes: tuple[PlatformAlertOutcome, ...], team_id: int) -> SourceBatchEvaluation:
+ """The outcomes that fit the platform's batch payload bound. A check left out keeps its due time,
+ so a later tick evaluates it again. Billing fetches once per organization, so evaluating the
+ whole batch before trimming costs no extra billing call."""
+ kept: list[PlatformAlertOutcome] = []
+ spent = 0
+ for outcome in outcomes:
+ size = payload_bytes(outcome)
+ if size > MAX_BATCH_PAYLOAD_BYTES:
+ # It never fits, so it would stay due forever without anyone seeing why.
+ logger.error(
+ "Billing alert outcome exceeds the whole batch payload budget",
+ team_id=team_id,
+ configuration_id=str(outcome.configuration_id),
+ size_bytes=size,
+ budget_bytes=MAX_BATCH_PAYLOAD_BYTES,
+ )
+ if spent + size > MAX_BATCH_PAYLOAD_BYTES:
+ # Skip rather than stop, so one oversized outcome cannot hold back every check after it.
+ continue
+ spent += size
+ kept.append(outcome)
+ deferred = len(outcomes) - len(kept)
+ if deferred:
+ logger.warning(
+ "Deferred billing alert checks over the batch payload byte budget",
+ team_id=team_id,
+ recorded=len(kept),
+ deferred=deferred,
+ budget_bytes=MAX_BATCH_PAYLOAD_BYTES,
+ )
+ safe_record(increment_checks_deferred, SOURCE, deferred)
+ return SourceBatchEvaluation(outcomes=tuple(kept), deliveries=(), omitted=deferred)
+
+
+def _decide(
+ check: PlatformAlertCheckInput,
+ alert: BillingAlertConfiguration | None,
+ now: datetime,
+ statuses: dict[UUID, _BillingStatus],
+) -> PlatformAlertOutcome:
+ snapshot = _snapshot(check, alert)
+ if alert is None:
+ # The billing alert is gone or has moved to another team, so this copy can never run again.
+ return _skipped(check, snapshot, now=now, disable=True)
+ if not alert.enabled:
+ return _skipped(check, snapshot, now=now)
+
+ # A snooze set on the billing row after the backfill copied it lives only there. The platform
+ # mutes rather than skips, so the check still tracks the total.
+ muted = alert.snoozed_until is not None and alert.snoozed_until > now
+ status = _billing_status(alert, statuses)
+ if status.response is None:
+ failed = CheckInput(threshold_breached=False, error_message=FETCH_FAILED, is_transient_error=True, muted=muted)
+ return _verdict(check, snapshot, failed, now=now, skip=SkipReason.QUERY_FAILED)
+
+ try:
+ amount = current_period_amount(alert, status.response)
+ except BillingAlertEvaluationError as error:
+ # Billing's response, not the alert, is at fault, so it must not count toward breaking the copy:
+ # a broken copy stays out of discovery after billing recovers.
+ failed = CheckInput(threshold_breached=False, error_message=str(error), is_transient_error=True, muted=muted)
+ return _verdict(check, snapshot, failed, now=now, skip=SkipReason.QUERY_FAILED)
+
+ if amount is None:
+ inconclusive = CheckInput(threshold_breached=False, is_inconclusive=True, muted=muted)
+ return _verdict(check, snapshot, inconclusive, now=now, query_duration_ms=status.duration_ms)
+ threshold = alert.threshold_value or Decimal("0")
+ return _verdict(
+ check,
+ snapshot,
+ CheckInput(threshold_breached=amount >= threshold, muted=muted),
+ now=now,
+ value=float(amount),
+ query_duration_ms=status.duration_ms,
+ )
+
+
+def _snapshot(check: PlatformAlertCheckInput, alert: BillingAlertConfiguration | None) -> AlertSnapshot:
+ # A billing alert does not group, so its state is the one instance on the empty key.
+ instance = check.instance()
+ return AlertSnapshot(
+ state=AlertState(instance.state),
+ cooldown=timedelta(minutes=check.cooldown_minutes),
+ last_notified_at=instance.last_notified_at,
+ # The billing row's snooze, because the copy keeps the one the backfill saw after a user clears it.
+ snooze_until=alert.snoozed_until if alert is not None else instance.snooze_until,
+ consecutive_failures=check.consecutive_failures,
+ firing_started_at=instance.firing_started_at,
+ )
+
+
+def _billing_status(alert: BillingAlertConfiguration, statuses: dict[UUID, _BillingStatus]) -> _BillingStatus:
+ """Fetches an organization's billing status once per batch key, however many of its alerts are due."""
+ organization_id = alert.organization_id
+ if organization_id not in statuses:
+ try:
+ response, duration_ms = fetch_billing_data(alert, alert.organization)
+ statuses[organization_id] = _BillingStatus(response=response, duration_ms=duration_ms)
+ except Exception as error:
+ logger.warning("Platform billing data fetch failed", organization_id=str(organization_id), error=str(error))
+ statuses[organization_id] = _BillingStatus(response=None)
+ return statuses[organization_id]
+
+
+def _verdict(
+ check: PlatformAlertCheckInput,
+ snapshot: AlertSnapshot,
+ check_input: CheckInput,
+ *,
+ now: datetime,
+ value: float | None = None,
+ query_duration_ms: int | None = None,
+ skip: SkipReason | None = None,
+) -> PlatformAlertOutcome:
+ outcome = evaluate_alert_check(snapshot, check_input, now, policy=PLATFORM_DEFAULT_ALERT_POLICY)
+ return _recorded(
+ check,
+ snapshot,
+ outcome,
+ now=now,
+ notification=outcome.notification,
+ muted_notification=outcome.muted_notification,
+ notified=outcome.update_last_notified_at,
+ value=value,
+ error_message=outcome.error_message,
+ query_duration_ms=query_duration_ms,
+ skip=skip,
+ disable=outcome.disable,
+ )
+
+
+def _skipped(
+ check: PlatformAlertCheckInput, snapshot: AlertSnapshot, *, now: datetime, disable: bool = False
+) -> PlatformAlertOutcome:
+ """A check that reached no verdict. Recorded all the same, so its schedule moves on."""
+ unchanged = ControlPlaneOutcome(new_state=snapshot.state, consecutive_failures=snapshot.consecutive_failures)
+ return _recorded(check, snapshot, unchanged, now=now, skip=SkipReason.SOURCE_RULE, disable=disable)
+
+
+def _recorded(
+ check: PlatformAlertCheckInput,
+ snapshot: AlertSnapshot,
+ outcome: Outcome,
+ *,
+ now: datetime,
+ notification: NotificationAction = NotificationAction.NONE,
+ muted_notification: NotificationAction = NotificationAction.NONE,
+ notified: bool = False,
+ value: float | None = None,
+ error_message: str | None = None,
+ query_duration_ms: int | None = None,
+ skip: SkipReason | None = None,
+ disable: bool = False,
+) -> PlatformAlertOutcome:
+ """The one place an outcome is built. Every path goes through the firing decision, because
+ an outcome without an episode clears the start of a firing the alert is still in."""
+ safe_record(increment_checks, SOURCE, notification.value)
+ if skip is not None:
+ safe_record(increment_checks_skipped, SOURCE, skip.value)
+ if snapshot.state != outcome.new_state:
+ safe_record(increment_state_transition, SOURCE, snapshot.state.value, outcome.new_state.value)
+ failed = skip == SkipReason.QUERY_FAILED
+ return PlatformAlertOutcome.ungrouped(
+ configuration_id=check.id,
+ evaluation_key=f"slot:{slot_of(check.next_check_at, now)}",
+ consecutive_failures=outcome.consecutive_failures,
+ kind=NOTIFICATION_EVENT_KINDS[notification],
+ new_state=outcome.new_state.value,
+ notified=notified,
+ failed=failed,
+ firing_episode=decide_firing_episode(snapshot, outcome, now, policy=PLATFORM_DEFAULT_ALERT_POLICY),
+ value=value,
+ muted_notification="" if muted_notification == NotificationAction.NONE else muted_notification.value,
+ error_message=error_message,
+ query_duration_ms=query_duration_ms,
+ disable=disable,
+ skipped=skip is not None and not failed,
+ # A batch key is one team, and so one organization, so one group keeps a team's alerts in
+ # one slot and one billing call. A broken alert is never discovered again.
+ schedule_group=None if outcome.new_state == AlertState.BROKEN else SOURCE,
+ )
diff --git a/products/billing_alerts/backend/temporal/__init__.py b/products/billing_alerts/backend/temporal/__init__.py
index 0166dbf5747a..20198820848c 100644
--- a/products/billing_alerts/backend/temporal/__init__.py
+++ b/products/billing_alerts/backend/temporal/__init__.py
@@ -2,6 +2,10 @@ from products.billing_alerts.backend.temporal.activities import (
discover_due_billing_alerts_activity,
evaluate_billing_alert_batch_activity,
)
+from products.billing_alerts.backend.temporal.platform_evaluate import (
+ PLATFORM_EVALUATION_ACTIVITIES,
+ PLATFORM_EVALUATION_WORKFLOWS,
+)
from products.billing_alerts.backend.temporal.workflows import (
CheckBillingAlertBatchWorkflow,
ScheduleDueBillingAlertChecksWorkflow,
@@ -12,6 +16,13 @@ WORKFLOWS = [
CheckBillingAlertBatchWorkflow,
]
+__all__ = [
+ "ACTIVITIES",
+ "PLATFORM_EVALUATION_ACTIVITIES",
+ "PLATFORM_EVALUATION_WORKFLOWS",
+ "WORKFLOWS",
+]
+
ACTIVITIES = [
discover_due_billing_alerts_activity,
evaluate_billing_alert_batch_activity,
diff --git a/products/billing_alerts/backend/temporal/platform_evaluate.py b/products/billing_alerts/backend/temporal/platform_evaluate.py
new file mode 100644
index 000000000000..afe26afbed52
--- /dev/null
+++ b/products/billing_alerts/backend/temporal/platform_evaluate.py
@@ -0,0 +1,77 @@
+"""Temporal wiring for billing alert evaluation on the shared alerts platform.
+
+The evaluation itself lives in `products/billing_alerts/backend/platform_source_cycle.py`, so it
+stays a plain function a test can call without Temporal.
+"""
+
+import datetime as dt
+
+from temporalio import activity, workflow
+from temporalio.common import RetryPolicy
+
+from posthog.temporal.common.base import PostHogWorkflow
+
+with workflow.unsafe.imports_passed_through():
+ from posthog.temporal.common.utils import close_db_connections
+
+ from products.alerts_platform.backend.facade.contracts import (
+ RECORD_OUTCOMES_ACTIVITY,
+ SourceBatchEvaluation,
+ SourceEvaluationInputs,
+ SourceOutcomeInputs,
+ )
+
+WORKFLOW_NAME = "billing-alert-platform-evaluate"
+
+# A batch key is one team, so a batch makes one billing service call and the budget covers a slow one.
+EVALUATE_START_TO_CLOSE = dt.timedelta(seconds=60)
+EVALUATE_SCHEDULE_TO_CLOSE = EVALUATE_START_TO_CLOSE + dt.timedelta(seconds=20)
+RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)
+RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)
+# What the source needs from its binding's `evaluation_timeout`.
+EVALUATION_BUDGET = EVALUATE_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE
+
+
+@close_db_connections
+def evaluate_billing_alerts_activity(inputs: SourceEvaluationInputs) -> SourceBatchEvaluation:
+ # Imported in the activity body, because the workflow below forces this module to evaluate
+ # inside Temporal's sandbox, which a Django model import trips.
+ from products.billing_alerts.backend.platform_source_cycle import evaluate_billing_batch # noqa: PLC0415
+
+ return evaluate_billing_batch(
+ inputs.batch_key.team_id, inputs.batch_key.slot, dt.datetime.fromisoformat(inputs.cutoff)
+ )
+
+
[email protected](name=WORKFLOW_NAME)
+class BillingAlertPlatformEvaluateWorkflow(PostHogWorkflow):
+ """Evaluates one batch key and records what it decided. Delivers nothing, and writes only the
+ shared platform's own rows: billing's own stack owns the billing tables."""
+
+ inputs_cls = SourceEvaluationInputs
+
+ @workflow.run
+ async def run(self, inputs: SourceEvaluationInputs) -> int:
+ # One attempt. A failed batch keeps its due time, so the next tick reaches it anyway.
+ evaluation = await workflow.execute_activity(
+ evaluate_billing_alerts_activity,
+ inputs,
+ start_to_close_timeout=EVALUATE_START_TO_CLOSE,
+ schedule_to_close_timeout=EVALUATE_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ if not evaluation.outcomes:
+ return 0
+ return await workflow.execute_activity(
+ RECORD_OUTCOMES_ACTIVITY,
+ SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=evaluation.outcomes),
+ start_to_close_timeout=RECORD_START_TO_CLOSE,
+ schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ result_type=int,
+ )
+
+
+PLATFORM_EVALUATION_WORKFLOWS = [BillingAlertPlatformEvaluateWorkflow]
+PLATFORM_EVALUATION_ACTIVITIES = [evaluate_billing_alerts_activity]
diff --git a/products/billing_alerts/backend/test/test_platform_source_cycle.py b/products/billing_alerts/backend/test/test_platform_source_cycle.py
new file mode 100644
index 000000000000..58cd7870bed9
--- /dev/null
+++ b/products/billing_alerts/backend/test/test_platform_source_cycle.py
@@ -0,0 +1,213 @@
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from typing import Any
+from uuid import UUID, uuid4
+
+import time_machine
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from posthog.models.scoping import team_scope
+
+from products.alerts_platform.backend.facade.api import record_outcomes, slot_of, upsert_configuration
+from products.alerts_platform.backend.facade.contracts import (
+ AlertEventKind,
+ CheckFailure,
+ GroupOutcome,
+ PlatformAlertOutcome,
+ PlatformAlertUpsert,
+ SourceBatchEvaluation,
+ SourceKind,
+)
+from products.alerts_platform.backend.facade.temporal import source_evaluation_timeout
+from products.alerts_platform.backend.facade.testing import (
+ configuration,
+ count_still_due,
+ create_configuration,
+ set_due_at,
+)
+from products.billing_alerts.backend.models import BillingAlertConfiguration, BillingAlertEvaluationClaim
+from products.billing_alerts.backend.platform_source_cycle import evaluate_billing_batch
+from products.billing_alerts.backend.temporal.platform_evaluate import EVALUATION_BUDGET
+
+_MODULE = "products.billing_alerts.backend.platform_source_cycle"
+CUTOFF = datetime(2026, 6, 23, 12, 0, tzinfo=UTC)
+DUE_AT = CUTOFF - timedelta(minutes=1)
+
+
+def _billing_response(current: int | None) -> dict[str, Any]:
+ customer: dict[str, Any] = {}
+ if current is not None:
+ customer["current_total_amount_usd_after_discount"] = str(current)
+ return {"customer": customer}
+
+
+def _decided(outcome: PlatformAlertOutcome) -> GroupOutcome | CheckFailure:
+ """The one verdict an ungrouped billing check reports, or its failure."""
+ return outcome.failure or outcome.groups[0]
+
+
+@time_machine.travel(CUTOFF, tick=False)
+class TestPlatformBillingEvaluation(BaseTest):
+ def _alert(self, **overrides: Any) -> BillingAlertConfiguration:
+ fields: dict[str, Any] = {
+ "organization_id": self.organization.id,
+ "team_id": self.team.id,
+ "created_by_id": self.user.id,
+ "name": "Period spend cap",
+ "metric": BillingAlertConfiguration.Metric.SPEND,
+ "threshold_type": BillingAlertConfiguration.ThresholdType.ABSOLUTE_VALUE,
+ "threshold_value": Decimal("100"),
+ }
+ fields.update(overrides)
+ return BillingAlertConfiguration.objects.create(**fields)
+
+ def _copy(self, alert: BillingAlertConfiguration | None) -> UUID:
+ with team_scope(self.team.id):
+ return create_configuration(
+ team=self.team,
+ name="Period spend cap",
+ source_kind=SourceKind.BILLING.value,
+ check_interval_minutes=60,
+ cooldown_minutes=0,
+ next_check_at=DUE_AT,
+ legacy_configuration_id=alert.id if alert else uuid4(),
+ ).id
+
+ def _evaluate(self, *responses: dict[str, Any] | Exception) -> tuple[SourceBatchEvaluation, Any]:
+ side_effect = [r if isinstance(r, Exception) else (r, 12) for r in responses]
+ with patch(f"{_MODULE}.fetch_billing_data", side_effect=side_effect) as fetch:
+ evaluation = evaluate_billing_batch(self.team.id, slot_of(DUE_AT, CUTOFF), CUTOFF)
+ return evaluation, fetch
+
+ def test_a_breach_fires_on_the_platform_and_leaves_billings_tables_alone(self) -> None:
+ alert = self._alert()
+ before = BillingAlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id)
+ self._copy(alert)
+
+ evaluation, _ = self._evaluate(_billing_response(150))
+
+ (outcome,) = evaluation.outcomes
+ (group,) = outcome.groups
+ assert (group.kind, group.new_state, group.value, outcome.evaluation_key) == (
+ AlertEventKind.FIRING,
+ "firing",
+ 150.0,
+ f"slot:{slot_of(DUE_AT, CUTOFF)}",
+ )
+ assert evaluation.deliveries == ()
+ assert (
+ BillingAlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id)
+ == before
+ )
+ assert not BillingAlertEvaluationClaim.objects.filter(alert=alert).exists()
+
+ @parameterized.expand(
+ [
+ ("no_total_yet", _billing_response(None)),
+ ("billing_service_down", ConnectionError("billing service timed out")),
+ ("billing_returns_no_customer", {}),
+ ("billing_returns_an_unreadable_amount", {"customer": {"current_total_amount_usd_after_discount": "n/a"}}),
+ ]
+ )
+ def test_a_check_that_reads_no_total_leaves_a_firing_alert_firing_quietly(
+ self, _name: str, response: dict[str, Any] | Exception
+ ) -> None:
+ configuration_id = self._copy(self._alert())
+ fired, _ = self._evaluate(_billing_response(150))
+ with team_scope(self.team.id):
+ record_outcomes(self.team.id, fired.outcomes, CUTOFF)
+ set_due_at(configuration_id, DUE_AT)
+
+ evaluation, _ = self._evaluate(response)
+
+ (outcome,) = evaluation.outcomes
+ decided = _decided(outcome)
+ assert (decided.kind, decided.new_state, outcome.consecutive_failures, outcome.disable) == (
+ AlertEventKind.CHECK,
+ "firing",
+ 0,
+ False,
+ )
+
+ def test_a_snooze_cleared_on_the_billing_alert_after_the_backfill_no_longer_holds_the_fire(self) -> None:
+ alert = self._alert()
+ upsert_configuration(
+ PlatformAlertUpsert(
+ legacy_configuration_id=alert.id,
+ team_id=self.team.id,
+ name=alert.name,
+ enabled=True,
+ source_kind=SourceKind.BILLING,
+ source_config={},
+ check_interval_minutes=60,
+ evaluation_periods=1,
+ datapoints_to_alarm=1,
+ cooldown_minutes=0,
+ schedule_restriction=None,
+ next_check_at=DUE_AT,
+ # The snooze the backfill copied, since cleared on the billing alert.
+ snooze_until=CUTOFF + timedelta(hours=2),
+ )
+ )
+
+ evaluation, _ = self._evaluate(_billing_response(150))
+
+ (outcome,) = evaluation.outcomes
+ (group,) = outcome.groups
+ assert (group.kind, group.muted_notification) == (AlertEventKind.FIRING, "")
+
+ def test_a_snooze_set_on_the_billing_alert_holds_the_fire(self) -> None:
+ self._copy(self._alert(snoozed_until=CUTOFF + timedelta(hours=2)))
+
+ evaluation, _ = self._evaluate(_billing_response(150))
+
+ (outcome,) = evaluation.outcomes
+ (group,) = outcome.groups
+ assert (group.kind, group.muted_notification) == (AlertEventKind.CHECK, "fire")
+
+ def test_one_billing_call_serves_every_alert_of_an_organization_in_a_batch(self) -> None:
+ configuration_ids = [self._copy(self._alert(name="First")), self._copy(self._alert(name="Second"))]
+
+ evaluation, fetch = self._evaluate(_billing_response(150))
+ with team_scope(self.team.id):
+ record_outcomes(self.team.id, evaluation.outcomes, CUTOFF)
+ next_checks = {configuration(c).next_check_at for c in configuration_ids}
+
+ assert fetch.call_count == 1
+ assert [o.groups[0].kind for o in evaluation.outcomes] == [AlertEventKind.FIRING, AlertEventKind.FIRING]
+ assert len(next_checks) == 1
+
+ @parameterized.expand([("gone", None, True), ("disabled", {"enabled": False}, False)])
+ def test_a_copy_of_an_alert_that_cannot_run_skips_without_a_billing_call(
+ self, _name: str, overrides: dict[str, Any] | None, disables_the_copy: bool
+ ) -> None:
+ self._copy(None if overrides is None else self._alert(**overrides))
+
+ evaluation, fetch = self._evaluate()
+
+ fetch.assert_not_called()
+ (outcome,) = evaluation.outcomes
+ (group,) = outcome.groups
+ assert (group.kind, outcome.disable) == (AlertEventKind.CHECK, disables_the_copy)
+
+ def test_checks_past_the_payload_bound_are_left_due(self) -> None:
+ configuration_ids = [self._copy(self._alert(name=f"Bound {n}")) for n in range(3)]
+
+ # The first outcome alone overruns a 25-byte budget; the two after it fit.
+ with (
+ patch(f"{_MODULE}.payload_bytes", side_effect=[30, 10, 10]),
+ patch(f"{_MODULE}.MAX_BATCH_PAYLOAD_BYTES", 25),
+ ):
+ evaluation, _ = self._evaluate(_billing_response(150))
+ with team_scope(self.team.id):
+ record_outcomes(self.team.id, evaluation.outcomes, CUTOFF)
+ still_due = count_still_due(configuration_ids, at=CUTOFF)
+
+ assert (len(evaluation.outcomes), evaluation.omitted, still_due) == (2, 1, 1)
+
+
+def test_the_binding_holds_the_whole_batch() -> None:
+ assert source_evaluation_timeout(SourceKind.BILLING) > EVALUATION_BUDGET
diff --git a/tach.toml b/tach.toml
index a292cc3f57e7..0809bb9ca260 100644
--- a/tach.toml
+++ b/tach.toml
@@ -1189,11 +1189,14 @@ from = [
"products.experiments",
]
-# Billing alerts registers product-owned workflows and a schedule with the shared billing worker.
+# Billing alerts registers product-owned workflows and a schedule with the shared billing worker,
+# and its platform evaluation with the alerts platform's evaluation worker.
[[interfaces]]
expose = [
"backend\\.temporal\\.ACTIVITIES",
+ "backend\\.temporal\\.PLATFORM_EVALUATION_ACTIVITIES",
+ "backend\\.temporal\\.PLATFORM_EVALUATION_WORKFLOWS",
"backend\\.temporal\\.WORKFLOWS",
"backend\\.temporal\\.schedule\\.create_schedule_due_billing_alert_checks_schedule",
]