Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): evaluate statistical detector insight alerts on the platform (#119947)

Jimmy Zhu210c7090948e
diff --git a/docs/internal/alerts-platform-temporal.md b/docs/internal/alerts-platform-temporal.md
index d75b81b64973..fed87960a858 100644
--- a/docs/internal/alerts-platform-temporal.md
+++ b/docs/internal/alerts-platform-temporal.md
@@ -508,7 +508,7 @@ Insight alerts are copied the same way:
python manage.py backfill_platform_insight_alert_configurations
```
-It copies threshold alerts on a 15-minute or slower cadence only, and skips detector alerts and the real-time cadence.
+It copies threshold and statistical detector alerts on a 15-minute or slower cadence only, and skips AI detector alerts and the real-time cadence.
Run it only after the evaluation worker's chart sets `CLICKHOUSE_ALERTS_PLATFORM_INSIGHT_USER` and its token file.
Insight checks tag their queries with `ClickHouseUser.ALERTS_PLATFORM_INSIGHT`, a user of their own, so the parallel run never takes from the per-user budget of the user that production insight alerts query as.
Without that env the tag resolves to the worker's default user, which other workloads on the same servers already push against its concurrent query limit.
diff --git a/products/alerts/backend/platform_alert_backfill.py b/products/alerts/backend/platform_alert_backfill.py
index 99a675e688fb..0ec56b6a7db7 100644
--- a/products/alerts/backend/platform_alert_backfill.py
+++ b/products/alerts/backend/platform_alert_backfill.py
@@ -59,6 +59,7 @@ def _upsert(alert: AlertConfiguration) -> PlatformAlertUpsert | None:
SOURCE_CONDITION_KEY: {
"threshold": alert.threshold.configuration if alert.threshold else None,
"comparison": alert.condition,
+ "detector": alert.detector_config,
},
},
check_interval_minutes=check_interval_minutes,
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 45d963b3aeb5..b20409737d5c 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -37,6 +37,7 @@ from products.alerts.backend.evaluation import check_alert_for_insight
from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, check_alert_per_series
from products.alerts.backend.insight_alert_state_machine import INSIGHT_ALERT_POLICY, insight_snapshot
+from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts_platform.backend.facade.api import due_check_ids, due_checks, slot_of
@@ -124,9 +125,9 @@ _EVENT_IDS_BY_KIND: Final[dict[str, str]] = {
def is_evaluated_on_the_platform(alert: AlertConfiguration) -> bool:
- """Threshold alerts on a 15-minute or slower cadence. A detector alert makes its own decision,
- and the LLM detector makes a charged model call, so evaluating either in parallel costs twice."""
- return not alert.detector_config and alert.calculation_interval in EVALUATED_INTERVALS
+ """Threshold and statistical detector alerts on a 15-minute or slower cadence. The AI detector
+ makes a charged model call, so evaluating it in parallel costs twice."""
+ return not is_llm_detector_config(alert.detector_config) and alert.calculation_interval in EVALUATED_INTERVALS
def plan_insight_batch(
diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.py
index 0363d6b4b395..ae74bde52ffd 100644
--- a/products/alerts/backend/test/test_platform_alert_backfill.py
+++ b/products/alerts/backend/test/test_platform_alert_backfill.py
@@ -51,29 +51,33 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
)
return {view.legacy_configuration_id: view for view in page.configurations}
- def test_only_15_minute_and_slower_threshold_alerts_are_copied_on_their_recurrence(self) -> None:
+ def test_only_15_minute_and_slower_alerts_without_the_ai_detector_are_copied_on_their_recurrence(self) -> None:
quarter_hourly = self._alert(calculation_interval=AlertCalculationInterval.EVERY_15_MINUTES.value)
hourly = self._alert(calculation_interval=AlertCalculationInterval.HOURLY.value, schedule_start_time="09:30")
daily = self._alert(schedule_start_time="09:30")
+ detector = self._alert(detector_config={"type": "zscore"})
self._alert(calculation_interval=AlertCalculationInterval.REAL_TIME.value)
- self._alert(detector_config={"type": "zscore"})
+ self._alert(detector_config={"type": "llm"})
unparseable = self._alert(schedule_start_time="25:99")
counts = backfill_platform_insight_alert_configurations(team_id=self.team.id)
again = backfill_platform_insight_alert_configurations(team_id=self.team.id)
- assert (counts.created, counts.skipped, counts.failed, again.created, again.updated) == (3, 2, 1, 0, 3)
+ assert (counts.created, counts.skipped, counts.failed, again.created, again.updated) == (4, 2, 1, 0, 4)
copies = self._copies()
assert unparseable.id not in copies
assert {key: (v.check_interval_minutes, v.recurrence_unit, v.anchor_time) for key, v in copies.items()} == {
quarter_hourly.id: (15, None, None),
hourly.id: (60, None, None),
daily.id: (60 * 24, "day", "09:30"),
+ detector.id: (60 * 24, "day", None),
}
assert copies[daily.id].source_config["condition"] == {
"threshold": THRESHOLD,
"comparison": {"type": "absolute_value"},
+ "detector": None,
}
+ assert copies[detector.id].source_config["condition"]["detector"] == {"type": "zscore"}
def test_a_sample_copies_the_same_alerts_on_every_run_and_widening_it_only_adds(self) -> None:
inside = self._alert(id=UUID(int=5))
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index b475c5070221..240f7121caf1 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -147,8 +147,11 @@ class TestPlatformInsightEvaluation(APIBaseTest):
self.deliveries = evaluation.deliveries if evaluation is not None else ()
return (evaluation.outcomes[0] if evaluation is not None else None), query
- def test_a_breach_fires_on_the_platform_and_leaves_the_production_alert_alone(self) -> None:
- alert = self._alert()
+ @parameterized.expand([("threshold", {}), ("detector", {"detector_config": {"type": "zscore"}})])
+ def test_a_breach_fires_on_the_platform_and_leaves_the_production_alert_alone(
+ self, _name: str, alert_overrides: dict[str, Any]
+ ) -> None:
+ alert = self._alert(**alert_overrides)
before = AlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id)
configuration = self._copy(alert)
@@ -209,7 +212,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):
@parameterized.expand(
[
("real_time", {"calculation_interval": AlertCalculationInterval.REAL_TIME.value}, "not_firing"),
- ("detector", {"detector_config": {"type": "zscore"}}, "not_firing"),
+ ("ai_detector", {"detector_config": {"type": "llm"}}, "not_firing"),
("disabled", {"enabled": False}, "not_firing"),
("snoozed", {"snoozed_until": CUTOFF + timedelta(hours=1)}, "snoozed"),
]