Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): leave ai detector alerts out of the insight parallel run (#125088)

Jimmy Zhu76ccdc4e3eb7
diff --git a/docs/internal/alerts-platform-temporal.md b/docs/internal/alerts-platform-temporal.md
index bf055da32ba7..76fd8ec94b2a 100644
--- a/docs/internal/alerts-platform-temporal.md
+++ b/docs/internal/alerts-platform-temporal.md
@@ -518,7 +518,10 @@ It copies every insight alert, on every cadence.
A real-time check gets production's real-time budget: a 7-minute evaluation with one retry inside it.
An AI detector check runs on the AI queue, whose worker holds the model credentials, with production's retries: a check whose model call reaches no verdict is retried, and only its last attempt records the error.
Every other check gets one 12-minute attempt.
-An AI detector check calls the model on both stacks while both run, and each call is billed to the project, so copy AI detector alerts only for internal projects until the insight cutover.
+An AI detector check calls the model on both stacks while both run, and each call is billed to the project, so the backfill leaves AI detector alerts out.
+Pass `--include-ai-detector` to copy them, with `--team-id` or `--ids-file` for an internal project, until the insight cutover.
+An AI detector alert created or turned back on later waits for the next backfill run with the flag.
+A copy made without the flag is switched off at its next check when its alert has since moved to the AI detector.
The AI queue's worker needs `CLICKHOUSE_ALERTS_PLATFORM_INSIGHT_USER` and its token file too, or those checks query as the worker's own ClickHouse user.
Run it only after the evaluation worker's chart sets `CLICKHOUSE_ALERTS_PLATFORM_INSIGHT_USER` and its token file.
Insight checks tag their queries with `ClickHouseUser.ALERTS_PLATFORM_INSIGHT`, a user of their own, so the parallel run never takes from the per-user budget of the user that production insight alerts query as.
diff --git a/products/alerts/backend/management/commands/backfill_platform_insight_alert_configurations.py b/products/alerts/backend/management/commands/backfill_platform_insight_alert_configurations.py
index c63aa2fbca17..7a610a8d745b 100644
--- a/products/alerts/backend/management/commands/backfill_platform_insight_alert_configurations.py
+++ b/products/alerts/backend/management/commands/backfill_platform_insight_alert_configurations.py
@@ -1,4 +1,5 @@
from collections.abc import Collection
+from typing import Any
from uuid import UUID
from products.alerts.backend.platform_alert_backfill import (
@@ -13,12 +14,29 @@ class Command(PlatformBackfillCommand):
help = "Copy insight alert configurations into the shared alert tables for a parallel evaluation."
source_name = "insight"
sample_unit = "alert"
+ include_ai_detector = False
+
+ def add_arguments(self, parser: Any) -> None:
+ super().add_arguments(parser)
+ parser.add_argument(
+ "--include-ai-detector",
+ action="store_true",
+ help="Also copy AI detector alerts. Each of their checks makes a model call the project pays for, "
+ "so a copy doubles that cost while both stacks run.",
+ )
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ self.include_ai_detector = options["include_ai_detector"]
+ super().handle(*args, **options)
def backfill(
self, *, team_id: int | None, sample_percent: int, alert_ids: Collection[UUID] | None
) -> BackfillCounts:
return backfill_platform_insight_alert_configurations(
- team_id=team_id, sample_percent=sample_percent, alert_ids=alert_ids
+ team_id=team_id,
+ sample_percent=sample_percent,
+ alert_ids=alert_ids,
+ include_ai_detector=self.include_ai_detector,
)
def disable(self, *, team_id: int | None, alert_ids: Collection[UUID] | None) -> int:
diff --git a/products/alerts/backend/platform_alert_backfill.py b/products/alerts/backend/platform_alert_backfill.py
index eb03b2ef3165..40101547d5ed 100644
--- a/products/alerts/backend/platform_alert_backfill.py
+++ b/products/alerts/backend/platform_alert_backfill.py
@@ -14,6 +14,7 @@ import structlog
from posthog.dataclasses import frozen
+from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts.backend.platform_source_cycle import platform_upsert_of
from products.alerts_platform.backend.facade.api import disable_configurations, upsert_configuration, upsert_if_running
@@ -37,13 +38,20 @@ def is_sampled(alert_id: UUID, sample_percent: int) -> bool:
def backfill_platform_insight_alert_configurations(
- *, team_id: int | None = None, sample_percent: int = 100, alert_ids: Collection[UUID] | None = None
+ *,
+ team_id: int | None = None,
+ sample_percent: int = 100,
+ alert_ids: Collection[UUID] | None = None,
+ include_ai_detector: bool = False,
) -> BackfillCounts:
"""Copies every insight alert the platform evaluates in parallel, or one team's, or a sample, or named alerts.
Every copy adds ClickHouse load beside production's, so a rollout copies a sample first and
widens it while the parallel run's query load stays inside its budget. Named alerts let a
rollout copy the ones a comparison still lacks evidence for.
+
+ AI detector alerts are left out unless `include_ai_detector` is set. Each of their checks makes
+ a model call that the project pays for, so a parallel copy doubles that cost.
"""
if not 1 <= sample_percent <= 100:
raise ValueError("sample_percent must be between 1 and 100")
@@ -58,7 +66,10 @@ def backfill_platform_insight_alert_configurations(
skipped = 0
failed = 0
for alert in source.iterator():
- upsert = platform_upsert_of(alert) if is_sampled(alert.id, sample_percent) else None
+ copied = is_sampled(alert.id, sample_percent) and (
+ include_ai_detector or not is_llm_detector_config(alert.detector_config)
+ )
+ upsert = platform_upsert_of(alert) if copied else None
if upsert is None:
skipped += 1
continue
@@ -102,7 +113,12 @@ def follow_insight_alert(alert: AlertConfiguration) -> None:
Checks only read enabled copies, so no check would ever see this alert otherwise, and it would
stay off the platform until the next backfill.
+
+ An AI detector alert is never followed. The backfill copies those only when an operator asks,
+ and a copy's stored detector is what records that request, so following would overwrite it.
"""
+ if is_llm_detector_config(alert.detector_config):
+ return
upsert = platform_upsert_of(alert)
if upsert is not None:
upsert_if_running(upsert, revive=True)
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 1f7c6928372b..8b240166c0fa 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -521,6 +521,10 @@ def _decide(
if alert is None:
# The production alert is gone, so this copy can never be compared again.
return _skipped(check, snapshot, now=now, disable=True)
+ if _ai_detector_not_copied(check, alert):
+ # Before the refresh, which would write the AI detector onto the copy and opt it in. The
+ # copy is switched off, because it would otherwise skip on its stale schedule every tick.
+ return _skipped(check, snapshot, now=now, disable=True)
# Before any skip, because a skip advances the check by the copy's schedule.
check = _following(check, alert)
if not alert.enabled or alert.insight.deleted or not is_evaluated_on_the_platform(alert):
@@ -600,6 +604,18 @@ def _decide(
)
+def _ai_detector_not_copied(check: PlatformAlertCheckInput, alert: AlertConfiguration) -> bool:
+ """Whether a parallel-run copy made without the AI detector would now run one.
+
+ A check reads the alert's live detector, so an alert switched to the AI detector after the
+ backfill copied it would make a model call the project pays for twice. The backfill copies
+ AI detector alerts only on request, and a copy it made that way stores the AI detector.
+ """
+ if check.owned_by_platform or not is_llm_detector_config(alert.detector_config):
+ return False
+ return not is_llm_detector_config(check.condition.get("detector"))
+
+
def _following(check: PlatformAlertCheckInput, alert: AlertConfiguration) -> PlatformAlertCheckInput:
"""The check with its alert's settings as they are now, written back to the copy first.
diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.py
index 0c10f5a8349a..017b4f378aaa 100644
--- a/products/alerts/backend/test/test_platform_alert_backfill.py
+++ b/products/alerts/backend/test/test_platform_alert_backfill.py
@@ -1,8 +1,11 @@
+from io import StringIO
from typing import Any
from uuid import UUID
from posthog.test.base import APIBaseTest
+from django.core.management import call_command
+
from parameterized import parameterized
from posthog.schema import ChartDisplayType, EventsNode, IntervalType, TrendsFilter, TrendsQuery
@@ -64,9 +67,18 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
unparseable = self._alert(schedule_start_time="25:99")
counts = backfill_platform_insight_alert_configurations(team_id=self.team.id)
- again = backfill_platform_insight_alert_configurations(team_id=self.team.id)
+ assert (counts.created, counts.skipped, counts.failed) == (5, 1, 1)
+ assert ai_detector.id not in self._copies()
+
+ call_command(
+ "backfill_platform_insight_alert_configurations",
+ team_id=self.team.id,
+ include_ai_detector=True,
+ stdout=StringIO(),
+ )
+ again = backfill_platform_insight_alert_configurations(team_id=self.team.id, include_ai_detector=True)
- assert (counts.created, counts.skipped, counts.failed, again.created, again.updated) == (6, 0, 1, 0, 6)
+ assert (again.created, again.updated) == (0, 6)
copies = self._copies()
assert unparseable.id not in copies
assert {key: (v.check_interval_minutes, v.recurrence_unit, v.anchor_time) for key, v in copies.items()} == {
@@ -113,13 +125,33 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
backfill_platform_insight_alert_configurations(team_id=self.team.id)
assert {view.enabled for view in self._copies().values()} == {True}
- @parameterized.expand([("a_team_on_the_platform", True), ("a_team_off_the_platform", False)])
- def test_a_created_or_reenabled_alert_follows_its_team_onto_the_platform(self, _name: str, running: bool) -> None:
+ @parameterized.expand(
+ [
+ ("a_team_on_the_platform", True, False, None, "renamed"),
+ ("a_team_off_the_platform", False, False, None, None),
+ ("an_ai_alert", True, False, {"type": "llm"}, None),
+ ("an_alert_switched_to_ai_after_its_copy", True, True, {"type": "llm"}, "alert"),
+ ]
+ )
+ def test_a_created_or_reenabled_alert_follows_its_team_onto_the_platform(
+ self,
+ _name: str,
+ running: bool,
+ copied_before: bool,
+ detector: dict[str, str] | None,
+ copy_name: str | None,
+ ) -> None:
if running:
self._alert()
backfill_platform_insight_alert_configurations(team_id=self.team.id)
created = self._alert()
+ if copied_before:
+ backfill_platform_insight_alert_configurations(alert_ids=[created.id])
+ created.name = "renamed"
+ created.detector_config = detector
+ created.save()
follow_insight_alert(created)
- assert (created.id in self._copies()) is running
+ copy = self._copies().get(created.id)
+ assert (copy.name if copy else None) == copy_name
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index fb7536793487..18b60905d340 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -59,6 +59,7 @@ from products.alerts.backend.test.insight_alerts import create_insight_alert
from products.alerts_platform.backend.facade import testing as platform_testing
from products.alerts_platform.backend.facade.api import list_configurations, record_outcomes, slot_of
from products.alerts_platform.backend.facade.contracts import (
+ SOURCE_CONDITION_KEY,
AlertBatchKey,
AlertEventKind,
Grouping,
@@ -96,7 +97,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):
"team_id": self.team.id,
"name": "alert",
"source_kind": SourceKind.INSIGHT,
- "source_config": {},
+ "source_config": {SOURCE_CONDITION_KEY: {"detector": alert.detector_config}} if alert else {},
"check_interval_minutes": 60 * 24,
"recurrence_unit": "day",
"next_check_at": CUTOFF - timedelta(minutes=1),
@@ -464,21 +465,28 @@ class TestPlatformInsightEvaluation(APIBaseTest):
@parameterized.expand(
[
- ("disabled", {"enabled": False}, "not_firing"),
- ("snoozed", {"snoozed_until": CUTOFF + timedelta(hours=1)}, "snoozed"),
+ ("disabled", {"enabled": False}, None, "not_firing"),
+ ("snoozed", {"snoozed_until": CUTOFF + timedelta(hours=1)}, None, "snoozed"),
+ ("switched_to_ai_after_its_copy", {}, {"type": "llm"}, "not_firing"),
]
)
def test_a_check_production_would_not_run_is_recorded_without_a_query(
- self, _name: str, overrides: dict[str, Any], state: str
+ self, _name: str, overrides: dict[str, Any], detector_after_copy: dict[str, str] | None, state: str
) -> None:
- outcome, query = self._evaluate(self._copy(self._alert(**overrides)))
+ alert = self._alert(**overrides)
+ configuration = self._copy(alert)
+ if detector_after_copy is not None:
+ alert.detector_config = detector_after_copy
+ alert.save()
+
+ outcome, query = self._evaluate(configuration, uses_llm_detector=detector_after_copy is not None)
query.assert_not_called()
assert outcome is not None
assert (outcome.groups[0].kind, outcome.groups[0].new_state, outcome.disable) == (
AlertEventKind.CHECK,
state,
- False,
+ detector_after_copy is not None,
)
@parameterized.expand(