:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): shadow-evaluate insight alerts the platform owns (#119973)
Jimmy Zhu2be9b5681745
diff --git a/posthog/tasks/alerts/test/test_utils.py b/posthog/tasks/alerts/test/test_utils.pyindex baad00ef2d86..dce2f4ed560c 100644--- a/posthog/tasks/alerts/test/test_utils.py+++ b/posthog/tasks/alerts/test/test_utils.py@@ -24,6 +24,7 @@ from posthog.tasks.alerts.utils import (from products.alerts.backend.models.alert import AlertCheck, AlertConfigurationfrom products.alerts_platform.backend.facade.contracts import ActiveAlertDestination+from products.alerts_platform.backend.facade.enums import PlatformAlertOwnerfrom products.product_analytics.backend.facade.models import Insight@@ -359,6 +360,13 @@ class TestDispatchAlertInsightChart(APIBaseTest):mock_prepare.assert_not_called()assert mock_send.call_args.kwargs["extra_properties"]["insight_chart_url"] == "https://supplied/url"+ @patch("posthog.tasks.alerts.utils.send_notifications_for_breaches", return_value=[])+ def test_alert_the_platform_took_over_after_it_was_loaded_sends_nothing(self, mock_send: MagicMock) -> None:+ AlertConfiguration.objects.filter(id=self.alert.id).update(owner=PlatformAlertOwner.PLATFORM)++ assert dispatch_alert_notification(self.alert, self.alert_check, ["breach"], render_chart=False) is None+ mock_send.assert_not_called()+class TestDisableInvalidAlert(APIBaseTest):def setUp(self) -> None:@@ -382,10 +390,22 @@ class TestDisableInvalidAlert(APIBaseTest):assert check.targets_notified == {}assert check.notification_sent_at is None+ @parameterized.expand([(PlatformAlertOwner.SOURCE,), (PlatformAlertOwner.PLATFORM,)])@patch("posthog.tasks.alerts.utils.send_alert_email")- def test_disable_invalid_alert_records_receipts_after_send(self, mock_send: MagicMock) -> None:+ def test_disable_invalid_alert_emails_only_for_an_alert_the_insight_stack_owns(+ self, owner: PlatformAlertOwner, mock_send: MagicMock+ ) -> None:+ self.alert.owner = owner+ self.alert.save(update_fields=["owner"])+check = disable_invalid_alert(self.alert, reason="bad config")check.refresh_from_db()- assert check.targets_notified == {"users": [self.user.email], "destinations": []}- mock_send.assert_called_once()+ self.alert.refresh_from_db()+ assert self.alert.enabled is False+ if owner == PlatformAlertOwner.SOURCE:+ assert check.targets_notified == {"users": [self.user.email], "destinations": []}+ mock_send.assert_called_once()+ else:+ assert check.targets_notified == {}+ mock_send.assert_not_called()diff --git a/posthog/tasks/alerts/utils.py b/posthog/tasks/alerts/utils.pyindex 286e99879ba6..65492f95fab6 100644--- a/posthog/tasks/alerts/utils.py+++ b/posthog/tasks/alerts/utils.py@@ -3,6 +3,7 @@ from contextlib import ExitStackfrom datetime import UTC, datetime, timedeltafrom zoneinfo import ZoneInfo+from django.db.models import Qfrom django.utils import timezoneimport pytz@@ -36,6 +37,7 @@ from products.alerts.backend.insight_alert_state_machine import (from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, derive_detector_event_fieldsfrom products.alerts_platform.backend.facade.contracts import AlertDeliveryfrom products.alerts_platform.backend.facade.delivery_slo import alert_delivery_slo+from products.alerts_platform.backend.facade.enums import PlatformAlertOwnerfrom products.alerts_platform.backend.facade.scheduling import (EVERY_15_MINUTES_CADENCE_MINUTES as EVERY_15_MINUTES_CADENCE_MINUTES,REAL_TIME_CADENCE_MINUTES as REAL_TIME_CADENCE_MINUTES,@@ -49,6 +51,10 @@ logger = structlog.get_logger(__name__)INSIGHT_ALERT_FIRING_EVENT = "$insight_alert_firing"+# How long the insight stack keeps evaluating an alert after the platform takes it over, so the two+# stacks' checks can be compared. It sends nothing for the alert in that time.+PLATFORM_SHADOW = timedelta(days=7)+# TTL for the tokenized chart URL embedded in Slack. Slack fetches the image at delivery,# but the URL must stay resolvable while people scroll back to the message; 30 days matches# the delivery-URL TTL used for task chart artifacts (products/tasks living_artifacts).@@ -89,6 +95,15 @@ NON_TIME_SERIES_DISPLAY_TYPES = {}+def evaluated_by_insight_stack_q(now: datetime) -> Q:+ """The alerts the insight stack still checks: every alert it owns, and a platform-owned alert inside its shadow."""+ # A platform-owned row with no cutover time has no shadow to finish, so it is not evaluated.+ return ~(+ Q(owner=PlatformAlertOwner.PLATFORM.value)+ & (Q(owner_changed_at__isnull=True) | Q(owner_changed_at__lt=now - PLATFORM_SHADOW))+ )++def is_non_time_series_trend(query: TrendsQuery) -> bool:display = query.trendsFilter.display if query.trendsFilter else Nonereturn display in NON_TIME_SERIES_DISPLAY_TYPES@@ -453,7 +468,7 @@ def dispatch_alert_notification("""Route an AlertCheck to the correct notification sender.Returns the delivery receipts the notification produced, or None if nothing was sent- (NOT_FIRING, or ERRORED with a non-dict error payload). Callers pass the returned+ (NOT_FIRING, a platform-owned alert, or ERRORED with a non-dict error payload). Callers pass the returnedreceipts to record_alert_delivery so the `targets_notified` sentinel reflects reality,never claiming delivery for a state that didn't actually send.@@ -472,6 +487,16 @@ def dispatch_alert_notification(ValueError: state is FIRING but breaches is None/empty.AssertionError: unknown state — surfaces a missing AlertState branch loudly."""+ # An investigation dispatches minutes after it loaded `alert`, and the platform can take the+ # alert over in that time, so the stored owner decides.+ if (+ alert.platform_owned+ or AlertConfiguration.objects.filter(+ team_id=alert.team_id, id=alert.id, owner=PlatformAlertOwner.PLATFORM+ ).exists()+ ):+ logger.info("Alert is owned by the platform, nothing to send", alert_id=alert.id)+ return Nonekey = idempotency_key or str(alert_check.id)with ExitStack() as stack:if alert_check.state == AlertState.FIRING:@@ -588,13 +613,20 @@ def record_alert_delivery("users": [delivery.target for delivery in deliveries if delivery.channel == "email"],"destinations": serialize_deliveries([d for d in deliveries if d.channel != "email"]),}- alert_check.notification_sent_at = recorded_at- alert_check.save(update_fields=["targets_notified", "notification_sent_at"])- alert.last_notified_at = recorded_at- alert.save(update_fields=["last_notified_at"])+ stamp_alert_notified(alert, alert_check, recorded_at, extra_check_fields=["targets_notified"])return True+def stamp_alert_notified(+ alert: AlertConfiguration, alert_check: AlertCheck, at: datetime, *, extra_check_fields: Collection[str] = ()+) -> None:+ """Marks the check as notified and moves the alert's last notification time to `at`."""+ alert_check.notification_sent_at = at+ alert_check.save(update_fields=[*extra_check_fields, "notification_sent_at"])+ alert.last_notified_at = at+ alert.save(update_fields=["last_notified_at"])++def add_alert_check(alert: AlertConfiguration,evaluation_result: AlertEvaluationResult | None,@@ -693,7 +725,7 @@ def disable_invalid_alert(state=AlertState.ERRORED,error=error,)- if targets_to_notify and notify_subscribers:+ if targets_to_notify and notify_subscribers and not alert.platform_owned:deliveries = send_notifications_for_disabled(alert, reason, targets_to_notify)record_alert_delivery(alert, alert_check, deliveries)return alert_checkdiff --git a/posthog/temporal/alerts/activities.py b/posthog/temporal/alerts/activities.pyindex 0e0b834a8f64..7ba6b616f85d 100644--- a/posthog/temporal/alerts/activities.py+++ b/posthog/temporal/alerts/activities.py@@ -46,11 +46,13 @@ from posthog.tasks.alerts.utils import (add_alert_check,disable_invalid_alert,dispatch_alert_notification,+ evaluated_by_insight_stack_q,get_alert_error_notification_recipients,next_check_time,next_scheduled_check_time,record_alert_delivery,skip_because_of_weekend,+ stamp_alert_notified,)from posthog.temporal.alerts.admission import (admit_evaluation_slots,@@ -165,6 +167,7 @@ async def retrieve_due_alerts(inputs: ScheduleDueAlertChecksWorkflowInputs | Non).filter(Q(snoozed_until__isnull=True) | Q(snoozed_until__lt=polled_at)).filter(insight__deleted=False)+ .filter(evaluated_by_insight_stack_q(polled_at)))# A check stays due until it finishes, so admitted alerts would otherwise be handed out again.in_flight = inflight_alert_ids()@@ -616,7 +619,21 @@ def _write_errored_alert_check(alert: AlertConfiguration, error: dict) -> tuple[Both evaluate_alert's failure path and the retry-exhausted record_failed_evaluation activity gothrough here, so the errored-check write stays in one place."""- return add_alert_check(alert, None, error)+ alert_check, should_notify = add_alert_check(alert, None, error)+ return alert_check, _sends_from_insight_stack(alert, alert_check, should_notify)+++def _sends_from_insight_stack(alert: AlertConfiguration, alert_check: AlertCheck, should_notify: bool) -> bool:+ """Whether the insight stack sends the check's notification. It sends nothing for a platform-owned alert.++ A held send is still recorded as sent, so `last_notified_at` moves as it would have and the+ shadow stays comparable with the platform. The stamp on the check also keeps the investigation+ safety net from sending it later.+ """+ if not (should_notify and alert.platform_owned):+ return should_notify+ stamp_alert_notified(alert, alert_check, datetime.now(UTC))+ return False@temporalio.activity.defn@@ -748,7 +765,9 @@ async def evaluate_alert(inputs: EvaluateAlertActivityInputs) -> EvaluateAlertRealert, invalid_configuration, notify_subscribers=False, error_code="invalid_configuration")return EvaluateAlertResult(- alert_check_id=str(alert_check.id), should_notify=True, new_state=AlertState.ERRORED+ alert_check_id=str(alert_check.id),+ should_notify=_sends_from_insight_stack(alert, alert_check, True),+ new_state=AlertState.ERRORED,)if error is not None:alert_check, should_notify = _write_errored_alert_check(alert, error)@@ -761,6 +780,13 @@ async def evaluate_alert(inputs: EvaluateAlertActivityInputs) -> EvaluateAlertRereturn EvaluateAlertResult(alert_check_id=str(alert_check.id), should_notify=False, new_state=AlertState(alert.state))+ if alert.platform_owned:+ # No investigation either: it exists to gate a notification, and each one costs a model call.+ return EvaluateAlertResult(+ alert_check_id=str(alert_check.id),+ should_notify=_sends_from_insight_stack(alert, alert_check, should_notify),+ new_state=AlertState(alert_check.state),+ )investigation = decide_investigation(alert, alert_check)if investigation.should_investigate and claim_investigation_slot(alert, alert_check):@@ -902,6 +928,8 @@ def _evaluation_fingerprint(alert: AlertConfiguration) -> str:"schedule_start_time","schedule_restriction","snoozed_until",+ "owner",+ "owner_changed_at",)values = [*(getattr(alert, field) for field in fields),diff --git a/posthog/temporal/tests/test_alerts_activities.py b/posthog/temporal/tests/test_alerts_activities.pyindex 50927812ca54..76f3c37d4740 100644--- a/posthog/temporal/tests/test_alerts_activities.py+++ b/posthog/temporal/tests/test_alerts_activities.py@@ -85,6 +85,7 @@ from products.alerts.backend.facade.api import ()from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, InvestigationStatus, Thresholdfrom products.alerts_platform.backend.facade.contracts import AlertDelivery+from products.alerts_platform.backend.facade.enums import PlatformAlertOwnerfrom products.product_analytics.backend.facade.models import Insight@@ -1020,11 +1021,12 @@ class TestEvaluateAlert:assert check.targets_notified == {}@pytest.mark.parametrize(- "investigated,previous_verdict,expect_investigation,expect_notify,expect_held",+ "investigated,previous_verdict,owner,expect_investigation,expect_notify,expect_held",[- (1, "false_positive", True, True, False),- (MAX_INVESTIGATIONS_PER_EPISODE, "false_positive", False, False, True),- (MAX_INVESTIGATIONS_PER_EPISODE, "true_positive", False, True, False),+ (1, "false_positive", PlatformAlertOwner.SOURCE, True, True, False),+ (MAX_INVESTIGATIONS_PER_EPISODE, "false_positive", PlatformAlertOwner.SOURCE, False, False, True),+ (MAX_INVESTIGATIONS_PER_EPISODE, "true_positive", PlatformAlertOwner.SOURCE, False, True, False),+ (1, "false_positive", PlatformAlertOwner.PLATFORM, False, False, False),],)async def test_gated_alert_holds_each_fire_on_a_verdict(@@ -1032,6 +1034,7 @@ class TestEvaluateAlert:ateam,investigated: int,previous_verdict: str,+ owner: PlatformAlertOwner,expect_investigation: bool,expect_notify: bool,expect_held: bool,@@ -1041,7 +1044,10 @@ class TestEvaluateAlert:@sync_to_asyncdef _seed_episode() -> None:AlertConfiguration.objects.filter(id=alert.id).update(- investigation_agent_enabled=True, investigation_gates_notifications=True+ investigation_agent_enabled=True,+ investigation_gates_notifications=True,+ owner=owner,+ owner_changed_at=datetime.now(UTC),)episode_start = datetime.now(UTC) - timedelta(days=5)AlertCheck.objects.create(alert_configuration=alert, state=AlertState.NOT_FIRING)@@ -1070,6 +1076,10 @@ class TestEvaluateAlert:assert result.should_notify is expect_notifycheck = await sync_to_async(AlertCheck.objects.get)(pk=result.alert_check_id)assert check.notification_suppressed_by_agent is expect_held+ if owner == PlatformAlertOwner.PLATFORM:+ await alert.arefresh_from_db()+ assert check.notification_sent_at is not None+ assert alert.last_notified_at == check.notification_sent_atasync def test_evaluate_errored_when_permanent_exception(self, alert) -> None:with patch(@@ -1318,7 +1328,7 @@ class TestEvaluateAlert:@pytest.mark.asyncio@pytest.mark.django_dbclass TestRecordFailedEvaluation:- @pytest.mark.parametrize("change", [None, "detector", "insight", "threshold"])+ @pytest.mark.parametrize("change", [None, "detector", "insight", "threshold", "owner"])async def test_failure_only_records_against_the_prepared_inputs(self, alert, change) -> None:env = ActivityEnvironment()prepared = await env.run(prepare_alert, PrepareAlertActivityInputs(alert_id=str(alert.id)))@@ -1334,6 +1344,10 @@ class TestRecordFailedEvaluation:await sync_to_async(Threshold.objects.filter(id=alert.threshold_id).update)(configuration={"type": "absolute", "bounds": {"upper": 200.0}})+ elif change == "owner":+ await sync_to_async(AlertConfiguration.objects.filter(id=alert.id).update)(+ owner=PlatformAlertOwner.PLATFORM, owner_changed_at=datetime.now(UTC)+ )before = await sync_to_async(AlertConfiguration.objects.get)(id=alert.id)due_before = before.next_check_atresult = await env.run(@@ -1371,6 +1385,7 @@ class TestRecordFailedEvaluation:),)assert result_again.alert_check_id is not None+ assert result_again.should_notify is (change != "owner")@pytest.mark.parametrize("error_type,expected_outcomes",@@ -1440,8 +1455,16 @@ class TestRecordFailedEvaluation:@pytest.mark.asyncio@pytest.mark.django_dbclass TestNotifyAlert:- async def test_noop_when_not_firing(self, alert_with_user) -> None:- check = await _create_alert_check(alert_with_user, state=AlertState.NOT_FIRING)+ @pytest.mark.parametrize(+ "state,owner",+ [+ (AlertState.NOT_FIRING, PlatformAlertOwner.SOURCE),+ (AlertState.FIRING, PlatformAlertOwner.PLATFORM),+ ],+ )+ async def test_noop_when_not_firing_or_platform_owned(self, alert_with_user, state, owner) -> None:+ await sync_to_async(AlertConfiguration.objects.filter(id=alert_with_user.id).update)(owner=owner)+ check = await _create_alert_check(alert_with_user, state=state)with (patch("posthog.tasks.alerts.utils.send_notifications_for_breaches") as mock_breaches,@@ -1450,7 +1473,9 @@ class TestNotifyAlert:env = ActivityEnvironment()await env.run(notify_alert,- NotifyAlertActivityInputs(alert_id=str(alert_with_user.id), alert_check_id=str(check.id)),+ NotifyAlertActivityInputs(+ alert_id=str(alert_with_user.id), alert_check_id=str(check.id), breaches=["value above threshold"]+ ),)mock_breaches.assert_not_called()diff --git a/posthog/temporal/tests/test_alerts_retrieval.py b/posthog/temporal/tests/test_alerts_retrieval.pyindex d04ce4eb7a25..80d7949da082 100644--- a/posthog/temporal/tests/test_alerts_retrieval.py+++ b/posthog/temporal/tests/test_alerts_retrieval.py@@ -18,6 +18,9 @@ from posthog.temporal.alerts.admission import INFLIGHT_KEY, SLOT_LEASE_SECONDS,from posthog.temporal.alerts.types import AlertInfo, ScheduleDueAlertChecksWorkflowInputsfrom posthog.temporal.tests.test_alerts_activities import _create_alert+from products.alerts.backend.models.alert import AlertConfiguration+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner+@pytest.fixture(autouse=True)def clear_inflight_slots():@@ -93,14 +96,31 @@ async def test_retrieve_due_alerts_excludes_future_checks_and_applies_the_documenext_check_at=next_check_at,)alert_label_by_id[str(alert.id)] = label+ # Platform-owned and aged real-time, so each would lead the result if the insight stack still checked it.+ platform_owned_specs = [+ ("platform_owned_in_shadow", datetime(2026, 9, 5, tzinfo=UTC)),+ ("platform_owned_after_shadow", datetime(2026, 9, 2, tzinfo=UTC)),+ ("platform_owned_without_cutover_time", None),+ ]+ for label, owner_changed_at in platform_owned_specs:+ alert = await _create_alert(+ ateam,+ calculation_interval=AlertCalculationInterval.REAL_TIME.value,+ next_check_at=datetime(2026, 9, 10, 11, 20, tzinfo=UTC),+ )+ await sync_to_async(AlertConfiguration.objects.filter(id=alert.id).update)(+ owner=PlatformAlertOwner.PLATFORM, owner_changed_at=owner_changed_at+ )+ alert_label_by_id[str(alert.id)] = labelwith time_machine.travel("2026-09-10T12:00:00Z", tick=False):alerts = await ActivityEnvironment().run(retrieve_due_alerts,- ScheduleDueAlertChecksWorkflowInputs(max_alerts_per_run=len(alert_specs)),+ ScheduleDueAlertChecksWorkflowInputs(max_alerts_per_run=len(alert_label_by_id)),)assert [alert_label_by_id[alert.alert_id] for alert in alerts] == [+ "platform_owned_in_shadow","aged_real_time","fresh_real_time","fresh_15_minutes",diff --git a/products/alerts/backend/models/alert.py b/products/alerts/backend/models/alert.pyindex a9ec7fa6b1f0..862425046ae1 100644--- a/products/alerts/backend/models/alert.py+++ b/products/alerts/backend/models/alert.py@@ -218,6 +218,12 @@ class AlertConfiguration(ModelActivityMixin, CreatedMetaFields, UUIDTModel):def __str__(self) -> str:return f"{self.name} (Team: {self.team})"+ @property+ def platform_owned(self) -> bool:+ """The platform evaluates and delivers this alert. The insight stack may still evaluate it to+ compare, but it sends nothing for it."""+ return self.owner == PlatformAlertOwner.PLATFORM+@propertydef is_high_frequency_interval(self) -> bool:# Real-time counts as high frequency so its checks always run fresh ClickHouse