Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): shadow-evaluate insight alerts the platform owns (#119973)

Jimmy Zhu2be9b5681745
diff --git a/posthog/tasks/alerts/test/test_utils.py b/posthog/tasks/alerts/test/test_utils.py
index baad00ef2d86..dce2f4ed560c 100644
--- a/posthog/tasks/alerts/test/test_utils.py
+++ b/posthog/tasks/alerts/test/test_utils.py
@@ -24,6 +24,7 @@ from posthog.tasks.alerts.utils import (
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration
from products.alerts_platform.backend.facade.contracts import ActiveAlertDestination
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
from products.product_analytics.backend.facade.models import Insight
@@ -359,6 +360,13 @@ class TestDispatchAlertInsightChart(APIBaseTest):
mock_prepare.assert_not_called()
assert mock_send.call_args.kwargs["extra_properties"]["insight_chart_url"] == "https://supplied/url"
+ @patch("posthog.tasks.alerts.utils.send_notifications_for_breaches", return_value=[])
+ def test_alert_the_platform_took_over_after_it_was_loaded_sends_nothing(self, mock_send: MagicMock) -> None:
+ AlertConfiguration.objects.filter(id=self.alert.id).update(owner=PlatformAlertOwner.PLATFORM)
+
+ assert dispatch_alert_notification(self.alert, self.alert_check, ["breach"], render_chart=False) is None
+ mock_send.assert_not_called()
+
class TestDisableInvalidAlert(APIBaseTest):
def setUp(self) -> None:
@@ -382,10 +390,22 @@ class TestDisableInvalidAlert(APIBaseTest):
assert check.targets_notified == {}
assert check.notification_sent_at is None
+ @parameterized.expand([(PlatformAlertOwner.SOURCE,), (PlatformAlertOwner.PLATFORM,)])
@patch("posthog.tasks.alerts.utils.send_alert_email")
- def test_disable_invalid_alert_records_receipts_after_send(self, mock_send: MagicMock) -> None:
+ def test_disable_invalid_alert_emails_only_for_an_alert_the_insight_stack_owns(
+ self, owner: PlatformAlertOwner, mock_send: MagicMock
+ ) -> None:
+ self.alert.owner = owner
+ self.alert.save(update_fields=["owner"])
+
check = disable_invalid_alert(self.alert, reason="bad config")
check.refresh_from_db()
- assert check.targets_notified == {"users": [self.user.email], "destinations": []}
- mock_send.assert_called_once()
+ self.alert.refresh_from_db()
+ assert self.alert.enabled is False
+ if owner == PlatformAlertOwner.SOURCE:
+ assert check.targets_notified == {"users": [self.user.email], "destinations": []}
+ mock_send.assert_called_once()
+ else:
+ assert check.targets_notified == {}
+ mock_send.assert_not_called()
diff --git a/posthog/tasks/alerts/utils.py b/posthog/tasks/alerts/utils.py
index 286e99879ba6..65492f95fab6 100644
--- a/posthog/tasks/alerts/utils.py
+++ b/posthog/tasks/alerts/utils.py
@@ -3,6 +3,7 @@ from contextlib import ExitStack
from datetime import UTC, datetime, timedelta
from zoneinfo import ZoneInfo
+from django.db.models import Q
from django.utils import timezone
import pytz
@@ -36,6 +37,7 @@ from products.alerts.backend.insight_alert_state_machine import (
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, derive_detector_event_fields
from products.alerts_platform.backend.facade.contracts import AlertDelivery
from products.alerts_platform.backend.facade.delivery_slo import alert_delivery_slo
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
from products.alerts_platform.backend.facade.scheduling import (
EVERY_15_MINUTES_CADENCE_MINUTES as EVERY_15_MINUTES_CADENCE_MINUTES,
REAL_TIME_CADENCE_MINUTES as REAL_TIME_CADENCE_MINUTES,
@@ -49,6 +51,10 @@ logger = structlog.get_logger(__name__)
INSIGHT_ALERT_FIRING_EVENT = "$insight_alert_firing"
+# How long the insight stack keeps evaluating an alert after the platform takes it over, so the two
+# stacks' checks can be compared. It sends nothing for the alert in that time.
+PLATFORM_SHADOW = timedelta(days=7)
+
# TTL for the tokenized chart URL embedded in Slack. Slack fetches the image at delivery,
# but the URL must stay resolvable while people scroll back to the message; 30 days matches
# the delivery-URL TTL used for task chart artifacts (products/tasks living_artifacts).
@@ -89,6 +95,15 @@ NON_TIME_SERIES_DISPLAY_TYPES = {
}
+def evaluated_by_insight_stack_q(now: datetime) -> Q:
+ """The alerts the insight stack still checks: every alert it owns, and a platform-owned alert inside its shadow."""
+ # A platform-owned row with no cutover time has no shadow to finish, so it is not evaluated.
+ return ~(
+ Q(owner=PlatformAlertOwner.PLATFORM.value)
+ & (Q(owner_changed_at__isnull=True) | Q(owner_changed_at__lt=now - PLATFORM_SHADOW))
+ )
+
+
def is_non_time_series_trend(query: TrendsQuery) -> bool:
display = query.trendsFilter.display if query.trendsFilter else None
return display in NON_TIME_SERIES_DISPLAY_TYPES
@@ -453,7 +468,7 @@ def dispatch_alert_notification(
"""Route an AlertCheck to the correct notification sender.
Returns the delivery receipts the notification produced, or None if nothing was sent
- (NOT_FIRING, or ERRORED with a non-dict error payload). Callers pass the returned
+ (NOT_FIRING, a platform-owned alert, or ERRORED with a non-dict error payload). Callers pass the returned
receipts to record_alert_delivery so the `targets_notified` sentinel reflects reality,
never claiming delivery for a state that didn't actually send.
@@ -472,6 +487,16 @@ def dispatch_alert_notification(
ValueError: state is FIRING but breaches is None/empty.
AssertionError: unknown state — surfaces a missing AlertState branch loudly.
"""
+ # An investigation dispatches minutes after it loaded `alert`, and the platform can take the
+ # alert over in that time, so the stored owner decides.
+ if (
+ alert.platform_owned
+ or AlertConfiguration.objects.filter(
+ team_id=alert.team_id, id=alert.id, owner=PlatformAlertOwner.PLATFORM
+ ).exists()
+ ):
+ logger.info("Alert is owned by the platform, nothing to send", alert_id=alert.id)
+ return None
key = idempotency_key or str(alert_check.id)
with ExitStack() as stack:
if alert_check.state == AlertState.FIRING:
@@ -588,13 +613,20 @@ def record_alert_delivery(
"users": [delivery.target for delivery in deliveries if delivery.channel == "email"],
"destinations": serialize_deliveries([d for d in deliveries if d.channel != "email"]),
}
- alert_check.notification_sent_at = recorded_at
- alert_check.save(update_fields=["targets_notified", "notification_sent_at"])
- alert.last_notified_at = recorded_at
- alert.save(update_fields=["last_notified_at"])
+ stamp_alert_notified(alert, alert_check, recorded_at, extra_check_fields=["targets_notified"])
return True
+def stamp_alert_notified(
+ alert: AlertConfiguration, alert_check: AlertCheck, at: datetime, *, extra_check_fields: Collection[str] = ()
+) -> None:
+ """Marks the check as notified and moves the alert's last notification time to `at`."""
+ alert_check.notification_sent_at = at
+ alert_check.save(update_fields=[*extra_check_fields, "notification_sent_at"])
+ alert.last_notified_at = at
+ alert.save(update_fields=["last_notified_at"])
+
+
def add_alert_check(
alert: AlertConfiguration,
evaluation_result: AlertEvaluationResult | None,
@@ -693,7 +725,7 @@ def disable_invalid_alert(
state=AlertState.ERRORED,
error=error,
)
- if targets_to_notify and notify_subscribers:
+ if targets_to_notify and notify_subscribers and not alert.platform_owned:
deliveries = send_notifications_for_disabled(alert, reason, targets_to_notify)
record_alert_delivery(alert, alert_check, deliveries)
return alert_check
diff --git a/posthog/temporal/alerts/activities.py b/posthog/temporal/alerts/activities.py
index 0e0b834a8f64..7ba6b616f85d 100644
--- a/posthog/temporal/alerts/activities.py
+++ b/posthog/temporal/alerts/activities.py
@@ -46,11 +46,13 @@ from posthog.tasks.alerts.utils import (
add_alert_check,
disable_invalid_alert,
dispatch_alert_notification,
+ evaluated_by_insight_stack_q,
get_alert_error_notification_recipients,
next_check_time,
next_scheduled_check_time,
record_alert_delivery,
skip_because_of_weekend,
+ stamp_alert_notified,
)
from posthog.temporal.alerts.admission import (
admit_evaluation_slots,
@@ -165,6 +167,7 @@ async def retrieve_due_alerts(inputs: ScheduleDueAlertChecksWorkflowInputs | Non
)
.filter(Q(snoozed_until__isnull=True) | Q(snoozed_until__lt=polled_at))
.filter(insight__deleted=False)
+ .filter(evaluated_by_insight_stack_q(polled_at))
)
# A check stays due until it finishes, so admitted alerts would otherwise be handed out again.
in_flight = inflight_alert_ids()
@@ -616,7 +619,21 @@ def _write_errored_alert_check(alert: AlertConfiguration, error: dict) -> tuple[
Both evaluate_alert's failure path and the retry-exhausted record_failed_evaluation activity go
through here, so the errored-check write stays in one place.
"""
- return add_alert_check(alert, None, error)
+ alert_check, should_notify = add_alert_check(alert, None, error)
+ return alert_check, _sends_from_insight_stack(alert, alert_check, should_notify)
+
+
+def _sends_from_insight_stack(alert: AlertConfiguration, alert_check: AlertCheck, should_notify: bool) -> bool:
+ """Whether the insight stack sends the check's notification. It sends nothing for a platform-owned alert.
+
+ A held send is still recorded as sent, so `last_notified_at` moves as it would have and the
+ shadow stays comparable with the platform. The stamp on the check also keeps the investigation
+ safety net from sending it later.
+ """
+ if not (should_notify and alert.platform_owned):
+ return should_notify
+ stamp_alert_notified(alert, alert_check, datetime.now(UTC))
+ return False
@temporalio.activity.defn
@@ -748,7 +765,9 @@ async def evaluate_alert(inputs: EvaluateAlertActivityInputs) -> EvaluateAlertRe
alert, invalid_configuration, notify_subscribers=False, error_code="invalid_configuration"
)
return EvaluateAlertResult(
- alert_check_id=str(alert_check.id), should_notify=True, new_state=AlertState.ERRORED
+ alert_check_id=str(alert_check.id),
+ should_notify=_sends_from_insight_stack(alert, alert_check, True),
+ new_state=AlertState.ERRORED,
)
if error is not None:
alert_check, should_notify = _write_errored_alert_check(alert, error)
@@ -761,6 +780,13 @@ async def evaluate_alert(inputs: EvaluateAlertActivityInputs) -> EvaluateAlertRe
return EvaluateAlertResult(
alert_check_id=str(alert_check.id), should_notify=False, new_state=AlertState(alert.state)
)
+ if alert.platform_owned:
+ # No investigation either: it exists to gate a notification, and each one costs a model call.
+ return EvaluateAlertResult(
+ alert_check_id=str(alert_check.id),
+ should_notify=_sends_from_insight_stack(alert, alert_check, should_notify),
+ new_state=AlertState(alert_check.state),
+ )
investigation = decide_investigation(alert, alert_check)
if investigation.should_investigate and claim_investigation_slot(alert, alert_check):
@@ -902,6 +928,8 @@ def _evaluation_fingerprint(alert: AlertConfiguration) -> str:
"schedule_start_time",
"schedule_restriction",
"snoozed_until",
+ "owner",
+ "owner_changed_at",
)
values = [
*(getattr(alert, field) for field in fields),
diff --git a/posthog/temporal/tests/test_alerts_activities.py b/posthog/temporal/tests/test_alerts_activities.py
index 50927812ca54..76f3c37d4740 100644
--- a/posthog/temporal/tests/test_alerts_activities.py
+++ b/posthog/temporal/tests/test_alerts_activities.py
@@ -85,6 +85,7 @@ from products.alerts.backend.facade.api import (
)
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, InvestigationStatus, Threshold
from products.alerts_platform.backend.facade.contracts import AlertDelivery
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
from products.product_analytics.backend.facade.models import Insight
@@ -1020,11 +1021,12 @@ class TestEvaluateAlert:
assert check.targets_notified == {}
@pytest.mark.parametrize(
- "investigated,previous_verdict,expect_investigation,expect_notify,expect_held",
+ "investigated,previous_verdict,owner,expect_investigation,expect_notify,expect_held",
[
- (1, "false_positive", True, True, False),
- (MAX_INVESTIGATIONS_PER_EPISODE, "false_positive", False, False, True),
- (MAX_INVESTIGATIONS_PER_EPISODE, "true_positive", False, True, False),
+ (1, "false_positive", PlatformAlertOwner.SOURCE, True, True, False),
+ (MAX_INVESTIGATIONS_PER_EPISODE, "false_positive", PlatformAlertOwner.SOURCE, False, False, True),
+ (MAX_INVESTIGATIONS_PER_EPISODE, "true_positive", PlatformAlertOwner.SOURCE, False, True, False),
+ (1, "false_positive", PlatformAlertOwner.PLATFORM, False, False, False),
],
)
async def test_gated_alert_holds_each_fire_on_a_verdict(
@@ -1032,6 +1034,7 @@ class TestEvaluateAlert:
ateam,
investigated: int,
previous_verdict: str,
+ owner: PlatformAlertOwner,
expect_investigation: bool,
expect_notify: bool,
expect_held: bool,
@@ -1041,7 +1044,10 @@ class TestEvaluateAlert:
@sync_to_async
def _seed_episode() -> None:
AlertConfiguration.objects.filter(id=alert.id).update(
- investigation_agent_enabled=True, investigation_gates_notifications=True
+ investigation_agent_enabled=True,
+ investigation_gates_notifications=True,
+ owner=owner,
+ owner_changed_at=datetime.now(UTC),
)
episode_start = datetime.now(UTC) - timedelta(days=5)
AlertCheck.objects.create(alert_configuration=alert, state=AlertState.NOT_FIRING)
@@ -1070,6 +1076,10 @@ class TestEvaluateAlert:
assert result.should_notify is expect_notify
check = await sync_to_async(AlertCheck.objects.get)(pk=result.alert_check_id)
assert check.notification_suppressed_by_agent is expect_held
+ if owner == PlatformAlertOwner.PLATFORM:
+ await alert.arefresh_from_db()
+ assert check.notification_sent_at is not None
+ assert alert.last_notified_at == check.notification_sent_at
async def test_evaluate_errored_when_permanent_exception(self, alert) -> None:
with patch(
@@ -1318,7 +1328,7 @@ class TestEvaluateAlert:
@pytest.mark.asyncio
@pytest.mark.django_db
class TestRecordFailedEvaluation:
- @pytest.mark.parametrize("change", [None, "detector", "insight", "threshold"])
+ @pytest.mark.parametrize("change", [None, "detector", "insight", "threshold", "owner"])
async def test_failure_only_records_against_the_prepared_inputs(self, alert, change) -> None:
env = ActivityEnvironment()
prepared = await env.run(prepare_alert, PrepareAlertActivityInputs(alert_id=str(alert.id)))
@@ -1334,6 +1344,10 @@ class TestRecordFailedEvaluation:
await sync_to_async(Threshold.objects.filter(id=alert.threshold_id).update)(
configuration={"type": "absolute", "bounds": {"upper": 200.0}}
)
+ elif change == "owner":
+ await sync_to_async(AlertConfiguration.objects.filter(id=alert.id).update)(
+ owner=PlatformAlertOwner.PLATFORM, owner_changed_at=datetime.now(UTC)
+ )
before = await sync_to_async(AlertConfiguration.objects.get)(id=alert.id)
due_before = before.next_check_at
result = await env.run(
@@ -1371,6 +1385,7 @@ class TestRecordFailedEvaluation:
),
)
assert result_again.alert_check_id is not None
+ assert result_again.should_notify is (change != "owner")
@pytest.mark.parametrize(
"error_type,expected_outcomes",
@@ -1440,8 +1455,16 @@ class TestRecordFailedEvaluation:
@pytest.mark.asyncio
@pytest.mark.django_db
class TestNotifyAlert:
- async def test_noop_when_not_firing(self, alert_with_user) -> None:
- check = await _create_alert_check(alert_with_user, state=AlertState.NOT_FIRING)
+ @pytest.mark.parametrize(
+ "state,owner",
+ [
+ (AlertState.NOT_FIRING, PlatformAlertOwner.SOURCE),
+ (AlertState.FIRING, PlatformAlertOwner.PLATFORM),
+ ],
+ )
+ async def test_noop_when_not_firing_or_platform_owned(self, alert_with_user, state, owner) -> None:
+ await sync_to_async(AlertConfiguration.objects.filter(id=alert_with_user.id).update)(owner=owner)
+ check = await _create_alert_check(alert_with_user, state=state)
with (
patch("posthog.tasks.alerts.utils.send_notifications_for_breaches") as mock_breaches,
@@ -1450,7 +1473,9 @@ class TestNotifyAlert:
env = ActivityEnvironment()
await env.run(
notify_alert,
- NotifyAlertActivityInputs(alert_id=str(alert_with_user.id), alert_check_id=str(check.id)),
+ NotifyAlertActivityInputs(
+ alert_id=str(alert_with_user.id), alert_check_id=str(check.id), breaches=["value above threshold"]
+ ),
)
mock_breaches.assert_not_called()
diff --git a/posthog/temporal/tests/test_alerts_retrieval.py b/posthog/temporal/tests/test_alerts_retrieval.py
index d04ce4eb7a25..80d7949da082 100644
--- a/posthog/temporal/tests/test_alerts_retrieval.py
+++ b/posthog/temporal/tests/test_alerts_retrieval.py
@@ -18,6 +18,9 @@ from posthog.temporal.alerts.admission import INFLIGHT_KEY, SLOT_LEASE_SECONDS,
from posthog.temporal.alerts.types import AlertInfo, ScheduleDueAlertChecksWorkflowInputs
from posthog.temporal.tests.test_alerts_activities import _create_alert
+from products.alerts.backend.models.alert import AlertConfiguration
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
+
@pytest.fixture(autouse=True)
def clear_inflight_slots():
@@ -93,14 +96,31 @@ async def test_retrieve_due_alerts_excludes_future_checks_and_applies_the_docume
next_check_at=next_check_at,
)
alert_label_by_id[str(alert.id)] = label
+ # Platform-owned and aged real-time, so each would lead the result if the insight stack still checked it.
+ platform_owned_specs = [
+ ("platform_owned_in_shadow", datetime(2026, 9, 5, tzinfo=UTC)),
+ ("platform_owned_after_shadow", datetime(2026, 9, 2, tzinfo=UTC)),
+ ("platform_owned_without_cutover_time", None),
+ ]
+ for label, owner_changed_at in platform_owned_specs:
+ alert = await _create_alert(
+ ateam,
+ calculation_interval=AlertCalculationInterval.REAL_TIME.value,
+ next_check_at=datetime(2026, 9, 10, 11, 20, tzinfo=UTC),
+ )
+ await sync_to_async(AlertConfiguration.objects.filter(id=alert.id).update)(
+ owner=PlatformAlertOwner.PLATFORM, owner_changed_at=owner_changed_at
+ )
+ alert_label_by_id[str(alert.id)] = label
with time_machine.travel("2026-09-10T12:00:00Z", tick=False):
alerts = await ActivityEnvironment().run(
retrieve_due_alerts,
- ScheduleDueAlertChecksWorkflowInputs(max_alerts_per_run=len(alert_specs)),
+ ScheduleDueAlertChecksWorkflowInputs(max_alerts_per_run=len(alert_label_by_id)),
)
assert [alert_label_by_id[alert.alert_id] for alert in alerts] == [
+ "platform_owned_in_shadow",
"aged_real_time",
"fresh_real_time",
"fresh_15_minutes",
diff --git a/products/alerts/backend/models/alert.py b/products/alerts/backend/models/alert.py
index a9ec7fa6b1f0..862425046ae1 100644
--- a/products/alerts/backend/models/alert.py
+++ b/products/alerts/backend/models/alert.py
@@ -218,6 +218,12 @@ class AlertConfiguration(ModelActivityMixin, CreatedMetaFields, UUIDTModel):
def __str__(self) -> str:
return f"{self.name} (Team: {self.team})"
+ @property
+ def platform_owned(self) -> bool:
+ """The platform evaluates and delivers this alert. The insight stack may still evaluate it to
+ compare, but it sends nothing for it."""
+ return self.owner == PlatformAlertOwner.PLATFORM
+
@property
def is_high_frequency_interval(self) -> bool:
# Real-time counts as high frequency so its checks always run fresh ClickHouse