:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): let the cutover move firing insight alerts (#129326)
Jimmy Zhufd838489cdd0
diff --git a/products/alerts/backend/platform_cutover.py b/products/alerts/backend/platform_cutover.pyindex 4179b18ea981..a53946fdd1d1 100644--- a/products/alerts/backend/platform_cutover.py+++ b/products/alerts/backend/platform_cutover.py@@ -19,6 +19,7 @@ from posthog.dataclasses import frozenfrom posthog.schema_enums import AlertStatefrom posthog.tasks.alerts.utils import due_insight_alerts_qfrom posthog.temporal.alerts.investigation import IN_FLIGHT_INVESTIGATION_HORIZON+from posthog.temporal.alerts.retry_policy import alert_timeoutsfrom products.alerts.backend.facade.destinations import choose_delivery_pathfrom products.alerts.backend.insight_alert_state_machine import apply_invalid_configuration, apply_platform_state@@ -34,8 +35,10 @@ from products.alerts_platform.backend.facade.contracts import (from products.alerts_platform.backend.facade.delivery import has_legacy_event_emitterfrom products.alerts_platform.backend.facade.enums import PlatformAlertOwner, PlatformAlertState-# A firing never moves, so only these states are carried over.PLATFORM_HANDOVER_STATE: Final[dict[str, PlatformAlertState]] = {+ # The platform carries the firing with its start unknown. An insight alert notifies on every+ # firing check, so the platform's next check sends what the insight stack would have sent.+ AlertState.FIRING: PlatformAlertState.FIRING,AlertState.NOT_FIRING: PlatformAlertState.NOT_FIRING,# The platform keeps a snooze on the copy's `snooze_until`, which the copy takes from this alert.AlertState.SNOOZED: PlatformAlertState.NOT_FIRING,@@ -73,8 +76,6 @@ class CutoverSkip(StrEnum):ENVIRONMENT = "environment"# The platform cannot copy it, such as an alert with no cadence.NOT_ON_PLATFORM = "not_on_platform"- # The two stacks key a firing differently, so a firing moves only after it resolves.- FIRING = "firing"# The insight scheduler owes this alert a check, which may already be running and could notify# after the move.CHECK_DUE = "check_due"@@ -86,6 +87,9 @@ class CutoverSkip(StrEnum):# The insight stack's agent still runs for a recent firing, or its held notification is unsent.# Moved now, that notification could not go out, and the platform would start a second agent.INVESTIGATION_UNSETTLED = "investigation_unsettled"+ # A recent check fired and its workflow can still notify. Moved now, that notification would be+ # dropped, and nobody would hear of the firing if the platform's next check clears.+ NOTIFICATION_PENDING = "notification_pending"@frozen@@ -124,6 +128,19 @@ def _investigation_unsettled(alert: AlertConfiguration, at: datetime) -> bool:)+def _firing_notification_pending(alert: AlertConfiguration, at: datetime) -> bool:+ # A check's workflow notifies before its execution timeout ends, so past that the insight stack+ # has sent the notification or given up on it.+ horizon = alert_timeouts(alert.calculation_interval).workflow_execution+ return AlertCheck.objects.filter(+ alert_configuration=alert,+ created_at__gte=at - horizon,+ state=AlertState.FIRING,+ notification_sent_at__isnull=True,+ notification_suppressed_by_agent=False,+ ).exists()++def cut_over_team(team_id: int, *, now: datetime | None = None) -> tuple[AlertCutover, ...]:"""Hands every insight alert of one team to the platform, or says why one stays.@@ -168,12 +185,12 @@ def _cut_over_alert(team_id: int, alert_id: UUID, now: datetime | None) -> Cutovreturn CutoverSkip.NOT_ON_PLATFORMif not delivers_as_it_does_today(alert):return CutoverSkip.NO_HOG_FUNCTION_PATH- if alert.state == AlertState.FIRING:- return CutoverSkip.FIRINGif _insight_check_owed(team_id, alert.id, at):return CutoverSkip.CHECK_DUEif _investigation_unsettled(alert, at):return CutoverSkip.INVESTIGATION_UNSETTLED+ if _firing_notification_pending(alert, at):+ return CutoverSkip.NOTIFICATION_PENDINGif not hand_over(alert, at):return CutoverSkip.PLATFORM_FIRINGreturn Nonediff --git a/products/alerts/backend/test/test_platform_cutover.py b/products/alerts/backend/test/test_platform_cutover.pyindex 2aea7c2ddf6c..19730b414d49 100644--- a/products/alerts/backend/test/test_platform_cutover.py+++ b/products/alerts/backend/test/test_platform_cutover.py@@ -30,6 +30,7 @@ class TestInsightPlatformCutover(APIBaseTest):@parameterized.expand([("quiet", {}, "not_firing"),+ ("firing", {"state": AlertState.FIRING}, "firing"),("errored", {"state": AlertState.ERRORED}, "errored"),("snoozed_keeps_checking_muted",@@ -69,7 +70,6 @@ class TestInsightPlatformCutover(APIBaseTest):@parameterized.expand([- ("firing", {"state": AlertState.FIRING}, CutoverSkip.FIRING),("check_due", {"next_check_at": NOW}, CutoverSkip.CHECK_DUE),("never_scheduled", {"next_check_at": None}, CutoverSkip.CHECK_DUE),]@@ -103,10 +103,13 @@ class TestInsightPlatformCutover(APIBaseTest):),("notification_sent", InvestigationStatus.DONE, NOW, timedelta(minutes=10), None),("past_the_horizon", InvestigationStatus.RUNNING, None, timedelta(hours=2), None),+ ("firing_not_yet_notified", None, None, timedelta(minutes=5), CutoverSkip.NOTIFICATION_PENDING),+ ("firing_notified", None, NOW, timedelta(minutes=5), None),+ ("firing_past_its_workflow", None, None, timedelta(minutes=30), None),])- def test_an_alert_moves_only_once_its_investigation_settles(- self, _name: str, status: str, sent_at: datetime | None, age: timedelta, reason: CutoverSkip | None+ def test_an_alert_moves_only_once_its_last_firing_settles(+ self, _name: str, status: str | None, sent_at: datetime | None, age: timedelta, reason: CutoverSkip | None) -> None:alert = self._alert(investigation_agent_enabled=True, detector_config={"type": "zscore"})check = AlertCheck.objects.create(