Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): let the cutover move firing insight alerts (#129326)

Jimmy Zhufd838489cdd0
diff --git a/products/alerts/backend/platform_cutover.py b/products/alerts/backend/platform_cutover.py
index 4179b18ea981..a53946fdd1d1 100644
--- a/products/alerts/backend/platform_cutover.py
+++ b/products/alerts/backend/platform_cutover.py
@@ -19,6 +19,7 @@ from posthog.dataclasses import frozen
from posthog.schema_enums import AlertState
from posthog.tasks.alerts.utils import due_insight_alerts_q
from posthog.temporal.alerts.investigation import IN_FLIGHT_INVESTIGATION_HORIZON
+from posthog.temporal.alerts.retry_policy import alert_timeouts
from products.alerts.backend.facade.destinations import choose_delivery_path
from products.alerts.backend.insight_alert_state_machine import apply_invalid_configuration, apply_platform_state
@@ -34,8 +35,10 @@ from products.alerts_platform.backend.facade.contracts import (
from products.alerts_platform.backend.facade.delivery import has_legacy_event_emitter
from products.alerts_platform.backend.facade.enums import PlatformAlertOwner, PlatformAlertState
-# A firing never moves, so only these states are carried over.
PLATFORM_HANDOVER_STATE: Final[dict[str, PlatformAlertState]] = {
+ # The platform carries the firing with its start unknown. An insight alert notifies on every
+ # firing check, so the platform's next check sends what the insight stack would have sent.
+ AlertState.FIRING: PlatformAlertState.FIRING,
AlertState.NOT_FIRING: PlatformAlertState.NOT_FIRING,
# The platform keeps a snooze on the copy's `snooze_until`, which the copy takes from this alert.
AlertState.SNOOZED: PlatformAlertState.NOT_FIRING,
@@ -73,8 +76,6 @@ class CutoverSkip(StrEnum):
ENVIRONMENT = "environment"
# The platform cannot copy it, such as an alert with no cadence.
NOT_ON_PLATFORM = "not_on_platform"
- # The two stacks key a firing differently, so a firing moves only after it resolves.
- FIRING = "firing"
# The insight scheduler owes this alert a check, which may already be running and could notify
# after the move.
CHECK_DUE = "check_due"
@@ -86,6 +87,9 @@ class CutoverSkip(StrEnum):
# The insight stack's agent still runs for a recent firing, or its held notification is unsent.
# Moved now, that notification could not go out, and the platform would start a second agent.
INVESTIGATION_UNSETTLED = "investigation_unsettled"
+ # A recent check fired and its workflow can still notify. Moved now, that notification would be
+ # dropped, and nobody would hear of the firing if the platform's next check clears.
+ NOTIFICATION_PENDING = "notification_pending"
@frozen
@@ -124,6 +128,19 @@ def _investigation_unsettled(alert: AlertConfiguration, at: datetime) -> bool:
)
+def _firing_notification_pending(alert: AlertConfiguration, at: datetime) -> bool:
+ # A check's workflow notifies before its execution timeout ends, so past that the insight stack
+ # has sent the notification or given up on it.
+ horizon = alert_timeouts(alert.calculation_interval).workflow_execution
+ return AlertCheck.objects.filter(
+ alert_configuration=alert,
+ created_at__gte=at - horizon,
+ state=AlertState.FIRING,
+ notification_sent_at__isnull=True,
+ notification_suppressed_by_agent=False,
+ ).exists()
+
+
def cut_over_team(team_id: int, *, now: datetime | None = None) -> tuple[AlertCutover, ...]:
"""Hands every insight alert of one team to the platform, or says why one stays.
@@ -168,12 +185,12 @@ def _cut_over_alert(team_id: int, alert_id: UUID, now: datetime | None) -> Cutov
return CutoverSkip.NOT_ON_PLATFORM
if not delivers_as_it_does_today(alert):
return CutoverSkip.NO_HOG_FUNCTION_PATH
- if alert.state == AlertState.FIRING:
- return CutoverSkip.FIRING
if _insight_check_owed(team_id, alert.id, at):
return CutoverSkip.CHECK_DUE
if _investigation_unsettled(alert, at):
return CutoverSkip.INVESTIGATION_UNSETTLED
+ if _firing_notification_pending(alert, at):
+ return CutoverSkip.NOTIFICATION_PENDING
if not hand_over(alert, at):
return CutoverSkip.PLATFORM_FIRING
return None
diff --git a/products/alerts/backend/test/test_platform_cutover.py b/products/alerts/backend/test/test_platform_cutover.py
index 2aea7c2ddf6c..19730b414d49 100644
--- a/products/alerts/backend/test/test_platform_cutover.py
+++ b/products/alerts/backend/test/test_platform_cutover.py
@@ -30,6 +30,7 @@ class TestInsightPlatformCutover(APIBaseTest):
@parameterized.expand(
[
("quiet", {}, "not_firing"),
+ ("firing", {"state": AlertState.FIRING}, "firing"),
("errored", {"state": AlertState.ERRORED}, "errored"),
(
"snoozed_keeps_checking_muted",
@@ -69,7 +70,6 @@ class TestInsightPlatformCutover(APIBaseTest):
@parameterized.expand(
[
- ("firing", {"state": AlertState.FIRING}, CutoverSkip.FIRING),
("check_due", {"next_check_at": NOW}, CutoverSkip.CHECK_DUE),
("never_scheduled", {"next_check_at": None}, CutoverSkip.CHECK_DUE),
]
@@ -103,10 +103,13 @@ class TestInsightPlatformCutover(APIBaseTest):
),
("notification_sent", InvestigationStatus.DONE, NOW, timedelta(minutes=10), None),
("past_the_horizon", InvestigationStatus.RUNNING, None, timedelta(hours=2), None),
+ ("firing_not_yet_notified", None, None, timedelta(minutes=5), CutoverSkip.NOTIFICATION_PENDING),
+ ("firing_notified", None, NOW, timedelta(minutes=5), None),
+ ("firing_past_its_workflow", None, None, timedelta(minutes=30), None),
]
)
- def test_an_alert_moves_only_once_its_investigation_settles(
- self, _name: str, status: str, sent_at: datetime | None, age: timedelta, reason: CutoverSkip | None
+ def test_an_alert_moves_only_once_its_last_firing_settles(
+ self, _name: str, status: str | None, sent_at: datetime | None, age: timedelta, reason: CutoverSkip | None
) -> None:
alert = self._alert(investigation_agent_enabled=True, detector_config={"type": "zscore"})
check = AlertCheck.objects.create(