Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): add per-team commands that hand insight alerts to the platform and back (#120636)

Jimmy Zhu8fdec7a1b27f
diff --git a/posthog/tasks/alerts/utils.py b/posthog/tasks/alerts/utils.py
index 65492f95fab6..73ed7d813baa 100644
--- a/posthog/tasks/alerts/utils.py
+++ b/posthog/tasks/alerts/utils.py
@@ -104,6 +104,17 @@ def evaluated_by_insight_stack_q(now: datetime) -> Q:
)
+def due_insight_alerts_q(now: datetime) -> Q:
+ """The alerts the insight scheduler owes a check at `now`."""
+ return (
+ Q(enabled=True)
+ & (Q(next_check_at__lte=now) | Q(next_check_at__isnull=True))
+ & (Q(snoozed_until__isnull=True) | Q(snoozed_until__lt=now))
+ & Q(insight__deleted=False)
+ & evaluated_by_insight_stack_q(now)
+ )
+
+
def is_non_time_series_trend(query: TrendsQuery) -> bool:
display = query.trendsFilter.display if query.trendsFilter else None
return display in NON_TIME_SERIES_DISPLAY_TYPES
diff --git a/posthog/temporal/alerts/activities.py b/posthog/temporal/alerts/activities.py
index 7ba6b616f85d..362327e36226 100644
--- a/posthog/temporal/alerts/activities.py
+++ b/posthog/temporal/alerts/activities.py
@@ -14,7 +14,7 @@ from typing import ParamSpec, TypeVar
from django.core.serializers.json import DjangoJSONEncoder
from django.db import transaction
-from django.db.models import Case, Count, F, IntegerField, Min, Q, Value, When, Window
+from django.db.models import Case, Count, F, IntegerField, Min, Value, When, Window
from django.db.models.functions import Coalesce, RowNumber
import structlog
@@ -46,7 +46,7 @@ from posthog.tasks.alerts.utils import (
add_alert_check,
disable_invalid_alert,
dispatch_alert_notification,
- evaluated_by_insight_stack_q,
+ due_insight_alerts_q,
get_alert_error_notification_recipients,
next_check_time,
next_scheduled_check_time,
@@ -161,14 +161,7 @@ async def retrieve_due_alerts(inputs: ScheduleDueAlertChecksWorkflowInputs | Non
output_field=IntegerField(),
)
- due_alerts_query = (
- AlertConfiguration.objects.filter(
- Q(enabled=True, next_check_at__lte=polled_at) | Q(enabled=True, next_check_at__isnull=True)
- )
- .filter(Q(snoozed_until__isnull=True) | Q(snoozed_until__lt=polled_at))
- .filter(insight__deleted=False)
- .filter(evaluated_by_insight_stack_q(polled_at))
- )
+ due_alerts_query = AlertConfiguration.objects.filter(due_insight_alerts_q(polled_at))
# A check stays due until it finishes, so admitted alerts would otherwise be handed out again.
in_flight = inflight_alert_ids()
selectable_query = due_alerts_query.exclude(id__in=in_flight) if in_flight else due_alerts_query
diff --git a/products/alerts/backend/facade/contracts.py b/products/alerts/backend/facade/contracts.py
index 1c3bc23defbb..fbf6a3f5a08d 100644
--- a/products/alerts/backend/facade/contracts.py
+++ b/products/alerts/backend/facade/contracts.py
@@ -81,6 +81,8 @@ class HogFunctionPathReason(StrEnum):
# body the HogFunction sends.
NEW_WEBHOOK_BODY = "new_webhook_body"
WEBHOOK_BODY_UNKNOWN = "webhook_body_unknown"
+ # Native insight delivery sends no incident actions, so a PagerDuty destination would open none.
+ PAGERDUTY_NOT_NATIVE = "pagerduty_not_native"
@frozen
diff --git a/products/alerts/backend/insight_alert_state_machine.py b/products/alerts/backend/insight_alert_state_machine.py
index 00acef4d46c5..01e577d5aff7 100644
--- a/products/alerts/backend/insight_alert_state_machine.py
+++ b/products/alerts/backend/insight_alert_state_machine.py
@@ -121,5 +121,10 @@ def apply_invalid_configuration(alert: AlertConfiguration) -> list[str]:
]
+def apply_platform_state(alert: AlertConfiguration, state: InsightAlertState) -> list[str]:
+ """Takes a state the platform holds for the alert, such as the one it hands back with ownership."""
+ return apply_outcome(alert, ControlPlaneOutcome(new_state=platform_state_of(state), consecutive_failures=0))
+
+
def should_notify(outcome: AlertCheckOutcome) -> bool:
return outcome.notification != NotificationAction.NONE
diff --git a/products/alerts/backend/logic/delivery_path.py b/products/alerts/backend/logic/delivery_path.py
index d1a7ebea7eb5..786745ffb8a0 100644
--- a/products/alerts/backend/logic/delivery_path.py
+++ b/products/alerts/backend/logic/delivery_path.py
@@ -20,7 +20,7 @@ from products.alerts.backend.facade.contracts import (
DestinationEligibility,
HogFunctionPathReason,
)
-from products.alerts.backend.logic.destination_configs import WebhookDestination
+from products.alerts.backend.logic.destination_configs import PagerDutyDestination, WebhookDestination
from products.alerts.backend.logic.destination_eligibility import (
bucket_for,
classify_alert_destinations,
@@ -72,6 +72,11 @@ class _TeamDestinations:
def choice(self, destinations: Sequence[DestinationEligibility]) -> DeliveryPathChoice:
if any(bucket_for(destination) != DestinationBucket.STOCK for destination in destinations):
return DeliveryPathChoice(reason=HogFunctionPathReason.CUSTOMIZED)
+ # Per alert, so the team's other alerts still deliver natively.
+ if self._source == SourceKind.INSIGHT and any(
+ destination.template_id == PagerDutyDestination.template_id for destination in destinations
+ ):
+ return DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE)
if any(destination.template_id == WebhookDestination.template_id for destination in destinations):
if self._native_webhook_body is None:
return DeliveryPathChoice(reason=HogFunctionPathReason.WEBHOOK_BODY_UNKNOWN)
diff --git a/products/alerts/backend/management/commands/cut_over_insight_alerts_to_platform.py b/products/alerts/backend/management/commands/cut_over_insight_alerts_to_platform.py
new file mode 100644
index 000000000000..43e95e6dd0e6
--- /dev/null
+++ b/products/alerts/backend/management/commands/cut_over_insight_alerts_to_platform.py
@@ -0,0 +1,41 @@
+from collections import Counter
+from typing import Any
+
+from django.core.management.base import BaseCommand
+
+from products.alerts.backend.facade.destinations import team_delivery_paths
+from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
+from products.alerts.backend.models.alert import AlertConfiguration
+from products.alerts.backend.platform_cutover import cut_over_team
+from products.alerts_platform.backend.facade.contracts import SourceKind
+from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENT
+
+
+class Command(BaseCommand):
+ help = "Hand one team's insight alerts to the alerts platform. Safe to run again."
+
+ def add_arguments(self, parser: Any) -> None:
+ parser.add_argument("--team-id", type=int, required=True, help="The team to move.")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ team_id = options["team_id"]
+ alert_ids = [
+ str(alert_id)
+ for alert_id in AlertConfiguration.objects.filter(team_id=team_id).values_list("id", flat=True)
+ ]
+ paths = team_delivery_paths(
+ source=SourceKind.INSIGHT,
+ team_id=team_id,
+ alert_ids=alert_ids,
+ allowed_event_ids=[LEGACY_INSIGHT_ALERT_EVENT, INSIGHT_ALERT_ERRORED_EVENT_ID],
+ )
+ for alert_id, choice in paths.items():
+ if choice.reason is not None:
+ self.stdout.write(f"alert={alert_id} path={choice.path} reason={choice.reason}")
+ results = cut_over_team(team_id)
+ for result in results:
+ if result.skipped is not None:
+ self.stdout.write(f"alert={result.alert_id} skipped={result.skipped}")
+ delivery = Counter(str(choice.path) for choice in paths.values())
+ outcomes = Counter(str(result.skipped or "moved") for result in results)
+ self.stdout.write(f"team={team_id} {dict(outcomes) or 'no alerts'} paths={dict(delivery)}")
diff --git a/products/alerts/backend/management/commands/roll_back_insight_alerts_from_platform.py b/products/alerts/backend/management/commands/roll_back_insight_alerts_from_platform.py
new file mode 100644
index 000000000000..ccb237ef0d07
--- /dev/null
+++ b/products/alerts/backend/management/commands/roll_back_insight_alerts_from_platform.py
@@ -0,0 +1,21 @@
+from collections import Counter
+from typing import Any
+
+from django.core.management.base import BaseCommand
+
+from products.alerts.backend.platform_cutover import roll_back_team
+
+
+class Command(BaseCommand):
+ help = "Hand one team's insight alerts back from the alerts platform. Safe to run again."
+
+ def add_arguments(self, parser: Any) -> None:
+ parser.add_argument("--team-id", type=int, required=True, help="The team to move back.")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ results = roll_back_team(options["team_id"])
+ for result in results:
+ if result.skipped is not None:
+ self.stdout.write(f"alert={result.alert_id} skipped={result.skipped}")
+ outcomes = Counter(str(result.skipped or "moved_back") for result in results)
+ self.stdout.write(f"team={options['team_id']} {dict(outcomes) or 'no alerts'}")
diff --git a/products/alerts/backend/platform_cutover.py b/products/alerts/backend/platform_cutover.py
new file mode 100644
index 000000000000..3701cc2edf9b
--- /dev/null
+++ b/products/alerts/backend/platform_cutover.py
@@ -0,0 +1,232 @@
+"""Hands a team's insight alerts to the alerts platform and back, one alert at a time.
+
+Each alert moves in its own transaction, under its own row lock. An alert that cannot move now is
+skipped with a reason, and a later run picks it up. The command is safe to run again.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Final
+from uuid import UUID
+
+from django.db import transaction
+from django.utils import timezone
+
+from posthog.dataclasses import frozen
+from posthog.schema_enums import AlertCalculationInterval, AlertState
+from posthog.tasks.alerts.utils import due_insight_alerts_q
+
+from products.alerts.backend.facade.destinations import choose_delivery_path
+from products.alerts.backend.insight_alert_state_machine import apply_invalid_configuration, apply_platform_state
+from products.alerts.backend.models.alert import AlertConfiguration
+from products.alerts.backend.platform_source_cycle import DELIVERY_EVENT_IDS, platform_upsert_of
+from products.alerts_platform.backend.facade.api import give_back_ownership, take_ownership, upsert_configuration
+from products.alerts_platform.backend.facade.contracts import (
+ DeliveryPath,
+ HandbackRefusal,
+ PlatformOwnershipHandover,
+ SourceKind,
+)
+from products.alerts_platform.backend.facade.delivery import has_legacy_event_emitter
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner, PlatformAlertState
+
+# A firing never moves, so only these states are carried over.
+PLATFORM_HANDOVER_STATE: Final[dict[str, PlatformAlertState]] = {
+ AlertState.NOT_FIRING: PlatformAlertState.NOT_FIRING,
+ # The platform keeps a snooze on the copy's `snooze_until`, which the copy takes from this alert.
+ AlertState.SNOOZED: PlatformAlertState.NOT_FIRING,
+ AlertState.ERRORED: PlatformAlertState.ERRORED,
+}
+# Insight has no broken state. A configuration the platform found broken is one the insight stack
+# reports as errored and disables.
+_INSIGHT_STATE_OF_PLATFORM: Final[dict[PlatformAlertState, AlertState]] = {
+ PlatformAlertState.NOT_FIRING: AlertState.NOT_FIRING,
+ PlatformAlertState.SNOOZED: AlertState.NOT_FIRING,
+ PlatformAlertState.ERRORED: AlertState.ERRORED,
+ PlatformAlertState.BROKEN: AlertState.ERRORED,
+}
+
+
+class CutoverSkip(StrEnum):
+ """Why an alert stays with the insight stack for now."""
+
+ ALREADY_MOVED = "already_moved"
+ # The platform does not evaluate it in parallel, such as a detector or a real-time alert.
+ NOT_ON_PLATFORM = "not_on_platform"
+ # The investigation agent reads and writes the insight stack's checks, which the platform does not keep.
+ INVESTIGATION = "investigation"
+ # The two stacks key a firing differently, so a firing moves only after it resolves.
+ FIRING = "firing"
+ # The insight scheduler owes this alert a check, which may already be running and could notify
+ # after the move.
+ CHECK_DUE = "check_due"
+ # A team on live delivery can have the platform copy inside a firing of its own.
+ PLATFORM_FIRING = "platform_firing"
+ # Its destinations need the HogFunction path, and nothing emits the insight events that path
+ # sends, so the platform would deliver it natively and lose what its destinations add.
+ NO_HOG_FUNCTION_PATH = "no_hog_function_path"
+
+
+@frozen
+class AlertCutover:
+ alert_id: UUID
+ skipped: CutoverSkip | None
+
+
+def _team_alert_ids(team_id: int) -> list[UUID]:
+ return list(
+ AlertConfiguration.objects.filter(team_id=team_id).order_by("created_at", "id").values_list("id", flat=True)
+ )
+
+
+def _insight_check_owed(team_id: int, alert_id: UUID, at: datetime) -> bool:
+ # A check advances `next_check_at` when it saves, so an alert the scheduler does not owe a check
+ # under the caller's lock has no check between reading it and notifying for it.
+ return AlertConfiguration.objects.filter(team_id=team_id, id=alert_id).filter(due_insight_alerts_q(at)).exists()
+
+
+def cut_over_team(team_id: int, *, now: datetime | None = None) -> tuple[AlertCutover, ...]:
+ """Hands every insight alert of one team to the platform, or says why one stays.
+
+ No destination holds an alert back: delivery sends a customized one through its HogFunction.
+ """
+ return tuple(
+ AlertCutover(alert_id=alert_id, skipped=_cut_over_alert(team_id, alert_id, now))
+ for alert_id in _team_alert_ids(team_id)
+ )
+
+
+def _is_threshold_alert(alert: AlertConfiguration) -> bool:
+ """The platform evaluates detector and real-time alerts in parallel, but cannot own them yet."""
+ return not alert.detector_config and alert.calculation_interval != AlertCalculationInterval.REAL_TIME
+
+
+def delivers_as_it_does_today(alert: AlertConfiguration) -> bool:
+ """Whether the platform can deliver the alert the way its destinations expect: natively, or
+ through its HogFunctions when the insight events those listen on are emitted."""
+ if has_legacy_event_emitter(SourceKind.INSIGHT):
+ return True
+ path = choose_delivery_path(
+ source=SourceKind.INSIGHT,
+ team_id=alert.team_id,
+ alert_id=str(alert.id),
+ allowed_event_ids=DELIVERY_EVENT_IDS,
+ )
+ return path == DeliveryPath.NATIVE
+
+
+def _cut_over_alert(team_id: int, alert_id: UUID, now: datetime | None) -> CutoverSkip | None:
+ with transaction.atomic():
+ # `of` keeps the lock off the nullable threshold join, which PostgreSQL refuses to lock.
+ alert = (
+ AlertConfiguration.objects.select_for_update(no_key=True, of=("self",))
+ .select_related("threshold")
+ .get(team_id=team_id, id=alert_id)
+ )
+ # Read after the lock, so a check that became due while earlier alerts moved is seen as due.
+ at = now or timezone.now()
+ if alert.platform_owned:
+ return CutoverSkip.ALREADY_MOVED
+ upsert = platform_upsert_of(alert)
+ if upsert is None or not _is_threshold_alert(alert):
+ return CutoverSkip.NOT_ON_PLATFORM
+ if alert.investigation_agent_enabled:
+ return CutoverSkip.INVESTIGATION
+ if not delivers_as_it_does_today(alert):
+ return CutoverSkip.NO_HOG_FUNCTION_PATH
+ if alert.state == AlertState.FIRING:
+ return CutoverSkip.FIRING
+ if _insight_check_owed(team_id, alert.id, at):
+ return CutoverSkip.CHECK_DUE
+
+ savepoint = transaction.savepoint()
+ upsert_configuration(upsert)
+ taken = take_ownership(
+ PlatformOwnershipHandover(
+ legacy_configuration_id=alert.id,
+ team_id=alert.team_id,
+ state=PLATFORM_HANDOVER_STATE[alert.state],
+ last_notified_at=alert.last_notified_at,
+ consecutive_failures=0,
+ next_check_at=alert.next_check_at or at,
+ at=at,
+ )
+ )
+ if not taken:
+ # The copy stays as live delivery left it. A copy disabled here would drop out of
+ # discovery and never resolve the firing it is inside.
+ transaction.savepoint_rollback(savepoint)
+ return CutoverSkip.PLATFORM_FIRING
+ transaction.savepoint_commit(savepoint)
+ AlertConfiguration.objects.filter(team_id=alert.team_id, id=alert.id).update(
+ owner=PlatformAlertOwner.PLATFORM, owner_changed_at=at
+ )
+ return None
+
+
+class RollbackSkip(StrEnum):
+ """Why an alert stays with the platform for now."""
+
+ NOT_MOVED = "not_moved"
+ # During the shadow the insight stack still checks the alert, and a check it started would save
+ # over the state carried back.
+ INSIGHT_CHECK_DUE = "insight_check_due"
+ PLATFORM_FIRING = "platform_firing"
+ PLATFORM_CHECK_DUE = "platform_check_due"
+ NOT_PORTABLE = "not_portable"
+
+
+_ROLLBACK_SKIP: Final[dict[HandbackRefusal, RollbackSkip]] = {
+ HandbackRefusal.FIRING: RollbackSkip.PLATFORM_FIRING,
+ HandbackRefusal.CHECK_DUE: RollbackSkip.PLATFORM_CHECK_DUE,
+ HandbackRefusal.NOT_PORTABLE: RollbackSkip.NOT_PORTABLE,
+}
+
+
+@frozen
+class AlertRollback:
+ alert_id: UUID
+ skipped: RollbackSkip | None
+
+
+def roll_back_team(team_id: int, *, now: datetime | None = None) -> tuple[AlertRollback, ...]:
+ """Hands every insight alert of one team back from the platform, or says why one stays."""
+ return tuple(
+ AlertRollback(alert_id=alert_id, skipped=_roll_back_alert(team_id, alert_id, now))
+ for alert_id in _team_alert_ids(team_id)
+ )
+
+
+def _roll_back_alert(team_id: int, alert_id: UUID, now: datetime | None) -> RollbackSkip | None:
+ with transaction.atomic():
+ alert = AlertConfiguration.objects.select_for_update(no_key=True).get(team_id=team_id, id=alert_id)
+ # Read after the lock, for the same reason as the cutover.
+ at = now or timezone.now()
+ if not alert.platform_owned:
+ return RollbackSkip.NOT_MOVED
+ if _insight_check_owed(team_id, alert.id, at):
+ return RollbackSkip.INSIGHT_CHECK_DUE
+
+ handback = give_back_ownership(
+ team_id=alert.team_id, legacy_configuration_id=alert.id, at=at, source_runs_calendar_schedules=True
+ )
+ if isinstance(handback, HandbackRefusal):
+ return _ROLLBACK_SKIP[handback]
+
+ # A snooze lives on this alert, so an active one stays its state.
+ snoozed = alert.snoozed_until is not None and alert.snoozed_until > at
+ update_fields = apply_platform_state(
+ alert, AlertState.SNOOZED if snoozed else _INSIGHT_STATE_OF_PLATFORM[handback.state]
+ )
+ if not handback.enabled and alert.enabled:
+ # The platform switches a copy off only for a configuration it cannot run, which the
+ # insight stack reports the same way, so it does not resume the alert.
+ update_fields.extend(apply_invalid_configuration(alert))
+ alert.owner = PlatformAlertOwner.SOURCE
+ alert.owner_changed_at = at
+ alert.last_notified_at = handback.last_notified_at
+ alert.next_check_at = handback.next_check_at
+ alert.save(update_fields=[*update_fields, "owner", "owner_changed_at", "last_notified_at", "next_check_at"])
+ return None
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 0db5f9e65f35..0d0a592d521b 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -140,6 +140,8 @@ _EVENT_IDS_BY_KIND: Final[dict[str, str]] = {
AlertEventKind.FIRING.value: LEGACY_INSIGHT_ALERT_EVENT,
AlertEventKind.ERRORED.value: INSIGHT_ALERT_ERRORED_EVENT_ID,
}
+# Every event an insight alert's destinations can listen on, for a question about all of them.
+DELIVERY_EVENT_IDS: Final = tuple(_EVENT_IDS_BY_KIND.values())
def is_evaluated_on_the_platform(alert: AlertConfiguration) -> bool:
@@ -446,6 +448,12 @@ def _decide(
check = _following(check, alert)
if not alert.enabled or alert.insight.deleted or not is_evaluated_on_the_platform(alert):
return _skipped(check, snapshot, now=now)
+ # As production: a plan that lost a cadence stops the alert, because the API checks it only on a write.
+ entitlement_error = AlertConfiguration.interval_entitlement_error(
+ calculation_interval=alert.calculation_interval, organization=alert.team.organization
+ )
+ if entitlement_error:
+ return _broken(check, snapshot, now=now, message=entitlement_error)
if skip_because_of_weekend(alert) or is_utc_datetime_blocked(alert, now):
return _skipped(check, snapshot, now=now)
if is_llm_detector_config(alert.detector_config) != uses_llm_detector:
diff --git a/products/alerts/backend/test/test_delivery_path.py b/products/alerts/backend/test/test_delivery_path.py
new file mode 100644
index 000000000000..33a323252329
--- /dev/null
+++ b/products/alerts/backend/test/test_delivery_path.py
@@ -0,0 +1,33 @@
+from uuid import uuid4
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from products.alerts.backend.facade.contracts import DeliveryPathChoice, DestinationEligibility, HogFunctionPathReason
+from products.alerts.backend.logic.delivery_path import _TeamDestinations
+from products.alerts_platform.backend.facade.contracts import DeliveryPath, SourceKind
+
+
+def _destination(template_id: str) -> DestinationEligibility:
+ return DestinationEligibility(hog_function_id=uuid4(), template_id=template_id, hold_backs=(), code_hash="")
+
+
+class TestDeliveryPathChoice(SimpleTestCase):
+ @parameterized.expand(
+ [
+ # Native insight delivery sends no incident actions, so the function keeps opening them.
+ (SourceKind.INSIGHT, DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE)),
+ (SourceKind.LOGS, DeliveryPathChoice()),
+ ]
+ )
+ def test_a_stock_pagerduty_destination_keeps_its_hog_function_only_for_insight(
+ self, source: SourceKind, expected: DeliveryPathChoice
+ ) -> None:
+ team = _TeamDestinations(source=source, team_id=1, allowed_event_ids=())
+
+ paged = team.choice([_destination("template-pagerduty"), _destination("template-slack")])
+ # Another alert of the same team, so the choice is per alert.
+ other = team.choice([_destination("template-slack")])
+
+ assert (paged, other.path) == (expected, DeliveryPath.NATIVE)
diff --git a/products/alerts/backend/test/test_platform_cutover.py b/products/alerts/backend/test/test_platform_cutover.py
new file mode 100644
index 000000000000..b265bd209e68
--- /dev/null
+++ b/products/alerts/backend/test/test_platform_cutover.py
@@ -0,0 +1,173 @@
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from posthog.test.base import APIBaseTest
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from posthog.models.scoping import team_scope
+from posthog.schema_enums import AlertCalculationInterval, AlertState
+
+from products.alerts.backend.models.alert import AlertConfiguration
+from products.alerts.backend.platform_cutover import CutoverSkip, RollbackSkip, cut_over_team, roll_back_team
+from products.alerts.backend.test.insight_alerts import create_insight_alert
+from products.alerts_platform.backend.facade import testing as platform_testing
+from products.alerts_platform.backend.facade.contracts import DeliveryPath
+
+NOW = datetime(2026, 10, 14, 12, 0, tzinfo=UTC)
+_CUTOVER = "products.alerts.backend.platform_cutover"
+
+
+class TestInsightPlatformCutover(APIBaseTest):
+ def _alert(self, **overrides: Any) -> AlertConfiguration:
+ return create_insight_alert(
+ self.team,
+ **{"next_check_at": NOW + timedelta(minutes=3), "last_notified_at": NOW - timedelta(days=1), **overrides},
+ )
+
+ @parameterized.expand(
+ [
+ ("quiet", {}, "not_firing"),
+ ("errored", {"state": AlertState.ERRORED}, "errored"),
+ (
+ "snoozed_keeps_checking_muted",
+ {"state": AlertState.SNOOZED, "snoozed_until": NOW + timedelta(hours=1)},
+ "not_firing",
+ ),
+ ("disabled", {"enabled": False, "next_check_at": None}, "not_firing"),
+ ]
+ )
+ def test_an_alert_moves_with_its_state_and_schedule(
+ self, _name: str, overrides: dict[str, Any], platform_state: str
+ ) -> None:
+ alert = self._alert(**overrides)
+
+ (result,) = cut_over_team(self.team.id, now=NOW)
+
+ assert result.skipped is None
+ alert.refresh_from_db()
+ assert (alert.owner, alert.owner_changed_at) == ("platform", NOW)
+ with team_scope(self.team.id):
+ copy = platform_testing.copy_of(alert.id)
+ instance = platform_testing.alert_for(copy.id)
+ assert (copy.owner, copy.next_check_at) == (
+ "platform",
+ max(alert.next_check_at or NOW, NOW + timedelta(minutes=1)),
+ )
+ assert instance is not None
+ assert (instance.state, instance.last_notified_at) == (platform_state, alert.last_notified_at)
+
+ @parameterized.expand(
+ [
+ ("firing", {"state": AlertState.FIRING}, CutoverSkip.FIRING),
+ ("check_due", {"next_check_at": NOW}, CutoverSkip.CHECK_DUE),
+ ("never_scheduled", {"next_check_at": None}, CutoverSkip.CHECK_DUE),
+ ("detector", {"detector_config": {"type": "zscore"}}, CutoverSkip.NOT_ON_PLATFORM),
+ (
+ "real_time",
+ {"calculation_interval": AlertCalculationInterval.REAL_TIME.value},
+ CutoverSkip.NOT_ON_PLATFORM,
+ ),
+ ("investigated", {"investigation_agent_enabled": True}, CutoverSkip.INVESTIGATION),
+ ]
+ )
+ def test_an_alert_that_cannot_move_now_stays_with_insight(
+ self, _name: str, overrides: dict[str, Any], reason: CutoverSkip
+ ) -> None:
+ alert = self._alert(**overrides)
+
+ (result,) = cut_over_team(self.team.id, now=NOW)
+
+ assert result.skipped == reason
+ alert.refresh_from_db()
+ assert alert.owner == "source"
+
+ @parameterized.expand([("events_emitted", True, None), ("nothing_emits", False, CutoverSkip.NO_HOG_FUNCTION_PATH)])
+ def test_an_alert_on_the_hog_function_path_moves_only_while_its_events_are_emitted(
+ self, _name: str, emitter: bool, reason: CutoverSkip | None
+ ) -> None:
+ alert = self._alert()
+
+ with (
+ patch(f"{_CUTOVER}.choose_delivery_path", return_value=DeliveryPath.HOG_FUNCTION),
+ patch(f"{_CUTOVER}.has_legacy_event_emitter", return_value=emitter),
+ ):
+ (result,) = cut_over_team(self.team.id, now=NOW)
+
+ assert result.skipped == reason
+ alert.refresh_from_db()
+ assert alert.owner == ("source" if reason else "platform")
+
+ def test_a_second_run_leaves_a_moved_alert_alone(self) -> None:
+ self._alert()
+ cut_over_team(self.team.id, now=NOW)
+
+ (result,) = cut_over_team(self.team.id, now=NOW + timedelta(minutes=1))
+
+ assert result.skipped == CutoverSkip.ALREADY_MOVED
+
+ def _moved(self, **overrides: Any) -> AlertConfiguration:
+ alert = self._alert(**overrides)
+ cut_over_team(self.team.id, now=NOW)
+ alert.refresh_from_db()
+ return alert
+
+ @parameterized.expand(
+ [
+ ("quiet", {}, None, False, AlertState.NOT_FIRING),
+ ("errored", {"state": AlertState.ERRORED}, None, False, AlertState.ERRORED),
+ # A snooze lives on the insight alert, so one set after the move survives the handback.
+ ("snoozed_since", {}, NOW + timedelta(days=30), False, AlertState.SNOOZED),
+ # The insight stack must not resume an alert the platform stopped.
+ ("switched_off_by_the_platform", {"state": AlertState.ERRORED}, None, True, AlertState.ERRORED),
+ ]
+ )
+ def test_rolling_back_after_the_shadow_carries_the_platform_state_back(
+ self,
+ _name: str,
+ overrides: dict[str, Any],
+ snoozed_until: datetime | None,
+ switched_off: bool,
+ state: AlertState,
+ ) -> None:
+ alert = self._moved(**overrides)
+ if snoozed_until is not None:
+ AlertConfiguration.objects.filter(id=alert.id).update(snoozed_until=snoozed_until)
+ if switched_off:
+ with team_scope(self.team.id):
+ platform_testing.set_enabled(platform_testing.copy_of(alert.id).id, False)
+ later = NOW + timedelta(days=8)
+ with team_scope(self.team.id):
+ platform_testing.set_due_at(platform_testing.copy_of(alert.id).id, later + timedelta(minutes=2))
+
+ (result,) = roll_back_team(self.team.id, now=later)
+
+ assert result.skipped is None
+ alert.refresh_from_db()
+ assert (alert.owner, alert.owner_changed_at, alert.state, alert.next_check_at, alert.enabled) == (
+ "source",
+ later,
+ state,
+ later + timedelta(minutes=2),
+ not switched_off,
+ )
+ with team_scope(self.team.id):
+ assert platform_testing.copy_of(alert.id).owner == "source"
+
+ @parameterized.expand(
+ [
+ ("never_moved", False, NOW + timedelta(days=8), RollbackSkip.NOT_MOVED),
+ ("insight_check_due_during_shadow", True, NOW + timedelta(minutes=10), RollbackSkip.INSIGHT_CHECK_DUE),
+ ]
+ )
+ def test_an_alert_that_cannot_move_back_now_stays(
+ self, _name: str, moved: bool, at: datetime, reason: RollbackSkip
+ ) -> None:
+ alert = self._moved() if moved else self._alert()
+
+ (result,) = roll_back_team(self.team.id, now=at)
+
+ assert result.skipped == reason
+ alert.refresh_from_db()
+ assert alert.owner == ("platform" if moved else "source")
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index 168675ca3439..11c21df604e2 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -14,6 +14,7 @@ from parameterized import parameterized
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import get_query_tags
+from posthog.constants import AvailableFeature
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded
from posthog.models.scoping import team_scope
from posthog.redis import get_client
@@ -67,6 +68,11 @@ class TestPlatformInsightEvaluation(APIBaseTest):
super().setUp()
get_client().delete(INFLIGHT_KEY)
self.addCleanup(get_client().delete, INFLIGHT_KEY)
+ self.organization.available_product_features = [
+ {"key": AvailableFeature.HIGH_FREQUENCY_ALERTS, "name": "High-frequency alerts"},
+ {"key": AvailableFeature.REAL_TIME_ALERTS, "name": "Real-time alerts"},
+ ]
+ self.organization.save()
def _alert(self, **overrides: Any) -> AlertConfiguration:
return create_insight_alert(self.team, **{"next_check_at": CUTOFF - timedelta(minutes=1), **overrides})
@@ -370,6 +376,18 @@ class TestPlatformInsightEvaluation(APIBaseTest):
False,
)
+ def test_a_cadence_the_plan_no_longer_includes_errors_and_is_disabled(self) -> None:
+ self.organization.available_product_features = []
+ self.organization.save()
+
+ outcome, query = self._evaluate(
+ self._copy(self._alert(calculation_interval=AlertCalculationInterval.EVERY_15_MINUTES.value)),
+ result=AlertEvaluationResult(value=1.0, breaches=[]),
+ )
+
+ assert outcome is not None and outcome.failure is not None
+ assert (outcome.failure.kind, outcome.disable, query.called) == (AlertEventKind.ERRORED, True, False)
+
@parameterized.expand(
[
("bad_query_shape", AlertExtractionError("bad query shape")),
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.py
index 3e388ae4cf8e..f8dd1581a83f 100644
--- a/products/alerts_platform/backend/facade/api.py
+++ b/products/alerts_platform/backend/facade/api.py
@@ -87,11 +87,14 @@ def take_ownership(handover: PlatformOwnershipHandover) -> bool:
def give_back_ownership(
- *, team_id: int, legacy_configuration_id: UUID, at: datetime
+ *, team_id: int, legacy_configuration_id: UUID, at: datetime, source_runs_calendar_schedules: bool = False
) -> PlatformHandback | HandbackRefusal:
"""Hands a copied alert back to its source, or says why the platform keeps it."""
return platform_lifecycle.give_back_ownership(
- team_id=team_id, legacy_configuration_id=legacy_configuration_id, at=at
+ team_id=team_id,
+ legacy_configuration_id=legacy_configuration_id,
+ at=at,
+ source_runs_calendar_schedules=source_runs_calendar_schedules,
)
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index ed6ba9b0457a..7c2eefb4ac57 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -385,7 +385,8 @@ class HandbackRefusal(StrEnum):
FIRING = "firing"
# A platform check may be running and could record or notify after the handback.
CHECK_DUE = "check_due"
- # A handback carries one alert on an interval cadence. A calendar schedule or grouped alerts stay.
+ # A handback carries one alert. Grouped alerts stay, and so does a calendar schedule its source
+ # does not run.
NOT_PORTABLE = "not_portable"
@@ -397,6 +398,8 @@ class PlatformHandback:
last_notified_at: datetime | None
consecutive_failures: int
next_check_at: datetime | None
+ # False when the platform switched the copy off, such as for a configuration it cannot run.
+ enabled: bool = True
@frozen
diff --git a/products/alerts_platform/backend/facade/delivery.py b/products/alerts_platform/backend/facade/delivery.py
index 291016d7bd7a..343698586c9a 100644
--- a/products/alerts_platform/backend/facade/delivery.py
+++ b/products/alerts_platform/backend/facade/delivery.py
@@ -60,6 +60,12 @@ def register_legacy_event_emitter(source: SourceKind, emitter: LegacyEventEmitte
delivery_paths.register_emitter(source, emitter)
+def has_legacy_event_emitter(source: SourceKind) -> bool:
+ """Whether the source emits its own events for the HogFunction path. Without an emitter, an alert
+ on that path would deliver natively, so its customized destinations would lose what they add."""
+ return delivery_paths.emitter_for(source) is not None
+
+
def native_sends_legacy_webhook_body(source: SourceKind, team: Team) -> bool | None:
"""Whether a stock webhook receives the body its HogFunction sent when delivered natively. None when unknown."""
return legacy_webhook.native_sends_legacy_body(source, team)
diff --git a/products/alerts_platform/backend/facade/testing.py b/products/alerts_platform/backend/facade/testing.py
index 3b4852338a38..92fa67cec176 100644
--- a/products/alerts_platform/backend/facade/testing.py
+++ b/products/alerts_platform/backend/facade/testing.py
@@ -105,6 +105,15 @@ def set_check_status(configuration_id: UUID, check_status: str) -> None:
PlatformAlertConfiguration.objects.filter(id=configuration_id).update(check_status=check_status)
+def set_enabled(configuration_id: UUID, enabled: bool) -> None:
+ """Switch a configuration on or off, as a broken check the platform owns leaves it.
+
+ A test door with no production caller. The caller holds the team scope.
+ """
+ # nosemgrep: idor-lookup-without-team, platform-alert-state-direct-mutation — test door, see the docstring
+ PlatformAlertConfiguration.objects.filter(id=configuration_id).update(enabled=enabled)
+
+
def record_check(
configuration_id: UUID,
*,
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index 8aa053f267f0..31ea542cc139 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -938,10 +938,13 @@ def take_ownership(handover: PlatformOwnershipHandover) -> bool:
def give_back_ownership(
- *, team_id: int, legacy_configuration_id: UUID, at: datetime
+ *, team_id: int, legacy_configuration_id: UUID, at: datetime, source_runs_calendar_schedules: bool = False
) -> PlatformHandback | HandbackRefusal:
"""Hands a copied alert back to its source, or says why the platform keeps it.
+ A source that runs calendar schedules itself takes back an alert on one. Any other source gets
+ only interval cadences back.
+
The caller holds the source's row lock and writes the source's owner and state in the same
transaction. Raises `PlatformAlertConfiguration.DoesNotExist` when the alert was never copied,
and `ValueError` when the platform does not own it, because both owners change together.
@@ -952,9 +955,10 @@ def give_back_ownership(
if configuration.owner != PlatformAlertOwner.PLATFORM:
raise ValueError("the platform does not own this alert, so its owner disagrees with its source's")
alerts = list(PlatformAlert.objects.for_team(team_id).filter(configuration=configuration))
+ calendar = configuration.recurrence_unit is not None and not source_runs_calendar_schedules
# A grouped copy with no group open yet still splits its results, which its source cannot do.
if (
- configuration.recurrence_unit is not None
+ calendar
or Grouping.from_stored(configuration.grouping).mode != GroupingMode.SINGLE
or any(alert.grouping_key for alert in alerts)
):
@@ -973,6 +977,7 @@ def give_back_ownership(
last_notified_at=alert.last_notified_at if alert is not None else None,
consecutive_failures=configuration.consecutive_failures,
next_check_at=configuration.next_check_at,
+ enabled=configuration.enabled,
)
configuration.owner = PlatformAlertOwner.SOURCE
configuration.owner_changed_at = at