:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): deliver to AlertDestination rows for platform-owned destinations (#125203)
Jimmy Zhu4ddf64b819f0
diff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.pyindex 5b03f967d083..2c92eb6cfe9a 100644--- a/products/alerts_platform/backend/delivery/evaluation.py+++ b/products/alerts_platform/backend/delivery/evaluation.py@@ -10,6 +10,7 @@ from dataclasses import replacefrom datetime import UTC, datetimefrom functools import cachefrom typing import Any, Final+from uuid import UUIDimport structlog@@ -23,7 +24,7 @@ from products.alerts_platform.backend.delivery.discord import DiscordTransportfrom products.alerts_platform.backend.delivery.dispatch import deliverfrom products.alerts_platform.backend.delivery.email import EmailTransportfrom products.alerts_platform.backend.delivery.in_app import InAppTransport-from products.alerts_platform.backend.delivery.message import AlertMessage+from products.alerts_platform.backend.delivery.message import INCIDENT_KINDS, AlertMessagefrom products.alerts_platform.backend.delivery.pagerduty import PagerDutyTransport, source_incident_keyfrom products.alerts_platform.backend.delivery.slack import SlackTransportfrom products.alerts_platform.backend.delivery.team_flags import team_flag_enabled@@ -43,9 +44,16 @@ from products.alerts_platform.backend.facade.contracts import (IncidentAction,LegacyEventEmission,MessageLink,+ StoredAlertDestination,)+from products.alerts_platform.backend.facade.enums import PlatformAlertOwnerfrom products.alerts_platform.backend.logic import delivery_holds+from products.alerts_platform.backend.logic.alert_destinations import (+ INCIDENT_DESTINATION_TYPES,+ enabled_alert_destinations,+)from products.alerts_platform.backend.logic.platform_alert_events import announced_kinds, announcement+from products.alerts_platform.backend.models import PlatformAlertConfigurationfrom products.alerts_platform.backend.temporal.metrics import DeliveryOutcomeLabel, increment_deliveries, safe_recordlogger = structlog.get_logger(__name__)@@ -181,12 +189,14 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:)if announced is None:return DeliveryOutcome(live=True, sent=0, skipped_without_transport=0)- native_announced, emissions = _split_by_path(request, announced)+ platform_destinations = _platform_destinations(request)+ native_announced, emissions = _split_by_path(request, announced, platform_destinations)emitter = delivery_paths.emitter_for(request.source) if emissions else Nonenotice_keys: dict[str, str] = {}if request.parts > 1 and request.overflowed:notice_keys = _overflow_notice_keys(request,+ platform_destinations,announced_kinds(request.team_id,request.configuration_id,@@ -214,7 +224,7 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:noticed: set[str] = set()native_by_event = _by_subscription(request, native_announced)for event_id, all_transitions in _by_subscription(request, announced).items():- for subscribed_destination in _destinations(request, event_id):+ for subscribed_destination in _destinations(request, event_id, platform_destinations):transitions = (native_by_event.get(event_id, ()) if subscribed_destination.receives_emitted_events else all_transitions)@@ -370,7 +380,9 @@ def _count_emissions(request: AlertDeliveryRequest, outcome: DeliveryOutcomeLabedef _split_by_path(- request: AlertDeliveryRequest, announced: EvaluationAnnouncement+ request: AlertDeliveryRequest,+ announced: EvaluationAnnouncement,+ platform_destinations: "_PlatformDestinations | None",) -> tuple[EvaluationAnnouncement, tuple[LegacyEventEmission, ...]]:"""The transitions HogFunction destinations receive natively, and the ones the source emits for them.@@ -380,7 +392,12 @@ def _split_by_path(A transition inside a firing takes the path its firing took first, so a firing and its resolvenever split across paths. Any other transition takes the path its destinations allow now."""- if not request.owned_by_platform or delivery_paths.emitter_for(request.source) is None:+ # Destinations in `AlertDestination` have no HogFunction to emit for.+ if (+ not request.owned_by_platform+ or platform_destinations is not None+ or delivery_paths.emitter_for(request.source) is None+ ):return announced, ()# Read once per delivery, because the chooser reads every destination of the alert.@@ -422,7 +439,11 @@ def _split_by_path(return replace(announced, transitions=tuple(native)), tuple(emissions)-def _overflow_notice_keys(request: AlertDeliveryRequest, kinds: dict[str, AlertEventKind]) -> dict[str, str]:+def _overflow_notice_keys(+ request: AlertDeliveryRequest,+ platform_destinations: "_PlatformDestinations | None",+ kinds: dict[str, AlertEventKind],+) -> dict[str, str]:"""For each destination, the grouping key whose part tells it about the overflow.Read from the kinds of the whole announcement, so every part agrees, and a destination whose@@ -437,7 +458,7 @@ def _overflow_notice_keys(request: AlertDeliveryRequest, kinds: dict[str, AlertEfirst_by_event_id[event_id] = min(first_by_event_id.get(event_id, grouping_key), grouping_key)keys: dict[str, str] = {}for event_id, first in first_by_event_id.items():- for subscribed_destination in _destinations(request, event_id):+ for subscribed_destination in _destinations(request, event_id, platform_destinations):destination = json.dumps(subscribed_destination.data, sort_keys=True)keys[destination] = min(keys.get(destination, first), first)return keys@@ -489,12 +510,57 @@ class _SubscribedDestination:receives_emitted_events: bool-def _destinations(request: AlertDeliveryRequest, event_id: str) -> list[_SubscribedDestination]:+@frozen+class _PlatformDestinations:+ """An alert's enabled `AlertDestination` rows, with the kind each of its event ids announces."""++ destinations: tuple[StoredAlertDestination, ...]+ kind_by_event_id: dict[str, AlertEventKind]+ incident_event_ids: frozenset[str]+++def _platform_destinations(request: AlertDeliveryRequest) -> _PlatformDestinations | None:+ """The alert's destinations when the platform owns them, or None when its source's HogFunctions+ hold them. Read once per delivery, and after the check, so a destination removed in between is+ not sent to. Only an alert the platform owns can have its destinations on the platform."""+ if not request.owned_by_platform:+ return None+ owner = (+ PlatformAlertConfiguration.objects.for_team(request.team_id)+ .filter(id=request.configuration_id)+ .values_list("destinations_owner", flat=True)+ .first()+ )+ if owner != PlatformAlertOwner.PLATFORM:+ return None+ incident_kinds = {+ event_id: INCIDENT_KINDS[IncidentAction(action)]+ for action, event_id in request.event_ids_by_incident_action.items()+ }+ return _PlatformDestinations(+ destinations=enabled_alert_destinations(+ team_id=request.team_id, configuration_id=UUID(request.configuration_id)+ ),+ kind_by_event_id={+ **{event_id: AlertEventKind(kind) for kind, event_id in request.event_ids_by_kind.items()},+ **incident_kinds,+ },+ incident_event_ids=frozenset(incident_kinds),+ )+++def _destinations(+ request: AlertDeliveryRequest,+ event_id: str,+ platform_destinations: "_PlatformDestinations | None",+) -> list[_SubscribedDestination]:"""Who subscribed to this event, resolved now rather than pinned when the check ran.A destination removed between the check and the send is not sent to, which a pinned setwould get wrong."""+ if platform_destinations is not None:+ return _platform_subscribers(event_id, platform_destinations)groups = list_alert_destination_groups(team_id=request.team_id,alert_id=request.destination_alert_id,@@ -509,3 +575,22 @@ def _destinations(request: AlertDeliveryRequest, event_id: str) -> list[_Subscrifor group in groupsif group.fully_enabled]+++def _platform_subscribers(event_id: str, platform: _PlatformDestinations) -> list[_SubscribedDestination]:+ """The `AlertDestination` rows that hear this event.++ A message event reaches every destination but an incident manager that asked for its kind. An+ incident event reaches every incident manager, whatever kinds it lists: one that heard a+ trigger and not its resolve would keep the incident open.+ """+ kind = platform.kind_by_event_id.get(event_id)+ if kind is None:+ return []+ incident = event_id in platform.incident_event_ids+ return [+ _SubscribedDestination(id=str(destination.id), data=destination.data, receives_emitted_events=False)+ for destination in platform.destinations+ if (destination.data["type"] in INCIDENT_DESTINATION_TYPES) == incident+ and (incident or kind in destination.event_kinds)+ ]diff --git a/products/alerts_platform/backend/delivery/message.py b/products/alerts_platform/backend/delivery/message.pyindex f0172f7aae40..1ae83291a6da 100644--- a/products/alerts_platform/backend/delivery/message.py+++ b/products/alerts_platform/backend/delivery/message.py@@ -51,7 +51,7 @@ _SOURCE_LABELS: Final[dict[SourceKind, str]] = {}# What a held row says, when cooldown or mute kept its announcement and only its incident moved.-_INCIDENT_KINDS: Final[dict[IncidentAction, AlertEventKind]] = {+INCIDENT_KINDS: Final[dict[IncidentAction, AlertEventKind]] = {IncidentAction.TRIGGER: AlertEventKind.FIRING,IncidentAction.RESOLVE: AlertEventKind.RESOLVED,}@@ -178,7 +178,7 @@ def build_message("""kind = transition.kindif kind == AlertEventKind.CHECK and incident_action is not None:- kind = _INCIDENT_KINDS[incident_action]+ kind = INCIDENT_KINDS[incident_action]headline = _HEADLINES.get(kind)if headline is None:# `announcement` returns a check row only for a group whose incident moved, so a checkdiff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.pyindex 8d9e6193d95b..85774ea367fa 100644--- a/products/alerts_platform/backend/facade/api.py+++ b/products/alerts_platform/backend/facade/api.py@@ -359,6 +359,11 @@ def list_alert_destinations(*, team_id: int, configuration_id: UUID) -> tuple[Streturn alert_destinations.list_alert_destinations(team_id=team_id, configuration_id=configuration_id)+def has_incident_destination(*, team_id: int, configuration_id: UUID) -> bool:+ """Whether an alert whose destinations the platform owns has an enabled incident manager."""+ return alert_destinations.has_incident_destination(team_id=team_id, configuration_id=configuration_id)++def delete_alert_destinations(*, team_id: int, configuration_id: UUID, destination_ids: Collection[UUID]) -> int:"""Deletes the named destinations of one alert. An id of another alert is left alone."""return alert_destinations.delete_alert_destinations(diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.pyindex ce0b92c35e4d..86affd107826 100644--- a/products/alerts_platform/backend/facade/contracts.py+++ b/products/alerts_platform/backend/facade/contracts.py@@ -1082,7 +1082,13 @@ class NativeAlertLifecycle:"""check_interval_minutes: int- remove_destinations: NativeAlertRemoveDestinations+ # Removes the source's HogFunction destinations of a deleted alert. None for a kind whose+ # destinations are `AlertDestination` rows from the start, which go with the configuration.+ remove_destinations: NativeAlertRemoveDestinations | None = None++ @property+ def platform_destinations(self) -> bool:+ return self.remove_destinations is None@frozendiff --git a/products/alerts_platform/backend/logic/alert_destinations.py b/products/alerts_platform/backend/logic/alert_destinations.pyindex a4af08c7d54d..2c80e53cbe40 100644--- a/products/alerts_platform/backend/logic/alert_destinations.py+++ b/products/alerts_platform/backend/logic/alert_destinations.py@@ -22,8 +22,12 @@ from products.alerts_platform.backend.facade.contracts import (NewAlertDestination,StoredAlertDestination,)+from products.alerts_platform.backend.facade.enums import PlatformAlertOwnerfrom products.alerts_platform.backend.models import AlertDestination, PlatformAlertConfiguration+# The destination types that open and close incidents rather than post messages.+INCIDENT_DESTINATION_TYPES: Final = frozenset({DestinationType.PAGERDUTY})+_REQUIRED_FIELDS: Final[dict[DestinationType, tuple[str, ...]]] = {DestinationType.SLACK: ("slack_workspace_id", "slack_channel_id"),DestinationType.WEBHOOK: ("webhook_url",),@@ -72,6 +76,9 @@ def create_alert_destinations(configuration = (PlatformAlertConfiguration.objects.for_team(team_id).select_for_update().get(id=configuration_id))+ # Delivery reads these rows only for such an alert. On any other, they would never send.+ if configuration.destinations_owner != PlatformAlertOwner.PLATFORM:+ raise AlertDestinationValidationError("This alert's destinations are managed by its product.")_refuse_duplicates(team_id, configuration_id, destinations)rows = [AlertDestination.objects.for_team(team_id).create(@@ -110,6 +117,25 @@ def delete_alert_destinations(*, team_id: int, configuration_id: UUID, destinatireturn deleted+def enabled_alert_destinations(*, team_id: int, configuration_id: UUID) -> tuple[StoredAlertDestination, ...]:+ """The destinations delivery sends to. A disabled one is left out before its address is decrypted."""+ return tuple(+ _stored(row)+ for row in AlertDestination.objects.for_team(team_id)+ .filter(configuration_id=configuration_id, enabled=True)+ .order_by("created_at", "id")+ )+++def has_incident_destination(*, team_id: int, configuration_id: UUID) -> bool:+ """Whether the alert has an enabled incident manager, so that a firing that opens or closes needs a delivery."""+ return (+ AlertDestination.objects.for_team(team_id)+ .filter(configuration_id=configuration_id, type__in=INCIDENT_DESTINATION_TYPES, enabled=True)+ .exists()+ )++def _refuse_duplicates(team_id: int, configuration_id: UUID, destinations: Sequence[NewAlertDestination]) -> None:"""A second destination with the same address would send every message twice."""seen = [diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.pyindex 366d88675554..be323f7e025a 100644--- a/products/alerts_platform/backend/logic/platform_lifecycle.py+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py@@ -828,6 +828,7 @@ def create_native_configuration(source_kind: SourceKind,edit: PlatformAlertConfigurationEdit,check_interval_minutes: int,+ destinations_owner: PlatformAlertOwner,now: datetime,) -> PlatformAlertConfiguration:"""Creates an alert the platform owns from the start, for a kind whose source keeps no row for it."""@@ -837,6 +838,7 @@ def create_native_configuration(check_interval_minutes=check_interval_minutes,next_check_at=now,owner=PlatformAlertOwner.PLATFORM,+ destinations_owner=destinations_owner,)_assign_edit(configuration, edit)# The model's own save logs the creation.diff --git a/products/alerts_platform/backend/logic/platform_writes.py b/products/alerts_platform/backend/logic/platform_writes.pyindex f5c1d410cee3..637b3d91ebce 100644--- a/products/alerts_platform/backend/logic/platform_writes.py+++ b/products/alerts_platform/backend/logic/platform_writes.py@@ -178,6 +178,9 @@ def create_configuration(source_kind=source_kind,edit=validated,check_interval_minutes=spec.native.check_interval_minutes,+ destinations_owner=PlatformAlertOwner.PLATFORM+ if spec.native.platform_destinations+ else PlatformAlertOwner.SOURCE,now=django_timezone.now(),)else:@@ -356,10 +359,12 @@ def _delete_native(team_id: int, configuration_id: UUID, native: NativeAlertLifeconfigurations = PlatformAlertConfiguration.objects.for_team(team_id, canonical=True).filter(id=configuration_id)# Before the destinations go, so an incident manager still gets the resolve for an open firing._close_firings_now(lambda: configurations.select_for_update().first())- # In the delete's transaction, so a delete that fails keeps the alert's destinations.+ # In the delete's transaction, so a delete that fails keeps the alert's destinations. An+ # `AlertDestination` row goes with its configuration, so there is nothing to remove first.+ remove = native.remove_destinations_delete_each(configurations,- before_delete=lambda: native.remove_destinations(team_id=team_id, configuration_id=configuration_id),+ before_delete=None if remove is None else lambda: remove(team_id=team_id, configuration_id=configuration_id),)diff --git a/products/alerts_platform/backend/tests/test_alert_destinations.py b/products/alerts_platform/backend/tests/test_alert_destinations.pyindex 1e100886ede6..24b09f16960f 100644--- a/products/alerts_platform/backend/tests/test_alert_destinations.py+++ b/products/alerts_platform/backend/tests/test_alert_destinations.py@@ -21,6 +21,7 @@ from products.alerts_platform.backend.facade.contracts import (NewAlertDestination,SourceKind,)+from products.alerts_platform.backend.facade.enums import PlatformAlertOwnerfrom products.alerts_platform.backend.models import PlatformAlertConfigurationWEBHOOK_URL = "https://example.com/hooks/secret-path"@@ -33,9 +34,13 @@ class TestAlertDestinations(BaseTest):super().setUp()self.configuration = self._configuration()- def _configuration(self) -> PlatformAlertConfiguration:+ def _configuration(self, destinations_owner: str = PlatformAlertOwner.PLATFORM) -> PlatformAlertConfiguration:return PlatformAlertConfiguration.objects.for_team(self.team.id).create(- team=self.team, name="API errors", source_kind=SourceKind.LOGS, check_interval_minutes=5+ team=self.team,+ name="API errors",+ source_kind=SourceKind.LOGS,+ check_interval_minutes=5,+ destinations_owner=destinations_owner,)def _create(self, *destinations: NewAlertDestination, configuration: Any = None) -> Any:@@ -100,6 +105,14 @@ class TestAlertDestinations(BaseTest):assert error.value.field == fieldassert list_alert_destinations(team_id=self.team.id, configuration_id=self.configuration.id) == ()+ def test_an_alert_whose_product_holds_its_destinations_takes_none_here(self) -> None:+ configuration = self._configuration(destinations_owner=PlatformAlertOwner.SOURCE)++ with pytest.raises(AlertDestinationValidationError):+ self._create(NewAlertDestination(data=WEBHOOK, event_kinds=FIRING), configuration=configuration)++ assert list_alert_destinations(team_id=self.team.id, configuration_id=configuration.id) == ()+@parameterized.expand([("same_webhook", WEBHOOK, dict(WEBHOOK)),diff --git a/products/alerts_platform/backend/tests/test_delivery_evaluation.py b/products/alerts_platform/backend/tests/test_delivery_evaluation.pyindex 5e17002b778d..047b5fad3258 100644--- a/products/alerts_platform/backend/tests/test_delivery_evaluation.py+++ b/products/alerts_platform/backend/tests/test_delivery_evaluation.py@@ -19,6 +19,7 @@ from products.alerts_platform.backend.delivery.legacy_webhook import LEGACY_WEBHfrom products.alerts_platform.backend.delivery.message import AlertMessagefrom products.alerts_platform.backend.delivery.thread_store import ThreadBusyfrom products.alerts_platform.backend.delivery.transport import DeliveryError, MessageHandle+from products.alerts_platform.backend.facade.api import create_alert_destinationsfrom products.alerts_platform.backend.facade.contracts import (AlertDeliveryRequest,AlertDestinationData,@@ -31,9 +32,10 @@ from products.alerts_platform.backend.facade.contracts import (IncidentAction,LegacyEventEmission,MessageLink,+ NewAlertDestination,SourceKind,)-from products.alerts_platform.backend.facade.enums import HoldRelease+from products.alerts_platform.backend.facade.enums import HoldRelease, PlatformAlertOwnerfrom products.alerts_platform.backend.models import PlatformAlertConfiguration, PlatformAlertDeliveryHoldfrom products.alerts_platform.backend.tests.recorded_metrics import recorded_metrics@@ -371,6 +373,51 @@ class TestDeliverEvaluation(APIBaseTest):assert [m.headline for channel, m in RecordingTransport.sends if channel == "C-1"] == chat_headlinesassert [m.incident_action for channel, m in RecordingTransport.sends if channel == "R-1"] == [action]+ @parameterized.expand(+ [+ ("firing", AlertEventKind.FIRING, IncidentAction.TRIGGER, "Log alert 'API errors' is firing"),+ ("resolved", AlertEventKind.RESOLVED, IncidentAction.RESOLVE, "Log alert 'API errors' is resolved"),+ ]+ )+ def test_an_alert_whose_destinations_the_platform_owns_sends_to_its_enabled_rows(+ self, _name: str, kind: AlertEventKind, action: IncidentAction, headline: str+ ) -> None:+ configuration = PlatformAlertConfiguration.objects.for_team(self.team.id).create(+ id=CONFIGURATION_ID,+ team=self.team,+ name="API errors",+ source_kind=SourceKind.LOGS,+ check_interval_minutes=5,+ destinations_owner=PlatformAlertOwner.PLATFORM,+ )+ both = (AlertEventKind.FIRING, AlertEventKind.RESOLVED)+ create_alert_destinations(+ team_id=self.team.id,+ configuration_id=configuration.id,+ destinations=(+ NewAlertDestination(data=SLACK, event_kinds=both),+ NewAlertDestination(+ data=cast(AlertDestinationData, {**SLACK, "slack_channel_id": "C-off"}),+ event_kinds=both,+ enabled=False,+ ),+ NewAlertDestination(data=PAGERDUTY, event_kinds=(AlertEventKind.FIRING,)),+ ),+ )+ hog_function = [_group(cast(AlertDestinationData, {**SLACK, "slack_channel_id": "C-hog"}))]++ self._run(+ _announcement(_transition(kind)),+ {FIRING_EVENT: hog_function, RESOLVED_EVENT: hog_function, OPENED_EVENT: hog_function},+ incident_actions={"": action},+ owned_by_platform=True,+ )++ assert [(channel, m.headline) for channel, m in RecordingTransport.sends if channel != "R-1"] == [+ ("C-1", headline)+ ]+ assert [m.incident_action for channel, m in RecordingTransport.sends if channel == "R-1"] == [action]+@parameterized.expand([("owned_by_the_platform", True, "posthog-alert-legacy-1"), ("live_delivery", False, None)])def test_only_an_alert_the_platform_owns_resolves_under_its_source_incident_key(self, _name: str, owned_by_platform: bool, expected: str | None