Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): deliver to AlertDestination rows for platform-owned destinations (#125203)

Jimmy Zhu4ddf64b819f0
diff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.py
index 5b03f967d083..2c92eb6cfe9a 100644
--- a/products/alerts_platform/backend/delivery/evaluation.py
+++ b/products/alerts_platform/backend/delivery/evaluation.py
@@ -10,6 +10,7 @@ from dataclasses import replace
from datetime import UTC, datetime
from functools import cache
from typing import Any, Final
+from uuid import UUID
import structlog
@@ -23,7 +24,7 @@ from products.alerts_platform.backend.delivery.discord import DiscordTransport
from products.alerts_platform.backend.delivery.dispatch import deliver
from products.alerts_platform.backend.delivery.email import EmailTransport
from products.alerts_platform.backend.delivery.in_app import InAppTransport
-from products.alerts_platform.backend.delivery.message import AlertMessage
+from products.alerts_platform.backend.delivery.message import INCIDENT_KINDS, AlertMessage
from products.alerts_platform.backend.delivery.pagerduty import PagerDutyTransport, source_incident_key
from products.alerts_platform.backend.delivery.slack import SlackTransport
from products.alerts_platform.backend.delivery.team_flags import team_flag_enabled
@@ -43,9 +44,16 @@ from products.alerts_platform.backend.facade.contracts import (
IncidentAction,
LegacyEventEmission,
MessageLink,
+ StoredAlertDestination,
)
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
from products.alerts_platform.backend.logic import delivery_holds
+from products.alerts_platform.backend.logic.alert_destinations import (
+ INCIDENT_DESTINATION_TYPES,
+ enabled_alert_destinations,
+)
from products.alerts_platform.backend.logic.platform_alert_events import announced_kinds, announcement
+from products.alerts_platform.backend.models import PlatformAlertConfiguration
from products.alerts_platform.backend.temporal.metrics import DeliveryOutcomeLabel, increment_deliveries, safe_record
logger = structlog.get_logger(__name__)
@@ -181,12 +189,14 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
)
if announced is None:
return DeliveryOutcome(live=True, sent=0, skipped_without_transport=0)
- native_announced, emissions = _split_by_path(request, announced)
+ platform_destinations = _platform_destinations(request)
+ native_announced, emissions = _split_by_path(request, announced, platform_destinations)
emitter = delivery_paths.emitter_for(request.source) if emissions else None
notice_keys: dict[str, str] = {}
if request.parts > 1 and request.overflowed:
notice_keys = _overflow_notice_keys(
request,
+ platform_destinations,
announced_kinds(
request.team_id,
request.configuration_id,
@@ -214,7 +224,7 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
noticed: set[str] = set()
native_by_event = _by_subscription(request, native_announced)
for event_id, all_transitions in _by_subscription(request, announced).items():
- for subscribed_destination in _destinations(request, event_id):
+ for subscribed_destination in _destinations(request, event_id, platform_destinations):
transitions = (
native_by_event.get(event_id, ()) if subscribed_destination.receives_emitted_events else all_transitions
)
@@ -370,7 +380,9 @@ def _count_emissions(request: AlertDeliveryRequest, outcome: DeliveryOutcomeLabe
def _split_by_path(
- request: AlertDeliveryRequest, announced: EvaluationAnnouncement
+ request: AlertDeliveryRequest,
+ announced: EvaluationAnnouncement,
+ platform_destinations: "_PlatformDestinations | None",
) -> tuple[EvaluationAnnouncement, tuple[LegacyEventEmission, ...]]:
"""The transitions HogFunction destinations receive natively, and the ones the source emits for them.
@@ -380,7 +392,12 @@ def _split_by_path(
A transition inside a firing takes the path its firing took first, so a firing and its resolve
never split across paths. Any other transition takes the path its destinations allow now.
"""
- if not request.owned_by_platform or delivery_paths.emitter_for(request.source) is None:
+ # Destinations in `AlertDestination` have no HogFunction to emit for.
+ if (
+ not request.owned_by_platform
+ or platform_destinations is not None
+ or delivery_paths.emitter_for(request.source) is None
+ ):
return announced, ()
# Read once per delivery, because the chooser reads every destination of the alert.
@@ -422,7 +439,11 @@ def _split_by_path(
return replace(announced, transitions=tuple(native)), tuple(emissions)
-def _overflow_notice_keys(request: AlertDeliveryRequest, kinds: dict[str, AlertEventKind]) -> dict[str, str]:
+def _overflow_notice_keys(
+ request: AlertDeliveryRequest,
+ platform_destinations: "_PlatformDestinations | None",
+ kinds: dict[str, AlertEventKind],
+) -> dict[str, str]:
"""For each destination, the grouping key whose part tells it about the overflow.
Read from the kinds of the whole announcement, so every part agrees, and a destination whose
@@ -437,7 +458,7 @@ def _overflow_notice_keys(request: AlertDeliveryRequest, kinds: dict[str, AlertE
first_by_event_id[event_id] = min(first_by_event_id.get(event_id, grouping_key), grouping_key)
keys: dict[str, str] = {}
for event_id, first in first_by_event_id.items():
- for subscribed_destination in _destinations(request, event_id):
+ for subscribed_destination in _destinations(request, event_id, platform_destinations):
destination = json.dumps(subscribed_destination.data, sort_keys=True)
keys[destination] = min(keys.get(destination, first), first)
return keys
@@ -489,12 +510,57 @@ class _SubscribedDestination:
receives_emitted_events: bool
-def _destinations(request: AlertDeliveryRequest, event_id: str) -> list[_SubscribedDestination]:
+@frozen
+class _PlatformDestinations:
+ """An alert's enabled `AlertDestination` rows, with the kind each of its event ids announces."""
+
+ destinations: tuple[StoredAlertDestination, ...]
+ kind_by_event_id: dict[str, AlertEventKind]
+ incident_event_ids: frozenset[str]
+
+
+def _platform_destinations(request: AlertDeliveryRequest) -> _PlatformDestinations | None:
+ """The alert's destinations when the platform owns them, or None when its source's HogFunctions
+ hold them. Read once per delivery, and after the check, so a destination removed in between is
+ not sent to. Only an alert the platform owns can have its destinations on the platform."""
+ if not request.owned_by_platform:
+ return None
+ owner = (
+ PlatformAlertConfiguration.objects.for_team(request.team_id)
+ .filter(id=request.configuration_id)
+ .values_list("destinations_owner", flat=True)
+ .first()
+ )
+ if owner != PlatformAlertOwner.PLATFORM:
+ return None
+ incident_kinds = {
+ event_id: INCIDENT_KINDS[IncidentAction(action)]
+ for action, event_id in request.event_ids_by_incident_action.items()
+ }
+ return _PlatformDestinations(
+ destinations=enabled_alert_destinations(
+ team_id=request.team_id, configuration_id=UUID(request.configuration_id)
+ ),
+ kind_by_event_id={
+ **{event_id: AlertEventKind(kind) for kind, event_id in request.event_ids_by_kind.items()},
+ **incident_kinds,
+ },
+ incident_event_ids=frozenset(incident_kinds),
+ )
+
+
+def _destinations(
+ request: AlertDeliveryRequest,
+ event_id: str,
+ platform_destinations: "_PlatformDestinations | None",
+) -> list[_SubscribedDestination]:
"""Who subscribed to this event, resolved now rather than pinned when the check ran.
A destination removed between the check and the send is not sent to, which a pinned set
would get wrong.
"""
+ if platform_destinations is not None:
+ return _platform_subscribers(event_id, platform_destinations)
groups = list_alert_destination_groups(
team_id=request.team_id,
alert_id=request.destination_alert_id,
@@ -509,3 +575,22 @@ def _destinations(request: AlertDeliveryRequest, event_id: str) -> list[_Subscri
for group in groups
if group.fully_enabled
]
+
+
+def _platform_subscribers(event_id: str, platform: _PlatformDestinations) -> list[_SubscribedDestination]:
+ """The `AlertDestination` rows that hear this event.
+
+ A message event reaches every destination but an incident manager that asked for its kind. An
+ incident event reaches every incident manager, whatever kinds it lists: one that heard a
+ trigger and not its resolve would keep the incident open.
+ """
+ kind = platform.kind_by_event_id.get(event_id)
+ if kind is None:
+ return []
+ incident = event_id in platform.incident_event_ids
+ return [
+ _SubscribedDestination(id=str(destination.id), data=destination.data, receives_emitted_events=False)
+ for destination in platform.destinations
+ if (destination.data["type"] in INCIDENT_DESTINATION_TYPES) == incident
+ and (incident or kind in destination.event_kinds)
+ ]
diff --git a/products/alerts_platform/backend/delivery/message.py b/products/alerts_platform/backend/delivery/message.py
index f0172f7aae40..1ae83291a6da 100644
--- a/products/alerts_platform/backend/delivery/message.py
+++ b/products/alerts_platform/backend/delivery/message.py
@@ -51,7 +51,7 @@ _SOURCE_LABELS: Final[dict[SourceKind, str]] = {
}
# What a held row says, when cooldown or mute kept its announcement and only its incident moved.
-_INCIDENT_KINDS: Final[dict[IncidentAction, AlertEventKind]] = {
+INCIDENT_KINDS: Final[dict[IncidentAction, AlertEventKind]] = {
IncidentAction.TRIGGER: AlertEventKind.FIRING,
IncidentAction.RESOLVE: AlertEventKind.RESOLVED,
}
@@ -178,7 +178,7 @@ def build_message(
"""
kind = transition.kind
if kind == AlertEventKind.CHECK and incident_action is not None:
- kind = _INCIDENT_KINDS[incident_action]
+ kind = INCIDENT_KINDS[incident_action]
headline = _HEADLINES.get(kind)
if headline is None:
# `announcement` returns a check row only for a group whose incident moved, so a check
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.py
index 8d9e6193d95b..85774ea367fa 100644
--- a/products/alerts_platform/backend/facade/api.py
+++ b/products/alerts_platform/backend/facade/api.py
@@ -359,6 +359,11 @@ def list_alert_destinations(*, team_id: int, configuration_id: UUID) -> tuple[St
return alert_destinations.list_alert_destinations(team_id=team_id, configuration_id=configuration_id)
+def has_incident_destination(*, team_id: int, configuration_id: UUID) -> bool:
+ """Whether an alert whose destinations the platform owns has an enabled incident manager."""
+ return alert_destinations.has_incident_destination(team_id=team_id, configuration_id=configuration_id)
+
+
def delete_alert_destinations(*, team_id: int, configuration_id: UUID, destination_ids: Collection[UUID]) -> int:
"""Deletes the named destinations of one alert. An id of another alert is left alone."""
return alert_destinations.delete_alert_destinations(
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index ce0b92c35e4d..86affd107826 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -1082,7 +1082,13 @@ class NativeAlertLifecycle:
"""
check_interval_minutes: int
- remove_destinations: NativeAlertRemoveDestinations
+ # Removes the source's HogFunction destinations of a deleted alert. None for a kind whose
+ # destinations are `AlertDestination` rows from the start, which go with the configuration.
+ remove_destinations: NativeAlertRemoveDestinations | None = None
+
+ @property
+ def platform_destinations(self) -> bool:
+ return self.remove_destinations is None
@frozen
diff --git a/products/alerts_platform/backend/logic/alert_destinations.py b/products/alerts_platform/backend/logic/alert_destinations.py
index a4af08c7d54d..2c80e53cbe40 100644
--- a/products/alerts_platform/backend/logic/alert_destinations.py
+++ b/products/alerts_platform/backend/logic/alert_destinations.py
@@ -22,8 +22,12 @@ from products.alerts_platform.backend.facade.contracts import (
NewAlertDestination,
StoredAlertDestination,
)
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
from products.alerts_platform.backend.models import AlertDestination, PlatformAlertConfiguration
+# The destination types that open and close incidents rather than post messages.
+INCIDENT_DESTINATION_TYPES: Final = frozenset({DestinationType.PAGERDUTY})
+
_REQUIRED_FIELDS: Final[dict[DestinationType, tuple[str, ...]]] = {
DestinationType.SLACK: ("slack_workspace_id", "slack_channel_id"),
DestinationType.WEBHOOK: ("webhook_url",),
@@ -72,6 +76,9 @@ def create_alert_destinations(
configuration = (
PlatformAlertConfiguration.objects.for_team(team_id).select_for_update().get(id=configuration_id)
)
+ # Delivery reads these rows only for such an alert. On any other, they would never send.
+ if configuration.destinations_owner != PlatformAlertOwner.PLATFORM:
+ raise AlertDestinationValidationError("This alert's destinations are managed by its product.")
_refuse_duplicates(team_id, configuration_id, destinations)
rows = [
AlertDestination.objects.for_team(team_id).create(
@@ -110,6 +117,25 @@ def delete_alert_destinations(*, team_id: int, configuration_id: UUID, destinati
return deleted
+def enabled_alert_destinations(*, team_id: int, configuration_id: UUID) -> tuple[StoredAlertDestination, ...]:
+ """The destinations delivery sends to. A disabled one is left out before its address is decrypted."""
+ return tuple(
+ _stored(row)
+ for row in AlertDestination.objects.for_team(team_id)
+ .filter(configuration_id=configuration_id, enabled=True)
+ .order_by("created_at", "id")
+ )
+
+
+def has_incident_destination(*, team_id: int, configuration_id: UUID) -> bool:
+ """Whether the alert has an enabled incident manager, so that a firing that opens or closes needs a delivery."""
+ return (
+ AlertDestination.objects.for_team(team_id)
+ .filter(configuration_id=configuration_id, type__in=INCIDENT_DESTINATION_TYPES, enabled=True)
+ .exists()
+ )
+
+
def _refuse_duplicates(team_id: int, configuration_id: UUID, destinations: Sequence[NewAlertDestination]) -> None:
"""A second destination with the same address would send every message twice."""
seen = [
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index 366d88675554..be323f7e025a 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -828,6 +828,7 @@ def create_native_configuration(
source_kind: SourceKind,
edit: PlatformAlertConfigurationEdit,
check_interval_minutes: int,
+ destinations_owner: PlatformAlertOwner,
now: datetime,
) -> PlatformAlertConfiguration:
"""Creates an alert the platform owns from the start, for a kind whose source keeps no row for it."""
@@ -837,6 +838,7 @@ def create_native_configuration(
check_interval_minutes=check_interval_minutes,
next_check_at=now,
owner=PlatformAlertOwner.PLATFORM,
+ destinations_owner=destinations_owner,
)
_assign_edit(configuration, edit)
# The model's own save logs the creation.
diff --git a/products/alerts_platform/backend/logic/platform_writes.py b/products/alerts_platform/backend/logic/platform_writes.py
index f5c1d410cee3..637b3d91ebce 100644
--- a/products/alerts_platform/backend/logic/platform_writes.py
+++ b/products/alerts_platform/backend/logic/platform_writes.py
@@ -178,6 +178,9 @@ def create_configuration(
source_kind=source_kind,
edit=validated,
check_interval_minutes=spec.native.check_interval_minutes,
+ destinations_owner=PlatformAlertOwner.PLATFORM
+ if spec.native.platform_destinations
+ else PlatformAlertOwner.SOURCE,
now=django_timezone.now(),
)
else:
@@ -356,10 +359,12 @@ def _delete_native(team_id: int, configuration_id: UUID, native: NativeAlertLife
configurations = PlatformAlertConfiguration.objects.for_team(team_id, canonical=True).filter(id=configuration_id)
# Before the destinations go, so an incident manager still gets the resolve for an open firing.
_close_firings_now(lambda: configurations.select_for_update().first())
- # In the delete's transaction, so a delete that fails keeps the alert's destinations.
+ # In the delete's transaction, so a delete that fails keeps the alert's destinations. An
+ # `AlertDestination` row goes with its configuration, so there is nothing to remove first.
+ remove = native.remove_destinations
_delete_each(
configurations,
- before_delete=lambda: native.remove_destinations(team_id=team_id, configuration_id=configuration_id),
+ before_delete=None if remove is None else lambda: remove(team_id=team_id, configuration_id=configuration_id),
)
diff --git a/products/alerts_platform/backend/tests/test_alert_destinations.py b/products/alerts_platform/backend/tests/test_alert_destinations.py
index 1e100886ede6..24b09f16960f 100644
--- a/products/alerts_platform/backend/tests/test_alert_destinations.py
+++ b/products/alerts_platform/backend/tests/test_alert_destinations.py
@@ -21,6 +21,7 @@ from products.alerts_platform.backend.facade.contracts import (
NewAlertDestination,
SourceKind,
)
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
from products.alerts_platform.backend.models import PlatformAlertConfiguration
WEBHOOK_URL = "https://example.com/hooks/secret-path"
@@ -33,9 +34,13 @@ class TestAlertDestinations(BaseTest):
super().setUp()
self.configuration = self._configuration()
- def _configuration(self) -> PlatformAlertConfiguration:
+ def _configuration(self, destinations_owner: str = PlatformAlertOwner.PLATFORM) -> PlatformAlertConfiguration:
return PlatformAlertConfiguration.objects.for_team(self.team.id).create(
- team=self.team, name="API errors", source_kind=SourceKind.LOGS, check_interval_minutes=5
+ team=self.team,
+ name="API errors",
+ source_kind=SourceKind.LOGS,
+ check_interval_minutes=5,
+ destinations_owner=destinations_owner,
)
def _create(self, *destinations: NewAlertDestination, configuration: Any = None) -> Any:
@@ -100,6 +105,14 @@ class TestAlertDestinations(BaseTest):
assert error.value.field == field
assert list_alert_destinations(team_id=self.team.id, configuration_id=self.configuration.id) == ()
+ def test_an_alert_whose_product_holds_its_destinations_takes_none_here(self) -> None:
+ configuration = self._configuration(destinations_owner=PlatformAlertOwner.SOURCE)
+
+ with pytest.raises(AlertDestinationValidationError):
+ self._create(NewAlertDestination(data=WEBHOOK, event_kinds=FIRING), configuration=configuration)
+
+ assert list_alert_destinations(team_id=self.team.id, configuration_id=configuration.id) == ()
+
@parameterized.expand(
[
("same_webhook", WEBHOOK, dict(WEBHOOK)),
diff --git a/products/alerts_platform/backend/tests/test_delivery_evaluation.py b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
index 5e17002b778d..047b5fad3258 100644
--- a/products/alerts_platform/backend/tests/test_delivery_evaluation.py
+++ b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
@@ -19,6 +19,7 @@ from products.alerts_platform.backend.delivery.legacy_webhook import LEGACY_WEBH
from products.alerts_platform.backend.delivery.message import AlertMessage
from products.alerts_platform.backend.delivery.thread_store import ThreadBusy
from products.alerts_platform.backend.delivery.transport import DeliveryError, MessageHandle
+from products.alerts_platform.backend.facade.api import create_alert_destinations
from products.alerts_platform.backend.facade.contracts import (
AlertDeliveryRequest,
AlertDestinationData,
@@ -31,9 +32,10 @@ from products.alerts_platform.backend.facade.contracts import (
IncidentAction,
LegacyEventEmission,
MessageLink,
+ NewAlertDestination,
SourceKind,
)
-from products.alerts_platform.backend.facade.enums import HoldRelease
+from products.alerts_platform.backend.facade.enums import HoldRelease, PlatformAlertOwner
from products.alerts_platform.backend.models import PlatformAlertConfiguration, PlatformAlertDeliveryHold
from products.alerts_platform.backend.tests.recorded_metrics import recorded_metrics
@@ -371,6 +373,51 @@ class TestDeliverEvaluation(APIBaseTest):
assert [m.headline for channel, m in RecordingTransport.sends if channel == "C-1"] == chat_headlines
assert [m.incident_action for channel, m in RecordingTransport.sends if channel == "R-1"] == [action]
+ @parameterized.expand(
+ [
+ ("firing", AlertEventKind.FIRING, IncidentAction.TRIGGER, "Log alert 'API errors' is firing"),
+ ("resolved", AlertEventKind.RESOLVED, IncidentAction.RESOLVE, "Log alert 'API errors' is resolved"),
+ ]
+ )
+ def test_an_alert_whose_destinations_the_platform_owns_sends_to_its_enabled_rows(
+ self, _name: str, kind: AlertEventKind, action: IncidentAction, headline: str
+ ) -> None:
+ configuration = PlatformAlertConfiguration.objects.for_team(self.team.id).create(
+ id=CONFIGURATION_ID,
+ team=self.team,
+ name="API errors",
+ source_kind=SourceKind.LOGS,
+ check_interval_minutes=5,
+ destinations_owner=PlatformAlertOwner.PLATFORM,
+ )
+ both = (AlertEventKind.FIRING, AlertEventKind.RESOLVED)
+ create_alert_destinations(
+ team_id=self.team.id,
+ configuration_id=configuration.id,
+ destinations=(
+ NewAlertDestination(data=SLACK, event_kinds=both),
+ NewAlertDestination(
+ data=cast(AlertDestinationData, {**SLACK, "slack_channel_id": "C-off"}),
+ event_kinds=both,
+ enabled=False,
+ ),
+ NewAlertDestination(data=PAGERDUTY, event_kinds=(AlertEventKind.FIRING,)),
+ ),
+ )
+ hog_function = [_group(cast(AlertDestinationData, {**SLACK, "slack_channel_id": "C-hog"}))]
+
+ self._run(
+ _announcement(_transition(kind)),
+ {FIRING_EVENT: hog_function, RESOLVED_EVENT: hog_function, OPENED_EVENT: hog_function},
+ incident_actions={"": action},
+ owned_by_platform=True,
+ )
+
+ assert [(channel, m.headline) for channel, m in RecordingTransport.sends if channel != "R-1"] == [
+ ("C-1", headline)
+ ]
+ assert [m.incident_action for channel, m in RecordingTransport.sends if channel == "R-1"] == [action]
+
@parameterized.expand([("owned_by_the_platform", True, "posthog-alert-legacy-1"), ("live_delivery", False, None)])
def test_only_an_alert_the_platform_owns_resolves_under_its_source_incident_key(
self, _name: str, owned_by_platform: bool, expected: str | None