:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): open and resolve pagerduty incidents for insight alerts (#127898)
Jimmy Zhue9675d5ae6c1
diff --git a/docs/internal/alerts-platform-temporal.md b/docs/internal/alerts-platform-temporal.mdindex 8fcb456137f1..8837a362971b 100644--- a/docs/internal/alerts-platform-temporal.md+++ b/docs/internal/alerts-platform-temporal.md@@ -421,6 +421,9 @@ The legacy logs stack's `incident_edge` wraps the same rule.`<configuration_id>:<grouping_key>:<episode_started_at>`, so a resolve closes the incident of its ownfiring episode. It names the platform in `source`, because a team on the pilot also gets the HogFunctionpath's incident.+- An insight alert announces no resolve, but it still sends incident actions for an alert the platform owns.+ A PagerDuty destination on `$insight_alert_incident_opened` and `$insight_alert_incident_closed` opens and resolves its incident.+ One on `$insight_alert_firing` keeps its HogFunction, which only triggers.A fire a mute swallowed is still owed an announcement.`_firing_is_unannounced` in `facade/lifecycle.py` decides that, and its docstring holds the rule.diff --git a/products/alerts/backend/facade/contracts.py b/products/alerts/backend/facade/contracts.pyindex 29c355e927e3..50b8222feabf 100644--- a/products/alerts/backend/facade/contracts.py+++ b/products/alerts/backend/facade/contracts.py@@ -15,13 +15,27 @@ from uuid import UUIDfrom posthog.dataclasses import frozenfrom posthog.schema_enums import AlertState-from products.alerts_platform.backend.facade.contracts import DeliveryPath, DestinationType+from products.alerts_platform.backend.facade.contracts import DeliveryPath, DestinationType, IncidentActionfrom products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENT# The internal event an insight alert's destinations fire on. Do not take it from# `posthog.tasks.alerts.utils` instead, because that module imports this product's facade.INSIGHT_ALERT_EVENT_IDS: Final[tuple[str, ...]] = (LEGACY_INSIGHT_ALERT_EVENT,)+# Every move into and out of a firing, so each trigger gets its resolve while messages still announce none.+INSIGHT_ALERT_INCIDENT_OPENED_EVENT: Final = "$insight_alert_incident_opened"+INSIGHT_ALERT_INCIDENT_CLOSED_EVENT: Final = "$insight_alert_incident_closed"+INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION: Final[dict[str, str]] = {+ IncidentAction.TRIGGER.value: INSIGHT_ALERT_INCIDENT_OPENED_EVENT,+ IncidentAction.RESOLVE.value: INSIGHT_ALERT_INCIDENT_CLOSED_EVENT,+}+# Every event an insight alert's own destinations can listen on. The generic HogFunction API treats+# the incident events as managed and refuses to edit them, so the insight API must be able to remove them.+INSIGHT_ALERT_DESTINATION_EVENT_IDS: Final[tuple[str, ...]] = (+ *INSIGHT_ALERT_EVENT_IDS,+ *INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values(),+)+# Slack only, because `alert:write` is grantable to a sandboxed agent. A connected workspace is a# destination an admin chose, while every other transport takes a URL the caller supplies.INSIGHT_ALERT_DESTINATION_TYPES: Final[tuple[DestinationType, ...]] = (DestinationType.SLACK,)@@ -82,7 +96,7 @@ class HogFunctionPathReason(StrEnum):# body the HogFunction sends.NEW_WEBHOOK_BODY = "new_webhook_body"WEBHOOK_BODY_UNKNOWN = "webhook_body_unknown"- # Native insight delivery sends no incident actions, so a PagerDuty destination would open none.+ # Native delivery sends PagerDuty incident actions only, never the firing event.PAGERDUTY_NOT_NATIVE = "pagerduty_not_native"@@ -104,6 +118,7 @@ class DestinationEligibility:hold_backs: tuple[DestinationHoldBack, ...]# The md5 of the function's code, so the same code can be recognized across functions and teams.code_hash: str+ event_ids: frozenset[str]@frozendiff --git a/products/alerts/backend/logic/delivery_path.py b/products/alerts/backend/logic/delivery_path.pyindex 786745ffb8a0..18b43342092a 100644--- a/products/alerts/backend/logic/delivery_path.py+++ b/products/alerts/backend/logic/delivery_path.py@@ -29,6 +29,7 @@ from products.alerts.backend.logic.destination_eligibility import ()from products.alerts_platform.backend.facade.contracts import DeliveryPath, SourceKindfrom products.alerts_platform.backend.facade.delivery import native_sends_legacy_webhook_body+from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENTlogger = structlog.get_logger(__name__)@@ -74,7 +75,9 @@ class _TeamDestinations:return DeliveryPathChoice(reason=HogFunctionPathReason.CUSTOMIZED)# Per alert, so the team's other alerts still deliver natively.if self._source == SourceKind.INSIGHT and any(- destination.template_id == PagerDutyDestination.template_id for destination in destinations+ destination.template_id == PagerDutyDestination.template_id+ and LEGACY_INSIGHT_ALERT_EVENT in destination.event_ids+ for destination in destinations):return DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE)if any(destination.template_id == WebhookDestination.template_id for destination in destinations):diff --git a/products/alerts/backend/logic/destination_eligibility.py b/products/alerts/backend/logic/destination_eligibility.pyindex a45378fcb0f7..8ae18d1ce4a2 100644--- a/products/alerts/backend/logic/destination_eligibility.py+++ b/products/alerts/backend/logic/destination_eligibility.py@@ -174,6 +174,7 @@ def _classify(function, first_revisions.get(function.id), _template_stock(source_stock, function.template_id)),code_hash=hashlib.md5((function.hog or "").encode(), usedforsecurity=False).hexdigest(),+ event_ids=_filter_event_ids(function.filters),)for function in functions)@@ -234,6 +235,13 @@ def unbound_alert_destinations(return tuple(function_id for function_id, filters in candidates if not _bound_to_one_alert(filters))+def _filter_event_ids(filters: Any) -> frozenset[str]:+ events = filters.get("events") if isinstance(filters, dict) else None+ return frozenset(+ event["id"] for event in events or () if isinstance(event, dict) and isinstance(event.get("id"), str)+ )++def _named_alert_ids(filters: Any) -> set[str]:# Matches `owned_alert_destinations_qs`, which reads an alert id under any operator.properties = filters.get("properties") if isinstance(filters, dict) else Nonediff --git a/products/alerts/backend/platform_alert_legacy_events.py b/products/alerts/backend/platform_alert_legacy_events.pyindex f4ea948604b4..27c597ee2444 100644--- a/products/alerts/backend/platform_alert_legacy_events.py+++ b/products/alerts/backend/platform_alert_legacy_events.py@@ -1,6 +1,6 @@-"""The `$insight_alert_firing` events the platform emits for an insight alert it owns on the HogFunction path.+"""The `$insight_alert_*` events the platform emits for an insight alert it owns on the HogFunction path.-The alert's HogFunctions filter on this event and render these properties, so the platform+The alert's HogFunctions filter on these events and render these properties, so the platformreproduces what `trigger_alert_hog_functions` sends: the same names, read from the same alert,with the breach text and chart URL the check recorded beside its verdict.@@ -19,7 +19,11 @@ import structlogfrom posthog.schema_enums import AlertStatefrom posthog.tasks.alerts.utils import RenderedInsightChart, detector_verdict_event_fields, insight_chart_url-from products.alerts.backend.facade.contracts import AlertInternalEvent+from products.alerts.backend.facade.contracts import (+ INSIGHT_ALERT_INCIDENT_CLOSED_EVENT,+ INSIGHT_ALERT_INCIDENT_OPENED_EVENT,+ AlertInternalEvent,+)from products.alerts.backend.logic.destinations import emit_alert_internal_eventsfrom products.alerts.backend.models.alert import AlertConfigurationfrom products.alerts.backend.platform_alert_slack import insight_transition_properties@@ -28,6 +32,7 @@ from products.alerts_platform.backend.facade.contracts import (AlertEventKind,AnnouncedTransition,EvaluationAnnouncement,+ IncidentAction,LegacyEventEmission,MessageImage,)@@ -42,12 +47,7 @@ FLUSH_TIMEOUT_SECONDS: Final = 5.0def emit_insight_legacy_events(*, request: AlertDeliveryRequest, announcement: EvaluationAnnouncement, emissions: Sequence[LegacyEventEmission]) -> None:- firings = [- emission.transition- for emission in emissions- if emission.sends_message and emission.transition.kind == AlertEventKind.FIRING- ]- if not firings:+ if not any(_announces_firing(emission) or emission.incident_action is not None for emission in emissions):returnalert = (AlertConfiguration.objects.select_related("team", "insight")@@ -57,14 +57,24 @@ def emit_insight_legacy_events(# A deleted alert takes its destinations with it, so nobody is left to receive the event.if alert is None:return+ events: list[AlertInternalEvent] = []+ for emission in emissions:+ if _announces_firing(emission):+ events.append(_firing_event(alert, emission.transition))+ if emission.incident_action is not None:+ events.append(_incident_event(alert, emission.transition, emission.incident_action))emit_alert_internal_events(team_id=request.team_id,alert_id=request.destination_alert_id,- events=[_firing_event(alert, transition) for transition in firings],+ events=events,flush_timeout_seconds=FLUSH_TIMEOUT_SECONDS,)+def _announces_firing(emission: LegacyEventEmission) -> bool:+ return emission.sends_message and emission.transition.kind == AlertEventKind.FIRING++def insight_chart_url_of(team_id: int, source_detail: dict[str, Any]) -> str | None:"""The URL that embeds the chart a check rendered, minted now from the ids the check stored."""asset_id = source_detail.get("insight_chart_asset_id")@@ -98,6 +108,21 @@ def _formatting_properties(transition: AnnouncedTransition) -> dict[str, Any]:def _firing_event(alert: AlertConfiguration, transition: AnnouncedTransition) -> AlertInternalEvent:+ return _event(alert, transition, LEGACY_INSIGHT_ALERT_EVENT, _firing_properties(alert, transition))+++def _incident_event(+ alert: AlertConfiguration, transition: AnnouncedTransition, action: IncidentAction+) -> AlertInternalEvent:+ if action == IncidentAction.TRIGGER:+ return _event(alert, transition, INSIGHT_ALERT_INCIDENT_OPENED_EVENT, _firing_properties(alert, transition))+ properties = alert.internal_event_properties(+ state=AlertState.NOT_FIRING.value, last_checked_at=transition.occurred_at+ )+ return _event(alert, transition, INSIGHT_ALERT_INCIDENT_CLOSED_EVENT, properties)+++def _firing_properties(alert: AlertConfiguration, transition: AnnouncedTransition) -> dict[str, Any]:detail = transition.source_detailproperties: dict[str, Any] = {# From the transition, not the alert row: the platform never writes `state` or@@ -111,13 +136,19 @@ def _firing_event(alert: AlertConfiguration, transition: AnnouncedTransition) ->}if chart_url := insight_chart_url_of(alert.team_id, detail):properties["insight_chart_url"] = chart_url+ return properties+++def _event(+ alert: AlertConfiguration, transition: AnnouncedTransition, event_name: str, properties: dict[str, Any]+) -> AlertInternalEvent:# The occurrence is in the seed too, so two firings recorded without an evaluation key keep distinct ids.seed = (- f"{alert.id}|{transition.grouping_key}|{LEGACY_INSIGHT_ALERT_EVENT}|{transition.evaluation_key}|"+ f"{alert.id}|{transition.grouping_key}|{event_name}|{transition.evaluation_key}|"f"{transition.occurred_at.isoformat()}")return AlertInternalEvent(- event_name=LEGACY_INSIGHT_ALERT_EVENT,+ event_name=event_name,properties=properties,timestamp=transition.occurred_at,uuid=str(uuid5(NAMESPACE_URL, seed)),diff --git a/products/alerts/backend/platform_source_config_schema.py b/products/alerts/backend/platform_source_config_schema.pyindex f67dacc44b5a..3689e15e88a1 100644--- a/products/alerts/backend/platform_source_config_schema.py+++ b/products/alerts/backend/platform_source_config_schema.py@@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, Fieldfrom posthog.schema import AlertCondition, InsightThreshold+from products.alerts.backend.facade.contracts import INSIGHT_EVENT_IDS_BY_INCIDENT_ACTIONfrom products.alerts.backend.platform_ownership import mirror_platform_editfrom products.alerts_platform.backend.facade.contracts import (GroupingMode,@@ -63,4 +64,5 @@ INSIGHT_SOURCE_CONFIG = SourceConfigSpec(model=InsightPlatformAlertSourceConfig,validate=validate_insight_edit,mirror=mirror_platform_edit,+ incident_event_ids=INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION,)diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex 226bceff110b..91789ed531fb 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -44,6 +44,7 @@ from products.alerts.backend.evaluation import check_alert_for_insightfrom products.alerts.backend.evaluation.comparator import MAX_BREACH_MESSAGESfrom products.alerts.backend.evaluation.contract import AlertExtractionErrorfrom products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, check_alert_per_series+from products.alerts.backend.facade.contracts import INSIGHT_EVENT_IDS_BY_INCIDENT_ACTIONfrom products.alerts.backend.insight_alert_state_machine import INSIGHT_ALERT_POLICY, insight_snapshotfrom products.alerts.backend.judge.contract import (LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,@@ -53,6 +54,7 @@ from products.alerts.backend.judge.contract import ()from products.alerts.backend.llm_detector_limits import is_llm_detector_configfrom products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID+from products.alerts.backend.logic.destinations import configured_destination_template_idsfrom products.alerts.backend.models.alert import AlertConfigurationfrom products.alerts.backend.platform_alert_wording import insight_alert_urlfrom products.alerts.backend.platform_chart_detail import CHART_PENDING_KEY, RenderedChart@@ -62,6 +64,7 @@ from products.alerts_platform.backend.facade.contracts import (AlertDeliveryRequest,AlertEventKind,GroupingMode,+ IncidentAction,InstanceCheckState,PlatformAlertCheckInput,PlatformAlertOutcome,@@ -89,6 +92,7 @@ from products.alerts_platform.backend.facade.lifecycle import (apply_snooze,apply_unsnooze,decide_firing_episode,+ decide_incident_action,evaluate_alert_check,)from products.alerts_platform.backend.facade.platform_metrics import (@@ -547,17 +551,31 @@ def _evaluation(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> AlertDeliveryRequest | None:+ """The delivery for a recorded outcome, or None when it neither announces nor moves a firing.++ A firing that opens or closes needs a delivery even though no resolve is announced, because a+ paging destination needs one resolve for every trigger. An alert with no such destination gets+ no incident action, so its edges start no delivery.+ """kinds = [group.kind for group in outcome.groups]if outcome.failure is not None:kinds.append(outcome.failure.kind)- if not any(kind.value in _EVENT_IDS_BY_KIND for kind in kinds):- return None+ sends_messages = any(kind.value in _EVENT_IDS_BY_KIND for kind in kinds)# Destinations are matched on the legacy alert's id, so a copy without one has nobody to send to.if check.legacy_configuration_id is None:return Nonelegacy_id = str(check.legacy_configuration_id)if not check.owned_by_platform and legacy_id not in LIVE_DELIVERY_INSIGHT_ALERT_IDS:return None+ actions = _incident_actions(check, outcome)+ if actions and not configured_destination_template_ids(+ team_id=check.team_id,+ alert_id=legacy_id,+ allowed_event_ids=tuple(INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values()),+ ):+ actions = {}+ if not sends_messages and not actions:+ return Nonereturn AlertDeliveryRequest(source=SourceKind.INSIGHT,team_id=check.team_id,@@ -566,14 +584,36 @@ def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) ->evaluation_key=outcome.evaluation_key,destination_alert_id=legacy_id,event_ids_by_kind=_EVENT_IDS_BY_KIND,+ event_ids_by_incident_action=INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION,+ incident_actions=actions,+ sends_messages=sends_messages,source_alert_url=_alert_url(check, legacy_id),owned_by_platform=check.owned_by_platform,overflowed=outcome.overflowed,overflow_is_lower_bound=outcome.overflow_is_lower_bound,- parts=delivery_parts(outcome),+ parts=delivery_parts(outcome, actions),)+def _incident_actions(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> dict[str, IncidentAction]:+ """Whether each group's firing opened or closed, read from the states rather than the announcement.++ A failed check moves no instance, so its incident stays open with its firing. Closing it would+ leave a re-enabled alert that still breaches firing with no incident.+ """+ policy = INSIGHT_ALERT_POLICY if check.grouping.mode == GroupingMode.SINGLE else GROUPED_INSIGHT_ALERT_POLICY+ return {+ group.grouping_key: action+ for group in outcome.groups+ if (+ action := decide_incident_action(+ AlertState(check.instance(group.grouping_key).state), AlertState(group.new_state), policy=policy+ )+ )+ is not None+ }++def _alert_url(check: PlatformAlertCheckInput, legacy_id: str) -> str | None:"""The alert's page on its insight, so a native message links where the insight stack's does."""insight_short_id = (diff --git a/products/alerts/backend/presentation/views/alert.py b/products/alerts/backend/presentation/views/alert.pyindex 752eff65f270..48709aab1880 100644--- a/products/alerts/backend/presentation/views/alert.py+++ b/products/alerts/backend/presentation/views/alert.py@@ -93,6 +93,7 @@ from products.alerts.backend.facade.api import (llm_detector_access_error,)from products.alerts.backend.facade.contracts import (+ INSIGHT_ALERT_DESTINATION_EVENT_IDS,INSIGHT_ALERT_DESTINATION_TYPES,INSIGHT_ALERT_EVENT_IDS,MovedAlertCheck,@@ -1953,7 +1954,7 @@ class AlertViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):soft_delete_alert_destinations(team_id=self.team_id,alert_id=str(alert.id),- allowed_event_ids=INSIGHT_ALERT_EVENT_IDS,+ allowed_event_ids=INSIGHT_ALERT_DESTINATION_EVENT_IDS,hog_function_ids=hog_function_ids,)except AlertDestinationValidationError as error:diff --git a/products/alerts/backend/test/test_delivery_path.py b/products/alerts/backend/test/test_delivery_path.pyindex 33a323252329..f8122b2af537 100644--- a/products/alerts/backend/test/test_delivery_path.py+++ b/products/alerts/backend/test/test_delivery_path.py@@ -9,25 +9,35 @@ from products.alerts.backend.logic.delivery_path import _TeamDestinationsfrom products.alerts_platform.backend.facade.contracts import DeliveryPath, SourceKind-def _destination(template_id: str) -> DestinationEligibility:- return DestinationEligibility(hog_function_id=uuid4(), template_id=template_id, hold_backs=(), code_hash="")+def _destination(template_id: str, event_id: str) -> DestinationEligibility:+ return DestinationEligibility(+ hog_function_id=uuid4(),+ template_id=template_id,+ hold_backs=(),+ code_hash="",+ event_ids=frozenset({event_id}),+ )class TestDeliveryPathChoice(SimpleTestCase):@parameterized.expand([- # Native insight delivery sends no incident actions, so the function keeps opening them.- (SourceKind.INSIGHT, DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE)),- (SourceKind.LOGS, DeliveryPathChoice()),+ (+ SourceKind.INSIGHT,+ "$insight_alert_firing",+ DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE),+ ),+ (SourceKind.INSIGHT, "$insight_alert_incident_opened", DeliveryPathChoice()),+ (SourceKind.LOGS, "$logs_alert_incident_opened", DeliveryPathChoice()),])- def test_a_stock_pagerduty_destination_keeps_its_hog_function_only_for_insight(- self, source: SourceKind, expected: DeliveryPathChoice+ def test_a_stock_pagerduty_destination_keeps_its_hog_function_only_on_the_insight_firing_event(+ self, source: SourceKind, event_id: str, expected: DeliveryPathChoice) -> None:team = _TeamDestinations(source=source, team_id=1, allowed_event_ids=())- paged = team.choice([_destination("template-pagerduty"), _destination("template-slack")])+ paged = team.choice([_destination("template-pagerduty", event_id), _destination("template-slack", event_id)])# Another alert of the same team, so the choice is per alert.- other = team.choice([_destination("template-slack")])+ other = team.choice([_destination("template-slack", event_id)])assert (paged, other.path) == (expected, DeliveryPath.NATIVE)diff --git a/products/alerts/backend/test/test_destination_eligibility.py b/products/alerts/backend/test/test_destination_eligibility.pyindex bae986af8f6f..b5631020db5a 100644--- a/products/alerts/backend/test/test_destination_eligibility.py+++ b/products/alerts/backend/test/test_destination_eligibility.py@@ -11,6 +11,8 @@ from parameterized import parameterizedfrom products.alerts.backend.facade.contracts import (INSIGHT_ALERT_EVENT_IDS,+ INSIGHT_ALERT_INCIDENT_CLOSED_EVENT,+ INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION,DestinationBucket,DestinationEligibility,DestinationHoldBack,@@ -20,10 +22,17 @@ from products.alerts.backend.facade.destinations import (classify_alert_destinations,unbound_alert_destinations,)+from products.alerts.backend.logic.delivery_path import delivery_pathfrom products.alerts.backend.logic.destination_bucket_report import bucketed_destinationsfrom products.alerts.backend.logic.destination_eligibility import LOGS_STOCK_INPUT_VALUES, STOCK_INPUT_VALUESfrom products.alerts.backend.logic.insight_alert_destinations import build_insight_alert_slack_config-from products.alerts_platform.backend.facade.contracts import AlertDestinationData, DestinationType+from products.alerts_platform.backend.facade.contracts import (+ AlertDestinationData,+ DeliveryPath,+ DestinationType,+ SourceKind,+)+from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENTfrom products.cdp.backend.models.hog_functions.hog_function import HogFunctionfrom products.cdp.backend.models.hog_functions.hog_function_revision import HogFunctionRevision@@ -64,6 +73,7 @@ class TestDestinationEligibility(APIBaseTest):secret_keys: frozenset[str] = frozenset(),alert_id: str | None = ALERT_ID,enabled: bool = True,+ event: str = EVENT,**fields: Any,) -> HogFunction:return HogFunction.objects.create(@@ -77,7 +87,7 @@ class TestDestinationEligibility(APIBaseTest):hog="return event",filters={"source": "internal-events",- "events": [{"id": EVENT, "type": "events"}],+ "events": [{"id": event, "type": "events"}],"properties": [{"key": "alert_id", "value": alert_id, "operator": "exact", "type": "event"}]if alert_idelse [],@@ -273,6 +283,26 @@ class TestDestinationEligibility(APIBaseTest):assert self._classify() == (DestinationHoldBack.REFUSED_ADDRESS,)+ @parameterized.expand(+ [+ ("the_firing_event", LEGACY_INSIGHT_ALERT_EVENT, DeliveryPath.HOG_FUNCTION),+ ("an_incident_event", INSIGHT_ALERT_INCIDENT_CLOSED_EVENT, DeliveryPath.NATIVE),+ ]+ )+ def test_an_insight_pagerduty_function_is_native_only_on_the_incident_events(+ self, _name: str, event: str, expected: DeliveryPath+ ) -> None:+ self._function("template-pagerduty", PAGERDUTY_INPUTS, event=event)++ path = delivery_path(+ source=SourceKind.INSIGHT,+ team_id=self.team.id,+ alert_id=ALERT_ID,+ allowed_event_ids=[LEGACY_INSIGHT_ALERT_EVENT, *INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values()],+ )++ assert path == expected+def test_a_signed_webhook_is_held_back(self) -> None:self._function("template-webhook",@@ -469,7 +499,11 @@ class TestBucketFor(SimpleTestCase):self, _name: str, hold_backs: tuple[DestinationHoldBack, ...], expected: DestinationBucket) -> None:eligibility = DestinationEligibility(- hog_function_id=UUID(int=1), template_id="template-webhook", hold_backs=hold_backs, code_hash=""+ hog_function_id=UUID(int=1),+ template_id="template-webhook",+ hold_backs=hold_backs,+ code_hash="",+ event_ids=frozenset(),)assert bucket_for(eligibility) == expecteddiff --git a/products/alerts/backend/test/test_platform_alert_legacy_events.py b/products/alerts/backend/test/test_platform_alert_legacy_events.pyindex 766e52b0895a..29c8c651f870 100644--- a/products/alerts/backend/test/test_platform_alert_legacy_events.py+++ b/products/alerts/backend/test/test_platform_alert_legacy_events.py@@ -18,6 +18,7 @@ from products.alerts_platform.backend.facade.contracts import (AlertEventKind,AnnouncedTransition,EvaluationAnnouncement,+ IncidentAction,LegacyEventEmission,MessageImage,SourceKind,@@ -75,6 +76,7 @@ class TestInsightLegacyEvents(APIBaseTest):*,sends_message: bool = True,detail: dict[str, Any] | None = None,+ incident_action: IncidentAction | None = None,) -> LegacyEventEmission:transition = AnnouncedTransition(grouping_key="",@@ -89,7 +91,7 @@ class TestInsightLegacyEvents(APIBaseTest):evaluation_key="slot:1",source_detail=detail or {},)- return LegacyEventEmission(transition=transition, sends_message=sends_message, incident_action=None)+ return LegacyEventEmission(transition=transition, sends_message=sends_message, incident_action=incident_action)@parameterized.expand([@@ -168,6 +170,41 @@ class TestInsightLegacyEvents(APIBaseTest):) -> None:assert self._emit(self._emission(kind, sends_message=sends_message), alert_id=alert_id) == []+ @parameterized.expand(+ [+ (+ "an_opened_firing",+ AlertEventKind.FIRING,+ True,+ IncidentAction.TRIGGER,+ ["$insight_alert_firing", "$insight_alert_incident_opened"],+ ),+ (+ "a_silent_resolve",+ AlertEventKind.CHECK,+ False,+ IncidentAction.RESOLVE,+ ["$insight_alert_incident_closed"],+ ),+ ]+ )+ def test_an_incident_edge_emits_its_incident_event_beside_any_message(+ self,+ _name: str,+ kind: AlertEventKind,+ sends_message: bool,+ action: IncidentAction,+ expected: list[str],+ ) -> None:+ events = self._emit(+ self._emission(+ kind, sends_message=sends_message, detail={"breaches": ["above 100"]}, incident_action=action+ )+ )++ assert [event.event_name for event in events] == expected+ assert all(event.properties["alert_id"] == str(self.alert.id) for event in events)+class TestInsightChartImage(SimpleTestCase):@parameterized.expand(diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex f1b1c8958059..507c40828edb 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -71,12 +71,14 @@ from products.alerts_platform.backend.facade.contracts import (AlertEventKind,Grouping,GroupingMode,+ IncidentAction,PlatformAlertOutcome,PlatformConfigurationSnapshot,SourceEvaluationInputs,SourceKind,)from products.alerts_platform.backend.facade.enums import PlatformAlertOwner+from products.cdp.backend.models.hog_functions.hog_function import HogFunction_MODULE = "products.alerts.backend.platform_source_cycle"# A Wednesday, so no weekend rule applies.@@ -502,18 +504,44 @@ class TestPlatformInsightEvaluation(APIBaseTest):@parameterized.expand([- ("allowlisted", True, False, False),- ("not_allowlisted", False, False, False),- ("allowlisted_resolve", True, True, False),- ("platform_owned", False, False, True),+ # name, allowlisted, was_firing, owned, paged, breaks, (incident_actions, sends_messages) or no delivery+ ("allowlisted", True, False, False, False, False, ({}, True)),+ ("not_allowlisted", False, False, False, False, False, None),+ ("allowlisted_resolve", True, True, False, False, False, None),+ ("platform_owned", False, False, True, False, False, ({}, True)),+ ("paged_firing", False, False, True, True, False, ({"": IncidentAction.TRIGGER}, True)),+ ("paged_resolve", False, True, True, True, False, ({"": IncidentAction.RESOLVE}, False)),+ ("paged_firing_that_breaks", False, True, True, True, True, ({}, True)),+ ("paged_but_not_allowlisted", False, False, False, True, False, None),])- def test_only_an_allowlisted_or_platform_owned_firing_or_failure_asks_for_a_delivery(- self, _name: str, allowlisted: bool, was_firing: bool, owned: bool+ def test_only_an_allowlisted_or_platform_owned_firing_failure_or_incident_edge_asks_for_a_delivery(+ self,+ _name: str,+ allowlisted: bool,+ was_firing: bool,+ owned: bool,+ paged: bool,+ breaks: bool,+ delivered: tuple[dict[str, IncidentAction], bool] | None,) -> None:alert = self._alert()configuration = self._copy(alert, **({"owner": PlatformAlertOwner.PLATFORM} if owned else {}))allowlist = frozenset({str(alert.id)}) if allowlisted else frozenset()+ if paged:+ HogFunction.objects.create(+ team=self.team,+ type="internal_destination",+ name="PagerDuty",+ template_id="template-pagerduty",+ hog="return event",+ enabled=True,+ filters={+ "source": "internal-events",+ "events": [{"id": "$insight_alert_incident_closed", "type": "events"}],+ "properties": [{"key": "alert_id", "value": str(alert.id), "operator": "exact", "type": "event"}],+ },+ )if was_firing:fired, _ = self._evaluate(configuration, result=AlertEvaluationResult(value=150.0, breaches=["above 100"]))assert fired is not None@@ -524,26 +552,35 @@ class TestPlatformInsightEvaluation(APIBaseTest):with patch(f"{_MODULE}.LIVE_DELIVERY_INSIGHT_ALERT_IDS", allowlist):self._evaluate(configuration,- result=AlertEvaluationResult(value=50.0, breaches=[] if was_firing else ["above 100"]),+ result=AlertExtractionError("bad query shape")+ if breaks+ else AlertEvaluationResult(value=50.0, breaches=[] if was_firing else ["above 100"]),)- expected = [- (- str(configuration.id),- str(alert.id),- {"firing": "$insight_alert_firing", "errored": INSIGHT_ALERT_ERRORED_EVENT_ID},- owned,- )- ]+ expected = (+ [+ (+ str(configuration.id),+ str(alert.id),+ {"firing": "$insight_alert_firing", "errored": INSIGHT_ALERT_ERRORED_EVENT_ID},+ owned,+ *delivered,+ )+ ]+ if delivered is not None+ else []+ )assert [(delivery.configuration_id,delivery.destination_alert_id,delivery.event_ids_by_kind,delivery.owned_by_platform,+ delivery.incident_actions,+ delivery.sends_messages,)for delivery in self.deliveries- ] == (expected if (allowlisted or owned) and not was_firing else [])+ ] == expected# A native message links to the alert on its insight, where the insight stack's messages link.assert all(delivery.source_alert_url == insight_alert_url(self.team.id, alert.insight.short_id, str(alert.id))diff --git a/products/alerts/backend/tests/api/test_alert.py b/products/alerts/backend/tests/api/test_alert.pyindex fdc74e6ecfe5..604202839017 100644--- a/products/alerts/backend/tests/api/test_alert.py+++ b/products/alerts/backend/tests/api/test_alert.py@@ -27,7 +27,7 @@ from posthog.models.team import Teamfrom posthog.models.utils import generate_random_token_personal, hash_key_valuefrom products.alerts.backend.facade.api import LLMDetectorUnavailableError-from products.alerts.backend.facade.contracts import INSIGHT_ALERT_EVENT_IDS+from products.alerts.backend.facade.contracts import INSIGHT_ALERT_EVENT_IDS, INSIGHT_EVENT_IDS_BY_INCIDENT_ACTIONfrom products.alerts.backend.facade.destinations import MAX_DESTINATIONS_PER_ALERT, count_active_alert_destinationsfrom products.alerts.backend.judge.verdict import LLMDetectionVerdictfrom products.alerts.backend.logic.insight_alert_destinations import SLACK_TEMPLATE_ID@@ -2919,6 +2919,34 @@ class TestAlertDestinations(APIBaseTest):assert response.status_code == status.HTTP_204_NO_CONTENT, response.contentassert self._active_destination_count() == 0+ def test_an_incident_destination_is_removed_through_the_insight_api(self) -> None:+ hog_function_ids = [+ str(+ HogFunction.objects.create(+ team=self.team,+ type="internal_destination",+ name="PagerDuty",+ template_id="template-pagerduty",+ hog="return event",+ enabled=True,+ inputs={"routing_key": {"value": "a" * 32}},+ filters={+ "source": "internal-events",+ "events": [{"id": event_id, "type": "events"}],+ "properties": [+ {"key": "alert_id", "value": str(self.alert.id), "operator": "exact", "type": "event"}+ ],+ },+ ).id+ )+ for event_id in INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values()+ ]++ response = self.client.post(f"{self.url}delete/", {"hog_function_ids": hog_function_ids}, format="json")++ assert response.status_code == status.HTTP_204_NO_CONTENT, response.content+ assert not HogFunction.objects.filter(id__in=hog_function_ids, deleted=False).exists()+def test_an_alert_stops_taking_destinations_at_the_cap(self) -> None:for index in range(MAX_DESTINATIONS_PER_ALERT):response = self.client.post(self.url, self._slack_payload(slack_channel_id=f"C{index}"), format="json")