Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): open and resolve pagerduty incidents for insight alerts (#127898)

Jimmy Zhue9675d5ae6c1
diff --git a/docs/internal/alerts-platform-temporal.md b/docs/internal/alerts-platform-temporal.md
index 8fcb456137f1..8837a362971b 100644
--- a/docs/internal/alerts-platform-temporal.md
+++ b/docs/internal/alerts-platform-temporal.md
@@ -421,6 +421,9 @@ The legacy logs stack's `incident_edge` wraps the same rule.
`<configuration_id>:<grouping_key>:<episode_started_at>`, so a resolve closes the incident of its own
firing episode. It names the platform in `source`, because a team on the pilot also gets the HogFunction
path's incident.
+- An insight alert announces no resolve, but it still sends incident actions for an alert the platform owns.
+ A PagerDuty destination on `$insight_alert_incident_opened` and `$insight_alert_incident_closed` opens and resolves its incident.
+ One on `$insight_alert_firing` keeps its HogFunction, which only triggers.
A fire a mute swallowed is still owed an announcement.
`_firing_is_unannounced` in `facade/lifecycle.py` decides that, and its docstring holds the rule.
diff --git a/products/alerts/backend/facade/contracts.py b/products/alerts/backend/facade/contracts.py
index 29c355e927e3..50b8222feabf 100644
--- a/products/alerts/backend/facade/contracts.py
+++ b/products/alerts/backend/facade/contracts.py
@@ -15,13 +15,27 @@ from uuid import UUID
from posthog.dataclasses import frozen
from posthog.schema_enums import AlertState
-from products.alerts_platform.backend.facade.contracts import DeliveryPath, DestinationType
+from products.alerts_platform.backend.facade.contracts import DeliveryPath, DestinationType, IncidentAction
from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENT
# The internal event an insight alert's destinations fire on. Do not take it from
# `posthog.tasks.alerts.utils` instead, because that module imports this product's facade.
INSIGHT_ALERT_EVENT_IDS: Final[tuple[str, ...]] = (LEGACY_INSIGHT_ALERT_EVENT,)
+# Every move into and out of a firing, so each trigger gets its resolve while messages still announce none.
+INSIGHT_ALERT_INCIDENT_OPENED_EVENT: Final = "$insight_alert_incident_opened"
+INSIGHT_ALERT_INCIDENT_CLOSED_EVENT: Final = "$insight_alert_incident_closed"
+INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION: Final[dict[str, str]] = {
+ IncidentAction.TRIGGER.value: INSIGHT_ALERT_INCIDENT_OPENED_EVENT,
+ IncidentAction.RESOLVE.value: INSIGHT_ALERT_INCIDENT_CLOSED_EVENT,
+}
+# Every event an insight alert's own destinations can listen on. The generic HogFunction API treats
+# the incident events as managed and refuses to edit them, so the insight API must be able to remove them.
+INSIGHT_ALERT_DESTINATION_EVENT_IDS: Final[tuple[str, ...]] = (
+ *INSIGHT_ALERT_EVENT_IDS,
+ *INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values(),
+)
+
# Slack only, because `alert:write` is grantable to a sandboxed agent. A connected workspace is a
# destination an admin chose, while every other transport takes a URL the caller supplies.
INSIGHT_ALERT_DESTINATION_TYPES: Final[tuple[DestinationType, ...]] = (DestinationType.SLACK,)
@@ -82,7 +96,7 @@ class HogFunctionPathReason(StrEnum):
# body the HogFunction sends.
NEW_WEBHOOK_BODY = "new_webhook_body"
WEBHOOK_BODY_UNKNOWN = "webhook_body_unknown"
- # Native insight delivery sends no incident actions, so a PagerDuty destination would open none.
+ # Native delivery sends PagerDuty incident actions only, never the firing event.
PAGERDUTY_NOT_NATIVE = "pagerduty_not_native"
@@ -104,6 +118,7 @@ class DestinationEligibility:
hold_backs: tuple[DestinationHoldBack, ...]
# The md5 of the function's code, so the same code can be recognized across functions and teams.
code_hash: str
+ event_ids: frozenset[str]
@frozen
diff --git a/products/alerts/backend/logic/delivery_path.py b/products/alerts/backend/logic/delivery_path.py
index 786745ffb8a0..18b43342092a 100644
--- a/products/alerts/backend/logic/delivery_path.py
+++ b/products/alerts/backend/logic/delivery_path.py
@@ -29,6 +29,7 @@ from products.alerts.backend.logic.destination_eligibility import (
)
from products.alerts_platform.backend.facade.contracts import DeliveryPath, SourceKind
from products.alerts_platform.backend.facade.delivery import native_sends_legacy_webhook_body
+from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENT
logger = structlog.get_logger(__name__)
@@ -74,7 +75,9 @@ class _TeamDestinations:
return DeliveryPathChoice(reason=HogFunctionPathReason.CUSTOMIZED)
# Per alert, so the team's other alerts still deliver natively.
if self._source == SourceKind.INSIGHT and any(
- destination.template_id == PagerDutyDestination.template_id for destination in destinations
+ destination.template_id == PagerDutyDestination.template_id
+ and LEGACY_INSIGHT_ALERT_EVENT in destination.event_ids
+ for destination in destinations
):
return DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE)
if any(destination.template_id == WebhookDestination.template_id for destination in destinations):
diff --git a/products/alerts/backend/logic/destination_eligibility.py b/products/alerts/backend/logic/destination_eligibility.py
index a45378fcb0f7..8ae18d1ce4a2 100644
--- a/products/alerts/backend/logic/destination_eligibility.py
+++ b/products/alerts/backend/logic/destination_eligibility.py
@@ -174,6 +174,7 @@ def _classify(
function, first_revisions.get(function.id), _template_stock(source_stock, function.template_id)
),
code_hash=hashlib.md5((function.hog or "").encode(), usedforsecurity=False).hexdigest(),
+ event_ids=_filter_event_ids(function.filters),
)
for function in functions
)
@@ -234,6 +235,13 @@ def unbound_alert_destinations(
return tuple(function_id for function_id, filters in candidates if not _bound_to_one_alert(filters))
+def _filter_event_ids(filters: Any) -> frozenset[str]:
+ events = filters.get("events") if isinstance(filters, dict) else None
+ return frozenset(
+ event["id"] for event in events or () if isinstance(event, dict) and isinstance(event.get("id"), str)
+ )
+
+
def _named_alert_ids(filters: Any) -> set[str]:
# Matches `owned_alert_destinations_qs`, which reads an alert id under any operator.
properties = filters.get("properties") if isinstance(filters, dict) else None
diff --git a/products/alerts/backend/platform_alert_legacy_events.py b/products/alerts/backend/platform_alert_legacy_events.py
index f4ea948604b4..27c597ee2444 100644
--- a/products/alerts/backend/platform_alert_legacy_events.py
+++ b/products/alerts/backend/platform_alert_legacy_events.py
@@ -1,6 +1,6 @@
-"""The `$insight_alert_firing` events the platform emits for an insight alert it owns on the HogFunction path.
+"""The `$insight_alert_*` events the platform emits for an insight alert it owns on the HogFunction path.
-The alert's HogFunctions filter on this event and render these properties, so the platform
+The alert's HogFunctions filter on these events and render these properties, so the platform
reproduces what `trigger_alert_hog_functions` sends: the same names, read from the same alert,
with the breach text and chart URL the check recorded beside its verdict.
@@ -19,7 +19,11 @@ import structlog
from posthog.schema_enums import AlertState
from posthog.tasks.alerts.utils import RenderedInsightChart, detector_verdict_event_fields, insight_chart_url
-from products.alerts.backend.facade.contracts import AlertInternalEvent
+from products.alerts.backend.facade.contracts import (
+ INSIGHT_ALERT_INCIDENT_CLOSED_EVENT,
+ INSIGHT_ALERT_INCIDENT_OPENED_EVENT,
+ AlertInternalEvent,
+)
from products.alerts.backend.logic.destinations import emit_alert_internal_events
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts.backend.platform_alert_slack import insight_transition_properties
@@ -28,6 +32,7 @@ from products.alerts_platform.backend.facade.contracts import (
AlertEventKind,
AnnouncedTransition,
EvaluationAnnouncement,
+ IncidentAction,
LegacyEventEmission,
MessageImage,
)
@@ -42,12 +47,7 @@ FLUSH_TIMEOUT_SECONDS: Final = 5.0
def emit_insight_legacy_events(
*, request: AlertDeliveryRequest, announcement: EvaluationAnnouncement, emissions: Sequence[LegacyEventEmission]
) -> None:
- firings = [
- emission.transition
- for emission in emissions
- if emission.sends_message and emission.transition.kind == AlertEventKind.FIRING
- ]
- if not firings:
+ if not any(_announces_firing(emission) or emission.incident_action is not None for emission in emissions):
return
alert = (
AlertConfiguration.objects.select_related("team", "insight")
@@ -57,14 +57,24 @@ def emit_insight_legacy_events(
# A deleted alert takes its destinations with it, so nobody is left to receive the event.
if alert is None:
return
+ events: list[AlertInternalEvent] = []
+ for emission in emissions:
+ if _announces_firing(emission):
+ events.append(_firing_event(alert, emission.transition))
+ if emission.incident_action is not None:
+ events.append(_incident_event(alert, emission.transition, emission.incident_action))
emit_alert_internal_events(
team_id=request.team_id,
alert_id=request.destination_alert_id,
- events=[_firing_event(alert, transition) for transition in firings],
+ events=events,
flush_timeout_seconds=FLUSH_TIMEOUT_SECONDS,
)
+def _announces_firing(emission: LegacyEventEmission) -> bool:
+ return emission.sends_message and emission.transition.kind == AlertEventKind.FIRING
+
+
def insight_chart_url_of(team_id: int, source_detail: dict[str, Any]) -> str | None:
"""The URL that embeds the chart a check rendered, minted now from the ids the check stored."""
asset_id = source_detail.get("insight_chart_asset_id")
@@ -98,6 +108,21 @@ def _formatting_properties(transition: AnnouncedTransition) -> dict[str, Any]:
def _firing_event(alert: AlertConfiguration, transition: AnnouncedTransition) -> AlertInternalEvent:
+ return _event(alert, transition, LEGACY_INSIGHT_ALERT_EVENT, _firing_properties(alert, transition))
+
+
+def _incident_event(
+ alert: AlertConfiguration, transition: AnnouncedTransition, action: IncidentAction
+) -> AlertInternalEvent:
+ if action == IncidentAction.TRIGGER:
+ return _event(alert, transition, INSIGHT_ALERT_INCIDENT_OPENED_EVENT, _firing_properties(alert, transition))
+ properties = alert.internal_event_properties(
+ state=AlertState.NOT_FIRING.value, last_checked_at=transition.occurred_at
+ )
+ return _event(alert, transition, INSIGHT_ALERT_INCIDENT_CLOSED_EVENT, properties)
+
+
+def _firing_properties(alert: AlertConfiguration, transition: AnnouncedTransition) -> dict[str, Any]:
detail = transition.source_detail
properties: dict[str, Any] = {
# From the transition, not the alert row: the platform never writes `state` or
@@ -111,13 +136,19 @@ def _firing_event(alert: AlertConfiguration, transition: AnnouncedTransition) ->
}
if chart_url := insight_chart_url_of(alert.team_id, detail):
properties["insight_chart_url"] = chart_url
+ return properties
+
+
+def _event(
+ alert: AlertConfiguration, transition: AnnouncedTransition, event_name: str, properties: dict[str, Any]
+) -> AlertInternalEvent:
# The occurrence is in the seed too, so two firings recorded without an evaluation key keep distinct ids.
seed = (
- f"{alert.id}|{transition.grouping_key}|{LEGACY_INSIGHT_ALERT_EVENT}|{transition.evaluation_key}|"
+ f"{alert.id}|{transition.grouping_key}|{event_name}|{transition.evaluation_key}|"
f"{transition.occurred_at.isoformat()}"
)
return AlertInternalEvent(
- event_name=LEGACY_INSIGHT_ALERT_EVENT,
+ event_name=event_name,
properties=properties,
timestamp=transition.occurred_at,
uuid=str(uuid5(NAMESPACE_URL, seed)),
diff --git a/products/alerts/backend/platform_source_config_schema.py b/products/alerts/backend/platform_source_config_schema.py
index f67dacc44b5a..3689e15e88a1 100644
--- a/products/alerts/backend/platform_source_config_schema.py
+++ b/products/alerts/backend/platform_source_config_schema.py
@@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field
from posthog.schema import AlertCondition, InsightThreshold
+from products.alerts.backend.facade.contracts import INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION
from products.alerts.backend.platform_ownership import mirror_platform_edit
from products.alerts_platform.backend.facade.contracts import (
GroupingMode,
@@ -63,4 +64,5 @@ INSIGHT_SOURCE_CONFIG = SourceConfigSpec(
model=InsightPlatformAlertSourceConfig,
validate=validate_insight_edit,
mirror=mirror_platform_edit,
+ incident_event_ids=INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION,
)
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 226bceff110b..91789ed531fb 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -44,6 +44,7 @@ from products.alerts.backend.evaluation import check_alert_for_insight
from products.alerts.backend.evaluation.comparator import MAX_BREACH_MESSAGES
from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, check_alert_per_series
+from products.alerts.backend.facade.contracts import INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION
from products.alerts.backend.insight_alert_state_machine import INSIGHT_ALERT_POLICY, insight_snapshot
from products.alerts.backend.judge.contract import (
LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,
@@ -53,6 +54,7 @@ from products.alerts.backend.judge.contract import (
)
from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
+from products.alerts.backend.logic.destinations import configured_destination_template_ids
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts.backend.platform_alert_wording import insight_alert_url
from products.alerts.backend.platform_chart_detail import CHART_PENDING_KEY, RenderedChart
@@ -62,6 +64,7 @@ from products.alerts_platform.backend.facade.contracts import (
AlertDeliveryRequest,
AlertEventKind,
GroupingMode,
+ IncidentAction,
InstanceCheckState,
PlatformAlertCheckInput,
PlatformAlertOutcome,
@@ -89,6 +92,7 @@ from products.alerts_platform.backend.facade.lifecycle import (
apply_snooze,
apply_unsnooze,
decide_firing_episode,
+ decide_incident_action,
evaluate_alert_check,
)
from products.alerts_platform.backend.facade.platform_metrics import (
@@ -547,17 +551,31 @@ def _evaluation(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -
def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> AlertDeliveryRequest | None:
+ """The delivery for a recorded outcome, or None when it neither announces nor moves a firing.
+
+ A firing that opens or closes needs a delivery even though no resolve is announced, because a
+ paging destination needs one resolve for every trigger. An alert with no such destination gets
+ no incident action, so its edges start no delivery.
+ """
kinds = [group.kind for group in outcome.groups]
if outcome.failure is not None:
kinds.append(outcome.failure.kind)
- if not any(kind.value in _EVENT_IDS_BY_KIND for kind in kinds):
- return None
+ sends_messages = any(kind.value in _EVENT_IDS_BY_KIND for kind in kinds)
# Destinations are matched on the legacy alert's id, so a copy without one has nobody to send to.
if check.legacy_configuration_id is None:
return None
legacy_id = str(check.legacy_configuration_id)
if not check.owned_by_platform and legacy_id not in LIVE_DELIVERY_INSIGHT_ALERT_IDS:
return None
+ actions = _incident_actions(check, outcome)
+ if actions and not configured_destination_template_ids(
+ team_id=check.team_id,
+ alert_id=legacy_id,
+ allowed_event_ids=tuple(INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values()),
+ ):
+ actions = {}
+ if not sends_messages and not actions:
+ return None
return AlertDeliveryRequest(
source=SourceKind.INSIGHT,
team_id=check.team_id,
@@ -566,14 +584,36 @@ def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) ->
evaluation_key=outcome.evaluation_key,
destination_alert_id=legacy_id,
event_ids_by_kind=_EVENT_IDS_BY_KIND,
+ event_ids_by_incident_action=INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION,
+ incident_actions=actions,
+ sends_messages=sends_messages,
source_alert_url=_alert_url(check, legacy_id),
owned_by_platform=check.owned_by_platform,
overflowed=outcome.overflowed,
overflow_is_lower_bound=outcome.overflow_is_lower_bound,
- parts=delivery_parts(outcome),
+ parts=delivery_parts(outcome, actions),
)
+def _incident_actions(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> dict[str, IncidentAction]:
+ """Whether each group's firing opened or closed, read from the states rather than the announcement.
+
+ A failed check moves no instance, so its incident stays open with its firing. Closing it would
+ leave a re-enabled alert that still breaches firing with no incident.
+ """
+ policy = INSIGHT_ALERT_POLICY if check.grouping.mode == GroupingMode.SINGLE else GROUPED_INSIGHT_ALERT_POLICY
+ return {
+ group.grouping_key: action
+ for group in outcome.groups
+ if (
+ action := decide_incident_action(
+ AlertState(check.instance(group.grouping_key).state), AlertState(group.new_state), policy=policy
+ )
+ )
+ is not None
+ }
+
+
def _alert_url(check: PlatformAlertCheckInput, legacy_id: str) -> str | None:
"""The alert's page on its insight, so a native message links where the insight stack's does."""
insight_short_id = (
diff --git a/products/alerts/backend/presentation/views/alert.py b/products/alerts/backend/presentation/views/alert.py
index 752eff65f270..48709aab1880 100644
--- a/products/alerts/backend/presentation/views/alert.py
+++ b/products/alerts/backend/presentation/views/alert.py
@@ -93,6 +93,7 @@ from products.alerts.backend.facade.api import (
llm_detector_access_error,
)
from products.alerts.backend.facade.contracts import (
+ INSIGHT_ALERT_DESTINATION_EVENT_IDS,
INSIGHT_ALERT_DESTINATION_TYPES,
INSIGHT_ALERT_EVENT_IDS,
MovedAlertCheck,
@@ -1953,7 +1954,7 @@ class AlertViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
soft_delete_alert_destinations(
team_id=self.team_id,
alert_id=str(alert.id),
- allowed_event_ids=INSIGHT_ALERT_EVENT_IDS,
+ allowed_event_ids=INSIGHT_ALERT_DESTINATION_EVENT_IDS,
hog_function_ids=hog_function_ids,
)
except AlertDestinationValidationError as error:
diff --git a/products/alerts/backend/test/test_delivery_path.py b/products/alerts/backend/test/test_delivery_path.py
index 33a323252329..f8122b2af537 100644
--- a/products/alerts/backend/test/test_delivery_path.py
+++ b/products/alerts/backend/test/test_delivery_path.py
@@ -9,25 +9,35 @@ from products.alerts.backend.logic.delivery_path import _TeamDestinations
from products.alerts_platform.backend.facade.contracts import DeliveryPath, SourceKind
-def _destination(template_id: str) -> DestinationEligibility:
- return DestinationEligibility(hog_function_id=uuid4(), template_id=template_id, hold_backs=(), code_hash="")
+def _destination(template_id: str, event_id: str) -> DestinationEligibility:
+ return DestinationEligibility(
+ hog_function_id=uuid4(),
+ template_id=template_id,
+ hold_backs=(),
+ code_hash="",
+ event_ids=frozenset({event_id}),
+ )
class TestDeliveryPathChoice(SimpleTestCase):
@parameterized.expand(
[
- # Native insight delivery sends no incident actions, so the function keeps opening them.
- (SourceKind.INSIGHT, DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE)),
- (SourceKind.LOGS, DeliveryPathChoice()),
+ (
+ SourceKind.INSIGHT,
+ "$insight_alert_firing",
+ DeliveryPathChoice(reason=HogFunctionPathReason.PAGERDUTY_NOT_NATIVE),
+ ),
+ (SourceKind.INSIGHT, "$insight_alert_incident_opened", DeliveryPathChoice()),
+ (SourceKind.LOGS, "$logs_alert_incident_opened", DeliveryPathChoice()),
]
)
- def test_a_stock_pagerduty_destination_keeps_its_hog_function_only_for_insight(
- self, source: SourceKind, expected: DeliveryPathChoice
+ def test_a_stock_pagerduty_destination_keeps_its_hog_function_only_on_the_insight_firing_event(
+ self, source: SourceKind, event_id: str, expected: DeliveryPathChoice
) -> None:
team = _TeamDestinations(source=source, team_id=1, allowed_event_ids=())
- paged = team.choice([_destination("template-pagerduty"), _destination("template-slack")])
+ paged = team.choice([_destination("template-pagerduty", event_id), _destination("template-slack", event_id)])
# Another alert of the same team, so the choice is per alert.
- other = team.choice([_destination("template-slack")])
+ other = team.choice([_destination("template-slack", event_id)])
assert (paged, other.path) == (expected, DeliveryPath.NATIVE)
diff --git a/products/alerts/backend/test/test_destination_eligibility.py b/products/alerts/backend/test/test_destination_eligibility.py
index bae986af8f6f..b5631020db5a 100644
--- a/products/alerts/backend/test/test_destination_eligibility.py
+++ b/products/alerts/backend/test/test_destination_eligibility.py
@@ -11,6 +11,8 @@ from parameterized import parameterized
from products.alerts.backend.facade.contracts import (
INSIGHT_ALERT_EVENT_IDS,
+ INSIGHT_ALERT_INCIDENT_CLOSED_EVENT,
+ INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION,
DestinationBucket,
DestinationEligibility,
DestinationHoldBack,
@@ -20,10 +22,17 @@ from products.alerts.backend.facade.destinations import (
classify_alert_destinations,
unbound_alert_destinations,
)
+from products.alerts.backend.logic.delivery_path import delivery_path
from products.alerts.backend.logic.destination_bucket_report import bucketed_destinations
from products.alerts.backend.logic.destination_eligibility import LOGS_STOCK_INPUT_VALUES, STOCK_INPUT_VALUES
from products.alerts.backend.logic.insight_alert_destinations import build_insight_alert_slack_config
-from products.alerts_platform.backend.facade.contracts import AlertDestinationData, DestinationType
+from products.alerts_platform.backend.facade.contracts import (
+ AlertDestinationData,
+ DeliveryPath,
+ DestinationType,
+ SourceKind,
+)
+from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENT
from products.cdp.backend.models.hog_functions.hog_function import HogFunction
from products.cdp.backend.models.hog_functions.hog_function_revision import HogFunctionRevision
@@ -64,6 +73,7 @@ class TestDestinationEligibility(APIBaseTest):
secret_keys: frozenset[str] = frozenset(),
alert_id: str | None = ALERT_ID,
enabled: bool = True,
+ event: str = EVENT,
**fields: Any,
) -> HogFunction:
return HogFunction.objects.create(
@@ -77,7 +87,7 @@ class TestDestinationEligibility(APIBaseTest):
hog="return event",
filters={
"source": "internal-events",
- "events": [{"id": EVENT, "type": "events"}],
+ "events": [{"id": event, "type": "events"}],
"properties": [{"key": "alert_id", "value": alert_id, "operator": "exact", "type": "event"}]
if alert_id
else [],
@@ -273,6 +283,26 @@ class TestDestinationEligibility(APIBaseTest):
assert self._classify() == (DestinationHoldBack.REFUSED_ADDRESS,)
+ @parameterized.expand(
+ [
+ ("the_firing_event", LEGACY_INSIGHT_ALERT_EVENT, DeliveryPath.HOG_FUNCTION),
+ ("an_incident_event", INSIGHT_ALERT_INCIDENT_CLOSED_EVENT, DeliveryPath.NATIVE),
+ ]
+ )
+ def test_an_insight_pagerduty_function_is_native_only_on_the_incident_events(
+ self, _name: str, event: str, expected: DeliveryPath
+ ) -> None:
+ self._function("template-pagerduty", PAGERDUTY_INPUTS, event=event)
+
+ path = delivery_path(
+ source=SourceKind.INSIGHT,
+ team_id=self.team.id,
+ alert_id=ALERT_ID,
+ allowed_event_ids=[LEGACY_INSIGHT_ALERT_EVENT, *INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values()],
+ )
+
+ assert path == expected
+
def test_a_signed_webhook_is_held_back(self) -> None:
self._function(
"template-webhook",
@@ -469,7 +499,11 @@ class TestBucketFor(SimpleTestCase):
self, _name: str, hold_backs: tuple[DestinationHoldBack, ...], expected: DestinationBucket
) -> None:
eligibility = DestinationEligibility(
- hog_function_id=UUID(int=1), template_id="template-webhook", hold_backs=hold_backs, code_hash=""
+ hog_function_id=UUID(int=1),
+ template_id="template-webhook",
+ hold_backs=hold_backs,
+ code_hash="",
+ event_ids=frozenset(),
)
assert bucket_for(eligibility) == expected
diff --git a/products/alerts/backend/test/test_platform_alert_legacy_events.py b/products/alerts/backend/test/test_platform_alert_legacy_events.py
index 766e52b0895a..29c8c651f870 100644
--- a/products/alerts/backend/test/test_platform_alert_legacy_events.py
+++ b/products/alerts/backend/test/test_platform_alert_legacy_events.py
@@ -18,6 +18,7 @@ from products.alerts_platform.backend.facade.contracts import (
AlertEventKind,
AnnouncedTransition,
EvaluationAnnouncement,
+ IncidentAction,
LegacyEventEmission,
MessageImage,
SourceKind,
@@ -75,6 +76,7 @@ class TestInsightLegacyEvents(APIBaseTest):
*,
sends_message: bool = True,
detail: dict[str, Any] | None = None,
+ incident_action: IncidentAction | None = None,
) -> LegacyEventEmission:
transition = AnnouncedTransition(
grouping_key="",
@@ -89,7 +91,7 @@ class TestInsightLegacyEvents(APIBaseTest):
evaluation_key="slot:1",
source_detail=detail or {},
)
- return LegacyEventEmission(transition=transition, sends_message=sends_message, incident_action=None)
+ return LegacyEventEmission(transition=transition, sends_message=sends_message, incident_action=incident_action)
@parameterized.expand(
[
@@ -168,6 +170,41 @@ class TestInsightLegacyEvents(APIBaseTest):
) -> None:
assert self._emit(self._emission(kind, sends_message=sends_message), alert_id=alert_id) == []
+ @parameterized.expand(
+ [
+ (
+ "an_opened_firing",
+ AlertEventKind.FIRING,
+ True,
+ IncidentAction.TRIGGER,
+ ["$insight_alert_firing", "$insight_alert_incident_opened"],
+ ),
+ (
+ "a_silent_resolve",
+ AlertEventKind.CHECK,
+ False,
+ IncidentAction.RESOLVE,
+ ["$insight_alert_incident_closed"],
+ ),
+ ]
+ )
+ def test_an_incident_edge_emits_its_incident_event_beside_any_message(
+ self,
+ _name: str,
+ kind: AlertEventKind,
+ sends_message: bool,
+ action: IncidentAction,
+ expected: list[str],
+ ) -> None:
+ events = self._emit(
+ self._emission(
+ kind, sends_message=sends_message, detail={"breaches": ["above 100"]}, incident_action=action
+ )
+ )
+
+ assert [event.event_name for event in events] == expected
+ assert all(event.properties["alert_id"] == str(self.alert.id) for event in events)
+
class TestInsightChartImage(SimpleTestCase):
@parameterized.expand(
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index f1b1c8958059..507c40828edb 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -71,12 +71,14 @@ from products.alerts_platform.backend.facade.contracts import (
AlertEventKind,
Grouping,
GroupingMode,
+ IncidentAction,
PlatformAlertOutcome,
PlatformConfigurationSnapshot,
SourceEvaluationInputs,
SourceKind,
)
from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
+from products.cdp.backend.models.hog_functions.hog_function import HogFunction
_MODULE = "products.alerts.backend.platform_source_cycle"
# A Wednesday, so no weekend rule applies.
@@ -502,18 +504,44 @@ class TestPlatformInsightEvaluation(APIBaseTest):
@parameterized.expand(
[
- ("allowlisted", True, False, False),
- ("not_allowlisted", False, False, False),
- ("allowlisted_resolve", True, True, False),
- ("platform_owned", False, False, True),
+ # name, allowlisted, was_firing, owned, paged, breaks, (incident_actions, sends_messages) or no delivery
+ ("allowlisted", True, False, False, False, False, ({}, True)),
+ ("not_allowlisted", False, False, False, False, False, None),
+ ("allowlisted_resolve", True, True, False, False, False, None),
+ ("platform_owned", False, False, True, False, False, ({}, True)),
+ ("paged_firing", False, False, True, True, False, ({"": IncidentAction.TRIGGER}, True)),
+ ("paged_resolve", False, True, True, True, False, ({"": IncidentAction.RESOLVE}, False)),
+ ("paged_firing_that_breaks", False, True, True, True, True, ({}, True)),
+ ("paged_but_not_allowlisted", False, False, False, True, False, None),
]
)
- def test_only_an_allowlisted_or_platform_owned_firing_or_failure_asks_for_a_delivery(
- self, _name: str, allowlisted: bool, was_firing: bool, owned: bool
+ def test_only_an_allowlisted_or_platform_owned_firing_failure_or_incident_edge_asks_for_a_delivery(
+ self,
+ _name: str,
+ allowlisted: bool,
+ was_firing: bool,
+ owned: bool,
+ paged: bool,
+ breaks: bool,
+ delivered: tuple[dict[str, IncidentAction], bool] | None,
) -> None:
alert = self._alert()
configuration = self._copy(alert, **({"owner": PlatformAlertOwner.PLATFORM} if owned else {}))
allowlist = frozenset({str(alert.id)}) if allowlisted else frozenset()
+ if paged:
+ HogFunction.objects.create(
+ team=self.team,
+ type="internal_destination",
+ name="PagerDuty",
+ template_id="template-pagerduty",
+ hog="return event",
+ enabled=True,
+ filters={
+ "source": "internal-events",
+ "events": [{"id": "$insight_alert_incident_closed", "type": "events"}],
+ "properties": [{"key": "alert_id", "value": str(alert.id), "operator": "exact", "type": "event"}],
+ },
+ )
if was_firing:
fired, _ = self._evaluate(configuration, result=AlertEvaluationResult(value=150.0, breaches=["above 100"]))
assert fired is not None
@@ -524,26 +552,35 @@ class TestPlatformInsightEvaluation(APIBaseTest):
with patch(f"{_MODULE}.LIVE_DELIVERY_INSIGHT_ALERT_IDS", allowlist):
self._evaluate(
configuration,
- result=AlertEvaluationResult(value=50.0, breaches=[] if was_firing else ["above 100"]),
+ result=AlertExtractionError("bad query shape")
+ if breaks
+ else AlertEvaluationResult(value=50.0, breaches=[] if was_firing else ["above 100"]),
)
- expected = [
- (
- str(configuration.id),
- str(alert.id),
- {"firing": "$insight_alert_firing", "errored": INSIGHT_ALERT_ERRORED_EVENT_ID},
- owned,
- )
- ]
+ expected = (
+ [
+ (
+ str(configuration.id),
+ str(alert.id),
+ {"firing": "$insight_alert_firing", "errored": INSIGHT_ALERT_ERRORED_EVENT_ID},
+ owned,
+ *delivered,
+ )
+ ]
+ if delivered is not None
+ else []
+ )
assert [
(
delivery.configuration_id,
delivery.destination_alert_id,
delivery.event_ids_by_kind,
delivery.owned_by_platform,
+ delivery.incident_actions,
+ delivery.sends_messages,
)
for delivery in self.deliveries
- ] == (expected if (allowlisted or owned) and not was_firing else [])
+ ] == expected
# A native message links to the alert on its insight, where the insight stack's messages link.
assert all(
delivery.source_alert_url == insight_alert_url(self.team.id, alert.insight.short_id, str(alert.id))
diff --git a/products/alerts/backend/tests/api/test_alert.py b/products/alerts/backend/tests/api/test_alert.py
index fdc74e6ecfe5..604202839017 100644
--- a/products/alerts/backend/tests/api/test_alert.py
+++ b/products/alerts/backend/tests/api/test_alert.py
@@ -27,7 +27,7 @@ from posthog.models.team import Team
from posthog.models.utils import generate_random_token_personal, hash_key_value
from products.alerts.backend.facade.api import LLMDetectorUnavailableError
-from products.alerts.backend.facade.contracts import INSIGHT_ALERT_EVENT_IDS
+from products.alerts.backend.facade.contracts import INSIGHT_ALERT_EVENT_IDS, INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION
from products.alerts.backend.facade.destinations import MAX_DESTINATIONS_PER_ALERT, count_active_alert_destinations
from products.alerts.backend.judge.verdict import LLMDetectionVerdict
from products.alerts.backend.logic.insight_alert_destinations import SLACK_TEMPLATE_ID
@@ -2919,6 +2919,34 @@ class TestAlertDestinations(APIBaseTest):
assert response.status_code == status.HTTP_204_NO_CONTENT, response.content
assert self._active_destination_count() == 0
+ def test_an_incident_destination_is_removed_through_the_insight_api(self) -> None:
+ hog_function_ids = [
+ str(
+ HogFunction.objects.create(
+ team=self.team,
+ type="internal_destination",
+ name="PagerDuty",
+ template_id="template-pagerduty",
+ hog="return event",
+ enabled=True,
+ inputs={"routing_key": {"value": "a" * 32}},
+ filters={
+ "source": "internal-events",
+ "events": [{"id": event_id, "type": "events"}],
+ "properties": [
+ {"key": "alert_id", "value": str(self.alert.id), "operator": "exact", "type": "event"}
+ ],
+ },
+ ).id
+ )
+ for event_id in INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION.values()
+ ]
+
+ response = self.client.post(f"{self.url}delete/", {"hog_function_ids": hog_function_ids}, format="json")
+
+ assert response.status_code == status.HTTP_204_NO_CONTENT, response.content
+ assert not HogFunction.objects.filter(id__in=hog_function_ids, deleted=False).exists()
+
def test_an_alert_stops_taking_destinations_at_the_cap(self) -> None:
for index in range(MAX_DESTINATIONS_PER_ALERT):
response = self.client.post(self.url, self._slack_payload(slack_channel_id=f"C{index}"), format="json")