Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): keep subscriber email and in-app native on the HogFunction path (#120001)

Jimmy Zhub968574efa69
diff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.py
index 745782b170b2..b0f3b1c55b3c 100644
--- a/products/alerts_platform/backend/delivery/evaluation.py
+++ b/products/alerts_platform/backend/delivery/evaluation.py
@@ -7,6 +7,7 @@ is what decides which destinations there are and which transport each one takes.
import json
from collections.abc import Callable
from dataclasses import replace
+from datetime import UTC, datetime
from functools import cache
from typing import Any, Final
@@ -27,7 +28,7 @@ from products.alerts_platform.backend.delivery.slack import SlackTransport
from products.alerts_platform.backend.delivery.team_flags import team_flag_enabled
from products.alerts_platform.backend.delivery.teams import TeamsTransport
from products.alerts_platform.backend.delivery.telemetry import record_delivery
-from products.alerts_platform.backend.delivery.thread_store import DatabaseThreadStore, ThreadBusy
+from products.alerts_platform.backend.delivery.thread_store import DatabaseThreadStore, ThreadBusy, ThreadKey
from products.alerts_platform.backend.delivery.transport import DeliveryError, DeliveryTransport
from products.alerts_platform.backend.delivery.webhook import WebhookTransport
from products.alerts_platform.backend.facade.contracts import (
@@ -45,6 +46,11 @@ from products.alerts_platform.backend.logic.platform_alert_events import announc
logger = structlog.get_logger(__name__)
+# Emitted events have no conversation, so each delivery part keeps one row in the thread store
+# under this provider and a fixed episode. The row only records which evaluations already emitted.
+_EMISSION_PROVIDER: Final = "legacy_events"
+_EMISSION_EPISODE: Final = datetime(1970, 1, 1, tzinfo=UTC)
+
# Whether the platform may contact a team's destinations for an alert its source still owns. Only
# the platform reads it: the source keeps delivering regardless, so a team with this on receives
# the legacy message and the native one. An alert the platform owns never reads it, because the
@@ -73,8 +79,9 @@ class DeliveryOutcome:
live: bool
sent: int
skipped_without_transport: int
- # Transitions handed to the source to emit its own events, on the HogFunction path.
- emitted: int = 0
+ # The broker took the events, which does not mean a HogFunction ran.
+ emitted_transitions: int = 0
+ emitted_events: int = 0
def destinations_are_live(team: Team) -> bool:
@@ -204,7 +211,7 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
)
if announced is None:
return DeliveryOutcome(live=True, sent=0, skipped_without_transport=0)
- announced, emissions = _split_by_path(request, announced)
+ native_announced, emissions = _split_by_path(request, announced)
emitter = delivery_paths.emitter_for(request.source) if emissions else None
notice_keys: dict[str, str] = {}
if request.parts > 1 and request.overflowed:
@@ -234,8 +241,14 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
message_event_ids = set(request.event_ids_by_kind.values())
# Destinations already told about the overflow, so each hears it once, on its first message.
noticed: set[str] = set()
- for event_id, transitions in _by_subscription(request, announced).items():
+ native_by_event = _by_subscription(request, native_announced)
+ for event_id, all_transitions in _by_subscription(request, announced).items():
for subscribed_destination in _destinations(request, event_id):
+ transitions = (
+ native_by_event.get(event_id, ()) if subscribed_destination.receives_emitted_events else all_transitions
+ )
+ if not transitions:
+ continue
target = subscribed_destination.data
overflowed = 0
destination = json.dumps(target, sort_keys=True)
@@ -292,18 +305,39 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
else:
sent += 1
# Emitted last and only without a held thread, because the workflow runs a held delivery again
- # in full, and a second emission sends every HogFunction message a second time.
+ # in full. A native send that fails also reruns the whole delivery, so the store records the
+ # emission, and a rerun does not send every HogFunction message a second time.
+ emitted: tuple[LegacyEventEmission, ...] = ()
if emitter is not None and not busy:
try:
- emitter(request=request, announcement=announced, emissions=emissions)
- except Exception as error:
- logger.warning(
- "alerts_platform_legacy_event_emission_failed",
- team_id=request.team_id,
- configuration_id=request.configuration_id,
- exc_info=True,
+ emission = thread_store.claim(
+ ThreadKey(
+ configuration_id=request.configuration_id,
+ grouping_key="",
+ provider=_EMISSION_PROVIDER,
+ channel_target=str(request.part),
+ episode_started_at=_EMISSION_EPISODE,
+ ),
+ request.evaluation_key,
)
- failures.append(f"legacy events: {type(error).__name__}")
+ except ThreadBusy as error:
+ busy.append(str(error))
+ emission = None
+ if emission is not None:
+ try:
+ emitter(request=request, announcement=announced, emissions=emissions)
+ except Exception as error:
+ thread_store.release(emission)
+ logger.warning(
+ "alerts_platform_legacy_event_emission_failed",
+ team_id=request.team_id,
+ configuration_id=request.configuration_id,
+ exc_info=True,
+ )
+ failures.append(f"legacy events: {type(error).__name__}")
+ else:
+ thread_store.delivered(emission, None)
+ emitted = emissions
# A held thread wins over a failure. The workflow waits a held thread out and then runs the
# whole delivery again, which retries the failed destinations too. A failure raised instead
# spends the activity's retries inside the claim's TTL, and the held message is lost.
@@ -312,13 +346,19 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
raise ThreadBusy("; ".join([*busy, *failures]))
if failures:
raise DeliveryError(f"{len(failures)} destination(s) failed: " + "; ".join(failures)) from first_refusal
- return DeliveryOutcome(live=True, sent=sent, skipped_without_transport=skipped, emitted=len(emissions))
+ return DeliveryOutcome(
+ live=True,
+ sent=sent,
+ skipped_without_transport=skipped,
+ emitted_transitions=len(emitted),
+ emitted_events=sum(emission.event_count for emission in emitted),
+ )
def _split_by_path(
request: AlertDeliveryRequest, announced: EvaluationAnnouncement
) -> tuple[EvaluationAnnouncement, tuple[LegacyEventEmission, ...]]:
- """The transitions to deliver natively, and the ones the source emits for its HogFunctions.
+ """The transitions HogFunction destinations receive natively, and the ones the source emits for them.
Only an alert the platform owns can take the HogFunction path. While the source owns it, its
own stack emits those events, and emitting them here too would send every message twice.
@@ -431,6 +471,8 @@ class _SubscribedDestination:
# The HogFunction that subscribed this destination to the event, which its sends are recorded under.
id: str | None
data: AlertDestinationData
+ # Whether the destination hears the events the source emits for its HogFunctions.
+ receives_emitted_events: bool
def _destinations(request: AlertDeliveryRequest, event_id: str) -> list[_SubscribedDestination]:
@@ -445,7 +487,11 @@ def _destinations(request: AlertDeliveryRequest, event_id: str) -> list[_Subscri
allowed_event_ids=[event_id],
)
return [
- _SubscribedDestination(id=str(min(group.hog_function_ids)) if group.hog_function_ids else None, data=group.data)
+ _SubscribedDestination(
+ id=str(min(group.hog_function_ids)) if group.hog_function_ids else None,
+ data=group.data,
+ receives_emitted_events=group.receives_emitted_events,
+ )
for group in groups
if group.fully_enabled
]
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index 52238a5949ae..175e2fa84ca6 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -1101,12 +1101,18 @@ class AlertDestinationConfig:
@frozen
class AlertDestinationGroup:
- """The HogFunctions that together make up one destination a person configured."""
+ """One destination an alert delivers to: the HogFunctions a person configured, or a destination
+ with none, such as an insight alert's subscriber email."""
hog_function_ids: tuple[UUID, ...]
data: AlertDestinationData
fully_enabled: bool
+ @property
+ def receives_emitted_events(self) -> bool:
+ """Whether the source's emitted internal events reach it. Only a HogFunction listens to them."""
+ return bool(self.hog_function_ids)
+
@frozen
class OwnedAlertDestination:
@@ -1228,6 +1234,10 @@ class LegacyEventEmission:
# The source emits the incident event for this action too, when it is set.
incident_action: IncidentAction | None
+ @property
+ def event_count(self) -> int:
+ return int(self.sends_message) + int(self.incident_action is not None)
+
class LegacyEventEmitter(Protocol):
"""Emits the internal events a source's own stack emitted, for transitions on the HogFunction path.
diff --git a/products/alerts_platform/backend/temporal/workflows.py b/products/alerts_platform/backend/temporal/workflows.py
index 516a978c0a04..46ca4fe28e18 100644
--- a/products/alerts_platform/backend/temporal/workflows.py
+++ b/products/alerts_platform/backend/temporal/workflows.py
@@ -176,7 +176,8 @@ async def alerts_platform_deliver_preview_activity(request: AlertDeliveryRequest
evaluation_key=request.evaluation_key,
sent=outcome.sent,
skipped_without_transport=outcome.skipped_without_transport,
- emitted=outcome.emitted,
+ emitted_transitions=outcome.emitted_transitions,
+ emitted_events=outcome.emitted_events,
)
diff --git a/products/alerts_platform/backend/tests/test_delivery_evaluation.py b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
index 2b0deb25f4e2..251a7e5f816b 100644
--- a/products/alerts_platform/backend/tests/test_delivery_evaluation.py
+++ b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
@@ -1,3 +1,4 @@
+from contextlib import ExitStack
from dataclasses import replace
from datetime import UTC, datetime
from typing import Any, cast
@@ -54,6 +55,16 @@ SECOND_WEBHOOK = cast(
TEAMS = cast(AlertDestinationData, {"type": DestinationType.TEAMS, "webhook_url": "https://example.com/teams"})
PAGERDUTY = cast(AlertDestinationData, {"type": DestinationType.PAGERDUTY, "pagerduty_routing_key": "R-1"})
UNSUPPORTED = cast(AlertDestinationData, {"type": "opsgenie"})
+HOG_FUNCTION_SLACK = AlertDestinationGroup(
+ hog_function_ids=(UUID("01900000-0000-0000-0000-0000000000a1"),), data=SLACK, fully_enabled=True
+)
+SUBSCRIBER = AlertDestinationGroup(
+ hog_function_ids=(),
+ data=cast(
+ AlertDestinationData, {"type": DestinationType.SLACK, "slack_workspace_id": 1, "slack_channel_id": "C-2"}
+ ),
+ fully_enabled=True,
+)
class RecordingTransport:
@@ -631,19 +642,26 @@ class TestDeliveryPath(APIBaseTest):
owned_by_platform: bool = True,
incident_actions: dict[str, IncidentAction] | None = None,
emitter: Any = None,
+ groups: list[AlertDestinationGroup] | None = None,
+ transport: type = RecordingTransport,
+ real_thread_store: bool = False,
) -> Any:
- with (
- patch(
- "products.alerts_platform.backend.delivery.team_flags.posthoganalytics.feature_enabled",
- return_value=True,
- ),
- patch(f"{_MODULE}.announcement", return_value=announced),
- patch(f"{_MODULE}.list_alert_destination_groups", return_value=[_group(SLACK)]),
- patch(f"{_MODULE}.DatabaseThreadStore"),
- patch.dict(f"{_MODULE}._TRANSPORTS", {DestinationType.SLACK: RecordingTransport}),
- patch.dict(delivery_paths._emitters, {SourceKind.LOGS: emitter or self._emit}),
- patch.object(delivery_paths, "_chooser", chooser),
- ):
+ with ExitStack() as stack:
+ stack.enter_context(
+ patch(
+ "products.alerts_platform.backend.delivery.team_flags.posthoganalytics.feature_enabled",
+ return_value=True,
+ )
+ )
+ stack.enter_context(patch(f"{_MODULE}.announcement", return_value=announced))
+ stack.enter_context(
+ patch(f"{_MODULE}.list_alert_destination_groups", return_value=groups or [HOG_FUNCTION_SLACK])
+ )
+ if not real_thread_store:
+ stack.enter_context(patch(f"{_MODULE}.DatabaseThreadStore"))
+ stack.enter_context(patch.dict(f"{_MODULE}._TRANSPORTS", {DestinationType.SLACK: transport}))
+ stack.enter_context(patch.dict(delivery_paths._emitters, {SourceKind.LOGS: emitter or self._emit}))
+ stack.enter_context(patch.object(delivery_paths, "_chooser", chooser))
return deliver_evaluation(
_request(
self.team.id,
@@ -661,32 +679,39 @@ class TestDeliveryPath(APIBaseTest):
def _fails(**_kwargs: Any) -> DeliveryPath:
raise RuntimeError("classifier down")
+ # The subscriber group stands in for email and in-app, which are no HogFunction and never see an emitted event.
@parameterized.expand(
[
- ("stock_destinations_send_natively", True, DeliveryPath.NATIVE, (0, 1)),
- ("customized_destinations_emit_for_their_hog_functions", True, DeliveryPath.HOG_FUNCTION, (1, 0)),
+ ("stock_destinations_send_natively", True, DeliveryPath.NATIVE, 0, ["C-1", "C-2"]),
+ ("customized_destinations_emit_for_their_hog_functions", True, DeliveryPath.HOG_FUNCTION, 1, ["C-2"]),
# The source's own stack still emits these events, so emitting them here would send twice.
- ("an_alert_the_source_owns_never_emits", False, DeliveryPath.HOG_FUNCTION, (0, 1)),
- ("a_chooser_that_fails_emits", True, None, (1, 0)),
+ ("an_alert_the_source_owns_never_emits", False, DeliveryPath.HOG_FUNCTION, 0, ["C-1", "C-2"]),
+ ("a_chooser_that_fails_emits", True, None, 1, ["C-2"]),
]
)
def test_an_alert_delivers_through_the_path_its_destinations_allow(
- self, _name: str, owned_by_platform: bool, path: DeliveryPath | None, expected: tuple[int, int]
+ self, _name: str, owned_by_platform: bool, path: DeliveryPath | None, emitted: int, native_channels: list[str]
) -> None:
chooser = self._chooses(path) if path else self._fails
outcome = self._run(
- _announcement(_transition(AlertEventKind.FIRING)), chooser, owned_by_platform=owned_by_platform
+ _announcement(_transition(AlertEventKind.FIRING)),
+ chooser,
+ owned_by_platform=owned_by_platform,
+ groups=[HOG_FUNCTION_SLACK, SUBSCRIBER],
)
- assert (len(self.emitted), len(RecordingTransport.sends)) == expected
- assert outcome.emitted == expected[0]
+ assert len(self.emitted) == emitted
+ assert sorted(channel for channel, _ in RecordingTransport.sends) == native_channels
+ assert (outcome.emitted_transitions, outcome.emitted_events) == (emitted, emitted)
def test_a_firing_and_its_resolve_take_the_path_the_firing_took(self) -> None:
- self._run(
+ fired = self._run(
_announcement(_transition(AlertEventKind.FIRING)),
self._chooses(DeliveryPath.HOG_FUNCTION),
incident_actions={"": IncidentAction.TRIGGER},
)
+ # One transition, emitted as its message event and its incident event.
+ assert (fired.emitted_transitions, fired.emitted_events) == (1, 2)
# Every destination became stock mid-firing. The resolve still closes what the HogFunction opened.
self._run(
_announcement(_transition(AlertEventKind.RESOLVED)),
@@ -717,3 +742,27 @@ class TestDeliveryPath(APIBaseTest):
self._chooses(DeliveryPath.HOG_FUNCTION),
emitter=refuse,
)
+
+ def test_a_rerun_after_a_refused_subscriber_send_emits_once(self) -> None:
+ for _ in range(2):
+ with pytest.raises(DeliveryError):
+ self._run(
+ _announcement(_transition(AlertEventKind.FIRING)),
+ self._chooses(DeliveryPath.HOG_FUNCTION),
+ groups=[HOG_FUNCTION_SLACK, SUBSCRIBER],
+ transport=RefusingTransport,
+ real_thread_store=True,
+ )
+
+ assert len(self.emitted) == 1
+
+ def test_a_held_subscriber_thread_holds_the_emission_for_the_rerun(self) -> None:
+ with pytest.raises(ThreadBusy):
+ self._run(
+ _announcement(_transition(AlertEventKind.FIRING)),
+ self._chooses(DeliveryPath.HOG_FUNCTION),
+ groups=[HOG_FUNCTION_SLACK, SUBSCRIBER],
+ transport=HeldTransport,
+ )
+
+ assert self.emitted == []