Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): emit $alert_transition for each announced alert transition (#119944)

Jimmy Zhued0a80b0d635
diff --git a/products/alerts/backend/apps.py b/products/alerts/backend/apps.py
index 3a313c3cd976..e87a14555465 100644
--- a/products/alerts/backend/apps.py
+++ b/products/alerts/backend/apps.py
@@ -17,12 +17,15 @@ class AlertsConfig(AppConfig):
# The platform cannot import this product, so it reaches this product's destinations here.
from products.alerts.backend.facade.destinations import ( # noqa: PLC0415
choose_delivery_path,
+ emit_alert_transition_events,
list_delivery_destination_groups,
)
from products.alerts_platform.backend.facade.delivery import ( # noqa: PLC0415
register_delivery_path_chooser,
register_destination_resolver,
+ register_transition_event_producer,
)
register_destination_resolver(list_delivery_destination_groups)
register_delivery_path_chooser(choose_delivery_path)
+ register_transition_event_producer(emit_alert_transition_events)
diff --git a/products/alerts/backend/facade/destinations.py b/products/alerts/backend/facade/destinations.py
index 5ec928cf8ce6..2c914e60fc2f 100644
--- a/products/alerts/backend/facade/destinations.py
+++ b/products/alerts/backend/facade/destinations.py
@@ -16,6 +16,7 @@ from uuid import UUID
from posthog.kafka_client.client import ProduceResult
from products.alerts_platform.backend.facade.contracts import (
+ ALERT_TRANSITION_EVENT,
ActiveAlertDestination,
AlertDelivery,
AlertDestinationConfig,
@@ -26,6 +27,7 @@ from products.alerts_platform.backend.facade.contracts import (
EventKindSpec,
OwnedAlertDestination,
SourceKind,
+ TransitionEvent,
)
from ..logic import (
@@ -47,6 +49,9 @@ from .contracts import (
ALERT_NOTIFICATION_FLUSH_TIMEOUT_SECONDS: Final = 10.0
+# Below the emitting activity's own timeout, which a flush must not use up.
+TRANSITION_EVENT_FLUSH_TIMEOUT_SECONDS: Final = 5.0
+
# Each destination is another message every time the alert fires, so a caller in a loop is capped.
MAX_DESTINATIONS_PER_ALERT: Final = 5
@@ -240,6 +245,24 @@ def emit_alert_internal_events(
)
+def emit_alert_transition_events(*, team_id: int, configuration_id: str, events: Sequence[TransitionEvent]) -> None:
+ """The platform's `$alert_transition` events. Raises `AlertInternalEventsNotDelivered` when the broker took not all of them."""
+ destinations.emit_alert_internal_events(
+ team_id=team_id,
+ alert_id=configuration_id,
+ events=[
+ AlertInternalEvent(
+ event_name=ALERT_TRANSITION_EVENT,
+ properties=event.properties,
+ timestamp=event.timestamp,
+ uuid=event.uuid,
+ )
+ for event in events
+ ],
+ flush_timeout_seconds=TRANSITION_EVENT_FLUSH_TIMEOUT_SECONDS,
+ )
+
+
def list_delivery_destination_groups(
*, team_id: int, alert_id: str, allowed_event_ids: Collection[str]
) -> list[AlertDestinationGroup]:
diff --git a/products/alerts/backend/logic/destinations.py b/products/alerts/backend/logic/destinations.py
index e9c27296b66a..e7bd34bc8626 100644
--- a/products/alerts/backend/logic/destinations.py
+++ b/products/alerts/backend/logic/destinations.py
@@ -447,7 +447,9 @@ def produce_alert_internal_event(
properties: dict[str, Any],
timestamp: datetime | None = None,
uuid: str | None = None,
+ alert_id: str | None = None,
) -> ProduceResult | None:
+ """`alert_id` names the alert in a failure report, for an event that carries it under another property."""
try:
return produce_internal_event(
team_id=team_id,
@@ -461,7 +463,7 @@ def produce_alert_internal_event(
)
except Exception as error:
context = {
- "alert_id": properties.get("alert_id"),
+ "alert_id": alert_id or properties.get("alert_id"),
"event_name": event_name,
"feature": "alerts",
"team_id": team_id,
@@ -517,6 +519,7 @@ def emit_alert_internal_events(
properties=event.properties,
timestamp=event.timestamp,
uuid=event.uuid,
+ alert_id=alert_id,
),
)
for event in events
diff --git a/products/alerts/backend/test/test_destinations.py b/products/alerts/backend/test/test_destinations.py
index d59aad3693e3..15fc1732768f 100644
--- a/products/alerts/backend/test/test_destinations.py
+++ b/products/alerts/backend/test/test_destinations.py
@@ -708,20 +708,27 @@ class TestAlertInternalEventDelivery(APIBaseTest):
class TestProduceAlertInternalEvent(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("id_in_properties", {"alert_id": "alert-1"}, None, "alert-1"),
+ ("id_passed_for_an_event_without_it", {"alert_configuration_id": "cfg-1"}, "cfg-1", "cfg-1"),
+ ]
+ )
@patch("products.alerts.backend.logic.destinations.capture_exception")
@patch(
"products.alerts.backend.logic.destinations.produce_internal_event",
side_effect=RuntimeError("broker down"),
)
- def test_a_producer_failure_is_swallowed_so_a_batch_caller_still_saves(
- self, _produce_internal_event, capture_exception
+ def test_a_producer_failure_is_swallowed_and_names_its_alert(
+ self, _name, properties, alert_id, reported_alert_id, _produce_internal_event, capture_exception
) -> None:
result = produce_alert_internal_event(
- team_id=1, event_name="$logs_alert_firing", properties={"alert_id": "alert-1"}
+ team_id=1, event_name="$logs_alert_firing", properties=properties, alert_id=alert_id
)
assert result is None
capture_exception.assert_called_once()
+ assert capture_exception.call_args.args[1]["alert_id"] == reported_alert_id
class TestFlushAlertInternalEvents(SimpleTestCase):
diff --git a/products/alerts/backend/test/test_transition_event_reservation.py b/products/alerts/backend/test/test_transition_event_reservation.py
new file mode 100644
index 000000000000..b050f9c779d2
--- /dev/null
+++ b/products/alerts/backend/test/test_transition_event_reservation.py
@@ -0,0 +1,9 @@
+from django.test import SimpleTestCase
+
+from products.alerts_platform.backend.facade.contracts import ALERT_TRANSITION_EVENT
+from products.cdp.backend.internal_events import is_reserved_internal_event
+
+
+class TestTransitionEventReservation(SimpleTestCase):
+ def test_the_event_the_platform_emits_is_closed_to_user_created_destinations(self) -> None:
+ assert is_reserved_internal_event(ALERT_TRANSITION_EVENT)
diff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.py
index d3124190b9fc..5f941f1b90b6 100644
--- a/products/alerts_platform/backend/delivery/evaluation.py
+++ b/products/alerts_platform/backend/delivery/evaluation.py
@@ -15,7 +15,7 @@ import structlog
from posthog.dataclasses import frozen
from posthog.models import Team
-from products.alerts_platform.backend.delivery import delivery_paths, episode_paths, legacy_webhook
+from products.alerts_platform.backend.delivery import delivery_paths, episode_paths, legacy_webhook, transition_events
from products.alerts_platform.backend.delivery.destinations import list_alert_destination_groups
from products.alerts_platform.backend.delivery.discord import DiscordTransport
from products.alerts_platform.backend.delivery.dispatch import deliver
@@ -146,6 +146,47 @@ class _WebhookTransports:
)
+def emit_transition_events(request: AlertDeliveryRequest) -> int:
+ """Emits one `$alert_transition` per transition the evaluation announced, and returns how many.
+
+ Only where native delivery would contact the team, so a workflow never hears about an alert
+ whose source still delivers on its own. Every transition goes out whatever path its destinations
+ take, because a workflow is a separate subscription, not a destination.
+ """
+ if not request.sends_messages:
+ return 0
+ team = Team.objects.filter(id=request.team_id).first()
+ if team is None or not transition_events_enabled(team):
+ return 0
+ if not (request.owned_by_platform or destinations_are_live(team)):
+ return 0
+ announced = announcement(
+ request.team_id,
+ request.configuration_id,
+ request.evaluation_key,
+ source=request.source,
+ part=request.part,
+ parts=request.parts,
+ )
+ if announced is None:
+ return 0
+ events = transition_events.transition_events(request, announced)
+ if events:
+ transition_events.producer()(team_id=request.team_id, configuration_id=request.configuration_id, events=events)
+ return len(events)
+
+
+def transition_events_enabled(team: Team) -> bool:
+ """Fail closed: a flag-eval blip leaves the team without the event, which is where it started."""
+ try:
+ return bool(team_flag_enabled(transition_events.TRANSITION_EVENT_FLAG, team))
+ except Exception:
+ logger.warning(
+ "alerts_platform_transition_event_flag_check_failed_defaulting_off", team_id=team.id, exc_info=True
+ )
+ return False
+
+
def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
"""Sends what one evaluation decided, to every destination configured for it."""
team = Team.objects.filter(id=request.team_id).first()
diff --git a/products/alerts_platform/backend/delivery/transition_events.py b/products/alerts_platform/backend/delivery/transition_events.py
new file mode 100644
index 000000000000..8c0c9e0f0e25
--- /dev/null
+++ b/products/alerts_platform/backend/delivery/transition_events.py
@@ -0,0 +1,110 @@
+"""The `$alert_transition` event: one per announced transition, for a workflow to subscribe to.
+
+The platform imports neither the product that produces internal events nor any source, so both
+plug in here when Django starts:
+
+- the product that owns alert destinations registers the producer;
+- a source registers its facts, which carry the property names its own alert events used.
+
+Imported from `ready()`, so it stays light: no models, no query layer, no transports.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Final
+from uuid import NAMESPACE_URL, uuid5
+
+from products.alerts_platform.backend.facade.contracts import (
+ AlertDeliveryRequest,
+ AlertEventKind,
+ AnnouncedTransition,
+ EvaluationAnnouncement,
+ SourceFacts,
+ SourceKind,
+ TransitionEvent,
+ TransitionEventProducer,
+)
+
+# Whether the platform emits `$alert_transition` for a team's alerts. Read only where delivery
+# already contacts the team's destinations, and a failed read emits nothing.
+TRANSITION_EVENT_FLAG: Final = "alert-platform-transition-event"
+
+_FAILURE_KINDS: Final = frozenset({AlertEventKind.ERRORED, AlertEventKind.BROKEN})
+
+_producer: TransitionEventProducer | None = None
+_source_facts: dict[SourceKind, SourceFacts] = {}
+
+
+def register_producer(producer: TransitionEventProducer) -> None:
+ global _producer
+ if _producer is not None and _producer is not producer:
+ raise RuntimeError("A different transition event producer is already registered.")
+ _producer = producer
+
+
+def register_source_facts(source: SourceKind, facts: SourceFacts) -> None:
+ registered = _source_facts.get(source)
+ if registered is not None and registered is not facts:
+ raise RuntimeError(f"Different source facts are already registered for {source} alerts.")
+ _source_facts[source] = facts
+
+
+def producer() -> TransitionEventProducer:
+ if _producer is None:
+ raise RuntimeError("No transition event producer is registered.")
+ return _producer
+
+
+def transition_events(request: AlertDeliveryRequest, announced: EvaluationAnnouncement) -> list[TransitionEvent]:
+ """One event per transition the evaluation announced.
+
+ A source with no registered facts gets an empty `source` block, so its events still carry
+ every platform field.
+ """
+ facts = _source_facts.get(request.source)
+ return [
+ TransitionEvent(
+ properties=_properties(
+ request,
+ announced,
+ transition,
+ facts(request=request, announcement=announced, transition=transition) if facts is not None else {},
+ ),
+ timestamp=transition.occurred_at,
+ uuid=_event_uuid(request, transition),
+ )
+ for transition in announced.transitions
+ ]
+
+
+def _properties(
+ request: AlertDeliveryRequest,
+ announced: EvaluationAnnouncement,
+ transition: AnnouncedTransition,
+ source_facts: dict[str, Any],
+) -> dict[str, Any]:
+ failed = transition.kind in _FAILURE_KINDS
+ return {
+ "alert_configuration_id": announced.configuration_id,
+ "alert_name": announced.alert_name,
+ "source_kind": announced.source.value,
+ "transition": transition.kind.value,
+ "occurred_at": transition.occurred_at.isoformat(),
+ "episode_started_at": transition.episode_started_at.isoformat() if transition.episode_started_at else None,
+ "value": transition.value,
+ "labels": transition.labels,
+ "grouping_key": transition.grouping_key,
+ "error_message": transition.error_message if failed else None,
+ "consecutive_failures": announced.consecutive_failures if failed else None,
+ "url": request.source_alert_url or "",
+ "source": source_facts,
+ }
+
+
+def _event_uuid(request: AlertDeliveryRequest, transition: AnnouncedTransition) -> str:
+ # The occurrence is in the seed too, so two transitions recorded without an evaluation key keep distinct ids.
+ seed = (
+ f"{request.team_id}|{request.configuration_id}|{request.evaluation_key}|{transition.grouping_key}|"
+ f"{transition.kind.value}|{transition.occurred_at.isoformat()}"
+ )
+ return str(uuid5(NAMESPACE_URL, seed))
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index a5343772f8ee..be1c3bfd5aac 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -1202,6 +1202,44 @@ class LegacyEventEmitter(Protocol):
) -> None: ...
+# The internal event a workflow subscribes to for an alert's transitions. One name for every source.
+ALERT_TRANSITION_EVENT: Final = "$alert_transition"
+
+
+@frozen
+class TransitionEvent:
+ """One `$alert_transition` event."""
+
+ properties: dict[str, Any]
+ timestamp: datetime
+ # Stable across attempts, so a receiver can drop an event a retry emits again.
+ uuid: str
+
+
+class TransitionEventProducer(Protocol):
+ """Puts `$alert_transition` events on the internal events topic.
+
+ Returns only once the broker has every event, and raises otherwise, so delivery can retry.
+ """
+
+ def __call__(self, *, team_id: int, configuration_id: str, events: Sequence[TransitionEvent]) -> None: ...
+
+
+class SourceFacts(Protocol):
+ """A source's facts about one transition, under the property names its own alert events used.
+
+ A HogFunction ported to a workflow renders these names, so they must match the legacy events.
+ """
+
+ def __call__(
+ self,
+ *,
+ request: AlertDeliveryRequest,
+ announcement: EvaluationAnnouncement,
+ transition: AnnouncedTransition,
+ ) -> dict[str, Any]: ...
+
+
# Comparison against a source's own stack.
#
# What a comparison reads, and what a source must supply for one.
diff --git a/products/alerts_platform/backend/facade/delivery.py b/products/alerts_platform/backend/facade/delivery.py
index 9c5cd92e6864..291016d7bd7a 100644
--- a/products/alerts_platform/backend/facade/delivery.py
+++ b/products/alerts_platform/backend/facade/delivery.py
@@ -10,6 +10,7 @@ from products.alerts_platform.backend.delivery import (
describers,
destinations,
legacy_webhook,
+ transition_events,
)
from products.alerts_platform.backend.facade.contracts import (
AlertDestinationData,
@@ -18,7 +19,9 @@ from products.alerts_platform.backend.facade.contracts import (
LegacyEventEmitter,
LegacyWebhookBody,
SourceDescriber,
+ SourceFacts,
SourceKind,
+ TransitionEventProducer,
)
if TYPE_CHECKING:
@@ -60,3 +63,13 @@ def register_legacy_event_emitter(source: SourceKind, emitter: LegacyEventEmitte
def native_sends_legacy_webhook_body(source: SourceKind, team: Team) -> bool | None:
"""Whether a stock webhook receives the body its HogFunction sent when delivered natively. None when unknown."""
return legacy_webhook.native_sends_legacy_body(source, team)
+
+
+def register_transition_event_producer(producer: TransitionEventProducer) -> None:
+ """The product that produces internal events puts the platform's `$alert_transition` events on the topic."""
+ transition_events.register_producer(producer)
+
+
+def register_source_facts(source: SourceKind, facts: SourceFacts) -> None:
+ """A source's facts ride on each `$alert_transition` under the property names its own alert events used."""
+ transition_events.register_source_facts(source, facts)
diff --git a/products/alerts_platform/backend/temporal/metrics.py b/products/alerts_platform/backend/temporal/metrics.py
index c6a05c1934e4..5f7fb5ddfe55 100644
--- a/products/alerts_platform/backend/temporal/metrics.py
+++ b/products/alerts_platform/backend/temporal/metrics.py
@@ -172,6 +172,13 @@ def increment_root_edit_failures(provider: str) -> None:
).add(1)
+def increment_transition_events_emitted(source: str, count: int) -> None:
+ get_metric_meter({"source": source}).create_counter(
+ "alerts_platform_transition_events_emitted_total",
+ "$alert_transition events put on the internal events topic",
+ ).add(count)
+
+
def increment_deliveries_previewed(source: str) -> None:
get_metric_meter({"source": source}).create_counter(
"alerts_platform_deliveries_previewed_total",
diff --git a/products/alerts_platform/backend/temporal/workflows.py b/products/alerts_platform/backend/temporal/workflows.py
index 89814b23fb7b..516a978c0a04 100644
--- a/products/alerts_platform/backend/temporal/workflows.py
+++ b/products/alerts_platform/backend/temporal/workflows.py
@@ -22,6 +22,7 @@ from posthog.temporal.common.logger import get_write_only_logger
from products.alerts_platform.backend.temporal.metrics import (
increment_deliveries_previewed,
increment_discovery_omitted,
+ increment_transition_events_emitted,
safe_record,
)
from products.alerts_platform.backend.temporal.outcomes import alerts_platform_record_outcomes_activity
@@ -36,7 +37,7 @@ with workflow.unsafe.imports_passed_through():
from posthog.sync import database_sync_to_async_pool
- from products.alerts_platform.backend.delivery.evaluation import deliver_evaluation
+ from products.alerts_platform.backend.delivery.evaluation import deliver_evaluation, emit_transition_events
from products.alerts_platform.backend.delivery.thread_store import PENDING_CLAIM_TTL, ThreadBusy
from products.alerts_platform.backend.facade.contracts import (
AlertDeliveryRequest,
@@ -80,11 +81,14 @@ DELIVERY_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=30)
# DELIVERY_SCHEDULE_TO_CLOSE, which is shorter than PENDING_CLAIM_TTL. A claim that a dead
# attempt left looks fresh to every one of those retries.
BUSY_RETRY_DELAY = dt.timedelta(seconds=20)
+# Delivery runs started before the transition events existed replay without them.
+TRANSITION_EVENTS_PATCH = "alerts-platform-transition-events"
# From the first busy result to the start of a later attempt. An attempt that starts this late
# finds every claim the first one saw already stale, so a thread still held is a newer send's.
BUSY_WAIT_LIMIT = PENDING_CLAIM_TTL + BUSY_RETRY_DELAY
-# The last attempt can start one attempt and one delay past the limit.
-DELIVERY_EXECUTION_TIMEOUT = 3 * DELIVERY_SCHEDULE_TO_CLOSE + BUSY_WAIT_LIMIT + BUSY_RETRY_DELAY
+# One schedule-to-close for the transition events and three for the send attempts. The last send
+# attempt can start one attempt and one delay past the limit.
+DELIVERY_EXECUTION_TIMEOUT = 4 * DELIVERY_SCHEDULE_TO_CLOSE + BUSY_WAIT_LIMIT + BUSY_RETRY_DELAY
@frozen
@@ -114,6 +118,24 @@ async def alerts_platform_deliver_activity() -> None:
pass
+async def alerts_platform_emit_transitions_activity(request: AlertDeliveryRequest) -> None:
+ """Emits the `$alert_transition` events of one evaluation, apart from native delivery.
+
+ Its own activity, so a held thread or a failing destination never delays or repeats them.
+ """
+ emitted = await database_sync_to_async_pool(emit_transition_events)(request)
+ if emitted:
+ await LOGGER.ainfo(
+ "alerts_platform_transition_events_emitted",
+ source=request.source.value,
+ configuration_id=request.configuration_id,
+ evaluation_key=request.evaluation_key,
+ emitted=emitted,
+ )
+ safe_record(increment_transition_events_emitted, request.source.value, emitted)
+
+
@activity.defn
async def alerts_platform_deliver_preview_activity(request: AlertDeliveryRequest) -> None:
"""Sends what one evaluation decided, once destinations are live for the team.
@@ -164,24 +186,41 @@ class AlertsPlatformDeliverPreviewWorkflow(PostHogWorkflow):
@workflow.run
async def run(self, inputs: AlertDeliveryRequest) -> None:
- """Delivers each part in turn, so a split evaluation's sends to one destination never overlap.
+ """Emits every part's transition events, then delivers each part in turn.
- A part that fails does not stop the parts after it. The first failure fails the run once
- every part has had its attempts.
+ Emission comes first for every part, so a held thread on one part never delays another
+ part's events. Parts deliver in turn, so a split evaluation's sends to one destination
+ never overlap. A failure does not stop the steps after it. The first failure fails the
+ run once every part has had its attempts, a failed send before a failed emission.
"""
- if inputs.parts <= 1:
- await self._deliver(inputs)
- return
- failures: list[ActivityError] = []
- for part in range(inputs.parts):
+ parts = [inputs] if inputs.parts <= 1 else [replace(inputs, part=part) for part in range(inputs.parts)]
+ emission_failures: list[ActivityError] = []
+ if workflow.patched(TRANSITION_EVENTS_PATCH):
+ for request in parts:
+ try:
+ await self._emit(request)
+ except ActivityError as error:
+ emission_failures.append(error)
+ send_failures: list[ActivityError] = []
+ for request in parts:
try:
- await self._deliver(replace(inputs, part=part))
+ await self._send(request)
except ActivityError as error:
- failures.append(error)
+ send_failures.append(error)
+ failures = [*send_failures, *emission_failures]
if failures:
raise failures[0]
- async def _deliver(self, request: AlertDeliveryRequest) -> None:
+ async def _emit(self, request: AlertDeliveryRequest) -> None:
+ await workflow.execute_activity(
+ alerts_platform_emit_transitions_activity,
+ request,
+ start_to_close_timeout=DELIVERY_START_TO_CLOSE,
+ schedule_to_close_timeout=DELIVERY_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+
+ async def _send(self, request: AlertDeliveryRequest) -> None:
busy_deadline: dt.datetime | None = None
while True:
started_at = workflow.now()
@@ -507,4 +546,5 @@ DELIVERY_WORKFLOWS: list[type[PostHogWorkflow]] = [AlertsPlatformDeliverWorkflow
DELIVERY_ACTIVITIES: list[Callable[..., object]] = [
alerts_platform_deliver_activity,
alerts_platform_deliver_preview_activity,
+ alerts_platform_emit_transitions_activity,
]
diff --git a/products/alerts_platform/backend/tests/test_delivery_evaluation.py b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
index f0f45ed84841..0986192da5fb 100644
--- a/products/alerts_platform/backend/tests/test_delivery_evaluation.py
+++ b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
@@ -8,9 +8,13 @@ from unittest.mock import patch
from parameterized import parameterized
-from products.alerts_platform.backend.delivery import delivery_paths, legacy_webhook
+from products.alerts_platform.backend.delivery import delivery_paths, legacy_webhook, transition_events
from products.alerts_platform.backend.delivery.destinations import list_alert_destination_groups
-from products.alerts_platform.backend.delivery.evaluation import LIVE_DELIVERY_FLAG, deliver_evaluation
+from products.alerts_platform.backend.delivery.evaluation import (
+ LIVE_DELIVERY_FLAG,
+ deliver_evaluation,
+ emit_transition_events,
+)
from products.alerts_platform.backend.delivery.legacy_webhook import LEGACY_WEBHOOK_BODY_FLAG
from products.alerts_platform.backend.delivery.message import AlertMessage
from products.alerts_platform.backend.delivery.thread_store import ThreadBusy
@@ -27,6 +31,7 @@ from products.alerts_platform.backend.facade.contracts import (
IncidentAction,
LegacyEventEmission,
SourceKind,
+ TransitionEvent,
)
from products.alerts_platform.backend.models import PlatformAlertConfiguration
@@ -476,16 +481,112 @@ class TestDeliverEvaluation(APIBaseTest):
# which would send grandfathered teams the Alertmanager body without any error.
assert legacy_webhook.body_for(SourceKind.LOGS) is not None
assert delivery_paths.emitter_for(SourceKind.LOGS) is not None
+ assert SourceKind.LOGS in transition_events._source_facts
def test_the_destination_owner_registers_its_lookup_at_startup(self) -> None:
# Every other case here patches the lookup, so none of them notices a missing registration.
assert delivery_paths._chooser is not None
+ assert transition_events._producer is not None
assert (
list_alert_destination_groups(team_id=self.team.id, alert_id="unknown", allowed_event_ids=[FIRING_EVENT])
== []
)
+class TestTransitionEvents(APIBaseTest):
+ def _emit(
+ self,
+ announced: EvaluationAnnouncement | None,
+ *,
+ transition_flag: bool = True,
+ live: bool = False,
+ owned_by_platform: bool = True,
+ sends_messages: bool = True,
+ facts: Any = None,
+ ) -> list[TransitionEvent]:
+ produced: list[TransitionEvent] = []
+
+ def produce(*, team_id: int, configuration_id: str, events: list[TransitionEvent]) -> None:
+ produced.extend(events)
+
+ def flag_enabled(flag: str, *args: Any, **kwargs: Any) -> bool:
+ return transition_flag if flag == transition_events.TRANSITION_EVENT_FLAG else live
+
+ with (
+ patch(
+ "products.alerts_platform.backend.delivery.team_flags.posthoganalytics.feature_enabled",
+ side_effect=flag_enabled,
+ ),
+ patch(f"{_MODULE}.announcement", return_value=announced),
+ patch.object(transition_events, "_producer", produce),
+ patch.dict(transition_events._source_facts, {SourceKind.LOGS: facts} if facts else {}, clear=True),
+ ):
+ emitted = emit_transition_events(
+ _request(self.team.id, owned_by_platform=owned_by_platform, sends_messages=sends_messages)
+ )
+ assert emitted == len(produced)
+ return produced
+
+ @parameterized.expand(
+ [
+ ("platform_owned", {}, 1),
+ ("flag_off", {"transition_flag": False}, 0),
+ ("source_still_delivers", {"owned_by_platform": False}, 0),
+ ("source_owned_on_live_delivery", {"owned_by_platform": False, "live": True}, 1),
+ ("incident_only_delivery", {"sends_messages": False}, 0),
+ ]
+ )
+ def test_an_alert_emits_only_where_the_platform_contacts_its_team(
+ self, _name: str, gates: dict[str, bool], expected: int
+ ) -> None:
+ assert len(self._emit(_announcement(_transition(AlertEventKind.FIRING)), **gates)) == expected
+
+ def test_each_transition_carries_the_platform_fields_and_the_source_facts(self) -> None:
+ firing = replace(_transition(AlertEventKind.FIRING), value=12.0, labels={"service": "api"})
+ errored = replace(_transition(AlertEventKind.ERRORED, episode_started_at=None), error_message="timeout")
+ announced = replace(_announcement(firing, errored), consecutive_failures=2)
+
+ def facts(*, transition: AnnouncedTransition, **_kwargs: Any) -> dict[str, Any]:
+ return {"alert_id": "legacy-1", "result_count": transition.value}
+
+ events = self._emit(announced, facts=facts)
+
+ assert [event.properties for event in events] == [
+ {
+ "alert_configuration_id": "cfg-1",
+ "alert_name": "API errors",
+ "source_kind": "logs",
+ "transition": "firing",
+ "occurred_at": FIRING.isoformat(),
+ "episode_started_at": FIRING.isoformat(),
+ "value": 12.0,
+ "labels": {"service": "api"},
+ "grouping_key": "",
+ "error_message": None,
+ "consecutive_failures": None,
+ "url": "",
+ "source": {"alert_id": "legacy-1", "result_count": 12.0},
+ },
+ {
+ "alert_configuration_id": "cfg-1",
+ "alert_name": "API errors",
+ "source_kind": "logs",
+ "transition": "errored",
+ "occurred_at": FIRING.isoformat(),
+ "episode_started_at": None,
+ "value": None,
+ "labels": {},
+ "grouping_key": "",
+ "error_message": "timeout",
+ "consecutive_failures": 2,
+ "url": "",
+ "source": {"alert_id": "legacy-1", "result_count": None},
+ },
+ ]
+ assert [event.uuid for event in events] == [event.uuid for event in self._emit(announced, facts=facts)]
+ assert events[0].uuid != events[1].uuid
+
+
class TestDeliveryPath(APIBaseTest):
def setUp(self) -> None:
super().setUp()
diff --git a/products/alerts_platform/backend/tests/test_temporal_workflows.py b/products/alerts_platform/backend/tests/test_temporal_workflows.py
index eda758f55949..635b52e81c48 100644
--- a/products/alerts_platform/backend/tests/test_temporal_workflows.py
+++ b/products/alerts_platform/backend/tests/test_temporal_workflows.py
@@ -81,6 +81,11 @@ class CloseAfterChildStartWorkflow:
await workflow.wait_condition(lambda: False)
[email protected](name="alerts_platform_emit_transitions_activity")
+async def emit_nothing(request: AlertDeliveryRequest) -> None:
+ pass
+
+
@pytest.fixture(scope="module")
def sdk_metrics() -> MetricBuffer:
return MetricBuffer(10000)
@@ -346,7 +351,7 @@ async def test_delivery_waits_out_a_held_thread_and_fails_when_it_never_frees(
environment.client,
task_queue=settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
workflows=DELIVERY_WORKFLOWS,
- activities=[held_thread],
+ activities=[held_thread, emit_nothing],
workflow_runner=UnsandboxedWorkflowRunner(),
):
handle = await environment.client.start_workflow(
@@ -374,14 +379,18 @@ async def test_delivery_waits_out_a_held_thread_and_fails_when_it_never_frees(
@pytest.mark.asyncio
-async def test_a_split_delivery_sends_every_part_in_turn_even_after_one_fails(
+async def test_a_split_delivery_emits_every_part_then_sends_every_part_in_turn_even_after_one_fails(
environment: WorkflowEnvironment,
) -> None:
- sent_parts: list[int] = []
+ steps: list[str] = []
+
+ @activity.defn(name="alerts_platform_emit_transitions_activity")
+ async def emit(request: AlertDeliveryRequest) -> None:
+ steps.append(f"emit {request.part}")
@activity.defn(name="alerts_platform_deliver_preview_activity")
async def deliver(request: AlertDeliveryRequest) -> None:
- sent_parts.append(request.part)
+ steps.append(f"send {request.part}")
if request.part == 0:
raise ApplicationError("channel refused", non_retryable=True)
@@ -389,7 +398,7 @@ async def test_a_split_delivery_sends_every_part_in_turn_even_after_one_fails(
environment.client,
task_queue=settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
workflows=DELIVERY_WORKFLOWS,
- activities=[deliver],
+ activities=[deliver, emit],
workflow_runner=UnsandboxedWorkflowRunner(),
):
handle = await environment.client.start_workflow(
@@ -411,7 +420,50 @@ async def test_a_split_delivery_sends_every_part_in_turn_even_after_one_fails(
await handle.result()
assert isinstance(failure.value.cause, ActivityError)
- assert sent_parts == [0, 1, 2]
+ assert steps == ["emit 0", "emit 1", "emit 2", "send 0", "send 1", "send 2"]
+
+
+async def test_a_failed_transition_emission_still_sends_the_messages_then_fails_the_run(
+ environment: WorkflowEnvironment,
+) -> None:
+ calls: list[str] = []
+
+ @activity.defn(name="alerts_platform_emit_transitions_activity")
+ async def emit(request: AlertDeliveryRequest) -> None:
+ calls.append("emit")
+ raise ApplicationError("broker refused", non_retryable=True)
+
+ @activity.defn(name="alerts_platform_deliver_preview_activity")
+ async def deliver(request: AlertDeliveryRequest) -> None:
+ calls.append("deliver")
+
+ async with Worker(
+ environment.client,
+ task_queue=settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
+ workflows=DELIVERY_WORKFLOWS,
+ activities=[emit, deliver],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await environment.client.start_workflow(
+ "alerts-platform-deliver-preview",
+ AlertDeliveryRequest(
+ source=SourceKind.LOGS,
+ team_id=1,
+ configuration_id="cfg-1",
+ evaluation_key="eval-1",
+ destination_alert_id="legacy-1",
+ event_ids_by_kind={},
+ ),
+ id=str(uuid.uuid4()),
+ task_queue=settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
+ execution_timeout=DELIVERY_EXECUTION_TIMEOUT,
+ )
+ with pytest.raises(WorkflowFailureError) as failure:
+ await handle.result()
+
+ assert isinstance(failure.value.cause, ActivityError)
+ assert calls == ["emit", "deliver"]
@pytest.mark.parametrize("timeout_type", [TimeoutType.START_TO_CLOSE, TimeoutType.SCHEDULE_TO_CLOSE])
diff --git a/products/cdp/backend/api/test/test_hog_function.py b/products/cdp/backend/api/test/test_hog_function.py
index 9f507fb30e72..8077939ecce4 100644
--- a/products/cdp/backend/api/test/test_hog_function.py
+++ b/products/cdp/backend/api/test/test_hog_function.py
@@ -218,15 +218,16 @@ class TestHogFunctionAPIWithoutAvailableFeature(ClickhouseTestMixin, APIBaseTest
self.assertEqual(response.json()["attr"], "filters")
self.assertIn("managed through the alert API", response.json()["detail"])
- def test_generic_api_cannot_subscribe_to_reserved_internal_events(self):
+ @parameterized.expand([("$workflow_step_resume",), ("$alert_transition",)])
+ def test_generic_api_cannot_subscribe_to_reserved_internal_events(self, event_id: str):
response = self.client.post(
f"/api/projects/{self.team.id}/hog_functions/",
data={
- "name": "Workflow result exfiltration",
+ "name": "Reserved event exfiltration",
"hog": "fetch('https://example.com');",
"type": "internal_destination",
"enabled": True,
- "filters": {"events": [{"id": "$workflow_step_resume", "type": "events"}]},
+ "filters": {"events": [{"id": event_id, "type": "events"}]},
},
)
diff --git a/products/cdp/backend/internal_events.py b/products/cdp/backend/internal_events.py
index b2add5ecd302..893b628c9709 100644
--- a/products/cdp/backend/internal_events.py
+++ b/products/cdp/backend/internal_events.py
@@ -25,10 +25,13 @@ _MANAGED_ALERT_EVENT = re.compile(MANAGED_ALERT_EVENT_PATTERN)
WORKFLOW_STEP_RESUME_EVENT = "$workflow_step_resume"
+# The alerts platform's transition event. Its properties carry the source product's alert data.
+ALERT_TRANSITION_EVENT = "$alert_transition"
+
# Internal events that only the product which emits them may consume. A user-created
# destination cannot subscribe to one, because its properties carry that product's data and
# forwarding them off-platform would bypass the read permission the product enforces on it.
-RESERVED_INTERNAL_EVENTS = frozenset({WORKFLOW_STEP_RESUME_EVENT})
+RESERVED_INTERNAL_EVENTS = frozenset({WORKFLOW_STEP_RESUME_EVENT, ALERT_TRANSITION_EVENT})
def is_reserved_internal_event(event_name: object) -> bool:
diff --git a/products/logs/backend/apps.py b/products/logs/backend/apps.py
index 6246504e20ca..13fd2994d36c 100644
--- a/products/logs/backend/apps.py
+++ b/products/logs/backend/apps.py
@@ -14,17 +14,21 @@ class LogsConfig(AppConfig):
# HogQL into django.setup() for every process type.
# - The alerts platform cannot import this product, so native alert messages get the logs
# wording, grandfathered webhooks the legacy logs body, alerts on the HogFunction path
- # their `$logs_alert_*` events, and the platform API the logs `source_config` schema,
- # through the functions registered below.
+ # their `$logs_alert_*` events, `$alert_transition` the logs source facts, and the platform
+ # API the logs `source_config` schema, through the functions registered below.
from products.alerts_platform.backend.facade.contracts import SourceKind # noqa: PLC0415
from products.alerts_platform.backend.facade.delivery import ( # noqa: PLC0415
register_legacy_event_emitter,
register_legacy_webhook_body,
register_source_describer,
+ register_source_facts,
)
from products.alerts_platform.backend.facade.source_configs import register_source_config # noqa: PLC0415
from products.logs.backend import activity_logging # noqa: F401, PLC0415
- from products.logs.backend.platform_alert_legacy_events import emit_logs_legacy_events # noqa: PLC0415
+ from products.logs.backend.platform_alert_legacy_events import ( # noqa: PLC0415
+ emit_logs_legacy_events,
+ logs_source_facts,
+ )
from products.logs.backend.platform_alert_wording import ( # noqa: PLC0415
describe_logs_transition,
legacy_logs_webhook_body,
@@ -34,4 +38,5 @@ class LogsConfig(AppConfig):
register_source_describer(SourceKind.LOGS, describe_logs_transition)
register_legacy_webhook_body(SourceKind.LOGS, legacy_logs_webhook_body)
register_legacy_event_emitter(SourceKind.LOGS, emit_logs_legacy_events)
+ register_source_facts(SourceKind.LOGS, logs_source_facts)
register_source_config(SourceKind.LOGS, load_logs_source_config)
diff --git a/products/logs/backend/platform_alert_legacy_events.py b/products/logs/backend/platform_alert_legacy_events.py
index bf302e745008..3eebe3685188 100644
--- a/products/logs/backend/platform_alert_legacy_events.py
+++ b/products/logs/backend/platform_alert_legacy_events.py
@@ -66,6 +66,19 @@ def emit_logs_legacy_events(
)
+def logs_source_facts(
+ *, request: AlertDeliveryRequest, announcement: EvaluationAnnouncement, transition: AnnouncedTransition
+) -> dict[str, Any]:
+ """The properties of the `$logs_alert_*` message event for this transition, for `$alert_transition` to carry."""
+ alert = _Alert(
+ team_id=request.team_id,
+ alert_id=request.destination_alert_id,
+ alert_name=announcement.alert_name,
+ consecutive_failures=announcement.consecutive_failures,
+ )
+ return _message_event(in_utc(transition), alert).properties
+
+
@frozen
class _Alert:
team_id: int