:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts-platform): send one follow-up about a firing that already went out (#127908)
Jimmy Zhudb4fadf1a6d9
diff --git a/products/alerts/backend/tests/api/test_alert_platform_owned.py b/products/alerts/backend/tests/api/test_alert_platform_owned.pyindex f05ce8317d40..36c5fa30a183 100644--- a/products/alerts/backend/tests/api/test_alert_platform_owned.py+++ b/products/alerts/backend/tests/api/test_alert_platform_owned.py@@ -26,11 +26,12 @@ from products.alerts.backend.models.alert import AlertCheck, AlertConfigurationfrom products.alerts.backend.platform_cutover import cut_over_teamfrom products.alerts.backend.test.insight_alerts import create_insight_alertfrom products.alerts_platform.backend.facade import testing as platform_testing-from products.alerts_platform.backend.facade.api import get_configuration, update_configuration+from products.alerts_platform.backend.facade.api import get_configuration, record_verdict_change, update_configurationfrom products.alerts_platform.backend.facade.contracts import (InvalidPlatformAlertEdit,PlatformAlertConfigurationEdit,SourceKind,+ VerdictChange,)@@ -77,6 +78,22 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):source_detail=detail,)+ def _follow_up(self, minutes_ago: int, *, firing_minutes_ago: int) -> None:+ firing_at = self._at(firing_minutes_ago)+ with team_scope(self.team.id):+ platform_testing.start_firing(self.copy_id, team_id=self.team.id, at=firing_at)+ request = record_verdict_change(+ VerdictChange(+ team_id=self.team.id,+ configuration_id=self.copy_id,+ firing_evaluation_key=f"slot:{firing_at.isoformat()}",+ episode_started_at=firing_at,+ firing_event_id="$insight_alert_firing",+ ),+ now=self._at(minutes_ago),+ )+ assert request is not None+def _own_check(self, minutes_ago: int, date: str, rationale: str) -> None:check = AlertCheck.objects.create(alert_configuration=self.alert,@@ -100,8 +117,7 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):@parameterized.expand([("list",), ("retrieve",)])def test_a_moved_alert_shows_the_platform_state_and_last_check(self, action: str) -> None:self._record(30, "firing", value=150.0)- with team_scope(self.team.id):- platform_testing.start_firing(self.copy_id, team_id=self.team.id, at=self._at(30))+ self._follow_up(10, firing_minutes_ago=30)row = self._row(action)@@ -113,6 +129,7 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):self._record(30, "errored", error="Query timed out")# A snoozed check evaluates nothing, so it measures no value either.self._record(15, "snoozed")+ self._follow_up(10, firing_minutes_ago=60)row = self._row("retrieve", "?checks_limit=2")diff --git a/products/alerts_platform/backend/comparison/platform_history.py b/products/alerts_platform/backend/comparison/platform_history.pyindex 7ab8029b4291..64ef53b66039 100644--- a/products/alerts_platform/backend/comparison/platform_history.py+++ b/products/alerts_platform/backend/comparison/platform_history.py@@ -20,6 +20,7 @@ from posthog.clickhouse.client import query_with_columnsfrom posthog.clickhouse.query_tagging import Feature, Product, tag_queriesfrom products.alerts_platform.backend.facade.contracts import GroupingMode, PlatformCheck, SourceKind+from products.alerts_platform.backend.logic.platform_alert_events import CHECKS_ONLYfrom products.alerts_platform.backend.models import PlatformAlertConfigurationfrom products.alerts_platform.backend.models.platform_alert_events_sql import PLATFORM_ALERT_EVENTS_TABLE@@ -41,6 +42,7 @@ WHERE team_id = %(team_id)sAND configuration_id IN %(configuration_ids)sAND occurred_at >= %(since)sAND occurred_at < %(until)s+ AND {CHECKS_ONLY}ORDER BY configuration_id, alert_id, occurred_at, evaluation_keyLIMIT 1 BY configuration_id, alert_id, evaluation_keyLIMIT %(limit)sdiff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.pyindex 396dbb0592ce..7b2d0670a026 100644--- a/products/alerts_platform/backend/delivery/evaluation.py+++ b/products/alerts_platform/backend/delivery/evaluation.py@@ -611,7 +611,7 @@ def _platform_subscribers(event_id: str, platform: _PlatformDestinations) -> lis_SubscribedDestination(id=str(destination.id), data=destination.data, receives_emitted_events=False)for destination in platform.destinationsif (destination.data["type"] in INCIDENT_DESTINATION_TYPES) == incident- and (incident or kind in destination.event_kinds)+ and (incident or kind.delivered_as in destination.event_kinds)]# The source's recipients read messages. An incident edge is for an incident manager only.if platform.recipients and not incident:diff --git a/products/alerts_platform/backend/delivery/in_app.py b/products/alerts_platform/backend/delivery/in_app.pyindex 6a2149a72528..25483f7602f2 100644--- a/products/alerts_platform/backend/delivery/in_app.py+++ b/products/alerts_platform/backend/delivery/in_app.py@@ -59,7 +59,7 @@ class InAppTransport:user_ids = target.get("in_app_user_ids") or []if not user_ids:raise DeliveryError("This in-app destination has no recipients.")- notification_type = _NOTIFICATION_TYPES.get(message.transition.kind)+ notification_type = _NOTIFICATION_TYPES.get(message.transition.kind.delivered_as)if notification_type is None:raise DeliveryError(f"In-app notifications have no type for {message.transition.kind.value} alerts.")key = transition_delivery_key(message)diff --git a/products/alerts_platform/backend/delivery/message.py b/products/alerts_platform/backend/delivery/message.pyindex 1ae83291a6da..4caa56a4b8b3 100644--- a/products/alerts_platform/backend/delivery/message.py+++ b/products/alerts_platform/backend/delivery/message.py@@ -29,6 +29,7 @@ _HEADLINES: Final[dict[AlertEventKind, str]] = {AlertEventKind.RESOLVED: "{kind} alert '{name}' is resolved",AlertEventKind.ERRORED: "{kind} alert '{name}' could not be checked",AlertEventKind.BROKEN: "{kind} alert '{name}' is turned off",+ AlertEventKind.VERDICT_CHANGED: "{kind} alert '{name}' has a new verdict",}# The symbols the legacy messages use, so a pilot team sees the same state at a glance on both paths.@@ -37,6 +38,7 @@ _SYMBOLS: Final[dict[AlertEventKind, str]] = {AlertEventKind.RESOLVED: "🟢",AlertEventKind.ERRORED: "🟡",AlertEventKind.BROKEN: "⚠️",+ AlertEventKind.VERDICT_CHANGED: "🔄",}UNTRACKED_GROUPS_LABEL: Final = "Untracked groups"@@ -191,6 +193,9 @@ def build_message(failure_kinds = (AlertEventKind.ERRORED, AlertEventKind.BROKEN)if transition.kind in failure_kinds:details = list(_failure_details(transition, announcement.consecutive_failures))+ elif transition.kind == AlertEventKind.VERDICT_CHANGED:+ # The row records no measurement, so the breach details would state the threshold without a value.+ details = list(description.details)else:details = list(description.details or _breach_details(transition))if overflowed:diff --git a/products/alerts_platform/backend/delivery/slack.py b/products/alerts_platform/backend/delivery/slack.pyindex ed40f3973eba..43dc31cb14f1 100644--- a/products/alerts_platform/backend/delivery/slack.py+++ b/products/alerts_platform/backend/delivery/slack.py@@ -215,7 +215,7 @@ class SlackTransport:# the thread. So a resolve and a repeat firing also go to the channel, where legacy# posted each one as a new message.reply_broadcast=in_reply_to is not None- and message.transition.kind in (AlertEventKind.RESOLVED, AlertEventKind.FIRING),+ and message.transition.kind.delivered_as in (AlertEventKind.RESOLVED, AlertEventKind.FIRING),**({"attachments": content.attachments} if content.attachments else {}),)diff --git a/products/alerts_platform/backend/delivery/webhook.py b/products/alerts_platform/backend/delivery/webhook.pyindex 9947e2a77411..76a4ecdc516a 100644--- a/products/alerts_platform/backend/delivery/webhook.py+++ b/products/alerts_platform/backend/delivery/webhook.py@@ -29,6 +29,7 @@ WEBHOOK_VERSION: Final = "2"_NOT_ENDED: Final = "0001-01-01T00:00:00Z"_BREACH_KINDS: Final = (AlertEventKind.FIRING, AlertEventKind.RESOLVED)+_ENDED_KINDS: Final = (AlertEventKind.RESOLVED, AlertEventKind.VERDICT_CHANGED)def _fingerprint(configuration_id: str, transition: AnnouncedTransition) -> str:@@ -54,8 +55,9 @@ def alertmanager_body(message: AlertMessage) -> dict[str, Any]:transition = message.transitionconfiguration_id = message.configuration_id# Alertmanager has only these two statuses. A failed or a turned-off check goes out as- # firing, and `posthog_event_kind` is what tells it apart from a breach.- status = "resolved" if transition.kind == AlertEventKind.RESOLVED else "firing"+ # firing, and `posthog_event_kind` is what tells it apart from a breach. A follow-up goes out+ # already ended under its own fingerprint, so it can neither open nor reopen anything.+ status = "resolved" if transition.kind in _ENDED_KINDS else "firing"group_labels = {"alertname": message.alert_name,"posthog_configuration_id": configuration_id,diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.pyindex fa18022f4591..ca947300a00c 100644--- a/products/alerts_platform/backend/facade/api.py+++ b/products/alerts_platform/backend/facade/api.py@@ -34,6 +34,7 @@ from products.alerts_platform.backend.facade.contracts import (PlatformOwnershipHandover,SourceKind,StoredAlertDestination,+ VerdictChange,)from products.alerts_platform.backend.facade.enums import HoldRelease, HoldReleasedByfrom products.alerts_platform.backend.logic import (@@ -43,6 +44,7 @@ from products.alerts_platform.backend.logic import (platform_lifecycle,platform_reads,platform_writes,+ verdict_changes,)@@ -84,6 +86,17 @@ def reschedule_hold(address: HeldDeliveryAddress, *, release_after: datetime) ->return delivery_holds.reschedule_hold(address, release_after=release_after)+def record_verdict_change(change: VerdictChange, *, now: datetime) -> AlertDeliveryRequest | None:+ """Records one follow-up about a firing that already went out, and returns the delivery to start+ with `start_deliveries`, or None when there is nothing to send.++ Writes only the follow-up's history row, so it leaves state, schedule and holds alone. A second+ call for the same firing starts the same delivery, whose id and threads send it once. Raises+ `VerdictChangeNotRecorded` when the row could not be written, which a retry can recover.+ """+ return verdict_changes.record_verdict_change(change, now=now)++def delivery_held_back(address: HeldDeliveryAddress) -> bool:"""Whether an evaluation's delivery is held and not released with a send, so it has not gone out."""return delivery_holds.blocks_delivery(address.team_id, address.configuration_id, address.evaluation_key)@@ -95,7 +108,8 @@ def record_verdict_changed("""Announces that a source's later verdict contradicts the one a delivered firing carried, inthat firing's thread. Returns the request the caller starts, or None when nothing is to be sent.- The platform has no verdict-changed kind to record it under, so this records nothing and returns None.+ Records nothing and returns None until the source can word a follow-up, which+ `record_verdict_change` then records and delivers."""return Nonediff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.pyindex 0a06726a10f0..b343e7259137 100644--- a/products/alerts_platform/backend/facade/contracts.py+++ b/products/alerts_platform/backend/facade/contracts.py@@ -7,6 +7,7 @@ contract check watches this file to decide whether they must retest.from __future__ import annotations+import hashlibfrom collections.abc import Callable, Collection, Mapping, Sequencefrom dataclasses import asdict, fieldfrom datetime import UTC, datetime@@ -544,6 +545,48 @@ class AlertEventKind(StrEnum):RESOLVED = "resolved"ERRORED = "errored"BROKEN = "broken"+ # A follow-up about a firing that already went out. History readers leave it out of the checks.+ VERDICT_CHANGED = "verdict_changed"++ @property+ def delivered_as(self) -> AlertEventKind:+ """The kind a destination hears this as. No destination can list a follow-up, so a follow-up+ reaches the destinations of the firing it follows, the way that firing did."""+ return AlertEventKind.FIRING if self is AlertEventKind.VERDICT_CHANGED else self+++@frozen+class VerdictChange:+ """A source's request to send one follow-up about a firing that already went out.++ `firing_event_id` is the event the firing's destinations listen on, so a destination that heard+ the firing hears the follow-up. `source_detail` is what the source's describer words the+ follow-up from. The platform stores it unread, as on a check.+ """++ team_id: int+ configuration_id: UUID+ firing_evaluation_key: str+ episode_started_at: datetime | None+ firing_event_id: str+ source_alert_url: str | None = None+ grouping_key: str = ""+ source_detail: dict[str, Any] = field(default_factory=dict)++ @property+ def evaluation_key(self) -> str:+ # A key of its own gives the follow-up its own delivery, and gets it past the thread store,+ # which skips a key a thread already carried. Per group, because two groups' follow-ups+ # under one key share a delivery, which would miss a row written after it read the history.+ # The group goes in as a digest, because a grouping key alone can fill a thread's key column.+ key = f"{self.firing_evaluation_key}#verdict"+ if not self.grouping_key:+ return key+ return f"{key}:{hashlib.sha256(self.grouping_key.encode()).hexdigest()[:16]}"+++class VerdictChangeNotRecorded(Exception):+ """ClickHouse refused a follow-up's history row, so no delivery could read it back. Retryable."""@frozendiff --git a/products/alerts_platform/backend/logic/platform_alert_events.py b/products/alerts_platform/backend/logic/platform_alert_events.pyindex c9241002a788..4f039596da88 100644--- a/products/alerts_platform/backend/logic/platform_alert_events.py+++ b/products/alerts_platform/backend/logic/platform_alert_events.py@@ -293,6 +293,11 @@ def _snapshot(raw: str) -> dict[str, Any]:return loaded if isinstance(loaded, dict) else {}+# Every read of checks leaves out follow-ups, which are messages about a check rather than checks.+# Only `announcement` reads them, by their own evaluation key.+CHECKS_ONLY: Final = f"kind != '{AlertEventKind.VERDICT_CHANGED.value}'"++# The inner read deduplicates on the pair the writer names, as `announcement` does. The outer one# keeps the checks that moved the alert and the newest check, so a page of alerts that check every# minute returns a few rows each rather than a day of them.@@ -307,6 +312,7 @@ FROM (WHERE team_id = %(team_id)sAND configuration_id IN %(configuration_ids)sAND grouping_key = ''+ AND {CHECKS_ONLY}AND occurred_at >= %(since)sAND occurred_at < %(until)sORDER BY configuration_id, alert_id, occurred_at, evaluation_key@@ -384,6 +390,7 @@ FROM (WHERE team_id = %(team_id)sAND configuration_id = %(configuration_id)sAND grouping_key = ''+ AND {CHECKS_ONLY}AND occurred_at >= %(since)sAND (previous_state != state OR error_message != '')ORDER BY alert_id, occurred_at, evaluation_key@@ -447,6 +454,7 @@ FROM (WHERE team_id = %(team_id)sAND configuration_id = %(configuration_id)sAND grouping_key = ''+ AND {CHECKS_ONLY}AND occurred_at >= toDateTime64(%(since)s, 6, 'UTC')AND occurred_at < toDateTime64(%(until)s, 6, 'UTC')ORDER BY alert_id, occurred_at, evaluation_key@@ -464,6 +472,7 @@ FROM (WHERE team_id = %(team_id)sAND configuration_id = %(configuration_id)sAND grouping_key = ''+ AND {CHECKS_ONLY}AND occurred_at >= toDateTime64(%(since)s, 6, 'UTC')AND occurred_at < toDateTime64(%(until)s, 6, 'UTC')LIMIT 1 BY alert_id, evaluation_key@@ -522,6 +531,7 @@ FROM (WHERE team_id = %(team_id)sAND configuration_id IN %(configuration_ids)sAND grouping_key = ''+ AND {CHECKS_ONLY}AND occurred_at >= toDateTime64(%(since)s, 6, 'UTC'){has_paths}ORDER BY configuration_id, alert_id, occurred_at, evaluation_keyLIMIT 1 BY alert_id, evaluation_key@@ -577,6 +587,7 @@ FROM {PLATFORM_ALERT_EVENTS_TABLE}WHERE team_id = %(team_id)sAND configuration_id IN %(configuration_ids)sAND grouping_key = ''+ AND {CHECKS_ONLY}GROUP BY configuration_id"""diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.pyindex e4d7f5e18481..812eb4ffe5fe 100644--- a/products/alerts_platform/backend/logic/platform_lifecycle.py+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py@@ -439,11 +439,12 @@ def event_row()-def record_history(team_id: int, rows: Sequence[PlatformAlertEventRow]) -> None:+def record_history(team_id: int, rows: Sequence[PlatformAlertEventRow]) -> int:recorded = insert_events(team_id, rows)if recorded < len(rows):# One batch holds one source's checks.safe_record(increment_history_rows_dropped, rows[0].source_kind, len(rows) - recorded)+ return recordeddef _shard_offset_seconds(configuration: PlatformAlertConfiguration, outcome: PlatformAlertOutcome) -> int:diff --git a/products/alerts_platform/backend/logic/verdict_changes.py b/products/alerts_platform/backend/logic/verdict_changes.pynew file mode 100644index 000000000000..db22047b7c8a--- /dev/null+++ b/products/alerts_platform/backend/logic/verdict_changes.py@@ -0,0 +1,71 @@+"""A follow-up about a firing that already went out, such as an investigation that changed its verdict.++A check announces once, and history is append-only, so a verdict that arrives after the firing's+message cannot change what that row said. The follow-up gets a row of its own, under its own+evaluation key, with the firing's episode. That episode puts the reply in the firing's thread and+sends it on the firing's delivery path.++The write is that row and nothing else. It moves no state, no schedule and no notified time,+because the check that fired already moved them.+"""++from datetime import datetime++from products.alerts_platform.backend.facade.contracts import (+ AlertDeliveryRequest,+ AlertEventKind,+ SourceKind,+ VerdictChange,+ VerdictChangeNotRecorded,+)+from products.alerts_platform.backend.logic import delivery_holds+from products.alerts_platform.backend.logic.platform_lifecycle import RULE_METADATA_FIELDS, event_row, record_history+from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration+++def record_verdict_change(change: VerdictChange, *, now: datetime) -> AlertDeliveryRequest | None:+ """Records the follow-up's row and returns the delivery to start, or None when there is nothing to send.++ None when the configuration or its group is gone, or when the firing is still held or was+ suppressed. Raises `VerdictChangeNotRecorded` when ClickHouse refused the row.+ """+ alert = (+ PlatformAlert.objects.for_team(change.team_id)+ .select_related("configuration")+ .defer(*(f"configuration__{name}" for name in RULE_METADATA_FIELDS))+ .filter(configuration_id=change.configuration_id, grouping_key=change.grouping_key)+ .first()+ )+ if alert is None:+ return None+ # A reply to a firing nobody received would arrive with no message above it.+ if delivery_holds.blocks_delivery(change.team_id, str(change.configuration_id), change.firing_evaluation_key):+ return None+ configuration = alert.configuration++ row = event_row(+ configuration,+ alert=alert,+ evaluation_key=change.evaluation_key,+ kind=AlertEventKind.VERDICT_CHANGED.value,+ previous_state=alert.state,+ state=alert.state,+ episode_started_at=change.episode_started_at,+ labels=alert.labels,+ consecutive_failures=configuration.consecutive_failures,+ now=now,+ source_detail=change.source_detail,+ )+ if not record_history(change.team_id, [row]):+ raise VerdictChangeNotRecorded(f"no history row for {change.evaluation_key}")+ return AlertDeliveryRequest(+ source=SourceKind(configuration.source_kind),+ team_id=change.team_id,+ configuration_id=str(configuration.id),+ evaluation_key=change.evaluation_key,+ destination_alert_id=str(configuration.destination_alert_id),+ # The firing's event, so a HogFunction that sent the firing sends the follow-up, as legacy does.+ event_ids_by_kind={AlertEventKind.VERDICT_CHANGED.value: change.firing_event_id},+ source_alert_url=change.source_alert_url,+ owned_by_platform=configuration.owner == PlatformAlertConfiguration.Owner.PLATFORM,+ )diff --git a/products/alerts_platform/backend/tests/test_comparison_platform_history.py b/products/alerts_platform/backend/tests/test_comparison_platform_history.pyindex 67ca98cf6113..2f1d34c461e6 100644--- a/products/alerts_platform/backend/tests/test_comparison_platform_history.py+++ b/products/alerts_platform/backend/tests/test_comparison_platform_history.py@@ -1,3 +1,4 @@+from dataclasses import replacefrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom uuid import uuid4@@ -52,11 +53,11 @@ class TestReadPlatformChecks(ClickhouseTestMixin, APIBaseTest):def _row(self, configuration: PlatformAlertConfiguration, *, evaluation_key: str, **overrides) -> PlatformAlertEventRow:- return PlatformAlertEventRow(+ row = PlatformAlertEventRow(team_id=self.team.id,source_kind="logs",configuration_id=configuration.id,- alert_id=overrides.pop("alert_id", uuid4()),+ alert_id=uuid4(),grouping_key="",evaluation_key=evaluation_key,kind="firing",@@ -73,8 +74,8 @@ class TestReadPlatformChecks(ClickhouseTestMixin, APIBaseTest):consecutive_failures=0,muted_notification="",occurred_at=OCCURRED_AT,- **overrides,)+ return replace(row, **overrides)def _record(self, configuration: PlatformAlertConfiguration, *, evaluation_key: str, **overrides) -> None:insert_events(self.team.id, [self._row(configuration, evaluation_key=evaluation_key, **overrides)])@@ -158,16 +159,18 @@ class TestReadPlatformChecks(ClickhouseTestMixin, APIBaseTest):@parameterized.expand([- ("another_source", {"source_kind": "insight"}),- ("a_grouped_configuration", {"grouping": {"mode": "by_result_labels", "keys": ["service_name"]}}),+ ("another_source", {"source_kind": "insight"}, {}),+ ("a_grouped_configuration", {"grouping": {"mode": "by_result_labels", "keys": ["service_name"]}}, {}),+ ("a_verdict_follow_up", {}, {"kind": "verdict_changed", "previous_state": "firing"}),])def test_a_check_with_no_comparable_verdict_is_not_in_a_logs_comparison(- self, _name: str, overrides: dict[str, Any]+ self, _name: str, overrides: dict[str, Any], row_overrides: dict[str, Any]) -> None:self._record(self._configuration(legacy_configuration_id=uuid4(), **overrides),evaluation_key="run:2026-09-30T11:59:00+00:00",+ **row_overrides,)assert self._read() == []diff --git a/products/alerts_platform/backend/tests/test_delivery_webhook.py b/products/alerts_platform/backend/tests/test_delivery_webhook.pyindex 709c1aae7331..7c2d210a168a 100644--- a/products/alerts_platform/backend/tests/test_delivery_webhook.py+++ b/products/alerts_platform/backend/tests/test_delivery_webhook.py@@ -90,6 +90,13 @@ class TestAlertmanagerBody(SimpleTestCase):("resolved", AlertEventKind.RESOLVED, "resolved", "2026-09-30T09:00:00+00:00", "2026-09-30T10:00:00+00:00"),("errored", AlertEventKind.ERRORED, "firing", "2026-09-30T10:00:00+00:00", "0001-01-01T00:00:00Z"),("broken", AlertEventKind.BROKEN, "firing", "2026-09-30T10:00:00+00:00", "0001-01-01T00:00:00Z"),+ (+ "verdict_changed",+ AlertEventKind.VERDICT_CHANGED,+ "resolved",+ "2026-09-30T09:00:00+00:00",+ "2026-09-30T10:00:00+00:00",+ ),])def test_each_kind_maps_onto_an_alertmanager_status(@@ -127,6 +134,8 @@ class TestAlertmanagerBody(SimpleTestCase):assert fingerprint(AlertEventKind.FIRING) == fingerprint(AlertEventKind.RESOLVED)# A failed check sent as firing would otherwise be closed by the breach's resolve.assert fingerprint(AlertEventKind.ERRORED) != fingerprint(AlertEventKind.FIRING)+ # A follow-up goes out ended, so on the breach's fingerprint it would close a firing that is still open.+ assert fingerprint(AlertEventKind.VERDICT_CHANGED) != fingerprint(AlertEventKind.FIRING)assert fingerprint(AlertEventKind.FIRING, "search") != fingerprint(AlertEventKind.FIRING)diff --git a/products/alerts_platform/backend/tests/test_verdict_changes.py b/products/alerts_platform/backend/tests/test_verdict_changes.pynew file mode 100644index 000000000000..b2db3a5f535e--- /dev/null+++ b/products/alerts_platform/backend/tests/test_verdict_changes.py@@ -0,0 +1,229 @@+from datetime import UTC, datetime, timedelta+from typing import Any, cast+from uuid import uuid4++import pytest+from posthog.test.base import APIBaseTest, ClickhouseTestMixin+from unittest.mock import patch++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.models.scoping import team_scope++from products.alerts_platform.backend.delivery.message import AlertMessage+from products.alerts_platform.backend.delivery.transport import MessageHandle+from products.alerts_platform.backend.facade import testing as platform_testing+from products.alerts_platform.backend.facade.api import create_alert_destinations, record_verdict_change+from products.alerts_platform.backend.facade.contracts import (+ AlertDeliveryRequest,+ AlertDestinationData,+ AlertEventKind,+ DestinationType,+ NewAlertDestination,+ SourceKind,+ VerdictChange,+ VerdictChangeNotRecorded,+)+from products.alerts_platform.backend.facade.enums import HoldRelease, PlatformAlertOwner+from products.alerts_platform.backend.logic.platform_lifecycle import event_row, record_history+from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration, PlatformAlertDeliveryHold++FIRING_AT = datetime(2026, 10, 19, 9, tzinfo=UTC)+FIRING_KEY = f"slot:{FIRING_AT.isoformat()}"+FIRING_EVENT = "$metrics_alert_firing"+SLACK = cast(AlertDestinationData, {"type": DestinationType.SLACK, "slack_workspace_id": 1, "slack_channel_id": "C-1"})+++class ThreadedTransport:+ sends: list[tuple[AlertMessage, MessageHandle | None]] = []+ provider = "slack"++ def channel_target(self, target: AlertDestinationData) -> str:+ return str(target.get("slack_channel_id", ""))++ def deliver(+ self, *, team_id: int, target: AlertDestinationData, message: AlertMessage, in_reply_to: Any = None+ ) -> MessageHandle | None:+ ThreadedTransport.sends.append((message, in_reply_to))+ return MessageHandle(external_ref={"ts": str(len(ThreadedTransport.sends))})+++class TestRecordVerdictChange(ClickhouseTestMixin, APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ ThreadedTransport.sends = []+ self.next_check_at = FIRING_AT + timedelta(minutes=5)+ with team_scope(self.team.id):+ self.configuration = PlatformAlertConfiguration.objects.create(+ team=self.team,+ name="Checkout latency",+ source_kind=PlatformAlertConfiguration.SourceKind.METRICS,+ source_config={},+ check_interval_minutes=5,+ next_check_at=self.next_check_at,+ owner=PlatformAlertOwner.PLATFORM,+ destinations_owner=PlatformAlertOwner.PLATFORM,+ )+ platform_testing.start_firing(self.configuration.id, team_id=self.team.id, at=FIRING_AT)+ PlatformAlert.objects.filter(configuration=self.configuration).update(last_notified_at=FIRING_AT)+ create_alert_destinations(+ team_id=self.team.id,+ configuration_id=self.configuration.id,+ destinations=(NewAlertDestination(data=SLACK, event_kinds=(AlertEventKind.FIRING,)),),+ )++ def _deliver(self, request: AlertDeliveryRequest) -> None:+ with patch.dict(+ "products.alerts_platform.backend.delivery.evaluation._TRANSPORTS",+ {DestinationType.SLACK: ThreadedTransport},+ ):+ platform_testing.deliver(request)++ def test_a_follow_up_replies_once_in_the_thread_of_the_firing_it_follows(self) -> None:+ with team_scope(self.team.id):+ alert = PlatformAlert.objects.get(configuration=self.configuration)+ firing_row = event_row(+ self.configuration,+ alert=alert,+ evaluation_key=FIRING_KEY,+ kind=AlertEventKind.FIRING.value,+ previous_state="not_firing",+ state="firing",+ episode_started_at=FIRING_AT,+ consecutive_failures=0,+ now=FIRING_AT,+ )+ record_history(self.team.id, [firing_row])+ self._deliver(+ AlertDeliveryRequest(+ source=SourceKind.METRICS,+ team_id=self.team.id,+ configuration_id=str(self.configuration.id),+ evaluation_key=FIRING_KEY,+ destination_alert_id=str(self.configuration.id),+ event_ids_by_kind={AlertEventKind.FIRING.value: FIRING_EVENT},+ owned_by_platform=True,+ )+ )++ with team_scope(self.team.id):+ follow_up = record_verdict_change(+ VerdictChange(+ team_id=self.team.id,+ configuration_id=self.configuration.id,+ firing_evaluation_key=FIRING_KEY,+ episode_started_at=FIRING_AT,+ firing_event_id=FIRING_EVENT,+ ),+ now=FIRING_AT + timedelta(minutes=20),+ )+ assert follow_up is not None+ self._deliver(follow_up)+ self._deliver(follow_up)++ [(firing, opened_in), (reply, replied_to)] = ThreadedTransport.sends+ assert (firing.transition.kind, opened_in) == (AlertEventKind.FIRING, None)+ assert reply.headline == "Metric alert 'Checkout latency' has a new verdict"+ assert replied_to == MessageHandle(external_ref={"ts": "1"})+ with team_scope(self.team.id):+ configuration = PlatformAlertConfiguration.objects.get(id=self.configuration.id)+ alert = PlatformAlert.objects.get(configuration=self.configuration)+ assert (configuration.next_check_at, alert.state, alert.last_notified_at) == (+ self.next_check_at,+ "firing",+ FIRING_AT,+ )++ @parameterized.expand([("still_held", None), ("suppressed", HoldRelease.SUPPRESS)])+ def test_a_firing_nobody_received_gets_no_follow_up(self, _name: str, release: HoldRelease | None) -> None:+ with team_scope(self.team.id):+ PlatformAlertDeliveryHold.objects.create(+ team=self.team,+ configuration=self.configuration,+ evaluation_key=FIRING_KEY,+ reason="investigation",+ request={},+ held_at=FIRING_AT,+ release_after=FIRING_AT + timedelta(minutes=90),+ released_at=FIRING_AT if release else None,+ release_action=release,+ )+ follow_up = record_verdict_change(+ VerdictChange(+ team_id=self.team.id,+ configuration_id=self.configuration.id,+ firing_evaluation_key=FIRING_KEY,+ episode_started_at=FIRING_AT,+ firing_event_id=FIRING_EVENT,+ ),+ now=FIRING_AT + timedelta(minutes=20),+ )++ assert follow_up is None++ def test_a_follow_up_clickhouse_refused_raises_so_the_caller_can_retry(self) -> None:+ with (+ team_scope(self.team.id),+ patch("products.alerts_platform.backend.logic.platform_lifecycle.insert_events", return_value=0),+ pytest.raises(VerdictChangeNotRecorded),+ ):+ record_verdict_change(+ VerdictChange(+ team_id=self.team.id,+ configuration_id=self.configuration.id,+ firing_evaluation_key=FIRING_KEY,+ episode_started_at=FIRING_AT,+ firing_event_id=FIRING_EVENT,+ ),+ now=FIRING_AT + timedelta(minutes=20),+ )++ def test_a_grouped_follow_up_names_its_group(self) -> None:+ change = VerdictChange(+ team_id=self.team.id,+ configuration_id=self.configuration.id,+ firing_evaluation_key=FIRING_KEY,+ episode_started_at=FIRING_AT,+ firing_event_id=FIRING_EVENT,+ grouping_key="checkout",+ )+ with team_scope(self.team.id):+ platform_testing.start_firing(+ self.configuration.id, team_id=self.team.id, at=FIRING_AT, grouping_key="checkout"+ )+ PlatformAlert.objects.filter(configuration=self.configuration, grouping_key="checkout").update(+ labels={"service_name": "checkout"}+ )+ with patch(+ "products.alerts_platform.backend.logic.verdict_changes.record_history", return_value=1+ ) as recorded:+ follow_up = record_verdict_change(change, now=FIRING_AT + timedelta(minutes=20))++ [[_team_id, [row]], _] = recorded.call_args+ assert (row.grouping_key, row.labels, row.evaluation_key) == (+ "checkout",+ {"service_name": "checkout"},+ change.evaluation_key,+ )+ assert follow_up is not None and follow_up.evaluation_key == change.evaluation_key+++class TestVerdictChangeKey(SimpleTestCase):+ def test_each_group_gets_a_key_that_fits_a_thread(self) -> None:+ def key(grouping_key: str) -> str:+ return VerdictChange(+ team_id=1,+ configuration_id=uuid4(),+ firing_evaluation_key=FIRING_KEY,+ episode_started_at=FIRING_AT,+ firing_event_id=FIRING_EVENT,+ grouping_key=grouping_key,+ ).evaluation_key++ longest = key("a" * 255)++ assert len(longest) <= 255+ assert longest != key("a" * 254 + "b")+ assert key("") == f"{FIRING_KEY}#verdict"