Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts-platform): send one follow-up about a firing that already went out (#127908)

Jimmy Zhudb4fadf1a6d9
diff --git a/products/alerts/backend/tests/api/test_alert_platform_owned.py b/products/alerts/backend/tests/api/test_alert_platform_owned.py
index f05ce8317d40..36c5fa30a183 100644
--- a/products/alerts/backend/tests/api/test_alert_platform_owned.py
+++ b/products/alerts/backend/tests/api/test_alert_platform_owned.py
@@ -26,11 +26,12 @@ from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration
from products.alerts.backend.platform_cutover import cut_over_team
from products.alerts.backend.test.insight_alerts import create_insight_alert
from products.alerts_platform.backend.facade import testing as platform_testing
-from products.alerts_platform.backend.facade.api import get_configuration, update_configuration
+from products.alerts_platform.backend.facade.api import get_configuration, record_verdict_change, update_configuration
from products.alerts_platform.backend.facade.contracts import (
InvalidPlatformAlertEdit,
PlatformAlertConfigurationEdit,
SourceKind,
+ VerdictChange,
)
@@ -77,6 +78,22 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):
source_detail=detail,
)
+ def _follow_up(self, minutes_ago: int, *, firing_minutes_ago: int) -> None:
+ firing_at = self._at(firing_minutes_ago)
+ with team_scope(self.team.id):
+ platform_testing.start_firing(self.copy_id, team_id=self.team.id, at=firing_at)
+ request = record_verdict_change(
+ VerdictChange(
+ team_id=self.team.id,
+ configuration_id=self.copy_id,
+ firing_evaluation_key=f"slot:{firing_at.isoformat()}",
+ episode_started_at=firing_at,
+ firing_event_id="$insight_alert_firing",
+ ),
+ now=self._at(minutes_ago),
+ )
+ assert request is not None
+
def _own_check(self, minutes_ago: int, date: str, rationale: str) -> None:
check = AlertCheck.objects.create(
alert_configuration=self.alert,
@@ -100,8 +117,7 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):
@parameterized.expand([("list",), ("retrieve",)])
def test_a_moved_alert_shows_the_platform_state_and_last_check(self, action: str) -> None:
self._record(30, "firing", value=150.0)
- with team_scope(self.team.id):
- platform_testing.start_firing(self.copy_id, team_id=self.team.id, at=self._at(30))
+ self._follow_up(10, firing_minutes_ago=30)
row = self._row(action)
@@ -113,6 +129,7 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):
self._record(30, "errored", error="Query timed out")
# A snoozed check evaluates nothing, so it measures no value either.
self._record(15, "snoozed")
+ self._follow_up(10, firing_minutes_ago=60)
row = self._row("retrieve", "?checks_limit=2")
diff --git a/products/alerts_platform/backend/comparison/platform_history.py b/products/alerts_platform/backend/comparison/platform_history.py
index 7ab8029b4291..64ef53b66039 100644
--- a/products/alerts_platform/backend/comparison/platform_history.py
+++ b/products/alerts_platform/backend/comparison/platform_history.py
@@ -20,6 +20,7 @@ from posthog.clickhouse.client import query_with_columns
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from products.alerts_platform.backend.facade.contracts import GroupingMode, PlatformCheck, SourceKind
+from products.alerts_platform.backend.logic.platform_alert_events import CHECKS_ONLY
from products.alerts_platform.backend.models import PlatformAlertConfiguration
from products.alerts_platform.backend.models.platform_alert_events_sql import PLATFORM_ALERT_EVENTS_TABLE
@@ -41,6 +42,7 @@ WHERE team_id = %(team_id)s
AND configuration_id IN %(configuration_ids)s
AND occurred_at >= %(since)s
AND occurred_at < %(until)s
+ AND {CHECKS_ONLY}
ORDER BY configuration_id, alert_id, occurred_at, evaluation_key
LIMIT 1 BY configuration_id, alert_id, evaluation_key
LIMIT %(limit)s
diff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.py
index 396dbb0592ce..7b2d0670a026 100644
--- a/products/alerts_platform/backend/delivery/evaluation.py
+++ b/products/alerts_platform/backend/delivery/evaluation.py
@@ -611,7 +611,7 @@ def _platform_subscribers(event_id: str, platform: _PlatformDestinations) -> lis
_SubscribedDestination(id=str(destination.id), data=destination.data, receives_emitted_events=False)
for destination in platform.destinations
if (destination.data["type"] in INCIDENT_DESTINATION_TYPES) == incident
- and (incident or kind in destination.event_kinds)
+ and (incident or kind.delivered_as in destination.event_kinds)
]
# The source's recipients read messages. An incident edge is for an incident manager only.
if platform.recipients and not incident:
diff --git a/products/alerts_platform/backend/delivery/in_app.py b/products/alerts_platform/backend/delivery/in_app.py
index 6a2149a72528..25483f7602f2 100644
--- a/products/alerts_platform/backend/delivery/in_app.py
+++ b/products/alerts_platform/backend/delivery/in_app.py
@@ -59,7 +59,7 @@ class InAppTransport:
user_ids = target.get("in_app_user_ids") or []
if not user_ids:
raise DeliveryError("This in-app destination has no recipients.")
- notification_type = _NOTIFICATION_TYPES.get(message.transition.kind)
+ notification_type = _NOTIFICATION_TYPES.get(message.transition.kind.delivered_as)
if notification_type is None:
raise DeliveryError(f"In-app notifications have no type for {message.transition.kind.value} alerts.")
key = transition_delivery_key(message)
diff --git a/products/alerts_platform/backend/delivery/message.py b/products/alerts_platform/backend/delivery/message.py
index 1ae83291a6da..4caa56a4b8b3 100644
--- a/products/alerts_platform/backend/delivery/message.py
+++ b/products/alerts_platform/backend/delivery/message.py
@@ -29,6 +29,7 @@ _HEADLINES: Final[dict[AlertEventKind, str]] = {
AlertEventKind.RESOLVED: "{kind} alert '{name}' is resolved",
AlertEventKind.ERRORED: "{kind} alert '{name}' could not be checked",
AlertEventKind.BROKEN: "{kind} alert '{name}' is turned off",
+ AlertEventKind.VERDICT_CHANGED: "{kind} alert '{name}' has a new verdict",
}
# The symbols the legacy messages use, so a pilot team sees the same state at a glance on both paths.
@@ -37,6 +38,7 @@ _SYMBOLS: Final[dict[AlertEventKind, str]] = {
AlertEventKind.RESOLVED: "🟢",
AlertEventKind.ERRORED: "🟡",
AlertEventKind.BROKEN: "⚠️",
+ AlertEventKind.VERDICT_CHANGED: "🔄",
}
UNTRACKED_GROUPS_LABEL: Final = "Untracked groups"
@@ -191,6 +193,9 @@ def build_message(
failure_kinds = (AlertEventKind.ERRORED, AlertEventKind.BROKEN)
if transition.kind in failure_kinds:
details = list(_failure_details(transition, announcement.consecutive_failures))
+ elif transition.kind == AlertEventKind.VERDICT_CHANGED:
+ # The row records no measurement, so the breach details would state the threshold without a value.
+ details = list(description.details)
else:
details = list(description.details or _breach_details(transition))
if overflowed:
diff --git a/products/alerts_platform/backend/delivery/slack.py b/products/alerts_platform/backend/delivery/slack.py
index ed40f3973eba..43dc31cb14f1 100644
--- a/products/alerts_platform/backend/delivery/slack.py
+++ b/products/alerts_platform/backend/delivery/slack.py
@@ -215,7 +215,7 @@ class SlackTransport:
# the thread. So a resolve and a repeat firing also go to the channel, where legacy
# posted each one as a new message.
reply_broadcast=in_reply_to is not None
- and message.transition.kind in (AlertEventKind.RESOLVED, AlertEventKind.FIRING),
+ and message.transition.kind.delivered_as in (AlertEventKind.RESOLVED, AlertEventKind.FIRING),
**({"attachments": content.attachments} if content.attachments else {}),
)
diff --git a/products/alerts_platform/backend/delivery/webhook.py b/products/alerts_platform/backend/delivery/webhook.py
index 9947e2a77411..76a4ecdc516a 100644
--- a/products/alerts_platform/backend/delivery/webhook.py
+++ b/products/alerts_platform/backend/delivery/webhook.py
@@ -29,6 +29,7 @@ WEBHOOK_VERSION: Final = "2"
_NOT_ENDED: Final = "0001-01-01T00:00:00Z"
_BREACH_KINDS: Final = (AlertEventKind.FIRING, AlertEventKind.RESOLVED)
+_ENDED_KINDS: Final = (AlertEventKind.RESOLVED, AlertEventKind.VERDICT_CHANGED)
def _fingerprint(configuration_id: str, transition: AnnouncedTransition) -> str:
@@ -54,8 +55,9 @@ def alertmanager_body(message: AlertMessage) -> dict[str, Any]:
transition = message.transition
configuration_id = message.configuration_id
# Alertmanager has only these two statuses. A failed or a turned-off check goes out as
- # firing, and `posthog_event_kind` is what tells it apart from a breach.
- status = "resolved" if transition.kind == AlertEventKind.RESOLVED else "firing"
+ # firing, and `posthog_event_kind` is what tells it apart from a breach. A follow-up goes out
+ # already ended under its own fingerprint, so it can neither open nor reopen anything.
+ status = "resolved" if transition.kind in _ENDED_KINDS else "firing"
group_labels = {
"alertname": message.alert_name,
"posthog_configuration_id": configuration_id,
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.py
index fa18022f4591..ca947300a00c 100644
--- a/products/alerts_platform/backend/facade/api.py
+++ b/products/alerts_platform/backend/facade/api.py
@@ -34,6 +34,7 @@ from products.alerts_platform.backend.facade.contracts import (
PlatformOwnershipHandover,
SourceKind,
StoredAlertDestination,
+ VerdictChange,
)
from products.alerts_platform.backend.facade.enums import HoldRelease, HoldReleasedBy
from products.alerts_platform.backend.logic import (
@@ -43,6 +44,7 @@ from products.alerts_platform.backend.logic import (
platform_lifecycle,
platform_reads,
platform_writes,
+ verdict_changes,
)
@@ -84,6 +86,17 @@ def reschedule_hold(address: HeldDeliveryAddress, *, release_after: datetime) ->
return delivery_holds.reschedule_hold(address, release_after=release_after)
+def record_verdict_change(change: VerdictChange, *, now: datetime) -> AlertDeliveryRequest | None:
+ """Records one follow-up about a firing that already went out, and returns the delivery to start
+ with `start_deliveries`, or None when there is nothing to send.
+
+ Writes only the follow-up's history row, so it leaves state, schedule and holds alone. A second
+ call for the same firing starts the same delivery, whose id and threads send it once. Raises
+ `VerdictChangeNotRecorded` when the row could not be written, which a retry can recover.
+ """
+ return verdict_changes.record_verdict_change(change, now=now)
+
+
def delivery_held_back(address: HeldDeliveryAddress) -> bool:
"""Whether an evaluation's delivery is held and not released with a send, so it has not gone out."""
return delivery_holds.blocks_delivery(address.team_id, address.configuration_id, address.evaluation_key)
@@ -95,7 +108,8 @@ def record_verdict_changed(
"""Announces that a source's later verdict contradicts the one a delivered firing carried, in
that firing's thread. Returns the request the caller starts, or None when nothing is to be sent.
- The platform has no verdict-changed kind to record it under, so this records nothing and returns None.
+ Records nothing and returns None until the source can word a follow-up, which
+ `record_verdict_change` then records and delivers.
"""
return None
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index 0a06726a10f0..b343e7259137 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -7,6 +7,7 @@ contract check watches this file to decide whether they must retest.
from __future__ import annotations
+import hashlib
from collections.abc import Callable, Collection, Mapping, Sequence
from dataclasses import asdict, field
from datetime import UTC, datetime
@@ -544,6 +545,48 @@ class AlertEventKind(StrEnum):
RESOLVED = "resolved"
ERRORED = "errored"
BROKEN = "broken"
+ # A follow-up about a firing that already went out. History readers leave it out of the checks.
+ VERDICT_CHANGED = "verdict_changed"
+
+ @property
+ def delivered_as(self) -> AlertEventKind:
+ """The kind a destination hears this as. No destination can list a follow-up, so a follow-up
+ reaches the destinations of the firing it follows, the way that firing did."""
+ return AlertEventKind.FIRING if self is AlertEventKind.VERDICT_CHANGED else self
+
+
+@frozen
+class VerdictChange:
+ """A source's request to send one follow-up about a firing that already went out.
+
+ `firing_event_id` is the event the firing's destinations listen on, so a destination that heard
+ the firing hears the follow-up. `source_detail` is what the source's describer words the
+ follow-up from. The platform stores it unread, as on a check.
+ """
+
+ team_id: int
+ configuration_id: UUID
+ firing_evaluation_key: str
+ episode_started_at: datetime | None
+ firing_event_id: str
+ source_alert_url: str | None = None
+ grouping_key: str = ""
+ source_detail: dict[str, Any] = field(default_factory=dict)
+
+ @property
+ def evaluation_key(self) -> str:
+ # A key of its own gives the follow-up its own delivery, and gets it past the thread store,
+ # which skips a key a thread already carried. Per group, because two groups' follow-ups
+ # under one key share a delivery, which would miss a row written after it read the history.
+ # The group goes in as a digest, because a grouping key alone can fill a thread's key column.
+ key = f"{self.firing_evaluation_key}#verdict"
+ if not self.grouping_key:
+ return key
+ return f"{key}:{hashlib.sha256(self.grouping_key.encode()).hexdigest()[:16]}"
+
+
+class VerdictChangeNotRecorded(Exception):
+ """ClickHouse refused a follow-up's history row, so no delivery could read it back. Retryable."""
@frozen
diff --git a/products/alerts_platform/backend/logic/platform_alert_events.py b/products/alerts_platform/backend/logic/platform_alert_events.py
index c9241002a788..4f039596da88 100644
--- a/products/alerts_platform/backend/logic/platform_alert_events.py
+++ b/products/alerts_platform/backend/logic/platform_alert_events.py
@@ -293,6 +293,11 @@ def _snapshot(raw: str) -> dict[str, Any]:
return loaded if isinstance(loaded, dict) else {}
+# Every read of checks leaves out follow-ups, which are messages about a check rather than checks.
+# Only `announcement` reads them, by their own evaluation key.
+CHECKS_ONLY: Final = f"kind != '{AlertEventKind.VERDICT_CHANGED.value}'"
+
+
# The inner read deduplicates on the pair the writer names, as `announcement` does. The outer one
# keeps the checks that moved the alert and the newest check, so a page of alerts that check every
# minute returns a few rows each rather than a day of them.
@@ -307,6 +312,7 @@ FROM (
WHERE team_id = %(team_id)s
AND configuration_id IN %(configuration_ids)s
AND grouping_key = ''
+ AND {CHECKS_ONLY}
AND occurred_at >= %(since)s
AND occurred_at < %(until)s
ORDER BY configuration_id, alert_id, occurred_at, evaluation_key
@@ -384,6 +390,7 @@ FROM (
WHERE team_id = %(team_id)s
AND configuration_id = %(configuration_id)s
AND grouping_key = ''
+ AND {CHECKS_ONLY}
AND occurred_at >= %(since)s
AND (previous_state != state OR error_message != '')
ORDER BY alert_id, occurred_at, evaluation_key
@@ -447,6 +454,7 @@ FROM (
WHERE team_id = %(team_id)s
AND configuration_id = %(configuration_id)s
AND grouping_key = ''
+ AND {CHECKS_ONLY}
AND occurred_at >= toDateTime64(%(since)s, 6, 'UTC')
AND occurred_at < toDateTime64(%(until)s, 6, 'UTC')
ORDER BY alert_id, occurred_at, evaluation_key
@@ -464,6 +472,7 @@ FROM (
WHERE team_id = %(team_id)s
AND configuration_id = %(configuration_id)s
AND grouping_key = ''
+ AND {CHECKS_ONLY}
AND occurred_at >= toDateTime64(%(since)s, 6, 'UTC')
AND occurred_at < toDateTime64(%(until)s, 6, 'UTC')
LIMIT 1 BY alert_id, evaluation_key
@@ -522,6 +531,7 @@ FROM (
WHERE team_id = %(team_id)s
AND configuration_id IN %(configuration_ids)s
AND grouping_key = ''
+ AND {CHECKS_ONLY}
AND occurred_at >= toDateTime64(%(since)s, 6, 'UTC'){has_paths}
ORDER BY configuration_id, alert_id, occurred_at, evaluation_key
LIMIT 1 BY alert_id, evaluation_key
@@ -577,6 +587,7 @@ FROM {PLATFORM_ALERT_EVENTS_TABLE}
WHERE team_id = %(team_id)s
AND configuration_id IN %(configuration_ids)s
AND grouping_key = ''
+ AND {CHECKS_ONLY}
GROUP BY configuration_id
"""
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index e4d7f5e18481..812eb4ffe5fe 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -439,11 +439,12 @@ def event_row(
)
-def record_history(team_id: int, rows: Sequence[PlatformAlertEventRow]) -> None:
+def record_history(team_id: int, rows: Sequence[PlatformAlertEventRow]) -> int:
recorded = insert_events(team_id, rows)
if recorded < len(rows):
# One batch holds one source's checks.
safe_record(increment_history_rows_dropped, rows[0].source_kind, len(rows) - recorded)
+ return recorded
def _shard_offset_seconds(configuration: PlatformAlertConfiguration, outcome: PlatformAlertOutcome) -> int:
diff --git a/products/alerts_platform/backend/logic/verdict_changes.py b/products/alerts_platform/backend/logic/verdict_changes.py
new file mode 100644
index 000000000000..db22047b7c8a
--- /dev/null
+++ b/products/alerts_platform/backend/logic/verdict_changes.py
@@ -0,0 +1,71 @@
+"""A follow-up about a firing that already went out, such as an investigation that changed its verdict.
+
+A check announces once, and history is append-only, so a verdict that arrives after the firing's
+message cannot change what that row said. The follow-up gets a row of its own, under its own
+evaluation key, with the firing's episode. That episode puts the reply in the firing's thread and
+sends it on the firing's delivery path.
+
+The write is that row and nothing else. It moves no state, no schedule and no notified time,
+because the check that fired already moved them.
+"""
+
+from datetime import datetime
+
+from products.alerts_platform.backend.facade.contracts import (
+ AlertDeliveryRequest,
+ AlertEventKind,
+ SourceKind,
+ VerdictChange,
+ VerdictChangeNotRecorded,
+)
+from products.alerts_platform.backend.logic import delivery_holds
+from products.alerts_platform.backend.logic.platform_lifecycle import RULE_METADATA_FIELDS, event_row, record_history
+from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration
+
+
+def record_verdict_change(change: VerdictChange, *, now: datetime) -> AlertDeliveryRequest | None:
+ """Records the follow-up's row and returns the delivery to start, or None when there is nothing to send.
+
+ None when the configuration or its group is gone, or when the firing is still held or was
+ suppressed. Raises `VerdictChangeNotRecorded` when ClickHouse refused the row.
+ """
+ alert = (
+ PlatformAlert.objects.for_team(change.team_id)
+ .select_related("configuration")
+ .defer(*(f"configuration__{name}" for name in RULE_METADATA_FIELDS))
+ .filter(configuration_id=change.configuration_id, grouping_key=change.grouping_key)
+ .first()
+ )
+ if alert is None:
+ return None
+ # A reply to a firing nobody received would arrive with no message above it.
+ if delivery_holds.blocks_delivery(change.team_id, str(change.configuration_id), change.firing_evaluation_key):
+ return None
+ configuration = alert.configuration
+
+ row = event_row(
+ configuration,
+ alert=alert,
+ evaluation_key=change.evaluation_key,
+ kind=AlertEventKind.VERDICT_CHANGED.value,
+ previous_state=alert.state,
+ state=alert.state,
+ episode_started_at=change.episode_started_at,
+ labels=alert.labels,
+ consecutive_failures=configuration.consecutive_failures,
+ now=now,
+ source_detail=change.source_detail,
+ )
+ if not record_history(change.team_id, [row]):
+ raise VerdictChangeNotRecorded(f"no history row for {change.evaluation_key}")
+ return AlertDeliveryRequest(
+ source=SourceKind(configuration.source_kind),
+ team_id=change.team_id,
+ configuration_id=str(configuration.id),
+ evaluation_key=change.evaluation_key,
+ destination_alert_id=str(configuration.destination_alert_id),
+ # The firing's event, so a HogFunction that sent the firing sends the follow-up, as legacy does.
+ event_ids_by_kind={AlertEventKind.VERDICT_CHANGED.value: change.firing_event_id},
+ source_alert_url=change.source_alert_url,
+ owned_by_platform=configuration.owner == PlatformAlertConfiguration.Owner.PLATFORM,
+ )
diff --git a/products/alerts_platform/backend/tests/test_comparison_platform_history.py b/products/alerts_platform/backend/tests/test_comparison_platform_history.py
index 67ca98cf6113..2f1d34c461e6 100644
--- a/products/alerts_platform/backend/tests/test_comparison_platform_history.py
+++ b/products/alerts_platform/backend/tests/test_comparison_platform_history.py
@@ -1,3 +1,4 @@
+from dataclasses import replace
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
@@ -52,11 +53,11 @@ class TestReadPlatformChecks(ClickhouseTestMixin, APIBaseTest):
def _row(
self, configuration: PlatformAlertConfiguration, *, evaluation_key: str, **overrides
) -> PlatformAlertEventRow:
- return PlatformAlertEventRow(
+ row = PlatformAlertEventRow(
team_id=self.team.id,
source_kind="logs",
configuration_id=configuration.id,
- alert_id=overrides.pop("alert_id", uuid4()),
+ alert_id=uuid4(),
grouping_key="",
evaluation_key=evaluation_key,
kind="firing",
@@ -73,8 +74,8 @@ class TestReadPlatformChecks(ClickhouseTestMixin, APIBaseTest):
consecutive_failures=0,
muted_notification="",
occurred_at=OCCURRED_AT,
- **overrides,
)
+ return replace(row, **overrides)
def _record(self, configuration: PlatformAlertConfiguration, *, evaluation_key: str, **overrides) -> None:
insert_events(self.team.id, [self._row(configuration, evaluation_key=evaluation_key, **overrides)])
@@ -158,16 +159,18 @@ class TestReadPlatformChecks(ClickhouseTestMixin, APIBaseTest):
@parameterized.expand(
[
- ("another_source", {"source_kind": "insight"}),
- ("a_grouped_configuration", {"grouping": {"mode": "by_result_labels", "keys": ["service_name"]}}),
+ ("another_source", {"source_kind": "insight"}, {}),
+ ("a_grouped_configuration", {"grouping": {"mode": "by_result_labels", "keys": ["service_name"]}}, {}),
+ ("a_verdict_follow_up", {}, {"kind": "verdict_changed", "previous_state": "firing"}),
]
)
def test_a_check_with_no_comparable_verdict_is_not_in_a_logs_comparison(
- self, _name: str, overrides: dict[str, Any]
+ self, _name: str, overrides: dict[str, Any], row_overrides: dict[str, Any]
) -> None:
self._record(
self._configuration(legacy_configuration_id=uuid4(), **overrides),
evaluation_key="run:2026-09-30T11:59:00+00:00",
+ **row_overrides,
)
assert self._read() == []
diff --git a/products/alerts_platform/backend/tests/test_delivery_webhook.py b/products/alerts_platform/backend/tests/test_delivery_webhook.py
index 709c1aae7331..7c2d210a168a 100644
--- a/products/alerts_platform/backend/tests/test_delivery_webhook.py
+++ b/products/alerts_platform/backend/tests/test_delivery_webhook.py
@@ -90,6 +90,13 @@ class TestAlertmanagerBody(SimpleTestCase):
("resolved", AlertEventKind.RESOLVED, "resolved", "2026-09-30T09:00:00+00:00", "2026-09-30T10:00:00+00:00"),
("errored", AlertEventKind.ERRORED, "firing", "2026-09-30T10:00:00+00:00", "0001-01-01T00:00:00Z"),
("broken", AlertEventKind.BROKEN, "firing", "2026-09-30T10:00:00+00:00", "0001-01-01T00:00:00Z"),
+ (
+ "verdict_changed",
+ AlertEventKind.VERDICT_CHANGED,
+ "resolved",
+ "2026-09-30T09:00:00+00:00",
+ "2026-09-30T10:00:00+00:00",
+ ),
]
)
def test_each_kind_maps_onto_an_alertmanager_status(
@@ -127,6 +134,8 @@ class TestAlertmanagerBody(SimpleTestCase):
assert fingerprint(AlertEventKind.FIRING) == fingerprint(AlertEventKind.RESOLVED)
# A failed check sent as firing would otherwise be closed by the breach's resolve.
assert fingerprint(AlertEventKind.ERRORED) != fingerprint(AlertEventKind.FIRING)
+ # A follow-up goes out ended, so on the breach's fingerprint it would close a firing that is still open.
+ assert fingerprint(AlertEventKind.VERDICT_CHANGED) != fingerprint(AlertEventKind.FIRING)
assert fingerprint(AlertEventKind.FIRING, "search") != fingerprint(AlertEventKind.FIRING)
diff --git a/products/alerts_platform/backend/tests/test_verdict_changes.py b/products/alerts_platform/backend/tests/test_verdict_changes.py
new file mode 100644
index 000000000000..b2db3a5f535e
--- /dev/null
+++ b/products/alerts_platform/backend/tests/test_verdict_changes.py
@@ -0,0 +1,229 @@
+from datetime import UTC, datetime, timedelta
+from typing import Any, cast
+from uuid import uuid4
+
+import pytest
+from posthog.test.base import APIBaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.models.scoping import team_scope
+
+from products.alerts_platform.backend.delivery.message import AlertMessage
+from products.alerts_platform.backend.delivery.transport import MessageHandle
+from products.alerts_platform.backend.facade import testing as platform_testing
+from products.alerts_platform.backend.facade.api import create_alert_destinations, record_verdict_change
+from products.alerts_platform.backend.facade.contracts import (
+ AlertDeliveryRequest,
+ AlertDestinationData,
+ AlertEventKind,
+ DestinationType,
+ NewAlertDestination,
+ SourceKind,
+ VerdictChange,
+ VerdictChangeNotRecorded,
+)
+from products.alerts_platform.backend.facade.enums import HoldRelease, PlatformAlertOwner
+from products.alerts_platform.backend.logic.platform_lifecycle import event_row, record_history
+from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration, PlatformAlertDeliveryHold
+
+FIRING_AT = datetime(2026, 10, 19, 9, tzinfo=UTC)
+FIRING_KEY = f"slot:{FIRING_AT.isoformat()}"
+FIRING_EVENT = "$metrics_alert_firing"
+SLACK = cast(AlertDestinationData, {"type": DestinationType.SLACK, "slack_workspace_id": 1, "slack_channel_id": "C-1"})
+
+
+class ThreadedTransport:
+ sends: list[tuple[AlertMessage, MessageHandle | None]] = []
+ provider = "slack"
+
+ def channel_target(self, target: AlertDestinationData) -> str:
+ return str(target.get("slack_channel_id", ""))
+
+ def deliver(
+ self, *, team_id: int, target: AlertDestinationData, message: AlertMessage, in_reply_to: Any = None
+ ) -> MessageHandle | None:
+ ThreadedTransport.sends.append((message, in_reply_to))
+ return MessageHandle(external_ref={"ts": str(len(ThreadedTransport.sends))})
+
+
+class TestRecordVerdictChange(ClickhouseTestMixin, APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ ThreadedTransport.sends = []
+ self.next_check_at = FIRING_AT + timedelta(minutes=5)
+ with team_scope(self.team.id):
+ self.configuration = PlatformAlertConfiguration.objects.create(
+ team=self.team,
+ name="Checkout latency",
+ source_kind=PlatformAlertConfiguration.SourceKind.METRICS,
+ source_config={},
+ check_interval_minutes=5,
+ next_check_at=self.next_check_at,
+ owner=PlatformAlertOwner.PLATFORM,
+ destinations_owner=PlatformAlertOwner.PLATFORM,
+ )
+ platform_testing.start_firing(self.configuration.id, team_id=self.team.id, at=FIRING_AT)
+ PlatformAlert.objects.filter(configuration=self.configuration).update(last_notified_at=FIRING_AT)
+ create_alert_destinations(
+ team_id=self.team.id,
+ configuration_id=self.configuration.id,
+ destinations=(NewAlertDestination(data=SLACK, event_kinds=(AlertEventKind.FIRING,)),),
+ )
+
+ def _deliver(self, request: AlertDeliveryRequest) -> None:
+ with patch.dict(
+ "products.alerts_platform.backend.delivery.evaluation._TRANSPORTS",
+ {DestinationType.SLACK: ThreadedTransport},
+ ):
+ platform_testing.deliver(request)
+
+ def test_a_follow_up_replies_once_in_the_thread_of_the_firing_it_follows(self) -> None:
+ with team_scope(self.team.id):
+ alert = PlatformAlert.objects.get(configuration=self.configuration)
+ firing_row = event_row(
+ self.configuration,
+ alert=alert,
+ evaluation_key=FIRING_KEY,
+ kind=AlertEventKind.FIRING.value,
+ previous_state="not_firing",
+ state="firing",
+ episode_started_at=FIRING_AT,
+ consecutive_failures=0,
+ now=FIRING_AT,
+ )
+ record_history(self.team.id, [firing_row])
+ self._deliver(
+ AlertDeliveryRequest(
+ source=SourceKind.METRICS,
+ team_id=self.team.id,
+ configuration_id=str(self.configuration.id),
+ evaluation_key=FIRING_KEY,
+ destination_alert_id=str(self.configuration.id),
+ event_ids_by_kind={AlertEventKind.FIRING.value: FIRING_EVENT},
+ owned_by_platform=True,
+ )
+ )
+
+ with team_scope(self.team.id):
+ follow_up = record_verdict_change(
+ VerdictChange(
+ team_id=self.team.id,
+ configuration_id=self.configuration.id,
+ firing_evaluation_key=FIRING_KEY,
+ episode_started_at=FIRING_AT,
+ firing_event_id=FIRING_EVENT,
+ ),
+ now=FIRING_AT + timedelta(minutes=20),
+ )
+ assert follow_up is not None
+ self._deliver(follow_up)
+ self._deliver(follow_up)
+
+ [(firing, opened_in), (reply, replied_to)] = ThreadedTransport.sends
+ assert (firing.transition.kind, opened_in) == (AlertEventKind.FIRING, None)
+ assert reply.headline == "Metric alert 'Checkout latency' has a new verdict"
+ assert replied_to == MessageHandle(external_ref={"ts": "1"})
+ with team_scope(self.team.id):
+ configuration = PlatformAlertConfiguration.objects.get(id=self.configuration.id)
+ alert = PlatformAlert.objects.get(configuration=self.configuration)
+ assert (configuration.next_check_at, alert.state, alert.last_notified_at) == (
+ self.next_check_at,
+ "firing",
+ FIRING_AT,
+ )
+
+ @parameterized.expand([("still_held", None), ("suppressed", HoldRelease.SUPPRESS)])
+ def test_a_firing_nobody_received_gets_no_follow_up(self, _name: str, release: HoldRelease | None) -> None:
+ with team_scope(self.team.id):
+ PlatformAlertDeliveryHold.objects.create(
+ team=self.team,
+ configuration=self.configuration,
+ evaluation_key=FIRING_KEY,
+ reason="investigation",
+ request={},
+ held_at=FIRING_AT,
+ release_after=FIRING_AT + timedelta(minutes=90),
+ released_at=FIRING_AT if release else None,
+ release_action=release,
+ )
+ follow_up = record_verdict_change(
+ VerdictChange(
+ team_id=self.team.id,
+ configuration_id=self.configuration.id,
+ firing_evaluation_key=FIRING_KEY,
+ episode_started_at=FIRING_AT,
+ firing_event_id=FIRING_EVENT,
+ ),
+ now=FIRING_AT + timedelta(minutes=20),
+ )
+
+ assert follow_up is None
+
+ def test_a_follow_up_clickhouse_refused_raises_so_the_caller_can_retry(self) -> None:
+ with (
+ team_scope(self.team.id),
+ patch("products.alerts_platform.backend.logic.platform_lifecycle.insert_events", return_value=0),
+ pytest.raises(VerdictChangeNotRecorded),
+ ):
+ record_verdict_change(
+ VerdictChange(
+ team_id=self.team.id,
+ configuration_id=self.configuration.id,
+ firing_evaluation_key=FIRING_KEY,
+ episode_started_at=FIRING_AT,
+ firing_event_id=FIRING_EVENT,
+ ),
+ now=FIRING_AT + timedelta(minutes=20),
+ )
+
+ def test_a_grouped_follow_up_names_its_group(self) -> None:
+ change = VerdictChange(
+ team_id=self.team.id,
+ configuration_id=self.configuration.id,
+ firing_evaluation_key=FIRING_KEY,
+ episode_started_at=FIRING_AT,
+ firing_event_id=FIRING_EVENT,
+ grouping_key="checkout",
+ )
+ with team_scope(self.team.id):
+ platform_testing.start_firing(
+ self.configuration.id, team_id=self.team.id, at=FIRING_AT, grouping_key="checkout"
+ )
+ PlatformAlert.objects.filter(configuration=self.configuration, grouping_key="checkout").update(
+ labels={"service_name": "checkout"}
+ )
+ with patch(
+ "products.alerts_platform.backend.logic.verdict_changes.record_history", return_value=1
+ ) as recorded:
+ follow_up = record_verdict_change(change, now=FIRING_AT + timedelta(minutes=20))
+
+ [[_team_id, [row]], _] = recorded.call_args
+ assert (row.grouping_key, row.labels, row.evaluation_key) == (
+ "checkout",
+ {"service_name": "checkout"},
+ change.evaluation_key,
+ )
+ assert follow_up is not None and follow_up.evaluation_key == change.evaluation_key
+
+
+class TestVerdictChangeKey(SimpleTestCase):
+ def test_each_group_gets_a_key_that_fits_a_thread(self) -> None:
+ def key(grouping_key: str) -> str:
+ return VerdictChange(
+ team_id=1,
+ configuration_id=uuid4(),
+ firing_evaluation_key=FIRING_KEY,
+ episode_started_at=FIRING_AT,
+ firing_event_id=FIRING_EVENT,
+ grouping_key=grouping_key,
+ ).evaluation_key
+
+ longest = key("a" * 255)
+
+ assert len(longest) <= 255
+ assert longest != key("a" * 254 + "b")
+ assert key("") == f"{FIRING_KEY}#verdict"