Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): drop the groups of a check that ran under an older grouping (#118297)

Jimmy Zhue3cd6f9c79df
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 5218102ebde1..45d963b3aeb5 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -418,7 +418,10 @@ def _decide_grouped(
for verdict in decision.verdicts:
_record_metrics(verdict.outcome.notification, verdict.snapshot.state.value, verdict.group.new_state)
return decision.as_outcome(
- configuration_id=check.id, evaluation_key=_evaluation_key(check, now), query_duration_ms=duration_ms
+ configuration_id=check.id,
+ evaluation_key=_evaluation_key(check, now),
+ grouping=check.grouping,
+ query_duration_ms=duration_ms,
)
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index d9973b2a33fe..b1d9dd50ce36 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -160,6 +160,14 @@ class Grouping:
on_overflow=OnOverflow(stored.get("on_overflow", OnOverflow.DEGRADE_VISIBLE)),
)
+ def splits_like(self, other: Grouping) -> bool:
+ """Whether the two key a check's results the same way, so their instances are interchangeable.
+
+ `max_instances` and `on_overflow` are left out, because they bound how many groups are kept and
+ change no group's key.
+ """
+ return (self.mode, self.keys) == (other.mode, other.keys)
+
def to_stored(self) -> dict[str, Any]:
return {
"mode": self.mode.value,
@@ -545,6 +553,10 @@ class PlatformAlertOutcome:
# A team's checks with the same group are due in one slot, so a source that answers them with
# one query can do so. None spreads the configuration by its own id.
schedule_group: str | None = None
+ # The grouping the check ran under. An edit can change it while the check is in flight, and the
+ # groups of a stale check are keyed for a grouping the configuration no longer has, so they are
+ # dropped. None when the outcome carries no groups to drop.
+ grouping: Grouping | None = None
@classmethod
def ungrouped(
@@ -598,6 +610,7 @@ class PlatformAlertOutcome:
evaluation_key=evaluation_key,
consecutive_failures=consecutive_failures,
groups=groups,
+ grouping=Grouping(),
failure=failure,
error_message=error_message,
query_duration_ms=query_duration_ms,
diff --git a/products/alerts_platform/backend/facade/grouping.py b/products/alerts_platform/backend/facade/grouping.py
index bd4544b98d34..a88fbeab50e4 100644
--- a/products/alerts_platform/backend/facade/grouping.py
+++ b/products/alerts_platform/backend/facade/grouping.py
@@ -18,6 +18,7 @@ from posthog.dataclasses import frozen
from products.alerts_platform.backend.facade.contracts import (
AlertEventKind,
+ Grouping,
GroupOutcome,
InstanceCheckState,
PlatformAlertCheckInput,
@@ -154,7 +155,7 @@ class GroupDecision:
evicted: frozenset[str] = frozenset()
def as_outcome(
- self, *, configuration_id: UUID, evaluation_key: str, query_duration_ms: int | None
+ self, *, configuration_id: UUID, evaluation_key: str, grouping: Grouping, query_duration_ms: int | None
) -> PlatformAlertOutcome:
"""The outcome of a grouped check that ran. A query that ran reset the failure count."""
return PlatformAlertOutcome(
@@ -162,6 +163,7 @@ class GroupDecision:
evaluation_key=evaluation_key,
consecutive_failures=0,
groups=tuple(verdict.group for verdict in self.verdicts),
+ grouping=grouping,
query_duration_ms=query_duration_ms,
overflowed=self.overflowed,
evicted=tuple(sorted(self.evicted)),
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index 10645b791485..db216e5592a5 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -5,6 +5,7 @@ and every write to these rows, stays here. A source never holds one of these mod
"""
from collections.abc import Collection, Sequence
+from dataclasses import replace
from datetime import datetime, timedelta
from typing import Final
from uuid import UUID
@@ -421,6 +422,22 @@ def _check_status(configuration: PlatformAlertConfiguration, outcome: PlatformAl
return PlatformAlertConfiguration.CheckStatus.OK.value
+def _without_stale_groups(
+ configuration: PlatformAlertConfiguration, outcome: PlatformAlertOutcome
+) -> PlatformAlertOutcome:
+ """The outcome, or the outcome with its groups dropped when the check ran under another grouping.
+
+ An edit that changes the grouping deletes the instances, so recording a stale check's groups
+ would bring back instances keyed for the old grouping and announce them. The rest of the
+ outcome still holds: the check ran, so its schedule and status advance.
+ """
+ if outcome.grouping is None:
+ return outcome
+ if outcome.grouping.splits_like(Grouping.from_stored(configuration.grouping)):
+ return outcome
+ return replace(outcome, groups=(), overflowed=0, evicted=())
+
+
def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now: datetime) -> int:
"""Persists a batch's decisions and advances each configuration's schedule.
@@ -456,6 +473,8 @@ def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now:
)
if not configurations:
return 0
+ for configuration in configurations:
+ by_id[str(configuration.id)] = _without_stale_groups(configuration, by_id[str(configuration.id)])
# In outcome order, because the source put the groups it most wants kept first.
returned = [
_InstanceKey(configuration_id=str(configuration.id), grouping_key=group.grouping_key)
diff --git a/products/alerts_platform/backend/logic/platform_writes.py b/products/alerts_platform/backend/logic/platform_writes.py
index 4fc9a6896281..4291cc8a5c98 100644
--- a/products/alerts_platform/backend/logic/platform_writes.py
+++ b/products/alerts_platform/backend/logic/platform_writes.py
@@ -106,7 +106,7 @@ def update_configuration(
current = edit_of(configuration)
edit = _validated(SourceKind(configuration.source_kind), current, change(current))
- regrouped = (edit.grouping.mode, edit.grouping.keys) != (current.grouping.mode, current.grouping.keys)
+ regrouped = not edit.grouping.splits_like(current.grouping)
platform_lifecycle.write_edit(configuration, edit, regrouped=regrouped, now=django_timezone.now())
return platform_reads.get_configuration(
diff --git a/products/alerts_platform/backend/tests/test_platform_lifecycle.py b/products/alerts_platform/backend/tests/test_platform_lifecycle.py
index f4158eab5d9e..87c8bb4ad78e 100644
--- a/products/alerts_platform/backend/tests/test_platform_lifecycle.py
+++ b/products/alerts_platform/backend/tests/test_platform_lifecycle.py
@@ -422,6 +422,26 @@ class TestPlatformAlertLifecycle(ClickhouseTestMixin, APIBaseTest):
)
assert (keys, rows) == (["api"], [("api",)])
+ @parameterized.expand([("stale", ("severity_text",), []), ("current", ("service_name",), ["api", "web"])])
+ def test_a_check_that_ran_under_another_grouping_records_no_groups(
+ self, _name: str, stored_keys: tuple[str, ...], expected_keys: list[str]
+ ) -> None:
+ checked_under = Grouping(mode=GroupingMode.BY_RESULT_LABELS, keys=("service_name",))
+ with team_scope(self.team.id):
+ PlatformAlertConfiguration.objects.filter(id=self.configuration.id).update(
+ grouping=Grouping(mode=GroupingMode.BY_RESULT_LABELS, keys=stored_keys).to_stored()
+ )
+
+ self._record(groups=(self._group("api", state="firing"), self._group("web")), grouping=checked_under)
+
+ with team_scope(self.team.id):
+ keys = sorted(
+ PlatformAlert.objects.filter(configuration=self.configuration).values_list("grouping_key", flat=True)
+ )
+ due = PlatformAlertConfiguration.objects.get(id=self.configuration.id).next_check_at
+ assert keys == expected_keys
+ assert due is not None and due > self.cutoff
+
@parameterized.expand(
[
("idle_past_the_window", {}, False),
@@ -910,7 +930,9 @@ def test_a_full_grouped_check_with_hostile_labels_fits_one_payload() -> None:
policy=LOGS_ALERT_POLICY,
now=datetime(2026, 10, 15, tzinfo=UTC),
)
- outcome = decision.as_outcome(configuration_id=check.id, evaluation_key="window:x", query_duration_ms=1)
+ outcome = decision.as_outcome(
+ configuration_id=check.id, evaluation_key="window:x", grouping=check.grouping, query_duration_ms=1
+ )
assert len(outcome.groups) == MAX_INSTANCES_CEILING
assert payload_bytes(outcome) < MAX_BATCH_PAYLOAD_BYTES
diff --git a/products/logs/backend/alert_source_cycle.py b/products/logs/backend/alert_source_cycle.py
index fb52ceee00c4..4d0461623c1a 100644
--- a/products/logs/backend/alert_source_cycle.py
+++ b/products/logs/backend/alert_source_cycle.py
@@ -851,6 +851,7 @@ def _evaluate_grouped(
recorded = decision.as_outcome(
configuration_id=check.id,
evaluation_key=_evaluation_key(check, date_to),
+ grouping=check.grouping,
query_duration_ms=result.query_duration_ms,
)
return recorded, _request(