Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): store and show how many groups an alert could not track (#119979)

Jimmy Zhu4e8ef20f01dc
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 0d0a592d521b..10bda809183d 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -381,6 +381,7 @@ def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) ->
event_ids_by_kind=_EVENT_IDS_BY_KIND,
owned_by_platform=check.owned_by_platform,
overflowed=outcome.overflowed,
+ overflow_is_lower_bound=outcome.overflow_is_lower_bound,
parts=delivery_parts(outcome),
)
@@ -639,6 +640,7 @@ def _decide_grouped(
evaluation_key=_evaluation_key(check, now),
grouping=check.grouping,
query_duration_ms=duration_ms,
+ results_cut=evaluated.truncated,
)
diff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.py
index 77bedd2a54b8..f0a1d9e1db8e 100644
--- a/products/alerts_platform/backend/delivery/evaluation.py
+++ b/products/alerts_platform/backend/delivery/evaluation.py
@@ -246,7 +246,12 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
evaluation_key=request.evaluation_key,
target=target,
destination_id=subscribed_destination.id,
- announcement=replace(announced, transitions=transitions, overflowed=overflowed),
+ announcement=replace(
+ announced,
+ transitions=transitions,
+ overflowed=overflowed,
+ overflow_is_lower_bound=request.overflow_is_lower_bound,
+ ),
incident_action=action_by_event_id.get(event_id),
source_alert_url=request.source_alert_url,
)
diff --git a/products/alerts_platform/backend/delivery/message.py b/products/alerts_platform/backend/delivery/message.py
index 412941f3744d..d5eba7405616 100644
--- a/products/alerts_platform/backend/delivery/message.py
+++ b/products/alerts_platform/backend/delivery/message.py
@@ -179,10 +179,13 @@ def build_message(
else:
details = list(description.details or _breach_details(transition))
if overflowed:
+ untracked = pluralize(overflowed, "more group")
+ if announcement.overflow_is_lower_bound:
+ untracked = f"At least {untracked}"
details.append(
MessageDetail(
label="Untracked groups",
- value=f"{pluralize(overflowed, 'more group')} not tracked because this alert is at its group limit",
+ value=f"{untracked} not tracked because this alert is at its group limit",
)
)
return AlertMessage(
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index 4ef3660e9fd5..9ee011bb223b 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -605,6 +605,8 @@ class PlatformAlertOutcome:
skipped: bool = False
# New groups `max_instances` had no room for, which the delivery names so a cap is never silent.
overflowed: int = 0
+ # The source cut its results before admission, so `overflowed` counts only the groups it saw.
+ overflow_is_lower_bound: bool = False
# Resting instances whose slot a new group took. The write deletes them before it creates rows.
evicted: tuple[str, ...] = ()
error_message: str | None = None
@@ -761,6 +763,7 @@ class EvaluationAnnouncement:
# Groups the configuration had no room for. The first message names them, so a cap never drops
# a group without anyone being told.
overflowed: int = 0
+ overflow_is_lower_bound: bool = False
@frozen
@@ -803,6 +806,7 @@ class AlertDeliveryRequest:
source_alert_url: str | None = None
owned_by_platform: bool = False
overflowed: int = 0
+ overflow_is_lower_bound: bool = False
# How many activities the delivery workflow splits an evaluation into, by `MAX_TRANSITIONS_PER_DELIVERY`,
# and which of them one activity sends. The workflow sets `part`.
parts: int = 1
@@ -850,6 +854,7 @@ class PlatformAlertSnapshot:
firing_started_at: datetime | None
last_notified_at: datetime | None
snooze_until: datetime | None
+ labels: dict[str, str] = field(default_factory=dict)
@frozen
@@ -878,6 +883,8 @@ class PlatformAlertConfigurationView:
grouping: Grouping
snooze_until: datetime | None
check_status: str
+ overflowed_groups: int
+ overflow_is_lower_bound: bool
owner: str
created_at: datetime
updated_at: datetime
diff --git a/products/alerts_platform/backend/facade/grouping.py b/products/alerts_platform/backend/facade/grouping.py
index b8dac220a56e..385128156768 100644
--- a/products/alerts_platform/backend/facade/grouping.py
+++ b/products/alerts_platform/backend/facade/grouping.py
@@ -5,6 +5,7 @@ every source, so it lives here once: run the machine per group, leave out a new
nothing, put firing groups first, and admit what `max_instances` has room for.
"""
+import re
import json
import math
import hashlib
@@ -47,16 +48,9 @@ def is_hashed_grouping_key(key: str) -> bool:
return key.startswith(_HASHED_PREFIX)
-def _storable(key: str) -> bool:
- # A PostgreSQL text column rejects NUL, and UTF-8 cannot encode a lone surrogate. Either one
- # aborts the transaction that records the whole batch.
- if "\x00" in key:
- return False
- try:
- key.encode()
- except UnicodeEncodeError:
- return False
- return True
+# PostgreSQL text and JSONB reject NUL, and UTF-8 cannot encode a surrogate. Either one aborts the
+# transaction that records the whole batch.
+_UNSTORABLE: Final = re.compile("[\x00\ud800-\udfff]")
def bounded_grouping_key(key: str) -> str:
@@ -66,7 +60,7 @@ def bounded_grouping_key(key: str) -> str:
Every other key is hashed. A literal key in the hash namespace is hashed too, because otherwise
it would share a row with the long key whose digest it spells.
"""
- if len(key) <= MAX_GROUPING_KEY_LENGTH and not is_hashed_grouping_key(key) and _storable(key):
+ if len(key) <= MAX_GROUPING_KEY_LENGTH and not is_hashed_grouping_key(key) and not _UNSTORABLE.search(key):
return key
return f"{_HASHED_PREFIX}{hashlib.sha256(key.encode('utf-8', 'surrogatepass')).hexdigest()}"
@@ -81,6 +75,8 @@ _TRUNCATED: Final = "…"
def _bounded_label(value: str) -> str:
+ # A label is display only, so a character the database cannot store shows as a replacement.
+ value = _UNSTORABLE.sub("\ufffd", value)
if _encoded_length(value) <= MAX_LABEL_BYTES:
return value
budget = MAX_LABEL_BYTES - _encoded_length(_TRUNCATED)
@@ -98,7 +94,7 @@ def _encoded_length(value: str) -> int:
def bounded_labels(labels: dict[str, str]) -> dict[str, str]:
- """The labels a group may carry: the first `MAX_GROUP_LABELS`, each name and value cut to size.
+ """The labels a group may carry: the first `MAX_GROUP_LABELS`, each name and value storable and cut to size.
Display only. The grouping key is derived from the full values, so a cut label never merges two groups.
"""
@@ -160,9 +156,19 @@ class GroupDecision:
evicted: frozenset[str] = frozenset()
def as_outcome(
- self, *, configuration_id: UUID, evaluation_key: str, grouping: Grouping, query_duration_ms: int | None
+ self,
+ *,
+ configuration_id: UUID,
+ evaluation_key: str,
+ grouping: Grouping,
+ query_duration_ms: int | None,
+ results_cut: bool,
) -> PlatformAlertOutcome:
- """The outcome of a grouped check that ran. A query that ran reset the failure count."""
+ """The outcome of a grouped check that ran. A query that ran reset the failure count.
+
+ `results_cut` says the source dropped results before they were observed. Groups it dropped
+ may also have been turned away, so a non-zero overflow is then only a lower bound.
+ """
return PlatformAlertOutcome(
configuration_id=configuration_id,
evaluation_key=evaluation_key,
@@ -171,6 +177,7 @@ class GroupDecision:
grouping=grouping,
query_duration_ms=query_duration_ms,
overflowed=self.overflowed,
+ overflow_is_lower_bound=results_cut and self.overflowed > 0,
evicted=tuple(sorted(self.evicted)),
)
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index 31ea542cc139..d0ae655888af 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -476,7 +476,7 @@ def _without_stale_groups(
return outcome
if outcome.grouping.splits_like(Grouping.from_stored(configuration.grouping)):
return outcome
- return replace(outcome, groups=(), overflowed=0, evicted=())
+ return replace(outcome, groups=(), overflowed=0, overflow_is_lower_bound=False, evicted=())
_HELD_BY_MUTE: Final = {
@@ -561,15 +561,22 @@ def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now:
rows: list[PlatformAlertEventRow] = []
for configuration in configurations:
outcome = by_id[str(configuration.id)]
+ turned_away = 0
for reported in outcome.groups:
alert = alerts.get(
_InstanceKey(configuration_id=str(configuration.id), grouping_key=reported.grouping_key)
)
if alert is None:
+ # `max_instances` fell after the source admitted this group, so it overflowed here instead.
+ turned_away += 1
continue
group = _held_by_mute(configuration, alert, reported, now)
touched.append(alert)
alert.last_seen_at = now
+ # A hashed key's group that a check did not return arrives without labels, and its
+ # labels have not changed, so the stored ones stay.
+ if group.labels:
+ alert.labels = group.labels
# Before either row is mutated, so the history row keeps the state the check found.
rows.append(
_event_row(
@@ -606,6 +613,9 @@ def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now:
)
configuration.check_status = _check_status(configuration, outcome)
configuration.consecutive_failures = outcome.consecutive_failures
+ if outcome.failure is None and not outcome.skipped:
+ configuration.overflowed_groups = outcome.overflowed + turned_away
+ configuration.overflow_is_lower_bound = outcome.overflow_is_lower_bound
if outcome.disable:
configuration.enabled = False
configuration.next_check_at = advance_schedule(
@@ -623,10 +633,18 @@ def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now:
# Batched, because a grouped check writes a row per group rather than one per configuration.
PlatformAlert.objects.for_team(team_id).bulk_update(
- touched, ["state", "last_notified_at", "firing_started_at", "last_seen_at"], batch_size=500
+ touched, ["state", "last_notified_at", "firing_started_at", "last_seen_at", "labels"], batch_size=500
)
PlatformAlertConfiguration.objects.for_team(team_id).bulk_update(
- configurations, ["check_status", "consecutive_failures", "enabled", "next_check_at"]
+ configurations,
+ [
+ "check_status",
+ "consecutive_failures",
+ "enabled",
+ "next_check_at",
+ "overflowed_groups",
+ "overflow_is_lower_bound",
+ ],
)
# `on_commit` rather than a statement after the block, so a caller that wraps this in its
# own `atomic()` cannot leave history for state its rollback removed.
@@ -706,8 +724,7 @@ def upsert_configuration(upsert: PlatformAlertUpsert, *, revive: bool = False) -
# episode nobody was told about.
revived = existing is not None and upsert.enabled and (revive or not existing["enabled"])
if revived:
- defaults["consecutive_failures"] = 0
- defaults["check_status"] = PlatformAlertConfiguration.CheckStatus.OK
+ defaults.update(_CLEAN_START)
# After any check already running, so an outcome it read before the source came back
# on, such as a stop, finds the row no longer due and records nothing.
defaults["next_check_at"] = max(
@@ -725,6 +742,16 @@ def upsert_configuration(upsert: PlatformAlertUpsert, *, revive: bool = False) -
return created
+# What one check found about its groups. It says nothing once the alert regroups or comes back on.
+_NO_OVERFLOW: Final = {"overflowed_groups": 0, "overflow_is_lower_bound": False}
+# The state an alert that comes back on starts from, whichever path turns it back on.
+_CLEAN_START: Final = {
+ "consecutive_failures": 0,
+ "check_status": PlatformAlertConfiguration.CheckStatus.OK,
+ **_NO_OVERFLOW,
+}
+
+
def write_edit(
configuration: PlatformAlertConfiguration,
edit: PlatformAlertConfigurationEdit,
@@ -757,13 +784,16 @@ def write_edit(
"cooldown_minutes",
"updated_at",
]
+ reenabled = edit.enabled and not configuration.enabled
# An alert turned back on starts clean, as a revived copy does. A BROKEN one would otherwise stay
# out of discovery for good.
- if edit.enabled and not configuration.enabled:
- configuration.consecutive_failures = 0
- configuration.check_status = PlatformAlertConfiguration.CheckStatus.OK
+ cleared = _CLEAN_START if reenabled else _NO_OVERFLOW if regrouped else {}
+ for field, value in cleared.items():
+ setattr(configuration, field, value)
+ update_fields += list(cleared)
+ if reenabled:
configuration.next_check_at = now + _REVIVE_FENCE
- update_fields += ["consecutive_failures", "check_status", "next_check_at"]
+ update_fields.append("next_check_at")
configuration.name = edit.name
configuration.enabled = edit.enabled
configuration.source_config = edit.source_config
diff --git a/products/alerts_platform/backend/logic/platform_reads.py b/products/alerts_platform/backend/logic/platform_reads.py
index b95bfad77a21..1e35366a2ce0 100644
--- a/products/alerts_platform/backend/logic/platform_reads.py
+++ b/products/alerts_platform/backend/logic/platform_reads.py
@@ -48,6 +48,7 @@ def instance_view(alert: PlatformAlert, configuration: PlatformAlertConfiguratio
firing_started_at=alert.firing_started_at,
last_notified_at=alert.last_notified_at,
snooze_until=seen.snooze_until,
+ labels=alert.labels,
)
@@ -71,6 +72,8 @@ def _configuration_view(configuration: PlatformAlertConfiguration) -> PlatformAl
grouping=Grouping.from_stored(configuration.grouping),
snooze_until=configuration.snooze_until,
check_status=configuration.check_status,
+ overflowed_groups=configuration.overflowed_groups,
+ overflow_is_lower_bound=configuration.overflow_is_lower_bound,
owner=configuration.owner,
created_at=configuration.created_at,
updated_at=configuration.updated_at,
diff --git a/products/alerts_platform/backend/migrations/0019_overflow_and_instance_labels.py b/products/alerts_platform/backend/migrations/0019_overflow_and_instance_labels.py
new file mode 100644
index 000000000000..10cc42a73704
--- /dev/null
+++ b/products/alerts_platform/backend/migrations/0019_overflow_and_instance_labels.py
@@ -0,0 +1,27 @@
+# Generated by Django 5.2.18 on 2026-10-20 00:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("alerts_platform", "0018_alert_destination"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="platformalert",
+ name="labels",
+ field=models.JSONField(db_default={}, default=dict),
+ ),
+ migrations.AddField(
+ model_name="platformalertconfiguration",
+ name="overflow_is_lower_bound",
+ field=models.BooleanField(db_default=False, default=False),
+ ),
+ migrations.AddField(
+ model_name="platformalertconfiguration",
+ name="overflowed_groups",
+ field=models.PositiveIntegerField(db_default=0, default=0),
+ ),
+ ]
diff --git a/products/alerts_platform/backend/migrations/max_migration.txt b/products/alerts_platform/backend/migrations/max_migration.txt
index b0b4c0094007..fe91e7572cbf 100644
--- a/products/alerts_platform/backend/migrations/max_migration.txt
+++ b/products/alerts_platform/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0018_alert_destination
+0019_overflow_and_instance_labels
diff --git a/products/alerts_platform/backend/models/platform_alert.py b/products/alerts_platform/backend/models/platform_alert.py
index 268b3c18b950..83c05b8031b5 100644
--- a/products/alerts_platform/backend/models/platform_alert.py
+++ b/products/alerts_platform/backend/models/platform_alert.py
@@ -95,6 +95,11 @@ class PlatformAlertConfiguration(ModelActivityMixin, TeamScopedRootMixin, UUIDMo
check_status = models.CharField(
max_length=16, choices=CheckStatus.choices, default=CheckStatus.OK.value, db_default="ok"
)
+ # New groups the last check that ran turned away at `max_instances`. A failed or skipped check
+ # leaves both as they were, because it measured no groups.
+ overflowed_groups = models.PositiveIntegerField(default=0, db_default=0)
+ # The source cut its results before admission saw them all, so more groups may have been turned away.
+ overflow_is_lower_bound = models.BooleanField(default=False, db_default=False)
# The row this was copied from, so a backfill can run twice and so a comparison can line
# an evaluation up against the one the source's own stack produced.
@@ -177,6 +182,9 @@ class PlatformAlert(TeamScopedRootMixin, UUIDModel):
# The last check that returned this group. Reaping reads it to free the slot of a group that
# stopped appearing. Null for a row no check has recorded since the column existed.
last_seen_at = models.DateTimeField(null=True, blank=True)
+ # The result labels whose values make `grouping_key`. Kept because a hashed key cannot be decoded
+ # back into them.
+ labels = models.JSONField(default=dict, db_default={})
class Meta:
db_table = "alerts_platformalert"
diff --git a/products/alerts_platform/backend/presentation/views/platform_alert.py b/products/alerts_platform/backend/presentation/views/platform_alert.py
index d5fa683e7ede..20e688a80e11 100644
--- a/products/alerts_platform/backend/presentation/views/platform_alert.py
+++ b/products/alerts_platform/backend/presentation/views/platform_alert.py
@@ -79,6 +79,12 @@ class PlatformAlertSerializer(serializers.Serializer):
allow_null=True,
help_text="Time until which notifications are snoozed. Null when not snoozed.",
)
+ labels = serializers.DictField(
+ child=serializers.CharField(),
+ read_only=True,
+ help_text="Result labels and their values for the group this instance tracks, such as "
+ '{"service_name": "checkout"}. Empty when the source does not group results.',
+ )
class GroupingSerializer(serializers.Serializer):
@@ -190,6 +196,16 @@ class PlatformAlertConfigurationSerializer(serializers.Serializer):
help_text="Whether the alert's checks succeed. Holds for every instance, because a failed check "
"names no group.",
)
+ overflowed_groups = serializers.IntegerField(
+ read_only=True,
+ help_text="New groups the last successful check could not track because the alert already held "
+ "grouping.max_instances instances. 0 when every group fit.",
+ )
+ overflow_is_lower_bound = serializers.BooleanField(
+ read_only=True,
+ help_text="Whether the last successful check cut its results before counting overflowed_groups. "
+ "When true, more groups than overflowed_groups may be untracked.",
+ )
owner = serializers.ChoiceField(
choices=PlatformAlertOwner.choices,
read_only=True,
diff --git a/products/alerts_platform/backend/tests/api/test_platform_alert.py b/products/alerts_platform/backend/tests/api/test_platform_alert.py
index d7bf826b78fb..de9d75b21411 100644
--- a/products/alerts_platform/backend/tests/api/test_platform_alert.py
+++ b/products/alerts_platform/backend/tests/api/test_platform_alert.py
@@ -74,6 +74,10 @@ class TestPlatformAlertAPI(APIBaseTest):
state=PlatformAlert.State.FIRING,
firing_started_at=firing_started_at,
last_notified_at=firing_started_at,
+ labels={"service_name": "checkout"},
+ )
+ PlatformAlertConfiguration.objects.filter(id=configuration.id).update(
+ overflowed_groups=2, overflow_is_lower_bound=True
)
list_response = self.client.get(f"/api/projects/{self.team.id}/platform_alerts/")
@@ -81,6 +85,7 @@ class TestPlatformAlertAPI(APIBaseTest):
results = list_response.json()["results"]
assert [r["id"] for r in results] == [str(configuration.id)]
assert results[0]["legacy_configuration_id"] == str(legacy_configuration_id)
+ assert (results[0]["overflowed_groups"], results[0]["overflow_is_lower_bound"]) == (2, True)
assert results[0]["alerts"] == [
{
"id": str(alert.id),
@@ -89,6 +94,7 @@ class TestPlatformAlertAPI(APIBaseTest):
"firing_started_at": "2026-09-16T10:00:00Z",
"last_notified_at": "2026-09-16T10:00:00Z",
"snooze_until": None,
+ "labels": {"service_name": "checkout"},
}
]
@@ -276,6 +282,9 @@ class TestPlatformAlertAPI(APIBaseTest):
configuration = self._create_configuration(self.team, "API errors", owner="platform")
with team_scope(self.team.id):
PlatformAlert.objects.create(team=self.team, configuration=configuration, state=PlatformAlert.State.FIRING)
+ PlatformAlertConfiguration.objects.filter(id=configuration.id).update(
+ overflowed_groups=4, overflow_is_lower_bound=True
+ )
PlatformAlertThread.objects.create(
team=self.team,
configuration=configuration,
@@ -299,6 +308,7 @@ class TestPlatformAlertAPI(APIBaseTest):
"on_overflow": "degrade_visible",
}
assert (body["name"], body["alerts"]) == ("API errors", [])
+ assert (body["overflowed_groups"], body["overflow_is_lower_bound"]) == (0, False)
with team_scope(self.team.id):
assert not PlatformAlertThread.objects.filter(configuration=configuration).exists()
diff --git a/products/alerts_platform/backend/tests/test_delivery_evaluation.py b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
index a98207caf1cb..4e8ae5104343 100644
--- a/products/alerts_platform/backend/tests/test_delivery_evaluation.py
+++ b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
@@ -107,6 +107,7 @@ def _request(
incident_actions: dict[str, IncidentAction] | None = None,
owned_by_platform: bool = False,
overflowed: int = 0,
+ overflow_is_lower_bound: bool = False,
parts: int = 1,
part: int = 0,
) -> AlertDeliveryRequest:
@@ -122,6 +123,7 @@ def _request(
event_ids_by_incident_action={"trigger": OPENED_EVENT, "resolve": CLOSED_EVENT},
owned_by_platform=owned_by_platform,
overflowed=overflowed,
+ overflow_is_lower_bound=overflow_is_lower_bound,
parts=parts,
part=part,
)
@@ -174,6 +176,7 @@ class TestDeliverEvaluation(APIBaseTest):
incident_actions: dict[str, IncidentAction] | None = None,
owned_by_platform: bool = False,
overflowed: int = 0,
+ overflow_is_lower_bound: bool = False,
parts: int = 1,
part: int = 0,
kinds: dict[str, AlertEventKind] | None = None,
@@ -208,6 +211,7 @@ class TestDeliverEvaluation(APIBaseTest):
incident_actions=incident_actions,
owned_by_platform=owned_by_platform,
overflowed=overflowed,
+ overflow_is_lower_bound=overflow_is_lower_bound,
parts=parts,
part=part,
)
@@ -251,22 +255,38 @@ class TestDeliverEvaluation(APIBaseTest):
[str(firing_function), str(resolved_function)]
)
- def test_a_destination_subscribed_to_both_hears_about_both(self) -> None:
+ @parameterized.expand(
+ [
+ ("exact", False, "3 more groups not tracked because this alert is at its group limit"),
+ ("lower_bound", True, "At least 3 more groups not tracked because this alert is at its group limit"),
+ ]
+ )
+ def test_a_destination_subscribed_to_both_hears_about_both(
+ self, _name: str, overflow_is_lower_bound: bool, notice: str
+ ) -> None:
announced = _announcement(
_transition(AlertEventKind.FIRING, "checkout"),
_transition(AlertEventKind.RESOLVED, "search"),
)
- self._run(announced, {FIRING_EVENT: [_group(SLACK)], RESOLVED_EVENT: [_group(SLACK)]}, overflowed=3)
+ self._run(
+ announced,
+ {FIRING_EVENT: [_group(SLACK)], RESOLVED_EVENT: [_group(SLACK)]},
+ overflowed=3,
+ overflow_is_lower_bound=overflow_is_lower_bound,
+ )
assert sorted(m.headline for _, m in RecordingTransport.sends) == [
"Log alert 'API errors' is firing",
"Log alert 'API errors' is resolved",
]
overflow_notices = [
- detail for _, m in RecordingTransport.sends for detail in m.details if detail.label == "Untracked groups"
+ detail.value
+ for _, m in RecordingTransport.sends
+ for detail in m.details
+ if detail.label == "Untracked groups"
]
- assert len(overflow_notices) == 1
+ assert overflow_notices == [notice]
def test_each_destination_hears_the_overflow_once_across_the_parts_of_a_split_delivery(self) -> None:
# Sorted by key, the firing lands in part 0 and the resolve in part 1, so a destination
diff --git a/products/alerts_platform/backend/tests/test_platform_lifecycle.py b/products/alerts_platform/backend/tests/test_platform_lifecycle.py
index 602b98d6be60..7e2ee760896b 100644
--- a/products/alerts_platform/backend/tests/test_platform_lifecycle.py
+++ b/products/alerts_platform/backend/tests/test_platform_lifecycle.py
@@ -16,6 +16,7 @@ from products.alerts_platform.backend.facade import testing as platform_testing
from products.alerts_platform.backend.facade.api import (
disable_configurations,
due_checks,
+ get_configuration,
give_back_ownership,
record_outcomes,
slot_of,
@@ -44,6 +45,7 @@ from products.alerts_platform.backend.facade.enums import PlatformAlertOwner, Pl
from products.alerts_platform.backend.facade.grouping import (
MAX_TRANSITIONS_PER_DELIVERY,
GroupObservation,
+ bounded_labels,
decide_groups,
delivery_parts,
)
@@ -509,17 +511,77 @@ class TestPlatformAlertLifecycle(ClickhouseTestMixin, APIBaseTest):
grouping=Grouping(mode=GroupingMode.BY_RESULT_LABELS, keys=("service",), max_instances=1).to_stored()
)
- self._record(groups=(self._group("api"), self._group("web")))
+ self._record(groups=(self._group("api"), self._group("web")), overflowed=2)
with team_scope(self.team.id):
keys = list(
PlatformAlert.objects.filter(configuration=self.configuration).values_list("grouping_key", flat=True)
)
+ self.configuration.refresh_from_db()
rows = sync_execute(
"SELECT grouping_key FROM platform_alert_events WHERE team_id = %(team_id)s AND configuration_id = %(configuration_id)s",
{"team_id": self.team.id, "configuration_id": self.configuration.id},
)
- assert (keys, rows) == (["api"], [("api",)])
+ # The source turned away 2 groups, and the write turned away `web`.
+ assert (keys, rows, self.configuration.overflowed_groups) == (["api"], [("api",)], 3)
+
+ @parameterized.expand(
+ [
+ ("cleared_by_a_check_that_fits", {}, (0, False)),
+ (
+ "kept_by_a_failed_check",
+ {
+ "groups": (),
+ "failure": CheckFailure(kind=AlertEventKind.ERRORED, new_state="errored", notified=False),
+ "consecutive_failures": 1,
+ },
+ (3, True),
+ ),
+ ("kept_by_a_skipped_check", {"groups": (), "skipped": True}, (3, True)),
+ ]
+ )
+ def test_the_configuration_shows_the_overflow_of_the_last_check_that_ran(
+ self, _name: str, next_check: dict[str, Any], expected: tuple[int, bool]
+ ) -> None:
+ def shown() -> tuple[int, bool]:
+ with team_scope(self.team.id):
+ view = get_configuration(
+ team_id=self.team.id, source_kinds=[SourceKind.LOGS.value], configuration_id=self.configuration.id
+ )
+ assert view is not None
+ return view.overflowed_groups, view.overflow_is_lower_bound
+
+ self._record(overflowed=3, overflow_is_lower_bound=True)
+ assert shown() == (3, True)
+
+ self._record(at=self._next_due(), **next_check)
+
+ assert shown() == expected
+
+ def test_an_instance_keeps_its_labels_when_a_check_returns_its_group_without_them(self) -> None:
+ labelled = GroupOutcome(
+ grouping_key="api",
+ kind=AlertEventKind.CHECK,
+ new_state="not_firing",
+ notified=False,
+ labels={"service_name": "api"},
+ )
+ self._record(groups=(labelled,))
+ assert self._alert("api").labels == {"service_name": "api"}
+
+ self._record(at=self._next_due(), groups=(self._group("api"),))
+
+ assert self._alert("api").labels == {"service_name": "api"}
+
+ @parameterized.expand([("nul", "\x00"), ("lone_surrogate", "\ud800")])
+ def test_an_instance_stores_labels_the_database_cannot_hold_as_replacement_characters(
+ self, _name: str, unstorable: str
+ ) -> None:
+ labels = bounded_labels({f"service{unstorable}name": f"api{unstorable}"})
+
+ self._record(groups=(dataclasses.replace(self._group("api"), labels=labels),))
+
+ assert self._alert("api").labels == {"service\ufffdname": "api\ufffd"}
@parameterized.expand([("stale", ("severity_text",), []), ("current", ("service_name",), ["api", "web"])])
def test_a_check_that_ran_under_another_grouping_records_no_groups(
@@ -1030,7 +1092,11 @@ def test_a_full_grouped_check_with_hostile_labels_fits_one_payload() -> None:
now=datetime(2026, 10, 15, tzinfo=UTC),
)
outcome = decision.as_outcome(
- configuration_id=check.id, evaluation_key="window:x", grouping=check.grouping, query_duration_ms=1
+ configuration_id=check.id,
+ evaluation_key="window:x",
+ grouping=check.grouping,
+ query_duration_ms=1,
+ results_cut=False,
)
assert len(outcome.groups) == MAX_INSTANCES_CEILING
diff --git a/products/alerts_platform/frontend/PlatformAlerts.stories.tsx b/products/alerts_platform/frontend/PlatformAlerts.stories.tsx
index 679716337a56..41b538607043 100644
--- a/products/alerts_platform/frontend/PlatformAlerts.stories.tsx
+++ b/products/alerts_platform/frontend/PlatformAlerts.stories.tsx
@@ -30,6 +30,8 @@ const CONFIGURATION: PlatformAlertConfigurationApi = {
grouping: { mode: 'single', keys: [], max_instances: 100, on_overflow: 'degrade_visible' },
snooze_until: null,
check_status: 'ok',
+ overflowed_groups: 0,
+ overflow_is_lower_bound: false,
owner: 'source',
created_at: '2026-09-20T09:00:00Z',
updated_at: '2026-10-01T09:00:00Z',
@@ -41,6 +43,7 @@ const CONFIGURATION: PlatformAlertConfigurationApi = {
firing_started_at: '2026-10-05T11:42:00Z',
last_notified_at: '2026-10-05T11:42:05Z',
snooze_until: null,
+ labels: {},
},
],
}
diff --git a/products/alerts_platform/frontend/generated/api.schemas.ts b/products/alerts_platform/frontend/generated/api.schemas.ts
index 3d1b29acf6e6..40b879a40329 100644
--- a/products/alerts_platform/frontend/generated/api.schemas.ts
+++ b/products/alerts_platform/frontend/generated/api.schemas.ts
@@ -127,6 +127,11 @@ export const BillingAlertConfigurationStateEnumApi = {
Broken: 'broken',
} as const
+/**
+ * Result labels and their values for the group this instance tracks, such as {"service_name": "checkout"}. Empty when the source does not group results.
+ */
+export type PlatformAlertApiLabels = { [key: string]: string }
+
export interface PlatformAlertApi {
/** Unique identifier of this alert instance. */
readonly id: string
@@ -155,6 +160,8 @@ export interface PlatformAlertApi {
* @nullable
*/
readonly snooze_until: string | null
+ /** Result labels and their values for the group this instance tracks, such as {"service_name": "checkout"}. Empty when the source does not group results. */
+ readonly labels: PlatformAlertApiLabels
}
/**
@@ -222,6 +229,10 @@ export interface PlatformAlertConfigurationApi {
* * `errored` - Errored
* * `broken` - Broken */
readonly check_status: PlatformAlertCheckStatusEnumApi
+ /** New groups the last successful check could not track because the alert already held grouping.max_instances instances. 0 when every group fit. */
+ readonly overflowed_groups: number
+ /** Whether the last successful check cut its results before counting overflowed_groups. When true, more groups than overflowed_groups may be untracked. */
+ readonly overflow_is_lower_bound: boolean
/** Which stack evaluates and delivers the alert. Only an alert the platform owns can be edited here. The source product's own API edits the rest.
*
* * `source` - Source
diff --git a/products/alerts_platform/frontend/platformAlertFormat.test.ts b/products/alerts_platform/frontend/platformAlertFormat.test.ts
index 380c5f4a354c..7482dc301b45 100644
--- a/products/alerts_platform/frontend/platformAlertFormat.test.ts
+++ b/products/alerts_platform/frontend/platformAlertFormat.test.ts
@@ -31,6 +31,8 @@ function makeConfiguration(overrides: Partial<PlatformAlertConfigurationApi> = {
grouping: { mode: 'single', keys: [], max_instances: 100, on_overflow: 'degrade_visible' },
snooze_until: null,
check_status: 'ok',
+ overflowed_groups: 0,
+ overflow_is_lower_bound: false,
owner: 'source',
created_at: '2026-10-01T00:00:00Z',
updated_at: '2026-10-01T00:00:00Z',
@@ -47,6 +49,7 @@ function makeAlert(state: PlatformAlertStateApi): PlatformAlertApi {
firing_started_at: null,
last_notified_at: null,
snooze_until: null,
+ labels: {},
}
}
diff --git a/products/logs/backend/alert_source_cycle.py b/products/logs/backend/alert_source_cycle.py
index af7e054e70cc..4b55b4ff9f21 100644
--- a/products/logs/backend/alert_source_cycle.py
+++ b/products/logs/backend/alert_source_cycle.py
@@ -478,6 +478,7 @@ def _request(
else None,
owned_by_platform=check.owned_by_platform,
overflowed=recorded.overflowed,
+ overflow_is_lower_bound=recorded.overflow_is_lower_bound,
parts=delivery_parts(recorded, actions),
)
@@ -857,6 +858,7 @@ def _evaluate_grouped(
evaluation_key=_evaluation_key(check, date_to),
grouping=check.grouping,
query_duration_ms=result.query_duration_ms,
+ results_cut=result.truncated,
)
return recorded, _request(
check,
diff --git a/products/logs/backend/test/test_alert_evaluate.py b/products/logs/backend/test/test_alert_evaluate.py
index d0727771d2b7..3a7255b83187 100644
--- a/products/logs/backend/test/test_alert_evaluate.py
+++ b/products/logs/backend/test/test_alert_evaluate.py
@@ -416,7 +416,12 @@ class TestLogsAlertEvaluation(APIBaseTest):
("api", AlertEventKind.FIRING),
("worker", AlertEventKind.FIRING),
]
- assert delivery.overflowed == 1
+ # The query cut `web` and anything after it, so more breaching groups may have been turned away.
+ assert (outcome.overflow_is_lower_bound, delivery.overflowed, delivery.overflow_is_lower_bound) == (
+ True,
+ 1,
+ True,
+ )
with team_scope(self.team.id):
platform_testing.set_due_at(configuration.id, self.cutoff - timedelta(minutes=1))
@@ -429,22 +434,28 @@ class TestLogsAlertEvaluation(APIBaseTest):
}
self._record(second)
- def run_after(counts: dict[str, int]) -> tuple[dict[str, AlertEventKind], int]:
+ def run_after(counts: dict[str, int]) -> tuple[dict[str, AlertEventKind], int, bool]:
with team_scope(self.team.id):
platform_testing.set_due_at(configuration.id, self.cutoff - timedelta(minutes=1))
evaluation = self._run_grouped(configuration, counts)
self._record(evaluation)
(outcome,) = evaluation.outcomes
- return {group.labels["service_name"]: group.kind for group in outcome.groups}, outcome.overflowed
+ return (
+ {group.labels["service_name"]: group.kind for group in outcome.groups},
+ outcome.overflowed,
+ outcome.overflow_is_lower_bound,
+ )
# The resolved group is in this check's results, so it keeps its slot and the new breach overflows.
assert run_after({"api": 500, "worker": 3, "cron": 300}) == (
{"api": AlertEventKind.CHECK, "worker": AlertEventKind.CHECK},
1,
+ False,
)
assert run_after({"api": 500, "cron": 300}) == (
{"api": AlertEventKind.CHECK, "cron": AlertEventKind.FIRING},
0,
+ False,
)
with team_scope(self.team.id):
evicted = platform_testing.alert_for(
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 1988001e36cd..8e750621bbec 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -73917,6 +73917,11 @@ export namespace Schemas {
Platform: 'platform',
} as const;
+ /**
+ * Result labels and their values for the group this instance tracks, such as {"service_name": "checkout"}. Empty when the source does not group results.
+ */
+ export type PlatformAlertLabels = {[key: string]: string};
+
export interface PlatformAlert {
/** Unique identifier of this alert instance. */
readonly id: string;
@@ -73945,6 +73950,8 @@ export namespace Schemas {
* @nullable
*/
readonly snooze_until: string | null;
+ /** Result labels and their values for the group this instance tracks, such as {"service_name": "checkout"}. Empty when the source does not group results. */
+ readonly labels: PlatformAlertLabels;
}
/**
@@ -74012,6 +74019,10 @@ export namespace Schemas {
* * `errored` - Errored
* * `broken` - Broken */
readonly check_status: PlatformAlertCheckStatusEnum;
+ /** New groups the last successful check could not track because the alert already held grouping.max_instances instances. 0 when every group fit. */
+ readonly overflowed_groups: number;
+ /** Whether the last successful check cut its results before counting overflowed_groups. When true, more groups than overflowed_groups may be untracked. */
+ readonly overflow_is_lower_bound: boolean;
/** Which stack evaluates and delivers the alert. Only an alert the platform owns can be edited here. The source product's own API edits the rest.
*
* * `source` - Source