Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): read moved detector alerts' history from the platform (#123630)

Jimmy Zhu75c9da6e75ec
diff --git a/products/alerts/backend/evaluation/detector.py b/products/alerts/backend/evaluation/detector.py
index a486bf5bd010..9e85b859179c 100644
--- a/products/alerts/backend/evaluation/detector.py
+++ b/products/alerts/backend/evaluation/detector.py
@@ -1,4 +1,5 @@
from collections.abc import Callable
+from datetime import datetime
from typing import Any, Optional, cast
from zoneinfo import ZoneInfo
@@ -48,6 +49,7 @@ from products.alerts.backend.evaluation.delay import (
apply_evaluation_delay,
validate_evaluation_delay,
)
+from products.alerts.backend.facade.contracts import MovedAlertCheck
from products.alerts.backend.judge import (
EarlierFlag,
JudgeAttribution,
@@ -222,24 +224,38 @@ def _series_context(
MAX_EARLIER_CHECKS = 500
-def _earlier_flags(alert_id: Any) -> tuple[EarlierFlag, ...]:
+def _earlier_flags(alert: AlertConfiguration) -> tuple[EarlierFlag, ...]:
"""The points that earlier AI checks of this alert fired on, with the reason each check gave.
Statistical checks and errored checks are left out: only an AI verdict has a reason the
model can read. When two checks fired on the same date, the newer reason wins.
+
+ After the platform takes the alert over, its checks come from the platform's history.
"""
- checks = (
- AlertCheck.objects.filter(
- alert_configuration_id=alert_id,
- error__isnull=True,
- triggered_metadata__detector_type=DetectorType.LLM.value,
- triggered_dates__isnull=False,
- )
- .order_by("-created_at")
- .values_list("triggered_dates", "triggered_metadata")[:MAX_EARLIER_CHECKS]
+ moved_at = alert.platform_owned_since
+ own_checks = AlertCheck.objects.filter(
+ alert_configuration_id=alert.id,
+ error__isnull=True,
+ triggered_metadata__detector_type=DetectorType.LLM.value,
+ triggered_dates__isnull=False,
+ )
+ if moved_at is not None:
+ own_checks = own_checks.filter(created_at__lt=moved_at)
+ fired: list[tuple[Any, Any]] = list(
+ own_checks.order_by("-created_at").values_list("triggered_dates", "triggered_metadata")[:MAX_EARLIER_CHECKS]
)
+ if moved_at is not None:
+ # The platform's checks are all newer than the alert's own, so they go first.
+ fired = [
+ *(
+ (list(check.triggered_dates), check.triggered_metadata)
+ for check in _platform_fired_checks(alert, moved_at)
+ ),
+ *fired,
+ ][:MAX_EARLIER_CHECKS]
+
by_date: dict[str, str] = {}
- for triggered_dates, metadata in checks:
+ for triggered_dates, metadata in fired:
rationale = str((metadata or {}).get("rationale") or "")
for date in triggered_dates if isinstance(triggered_dates, list) else []:
if isinstance(date, str):
@@ -247,6 +263,31 @@ def _earlier_flags(alert_id: Any) -> tuple[EarlierFlag, ...]:
return tuple(EarlierFlag(date=date, rationale=rationale) for date, rationale in sorted(by_date.items()))
+def _saved_alert_flags(team: Team, alert_id: Any) -> tuple[EarlierFlag, ...]:
+ alert = AlertConfiguration.objects.filter(team_id=team.id, id=alert_id).first()
+ return _earlier_flags(alert) if alert is not None else ()
+
+
+def _platform_fired_checks(alert: AlertConfiguration, moved_at: datetime) -> list[MovedAlertCheck]:
+ """The AI checks of a moved alert that fired, newest first."""
+ # Imported here because the platform reads import the evaluation package this module is part of.
+ from products.alerts.backend.platform_alert_reads import moved_alert_fired_checks # noqa: PLC0415
+
+ checks = moved_alert_fired_checks(
+ team_id=alert.team_id,
+ moved_at={alert.id: moved_at},
+ not_before=None,
+ limit=MAX_EARLIER_CHECKS,
+ ai_verdicts_only=True,
+ ).get(alert.id, ())
+ return [
+ check
+ for check in checks
+ if check.error_message is None
+ and (check.triggered_metadata or {}).get("detector_type") == DetectorType.LLM.value
+ ]
+
+
def _judge_attribution(
*, team: Team | None, user: User | None, evaluation_id: str | None, is_agent_billable: bool
) -> JudgeAttribution:
@@ -388,9 +429,7 @@ def evaluate_with_detector(
metric_description = _metric_description(
insight, series_index, _effective_date_range(result, detector_config), alert_config
)
- earlier_flags = (
- _earlier_flags(alert.id) if alert is not None and detector_type_str == DetectorType.LLM.value else ()
- )
+ earlier_flags = _earlier_flags(alert) if alert is not None and detector_type_str == DetectorType.LLM.value else ()
def score(series: ComparableSeries) -> _ScoredSeries:
return _score_series(
@@ -602,11 +641,9 @@ def simulate_detector_on_insight(
user=user,
score=score,
is_agent_billable=is_agent_billable,
- earlier_flags=(
- _earlier_flags(alert_id)
- if score and alert_id is not None and detector_type_str == DetectorType.LLM.value
- else ()
- ),
+ earlier_flags=_saved_alert_flags(team, alert_id)
+ if score and alert_id is not None and detector_type_str == DetectorType.LLM.value
+ else (),
)
if result.is_breakdown:
diff --git a/products/alerts/backend/evaluation/test/test_detector_context.py b/products/alerts/backend/evaluation/test/test_detector_context.py
index 1ebb26209d46..05fe48d80a9b 100644
--- a/products/alerts/backend/evaluation/test/test_detector_context.py
+++ b/products/alerts/backend/evaluation/test/test_detector_context.py
@@ -236,4 +236,4 @@ class TestEarlierFlags(BaseTest):
check(alert, None)
check(other_alert, ["2026-01-05"])
- assert _earlier_flags(alert.id) == (EarlierFlag(date="2026-01-02", rationale="Flagged ['2026-01-02']."),)
+ assert _earlier_flags(alert) == (EarlierFlag(date="2026-01-02", rationale="Flagged ['2026-01-02']."),)
diff --git a/products/alerts/backend/facade/contracts.py b/products/alerts/backend/facade/contracts.py
index 8e0ad562694c..0fadf08ad53d 100644
--- a/products/alerts/backend/facade/contracts.py
+++ b/products/alerts/backend/facade/contracts.py
@@ -174,6 +174,14 @@ class MovedAlertCheck:
value: float | None
state: AlertState
error_message: str | None
+ # Names a failed check's reason, for the reasons the history shows by their code.
+ error_code: str | None = None
+ # A detector check's own output, as production keeps it on its check. Empty for a threshold check.
+ anomaly_scores: tuple[float | None, ...] = ()
+ triggered_points: tuple[int, ...] = ()
+ triggered_dates: tuple[str, ...] = ()
+ interval: str | None = None
+ triggered_metadata: dict[str, Any] | None = None
@frozen
diff --git a/products/alerts/backend/facade/platform.py b/products/alerts/backend/facade/platform.py
index fcb580ebc293..1a96c679b1a7 100644
--- a/products/alerts/backend/facade/platform.py
+++ b/products/alerts/backend/facade/platform.py
@@ -8,7 +8,11 @@ import uuid
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts.backend.platform_alert_backfill import follow_insight_alert as follow_alert
-from products.alerts.backend.platform_alert_reads import moved_alert_check_page, moved_alert_statuses
+from products.alerts.backend.platform_alert_reads import (
+ moved_alert_check_page,
+ moved_alert_fired_checks,
+ moved_alert_statuses,
+)
from products.alerts.backend.platform_ownership import (
close_moved_alert_firings,
delete_moved_alert_copy,
@@ -31,6 +35,7 @@ __all__ = [
"follow_insight_alert",
"hand_over_new_alert",
"moved_alert_check_page",
+ "moved_alert_fired_checks",
"moved_alert_statuses",
"write_moved_alert_edit",
]
diff --git a/products/alerts/backend/models/alert.py b/products/alerts/backend/models/alert.py
index 2ecbd72f5339..af341988bb35 100644
--- a/products/alerts/backend/models/alert.py
+++ b/products/alerts/backend/models/alert.py
@@ -237,6 +237,15 @@ class AlertConfiguration(ModelActivityMixin, CreatedMetaFields, UUIDTModel):
compare, but it sends nothing for it."""
return self.owner == PlatformAlertOwner.PLATFORM
+ @property
+ def platform_owned_since(self) -> datetime | None:
+ """When the platform took this alert over, or None while the insight stack owns it.
+
+ The alert's own checks before then are its history. Its own checks after then are the shadow's,
+ which sent nothing, so the platform's checks replace them.
+ """
+ return self.owner_changed_at if self.platform_owned else None
+
@property
def is_high_frequency_interval(self) -> bool:
# Real-time counts as high frequency so its checks always run fresh ClickHouse
diff --git a/products/alerts/backend/platform_alert_reads.py b/products/alerts/backend/platform_alert_reads.py
index b39022e572a5..610cc3e5ebde 100644
--- a/products/alerts/backend/platform_alert_reads.py
+++ b/products/alerts/backend/platform_alert_reads.py
@@ -6,9 +6,9 @@ moving. The platform's runtime rows and check history say where the alert stands
from __future__ import annotations
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
from datetime import UTC, datetime, timedelta
-from typing import Final
+from typing import Any, Final
from uuid import NAMESPACE_URL, UUID, uuid5
from posthog.exceptions_capture import capture_exception
@@ -24,14 +24,22 @@ from products.alerts.backend.facade.contracts import (
)
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts.backend.platform_cutover import insight_state_of_platform
+from products.alerts.backend.platform_source_cycle import ERROR_CODE_KEY
from products.alerts_platform.backend.facade.api import (
check_page_by_legacy_id,
+ checks_with_detail_by_legacy_id,
last_check_by_legacy_id,
runtime_by_legacy_id,
)
from products.alerts_platform.backend.facade.contracts import PlatformCheckEntry
from products.alerts_platform.backend.facade.enums import PlatformAlertOwner, PlatformAlertState
+# The detail key a detector check that fired on a point stores its dates under.
+FIRED_DATES_KEY: Final = "triggered_dates"
+
+# Only an AI verdict has a reason, so a check that holds one is an AI detector's.
+AI_REASON_PATH: Final = ("triggered_metadata", "rationale")
+
_EARLIEST: Final = datetime(1970, 1, 1, tzinfo=UTC)
_LATEST: Final = datetime(2100, 1, 1, tzinfo=UTC)
@@ -109,7 +117,49 @@ def moved_alert_check_page(
return MovedAlertCheckPage(total=page.total, checks=tuple(_as_check(alert_id, check) for check in page.checks))
+def moved_alert_fired_checks(
+ *,
+ team_id: int,
+ moved_at: Mapping[UUID, datetime],
+ not_before: datetime | None,
+ limit: int,
+ ai_verdicts_only: bool = False,
+) -> dict[UUID, tuple[MovedAlertCheck, ...]]:
+ """Each moved alert's newest `limit` detector checks that fired on a point, newest first, from the
+ platform's history since the move and not before `not_before`.
+
+ `moved_at` holds each alert's `platform_owned_since`. `ai_verdicts_only` keeps the AI detector's
+ checks, before the limit applies, so statistical checks from another detector cannot use it up. The platform's checks before the move were
+ a parallel run, not the alert's own. A ClickHouse failure leaves out every alert, so a caller
+ shows what it has without the platform.
+ """
+ if not moved_at:
+ return {}
+ since = {alert_id: max(at, not_before) if not_before is not None else at for alert_id, at in moved_at.items()}
+ try:
+ checks = checks_with_detail_by_legacy_id(
+ team_id=team_id,
+ legacy_configuration_ids=list(since),
+ detail_paths=[(FIRED_DATES_KEY,), *([AI_REASON_PATH] if ai_verdicts_only else [])],
+ since=min(since.values()),
+ limit=limit,
+ )
+ except Exception as error:
+ capture_exception(error, additional_properties={"team_id": team_id})
+ return {}
+ return {
+ alert_id: tuple(
+ _as_check(alert_id, check) for check in entries if ensure_utc(check.occurred_at) >= since[alert_id]
+ )
+ for alert_id, entries in checks.items()
+ }
+
+
def _as_check(alert_id: UUID, check: PlatformCheckEntry) -> MovedAlertCheck:
+ detail = check.source_detail
+ metadata = detail.get("triggered_metadata")
+ interval = detail.get("interval")
+ error_code = detail.get(ERROR_CODE_KEY)
return MovedAlertCheck(
id=uuid5(NAMESPACE_URL, f"{alert_id}|{check.evaluation_key}"),
checked_at=ensure_utc(check.occurred_at),
@@ -119,4 +169,18 @@ def _as_check(alert_id: UUID, check: PlatformCheckEntry) -> MovedAlertCheck:
if check.state == PlatformAlertState.SNOOZED
else insight_state_of_platform(check.state, snoozed_until=None, at=check.occurred_at),
error_message=check.error_message or None,
+ error_code=error_code if isinstance(error_code, str) else None,
+ anomaly_scores=tuple(_listed(detail, "anomaly_scores", (int, float, type(None)))),
+ triggered_points=tuple(_listed(detail, "triggered_points", (int,))),
+ triggered_dates=tuple(_listed(detail, FIRED_DATES_KEY, (str,))),
+ interval=interval if isinstance(interval, str) else None,
+ triggered_metadata=metadata if isinstance(metadata, dict) else None,
)
+
+
+def _listed(detail: dict[str, Any], key: str, types: tuple[type, ...]) -> list[Any]:
+ """A list the check stored, keeping only items of the expected types, because history outlives the code that wrote it."""
+ values = detail.get(key)
+ if not isinstance(values, list):
+ return []
+ return [value for value in values if isinstance(value, types) and not isinstance(value, bool)]
diff --git a/products/alerts/backend/presentation/views/alert.py b/products/alerts/backend/presentation/views/alert.py
index 96f8391ab572..38237895ba8d 100644
--- a/products/alerts/backend/presentation/views/alert.py
+++ b/products/alerts/backend/presentation/views/alert.py
@@ -1,11 +1,13 @@
import json
import uuid
from datetime import datetime
-from typing import Annotated, Any, cast
+from functools import reduce
+from operator import or_
+from typing import Annotated, Any, Final, cast
from zoneinfo import ZoneInfo
from django.db import transaction
-from django.db.models import OuterRef, Prefetch, QuerySet, Subquery
+from django.db.models import OuterRef, Prefetch, Q, QuerySet, Subquery
from django.utils import timezone
import pydantic
@@ -112,6 +114,7 @@ from products.alerts.backend.facade.platform import (
follow_insight_alert,
hand_over_new_alert,
moved_alert_check_page,
+ moved_alert_fired_checks,
moved_alert_statuses,
write_moved_alert_edit,
)
@@ -1485,6 +1488,11 @@ def _show_platform_status(alerts: list[AlertConfiguration], *, now: datetime) ->
alert.last_value = moved_status.last_check.value # type: ignore[attr-defined]
+# How many fired checks of one moved alert the list returns for the chart's dots. The window holds a
+# check every 15 minutes for a 15-minute alert, and one alert that keeps firing must not fill the response.
+MOVED_ALERT_FIRED_CHECKS_LIMIT: Final = 500
+
+
def _as_alert_check(alert: AlertConfiguration, check: MovedAlertCheck) -> AlertCheck:
# Never saved. It carries what the check serializer shows.
return AlertCheck(
@@ -1494,8 +1502,15 @@ def _as_alert_check(alert: AlertConfiguration, check: MovedAlertCheck) -> AlertC
calculated_value=check.value,
condition=alert.condition,
state=check.state,
- error={"message": check.error_message} if check.error_message else None,
+ error=({"code": check.error_code} if check.error_code else {}) | {"message": check.error_message}
+ if check.error_message
+ else None,
targets_notified={},
+ anomaly_scores=list(check.anomaly_scores) or None,
+ triggered_points=list(check.triggered_points) or None,
+ triggered_dates=list(check.triggered_dates) or None,
+ interval=check.interval,
+ triggered_metadata=check.triggered_metadata,
)
@@ -1749,16 +1764,34 @@ class AlertViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
# Prefetch firing checks for anomaly point display on chart.
if alerts:
alert_ids = [a.id for a in alerts]
+ moved_at = {alert.id: at for alert in alerts if (at := alert.platform_owned_since) is not None}
firing_checks = (
AlertCheck.objects.filter(
alert_configuration_id__in=alert_ids,
triggered_points__isnull=False,
)
.exclude(triggered_points=[])
+ # A moved alert's own checks after the move are the shadow's, so the platform's replace them.
+ .exclude(
+ reduce(
+ or_,
+ (Q(alert_configuration_id=alert_id, created_at__gte=at) for alert_id, at in moved_at.items()),
+ Q(pk__in=[]),
+ )
+ )
.select_related("investigation_notebook")
.order_by("-created_at")
)
+ moved_checks = moved_alert_fired_checks(
+ team_id=self.team_id,
+ moved_at=moved_at,
+ not_before=timezone.now() - AlertCheck.RETENTION,
+ limit=MOVED_ALERT_FIRED_CHECKS_LIMIT,
+ )
+ alerts_by_id = {alert.id: alert for alert in alerts}
checks_by_alert: dict[str, list] = {str(a.id): [] for a in alerts}
+ for alert_id, checks in moved_checks.items():
+ checks_by_alert[str(alert_id)] = [_as_alert_check(alerts_by_id[alert_id], check) for check in checks]
for check in firing_checks:
checks_by_alert.setdefault(str(check.alert_configuration_id), []).append(check)
for alert in alerts:
diff --git a/products/alerts/backend/tests/api/test_alert_platform_owned.py b/products/alerts/backend/tests/api/test_alert_platform_owned.py
index 7334a5245cd5..dc2c2ce9d2aa 100644
--- a/products/alerts/backend/tests/api/test_alert_platform_owned.py
+++ b/products/alerts/backend/tests/api/test_alert_platform_owned.py
@@ -14,8 +14,14 @@ from posthog.constants import AvailableFeature
from posthog.models.scoping import team_scope
from posthog.schema_enums import AlertCalculationInterval
+from products.alerts.backend.evaluation.detector import _earlier_flags
from products.alerts.backend.facade.api import snooze_alert_from_slack
from products.alerts.backend.facade.contracts import PlatformCannotOwn
+from products.alerts.backend.judge import (
+ LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,
+ LLM_DETECTOR_UNAVAILABLE_MESSAGE,
+ EarlierFlag,
+)
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration
from products.alerts.backend.platform_cutover import cut_over_team
from products.alerts.backend.test.insight_alerts import create_insight_alert
@@ -51,7 +57,15 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):
def _at(self, minutes_ago: int) -> datetime:
return self.now - timedelta(minutes=minutes_ago)
- def _record(self, minutes_ago: int, state: str, *, value: float | None = None, error: str | None = None) -> None:
+ def _record(
+ self,
+ minutes_ago: int,
+ state: str,
+ *,
+ value: float | None = None,
+ error: str | None = None,
+ detail: dict[str, Any] | None = None,
+ ) -> None:
platform_testing.record_check(
self.copy_id,
team_id=self.team.id,
@@ -60,7 +74,18 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):
state=state,
value=value,
error_message=error,
+ source_detail=detail,
+ )
+
+ def _own_check(self, minutes_ago: int, date: str, rationale: str) -> None:
+ check = AlertCheck.objects.create(
+ alert_configuration=self.alert,
+ state="Firing",
+ triggered_points=[9],
+ triggered_dates=[date],
+ triggered_metadata={"detector_type": "llm", "rationale": rationale},
)
+ AlertCheck.objects.filter(id=check.id).update(created_at=self._at(minutes_ago))
def _row(self, action: str, query: str = "") -> dict[str, Any]:
if action == "list":
@@ -108,6 +133,82 @@ class TestPlatformOwnedInsightAlert(ClickhouseTestMixin, APIBaseTest):
up_to_a_check = self._row("retrieve", f"?checks_date_to={_iso(self._at(60))}")
assert [check["created_at"] for check in up_to_a_check["checks"]] == [_iso(self._at(60)), _iso(self._at(90))]
+ def test_a_moved_detector_alert_shows_its_detector_output_from_the_platform(self) -> None:
+ detail = {
+ "anomaly_scores": [0.99],
+ "triggered_points": [9],
+ "triggered_dates": ["2026-10-20"],
+ "interval": "day",
+ "triggered_metadata": {"detector_type": "llm", "rationale": "Signups fell.", "kind": "drop"},
+ }
+ self._record(30, "firing", value=150.0, detail=detail)
+ # History outlives the code that wrote it, so a value of another shape is left out.
+ self._record(20, "firing", value=150.0, detail={**detail, "triggered_dates": "2026-10-21"})
+
+ # The history explains a check the model could not answer, as it does for the alert's own checks.
+ self._record(
+ 10,
+ "errored",
+ error=LLM_DETECTOR_UNAVAILABLE_MESSAGE,
+ detail={"error_code": LLM_DETECTOR_UNAVAILABLE_ERROR_CODE},
+ )
+
+ unanswered, newest, oldest = self._row("retrieve", "?checks_limit=3")["checks"]
+
+ assert {key: oldest[key] for key in detail} == detail
+ assert newest["triggered_dates"] is None
+ assert unanswered["error"] == {
+ "code": LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,
+ "message": LLM_DETECTOR_UNAVAILABLE_MESSAGE,
+ }
+
+ def test_a_moved_detector_alert_marks_the_points_both_stacks_fired_on(self) -> None:
+ self._own_check(60 * 24, "2026-10-18", "Before the move.")
+ # The shadow's check after the move sent nothing, so the platform's verdict replaces it.
+ self._own_check(-30, "2026-10-19", "The shadow.")
+ # Before the move, the platform's copy only ran in parallel, so its verdicts are not the alert's.
+ self._record(60, "firing", detail={"triggered_points": [9], "triggered_dates": ["2026-10-17"]})
+ self._record(-30, "firing", detail={"triggered_points": [9], "triggered_dates": ["2026-10-20"]})
+ self._record(-20, "not_firing", detail={"anomaly_scores": [0.1]})
+
+ row = self._row("list")
+
+ assert [check["triggered_dates"] for check in row["checks"]] == [["2026-10-20"], ["2026-10-18"]]
+
+ @parameterized.expand(
+ [
+ ("every_check", 500, ["2026-10-18", "2026-10-20"]),
+ # A newer statistical check must not use up the limit and push out the AI verdict.
+ ("only_the_newest", 1, ["2026-10-20"]),
+ ]
+ )
+ def test_a_moved_ai_alert_reads_its_earlier_flags_from_both_stacks(
+ self, _name: str, limit: int, dates: list[str]
+ ) -> None:
+ self._own_check(60 * 24, "2026-10-18", "Before the move.")
+ self._own_check(-30, "2026-10-19", "The shadow.")
+ llm = {"detector_type": "llm"}
+ # Before the move, the platform's copy only ran in parallel, so its verdicts are not the alert's.
+ self._record(60, "firing", detail={"triggered_dates": ["2026-10-17"], "triggered_metadata": llm})
+ self._record(
+ -30,
+ "firing",
+ detail={"triggered_dates": ["2026-10-20"], "triggered_metadata": {**llm, "rationale": "After."}},
+ )
+ self._record(
+ -25, "firing", detail={"triggered_dates": ["2026-10-21"], "triggered_metadata": {"detector_type": "zscore"}}
+ )
+ self._record(
+ -20, "errored", error="boom", detail={"triggered_dates": ["2026-10-22"], "triggered_metadata": llm}
+ )
+ self.alert.refresh_from_db()
+
+ with patch("products.alerts.backend.evaluation.detector.MAX_EARLIER_CHECKS", limit):
+ flags = _earlier_flags(self.alert)
+
+ reasons = {"2026-10-18": "Before the move.", "2026-10-20": "After."}
+ assert flags == tuple(EarlierFlag(date=date, rationale=reasons[date]) for date in dates)
+
def test_a_platform_history_failure_falls_back_to_the_alerts_own_checks(self) -> None:
self._record(30, "firing", value=150.0)
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.py
index 0935bc76db70..14e0e3a06b4d 100644
--- a/products/alerts_platform/backend/facade/api.py
+++ b/products/alerts_platform/backend/facade/api.py
@@ -24,6 +24,7 @@ from products.alerts_platform.backend.facade.contracts import (
PlatformAlertRuntime,
PlatformAlertSnapshot,
PlatformAlertUpsert,
+ PlatformCheckEntry,
PlatformCheckEventPage,
PlatformCheckPage,
PlatformHandback,
@@ -166,6 +167,28 @@ def check_page_by_legacy_id(
)
+def checks_with_detail_by_legacy_id(
+ *,
+ team_id: int,
+ legacy_configuration_ids: Collection[UUID],
+ detail_paths: Sequence[Sequence[str]],
+ since: datetime,
+ limit: int,
+) -> dict[UUID, tuple[PlatformCheckEntry, ...]]:
+ """Each named owned alert's newest `limit` checks since `since` whose source detail holds
+ every one of `detail_paths`, newest first, keyed by the source's id.
+
+ Raises when ClickHouse fails, so a caller can fall back to what it shows without the platform.
+ """
+ return platform_reads.checks_with_detail_by_legacy_id(
+ team_id=team_id,
+ legacy_configuration_ids=legacy_configuration_ids,
+ detail_paths=detail_paths,
+ since=since,
+ limit=limit,
+ )
+
+
def history_by_legacy_id(
*, team_id: int, legacy_configuration_ids: Collection[UUID], since: datetime, until: datetime
) -> dict[UUID, PlatformAlertHistory]:
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index 0000f2301425..3cf7039c43ce 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -457,6 +457,8 @@ class PlatformCheckEntry:
value: float | None
# None when the check succeeded.
error_message: str | None
+ # What the source recorded beside the verdict. The platform reads only which keys it holds.
+ source_detail: dict[str, Any] = field(default_factory=dict)
@frozen
diff --git a/products/alerts_platform/backend/facade/testing.py b/products/alerts_platform/backend/facade/testing.py
index ee7493b7ad01..6d3f4a2e42c3 100644
--- a/products/alerts_platform/backend/facade/testing.py
+++ b/products/alerts_platform/backend/facade/testing.py
@@ -142,6 +142,7 @@ def record_check(
value: float | None = None,
condition: dict[str, Any] | None = None,
query_duration_ms: int | None = None,
+ source_detail: dict[str, Any] | None = None,
) -> None:
"""Write one ungrouped check to the platform's ClickHouse history, as a check that ran at `at`."""
insert_events(
@@ -168,6 +169,7 @@ def record_check(
consecutive_failures=0,
muted_notification="",
occurred_at=at,
+ source_detail=source_detail or {},
)
],
)
diff --git a/products/alerts_platform/backend/logic/platform_alert_events.py b/products/alerts_platform/backend/logic/platform_alert_events.py
index a1bc04a03aee..048e4b10a414 100644
--- a/products/alerts_platform/backend/logic/platform_alert_events.py
+++ b/products/alerts_platform/backend/logic/platform_alert_events.py
@@ -425,10 +425,23 @@ def notable_checks(team_id: int, configuration_id: UUID, *, since: datetime, lim
)
+def _check_entry(
+ occurred_at: datetime, evaluation_key: str, state: str, value: float | None, error_message: str, source_detail: str
+) -> PlatformCheckEntry:
+ return PlatformCheckEntry(
+ occurred_at=occurred_at,
+ evaluation_key=evaluation_key,
+ state=PlatformAlertState(state),
+ value=value,
+ error_message=error_message or None,
+ source_detail=_snapshot(source_detail),
+ )
+
+
_CHECKS_SQL = f"""
-SELECT occurred_at, evaluation_key, state, value, error_message, count() OVER () AS total
+SELECT occurred_at, evaluation_key, state, value, error_message, count() OVER () AS total, source_detail
FROM (
- SELECT alert_id, evaluation_key, occurred_at, state, value, error_message
+ SELECT alert_id, evaluation_key, occurred_at, state, value, error_message, source_detail
FROM {PLATFORM_ALERT_EVENTS_TABLE}
WHERE team_id = %(team_id)s
AND configuration_id = %(configuration_id)s
@@ -486,18 +499,75 @@ def check_page(
return PlatformCheckPage(
total=total,
checks=tuple(
- PlatformCheckEntry(
- occurred_at=occurred_at,
- evaluation_key=evaluation_key,
- state=PlatformAlertState(state),
- value=value,
- error_message=error_message or None,
- )
- for occurred_at, evaluation_key, state, value, error_message, _ in rows
+ _check_entry(occurred_at, evaluation_key, state, value, error_message, source_detail)
+ for occurred_at, evaluation_key, state, value, error_message, _, source_detail in rows
),
)
+# Deduplicated as `_CHECKS_SQL` is. `JSONHas` is false for the empty string a check without detail
+# stores, so only checks whose source recorded every path are read. The paths filter before the
+# limit, so checks a caller does not want never use it up.
+def _checks_with_detail_sql(paths: Sequence[Sequence[str]]) -> str:
+ has_paths = "".join(
+ f"\n AND JSONHas(source_detail, {', '.join(f'%(path_{i}_{j})s' for j in range(len(path)))})"
+ for i, path in enumerate(paths)
+ )
+ return f"""
+SELECT configuration_id, occurred_at, evaluation_key, state, value, error_message, source_detail
+FROM (
+ SELECT configuration_id, alert_id, evaluation_key, occurred_at, state, value, error_message, source_detail
+ FROM {PLATFORM_ALERT_EVENTS_TABLE}
+ WHERE team_id = %(team_id)s
+ AND configuration_id IN %(configuration_ids)s
+ AND grouping_key = ''
+ AND occurred_at >= toDateTime64(%(since)s, 6, 'UTC'){has_paths}
+ ORDER BY configuration_id, alert_id, occurred_at, evaluation_key
+ LIMIT 1 BY alert_id, evaluation_key
+)
+ORDER BY configuration_id, occurred_at DESC, evaluation_key DESC
+LIMIT %(limit)s BY configuration_id
+"""
+
+
+def checks_with_detail(
+ team_id: int,
+ configuration_ids: Collection[UUID],
+ *,
+ detail_paths: Sequence[Sequence[str]],
+ since: datetime,
+ limit: int,
+) -> dict[UUID, tuple[PlatformCheckEntry, ...]]:
+ """Each configuration's newest `limit` checks since `since` whose source detail holds every one
+ of `detail_paths`, newest first, in one query. A path is a list of keys into nested objects.
+
+ Only the ungrouped instance is read, as in `history`. A configuration with no such check is left
+ out. Raises on a ClickHouse failure, because only the caller knows what it can show instead.
+ """
+ if not configuration_ids:
+ return {}
+ tag_queries(product=Product.PLATFORM_AND_SUPPORT, feature=Feature.ALERTING)
+ # Not `_execute`, for the same reason as `history`.
+ rows = sync_execute(
+ _checks_with_detail_sql(detail_paths),
+ {
+ "team_id": team_id,
+ "configuration_ids": list(configuration_ids),
+ "since": _microseconds(since),
+ "limit": limit,
+ **{f"path_{i}_{j}": key for i, path in enumerate(detail_paths) for j, key in enumerate(path)},
+ },
+ team_id=team_id,
+ settings=_READ_BUDGET,
+ )
+ checks: dict[UUID, list[PlatformCheckEntry]] = {}
+ for configuration_id, occurred_at, evaluation_key, state, value, error_message, source_detail in rows:
+ checks.setdefault(configuration_id, []).append(
+ _check_entry(occurred_at, evaluation_key, state, value, error_message, source_detail)
+ )
+ return {configuration_id: tuple(entries) for configuration_id, entries in checks.items()}
+
+
# Deduplication does not matter here, because two copies of one check are identical rows. The tuple
# keeps a NULL value, which `argMax` would otherwise skip in favor of an older check's.
_LAST_CHECKS_SQL = f"""
diff --git a/products/alerts_platform/backend/logic/platform_reads.py b/products/alerts_platform/backend/logic/platform_reads.py
index 1e35366a2ce0..6b39487353e0 100644
--- a/products/alerts_platform/backend/logic/platform_reads.py
+++ b/products/alerts_platform/backend/logic/platform_reads.py
@@ -18,6 +18,7 @@ from products.alerts_platform.backend.facade.contracts import (
PlatformAlertHistory,
PlatformAlertRuntime,
PlatformAlertSnapshot,
+ PlatformCheckEntry,
PlatformCheckEventPage,
PlatformCheckPage,
PlatformLastCheck,
@@ -196,6 +197,17 @@ def runtime_by_legacy_id(
}
+def _owned_by_legacy_id(team_id: int, legacy_configuration_ids: Collection[UUID]) -> dict[UUID, UUID]:
+ """The source's id of each named alert the platform owns, keyed by the platform's id."""
+ return {
+ platform_id: legacy_id
+ for platform_id, legacy_id in PlatformAlertConfiguration.objects.for_team(team_id)
+ .filter(legacy_configuration_id__in=list(legacy_configuration_ids), owner=PlatformAlertOwner.PLATFORM)
+ .values_list("id", "legacy_configuration_id")
+ if legacy_id is not None
+ }
+
+
def history_by_legacy_id(
*, team_id: int, legacy_configuration_ids: Collection[UUID], since: datetime, until: datetime
) -> dict[UUID, PlatformAlertHistory]:
@@ -207,13 +219,7 @@ def history_by_legacy_id(
"""
if not legacy_configuration_ids:
return {}
- legacy_by_platform_id = {
- platform_id: legacy_id
- for platform_id, legacy_id in PlatformAlertConfiguration.objects.for_team(team_id)
- .filter(legacy_configuration_id__in=list(legacy_configuration_ids), owner=PlatformAlertOwner.PLATFORM)
- .values_list("id", "legacy_configuration_id")
- if legacy_id is not None
- }
+ legacy_by_platform_id = _owned_by_legacy_id(team_id, legacy_configuration_ids)
histories = platform_alert_events.history(team_id, legacy_by_platform_id, since=since, until=until)
return {legacy_by_platform_id[platform_id]: history for platform_id, history in histories.items()}
@@ -227,13 +233,7 @@ def last_check_by_legacy_id(
"""
if not legacy_configuration_ids:
return {}
- legacy_by_platform_id = {
- platform_id: legacy_id
- for platform_id, legacy_id in PlatformAlertConfiguration.objects.for_team(team_id)
- .filter(legacy_configuration_id__in=list(legacy_configuration_ids), owner=PlatformAlertOwner.PLATFORM)
- .values_list("id", "legacy_configuration_id")
- if legacy_id is not None
- }
+ legacy_by_platform_id = _owned_by_legacy_id(team_id, legacy_configuration_ids)
checks = platform_alert_events.last_checks(team_id, legacy_by_platform_id)
return {legacy_by_platform_id[platform_id]: check for platform_id, check in checks.items()}
@@ -267,3 +267,25 @@ def check_page_by_legacy_id(
if platform_id is None:
return None
return platform_alert_events.check_page(team_id, platform_id, since=since, until=until, limit=limit, offset=offset)
+
+
+def checks_with_detail_by_legacy_id(
+ *,
+ team_id: int,
+ legacy_configuration_ids: Collection[UUID],
+ detail_paths: Sequence[Sequence[str]],
+ since: datetime,
+ limit: int,
+) -> dict[UUID, tuple[PlatformCheckEntry, ...]]:
+ """Each named alert's newest `limit` checks since `since` whose source detail holds every one of `detail_paths`,
+ keyed by the source's id.
+
+ An alert the platform does not own is left out, as in `runtime_by_legacy_id`.
+ """
+ if not legacy_configuration_ids:
+ return {}
+ legacy_by_platform_id = _owned_by_legacy_id(team_id, legacy_configuration_ids)
+ checks = platform_alert_events.checks_with_detail(
+ team_id, legacy_by_platform_id, detail_paths=detail_paths, since=since, limit=limit
+ )
+ return {legacy_by_platform_id[platform_id]: entries for platform_id, entries in checks.items()}