Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): keep detector fields on platform insight check history (#123628)

Jimmy Zhu79c2396b9ee3
diff --git a/posthog/tasks/alerts/test/test_utils.py b/posthog/tasks/alerts/test/test_utils.py
index 394958041b3f..54da670e2b7e 100644
--- a/posthog/tasks/alerts/test/test_utils.py
+++ b/posthog/tasks/alerts/test/test_utils.py
@@ -48,10 +48,7 @@ class TestAlertUtils:
]
)
def test_detector_verdict_event_fields(self, _name: str, metadata: dict | None, expected: dict) -> None:
- alert_check = MagicMock(spec=AlertCheck)
- alert_check.triggered_metadata = metadata
-
- assert detector_verdict_event_fields(alert_check) == expected
+ assert detector_verdict_event_fields(metadata) == expected
def test_next_check_time_advances_by_2_minutes(self) -> None:
alert = MagicMock(spec=AlertConfiguration)
diff --git a/posthog/tasks/alerts/utils.py b/posthog/tasks/alerts/utils.py
index 3232c3b48fe8..f9ee2576966c 100644
--- a/posthog/tasks/alerts/utils.py
+++ b/posthog/tasks/alerts/utils.py
@@ -493,14 +493,14 @@ def render_alert_insight_chart_url(alert: AlertConfiguration, render_user: User
return insight_chart_url(team_id=alert.team_id, chart=chart) if chart is not None else None
-def detector_verdict_event_fields(alert_check: AlertCheck) -> dict[str, str]:
+def detector_verdict_event_fields(triggered_metadata: dict | None) -> dict[str, str]:
"""The model's verdict, as its own `$insight_alert_firing` properties.
The rationale is already inside the breach text, but a destination that wants to place
it on its own, in its own Slack block or its own webhook field, cannot split it back
out of a sentence. Absent for every detector type that reports no verdict.
"""
- metadata = alert_check.triggered_metadata or {}
+ metadata = triggered_metadata or {}
fields = {"anomaly_rationale": metadata.get("rationale"), "anomaly_kind": metadata.get("kind")}
return {key: str(value) for key, value in fields.items() if value}
@@ -591,7 +591,10 @@ def dispatch_alert_notification(
"caller must pass the breaches list from AlertEvaluationResult"
)
logger.info("Sending alert firing notifications", alert_id=alert.id)
- properties = {**detector_verdict_event_fields(alert_check), **(extra_properties or {})}
+ properties = {
+ **detector_verdict_event_fields(alert_check.triggered_metadata),
+ **(extra_properties or {}),
+ }
# Attach the chart for any firing alert whose caller did not already supply
# one, so ordinary threshold and anomaly alerts get the chart too, not just
# the anomaly investigation path (which renders it early and passes it in).
diff --git a/products/alerts/backend/platform_alert_legacy_events.py b/products/alerts/backend/platform_alert_legacy_events.py
index ec6a327c709a..bf070a502f80 100644
--- a/products/alerts/backend/platform_alert_legacy_events.py
+++ b/products/alerts/backend/platform_alert_legacy_events.py
@@ -15,7 +15,7 @@ from typing import Any, Final
from uuid import NAMESPACE_URL, uuid5
from posthog.schema_enums import AlertState
-from posthog.tasks.alerts.utils import RenderedInsightChart, insight_chart_url
+from posthog.tasks.alerts.utils import RenderedInsightChart, detector_verdict_event_fields, insight_chart_url
from products.alerts.backend.facade.contracts import AlertInternalEvent
from products.alerts.backend.logic.destinations import emit_alert_internal_events
@@ -84,6 +84,7 @@ def _firing_event(alert: AlertConfiguration, transition: AnnouncedTransition) ->
properties: dict[str, Any] = {
**alert.internal_event_properties(state=AlertState.FIRING.value, last_checked_at=transition.occurred_at),
"breaches": "\n".join(detail.get("breaches") or []),
+ **detector_verdict_event_fields(detail.get("triggered_metadata")),
}
if chart_url := insight_chart_url_of(alert.team_id, detail):
properties["insight_chart_url"] = chart_url
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 95b694dee9f7..0d8aa7d4e60c 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -32,7 +32,12 @@ from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.errors import CH_TRANSIENT_ERRORS, QueryErrorCategory, classify_query_error
from posthog.schema_enums import AlertCalculationInterval
from posthog.tasks.alerts.schedule_restriction import is_utc_datetime_blocked
-from posthog.tasks.alerts.utils import RenderedInsightChart, render_alert_insight_chart, skip_because_of_weekend
+from posthog.tasks.alerts.utils import (
+ AlertEvaluationResult,
+ RenderedInsightChart,
+ render_alert_insight_chart,
+ skip_because_of_weekend,
+)
from posthog.temporal.alerts.admission import admit_evaluation_slots, release_evaluation_slot
from products.alerts.backend.evaluation import check_alert_for_insight
@@ -40,6 +45,7 @@ from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, check_alert_per_series
from products.alerts.backend.insight_alert_state_machine import INSIGHT_ALERT_POLICY, insight_snapshot
from products.alerts.backend.judge.contract import (
+ LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,
LLM_DETECTOR_UNAVAILABLE_MESSAGE,
LLMDetectorMisconfiguredError,
LLMDetectorUnavailableError,
@@ -115,6 +121,9 @@ QUERY_ID_PREFIX = "alerts-platform-insight:"
# matches on it to set those checks aside.
CAPACITY_REJECTED = "ClickHouse refused the query for capacity"
+# The detail key of the code that names a failed check's reason, as production keeps it on its check's error.
+ERROR_CODE_KEY: Final = "error_code"
+
SLOT_KEY_PREFIX = "slot:"
EVALUATED_INTERVALS = frozenset(
@@ -513,6 +522,8 @@ def _decide(
now=now,
skip=SkipReason.QUERY_FAILED,
context=_context(alert),
+ # History shows this reason by its code, and an errored check that repeats announces nothing.
+ check_detail={ERROR_CODE_KEY: LLM_DETECTOR_UNAVAILABLE_ERROR_CODE},
)
except (AlertExtractionError, LLMDetectorMisconfiguredError) as error:
return _broken(check, snapshot, alert, now=now, message=str(error))
@@ -530,6 +541,7 @@ def _decide(
value=result.value,
query_duration_ms=duration_ms,
firing=(alert, result.breaches or []),
+ check_detail=detector_detail(result) if alert.detector_config else None,
)
@@ -716,8 +728,8 @@ def _context(alert: AlertConfiguration) -> dict[str, Any]:
return {"insight_name": alert.insight.name, "insight_short_id": alert.insight.short_id}
-def _clipped(breach: str) -> str:
- return breach if len(breach) <= MAX_BREACH_CHARS else breach[: MAX_BREACH_CHARS - 1] + "…"
+def _clipped(text: str, limit: int = MAX_BREACH_CHARS) -> str:
+ return text if len(text) <= limit else text[: limit - 1] + "…"
def _detail(alert: AlertConfiguration, breaches: Sequence[str], chart: RenderedInsightChart | None) -> dict[str, Any]:
@@ -736,6 +748,33 @@ def _detail(alert: AlertConfiguration, breaches: Sequence[str], chart: RenderedI
return detail
+# A scheduled check scores only its latest point, so a detector reports one score and at most one
+# fired point. These bound a detector that reports more, because every check's detail rides in the
+# outcome Temporal carries.
+MAX_DETECTOR_POINTS: Final = 10
+MAX_RATIONALE_CHARS: Final = 1000
+
+
+def detector_detail(result: AlertEvaluationResult) -> dict[str, Any]:
+ """What production keeps on a detector check beside its verdict, under the same names: the
+ score, kind and reason history shows, and the fired points the insight chart marks.
+
+ Kept on every detector check, not only on one that sends, because production shows them on every check.
+ """
+ detail: dict[str, Any] = {
+ "anomaly_scores": (result.anomaly_scores or [])[-MAX_DETECTOR_POINTS:],
+ "triggered_points": (result.triggered_points or [])[-MAX_DETECTOR_POINTS:],
+ "triggered_dates": (result.triggered_dates or [])[-MAX_DETECTOR_POINTS:],
+ "interval": result.interval,
+ "triggered_metadata": dict(result.triggered_metadata or {}),
+ }
+ rationale = detail["triggered_metadata"].get("rationale")
+ if isinstance(rationale, str):
+ detail["triggered_metadata"]["rationale"] = _clipped(rationale, MAX_RATIONALE_CHARS)
+ # Empty values are left out, so a reader finds `triggered_dates` only on a check that fired on a point.
+ return {key: value for key, value in detail.items() if value}
+
+
def _verdict(
check: PlatformAlertCheckInput,
snapshot: AlertSnapshot,
@@ -747,6 +786,7 @@ def _verdict(
skip: SkipReason | None = None,
firing: tuple[AlertConfiguration, Sequence[str]] | None = None,
context: dict[str, Any] | None = None,
+ check_detail: dict[str, Any] | None = None,
) -> PlatformAlertOutcome:
outcome = evaluate_alert_check(snapshot, check_input, now, policy=INSIGHT_ALERT_POLICY)
source_detail: dict[str, Any] | None
@@ -756,6 +796,9 @@ def _verdict(
source_detail = context
else:
source_detail = None
+ # Kept on every check, including one that announces nothing.
+ if check_detail:
+ source_detail = {**(source_detail or {}), **check_detail}
return _recorded(
check,
snapshot,
diff --git a/products/alerts/backend/test/test_platform_alert_legacy_events.py b/products/alerts/backend/test/test_platform_alert_legacy_events.py
index a831fc1e4256..69193180b1f5 100644
--- a/products/alerts/backend/test/test_platform_alert_legacy_events.py
+++ b/products/alerts/backend/test/test_platform_alert_legacy_events.py
@@ -90,10 +90,15 @@ class TestInsightLegacyEvents(APIBaseTest):
{"insight_chart_asset_id": 7, "insight_chart_render_user_id": 3},
{"insight_chart_url": CHART_URL},
),
+ (
+ "with_an_ai_verdict",
+ {"triggered_metadata": {"detector_type": "llm", "rationale": "Signups fell to 12.", "kind": "drop"}},
+ {"anomaly_rationale": "Signups fell to 12.", "anomaly_kind": "drop"},
+ ),
]
)
def test_a_firing_emits_the_properties_the_insight_stack_sends(
- self, _name: str, stored: dict[str, int], sent: dict[str, str]
+ self, _name: str, stored: dict[str, Any], sent: dict[str, str]
) -> None:
breaches = ["The insight value (150) is more than upper threshold (100)", "A second series breached"]
with patch("posthog.tasks.alerts.utils.produce_alert_internal_event") as produce:
@@ -105,7 +110,7 @@ class TestInsightLegacyEvents(APIBaseTest):
assert event.event_name == produce.call_args.kwargs["event_name"]
assert event.properties == produce.call_args.kwargs["properties"]
# The URL opens the chart without a login, so it is minted for the user that rendered it.
- if stored:
+ if "insight_chart_asset_id" in stored:
assert mint.call_args.kwargs["chart"] == RenderedInsightChart(asset_id=7, render_user_id=3)
@parameterized.expand(
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index 2117dfd583a8..1a85b2a6d97a 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -25,6 +25,7 @@ from posthog.temporal.alerts.admission import inflight_alert_ids, release_evalua
from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, SeriesEvaluation
from products.alerts.backend.judge.contract import (
+ LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,
LLM_DETECTOR_UNAVAILABLE_MESSAGE,
LLMDetectorMisconfiguredError,
LLMDetectorUnavailableError,
@@ -36,6 +37,8 @@ from products.alerts.backend.platform_source_cycle import (
CAPACITY_REJECTED,
INFLIGHT_KEY,
MAX_BREACH_CHARS,
+ MAX_DETECTOR_POINTS,
+ MAX_RATIONALE_CHARS,
evaluate_insight_check,
plan_insight_batch,
readmit_retried_check,
@@ -223,6 +226,60 @@ class TestPlatformInsightEvaluation(APIBaseTest):
# A render costs up to a minute and a half, so the parallel run must never pay for one.
assert render.called is (owner == "platform")
+ @parameterized.expand(
+ [
+ (
+ "quiet_ai_check",
+ {"type": "llm"},
+ AlertEvaluationResult(
+ value=50.0,
+ breaches=[],
+ interval="day",
+ triggered_metadata={"detector_type": "llm", "rationale": "x" * 5000, "kind": "none"},
+ ),
+ {
+ "interval": "day",
+ "triggered_metadata": {
+ "detector_type": "llm",
+ "rationale": "x" * (MAX_RATIONALE_CHARS - 1) + "…",
+ "kind": "none",
+ },
+ },
+ ),
+ (
+ "firing_statistical_check",
+ {"type": "zscore"},
+ AlertEvaluationResult(
+ value=150.0,
+ breaches=["Anomaly detected"],
+ anomaly_scores=[0.1] * 30,
+ triggered_points=[89],
+ triggered_dates=["2026-09-15"],
+ interval="day",
+ triggered_metadata={"detector_type": "zscore", "series_index": 0},
+ ),
+ {
+ "breaches": ["Anomaly detected"],
+ "anomaly_scores": [0.1] * MAX_DETECTOR_POINTS,
+ "triggered_points": [89],
+ "triggered_dates": ["2026-09-15"],
+ "interval": "day",
+ "triggered_metadata": {"detector_type": "zscore", "series_index": 0},
+ },
+ ),
+ ]
+ )
+ def test_every_detector_check_records_what_production_keeps_on_its_check(
+ self, _name: str, detector_config: dict[str, Any], result: AlertEvaluationResult, expected: dict[str, Any]
+ ) -> None:
+ alert = self._alert(detector_config=detector_config)
+ configuration = self._copy(alert)
+
+ outcome, _ = self._evaluate(configuration, result=result, uses_llm_detector=detector_config["type"] == "llm")
+
+ assert outcome is not None
+ assert {key: outcome.groups[0].source_detail.get(key) for key in expected} == expected
+
@parameterized.expand(
[
("renamed", {"name": "Signups dropped"}, ("Signups dropped", 60 * 24, "day", None)),
@@ -507,6 +564,8 @@ class TestPlatformInsightEvaluation(APIBaseTest):
LLM_DETECTOR_UNAVAILABLE_MESSAGE,
False,
)
+ # History shows this reason by its code, as it does for the alert's own checks.
+ assert outcome.failure.source_detail["error_code"] == LLM_DETECTOR_UNAVAILABLE_ERROR_CODE
@override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS=1)
def test_a_retry_stops_the_timed_out_query_and_runs_only_inside_the_pool(self) -> None: