:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): carry what an insight check sent beside its verdict (#120019)
Jimmy Zhuf0f9001274ff
diff --git a/posthog/tasks/alerts/utils.py b/posthog/tasks/alerts/utils.pyindex 73ed7d813baa..5cd76688c170 100644--- a/posthog/tasks/alerts/utils.py+++ b/posthog/tasks/alerts/utils.py@@ -426,6 +426,29 @@ def prepare_alert_insight_chart_url().exists()if not pending:return None+ return render_alert_insight_chart_url(alert, render_user)+++@frozen+class RenderedInsightChart:+ """A chart rendered for an alert, by id. The URL that opens it is minted from these when a message+ goes out, because that URL opens the chart without a login and must not be stored."""++ asset_id: int+ render_user_id: int+++def render_alert_insight_chart(+ alert: AlertConfiguration, render_user: User | None = None+) -> RenderedInsightChart | None:+ """Render the alerted insight for its Slack destinations, or None when none would show it.++ Best-effort, like `prepare_alert_insight_chart_url`: any failure returns None. The render user+ defaults the way that function's does.+ """+ render_user = render_user or _resolve_alert_render_user(alert)+ if render_user is None:+ return Nonetry:destinations = list_active_alert_destinations(team_id=alert.team_id, alert_id=str(alert.id), allowed_event_ids=(INSIGHT_ALERT_FIRING_EVENT,)@@ -448,14 +471,35 @@ def prepare_alert_insight_chart_url(exception=asset.exception,)return None+ return RenderedInsightChart(asset_id=asset.id, render_user_id=render_user.id)+ except Exception:+ logger.exception("alerts.insight_chart_render_failed", alert_id=str(alert.id))+ return None+++def insight_chart_url(*, team_id: int, chart: RenderedInsightChart) -> str | None:+ """The URL a message embeds for a rendered chart, or None when the asset is gone.++ The ids may come back from storage, so the asset is pinned to the user that rendered it.+ """+ try:return exports.get_delivery_image_url(- team_id=alert.team_id, asset_id=asset.id, expiry_delta=_INSIGHT_CHART_URL_TTL+ team_id=team_id,+ asset_id=chart.asset_id,+ expiry_delta=_INSIGHT_CHART_URL_TTL,+ created_by_id=chart.render_user_id,)except Exception:- logger.exception("alerts.insight_chart_render_failed", alert_id=str(alert.id))+ logger.exception("alerts.insight_chart_url_failed", asset_id=chart.asset_id)return None+def render_alert_insight_chart_url(alert: AlertConfiguration, render_user: User | None = None) -> str | None:+ """Render the alerted insight and mint the URL that embeds it, or None when no chart applies."""+ chart = render_alert_insight_chart(alert, render_user)+ return insight_chart_url(team_id=alert.team_id, chart=chart) if chart is not None else None++def detector_verdict_event_fields(alert_check: AlertCheck) -> dict[str, str]:"""The model's verdict, as its own `$insight_alert_firing` properties.diff --git a/posthog/temporal/tests/ai/test_anomaly_investigation_verdict.py b/posthog/temporal/tests/ai/test_anomaly_investigation_verdict.pyindex 71e3d7400d71..e6f4b89ac57d 100644--- a/posthog/temporal/tests/ai/test_anomaly_investigation_verdict.py+++ b/posthog/temporal/tests/ai/test_anomaly_investigation_verdict.py@@ -201,11 +201,13 @@ class TestInvestigationVerdictPersistence(NonAtomicBaseTest):model="test-model",)await sync_to_async(self._create_slack_destination)()+ # A render records the user it ran as, and the URL is minted only for that user's asset.asset = await sync_to_async(ExportedAsset.objects.create)(team=self.team,insight=self.insight,export_format=ExportedAsset.ExportFormat.PNG,content=b"png-bytes",+ created_by=self.user,)mock_render.return_value = (asset, b"png-bytes")diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex 10bda809183d..68b0bd491f61 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -18,10 +18,10 @@ same set of checks.import timeimport asyncio-from collections.abc import Collection+from collections.abc import Collection, Sequencefrom dataclasses import replacefrom datetime import UTC, datetime-from typing import Final+from typing import Any, Finalfrom django.conf import settings@@ -32,7 +32,7 @@ from posthog.clickhouse.query_tagging import Feature, Product, tag_queriesfrom posthog.errors import CH_TRANSIENT_ERRORS, QueryErrorCategory, classify_query_errorfrom posthog.schema_enums import AlertCalculationIntervalfrom posthog.tasks.alerts.schedule_restriction import is_utc_datetime_blocked-from posthog.tasks.alerts.utils import skip_because_of_weekend+from posthog.tasks.alerts.utils import RenderedInsightChart, render_alert_insight_chart, skip_because_of_weekendfrom posthog.temporal.alerts.admission import admit_evaluation_slots, release_evaluation_slotfrom products.alerts.backend.evaluation import check_alert_for_insight@@ -516,6 +516,7 @@ def _decide(now=now,value=result.value,query_duration_ms=duration_ms,+ firing=(alert, result.breaches or []),)@@ -589,6 +590,11 @@ _FAILED_CHECKS: Final = frozenset({SkipReason.QUERY_FAILED, SkipReason.BROKEN_CO# The one label a grouped insight alert splits on: each breakdown value is a group.BREAKDOWN_GROUPING_KEY = "breakdown"+# How much of what a check found one group's detail keeps. Production lists one breach per series,+# and every group's detail rides in the outcome Temporal carries, so a long label must not grow it.+MAX_DETAIL_BREACHES: Final = 10+MAX_BREACH_CHARS: Final = 500+# A breakdown value can be muted on its own, which production cannot express, so the platform's# mute applies: the value is still judged and only its announcement is held. A snooze of the# production alert never reaches here, because `_decide` records that check without a query.@@ -635,6 +641,30 @@ def _decide_grouped()for verdict in decision.verdicts:_record_metrics(verdict.outcome.notification, verdict.snapshot.state.value, verdict.group.new_state)+ if any(verdict.outcome.notification == NotificationAction.FIRE for verdict in decision.verdicts):+ # The row `decide_groups` judged for each key: the first, unless a later one breaches and it did not.+ breaches: dict[str, list[str]] = {}+ for series in evaluated.series:+ key, found = bounded_grouping_key(series.identity), series.result.breaches or []+ if key not in breaches or (found and not breaches[key]):+ breaches[key] = found+ # One render serves every value that fires, because they all show the same insight.+ chart = _chart(check, alert)+ decision = replace(+ decision,+ verdicts=tuple(+ replace(+ verdict,+ group=replace(+ verdict.group,+ source_detail=_detail(breaches.get(verdict.group.grouping_key, []), chart),+ ),+ )+ if verdict.outcome.notification == NotificationAction.FIRE+ else verdict+ for verdict in decision.verdicts+ ),+ )return decision.as_outcome(configuration_id=check.id,evaluation_key=_evaluation_key(check, now),@@ -644,6 +674,36 @@ def _decide_grouped()+def _firing_detail(+ check: PlatformAlertCheckInput, alert: AlertConfiguration, breaches: Sequence[str]+) -> dict[str, Any]:+ """What production's `$insight_alert_firing` carries beside the verdict, which only the check can know."""+ return _detail(breaches, _chart(check, alert))+++def _chart(check: PlatformAlertCheckInput, alert: AlertConfiguration) -> RenderedInsightChart | None:+ # A render takes up to a minute and a half, so only an alert the platform delivers for pays+ # for one. Production renders no chart for a real-time alert either.+ if check.owned_by_platform and alert.calculation_interval != AlertCalculationInterval.REAL_TIME:+ return render_alert_insight_chart(alert)+ return None+++def _clipped(breach: str) -> str:+ return breach if len(breach) <= MAX_BREACH_CHARS else breach[: MAX_BREACH_CHARS - 1] + "…"+++def _detail(breaches: Sequence[str], chart: RenderedInsightChart | None) -> dict[str, Any]:+ # Bounded, because the outcome crosses into Temporal and a breach names user-written labels.+ detail: dict[str, Any] = {"breaches": [_clipped(breach) for breach in breaches[:MAX_DETAIL_BREACHES]]}+ if chart is not None:+ # The ids rather than a URL, because the URL opens the chart without a login and history keeps+ # this for its whole retention. A delivery mints the URL when it sends.+ detail["insight_chart_asset_id"] = chart.asset_id+ detail["insight_chart_render_user_id"] = chart.render_user_id+ return detail++def _verdict(check: PlatformAlertCheckInput,snapshot: AlertSnapshot,@@ -653,8 +713,14 @@ def _verdict(value: float | None = None,query_duration_ms: int | None = None,skip: SkipReason | None = None,+ firing: tuple[AlertConfiguration, Sequence[str]] | None = None,) -> PlatformAlertOutcome:outcome = evaluate_alert_check(snapshot, check_input, now, policy=INSIGHT_ALERT_POLICY)+ source_detail = (+ _firing_detail(check, *firing)+ if firing is not None and outcome.notification == NotificationAction.FIRE+ else None+ )return _recorded(check,snapshot,@@ -667,6 +733,7 @@ def _verdict(query_duration_ms=query_duration_ms,skip=skip,disable=outcome.disable,+ source_detail=source_detail,)@@ -708,6 +775,7 @@ def _recorded(query_duration_ms: int | None = None,skip: SkipReason | None = None,disable: bool = False,+ source_detail: dict[str, Any] | None = None,) -> PlatformAlertOutcome:"""The one place an outcome is built. Every path goes through the firing decision, becausean outcome without an episode clears the start of a firing the alert is still in."""@@ -738,4 +806,5 @@ def _recorded(query_duration_ms=query_duration_ms,disable=disable,skipped=skipped,+ source_detail=source_detail,)diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex 11c21df604e2..843d8f836b1f 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -19,7 +19,7 @@ from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimifrom posthog.models.scoping import team_scopefrom posthog.redis import get_clientfrom posthog.schema_enums import AlertCalculationInterval-from posthog.tasks.alerts.utils import AlertEvaluationResult, next_check_time+from posthog.tasks.alerts.utils import AlertEvaluationResult, RenderedInsightChart, next_check_timefrom posthog.temporal.alerts.admission import inflight_alert_ids, release_evaluation_slotfrom products.alerts.backend.evaluation.contract import AlertExtractionError@@ -34,6 +34,7 @@ from products.alerts.backend.models.alert import AlertCheck, AlertConfigurationfrom products.alerts.backend.platform_source_cycle import (CAPACITY_REJECTED,INFLIGHT_KEY,+ MAX_BREACH_CHARS,evaluate_insight_check,plan_insight_batch,readmit_retried_check,@@ -94,7 +95,12 @@ class TestPlatformInsightEvaluation(APIBaseTest):)def _evaluate_per_series(- self, configuration: PlatformConfigurationSnapshot, values: dict[str, float], *, truncated: bool = False+ self,+ configuration: PlatformConfigurationSnapshot,+ values: dict[str, float] | list[tuple[str, float]],+ *,+ truncated: bool = False,+ breach: str = "above 100",) -> PlatformAlertOutcome:slot = slot_of(configuration.next_check_at, CUTOFF)expires_at = time.time() + 3600@@ -105,9 +111,9 @@ class TestPlatformInsightEvaluation(APIBaseTest):SeriesEvaluation(identity=identity,label="browser",- result=AlertEvaluationResult(value=value, breaches=["above 100"] if value > 100 else []),+ result=AlertEvaluationResult(value=value, breaches=[breach] if value > 100 else []),)- for identity, value in values.items()+ for identity, value in (values.items() if isinstance(values, dict) else values)),truncated=truncated,)@@ -132,6 +138,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):"Chrome": AlertEventKind.FIRING,"Firefox": AlertEventKind.FIRING,}+ assert {group.source_detail["breaches"][0] for group in fired.groups} == {"above 100"}with team_scope(self.team.id):platform_testing.set_due_at(configuration.id, CUTOFF - timedelta(minutes=1))@@ -140,6 +147,16 @@ class TestPlatformInsightEvaluation(APIBaseTest):expected = {"Chrome": "firing"} | ({"Firefox": missing_state} if missing_state else {})assert {group.grouping_key: group.new_state for group in resolved.groups} == expected+ def test_a_breakdown_value_reported_twice_keeps_the_breach_of_the_row_that_fired(self) -> None:+ grouping = Grouping(mode=GroupingMode.BY_RESULT_LABELS, keys=("breakdown",))+ configuration = self._copy(self._alert(), grouping=grouping.to_stored())++ fired = self._evaluate_per_series(configuration, [("Chrome", 150.0), ("Chrome", 40.0)], breach="x" * 2000)++ (group,) = fired.groups+ # The detail rides in the outcome Temporal carries, so a long breach is clipped.+ assert (group.kind, len(group.source_detail["breaches"][0])) == (AlertEventKind.FIRING, MAX_BREACH_CHARS)+def test_a_muted_breakdown_value_tracks_its_firing_without_announcing_it(self) -> None:grouping = Grouping(mode=GroupingMode.BY_RESULT_LABELS, keys=("breakdown",))configuration = self._copy(@@ -173,6 +190,34 @@ class TestPlatformInsightEvaluation(APIBaseTest):self.deliveries = evaluation.deliveries if evaluation is not None else ()return (evaluation.outcomes[0] if evaluation is not None else None), query+ @parameterized.expand(+ [+ ("source_owned", "source", {"breaches": ["above 100"]}),+ (+ "platform_owned",+ "platform",+ {"breaches": ["above 100"], "insight_chart_asset_id": 7, "insight_chart_render_user_id": 3},+ ),+ ]+ )+ def test_a_firing_records_what_production_sends_beside_it(+ self, _name: str, owner: str, expected: dict[str, Any]+ ) -> None:+ configuration = self._copy(self._alert(), owner=owner)++ with patch(+ f"{_MODULE}.render_alert_insight_chart",+ return_value=RenderedInsightChart(asset_id=7, render_user_id=3),+ ) as render:+ outcome, _ = self._evaluate(+ configuration, result=AlertEvaluationResult(value=150.0, breaches=["above 100"])+ )++ assert outcome is not None+ assert outcome.groups[0].source_detail == expected+ # A render costs up to a minute and a half, so the parallel run must never pay for one.+ assert render.called is (owner == "platform")+@parameterized.expand([("renamed", {"name": "Signups dropped"}, ("Signups dropped", 60 * 24, "day", None)),diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.pyindex 9ee011bb223b..f122442275b9 100644--- a/products/alerts_platform/backend/facade/contracts.py+++ b/products/alerts_platform/backend/facade/contracts.py@@ -566,6 +566,9 @@ class GroupOutcome:labels: dict[str, str] = field(default_factory=dict)# What a mute held back, so history separates a muted fire from a check that said nothing.muted_notification: str = ""+ # What the source's own stack sent beside this verdict, which the platform stores and returns+ # without reading, so a delivery on the HogFunction path can send what the source sent.+ source_detail: dict[str, Any] = field(default_factory=dict)@frozen@@ -645,6 +648,7 @@ class PlatformAlertOutcome:disable: bool = False,skipped: bool = False,schedule_group: str | None = None,+ source_detail: dict[str, Any] | None = None,) -> PlatformAlertOutcome:"""A check of a source that does not group: its one verdict on the empty key, or its failure."""failure = (@@ -670,6 +674,7 @@ class PlatformAlertOutcome:firing_episode=firing_episode,value=value,muted_notification=muted_notification,+ source_detail=source_detail or {},),))@@ -742,6 +747,7 @@ class AnnouncedTransition:occurred_at: datetime# The evaluation this transition came from, so a source can read back what it put in the key.evaluation_key: str = ""+ source_detail: dict[str, Any] = field(default_factory=dict)@frozendiff --git a/products/alerts_platform/backend/logic/platform_alert_events.py b/products/alerts_platform/backend/logic/platform_alert_events.pyindex d306db462c60..a1bc04a03aee 100644--- a/products/alerts_platform/backend/logic/platform_alert_events.py+++ b/products/alerts_platform/backend/logic/platform_alert_events.py@@ -14,7 +14,7 @@ batch and the engine only remembers a bounded window of them.import jsonimport hashlibfrom collections.abc import Collection, Sequence-from dataclasses import fields+from dataclasses import field, fieldsfrom datetime import UTC, datetimefrom typing import Any, Finalfrom uuid import UUID@@ -77,16 +77,18 @@ class PlatformAlertEventRow:consecutive_failures: intmuted_notification: stroccurred_at: datetime+ source_detail: dict[str, Any] = field(default_factory=dict)def as_row(self) -> dict[str, Any]:"""The row as ClickHouse takes it, derived from the fields so a new one cannot be missed.- Only the three the wire needs in another shape are named.+ Only the four the wire needs in another shape are named."""return {f.name: getattr(self, f.name) for f in fields(self)} | {"condition_snapshot": json.dumps(self.condition_snapshot),"source_config_snapshot": json.dumps(self.source_config_snapshot),"error_message": self.error_message or "",+ "source_detail": json.dumps(self.source_detail) if self.source_detail else "",}@@ -172,7 +174,7 @@ LIMIT 1 BY grouping_key"""_ANNOUNCEMENT_COLUMNS = """grouping_key, kind, episode_started_at, value, labels, condition_snapshot,- source_config_snapshot, error_message, occurred_at, alert_name, consecutive_failures"""+ source_config_snapshot, error_message, occurred_at, source_detail, alert_name, consecutive_failures"""_ANNOUNCEMENT_SQL = f"SELECT {_ANNOUNCEMENT_COLUMNS} {_ANNOUNCED_ROWS}"@@ -241,8 +243,9 @@ def announcement(error_message=error_message or None,occurred_at=occurred_at,evaluation_key=evaluation_key,+ source_detail=_snapshot(source_detail),)- for grouping_key, kind, episode_started_at, value, labels, condition_snapshot, source_config_snapshot, error_message, occurred_at, _, _ in rows+ for grouping_key, kind, episode_started_at, value, labels, condition_snapshot, source_config_snapshot, error_message, occurred_at, source_detail, _, _ in rows)# Evaluation-level, and the same on every row of one evaluation, so the first row carries it.*_, alert_name, consecutive_failures = rows[0]diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.pyindex d0ae655888af..2a5b600eb6e7 100644--- a/products/alerts_platform/backend/logic/platform_lifecycle.py+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py@@ -7,7 +7,7 @@ and every write to these rows, stays here. A source never holds one of these modfrom collections.abc import Collection, Sequencefrom dataclasses import replacefrom datetime import datetime, timedelta-from typing import Final+from typing import Any, Finalfrom uuid import UUIDfrom django.db import connection, transaction@@ -379,6 +379,7 @@ def _event_row(error_message=outcome.error_message,consecutive_failures=outcome.consecutive_failures,muted_notification=verdict.muted_notification,+ source_detail=group.source_detail if group else None,now=now,)@@ -399,6 +400,7 @@ def event_row(query_duration_ms: int | None = None,error_message: str | None = None,muted_notification: str = "",+ source_detail: dict[str, Any] | None = None,) -> PlatformAlertEventRow:"""One history row for a configuration, with the snapshots every row carries."""return PlatformAlertEventRow(@@ -422,6 +424,7 @@ def event_row(consecutive_failures=consecutive_failures,muted_notification=muted_notification,occurred_at=now,+ source_detail=source_detail or {},)diff --git a/products/alerts_platform/backend/tests/test_platform_announcement.py b/products/alerts_platform/backend/tests/test_platform_announcement.pyindex 76ec1039b3a3..faf122e5d50b 100644--- a/products/alerts_platform/backend/tests/test_platform_announcement.py+++ b/products/alerts_platform/backend/tests/test_platform_announcement.py@@ -65,7 +65,7 @@ class TestAnnouncement(ClickhouseTestMixin, APIBaseTest):def test_a_recorded_firing_comes_back_as_the_message_it_should_send(self) -> None:module = "products.alerts_platform.backend.logic.platform_alert_events"with patch(f"{module}.sync_execute", wraps=sync_execute) as executed:- insert_events(self.team.id, [self._row()])+ insert_events(self.team.id, [self._row(source_detail={"breaches": ["above 10"]})])result = self._announcement()# As `default`, the history write and read share that user's per-user concurrency limit@@ -85,6 +85,7 @@ class TestAnnouncement(ClickhouseTestMixin, APIBaseTest):# after the check it is rendered.assert transition.condition["threshold_count"] == 10assert transition.source_config["service_names"] == ["checkout"]+ assert transition.source_detail == {"breaches": ["above 10"]}def test_a_check_that_announced_nothing_reaches_no_destination(self) -> None:insert_events(self.team.id, [self._row(kind=AlertEventKind.CHECK)])diff --git a/products/alerts_platform/backend/tests/test_platform_lifecycle.py b/products/alerts_platform/backend/tests/test_platform_lifecycle.pyindex 7e2ee760896b..525ad96478cb 100644--- a/products/alerts_platform/backend/tests/test_platform_lifecycle.py+++ b/products/alerts_platform/backend/tests/test_platform_lifecycle.py@@ -56,7 +56,16 @@ from products.alerts_platform.backend.logic.platform_lifecycle import CONFIGURATfrom products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfigurationfrom products.alerts_platform.backend.models.platform_alert_events_sql import SHARDED_PLATFORM_ALERT_EVENTS_TABLE-_GROUP_FIELDS = ("kind", "new_state", "notified", "firing_episode", "value", "labels", "muted_notification")+_GROUP_FIELDS = (+ "kind",+ "new_state",+ "notified",+ "firing_episode",+ "value",+ "labels",+ "muted_notification",+ "source_detail",+)class TestPlatformAlertLifecycle(ClickhouseTestMixin, APIBaseTest):@@ -364,12 +373,14 @@ class TestPlatformAlertLifecycle(ClickhouseTestMixin, APIBaseTest):query_duration_ms=12,muted_notification="fire",firing_episode=FiringEpisode(started_at=self.cutoff, ended=False),+ source_detail={"breaches": ["above 10"]},)rows = sync_execute("""SELECT kind, previous_state, state, value, alert_name, muted_notification,- JSONExtractInt(condition_snapshot, 'threshold_count'), episode_started_at, source_kind+ JSONExtractInt(condition_snapshot, 'threshold_count'), episode_started_at, source_kind,+ JSONExtractArrayRaw(source_detail, 'breaches')FROM platform_alert_eventsWHERE team_id = %(team_id)s AND configuration_id = %(configuration_id)s""",@@ -377,7 +388,9 @@ class TestPlatformAlertLifecycle(ClickhouseTestMixin, APIBaseTest):)# `insert_events` never raises, so without reading a row back a broken write is invisible.- assert rows == [("firing", "not_firing", "firing", 47.0, "API errors", "fire", 10, self.cutoff, "logs")]+ assert rows == [+ ("firing", "not_firing", "firing", 47.0, "API errors", "fire", 10, self.cutoff, "logs", ['"above 10"'])+ ]def test_each_group_of_a_check_keeps_its_own_instance_and_history_row(self) -> None:self._record(