:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): run metrics investigations for platform-owned insight alerts (#127925)
Jimmy Zhu09d01bc802f9
diff --git a/posthog/tasks/alerts/metrics_investigation.py b/posthog/tasks/alerts/metrics_investigation.pyindex 80adf83b1785..1eaf195d5c3f 100644--- a/posthog/tasks/alerts/metrics_investigation.py+++ b/posthog/tasks/alerts/metrics_investigation.py@@ -14,6 +14,7 @@ ClickHouse queries and must never hold the row lock or affect the check outcome."""import math+from datetime import datetimefrom typing import TYPE_CHECKING, Any, NamedTupleimport structlog@@ -93,6 +94,11 @@ def run_metrics_alert_investigation(alert: AlertConfiguration, alert_check: Alerdef _run_investigation(alert: AlertConfiguration, alert_check: AlertCheck) -> str:+ return investigate_metrics_alert(alert, fired_at=alert_check.created_at)+++def investigate_metrics_alert(alert: AlertConfiguration, *, fired_at: datetime) -> str:+ """The summary of an investigation of the window around `fired_at`. Raises when it cannot run."""# Deferred: the facade pulls in HogQL machinery that must stay off the# alerts import path (matching how core defers facade.queries).from products.metrics.backend.facade.api import investigate_incident # noqa: PLC0415@@ -104,7 +110,7 @@ def _run_investigation(alert: AlertConfiguration, alert_check: AlertCheck) -> stteam=alert.team,context=IncidentContext(metric_name=target.metric_name,- fired_at=alert_check.created_at,+ fired_at=fired_at,service_name=target.service_name,),)diff --git a/products/alerts/backend/platform_investigation.py b/products/alerts/backend/platform_investigation.pyindex 25f99b211e05..d4ed1808e20e 100644--- a/products/alerts/backend/platform_investigation.py+++ b/products/alerts/backend/platform_investigation.py@@ -8,6 +8,8 @@ The decision is a read, so it can run inside an evaluation that a retry repeats.write, and runs once the evaluation is recorded."""+from collections.abc import Callable+from contextlib import AbstractContextManager, nullcontextfrom dataclasses import replacefrom datetime import UTC, datetimefrom typing import Any, Final@@ -15,7 +17,12 @@ from uuid import UUIDfrom django.db import connection, transaction+import structlog++from posthog.schema import AlertState as InsightAlertState+from posthog.dataclasses import frozen+from posthog.tasks.alerts.metrics_investigation import investigate_metrics_alert, should_investigate_metrics_alertfrom posthog.temporal.alerts.investigation import (InvestigationDecision,carried_verdict_suppresses,@@ -36,11 +43,14 @@ from products.alerts.backend.models.alert import AlertConfiguration, Investigatifrom products.alerts.backend.models.investigation import InsightAlertInvestigationfrom products.alerts.backend.platform_investigation_detail import (INVESTIGATION_REQUESTED_KEY,+ METRICS_INVESTIGATION_REQUESTED_KEY,InvestigationClaims,InvestigationStart,+ MetricsInvestigationRun,requested_investigation,)from products.alerts_platform.backend.facade.contracts import (+ AlertDeliveryRequest,AlertEventKind,HeldDelivery,HeldDeliveryAddress,@@ -51,8 +61,13 @@ from products.alerts_platform.backend.facade.contracts import (from products.alerts_platform.backend.facade.enums import HoldReleasedByfrom products.alerts_platform.backend.facade.lifecycle import AlertState, NotificationAction+logger = structlog.get_logger(__name__)+_CLAIM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))"HOLD_REASON: Final = "investigation"+METRICS_HOLD_REASON: Final = "metrics_investigation"+# A metrics investigation is a few bounded queries, so its firing waits minutes at most, not the agent's grace.+METRICS_HOLD_SECONDS: Final = 5 * 60_EVALUATED_KEYS: Final = ("anomaly_scores", "triggered_points", "triggered_dates", "interval", "triggered_metadata")# A verdict-change follow-up is recorded under the key of the firing it follows up plus this suffix,# so it gets a delivery of its own.@@ -144,7 +159,8 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEvA firing the agent investigates asks for its claim in `source_detail`, and a gated alert holds thefiring's delivery until the verdict. A firing past the budget is muted when the firing's last- verdict suppresses it. Only an owned check, because a parallel run must never pay for an agent run.+ verdict suppresses it. A metrics alert's firing asks for its metrics investigation, which holds+ nothing. Only an owned check, because a parallel run must never pay for an agent run."""if not check.owned_by_platform or check.legacy_configuration_id is None or len(evaluation.outcomes) != 1:return evaluation@@ -152,10 +168,36 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEv# A skipped check keeps a firing alert's state without evaluating it, so it has nothing to investigate.if outcome.skipped or len(outcome.groups) != 1 or outcome.groups[0].new_state != AlertState.FIRING.value:return evaluation- alert = AlertConfiguration.objects.filter(team_id=check.team_id, id=check.legacy_configuration_id).first()+ alert = (+ AlertConfiguration.objects.select_related("insight")+ .filter(team_id=check.team_id, id=check.legacy_configuration_id)+ .first()+ )if alert is None:return evaluationgroup = outcome.groups[0]+ was_firing = check.instance().state == AlertState.FIRING.value+ if should_investigate_metrics_alert(+ alert,+ previous_state=InsightAlertState.FIRING if was_firing else None,+ new_state=InsightAlertState.FIRING,+ ):+ group = replace(+ group, source_detail={**group.source_detail, METRICS_INVESTIGATION_REQUESTED_KEY: str(alert.id)}+ )+ # Held, as the insight stack sends a metrics firing with its explanation. The hold also lets+ # delivery drop the incident trigger of a firing that resolved while it waited.+ metrics_held = None+ if evaluation.deliveries:+ (request,) = evaluation.deliveries+ metrics_held = HeldDelivery(+ request=request, reason=METRICS_HOLD_REASON, release_after_seconds=METRICS_HOLD_SECONDS+ )+ return replace(+ evaluation,+ outcomes=(replace(outcome, groups=(group,), held_delivery=metrics_held),),+ deliveries=() if metrics_held else evaluation.deliveries,+ )decision = decide_investigation(alert,firing=True,@@ -184,13 +226,14 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEvdef claim_requested_investigations(team_id: int, outcomes: tuple[PlatformAlertOutcome, ...]) -> InvestigationClaims:- """Claims the investigation each recorded firing asked for, and returns the agent runs to start.+ """Claims the investigation each recorded firing asked for, and returns the runs to start.A refused claim releases the firing's held delivery by the firing's last verdict, as the insightstack mutes a check that runs no investigation of its own."""starts: list[InvestigationStart] = []deliveries = []+ metrics: list[MetricsInvestigationRun] = []for outcome in outcomes:requested = requested_investigation(outcome)if requested is None:@@ -200,6 +243,27 @@ def claim_requested_investigations(team_id: int, outcomes: tuple[PlatformAlertOu# The safety net sends the held firing at its deadline.continuegroup = outcome.groups[0]+ if requested.metrics:+ claim = claim_investigation(+ alert,+ configuration_id=outcome.configuration_id,+ evaluation_key=outcome.evaluation_key,+ episode_started_at=requested.episode_started_at,+ kind=InsightAlertInvestigation.Kind.METRICS,+ evaluated={"value": group.value},+ )+ if claim.claimed:+ metrics.append(+ MetricsInvestigationRun(+ team_id=team_id,+ investigation_id=str(claim.investigation_id),+ configuration_id=str(outcome.configuration_id),+ evaluation_key=outcome.evaluation_key,+ )+ )+ elif outcome.held_delivery is not None and (request := _release_metrics_hold(team_id, outcome)):+ deliveries.append(request)+ continueclaim = claim_investigation(alert,configuration_id=outcome.configuration_id,@@ -237,4 +301,67 @@ def claim_requested_investigations(team_id: int, outcomes: tuple[PlatformAlertOu)if request is not None:deliveries.append(request)- return InvestigationClaims(starts=tuple(starts), deliveries=tuple(deliveries))+ return InvestigationClaims(starts=tuple(starts), deliveries=tuple(deliveries), metrics=tuple(metrics))+++def _release_metrics_hold(team_id: int, outcome: PlatformAlertOutcome) -> AlertDeliveryRequest | None:+ return release_held(+ HeldDeliveryAddress(+ team_id=team_id, configuration_id=str(outcome.configuration_id), evaluation_key=outcome.evaluation_key+ ),+ suppress=False,+ released_by=HoldReleasedBy.CLAIM_REFUSED,+ )+++def release_metrics_firing(run: MetricsInvestigationRun) -> AlertDeliveryRequest | None:+ """Sends the firing a metrics investigation held, once the run ended or could not start. Returns+ the request the caller starts. A retry gets the same request back."""+ return release_held(+ HeldDeliveryAddress(+ team_id=run.team_id, configuration_id=run.configuration_id, evaluation_key=run.evaluation_key+ ),+ suppress=False,+ released_by=HoldReleasedBy.VERDICT,+ )+++def fail_unstarted_metrics_investigation(run: MetricsInvestigationRun) -> None:+ """Records that a granted metrics investigation never started, because its activity ran out of time."""+ InsightAlertInvestigation.objects.for_team(run.team_id).filter(+ id=run.investigation_id, status=InvestigationStatus.PENDING+ ).update(+ status=InvestigationStatus.FAILED, error={"message": "The investigation ran out of time before it started."}+ )+++def run_metrics_investigation(+ run: MetricsInvestigationRun, *, query_slot: Callable[[], AbstractContextManager[None]] = nullcontext+) -> None:+ """Runs a metrics alert's investigation once, and records its summary or its failure.++ The move from PENDING to RUNNING is the run's lease, so a retry of a timed-out attempt finds the+ row taken and runs nothing. A run that dies holds RUNNING until the in-flight horizon passes,+ as an agent run that dies does. Its failure never fails the activity, because the firing it+ explains has already gone out. `query_slot` holds a ClickHouse slot around the queries, so they+ share the evaluation's concurrency cap.+ """+ rows = InsightAlertInvestigation.objects.for_team(run.team_id).filter(id=run.investigation_id)+ if not rows.filter(status=InvestigationStatus.PENDING).update(status=InvestigationStatus.RUNNING):+ return+ investigation = rows.select_related("alert_configuration__insight", "alert_configuration__team").get()+ alert = investigation.alert_configuration+ running = rows.filter(status=InvestigationStatus.RUNNING)+ try:+ # The firing's start, because a metrics alert investigates only the check that started firing.+ with query_slot():+ summary = investigate_metrics_alert(+ alert, fired_at=investigation.episode_started_at or investigation.created_at+ )+ except Exception as error:+ logger.exception("platform_metrics_alert_investigation_failed", alert_id=str(alert.id))+ running.update(+ status=InvestigationStatus.FAILED, error={"message": (str(error) or type(error).__name__)[:1000]}+ )+ return+ running.update(status=InvestigationStatus.DONE, summary=summary)diff --git a/products/alerts/backend/platform_investigation_detail.py b/products/alerts/backend/platform_investigation_detail.pyindex c31b4db8a3f2..5020343ebcb8 100644--- a/products/alerts/backend/platform_investigation_detail.py+++ b/products/alerts/backend/platform_investigation_detail.py@@ -14,6 +14,8 @@ from products.alerts_platform.backend.facade.contracts import AlertDeliveryReque# The legacy alert whose agent investigates the firing. Recorded with the firing, so history shows# which checks asked for an investigation.INVESTIGATION_REQUESTED_KEY = "investigation_alert_id"+# The same, for a metrics alert, whose investigation is a few metric queries rather than an agent run.+METRICS_INVESTIGATION_REQUESTED_KEY = "metrics_investigation_alert_id"@frozen@@ -42,24 +44,38 @@ class InvestigationStart:}+@frozen+class MetricsInvestigationRun:+ """A metrics investigation the claim granted, which the workflow runs in an activity of its own."""++ team_id: int+ investigation_id: str+ # The firing the run explains, whose held delivery the run releases once it ends.+ configuration_id: str = ""+ evaluation_key: str = ""++@frozenclass InvestigationClaims:starts: tuple[InvestigationStart, ...] = ()# Held firings whose claim was refused, released by the firing's last verdict and due to start.deliveries: tuple[AlertDeliveryRequest, ...] = ()+ metrics: tuple[MetricsInvestigationRun, ...] = ()@frozenclass RequestedInvestigation:alert_id: strepisode_started_at: datetime | None+ metrics: bool = Falsedef requested_investigation(outcome: PlatformAlertOutcome) -> RequestedInvestigation | None:"""The investigation an outcome asks for, or None. Only an ungrouped firing asks for one."""for group in outcome.groups:- alert_id = group.source_detail.get(INVESTIGATION_REQUESTED_KEY)- if alert_id is not None:- started_at = group.firing_episode.started_at if group.firing_episode else None+ started_at = group.firing_episode.started_at if group.firing_episode else None+ if (alert_id := group.source_detail.get(INVESTIGATION_REQUESTED_KEY)) is not None:return RequestedInvestigation(alert_id=str(alert_id), episode_started_at=started_at)+ if (alert_id := group.source_detail.get(METRICS_INVESTIGATION_REQUESTED_KEY)) is not None:+ return RequestedInvestigation(alert_id=str(alert_id), episode_started_at=started_at, metrics=True)return Nonediff --git a/products/alerts/backend/platform_investigation_message.py b/products/alerts/backend/platform_investigation_message.pyindex f4ef2bf7b586..dcc1a3fd57cb 100644--- a/products/alerts/backend/platform_investigation_message.py+++ b/products/alerts/backend/platform_investigation_message.py@@ -96,7 +96,10 @@ def _read(team_id: int, transition: AnnouncedTransition) -> FiringInvestigationvalue = evaluated.get("value")breaches: tuple[str, ...] | None = Nonelink: str | None = None- if follow_up or released_by == HoldReleasedBy.VERDICT:+ # Only the agent's verdict rewords a firing. A metrics investigation explains the firing beside+ # its own breaches, so its release keeps them, also when it failed.+ agent = kind == InsightAlertInvestigation.Kind.AGENT+ if follow_up or (agent and released_by == HoldReleasedBy.VERDICT):previous_verdict = (rows.filter(episode_started_at=episode_started_at, created_at__lt=created_at, verdict__isnull=False).order_by("-created_at")@@ -116,8 +119,11 @@ def _read(team_id: int, transition: AnnouncedTransition) -> FiringInvestigation).lines)link = notebook_url(notebook_short_id) if notebook_short_id else None- elif released_by == HoldReleasedBy.SAFETY_NET:+ elif agent and released_by == HoldReleasedBy.SAFETY_NET:breaches = tuple(safety_net_breaches(triggered_dates=triggered_dates, value=value))+ elif not agent and status == InvestigationStatus.DONE and summary:+ # Every channel reads the breaches, so the explanation goes after them, as the agent's teaser does.+ breaches = (*(str(line) for line in transition.source_detail.get("breaches") or []), summary)return FiringInvestigation(status=status,diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex b3dcf41564eb..c5e51f5f79f1 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -330,22 +330,24 @@ def readmit_retried_check(return token if admitted else None-class _ClickHouseSlot:+class ClickHouseQuerySlot:"""Holds a ClickHouse slot around one attempt's query, and counts how long it waited for one.Raises `ClickHouseBudgetFull` when no slot comes free in time. `holder` names one attempt of onecheck, so a retry never takes over the slot of the attempt before it."""- def __init__(self, holder: str) -> None:+ def __init__(self, holder: str, *, wait_seconds: float | None = None) -> None:self.holder = holderself.waited_ms = 0+ # A caller with a budget of its own waits no longer than it, on the same shared cap.+ self._limit = _QUERY_LIMIT if wait_seconds is None else replace(_QUERY_LIMIT, retry_timeout=wait_seconds)@contextmanagerdef __call__(self) -> Iterator[None]:waiting_since = time.monotonic()try:- slot = _QUERY_LIMIT.use(self.holder, limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS)+ slot = self._limit.use(self.holder, limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS)except ConcurrencyLimitExceeded as error:raise ClickHouseBudgetFull() from errorexcept Exception as error:@@ -359,13 +361,13 @@ class _ClickHouseSlot:finally:if slot is not None:try:- _QUERY_LIMIT.release(slot)+ self._limit.release(slot)except Exception:# Best effort, so a Redis failure never replaces the query's result. The lease frees the slot.logger.exception("Could not release a ClickHouse slot", holder=self.holder)-def _with_capacity_retries[T](run: Callable[[], T], alert: AlertConfiguration, query_slot: _ClickHouseSlot) -> T:+def _with_capacity_retries[T](run: Callable[[], T], alert: AlertConfiguration, query_slot: ClickHouseQuerySlot) -> T:"""Runs a check's query, and runs it again after a ClickHouse refusal for load, on the retrypolicy production evaluates the alert's cadence with.@@ -744,7 +746,7 @@ def _decide(alert_calculation_interval=alert.calculation_interval,alert_config_type=(alert.config or {}).get("type"),)- query_slot = _ClickHouseSlot(f"{evaluation_id}:{attempt}")+ query_slot = ClickHouseQuerySlot(f"{evaluation_id}:{attempt}")if check.grouping.mode != GroupingMode.SINGLE:return _decide_grouped(check, snapshot, alert, now=now, query_slot=query_slot)started_at = time.monotonic()@@ -907,7 +909,7 @@ def _decide_grouped(alert: AlertConfiguration,*,now: datetime,- query_slot: _ClickHouseSlot,+ query_slot: ClickHouseQuerySlot,) -> PlatformAlertOutcome | None:"""One check of a breakdown alert, judged per breakdown value through the platform's decision.diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.pyindex adec0ea424ae..e25e84ab3674 100644--- a/products/alerts/backend/temporal/platform_evaluate.py+++ b/products/alerts/backend/temporal/platform_evaluate.py@@ -38,6 +38,7 @@ with workflow.unsafe.imports_passed_through():from products.alerts.backend.platform_investigation_detail import (InvestigationClaims,InvestigationStart,+ MetricsInvestigationRun,requested_investigation,)from products.alerts_platform.backend.facade.contracts import (@@ -95,6 +96,12 @@ STOP_QUERIES_START_TO_CLOSE = dt.timedelta(seconds=60)RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)CLAIM_START_TO_CLOSE = dt.timedelta(seconds=10)CLAIM_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=20)+# A metrics alert's investigation is a few bounded metric queries. One attempt, because the run+# leases its row, so a retry would find it taken.+METRICS_INVESTIGATION_START_TO_CLOSE = dt.timedelta(seconds=45)+# Time a run's queries can take, so the last run starts while the activity can still finish it.+METRICS_INVESTIGATION_MARGIN = dt.timedelta(seconds=15)+METRICS_INVESTIGATION_SCHEDULE_TO_CLOSE = METRICS_INVESTIGATION_START_TO_CLOSE + CHECK_QUEUE_TOLERANCERECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)# The pool admits once per round, so a batch that admits only once evaluates at most one pool of# checks per tick. Rounds refill the slots that finished checks free, the way production's@@ -104,7 +111,8 @@ ADMISSION_POLL_INTERVAL = dt.timedelta(seconds=2)# What the source needs from its binding's `evaluation_timeout`. The checks run side by side, so# the batch costs the admission rounds plus one check's budget, not one per check. The last check# can end while an earlier record still runs, so the tail holds two records and their failures,-# or one record and the stop of a failed check's queries.+# or one record and the stop of a failed check's queries. A firing's claim follows its record, and a+# failed check claims nothing.EVALUATION_BUDGET = (PLAN_SCHEDULE_TO_CLOSE+ ADMISSION_RUN_BUDGET@@ -112,7 +120,8 @@ EVALUATION_BUDGET = (+ max(CHART_SCHEDULE_TO_CLOSE, FAIL_SCHEDULE_TO_CLOSE)+ RECORD_SCHEDULE_TO_CLOSE+ CLAIM_SCHEDULE_TO_CLOSE- + max(FAIL_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE + CLAIM_SCHEDULE_TO_CLOSE, STOP_QUERIES_START_TO_CLOSE)+ + METRICS_INVESTIGATION_SCHEDULE_TO_CLOSE+ + max(FAIL_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE, STOP_QUERIES_START_TO_CLOSE))_PATCH_READMISSION = "platform-insight-readmission-2026-10"# Runs started before it recorded every outcome in one activity, and replay must schedule the same.@@ -130,6 +139,8 @@ _PATCH_RECORD_AS_FINISHED = "platform-insight-record-as-finished-2026-10"_PATCH_WAIT_FOR_POOL = "platform-insight-wait-for-pool-2026-10"# Runs started before it claimed investigations after a record, and replay must not schedule a claim._PATCH_INVESTIGATIONS = "platform-insight-investigations-2026-11"+# Runs started before it ran metrics investigations after a claim, and replay must not schedule one.+_PATCH_METRICS_INVESTIGATIONS = "platform-insight-metrics-investigations-2026-11"@frozen@@ -180,6 +191,11 @@ class ClaimInsightInvestigationsInputs:outcomes: tuple[PlatformAlertOutcome, ...]+@frozen+class MetricsInvestigationsInputs:+ runs: tuple[MetricsInvestigationRun, ...]++@frozenclass StopInsightQueriesInputs:team_id: int@@ -365,6 +381,35 @@ def claim_platform_insight_investigations_activity(inputs: ClaimInsightInvestigareturn claim_requested_investigations(inputs.team_id, inputs.outcomes)+@close_db_connections+def run_platform_metrics_investigations_activity(inputs: MetricsInvestigationsInputs) -> list[AlertDeliveryRequest]:+ """Runs the metrics investigations a claim granted, apart from the claim, so their queries never+ hold back an agent's start. Releases each run's held firing, and returns the deliveries to start."""+ from products.alerts.backend.platform_investigation import (+ fail_unstarted_metrics_investigation, # noqa: PLC0415+ release_metrics_firing, # noqa: PLC0415+ run_metrics_investigation, # noqa: PLC0415+ )+ from products.alerts.backend.platform_source_cycle import ClickHouseQuerySlot # noqa: PLC0415++ # Stops starting runs before the activity's own timeout, because Temporal does not stop a sync+ # loop that outlives it. A run already started finishes its own queries.+ deadline = time.monotonic() + (METRICS_INVESTIGATION_START_TO_CLOSE - METRICS_INVESTIGATION_MARGIN).total_seconds()+ released: list[AlertDeliveryRequest] = []+ for run in inputs.runs:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ fail_unstarted_metrics_investigation(run)+ else:+ # The slot wait counts against the same deadline, so a full ClickHouse budget cannot use up the activity.+ slot = ClickHouseQuerySlot(f"metrics-investigation:{run.investigation_id}", wait_seconds=remaining)+ run_metrics_investigation(run, query_slot=slot)+ if request := release_metrics_firing(run):+ released.append(request)+ return released++@activity.defn@close_db_connectionsdef render_platform_insight_chart_activity(inputs: InsightChartInputs) -> RenderedChart | None:@@ -794,6 +839,20 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):claims = await self._claim_investigations(inputs, outcomes)await asyncio.gather(*(self._start_investigation(start) for start in claims.starts))await start_deliveries(list(claims.deliveries))+ if workflow.patched(_PATCH_METRICS_INVESTIGATIONS) and claims.metrics:+ try:+ released = await workflow.execute_activity(+ run_platform_metrics_investigations_activity,+ MetricsInvestigationsInputs(runs=claims.metrics),+ start_to_close_timeout=METRICS_INVESTIGATION_START_TO_CLOSE,+ schedule_to_close_timeout=METRICS_INVESTIGATION_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ except ActivityError:+ # The held firings go out at their hold's deadline, and the rows stay in flight until the horizon passes.+ workflow.logger.warning("Could not run the metrics investigations of platform insight firings")+ else:+ await start_deliveries(released)async def _claim_investigations(self, inputs: SourceEvaluationInputs, outcomes: tuple[PlatformAlertOutcome, ...]@@ -843,6 +902,7 @@ PLATFORM_EVALUATION_ACTIVITIES = [evaluate_platform_insight_check_activity,render_platform_insight_chart_activity,claim_platform_insight_investigations_activity,+ run_platform_metrics_investigations_activity,fail_platform_insight_check_activity,stop_platform_insight_queries_activity,]diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.pyindex f08c9366e16a..eee5f49a1e27 100644--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py@@ -2,6 +2,7 @@ import uuidimport asyncioimport threadingfrom collections.abc import AsyncIterator+from dataclasses import replaceimport pytestfrom unittest.mock import patch@@ -21,6 +22,11 @@ from products.alerts.backend.platform_chart_detail import (CHART_RENDER_USER_ID_KEY,RenderedChart,)+from products.alerts.backend.platform_investigation_detail import (+ METRICS_INVESTIGATION_REQUESTED_KEY,+ InvestigationClaims,+ MetricsInvestigationRun,+)from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKSfrom products.alerts.backend.temporal.platform_evaluate import (CHECK_HEARTBEAT_TIMEOUT,@@ -28,11 +34,13 @@ from products.alerts.backend.temporal.platform_evaluate import (EVALUATION_BUDGET,REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,AdmittedInsightCheck,+ ClaimInsightInvestigationsInputs,FailedInsightCheckInputs,InsightAlertPlatformEvaluateWorkflow,InsightBatchPlanInputs,InsightChartInputs,InsightCheckInputs,+ MetricsInvestigationsInputs,StopInsightQueriesInputs,evaluate_platform_insight_llm_check_activity,)@@ -42,6 +50,7 @@ from products.alerts_platform.backend.facade.contracts import (AlertDeliveryRequest,AlertEventKind,GroupOutcome,+ HeldDelivery,PlatformAlertOutcome,SourceBatchEvaluation,SourceEvaluationInputs,@@ -591,3 +600,87 @@ async def test_a_record_that_fails_while_a_check_still_runs_records_again_at_theassert result == 2assert attempts[0] == [fast]assert sorted(configuration_id for attempt in attempts[1:] for configuration_id in attempt) == [fast, slow]+++async def test_a_metrics_firing_is_delivered_once_its_investigation_releases_it(+ environment: WorkflowEnvironment,+) -> None:+ released: list[AlertDeliveryRequest] = []++ @activity.defn(name="plan_platform_insight_checks_activity")+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:+ return [] if inputs.exclude else [AdmittedInsightCheck(configuration_id=ADMITTED[0], owned_by_platform=True)]++ @activity.defn(name="evaluate_platform_insight_check_activity")+ async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:+ outcome = _outcome(inputs.configuration_id, AlertEventKind.FIRING, "firing")+ group = outcome.groups[0]+ outcome = PlatformAlertOutcome(+ configuration_id=outcome.configuration_id,+ evaluation_key=outcome.evaluation_key,+ consecutive_failures=0,+ groups=(+ GroupOutcome(+ grouping_key=group.grouping_key,+ kind=group.kind,+ new_state=group.new_state,+ notified=group.notified,+ source_detail={METRICS_INVESTIGATION_REQUESTED_KEY: "legacy-alert"},+ ),+ ),+ )+ delivery = AlertDeliveryRequest(+ source=SourceKind.INSIGHT,+ team_id=1,+ configuration_id=inputs.configuration_id,+ evaluation_key=outcome.evaluation_key,+ destination_alert_id="legacy-alert",+ event_ids_by_kind={"firing": "$insight_alert_firing"},+ )+ # Held by the check, so only the investigation's release starts it.+ held = replace(+ outcome,+ held_delivery=HeldDelivery(request=delivery, reason="metrics_investigation", release_after_seconds=300),+ )+ released.append(delivery)+ return SourceBatchEvaluation(outcomes=(held,), deliveries=())++ @activity.defn(name=RECORD_OUTCOMES_ACTIVITY)+ async def record(inputs: SourceOutcomeInputs) -> int:+ return len(inputs.outcomes)++ @activity.defn(name="claim_platform_insight_investigations_activity")+ async def claim(inputs: ClaimInsightInvestigationsInputs) -> InvestigationClaims:+ return InvestigationClaims(metrics=(MetricsInvestigationRun(team_id=1, investigation_id="inv-1"),))++ @activity.defn(name="run_platform_metrics_investigations_activity")+ async def investigate(inputs: MetricsInvestigationsInputs) -> list[AlertDeliveryRequest]:+ return released++ async with Worker(+ environment.client,+ task_queue=QUEUE,+ workflows=[InsightAlertPlatformEvaluateWorkflow],+ activities=[plan, evaluate, record, claim, investigate],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ handle = await environment.client.start_workflow(+ InsightAlertPlatformEvaluateWorkflow.run,+ SourceEvaluationInputs(+ source=SourceKind.INSIGHT,+ cutoff="2026-09-16T10:00:00+00:00",+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),+ ),+ id=f"test-{uuid.uuid4()}",+ task_queue=QUEUE,+ )+ await handle.result()+ history = await handle.fetch_history()++ order = []+ for event in history.events:+ if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED:+ order.append(event.activity_task_scheduled_event_attributes.activity_type.name)+ elif event.event_type == EventType.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED:+ order.append("delivery")+ assert order.index("run_platform_metrics_investigations_activity") < order.index("delivery")diff --git a/products/alerts/backend/test/test_platform_investigation_message.py b/products/alerts/backend/test/test_platform_investigation_message.pyindex 0540d95a50b0..b64a56aa0c5e 100644--- a/products/alerts/backend/test/test_platform_investigation_message.py+++ b/products/alerts/backend/test/test_platform_investigation_message.py@@ -38,6 +38,7 @@ KEY = "slot:2026-10-20T09:00:00+00:00"OWN_BREACH = "Signups (50) is less than lower threshold (100)"ANOMALY_HEAD = "Anomaly detected on 2026-10-20."SUMMARY = "Signups fell after the **release**."+METRICS_SUMMARY = "queue.depth moved up to 420.00 (baseline 100.00, 4.2x)."TEASER = "Signups fell after the release."@@ -58,7 +59,14 @@ class TestInvestigatedFiringMessages(APIBaseTest):self.notebook_url = absolute_uri(f"/notebooks/{self.notebook.short_id}")def _investigation(- self, key: str, *, status: str = InvestigationStatus.DONE, verdict: str | None = None, at: datetime = FIRED_AT+ self,+ key: str,+ *,+ status: str = InvestigationStatus.DONE,+ verdict: str | None = None,+ at: datetime = FIRED_AT,+ kind: str = InsightAlertInvestigation.Kind.AGENT,+ summary: str | None = None,) -> None:with time_machine.travel(at, tick=False):InsightAlertInvestigation.objects.for_team(self.team.id).create(@@ -67,10 +75,10 @@ class TestInvestigatedFiringMessages(APIBaseTest):configuration_id=self.configuration_id,evaluation_key=key,episode_started_at=EPISODE,- kind=InsightAlertInvestigation.Kind.AGENT,+ kind=kind,status=status,verdict=verdict,- summary=SUMMARY if verdict else None,+ summary=summary or (SUMMARY if verdict else None),notebook_id=self.notebook.id if verdict else None,evaluated={"value": 50.0, "triggered_dates": ["2026-10-20"]},)@@ -192,6 +200,36 @@ class TestInvestigatedFiringMessages(APIBaseTest):description = describe_insight_transition(project_id=self.team.id, transition=self._transition())assert description.details == (MessageDetail(label="Threshold breached", value="\n".join(expected_breaches)),)+ @parameterized.expand(+ [+ ("released_after_its_run", InvestigationStatus.DONE, HoldReleasedBy.VERDICT, [OWN_BREACH, METRICS_SUMMARY]),+ (+ "sent_by_the_safety_net_after_it_failed",+ InvestigationStatus.FAILED,+ HoldReleasedBy.SAFETY_NET,+ [OWN_BREACH],+ ),+ ]+ )+ def test_a_metrics_firing_keeps_its_own_breaches_and_adds_its_explanation(+ self, _name: str, status: str, released_by: HoldReleasedBy, breaches: list[str]+ ) -> None:+ self._investigation(+ KEY,+ status=status,+ kind=InsightAlertInvestigation.Kind.METRICS,+ summary=METRICS_SUMMARY if status == InvestigationStatus.DONE else None,+ )+ self._released_by(released_by)++ properties = self._event_properties(self._transition())++ assert properties["breaches"] == "\n".join(breaches)+ assert properties.get("investigation_notebook_url") is None+ # Email, PagerDuty and webhooks read the same breaches, so they carry the explanation too.+ description = describe_insight_transition(project_id=self.team.id, transition=self._transition())+ assert description.details == (MessageDetail(label="Threshold breached", value="\n".join(breaches)),)+def test_a_changed_verdict_follows_up_with_the_change(self) -> None:self._investigation("slot:2026-10-20T08:00:00+00:00", verdict="true_positive", at=EPISODE)self._investigation(KEY, verdict="false_positive")diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex 03b0a29d6022..76edd5c5fdc2 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -68,7 +68,9 @@ from products.alerts.backend.platform_source_cycle import (from products.alerts.backend.temporal.platform_evaluate import (AdmittedInsightCheck,InsightBatchPlanInputs,+ MetricsInvestigationsInputs,plan_platform_insight_checks_activity,+ run_platform_metrics_investigations_activity,)from products.alerts.backend.test.insight_alerts import create_insight_alertfrom products.alerts_platform.backend.facade import testing as platform_testing@@ -1004,6 +1006,39 @@ class TestPlatformInsightEvaluation(APIBaseTest):assert outcome.held_delivery is Noneassert len(self.deliveries) == 1+ @parameterized.expand([("found", None, InvestigationStatus.DONE), ("failed", RuntimeError("no data"), "failed")])+ def test_a_metrics_firing_is_held_until_its_investigation_ends(+ self, _name: str, error: Exception | None, status: str+ ) -> None:+ alert = self._alert(investigation_agent_enabled=True, investigation_gates_notifications=True)+ alert.insight.query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue.depth"}]}+ alert.insight.save()+ configuration = self._copy(alert, owner=PlatformAlertOwner.PLATFORM)+ outcome = self._fire(configuration)+ record_outcomes(self.team.id, (outcome,), CUTOFF)++ claims = claim_requested_investigations(self.team.id, (outcome,))+ with patch(+ "products.alerts.backend.platform_investigation.investigate_metrics_alert",+ side_effect=error,+ return_value="queue.depth moved up.",+ ) as investigate:+ released = [+ run_platform_metrics_investigations_activity(MetricsInvestigationsInputs(runs=claims.metrics))+ for _ in range(2)+ ]++ assert outcome.held_delivery is not None+ assert self.deliveries == ()+ assert (claims.starts, len(claims.metrics)) == ((), 1)+ # Released once the run ends, whatever it found, and a retry gets the same request back.+ assert released == [[outcome.held_delivery.request]] * 2+ investigate.assert_called_once()+ assert investigate.call_args.kwargs["fired_at"] == CUTOFF+ investigation = InsightAlertInvestigation.objects.for_team(self.team.id).get()+ assert (investigation.kind, investigation.status) == (InsightAlertInvestigation.Kind.METRICS, status)+ assert investigation.summary == ("queue.depth moved up." if error is None else None)+def test_a_skipped_check_of_a_firing_alert_asks_for_no_investigation(self) -> None:_alert, configuration = self._investigated_alert(investigation_gates_notifications=True)record_outcomes(self.team.id, (self._fire(configuration),), CUTOFF)@@ -1017,3 +1052,22 @@ class TestPlatformInsightEvaluation(APIBaseTest):assert outcome is not Noneassert outcome.skippedassert requested_investigation(outcome) is None++ def test_a_metrics_run_past_its_activity_deadline_is_recorded_as_failed(self) -> None:+ alert = self._alert(investigation_agent_enabled=True)+ alert.insight.query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue.depth"}]}+ alert.insight.save()+ configuration = self._copy(alert, owner=PlatformAlertOwner.PLATFORM)+ outcome = self._fire(configuration)+ record_outcomes(self.team.id, (outcome,), CUTOFF)+ claims = claim_requested_investigations(self.team.id, (outcome,))++ with (+ patch("products.alerts.backend.temporal.platform_evaluate.time.monotonic", side_effect=[0.0, 10_000.0]),+ patch("products.alerts.backend.platform_investigation.investigate_metrics_alert") as investigate,+ ):+ run_platform_metrics_investigations_activity(MetricsInvestigationsInputs(runs=claims.metrics))++ investigate.assert_not_called()+ investigation = InsightAlertInvestigation.objects.for_team(self.team.id).get()+ assert investigation.status == InvestigationStatus.FAILEDdiff --git a/products/alerts_platform/backend/temporal/sources.py b/products/alerts_platform/backend/temporal/sources.pyindex 4d274c1a746f..eb44243286b7 100644--- a/products/alerts_platform/backend/temporal/sources.py+++ b/products/alerts_platform/backend/temporal/sources.py@@ -51,7 +51,7 @@ SOURCE_BINDINGS: dict[SourceKind, SourceBinding] = {SourceKind.INSIGHT: SourceBinding(workflow="insight-alert-platform-evaluate",task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,- evaluation_timeout=dt.timedelta(minutes=20),+ evaluation_timeout=dt.timedelta(minutes=22),# Not the pool size. A check frees its slot in seconds, so the pool turns over many times a# tick, and a limit equal to its size caps checks per minute far below what it can run.discovery_limit=settings.ALERTS_PLATFORM_INSIGHT_KEYS_PER_TICK,