Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): run metrics investigations for platform-owned insight alerts (#127925)

Jimmy Zhu09d01bc802f9
diff --git a/posthog/tasks/alerts/metrics_investigation.py b/posthog/tasks/alerts/metrics_investigation.py
index 80adf83b1785..1eaf195d5c3f 100644
--- a/posthog/tasks/alerts/metrics_investigation.py
+++ b/posthog/tasks/alerts/metrics_investigation.py
@@ -14,6 +14,7 @@ ClickHouse queries and must never hold the row lock or affect the check outcome.
"""
import math
+from datetime import datetime
from typing import TYPE_CHECKING, Any, NamedTuple
import structlog
@@ -93,6 +94,11 @@ def run_metrics_alert_investigation(alert: AlertConfiguration, alert_check: Aler
def _run_investigation(alert: AlertConfiguration, alert_check: AlertCheck) -> str:
+ return investigate_metrics_alert(alert, fired_at=alert_check.created_at)
+
+
+def investigate_metrics_alert(alert: AlertConfiguration, *, fired_at: datetime) -> str:
+ """The summary of an investigation of the window around `fired_at`. Raises when it cannot run."""
# Deferred: the facade pulls in HogQL machinery that must stay off the
# alerts import path (matching how core defers facade.queries).
from products.metrics.backend.facade.api import investigate_incident # noqa: PLC0415
@@ -104,7 +110,7 @@ def _run_investigation(alert: AlertConfiguration, alert_check: AlertCheck) -> st
team=alert.team,
context=IncidentContext(
metric_name=target.metric_name,
- fired_at=alert_check.created_at,
+ fired_at=fired_at,
service_name=target.service_name,
),
)
diff --git a/products/alerts/backend/platform_investigation.py b/products/alerts/backend/platform_investigation.py
index 25f99b211e05..d4ed1808e20e 100644
--- a/products/alerts/backend/platform_investigation.py
+++ b/products/alerts/backend/platform_investigation.py
@@ -8,6 +8,8 @@ The decision is a read, so it can run inside an evaluation that a retry repeats.
write, and runs once the evaluation is recorded.
"""
+from collections.abc import Callable
+from contextlib import AbstractContextManager, nullcontext
from dataclasses import replace
from datetime import UTC, datetime
from typing import Any, Final
@@ -15,7 +17,12 @@ from uuid import UUID
from django.db import connection, transaction
+import structlog
+
+from posthog.schema import AlertState as InsightAlertState
+
from posthog.dataclasses import frozen
+from posthog.tasks.alerts.metrics_investigation import investigate_metrics_alert, should_investigate_metrics_alert
from posthog.temporal.alerts.investigation import (
InvestigationDecision,
carried_verdict_suppresses,
@@ -36,11 +43,14 @@ from products.alerts.backend.models.alert import AlertConfiguration, Investigati
from products.alerts.backend.models.investigation import InsightAlertInvestigation
from products.alerts.backend.platform_investigation_detail import (
INVESTIGATION_REQUESTED_KEY,
+ METRICS_INVESTIGATION_REQUESTED_KEY,
InvestigationClaims,
InvestigationStart,
+ MetricsInvestigationRun,
requested_investigation,
)
from products.alerts_platform.backend.facade.contracts import (
+ AlertDeliveryRequest,
AlertEventKind,
HeldDelivery,
HeldDeliveryAddress,
@@ -51,8 +61,13 @@ from products.alerts_platform.backend.facade.contracts import (
from products.alerts_platform.backend.facade.enums import HoldReleasedBy
from products.alerts_platform.backend.facade.lifecycle import AlertState, NotificationAction
+logger = structlog.get_logger(__name__)
+
_CLAIM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))"
HOLD_REASON: Final = "investigation"
+METRICS_HOLD_REASON: Final = "metrics_investigation"
+# A metrics investigation is a few bounded queries, so its firing waits minutes at most, not the agent's grace.
+METRICS_HOLD_SECONDS: Final = 5 * 60
_EVALUATED_KEYS: Final = ("anomaly_scores", "triggered_points", "triggered_dates", "interval", "triggered_metadata")
# A verdict-change follow-up is recorded under the key of the firing it follows up plus this suffix,
# so it gets a delivery of its own.
@@ -144,7 +159,8 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEv
A firing the agent investigates asks for its claim in `source_detail`, and a gated alert holds the
firing's delivery until the verdict. A firing past the budget is muted when the firing's last
- verdict suppresses it. Only an owned check, because a parallel run must never pay for an agent run.
+ verdict suppresses it. A metrics alert's firing asks for its metrics investigation, which holds
+ nothing. Only an owned check, because a parallel run must never pay for an agent run.
"""
if not check.owned_by_platform or check.legacy_configuration_id is None or len(evaluation.outcomes) != 1:
return evaluation
@@ -152,10 +168,36 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEv
# A skipped check keeps a firing alert's state without evaluating it, so it has nothing to investigate.
if outcome.skipped or len(outcome.groups) != 1 or outcome.groups[0].new_state != AlertState.FIRING.value:
return evaluation
- alert = AlertConfiguration.objects.filter(team_id=check.team_id, id=check.legacy_configuration_id).first()
+ alert = (
+ AlertConfiguration.objects.select_related("insight")
+ .filter(team_id=check.team_id, id=check.legacy_configuration_id)
+ .first()
+ )
if alert is None:
return evaluation
group = outcome.groups[0]
+ was_firing = check.instance().state == AlertState.FIRING.value
+ if should_investigate_metrics_alert(
+ alert,
+ previous_state=InsightAlertState.FIRING if was_firing else None,
+ new_state=InsightAlertState.FIRING,
+ ):
+ group = replace(
+ group, source_detail={**group.source_detail, METRICS_INVESTIGATION_REQUESTED_KEY: str(alert.id)}
+ )
+ # Held, as the insight stack sends a metrics firing with its explanation. The hold also lets
+ # delivery drop the incident trigger of a firing that resolved while it waited.
+ metrics_held = None
+ if evaluation.deliveries:
+ (request,) = evaluation.deliveries
+ metrics_held = HeldDelivery(
+ request=request, reason=METRICS_HOLD_REASON, release_after_seconds=METRICS_HOLD_SECONDS
+ )
+ return replace(
+ evaluation,
+ outcomes=(replace(outcome, groups=(group,), held_delivery=metrics_held),),
+ deliveries=() if metrics_held else evaluation.deliveries,
+ )
decision = decide_investigation(
alert,
firing=True,
@@ -184,13 +226,14 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEv
def claim_requested_investigations(team_id: int, outcomes: tuple[PlatformAlertOutcome, ...]) -> InvestigationClaims:
- """Claims the investigation each recorded firing asked for, and returns the agent runs to start.
+ """Claims the investigation each recorded firing asked for, and returns the runs to start.
A refused claim releases the firing's held delivery by the firing's last verdict, as the insight
stack mutes a check that runs no investigation of its own.
"""
starts: list[InvestigationStart] = []
deliveries = []
+ metrics: list[MetricsInvestigationRun] = []
for outcome in outcomes:
requested = requested_investigation(outcome)
if requested is None:
@@ -200,6 +243,27 @@ def claim_requested_investigations(team_id: int, outcomes: tuple[PlatformAlertOu
# The safety net sends the held firing at its deadline.
continue
group = outcome.groups[0]
+ if requested.metrics:
+ claim = claim_investigation(
+ alert,
+ configuration_id=outcome.configuration_id,
+ evaluation_key=outcome.evaluation_key,
+ episode_started_at=requested.episode_started_at,
+ kind=InsightAlertInvestigation.Kind.METRICS,
+ evaluated={"value": group.value},
+ )
+ if claim.claimed:
+ metrics.append(
+ MetricsInvestigationRun(
+ team_id=team_id,
+ investigation_id=str(claim.investigation_id),
+ configuration_id=str(outcome.configuration_id),
+ evaluation_key=outcome.evaluation_key,
+ )
+ )
+ elif outcome.held_delivery is not None and (request := _release_metrics_hold(team_id, outcome)):
+ deliveries.append(request)
+ continue
claim = claim_investigation(
alert,
configuration_id=outcome.configuration_id,
@@ -237,4 +301,67 @@ def claim_requested_investigations(team_id: int, outcomes: tuple[PlatformAlertOu
)
if request is not None:
deliveries.append(request)
- return InvestigationClaims(starts=tuple(starts), deliveries=tuple(deliveries))
+ return InvestigationClaims(starts=tuple(starts), deliveries=tuple(deliveries), metrics=tuple(metrics))
+
+
+def _release_metrics_hold(team_id: int, outcome: PlatformAlertOutcome) -> AlertDeliveryRequest | None:
+ return release_held(
+ HeldDeliveryAddress(
+ team_id=team_id, configuration_id=str(outcome.configuration_id), evaluation_key=outcome.evaluation_key
+ ),
+ suppress=False,
+ released_by=HoldReleasedBy.CLAIM_REFUSED,
+ )
+
+
+def release_metrics_firing(run: MetricsInvestigationRun) -> AlertDeliveryRequest | None:
+ """Sends the firing a metrics investigation held, once the run ended or could not start. Returns
+ the request the caller starts. A retry gets the same request back."""
+ return release_held(
+ HeldDeliveryAddress(
+ team_id=run.team_id, configuration_id=run.configuration_id, evaluation_key=run.evaluation_key
+ ),
+ suppress=False,
+ released_by=HoldReleasedBy.VERDICT,
+ )
+
+
+def fail_unstarted_metrics_investigation(run: MetricsInvestigationRun) -> None:
+ """Records that a granted metrics investigation never started, because its activity ran out of time."""
+ InsightAlertInvestigation.objects.for_team(run.team_id).filter(
+ id=run.investigation_id, status=InvestigationStatus.PENDING
+ ).update(
+ status=InvestigationStatus.FAILED, error={"message": "The investigation ran out of time before it started."}
+ )
+
+
+def run_metrics_investigation(
+ run: MetricsInvestigationRun, *, query_slot: Callable[[], AbstractContextManager[None]] = nullcontext
+) -> None:
+ """Runs a metrics alert's investigation once, and records its summary or its failure.
+
+ The move from PENDING to RUNNING is the run's lease, so a retry of a timed-out attempt finds the
+ row taken and runs nothing. A run that dies holds RUNNING until the in-flight horizon passes,
+ as an agent run that dies does. Its failure never fails the activity, because the firing it
+ explains has already gone out. `query_slot` holds a ClickHouse slot around the queries, so they
+ share the evaluation's concurrency cap.
+ """
+ rows = InsightAlertInvestigation.objects.for_team(run.team_id).filter(id=run.investigation_id)
+ if not rows.filter(status=InvestigationStatus.PENDING).update(status=InvestigationStatus.RUNNING):
+ return
+ investigation = rows.select_related("alert_configuration__insight", "alert_configuration__team").get()
+ alert = investigation.alert_configuration
+ running = rows.filter(status=InvestigationStatus.RUNNING)
+ try:
+ # The firing's start, because a metrics alert investigates only the check that started firing.
+ with query_slot():
+ summary = investigate_metrics_alert(
+ alert, fired_at=investigation.episode_started_at or investigation.created_at
+ )
+ except Exception as error:
+ logger.exception("platform_metrics_alert_investigation_failed", alert_id=str(alert.id))
+ running.update(
+ status=InvestigationStatus.FAILED, error={"message": (str(error) or type(error).__name__)[:1000]}
+ )
+ return
+ running.update(status=InvestigationStatus.DONE, summary=summary)
diff --git a/products/alerts/backend/platform_investigation_detail.py b/products/alerts/backend/platform_investigation_detail.py
index c31b4db8a3f2..5020343ebcb8 100644
--- a/products/alerts/backend/platform_investigation_detail.py
+++ b/products/alerts/backend/platform_investigation_detail.py
@@ -14,6 +14,8 @@ from products.alerts_platform.backend.facade.contracts import AlertDeliveryReque
# The legacy alert whose agent investigates the firing. Recorded with the firing, so history shows
# which checks asked for an investigation.
INVESTIGATION_REQUESTED_KEY = "investigation_alert_id"
+# The same, for a metrics alert, whose investigation is a few metric queries rather than an agent run.
+METRICS_INVESTIGATION_REQUESTED_KEY = "metrics_investigation_alert_id"
@frozen
@@ -42,24 +44,38 @@ class InvestigationStart:
}
+@frozen
+class MetricsInvestigationRun:
+ """A metrics investigation the claim granted, which the workflow runs in an activity of its own."""
+
+ team_id: int
+ investigation_id: str
+ # The firing the run explains, whose held delivery the run releases once it ends.
+ configuration_id: str = ""
+ evaluation_key: str = ""
+
+
@frozen
class InvestigationClaims:
starts: tuple[InvestigationStart, ...] = ()
# Held firings whose claim was refused, released by the firing's last verdict and due to start.
deliveries: tuple[AlertDeliveryRequest, ...] = ()
+ metrics: tuple[MetricsInvestigationRun, ...] = ()
@frozen
class RequestedInvestigation:
alert_id: str
episode_started_at: datetime | None
+ metrics: bool = False
def requested_investigation(outcome: PlatformAlertOutcome) -> RequestedInvestigation | None:
"""The investigation an outcome asks for, or None. Only an ungrouped firing asks for one."""
for group in outcome.groups:
- alert_id = group.source_detail.get(INVESTIGATION_REQUESTED_KEY)
- if alert_id is not None:
- started_at = group.firing_episode.started_at if group.firing_episode else None
+ started_at = group.firing_episode.started_at if group.firing_episode else None
+ if (alert_id := group.source_detail.get(INVESTIGATION_REQUESTED_KEY)) is not None:
return RequestedInvestigation(alert_id=str(alert_id), episode_started_at=started_at)
+ if (alert_id := group.source_detail.get(METRICS_INVESTIGATION_REQUESTED_KEY)) is not None:
+ return RequestedInvestigation(alert_id=str(alert_id), episode_started_at=started_at, metrics=True)
return None
diff --git a/products/alerts/backend/platform_investigation_message.py b/products/alerts/backend/platform_investigation_message.py
index f4ef2bf7b586..dcc1a3fd57cb 100644
--- a/products/alerts/backend/platform_investigation_message.py
+++ b/products/alerts/backend/platform_investigation_message.py
@@ -96,7 +96,10 @@ def _read(team_id: int, transition: AnnouncedTransition) -> FiringInvestigation
value = evaluated.get("value")
breaches: tuple[str, ...] | None = None
link: str | None = None
- if follow_up or released_by == HoldReleasedBy.VERDICT:
+ # Only the agent's verdict rewords a firing. A metrics investigation explains the firing beside
+ # its own breaches, so its release keeps them, also when it failed.
+ agent = kind == InsightAlertInvestigation.Kind.AGENT
+ if follow_up or (agent and released_by == HoldReleasedBy.VERDICT):
previous_verdict = (
rows.filter(episode_started_at=episode_started_at, created_at__lt=created_at, verdict__isnull=False)
.order_by("-created_at")
@@ -116,8 +119,11 @@ def _read(team_id: int, transition: AnnouncedTransition) -> FiringInvestigation
).lines
)
link = notebook_url(notebook_short_id) if notebook_short_id else None
- elif released_by == HoldReleasedBy.SAFETY_NET:
+ elif agent and released_by == HoldReleasedBy.SAFETY_NET:
breaches = tuple(safety_net_breaches(triggered_dates=triggered_dates, value=value))
+ elif not agent and status == InvestigationStatus.DONE and summary:
+ # Every channel reads the breaches, so the explanation goes after them, as the agent's teaser does.
+ breaches = (*(str(line) for line in transition.source_detail.get("breaches") or []), summary)
return FiringInvestigation(
status=status,
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index b3dcf41564eb..c5e51f5f79f1 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -330,22 +330,24 @@ def readmit_retried_check(
return token if admitted else None
-class _ClickHouseSlot:
+class ClickHouseQuerySlot:
"""Holds a ClickHouse slot around one attempt's query, and counts how long it waited for one.
Raises `ClickHouseBudgetFull` when no slot comes free in time. `holder` names one attempt of one
check, so a retry never takes over the slot of the attempt before it.
"""
- def __init__(self, holder: str) -> None:
+ def __init__(self, holder: str, *, wait_seconds: float | None = None) -> None:
self.holder = holder
self.waited_ms = 0
+ # A caller with a budget of its own waits no longer than it, on the same shared cap.
+ self._limit = _QUERY_LIMIT if wait_seconds is None else replace(_QUERY_LIMIT, retry_timeout=wait_seconds)
@contextmanager
def __call__(self) -> Iterator[None]:
waiting_since = time.monotonic()
try:
- slot = _QUERY_LIMIT.use(self.holder, limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS)
+ slot = self._limit.use(self.holder, limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS)
except ConcurrencyLimitExceeded as error:
raise ClickHouseBudgetFull() from error
except Exception as error:
@@ -359,13 +361,13 @@ class _ClickHouseSlot:
finally:
if slot is not None:
try:
- _QUERY_LIMIT.release(slot)
+ self._limit.release(slot)
except Exception:
# Best effort, so a Redis failure never replaces the query's result. The lease frees the slot.
logger.exception("Could not release a ClickHouse slot", holder=self.holder)
-def _with_capacity_retries[T](run: Callable[[], T], alert: AlertConfiguration, query_slot: _ClickHouseSlot) -> T:
+def _with_capacity_retries[T](run: Callable[[], T], alert: AlertConfiguration, query_slot: ClickHouseQuerySlot) -> T:
"""Runs a check's query, and runs it again after a ClickHouse refusal for load, on the retry
policy production evaluates the alert's cadence with.
@@ -744,7 +746,7 @@ def _decide(
alert_calculation_interval=alert.calculation_interval,
alert_config_type=(alert.config or {}).get("type"),
)
- query_slot = _ClickHouseSlot(f"{evaluation_id}:{attempt}")
+ query_slot = ClickHouseQuerySlot(f"{evaluation_id}:{attempt}")
if check.grouping.mode != GroupingMode.SINGLE:
return _decide_grouped(check, snapshot, alert, now=now, query_slot=query_slot)
started_at = time.monotonic()
@@ -907,7 +909,7 @@ def _decide_grouped(
alert: AlertConfiguration,
*,
now: datetime,
- query_slot: _ClickHouseSlot,
+ query_slot: ClickHouseQuerySlot,
) -> PlatformAlertOutcome | None:
"""One check of a breakdown alert, judged per breakdown value through the platform's decision.
diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.py
index adec0ea424ae..e25e84ab3674 100644
--- a/products/alerts/backend/temporal/platform_evaluate.py
+++ b/products/alerts/backend/temporal/platform_evaluate.py
@@ -38,6 +38,7 @@ with workflow.unsafe.imports_passed_through():
from products.alerts.backend.platform_investigation_detail import (
InvestigationClaims,
InvestigationStart,
+ MetricsInvestigationRun,
requested_investigation,
)
from products.alerts_platform.backend.facade.contracts import (
@@ -95,6 +96,12 @@ STOP_QUERIES_START_TO_CLOSE = dt.timedelta(seconds=60)
RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)
CLAIM_START_TO_CLOSE = dt.timedelta(seconds=10)
CLAIM_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=20)
+# A metrics alert's investigation is a few bounded metric queries. One attempt, because the run
+# leases its row, so a retry would find it taken.
+METRICS_INVESTIGATION_START_TO_CLOSE = dt.timedelta(seconds=45)
+# Time a run's queries can take, so the last run starts while the activity can still finish it.
+METRICS_INVESTIGATION_MARGIN = dt.timedelta(seconds=15)
+METRICS_INVESTIGATION_SCHEDULE_TO_CLOSE = METRICS_INVESTIGATION_START_TO_CLOSE + CHECK_QUEUE_TOLERANCE
RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)
# The pool admits once per round, so a batch that admits only once evaluates at most one pool of
# checks per tick. Rounds refill the slots that finished checks free, the way production's
@@ -104,7 +111,8 @@ ADMISSION_POLL_INTERVAL = dt.timedelta(seconds=2)
# What the source needs from its binding's `evaluation_timeout`. The checks run side by side, so
# the batch costs the admission rounds plus one check's budget, not one per check. The last check
# can end while an earlier record still runs, so the tail holds two records and their failures,
-# or one record and the stop of a failed check's queries.
+# or one record and the stop of a failed check's queries. A firing's claim follows its record, and a
+# failed check claims nothing.
EVALUATION_BUDGET = (
PLAN_SCHEDULE_TO_CLOSE
+ ADMISSION_RUN_BUDGET
@@ -112,7 +120,8 @@ EVALUATION_BUDGET = (
+ max(CHART_SCHEDULE_TO_CLOSE, FAIL_SCHEDULE_TO_CLOSE)
+ RECORD_SCHEDULE_TO_CLOSE
+ CLAIM_SCHEDULE_TO_CLOSE
- + max(FAIL_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE + CLAIM_SCHEDULE_TO_CLOSE, STOP_QUERIES_START_TO_CLOSE)
+ + METRICS_INVESTIGATION_SCHEDULE_TO_CLOSE
+ + max(FAIL_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE, STOP_QUERIES_START_TO_CLOSE)
)
_PATCH_READMISSION = "platform-insight-readmission-2026-10"
# Runs started before it recorded every outcome in one activity, and replay must schedule the same.
@@ -130,6 +139,8 @@ _PATCH_RECORD_AS_FINISHED = "platform-insight-record-as-finished-2026-10"
_PATCH_WAIT_FOR_POOL = "platform-insight-wait-for-pool-2026-10"
# Runs started before it claimed investigations after a record, and replay must not schedule a claim.
_PATCH_INVESTIGATIONS = "platform-insight-investigations-2026-11"
+# Runs started before it ran metrics investigations after a claim, and replay must not schedule one.
+_PATCH_METRICS_INVESTIGATIONS = "platform-insight-metrics-investigations-2026-11"
@frozen
@@ -180,6 +191,11 @@ class ClaimInsightInvestigationsInputs:
outcomes: tuple[PlatformAlertOutcome, ...]
+@frozen
+class MetricsInvestigationsInputs:
+ runs: tuple[MetricsInvestigationRun, ...]
+
+
@frozen
class StopInsightQueriesInputs:
team_id: int
@@ -365,6 +381,35 @@ def claim_platform_insight_investigations_activity(inputs: ClaimInsightInvestiga
return claim_requested_investigations(inputs.team_id, inputs.outcomes)
+@close_db_connections
+def run_platform_metrics_investigations_activity(inputs: MetricsInvestigationsInputs) -> list[AlertDeliveryRequest]:
+ """Runs the metrics investigations a claim granted, apart from the claim, so their queries never
+ hold back an agent's start. Releases each run's held firing, and returns the deliveries to start."""
+ from products.alerts.backend.platform_investigation import (
+ fail_unstarted_metrics_investigation, # noqa: PLC0415
+ release_metrics_firing, # noqa: PLC0415
+ run_metrics_investigation, # noqa: PLC0415
+ )
+ from products.alerts.backend.platform_source_cycle import ClickHouseQuerySlot # noqa: PLC0415
+
+ # Stops starting runs before the activity's own timeout, because Temporal does not stop a sync
+ # loop that outlives it. A run already started finishes its own queries.
+ deadline = time.monotonic() + (METRICS_INVESTIGATION_START_TO_CLOSE - METRICS_INVESTIGATION_MARGIN).total_seconds()
+ released: list[AlertDeliveryRequest] = []
+ for run in inputs.runs:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ fail_unstarted_metrics_investigation(run)
+ else:
+ # The slot wait counts against the same deadline, so a full ClickHouse budget cannot use up the activity.
+ slot = ClickHouseQuerySlot(f"metrics-investigation:{run.investigation_id}", wait_seconds=remaining)
+ run_metrics_investigation(run, query_slot=slot)
+ if request := release_metrics_firing(run):
+ released.append(request)
+ return released
+
+
@activity.defn
@close_db_connections
def render_platform_insight_chart_activity(inputs: InsightChartInputs) -> RenderedChart | None:
@@ -794,6 +839,20 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
claims = await self._claim_investigations(inputs, outcomes)
await asyncio.gather(*(self._start_investigation(start) for start in claims.starts))
await start_deliveries(list(claims.deliveries))
+ if workflow.patched(_PATCH_METRICS_INVESTIGATIONS) and claims.metrics:
+ try:
+ released = await workflow.execute_activity(
+ run_platform_metrics_investigations_activity,
+ MetricsInvestigationsInputs(runs=claims.metrics),
+ start_to_close_timeout=METRICS_INVESTIGATION_START_TO_CLOSE,
+ schedule_to_close_timeout=METRICS_INVESTIGATION_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ except ActivityError:
+ # The held firings go out at their hold's deadline, and the rows stay in flight until the horizon passes.
+ workflow.logger.warning("Could not run the metrics investigations of platform insight firings")
+ else:
+ await start_deliveries(released)
async def _claim_investigations(
self, inputs: SourceEvaluationInputs, outcomes: tuple[PlatformAlertOutcome, ...]
@@ -843,6 +902,7 @@ PLATFORM_EVALUATION_ACTIVITIES = [
evaluate_platform_insight_check_activity,
render_platform_insight_chart_activity,
claim_platform_insight_investigations_activity,
+ run_platform_metrics_investigations_activity,
fail_platform_insight_check_activity,
stop_platform_insight_queries_activity,
]
diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.py
index f08c9366e16a..eee5f49a1e27 100644
--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py
+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py
@@ -2,6 +2,7 @@ import uuid
import asyncio
import threading
from collections.abc import AsyncIterator
+from dataclasses import replace
import pytest
from unittest.mock import patch
@@ -21,6 +22,11 @@ from products.alerts.backend.platform_chart_detail import (
CHART_RENDER_USER_ID_KEY,
RenderedChart,
)
+from products.alerts.backend.platform_investigation_detail import (
+ METRICS_INVESTIGATION_REQUESTED_KEY,
+ InvestigationClaims,
+ MetricsInvestigationRun,
+)
from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS
from products.alerts.backend.temporal.platform_evaluate import (
CHECK_HEARTBEAT_TIMEOUT,
@@ -28,11 +34,13 @@ from products.alerts.backend.temporal.platform_evaluate import (
EVALUATION_BUDGET,
REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,
AdmittedInsightCheck,
+ ClaimInsightInvestigationsInputs,
FailedInsightCheckInputs,
InsightAlertPlatformEvaluateWorkflow,
InsightBatchPlanInputs,
InsightChartInputs,
InsightCheckInputs,
+ MetricsInvestigationsInputs,
StopInsightQueriesInputs,
evaluate_platform_insight_llm_check_activity,
)
@@ -42,6 +50,7 @@ from products.alerts_platform.backend.facade.contracts import (
AlertDeliveryRequest,
AlertEventKind,
GroupOutcome,
+ HeldDelivery,
PlatformAlertOutcome,
SourceBatchEvaluation,
SourceEvaluationInputs,
@@ -591,3 +600,87 @@ async def test_a_record_that_fails_while_a_check_still_runs_records_again_at_the
assert result == 2
assert attempts[0] == [fast]
assert sorted(configuration_id for attempt in attempts[1:] for configuration_id in attempt) == [fast, slow]
+
+
+async def test_a_metrics_firing_is_delivered_once_its_investigation_releases_it(
+ environment: WorkflowEnvironment,
+) -> None:
+ released: list[AlertDeliveryRequest] = []
+
+ @activity.defn(name="plan_platform_insight_checks_activity")
+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:
+ return [] if inputs.exclude else [AdmittedInsightCheck(configuration_id=ADMITTED[0], owned_by_platform=True)]
+
+ @activity.defn(name="evaluate_platform_insight_check_activity")
+ async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
+ outcome = _outcome(inputs.configuration_id, AlertEventKind.FIRING, "firing")
+ group = outcome.groups[0]
+ outcome = PlatformAlertOutcome(
+ configuration_id=outcome.configuration_id,
+ evaluation_key=outcome.evaluation_key,
+ consecutive_failures=0,
+ groups=(
+ GroupOutcome(
+ grouping_key=group.grouping_key,
+ kind=group.kind,
+ new_state=group.new_state,
+ notified=group.notified,
+ source_detail={METRICS_INVESTIGATION_REQUESTED_KEY: "legacy-alert"},
+ ),
+ ),
+ )
+ delivery = AlertDeliveryRequest(
+ source=SourceKind.INSIGHT,
+ team_id=1,
+ configuration_id=inputs.configuration_id,
+ evaluation_key=outcome.evaluation_key,
+ destination_alert_id="legacy-alert",
+ event_ids_by_kind={"firing": "$insight_alert_firing"},
+ )
+ # Held by the check, so only the investigation's release starts it.
+ held = replace(
+ outcome,
+ held_delivery=HeldDelivery(request=delivery, reason="metrics_investigation", release_after_seconds=300),
+ )
+ released.append(delivery)
+ return SourceBatchEvaluation(outcomes=(held,), deliveries=())
+
+ @activity.defn(name=RECORD_OUTCOMES_ACTIVITY)
+ async def record(inputs: SourceOutcomeInputs) -> int:
+ return len(inputs.outcomes)
+
+ @activity.defn(name="claim_platform_insight_investigations_activity")
+ async def claim(inputs: ClaimInsightInvestigationsInputs) -> InvestigationClaims:
+ return InvestigationClaims(metrics=(MetricsInvestigationRun(team_id=1, investigation_id="inv-1"),))
+
+ @activity.defn(name="run_platform_metrics_investigations_activity")
+ async def investigate(inputs: MetricsInvestigationsInputs) -> list[AlertDeliveryRequest]:
+ return released
+
+ async with Worker(
+ environment.client,
+ task_queue=QUEUE,
+ workflows=[InsightAlertPlatformEvaluateWorkflow],
+ activities=[plan, evaluate, record, claim, investigate],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ handle = await environment.client.start_workflow(
+ InsightAlertPlatformEvaluateWorkflow.run,
+ SourceEvaluationInputs(
+ source=SourceKind.INSIGHT,
+ cutoff="2026-09-16T10:00:00+00:00",
+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),
+ ),
+ id=f"test-{uuid.uuid4()}",
+ task_queue=QUEUE,
+ )
+ await handle.result()
+ history = await handle.fetch_history()
+
+ order = []
+ for event in history.events:
+ if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED:
+ order.append(event.activity_task_scheduled_event_attributes.activity_type.name)
+ elif event.event_type == EventType.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED:
+ order.append("delivery")
+ assert order.index("run_platform_metrics_investigations_activity") < order.index("delivery")
diff --git a/products/alerts/backend/test/test_platform_investigation_message.py b/products/alerts/backend/test/test_platform_investigation_message.py
index 0540d95a50b0..b64a56aa0c5e 100644
--- a/products/alerts/backend/test/test_platform_investigation_message.py
+++ b/products/alerts/backend/test/test_platform_investigation_message.py
@@ -38,6 +38,7 @@ KEY = "slot:2026-10-20T09:00:00+00:00"
OWN_BREACH = "Signups (50) is less than lower threshold (100)"
ANOMALY_HEAD = "Anomaly detected on 2026-10-20."
SUMMARY = "Signups fell after the **release**."
+METRICS_SUMMARY = "queue.depth moved up to 420.00 (baseline 100.00, 4.2x)."
TEASER = "Signups fell after the release."
@@ -58,7 +59,14 @@ class TestInvestigatedFiringMessages(APIBaseTest):
self.notebook_url = absolute_uri(f"/notebooks/{self.notebook.short_id}")
def _investigation(
- self, key: str, *, status: str = InvestigationStatus.DONE, verdict: str | None = None, at: datetime = FIRED_AT
+ self,
+ key: str,
+ *,
+ status: str = InvestigationStatus.DONE,
+ verdict: str | None = None,
+ at: datetime = FIRED_AT,
+ kind: str = InsightAlertInvestigation.Kind.AGENT,
+ summary: str | None = None,
) -> None:
with time_machine.travel(at, tick=False):
InsightAlertInvestigation.objects.for_team(self.team.id).create(
@@ -67,10 +75,10 @@ class TestInvestigatedFiringMessages(APIBaseTest):
configuration_id=self.configuration_id,
evaluation_key=key,
episode_started_at=EPISODE,
- kind=InsightAlertInvestigation.Kind.AGENT,
+ kind=kind,
status=status,
verdict=verdict,
- summary=SUMMARY if verdict else None,
+ summary=summary or (SUMMARY if verdict else None),
notebook_id=self.notebook.id if verdict else None,
evaluated={"value": 50.0, "triggered_dates": ["2026-10-20"]},
)
@@ -192,6 +200,36 @@ class TestInvestigatedFiringMessages(APIBaseTest):
description = describe_insight_transition(project_id=self.team.id, transition=self._transition())
assert description.details == (MessageDetail(label="Threshold breached", value="\n".join(expected_breaches)),)
+ @parameterized.expand(
+ [
+ ("released_after_its_run", InvestigationStatus.DONE, HoldReleasedBy.VERDICT, [OWN_BREACH, METRICS_SUMMARY]),
+ (
+ "sent_by_the_safety_net_after_it_failed",
+ InvestigationStatus.FAILED,
+ HoldReleasedBy.SAFETY_NET,
+ [OWN_BREACH],
+ ),
+ ]
+ )
+ def test_a_metrics_firing_keeps_its_own_breaches_and_adds_its_explanation(
+ self, _name: str, status: str, released_by: HoldReleasedBy, breaches: list[str]
+ ) -> None:
+ self._investigation(
+ KEY,
+ status=status,
+ kind=InsightAlertInvestigation.Kind.METRICS,
+ summary=METRICS_SUMMARY if status == InvestigationStatus.DONE else None,
+ )
+ self._released_by(released_by)
+
+ properties = self._event_properties(self._transition())
+
+ assert properties["breaches"] == "\n".join(breaches)
+ assert properties.get("investigation_notebook_url") is None
+ # Email, PagerDuty and webhooks read the same breaches, so they carry the explanation too.
+ description = describe_insight_transition(project_id=self.team.id, transition=self._transition())
+ assert description.details == (MessageDetail(label="Threshold breached", value="\n".join(breaches)),)
+
def test_a_changed_verdict_follows_up_with_the_change(self) -> None:
self._investigation("slot:2026-10-20T08:00:00+00:00", verdict="true_positive", at=EPISODE)
self._investigation(KEY, verdict="false_positive")
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index 03b0a29d6022..76edd5c5fdc2 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -68,7 +68,9 @@ from products.alerts.backend.platform_source_cycle import (
from products.alerts.backend.temporal.platform_evaluate import (
AdmittedInsightCheck,
InsightBatchPlanInputs,
+ MetricsInvestigationsInputs,
plan_platform_insight_checks_activity,
+ run_platform_metrics_investigations_activity,
)
from products.alerts.backend.test.insight_alerts import create_insight_alert
from products.alerts_platform.backend.facade import testing as platform_testing
@@ -1004,6 +1006,39 @@ class TestPlatformInsightEvaluation(APIBaseTest):
assert outcome.held_delivery is None
assert len(self.deliveries) == 1
+ @parameterized.expand([("found", None, InvestigationStatus.DONE), ("failed", RuntimeError("no data"), "failed")])
+ def test_a_metrics_firing_is_held_until_its_investigation_ends(
+ self, _name: str, error: Exception | None, status: str
+ ) -> None:
+ alert = self._alert(investigation_agent_enabled=True, investigation_gates_notifications=True)
+ alert.insight.query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue.depth"}]}
+ alert.insight.save()
+ configuration = self._copy(alert, owner=PlatformAlertOwner.PLATFORM)
+ outcome = self._fire(configuration)
+ record_outcomes(self.team.id, (outcome,), CUTOFF)
+
+ claims = claim_requested_investigations(self.team.id, (outcome,))
+ with patch(
+ "products.alerts.backend.platform_investigation.investigate_metrics_alert",
+ side_effect=error,
+ return_value="queue.depth moved up.",
+ ) as investigate:
+ released = [
+ run_platform_metrics_investigations_activity(MetricsInvestigationsInputs(runs=claims.metrics))
+ for _ in range(2)
+ ]
+
+ assert outcome.held_delivery is not None
+ assert self.deliveries == ()
+ assert (claims.starts, len(claims.metrics)) == ((), 1)
+ # Released once the run ends, whatever it found, and a retry gets the same request back.
+ assert released == [[outcome.held_delivery.request]] * 2
+ investigate.assert_called_once()
+ assert investigate.call_args.kwargs["fired_at"] == CUTOFF
+ investigation = InsightAlertInvestigation.objects.for_team(self.team.id).get()
+ assert (investigation.kind, investigation.status) == (InsightAlertInvestigation.Kind.METRICS, status)
+ assert investigation.summary == ("queue.depth moved up." if error is None else None)
+
def test_a_skipped_check_of_a_firing_alert_asks_for_no_investigation(self) -> None:
_alert, configuration = self._investigated_alert(investigation_gates_notifications=True)
record_outcomes(self.team.id, (self._fire(configuration),), CUTOFF)
@@ -1017,3 +1052,22 @@ class TestPlatformInsightEvaluation(APIBaseTest):
assert outcome is not None
assert outcome.skipped
assert requested_investigation(outcome) is None
+
+ def test_a_metrics_run_past_its_activity_deadline_is_recorded_as_failed(self) -> None:
+ alert = self._alert(investigation_agent_enabled=True)
+ alert.insight.query = {"kind": "MetricsQuery", "clauses": [{"name": "a", "metricName": "queue.depth"}]}
+ alert.insight.save()
+ configuration = self._copy(alert, owner=PlatformAlertOwner.PLATFORM)
+ outcome = self._fire(configuration)
+ record_outcomes(self.team.id, (outcome,), CUTOFF)
+ claims = claim_requested_investigations(self.team.id, (outcome,))
+
+ with (
+ patch("products.alerts.backend.temporal.platform_evaluate.time.monotonic", side_effect=[0.0, 10_000.0]),
+ patch("products.alerts.backend.platform_investigation.investigate_metrics_alert") as investigate,
+ ):
+ run_platform_metrics_investigations_activity(MetricsInvestigationsInputs(runs=claims.metrics))
+
+ investigate.assert_not_called()
+ investigation = InsightAlertInvestigation.objects.for_team(self.team.id).get()
+ assert investigation.status == InvestigationStatus.FAILED
diff --git a/products/alerts_platform/backend/temporal/sources.py b/products/alerts_platform/backend/temporal/sources.py
index 4d274c1a746f..eb44243286b7 100644
--- a/products/alerts_platform/backend/temporal/sources.py
+++ b/products/alerts_platform/backend/temporal/sources.py
@@ -51,7 +51,7 @@ SOURCE_BINDINGS: dict[SourceKind, SourceBinding] = {
SourceKind.INSIGHT: SourceBinding(
workflow="insight-alert-platform-evaluate",
task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,
- evaluation_timeout=dt.timedelta(minutes=20),
+ evaluation_timeout=dt.timedelta(minutes=22),
# Not the pool size. A check frees its slot in seconds, so the pool turns over many times a
# tick, and a limit equal to its size caps checks per minute far below what it can run.
discovery_limit=settings.ALERTS_PLATFORM_INSIGHT_KEYS_PER_TICK,