:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): record insight alert investigations for platform evaluations (#123622)
Jimmy Zhu9c16efd0029a
diff --git a/posthog/temporal/alerts/investigation.py b/posthog/temporal/alerts/investigation.pyindex e6831503861c..282539f937df 100644--- a/posthog/temporal/alerts/investigation.py+++ b/posthog/temporal/alerts/investigation.py@@ -77,17 +77,22 @@ def decide_investigation(alert: AlertConfiguration, alert_check: AlertCheck) ->Cooldown is enforced by `claim_investigation_slot`, which does the read-then-writeinside the caller's transaction."""- if not alert.investigation_agent_enabled:+ if not investigates(alert, firing=alert_check.state == AlertState.FIRING):return InvestigationDecision()- if not alert.detector_config:- return InvestigationDecision()- if alert_check.state != AlertState.FIRING:- return InvestigationDecision()-episode = episode_investigations(alert, alert_check)- if episode.started >= MAX_INVESTIGATIONS_PER_EPISODE:- return InvestigationDecision(previous_verdict=episode.previous_verdict)- return InvestigationDecision(should_investigate=True, previous_verdict=episode.previous_verdict)+ return decide_within_budget(started=episode.started, previous_verdict=episode.previous_verdict)+++def investigates(alert: AlertConfiguration, *, firing: bool) -> bool:+ """Whether a check of `alert` is one the agent investigates at all, before its budget."""+ return bool(alert.investigation_agent_enabled and alert.detector_config and firing)+++def decide_within_budget(*, started: int, previous_verdict: str | None) -> InvestigationDecision:+ """The decision for an eligible check, given what its episode already started."""+ return InvestigationDecision(+ should_investigate=started < MAX_INVESTIGATIONS_PER_EPISODE, previous_verdict=previous_verdict+ )def carried_verdict_suppresses(alert: AlertConfiguration, decision: InvestigationDecision) -> bool:@@ -111,22 +116,28 @@ def claim_investigation_slot(alert: AlertConfiguration, alert_check: AlertCheck)Caller must run this inside a transaction so the read-then-write is atomic."""- now = datetime.now(UTC)- in_flight = Q(- investigation_status__in=[InvestigationStatus.PENDING, InvestigationStatus.RUNNING],- created_at__gte=now - IN_FLIGHT_INVESTIGATION_HORIZON,- )- within_cooldown = Q(- investigation_status=InvestigationStatus.DONE,- created_at__gte=now - investigation_cooldown(alert),- )recent = (AlertCheck.objects.filter(alert_configuration=alert).exclude(id=alert_check.id)- .filter(in_flight | within_cooldown)+ .filter(refusing_investigations(alert, datetime.now(UTC), status_field="investigation_status")))if recent.exists():AlertCheck.objects.filter(id=alert_check.id).update(investigation_status=InvestigationStatus.SKIPPED)return FalseAlertCheck.objects.filter(id=alert_check.id).update(investigation_status=InvestigationStatus.PENDING)return True+++def refusing_investigations(alert: AlertConfiguration, now: datetime, *, status_field: str) -> Q:+ """Earlier investigations of `alert` that refuse a new one: in flight, or done inside the cooldown.++ `status_field` names the status column, because each stack keeps its investigations in its own table.+ """+ cooldown = investigation_cooldown(alert)+ in_flight = Q(+ **{f"{status_field}__in": [InvestigationStatus.PENDING, InvestigationStatus.RUNNING]},+ created_at__gte=now - IN_FLIGHT_INVESTIGATION_HORIZON,+ )+ within_cooldown = Q(**{status_field: InvestigationStatus.DONE}, created_at__gte=now - cooldown)+ # The outer bound on its own lets the read scan one short range of the creation-time index.+ return Q(created_at__gte=now - max(IN_FLIGHT_INVESTIGATION_HORIZON, cooldown)) & (in_flight | within_cooldown)diff --git a/products/alerts/backend/investigation_episode.py b/products/alerts/backend/investigation_episode.pyindex 75208ef083ca..66a19d5e67d7 100644--- a/products/alerts/backend/investigation_episode.py+++ b/products/alerts/backend/investigation_episode.py@@ -15,7 +15,7 @@ from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration,# Statuses of a check whose investigation was started. SKIPPED never ran, so it costs no# budget; FAILED did run and costs one, which keeps a broken agent from retrying forever.-_STARTED_INVESTIGATION_STATUSES = [+STARTED_INVESTIGATION_STATUSES = [InvestigationStatus.PENDING,InvestigationStatus.RUNNING,InvestigationStatus.DONE,@@ -57,7 +57,7 @@ def episode_investigations(alert: AlertConfiguration, alert_check: AlertCheck) -.first())return EpisodeInvestigations(- started=earlier.filter(investigation_status__in=_STARTED_INVESTIGATION_STATUSES).count(),+ started=earlier.filter(investigation_status__in=STARTED_INVESTIGATION_STATUSES).count(),first_check_id=str(first_check_id or alert_check.id),previous_verdict=previous_verdict,)diff --git a/products/alerts/backend/migrations/0015_insight_alert_investigation.py b/products/alerts/backend/migrations/0015_insight_alert_investigation.pynew file mode 100644index 000000000000..b70b2aa36667--- /dev/null+++ b/products/alerts/backend/migrations/0015_insight_alert_investigation.py@@ -0,0 +1,125 @@+# Generated by Django 5.2.18 on 2026-10-26 19:46++import django.db.models.deletion+from django.db import migrations, models++import posthog.uuidt+++class Migration(migrations.Migration):+ dependencies = [+ ("alerts", "0014_strip_llm_detector_threshold_and_window"),+ ("notebooks", "0027_notebook_unlisted_visibility"),+ ("posthog", "1413_event_filter_pipeline_override"),+ ]++ operations = [+ migrations.CreateModel(+ name="InsightAlertInvestigation",+ fields=[+ (+ "id",+ models.UUIDField(+ default=posthog.uuidt.uuid7,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("configuration_id", models.UUIDField()),+ ("evaluation_key", models.CharField(max_length=255)),+ ("episode_started_at", models.DateTimeField(blank=True, null=True)),+ (+ "kind",+ models.CharField(+ choices=[("agent", "Agent"), ("metrics", "Metrics")],+ max_length=16,+ ),+ ),+ (+ "status",+ models.CharField(+ choices=[+ ("pending", "pending"),+ ("running", "running"),+ ("done", "done"),+ ("failed", "failed"),+ ("skipped", "skipped"),+ ],+ max_length=10,+ ),+ ),+ (+ "verdict",+ models.CharField(+ blank=True,+ choices=[+ ("true_positive", "true_positive"),+ ("false_positive", "false_positive"),+ ("inconclusive", "inconclusive"),+ ],+ max_length=20,+ null=True,+ ),+ ),+ ("summary", models.TextField(blank=True, null=True)),+ ("error", models.JSONField(blank=True, null=True)),+ ("evaluated", models.JSONField(default=dict)),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("updated_at", models.DateTimeField(auto_now=True)),+ (+ "alert_configuration",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="alerts.alertconfiguration",+ ),+ ),+ (+ "notebook",+ models.ForeignKey(+ blank=True,+ db_constraint=False,+ db_index=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="+",+ to="notebooks.notebook",+ ),+ ),+ (+ "team",+ models.ForeignKey(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="posthog.team",+ ),+ ),+ ],+ options={+ "db_table": "alerts_insightalertinvestigation",+ "indexes": [+ models.Index(+ fields=["configuration_id", "episode_started_at", "created_at"],+ name="insight_alert_inv_episode_idx",+ ),+ models.Index(+ condition=models.Q(("notebook__isnull", False)),+ fields=["notebook"],+ name="insight_alert_inv_notebook_idx",+ ),+ models.Index(+ fields=["configuration_id", "created_at"],+ name="insight_alert_inv_recent_idx",+ ),+ ],+ "constraints": [+ models.UniqueConstraint(+ fields=("configuration_id", "evaluation_key"),+ name="insight_alert_investigation_one_per_evaluation",+ )+ ],+ },+ ),+ ]diff --git a/products/alerts/backend/migrations/max_migration.txt b/products/alerts/backend/migrations/max_migration.txtindex 1e6d1cdb2ffd..4b481002c897 100644--- a/products/alerts/backend/migrations/max_migration.txt+++ b/products/alerts/backend/migrations/max_migration.txt@@ -1 +1 @@-0014_strip_llm_detector_threshold_and_window+0015_insight_alert_investigationdiff --git a/products/alerts/backend/models/__init__.py b/products/alerts/backend/models/__init__.pyindex 883ad1df8074..6200669fa419 100644--- a/products/alerts/backend/models/__init__.py+++ b/products/alerts/backend/models/__init__.py@@ -1,9 +1,11 @@from .alert import Alert, AlertCheck, AlertConfiguration, AlertSubscription, Threshold+from .investigation import InsightAlertInvestigation__all__ = ["Alert","AlertCheck","AlertConfiguration","AlertSubscription",+ "InsightAlertInvestigation","Threshold",]diff --git a/products/alerts/backend/models/investigation.py b/products/alerts/backend/models/investigation.pynew file mode 100644index 000000000000..52b8692bdb70--- /dev/null+++ b/products/alerts/backend/models/investigation.py@@ -0,0 +1,69 @@+from django.db import models++from posthog.models.scoping.root_mixin import TeamScopedRootMixin+from posthog.models.utils import UUIDModel++from products.alerts.backend.models.alert import AlertConfiguration, InvestigationStatus, InvestigationVerdict+++class InsightAlertInvestigation(TeamScopedRootMixin, UUIDModel):+ """One investigation of one platform evaluation of an insight alert.++ The platform never writes `AlertCheck`, so an alert the platform evaluates keeps its+ investigations here. A row exists for every evaluation that asked for one, including one the+ claim refused, which is recorded as SKIPPED and spends no budget.+ """++ class Kind(models.TextChoices):+ AGENT = "agent", "Agent"+ METRICS = "metrics", "Metrics"++ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")+ alert_configuration = models.ForeignKey(AlertConfiguration, on_delete=models.CASCADE, related_name="+")+ # The platform's configuration. Not a foreign key, because that model belongs to another product.+ configuration_id = models.UUIDField()+ evaluation_key = models.CharField(max_length=255)+ # The firing the evaluation belongs to, which is what the per-episode budget counts. Null for+ # the firing that was already open when the platform took the alert over.+ episode_started_at = models.DateTimeField(null=True, blank=True)++ kind = models.CharField(max_length=16, choices=Kind.choices)+ status = models.CharField(max_length=10, choices=InvestigationStatus.choices)+ verdict = models.CharField(max_length=20, choices=InvestigationVerdict.choices, null=True, blank=True)+ summary = models.TextField(null=True, blank=True)+ notebook = models.ForeignKey(+ "notebooks.Notebook",+ on_delete=models.SET_NULL,+ db_constraint=False,+ db_index=False,+ null=True,+ blank=True,+ related_name="+",+ )+ error = models.JSONField(null=True, blank=True)+ # What the evaluation measured, as the agent reads it: value, triggered dates and points,+ # anomaly scores, interval and triggered metadata.+ evaluated = models.JSONField(default=dict)++ created_at = models.DateTimeField(auto_now_add=True)+ updated_at = models.DateTimeField(auto_now=True)++ class Meta:+ db_table = "alerts_insightalertinvestigation"+ constraints = [+ models.UniqueConstraint(+ fields=["configuration_id", "evaluation_key"], name="insight_alert_investigation_one_per_evaluation"+ )+ ]+ indexes = [+ # The episode's budget and its last verdict.+ models.Index(+ fields=["configuration_id", "episode_started_at", "created_at"], name="insight_alert_inv_episode_idx"+ ),+ # Deleting a notebook nulls the link through this, and most rows have no notebook.+ models.Index(+ fields=["notebook"], name="insight_alert_inv_notebook_idx", condition=models.Q(notebook__isnull=False)+ ),+ # The claim's in-flight and cooldown checks.+ models.Index(fields=["configuration_id", "created_at"], name="insight_alert_inv_recent_idx"),+ ]diff --git a/products/alerts/backend/platform_investigation.py b/products/alerts/backend/platform_investigation.pynew file mode 100644index 000000000000..af37f5c41844--- /dev/null+++ b/products/alerts/backend/platform_investigation.py@@ -0,0 +1,107 @@+"""The investigation agent's decision and claim for an insight alert the platform evaluates.++The same rules as the insight stack's own (`posthog/temporal/alerts/investigation.py`), read from+`InsightAlertInvestigation` instead of `AlertCheck`. An episode is one firing, named by the+platform's `episode_started_at`, rather than a run of FIRING checks.++The decision is a read, so it can run inside an evaluation that a retry repeats. The claim is the+write, and runs once the evaluation is recorded.+"""++from datetime import UTC, datetime+from typing import Any, Final+from uuid import UUID++from django.db import connection, transaction++from posthog.dataclasses import frozen+from posthog.temporal.alerts.investigation import (+ InvestigationDecision,+ decide_within_budget,+ investigates,+ refusing_investigations,+)++from products.alerts.backend.investigation_episode import STARTED_INVESTIGATION_STATUSES+from products.alerts.backend.models.alert import AlertConfiguration, InvestigationStatus+from products.alerts.backend.models.investigation import InsightAlertInvestigation++_CLAIM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))"+++@frozen+class InvestigationClaim:+ investigation_id: UUID+ # False when an investigation in flight or inside the cooldown refused it.+ claimed: bool+++def decide_investigation(+ alert: AlertConfiguration,+ *,+ firing: bool,+ configuration_id: UUID,+ evaluation_key: str,+ episode_started_at: datetime | None,+) -> InvestigationDecision:+ """Whether this evaluation gets an investigation, and the firing's last verdict so far.++ `firing` is whether the evaluation left the group firing, which the platform's state names+ differently from the insight stack's.++ The cooldown and the in-flight check belong to the claim, which takes a lock for them.+ """+ if not investigates(alert, firing=firing):+ return InvestigationDecision()+ # Newest first, so the first verdict is the last one reached. One read, because a firing holds+ # a few rows. Its own row exists when a retry repeats a decision its claim already wrote.+ earlier = list(+ InsightAlertInvestigation.objects.for_team(alert.team_id)+ .filter(configuration_id=configuration_id, episode_started_at=episode_started_at)+ .exclude(evaluation_key=evaluation_key)+ .order_by("-created_at")+ .values_list("status", "verdict")+ )+ return decide_within_budget(+ started=sum(status in STARTED_INVESTIGATION_STATUSES for status, _ in earlier),+ previous_verdict=next((verdict for _, verdict in earlier if verdict is not None), None),+ )+++def claim_investigation(+ alert: AlertConfiguration,+ *,+ configuration_id: UUID,+ evaluation_key: str,+ episode_started_at: datetime | None,+ kind: InsightAlertInvestigation.Kind,+ evaluated: dict[str, Any],+) -> InvestigationClaim:+ """Records this evaluation's investigation, PENDING when it may run and SKIPPED when it may not.++ Refused by an investigation of the same alert that is still in flight, up to the horizon past+ which it is presumed dead, or one that finished inside the cooldown. The lock serializes two+ claims for one alert, so both cannot pass the check. A repeated claim returns the first one's+ answer, so a retry starts the same agent rather than a second.+ """+ with transaction.atomic():+ with connection.cursor() as cursor:+ cursor.execute(_CLAIM_LOCK_SQL, [f"insight-alert-investigation:{configuration_id}"])+ rows = InsightAlertInvestigation.objects.for_team(alert.team_id).filter(configuration_id=configuration_id)+ existing = rows.filter(evaluation_key=evaluation_key).first()+ if existing is not None:+ return InvestigationClaim(+ investigation_id=existing.id, claimed=existing.status != InvestigationStatus.SKIPPED+ )+ refused = rows.filter(refusing_investigations(alert, datetime.now(UTC), status_field="status")).exists()+ investigation = InsightAlertInvestigation.objects.for_team(alert.team_id).create(+ team_id=alert.team_id,+ alert_configuration=alert,+ configuration_id=configuration_id,+ evaluation_key=evaluation_key,+ episode_started_at=episode_started_at,+ kind=kind,+ status=InvestigationStatus.SKIPPED if refused else InvestigationStatus.PENDING,+ evaluated=evaluated,+ )+ return InvestigationClaim(investigation_id=investigation.id, claimed=not refused)diff --git a/products/alerts/backend/tests/test_platform_investigation.py b/products/alerts/backend/tests/test_platform_investigation.pynew file mode 100644index 000000000000..09fa6e7e98bd--- /dev/null+++ b/products/alerts/backend/tests/test_platform_investigation.py@@ -0,0 +1,156 @@+from datetime import UTC, datetime, timedelta+from uuid import UUID, uuid4++import time_machine+from posthog.test.base import BaseTest++from parameterized import parameterized++from posthog.schema_enums import AlertCalculationInterval+from posthog.temporal.alerts.investigation import MAX_INVESTIGATIONS_PER_EPISODE, InvestigationDecision++from products.alerts.backend.models.alert import AlertConfiguration, InvestigationStatus+from products.alerts.backend.models.investigation import InsightAlertInvestigation+from products.alerts.backend.platform_investigation import InvestigationClaim, claim_investigation, decide_investigation+from products.product_analytics.backend.facade.models import Insight++NOW = datetime(2026, 10, 20, 10, tzinfo=UTC)+EPISODE = NOW - timedelta(hours=6)+++class TestPlatformInvestigation(BaseTest):+ alert: AlertConfiguration+ configuration_id: UUID++ @classmethod+ def setUpTestData(cls) -> None:+ super().setUpTestData()+ cls.alert = AlertConfiguration.objects.create(+ team=cls.team,+ insight=Insight.objects.create(team=cls.team, name="Signups"),+ name="Signups dropped",+ detector_config={"type": "zscore", "threshold": 0.95, "window": 30},+ investigation_agent_enabled=True,+ calculation_interval=AlertCalculationInterval.HOURLY,+ )+ cls.configuration_id = uuid4()++ def _investigation(+ self,+ *,+ at: datetime,+ status: str = InvestigationStatus.DONE,+ verdict: str | None = None,+ episode_started_at: datetime | None = EPISODE,+ key: str | None = None,+ ) -> None:+ with time_machine.travel(at, tick=False):+ InsightAlertInvestigation.objects.for_team(self.team.id).create(+ team=self.team,+ alert_configuration=self.alert,+ configuration_id=self.configuration_id,+ evaluation_key=key or f"slot:{at.isoformat()}",+ episode_started_at=episode_started_at,+ kind=InsightAlertInvestigation.Kind.AGENT,+ status=status,+ verdict=verdict,+ )++ def _decide(self, *, firing: bool = True, key: str = "slot:now") -> InvestigationDecision:+ return decide_investigation(+ self.alert,+ firing=firing,+ configuration_id=self.configuration_id,+ evaluation_key=key,+ episode_started_at=EPISODE,+ )++ def _claim(self, key: str = "slot:now") -> InvestigationClaim:+ with time_machine.travel(NOW, tick=False):+ return claim_investigation(+ self.alert,+ configuration_id=self.configuration_id,+ evaluation_key=key,+ episode_started_at=EPISODE,+ kind=InsightAlertInvestigation.Kind.AGENT,+ evaluated={"value": 3.0},+ )++ @parameterized.expand([("first", 0, True), ("last", MAX_INVESTIGATIONS_PER_EPISODE - 1, True), ("spent", 3, False)])+ def test_a_firing_investigates_until_its_budget_is_spent(self, _name: str, started: int, expected: bool) -> None:+ for index in range(started):+ self._investigation(at=EPISODE + timedelta(hours=index), verdict="false_positive")++ decision = self._decide()++ assert decision.should_investigate is expected+ assert decision.previous_verdict == ("false_positive" if started else None)++ def test_only_started_investigations_of_this_firing_spend_its_budget(self) -> None:+ for index in range(MAX_INVESTIGATIONS_PER_EPISODE):+ self._investigation(at=EPISODE + timedelta(hours=index), status=InvestigationStatus.SKIPPED)+ self._investigation(+ at=EPISODE - timedelta(days=1, hours=index),+ episode_started_at=EPISODE - timedelta(days=1),+ verdict="true_positive",+ )+ # A retry repeats the decision after its own claim wrote a row, which must not count against it.+ self._investigation(at=NOW, status=InvestigationStatus.PENDING, verdict="true_positive", key="slot:now")++ decision = self._decide()++ assert decision.should_investigate is True+ assert decision.previous_verdict is None++ def test_a_check_that_is_not_firing_never_investigates(self) -> None:+ assert self._decide(firing=False).should_investigate is False++ @parameterized.expand(+ [+ ("nothing recent", None, timedelta(0), AlertCalculationInterval.HOURLY, True),+ (+ "done inside the cooldown",+ InvestigationStatus.DONE,+ timedelta(minutes=10),+ AlertCalculationInterval.HOURLY,+ False,+ ),+ (+ "done after the cooldown",+ InvestigationStatus.DONE,+ timedelta(hours=2),+ AlertCalculationInterval.HOURLY,+ True,+ ),+ # Half an hour is far past a real-time alert's cooldown, and the agent is still running.+ (+ "in flight past the cooldown",+ InvestigationStatus.RUNNING,+ timedelta(minutes=30),+ AlertCalculationInterval.REAL_TIME,+ False,+ ),+ ]+ )+ def test_a_claim_is_refused_by_an_investigation_in_flight_or_inside_the_cooldown(+ self, _name: str, earlier: str | None, age: timedelta, interval: str, claimed: bool+ ) -> None:+ self.alert.calculation_interval = interval+ if earlier is not None:+ self._investigation(at=NOW - age, status=earlier)++ claim = self._claim()++ assert claim.claimed is claimed+ row = InsightAlertInvestigation.objects.for_team(self.team.id).get(id=claim.investigation_id)+ assert row.status == (InvestigationStatus.PENDING if claimed else InvestigationStatus.SKIPPED)++ @parameterized.expand([("claimed", None, True), ("refused", InvestigationStatus.RUNNING, False)])+ def test_a_repeated_claim_returns_the_first_answer(self, _name: str, earlier: str | None, claimed: bool) -> None:+ if earlier is not None:+ self._investigation(at=NOW - timedelta(minutes=5), status=earlier)+ first = self._claim()+ InsightAlertInvestigation.objects.for_team(self.team.id).exclude(id=first.investigation_id).delete()++ assert self._claim() == first+ assert first.claimed is claimed