Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): let detector insight alerts cut over to the platform (#123632)

Jimmy Zhu8bd2980ba7f1
diff --git a/products/alerts/backend/platform_alert_wording.py b/products/alerts/backend/platform_alert_wording.py
index 574c0ef57041..1da56ed23278 100644
--- a/products/alerts/backend/platform_alert_wording.py
+++ b/products/alerts/backend/platform_alert_wording.py
@@ -13,6 +13,7 @@ from posthog.utils import absolute_uri
from products.alerts.backend.logic.insight_alert_destinations import snooze_select
from products.alerts_platform.backend.facade.contracts import (
+ SOURCE_CONDITION_KEY,
AlertEventKind,
AnnouncedTransition,
MessageDetail,
@@ -31,7 +32,7 @@ def describe_insight_transition(*, project_id: int, transition: AnnouncedTransit
insight_short_id = detail.get("insight_short_id")
return SourceDescription(
# One detail for every breach, because a transport such as PagerDuty keys its details by label.
- details=(MessageDetail(label="Threshold breached", value="\n".join(breaches)),) if breaches else (),
+ details=(MessageDetail(label=_breach_label(transition), value="\n".join(breaches)),) if breaches else (),
context=(f"Insight: {insight_name}",) if insight_name else (),
data_link=MessageLink(label="View insight", url=insight_url(project_id, insight_short_id))
if insight_short_id
@@ -40,6 +41,11 @@ def describe_insight_transition(*, project_id: int, transition: AnnouncedTransit
)
+def _breach_label(transition: AnnouncedTransition) -> str:
+ condition = transition.source_config.get(SOURCE_CONDITION_KEY)
+ return "Anomaly detected" if isinstance(condition, dict) and condition.get("detector") else "Threshold breached"
+
+
def insight_url(project_id: int, insight_short_id: str) -> str:
return absolute_uri(f"/project/{project_id}/insights/{insight_short_id}")
diff --git a/products/alerts/backend/platform_cutover.py b/products/alerts/backend/platform_cutover.py
index acf8b936b38a..60a452c4c00f 100644
--- a/products/alerts/backend/platform_cutover.py
+++ b/products/alerts/backend/platform_cutover.py
@@ -69,7 +69,7 @@ class CutoverSkip(StrEnum):
ALREADY_MOVED = "already_moved"
# Platform rows live on the project's root team, so an alert in a child environment stays.
ENVIRONMENT = "environment"
- # The platform does not evaluate it in parallel, such as a detector or a real-time alert.
+ # The platform evaluates it only in parallel, such as a real-time alert.
NOT_ON_PLATFORM = "not_on_platform"
# The investigation agent reads and writes the insight stack's checks, which the platform does not keep.
INVESTIGATION = "investigation"
@@ -114,9 +114,9 @@ def cut_over_team(team_id: int, *, now: datetime | None = None) -> tuple[AlertCu
)
-def _is_threshold_alert(alert: AlertConfiguration) -> bool:
- """The platform evaluates detector and real-time alerts in parallel, but cannot own them yet."""
- return not alert.detector_config and alert.calculation_interval != AlertCalculationInterval.REAL_TIME
+def _is_scheduled(alert: AlertConfiguration) -> bool:
+ """The platform evaluates real-time alerts in parallel, but cannot own them yet."""
+ return alert.calculation_interval != AlertCalculationInterval.REAL_TIME
def delivers_as_it_does_today(alert: AlertConfiguration) -> bool:
@@ -167,10 +167,9 @@ def in_an_environment(alert: AlertConfiguration) -> bool:
def can_move(alert: AlertConfiguration) -> bool:
- """Whether the platform can own the alert: a threshold alert it evaluates, which no investigation agent reads."""
- return (
- platform_upsert_of(alert) is not None and _is_threshold_alert(alert) and not alert.investigation_agent_enabled
- )
+ """Whether the platform can own the alert: a scheduled alert it evaluates, with a threshold or a
+ detector, which no investigation agent reads."""
+ return platform_upsert_of(alert) is not None and _is_scheduled(alert) and not alert.investigation_agent_enabled
def has_begun_cutover(team_id: int) -> bool:
diff --git a/products/alerts/backend/platform_ownership.py b/products/alerts/backend/platform_ownership.py
index 6d8c6c76a33a..a5619423cbc7 100644
--- a/products/alerts/backend/platform_ownership.py
+++ b/products/alerts/backend/platform_ownership.py
@@ -15,6 +15,7 @@ from uuid import UUID
from products.alerts.backend.evaluation.validation import validate_alert_config
from products.alerts.backend.facade.contracts import PlatformCannotOwn
from products.alerts.backend.insight_alert_state_machine import apply_disable, apply_enable, apply_threshold_change
+from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts.backend.models.alert import AlertConfiguration, Threshold
from products.alerts.backend.platform_cutover import (
can_move,
@@ -52,13 +53,12 @@ def on_created(alert: AlertConfiguration, at: datetime) -> bool:
def on_updated(alert: AlertConfiguration, *, snooze_changed: bool) -> None:
"""Writes an edit the insight API saved onto the platform's copy.
- Raises `PlatformCannotOwn` for an edit the platform cannot evaluate, such as a detector, and
+ Raises `PlatformCannotOwn` for an edit the platform cannot evaluate, such as a real-time check, and
`InvalidPlatformAlertEdit` for one the platform's schema rejects.
"""
if not can_move(alert):
raise PlatformCannotOwn(
- "The alerts platform runs this alert, and it cannot run detectors, real-time checks or the "
- "investigation agent yet."
+ "The alerts platform runs this alert, and it cannot run real-time checks or the investigation agent yet."
)
update_owned_copy(
team_id=alert.team_id,
@@ -149,7 +149,19 @@ def _write_insight_row(alert: AlertConfiguration, edit: PlatformAlertConfigurati
# The inverse of `platform_upsert_of`.
if edit.source_config["insight_id"] != alert.insight_id:
raise InvalidPlatformAlertEdit("source_config", "An insight alert cannot move to another insight.")
+ if edit.enabled and not alert.enabled and is_llm_detector_config(alert.detector_config):
+ # Turning an AI alert back on adds to its project's capped model spend, which only the
+ # insight API's write admission checks.
+ raise InvalidPlatformAlertEdit(
+ "enabled",
+ "An AI detector alert can't be turned back on through this API. Turn it on in the insight's alert settings.",
+ )
condition = edit.source_config[SOURCE_CONDITION_KEY]
+ if condition.get("detector") != alert.detector_config:
+ # A detector config is checked against its insight and its access rules in the insight API only.
+ raise InvalidPlatformAlertEdit(
+ "source_config", "The detector can't change through this API. Change it in the insight's alert settings."
+ )
try:
# The rules the insight API applies, so the insight stack never holds a pair it cannot check.
validate_alert_config(
diff --git a/products/alerts/backend/platform_source_config_schema.py b/products/alerts/backend/platform_source_config_schema.py
index 9ee564e45054..f67dacc44b5a 100644
--- a/products/alerts/backend/platform_source_config_schema.py
+++ b/products/alerts/backend/platform_source_config_schema.py
@@ -5,6 +5,8 @@ Loaded on first use through `load_insight_source_config`, never from `ready()`,
on `posthog.schema`.
"""
+from typing import Any
+
from pydantic import BaseModel, ConfigDict, Field
from posthog.schema import AlertCondition, InsightThreshold
@@ -28,8 +30,10 @@ class InsightPlatformAlertCondition(BaseModel):
...,
description="How the value is compared: as it is, or as an increase or decrease from the previous interval.",
)
- detector: None = Field(
- default=None, description="Always null. The platform owns threshold alerts only, so no detector applies."
+ detector: dict[str, Any] | None = Field(
+ default=None,
+ description="The anomaly detector the alert uses in place of its threshold, or null for a threshold alert. "
+ "It can't change through this API.",
)
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 0d8aa7d4e60c..c645cca03f83 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -307,7 +307,7 @@ def legacy_alerts_of(
if check.legacy_configuration_id is not None
}
alerts = AlertConfiguration.objects.filter(team_id=team_id, id__in=list(legacy_ids)).only(
- "id", "calculation_interval", "detector_config"
+ "id", "calculation_interval", "detector_config", "owner"
)
return {legacy_ids[alert.id]: alert for alert in alerts}
diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.py
index 8dce035fb75e..ef990d0cdc93 100644
--- a/products/alerts/backend/temporal/platform_evaluate.py
+++ b/products/alerts/backend/temporal/platform_evaluate.py
@@ -96,6 +96,8 @@ class AdmittedInsightCheck:
configuration_id: str
real_time: bool = False
uses_llm_detector: bool = False
+ # The platform owns the alert, so this check is the one that notifies.
+ owned_by_platform: bool = False
@frozen
@@ -105,6 +107,7 @@ class InsightCheckInputs:
held_until: float
# The worker this check was routed to. Only the AI queue's worker holds the model credentials.
uses_llm_detector: bool = False
+ owned_by_platform: bool = False
@activity.defn
@@ -165,6 +168,7 @@ def plan_platform_insight_checks_activity(inputs: InsightBatchPlanInputs) -> lis
configuration_id=configuration_id,
real_time=alert is not None and alert.calculation_interval == AlertCalculationInterval.REAL_TIME,
uses_llm_detector=alert is not None and is_llm_detector_config(alert.detector_config),
+ owned_by_platform=alert is not None and alert.platform_owned,
)
)
return checks
@@ -216,19 +220,21 @@ def evaluate_platform_insight_check_activity(inputs: InsightCheckInputs) -> Sour
async def evaluate_platform_insight_llm_check_activity(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
"""An AI detector check. It runs on the AI queue, because only that worker holds the model
credentials, on the pool production's AI checks use, so the worker never runs more checks than
- the judge has model-call slots. `PLATFORM_LLM_CHECKS` caps how many of its threads this takes."""
+ the judge has model-call slots. `PLATFORM_LLM_CHECKS` caps how many of its threads a parallel
+ check takes. A check of an alert the platform owns is production's own, so it skips that cap,
+ as production's checks do."""
from posthog.sync import database_sync_to_async # noqa: PLC0415
from products.alerts.backend.judge.executor import LLM_EVALUATE_EXECUTOR # noqa: PLC0415
from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS # noqa: PLC0415
info = activity.info()
+ evaluate = database_sync_to_async(_evaluate_check, thread_sensitive=False, executor=LLM_EVALUATE_EXECUTOR)
+ final_attempt = info.attempt >= LLM_CHECK_RETRY_POLICY.maximum_attempts
+ if inputs.owned_by_platform:
+ return await evaluate(inputs, info, final_attempt=final_attempt)
await PLATFORM_LLM_CHECKS.acquire()
- thread = asyncio.ensure_future(
- database_sync_to_async(_evaluate_check, thread_sensitive=False, executor=LLM_EVALUATE_EXECUTOR)(
- inputs, info, final_attempt=info.attempt >= LLM_CHECK_RETRY_POLICY.maximum_attempts
- )
- )
+ thread = asyncio.ensure_future(evaluate(inputs, info, final_attempt=final_attempt))
thread.add_done_callback(_release_platform_llm_check)
# Shielded, so a cancelled attempt leaves the thread holding its place in the cap until it ends.
return await asyncio.shield(thread)
@@ -323,7 +329,10 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
self, inputs: SourceEvaluationInputs, admitted: AdmittedInsightCheck, expires_at: float
) -> SourceBatchEvaluation | None:
check_inputs = InsightCheckInputs(
- evaluation=inputs, configuration_id=admitted.configuration_id, held_until=expires_at
+ evaluation=inputs,
+ configuration_id=admitted.configuration_id,
+ held_until=expires_at,
+ owned_by_platform=admitted.owned_by_platform,
)
if admitted.uses_llm_detector and workflow.patched(_PATCH_AI_DETECTOR_QUEUE):
return await workflow.execute_activity(
diff --git a/products/alerts/backend/test/test_platform_alert_wording.py b/products/alerts/backend/test/test_platform_alert_wording.py
index 1f8cb0996e55..03d6d808d1bb 100644
--- a/products/alerts/backend/test/test_platform_alert_wording.py
+++ b/products/alerts/backend/test/test_platform_alert_wording.py
@@ -55,6 +55,22 @@ class TestDescribeInsightTransition(SimpleTestCase):
slack_elements=(snooze_select("01900000-0000-0000-0000-000000000001"),),
),
),
+ (
+ "a_detector_firing_names_its_anomaly",
+ AlertEventKind.FIRING,
+ {**DETAIL, "breaches": ["Anomaly detected in Signups: value 50.00 using AI detector"]},
+ SourceDescription(
+ details=(
+ MessageDetail(
+ label="Anomaly detected", value="Anomaly detected in Signups: value 50.00 using AI detector"
+ ),
+ ),
+ context=("Insight: Signups",),
+ data_link=LINK,
+ slack_elements=(snooze_select("01900000-0000-0000-0000-000000000001"),),
+ ),
+ {"condition": {"threshold": None, "comparison": {}, "detector": {"type": "llm"}}},
+ ),
# The platform states a failure in its own words, so only the insight's name and link are added.
(
"an_error_keeps_the_platform_failure_details",
@@ -67,7 +83,12 @@ class TestDescribeInsightTransition(SimpleTestCase):
]
)
def test_the_insight_words_a_native_message(
- self, _name: str, kind: AlertEventKind, detail: dict[str, Any], expected: SourceDescription
+ self,
+ _name: str,
+ kind: AlertEventKind,
+ detail: dict[str, Any],
+ expected: SourceDescription,
+ source_config: dict[str, Any] | None = None,
) -> None:
transition = AnnouncedTransition(
grouping_key="",
@@ -76,7 +97,7 @@ class TestDescribeInsightTransition(SimpleTestCase):
value=50.0,
labels={},
condition={},
- source_config={},
+ source_config=source_config or {},
error_message=None,
occurred_at=datetime(2026, 10, 20, 9, 0, tzinfo=UTC),
source_detail=detail,
diff --git a/products/alerts/backend/test/test_platform_cutover.py b/products/alerts/backend/test/test_platform_cutover.py
index 7456cbcafdb5..ae66e7ee0992 100644
--- a/products/alerts/backend/test/test_platform_cutover.py
+++ b/products/alerts/backend/test/test_platform_cutover.py
@@ -37,6 +37,8 @@ class TestInsightPlatformCutover(APIBaseTest):
"not_firing",
),
("disabled", {"enabled": False, "next_check_at": None}, "not_firing"),
+ ("statistical_detector", {"detector_config": {"type": "zscore"}}, "not_firing"),
+ ("ai_detector", {"detector_config": {"type": "llm"}}, "not_firing"),
]
)
def test_an_alert_moves_with_its_state_and_schedule(
@@ -64,7 +66,6 @@ class TestInsightPlatformCutover(APIBaseTest):
("firing", {"state": AlertState.FIRING}, CutoverSkip.FIRING),
("check_due", {"next_check_at": NOW}, CutoverSkip.CHECK_DUE),
("never_scheduled", {"next_check_at": None}, CutoverSkip.CHECK_DUE),
- ("detector", {"detector_config": {"type": "zscore"}}, CutoverSkip.NOT_ON_PLATFORM),
(
"real_time",
{"calculation_interval": AlertCalculationInterval.REAL_TIME.value},
diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.py
index 4a1a09422776..ae55843f015f 100644
--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py
+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py
@@ -303,9 +303,10 @@ async def test_a_cancelled_platform_ai_check_keeps_its_place_in_the_cap_until_it
def blocking(inputs: InsightCheckInputs, *_args: object, **_kwargs: object) -> None:
started.append(inputs.configuration_id)
- release.wait(timeout=30)
+ if not inputs.owned_by_platform:
+ release.wait(timeout=30)
- def check(configuration_id: str) -> InsightCheckInputs:
+ def check(configuration_id: str, *, owned: bool = False) -> InsightCheckInputs:
return InsightCheckInputs(
evaluation=SourceEvaluationInputs(
source=SourceKind.INSIGHT,
@@ -315,6 +316,7 @@ async def test_a_cancelled_platform_ai_check_keeps_its_place_in_the_cap_until_it
configuration_id=configuration_id,
held_until=0.0,
uses_llm_detector=True,
+ owned_by_platform=owned,
)
first, second = ActivityEnvironment(), ActivityEnvironment()
@@ -335,6 +337,12 @@ async def test_a_cancelled_platform_ai_check_keeps_its_place_in_the_cap_until_it
await cancelled
# Its thread still runs, so the parallel run still holds both of its places.
assert PLATFORM_LLM_CHECKS.locked()
+ # A check of an alert the platform owns is production's own, so the full cap does not hold it back.
+ async with asyncio.timeout(10):
+ await ActivityEnvironment().run(
+ evaluate_platform_insight_llm_check_activity, check("owned", owned=True)
+ )
+ assert "owned" in started
release.set()
await running
diff --git a/products/alerts/backend/tests/api/test_alert_platform_owned.py b/products/alerts/backend/tests/api/test_alert_platform_owned.py
index dc2c2ce9d2aa..cdbd4e832b46 100644
--- a/products/alerts/backend/tests/api/test_alert_platform_owned.py
+++ b/products/alerts/backend/tests/api/test_alert_platform_owned.py
@@ -268,24 +268,40 @@ class TestPlatformOwnedInsightAlertWrites(APIBaseTest):
{"upper": 200},
)
- def test_a_platform_edit_the_insight_stack_cannot_check_is_refused_on_both_rows(self) -> None:
- def percentage_bound(stored: PlatformAlertConfigurationEdit) -> PlatformAlertConfigurationEdit:
- threshold = {"type": "percentage", "bounds": {"upper": 0.5}}
- condition = {**stored.source_config["condition"], "threshold": threshold}
+ @parameterized.expand(
+ [
+ ("percentage_bound", {"threshold": {"type": "percentage", "bounds": {"upper": 0.5}}}),
+ # The insight API checks a detector against its insight and its access rules, so only it may set one.
+ ("detector", {"detector": {"type": "zscore"}}),
+ ]
+ )
+ def test_a_platform_edit_the_insight_stack_cannot_check_is_refused_on_both_rows(
+ self, _name: str, condition_change: dict[str, Any]
+ ) -> None:
+ stored_config = self._copy().source_config
+
+ def edit(stored: PlatformAlertConfigurationEdit) -> PlatformAlertConfigurationEdit:
+ condition = {**stored.source_config["condition"], **condition_change}
return replace(stored, source_config={**stored.source_config, "condition": condition})
with pytest.raises(InvalidPlatformAlertEdit):
- self._edit_on_the_platform(percentage_bound)
+ self._edit_on_the_platform(edit)
self.alert.refresh_from_db()
assert self.alert.threshold is not None
- assert (
- self.alert.threshold.configuration["type"],
- self._copy().source_config["condition"]["threshold"]["type"],
- ) == (
- "absolute",
- "absolute",
- )
+ assert (self.alert.threshold.configuration["type"], self.alert.detector_config) == ("absolute", None)
+ assert self._copy().source_config == stored_config
+
+ def test_a_platform_edit_cannot_turn_an_ai_alert_back_on(self) -> None:
+ self._edit_on_the_platform(lambda stored: replace(stored, enabled=False))
+ AlertConfiguration.objects.filter(id=self.alert.id).update(detector_config={"type": "llm"})
+
+ with pytest.raises(InvalidPlatformAlertEdit) as refused:
+ self._edit_on_the_platform(lambda stored: replace(stored, enabled=True))
+
+ assert refused.value.field_name == "enabled"
+ self.alert.refresh_from_db()
+ assert (self.alert.enabled, self._copy().enabled) == (False, False)
def test_a_snooze_from_slack_mutes_the_platform_copy(self) -> None:
outcome = snooze_alert_from_slack(self.alert.id, user=self.user, duration="1d")
diff --git a/products/alerts_platform/frontend/generated/api.schemas.ts b/products/alerts_platform/frontend/generated/api.schemas.ts
index 40b879a40329..e37549fb5365 100644
--- a/products/alerts_platform/frontend/generated/api.schemas.ts
+++ b/products/alerts_platform/frontend/generated/api.schemas.ts
@@ -775,16 +775,18 @@ export interface AlertConditionApi {
type: AlertConditionTypeApi
}
+/**
+ * The anomaly detector the alert uses in place of its threshold, or null for a threshold alert. It can't change through this API.
+ */
+export type InsightPlatformAlertConditionApiDetector = { [key: string]: unknown } | null
+
export interface InsightPlatformAlertConditionApi {
/** The bounds the insight's value is checked against, absolute or as a percentage change. */
threshold: InsightThresholdApi | null
/** How the value is compared: as it is, or as an increase or decrease from the previous interval. */
comparison: AlertConditionApi
- /**
- * Always null. The platform owns threshold alerts only, so no detector applies.
- * @nullable
- */
- detector?: null
+ /** The anomaly detector the alert uses in place of its threshold, or null for a threshold alert. It can't change through this API. */
+ detector?: InsightPlatformAlertConditionApiDetector
}
export interface InsightPlatformAlertSourceConfigApi {
diff --git a/products/alerts_platform/frontend/generated/api.zod.ts b/products/alerts_platform/frontend/generated/api.zod.ts
index e1c621b18915..12232bd064b8 100644
--- a/products/alerts_platform/frontend/generated/api.zod.ts
+++ b/products/alerts_platform/frontend/generated/api.zod.ts
@@ -261,10 +261,10 @@ export const PlatformAlertsUpdateBody = /* @__PURE__ */ zod.union([
'How the value is compared: as it is, or as an increase or decrease from the previous interval.'
),
detector: zod
- .null()
+ .union([zod.record(zod.string(), zod.unknown()), zod.null()])
.optional()
.describe(
- 'Always null. The platform owns threshold alerts only, so no detector applies.'
+ "The anomaly detector the alert uses in place of its threshold, or null for a threshold alert. It can't change through this API."
),
})
.describe("The bound the insight's value is checked against."),
@@ -579,10 +579,10 @@ export const PlatformAlertsPartialUpdateBody = /* @__PURE__ */ zod.union([
'How the value is compared: as it is, or as an increase or decrease from the previous interval.'
),
detector: zod
- .null()
+ .union([zod.record(zod.string(), zod.unknown()), zod.null()])
.optional()
.describe(
- 'Always null. The platform owns threshold alerts only, so no detector applies.'
+ "The anomaly detector the alert uses in place of its threshold, or null for a threshold alert. It can't change through this API."
),
})
.describe("The bound the insight's value is checked against."),
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 3e81a380da6a..84b24649db3d 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -64553,16 +64553,18 @@ export namespace Schemas {
depth?: InsightDataChecksDepthEnum;
}
+ /**
+ * The anomaly detector the alert uses in place of its threshold, or null for a threshold alert. It can't change through this API.
+ */
+ export type InsightPlatformAlertConditionDetector = { [key: string]: unknown } | null;
+
export interface InsightPlatformAlertCondition {
/** The bounds the insight's value is checked against, absolute or as a percentage change. */
threshold: InsightThreshold | null;
/** How the value is compared: as it is, or as an increase or decrease from the previous interval. */
comparison: AlertCondition;
- /**
- * Always null. The platform owns threshold alerts only, so no detector applies.
- * @nullable
- */
- detector?: null;
+ /** The anomaly detector the alert uses in place of its threshold, or null for a threshold alert. It can't change through this API. */
+ detector?: InsightPlatformAlertConditionDetector;
}
/**