Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): evaluate ai detector insight alerts on the platform (#119949)

Jimmy Zhu82b368748508
diff --git a/docs/internal/alerts-platform-temporal.md b/docs/internal/alerts-platform-temporal.md
index b543e57d09b3..f36d25f06857 100644
--- a/docs/internal/alerts-platform-temporal.md
+++ b/docs/internal/alerts-platform-temporal.md
@@ -508,9 +508,12 @@ Insight alerts are copied the same way:
python manage.py backfill_platform_insight_alert_configurations
```
-It copies threshold and statistical detector alerts on every cadence, and skips AI detector alerts.
+It copies every insight alert, on every cadence.
A real-time check gets production's real-time budget: a 7-minute evaluation with one retry inside it.
+An AI detector check runs on the AI queue, whose worker holds the model credentials, with production's retries: a check whose model call reaches no verdict is retried, and only its last attempt records the error.
Every other check gets one 12-minute attempt.
+An AI detector check calls the model on both stacks while both run, and each call is billed to the project, so copy AI detector alerts only for internal projects until the insight cutover.
+The AI queue's worker needs `CLICKHOUSE_ALERTS_PLATFORM_INSIGHT_USER` and its token file too, or those checks query as the worker's own ClickHouse user.
Run it only after the evaluation worker's chart sets `CLICKHOUSE_ALERTS_PLATFORM_INSIGHT_USER` and its token file.
Insight checks tag their queries with `ClickHouseUser.ALERTS_PLATFORM_INSIGHT`, a user of their own, so the parallel run never takes from the per-user budget of the user that production insight alerts query as.
Without that env the tag resolves to the worker's default user, which other workloads on the same servers already push against its concurrent query limit.
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 2bd31af97f65..2904f30043dd 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -153,6 +153,7 @@ from posthog.temporal.weekly_digest import (
)
from products.alerts.backend.facade.temporal import (
+ PLATFORM_AI_QUEUE_ACTIVITIES as INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES,
PLATFORM_EVALUATION_ACTIVITIES as INSIGHT_PLATFORM_EVALUATION_ACTIVITIES,
PLATFORM_EVALUATION_WORKFLOWS as INSIGHT_PLATFORM_EVALUATION_WORKFLOWS,
)
@@ -502,7 +503,7 @@ _task_queue_specs = [
(
settings.MAX_AI_TASK_QUEUE,
AI_WORKFLOWS,
- AI_ACTIVITIES + ALERT_AI_QUEUE_ACTIVITIES,
+ AI_ACTIVITIES + ALERT_AI_QUEUE_ACTIVITIES + INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES,
),
(
settings.TEST_TASK_QUEUE,
diff --git a/posthog/management/commands/test/test_start_temporal_worker.py b/posthog/management/commands/test/test_start_temporal_worker.py
index 776a7fa0259d..e7e3e4be983d 100644
--- a/posthog/management/commands/test/test_start_temporal_worker.py
+++ b/posthog/management/commands/test/test_start_temporal_worker.py
@@ -10,6 +10,7 @@ from posthog.management.commands.start_temporal_worker import (
AI_WORKFLOWS,
ALERT_AI_QUEUE_ACTIVITIES,
DATA_SYNC_WORKFLOWS,
+ INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES,
WA_DIGEST_ACTIVITIES,
WA_DIGEST_WORKFLOWS,
WEEKLY_DIGEST_WORKFLOWS,
@@ -125,3 +126,4 @@ def test_ai_queue_registers_the_alert_evaluate_activity() -> None:
assert len(entries) == 1
assert ALERT_AI_QUEUE_ACTIVITIES
assert set(ALERT_AI_QUEUE_ACTIVITIES) <= set(entries[0])
+ assert set(INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES) <= set(entries[0])
diff --git a/posthog/temporal/alerts/activities.py b/posthog/temporal/alerts/activities.py
index 71963cbbae33..0e0b834a8f64 100644
--- a/posthog/temporal/alerts/activities.py
+++ b/posthog/temporal/alerts/activities.py
@@ -93,7 +93,7 @@ from products.alerts.backend.evaluation.validation import validate_alert_config,
from products.alerts.backend.facade.api import (
LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,
LLM_DETECTOR_UNAVAILABLE_MESSAGE,
- MAX_CONCURRENT_MODEL_CALLS,
+ LLM_EVALUATE_EXECUTOR,
LLMDetectorMisconfiguredError,
LLMDetectorUnavailableError,
is_llm_detector_config,
@@ -131,14 +131,6 @@ async def _run_control(
_NOTIFICATION_DELIVERY_EXECUTOR = ThreadPoolExecutor(max_workers=10, thread_name_prefix="insight-alert-delivery")
-# AI-detector checks hold a thread for the model call, up to a minute each. On the shared
-# default pool that would let a slow model stall every alert's database work on the worker,
-# so they run on their own pool, sized to the detector's own concurrency bound. A check
-# past that bound queues here without holding any thread.
-_LLM_EVALUATE_EXECUTOR = ThreadPoolExecutor(
- max_workers=MAX_CONCURRENT_MODEL_CALLS, thread_name_prefix="insight-alert-llm-evaluate"
-)
-
@frozen
class _RetrievedAlerts:
@@ -819,7 +811,7 @@ async def evaluate_alert(inputs: EvaluateAlertActivityInputs) -> EvaluateAlertRe
# tick, which prepares and routes it again.
logger.info("alerts.skip_detector_type_changed", alert_id=inputs.alert_id)
return EvaluateAlertResult(alert_check_id=None, should_notify=False, new_state=AlertState(alert.state))
- executor = _LLM_EVALUATE_EXECUTOR if uses_llm_detector else None
+ executor = LLM_EVALUATE_EXECUTOR if uses_llm_detector else None
# Shielded, so cancelling the attempt leaves this future to complete when the thread exits,
# which is what _stop_work waits for.
thread = asyncio.ensure_future(
diff --git a/products/alerts/backend/facade/api.py b/products/alerts/backend/facade/api.py
index f3668384eb91..16e622f385c4 100644
--- a/products/alerts/backend/facade/api.py
+++ b/products/alerts/backend/facade/api.py
@@ -37,6 +37,7 @@ from products.alerts.backend.judge.contract import (
LLMDetectorMisconfiguredError,
LLMDetectorUnavailableError,
)
+from products.alerts.backend.judge.executor import LLM_EVALUATE_EXECUTOR
from products.alerts.backend.llm_detector_limits import (
LLMAlertRefusal,
LLMAlertWrite,
@@ -214,6 +215,7 @@ def list_firing_alerts_created_by(
__all__ = [
"LLM_DETECTOR_UNAVAILABLE_ERROR_CODE",
"LLM_DETECTOR_UNAVAILABLE_MESSAGE",
+ "LLM_EVALUATE_EXECUTOR",
"MAX_CONCURRENT_MODEL_CALLS",
"MAX_PROMPT_POINTS",
"LLMAlertRefusal",
diff --git a/products/alerts/backend/facade/temporal.py b/products/alerts/backend/facade/temporal.py
index 907d1b214611..0bccb1e8c67c 100644
--- a/products/alerts/backend/facade/temporal.py
+++ b/products/alerts/backend/facade/temporal.py
@@ -1,11 +1,13 @@
"""Temporal workflows and activities this product registers on the alerts platform's queues."""
from products.alerts.backend.temporal.platform_evaluate import (
+ PLATFORM_AI_QUEUE_ACTIVITIES,
PLATFORM_EVALUATION_ACTIVITIES,
PLATFORM_EVALUATION_WORKFLOWS,
)
__all__ = [
+ "PLATFORM_AI_QUEUE_ACTIVITIES",
"PLATFORM_EVALUATION_ACTIVITIES",
"PLATFORM_EVALUATION_WORKFLOWS",
]
diff --git a/products/alerts/backend/judge/executor.py b/products/alerts/backend/judge/executor.py
new file mode 100644
index 000000000000..1d30518eb208
--- /dev/null
+++ b/products/alerts/backend/judge/executor.py
@@ -0,0 +1,12 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from products.alerts.backend.judge.contract import MAX_CONCURRENT_MODEL_CALLS
+
+# AI-detector checks hold a thread for the model call, up to a minute each. On the shared
+# default pool that would let a slow model stall every alert's database work on the worker,
+# so they run on their own pool, sized to the detector's own concurrency bound. A check
+# past that bound queues here without holding any thread. The platform's parallel run uses
+# this pool too, so the worker never runs more AI checks than the judge has model-call slots.
+LLM_EVALUATE_EXECUTOR = ThreadPoolExecutor(
+ max_workers=MAX_CONCURRENT_MODEL_CALLS, thread_name_prefix="insight-alert-llm-evaluate"
+)
diff --git a/products/alerts/backend/judge/llm.py b/products/alerts/backend/judge/llm.py
index 8c253eb4d2de..4460d5f3df2b 100644
--- a/products/alerts/backend/judge/llm.py
+++ b/products/alerts/backend/judge/llm.py
@@ -69,8 +69,8 @@ REQUEST_TIMEOUT_SECONDS = 60.0
# the same message need room too.
MAX_RATIONALE_CHARS = 600
-# The evaluate activity runs AI checks on a dedicated executor of MAX_CONCURRENT_MODEL_CALLS
-# threads (see posthog/temporal/alerts/activities.py), so a check there never waits here;
+# The evaluate activities run AI checks on a dedicated executor of MAX_CONCURRENT_MODEL_CALLS
+# threads (see products/alerts/backend/judge/executor.py), so a check there never waits here;
# the wait covers the API's simulate path, whose request threads are its own pool. The wait
# is short so a full pool fails a request fast instead of holding its thread.
MODEL_CALL_SLOT_WAIT_SECONDS = 5.0
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 54b70f55c54e..f2b964360137 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -16,6 +16,7 @@ same set of checks.
"""
import time
+import asyncio
from collections.abc import Collection
from dataclasses import replace
from datetime import UTC, datetime
@@ -37,6 +38,11 @@ from products.alerts.backend.evaluation import check_alert_for_insight
from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, check_alert_per_series
from products.alerts.backend.insight_alert_state_machine import INSIGHT_ALERT_POLICY, insight_snapshot
+from products.alerts.backend.judge.contract import (
+ LLM_DETECTOR_UNAVAILABLE_MESSAGE,
+ LLMDetectorMisconfiguredError,
+ LLMDetectorUnavailableError,
+)
from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
from products.alerts.backend.models.alert import AlertConfiguration
@@ -86,6 +92,11 @@ logger = structlog.get_logger(__name__)
INFLIGHT_KEY = "alerts:platform:insight:evaluations:inflight"
+# How many threads of the shared AI pool the parallel run may hold, so a stalled platform check
+# never takes more than this from production. A thread counts until it ends, not until Temporal
+# gives up on its attempt. Created per worker process and used only from the activity's event loop.
+PLATFORM_LLM_CHECKS = asyncio.Semaphore(2)
+
# Every query the parallel run sends starts with this, so its ClickHouse cost can be read apart from
# production's in `query_log`. Production's other tags are kept, so workload management treats both
# the same way.
@@ -124,9 +135,9 @@ _EVENT_IDS_BY_KIND: Final[dict[str, str]] = {
def is_evaluated_on_the_platform(alert: AlertConfiguration) -> bool:
- """Threshold and statistical detector alerts on every cadence. The AI detector makes a charged
- model call, so evaluating it in parallel costs twice."""
- return not is_llm_detector_config(alert.detector_config) and alert.calculation_interval in EVALUATED_INTERVALS
+ """Every alert production evaluates. An AI detector check makes a charged model call, so while
+ both stacks run, each of those checks is paid for twice."""
+ return alert.calculation_interval in EVALUATED_INTERVALS
def plan_insight_batch(
@@ -220,7 +231,7 @@ def legacy_alerts_of(
if check.legacy_configuration_id is not None
}
alerts = AlertConfiguration.objects.filter(team_id=team_id, id__in=list(legacy_ids)).only(
- "id", "calculation_interval"
+ "id", "calculation_interval", "detector_config"
)
return {legacy_ids[alert.id]: alert for alert in alerts}
@@ -233,13 +244,21 @@ def evaluate_insight_check(
*,
held_until: float,
evaluation_id: str,
+ attempt: int = 1,
+ final_attempt: bool = True,
+ uses_llm_detector: bool = False,
) -> SourceBatchEvaluation | None:
"""Decides one admitted check, then frees its slot.
Returns the outcome with its delivery, if it asks for one. Returns None when the configuration
is no longer due, because an earlier attempt already recorded it or it was edited out of the
- batch. Writes nothing.
+ batch, or when its detector no longer matches the worker it was routed to. Writes nothing.
+
+ `evaluation_id` names the check across its attempts, so the AI detector reuses a verdict an
+ earlier attempt paid for. An AI detector that reaches no verdict raises
+ `LLMDetectorUnavailableError` for a retry, and keeps the slot for it, unless `final_attempt`.
"""
+ retrying = False
try:
checks = due_checks(team_id, SourceKind.INSIGHT.value, slot, cutoff, configuration_ids=[configuration_id])
if not checks:
@@ -250,11 +269,25 @@ def evaluate_insight_check(
lag_ms = int((datetime.now(UTC) - checks[0].next_check_at).total_seconds() * 1000)
if lag_ms > 0:
safe_record(record_scheduler_lag, SourceKind.INSIGHT.value, lag_ms)
- outcome = _decide(checks[0], cutoff, evaluation_id=evaluation_id)
+ try:
+ outcome = _decide(
+ checks[0],
+ cutoff,
+ evaluation_id=evaluation_id,
+ attempt=attempt,
+ final_attempt=final_attempt,
+ uses_llm_detector=uses_llm_detector,
+ )
+ except LLMDetectorUnavailableError:
+ retrying = True
+ raise
+ if outcome is None:
+ return None
delivery = _delivery(checks[0], outcome)
return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,) if delivery is not None else ())
finally:
- release_evaluation_slot(configuration_id, held_until=held_until, key=INFLIGHT_KEY)
+ if not retrying:
+ release_evaluation_slot(configuration_id, held_until=held_until, key=INFLIGHT_KEY)
def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> AlertDeliveryRequest | None:
@@ -324,7 +357,15 @@ def slot_of_evaluation_key(evaluation_key: str) -> datetime | None:
return None
-def _decide(check: PlatformAlertCheckInput, now: datetime, *, evaluation_id: str) -> PlatformAlertOutcome:
+def _decide(
+ check: PlatformAlertCheckInput,
+ now: datetime,
+ *,
+ evaluation_id: str,
+ attempt: int,
+ final_attempt: bool,
+ uses_llm_detector: bool,
+) -> PlatformAlertOutcome | None:
snapshot = _snapshot(check)
alert = _legacy_alert(check)
if alert is None:
@@ -334,6 +375,10 @@ def _decide(check: PlatformAlertCheckInput, now: datetime, *, evaluation_id: str
return _skipped(check, snapshot, now=now)
if skip_because_of_weekend(alert) or is_utc_datetime_blocked(alert, now):
return _skipped(check, snapshot, now=now)
+ if is_llm_detector_config(alert.detector_config) != uses_llm_detector:
+ # As production: an edit since the plan means this worker may not hold the model
+ # credentials. No outcome keeps the check due, so the next tick plans and routes it again.
+ return None
# Production gates evaluation on a snooze and holds the alert in SNOOZED, then clears it to
# NOT_FIRING on the first check after the snooze ends. The legacy row is read for the snooze,
@@ -350,7 +395,7 @@ def _decide(check: PlatformAlertCheckInput, now: datetime, *, evaluation_id: str
tag_queries(
ch_user=ClickHouseUser.ALERTS_PLATFORM_INSIGHT,
team_id=alert.team_id,
- client_query_id=f"{QUERY_ID_PREFIX}{evaluation_id}",
+ client_query_id=f"{QUERY_ID_PREFIX}{evaluation_id}:{attempt}",
alert_config_id=str(alert.id),
product=Product.PRODUCT_ANALYTICS,
feature=Feature.ALERTING,
@@ -362,7 +407,19 @@ def _decide(check: PlatformAlertCheckInput, now: datetime, *, evaluation_id: str
started_at = time.monotonic()
try:
result = check_alert_for_insight(alert, evaluation_id=evaluation_id)
- except AlertExtractionError as error:
+ except LLMDetectorUnavailableError:
+ if not final_attempt:
+ # As production: a judge that reached no verdict gets another attempt, and only the
+ # last one records the failure.
+ raise
+ return _verdict(
+ check,
+ snapshot,
+ CheckInput(threshold_breached=False, error_message=LLM_DETECTOR_UNAVAILABLE_MESSAGE),
+ now=now,
+ skip=SkipReason.QUERY_FAILED,
+ )
+ except (AlertExtractionError, LLMDetectorMisconfiguredError) as error:
return _broken(check, snapshot, now=now, message=str(error))
except Exception as error:
return _query_failed(check, snapshot, error, now=now)
diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.py
index 3e061c3d4d25..8dce035fb75e 100644
--- a/products/alerts/backend/temporal/platform_evaluate.py
+++ b/products/alerts/backend/temporal/platform_evaluate.py
@@ -11,6 +11,7 @@ another would put the last check minutes behind the first.
import asyncio
import datetime as dt
from collections.abc import Callable
+from dataclasses import replace
from temporalio import activity, workflow
from temporalio.common import RetryPolicy
@@ -20,6 +21,8 @@ from posthog.dataclasses import frozen
from posthog.temporal.common.base import PostHogWorkflow
with workflow.unsafe.imports_passed_through():
+ from django.conf import settings
+
from posthog.schema_enums import AlertCalculationInterval
from posthog.temporal.alerts.retry_policy import alert_timeouts
from posthog.temporal.common.utils import close_db_connections
@@ -51,6 +54,9 @@ _REAL_TIME_TIMEOUTS = alert_timeouts(AlertCalculationInterval.REAL_TIME)
REAL_TIME_CHECK_START_TO_CLOSE = _REAL_TIME_TIMEOUTS.evaluate_start_to_close
REAL_TIME_CHECK_SCHEDULE_TO_CLOSE = _REAL_TIME_TIMEOUTS.activity_schedule_to_close + CHECK_QUEUE_TOLERANCE
REAL_TIME_CHECK_RETRY_POLICY = _REAL_TIME_TIMEOUTS.evaluate_retry_policy
+# Production's retries for an AI detector check, inside the default budget. A judge that reaches
+# no verdict raises for another attempt, and the last attempt records the failure.
+LLM_CHECK_RETRY_POLICY = alert_timeouts(None).evaluate_retry_policy
RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)
RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)
# The pool admits once per round, so a batch that admits only once evaluates at most one pool of
@@ -71,6 +77,9 @@ _PATCH_READMISSION = "platform-insight-readmission-2026-10"
_PATCH_CHUNKED_RECORD = "platform-insight-chunked-record-2026-10"
# Runs started before it planned bare ids, and gave every check the default budget.
_PATCH_CHECK_BUDGETS = "platform-insight-check-budgets-2026-10"
+# Runs started before it evaluated every check on this queue. A plan can return an AI detector
+# check into such a run during a deploy, and replay must schedule the activity that run recorded.
+_PATCH_AI_DETECTOR_QUEUE = "platform-insight-ai-detector-queue-2026-10"
@frozen
@@ -86,6 +95,7 @@ class AdmittedInsightCheck:
configuration_id: str
real_time: bool = False
+ uses_llm_detector: bool = False
@frozen
@@ -93,6 +103,8 @@ class InsightCheckInputs:
evaluation: SourceEvaluationInputs
configuration_id: str
held_until: float
+ # The worker this check was routed to. Only the AI queue's worker holds the model credentials.
+ uses_llm_detector: bool = False
@activity.defn
@@ -138,32 +150,34 @@ def plan_platform_insight_round_activity(inputs: InsightBatchPlanInputs) -> list
def plan_platform_insight_checks_activity(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:
"""`plan_platform_insight_round_activity` with each check's cadence, under its own name, because
a run that recorded the old name must decode bare ids on replay."""
+ from products.alerts.backend.llm_detector_limits import is_llm_detector_config # noqa: PLC0415
from products.alerts.backend.platform_source_cycle import legacy_alerts_of, plan_insight_batch # noqa: PLC0415
key = inputs.evaluation.batch_key
cutoff = dt.datetime.fromisoformat(inputs.evaluation.cutoff)
admitted = plan_insight_batch(key.team_id, key.slot, cutoff, expires_at=inputs.expires_at, exclude=inputs.exclude)
alerts = legacy_alerts_of(key.team_id, key.slot, cutoff, admitted)
- return [
- AdmittedInsightCheck(
- configuration_id=configuration_id,
- real_time=configuration_id in alerts
- and alerts[configuration_id].calculation_interval == AlertCalculationInterval.REAL_TIME,
+ checks: list[AdmittedInsightCheck] = []
+ for configuration_id in admitted:
+ alert = alerts.get(configuration_id)
+ checks.append(
+ AdmittedInsightCheck(
+ configuration_id=configuration_id,
+ real_time=alert is not None and alert.calculation_interval == AlertCalculationInterval.REAL_TIME,
+ uses_llm_detector=alert is not None and is_llm_detector_config(alert.detector_config),
+ )
)
- for configuration_id in admitted
- ]
+ return checks
-@close_db_connections
-def evaluate_platform_insight_check_activity(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
- """Sync, so the thread this blocks is a slot Temporal is accounting for."""
+def _evaluate_check(
+ inputs: InsightCheckInputs, info: activity.Info, *, final_attempt: bool
+) -> SourceBatchEvaluation | None:
from products.alerts.backend.platform_source_cycle import ( # noqa: PLC0415
evaluate_insight_check,
readmit_retried_check,
)
- info = activity.info()
key = inputs.evaluation.batch_key
held_until: float | None = inputs.held_until
if info.attempt > 1:
@@ -183,10 +197,52 @@ def evaluate_platform_insight_check_activity(inputs: InsightCheckInputs) -> Sour
dt.datetime.fromisoformat(inputs.evaluation.cutoff),
inputs.configuration_id,
held_until=held_until,
- evaluation_id=f"{info.workflow_run_id}:{info.activity_id}:{info.attempt}",
+ evaluation_id=f"{info.workflow_run_id}:{info.activity_id}",
+ attempt=info.attempt,
+ final_attempt=final_attempt,
+ uses_llm_detector=inputs.uses_llm_detector,
)
+@close_db_connections
+def evaluate_platform_insight_check_activity(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
+ """Sync, so the thread this blocks is a slot Temporal is accounting for."""
+ return _evaluate_check(inputs, activity.info(), final_attempt=True)
+
+
+@close_db_connections
+async def evaluate_platform_insight_llm_check_activity(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
+ """An AI detector check. It runs on the AI queue, because only that worker holds the model
+ credentials, on the pool production's AI checks use, so the worker never runs more checks than
+ the judge has model-call slots. `PLATFORM_LLM_CHECKS` caps how many of its threads this takes."""
+ from posthog.sync import database_sync_to_async # noqa: PLC0415
+
+ from products.alerts.backend.judge.executor import LLM_EVALUATE_EXECUTOR # noqa: PLC0415
+ from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS # noqa: PLC0415
+
+ info = activity.info()
+ await PLATFORM_LLM_CHECKS.acquire()
+ thread = asyncio.ensure_future(
+ database_sync_to_async(_evaluate_check, thread_sensitive=False, executor=LLM_EVALUATE_EXECUTOR)(
+ inputs, info, final_attempt=info.attempt >= LLM_CHECK_RETRY_POLICY.maximum_attempts
+ )
+ )
+ thread.add_done_callback(_release_platform_llm_check)
+ # Shielded, so a cancelled attempt leaves the thread holding its place in the cap until it ends.
+ return await asyncio.shield(thread)
+
+
+def _release_platform_llm_check(thread: "asyncio.Future[SourceBatchEvaluation | None]") -> None:
+ from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS # noqa: PLC0415
+
+ PLATFORM_LLM_CHECKS.release()
+ # Read the error of a thread nobody awaits any more, so asyncio does not log it as never retrieved.
+ if not thread.cancelled():
+ thread.exception()
+
+
@workflow.defn(name="insight-alert-platform-evaluate")
class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
"""Evaluates one batch key, records what it decided, and starts the deliveries its checks asked
@@ -269,6 +325,15 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
check_inputs = InsightCheckInputs(
evaluation=inputs, configuration_id=admitted.configuration_id, held_until=expires_at
)
+ if admitted.uses_llm_detector and workflow.patched(_PATCH_AI_DETECTOR_QUEUE):
+ return await workflow.execute_activity(
+ evaluate_platform_insight_llm_check_activity,
+ replace(check_inputs, uses_llm_detector=True),
+ task_queue=settings.MAX_AI_TASK_QUEUE,
+ start_to_close_timeout=CHECK_START_TO_CLOSE,
+ schedule_to_close_timeout=CHECK_SCHEDULE_TO_CLOSE,
+ retry_policy=LLM_CHECK_RETRY_POLICY,
+ )
if admitted.real_time:
return await workflow.execute_activity(
evaluate_platform_insight_check_activity,
@@ -332,6 +397,8 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
PLATFORM_EVALUATION_WORKFLOWS = [InsightAlertPlatformEvaluateWorkflow]
+# Registered on the AI queue's worker, which holds the model credentials.
+PLATFORM_AI_QUEUE_ACTIVITIES = [evaluate_platform_insight_llm_check_activity]
PLATFORM_EVALUATION_ACTIVITIES = [
plan_platform_insight_batch_activity,
plan_platform_insight_round_activity,
diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.py
index fbe12c195973..72cb65e77758 100644
--- a/products/alerts/backend/test/test_platform_alert_backfill.py
+++ b/products/alerts/backend/test/test_platform_alert_backfill.py
@@ -51,19 +51,19 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
)
return {view.legacy_configuration_id: view for view in page.configurations}
- def test_only_alerts_without_the_ai_detector_are_copied_on_their_recurrence(self) -> None:
+ def test_every_alert_is_copied_on_its_recurrence(self) -> None:
real_time = self._alert(calculation_interval=AlertCalculationInterval.REAL_TIME.value)
quarter_hourly = self._alert(calculation_interval=AlertCalculationInterval.EVERY_15_MINUTES.value)
hourly = self._alert(calculation_interval=AlertCalculationInterval.HOURLY.value, schedule_start_time="09:30")
daily = self._alert(schedule_start_time="09:30")
detector = self._alert(detector_config={"type": "zscore"})
- self._alert(detector_config={"type": "llm"})
+ ai_detector = self._alert(detector_config={"type": "llm"})
unparseable = self._alert(schedule_start_time="25:99")
counts = backfill_platform_insight_alert_configurations(team_id=self.team.id)
again = backfill_platform_insight_alert_configurations(team_id=self.team.id)
- assert (counts.created, counts.skipped, counts.failed, again.created, again.updated) == (5, 1, 1, 0, 5)
+ assert (counts.created, counts.skipped, counts.failed, again.created, again.updated) == (6, 0, 1, 0, 6)
copies = self._copies()
assert unparseable.id not in copies
assert {key: (v.check_interval_minutes, v.recurrence_unit, v.anchor_time) for key, v in copies.items()} == {
@@ -72,6 +72,7 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
hourly.id: (60, None, None),
daily.id: (60 * 24, "day", "09:30"),
detector.id: (60 * 24, "day", None),
+ ai_detector.id: (60 * 24, "day", None),
}
assert copies[daily.id].source_config["condition"] == {
"threshold": THRESHOLD,
diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.py
index 0413360b7adf..4a1a09422776 100644
--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py
+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py
@@ -1,4 +1,6 @@
import uuid
+import asyncio
+import threading
from collections.abc import AsyncIterator
import pytest
@@ -10,15 +12,17 @@ import pytest_asyncio
from temporalio import activity, converter
from temporalio.api.enums.v1 import EventType
from temporalio.exceptions import ApplicationError
-from temporalio.testing import WorkflowEnvironment
+from temporalio.testing import ActivityEnvironment, WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS
from products.alerts.backend.temporal.platform_evaluate import (
EVALUATION_BUDGET,
AdmittedInsightCheck,
InsightAlertPlatformEvaluateWorkflow,
InsightBatchPlanInputs,
InsightCheckInputs,
+ evaluate_platform_insight_llm_check_activity,
)
from products.alerts_platform.backend.facade.contracts import (
RECORD_OUTCOMES_ACTIVITY,
@@ -36,6 +40,7 @@ from products.alerts_platform.backend.facade.temporal import source_evaluation_t
QUEUE = "test-platform-insight-evaluation"
ADMITTED = ("00000000-0000-0000-0000-000000000001", "00000000-0000-0000-0000-000000000002")
+AI_CHECK = "00000000-0000-0000-0000-000000000003"
@pytest_asyncio.fixture
@@ -47,12 +52,13 @@ async def environment() -> AsyncIterator[WorkflowEnvironment]:
async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(environment: WorkflowEnvironment) -> None:
recorded: list[SourceOutcomeInputs] = []
- # Only the real-time check has a retry, so the same first failure loses the other one.
+ # Only the real-time and AI checks have a retry, so the same first failure loses the other one.
@activity.defn(name="plan_platform_insight_checks_activity")
async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:
admitted = [
AdmittedInsightCheck(configuration_id=ADMITTED[0]),
AdmittedInsightCheck(configuration_id=ADMITTED[1], real_time=True),
+ AdmittedInsightCheck(configuration_id=AI_CHECK, uses_llm_detector=True),
]
return [check for check in admitted if check.configuration_id not in inputs.exclude]
@@ -60,6 +66,18 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
if activity.info().attempt == 1:
raise ApplicationError("worker lost")
+ return _fired(inputs)
+
+ # Registered only on the AI queue, so an AI check routed anywhere else never runs.
+ @activity.defn(name="evaluate_platform_insight_llm_check_activity")
+ async def evaluate_on_the_ai_queue(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
+ if not inputs.uses_llm_detector:
+ raise ApplicationError("routed without the AI flag", non_retryable=True)
+ if activity.info().attempt == 1:
+ raise ApplicationError("model timed out")
+ return _fired(inputs)
+
+ def _fired(inputs: InsightCheckInputs) -> SourceBatchEvaluation:
outcome = PlatformAlertOutcome(
configuration_id=uuid.UUID(inputs.configuration_id),
evaluation_key="slot:2026-09-16T10:00:00+00:00",
@@ -81,12 +99,15 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
recorded.append(inputs)
return len(inputs.outcomes)
- async with Worker(
- environment.client,
- task_queue=QUEUE,
- workflows=[InsightAlertPlatformEvaluateWorkflow],
- activities=[plan, evaluate, record],
- workflow_runner=UnsandboxedWorkflowRunner(),
+ async with (
+ Worker(
+ environment.client,
+ task_queue=QUEUE,
+ workflows=[InsightAlertPlatformEvaluateWorkflow],
+ activities=[plan, evaluate, record],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ),
+ Worker(environment.client, task_queue=settings.MAX_AI_TASK_QUEUE, activities=[evaluate_on_the_ai_queue]),
):
handle = await environment.client.start_workflow(
InsightAlertPlatformEvaluateWorkflow.run,
@@ -102,15 +123,16 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
result = await handle.result()
history = await handle.fetch_history()
- assert result == 1
- assert [[str(o.configuration_id) for o in batch.outcomes] for batch in recorded] == [[ADMITTED[1]]]
+ assert result == 2
+ assert [sorted(str(o.configuration_id) for o in batch.outcomes) for batch in recorded] == [[ADMITTED[1], AI_CHECK]]
children = [
event.child_workflow_execution_started_event_attributes
for event in history.events
if event.event_type == EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED
]
- assert [child.workflow_execution.workflow_id for child in children] == [
- f"alerts-deliver-preview-{ADMITTED[1]}:slot:2026-09-16T10:00:00+00:00"
+ assert sorted(child.workflow_execution.workflow_id for child in children) == [
+ f"alerts-deliver-preview-{configuration_id}:slot:2026-09-16T10:00:00+00:00"
+ for configuration_id in (ADMITTED[1], AI_CHECK)
]
child = await environment.client.get_workflow_handle(children[0].workflow_execution.workflow_id).describe()
assert child.task_queue == settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE
@@ -273,3 +295,51 @@ async def test_a_batch_refills_the_slots_its_finished_checks_free(
def test_the_binding_holds_the_whole_batch() -> None:
assert source_evaluation_timeout(SourceKind.INSIGHT) > EVALUATION_BUDGET
+
+
+async def test_a_cancelled_platform_ai_check_keeps_its_place_in_the_cap_until_its_thread_ends() -> None:
+ release = threading.Event()
+ started: list[str] = []
+
+ def blocking(inputs: InsightCheckInputs, *_args: object, **_kwargs: object) -> None:
+ started.append(inputs.configuration_id)
+ release.wait(timeout=30)
+
+ def check(configuration_id: str) -> InsightCheckInputs:
+ return InsightCheckInputs(
+ evaluation=SourceEvaluationInputs(
+ source=SourceKind.INSIGHT,
+ cutoff="2026-09-16T10:00:00+00:00",
+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),
+ ),
+ configuration_id=configuration_id,
+ held_until=0.0,
+ uses_llm_detector=True,
+ )
+
+ first, second = ActivityEnvironment(), ActivityEnvironment()
+ try:
+ with patch("products.alerts.backend.temporal.platform_evaluate._evaluate_check", blocking):
+ cancelled = asyncio.ensure_future(
+ first.run(evaluate_platform_insight_llm_check_activity, check(ADMITTED[0]))
+ )
+ running = asyncio.ensure_future(
+ second.run(evaluate_platform_insight_llm_check_activity, check(ADMITTED[1]))
+ )
+ async with asyncio.timeout(10):
+ while len(started) < 2:
+ await asyncio.sleep(0.01)
+
+ first.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await cancelled
+ # Its thread still runs, so the parallel run still holds both of its places.
+ assert PLATFORM_LLM_CHECKS.locked()
+
+ release.set()
+ await running
+ async with asyncio.timeout(10):
+ while PLATFORM_LLM_CHECKS.locked():
+ await asyncio.sleep(0.01)
+ finally:
+ release.set()
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index c6a44bb2d4ed..a73db98c36a5 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -3,6 +3,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import UUID
+import pytest
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch
@@ -21,6 +22,11 @@ from posthog.temporal.alerts.admission import inflight_alert_ids, release_evalua
from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, SeriesEvaluation
+from products.alerts.backend.judge.contract import (
+ LLM_DETECTOR_UNAVAILABLE_MESSAGE,
+ LLMDetectorMisconfiguredError,
+ LLMDetectorUnavailableError,
+)
from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration
from products.alerts.backend.platform_source_cycle import (
@@ -137,7 +143,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):
]
def _evaluate(
- self, configuration: PlatformConfigurationSnapshot, *, result: Any = None
+ self, configuration: PlatformConfigurationSnapshot, *, result: Any = None, uses_llm_detector: bool = False
) -> tuple[PlatformAlertOutcome | None, MagicMock]:
slot = slot_of(configuration.next_check_at, CUTOFF)
expires_at = time.time() + 3600
@@ -146,7 +152,13 @@ class TestPlatformInsightEvaluation(APIBaseTest):
returned = result if isinstance(result, AlertEvaluationResult) else None
with patch(f"{_MODULE}.check_alert_for_insight", side_effect=side_effect, return_value=returned) as query:
evaluation = evaluate_insight_check(
- self.team.id, slot, CUTOFF, str(configuration.id), held_until=expires_at, evaluation_id="test"
+ self.team.id,
+ slot,
+ CUTOFF,
+ str(configuration.id),
+ held_until=expires_at,
+ evaluation_id="test",
+ uses_llm_detector=uses_llm_detector,
)
self.deliveries = evaluation.deliveries if evaluation is not None else ()
return (evaluation.outcomes[0] if evaluation is not None else None), query
@@ -215,7 +227,6 @@ class TestPlatformInsightEvaluation(APIBaseTest):
@parameterized.expand(
[
- ("ai_detector", {"detector_config": {"type": "llm"}}, "not_firing"),
("disabled", {"enabled": False}, "not_firing"),
("snoozed", {"snoozed_until": CUTOFF + timedelta(hours=1)}, "snoozed"),
]
@@ -233,6 +244,20 @@ class TestPlatformInsightEvaluation(APIBaseTest):
False,
)
+ @parameterized.expand(
+ [
+ ("ai_detector_on_the_default_worker", {"detector_config": {"type": "llm"}}, False),
+ ("threshold_on_the_ai_worker", {}, True),
+ ]
+ )
+ def test_a_check_routed_for_another_detector_stays_due_without_a_query(
+ self, _name: str, overrides: dict[str, Any], uses_llm_detector: bool
+ ) -> None:
+ outcome, query = self._evaluate(self._copy(self._alert(**overrides)), uses_llm_detector=uses_llm_detector)
+
+ query.assert_not_called()
+ assert outcome is None
+
def test_a_copy_whose_production_alert_is_gone_is_disabled(self) -> None:
outcome, query = self._evaluate(self._copy(None))
@@ -256,8 +281,16 @@ class TestPlatformInsightEvaluation(APIBaseTest):
False,
)
- def test_an_alert_production_cannot_evaluate_as_configured_errors_and_is_disabled(self) -> None:
- outcome, _ = self._evaluate(self._copy(self._alert()), result=AlertExtractionError("bad query shape"))
+ @parameterized.expand(
+ [
+ ("bad_query_shape", AlertExtractionError("bad query shape")),
+ ("ai_consent_withdrawn", LLMDetectorMisconfiguredError("consent withdrawn")),
+ ]
+ )
+ def test_an_alert_production_cannot_evaluate_as_configured_errors_and_is_disabled(
+ self, _name: str, error: Exception
+ ) -> None:
+ outcome, _ = self._evaluate(self._copy(self._alert()), result=error)
assert outcome is not None
assert outcome.failure is not None and outcome.groups == ()
@@ -267,6 +300,50 @@ class TestPlatformInsightEvaluation(APIBaseTest):
True,
)
+ @parameterized.expand([("retried", False), ("last_attempt", True)])
+ def test_an_ai_detector_without_a_verdict_is_retried_and_only_its_last_attempt_records_an_error(
+ self, _name: str, final_attempt: bool
+ ) -> None:
+ configuration = self._copy(self._alert(detector_config={"type": "llm"}))
+ slot = slot_of(configuration.next_check_at, CUTOFF)
+ expires_at = time.time() + 3600
+ plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at)
+
+ with patch(f"{_MODULE}.check_alert_for_insight", side_effect=LLMDetectorUnavailableError("timed out")):
+ if not final_attempt:
+ with pytest.raises(LLMDetectorUnavailableError):
+ evaluate_insight_check(
+ self.team.id,
+ slot,
+ CUTOFF,
+ str(configuration.id),
+ held_until=expires_at,
+ evaluation_id="test",
+ final_attempt=False,
+ uses_llm_detector=True,
+ )
+ # The retry still holds the slot, so the pool cannot hand it to another check.
+ assert inflight_alert_ids(INFLIGHT_KEY) == {str(configuration.id)}
+ return
+ evaluation = evaluate_insight_check(
+ self.team.id,
+ slot,
+ CUTOFF,
+ str(configuration.id),
+ held_until=expires_at,
+ evaluation_id="test",
+ uses_llm_detector=True,
+ )
+
+ assert evaluation is not None
+ (outcome,) = evaluation.outcomes
+ assert outcome.failure is not None
+ assert (outcome.failure.kind, outcome.error_message, outcome.disable) == (
+ AlertEventKind.ERRORED,
+ LLM_DETECTOR_UNAVAILABLE_MESSAGE,
+ False,
+ )
+
@override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS=1)
def test_a_retry_stops_the_timed_out_query_and_runs_only_inside_the_pool(self) -> None:
held = self._copy(self._alert(), id=UUID(int=1))