:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): record platform insight checks as they finish (#123636)
Jimmy Zhub2d2f4b063b9
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex 62912e98b7bf..1f7c6928372b 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -277,13 +277,7 @@ def readmit_retried_check(would run beside it. The retry then moves the slot to its own token, so the pool never countsthe two attempts as one check that either can release."""- # Imported here: at module level it changes how many queries the access-control tests count.- from posthog.clickhouse.cancel import cancel_query_on_cluster # noqa: PLC0415-- try:- cancel_query_on_cluster(team_id, f"{QUERY_ID_PREFIX}{previous_query_id}")- except Exception:- logger.exception("Could not stop a timed-out platform insight check", configuration_id=configuration_id)+ _stop_query(team_id, configuration_id, previous_query_id)release_evaluation_slot(configuration_id, held_until=attempt_slot_token(held_until, attempt - 1), key=INFLIGHT_KEY)token = attempt_slot_token(held_until, attempt)admitted = admit_evaluation_slots(@@ -295,6 +289,16 @@ def readmit_retried_check(return token if admitted else None+def _stop_query(team_id: int, configuration_id: str, query_id: str) -> None:+ # Imported here: at module level it changes how many queries the access-control tests count.+ from posthog.clickhouse.cancel import cancel_query_on_cluster # noqa: PLC0415++ try:+ cancel_query_on_cluster(team_id, f"{QUERY_ID_PREFIX}{query_id}")+ except Exception:+ logger.exception("Could not stop a timed-out platform insight check", configuration_id=configuration_id)++def legacy_alerts_of(team_id: int, slot: str, cutoff: datetime, configuration_ids: Collection[str]) -> dict[str, AlertConfiguration]:@@ -361,16 +365,65 @@ def evaluate_insight_check(raiseif outcome is None:return None- if checks[0].legacy_configuration_id is not None:- # Production times each check by its own alert's id, so the copy checks at that minute.- outcome = replace(outcome, schedule_key=str(checks[0].legacy_configuration_id))- delivery = _delivery(checks[0], outcome)- return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,) if delivery is not None else ())+ return _evaluation(checks[0], outcome)finally:if not retrying:release_evaluation_slot(configuration_id, held_until=held_until, key=INFLIGHT_KEY)+def fail_insight_check(+ team_id: int,+ slot: str,+ cutoff: datetime,+ configuration_id: str,+ *,+ error_message: str,+ error_type: str | None = None,+) -> SourceBatchEvaluation | None:+ """The errored outcome of a check whose every attempt ended without one, such as a query that ran+ past its time budget, as production records an errored check when its retries run out.++ `error_type` names the class of the last attempt's error, as production reads it to word a+ judge that could not be reached.+ Returns None when the check is no longer due, or its production alert is gone or switched off.+ Writes no state.+ """+ checks = due_checks(team_id, SourceKind.INSIGHT.value, slot, cutoff, configuration_ids=[configuration_id])+ if not checks:+ return None+ alert = _legacy_alert(checks[0])+ # As production: switching an alert off mid-check is not a failure of the alert.+ if alert is None or not alert.enabled:+ return None+ if error_type == LLMDetectorUnavailableError.__name__:+ # As production: the raw transport error is not the owner's configuration, so it gets its own wording.+ error_message = LLM_DETECTOR_UNAVAILABLE_MESSAGE+ outcome = _verdict(+ checks[0],+ _snapshot(checks[0]),+ CheckInput(threshold_breached=False, error_message=error_message),+ now=cutoff,+ skip=SkipReason.QUERY_FAILED,+ context=_context(alert),+ )+ return _evaluation(checks[0], outcome)+++def stop_insight_queries(team_id: int, configuration_id: str, query_ids: Sequence[str]) -> None:+ """Stops the queries a failed check's attempts may still run. A timed-out attempt's thread+ keeps its query going, and nothing else stops it."""+ for query_id in query_ids:+ _stop_query(team_id, configuration_id, query_id)+++def _evaluation(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> SourceBatchEvaluation:+ if check.legacy_configuration_id is not None:+ # Production times each check by its own alert's id, so the copy checks at that minute.+ outcome = replace(outcome, schedule_key=str(check.legacy_configuration_id))+ delivery = _delivery(check, outcome)+ return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,) if delivery is not None else ())++def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> AlertDeliveryRequest | None:kinds = [group.kind for group in outcome.groups]if outcome.failure is not None:diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.pyindex e5158cef1f5a..e5f35ec57385 100644--- a/products/alerts/backend/temporal/platform_evaluate.py+++ b/products/alerts/backend/temporal/platform_evaluate.py@@ -26,6 +26,12 @@ with workflow.unsafe.imports_passed_through():from posthog.schema_enums import AlertCalculationIntervalfrom posthog.temporal.alerts.retry_policy import alert_timeouts+ from posthog.temporal.common.errors import (+ MAX_ERROR_MESSAGE_CHARS,+ truncate_for_temporal_payload,+ unwrap_temporal_cause,+ )+ from posthog.temporal.common.heartbeat_sync import HeartbeaterSyncfrom posthog.temporal.common.utils import close_db_connectionsfrom products.alerts.backend.platform_chart_detail import RenderedChart, pending_chart_alert_id, with_chart@@ -56,6 +62,8 @@ _REAL_TIME_TIMEOUTS = alert_timeouts(AlertCalculationInterval.REAL_TIME)REAL_TIME_CHECK_START_TO_CLOSE = _REAL_TIME_TIMEOUTS.evaluate_start_to_closeREAL_TIME_CHECK_SCHEDULE_TO_CLOSE = _REAL_TIME_TIMEOUTS.activity_schedule_to_close + CHECK_QUEUE_TOLERANCEREAL_TIME_CHECK_RETRY_POLICY = _REAL_TIME_TIMEOUTS.evaluate_retry_policy+# Production's real-time heartbeat, so a check whose worker died is retried inside its budget.+REAL_TIME_CHECK_HEARTBEAT_TIMEOUT = _REAL_TIME_TIMEOUTS.heartbeat_timeout# Production's retries for an AI detector check, inside the default budget. A judge that reaches# no verdict raises for another attempt, and the last attempt records the failure.LLM_CHECK_RETRY_POLICY = alert_timeouts(None).evaluate_retry_policy@@ -69,6 +77,11 @@ CHART_RETRY_POLICY = RetryPolicy(backoff_coefficient=2.0,maximum_interval=dt.timedelta(seconds=30),)+FAIL_START_TO_CLOSE = dt.timedelta(seconds=10)+FAIL_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=20)+# Stopping a query can wait on a slow ClickHouse, so it runs apart from the failed check's record+# and never holds it back. It is best effort, so it gets what the batch budget leaves.+STOP_QUERIES_START_TO_CLOSE = dt.timedelta(seconds=60)RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)# The pool admits once per round, so a batch that admits only once evaluates at most one pool of@@ -77,13 +90,16 @@ RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)ADMISSION_RUN_BUDGET = dt.timedelta(seconds=50)ADMISSION_POLL_INTERVAL = dt.timedelta(seconds=2)# What the source needs from its binding's `evaluation_timeout`. The checks run side by side, so-# the batch costs the admission rounds plus one check's budget, not one per check.+# the batch costs the admission rounds plus one check's budget, not one per check. The last check+# can end while an earlier record still runs, so the tail holds two records and their failures,+# or one record and the stop of a failed check's queries.EVALUATION_BUDGET = (PLAN_SCHEDULE_TO_CLOSE+ ADMISSION_RUN_BUDGET+ max(CHECK_SCHEDULE_TO_CLOSE, REAL_TIME_CHECK_SCHEDULE_TO_CLOSE)- + CHART_SCHEDULE_TO_CLOSE+ + max(CHART_SCHEDULE_TO_CLOSE, FAIL_SCHEDULE_TO_CLOSE)+ RECORD_SCHEDULE_TO_CLOSE+ + max(FAIL_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE, STOP_QUERIES_START_TO_CLOSE))_PATCH_READMISSION = "platform-insight-readmission-2026-10"# Runs started before it recorded every outcome in one activity, and replay must schedule the same.@@ -95,6 +111,8 @@ _PATCH_CHECK_BUDGETS = "platform-insight-check-budgets-2026-10"_PATCH_AI_DETECTOR_QUEUE = "platform-insight-ai-detector-queue-2026-10"# Runs started before it rendered charts inside the check, and replay must not schedule a render._PATCH_CHART_AFTER_CHECK = "platform-insight-chart-after-check-2026-10"+# Runs started before it recorded every outcome once the whole batch ended, and failed checks left none.+_PATCH_RECORD_AS_FINISHED = "platform-insight-record-as-finished-2026-10"@frozen@@ -131,6 +149,22 @@ class InsightCheckInputs:owned_by_platform: bool = False+@frozen+class FailedInsightCheckInputs:+ evaluation: SourceEvaluationInputs+ configuration_id: str+ error_message: str+ error_type: str | None = None+++@frozen+class StopInsightQueriesInputs:+ team_id: int+ configuration_id: str+ # The queries a failed check's attempts ran under, any of which may still run.+ query_ids: tuple[str, ...]++@activity.defn@close_db_connectionsdef plan_platform_insight_batch_activity(inputs: InsightBatchPlanInputs) -> list[str]:@@ -232,8 +266,37 @@ def _evaluate_check(@activity.defn@close_db_connectionsdef evaluate_platform_insight_check_activity(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:- """Sync, so the thread this blocks is a slot Temporal is accounting for."""- return _evaluate_check(inputs, activity.info(), final_attempt=True)+ """Sync, so the thread this blocks is a slot Temporal is accounting for. Heartbeats only when the+ workflow set a heartbeat timeout."""+ with HeartbeaterSync():+ return _evaluate_check(inputs, activity.info(), final_attempt=True)+++@close_db_connections+def fail_platform_insight_check_activity(inputs: FailedInsightCheckInputs) -> SourceBatchEvaluation | None:+ """The errored outcome of a check whose every attempt ended without one."""+ from products.alerts.backend.platform_source_cycle import fail_insight_check # noqa: PLC0415++ key = inputs.evaluation.batch_key+ return fail_insight_check(+ key.team_id,+ key.slot,+ dt.datetime.fromisoformat(inputs.evaluation.cutoff),+ inputs.configuration_id,+ error_message=inputs.error_message,+ error_type=inputs.error_type,+ )+++@close_db_connections+def stop_platform_insight_queries_activity(inputs: StopInsightQueriesInputs) -> None:+ """Stops the queries a failed check's attempts may still run, because a timed-out attempt's+ thread keeps its query going."""+ from products.alerts.backend.platform_source_cycle import stop_insight_queries # noqa: PLC0415++ stop_insight_queries(inputs.team_id, inputs.configuration_id, inputs.query_ids)@activity.defn@@ -303,6 +366,8 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):# From the workflow's clock, so a retried plan reuses the expiry and gets its slots back. A# check releases its slot only while the slot still carries this expiry.expires_at = (workflow.now() + source_evaluation_timeout(SourceKind.INSIGHT)).timestamp()+ if workflow.patched(_PATCH_READMISSION) and workflow.patched(_PATCH_RECORD_AS_FINISHED):+ return await self._evaluate_and_record_in_rounds(inputs, expires_at)if workflow.patched(_PATCH_READMISSION):settled = await self._evaluate_in_rounds(inputs, expires_at)else:@@ -329,8 +394,7 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):evaluations = await self._with_charts(inputs, evaluations)# A check that ran on new code under a run that started before the patch leaves its marker.evaluations = [with_chart(evaluation, None) for evaluation in evaluations]- outcomes: tuple[PlatformAlertOutcome, ...] = tuple(o for e in evaluations for o in e.outcomes)- deliveries: tuple[AlertDeliveryRequest, ...] = tuple(d for e in evaluations for d in e.deliveries)+ outcomes = tuple(outcome for evaluation in evaluations for outcome in evaluation.outcomes)if len(outcomes) < len(settled):workflow.logger.warning("%d of %d platform insight checks reached no outcome", len(settled) - len(outcomes), len(settled)@@ -338,21 +402,7 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):safe_record(increment_checks_unfinished, SourceKind.INSIGHT.value, len(settled) - len(outcomes))if not outcomes:return 0-- # Each check's result fits a payload on its own, but the batch's together may not. A chunk's- # deliveries start after its own record, so a later chunk that fails cannot strand them.- chunks = outcome_chunks(outcomes) if workflow.patched(_PATCH_CHUNKED_RECORD) else [outcomes]- for chunk in chunks:- await workflow.execute_activity(- RECORD_OUTCOMES_ACTIVITY,- SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=chunk),- start_to_close_timeout=RECORD_START_TO_CLOSE,- schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,- retry_policy=RetryPolicy(maximum_attempts=3),- )- # Started only after the record, because a delivery reads the history row the record writes.- recorded = {str(outcome.configuration_id) for outcome in chunk}- await start_deliveries([delivery for delivery in deliveries if delivery.configuration_id in recorded])+ await self._record(inputs, evaluations, chunked=workflow.patched(_PATCH_CHUNKED_RECORD))return len(outcomes)async def _with_charts(@@ -427,10 +477,11 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):check_inputs,start_to_close_timeout=REAL_TIME_CHECK_START_TO_CLOSE,schedule_to_close_timeout=REAL_TIME_CHECK_SCHEDULE_TO_CLOSE,+ heartbeat_timeout=REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,retry_policy=REAL_TIME_CHECK_RETRY_POLICY,)- # One attempt each. A check that does not finish leaves no outcome, so it keeps its due time- # and a later tick evaluates it again, the way a cohort the logs budget left out behaves.+ # One attempt each. In a run that records as checks finish, a check that does not finish is+ # recorded as errored. In an older run it leaves no outcome and keeps its due time.return await workflow.execute_activity(evaluate_platform_insight_check_activity,check_inputs,@@ -482,6 +533,226 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):)return await asyncio.gather(*started.values(), return_exceptions=True)+ async def _evaluate_and_record_in_rounds(self, inputs: SourceEvaluationInputs, expires_at: float) -> int:+ """`_evaluate_in_rounds`, but the checks that ended are recorded and delivered at each poll, so+ a slow check in the batch never holds back another's state, schedule or delivery. A check+ whose every attempt failed is recorded as errored. Returns how many outcomes it recorded.++ One record per poll rather than per check, because each record is a transaction and a+ ClickHouse insert of its own. A record that fails keeps its outcomes for the next poll, so a+ passing storage failure does not cost the checks still running.+ """+ started: dict[str, AdmittedInsightCheck] = {}+ unrecorded: dict[str, asyncio.Task[SourceBatchEvaluation | None]] = {}+ waiting: list[SourceBatchEvaluation] = []+ stopping: list[asyncio.Task[None]] = []+ recorded = 0+ admitting = True+ deadline = workflow.now() + ADMISSION_RUN_BUDGET+ next_record_at = workflow.now()+ while True:+ added = False+ if admitting:+ try:+ admitted = await self._plan_round(inputs, expires_at, exclude=tuple(started), budgeted=True)+ except ActivityError:+ if not started:+ raise+ workflow.logger.warning("Stopped admitting platform insight checks after a failed plan")+ admitting = False+ else:+ added = bool(admitted)+ for check in admitted:+ started[check.configuration_id] = check+ unrecorded[check.configuration_id] = asyncio.create_task(+ self._check_and_chart(inputs, check, expires_at)+ )+ finished = {configuration_id: task for configuration_id, task in unrecorded.items() if task.done()}+ if (finished or waiting) and (+ workflow.now() >= next_record_at or all(task.done() for task in unrecorded.values())+ ):+ next_record_at = workflow.now() + ADMISSION_POLL_INTERVAL+ for configuration_id in finished:+ del unrecorded[configuration_id]+ waiting.extend(+ await self._evaluations_of(+ inputs,+ {started[configuration_id]: task for configuration_id, task in finished.items()},+ stopping,+ )+ )+ recorded += await self._record_waiting(inputs, waiting, last_chance=False)+ if workflow.now() >= deadline:+ admitting = False+ if not unrecorded:+ # The checks a round added can end while earlier ones record, and free slots for more.+ if admitting and added:+ continue+ recorded += await self._record_waiting(inputs, waiting, last_chance=True)+ await asyncio.gather(*stopping, return_exceptions=True)+ return recorded+ running = [task for task in unrecorded.values() if not task.done()]+ if not running:+ continue+ # While admitting, wake at each poll to refill the pool. After that, wake when a check+ # ends, or when the checks that already ended are due their record. An idle timer would+ # add history events for the whole of a slow check.+ if admitting:+ timeout: float | None = ADMISSION_POLL_INTERVAL.total_seconds()+ elif len(running) < len(unrecorded) or waiting:+ timeout = max((next_record_at - workflow.now()).total_seconds(), 0)+ else:+ timeout = None+ await asyncio.wait(running, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)++ async def _check_and_chart(+ self, inputs: SourceEvaluationInputs, check: AdmittedInsightCheck, expires_at: float+ ) -> SourceBatchEvaluation | None:+ """The check, then its firing's chart, so a slow render holds back only its own check's record.++ No patch guards the render, because a run on this path started on code that renders charts."""+ evaluation = await self._check(inputs, check, expires_at)+ if evaluation is None:+ return None+ return (await self._with_charts(inputs, [evaluation]))[0]++ async def _evaluations_of(+ self,+ inputs: SourceEvaluationInputs,+ finished: dict[AdmittedInsightCheck, "asyncio.Task[SourceBatchEvaluation | None]"],+ stopping: list["asyncio.Task[None]"],+ ) -> list[SourceBatchEvaluation]:+ """What the finished checks decided, with an errored outcome for each check that failed."""+ evaluations: list[SourceBatchEvaluation] = []+ failures: dict[AdmittedInsightCheck, ActivityError] = {}+ for check, task in finished.items():+ error = task.exception()+ if isinstance(error, ActivityError):+ failures[check] = error+ elif error is not None:+ # Only a check's own failure is a missing outcome. A bug in this workflow ends the run.+ raise error+ elif (evaluation := task.result()) is not None:+ evaluations.append(evaluation)+ for check, error in failures.items():+ stopping.append(asyncio.create_task(self._stop_queries(inputs, check, error)))+ failed = await asyncio.gather(*(self._fail(inputs, check, error) for check, error in failures.items()))+ evaluations.extend(evaluation for evaluation in failed if evaluation is not None)+ unfinished = len(finished) - sum(len(evaluation.outcomes) for evaluation in evaluations)+ if unfinished > 0:+ safe_record(increment_checks_unfinished, SourceKind.INSIGHT.value, unfinished)+ return evaluations++ async def _fail(+ self, inputs: SourceEvaluationInputs, check: AdmittedInsightCheck, error: ActivityError+ ) -> SourceBatchEvaluation | None:+ # The reason the owner reads, unwrapped from Temporal's failure and bounded for the payload.+ cause = unwrap_temporal_cause(error)+ message = cause.message if cause is not None else str(error.cause or error)+ try:+ return await workflow.execute_activity(+ fail_platform_insight_check_activity,+ FailedInsightCheckInputs(+ evaluation=inputs,+ configuration_id=check.configuration_id,+ error_message=truncate_for_temporal_payload(message, MAX_ERROR_MESSAGE_CHARS),+ # Temporal rebuilds a remote error as an ApplicationError carrying the class name on `type`.+ error_type=cause.type if cause is not None else None,+ ),+ start_to_close_timeout=FAIL_START_TO_CLOSE,+ schedule_to_close_timeout=FAIL_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ except ActivityError:+ # The check stays due and the next tick evaluates it again, so the batch's other+ # outcomes still record.+ workflow.logger.warning("Could not record a failed platform insight check as errored")+ return None++ async def _stop_queries(+ self, inputs: SourceEvaluationInputs, check: AdmittedInsightCheck, error: ActivityError+ ) -> None:+ # Each attempt queries under `{run}:{activity}:{attempt}`, as `_evaluate_check` names it.+ evaluation_id = f"{workflow.info().run_id}:{error.activity_id}"+ try:+ await workflow.execute_activity(+ stop_platform_insight_queries_activity,+ StopInsightQueriesInputs(+ team_id=inputs.batch_key.team_id,+ configuration_id=check.configuration_id,+ query_ids=tuple(f"{evaluation_id}:{attempt}" for attempt in range(1, _maximum_attempts(check) + 1)),+ ),+ start_to_close_timeout=STOP_QUERIES_START_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ except ActivityError:+ workflow.logger.warning("Could not stop the queries of a failed platform insight check")++ async def _record_waiting(+ self, inputs: SourceEvaluationInputs, waiting: list[SourceBatchEvaluation], *, last_chance: bool+ ) -> int:+ """Records the waiting outcomes chunk by chunk, and takes each chunk that recorded out of+ `waiting`. A chunk that fails stays for the next poll, unless this is the last chance.+ Returns how many outcomes it recorded."""+ outcomes = tuple(outcome for evaluation in waiting for outcome in evaluation.outcomes)+ deliveries = tuple(delivery for evaluation in waiting for delivery in evaluation.deliveries)+ recorded = 0+ for chunk in outcome_chunks(outcomes) if outcomes else []:+ try:+ await self._record_chunk(inputs, chunk, deliveries)+ except ActivityError:+ if last_chance:+ raise+ workflow.logger.warning("Will record platform insight outcomes again at the next poll")+ break+ done = {str(outcome.configuration_id) for outcome in chunk}+ waiting[:] = [+ evaluation+ for evaluation in waiting+ if not any(str(outcome.configuration_id) in done for outcome in evaluation.outcomes)+ ]+ recorded += len(chunk)+ return recorded++ async def _record(+ self, inputs: SourceEvaluationInputs, evaluations: list[SourceBatchEvaluation], *, chunked: bool+ ) -> int:+ """Records the outcomes, then starts their deliveries. Returns how many outcomes it recorded."""+ outcomes = tuple(outcome for evaluation in evaluations for outcome in evaluation.outcomes)+ if not outcomes:+ return 0+ deliveries = tuple(delivery for evaluation in evaluations for delivery in evaluation.deliveries)+ for chunk in outcome_chunks(outcomes) if chunked else [outcomes]:+ await self._record_chunk(inputs, chunk, deliveries)+ return len(outcomes)++ async def _record_chunk(+ self,+ inputs: SourceEvaluationInputs,+ chunk: tuple[PlatformAlertOutcome, ...],+ deliveries: tuple[AlertDeliveryRequest, ...],+ ) -> None:+ # Each check's result fits a payload on its own, but the batch's together may not. A chunk's+ # deliveries start after its own record, so a later chunk that fails cannot strand them.+ await workflow.execute_activity(+ RECORD_OUTCOMES_ACTIVITY,+ SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=chunk),+ start_to_close_timeout=RECORD_START_TO_CLOSE,+ schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ # Started only after the record, because a delivery reads the history row the record writes.+ recorded = {str(outcome.configuration_id) for outcome in chunk}+ await start_deliveries([delivery for delivery in deliveries if delivery.configuration_id in recorded])+++def _maximum_attempts(check: AdmittedInsightCheck) -> int:+ if check.uses_llm_detector:+ return LLM_CHECK_RETRY_POLICY.maximum_attempts+ if check.real_time:+ return REAL_TIME_CHECK_RETRY_POLICY.maximum_attempts+ return 1+PLATFORM_EVALUATION_WORKFLOWS = [InsightAlertPlatformEvaluateWorkflow]# Registered on the AI queue's worker, which holds the model credentials.@@ -492,4 +763,6 @@ PLATFORM_EVALUATION_ACTIVITIES = [plan_platform_insight_checks_activity,evaluate_platform_insight_check_activity,render_platform_insight_chart_activity,+ fail_platform_insight_check_activity,+ stop_platform_insight_queries_activity,]diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.pyindex ae99755c490e..bc8b91fc40b3 100644--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py@@ -24,11 +24,14 @@ from products.alerts.backend.platform_chart_detail import (from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKSfrom products.alerts.backend.temporal.platform_evaluate import (EVALUATION_BUDGET,+ REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,AdmittedInsightCheck,+ FailedInsightCheckInputs,InsightAlertPlatformEvaluateWorkflow,InsightBatchPlanInputs,InsightChartInputs,InsightCheckInputs,+ StopInsightQueriesInputs,evaluate_platform_insight_llm_check_activity,)from products.alerts_platform.backend.facade.contracts import (@@ -56,8 +59,19 @@ async def environment() -> AsyncIterator[WorkflowEnvironment]:yield env+def _outcome(configuration_id: str, kind: AlertEventKind, state: str) -> PlatformAlertOutcome:+ return PlatformAlertOutcome(+ configuration_id=uuid.UUID(configuration_id),+ evaluation_key="slot:2026-09-16T10:00:00+00:00",+ consecutive_failures=0,+ groups=(GroupOutcome(grouping_key="", kind=kind, new_state=state, notified=kind != AlertEventKind.CHECK),),+ )++async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(environment: WorkflowEnvironment) -> None:recorded: list[SourceOutcomeInputs] = []+ failed: list[FailedInsightCheckInputs] = []+ stopped: list[StopInsightQueriesInputs] = []# Only the real-time and AI checks have a retry, so the same first failure loses the other one.@activity.defn(name="plan_platform_insight_checks_activity")@@ -85,12 +99,7 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(envreturn _fired(inputs)def _fired(inputs: InsightCheckInputs) -> SourceBatchEvaluation:- outcome = PlatformAlertOutcome(- configuration_id=uuid.UUID(inputs.configuration_id),- evaluation_key="slot:2026-09-16T10:00:00+00:00",- consecutive_failures=0,- groups=(GroupOutcome(grouping_key="", kind=AlertEventKind.FIRING, new_state="firing", notified=True),),- )+ outcome = _outcome(inputs.configuration_id, AlertEventKind.FIRING, "firing")delivery = AlertDeliveryRequest(source=SourceKind.INSIGHT,team_id=1,@@ -101,6 +110,17 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env)return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,))+ @activity.defn(name="fail_platform_insight_check_activity")+ async def fail(inputs: FailedInsightCheckInputs) -> SourceBatchEvaluation | None:+ failed.append(inputs)+ return SourceBatchEvaluation(+ outcomes=(_outcome(inputs.configuration_id, AlertEventKind.ERRORED, "errored"),), deliveries=()+ )++ @activity.defn(name="stop_platform_insight_queries_activity")+ async def stop(inputs: StopInsightQueriesInputs) -> None:+ stopped.append(inputs)+@activity.defn(name=RECORD_OUTCOMES_ACTIVITY)async def record(inputs: SourceOutcomeInputs) -> int:recorded.append(inputs)@@ -111,7 +131,7 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(envenvironment.client,task_queue=QUEUE,workflows=[InsightAlertPlatformEvaluateWorkflow],- activities=[plan, evaluate, record],+ activities=[plan, evaluate, fail, stop, record],workflow_runner=UnsandboxedWorkflowRunner(),),Worker(environment.client, task_queue=settings.MAX_AI_TASK_QUEUE, activities=[evaluate_on_the_ai_queue]),@@ -130,8 +150,24 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(envresult = await handle.result()history = await handle.fetch_history()- assert result == 2- assert [sorted(str(o.configuration_id) for o in batch.outcomes) for batch in recorded] == [[ADMITTED[1], AI_CHECK]]+ assert result == 3+ # The check with no retry left no outcome of its own, so it is recorded as errored, with the reason.+ assert [(inputs.configuration_id, inputs.error_message) for inputs in failed] == [(ADMITTED[0], "worker lost")]+ # Its one attempt's query may still run, which nothing else stops.+ assert [(inputs.configuration_id, len(inputs.query_ids)) for inputs in stopped] == [(ADMITTED[0], 1)]+ assert sorted(+ (str(o.configuration_id), o.groups[0].new_state) for batch in recorded for o in batch.outcomes+ ) == sorted([(ADMITTED[0], "errored"), (ADMITTED[1], "firing"), (AI_CHECK, "firing")])+ real_time_heartbeats = [+ event.activity_task_scheduled_event_attributes.heartbeat_timeout.ToTimedelta()+ for event in history.events+ if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED+ and event.activity_task_scheduled_event_attributes.activity_type.name+ == "evaluate_platform_insight_check_activity"+ and event.activity_task_scheduled_event_attributes.heartbeat_timeout.ToTimedelta()+ ]+ # Only the real-time check heartbeats, so a dead worker's check is retried inside its budget.+ assert real_time_heartbeats == [REAL_TIME_CHECK_HEARTBEAT_TIMEOUT]children = [event.child_workflow_execution_started_event_attributesfor event in history.events@@ -160,12 +196,7 @@ async def test_a_batch_over_the_payload_budget_records_in_chunks_and_starts_one_@activity.defn(name="evaluate_platform_insight_check_activity")async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:- outcome = PlatformAlertOutcome(- configuration_id=uuid.UUID(inputs.configuration_id),- evaluation_key="slot:2026-09-16T10:00:00+00:00",- consecutive_failures=0,- groups=(GroupOutcome(grouping_key="", kind=AlertEventKind.FIRING, new_state="firing", notified=True),),- )+ outcome = _outcome(inputs.configuration_id, AlertEventKind.FIRING, "firing")split = inputs.configuration_id == ADMITTED[0]delivery = AlertDeliveryRequest(source=SourceKind.INSIGHT,@@ -264,12 +295,7 @@ async def test_a_batch_refills_the_slots_its_finished_checks_free(@activity.defn(name="evaluate_platform_insight_check_activity")async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:evaluated.append(inputs.configuration_id)- outcome = PlatformAlertOutcome(- configuration_id=uuid.UUID(inputs.configuration_id),- evaluation_key="slot:2026-09-16T10:00:00+00:00",- consecutive_failures=0,- groups=(GroupOutcome(grouping_key="", kind=AlertEventKind.CHECK, new_state="not_firing", notified=False),),- )+ outcome = _outcome(inputs.configuration_id, AlertEventKind.CHECK, "not_firing")return SourceBatchEvaluation(outcomes=(outcome,), deliveries=())@activity.defn(name=RECORD_OUTCOMES_ACTIVITY)@@ -297,7 +323,64 @@ async def test_a_batch_refills_the_slots_its_finished_checks_free(assert result == len(expected)assert sorted(evaluated) == expected- assert [sorted(str(o.configuration_id) for o in batch.outcomes) for batch in recorded] == [expected]+ assert sorted(str(o.configuration_id) for batch in recorded for o in batch.outcomes) == expected+++async def test_a_slow_check_does_not_hold_back_the_record_of_a_check_that_finished(+ environment: WorkflowEnvironment,+) -> None:+ fast, slow = ADMITTED+ recorded: list[str] = []+ fast_recorded = asyncio.Event()+ slow_saw_the_record: list[bool] = []++ @activity.defn(name="plan_platform_insight_checks_activity")+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:+ return [+ AdmittedInsightCheck(configuration_id=configuration_id, real_time=configuration_id == fast)+ for configuration_id in ADMITTED+ if configuration_id not in inputs.exclude+ ]++ @activity.defn(name="evaluate_platform_insight_check_activity")+ async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:+ if inputs.configuration_id == slow:+ # A batch that records only once every check ended would never set this.+ try:+ await asyncio.wait_for(fast_recorded.wait(), timeout=10)+ slow_saw_the_record.append(True)+ except TimeoutError:+ slow_saw_the_record.append(False)+ return SourceBatchEvaluation(+ outcomes=(_outcome(inputs.configuration_id, AlertEventKind.CHECK, "not_firing"),), deliveries=()+ )++ @activity.defn(name=RECORD_OUTCOMES_ACTIVITY)+ async def record(inputs: SourceOutcomeInputs) -> int:+ recorded.extend(str(outcome.configuration_id) for outcome in inputs.outcomes)+ if fast in recorded:+ fast_recorded.set()+ return len(inputs.outcomes)++ async with Worker(+ environment.client,+ task_queue=QUEUE,+ workflows=[InsightAlertPlatformEvaluateWorkflow],+ activities=[plan, evaluate, record],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ result = await environment.client.execute_workflow(+ InsightAlertPlatformEvaluateWorkflow.run,+ SourceEvaluationInputs(+ source=SourceKind.INSIGHT,+ cutoff="2026-09-16T10:00:00+00:00",+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),+ ),+ id=f"test-{uuid.uuid4()}",+ task_queue=QUEUE,+ )++ assert (result, recorded, slow_saw_the_record) == (2, [fast, slow], [True])@pytest.mark.parametrize(@@ -433,3 +516,58 @@ async def test_a_cancelled_platform_ai_check_keeps_its_place_in_the_cap_until_itawait asyncio.sleep(0.01)finally:release.set()+++async def test_a_record_that_fails_while_a_check_still_runs_records_again_at_the_next_poll(+ environment: WorkflowEnvironment,+) -> None:+ fast, slow = ADMITTED+ attempts: list[list[str]] = []+ fast_attempted = asyncio.Event()++ @activity.defn(name="plan_platform_insight_checks_activity")+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:+ return [+ AdmittedInsightCheck(configuration_id=configuration_id)+ for configuration_id in ADMITTED+ if configuration_id not in inputs.exclude+ ]++ @activity.defn(name="evaluate_platform_insight_check_activity")+ async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:+ if inputs.configuration_id == slow:+ await asyncio.wait_for(fast_attempted.wait(), timeout=10)+ return SourceBatchEvaluation(+ outcomes=(_outcome(inputs.configuration_id, AlertEventKind.CHECK, "not_firing"),), deliveries=()+ )++ @activity.defn(name=RECORD_OUTCOMES_ACTIVITY)+ async def record(inputs: SourceOutcomeInputs) -> int:+ attempts.append(sorted(str(outcome.configuration_id) for outcome in inputs.outcomes))+ if len(attempts) == 1:+ fast_attempted.set()+ raise ApplicationError("database unavailable", non_retryable=True)+ return len(inputs.outcomes)++ async with Worker(+ environment.client,+ task_queue=QUEUE,+ workflows=[InsightAlertPlatformEvaluateWorkflow],+ activities=[plan, evaluate, record],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ result = await environment.client.execute_workflow(+ InsightAlertPlatformEvaluateWorkflow.run,+ SourceEvaluationInputs(+ source=SourceKind.INSIGHT,+ cutoff="2026-09-16T10:00:00+00:00",+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),+ ),+ id=f"test-{uuid.uuid4()}",+ task_queue=QUEUE,+ )++ # The failed record ended neither the batch nor the outcome it held.+ assert result == 2+ assert attempts[0] == [fast]+ assert sorted(configuration_id for attempt in attempts[1:] for configuration_id in attempt) == [fast, slow]diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex 46b763cab000..fb7536793487 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -41,11 +41,14 @@ from products.alerts.backend.platform_source_cycle import (MAX_BREACH_CHARS,MAX_DETECTOR_POINTS,MAX_RATIONALE_CHARS,+ QUERY_ID_PREFIX,ChartRenderPoolFull,evaluate_insight_check,+ fail_insight_check,plan_insight_batch,readmit_retried_check,render_firing_chart,+ stop_insight_queries,)from products.alerts.backend.temporal.platform_evaluate import (AdmittedInsightCheck,@@ -594,6 +597,63 @@ class TestPlatformInsightEvaluation(APIBaseTest):# History shows this reason by its code, as it does for the alert's own checks.assert outcome.failure.source_detail["error_code"] == LLM_DETECTOR_UNAVAILABLE_ERROR_CODE+ @parameterized.expand(+ [+ ("enabled", True, None, (AlertEventKind.ERRORED, "errored", "activity timed out", False, 1)),+ # The raw transport error is not the owner's configuration.+ (+ "judge_unreachable",+ True,+ "LLMDetectorUnavailableError",+ (AlertEventKind.ERRORED, "errored", LLM_DETECTOR_UNAVAILABLE_MESSAGE, False, 1),+ ),+ # As production: switching an alert off mid-check is not a failure of the alert.+ ("switched_off", False, None, None),+ ]+ )+ def test_a_check_whose_every_attempt_failed_records_an_error(+ self,+ _name: str,+ enabled: bool,+ error_type: str | None,+ expected: tuple[AlertEventKind, str, str, bool, int] | None,+ ) -> None:+ configuration = self._copy(self._alert(enabled=enabled), owner=PlatformAlertOwner.PLATFORM)++ evaluation = fail_insight_check(+ self.team.id,+ slot_of(configuration.next_check_at, CUTOFF),+ CUTOFF,+ str(configuration.id),+ error_message="activity timed out",+ error_type=error_type,+ )++ actual = None+ if evaluation is not None:+ (outcome,) = evaluation.outcomes+ assert outcome.failure is not None+ actual = (+ outcome.failure.kind,+ outcome.failure.new_state,+ outcome.error_message,+ outcome.disable,+ len(evaluation.deliveries),+ )+ assert actual == expected++ def test_a_failed_checks_queries_are_stopped_even_when_one_stop_fails(self) -> None:+ with patch(+ "posthog.clickhouse.cancel.cancel_query_on_cluster", side_effect=[TimeoutError("socket timed out"), None]+ ) as cancel:+ stop_insight_queries(self.team.id, "configuration", ("run:1:1", "run:1:2"))++ # A timed-out attempt's thread can still run its query, which nothing else stops.+ assert [call.args for call in cancel.call_args_list] == [+ (self.team.id, f"{QUERY_ID_PREFIX}run:1:1"),+ (self.team.id, f"{QUERY_ID_PREFIX}run:1:2"),+ ]+@override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS=1)def test_a_retry_stops_the_timed_out_query_and_runs_only_inside_the_pool(self) -> None:held = self._copy(self._alert(), id=UUID(int=1))