Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): record platform insight checks as they finish (#123636)

Jimmy Zhub2d2f4b063b9
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 62912e98b7bf..1f7c6928372b 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -277,13 +277,7 @@ def readmit_retried_check(
would run beside it. The retry then moves the slot to its own token, so the pool never counts
the two attempts as one check that either can release.
"""
- # Imported here: at module level it changes how many queries the access-control tests count.
- from posthog.clickhouse.cancel import cancel_query_on_cluster # noqa: PLC0415
-
- try:
- cancel_query_on_cluster(team_id, f"{QUERY_ID_PREFIX}{previous_query_id}")
- except Exception:
- logger.exception("Could not stop a timed-out platform insight check", configuration_id=configuration_id)
+ _stop_query(team_id, configuration_id, previous_query_id)
release_evaluation_slot(configuration_id, held_until=attempt_slot_token(held_until, attempt - 1), key=INFLIGHT_KEY)
token = attempt_slot_token(held_until, attempt)
admitted = admit_evaluation_slots(
@@ -295,6 +289,16 @@ def readmit_retried_check(
return token if admitted else None
+def _stop_query(team_id: int, configuration_id: str, query_id: str) -> None:
+ # Imported here: at module level it changes how many queries the access-control tests count.
+ from posthog.clickhouse.cancel import cancel_query_on_cluster # noqa: PLC0415
+
+ try:
+ cancel_query_on_cluster(team_id, f"{QUERY_ID_PREFIX}{query_id}")
+ except Exception:
+ logger.exception("Could not stop a timed-out platform insight check", configuration_id=configuration_id)
+
+
def legacy_alerts_of(
team_id: int, slot: str, cutoff: datetime, configuration_ids: Collection[str]
) -> dict[str, AlertConfiguration]:
@@ -361,16 +365,65 @@ def evaluate_insight_check(
raise
if outcome is None:
return None
- if checks[0].legacy_configuration_id is not None:
- # Production times each check by its own alert's id, so the copy checks at that minute.
- outcome = replace(outcome, schedule_key=str(checks[0].legacy_configuration_id))
- delivery = _delivery(checks[0], outcome)
- return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,) if delivery is not None else ())
+ return _evaluation(checks[0], outcome)
finally:
if not retrying:
release_evaluation_slot(configuration_id, held_until=held_until, key=INFLIGHT_KEY)
+def fail_insight_check(
+ team_id: int,
+ slot: str,
+ cutoff: datetime,
+ configuration_id: str,
+ *,
+ error_message: str,
+ error_type: str | None = None,
+) -> SourceBatchEvaluation | None:
+ """The errored outcome of a check whose every attempt ended without one, such as a query that ran
+ past its time budget, as production records an errored check when its retries run out.
+
+ `error_type` names the class of the last attempt's error, as production reads it to word a
+ judge that could not be reached.
+ Returns None when the check is no longer due, or its production alert is gone or switched off.
+ Writes no state.
+ """
+ checks = due_checks(team_id, SourceKind.INSIGHT.value, slot, cutoff, configuration_ids=[configuration_id])
+ if not checks:
+ return None
+ alert = _legacy_alert(checks[0])
+ # As production: switching an alert off mid-check is not a failure of the alert.
+ if alert is None or not alert.enabled:
+ return None
+ if error_type == LLMDetectorUnavailableError.__name__:
+ # As production: the raw transport error is not the owner's configuration, so it gets its own wording.
+ error_message = LLM_DETECTOR_UNAVAILABLE_MESSAGE
+ outcome = _verdict(
+ checks[0],
+ _snapshot(checks[0]),
+ CheckInput(threshold_breached=False, error_message=error_message),
+ now=cutoff,
+ skip=SkipReason.QUERY_FAILED,
+ context=_context(alert),
+ )
+ return _evaluation(checks[0], outcome)
+
+
+def stop_insight_queries(team_id: int, configuration_id: str, query_ids: Sequence[str]) -> None:
+ """Stops the queries a failed check's attempts may still run. A timed-out attempt's thread
+ keeps its query going, and nothing else stops it."""
+ for query_id in query_ids:
+ _stop_query(team_id, configuration_id, query_id)
+
+
+def _evaluation(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> SourceBatchEvaluation:
+ if check.legacy_configuration_id is not None:
+ # Production times each check by its own alert's id, so the copy checks at that minute.
+ outcome = replace(outcome, schedule_key=str(check.legacy_configuration_id))
+ delivery = _delivery(check, outcome)
+ return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,) if delivery is not None else ())
+
+
def _delivery(check: PlatformAlertCheckInput, outcome: PlatformAlertOutcome) -> AlertDeliveryRequest | None:
kinds = [group.kind for group in outcome.groups]
if outcome.failure is not None:
diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.py
index e5158cef1f5a..e5f35ec57385 100644
--- a/products/alerts/backend/temporal/platform_evaluate.py
+++ b/products/alerts/backend/temporal/platform_evaluate.py
@@ -26,6 +26,12 @@ with workflow.unsafe.imports_passed_through():
from posthog.schema_enums import AlertCalculationInterval
from posthog.temporal.alerts.retry_policy import alert_timeouts
+ from posthog.temporal.common.errors import (
+ MAX_ERROR_MESSAGE_CHARS,
+ truncate_for_temporal_payload,
+ unwrap_temporal_cause,
+ )
+ from posthog.temporal.common.heartbeat_sync import HeartbeaterSync
from posthog.temporal.common.utils import close_db_connections
from products.alerts.backend.platform_chart_detail import RenderedChart, pending_chart_alert_id, with_chart
@@ -56,6 +62,8 @@ _REAL_TIME_TIMEOUTS = alert_timeouts(AlertCalculationInterval.REAL_TIME)
REAL_TIME_CHECK_START_TO_CLOSE = _REAL_TIME_TIMEOUTS.evaluate_start_to_close
REAL_TIME_CHECK_SCHEDULE_TO_CLOSE = _REAL_TIME_TIMEOUTS.activity_schedule_to_close + CHECK_QUEUE_TOLERANCE
REAL_TIME_CHECK_RETRY_POLICY = _REAL_TIME_TIMEOUTS.evaluate_retry_policy
+# Production's real-time heartbeat, so a check whose worker died is retried inside its budget.
+REAL_TIME_CHECK_HEARTBEAT_TIMEOUT = _REAL_TIME_TIMEOUTS.heartbeat_timeout
# Production's retries for an AI detector check, inside the default budget. A judge that reaches
# no verdict raises for another attempt, and the last attempt records the failure.
LLM_CHECK_RETRY_POLICY = alert_timeouts(None).evaluate_retry_policy
@@ -69,6 +77,11 @@ CHART_RETRY_POLICY = RetryPolicy(
backoff_coefficient=2.0,
maximum_interval=dt.timedelta(seconds=30),
)
+FAIL_START_TO_CLOSE = dt.timedelta(seconds=10)
+FAIL_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=20)
+# Stopping a query can wait on a slow ClickHouse, so it runs apart from the failed check's record
+# and never holds it back. It is best effort, so it gets what the batch budget leaves.
+STOP_QUERIES_START_TO_CLOSE = dt.timedelta(seconds=60)
RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)
RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)
# The pool admits once per round, so a batch that admits only once evaluates at most one pool of
@@ -77,13 +90,16 @@ RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)
ADMISSION_RUN_BUDGET = dt.timedelta(seconds=50)
ADMISSION_POLL_INTERVAL = dt.timedelta(seconds=2)
# What the source needs from its binding's `evaluation_timeout`. The checks run side by side, so
-# the batch costs the admission rounds plus one check's budget, not one per check.
+# the batch costs the admission rounds plus one check's budget, not one per check. The last check
+# can end while an earlier record still runs, so the tail holds two records and their failures,
+# or one record and the stop of a failed check's queries.
EVALUATION_BUDGET = (
PLAN_SCHEDULE_TO_CLOSE
+ ADMISSION_RUN_BUDGET
+ max(CHECK_SCHEDULE_TO_CLOSE, REAL_TIME_CHECK_SCHEDULE_TO_CLOSE)
- + CHART_SCHEDULE_TO_CLOSE
+ + max(CHART_SCHEDULE_TO_CLOSE, FAIL_SCHEDULE_TO_CLOSE)
+ RECORD_SCHEDULE_TO_CLOSE
+ + max(FAIL_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE, STOP_QUERIES_START_TO_CLOSE)
)
_PATCH_READMISSION = "platform-insight-readmission-2026-10"
# Runs started before it recorded every outcome in one activity, and replay must schedule the same.
@@ -95,6 +111,8 @@ _PATCH_CHECK_BUDGETS = "platform-insight-check-budgets-2026-10"
_PATCH_AI_DETECTOR_QUEUE = "platform-insight-ai-detector-queue-2026-10"
# Runs started before it rendered charts inside the check, and replay must not schedule a render.
_PATCH_CHART_AFTER_CHECK = "platform-insight-chart-after-check-2026-10"
+# Runs started before it recorded every outcome once the whole batch ended, and failed checks left none.
+_PATCH_RECORD_AS_FINISHED = "platform-insight-record-as-finished-2026-10"
@frozen
@@ -131,6 +149,22 @@ class InsightCheckInputs:
owned_by_platform: bool = False
+@frozen
+class FailedInsightCheckInputs:
+ evaluation: SourceEvaluationInputs
+ configuration_id: str
+ error_message: str
+ error_type: str | None = None
+
+
+@frozen
+class StopInsightQueriesInputs:
+ team_id: int
+ configuration_id: str
+ # The queries a failed check's attempts ran under, any of which may still run.
+ query_ids: tuple[str, ...]
+
+
@activity.defn
@close_db_connections
def plan_platform_insight_batch_activity(inputs: InsightBatchPlanInputs) -> list[str]:
@@ -232,8 +266,37 @@ def _evaluate_check(
@activity.defn
@close_db_connections
def evaluate_platform_insight_check_activity(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
- """Sync, so the thread this blocks is a slot Temporal is accounting for."""
- return _evaluate_check(inputs, activity.info(), final_attempt=True)
+ """Sync, so the thread this blocks is a slot Temporal is accounting for. Heartbeats only when the
+ workflow set a heartbeat timeout."""
+ with HeartbeaterSync():
+ return _evaluate_check(inputs, activity.info(), final_attempt=True)
+
+
+@close_db_connections
+def fail_platform_insight_check_activity(inputs: FailedInsightCheckInputs) -> SourceBatchEvaluation | None:
+ """The errored outcome of a check whose every attempt ended without one."""
+ from products.alerts.backend.platform_source_cycle import fail_insight_check # noqa: PLC0415
+
+ key = inputs.evaluation.batch_key
+ return fail_insight_check(
+ key.team_id,
+ key.slot,
+ dt.datetime.fromisoformat(inputs.evaluation.cutoff),
+ inputs.configuration_id,
+ error_message=inputs.error_message,
+ error_type=inputs.error_type,
+ )
+
+
+@close_db_connections
+def stop_platform_insight_queries_activity(inputs: StopInsightQueriesInputs) -> None:
+ """Stops the queries a failed check's attempts may still run, because a timed-out attempt's
+ thread keeps its query going."""
+ from products.alerts.backend.platform_source_cycle import stop_insight_queries # noqa: PLC0415
+
+ stop_insight_queries(inputs.team_id, inputs.configuration_id, inputs.query_ids)
@activity.defn
@@ -303,6 +366,8 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
# From the workflow's clock, so a retried plan reuses the expiry and gets its slots back. A
# check releases its slot only while the slot still carries this expiry.
expires_at = (workflow.now() + source_evaluation_timeout(SourceKind.INSIGHT)).timestamp()
+ if workflow.patched(_PATCH_READMISSION) and workflow.patched(_PATCH_RECORD_AS_FINISHED):
+ return await self._evaluate_and_record_in_rounds(inputs, expires_at)
if workflow.patched(_PATCH_READMISSION):
settled = await self._evaluate_in_rounds(inputs, expires_at)
else:
@@ -329,8 +394,7 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
evaluations = await self._with_charts(inputs, evaluations)
# A check that ran on new code under a run that started before the patch leaves its marker.
evaluations = [with_chart(evaluation, None) for evaluation in evaluations]
- outcomes: tuple[PlatformAlertOutcome, ...] = tuple(o for e in evaluations for o in e.outcomes)
- deliveries: tuple[AlertDeliveryRequest, ...] = tuple(d for e in evaluations for d in e.deliveries)
+ outcomes = tuple(outcome for evaluation in evaluations for outcome in evaluation.outcomes)
if len(outcomes) < len(settled):
workflow.logger.warning(
"%d of %d platform insight checks reached no outcome", len(settled) - len(outcomes), len(settled)
@@ -338,21 +402,7 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
safe_record(increment_checks_unfinished, SourceKind.INSIGHT.value, len(settled) - len(outcomes))
if not outcomes:
return 0
-
- # Each check's result fits a payload on its own, but the batch's together may not. A chunk's
- # deliveries start after its own record, so a later chunk that fails cannot strand them.
- chunks = outcome_chunks(outcomes) if workflow.patched(_PATCH_CHUNKED_RECORD) else [outcomes]
- for chunk in chunks:
- await workflow.execute_activity(
- RECORD_OUTCOMES_ACTIVITY,
- SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=chunk),
- start_to_close_timeout=RECORD_START_TO_CLOSE,
- schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,
- retry_policy=RetryPolicy(maximum_attempts=3),
- )
- # Started only after the record, because a delivery reads the history row the record writes.
- recorded = {str(outcome.configuration_id) for outcome in chunk}
- await start_deliveries([delivery for delivery in deliveries if delivery.configuration_id in recorded])
+ await self._record(inputs, evaluations, chunked=workflow.patched(_PATCH_CHUNKED_RECORD))
return len(outcomes)
async def _with_charts(
@@ -427,10 +477,11 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
check_inputs,
start_to_close_timeout=REAL_TIME_CHECK_START_TO_CLOSE,
schedule_to_close_timeout=REAL_TIME_CHECK_SCHEDULE_TO_CLOSE,
+ heartbeat_timeout=REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,
retry_policy=REAL_TIME_CHECK_RETRY_POLICY,
)
- # One attempt each. A check that does not finish leaves no outcome, so it keeps its due time
- # and a later tick evaluates it again, the way a cohort the logs budget left out behaves.
+ # One attempt each. In a run that records as checks finish, a check that does not finish is
+ # recorded as errored. In an older run it leaves no outcome and keeps its due time.
return await workflow.execute_activity(
evaluate_platform_insight_check_activity,
check_inputs,
@@ -482,6 +533,226 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
)
return await asyncio.gather(*started.values(), return_exceptions=True)
+ async def _evaluate_and_record_in_rounds(self, inputs: SourceEvaluationInputs, expires_at: float) -> int:
+ """`_evaluate_in_rounds`, but the checks that ended are recorded and delivered at each poll, so
+ a slow check in the batch never holds back another's state, schedule or delivery. A check
+ whose every attempt failed is recorded as errored. Returns how many outcomes it recorded.
+
+ One record per poll rather than per check, because each record is a transaction and a
+ ClickHouse insert of its own. A record that fails keeps its outcomes for the next poll, so a
+ passing storage failure does not cost the checks still running.
+ """
+ started: dict[str, AdmittedInsightCheck] = {}
+ unrecorded: dict[str, asyncio.Task[SourceBatchEvaluation | None]] = {}
+ waiting: list[SourceBatchEvaluation] = []
+ stopping: list[asyncio.Task[None]] = []
+ recorded = 0
+ admitting = True
+ deadline = workflow.now() + ADMISSION_RUN_BUDGET
+ next_record_at = workflow.now()
+ while True:
+ added = False
+ if admitting:
+ try:
+ admitted = await self._plan_round(inputs, expires_at, exclude=tuple(started), budgeted=True)
+ except ActivityError:
+ if not started:
+ raise
+ workflow.logger.warning("Stopped admitting platform insight checks after a failed plan")
+ admitting = False
+ else:
+ added = bool(admitted)
+ for check in admitted:
+ started[check.configuration_id] = check
+ unrecorded[check.configuration_id] = asyncio.create_task(
+ self._check_and_chart(inputs, check, expires_at)
+ )
+ finished = {configuration_id: task for configuration_id, task in unrecorded.items() if task.done()}
+ if (finished or waiting) and (
+ workflow.now() >= next_record_at or all(task.done() for task in unrecorded.values())
+ ):
+ next_record_at = workflow.now() + ADMISSION_POLL_INTERVAL
+ for configuration_id in finished:
+ del unrecorded[configuration_id]
+ waiting.extend(
+ await self._evaluations_of(
+ inputs,
+ {started[configuration_id]: task for configuration_id, task in finished.items()},
+ stopping,
+ )
+ )
+ recorded += await self._record_waiting(inputs, waiting, last_chance=False)
+ if workflow.now() >= deadline:
+ admitting = False
+ if not unrecorded:
+ # The checks a round added can end while earlier ones record, and free slots for more.
+ if admitting and added:
+ continue
+ recorded += await self._record_waiting(inputs, waiting, last_chance=True)
+ await asyncio.gather(*stopping, return_exceptions=True)
+ return recorded
+ running = [task for task in unrecorded.values() if not task.done()]
+ if not running:
+ continue
+ # While admitting, wake at each poll to refill the pool. After that, wake when a check
+ # ends, or when the checks that already ended are due their record. An idle timer would
+ # add history events for the whole of a slow check.
+ if admitting:
+ timeout: float | None = ADMISSION_POLL_INTERVAL.total_seconds()
+ elif len(running) < len(unrecorded) or waiting:
+ timeout = max((next_record_at - workflow.now()).total_seconds(), 0)
+ else:
+ timeout = None
+ await asyncio.wait(running, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
+
+ async def _check_and_chart(
+ self, inputs: SourceEvaluationInputs, check: AdmittedInsightCheck, expires_at: float
+ ) -> SourceBatchEvaluation | None:
+ """The check, then its firing's chart, so a slow render holds back only its own check's record.
+
+ No patch guards the render, because a run on this path started on code that renders charts."""
+ evaluation = await self._check(inputs, check, expires_at)
+ if evaluation is None:
+ return None
+ return (await self._with_charts(inputs, [evaluation]))[0]
+
+ async def _evaluations_of(
+ self,
+ inputs: SourceEvaluationInputs,
+ finished: dict[AdmittedInsightCheck, "asyncio.Task[SourceBatchEvaluation | None]"],
+ stopping: list["asyncio.Task[None]"],
+ ) -> list[SourceBatchEvaluation]:
+ """What the finished checks decided, with an errored outcome for each check that failed."""
+ evaluations: list[SourceBatchEvaluation] = []
+ failures: dict[AdmittedInsightCheck, ActivityError] = {}
+ for check, task in finished.items():
+ error = task.exception()
+ if isinstance(error, ActivityError):
+ failures[check] = error
+ elif error is not None:
+ # Only a check's own failure is a missing outcome. A bug in this workflow ends the run.
+ raise error
+ elif (evaluation := task.result()) is not None:
+ evaluations.append(evaluation)
+ for check, error in failures.items():
+ stopping.append(asyncio.create_task(self._stop_queries(inputs, check, error)))
+ failed = await asyncio.gather(*(self._fail(inputs, check, error) for check, error in failures.items()))
+ evaluations.extend(evaluation for evaluation in failed if evaluation is not None)
+ unfinished = len(finished) - sum(len(evaluation.outcomes) for evaluation in evaluations)
+ if unfinished > 0:
+ safe_record(increment_checks_unfinished, SourceKind.INSIGHT.value, unfinished)
+ return evaluations
+
+ async def _fail(
+ self, inputs: SourceEvaluationInputs, check: AdmittedInsightCheck, error: ActivityError
+ ) -> SourceBatchEvaluation | None:
+ # The reason the owner reads, unwrapped from Temporal's failure and bounded for the payload.
+ cause = unwrap_temporal_cause(error)
+ message = cause.message if cause is not None else str(error.cause or error)
+ try:
+ return await workflow.execute_activity(
+ fail_platform_insight_check_activity,
+ FailedInsightCheckInputs(
+ evaluation=inputs,
+ configuration_id=check.configuration_id,
+ error_message=truncate_for_temporal_payload(message, MAX_ERROR_MESSAGE_CHARS),
+ # Temporal rebuilds a remote error as an ApplicationError carrying the class name on `type`.
+ error_type=cause.type if cause is not None else None,
+ ),
+ start_to_close_timeout=FAIL_START_TO_CLOSE,
+ schedule_to_close_timeout=FAIL_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ except ActivityError:
+ # The check stays due and the next tick evaluates it again, so the batch's other
+ # outcomes still record.
+ workflow.logger.warning("Could not record a failed platform insight check as errored")
+ return None
+
+ async def _stop_queries(
+ self, inputs: SourceEvaluationInputs, check: AdmittedInsightCheck, error: ActivityError
+ ) -> None:
+ # Each attempt queries under `{run}:{activity}:{attempt}`, as `_evaluate_check` names it.
+ evaluation_id = f"{workflow.info().run_id}:{error.activity_id}"
+ try:
+ await workflow.execute_activity(
+ stop_platform_insight_queries_activity,
+ StopInsightQueriesInputs(
+ team_id=inputs.batch_key.team_id,
+ configuration_id=check.configuration_id,
+ query_ids=tuple(f"{evaluation_id}:{attempt}" for attempt in range(1, _maximum_attempts(check) + 1)),
+ ),
+ start_to_close_timeout=STOP_QUERIES_START_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ except ActivityError:
+ workflow.logger.warning("Could not stop the queries of a failed platform insight check")
+
+ async def _record_waiting(
+ self, inputs: SourceEvaluationInputs, waiting: list[SourceBatchEvaluation], *, last_chance: bool
+ ) -> int:
+ """Records the waiting outcomes chunk by chunk, and takes each chunk that recorded out of
+ `waiting`. A chunk that fails stays for the next poll, unless this is the last chance.
+ Returns how many outcomes it recorded."""
+ outcomes = tuple(outcome for evaluation in waiting for outcome in evaluation.outcomes)
+ deliveries = tuple(delivery for evaluation in waiting for delivery in evaluation.deliveries)
+ recorded = 0
+ for chunk in outcome_chunks(outcomes) if outcomes else []:
+ try:
+ await self._record_chunk(inputs, chunk, deliveries)
+ except ActivityError:
+ if last_chance:
+ raise
+ workflow.logger.warning("Will record platform insight outcomes again at the next poll")
+ break
+ done = {str(outcome.configuration_id) for outcome in chunk}
+ waiting[:] = [
+ evaluation
+ for evaluation in waiting
+ if not any(str(outcome.configuration_id) in done for outcome in evaluation.outcomes)
+ ]
+ recorded += len(chunk)
+ return recorded
+
+ async def _record(
+ self, inputs: SourceEvaluationInputs, evaluations: list[SourceBatchEvaluation], *, chunked: bool
+ ) -> int:
+ """Records the outcomes, then starts their deliveries. Returns how many outcomes it recorded."""
+ outcomes = tuple(outcome for evaluation in evaluations for outcome in evaluation.outcomes)
+ if not outcomes:
+ return 0
+ deliveries = tuple(delivery for evaluation in evaluations for delivery in evaluation.deliveries)
+ for chunk in outcome_chunks(outcomes) if chunked else [outcomes]:
+ await self._record_chunk(inputs, chunk, deliveries)
+ return len(outcomes)
+
+ async def _record_chunk(
+ self,
+ inputs: SourceEvaluationInputs,
+ chunk: tuple[PlatformAlertOutcome, ...],
+ deliveries: tuple[AlertDeliveryRequest, ...],
+ ) -> None:
+ # Each check's result fits a payload on its own, but the batch's together may not. A chunk's
+ # deliveries start after its own record, so a later chunk that fails cannot strand them.
+ await workflow.execute_activity(
+ RECORD_OUTCOMES_ACTIVITY,
+ SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=chunk),
+ start_to_close_timeout=RECORD_START_TO_CLOSE,
+ schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ # Started only after the record, because a delivery reads the history row the record writes.
+ recorded = {str(outcome.configuration_id) for outcome in chunk}
+ await start_deliveries([delivery for delivery in deliveries if delivery.configuration_id in recorded])
+
+
+def _maximum_attempts(check: AdmittedInsightCheck) -> int:
+ if check.uses_llm_detector:
+ return LLM_CHECK_RETRY_POLICY.maximum_attempts
+ if check.real_time:
+ return REAL_TIME_CHECK_RETRY_POLICY.maximum_attempts
+ return 1
+
PLATFORM_EVALUATION_WORKFLOWS = [InsightAlertPlatformEvaluateWorkflow]
# Registered on the AI queue's worker, which holds the model credentials.
@@ -492,4 +763,6 @@ PLATFORM_EVALUATION_ACTIVITIES = [
plan_platform_insight_checks_activity,
evaluate_platform_insight_check_activity,
render_platform_insight_chart_activity,
+ fail_platform_insight_check_activity,
+ stop_platform_insight_queries_activity,
]
diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.py
index ae99755c490e..bc8b91fc40b3 100644
--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py
+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py
@@ -24,11 +24,14 @@ from products.alerts.backend.platform_chart_detail import (
from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS
from products.alerts.backend.temporal.platform_evaluate import (
EVALUATION_BUDGET,
+ REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,
AdmittedInsightCheck,
+ FailedInsightCheckInputs,
InsightAlertPlatformEvaluateWorkflow,
InsightBatchPlanInputs,
InsightChartInputs,
InsightCheckInputs,
+ StopInsightQueriesInputs,
evaluate_platform_insight_llm_check_activity,
)
from products.alerts_platform.backend.facade.contracts import (
@@ -56,8 +59,19 @@ async def environment() -> AsyncIterator[WorkflowEnvironment]:
yield env
+def _outcome(configuration_id: str, kind: AlertEventKind, state: str) -> PlatformAlertOutcome:
+ return PlatformAlertOutcome(
+ configuration_id=uuid.UUID(configuration_id),
+ evaluation_key="slot:2026-09-16T10:00:00+00:00",
+ consecutive_failures=0,
+ groups=(GroupOutcome(grouping_key="", kind=kind, new_state=state, notified=kind != AlertEventKind.CHECK),),
+ )
+
+
async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(environment: WorkflowEnvironment) -> None:
recorded: list[SourceOutcomeInputs] = []
+ failed: list[FailedInsightCheckInputs] = []
+ stopped: list[StopInsightQueriesInputs] = []
# Only the real-time and AI checks have a retry, so the same first failure loses the other one.
@activity.defn(name="plan_platform_insight_checks_activity")
@@ -85,12 +99,7 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
return _fired(inputs)
def _fired(inputs: InsightCheckInputs) -> SourceBatchEvaluation:
- outcome = PlatformAlertOutcome(
- configuration_id=uuid.UUID(inputs.configuration_id),
- evaluation_key="slot:2026-09-16T10:00:00+00:00",
- consecutive_failures=0,
- groups=(GroupOutcome(grouping_key="", kind=AlertEventKind.FIRING, new_state="firing", notified=True),),
- )
+ outcome = _outcome(inputs.configuration_id, AlertEventKind.FIRING, "firing")
delivery = AlertDeliveryRequest(
source=SourceKind.INSIGHT,
team_id=1,
@@ -101,6 +110,17 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
)
return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,))
+ @activity.defn(name="fail_platform_insight_check_activity")
+ async def fail(inputs: FailedInsightCheckInputs) -> SourceBatchEvaluation | None:
+ failed.append(inputs)
+ return SourceBatchEvaluation(
+ outcomes=(_outcome(inputs.configuration_id, AlertEventKind.ERRORED, "errored"),), deliveries=()
+ )
+
+ @activity.defn(name="stop_platform_insight_queries_activity")
+ async def stop(inputs: StopInsightQueriesInputs) -> None:
+ stopped.append(inputs)
+
@activity.defn(name=RECORD_OUTCOMES_ACTIVITY)
async def record(inputs: SourceOutcomeInputs) -> int:
recorded.append(inputs)
@@ -111,7 +131,7 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
environment.client,
task_queue=QUEUE,
workflows=[InsightAlertPlatformEvaluateWorkflow],
- activities=[plan, evaluate, record],
+ activities=[plan, evaluate, fail, stop, record],
workflow_runner=UnsandboxedWorkflowRunner(),
),
Worker(environment.client, task_queue=settings.MAX_AI_TASK_QUEUE, activities=[evaluate_on_the_ai_queue]),
@@ -130,8 +150,24 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
result = await handle.result()
history = await handle.fetch_history()
- assert result == 2
- assert [sorted(str(o.configuration_id) for o in batch.outcomes) for batch in recorded] == [[ADMITTED[1], AI_CHECK]]
+ assert result == 3
+ # The check with no retry left no outcome of its own, so it is recorded as errored, with the reason.
+ assert [(inputs.configuration_id, inputs.error_message) for inputs in failed] == [(ADMITTED[0], "worker lost")]
+ # Its one attempt's query may still run, which nothing else stops.
+ assert [(inputs.configuration_id, len(inputs.query_ids)) for inputs in stopped] == [(ADMITTED[0], 1)]
+ assert sorted(
+ (str(o.configuration_id), o.groups[0].new_state) for batch in recorded for o in batch.outcomes
+ ) == sorted([(ADMITTED[0], "errored"), (ADMITTED[1], "firing"), (AI_CHECK, "firing")])
+ real_time_heartbeats = [
+ event.activity_task_scheduled_event_attributes.heartbeat_timeout.ToTimedelta()
+ for event in history.events
+ if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED
+ and event.activity_task_scheduled_event_attributes.activity_type.name
+ == "evaluate_platform_insight_check_activity"
+ and event.activity_task_scheduled_event_attributes.heartbeat_timeout.ToTimedelta()
+ ]
+ # Only the real-time check heartbeats, so a dead worker's check is retried inside its budget.
+ assert real_time_heartbeats == [REAL_TIME_CHECK_HEARTBEAT_TIMEOUT]
children = [
event.child_workflow_execution_started_event_attributes
for event in history.events
@@ -160,12 +196,7 @@ async def test_a_batch_over_the_payload_budget_records_in_chunks_and_starts_one_
@activity.defn(name="evaluate_platform_insight_check_activity")
async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
- outcome = PlatformAlertOutcome(
- configuration_id=uuid.UUID(inputs.configuration_id),
- evaluation_key="slot:2026-09-16T10:00:00+00:00",
- consecutive_failures=0,
- groups=(GroupOutcome(grouping_key="", kind=AlertEventKind.FIRING, new_state="firing", notified=True),),
- )
+ outcome = _outcome(inputs.configuration_id, AlertEventKind.FIRING, "firing")
split = inputs.configuration_id == ADMITTED[0]
delivery = AlertDeliveryRequest(
source=SourceKind.INSIGHT,
@@ -264,12 +295,7 @@ async def test_a_batch_refills_the_slots_its_finished_checks_free(
@activity.defn(name="evaluate_platform_insight_check_activity")
async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
evaluated.append(inputs.configuration_id)
- outcome = PlatformAlertOutcome(
- configuration_id=uuid.UUID(inputs.configuration_id),
- evaluation_key="slot:2026-09-16T10:00:00+00:00",
- consecutive_failures=0,
- groups=(GroupOutcome(grouping_key="", kind=AlertEventKind.CHECK, new_state="not_firing", notified=False),),
- )
+ outcome = _outcome(inputs.configuration_id, AlertEventKind.CHECK, "not_firing")
return SourceBatchEvaluation(outcomes=(outcome,), deliveries=())
@activity.defn(name=RECORD_OUTCOMES_ACTIVITY)
@@ -297,7 +323,64 @@ async def test_a_batch_refills_the_slots_its_finished_checks_free(
assert result == len(expected)
assert sorted(evaluated) == expected
- assert [sorted(str(o.configuration_id) for o in batch.outcomes) for batch in recorded] == [expected]
+ assert sorted(str(o.configuration_id) for batch in recorded for o in batch.outcomes) == expected
+
+
+async def test_a_slow_check_does_not_hold_back_the_record_of_a_check_that_finished(
+ environment: WorkflowEnvironment,
+) -> None:
+ fast, slow = ADMITTED
+ recorded: list[str] = []
+ fast_recorded = asyncio.Event()
+ slow_saw_the_record: list[bool] = []
+
+ @activity.defn(name="plan_platform_insight_checks_activity")
+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:
+ return [
+ AdmittedInsightCheck(configuration_id=configuration_id, real_time=configuration_id == fast)
+ for configuration_id in ADMITTED
+ if configuration_id not in inputs.exclude
+ ]
+
+ @activity.defn(name="evaluate_platform_insight_check_activity")
+ async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
+ if inputs.configuration_id == slow:
+ # A batch that records only once every check ended would never set this.
+ try:
+ await asyncio.wait_for(fast_recorded.wait(), timeout=10)
+ slow_saw_the_record.append(True)
+ except TimeoutError:
+ slow_saw_the_record.append(False)
+ return SourceBatchEvaluation(
+ outcomes=(_outcome(inputs.configuration_id, AlertEventKind.CHECK, "not_firing"),), deliveries=()
+ )
+
+ @activity.defn(name=RECORD_OUTCOMES_ACTIVITY)
+ async def record(inputs: SourceOutcomeInputs) -> int:
+ recorded.extend(str(outcome.configuration_id) for outcome in inputs.outcomes)
+ if fast in recorded:
+ fast_recorded.set()
+ return len(inputs.outcomes)
+
+ async with Worker(
+ environment.client,
+ task_queue=QUEUE,
+ workflows=[InsightAlertPlatformEvaluateWorkflow],
+ activities=[plan, evaluate, record],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ result = await environment.client.execute_workflow(
+ InsightAlertPlatformEvaluateWorkflow.run,
+ SourceEvaluationInputs(
+ source=SourceKind.INSIGHT,
+ cutoff="2026-09-16T10:00:00+00:00",
+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),
+ ),
+ id=f"test-{uuid.uuid4()}",
+ task_queue=QUEUE,
+ )
+
+ assert (result, recorded, slow_saw_the_record) == (2, [fast, slow], [True])
@pytest.mark.parametrize(
@@ -433,3 +516,58 @@ async def test_a_cancelled_platform_ai_check_keeps_its_place_in_the_cap_until_it
await asyncio.sleep(0.01)
finally:
release.set()
+
+
+async def test_a_record_that_fails_while_a_check_still_runs_records_again_at_the_next_poll(
+ environment: WorkflowEnvironment,
+) -> None:
+ fast, slow = ADMITTED
+ attempts: list[list[str]] = []
+ fast_attempted = asyncio.Event()
+
+ @activity.defn(name="plan_platform_insight_checks_activity")
+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:
+ return [
+ AdmittedInsightCheck(configuration_id=configuration_id)
+ for configuration_id in ADMITTED
+ if configuration_id not in inputs.exclude
+ ]
+
+ @activity.defn(name="evaluate_platform_insight_check_activity")
+ async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
+ if inputs.configuration_id == slow:
+ await asyncio.wait_for(fast_attempted.wait(), timeout=10)
+ return SourceBatchEvaluation(
+ outcomes=(_outcome(inputs.configuration_id, AlertEventKind.CHECK, "not_firing"),), deliveries=()
+ )
+
+ @activity.defn(name=RECORD_OUTCOMES_ACTIVITY)
+ async def record(inputs: SourceOutcomeInputs) -> int:
+ attempts.append(sorted(str(outcome.configuration_id) for outcome in inputs.outcomes))
+ if len(attempts) == 1:
+ fast_attempted.set()
+ raise ApplicationError("database unavailable", non_retryable=True)
+ return len(inputs.outcomes)
+
+ async with Worker(
+ environment.client,
+ task_queue=QUEUE,
+ workflows=[InsightAlertPlatformEvaluateWorkflow],
+ activities=[plan, evaluate, record],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ result = await environment.client.execute_workflow(
+ InsightAlertPlatformEvaluateWorkflow.run,
+ SourceEvaluationInputs(
+ source=SourceKind.INSIGHT,
+ cutoff="2026-09-16T10:00:00+00:00",
+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),
+ ),
+ id=f"test-{uuid.uuid4()}",
+ task_queue=QUEUE,
+ )
+
+ # The failed record ended neither the batch nor the outcome it held.
+ assert result == 2
+ assert attempts[0] == [fast]
+ assert sorted(configuration_id for attempt in attempts[1:] for configuration_id in attempt) == [fast, slow]
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index 46b763cab000..fb7536793487 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -41,11 +41,14 @@ from products.alerts.backend.platform_source_cycle import (
MAX_BREACH_CHARS,
MAX_DETECTOR_POINTS,
MAX_RATIONALE_CHARS,
+ QUERY_ID_PREFIX,
ChartRenderPoolFull,
evaluate_insight_check,
+ fail_insight_check,
plan_insight_batch,
readmit_retried_check,
render_firing_chart,
+ stop_insight_queries,
)
from products.alerts.backend.temporal.platform_evaluate import (
AdmittedInsightCheck,
@@ -594,6 +597,63 @@ class TestPlatformInsightEvaluation(APIBaseTest):
# History shows this reason by its code, as it does for the alert's own checks.
assert outcome.failure.source_detail["error_code"] == LLM_DETECTOR_UNAVAILABLE_ERROR_CODE
+ @parameterized.expand(
+ [
+ ("enabled", True, None, (AlertEventKind.ERRORED, "errored", "activity timed out", False, 1)),
+ # The raw transport error is not the owner's configuration.
+ (
+ "judge_unreachable",
+ True,
+ "LLMDetectorUnavailableError",
+ (AlertEventKind.ERRORED, "errored", LLM_DETECTOR_UNAVAILABLE_MESSAGE, False, 1),
+ ),
+ # As production: switching an alert off mid-check is not a failure of the alert.
+ ("switched_off", False, None, None),
+ ]
+ )
+ def test_a_check_whose_every_attempt_failed_records_an_error(
+ self,
+ _name: str,
+ enabled: bool,
+ error_type: str | None,
+ expected: tuple[AlertEventKind, str, str, bool, int] | None,
+ ) -> None:
+ configuration = self._copy(self._alert(enabled=enabled), owner=PlatformAlertOwner.PLATFORM)
+
+ evaluation = fail_insight_check(
+ self.team.id,
+ slot_of(configuration.next_check_at, CUTOFF),
+ CUTOFF,
+ str(configuration.id),
+ error_message="activity timed out",
+ error_type=error_type,
+ )
+
+ actual = None
+ if evaluation is not None:
+ (outcome,) = evaluation.outcomes
+ assert outcome.failure is not None
+ actual = (
+ outcome.failure.kind,
+ outcome.failure.new_state,
+ outcome.error_message,
+ outcome.disable,
+ len(evaluation.deliveries),
+ )
+ assert actual == expected
+
+ def test_a_failed_checks_queries_are_stopped_even_when_one_stop_fails(self) -> None:
+ with patch(
+ "posthog.clickhouse.cancel.cancel_query_on_cluster", side_effect=[TimeoutError("socket timed out"), None]
+ ) as cancel:
+ stop_insight_queries(self.team.id, "configuration", ("run:1:1", "run:1:2"))
+
+ # A timed-out attempt's thread can still run its query, which nothing else stops.
+ assert [call.args for call in cancel.call_args_list] == [
+ (self.team.id, f"{QUERY_ID_PREFIX}run:1:1"),
+ (self.team.id, f"{QUERY_ID_PREFIX}run:1:2"),
+ ]
+
@override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS=1)
def test_a_retry_stops_the_timed_out_query_and_runs_only_inside_the_pool(self) -> None:
held = self._copy(self._alert(), id=UUID(int=1))