:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): evaluate real-time insight alerts on the platform (#119948)
Jimmy Zhu88e2c4045cf8
diff --git a/docs/internal/alerts-platform-temporal.md b/docs/internal/alerts-platform-temporal.mdindex fed87960a858..b543e57d09b3 100644--- a/docs/internal/alerts-platform-temporal.md+++ b/docs/internal/alerts-platform-temporal.md@@ -508,7 +508,9 @@ Insight alerts are copied the same way:python manage.py backfill_platform_insight_alert_configurations```-It copies threshold and statistical detector alerts on a 15-minute or slower cadence only, and skips AI detector alerts and the real-time cadence.+It copies threshold and statistical detector alerts on every cadence, and skips AI detector alerts.+A real-time check gets production's real-time budget: a 7-minute evaluation with one retry inside it.+Every other check gets one 12-minute attempt.Run it only after the evaluation worker's chart sets `CLICKHOUSE_ALERTS_PLATFORM_INSIGHT_USER` and its token file.Insight checks tag their queries with `ClickHouseUser.ALERTS_PLATFORM_INSIGHT`, a user of their own, so the parallel run never takes from the per-user budget of the user that production insight alerts query as.Without that env the tag resolves to the worker's default user, which other workloads on the same servers already push against its concurrent query limit.diff --git a/products/alerts/backend/platform_alert_backfill.py b/products/alerts/backend/platform_alert_backfill.pyindex 0ec56b6a7db7..beb8bf345678 100644--- a/products/alerts/backend/platform_alert_backfill.py+++ b/products/alerts/backend/platform_alert_backfill.py@@ -20,12 +20,14 @@ from products.alerts.backend.platform_source_cycle import is_evaluated_on_the_plfrom products.alerts_platform.backend.facade.api import disable_configurations, upsert_configurationfrom products.alerts_platform.backend.facade.contracts import SOURCE_CONDITION_KEY, PlatformAlertUpsert, SourceKindfrom products.alerts_platform.backend.facade.enums import PlatformAlertConfigurationRecurrenceUnit as RecurrenceUnit+from products.alerts_platform.backend.facade.scheduling import REAL_TIME_CADENCE_MINUTESlogger = structlog.get_logger(__name__)-# Minutes for the 15-minute and hourly cadences, a calendar unit for the rest. A calendar unit reads its cadence+# Minutes for the real-time, 15-minute and hourly cadences, a calendar unit for the rest. A calendar unit reads its cadence# from the unit, so its minutes only feed the shard offset, which an anchored unit ignores._RECURRENCE: dict[str, tuple[int, str | None]] = {+ AlertCalculationInterval.REAL_TIME: (REAL_TIME_CADENCE_MINUTES, None),AlertCalculationInterval.EVERY_15_MINUTES: (15, None),AlertCalculationInterval.HOURLY: (60, None),AlertCalculationInterval.DAILY: (60 * 24, RecurrenceUnit.DAY),diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex b20409737d5c..54b70f55c54e 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -97,10 +97,9 @@ CAPACITY_REJECTED = "ClickHouse refused the query for capacity"SLOT_KEY_PREFIX = "slot:"-# Real-time is the most expensive cadence and the one production holds to a tighter budget, so the-# parallel run leaves it out until it agrees with production elsewhere.EVALUATED_INTERVALS = frozenset({+ AlertCalculationInterval.REAL_TIME,AlertCalculationInterval.EVERY_15_MINUTES,AlertCalculationInterval.HOURLY,AlertCalculationInterval.DAILY,@@ -125,8 +124,8 @@ _EVENT_IDS_BY_KIND: Final[dict[str, str]] = {def is_evaluated_on_the_platform(alert: AlertConfiguration) -> bool:- """Threshold and statistical detector alerts on a 15-minute or slower cadence. The AI detector- makes a charged model call, so evaluating it in parallel costs twice."""+ """Threshold and statistical detector alerts on every cadence. The AI detector makes a charged+ model call, so evaluating it in parallel costs twice."""return not is_llm_detector_config(alert.detector_config) and alert.calculation_interval in EVALUATED_INTERVALS@@ -173,6 +172,59 @@ def plan_insight_batch(return tuple(admitted)+def attempt_slot_token(held_until: float, attempt: int) -> float:+ """The expiry an attempt holds its slot under. Each attempt gets its own, so a timed-out attempt+ whose thread finishes late cannot release the slot its retry holds. The offset is negligible+ against the expiry itself."""+ return held_until + (attempt - 1) / 1000+++def readmit_retried_check(+ team_id: int, configuration_id: str, *, previous_query_id: str, held_until: float, attempt: int+) -> float | None:+ """Prepares a retried check, and returns the slot token it holds, or None when the pool is full.++ A timed-out attempt's thread can still run its query, so the query is killed first, or the retry+ would run beside it. The retry then moves the slot to its own token, so the pool never counts+ the two attempts as one check that either can release.+ """+ # Imported here: at module level it changes how many queries the access-control tests count.+ from posthog.clickhouse.cancel import cancel_query_on_cluster # noqa: PLC0415++ try:+ cancel_query_on_cluster(team_id, f"{QUERY_ID_PREFIX}{previous_query_id}")+ except Exception:+ logger.exception("Could not stop a timed-out platform insight check", configuration_id=configuration_id)+ release_evaluation_slot(configuration_id, held_until=attempt_slot_token(held_until, attempt - 1), key=INFLIGHT_KEY)+ token = attempt_slot_token(held_until, attempt)+ admitted = admit_evaluation_slots(+ [configuration_id],+ limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS,+ expires_at=token,+ key=INFLIGHT_KEY,+ )+ return token if admitted else None+++def legacy_alerts_of(+ team_id: int, slot: str, cutoff: datetime, configuration_ids: Collection[str]+) -> dict[str, AlertConfiguration]:+ """The production alert behind each due copy that still has one, keyed by the copy's id."""+ if not configuration_ids:+ return {}+ legacy_ids = {+ check.legacy_configuration_id: str(check.id)+ for check in due_checks(+ team_id, SourceKind.INSIGHT.value, slot, cutoff, configuration_ids=list(configuration_ids)+ )+ if check.legacy_configuration_id is not None+ }+ alerts = AlertConfiguration.objects.filter(team_id=team_id, id__in=list(legacy_ids)).only(+ "id", "calculation_interval"+ )+ return {legacy_ids[alert.id]: alert for alert in alerts}++def evaluate_insight_check(team_id: int,slot: str,diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.pyindex 9ca5c6f8a353..3e061c3d4d25 100644--- a/products/alerts/backend/temporal/platform_evaluate.py+++ b/products/alerts/backend/temporal/platform_evaluate.py@@ -20,6 +20,7 @@ from posthog.dataclasses import frozenfrom posthog.temporal.common.base import PostHogWorkflowwith workflow.unsafe.imports_passed_through():+ from posthog.schema_enums import AlertCalculationIntervalfrom posthog.temporal.alerts.retry_policy import alert_timeoutsfrom posthog.temporal.common.utils import close_db_connections@@ -45,6 +46,11 @@ CHECK_START_TO_CLOSE = alert_timeouts(None).evaluate_start_to_close# check below its own budget.CHECK_QUEUE_TOLERANCE = dt.timedelta(minutes=1)CHECK_SCHEDULE_TO_CLOSE = CHECK_START_TO_CLOSE + CHECK_QUEUE_TOLERANCE+# Production's real-time budget: a shorter attempt, and one retry inside the same total.+_REAL_TIME_TIMEOUTS = alert_timeouts(AlertCalculationInterval.REAL_TIME)+REAL_TIME_CHECK_START_TO_CLOSE = _REAL_TIME_TIMEOUTS.evaluate_start_to_close+REAL_TIME_CHECK_SCHEDULE_TO_CLOSE = _REAL_TIME_TIMEOUTS.activity_schedule_to_close + CHECK_QUEUE_TOLERANCE+REAL_TIME_CHECK_RETRY_POLICY = _REAL_TIME_TIMEOUTS.evaluate_retry_policyRECORD_START_TO_CLOSE = dt.timedelta(seconds=8)RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)# The pool admits once per round, so a batch that admits only once evaluates at most one pool of@@ -54,10 +60,17 @@ ADMISSION_RUN_BUDGET = dt.timedelta(seconds=50)ADMISSION_POLL_INTERVAL = dt.timedelta(seconds=2)# What the source needs from its binding's `evaluation_timeout`. The checks run side by side, so# the batch costs the admission rounds plus one check's budget, not one per check.-EVALUATION_BUDGET = PLAN_SCHEDULE_TO_CLOSE + ADMISSION_RUN_BUDGET + CHECK_SCHEDULE_TO_CLOSE + RECORD_SCHEDULE_TO_CLOSE+EVALUATION_BUDGET = (+ PLAN_SCHEDULE_TO_CLOSE+ + ADMISSION_RUN_BUDGET+ + max(CHECK_SCHEDULE_TO_CLOSE, REAL_TIME_CHECK_SCHEDULE_TO_CLOSE)+ + RECORD_SCHEDULE_TO_CLOSE+)_PATCH_READMISSION = "platform-insight-readmission-2026-10"# Runs started before it recorded every outcome in one activity, and replay must schedule the same._PATCH_CHUNKED_RECORD = "platform-insight-chunked-record-2026-10"+# Runs started before it planned bare ids, and gave every check the default budget.+_PATCH_CHECK_BUDGETS = "platform-insight-check-budgets-2026-10"@frozen@@ -67,6 +80,14 @@ class InsightBatchPlanInputs:exclude: tuple[str, ...] = ()+@frozen+class AdmittedInsightCheck:+ """An admitted check, with what the workflow needs to give it production's budget."""++ configuration_id: str+ real_time: bool = False++@frozenclass InsightCheckInputs:evaluation: SourceEvaluationInputs@@ -112,20 +133,56 @@ def plan_platform_insight_round_activity(inputs: InsightBatchPlanInputs) -> list)+@close_db_connections+def plan_platform_insight_checks_activity(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:+ """`plan_platform_insight_round_activity` with each check's cadence, under its own name, because+ a run that recorded the old name must decode bare ids on replay."""+ from products.alerts.backend.platform_source_cycle import legacy_alerts_of, plan_insight_batch # noqa: PLC0415++ key = inputs.evaluation.batch_key+ cutoff = dt.datetime.fromisoformat(inputs.evaluation.cutoff)+ admitted = plan_insight_batch(key.team_id, key.slot, cutoff, expires_at=inputs.expires_at, exclude=inputs.exclude)+ alerts = legacy_alerts_of(key.team_id, key.slot, cutoff, admitted)+ return [+ AdmittedInsightCheck(+ configuration_id=configuration_id,+ real_time=configuration_id in alerts+ and alerts[configuration_id].calculation_interval == AlertCalculationInterval.REAL_TIME,+ )+ for configuration_id in admitted+ ]++@activity.defn@close_db_connectionsdef evaluate_platform_insight_check_activity(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:"""Sync, so the thread this blocks is a slot Temporal is accounting for."""- from products.alerts.backend.platform_source_cycle import evaluate_insight_check # noqa: PLC0415+ from products.alerts.backend.platform_source_cycle import ( # noqa: PLC0415+ evaluate_insight_check,+ readmit_retried_check,+ )info = activity.info()key = inputs.evaluation.batch_key+ held_until: float | None = inputs.held_until+ if info.attempt > 1:+ held_until = readmit_retried_check(+ key.team_id,+ inputs.configuration_id,+ previous_query_id=f"{info.workflow_run_id}:{info.activity_id}:{info.attempt - 1}",+ held_until=inputs.held_until,+ attempt=info.attempt,+ )+ if held_until is None:+ # No outcome keeps the check due, so a later tick evaluates it once the pool has room.+ return Nonereturn evaluate_insight_check(key.team_id,key.slot,dt.datetime.fromisoformat(inputs.evaluation.cutoff),inputs.configuration_id,- held_until=inputs.held_until,+ held_until=held_until,evaluation_id=f"{info.workflow_run_id}:{info.activity_id}:{info.attempt}",)@@ -149,7 +206,10 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):inputs, expires_at, exclude=(), activity_fn=plan_platform_insight_batch_activity)settled = await asyncio.gather(- *(self._check(inputs, configuration_id, expires_at) for configuration_id in admitted),+ *(+ self._check(inputs, AdmittedInsightCheck(configuration_id=configuration_id), expires_at)+ for configuration_id in admitted+ ),return_exceptions=True,)if not settled:@@ -204,18 +264,45 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):)async def _check(- self, inputs: SourceEvaluationInputs, configuration_id: str, expires_at: float+ self, inputs: SourceEvaluationInputs, admitted: AdmittedInsightCheck, expires_at: float) -> SourceBatchEvaluation | None:+ check_inputs = InsightCheckInputs(+ evaluation=inputs, configuration_id=admitted.configuration_id, held_until=expires_at+ )+ if admitted.real_time:+ return await workflow.execute_activity(+ evaluate_platform_insight_check_activity,+ check_inputs,+ start_to_close_timeout=REAL_TIME_CHECK_START_TO_CLOSE,+ schedule_to_close_timeout=REAL_TIME_CHECK_SCHEDULE_TO_CLOSE,+ retry_policy=REAL_TIME_CHECK_RETRY_POLICY,+ )# One attempt each. A check that does not finish leaves no outcome, so it keeps its due time# and a later tick evaluates it again, the way a cohort the logs budget left out behaves.return await workflow.execute_activity(evaluate_platform_insight_check_activity,- InsightCheckInputs(evaluation=inputs, configuration_id=configuration_id, held_until=expires_at),+ check_inputs,start_to_close_timeout=CHECK_START_TO_CLOSE,schedule_to_close_timeout=CHECK_SCHEDULE_TO_CLOSE,retry_policy=RetryPolicy(maximum_attempts=1),)+ async def _plan_round(+ self, inputs: SourceEvaluationInputs, expires_at: float, *, exclude: tuple[str, ...], budgeted: bool+ ) -> list[AdmittedInsightCheck]:+ if budgeted:+ return await workflow.execute_activity(+ plan_platform_insight_checks_activity,+ InsightBatchPlanInputs(evaluation=inputs, expires_at=expires_at, exclude=exclude),+ start_to_close_timeout=PLAN_START_TO_CLOSE,+ schedule_to_close_timeout=PLAN_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ admitted = await self._plan(+ inputs, expires_at, exclude=exclude, activity_fn=plan_platform_insight_round_activity+ )+ return [AdmittedInsightCheck(configuration_id=configuration_id) for configuration_id in admitted]+async def _evaluate_in_rounds(self, inputs: SourceEvaluationInputs, expires_at: float) -> list[SourceBatchEvaluation | None | BaseException]:@@ -223,19 +310,18 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):checks still run, or until the admission budget ends."""started: dict[str, asyncio.Task[SourceBatchEvaluation | None]] = {}deadline = workflow.now() + ADMISSION_RUN_BUDGET+ budgeted = workflow.patched(_PATCH_CHECK_BUDGETS)while True:try:- admitted = await self._plan(- inputs, expires_at, exclude=tuple(started), activity_fn=plan_platform_insight_round_activity- )+ admitted = await self._plan_round(inputs, expires_at, exclude=tuple(started), budgeted=budgeted)except ActivityError:if not started:raise# The started checks still record. Their outcomes are lost if this failure ends the run.workflow.logger.warning("Stopped admitting platform insight checks after a failed plan")break- for configuration_id in admitted:- started[configuration_id] = asyncio.create_task(self._check(inputs, configuration_id, expires_at))+ for check in admitted:+ started[check.configuration_id] = asyncio.create_task(self._check(inputs, check, expires_at))running = [task for task in started.values() if not task.done()]if not running or workflow.now() >= deadline:break@@ -249,5 +335,6 @@ PLATFORM_EVALUATION_WORKFLOWS = [InsightAlertPlatformEvaluateWorkflow]PLATFORM_EVALUATION_ACTIVITIES = [plan_platform_insight_batch_activity,plan_platform_insight_round_activity,+ plan_platform_insight_checks_activity,evaluate_platform_insight_check_activity,]diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.pyindex ae74bde52ffd..fbe12c195973 100644--- a/products/alerts/backend/test/test_platform_alert_backfill.py+++ b/products/alerts/backend/test/test_platform_alert_backfill.py@@ -51,22 +51,23 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):)return {view.legacy_configuration_id: view for view in page.configurations}- def test_only_15_minute_and_slower_alerts_without_the_ai_detector_are_copied_on_their_recurrence(self) -> None:+ def test_only_alerts_without_the_ai_detector_are_copied_on_their_recurrence(self) -> None:+ real_time = self._alert(calculation_interval=AlertCalculationInterval.REAL_TIME.value)quarter_hourly = self._alert(calculation_interval=AlertCalculationInterval.EVERY_15_MINUTES.value)hourly = self._alert(calculation_interval=AlertCalculationInterval.HOURLY.value, schedule_start_time="09:30")daily = self._alert(schedule_start_time="09:30")detector = self._alert(detector_config={"type": "zscore"})- self._alert(calculation_interval=AlertCalculationInterval.REAL_TIME.value)self._alert(detector_config={"type": "llm"})unparseable = self._alert(schedule_start_time="25:99")counts = backfill_platform_insight_alert_configurations(team_id=self.team.id)again = backfill_platform_insight_alert_configurations(team_id=self.team.id)- assert (counts.created, counts.skipped, counts.failed, again.created, again.updated) == (4, 2, 1, 0, 4)+ assert (counts.created, counts.skipped, counts.failed, again.created, again.updated) == (5, 1, 1, 0, 5)copies = self._copies()assert unparseable.id not in copiesassert {key: (v.check_interval_minutes, v.recurrence_unit, v.anchor_time) for key, v in copies.items()} == {+ real_time.id: (2, None, None),quarter_hourly.id: (15, None, None),hourly.id: (60, None, None),daily.id: (60 * 24, "day", "09:30"),diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.pyindex 579488ab3d5d..0413360b7adf 100644--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py@@ -15,6 +15,7 @@ from temporalio.worker import UnsandboxedWorkflowRunner, Workerfrom products.alerts.backend.temporal.platform_evaluate import (EVALUATION_BUDGET,+ AdmittedInsightCheck,InsightAlertPlatformEvaluateWorkflow,InsightBatchPlanInputs,InsightCheckInputs,@@ -46,14 +47,19 @@ async def environment() -> AsyncIterator[WorkflowEnvironment]:async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(environment: WorkflowEnvironment) -> None:recorded: list[SourceOutcomeInputs] = []- @activity.defn(name="plan_platform_insight_round_activity")- async def plan(inputs: InsightBatchPlanInputs) -> list[str]:- return [configuration_id for configuration_id in ADMITTED if configuration_id not in inputs.exclude]+ # Only the real-time check has a retry, so the same first failure loses the other one.+ @activity.defn(name="plan_platform_insight_checks_activity")+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:+ admitted = [+ AdmittedInsightCheck(configuration_id=ADMITTED[0]),+ AdmittedInsightCheck(configuration_id=ADMITTED[1], real_time=True),+ ]+ return [check for check in admitted if check.configuration_id not in inputs.exclude]@activity.defn(name="evaluate_platform_insight_check_activity")async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:- if inputs.configuration_id == ADMITTED[0]:- raise ApplicationError("query failed", non_retryable=True)+ if activity.info().attempt == 1:+ raise ApplicationError("worker lost")outcome = PlatformAlertOutcome(configuration_id=uuid.UUID(inputs.configuration_id),evaluation_key="slot:2026-09-16T10:00:00+00:00",@@ -115,9 +121,13 @@ async def test_a_batch_over_the_payload_budget_records_in_chunks_and_starts_one_) -> None:recorded: list[SourceOutcomeInputs] = []- @activity.defn(name="plan_platform_insight_round_activity")- async def plan(inputs: InsightBatchPlanInputs) -> list[str]:- return [configuration_id for configuration_id in ADMITTED if configuration_id not in inputs.exclude]+ @activity.defn(name="plan_platform_insight_checks_activity")+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:+ return [+ AdmittedInsightCheck(configuration_id=configuration_id)+ for configuration_id in ADMITTED+ if configuration_id not in inputs.exclude+ ]@activity.defn(name="evaluate_platform_insight_check_activity")async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:@@ -209,14 +219,18 @@ async def test_a_batch_refills_the_slots_its_finished_checks_free(recorded: list[SourceOutcomeInputs] = []# A pool of one slot: each round admits the first due check that this batch did not start.- @activity.defn(name="plan_platform_insight_round_activity")- async def plan(inputs: InsightBatchPlanInputs) -> list[str]:+ @activity.defn(name="plan_platform_insight_checks_activity")+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:plans.append(inputs.exclude)if fail_second_plan and len(plans) == 2:raise ApplicationError("redis unavailable", non_retryable=True)if set(inputs.exclude) - set(evaluated):return []- return [configuration_id for configuration_id in due if configuration_id not in inputs.exclude][:1]+ return [+ AdmittedInsightCheck(configuration_id=configuration_id)+ for configuration_id in due+ if configuration_id not in inputs.exclude+ ][:1]@activity.defn(name="evaluate_platform_insight_check_activity")async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex 240f7121caf1..c6a44bb2d4ed 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -1,6 +1,7 @@import timefrom datetime import UTC, datetime, timedeltafrom typing import Any+from uuid import UUIDfrom posthog.test.base import APIBaseTestfrom unittest.mock import MagicMock, patch@@ -16,6 +17,7 @@ from posthog.models.scoping import team_scopefrom posthog.redis import get_clientfrom posthog.schema_enums import AlertCalculationIntervalfrom posthog.tasks.alerts.utils import AlertEvaluationResult+from posthog.temporal.alerts.admission import inflight_alert_ids, release_evaluation_slotfrom products.alerts.backend.evaluation.contract import AlertExtractionErrorfrom products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, SeriesEvaluation@@ -26,10 +28,12 @@ from products.alerts.backend.platform_source_cycle import (INFLIGHT_KEY,evaluate_insight_check,plan_insight_batch,+ readmit_retried_check,)from products.alerts.backend.temporal.platform_evaluate import (+ AdmittedInsightCheck,InsightBatchPlanInputs,- plan_platform_insight_round_activity,+ plan_platform_insight_checks_activity,)from products.alerts.backend.test.insight_alerts import create_insight_alertfrom products.alerts_platform.backend.facade import testing as platform_testing@@ -211,7 +215,6 @@ class TestPlatformInsightEvaluation(APIBaseTest):@parameterized.expand([- ("real_time", {"calculation_interval": AlertCalculationInterval.REAL_TIME.value}, "not_firing"),("ai_detector", {"detector_config": {"type": "llm"}}, "not_firing"),("disabled", {"enabled": False}, "not_firing"),("snoozed", {"snoozed_until": CUTOFF + timedelta(hours=1)}, "snoozed"),@@ -264,20 +267,44 @@ class TestPlatformInsightEvaluation(APIBaseTest):True,)+ @override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS=1)+ def test_a_retry_stops_the_timed_out_query_and_runs_only_inside_the_pool(self) -> None:+ held = self._copy(self._alert(), id=UUID(int=1))+ waiting = self._copy(self._alert(), id=UUID(int=2))+ slot = slot_of(held.next_check_at, CUTOFF)+ expires_at = time.time() + 3600+ assert plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at) == (str(held.id),)++ with patch("posthog.clickhouse.cancel.cancel_query_on_cluster") as cancel:+ retried = readmit_retried_check(+ self.team.id, str(held.id), previous_query_id="run:1:1", held_until=expires_at, attempt=2+ )+ # The timed-out attempt's thread finishes late and releases the slot it took.+ release_evaluation_slot(str(held.id), held_until=expires_at, key=INFLIGHT_KEY)+ over_the_limit = readmit_retried_check(+ self.team.id, str(waiting.id), previous_query_id="run:2:1", held_until=expires_at, attempt=2+ )++ assert retried is not None and over_the_limit is None+ assert inflight_alert_ids(INFLIGHT_KEY) == {str(held.id)}+ cancel.assert_any_call(self.team.id, "alerts-platform-insight:run:1:1")+@override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS=1)def test_the_pool_defers_what_it_cannot_hold_until_a_check_frees_its_slot(self) -> None:- first, second = self._copy(self._alert()), self._copy(self._alert())+ first = self._copy(self._alert(), id=UUID(int=1))+ second = self._copy(self._alert(calculation_interval=AlertCalculationInterval.REAL_TIME.value), id=UUID(int=2))slot = slot_of(first.next_check_at, CUTOFF)expires_at = time.time() + 3600(admitted,) = plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at)+ assert admitted == str(first.id)assert plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at + 60) == ()with patch(f"{_MODULE}.check_alert_for_insight", return_value=AlertEvaluationResult(value=1.0, breaches=[])):evaluate_insight_check(self.team.id, slot, CUTOFF, admitted, held_until=expires_at, evaluation_id="test")- # Through the activity the workflow calls, so a dropped `exclude` there fails here too.- replanned = plan_platform_insight_round_activity(+ # Through the activity the workflow calls, so a dropped `exclude` or cadence there fails here too.+ replanned = plan_platform_insight_checks_activity(InsightBatchPlanInputs(evaluation=SourceEvaluationInputs(source=SourceKind.INSIGHT,@@ -288,4 +315,4 @@ class TestPlatformInsightEvaluation(APIBaseTest):exclude=(admitted,),))- assert replanned == list({str(first.id), str(second.id)} - {admitted})+ assert replanned == [AdmittedInsightCheck(configuration_id=str(second.id), real_time=True)]