:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(alerts-platform): retry capacity refusals and keep ai checks on their user (#127828)
Jimmy Zhu5567df380de7
diff --git a/posthog/clickhouse/client/execute.py b/posthog/clickhouse/client/execute.pyindex 875d27a978ae..0b5da3e96da0 100644--- a/posthog/clickhouse/client/execute.py+++ b/posthog/clickhouse/client/execute.py@@ -502,7 +502,9 @@ def sync_execute(add_fallback_query_tags(tags)- if tags.product == Product.MAX_AI or tags.service_name == "temporal-worker-max-ai":+ # A ch_user tag wins on the Max AI worker, because the worker also hosts work with its own+ # ClickHouse user and concurrency budget, such as the alerts platform's AI insight checks.+ if tags.product == Product.MAX_AI or (tags.service_name == "temporal-worker-max-ai" and not tags.ch_user):ch_user = ClickHouseUser.MAX_AIelif tags.product == Product.ENDPOINTS:ch_user = ClickHouseUser.ENDPOINTSdiff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.pyindex b978ab5677fb..434d01786590 100644--- a/posthog/clickhouse/client/test/test_execute.py+++ b/posthog/clickhouse/client/test/test_execute.py@@ -120,18 +120,44 @@ def test_llm_analytics_ch_user_routing(client_from_pool, product, kind, tag_id,@pytest.mark.parametrize(- "tagged_ch_user,feature,requested_ch_user,expected_ch_user",+ "tagged_ch_user,feature,service_name,requested_ch_user,expected_ch_user",[- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.DEFAULT, ClickHouseUser.ALERTS_PLATFORM_INSIGHT),- (None, None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.META, ClickHouseUser.META),- (ClickHouseUser.EXPERIMENTS, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.EXPERIMENTS),- (None, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.CACHE_WARMUP),+ (+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,+ None,+ None,+ ClickHouseUser.DEFAULT,+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,+ ),+ (None, None, None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),+ (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, None, ClickHouseUser.META, ClickHouseUser.META),+ (ClickHouseUser.EXPERIMENTS, Feature.CACHE_WARMUP, None, ClickHouseUser.DEFAULT, ClickHouseUser.EXPERIMENTS),+ (None, Feature.CACHE_WARMUP, None, ClickHouseUser.DEFAULT, ClickHouseUser.CACHE_WARMUP),+ (+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,+ None,+ "temporal-worker-max-ai",+ ClickHouseUser.DEFAULT,+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,+ ),+ (None, None, "temporal-worker-max-ai", ClickHouseUser.DEFAULT, ClickHouseUser.MAX_AI),],)-def test_tagged_ch_user_routing(client_from_pool, tagged_ch_user, feature, requested_ch_user, expected_ch_user):+def test_tagged_ch_user_routing(+ client_from_pool: MagicMock,+ tagged_ch_user: ClickHouseUser | None,+ feature: Feature | None,+ service_name: str | None,+ requested_ch_user: ClickHouseUser,+ expected_ch_user: ClickHouseUser,+) -> None:with tags_context(- product=Product.PRODUCT_ANALYTICS, feature=feature, kind="temporal", id="alerts", ch_user=tagged_ch_user+ product=Product.PRODUCT_ANALYTICS,+ feature=feature,+ kind="temporal",+ id="alerts",+ ch_user=tagged_ch_user,+ service_name=service_name,):sync_execute("SELECT 1", flush=False, ch_user=requested_ch_user)diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex bbff6244a1c5..226bceff110b 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -19,7 +19,7 @@ same set of checks.import jsonimport timeimport asyncio-from collections.abc import Collection, Iterator, Sequence+from collections.abc import Callable, Collection, Iterator, Sequencefrom contextlib import contextmanagerfrom dataclasses import replacefrom datetime import UTC, datetime, timedelta@@ -28,6 +28,7 @@ from typing import Any, Finalfrom django.conf import settingsimport structlog+from temporalio import activityfrom posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimit@@ -356,6 +357,59 @@ class _ClickHouseSlot:logger.exception("Could not release a ClickHouse slot", holder=self.holder)+def _with_capacity_retries[T](run: Callable[[], T], alert: AlertConfiguration, query_slot: _ClickHouseSlot) -> T:+ """Runs a check's query, and runs it again after a ClickHouse refusal for load, on the retry+ policy production evaluates the alert's cadence with.++ Production raises the refusal and Temporal retries its activity. A parallel-run check has one+ attempt, so it retries inside that attempt instead, and keeps its evaluation slot between tries.+ The wait between tries counts as slot wait, so the recorded query duration leaves it out.+ """+ policy = alert_timeouts(alert.calculation_interval).evaluate_retry_policy+ delay = policy.initial_interval.total_seconds()+ # Temporal caps an unset maximum interval at 100 times the initial one.+ longest = (policy.maximum_interval or 100 * policy.initial_interval).total_seconds()+ tries = policy.maximum_attempts+ for tried in range(1, tries + 1):+ try:+ return run()+ except Exception as error:+ if (+ tried == tries+ or classify_query_error(error) != QueryErrorCategory.RATE_LIMITED+ or not _attempt_outlasts(delay)+ ):+ raise+ time.sleep(delay)+ query_slot.waited_ms += int(delay * 1000)+ if not _attempt_outlasts(0):+ raise+ delay = min(delay * policy.backoff_coefficient, longest)+ raise AssertionError("The retry loop returns or raises on its last try")+++def _attempt_outlasts(seconds: float) -> bool:+ """Whether the Temporal attempt running this check is live and has `seconds` of its budget left.++ An attempt that timed out or was cancelled keeps its thread, and the failed-check cleanup has+ already stopped its queries, so a query it started after that would hold a ClickHouse slot that+ nothing frees. Outside an activity there is no budget to outlast.+ """+ if not activity.in_activity():+ return True+ if activity.is_cancelled():+ return False+ info = activity.info()+ # The attempt starts after it is scheduled, so its scheduled time gives a deadline no later than+ # the real one. Schedule-to-close counts from the first attempt, so it can end a retry sooner.+ deadlines = []+ if info.start_to_close_timeout is not None:+ deadlines.append(info.current_attempt_scheduled_time + info.start_to_close_timeout)+ if info.schedule_to_close_timeout is not None:+ deadlines.append(info.scheduled_time + info.schedule_to_close_timeout)+ return not deadlines or datetime.now(UTC) + timedelta(seconds=seconds) < min(deadlines)++def _stop_query(team_id: int, configuration_id: str, query_id: str) -> None:# Imported here: at module level it changes how many queries the access-control tests count.from posthog.clickhouse.cancel import cancel_query_on_cluster # noqa: PLC0415@@ -641,7 +695,11 @@ def _decide(return _decide_grouped(check, snapshot, alert, now=now, query_slot=query_slot)started_at = time.monotonic()try:- result = check_alert_for_insight(alert, evaluation_id=evaluation_id, query_slot=query_slot)+ result = _with_capacity_retries(+ lambda: check_alert_for_insight(alert, evaluation_id=evaluation_id, query_slot=query_slot),+ alert,+ query_slot,+ )except ClickHouseBudgetFull:return Noneexcept LLMDetectorUnavailableError:@@ -809,7 +867,9 @@ def _decide_grouped()started_at = time.monotonic()try:- evaluated = check_alert_per_series(alert, query_slot=query_slot)+ evaluated = _with_capacity_retries(+ lambda: check_alert_per_series(alert, query_slot=query_slot), alert, query_slot+ )except ClickHouseBudgetFull:return Noneexcept AlertExtractionError as error:diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex f7bc8e8e519a..f1b1c8958059 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -2,6 +2,7 @@ import jsonimport timefrom collections.abc import Callablefrom contextlib import AbstractContextManager+from dataclasses import replacefrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom uuid import UUID@@ -14,6 +15,7 @@ from unittest.mock import MagicMock, patchfrom django.test import override_settingsfrom parameterized import parameterized+from temporalio.testing import ActivityEnvironmentfrom posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.clickhouse.query_tagging import get_query_tags@@ -24,6 +26,7 @@ from posthog.redis import get_clientfrom posthog.schema_enums import AlertCalculationIntervalfrom posthog.tasks.alerts.utils import AlertEvaluationResult, RenderedInsightChart, next_check_timefrom posthog.temporal.alerts.admission import admit_evaluation_slots, inflight_alert_ids, release_evaluation_slot+from posthog.temporal.alerts.retry_policy import ALERT_EVALUATE_RETRY_POLICYfrom products.alerts.backend.evaluation.contract import AlertExtractionErrorfrom products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, SeriesEvaluation@@ -208,7 +211,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):slot = slot_of(configuration.next_check_at, CUTOFF)expires_at = time.time() + 3600assert plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at) == (str(configuration.id),)- side_effect = result if isinstance(result, Exception) else None+ side_effect = result if isinstance(result, Exception | list) else Nonereturned = result if isinstance(result, AlertEvaluationResult) else Nonewith patch(f"{_MODULE}.check_alert_for_insight", side_effect=side_effect, return_value=returned) as query:evaluation = evaluate_insight_check(@@ -595,12 +598,38 @@ class TestPlatformInsightEvaluation(APIBaseTest):@parameterized.expand([- ("too_many_queries", ClickHouseAtCapacity()),- ("cluster_memory_full", ClickHouseClusterMemoryLimitExceeded()),+ ("too_many_queries", ClickHouseAtCapacity(), None, ALERT_EVALUATE_RETRY_POLICY.maximum_attempts),+ (+ "cluster_memory_full",+ ClickHouseClusterMemoryLimitExceeded(),+ None,+ ALERT_EVALUATE_RETRY_POLICY.maximum_attempts,+ ),+ ("attempt_out_of_time", ClickHouseAtCapacity(), "start_to_close", 1),+ ("check_out_of_time", ClickHouseAtCapacity(), "schedule_to_close", 1),])- def test_a_check_clickhouse_refuses_for_load_leaves_the_alert_as_it_was(self, _name: str, error: Exception) -> None:- outcome, _ = self._evaluate(self._copy(self._alert()), result=error)+ def test_a_check_clickhouse_refuses_on_every_try_leaves_the_alert_as_it_was(+ self, _name: str, error: Exception, budget_spent: str | None, tries: int+ ) -> None:+ configuration = self._copy(self._alert())+ with patch(f"{_MODULE}.time.sleep"):+ if budget_spent is None:+ outcome, query = self._evaluate(configuration, result=error)+ else:+ now = datetime.now(UTC)+ spent = timedelta(minutes=13)+ environment = ActivityEnvironment()+ # A later attempt with its own start-to-close left, inside a schedule-to-close that ran out, or the reverse.+ environment.info = replace(+ environment.info,+ attempt=2,+ scheduled_time=now - spent if budget_spent == "schedule_to_close" else now,+ schedule_to_close_timeout=spent,+ current_attempt_scheduled_time=now - spent if budget_spent == "start_to_close" else now,+ start_to_close_timeout=timedelta(minutes=12),+ )+ outcome, query = environment.run(lambda: self._evaluate(configuration, result=error))assert outcome is not Noneassert (outcome.groups[0].kind, outcome.groups[0].new_state, outcome.error_message, outcome.disable) == (@@ -609,6 +638,17 @@ class TestPlatformInsightEvaluation(APIBaseTest):CAPACITY_REJECTED,False,)+ assert query.call_count == tries++ def test_a_check_clickhouse_refuses_once_records_the_retried_query_verdict(self) -> None:+ with patch(f"{_MODULE}.time.sleep"):+ outcome, query = self._evaluate(+ self._copy(self._alert()),+ result=[ClickHouseAtCapacity(), AlertEvaluationResult(value=150.0, breaches=["above 100"])],+ )++ assert outcome is not None+ assert (outcome.groups[0].new_state, outcome.error_message, query.call_count) == ("firing", None, 2)def test_a_cadence_the_plan_no_longer_includes_errors_and_is_disabled(self) -> None:self.organization.available_product_features = []