Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts-platform): retry capacity refusals and keep ai checks on their user (#127828)

Jimmy Zhu5567df380de7
diff --git a/posthog/clickhouse/client/execute.py b/posthog/clickhouse/client/execute.py
index 875d27a978ae..0b5da3e96da0 100644
--- a/posthog/clickhouse/client/execute.py
+++ b/posthog/clickhouse/client/execute.py
@@ -502,7 +502,9 @@ def sync_execute(
add_fallback_query_tags(tags)
- if tags.product == Product.MAX_AI or tags.service_name == "temporal-worker-max-ai":
+ # A ch_user tag wins on the Max AI worker, because the worker also hosts work with its own
+ # ClickHouse user and concurrency budget, such as the alerts platform's AI insight checks.
+ if tags.product == Product.MAX_AI or (tags.service_name == "temporal-worker-max-ai" and not tags.ch_user):
ch_user = ClickHouseUser.MAX_AI
elif tags.product == Product.ENDPOINTS:
ch_user = ClickHouseUser.ENDPOINTS
diff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.py
index b978ab5677fb..434d01786590 100644
--- a/posthog/clickhouse/client/test/test_execute.py
+++ b/posthog/clickhouse/client/test/test_execute.py
@@ -120,18 +120,44 @@ def test_llm_analytics_ch_user_routing(client_from_pool, product, kind, tag_id,
@pytest.mark.parametrize(
- "tagged_ch_user,feature,requested_ch_user,expected_ch_user",
+ "tagged_ch_user,feature,service_name,requested_ch_user,expected_ch_user",
[
- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.DEFAULT, ClickHouseUser.ALERTS_PLATFORM_INSIGHT),
- (None, None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),
- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.META, ClickHouseUser.META),
- (ClickHouseUser.EXPERIMENTS, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.EXPERIMENTS),
- (None, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.CACHE_WARMUP),
+ (
+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,
+ None,
+ None,
+ ClickHouseUser.DEFAULT,
+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,
+ ),
+ (None, None, None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),
+ (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, None, ClickHouseUser.META, ClickHouseUser.META),
+ (ClickHouseUser.EXPERIMENTS, Feature.CACHE_WARMUP, None, ClickHouseUser.DEFAULT, ClickHouseUser.EXPERIMENTS),
+ (None, Feature.CACHE_WARMUP, None, ClickHouseUser.DEFAULT, ClickHouseUser.CACHE_WARMUP),
+ (
+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,
+ None,
+ "temporal-worker-max-ai",
+ ClickHouseUser.DEFAULT,
+ ClickHouseUser.ALERTS_PLATFORM_INSIGHT,
+ ),
+ (None, None, "temporal-worker-max-ai", ClickHouseUser.DEFAULT, ClickHouseUser.MAX_AI),
],
)
-def test_tagged_ch_user_routing(client_from_pool, tagged_ch_user, feature, requested_ch_user, expected_ch_user):
+def test_tagged_ch_user_routing(
+ client_from_pool: MagicMock,
+ tagged_ch_user: ClickHouseUser | None,
+ feature: Feature | None,
+ service_name: str | None,
+ requested_ch_user: ClickHouseUser,
+ expected_ch_user: ClickHouseUser,
+) -> None:
with tags_context(
- product=Product.PRODUCT_ANALYTICS, feature=feature, kind="temporal", id="alerts", ch_user=tagged_ch_user
+ product=Product.PRODUCT_ANALYTICS,
+ feature=feature,
+ kind="temporal",
+ id="alerts",
+ ch_user=tagged_ch_user,
+ service_name=service_name,
):
sync_execute("SELECT 1", flush=False, ch_user=requested_ch_user)
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index bbff6244a1c5..226bceff110b 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -19,7 +19,7 @@ same set of checks.
import json
import time
import asyncio
-from collections.abc import Collection, Iterator, Sequence
+from collections.abc import Callable, Collection, Iterator, Sequence
from contextlib import contextmanager
from dataclasses import replace
from datetime import UTC, datetime, timedelta
@@ -28,6 +28,7 @@ from typing import Any, Final
from django.conf import settings
import structlog
+from temporalio import activity
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimit
@@ -356,6 +357,59 @@ class _ClickHouseSlot:
logger.exception("Could not release a ClickHouse slot", holder=self.holder)
+def _with_capacity_retries[T](run: Callable[[], T], alert: AlertConfiguration, query_slot: _ClickHouseSlot) -> T:
+ """Runs a check's query, and runs it again after a ClickHouse refusal for load, on the retry
+ policy production evaluates the alert's cadence with.
+
+ Production raises the refusal and Temporal retries its activity. A parallel-run check has one
+ attempt, so it retries inside that attempt instead, and keeps its evaluation slot between tries.
+ The wait between tries counts as slot wait, so the recorded query duration leaves it out.
+ """
+ policy = alert_timeouts(alert.calculation_interval).evaluate_retry_policy
+ delay = policy.initial_interval.total_seconds()
+ # Temporal caps an unset maximum interval at 100 times the initial one.
+ longest = (policy.maximum_interval or 100 * policy.initial_interval).total_seconds()
+ tries = policy.maximum_attempts
+ for tried in range(1, tries + 1):
+ try:
+ return run()
+ except Exception as error:
+ if (
+ tried == tries
+ or classify_query_error(error) != QueryErrorCategory.RATE_LIMITED
+ or not _attempt_outlasts(delay)
+ ):
+ raise
+ time.sleep(delay)
+ query_slot.waited_ms += int(delay * 1000)
+ if not _attempt_outlasts(0):
+ raise
+ delay = min(delay * policy.backoff_coefficient, longest)
+ raise AssertionError("The retry loop returns or raises on its last try")
+
+
+def _attempt_outlasts(seconds: float) -> bool:
+ """Whether the Temporal attempt running this check is live and has `seconds` of its budget left.
+
+ An attempt that timed out or was cancelled keeps its thread, and the failed-check cleanup has
+ already stopped its queries, so a query it started after that would hold a ClickHouse slot that
+ nothing frees. Outside an activity there is no budget to outlast.
+ """
+ if not activity.in_activity():
+ return True
+ if activity.is_cancelled():
+ return False
+ info = activity.info()
+ # The attempt starts after it is scheduled, so its scheduled time gives a deadline no later than
+ # the real one. Schedule-to-close counts from the first attempt, so it can end a retry sooner.
+ deadlines = []
+ if info.start_to_close_timeout is not None:
+ deadlines.append(info.current_attempt_scheduled_time + info.start_to_close_timeout)
+ if info.schedule_to_close_timeout is not None:
+ deadlines.append(info.scheduled_time + info.schedule_to_close_timeout)
+ return not deadlines or datetime.now(UTC) + timedelta(seconds=seconds) < min(deadlines)
+
+
def _stop_query(team_id: int, configuration_id: str, query_id: str) -> None:
# Imported here: at module level it changes how many queries the access-control tests count.
from posthog.clickhouse.cancel import cancel_query_on_cluster # noqa: PLC0415
@@ -641,7 +695,11 @@ def _decide(
return _decide_grouped(check, snapshot, alert, now=now, query_slot=query_slot)
started_at = time.monotonic()
try:
- result = check_alert_for_insight(alert, evaluation_id=evaluation_id, query_slot=query_slot)
+ result = _with_capacity_retries(
+ lambda: check_alert_for_insight(alert, evaluation_id=evaluation_id, query_slot=query_slot),
+ alert,
+ query_slot,
+ )
except ClickHouseBudgetFull:
return None
except LLMDetectorUnavailableError:
@@ -809,7 +867,9 @@ def _decide_grouped(
)
started_at = time.monotonic()
try:
- evaluated = check_alert_per_series(alert, query_slot=query_slot)
+ evaluated = _with_capacity_retries(
+ lambda: check_alert_per_series(alert, query_slot=query_slot), alert, query_slot
+ )
except ClickHouseBudgetFull:
return None
except AlertExtractionError as error:
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index f7bc8e8e519a..f1b1c8958059 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -2,6 +2,7 @@ import json
import time
from collections.abc import Callable
from contextlib import AbstractContextManager
+from dataclasses import replace
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import UUID
@@ -14,6 +15,7 @@ from unittest.mock import MagicMock, patch
from django.test import override_settings
from parameterized import parameterized
+from temporalio.testing import ActivityEnvironment
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import get_query_tags
@@ -24,6 +26,7 @@ from posthog.redis import get_client
from posthog.schema_enums import AlertCalculationInterval
from posthog.tasks.alerts.utils import AlertEvaluationResult, RenderedInsightChart, next_check_time
from posthog.temporal.alerts.admission import admit_evaluation_slots, inflight_alert_ids, release_evaluation_slot
+from posthog.temporal.alerts.retry_policy import ALERT_EVALUATE_RETRY_POLICY
from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, SeriesEvaluation
@@ -208,7 +211,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):
slot = slot_of(configuration.next_check_at, CUTOFF)
expires_at = time.time() + 3600
assert plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at) == (str(configuration.id),)
- side_effect = result if isinstance(result, Exception) else None
+ side_effect = result if isinstance(result, Exception | list) else None
returned = result if isinstance(result, AlertEvaluationResult) else None
with patch(f"{_MODULE}.check_alert_for_insight", side_effect=side_effect, return_value=returned) as query:
evaluation = evaluate_insight_check(
@@ -595,12 +598,38 @@ class TestPlatformInsightEvaluation(APIBaseTest):
@parameterized.expand(
[
- ("too_many_queries", ClickHouseAtCapacity()),
- ("cluster_memory_full", ClickHouseClusterMemoryLimitExceeded()),
+ ("too_many_queries", ClickHouseAtCapacity(), None, ALERT_EVALUATE_RETRY_POLICY.maximum_attempts),
+ (
+ "cluster_memory_full",
+ ClickHouseClusterMemoryLimitExceeded(),
+ None,
+ ALERT_EVALUATE_RETRY_POLICY.maximum_attempts,
+ ),
+ ("attempt_out_of_time", ClickHouseAtCapacity(), "start_to_close", 1),
+ ("check_out_of_time", ClickHouseAtCapacity(), "schedule_to_close", 1),
]
)
- def test_a_check_clickhouse_refuses_for_load_leaves_the_alert_as_it_was(self, _name: str, error: Exception) -> None:
- outcome, _ = self._evaluate(self._copy(self._alert()), result=error)
+ def test_a_check_clickhouse_refuses_on_every_try_leaves_the_alert_as_it_was(
+ self, _name: str, error: Exception, budget_spent: str | None, tries: int
+ ) -> None:
+ configuration = self._copy(self._alert())
+ with patch(f"{_MODULE}.time.sleep"):
+ if budget_spent is None:
+ outcome, query = self._evaluate(configuration, result=error)
+ else:
+ now = datetime.now(UTC)
+ spent = timedelta(minutes=13)
+ environment = ActivityEnvironment()
+ # A later attempt with its own start-to-close left, inside a schedule-to-close that ran out, or the reverse.
+ environment.info = replace(
+ environment.info,
+ attempt=2,
+ scheduled_time=now - spent if budget_spent == "schedule_to_close" else now,
+ schedule_to_close_timeout=spent,
+ current_attempt_scheduled_time=now - spent if budget_spent == "start_to_close" else now,
+ start_to_close_timeout=timedelta(minutes=12),
+ )
+ outcome, query = environment.run(lambda: self._evaluate(configuration, result=error))
assert outcome is not None
assert (outcome.groups[0].kind, outcome.groups[0].new_state, outcome.error_message, outcome.disable) == (
@@ -609,6 +638,17 @@ class TestPlatformInsightEvaluation(APIBaseTest):
CAPACITY_REJECTED,
False,
)
+ assert query.call_count == tries
+
+ def test_a_check_clickhouse_refuses_once_records_the_retried_query_verdict(self) -> None:
+ with patch(f"{_MODULE}.time.sleep"):
+ outcome, query = self._evaluate(
+ self._copy(self._alert()),
+ result=[ClickHouseAtCapacity(), AlertEvaluationResult(value=150.0, breaches=["above 100"])],
+ )
+
+ assert outcome is not None
+ assert (outcome.groups[0].new_state, outcome.error_message, query.call_count) == ("firing", None, 2)
def test_a_cadence_the_plan_no_longer_includes_errors_and_is_disabled(self) -> None:
self.organization.available_product_features = []