Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): heartbeat and retry hourly and slower platform insight checks (#126715)

Jimmy Zhua684666509ad
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 8a22dfa868fc..1a6e90c3b0d2 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -31,7 +31,7 @@ import structlog
from temporalio import activity
from posthog.clickhouse.client.connection import ClickHouseUser
-from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimit
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, RateLimit
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.errors import CH_TRANSIENT_ERRORS, QueryErrorCategory, classify_query_error
from posthog.query_cache.single_flight import skip_query_single_flight
@@ -312,10 +312,12 @@ def readmit_retried_check(
A timed-out attempt's thread can still run its query, so the query is killed first, or the retry
would run beside it. The retry then moves the slot to its own token, so the pool never counts
the two attempts as one check that either can release.
+
+ The earlier attempt keeps its ClickHouse slot. A kill can fail without an error, and a query that
+ still runs must stay counted. Its thread frees the slot when the query ends, and the lease frees
+ it if the worker died.
"""
_stop_query(team_id, configuration_id, previous_query_id)
- # The query ID names the attempt that held this slot, so its slot frees now rather than at its lease.
- _QUERY_LIMIT.release(ConcurrencySlot(running_tasks_key=QUERY_INFLIGHT_KEY, task_id=previous_query_id))
release_evaluation_slot(configuration_id, held_until=attempt_slot_token(held_until, attempt - 1), key=INFLIGHT_KEY)
token = attempt_slot_token(held_until, attempt)
admitted = admit_evaluation_slots(
diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.py
index b32797a376d6..ce6e8731e2f9 100644
--- a/products/alerts/backend/temporal/platform_evaluate.py
+++ b/products/alerts/backend/temporal/platform_evaluate.py
@@ -57,6 +57,11 @@ CHECK_START_TO_CLOSE = alert_timeouts(None).evaluate_start_to_close
# check below its own budget.
CHECK_QUEUE_TOLERANCE = dt.timedelta(minutes=1)
CHECK_SCHEDULE_TO_CLOSE = CHECK_START_TO_CLOSE + CHECK_QUEUE_TOLERANCE
+# Production's heartbeat and retries for an hourly or slower check. A worker that never runs its
+# task, such as one that gets it while it shuts down, stops heartbeating, so Temporal retries the
+# check on another worker after the heartbeat timeout instead of after the whole budget.
+CHECK_HEARTBEAT_TIMEOUT = alert_timeouts(None).heartbeat_timeout
+CHECK_RETRY_POLICY = alert_timeouts(None).evaluate_retry_policy
# Production's real-time budget: a shorter attempt, and one retry inside the same total.
_REAL_TIME_TIMEOUTS = alert_timeouts(AlertCalculationInterval.REAL_TIME)
REAL_TIME_CHECK_START_TO_CLOSE = _REAL_TIME_TIMEOUTS.evaluate_start_to_close
@@ -66,7 +71,7 @@ REAL_TIME_CHECK_RETRY_POLICY = _REAL_TIME_TIMEOUTS.evaluate_retry_policy
REAL_TIME_CHECK_HEARTBEAT_TIMEOUT = _REAL_TIME_TIMEOUTS.heartbeat_timeout
# Production's retries for an AI detector check, inside the default budget. A judge that reaches
# no verdict raises for another attempt, and the last attempt records the failure.
-LLM_CHECK_RETRY_POLICY = alert_timeouts(None).evaluate_retry_policy
+LLM_CHECK_RETRY_POLICY = CHECK_RETRY_POLICY
# A firing's chart renders in its own activity, after the check has released its pool slot. It
# outlasts the export render's own 90-second limit, so the render, not the activity, gives up first.
CHART_START_TO_CLOSE = dt.timedelta(minutes=2)
@@ -482,14 +487,15 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
heartbeat_timeout=REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,
retry_policy=REAL_TIME_CHECK_RETRY_POLICY,
)
- # One attempt each. In a run that records as checks finish, a check that does not finish is
- # recorded as errored. In an older run it leaves no outcome and keeps its due time.
+ # In a run that records as checks finish, a check whose every attempt fails is recorded as
+ # errored. In an older run it leaves no outcome and keeps its due time.
return await workflow.execute_activity(
evaluate_platform_insight_check_activity,
check_inputs,
start_to_close_timeout=CHECK_START_TO_CLOSE,
schedule_to_close_timeout=CHECK_SCHEDULE_TO_CLOSE,
- retry_policy=RetryPolicy(maximum_attempts=1),
+ heartbeat_timeout=CHECK_HEARTBEAT_TIMEOUT,
+ retry_policy=CHECK_RETRY_POLICY,
)
async def _plan_round(
@@ -754,11 +760,10 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
def _maximum_attempts(check: AdmittedInsightCheck) -> int:
- if check.uses_llm_detector:
- return LLM_CHECK_RETRY_POLICY.maximum_attempts
- if check.real_time:
+ # `_check` sends an AI detector check to its own queue under the default policy, whatever its cadence.
+ if check.real_time and not check.uses_llm_detector:
return REAL_TIME_CHECK_RETRY_POLICY.maximum_attempts
- return 1
+ return CHECK_RETRY_POLICY.maximum_attempts
PLATFORM_EVALUATION_WORKFLOWS = [InsightAlertPlatformEvaluateWorkflow]
diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.py
index aac68c3d8334..f08c9366e16a 100644
--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py
+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py
@@ -23,6 +23,8 @@ from products.alerts.backend.platform_chart_detail import (
)
from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS
from products.alerts.backend.temporal.platform_evaluate import (
+ CHECK_HEARTBEAT_TIMEOUT,
+ CHECK_RETRY_POLICY,
EVALUATION_BUDGET,
REAL_TIME_CHECK_HEARTBEAT_TIMEOUT,
AdmittedInsightCheck,
@@ -72,8 +74,9 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
recorded: list[SourceOutcomeInputs] = []
failed: list[FailedInsightCheckInputs] = []
stopped: list[StopInsightQueriesInputs] = []
+ hourly_attempts: list[int] = []
- # Only the real-time and AI checks have a retry, so the same first failure loses the other one.
+ # Every check retries a first failure. The first check fails every attempt, so it ends errored.
@activity.defn(name="plan_platform_insight_checks_activity")
async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:
admitted = [
@@ -85,7 +88,9 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
@activity.defn(name="evaluate_platform_insight_check_activity")
async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
- if activity.info().attempt == 1:
+ if inputs.configuration_id == ADMITTED[0]:
+ hourly_attempts.append(activity.info().attempt)
+ if activity.info().attempt == 1 or inputs.configuration_id == ADMITTED[0]:
raise ApplicationError("worker lost")
return _fired(inputs)
@@ -151,23 +156,25 @@ async def test_a_check_that_fails_does_not_stop_the_batch_recording_the_rest(env
history = await handle.fetch_history()
assert result == 3
- # The check with no retry left no outcome of its own, so it is recorded as errored, with the reason.
+ # The check whose every attempt failed left no outcome of its own, so it is recorded as errored, with the reason.
+ assert hourly_attempts == list(range(1, CHECK_RETRY_POLICY.maximum_attempts + 1))
assert [(inputs.configuration_id, inputs.error_message) for inputs in failed] == [(ADMITTED[0], "worker lost")]
- # Its one attempt's query may still run, which nothing else stops.
- assert [(inputs.configuration_id, len(inputs.query_ids)) for inputs in stopped] == [(ADMITTED[0], 1)]
+ # Each attempt's query may still run, which nothing else stops.
+ assert [(inputs.configuration_id, len(inputs.query_ids)) for inputs in stopped] == [
+ (ADMITTED[0], CHECK_RETRY_POLICY.maximum_attempts)
+ ]
assert sorted(
(str(o.configuration_id), o.groups[0].new_state) for batch in recorded for o in batch.outcomes
) == sorted([(ADMITTED[0], "errored"), (ADMITTED[1], "firing"), (AI_CHECK, "firing")])
- real_time_heartbeats = [
+ check_heartbeats = [
event.activity_task_scheduled_event_attributes.heartbeat_timeout.ToTimedelta()
for event in history.events
if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED
and event.activity_task_scheduled_event_attributes.activity_type.name
== "evaluate_platform_insight_check_activity"
- and event.activity_task_scheduled_event_attributes.heartbeat_timeout.ToTimedelta()
]
- # Only the real-time check heartbeats, so a dead worker's check is retried inside its budget.
- assert real_time_heartbeats == [REAL_TIME_CHECK_HEARTBEAT_TIMEOUT]
+ # Every cadence heartbeats, so a check whose worker never ran it is retried inside its budget.
+ assert sorted(check_heartbeats) == sorted([CHECK_HEARTBEAT_TIMEOUT, REAL_TIME_CHECK_HEARTBEAT_TIMEOUT])
children = [
event.child_workflow_execution_started_event_attributes
for event in history.events
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index ccb3ac6f1390..bfc0135d398c 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -852,7 +852,8 @@ class TestPlatformInsightEvaluation(APIBaseTest):
assert retried is not None and over_the_limit is None
assert inflight_alert_ids(INFLIGHT_KEY) == {str(held.id)}
- assert inflight_alert_ids(QUERY_INFLIGHT_KEY) == set()
+ # The kill may not have landed, so the timed-out query stays counted until its thread frees it.
+ assert inflight_alert_ids(QUERY_INFLIGHT_KEY) == {"run:1:1"}
cancel.assert_any_call(self.team.id, "alerts-platform-insight:run:1:1")
@override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHECKS=1)