:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(alerts): keep platform insight checks out of query single flights (#127919)
Jimmy Zhu6c695f394ecc
diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.pyindex 0252ec88f89d..af4837d9910f 100644--- a/posthog/hogql_queries/query_runner.py+++ b/posthog/hogql_queries/query_runner.py@@ -224,6 +224,7 @@ from posthog.query_cache.single_flight import (QUERY_SINGLE_FLIGHT_FLAG,FlightWait,QuerySingleFlight,+ query_single_flight_skipped,)from posthog.query_scan.flag import QueryScanFlag, QueryScanMode, get_query_scan_flagfrom posthog.query_scan.serve import hydrate_response_scan@@ -2738,7 +2739,8 @@ class QueryRunner(ABC, Generic[Q, R, CR]):# rules out serving a leader's entry too. An export never stores its result, so its leader# would have nothing to hand a follower.return (- self._query_single_flight_enabled+ not query_single_flight_skipped()+ and self._query_single_flight_enabledand not self.requires_fresh_calculation()and self.limit_context != LimitContext.EXPORT)diff --git a/posthog/hogql_queries/test/test_query_runner.py b/posthog/hogql_queries/test/test_query_runner.pyindex 9b3a2f2a7077..fcd6c750f82d 100644--- a/posthog/hogql_queries/test/test_query_runner.py+++ b/posthog/hogql_queries/test/test_query_runner.py@@ -1,6 +1,7 @@import jsonimport timeimport threading+from contextlib import nullcontextfrom datetime import UTC, datetime, timedeltafrom typing import Any, Literal, Optional, castfrom zoneinfo import ZoneInfo@@ -110,7 +111,13 @@ from posthog.query_cache.failures import (QueryFailureCache,WarmingQueryFailureCache,)-from posthog.query_cache.single_flight import QUERY_SINGLE_FLIGHT_FLAG, FlightWait, QuerySingleFlight, SharedFailure+from posthog.query_cache.single_flight import (+ QUERY_SINGLE_FLIGHT_FLAG,+ FlightWait,+ QuerySingleFlight,+ SharedFailure,+ skip_query_single_flight,+)from posthog.query_cache.storage import entry_redis_keyfrom posthog.query_scan.flag import QueryScanFlag, QueryScanModefrom posthog.shared_link_user import SharedLinkUser@@ -2851,15 +2858,23 @@ class TestQuerySingleFlightRunner(BaseTest):response = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS, cache_age_seconds=0)assert response.is_cached is True # a zero cache age window must not reject the leader's own write- @parameterized.expand([("flag_off", False, None), ("export", True, LimitContext.EXPORT)])- def test_runs_that_cannot_share_a_result_never_touch_the_flight(self, _name, flag_on, limit_context):+ @parameterized.expand(+ [+ ("flag_off", False, None, False),+ ("export", True, LimitContext.EXPORT, False),+ ("skipped_by_the_caller", True, None, True),+ ]+ )+ def test_runs_that_cannot_share_a_result_never_touch_the_flight(self, _name, flag_on, limit_context, skipped):runner_class = setup_test_query_runner_class()runner = runner_class(query={"some_attr": "bla"}, team=self.team, limit_context=limit_context)flags = _single_flight_flag if flag_on else (lambda *args, **kwargs: False)- with mock.patch.object(QuerySingleFlight, "acquire", autospec=True) as mock_acquire:+ with mock.patch.object(QuerySingleFlight, "acquire", autospec=True, return_value=False) as mock_acquire:with mock.patch("posthoganalytics.feature_enabled", side_effect=flags):- runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)+ with skip_query_single_flight() if skipped else nullcontext():+ response = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)mock_acquire.assert_not_called()+ assert response.is_cached is Falseclass TestRunnersBuildDatabaseOnce(ClickhouseTestMixin, APIBaseTest):diff --git a/posthog/query_cache/single_flight.py b/posthog/query_cache/single_flight.pyindex 5c9473e8f004..09739902dff0 100644--- a/posthog/query_cache/single_flight.py+++ b/posthog/query_cache/single_flight.py@@ -2,6 +2,9 @@ import jsonimport timeimport uuidimport threading+from collections.abc import Iterator+from contextlib import contextmanager+from contextvars import ContextVarfrom dataclasses import asdict, fieldsfrom datetime import datetimefrom typing import Any, Literal, Optional@@ -47,6 +50,27 @@ FLIGHT_MAX_POLL_INTERVAL = 1.0# Acquiring the lock drops the previous result, so the result can safely outlive a stalled follower.FLIGHT_RESULT_TTL = 60+_SKIPPED: ContextVar[bool] = ContextVar("query_single_flight_skipped", default=False)+++@contextmanager+def skip_query_single_flight() -> Iterator[None]:+ """Runs every blocking query in the block on its own, neither leading nor following a flight.++ For a caller that runs the same query beside a run in another flight partition. Both write one+ cache entry, so the other run can replace the leader's entry before a follower reads it, and the+ follower then has nothing to serve."""+ token = _SKIPPED.set(True)+ try:+ yield+ finally:+ _SKIPPED.reset(token)+++def query_single_flight_skipped() -> bool:+ return _SKIPPED.get()++# Acquiring the lock also drops the result the previous leader published, so a result can only ever# belong to the flight that is currently in progress or just ended._ACQUIRE_LOCK_SCRIPT = """diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex 91789ed531fb..8a22dfa868fc 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -34,6 +34,7 @@ from posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimitfrom posthog.clickhouse.query_tagging import Feature, Product, tag_queriesfrom posthog.errors import CH_TRANSIENT_ERRORS, QueryErrorCategory, classify_query_error+from posthog.query_cache.single_flight import skip_query_single_flightfrom posthog.schema_enums import AlertCalculationIntervalfrom posthog.tasks.alerts.schedule_restriction import is_utc_datetime_blockedfrom posthog.tasks.alerts.utils import AlertEvaluationResult, render_alert_insight_chart, skip_because_of_weekend@@ -478,14 +479,17 @@ def evaluate_insight_check(if lag_ms > 0:safe_record(record_scheduler_lag, SourceKind.INSIGHT.value, lag_ms)try:- outcome = _decide(- checks[0],- cutoff,- evaluation_id=evaluation_id,- attempt=attempt,- final_attempt=final_attempt,- uses_llm_detector=uses_llm_detector,- )+ # Other ClickHouse users, production's included, run this query in flights of their own but write+ # the same cache entry, so a platform follower can find their entry in place of its leader's.+ with skip_query_single_flight():+ outcome = _decide(+ checks[0],+ cutoff,+ evaluation_id=evaluation_id,+ attempt=attempt,+ final_attempt=final_attempt,+ uses_llm_detector=uses_llm_detector,+ )except LLMDetectorUnavailableError:retrying = Trueraisediff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex 507c40828edb..ccb3ac6f1390 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -22,6 +22,7 @@ from posthog.clickhouse.query_tagging import get_query_tagsfrom posthog.constants import AvailableFeaturefrom posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceededfrom posthog.models.scoping import team_scope+from posthog.query_cache.single_flight import query_single_flight_skippedfrom posthog.redis import get_clientfrom posthog.schema_enums import AlertCalculationIntervalfrom posthog.tasks.alerts.utils import AlertEvaluationResult, RenderedInsightChart, next_check_time@@ -213,7 +214,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):slot = slot_of(configuration.next_check_at, CUTOFF)expires_at = time.time() + 3600assert plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at) == (str(configuration.id),)- side_effect = result if isinstance(result, Exception | list) else None+ side_effect = None if isinstance(result, AlertEvaluationResult) else resultreturned = result if isinstance(result, AlertEvaluationResult) else Nonewith patch(f"{_MODULE}.check_alert_for_insight", side_effect=side_effect, return_value=returned) as query:evaluation = evaluate_insight_check(@@ -432,9 +433,16 @@ class TestPlatformInsightEvaluation(APIBaseTest):before = AlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id)configuration = self._copy(alert)- outcome, _ = self._evaluate(configuration, result=AlertEvaluationResult(value=150.0, breaches=["above 100"]))+ skipped_during_query: list[bool] = []++ def query(*args: Any, **kwargs: Any) -> AlertEvaluationResult:+ skipped_during_query.append(query_single_flight_skipped())+ return AlertEvaluationResult(value=150.0, breaches=["above 100"])++ outcome, _ = self._evaluate(configuration, result=query)assert outcome is not Noneassert get_query_tags().ch_user == ClickHouseUser.ALERTS_PLATFORM_INSIGHT+ assert skipped_during_query == [True]record_outcomes(self.team.id, (outcome,), CUTOFF)assert (outcome.groups[0].kind, outcome.groups[0].value, outcome.evaluation_key) == (