Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): keep platform insight checks out of query single flights (#127919)

Jimmy Zhu6c695f394ecc
diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py
index 0252ec88f89d..af4837d9910f 100644
--- a/posthog/hogql_queries/query_runner.py
+++ b/posthog/hogql_queries/query_runner.py
@@ -224,6 +224,7 @@ from posthog.query_cache.single_flight import (
QUERY_SINGLE_FLIGHT_FLAG,
FlightWait,
QuerySingleFlight,
+ query_single_flight_skipped,
)
from posthog.query_scan.flag import QueryScanFlag, QueryScanMode, get_query_scan_flag
from posthog.query_scan.serve import hydrate_response_scan
@@ -2738,7 +2739,8 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
# rules out serving a leader's entry too. An export never stores its result, so its leader
# would have nothing to hand a follower.
return (
- self._query_single_flight_enabled
+ not query_single_flight_skipped()
+ and self._query_single_flight_enabled
and not self.requires_fresh_calculation()
and self.limit_context != LimitContext.EXPORT
)
diff --git a/posthog/hogql_queries/test/test_query_runner.py b/posthog/hogql_queries/test/test_query_runner.py
index 9b3a2f2a7077..fcd6c750f82d 100644
--- a/posthog/hogql_queries/test/test_query_runner.py
+++ b/posthog/hogql_queries/test/test_query_runner.py
@@ -1,6 +1,7 @@
import json
import time
import threading
+from contextlib import nullcontext
from datetime import UTC, datetime, timedelta
from typing import Any, Literal, Optional, cast
from zoneinfo import ZoneInfo
@@ -110,7 +111,13 @@ from posthog.query_cache.failures import (
QueryFailureCache,
WarmingQueryFailureCache,
)
-from posthog.query_cache.single_flight import QUERY_SINGLE_FLIGHT_FLAG, FlightWait, QuerySingleFlight, SharedFailure
+from posthog.query_cache.single_flight import (
+ QUERY_SINGLE_FLIGHT_FLAG,
+ FlightWait,
+ QuerySingleFlight,
+ SharedFailure,
+ skip_query_single_flight,
+)
from posthog.query_cache.storage import entry_redis_key
from posthog.query_scan.flag import QueryScanFlag, QueryScanMode
from posthog.shared_link_user import SharedLinkUser
@@ -2851,15 +2858,23 @@ class TestQuerySingleFlightRunner(BaseTest):
response = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS, cache_age_seconds=0)
assert response.is_cached is True # a zero cache age window must not reject the leader's own write
- @parameterized.expand([("flag_off", False, None), ("export", True, LimitContext.EXPORT)])
- def test_runs_that_cannot_share_a_result_never_touch_the_flight(self, _name, flag_on, limit_context):
+ @parameterized.expand(
+ [
+ ("flag_off", False, None, False),
+ ("export", True, LimitContext.EXPORT, False),
+ ("skipped_by_the_caller", True, None, True),
+ ]
+ )
+ def test_runs_that_cannot_share_a_result_never_touch_the_flight(self, _name, flag_on, limit_context, skipped):
runner_class = setup_test_query_runner_class()
runner = runner_class(query={"some_attr": "bla"}, team=self.team, limit_context=limit_context)
flags = _single_flight_flag if flag_on else (lambda *args, **kwargs: False)
- with mock.patch.object(QuerySingleFlight, "acquire", autospec=True) as mock_acquire:
+ with mock.patch.object(QuerySingleFlight, "acquire", autospec=True, return_value=False) as mock_acquire:
with mock.patch("posthoganalytics.feature_enabled", side_effect=flags):
- runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)
+ with skip_query_single_flight() if skipped else nullcontext():
+ response = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)
mock_acquire.assert_not_called()
+ assert response.is_cached is False
class TestRunnersBuildDatabaseOnce(ClickhouseTestMixin, APIBaseTest):
diff --git a/posthog/query_cache/single_flight.py b/posthog/query_cache/single_flight.py
index 5c9473e8f004..09739902dff0 100644
--- a/posthog/query_cache/single_flight.py
+++ b/posthog/query_cache/single_flight.py
@@ -2,6 +2,9 @@ import json
import time
import uuid
import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
from dataclasses import asdict, fields
from datetime import datetime
from typing import Any, Literal, Optional
@@ -47,6 +50,27 @@ FLIGHT_MAX_POLL_INTERVAL = 1.0
# Acquiring the lock drops the previous result, so the result can safely outlive a stalled follower.
FLIGHT_RESULT_TTL = 60
+_SKIPPED: ContextVar[bool] = ContextVar("query_single_flight_skipped", default=False)
+
+
+@contextmanager
+def skip_query_single_flight() -> Iterator[None]:
+ """Runs every blocking query in the block on its own, neither leading nor following a flight.
+
+ For a caller that runs the same query beside a run in another flight partition. Both write one
+ cache entry, so the other run can replace the leader's entry before a follower reads it, and the
+ follower then has nothing to serve."""
+ token = _SKIPPED.set(True)
+ try:
+ yield
+ finally:
+ _SKIPPED.reset(token)
+
+
+def query_single_flight_skipped() -> bool:
+ return _SKIPPED.get()
+
+
# Acquiring the lock also drops the result the previous leader published, so a result can only ever
# belong to the flight that is currently in progress or just ended.
_ACQUIRE_LOCK_SCRIPT = """
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index 91789ed531fb..8a22dfa868fc 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -34,6 +34,7 @@ from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimit
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.errors import CH_TRANSIENT_ERRORS, QueryErrorCategory, classify_query_error
+from posthog.query_cache.single_flight import skip_query_single_flight
from posthog.schema_enums import AlertCalculationInterval
from posthog.tasks.alerts.schedule_restriction import is_utc_datetime_blocked
from posthog.tasks.alerts.utils import AlertEvaluationResult, render_alert_insight_chart, skip_because_of_weekend
@@ -478,14 +479,17 @@ def evaluate_insight_check(
if lag_ms > 0:
safe_record(record_scheduler_lag, SourceKind.INSIGHT.value, lag_ms)
try:
- outcome = _decide(
- checks[0],
- cutoff,
- evaluation_id=evaluation_id,
- attempt=attempt,
- final_attempt=final_attempt,
- uses_llm_detector=uses_llm_detector,
- )
+ # Other ClickHouse users, production's included, run this query in flights of their own but write
+ # the same cache entry, so a platform follower can find their entry in place of its leader's.
+ with skip_query_single_flight():
+ outcome = _decide(
+ checks[0],
+ cutoff,
+ evaluation_id=evaluation_id,
+ attempt=attempt,
+ final_attempt=final_attempt,
+ uses_llm_detector=uses_llm_detector,
+ )
except LLMDetectorUnavailableError:
retrying = True
raise
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index 507c40828edb..ccb3ac6f1390 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -22,6 +22,7 @@ from posthog.clickhouse.query_tagging import get_query_tags
from posthog.constants import AvailableFeature
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded
from posthog.models.scoping import team_scope
+from posthog.query_cache.single_flight import query_single_flight_skipped
from posthog.redis import get_client
from posthog.schema_enums import AlertCalculationInterval
from posthog.tasks.alerts.utils import AlertEvaluationResult, RenderedInsightChart, next_check_time
@@ -213,7 +214,7 @@ class TestPlatformInsightEvaluation(APIBaseTest):
slot = slot_of(configuration.next_check_at, CUTOFF)
expires_at = time.time() + 3600
assert plan_insight_batch(self.team.id, slot, CUTOFF, expires_at=expires_at) == (str(configuration.id),)
- side_effect = result if isinstance(result, Exception | list) else None
+ side_effect = None if isinstance(result, AlertEvaluationResult) else result
returned = result if isinstance(result, AlertEvaluationResult) else None
with patch(f"{_MODULE}.check_alert_for_insight", side_effect=side_effect, return_value=returned) as query:
evaluation = evaluate_insight_check(
@@ -432,9 +433,16 @@ class TestPlatformInsightEvaluation(APIBaseTest):
before = AlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id)
configuration = self._copy(alert)
- outcome, _ = self._evaluate(configuration, result=AlertEvaluationResult(value=150.0, breaches=["above 100"]))
+ skipped_during_query: list[bool] = []
+
+ def query(*args: Any, **kwargs: Any) -> AlertEvaluationResult:
+ skipped_during_query.append(query_single_flight_skipped())
+ return AlertEvaluationResult(value=150.0, breaches=["above 100"])
+
+ outcome, _ = self._evaluate(configuration, result=query)
assert outcome is not None
assert get_query_tags().ch_user == ClickHouseUser.ALERTS_PLATFORM_INSIGHT
+ assert skipped_during_query == [True]
record_outcomes(self.team.id, (outcome,), CUTOFF)
assert (outcome.groups[0].kind, outcome.groups[0].value, outcome.evaluation_key) == (