:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): pin each run's events route for all-time lookups (#120298)
Vasco de Krijger911d39f1aba7
diff --git a/posthog/hogql_queries/hogql_query_runner.py b/posthog/hogql_queries/hogql_query_runner.pyindex 8bc7ec64ea8e..bc279b13d85f 100644--- a/posthog/hogql_queries/hogql_query_runner.py+++ b/posthog/hogql_queries/hogql_query_runner.py@@ -68,9 +68,13 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):self._direct_engine: str | None = Noneself._managed_warehouse_sql_mode: ManagedWarehouseSQLMode | None = Nonesuper().__init__(*args, **kwargs)- if self.query.connectionId and self.modifiers.useNewEventsCluster:+ if self.modifiers.useNewEventsCluster and not self._reads_events_route():raise ExposedHogQLError("Native events cluster queries cannot use a warehouse connection.")+ def _reads_events_route(self) -> bool:+ # A direct connection runs on another database, so the events route never applies to it.+ return not self.query.connectionId+# Treat SQL query caching like day insightdef cache_target_age(self, last_refresh: Optional[datetime], lazy: bool = False) -> Optional[datetime]:if last_refresh is None:diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.pyindex 9ef68d4f9e88..b57681a9f285 100644--- a/posthog/hogql_queries/query_runner.py+++ b/posthog/hogql_queries/query_runner.py@@ -2,11 +2,11 @@ import hashlibimport inspectimport threadingfrom abc import ABC, abstractmethod-from collections.abc import Iterable, Sequence+from collections.abc import Callable, Iterable, Sequencefrom contextlib import ExitStack, suppressfrom datetime import UTC, datetime, timedeltafrom enum import StrEnum-from functools import cache, cached_property+from functools import cache, cached_property, wrapsfrom time import perf_counterfrom types import UnionTypefrom typing import (@@ -190,7 +190,15 @@ from posthog.hogql_queries.validation.validation import (run_validation_rules,)from posthog.models import Team, User-from posthog.models.event.new_events_schema import get_new_events_cluster_host, new_events_cluster_host_scope+from posthog.models.event.new_events_schema import (+ events_route_cache_modifiers,+ events_route_scope,+ get_new_events_cluster_host,+ new_events_cluster_host_scope,+ strip_events_route_modifiers,+ use_new_events_schema,+ with_pinned_events_route,+)from posthog.models.instance_setting import get_instance_settingfrom posthog.models.team import WeekStartDayfrom posthog.models.team.event_retention import events_retention_months_for_team@@ -1815,6 +1823,7 @@ R = TypeVar("R", bound=BaseModel)# Unfortunately inheritance is also not a thing here, because we lose this info in the schema.ts->.json->.py journeyCR = TypeVar("CR", bound=GenericCachedQueryResponse)SourceT = TypeVar("SourceT", bound=BaseModel)+_RunMethod = TypeVar("_RunMethod", bound=Callable[..., Any])def resolve_series_custom_name(series: Any, raw_label: str | None) -> str | None:@@ -1873,6 +1882,15 @@ def team_configs_for_cache_key(team: Team) -> dict[str, dict | str]:}+def _pin_events_route(run: _RunMethod) -> _RunMethod:+ @wraps(run)+ def pinned(self: "QueryRunner", *args: Any, **kwargs: Any) -> Any:+ with events_route_scope(self.modifiers):+ return run(self, *args, **kwargs)++ return cast(_RunMethod, pinned)++class QueryRunner(ABC, Generic[Q, R, CR]):query: Qresponse: R@@ -1934,6 +1952,8 @@ class QueryRunner(ABC, Generic[Q, R, CR]):_modifiers = modifiers or extract_modifiers(query)self.modifiers = create_default_modifiers_for_team(team, _modifiers)+ if self.modifiers.useNewEventsSchema is None:+ self.modifiers.useNewEventsSchema = use_new_events_schema(team.pk, self.modifiers)self.query = queryself.modifiers.webAnalyticsFirstPageviewFilters = resolve_first_pageview_filters_modifier(query, team, self.modifiers.webAnalyticsFirstPageviewFilters@@ -2081,6 +2101,20 @@ class QueryRunner(ABC, Generic[Q, R, CR]):def query_status_labels(self) -> list[str] | None:return None+ def _reads_events_route(self) -> bool:+ """True when this query reads the PostHog events table.++ Only then does the events route belong in the query's cache key and in its queued payload.+ """+ return True++ def _query_json_with_events_schema(self) -> dict[str, Any]:+ query_json = self.query.model_dump()+ # A query that reads no events table needs no pin, and pure Trino rejects modifiers the caller did not set.+ if self.modifiers.useNewEventsSchema and self._reads_events_route():+ return with_pinned_events_route(query_json, self.modifiers)+ return query_json+def enqueue_async_calculation(self,*,@@ -2116,7 +2150,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):sharing_configuration_id=sharing_configuration_id,insight_id=cache_manager.insight_id,dashboard_id=cache_manager.dashboard_id,- query_json=self.query.model_dump(),+ query_json=self._query_json_with_events_schema(),query_id=self.query_id or cache_manager.cache_key, # Use cache key as query ID to avoid duplicatescache_key=cache_manager.cache_key,labels=self.query_status_labels(),@@ -2383,6 +2417,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):return query_result, query_duration_ms@new_events_cluster_host_scope()+ @_pin_events_routedef run(self,execution_mode: ExecutionMode = ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE,@@ -3122,11 +3157,15 @@ class QueryRunner(ABC, Generic[Q, R, CR]):# note: to_dict already strips custom_name from series (see schema_helpers.py)query = to_dict(self.query)query.pop("tags", None)+ # The resolved modifiers key the events route, so requested values that resolve the same share one entry.+ query = strip_events_route_modifiers(query, drop_empty=True)++ cache_modifiers = events_route_cache_modifiers(to_dict(self.modifiers), reads_events=self._reads_events_route())payload = {"query_runner": self.__class__.__name__,"query": query,"team_id": self.team.pk,- "hogql_modifiers": to_dict(self.modifiers),+ "hogql_modifiers": cache_modifiers,"products_modifiers": self._products_modifiers_for_cache(),"limit_context": self._limit_context_aliased_for_cache,"timezone": self.team.timezone,diff --git a/posthog/hogql_queries/test/test_query_runner.py b/posthog/hogql_queries/test/test_query_runner.pyindex 138755e6c207..2930654676b5 100644--- a/posthog/hogql_queries/test/test_query_runner.py+++ b/posthog/hogql_queries/test/test_query_runner.py@@ -91,6 +91,7 @@ from posthog.hogql_queries.query_runner import (shared_insights_execution_mode,)from posthog.hogql_queries.utils.query_date_range import QueryDateRange+from posthog.models.event.new_events_schema import get_events_routefrom posthog.models.instance_setting import override_instance_configfrom posthog.models.organization import OrganizationMembershipfrom posthog.models.sharing_configuration import SharingConfiguration@@ -226,6 +227,24 @@ class TestQueryRunner(BaseTest):def setup_test_query_runner_class(self, base: type[QueryRunner] = QueryRunner):return setup_test_query_runner_class(base)+ @parameterized.expand([(False,), (True,)])+ def test_run_pins_its_events_route_for_lookups(self, events_json: bool) -> None:+ TestQueryRunner = self.setup_test_query_runner_class()+ routes: list[HogQLQueryModifiers | None] = []++ def calculate(_runner: QueryRunner) -> TheTestBasicQueryResponse:+ routes.append(get_events_route())+ return TheTestBasicQueryResponse(results=[])++ runner = TestQueryRunner(+ query={"some_attr": "bla"}, team=self.team, modifiers=HogQLQueryModifiers(useNewEventsSchema=events_json)+ )+ with mock.patch.object(TestQueryRunner, "_calculate", autospec=True, side_effect=calculate):+ runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)++ assert routes == [HogQLQueryModifiers(useNewEventsSchema=events_json)]+ assert get_events_route() is None+def test_sync_warning_attach_preserves_other_warning_kinds(self):# The accumulator attach replaces the response's warnings with the collected sync warnings.# Access control warnings ride the same shared field and were silently dropped whenever a@@ -693,7 +712,11 @@ class TestQueryRunner(BaseTest):self.assertEqual(first.query, HogQLQuery(query="SELECT 1"))self.assertEqual(first.get_cache_key(), second.get_cache_key())- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)+ @override_settings(+ PERSON_ON_EVENTS_OVERRIDE=False,+ PERSON_ON_EVENTS_V2_OVERRIDE=False,+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,+ )def test_cache_payload(self):TestQueryRunner = self.setup_test_query_runner_class()@@ -828,7 +851,11 @@ class TestQueryRunner(BaseTest):return runner.get_cache_key()@parameterized.expand([("new_team", False), ("existing_team", True)])- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)+ @override_settings(+ PERSON_ON_EVENTS_OVERRIDE=False,+ PERSON_ON_EVENTS_V2_OVERRIDE=False,+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,+ )def test_cache_key(self, _name: str, existing_team: bool) -> None:TestQueryRunner = self.setup_test_query_runner_class()team = Team.objects.create(organization=self.organization)@@ -854,7 +881,11 @@ class TestQueryRunner(BaseTest):assert tagged.get_cache_key() == untagged.get_cache_key()- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)+ @override_settings(+ PERSON_ON_EVENTS_OVERRIDE=False,+ PERSON_ON_EVENTS_V2_OVERRIDE=False,+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,+ )def test_cache_key_runner_subclass(self):TestQueryRunner = self.setup_test_query_runner_class()@@ -868,7 +899,11 @@ class TestQueryRunner(BaseTest):cache_key = self._cache_key_with_fixed_payload_team_id(runner)assert cache_key == f"cache_{team.pk}_9dcfced89edfbfcd1e0fd380380c7a5e4a9957386fc22172c1b83548a9af6d02"- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)+ @override_settings(+ PERSON_ON_EVENTS_OVERRIDE=False,+ PERSON_ON_EVENTS_V2_OVERRIDE=False,+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,+ )def test_cache_key_different_timezone(self):TestQueryRunner = self.setup_test_query_runner_class()team = Team.objects.create(organization=self.organization)@@ -1938,6 +1973,29 @@ class TestSharedInsightsExecutionMode(BaseTest):expected = sharing_configuration.pk if is_shared_viewer else Noneassert mock_enqueue.call_args.kwargs["sharing_configuration_id"] == expected+ @mock.patch("posthog.hogql_queries.query_runner.enqueue_process_query_task")+ def test_async_payload_pins_a_resolved_native_table(self, mock_enqueue: mock.MagicMock) -> None:+ with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True):+ direct = HogQLQueryRunner(query=HogQLQuery(query="SELECT 1"), team=self.team)+ direct.query = direct.query.model_copy(update={"connectionId": "direct"})+ with mock.patch.object(HogQLQueryRunner, "_validate_direct_connection"):+ direct.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)+ assert mock_enqueue.call_args.kwargs["query_json"] == direct.query.model_dump()+ assert "useNewEventsSchema" not in direct.get_cache_payload()["hogql_modifiers"]+ runner = HogQLQueryRunner(query=HogQLQuery(query="SELECT 1"), team=self.team)+ runner.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)+ with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False):+ worker = HogQLQueryRunner(query=mock_enqueue.call_args.kwargs["query_json"], team=self.team)+ runner.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)+ legacy = HogQLQueryRunner(query=HogQLQuery(query="SELECT 1"), team=self.team)+ legacy.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)++ assert worker.modifiers.useNewEventsSchema is True+ assert worker.get_cache_key() == runner.get_cache_key()+ assert mock_enqueue.call_args.kwargs["query_json"] == legacy.query.model_dump()+ assert legacy.get_cache_key() != runner.get_cache_key()+ assert "useNewEventsSchema" not in legacy.get_cache_payload()["hogql_modifiers"]+@pytest.mark.eeclass TestQueryRunnerAccessControlFingerprint(BaseTest):diff --git a/posthog/hogql_queries/utils/test/test_timestamp_utils.py b/posthog/hogql_queries/utils/test/test_timestamp_utils.pyindex 6a17c6287b81..f07b172fdf58 100644--- a/posthog/hogql_queries/utils/test/test_timestamp_utils.py+++ b/posthog/hogql_queries/utils/test/test_timestamp_utils.py@@ -2,7 +2,13 @@ import datetimefrom zoneinfo import ZoneInfoimport time_machine-from posthog.test.base import APIBaseTest, ClickhouseDestroyTablesMixin, _create_event, flush_persons_and_events+from posthog.test.base import (+ APIBaseTest,+ BaseTest,+ ClickhouseDestroyTablesMixin,+ _create_event,+ flush_persons_and_events,+)from unittest.mock import MagicMock, patchfrom django.core.cache import cache@@ -11,20 +17,25 @@ from django.test import override_settingsfrom dateutil import parserfrom parameterized import parameterized-from posthog.schema import ActionsNode, DataWarehouseNode, DateRange, EventsNode, IntervalType+from posthog.schema import ActionsNode, DataWarehouseNode, DateRange, EventsNode, HogQLQueryModifiers, IntervalTypefrom posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context+from posthog.clickhouse.workload import Workloadfrom posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom posthog.hogql_queries.utils.timestamp_utils import (EARLIEST_EVENT_TIMESTAMP,_coerce_to_datetime,+ _events_route_cache_key,+ _events_route_modifiers,_get_earliest_timestamp_cache_key,format_label_date,get_earliest_timestamp_from_series,get_earliest_timestamp_unfiltered,)+from posthog.models.event.new_events_schema import events_route_scope+from posthog.models.instance_setting import override_instance_configfrom posthog.models.team import WeekStartDayfrom products.actions.backend.models.action import Action@@ -526,7 +537,9 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):# TTL window. A cache hit must still pass through _coerce_to_datetime instead of# returning the raw value, otherwise downstream date math hits AttributeError/TypeError.node = EventsNode(event="$pageview")- cache_key = _get_earliest_timestamp_cache_key(self.team, node)+ cache_key = _events_route_cache_key(+ self.team, _events_route_modifiers(self.team), _get_earliest_timestamp_cache_key(self.team, node)+ )# team timezone is UTC in tests; "2021-03-15" is the pre-fix raw Date/String shape.cache.set(cache_key, "2021-03-15", timeout=3600)@@ -553,7 +566,7 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):def test_single_path_query_tags(self, _name, caller_product, caller_feature, expected_product, expected_feature):captured: dict[str, object] = {}- def capture(query, team, user=None):+ def capture(query, team, user=None, modifiers=None):tags = get_query_tags()captured["product"] = tags.productcaptured["feature"] = tags.feature@@ -572,3 +585,74 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):self.assertEqual(captured["product"], expected_product)self.assertEqual(captured["feature"], expected_feature)self.assertEqual(captured["lookup"], "earliest_timestamp")+++class TestEarliestTimestampRouting(BaseTest):+ def tearDown(self):+ super().tearDown()+ cache.clear()++ @parameterized.expand([("series",), ("unfiltered",)])+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False, CLICKHOUSE_NATIVE_EVENTS_HOST="native.example.com")+ def test_native_route_reads_the_native_cluster_and_skips_the_legacy_cache(self, lookup: str) -> None:+ node = EventsNode(event="$pageview")+ legacy = datetime.datetime(2022, 1, 1, tzinfo=ZoneInfo("UTC"))+ native = datetime.datetime(2020, 1, 1, tzinfo=ZoneInfo("UTC"))+ if lookup == "series":+ cache.set(_get_earliest_timestamp_cache_key(self.team, node), legacy)+ else:+ cache.set(f"earliest_timestamp_unfiltered_{self.team.pk}", legacy)++ def earliest() -> datetime.datetime:+ if lookup == "series":+ return get_earliest_timestamp_from_series(self.team, [node])+ return get_earliest_timestamp_unfiltered(self.team)++ with (+ override_instance_config("CLICKHOUSE_NATIVE_EVENTS_HOST", "native.example.com"),+ override_instance_config("CLICKHOUSE_NATIVE_EVENTS_TEAMS", [self.team.pk]),+ patch(+ "posthog.hogql.query.sync_execute", return_value=([(native,)], [("timestamp", "DateTime")])+ ) as execute,+ ):+ with events_route_scope(HogQLQueryModifiers(useNewEventsCluster=True, useNewEventsSchema=True)):+ self.assertEqual(earliest(), native)+ self.assertEqual(earliest(), native)+ self.assertEqual(earliest(), legacy)+ execute.assert_called_once()+ self.assertEqual(execute.call_args.kwargs["workload"], Workload.NEW_EVENTS_CLUSTER)+ self.assertIn("events_json", execute.call_args.args[0])++ def test_route_keys_never_collide_with_legacy_event_keys(self) -> None:+ legacy_events_json_event = datetime.datetime(2022, 1, 1, tzinfo=ZoneInfo("UTC"))+ cache.set(+ _get_earliest_timestamp_cache_key(self.team, EventsNode(event="events_json")), legacy_events_json_event+ )+ native = datetime.datetime(2020, 1, 1, tzinfo=ZoneInfo("UTC"))+ with (+ patch("posthog.hogql.query.sync_execute", return_value=([(native,)], [("timestamp", "DateTime")])),+ events_route_scope(HogQLQueryModifiers(useNewEventsSchema=True)),+ ):+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [EventsNode(event=None)]), native)++ @parameterized.expand([("allowlisted_project", True, {}), ("project_setting", False, {"useNewEventsSchema": True})])+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)+ def test_lookup_outside_a_run_keys_on_the_events_table_it_reads(+ self, _name: str, allowlisted: bool, team_modifiers: dict[str, bool]+ ) -> None:+ self.team.modifiers = team_modifiers+ node = EventsNode(event="$pageview")+ cache.set(+ _get_earliest_timestamp_cache_key(self.team, node), datetime.datetime(2022, 1, 1, tzinfo=ZoneInfo("UTC"))+ )+ native = datetime.datetime(2020, 1, 1, tzinfo=ZoneInfo("UTC"))+ with (+ override_instance_config(+ "CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.pk) if allowlisted else ""+ ),+ patch(+ "posthog.hogql.query.sync_execute", return_value=([(native,)], [("timestamp", "DateTime")])+ ) as execute,+ ):+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), native)+ self.assertIn("events_json", execute.call_args.args[0])diff --git a/posthog/hogql_queries/utils/timestamp_utils.py b/posthog/hogql_queries/utils/timestamp_utils.pyindex 0eac1f41a6ed..6419bf78f4e1 100644--- a/posthog/hogql_queries/utils/timestamp_utils.py+++ b/posthog/hogql_queries/utils/timestamp_utils.py@@ -18,12 +18,14 @@ from posthog.schema import (EventsNode,FunnelsDataWarehouseNode,GroupNode,+ HogQLQueryModifiers,LifecycleDataWarehouseNode,)from posthog.hogql import astfrom posthog.hogql.ast import SelectQueryfrom posthog.hogql.context import HogQLContext+from posthog.hogql.modifiers import create_default_modifiers_for_teamfrom posthog.hogql.property import action_to_exprfrom posthog.hogql.query import execute_hogql_query@@ -31,6 +33,7 @@ from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tfrom posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom posthog.models import Team, Userfrom posthog.models.event import DEFAULT_EARLIEST_TIME_DELTA+from posthog.models.event.new_events_schema import get_events_route, get_new_events_cluster_host, use_new_events_schemafrom posthog.models.team import WeekStartDayfrom posthog.utils import get_safe_cache@@ -135,6 +138,21 @@ def _get_earliest_timestamp_cache_key(raise ValueError(f"Unsupported node type: {type(node)}")+def _events_route_modifiers(team: Team) -> HogQLQueryModifiers:+ return create_default_modifiers_for_team(team, get_events_route())+++def _events_route_cache_key(team: Team, modifiers: HogQLQueryModifiers, cache_key: str) -> str:+ # Each events table and cluster holds its own copy of the events, so one route's result must not answer another.+ # The route goes in front, because legacy keys end in event names and table names that a suffix could collide with.+ native_table = use_new_events_schema(team.pk, modifiers)+ native_cluster = bool(modifiers.useNewEventsCluster)+ if not (native_table or native_cluster):+ return cache_key+ native_host = get_new_events_cluster_host() if native_cluster else None+ return f"events_route:{native_table}:{native_host}:{cache_key}"++def _coerce_to_datetime(value: Any, tz: tzinfo) -> datetime:"""Normalize a min(timestamp) result to a timezone-aware ``datetime``.@@ -197,7 +215,8 @@ def _get_earliest_timestamp_from_node("""# The cache is team-wide (keyed by team + node, not user): the value is table metadata, and row# access is still enforced on the insight's main query, so per-user entries aren't worth the misses.- cache_key = _get_earliest_timestamp_cache_key(team, node)+ modifiers = _events_route_modifiers(team)+ cache_key = _events_route_cache_key(team, modifiers, _get_earliest_timestamp_cache_key(team, node))cached_result = get_safe_cache(cache_key)if cached_result is not None:# Coerce on read too: entries cached before the coercion fix shipped hold a raw@@ -216,7 +235,7 @@ def _get_earliest_timestamp_from_node(earliest_timestamp = EARLIEST_EVENT_TIMESTAMPwith _earliest_timestamp_query_tags():- result = execute_hogql_query(query=query, team=team, user=user)+ result = execute_hogql_query(query=query, team=team, user=user, modifiers=modifiers)if result and len(result.results) > 0 and len(result.results[0]) > 0 and result.results[0][0] is not None:earliest_timestamp = _coerce_to_datetime(result.results[0][0], team.timezone_info)@@ -277,7 +296,8 @@ def get_earliest_timestamp_unfiltered(team: Team) -> datetime:legacy raw-SQL earliest-timestamp helper: floored at 2015-01-01, falling back tonow - DEFAULT_EARLIEST_TIME_DELTA when the team has no events."""- cache_key = f"earliest_timestamp_unfiltered_{team.pk}"+ modifiers = _events_route_modifiers(team)+ cache_key = _events_route_cache_key(team, modifiers, f"earliest_timestamp_unfiltered_{team.pk}")cached_result = get_safe_cache(cache_key)if cached_result is not None:return _coerce_to_datetime(cached_result, team.timezone_info)@@ -300,6 +320,7 @@ def get_earliest_timestamp_unfiltered(team: Team) -> datetime:team=team,# The read has no upper bound and no filter, so the first row in sort key order is the answer.context=HogQLContext(team_id=team.pk, order_events_reads_by_sort_key=True),+ modifiers=modifiers,)if result and len(result.results) > 0 and len(result.results[0]) > 0 and result.results[0][0] is not None:earliest_timestamp = _coerce_to_datetime(result.results[0][0], team.timezone_info)diff --git a/posthog/models/event/new_events_schema.py b/posthog/models/event/new_events_schema.pyindex 4400be21c41a..aae7ebbe24b4 100644--- a/posthog/models/event/new_events_schema.py+++ b/posthog/models/event/new_events_schema.py@@ -1,7 +1,7 @@from collections.abc import Iteratorfrom contextlib import contextmanagerfrom contextvars import ContextVar-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom django.conf import settings@@ -11,8 +11,68 @@ if TYPE_CHECKING:from posthog.schema import HogQLQueryModifiers-# TODO: Remove the per-query rollout option after the events cluster migration is complete.+# TODO: Remove the per-query rollout option, and the route helpers in this block, after the events cluster+# migration is complete.+EVENTS_ROUTE_MODIFIER_KEYS = ("useNewEventsSchema", "useNewEventsCluster")_query_host: ContextVar[list[str | None] | None] = ContextVar("new_events_cluster_host", default=None)+_query_route: ContextVar["HogQLQueryModifiers | None"] = ContextVar("new_events_query_route", default=None)+++def _copy_source_chain(query: dict[str, Any]) -> list[dict[str, Any]]:+ """Copies of the query and every source it wraps, outermost first. Other nested values stay shared."""+ nodes = [dict(query)]+ while isinstance(nodes[-1].get("source"), dict):+ nodes[-1]["source"] = dict(nodes[-1]["source"])+ nodes.append(nodes[-1]["source"])+ return nodes+++def strip_events_route_modifiers(query: dict[str, Any], *, drop_empty: bool = False) -> dict[str, Any]:+ """The serialized query without the requested events route.++ The resolved route keys the cache, so requested values that resolve to the same route must not split the+ cache key. With drop_empty, a modifiers field left empty is removed.+ """+ nodes = _copy_source_chain(query)+ for node in nodes:+ modifiers = node.get("modifiers")+ if isinstance(modifiers, dict):+ node["modifiers"] = {+ key: value for key, value in modifiers.items() if key not in EVENTS_ROUTE_MODIFIER_KEYS+ }+ if drop_empty and not node["modifiers"]:+ node.pop("modifiers")+ return nodes[0]+++def with_pinned_events_route(query: dict[str, Any], modifiers: "HogQLQueryModifiers") -> dict[str, Any]:+ """The serialized query with the resolved events route in place of the requested one.++ A worker rebuilds the runner from this payload. The resolved route goes into the innermost source, which+ every worker reads, so the worker reads the same table and writes the cache key the request reads.+ """+ if "modifiers" not in _copy_source_chain(query)[-1]:+ return query+ nodes = _copy_source_chain(strip_events_route_modifiers(query))+ route: dict[str, Any] = {"useNewEventsSchema": modifiers.useNewEventsSchema}+ if modifiers.useNewEventsCluster is not None:+ route["useNewEventsCluster"] = modifiers.useNewEventsCluster+ nodes[-1]["modifiers"] = {**(nodes[-1]["modifiers"] or {}), **route}+ return nodes[0]+++def events_route_cache_modifiers(modifiers: dict[str, Any], *, reads_events: bool) -> dict[str, Any]:+ """The resolved modifiers as they key the cache.++ A legacy-table query must keep the cache key it had before the native route existed, so the route values it+ resolves to are left out. A query that reads no events table is never keyed by one.+ """+ modifiers = dict(modifiers)+ if modifiers.get("useNewEventsSchema") is False or not reads_events:+ modifiers.pop("useNewEventsSchema", None)+ if not modifiers.get("useNewEventsCluster"):+ modifiers.pop("useNewEventsCluster", None)+ return modifiersdef get_new_events_cluster_host() -> str | None:@@ -39,6 +99,27 @@ def new_events_cluster_host_scope() -> Iterator[None]:_query_host.reset(token)+def get_events_route() -> "HogQLQueryModifiers | None":+ """The events cluster and table of the query run in progress, or None outside a query run."""+ return _query_route.get()+++@contextmanager+def events_route_scope(modifiers: "HogQLQueryModifiers") -> Iterator[None]:+ # Lookups inside a run, such as an all-time range's earliest event, must read the query's cluster and table.+ from posthog.schema import HogQLQueryModifiers # noqa: PLC0415++ token = _query_route.set(+ HogQLQueryModifiers(+ useNewEventsCluster=modifiers.useNewEventsCluster, useNewEventsSchema=modifiers.useNewEventsSchema+ )+ )+ try:+ yield+ finally:+ _query_route.reset(token)++def use_new_events_cluster(team_id: int | None, modifiers: "HogQLQueryModifiers | None") -> bool:from posthog.hogql.errors import ExposedHogQLError # noqa: PLC0415