Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): pin each run's events route for all-time lookups (#120298)

Vasco de Krijger911d39f1aba7
diff --git a/posthog/hogql_queries/hogql_query_runner.py b/posthog/hogql_queries/hogql_query_runner.py
index 8bc7ec64ea8e..bc279b13d85f 100644
--- a/posthog/hogql_queries/hogql_query_runner.py
+++ b/posthog/hogql_queries/hogql_query_runner.py
@@ -68,9 +68,13 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
self._direct_engine: str | None = None
self._managed_warehouse_sql_mode: ManagedWarehouseSQLMode | None = None
super().__init__(*args, **kwargs)
- if self.query.connectionId and self.modifiers.useNewEventsCluster:
+ if self.modifiers.useNewEventsCluster and not self._reads_events_route():
raise ExposedHogQLError("Native events cluster queries cannot use a warehouse connection.")
+ def _reads_events_route(self) -> bool:
+ # A direct connection runs on another database, so the events route never applies to it.
+ return not self.query.connectionId
+
# Treat SQL query caching like day insight
def cache_target_age(self, last_refresh: Optional[datetime], lazy: bool = False) -> Optional[datetime]:
if last_refresh is None:
diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py
index 9ef68d4f9e88..b57681a9f285 100644
--- a/posthog/hogql_queries/query_runner.py
+++ b/posthog/hogql_queries/query_runner.py
@@ -2,11 +2,11 @@ import hashlib
import inspect
import threading
from abc import ABC, abstractmethod
-from collections.abc import Iterable, Sequence
+from collections.abc import Callable, Iterable, Sequence
from contextlib import ExitStack, suppress
from datetime import UTC, datetime, timedelta
from enum import StrEnum
-from functools import cache, cached_property
+from functools import cache, cached_property, wraps
from time import perf_counter
from types import UnionType
from typing import (
@@ -190,7 +190,15 @@ from posthog.hogql_queries.validation.validation import (
run_validation_rules,
)
from posthog.models import Team, User
-from posthog.models.event.new_events_schema import get_new_events_cluster_host, new_events_cluster_host_scope
+from posthog.models.event.new_events_schema import (
+ events_route_cache_modifiers,
+ events_route_scope,
+ get_new_events_cluster_host,
+ new_events_cluster_host_scope,
+ strip_events_route_modifiers,
+ use_new_events_schema,
+ with_pinned_events_route,
+)
from posthog.models.instance_setting import get_instance_setting
from posthog.models.team import WeekStartDay
from posthog.models.team.event_retention import events_retention_months_for_team
@@ -1815,6 +1823,7 @@ R = TypeVar("R", bound=BaseModel)
# Unfortunately inheritance is also not a thing here, because we lose this info in the schema.ts->.json->.py journey
CR = TypeVar("CR", bound=GenericCachedQueryResponse)
SourceT = TypeVar("SourceT", bound=BaseModel)
+_RunMethod = TypeVar("_RunMethod", bound=Callable[..., Any])
def resolve_series_custom_name(series: Any, raw_label: str | None) -> str | None:
@@ -1873,6 +1882,15 @@ def team_configs_for_cache_key(team: Team) -> dict[str, dict | str]:
}
+def _pin_events_route(run: _RunMethod) -> _RunMethod:
+ @wraps(run)
+ def pinned(self: "QueryRunner", *args: Any, **kwargs: Any) -> Any:
+ with events_route_scope(self.modifiers):
+ return run(self, *args, **kwargs)
+
+ return cast(_RunMethod, pinned)
+
+
class QueryRunner(ABC, Generic[Q, R, CR]):
query: Q
response: R
@@ -1934,6 +1952,8 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
_modifiers = modifiers or extract_modifiers(query)
self.modifiers = create_default_modifiers_for_team(team, _modifiers)
+ if self.modifiers.useNewEventsSchema is None:
+ self.modifiers.useNewEventsSchema = use_new_events_schema(team.pk, self.modifiers)
self.query = query
self.modifiers.webAnalyticsFirstPageviewFilters = resolve_first_pageview_filters_modifier(
query, team, self.modifiers.webAnalyticsFirstPageviewFilters
@@ -2081,6 +2101,20 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
def query_status_labels(self) -> list[str] | None:
return None
+ def _reads_events_route(self) -> bool:
+ """True when this query reads the PostHog events table.
+
+ Only then does the events route belong in the query's cache key and in its queued payload.
+ """
+ return True
+
+ def _query_json_with_events_schema(self) -> dict[str, Any]:
+ query_json = self.query.model_dump()
+ # A query that reads no events table needs no pin, and pure Trino rejects modifiers the caller did not set.
+ if self.modifiers.useNewEventsSchema and self._reads_events_route():
+ return with_pinned_events_route(query_json, self.modifiers)
+ return query_json
+
def enqueue_async_calculation(
self,
*,
@@ -2116,7 +2150,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
sharing_configuration_id=sharing_configuration_id,
insight_id=cache_manager.insight_id,
dashboard_id=cache_manager.dashboard_id,
- query_json=self.query.model_dump(),
+ query_json=self._query_json_with_events_schema(),
query_id=self.query_id or cache_manager.cache_key, # Use cache key as query ID to avoid duplicates
cache_key=cache_manager.cache_key,
labels=self.query_status_labels(),
@@ -2383,6 +2417,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
return query_result, query_duration_ms
@new_events_cluster_host_scope()
+ @_pin_events_route
def run(
self,
execution_mode: ExecutionMode = ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE,
@@ -3122,11 +3157,15 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
# note: to_dict already strips custom_name from series (see schema_helpers.py)
query = to_dict(self.query)
query.pop("tags", None)
+ # The resolved modifiers key the events route, so requested values that resolve the same share one entry.
+ query = strip_events_route_modifiers(query, drop_empty=True)
+
+ cache_modifiers = events_route_cache_modifiers(to_dict(self.modifiers), reads_events=self._reads_events_route())
payload = {
"query_runner": self.__class__.__name__,
"query": query,
"team_id": self.team.pk,
- "hogql_modifiers": to_dict(self.modifiers),
+ "hogql_modifiers": cache_modifiers,
"products_modifiers": self._products_modifiers_for_cache(),
"limit_context": self._limit_context_aliased_for_cache,
"timezone": self.team.timezone,
diff --git a/posthog/hogql_queries/test/test_query_runner.py b/posthog/hogql_queries/test/test_query_runner.py
index 138755e6c207..2930654676b5 100644
--- a/posthog/hogql_queries/test/test_query_runner.py
+++ b/posthog/hogql_queries/test/test_query_runner.py
@@ -91,6 +91,7 @@ from posthog.hogql_queries.query_runner import (
shared_insights_execution_mode,
)
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
+from posthog.models.event.new_events_schema import get_events_route
from posthog.models.instance_setting import override_instance_config
from posthog.models.organization import OrganizationMembership
from posthog.models.sharing_configuration import SharingConfiguration
@@ -226,6 +227,24 @@ class TestQueryRunner(BaseTest):
def setup_test_query_runner_class(self, base: type[QueryRunner] = QueryRunner):
return setup_test_query_runner_class(base)
+ @parameterized.expand([(False,), (True,)])
+ def test_run_pins_its_events_route_for_lookups(self, events_json: bool) -> None:
+ TestQueryRunner = self.setup_test_query_runner_class()
+ routes: list[HogQLQueryModifiers | None] = []
+
+ def calculate(_runner: QueryRunner) -> TheTestBasicQueryResponse:
+ routes.append(get_events_route())
+ return TheTestBasicQueryResponse(results=[])
+
+ runner = TestQueryRunner(
+ query={"some_attr": "bla"}, team=self.team, modifiers=HogQLQueryModifiers(useNewEventsSchema=events_json)
+ )
+ with mock.patch.object(TestQueryRunner, "_calculate", autospec=True, side_effect=calculate):
+ runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)
+
+ assert routes == [HogQLQueryModifiers(useNewEventsSchema=events_json)]
+ assert get_events_route() is None
+
def test_sync_warning_attach_preserves_other_warning_kinds(self):
# The accumulator attach replaces the response's warnings with the collected sync warnings.
# Access control warnings ride the same shared field and were silently dropped whenever a
@@ -693,7 +712,11 @@ class TestQueryRunner(BaseTest):
self.assertEqual(first.query, HogQLQuery(query="SELECT 1"))
self.assertEqual(first.get_cache_key(), second.get_cache_key())
- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)
+ @override_settings(
+ PERSON_ON_EVENTS_OVERRIDE=False,
+ PERSON_ON_EVENTS_V2_OVERRIDE=False,
+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,
+ )
def test_cache_payload(self):
TestQueryRunner = self.setup_test_query_runner_class()
@@ -828,7 +851,11 @@ class TestQueryRunner(BaseTest):
return runner.get_cache_key()
@parameterized.expand([("new_team", False), ("existing_team", True)])
- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)
+ @override_settings(
+ PERSON_ON_EVENTS_OVERRIDE=False,
+ PERSON_ON_EVENTS_V2_OVERRIDE=False,
+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,
+ )
def test_cache_key(self, _name: str, existing_team: bool) -> None:
TestQueryRunner = self.setup_test_query_runner_class()
team = Team.objects.create(organization=self.organization)
@@ -854,7 +881,11 @@ class TestQueryRunner(BaseTest):
assert tagged.get_cache_key() == untagged.get_cache_key()
- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)
+ @override_settings(
+ PERSON_ON_EVENTS_OVERRIDE=False,
+ PERSON_ON_EVENTS_V2_OVERRIDE=False,
+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,
+ )
def test_cache_key_runner_subclass(self):
TestQueryRunner = self.setup_test_query_runner_class()
@@ -868,7 +899,11 @@ class TestQueryRunner(BaseTest):
cache_key = self._cache_key_with_fixed_payload_team_id(runner)
assert cache_key == f"cache_{team.pk}_9dcfced89edfbfcd1e0fd380380c7a5e4a9957386fc22172c1b83548a9af6d02"
- @override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)
+ @override_settings(
+ PERSON_ON_EVENTS_OVERRIDE=False,
+ PERSON_ON_EVENTS_V2_OVERRIDE=False,
+ CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False,
+ )
def test_cache_key_different_timezone(self):
TestQueryRunner = self.setup_test_query_runner_class()
team = Team.objects.create(organization=self.organization)
@@ -1938,6 +1973,29 @@ class TestSharedInsightsExecutionMode(BaseTest):
expected = sharing_configuration.pk if is_shared_viewer else None
assert mock_enqueue.call_args.kwargs["sharing_configuration_id"] == expected
+ @mock.patch("posthog.hogql_queries.query_runner.enqueue_process_query_task")
+ def test_async_payload_pins_a_resolved_native_table(self, mock_enqueue: mock.MagicMock) -> None:
+ with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True):
+ direct = HogQLQueryRunner(query=HogQLQuery(query="SELECT 1"), team=self.team)
+ direct.query = direct.query.model_copy(update={"connectionId": "direct"})
+ with mock.patch.object(HogQLQueryRunner, "_validate_direct_connection"):
+ direct.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)
+ assert mock_enqueue.call_args.kwargs["query_json"] == direct.query.model_dump()
+ assert "useNewEventsSchema" not in direct.get_cache_payload()["hogql_modifiers"]
+ runner = HogQLQueryRunner(query=HogQLQuery(query="SELECT 1"), team=self.team)
+ runner.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)
+ with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False):
+ worker = HogQLQueryRunner(query=mock_enqueue.call_args.kwargs["query_json"], team=self.team)
+ runner.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)
+ legacy = HogQLQueryRunner(query=HogQLQuery(query="SELECT 1"), team=self.team)
+ legacy.enqueue_async_calculation(cache_manager=mock.MagicMock(), user=self.user)
+
+ assert worker.modifiers.useNewEventsSchema is True
+ assert worker.get_cache_key() == runner.get_cache_key()
+ assert mock_enqueue.call_args.kwargs["query_json"] == legacy.query.model_dump()
+ assert legacy.get_cache_key() != runner.get_cache_key()
+ assert "useNewEventsSchema" not in legacy.get_cache_payload()["hogql_modifiers"]
+
@pytest.mark.ee
class TestQueryRunnerAccessControlFingerprint(BaseTest):
diff --git a/posthog/hogql_queries/utils/test/test_timestamp_utils.py b/posthog/hogql_queries/utils/test/test_timestamp_utils.py
index 6a17c6287b81..f07b172fdf58 100644
--- a/posthog/hogql_queries/utils/test/test_timestamp_utils.py
+++ b/posthog/hogql_queries/utils/test/test_timestamp_utils.py
@@ -2,7 +2,13 @@ import datetime
from zoneinfo import ZoneInfo
import time_machine
-from posthog.test.base import APIBaseTest, ClickhouseDestroyTablesMixin, _create_event, flush_persons_and_events
+from posthog.test.base import (
+ APIBaseTest,
+ BaseTest,
+ ClickhouseDestroyTablesMixin,
+ _create_event,
+ flush_persons_and_events,
+)
from unittest.mock import MagicMock, patch
from django.core.cache import cache
@@ -11,20 +17,25 @@ from django.test import override_settings
from dateutil import parser
from parameterized import parameterized
-from posthog.schema import ActionsNode, DataWarehouseNode, DateRange, EventsNode, IntervalType
+from posthog.schema import ActionsNode, DataWarehouseNode, DateRange, EventsNode, HogQLQueryModifiers, IntervalType
from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context
+from posthog.clickhouse.workload import Workload
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
from posthog.hogql_queries.utils.timestamp_utils import (
EARLIEST_EVENT_TIMESTAMP,
_coerce_to_datetime,
+ _events_route_cache_key,
+ _events_route_modifiers,
_get_earliest_timestamp_cache_key,
format_label_date,
get_earliest_timestamp_from_series,
get_earliest_timestamp_unfiltered,
)
+from posthog.models.event.new_events_schema import events_route_scope
+from posthog.models.instance_setting import override_instance_config
from posthog.models.team import WeekStartDay
from products.actions.backend.models.action import Action
@@ -526,7 +537,9 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):
# TTL window. A cache hit must still pass through _coerce_to_datetime instead of
# returning the raw value, otherwise downstream date math hits AttributeError/TypeError.
node = EventsNode(event="$pageview")
- cache_key = _get_earliest_timestamp_cache_key(self.team, node)
+ cache_key = _events_route_cache_key(
+ self.team, _events_route_modifiers(self.team), _get_earliest_timestamp_cache_key(self.team, node)
+ )
# team timezone is UTC in tests; "2021-03-15" is the pre-fix raw Date/String shape.
cache.set(cache_key, "2021-03-15", timeout=3600)
@@ -553,7 +566,7 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):
def test_single_path_query_tags(self, _name, caller_product, caller_feature, expected_product, expected_feature):
captured: dict[str, object] = {}
- def capture(query, team, user=None):
+ def capture(query, team, user=None, modifiers=None):
tags = get_query_tags()
captured["product"] = tags.product
captured["feature"] = tags.feature
@@ -572,3 +585,74 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):
self.assertEqual(captured["product"], expected_product)
self.assertEqual(captured["feature"], expected_feature)
self.assertEqual(captured["lookup"], "earliest_timestamp")
+
+
+class TestEarliestTimestampRouting(BaseTest):
+ def tearDown(self):
+ super().tearDown()
+ cache.clear()
+
+ @parameterized.expand([("series",), ("unfiltered",)])
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False, CLICKHOUSE_NATIVE_EVENTS_HOST="native.example.com")
+ def test_native_route_reads_the_native_cluster_and_skips_the_legacy_cache(self, lookup: str) -> None:
+ node = EventsNode(event="$pageview")
+ legacy = datetime.datetime(2022, 1, 1, tzinfo=ZoneInfo("UTC"))
+ native = datetime.datetime(2020, 1, 1, tzinfo=ZoneInfo("UTC"))
+ if lookup == "series":
+ cache.set(_get_earliest_timestamp_cache_key(self.team, node), legacy)
+ else:
+ cache.set(f"earliest_timestamp_unfiltered_{self.team.pk}", legacy)
+
+ def earliest() -> datetime.datetime:
+ if lookup == "series":
+ return get_earliest_timestamp_from_series(self.team, [node])
+ return get_earliest_timestamp_unfiltered(self.team)
+
+ with (
+ override_instance_config("CLICKHOUSE_NATIVE_EVENTS_HOST", "native.example.com"),
+ override_instance_config("CLICKHOUSE_NATIVE_EVENTS_TEAMS", [self.team.pk]),
+ patch(
+ "posthog.hogql.query.sync_execute", return_value=([(native,)], [("timestamp", "DateTime")])
+ ) as execute,
+ ):
+ with events_route_scope(HogQLQueryModifiers(useNewEventsCluster=True, useNewEventsSchema=True)):
+ self.assertEqual(earliest(), native)
+ self.assertEqual(earliest(), native)
+ self.assertEqual(earliest(), legacy)
+ execute.assert_called_once()
+ self.assertEqual(execute.call_args.kwargs["workload"], Workload.NEW_EVENTS_CLUSTER)
+ self.assertIn("events_json", execute.call_args.args[0])
+
+ def test_route_keys_never_collide_with_legacy_event_keys(self) -> None:
+ legacy_events_json_event = datetime.datetime(2022, 1, 1, tzinfo=ZoneInfo("UTC"))
+ cache.set(
+ _get_earliest_timestamp_cache_key(self.team, EventsNode(event="events_json")), legacy_events_json_event
+ )
+ native = datetime.datetime(2020, 1, 1, tzinfo=ZoneInfo("UTC"))
+ with (
+ patch("posthog.hogql.query.sync_execute", return_value=([(native,)], [("timestamp", "DateTime")])),
+ events_route_scope(HogQLQueryModifiers(useNewEventsSchema=True)),
+ ):
+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [EventsNode(event=None)]), native)
+
+ @parameterized.expand([("allowlisted_project", True, {}), ("project_setting", False, {"useNewEventsSchema": True})])
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
+ def test_lookup_outside_a_run_keys_on_the_events_table_it_reads(
+ self, _name: str, allowlisted: bool, team_modifiers: dict[str, bool]
+ ) -> None:
+ self.team.modifiers = team_modifiers
+ node = EventsNode(event="$pageview")
+ cache.set(
+ _get_earliest_timestamp_cache_key(self.team, node), datetime.datetime(2022, 1, 1, tzinfo=ZoneInfo("UTC"))
+ )
+ native = datetime.datetime(2020, 1, 1, tzinfo=ZoneInfo("UTC"))
+ with (
+ override_instance_config(
+ "CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.pk) if allowlisted else ""
+ ),
+ patch(
+ "posthog.hogql.query.sync_execute", return_value=([(native,)], [("timestamp", "DateTime")])
+ ) as execute,
+ ):
+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), native)
+ self.assertIn("events_json", execute.call_args.args[0])
diff --git a/posthog/hogql_queries/utils/timestamp_utils.py b/posthog/hogql_queries/utils/timestamp_utils.py
index 0eac1f41a6ed..6419bf78f4e1 100644
--- a/posthog/hogql_queries/utils/timestamp_utils.py
+++ b/posthog/hogql_queries/utils/timestamp_utils.py
@@ -18,12 +18,14 @@ from posthog.schema import (
EventsNode,
FunnelsDataWarehouseNode,
GroupNode,
+ HogQLQueryModifiers,
LifecycleDataWarehouseNode,
)
from posthog.hogql import ast
from posthog.hogql.ast import SelectQuery
from posthog.hogql.context import HogQLContext
+from posthog.hogql.modifiers import create_default_modifiers_for_team
from posthog.hogql.property import action_to_expr
from posthog.hogql.query import execute_hogql_query
@@ -31,6 +33,7 @@ from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, t
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
from posthog.models import Team, User
from posthog.models.event import DEFAULT_EARLIEST_TIME_DELTA
+from posthog.models.event.new_events_schema import get_events_route, get_new_events_cluster_host, use_new_events_schema
from posthog.models.team import WeekStartDay
from posthog.utils import get_safe_cache
@@ -135,6 +138,21 @@ def _get_earliest_timestamp_cache_key(
raise ValueError(f"Unsupported node type: {type(node)}")
+def _events_route_modifiers(team: Team) -> HogQLQueryModifiers:
+ return create_default_modifiers_for_team(team, get_events_route())
+
+
+def _events_route_cache_key(team: Team, modifiers: HogQLQueryModifiers, cache_key: str) -> str:
+ # Each events table and cluster holds its own copy of the events, so one route's result must not answer another.
+ # The route goes in front, because legacy keys end in event names and table names that a suffix could collide with.
+ native_table = use_new_events_schema(team.pk, modifiers)
+ native_cluster = bool(modifiers.useNewEventsCluster)
+ if not (native_table or native_cluster):
+ return cache_key
+ native_host = get_new_events_cluster_host() if native_cluster else None
+ return f"events_route:{native_table}:{native_host}:{cache_key}"
+
+
def _coerce_to_datetime(value: Any, tz: tzinfo) -> datetime:
"""Normalize a min(timestamp) result to a timezone-aware ``datetime``.
@@ -197,7 +215,8 @@ def _get_earliest_timestamp_from_node(
"""
# The cache is team-wide (keyed by team + node, not user): the value is table metadata, and row
# access is still enforced on the insight's main query, so per-user entries aren't worth the misses.
- cache_key = _get_earliest_timestamp_cache_key(team, node)
+ modifiers = _events_route_modifiers(team)
+ cache_key = _events_route_cache_key(team, modifiers, _get_earliest_timestamp_cache_key(team, node))
cached_result = get_safe_cache(cache_key)
if cached_result is not None:
# Coerce on read too: entries cached before the coercion fix shipped hold a raw
@@ -216,7 +235,7 @@ def _get_earliest_timestamp_from_node(
earliest_timestamp = EARLIEST_EVENT_TIMESTAMP
with _earliest_timestamp_query_tags():
- result = execute_hogql_query(query=query, team=team, user=user)
+ result = execute_hogql_query(query=query, team=team, user=user, modifiers=modifiers)
if result and len(result.results) > 0 and len(result.results[0]) > 0 and result.results[0][0] is not None:
earliest_timestamp = _coerce_to_datetime(result.results[0][0], team.timezone_info)
@@ -277,7 +296,8 @@ def get_earliest_timestamp_unfiltered(team: Team) -> datetime:
legacy raw-SQL earliest-timestamp helper: floored at 2015-01-01, falling back to
now - DEFAULT_EARLIEST_TIME_DELTA when the team has no events.
"""
- cache_key = f"earliest_timestamp_unfiltered_{team.pk}"
+ modifiers = _events_route_modifiers(team)
+ cache_key = _events_route_cache_key(team, modifiers, f"earliest_timestamp_unfiltered_{team.pk}")
cached_result = get_safe_cache(cache_key)
if cached_result is not None:
return _coerce_to_datetime(cached_result, team.timezone_info)
@@ -300,6 +320,7 @@ def get_earliest_timestamp_unfiltered(team: Team) -> datetime:
team=team,
# The read has no upper bound and no filter, so the first row in sort key order is the answer.
context=HogQLContext(team_id=team.pk, order_events_reads_by_sort_key=True),
+ modifiers=modifiers,
)
if result and len(result.results) > 0 and len(result.results[0]) > 0 and result.results[0][0] is not None:
earliest_timestamp = _coerce_to_datetime(result.results[0][0], team.timezone_info)
diff --git a/posthog/models/event/new_events_schema.py b/posthog/models/event/new_events_schema.py
index 4400be21c41a..aae7ebbe24b4 100644
--- a/posthog/models/event/new_events_schema.py
+++ b/posthog/models/event/new_events_schema.py
@@ -1,7 +1,7 @@
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from django.conf import settings
@@ -11,8 +11,68 @@ if TYPE_CHECKING:
from posthog.schema import HogQLQueryModifiers
-# TODO: Remove the per-query rollout option after the events cluster migration is complete.
+# TODO: Remove the per-query rollout option, and the route helpers in this block, after the events cluster
+# migration is complete.
+EVENTS_ROUTE_MODIFIER_KEYS = ("useNewEventsSchema", "useNewEventsCluster")
_query_host: ContextVar[list[str | None] | None] = ContextVar("new_events_cluster_host", default=None)
+_query_route: ContextVar["HogQLQueryModifiers | None"] = ContextVar("new_events_query_route", default=None)
+
+
+def _copy_source_chain(query: dict[str, Any]) -> list[dict[str, Any]]:
+ """Copies of the query and every source it wraps, outermost first. Other nested values stay shared."""
+ nodes = [dict(query)]
+ while isinstance(nodes[-1].get("source"), dict):
+ nodes[-1]["source"] = dict(nodes[-1]["source"])
+ nodes.append(nodes[-1]["source"])
+ return nodes
+
+
+def strip_events_route_modifiers(query: dict[str, Any], *, drop_empty: bool = False) -> dict[str, Any]:
+ """The serialized query without the requested events route.
+
+ The resolved route keys the cache, so requested values that resolve to the same route must not split the
+ cache key. With drop_empty, a modifiers field left empty is removed.
+ """
+ nodes = _copy_source_chain(query)
+ for node in nodes:
+ modifiers = node.get("modifiers")
+ if isinstance(modifiers, dict):
+ node["modifiers"] = {
+ key: value for key, value in modifiers.items() if key not in EVENTS_ROUTE_MODIFIER_KEYS
+ }
+ if drop_empty and not node["modifiers"]:
+ node.pop("modifiers")
+ return nodes[0]
+
+
+def with_pinned_events_route(query: dict[str, Any], modifiers: "HogQLQueryModifiers") -> dict[str, Any]:
+ """The serialized query with the resolved events route in place of the requested one.
+
+ A worker rebuilds the runner from this payload. The resolved route goes into the innermost source, which
+ every worker reads, so the worker reads the same table and writes the cache key the request reads.
+ """
+ if "modifiers" not in _copy_source_chain(query)[-1]:
+ return query
+ nodes = _copy_source_chain(strip_events_route_modifiers(query))
+ route: dict[str, Any] = {"useNewEventsSchema": modifiers.useNewEventsSchema}
+ if modifiers.useNewEventsCluster is not None:
+ route["useNewEventsCluster"] = modifiers.useNewEventsCluster
+ nodes[-1]["modifiers"] = {**(nodes[-1]["modifiers"] or {}), **route}
+ return nodes[0]
+
+
+def events_route_cache_modifiers(modifiers: dict[str, Any], *, reads_events: bool) -> dict[str, Any]:
+ """The resolved modifiers as they key the cache.
+
+ A legacy-table query must keep the cache key it had before the native route existed, so the route values it
+ resolves to are left out. A query that reads no events table is never keyed by one.
+ """
+ modifiers = dict(modifiers)
+ if modifiers.get("useNewEventsSchema") is False or not reads_events:
+ modifiers.pop("useNewEventsSchema", None)
+ if not modifiers.get("useNewEventsCluster"):
+ modifiers.pop("useNewEventsCluster", None)
+ return modifiers
def get_new_events_cluster_host() -> str | None:
@@ -39,6 +99,27 @@ def new_events_cluster_host_scope() -> Iterator[None]:
_query_host.reset(token)
+def get_events_route() -> "HogQLQueryModifiers | None":
+ """The events cluster and table of the query run in progress, or None outside a query run."""
+ return _query_route.get()
+
+
+@contextmanager
+def events_route_scope(modifiers: "HogQLQueryModifiers") -> Iterator[None]:
+ # Lookups inside a run, such as an all-time range's earliest event, must read the query's cluster and table.
+ from posthog.schema import HogQLQueryModifiers # noqa: PLC0415
+
+ token = _query_route.set(
+ HogQLQueryModifiers(
+ useNewEventsCluster=modifiers.useNewEventsCluster, useNewEventsSchema=modifiers.useNewEventsSchema
+ )
+ )
+ try:
+ yield
+ finally:
+ _query_route.reset(token)
+
+
def use_new_events_cluster(team_id: int | None, modifiers: "HogQLQueryModifiers | None") -> bool:
from posthog.hogql.errors import ExposedHogQLError # noqa: PLC0415