:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): carry native read settings on raw cohort and precompute sql (#128590)
Sandy Spicer041edb69ed96
diff --git a/posthog/models/event/new_events_schema.py b/posthog/models/event/new_events_schema.pyindex d8206ab206a2..91d1da4f8a11 100644--- a/posthog/models/event/new_events_schema.py+++ b/posthog/models/event/new_events_schema.py@@ -138,6 +138,25 @@ def events_read_table(use_new_events_schema: bool) -> str:return DISTRIBUTED_EVENTS_JSON_TABLE if use_new_events_schema else "events"+# The native table stores a dotted key `a.b` under the path `a%2Eb` (EVENTS_JSON_INSERT_SETTINGS), and ClickHouse+# prints the dot again only for a query that sets the first setting. The second keeps a `/` inside an object or+# array value unescaped, as the legacy text column returns it. The ClickHouse printer sets both on every HogQL+# query that reads the native table, so raw SQL that embeds or re-runs a printed query must send them itself.+EVENTS_JSON_READ_SETTINGS: dict[str, int] = {+ "json_type_escape_dots_in_keys": 1,+ "output_format_json_escape_forward_slashes": 0,+}+++def events_read_settings(use_new_events_schema: bool) -> dict[str, int]:+ """The ClickHouse settings raw (non-HogQL) SQL must send when it reads the native events table.++ Resolve the gate once per query, as `events_read_table` asks, so the settings match the table the SQL was+ printed against.+ """+ return dict(EVENTS_JSON_READ_SETTINGS) if use_new_events_schema else {}++def _allowlisted_team_ids() -> set[int]:raw = str(get_instance_setting("CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS") or "")return {int(part) for part in raw.split(",") if part.strip()}diff --git a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.pyindex 5557ba85a829..51282443b7b0 100644--- a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py+++ b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py@@ -49,7 +49,7 @@ from posthog.clickhouse.client import sync_executefrom posthog.clickhouse.preaggregation.sql import DISTRIBUTED_PREAGGREGATION_RESULTS_TABLEfrom posthog.clickhouse.query_tagging import tags_contextfrom posthog.dataclasses import frozen-from posthog.models.event.new_events_schema import use_new_events_schema+from posthog.models.event.new_events_schema import events_read_settings, use_new_events_schemafrom posthog.models.team import Teamfrom posthog.settings import DEBUG, HOGQL_INCREASED_MAX_EXECUTION_TIME, TESTfrom posthog.utils import relative_date_parse_with_delta_mapping@@ -195,6 +195,7 @@ def _get_insert_settings(spill_to_disk: bool = False,read_after_write: bool = True,overrides: Mapping[str, int | str] | None = None,+ modifiers: HogQLQueryModifiers | None = None,) -> dict:"""Build ClickHouse settings for preaggregation INSERT queries.@@ -215,6 +216,9 @@ def _get_insert_settings(`load_balancing` comes from the HogQL table that reads `table`, so the INSERT and thelater SELECT choose replicas the same way. A table that sets none uses the profile default.++ `modifiers` are the ones the INSERT's SELECT was printed with. The native read settings follow the+ events table that SELECT reads, which a per-query modifier can switch away from the team's default."""settings = get_default_hogql_global_settings(team_id=team_id).model_dump(exclude_none=True)settings.pop("readonly", None) # INSERTs need write access@@ -233,6 +237,7 @@ def _get_insert_settings(**_read_table_load_balancing(table),})+ settings.update(events_read_settings(use_new_events_schema(team_id, modifiers)))if spill_to_disk:# Spill heavy GROUP BYs to disk instead of OOMing. This is a spill# *threshold*, not a memory reservation — it lowers peak RAM (vs. holding@@ -1138,7 +1143,7 @@ def run_lazy_computation_insert(sync_execute(insert_sql,values,- settings=_get_insert_settings(team.id, query_info.table),+ settings=_get_insert_settings(team.id, query_info.table, modifiers=query_info.modifiers),))@@ -1939,6 +1944,7 @@ def ensure_precomputed(spill_to_disk=spill_to_disk,read_after_write=read_after_write,overrides=insert_settings,+ modifiers=modifiers,),))diff --git a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.pyindex 685aeeff83e8..7a604c86d4fa 100644--- a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py+++ b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py@@ -21,9 +21,9 @@ from posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.preaggregation.sql import DISTRIBUTED_PREAGGREGATION_RESULTS_TABLE-from posthog.models.event.new_events_schema import new_events_schema_for_request+from posthog.models.event.new_events_schema import events_read_settings, new_events_schema_for_requestfrom posthog.models.instance_setting import override_instance_config-from posthog.settings import HOGQL_INCREASED_MAX_EXECUTION_TIME+from posthog.settings import CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA, HOGQL_INCREASED_MAX_EXECUTION_TIMEfrom products.analytics_platform.backend.lazy_computation.computation_notifications import (job_channel,@@ -3568,6 +3568,23 @@ class TestInsertSettings(BaseTest):assert _get_insert_settings(self.team.pk, table)["insert_quorum"] == "auto"assert _get_insert_settings(self.team.pk, table, read_after_write=False)["insert_quorum"] == 0+ def test_insert_settings_carry_the_native_read_settings_for_a_switched_team(self):+ table = LazyComputationTable.PREAGGREGATION_RESULTS+ if not CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:+ assert "json_type_escape_dots_in_keys" not in _get_insert_settings(self.team.pk, table)+ with override_instance_config("CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.pk)):+ switched = _get_insert_settings(self.team.pk, table)+ assert switched["json_type_escape_dots_in_keys"] == 1+ assert switched["output_format_json_escape_forward_slashes"] == 0+ # The INSERT's SELECT is printed with the job's modifiers, so a per-query switch wins over the team's.+ forced_on = _get_insert_settings(self.team.pk, table, modifiers=HogQLQueryModifiers(useNewEventsSchema=True))+ assert forced_on["json_type_escape_dots_in_keys"] == 1+ with override_instance_config("CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.pk)):+ forced_off = _get_insert_settings(+ self.team.pk, table, modifiers=HogQLQueryModifiers(useNewEventsSchema=False)+ )+ assert "json_type_escape_dots_in_keys" not in forced_off+@parameterized.expand(_INSERT_LOAD_BALANCING_CASES)def test_insert_load_balancing_follows_the_read_table(self, table, expected_load_balancing):assert _get_insert_settings(self.team.pk, table).get("load_balancing") == expected_load_balancing@@ -3603,6 +3620,13 @@ class TestInsertSettingsAppliedToInserts(BaseTest):"auto",{"max_memory_usage": 7, "max_bytes_before_external_group_by": 3},),+ # The modifiers switch the SELECT's events table, and the read settings must follow it, not the team.+ (+ "modifier_switches_events_table",+ {"modifiers": HogQLQueryModifiers(useNewEventsSchema=not CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA)},+ "auto",+ events_read_settings(not CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA),+ ),])def test_manual_insert_path_passes_insert_settings_to_clickhouse(@@ -3632,6 +3656,7 @@ class TestInsertSettingsAppliedToInserts(BaseTest):spill_to_disk=extra_kwargs.get("spill_to_disk", False),read_after_write=extra_kwargs.get("read_after_write", True),overrides=extra_kwargs.get("insert_settings"),+ modifiers=extra_kwargs.get("modifiers"),)assert result.ready is True@@ -3696,7 +3721,12 @@ class TestInsertSettingsAppliedToInserts(BaseTest):},)assert isinstance(query, ast.SelectQuery)- query_info = LazyComputationQuery(query=query, table=LazyComputationTable.PREAGGREGATION_RESULTS)+ # Read the table the test leg does not read by default, so the settings only match when the INSERT+ # forwards the job's modifiers instead of asking the team.+ modifiers = HogQLQueryModifiers(useNewEventsSchema=not CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA)+ query_info = LazyComputationQuery(+ query=query, table=LazyComputationTable.PREAGGREGATION_RESULTS, modifiers=modifiers+ )with patch("products.analytics_platform.backend.lazy_computation.lazy_computation_executor.sync_execute"@@ -3704,9 +3734,11 @@ class TestInsertSettingsAppliedToInserts(BaseTest):run_lazy_computation_insert(self.team, job, query_info)assert mock_execute.call_count == 1- assert mock_execute.call_args.kwargs["settings"] == _get_insert_settings(- self.team.pk, LazyComputationTable.PREAGGREGATION_RESULTS+ settings = mock_execute.call_args.kwargs["settings"]+ assert settings == _get_insert_settings(+ self.team.pk, LazyComputationTable.PREAGGREGATION_RESULTS, modifiers=modifiers)+ assert ("json_type_escape_dots_in_keys" in settings) == (not CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA)class TestInsertsReportRowsWritten(BaseTest):diff --git a/products/cohorts/backend/models/test/test_util.py b/products/cohorts/backend/models/test/test_util.pyindex 9f8af39a640e..84049db7265f 100644--- a/products/cohorts/backend/models/test/test_util.py+++ b/products/cohorts/backend/models/test/test_util.py@@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patchfrom django.db import DEFAULT_DB_ALIAS, OperationalErrorfrom django.test import SimpleTestCase, override_settings+from django.utils import timezonefrom clickhouse_driver.errors import SocketTimeoutErrorfrom kombu.exceptions import OperationalError as KombuOperationalError@@ -27,8 +28,12 @@ from posthog.exceptions import (ClickHouseQuerySizeExceeded,ClickHouseQueryTimeOut,)+from posthog.models.event.new_events_schema import events_read_settings+from posthog.models.instance_setting import override_instance_configfrom posthog.models.person.sql import PERSON_STATIC_COHORT_TABLE+from posthog.settings import CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA+from products.cohorts.backend.models.calculation_history import CohortCalculationHistoryfrom products.cohorts.backend.models.cohort import Cohort, CohortOrEmptyfrom products.cohorts.backend.models.util import (COHORT_QUERY_TIMEOUT_SECONDS,@@ -36,11 +41,13 @@ from products.cohorts.backend.models.util import (ERROR_CODE_MESSAGES,CohortErrorCode,_recalculate_cohortpeople_for_team,+ _recalculate_cohortpeople_for_team_hogql,_sanitize_query_for_cohort,get_all_cohort_dependencies,get_friendly_error_message,get_nested_cohort_ids,get_static_cohort_size,+ insert_cohort_filter_actors_into_ch,insert_cohort_query_actors_into_ch,legacy_groups_to_filters,parse_error_code,@@ -728,6 +735,37 @@ class TestCohortUtils(BaseTest):self.assertIn("WHERE distinct_id IN", result)# Should include settingsself.assertIn("SETTINGS", result)+ if not CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:+ self.assertNotIn("json_type_escape_dots_in_keys", result)++ with (+ override_instance_config("CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.id)),+ patch("posthog.hogql_queries.query_runner.get_query_runner", mock_get_query_runner),+ ):+ switched = print_cohort_hogql_query(+ cohort, HogQLContext(team_id=self.team.id, enable_select_queries=True), team=self.team+ )+ self.assertIn("json_type_escape_dots_in_keys=1", switched)+ self.assertIn("output_format_json_escape_forward_slashes=0", switched)++ def test_recalculate_cohortpeople_hogql_sends_the_native_read_settings_for_a_switched_team(self):+ cohort = Cohort.objects.create(+ team=self.team,+ name="switched",+ groups=[{"properties": [{"key": "$browser", "value": "Chrome", "type": "person"}]}],+ )+ history = CohortCalculationHistory.objects.create(+ team=self.team, cohort=cohort, filters={}, started_at=timezone.now()+ )++ with (+ override_instance_config("CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.id)),+ patch("products.cohorts.backend.models.util.sync_execute", return_value=0) as execute,+ ):+ _recalculate_cohortpeople_for_team_hogql(cohort, 1, self.team, history)+ settings = execute.call_args.kwargs["settings"]+ self.assertEqual(settings["json_type_escape_dots_in_keys"], 1)+ self.assertEqual(settings["output_format_json_escape_forward_slashes"], 0)def test_print_cohort_hogql_query_with_select_star_raises_error(self):"""Test that SELECT * queries without explicit ID columns raise an error"""@@ -981,12 +1019,30 @@ class TestCohortUtils(BaseTest):"max_bytes_before_external_sort": 21 * 1024**3,"max_bytes_ratio_before_external_group_by": 0,"max_bytes_ratio_before_external_sort": 0,+ **events_read_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA),},)self.assertEqual(kwargs["workload"], Workload.OFFLINE)self.assertEqual(kwargs["ch_user"], ClickHouseUser.COHORTS)self.assertEqual(kwargs["team_id"], self.team.id)+ def test_insert_cohort_filter_actors_into_ch_sends_the_native_read_settings_for_a_switched_team(self):+ cohort = Cohort.objects.create(+ team=self.team,+ name="switched static",+ is_static=True,+ groups=[{"properties": [{"key": "$browser", "value": "Chrome", "type": "person"}]}],+ )++ with (+ override_instance_config("CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.id)),+ patch("products.cohorts.backend.models.util.sync_execute") as execute,+ ):+ insert_cohort_filter_actors_into_ch(cohort, team=self.team)+ settings = execute.call_args.kwargs["settings"]+ self.assertEqual(settings["json_type_escape_dots_in_keys"], 1)+ self.assertEqual(settings["output_format_json_escape_forward_slashes"], 0)+@parameterized.expand([(diff --git a/products/cohorts/backend/models/util.py b/products/cohorts/backend/models/util.pyindex 83ff601afb7b..a17f317abe1c 100644--- a/products/cohorts/backend/models/util.py+++ b/products/cohorts/backend/models/util.py@@ -45,6 +45,7 @@ from posthog.exceptions import ()from posthog.exceptions_capture import capture_exceptionfrom posthog.models import Team+from posthog.models.event.new_events_schema import events_read_settingsfrom posthog.models.person.sql import (DELETE_PERSON_FROM_STATIC_COHORT,INSERT_COHORT_ALL_PEOPLE_THROUGH_PERSON_ID,@@ -735,6 +736,9 @@ def print_cohort_hogql_query(cohort: Cohort, hogql_context: HogQLContext, *, tea# Format settings as key=value pairssettings_pairs = {k: str(v) for k, v in settings.model_dump().items() if v is not None}+ settings_pairs.update(+ {key: str(value) for key, value in events_read_settings(hogql_context.uses_new_events_schema()).items()}+ )settings_clause = ""if settings_pairs:settings_str = ", ".join([f"{key}={value}" for key, value in settings_pairs.items()])@@ -962,6 +966,7 @@ def _recalculate_cohortpeople_for_team_hogql(cohort: Cohort, pending_version: int, team: Team, history: CohortCalculationHistory) -> int:cohort_params: dict[str, Any]+ hogql_context: HogQLContext | None = Noneif cohort.is_static:cohort_query, cohort_params = format_static_cohort_query(cohort, 0, prepend="")elif not cohort.properties.values:@@ -1001,6 +1006,8 @@ def _recalculate_cohortpeople_for_team_hogql(**COHORT_QUERY_SPILL_SETTINGS,})+ if hogql_context is not None:+ settings.update(events_read_settings(hogql_context.uses_new_events_schema()))return sync_execute(recalculate_cohortpeople_sql,@@ -1368,6 +1375,9 @@ def insert_actors_into_cohort_by_query(settings={**COHORT_QUERY_TIMEOUT_SETTINGS,**COHORT_QUERY_SPILL_SETTINGS,+ # A SELECT compiled from cohort filters arrives without its SETTINGS clause (hogql_cohort_subquery_sql+ # trims it so the query can be embedded), so the native read settings travel on the INSERT.+ **events_read_settings(context.uses_new_events_schema()),},workload=Workload.OFFLINE,ch_user=ClickHouseUser.COHORTS,diff --git a/products/web_analytics/backend/hogql_queries/test/test_web_overview_lazy_precompute.py b/products/web_analytics/backend/hogql_queries/test/test_web_overview_lazy_precompute.pyindex db09246056aa..a3c21b7fc737 100644--- a/products/web_analytics/backend/hogql_queries/test/test_web_overview_lazy_precompute.py+++ b/products/web_analytics/backend/hogql_queries/test/test_web_overview_lazy_precompute.py@@ -30,7 +30,9 @@ from posthog.hogql.printer import prepare_and_print_astfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.query_tagging import tags_context+from posthog.models.instance_setting import override_instance_configfrom posthog.models.utils import uuid7+from posthog.settings import CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMAfrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResultfrom products.analytics_platform.backend.models.preaggregation_job import PreaggregationJob@@ -337,6 +339,32 @@ class TestWebOverviewLazyPrecompute(ClickhouseTestMixin, APIBaseTest):eligible,)+ @time_machine.travel("2024-01-15T12:00:00Z", tick=False)+ def test_channel_boundary_read_carries_the_native_read_settings_for_a_switched_team(self) -> None:+ def boundary_read_settings() -> dict:+ query = self._build_query(+ date_from="2024-01-03T11:00:00",+ date_to="2024-01-03T12:00:00",+ properties=[+ SessionPropertyFilter(key="$channel_type", value="Paid Search", operator=PropertyOperator.EXACT)+ ],+ )+ assert query.dateRange is not None+ query.dateRange.explicitDate = True+ runner = WebOverviewQueryRunner(team=self.team, query=query)+ with patch.object(overview_precompute, "sync_execute", return_value=[[0, 0, 0, 0.0, None]]) as execute:+ overview_precompute.execute_channel_period_read(+ runner, runner.query_date_range.date_from(), runner.query_date_range.date_to()+ )+ return execute.call_args.kwargs["settings"]++ if not CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:+ self.assertNotIn("json_type_escape_dots_in_keys", boundary_read_settings())+ with override_instance_config("CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA_TEAMS", str(self.team.pk)):+ switched = boundary_read_settings()+ self.assertEqual(switched["json_type_escape_dots_in_keys"], 1)+ self.assertEqual(switched["output_format_json_escape_forward_slashes"], 0)+@time_machine.travel("2024-01-15T12:00:00Z", tick=False)def test_sampling_falls_through(self):self._seed_two_sessions()diff --git a/products/web_analytics/backend/hogql_queries/web_overview_lazy_precompute.py b/products/web_analytics/backend/hogql_queries/web_overview_lazy_precompute.pyindex eaed72b7055a..264eb23d7d2b 100644--- a/products/web_analytics/backend/hogql_queries/web_overview_lazy_precompute.py+++ b/products/web_analytics/backend/hogql_queries/web_overview_lazy_precompute.py@@ -19,6 +19,7 @@ from posthog.clickhouse.preaggregation.web_overview_preaggregated_sql import ()from posthog.clickhouse.query_tagging import Feature, Product, tag_queriesfrom posthog.dataclasses import frozen+from posthog.models.event.new_events_schema import events_read_settingsfrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import (LazyComputationResult,@@ -569,7 +570,12 @@ FROM (+ ")")tag_queries(product=Product.WEB_ANALYTICS, feature=Feature.QUERY, query_type="web_overview_lazy_query")- rows = sync_execute(sql, params, settings=_READ_SETTINGS, team_id=runner.team.pk)+ rows = sync_execute(+ sql,+ params,+ settings={**_READ_SETTINGS, **events_read_settings(context.uses_new_events_schema())},+ team_id=runner.team.pk,+ )return list(rows[0]) if rows else [0, 0, 0, 0, 0]