:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): read dotted native property keys as direct subcolumns (#130464)
Sandy Spicer52e6d7c568f1
diff --git a/posthog/hogql/escape_sql.py b/posthog/hogql/escape_sql.pyindex d46905bbbe6b..1c415295794f 100644--- a/posthog/hogql/escape_sql.py+++ b/posthog/hogql/escape_sql.py@@ -345,6 +345,17 @@ def backquote_clickhouse_identifier(identifier: str) -> str:return "`{}`".format("".join(backquote_escape_chars_map.get(c, c) for c in identifier))+def escape_clickhouse_json_path(key: str) -> str:+ """Quote one segment of a JSON subcolumn path, spelling a `%` as the `\\x25` escape.++ ClickHouse decodes the escape inside backticks, so the path reads as a direct subcolumn. A literal `%` cannot+ appear in the SQL, because the client runs it through parameter substitution, where `%` opens a format directive.+ """+ if "%" not in key:+ return escape_clickhouse_identifier(key)+ return backquote_clickhouse_identifier(key).replace("%", "\\x25")++def escape_hogql_string(name: bool | float | int | str | list | tuple | date | datetime | UUID | UUIDT | None,timezone: Optional[str] = None,diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.pyindex 3c2384144c55..e7546263760f 100644--- a/posthog/hogql/printer/clickhouse.py+++ b/posthog/hogql/printer/clickhouse.py@@ -27,7 +27,12 @@ from posthog.hogql.database.schema.util.where_clause_extractor import (get_events_session_id_table_type,)from posthog.hogql.errors import ImpossibleASTError, InternalHogQLError, QueryError-from posthog.hogql.escape_sql import escape_clickhouse_identifier, escape_clickhouse_string, safe_identifier+from posthog.hogql.escape_sql import (+ escape_clickhouse_identifier,+ escape_clickhouse_json_path,+ escape_clickhouse_string,+ safe_identifier,+)from posthog.hogql.functions import ADD_OR_NULL_DATETIME_FUNCTIONS, FIRST_ARG_DATETIME_FUNCTIONSfrom posthog.hogql.functions.clickhouse.json import JSON_FUNCTIONSfrom posthog.hogql.functions.embed_text import resolve_embed_text@@ -1523,9 +1528,9 @@ class ClickHousePrinter(BasePrinter):# dot inside one key must not read as a path separator.keys = [key.replace(".", "%2E") for key in node.keys]- # An empty key is not a valid SQL identifier, and a `%` would break the SQL's own parameter placeholders,- # so those paths go to getSubcolumn as bound values.- if any(not key or "%" in key for key in keys):+ # An empty key is not a valid SQL identifier, so that path goes to getSubcolumn as a bound value. Every+ # other key prints as a direct subcolumn read: getSubcolumn() reads every subcolumn and filters afterwards.+ if any(not key for key in keys):if node.access_type == "sub_object":# getSubcolumn's path parser reads a backslash-escaped backtick, not SQL's doubled backtick.escaped_keys = (key.replace("\\", "\\\\").replace("`", "\\`") for key in keys)@@ -1538,5 +1543,5 @@ class ClickHousePrinter(BasePrinter):for index, key in enumerate(keys):separator = ".^" if node.access_type == "sub_object" and index == 0 else "."- expr = f"{expr}{separator}{escape_clickhouse_identifier(key)}"+ expr = f"{expr}{separator}{escape_clickhouse_json_path(key)}"return exprdiff --git a/posthog/hogql/printer/test/test_printer.py b/posthog/hogql/printer/test/test_printer.pyindex eda82a8c4fa0..ae6ff2bfa08f 100644--- a/posthog/hogql/printer/test/test_printer.py+++ b/posthog/hogql/printer/test/test_printer.py@@ -58,7 +58,7 @@ from posthog.hogql.printer import prepare_and_print_ast, prepare_ast_for_printinfrom posthog.hogql.property import property_to_exprfrom posthog.hogql.property_access_types import RestrictedPropertyfrom posthog.hogql.query import execute_hogql_query-from posthog.hogql.test.utils import json_dynamic_read_sql, json_dynamic_read_sql_from_parts+from posthog.hogql.test.utils import json_dynamic_read_sqlfrom posthog.clickhouse.client.execute import sync_executefrom posthog.clickhouse.events_json import TEMPORARY_PROPERTIES_JSON_TYPE@@ -139,13 +139,8 @@ class TestPrinter(BaseTest):return "events_json AS events" if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else "events"def _json_dynamic_subcolumn_expr(self, root: str, property_name: str) -> str:- if "%" in property_name or "." in property_name:- # A key the printer cannot inline is parameterized: every occurrence takes the next placeholder.- # A dotted key is bound with its dots escaped, so it reads as one path name rather than a nested path.- counter = iter(range(100))- next_placeholder = lambda: f"getSubcolumn({root}, %(hogql_val_{next(counter)})s)" # noqa: E731- return json_dynamic_read_sql_from_parts(next_placeholder, next_placeholder)- return self._json_dynamic_subcolumn_path_expr(root, [property_name])+ # A dotted key is one path name with its dots escaped, not a nested path.+ return self._json_dynamic_subcolumn_path_expr(root, [property_name.replace(".", "%2E")])def _json_dynamic_subcolumn_path_expr(self, root: str, property_path: list[str]) -> str:return json_dynamic_read_sql(root, property_path)@@ -981,10 +976,10 @@ class TestPrinter(BaseTest):self._expr("properties.`Account.client_id`", context),expected_client_id_sql,)- if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA:- self.assertEqual(set(context.values.values()), {"Account%2Eclient_id", "^`Account%2Eclient_id`"})- else:- self.assertEqual(context.values, {"hogql_val_0": "Account.client_id"})+ self.assertEqual(+ context.values,+ {} if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else {"hogql_val_0": "Account.client_id"},+ )context = HogQLContext(team_id=self.team.pk)expected_dynamic_sql = (@@ -1582,12 +1577,13 @@ class TestPrinter(BaseTest):self._select("SELECT event FROM events")@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)- def test_new_events_schema_percent_property_keys_use_bound_subcolumns(self) -> None:+ def test_new_events_schema_percent_property_keys_print_as_escaped_paths(self) -> None:context = HogQLContext(team_id=self.team.pk, enable_select_queries=True)for expression in ("JSONHas(properties, 'bad%key')", "JSONExtractRaw(properties, 'bad%key')"):printed = self._expr(expression, context)- self.assertIn("getSubcolumn(events.properties,", printed)+ self.assertIn("events.properties.`bad\\x25key`", printed)+ self.assertNotIn("getSubcolumn", printed)self.assertNotIn("JSONExtractKeysAndValuesRaw", printed)@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)@@ -1597,7 +1593,7 @@ class TestPrinter(BaseTest):self.assertEqual(self._expr("properties.`a.b`", context), self._json_dynamic_subcolumn_expr("events.properties", "a.b"))- self.assertEqual(set(context.values.values()), {"a%2Eb", "^`a%2Eb`"})+ self.assertEqual(context.values, {})nested_context = HogQLContext(team_id=self.team.pk, enable_select_queries=True)self.assertEqual(diff --git a/posthog/hogql/printer/test/test_property_access_control.py b/posthog/hogql/printer/test/test_property_access_control.pyindex f1b5d3cf2eaf..b1b3339cf619 100644--- a/posthog/hogql/printer/test/test_property_access_control.py+++ b/posthog/hogql/printer/test/test_property_access_control.py@@ -124,7 +124,7 @@ class TestRestrictPropertiesInHogQL(BaseTest):modifiers=HogQLQueryModifiers(useNewEventsSchema=True),)sql, _ = prepare_and_print_ast(parse_select(query), context=context, dialect="clickhouse")- assert "getSubcolumn" not in sql+ assert "customer" not in sqldef test_allowed_property_not_affected(self):PropertyAccessControl.objects.create(diff --git a/posthog/hogql/test/test_property.py b/posthog/hogql/test/test_property.pyindex 2b173feab858..6158807e6f26 100644--- a/posthog/hogql/test/test_property.py+++ b/posthog/hogql/test/test_property.py@@ -489,8 +489,8 @@ class TestProperty(BaseTest):@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)def test_property_to_expr_event_dotted_key_reads_one_flat_path(self):- # A filter key is one property name however many dots it holds, so the native read must bind the escaped- # path name rather than descend into `properties.a.b`.+ # A filter key is one property name however many dots it holds, so the native read must take the escaped+ # path name as one segment rather than descend into `properties.a.b`.expr = self._property_to_expr({"type": "event", "key": "a.b", "value": "x"})self.assertEqual(expr, self._parse_expr("properties.`a.b` = 'x'"))@@ -501,12 +501,12 @@ class TestProperty(BaseTest):)context = HogQLContext(team_id=self.team.pk, enable_select_queries=True)sql, _ = prepare_and_print_ast(query, context=context, dialect="clickhouse")- self.assertIn("getSubcolumn(events.properties, %(hogql_val_", sql)+ self.assertIn("events.properties.`a\\x252Eb`", sql)self.assertNotIn("events.properties.a", sql)- self.assertEqual(set(context.values.values()) - {"x"}, {"a%2Eb"})+ self.assertEqual(set(context.values.values()), {"x"})def test_property_string_expr_reads_dotted_key_as_one_flat_path_on_native_table(self):- # Raw-SQL readers must bind the escaped path name like the printer does, and stay safe for callers that+ # Raw-SQL readers spell the escaped path name inline like the printer does, and stay safe for callers that# run the SQL through `%` parameter substitution.expr, _ = get_property_string_expr("events", "a.b", "'a.b'", "properties", use_new_events_schema=True)self.assertNotIn("%", expr)diff --git a/posthog/hogql/test/utils.py b/posthog/hogql/test/utils.pyindex f0465c24254e..4bae78ced2f5 100644--- a/posthog/hogql/test/utils.py+++ b/posthog/hogql/test/utils.py@@ -10,7 +10,7 @@ from pydantic import BaseModelfrom posthog.schema import HogQLQueryModifiers-from posthog.hogql.escape_sql import escape_clickhouse_identifier+from posthog.hogql.escape_sql import escape_clickhouse_json_pathfrom posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.events_json import EVENTS_PROPERTIES_JSON_SUBCOLUMNS, PERSON_PROPERTIES_JSON_SUBCOLUMNS@@ -184,10 +184,9 @@ def json_dynamic_read_sql_from_parts() -> str:"""The ClickHouse SQL the resolver prints for a property read on the native-JSON events table.- `field` and `sub_object` are called once per occurrence, so a caller whose key is parameterized can hand out the- next `%(hogql_val_N)s` placeholder each time. The shape mirrors `_json_subcolumn_value_expr`: a non-empty- sub-object wins, a missing key reads as NULL, containers render as JSON. A declared path cannot hold NULL, so- its '' and [] defaults read as NULL as well.+ `field` and `sub_object` are called once per occurrence of the read. The shape mirrors+ `_json_subcolumn_value_expr`: a non-empty sub-object wins, a missing key reads as NULL, containers render as+ JSON. A declared path cannot hold NULL, so its '' and [] defaults read as NULL as well."""sub_object_read = f"toJSONString({sub_object()})" if with_sub_object else ""sub_object_again = f"toJSONString({sub_object()})" if with_sub_object else ""@@ -216,7 +215,7 @@ def json_dynamic_read_sql(root: str, path: Sequence[str | int], *, as_json: bool"""field = rootfor key in path:- field += f"[{key}]" if isinstance(key, int) else f".{escape_clickhouse_identifier(key)}"+ field += f"[{key}]" if isinstance(key, int) else f".{escape_clickhouse_json_path(key)}"if any(isinstance(key, int) for key in path):return json_dynamic_read_sql_from_parts(lambda: field, lambda: field, as_json=as_json, with_sub_object=False, declared=True@@ -225,7 +224,7 @@ def json_dynamic_read_sql(root: str, path: Sequence[str | int], *, as_json: boolPERSON_PROPERTIES_JSON_SUBCOLUMNS if root.endswith(".person_properties") else EVENTS_PROPERTIES_JSON_SUBCOLUMNS)declared = ".".join(str(key) for key in path) in declared_types- escaped = ".".join(escape_clickhouse_identifier(key) for key in path if isinstance(key, str))+ escaped = ".".join(escape_clickhouse_json_path(key) for key in path if isinstance(key, str))return json_dynamic_read_sql_from_parts(lambda: field, lambda: f"{root}.^{escaped}", as_json=as_json, declared=declared)diff --git a/posthog/models/property/util.py b/posthog/models/property/util.pyindex 2f096e7467a2..1a561f702de4 100644--- a/posthog/models/property/util.py+++ b/posthog/models/property/util.py@@ -8,7 +8,7 @@ from typing import Any, Literal, Optional, Union, castfrom posthog.hogql import astfrom posthog.hogql.database.s3_table import S3Table-from posthog.hogql.escape_sql import escape_clickhouse_identifier+from posthog.hogql.escape_sql import escape_clickhouse_json_pathfrom posthog.hogql.parser import parse_exprfrom posthog.hogql.visitor import TraversingVisitor@@ -54,7 +54,7 @@ def get_property_string_expr(if use_new_events_schema and table == "events":if materialised_table_column in ("properties", "person_properties"):- return _json_events_property_expr(property_name, var, f"{table_string}{column}")+ return _json_events_property_expr(property_name, f"{table_string}{column}")# The JSON events table has no mat_* columns at all; group columns there stay String blobs.allow_denormalized_props = False@@ -84,10 +84,10 @@ def _json_events_to_json_string_expr(value: str) -> str:return f"if(getSetting('output_format_json_escape_forward_slashes'), replaceAll({serialized}, '\\\\/', '/'), {serialized})"-def _json_events_property_expr(property_name: PropertyName, var: str, column_ref: str) -> tuple[str, bool]:- scalar_value = _json_events_subcolumn_expr(property_name, var, column_ref)+def _json_events_property_expr(property_name: PropertyName, column_ref: str) -> tuple[str, bool]:+ scalar_value = _json_events_subcolumn_expr(property_name, column_ref)object_value = _json_events_to_json_string_expr(- _json_events_subcolumn_expr(property_name, var, column_ref, sub_object=True)+ _json_events_subcolumn_expr(property_name, column_ref, sub_object=True))scalar_string = f"toString({scalar_value})"# Arrays and maps read as JSON text. Their plain text starts with '[' or '{', which a string can too, but a@@ -102,21 +102,12 @@ def _json_events_property_expr(property_name: PropertyName, var: str, column_refreturn f"ifNull({raw_value}, '')", False-def _json_events_subcolumn_expr(- property_name: PropertyName, var: str, column_ref: str, *, sub_object: bool = False-) -> str:- if "%" not in property_name and "." not in property_name:- separator = ".^" if sub_object else "."- return f"{column_ref}{separator}{escape_clickhouse_identifier(property_name)}"-+def _json_events_subcolumn_expr(property_name: PropertyName, column_ref: str, *, sub_object: bool = False) -> str:# A dot inside one key is stored as `%2E` (EVENTS_JSON_INSERT_SETTINGS), so it must not read as a path- # separator. `%` is spelled char(37) because callers run this SQL through parameter substitution, where a- # literal `%` is a format directive.- escaped_dots = f"replaceAll({var}, '.', concat(char(37), '2E'))"- escaped_backticks = f"replaceAll({escaped_dots}, char(96), concat(char(96), char(96)))"- quoted_subcolumn = f"concat(char(96), {escaped_backticks}, char(96))"- subcolumn = f"concat('^', {quoted_subcolumn})" if sub_object else escaped_dots- return f"getSubcolumn({column_ref}, {subcolumn})"+ # separator. The path prints inline with its `%` as a `\x25` escape, which passes unchanged through the+ # parameter substitution and str.format that callers run this SQL through.+ separator = ".^" if sub_object else "."+ return f"{column_ref}{separator}{escape_clickhouse_json_path(property_name.replace('.', '%2E'))}"def box_value(value: Any, remove_spaces=False) -> list[Any]: