:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): preserve empty native json property keys (#129448)
Sandy Spicerd8d492412939
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.mdindex fc80bc00e11b..4f6e1327813b 100644--- a/docs/published/handbook/engineering/databases/hogql-python.md+++ b/docs/published/handbook/engineering/databases/hogql-python.md@@ -201,6 +201,8 @@ DateTime property definitions keep their conversions when compared with strings,Boolean conversion reads the native path directly because object and array text cannot match `true` or `false`; numeric conversions already read native paths directly.For String and untyped properties, constants starting with `[` or `{` keep the full property read because arrays and objects serialize differently from scalars.Single-property reads serialize sub-objects with ClickHouse's JSON formatter; whole-document reads still remove declared defaults.+Empty property names, including nested path segments, use bound `getSubcolumn` calls because ClickHouse rejects empty SQL identifiers.+Bound sub-object paths escape backticks with backslashes rather than SQL identifier quoting.A JSON function on the native `properties` column needs JSON text, and rebuilding the whole document as text from every path is the slowest way to get it.A call with a constant first key, such as `JSONLength(properties, 'details')`, serializes only that key's value and runs the function on it. This includes calls through `toString(properties)` and keys stored among the temporary properties.String, floating-point and Boolean extraction skip the serialization as well: they read the native value and apply the function's conversion and missing-value rules, independently of property-definition types. A stored floating-point value keeps the parser for string extraction, because JSON number formatting decides the result. It reads as false for Boolean extraction and as `Double` for `JSONType`, because the serialized form of an integral value is an integer token.diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.pyindex 281d8a838ff1..3c2384144c55 100644--- a/posthog/hogql/printer/clickhouse.py+++ b/posthog/hogql/printer/clickhouse.py@@ -27,12 +27,7 @@ from posthog.hogql.database.schema.util.where_clause_extractor import (get_events_session_id_table_type,)from posthog.hogql.errors import ImpossibleASTError, InternalHogQLError, QueryError-from posthog.hogql.escape_sql import (- escape_clickhouse_identifier,- escape_clickhouse_string,- quote_clickhouse_identifier,- safe_identifier,-)+from posthog.hogql.escape_sql import escape_clickhouse_identifier, escape_clickhouse_string, safe_identifierfrom posthog.hogql.functions import ADD_OR_NULL_DATETIME_FUNCTIONS, FIRST_ARG_DATETIME_FUNCTIONSfrom posthog.hogql.functions.clickhouse.json import JSON_FUNCTIONSfrom posthog.hogql.functions.embed_text import resolve_embed_text@@ -1525,18 +1520,21 @@ class ClickHousePrinter(BasePrinter):expr = self.visit(node.expr)# The native table stores a dotted key as one path named with `%2E` (EVENTS_JSON_INSERT_SETTINGS), so a- # dot inside one key must not read as a path separator. The escaped key then takes the bound-parameter- # branch below, as `%` would otherwise break the SQL's own parameter placeholders.+ # dot inside one key must not read as a path separator.keys = [key.replace(".", "%2E") for key in node.keys]- if any("%" in key for key in keys):+ # An empty key is not a valid SQL identifier, and a `%` would break the SQL's own parameter placeholders,+ # so those paths go to getSubcolumn as bound values.+ if any(not key or "%" in key for key in keys):if node.access_type == "sub_object":- for key in keys:- subcolumn = f"^{quote_clickhouse_identifier(key)}"- expr = f"getSubcolumn({expr}, {self.context.add_value(subcolumn)})"- return expr- subcolumn = ".".join(keys)- return f"getSubcolumn({expr}, {self.context.add_value(subcolumn)})"+ # getSubcolumn's path parser reads a backslash-escaped backtick, not SQL's doubled backtick.+ escaped_keys = (key.replace("\\", "\\\\").replace("`", "\\`") for key in keys)+ subcolumns = [f"^`{key}`" for key in escaped_keys]+ else:+ subcolumns = [".".join(keys)]+ for subcolumn in subcolumns:+ expr = f"getSubcolumn({expr}, {self.context.add_value(subcolumn)})"+ return exprfor index, key in enumerate(keys):separator = ".^" if node.access_type == "sub_object" and index == 0 else "."diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.pyindex d629fe657e4b..28e155bc9d7b 100644--- a/posthog/hogql/transforms/test/test_property_types.py+++ b/posthog/hogql/transforms/test/test_property_types.py@@ -36,6 +36,7 @@ from posthog.schema import HogQLQueryModifiers, MaterializationModefrom posthog.hogql import astfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.database.database import Database+from posthog.hogql.escape_sql import escape_hogql_stringfrom posthog.hogql.functions.clickhouse.json import JSON_FUNCTIONSfrom posthog.hogql.functions.udfs import JSON_DROP_KEYS_CLICKHOUSE_NAMEfrom posthog.hogql.parser import parse_select@@ -140,6 +141,22 @@ class _NewEventsSchemaArraySubcolumnsHelpers:@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)class TestNewEventsSchemaArraySubcolumns(_NewEventsSchemaArraySubcolumnsHelpers, SimpleTestCase):+ @parameterized.expand(+ [+ ("extract", "JSONExtractString(properties, '')"),+ ("has", "JSONHas(properties, '')"),+ ("property", "properties['']"),+ ("nested_property", "properties.parent[''].child"),+ ("numeric", "toFloat(properties[''])"),+ ("person", "JSONExtract(poe.properties, '', 'String')"),+ ("wrapped", "JSONExtractString(toString(properties), '')"),+ ]+ )+ def test_empty_property_keys_do_not_print_empty_identifiers(self, _name: str, expression: str) -> None:+ printed = self._print_select(f"SELECT {expression} AS value FROM events")+ assert ".``" not in printed+ assert "getSubcolumn(" in printed+def test_json_function_rewrites_ignore_property_metadata(self) -> None:printed = self._print_select("SELECT JSONExtractRaw(properties, 'value') AS value FROM events",@@ -612,39 +629,57 @@ class TestNewEventsSchemaArraySubcolumns(_NewEventsSchemaArraySubcolumnsHelpers,@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)@pytest.mark.usefixtures("clickhouse_database")class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolumnsHelpers, SimpleTestCase):- def test_json_function_conversions_and_defaults_match_the_blob(self) -> None:- documents = [- "{}",- '{"value":null}',- '{"value":""}',- '{"value":"hello"}',- '{"value":"2.5"}',- '{"value":"1e3"}',- '{"value":true}',- '{"value":false}',- '{"value":1.5}',- '{"value":2.0}',- '{"value":1e21}',- '{"value":-1.5e-7}',- '{"value":-2}',- '{"value":0}',- '{"value":[1,"two"]}',- '{"value":{"name":"example"}}',+ @parameterized.expand(+ [+ ("ordinary", ["value"]),+ ("empty", [""]),+ ("empty_first", ["", "child"]),+ ("empty_middle", ["parent", "", "child"]),+ ("empty_last", ["parent", ""]),+ ("nested_percent", ["parent", "value%"]),+ ("empty_then_backtick", ["", "tick`key"]),+ ("percent_then_backtick", ["percent%", "tick`key"]),+ ("empty_then_backslash_backtick", ["", "slash\\`key"]),+ ("empty_then_control", ["", "line\nbreak\tkey"]),+ ]+ )+ def test_json_function_conversions_and_defaults_match_the_blob(self, _name: str, keys: list[str]) -> None:+ values = [+ "null",+ '""',+ '"hello"',+ '"2.5"',+ '"1e3"',+ "true",+ "false",+ "1.5",+ "2.0",+ "1e21",+ "-1.5e-7",+ "-2",+ "0",+ '[1,"two"]',+ '{"name":"example"}',]+ # Wrap the raw value text, because re-serializing it would rewrite number tokens such as `1e21`.+ prefix = "".join(f"{{{json.dumps(key)}:" for key in keys)+ suffix = "}" * len(keys)+ documents = ["{}", *(prefix + value + suffix for value in values)]+ path = ", ".join(escape_hogql_string(key) for key in keys)expressions = [- "JSONExtractString(properties, 'value')",- "JSONExtractInt(properties, 'value')",- "JSONExtractUInt(properties, 'value')",- "JSONExtractFloat(properties, 'value')",- "JSONExtractBool(properties, 'value')",- "JSONExtract(properties, 'value', 'Nullable(Int64)')",- "JSONExtract(properties, 'value', 'Nullable(Bool)')",- "JSONExtract(properties, 'value', 'String')",- "JSONExtract(properties, 'value', 'Nullable(String)')",- "JSONExtract(properties, 'value', 'Float64')",- "JSONExtract(properties, 'value', 'Nullable(Float64)')",- "JSONLength(properties, 'value')",- "JSONType(properties, 'value')",+ f"JSONExtractString(properties, {path})",+ f"JSONExtractInt(properties, {path})",+ f"JSONExtractUInt(properties, {path})",+ f"JSONExtractFloat(properties, {path})",+ f"JSONExtractBool(properties, {path})",+ f"JSONExtract(properties, {path}, 'Nullable(Int64)')",+ f"JSONExtract(properties, {path}, 'Nullable(Bool)')",+ f"JSONExtract(properties, {path}, 'String')",+ f"JSONExtract(properties, {path}, 'Nullable(String)')",+ f"JSONExtract(properties, {path}, 'Float64')",+ f"JSONExtract(properties, {path}, 'Nullable(Float64)')",+ f"JSONLength(properties, {path})",+ f"JSONType(properties, {path})",]query = "SELECT distinct_id, " + ", ".join(expressions) + " FROM events ORDER BY distinct_id"results = []