Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): preserve empty native json property keys (#129448)

Sandy Spicerd8d492412939
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.md
index fc80bc00e11b..4f6e1327813b 100644
--- a/docs/published/handbook/engineering/databases/hogql-python.md
+++ b/docs/published/handbook/engineering/databases/hogql-python.md
@@ -201,6 +201,8 @@ DateTime property definitions keep their conversions when compared with strings,
Boolean conversion reads the native path directly because object and array text cannot match `true` or `false`; numeric conversions already read native paths directly.
For String and untyped properties, constants starting with `[` or `{` keep the full property read because arrays and objects serialize differently from scalars.
Single-property reads serialize sub-objects with ClickHouse's JSON formatter; whole-document reads still remove declared defaults.
+Empty property names, including nested path segments, use bound `getSubcolumn` calls because ClickHouse rejects empty SQL identifiers.
+Bound sub-object paths escape backticks with backslashes rather than SQL identifier quoting.
A JSON function on the native `properties` column needs JSON text, and rebuilding the whole document as text from every path is the slowest way to get it.
A call with a constant first key, such as `JSONLength(properties, 'details')`, serializes only that key's value and runs the function on it. This includes calls through `toString(properties)` and keys stored among the temporary properties.
String, floating-point and Boolean extraction skip the serialization as well: they read the native value and apply the function's conversion and missing-value rules, independently of property-definition types. A stored floating-point value keeps the parser for string extraction, because JSON number formatting decides the result. It reads as false for Boolean extraction and as `Double` for `JSONType`, because the serialized form of an integral value is an integer token.
diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.py
index 281d8a838ff1..3c2384144c55 100644
--- a/posthog/hogql/printer/clickhouse.py
+++ b/posthog/hogql/printer/clickhouse.py
@@ -27,12 +27,7 @@ from posthog.hogql.database.schema.util.where_clause_extractor import (
get_events_session_id_table_type,
)
from posthog.hogql.errors import ImpossibleASTError, InternalHogQLError, QueryError
-from posthog.hogql.escape_sql import (
- escape_clickhouse_identifier,
- escape_clickhouse_string,
- quote_clickhouse_identifier,
- safe_identifier,
-)
+from posthog.hogql.escape_sql import escape_clickhouse_identifier, escape_clickhouse_string, safe_identifier
from posthog.hogql.functions import ADD_OR_NULL_DATETIME_FUNCTIONS, FIRST_ARG_DATETIME_FUNCTIONS
from posthog.hogql.functions.clickhouse.json import JSON_FUNCTIONS
from posthog.hogql.functions.embed_text import resolve_embed_text
@@ -1525,18 +1520,21 @@ class ClickHousePrinter(BasePrinter):
expr = self.visit(node.expr)
# The native table stores a dotted key as one path named with `%2E` (EVENTS_JSON_INSERT_SETTINGS), so a
- # dot inside one key must not read as a path separator. The escaped key then takes the bound-parameter
- # branch below, as `%` would otherwise break the SQL's own parameter placeholders.
+ # dot inside one key must not read as a path separator.
keys = [key.replace(".", "%2E") for key in node.keys]
- if any("%" in key for key in keys):
+ # An empty key is not a valid SQL identifier, and a `%` would break the SQL's own parameter placeholders,
+ # so those paths go to getSubcolumn as bound values.
+ if any(not key or "%" in key for key in keys):
if node.access_type == "sub_object":
- for key in keys:
- subcolumn = f"^{quote_clickhouse_identifier(key)}"
- expr = f"getSubcolumn({expr}, {self.context.add_value(subcolumn)})"
- return expr
- subcolumn = ".".join(keys)
- return f"getSubcolumn({expr}, {self.context.add_value(subcolumn)})"
+ # getSubcolumn's path parser reads a backslash-escaped backtick, not SQL's doubled backtick.
+ escaped_keys = (key.replace("\\", "\\\\").replace("`", "\\`") for key in keys)
+ subcolumns = [f"^`{key}`" for key in escaped_keys]
+ else:
+ subcolumns = [".".join(keys)]
+ for subcolumn in subcolumns:
+ expr = f"getSubcolumn({expr}, {self.context.add_value(subcolumn)})"
+ return expr
for index, key in enumerate(keys):
separator = ".^" if node.access_type == "sub_object" and index == 0 else "."
diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.py
index d629fe657e4b..28e155bc9d7b 100644
--- a/posthog/hogql/transforms/test/test_property_types.py
+++ b/posthog/hogql/transforms/test/test_property_types.py
@@ -36,6 +36,7 @@ from posthog.schema import HogQLQueryModifiers, MaterializationMode
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
+from posthog.hogql.escape_sql import escape_hogql_string
from posthog.hogql.functions.clickhouse.json import JSON_FUNCTIONS
from posthog.hogql.functions.udfs import JSON_DROP_KEYS_CLICKHOUSE_NAME
from posthog.hogql.parser import parse_select
@@ -140,6 +141,22 @@ class _NewEventsSchemaArraySubcolumnsHelpers:
@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
class TestNewEventsSchemaArraySubcolumns(_NewEventsSchemaArraySubcolumnsHelpers, SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("extract", "JSONExtractString(properties, '')"),
+ ("has", "JSONHas(properties, '')"),
+ ("property", "properties['']"),
+ ("nested_property", "properties.parent[''].child"),
+ ("numeric", "toFloat(properties[''])"),
+ ("person", "JSONExtract(poe.properties, '', 'String')"),
+ ("wrapped", "JSONExtractString(toString(properties), '')"),
+ ]
+ )
+ def test_empty_property_keys_do_not_print_empty_identifiers(self, _name: str, expression: str) -> None:
+ printed = self._print_select(f"SELECT {expression} AS value FROM events")
+ assert ".``" not in printed
+ assert "getSubcolumn(" in printed
+
def test_json_function_rewrites_ignore_property_metadata(self) -> None:
printed = self._print_select(
"SELECT JSONExtractRaw(properties, 'value') AS value FROM events",
@@ -612,39 +629,57 @@ class TestNewEventsSchemaArraySubcolumns(_NewEventsSchemaArraySubcolumnsHelpers,
@override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=True)
@pytest.mark.usefixtures("clickhouse_database")
class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolumnsHelpers, SimpleTestCase):
- def test_json_function_conversions_and_defaults_match_the_blob(self) -> None:
- documents = [
- "{}",
- '{"value":null}',
- '{"value":""}',
- '{"value":"hello"}',
- '{"value":"2.5"}',
- '{"value":"1e3"}',
- '{"value":true}',
- '{"value":false}',
- '{"value":1.5}',
- '{"value":2.0}',
- '{"value":1e21}',
- '{"value":-1.5e-7}',
- '{"value":-2}',
- '{"value":0}',
- '{"value":[1,"two"]}',
- '{"value":{"name":"example"}}',
+ @parameterized.expand(
+ [
+ ("ordinary", ["value"]),
+ ("empty", [""]),
+ ("empty_first", ["", "child"]),
+ ("empty_middle", ["parent", "", "child"]),
+ ("empty_last", ["parent", ""]),
+ ("nested_percent", ["parent", "value%"]),
+ ("empty_then_backtick", ["", "tick`key"]),
+ ("percent_then_backtick", ["percent%", "tick`key"]),
+ ("empty_then_backslash_backtick", ["", "slash\\`key"]),
+ ("empty_then_control", ["", "line\nbreak\tkey"]),
+ ]
+ )
+ def test_json_function_conversions_and_defaults_match_the_blob(self, _name: str, keys: list[str]) -> None:
+ values = [
+ "null",
+ '""',
+ '"hello"',
+ '"2.5"',
+ '"1e3"',
+ "true",
+ "false",
+ "1.5",
+ "2.0",
+ "1e21",
+ "-1.5e-7",
+ "-2",
+ "0",
+ '[1,"two"]',
+ '{"name":"example"}',
]
+ # Wrap the raw value text, because re-serializing it would rewrite number tokens such as `1e21`.
+ prefix = "".join(f"{{{json.dumps(key)}:" for key in keys)
+ suffix = "}" * len(keys)
+ documents = ["{}", *(prefix + value + suffix for value in values)]
+ path = ", ".join(escape_hogql_string(key) for key in keys)
expressions = [
- "JSONExtractString(properties, 'value')",
- "JSONExtractInt(properties, 'value')",
- "JSONExtractUInt(properties, 'value')",
- "JSONExtractFloat(properties, 'value')",
- "JSONExtractBool(properties, 'value')",
- "JSONExtract(properties, 'value', 'Nullable(Int64)')",
- "JSONExtract(properties, 'value', 'Nullable(Bool)')",
- "JSONExtract(properties, 'value', 'String')",
- "JSONExtract(properties, 'value', 'Nullable(String)')",
- "JSONExtract(properties, 'value', 'Float64')",
- "JSONExtract(properties, 'value', 'Nullable(Float64)')",
- "JSONLength(properties, 'value')",
- "JSONType(properties, 'value')",
+ f"JSONExtractString(properties, {path})",
+ f"JSONExtractInt(properties, {path})",
+ f"JSONExtractUInt(properties, {path})",
+ f"JSONExtractFloat(properties, {path})",
+ f"JSONExtractBool(properties, {path})",
+ f"JSONExtract(properties, {path}, 'Nullable(Int64)')",
+ f"JSONExtract(properties, {path}, 'Nullable(Bool)')",
+ f"JSONExtract(properties, {path}, 'String')",
+ f"JSONExtract(properties, {path}, 'Nullable(String)')",
+ f"JSONExtract(properties, {path}, 'Float64')",
+ f"JSONExtract(properties, {path}, 'Nullable(Float64)')",
+ f"JSONLength(properties, {path})",
+ f"JSONType(properties, {path})",
]
query = "SELECT distinct_id, " + ", ".join(expressions) + " FROM events ORDER BY distinct_id"
results = []