Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): compare typed properties with the empty string on native (#129818)

Sandy Spicerb79ff6f5421a
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.md
index f190b39d9f11..fc80bc00e11b 100644
--- a/docs/published/handbook/engineering/databases/hogql-python.md
+++ b/docs/published/handbook/engineering/databases/hogql-python.md
@@ -195,11 +195,11 @@ In practice, you should avoid both and access `person.properties.$browser`, whic
On the native JSON events table, HogQL compares non-empty string constants directly with declared String paths and casts Dynamic paths to Nullable(String) for properties with String or unknown definition types.
This keeps equality and membership predicates visible to the JSON skip index without failing on mixed scalar types.
-Numeric and Boolean properties on the native table compare as text when a string literal cannot be converted to the property's definition type.
+Numeric and Boolean properties on the native table compare as text when a string literal cannot be converted to the property's definition type, including the empty string and text that starts with `[` or `{`.
Boolean literals keep ClickHouse's case-insensitive spellings, including yes/no, on/off and enable/disable.
-DateTime property definitions keep their conversions when compared with strings, for both event and person properties.
+DateTime property definitions keep their conversions when compared with strings, for both event and person properties; only a comparison with the empty string alone compares as text, and an `IN` list that also holds dates keeps the conversion.
Boolean conversion reads the native path directly because object and array text cannot match `true` or `false`; numeric conversions already read native paths directly.
-Constants starting with `[` or `{` keep the full property read because arrays and objects serialize differently from scalars.
+For String and untyped properties, constants starting with `[` or `{` keep the full property read because arrays and objects serialize differently from scalars.
Single-property reads serialize sub-objects with ClickHouse's JSON formatter; whole-document reads still remove declared defaults.
A JSON function on the native `properties` column needs JSON text, and rebuilding the whole document as text from every path is the slowest way to get it.
A call with a constant first key, such as `JSONLength(properties, 'details')`, serializes only that key's value and runs the function on it. This includes calls through `toString(properties)` and keys stored among the temporary properties.
diff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.py
index 4955e9f6155e..c438276ce9be 100644
--- a/posthog/hogql/transforms/clickhouse_property_resolution.py
+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py
@@ -2141,8 +2141,8 @@ class ClickHousePropertyResolver(GroupedClauseTracker, CloningVisitor):
)
def _typed_property_to_compare_as_text(self, expr: ast.Expr, values: list[str]) -> _OptimizableProperty | None:
- """A Numeric or Boolean JSON property compared to text it cannot be read as, read as text the way the legacy
- table compares it.
+ """A Numeric, Boolean or DateTime JSON property compared to text it cannot be read as, read as text the way the
+ legacy table compares it.
A property definition keeps the type of the first value seen, so a Numeric key can still hold text that a
Float64 read never matches, and a Boolean key can hold text that the Bool read turns into NULL. Comparing that
@@ -2156,9 +2156,15 @@ class ClickHousePropertyResolver(GroupedClauseTracker, CloningVisitor):
field_type, property_name = single
property_type = ast.PropertyType(field_type=field_type, chain=[property_name])
semantic_type = runtime_type_from_constant_type(property_type.resolve_constant_type(self.context))
- if semantic_type.family not in ("float", "boolean"):
+ if semantic_type.family == "datetime":
+ # ClickHouse's best-effort parser accepts spellings a Python check would reject, so only a lone empty
+ # string counts as unreadable. A list that also holds dates keeps its conversion, because '2024-04-06'
+ # matches a stored '2024-04-06 00:00:00' only through it.
+ if values != [""]:
+ return None
+ elif semantic_type.family not in ("float", "boolean"):
return None
- if all(convert_literal_to_runtime_type(value, semantic_type) is not None for value in values):
+ elif all(convert_literal_to_runtime_type(value, semantic_type) is not None for value in values):
return None
source = resolve_materialized_property_source(field_type, property_name, self.context)
if source is None or not _is_dynamic_json_source(source):
@@ -2535,13 +2541,17 @@ class ClickHousePropertyResolver(GroupedClauseTracker, CloningVisitor):
values = [constant.value] if constant is not None else None
else:
values = self._extract_string_constants(node.right)
- if not values or any(not value or value.startswith(("[", "{")) for value in values):
+ if not values:
return None
# A user-written `toBool(properties.x)` over a String property keeps its cast and is handled by
# `_optimize_materialized_equals`. The swapper's Boolean cast over a Boolean-typed property is the one looked
# through here, so that text the property cannot be read as compares as text.
+ # A String property keeps its full read for '' and for constants that start with `[` or `{`, because a declared
+ # String column stores '' the same as a missing key, and arrays and objects serialize differently from scalars.
+ # A typed property compares those constants as text, because its typed read of them can only fail.
+ scalar_text = all(value and not value.startswith(("[", "{")) for value in values)
prop = None
- if not self._is_boolean_conversion(property_expr):
+ if scalar_text and not self._is_boolean_conversion(property_expr):
prop = self._materialized_string_property(property_expr, allow_dynamic_json=True)
prop = prop or self._typed_property_to_compare_as_text(property_expr, values)
if prop is None or prop.source.kind != "json_subcolumn":
diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.py
index d12294ab53e4..d629fe657e4b 100644
--- a/posthog/hogql/transforms/test/test_property_types.py
+++ b/posthog/hogql/transforms/test/test_property_types.py
@@ -930,12 +930,16 @@ class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolum
@parameterized.expand(
[
- ("text_equals", "properties.step = 'template'", [1, 0, 0, 0, 0, 0]),
- ("text_not_equals", "properties.step != 'template'", [0, 1, 1, 1, 1, 1]),
- ("text_in", "properties.step IN ('template', 'domain')", [1, 1, 0, 0, None, 0]),
- ("text_and_number_in", "properties.step IN ('template', '5')", [1, 0, 1, 1, None, 0]),
- ("number_equals", "properties.step = '5'", [0, 0, 1, 1, 0, 0]),
- ("non_finite_text", "properties.step = 'nan'", [0, 0, 0, 0, 0, 1]),
+ ("text_equals", "properties.step = 'template'", [1, 0, 0, 0, 0, 0, 0, 0]),
+ ("text_not_equals", "properties.step != 'template'", [0, 1, 1, 1, 1, 1, 1, 1]),
+ ("text_in", "properties.step IN ('template', 'domain')", [1, 1, 0, 0, None, 0, 0, 0]),
+ ("text_and_number_in", "properties.step IN ('template', '5')", [1, 0, 1, 1, None, 0, 0, 0]),
+ ("number_equals", "properties.step = '5'", [0, 0, 1, 1, 0, 0, 0, 0]),
+ ("non_finite_text", "properties.step = 'nan'", [0, 0, 0, 0, 0, 1, 0, 0]),
+ ("empty_equals", "properties.step = ''", [0, 0, 0, 0, 0, 0, 1, 0]),
+ ("empty_not_equals", "properties.step != ''", [1, 1, 1, 1, 1, 1, 0, 1]),
+ ("empty_in", "properties.step IN ('', '5')", [0, 0, 1, 1, None, 0, 1, 0]),
+ ("array_text_equals", "properties.step = '[5]'", [0, 0, 0, 0, 0, 0, 0, 1]),
]
)
def test_numeric_property_compared_to_text_matches_text(
@@ -956,6 +960,8 @@ class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolum
'{"step":"5"}',
"{}",
'{"step":"nan"}',
+ '{"step":""}',
+ '{"step":[5]}',
],
},
)
@@ -963,16 +969,17 @@ class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolum
@parameterized.expand(
[
- ("text_equals", "properties.opted_in = 'pending'", [1, 0, 0, 0, 0, 0]),
- ("text_not_equals", "properties.opted_in != 'pending'", [0, 1, 1, 1, 1, 1]),
- ("text_in", "properties.opted_in IN ('pending', 'declined')", [1, 1, 0, 0, None, 0]),
- ("text_and_boolean_in", "properties.opted_in IN ('pending', 'true')", [1, 0, 1, 1, None, 0]),
- ("boolean_text_equals", "properties.opted_in = 'true'", [0, 0, 1, 1, 0, 0]),
- ("boolean_digit_equals", "properties.opted_in = '1'", [0, 0, 1, 1, 0, 0]),
- ("boolean_yes_equals", "properties.opted_in = 'YeS'", [0, 0, 1, 1, 0, 0]),
- ("boolean_no_equals", "properties.opted_in = 'No'", [0, 0, 0, 0, 0, 1]),
- ("boolean_t_equals", "properties.opted_in = 'T'", [0, 0, 1, 1, 0, 0]),
- ("boolean_f_equals", "properties.opted_in = 'F'", [0, 0, 0, 0, 0, 1]),
+ ("text_equals", "properties.opted_in = 'pending'", [1, 0, 0, 0, 0, 0, 0]),
+ ("text_not_equals", "properties.opted_in != 'pending'", [0, 1, 1, 1, 1, 1, 1]),
+ ("text_in", "properties.opted_in IN ('pending', 'declined')", [1, 1, 0, 0, None, 0, 0]),
+ ("text_and_boolean_in", "properties.opted_in IN ('pending', 'true')", [1, 0, 1, 1, None, 0, 0]),
+ ("boolean_text_equals", "properties.opted_in = 'true'", [0, 0, 1, 1, 0, 0, 0]),
+ ("boolean_digit_equals", "properties.opted_in = '1'", [0, 0, 1, 1, 0, 0, 0]),
+ ("boolean_yes_equals", "properties.opted_in = 'YeS'", [0, 0, 1, 1, 0, 0, 0]),
+ ("boolean_no_equals", "properties.opted_in = 'No'", [0, 0, 0, 0, 0, 1, 0]),
+ ("boolean_t_equals", "properties.opted_in = 'T'", [0, 0, 1, 1, 0, 0, 0]),
+ ("boolean_f_equals", "properties.opted_in = 'F'", [0, 0, 0, 0, 0, 1, 0]),
+ ("empty_equals", "properties.opted_in = ''", [0, 0, 0, 0, 0, 0, 1]),
(
"boolean_true_aliases_in",
"countIf(properties.opted_in IN ('true', '1', 'T', 'Y', 'YeS', 'ON', 'Enable', 'Enabled'))",
@@ -1003,6 +1010,34 @@ class TestNewEventsSchemaArraySubcolumnsClickhouse(_NewEventsSchemaArraySubcolum
'{"opted_in":"true"}',
"{}",
'{"opted_in":false}',
+ '{"opted_in":""}',
+ ],
+ },
+ )
+ assert [row[0] for row in rows] == expected
+
+ @parameterized.expand(
+ [
+ ("empty_equals", "properties.signed_up_at = ''", [0, 1, 0, 0]),
+ ("empty_in", "properties.signed_up_at IN ('', '2024-05-06')", [0, None, None, 1]),
+ ]
+ )
+ def test_datetime_property_compared_to_empty_text_matches_text(
+ self, _name: str, predicate: str, expected: list[int]
+ ) -> None:
+ context = self._context(event_properties={"signed_up_at": {"type": "DateTime"}})
+ with patch("posthog.hogql.printer.utils.build_property_swapper"):
+ printed, _ = prepare_and_print_ast(parse_select(f"SELECT {predicate} FROM events"), context, "clickhouse")
+ rows = sync_execute(
+ "WITH events_json AS (SELECT 1 AS team_id, CAST(arrayJoin(%(documents)s), 'JSON') AS properties) "
+ + printed,
+ {
+ **context.values,
+ "documents": [
+ '{"signed_up_at":"2024-04-06 07:08:09"}',
+ '{"signed_up_at":""}',
+ "{}",
+ '{"signed_up_at":"2024-05-06 00:00:00"}',
],
},
)