Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): accept nullable input in json functions that return arrays or maps (#119345)

Vasco de Krijgercfaf2c0bcd61
diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.py
index f03a409b90a6..5acd6bd92990 100644
--- a/posthog/hogql/printer/clickhouse.py
+++ b/posthog/hogql/printer/clickhouse.py
@@ -32,6 +32,7 @@ from posthog.hogql.escape_sql import (
safe_identifier,
)
from posthog.hogql.functions import ADD_OR_NULL_DATETIME_FUNCTIONS, FIRST_ARG_DATETIME_FUNCTIONS
+from posthog.hogql.functions.clickhouse.json import JSON_FUNCTIONS
from posthog.hogql.functions.embed_text import resolve_embed_text
from posthog.hogql.functions.prompt_jev import is_decision_call
from posthog.hogql.functions.udfs import JSON_DROP_KEYS_CLICKHOUSE_NAME
@@ -210,6 +211,9 @@ class ClickHousePrinter(BasePrinter):
args.append(f"ifNull(toString({self.visit(arg)}), '')")
elif node.name == "toJSONString":
args = [self._visit_json_function_argument(arg) for arg in node_args]
+ # ClickHouse has no Nullable(Array) or Nullable(Map), so a nullable JSON input must become '' first.
+ elif self._returns_json_container(node) and self._json_input_needs_null_guard(node_args[0]):
+ args = [f"ifNull({self.visit(node_args[0])}, '')", *(self.visit(arg) for arg in node_args[1:])]
else:
args = [self.visit(arg) for arg in node_args]
@@ -333,6 +337,45 @@ class ClickHousePrinter(BasePrinter):
filter_part = f" FILTER (WHERE {self.visit(node.filter_expr)})" if node.filter_expr else ""
return f"{relevant_clickhouse_name}{params_part}{args_part}{filter_part}"
+ @staticmethod
+ def _returns_json_container(node: ast.Call) -> bool:
+ return (
+ node.name in JSON_FUNCTIONS
+ and bool(node.args)
+ and isinstance(node.type, ast.CallType)
+ and isinstance(node.type.return_type, (ast.ArrayType, ast.MapType))
+ )
+
+ def _json_input_needs_null_guard(self, node: ast.Expr) -> bool:
+ if self._reads_nullable_cte_column(node):
+ return True
+ input_type = node.type.resolve_constant_type(self.context) if node.type is not None else None
+ # A vacuous unknown is a plain NULL, which ClickHouse accepts as Nullable(Nothing) and returns as NULL.
+ if isinstance(input_type, ast.UnknownType) and not input_type.unanalyzable:
+ return False
+ return self._is_nullable(node)
+
+ def _reads_nullable_cte_column(self, node: ast.Expr) -> bool:
+ # A CTE column's own type keeps its nullability; the table rebuilt from the CTE drops it.
+ if isinstance(node, ast.Call):
+ return any(self._reads_nullable_cte_column(arg) for arg in node.args)
+ if isinstance(node, ast.Alias):
+ return self._reads_nullable_cte_column(node.expr)
+ field_type = resolve_field_type(node)
+ if not isinstance(field_type, ast.FieldType):
+ return False
+ table_type = field_type.table_type
+ if isinstance(table_type, ast.CTETableAliasType):
+ table_type = table_type.cte_table_type
+ if not isinstance(table_type, ast.CTETableType):
+ return False
+ try:
+ column_type = table_type.select_query_type.resolve_column_constant_type(field_type.name, self.context)
+ except Exception:
+ return True
+ is_null_literal = isinstance(column_type, ast.UnknownType) and not column_type.unanalyzable
+ return column_type.nullable and not is_null_literal
+
def _render_posthog_function_call(self, node: ast.Call, func_meta) -> str:
args = [self.visit(arg) for arg in node.args]
diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py
index fbf791cd87fe..cb8651c64f3e 100644
--- a/posthog/hogql/test/test_query.py
+++ b/posthog/hogql/test/test_query.py
@@ -1055,6 +1055,52 @@ class TestQuery(ClickhouseTestMixin, APIBaseTest):
self.assertEqual(response.results, [])
self.assertResponseMatchesSnapshot(response)
+ @parameterized.expand(
+ [
+ ("array_raw", "JSONExtractArrayRaw(properties.list)", ["1", "2"], []),
+ ("array_raw_to_string", "JSONExtractArrayRaw(toString(properties.list))", ["1", "2"], []),
+ ("keys", "JSONExtractKeys(properties.obj)", ["a", "b"], []),
+ ("keys_and_values_raw", "JSONExtractKeysAndValuesRaw(properties.map)", [("k", '"v"')], []),
+ ("keys_and_values", "JSONExtractKeysAndValues(properties.map, 'String')", [("k", "v")], []),
+ ("extract_array", "JSONExtract(properties.list, 'Array(Int64)')", [1, 2], []),
+ ("extract_map", "JSONExtract(properties.map, 'Map(String, String)')", {"k": "v"}, {}),
+ ("extract_scalar_keeps_null", "JSONExtract(properties.num, 'Nullable(Int64)')", 7, None),
+ ("null_literal_keeps_null", "JSONExtractArrayRaw(NULL)", None, None),
+ ("null_through_function_returns_empty", "JSONExtractArrayRaw(toString(NULL))", [], []),
+ ("cte_column", "JSONExtractArrayRaw(list)", ["1", "2"], [], "t"),
+ ("cte_to_string", "JSONExtractArrayRaw(toString(list))", ["1", "2"], [], "t"),
+ ("cte_unknown_function", "JSONExtractArrayRaw(list)", ["1", "2"], [], "t", "toValidUTF8(properties.list)"),
+ ("cte_equality_unchanged", "list = 'x'", False, None, "t"),
+ ("cte_inequality_unchanged", "list != 'x'", True, None, "t"),
+ ]
+ )
+ def test_json_container_functions_accept_nullable_input(
+ self,
+ _name: str,
+ expression: str,
+ expected_with: Any,
+ expected_without: Any,
+ source: str = "events",
+ cte_expression: str = "properties.list",
+ ) -> None:
+ _create_event(
+ distinct_id="d1",
+ event="with",
+ team=self.team,
+ properties={"list": [1, 2], "obj": {"a": 1, "b": 2}, "map": {"k": "v"}, "num": 7},
+ )
+ _create_event(distinct_id="d1", event="without", team=self.team, properties={})
+ flush_persons_and_events()
+
+ response = execute_hogql_query(
+ f"WITH t AS (SELECT event, {cte_expression} AS list FROM events) SELECT event, {expression} FROM {source}",
+ team=self.team,
+ pretty=False,
+ )
+
+ rows = sorted(response.results, key=lambda row: row[0])
+ self.assertEqual(rows, [("with", expected_with), ("without", expected_without)])
+
@pytest.mark.usefixtures("unittest_snapshot")
def test_hogql_proper_ifnull(self):
# latest_os_version is Nullable, splitByChar does not access Nullable argument