:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): accept nullable input in json functions that return arrays or maps (#119345)
Vasco de Krijgercfaf2c0bcd61
diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.pyindex f03a409b90a6..5acd6bd92990 100644--- a/posthog/hogql/printer/clickhouse.py+++ b/posthog/hogql/printer/clickhouse.py@@ -32,6 +32,7 @@ from posthog.hogql.escape_sql import (safe_identifier,)from posthog.hogql.functions import ADD_OR_NULL_DATETIME_FUNCTIONS, FIRST_ARG_DATETIME_FUNCTIONS+from posthog.hogql.functions.clickhouse.json import JSON_FUNCTIONSfrom posthog.hogql.functions.embed_text import resolve_embed_textfrom posthog.hogql.functions.prompt_jev import is_decision_callfrom posthog.hogql.functions.udfs import JSON_DROP_KEYS_CLICKHOUSE_NAME@@ -210,6 +211,9 @@ class ClickHousePrinter(BasePrinter):args.append(f"ifNull(toString({self.visit(arg)}), '')")elif node.name == "toJSONString":args = [self._visit_json_function_argument(arg) for arg in node_args]+ # ClickHouse has no Nullable(Array) or Nullable(Map), so a nullable JSON input must become '' first.+ elif self._returns_json_container(node) and self._json_input_needs_null_guard(node_args[0]):+ args = [f"ifNull({self.visit(node_args[0])}, '')", *(self.visit(arg) for arg in node_args[1:])]else:args = [self.visit(arg) for arg in node_args]@@ -333,6 +337,45 @@ class ClickHousePrinter(BasePrinter):filter_part = f" FILTER (WHERE {self.visit(node.filter_expr)})" if node.filter_expr else ""return f"{relevant_clickhouse_name}{params_part}{args_part}{filter_part}"+ @staticmethod+ def _returns_json_container(node: ast.Call) -> bool:+ return (+ node.name in JSON_FUNCTIONS+ and bool(node.args)+ and isinstance(node.type, ast.CallType)+ and isinstance(node.type.return_type, (ast.ArrayType, ast.MapType))+ )++ def _json_input_needs_null_guard(self, node: ast.Expr) -> bool:+ if self._reads_nullable_cte_column(node):+ return True+ input_type = node.type.resolve_constant_type(self.context) if node.type is not None else None+ # A vacuous unknown is a plain NULL, which ClickHouse accepts as Nullable(Nothing) and returns as NULL.+ if isinstance(input_type, ast.UnknownType) and not input_type.unanalyzable:+ return False+ return self._is_nullable(node)++ def _reads_nullable_cte_column(self, node: ast.Expr) -> bool:+ # A CTE column's own type keeps its nullability; the table rebuilt from the CTE drops it.+ if isinstance(node, ast.Call):+ return any(self._reads_nullable_cte_column(arg) for arg in node.args)+ if isinstance(node, ast.Alias):+ return self._reads_nullable_cte_column(node.expr)+ field_type = resolve_field_type(node)+ if not isinstance(field_type, ast.FieldType):+ return False+ table_type = field_type.table_type+ if isinstance(table_type, ast.CTETableAliasType):+ table_type = table_type.cte_table_type+ if not isinstance(table_type, ast.CTETableType):+ return False+ try:+ column_type = table_type.select_query_type.resolve_column_constant_type(field_type.name, self.context)+ except Exception:+ return True+ is_null_literal = isinstance(column_type, ast.UnknownType) and not column_type.unanalyzable+ return column_type.nullable and not is_null_literal+def _render_posthog_function_call(self, node: ast.Call, func_meta) -> str:args = [self.visit(arg) for arg in node.args]diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.pyindex fbf791cd87fe..cb8651c64f3e 100644--- a/posthog/hogql/test/test_query.py+++ b/posthog/hogql/test/test_query.py@@ -1055,6 +1055,52 @@ class TestQuery(ClickhouseTestMixin, APIBaseTest):self.assertEqual(response.results, [])self.assertResponseMatchesSnapshot(response)+ @parameterized.expand(+ [+ ("array_raw", "JSONExtractArrayRaw(properties.list)", ["1", "2"], []),+ ("array_raw_to_string", "JSONExtractArrayRaw(toString(properties.list))", ["1", "2"], []),+ ("keys", "JSONExtractKeys(properties.obj)", ["a", "b"], []),+ ("keys_and_values_raw", "JSONExtractKeysAndValuesRaw(properties.map)", [("k", '"v"')], []),+ ("keys_and_values", "JSONExtractKeysAndValues(properties.map, 'String')", [("k", "v")], []),+ ("extract_array", "JSONExtract(properties.list, 'Array(Int64)')", [1, 2], []),+ ("extract_map", "JSONExtract(properties.map, 'Map(String, String)')", {"k": "v"}, {}),+ ("extract_scalar_keeps_null", "JSONExtract(properties.num, 'Nullable(Int64)')", 7, None),+ ("null_literal_keeps_null", "JSONExtractArrayRaw(NULL)", None, None),+ ("null_through_function_returns_empty", "JSONExtractArrayRaw(toString(NULL))", [], []),+ ("cte_column", "JSONExtractArrayRaw(list)", ["1", "2"], [], "t"),+ ("cte_to_string", "JSONExtractArrayRaw(toString(list))", ["1", "2"], [], "t"),+ ("cte_unknown_function", "JSONExtractArrayRaw(list)", ["1", "2"], [], "t", "toValidUTF8(properties.list)"),+ ("cte_equality_unchanged", "list = 'x'", False, None, "t"),+ ("cte_inequality_unchanged", "list != 'x'", True, None, "t"),+ ]+ )+ def test_json_container_functions_accept_nullable_input(+ self,+ _name: str,+ expression: str,+ expected_with: Any,+ expected_without: Any,+ source: str = "events",+ cte_expression: str = "properties.list",+ ) -> None:+ _create_event(+ distinct_id="d1",+ event="with",+ team=self.team,+ properties={"list": [1, 2], "obj": {"a": 1, "b": 2}, "map": {"k": "v"}, "num": 7},+ )+ _create_event(distinct_id="d1", event="without", team=self.team, properties={})+ flush_persons_and_events()++ response = execute_hogql_query(+ f"WITH t AS (SELECT event, {cte_expression} AS list FROM events) SELECT event, {expression} FROM {source}",+ team=self.team,+ pretty=False,+ )++ rows = sorted(response.results, key=lambda row: row[0])+ self.assertEqual(rows, [("with", expected_with), ("without", expected_without)])+@pytest.mark.usefixtures("unittest_snapshot")def test_hogql_proper_ifnull(self):# latest_os_version is Nullable, splitByChar does not access Nullable argument