Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(hogql): compile trino JSONExtractKeysAndValues to fork functions (#130307)

Jovan Sakovic709b65ecce74
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.py
index 82e728495770..3243111ab7e9 100644
--- a/posthog/hogql/printer/test/test_trino_printer.py
+++ b/posthog/hogql/printer/test/test_trino_printer.py
@@ -643,6 +643,8 @@ def test_tracks_every_registered_function_without_a_trino_mapping(snapshot: Snap
("toTypeName(user_id)", "TRINO_FUNCTION_UNSUPPORTED"),
("bitNot(id)", "TRINO_FUNCTION_UNSUPPORTED"),
("range(1, 10, id)", "TRINO_RANGE_STEP_UNSUPPORTED"),
+ ("JSONExtractKeysAndValues(properties, 'Array(Float64)')", "TRINO_JSON_TARGET_TYPE_UNSUPPORTED"),
+ ("JSONExtractKeysAndValues(properties, user_id, 'Float64')", "TRINO_JSON_DYNAMIC_PATH_UNSUPPORTED"),
],
)
def test_rejects_unsupported_printer_function_variants(expression: str, feature_code: str) -> None:
@@ -2093,7 +2095,21 @@ def test_rejects_with_fill_before_printing_invalid_trino_sql() -> None:
("date_part('year', created_at)", 'EXTRACT(YEAR FROM "users"."created_at")'),
(
"JSONExtractKeysAndValues(properties, 'Float64')",
- 'map_entries(transform_values(CAST(json_extract("users"."properties", \'$\') AS MAP(VARCHAR, JSON))',
+ 'ch_json_extract_keys_and_values_float64(IF(typeof("users"."properties") = \'json\', '
+ 'json_format(json_extract("users"."properties", \'$\')), CAST("users"."properties" AS VARCHAR)), '
+ "'[]', false)",
+ ),
+ # ClickHouse reads 'Float' as Float32 and 'Int' as Int32.
+ ("JSONExtractKeysAndValues(properties, 'Float')", "ch_json_extract_keys_and_values_float32("),
+ (
+ "JSONExtractKeysAndValues(properties, 'metrics', 1, 'Int')",
+ "'[\"metrics\", 1]', false, -2147483648, 2147483647) AS ARRAY(ROW(VARCHAR, INTEGER)))",
+ ),
+ ("JSONExtractKeysAndValues(properties, 'Int64')", "false, -9223372036854775807 - 1, 9223372036854775807)"),
+ # A Nullable type keeps a JSON null as a NULL value instead of dropping the key.
+ (
+ "JSONExtractKeysAndValues(properties, 'Nullable(UInt8)')",
+ "'[]', true, 0, 255) AS ARRAY(ROW(VARCHAR, SMALLINT)))",
),
("['a', 'b'][2:]", "slice(ARRAY[%(hogql_val_0)s, %(hogql_val_1)s], 2, 2147483647)"),
],
diff --git a/posthog/hogql/printer/trino.py b/posthog/hogql/printer/trino.py
index 78b69cf5bd4e..0aea54e71a01 100644
--- a/posthog/hogql/printer/trino.py
+++ b/posthog/hogql/printer/trino.py
@@ -34,6 +34,7 @@ from posthog.hogql.printer.trino_functions import (
TRINO_VECTOR_REWRITES,
TRINO_WINDOW_ONLY_FUNCTIONS,
)
+from posthog.hogql.printer.trino_json import keys_and_values, keys_and_values_path, parse_keys_and_values_type
from posthog.hogql.printer.types import JoinExprResponse
from posthog.hogql.transforms.trino.errors import TrinoLoweringError
from posthog.hogql.transforms.trino.expressions import constant_integer
@@ -1122,7 +1123,7 @@ class TrinoPrinter(PostgresPrinter):
if name in {"jsonextractkeys", "jsonextractkeysandvaluesraw", "jsonhas", "jsonlength"}:
return self._visit_json_metadata(node)
if name == "jsonextractkeysandvalues":
- return self._visit_json_metadata(node)
+ return self._visit_json_extract_keys_and_values(node)
if name == "jsontype":
return self._visit_json_type(node)
if name == "tojsonstring":
@@ -2263,27 +2264,6 @@ class TrinoPrinter(PostgresPrinter):
path = self._dynamic_json_key_path(key)
extracted = f"json_extract({extracted}, {path})"
return f"({extracted} IS NOT NULL)"
- target_type: str | None = None
- if name == "jsonextractkeysandvalues":
- if not path_args or not isinstance(path_args[-1], ast.Constant) or not isinstance(path_args[-1].value, str):
- self._invalid_function_arguments(
- node, "JSONExtractKeysAndValues requires a constant target type in Trino mode."
- )
- target_types = {
- "bool": "BOOLEAN",
- "float64": "DOUBLE",
- "int64": "BIGINT",
- "string": "VARCHAR",
- "uint64": "DECIMAL(20, 0)",
- }
- target_type = target_types.get(path_args[-1].value.lower())
- if target_type is None:
- self._unsupported(
- "TRINO_JSON_TARGET_TYPE_UNSUPPORTED",
- f"JSONExtractKeysAndValues target type '{path_args[-1].value}' is not supported in Trino mode.",
- node,
- )
- path_args = path_args[:-1]
path_members: list[str | int] = []
for key in path_args:
if not isinstance(key, ast.Constant) or not isinstance(key.value, (str, int)):
@@ -2307,21 +2287,6 @@ class TrinoPrinter(PostgresPrinter):
if extracted:
return self._json_default(node, f"json_size({extracted}, '$')", "0")
return self._json_default(node, f"json_size({source}, {path})", "0")
- if name == "jsonextractkeysandvalues":
- raw = (
- f"CAST({extracted} AS MAP(VARCHAR, JSON))"
- if extracted
- else f"CAST(json_extract({source}, {path}) AS MAP(VARCHAR, JSON))"
- )
- value = "__hogql_json_value"
- assert target_type is not None
- converted = self._convert_json_scalar(value, target_type)
- result = (
- f"filter(map_entries(transform_values({raw}, (__hogql_json_key, {value}) -> {converted})), "
- "__hogql_entry -> __hogql_entry[2] IS NOT NULL)"
- )
- default = f"CAST(ARRAY[] AS ARRAY(ROW(VARCHAR, {target_type})))"
- return self._json_default(node, result, default)
raw = (
f"CAST({extracted} AS MAP(VARCHAR, JSON))"
if extracted
@@ -2329,6 +2294,35 @@ class TrinoPrinter(PostgresPrinter):
)
return self._json_default(node, f"map_keys({raw})", "CAST(ARRAY[] AS ARRAY(VARCHAR))")
+ def _visit_json_extract_keys_and_values(self, node: ast.Call) -> str:
+ type_arg = node.args[-1] if len(node.args) >= 2 else None
+ if not isinstance(type_arg, ast.Constant) or not isinstance(type_arg.value, str):
+ self._invalid_function_arguments(
+ node, "JSONExtractKeysAndValues requires a constant target type in Trino mode."
+ )
+ value_type = parse_keys_and_values_type(type_arg.value)
+ if value_type is None:
+ self._unsupported(
+ "TRINO_JSON_TARGET_TYPE_UNSUPPORTED",
+ f"JSONExtractKeysAndValues target type '{type_arg.value}' is not supported in Trino mode.",
+ node,
+ )
+ path: list[str | int] = []
+ for key in node.args[1:-1]:
+ if (
+ not isinstance(key, ast.Constant)
+ or isinstance(key.value, bool)
+ or not isinstance(key.value, (str, int))
+ ):
+ self._unsupported(
+ "TRINO_JSON_DYNAMIC_PATH_UNSUPPORTED",
+ "JSONExtractKeysAndValues requires a constant key path in Trino mode.",
+ node,
+ )
+ path.append(key.value)
+ path_sql = escape_trino_string(keys_and_values_path(path))
+ return keys_and_values(self.visit(node.args[0]), path_sql, value_type)
+
def _visit_json_type(self, node: ast.Call) -> str:
if not 1 <= len(node.args) <= 6:
self._invalid_function_arguments(node, "JSONType expects a JSON expression and up to five path items.")
diff --git a/posthog/hogql/printer/trino_json.py b/posthog/hogql/printer/trino_json.py
new file mode 100644
index 000000000000..7049427fd182
--- /dev/null
+++ b/posthog/hogql/printer/trino_json.py
@@ -0,0 +1,68 @@
+"""Calls to the PostHog Trino functions that reproduce ClickHouse's JSONExtractKeysAndValues."""
+
+import json
+
+from posthog.dataclasses import frozen
+
+
+@frozen
+class KeysAndValuesType:
+ # The suffix of the ch_json_extract_keys_and_values_* function for this type.
+ function: str
+ # The narrower Trino integer type to cast the function's BIGINT values to, if any.
+ cast: str | None
+ # The ClickHouse range a number must fall in, for the ch_..._int function.
+ integer_range: tuple[int, int] | None
+ nullable: bool
+
+
+_VALUE_TYPES: dict[str, tuple[str, str | None, tuple[int, int] | None]] = {
+ "bool": ("bool", None, None),
+ "float": ("float32", None, None),
+ "float32": ("float32", None, None),
+ "float64": ("float64", None, None),
+ "int": ("int", "INTEGER", (-(2**31), 2**31 - 1)),
+ "int8": ("int", "TINYINT", (-(2**7), 2**7 - 1)),
+ "int16": ("int", "SMALLINT", (-(2**15), 2**15 - 1)),
+ "int32": ("int", "INTEGER", (-(2**31), 2**31 - 1)),
+ "int64": ("int", None, (-(2**63), 2**63 - 1)),
+ "uint8": ("int", "SMALLINT", (0, 2**8 - 1)),
+ "uint16": ("int", "INTEGER", (0, 2**16 - 1)),
+ "uint32": ("int", None, (0, 2**32 - 1)),
+ "uint64": ("uint64", None, None),
+ "string": ("string", None, None),
+}
+
+
+def parse_keys_and_values_type(type_name: str) -> KeysAndValuesType | None:
+ name = type_name.strip()
+ nullable = name.lower().startswith("nullable(") and name.endswith(")")
+ if nullable:
+ name = name[len("nullable(") : -1].strip()
+ value_type = _VALUE_TYPES.get(name.lower())
+ if value_type is None:
+ return None
+ function, cast, integer_range = value_type
+ return KeysAndValuesType(function=function, cast=cast, integer_range=integer_range, nullable=nullable)
+
+
+def keys_and_values_path(path: list[str | int]) -> str:
+ """The path argument of the ch_json_extract_keys_and_values_* functions: a JSON array of keys and indexes."""
+ return json.dumps(path, ensure_ascii=False)
+
+
+def keys_and_values(source: str, path: str, value_type: KeysAndValuesType) -> str:
+ # The functions read JSON text. A native JSON value cannot be cast to VARCHAR when it holds an object,
+ # and the HogQL type cannot tell it from a JSON string, so choose by the runtime type.
+ document = f"IF(typeof({source}) = 'json', json_format(json_extract({source}, '$')), CAST({source} AS VARCHAR))"
+ arguments = f"{document}, {path}, {'true' if value_type.nullable else 'false'}"
+ if value_type.integer_range is not None:
+ low, high = value_type.integer_range
+ # BIGINT has no literal for its minimum value.
+ low_sql = f"{low + 1} - 1" if low == -(2**63) else str(low)
+ arguments += f", {low_sql}, {high}"
+ call = f"ch_json_extract_keys_and_values_{value_type.function}({arguments})"
+ if value_type.cast is None:
+ return call
+ # The function already applied the type's range, so this cast cannot fail.
+ return f"CAST({call} AS ARRAY(ROW(VARCHAR, {value_type.cast})))"
diff --git a/posthog/hogql/test/test_type_system.py b/posthog/hogql/test/test_type_system.py
index 45f4766fd756..4644f0c46de5 100644
--- a/posthog/hogql/test/test_type_system.py
+++ b/posthog/hogql/test/test_type_system.py
@@ -4,6 +4,8 @@ from uuid import UUID
import pytest
+from parameterized import parameterized
+
from posthog.hogql import ast
from posthog.hogql.constants import HogQLDialect
from posthog.hogql.context import HogQLContext
@@ -623,6 +625,24 @@ class TestHogQLTypeSystem:
ast.StringType(nullable=False),
)
+ @parameterized.expand(
+ [
+ ("Float32", ast.FloatType(nullable=False)),
+ ("Nullable(UInt8)", ast.IntegerType(nullable=True)),
+ ]
+ )
+ def test_resolver_infers_trino_json_extract_keys_and_values_clickhouse_type(
+ self, type_name: str, value_type: ast.ConstantType
+ ) -> None:
+ self._assert_first_column_type(
+ f"SELECT JSONExtractKeysAndValues('{{\"a\": 1}}', '{type_name}')",
+ ast.ArrayType(
+ nullable=False,
+ item_type=ast.TupleType(nullable=False, item_types=[ast.StringType(nullable=False), value_type]),
+ ),
+ dialect="trino",
+ )
+
def test_resolver_infers_json_helper_function_types(self) -> None:
self._assert_first_column_type("SELECT JSONHas('{\"num\": 42}', 'num')", ast.IntegerType(nullable=False))
self._assert_first_column_type(
diff --git a/posthog/hogql/type_system.py b/posthog/hogql/type_system.py
index 87d48083626f..f0d3a54d7d84 100644
--- a/posthog/hogql/type_system.py
+++ b/posthog/hogql/type_system.py
@@ -2145,7 +2145,7 @@ def _infer_json_extract_keys_and_values_type(
if not isinstance(type_arg, ast.Constant) or not isinstance(type_arg.value, str):
return None
- runtime_type = parse_sql_runtime_type(type_arg.value, dialect=dialect)
+ runtime_type = parse_sql_runtime_type(type_arg.value, dialect="clickhouse" if dialect == "trino" else dialect)
if runtime_type.family == "unknown":
return None