:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(hogql): compile trino JSONExtractKeysAndValues to fork functions (#130307)
Jovan Sakovic709b65ecce74
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.pyindex 82e728495770..3243111ab7e9 100644--- a/posthog/hogql/printer/test/test_trino_printer.py+++ b/posthog/hogql/printer/test/test_trino_printer.py@@ -643,6 +643,8 @@ def test_tracks_every_registered_function_without_a_trino_mapping(snapshot: Snap("toTypeName(user_id)", "TRINO_FUNCTION_UNSUPPORTED"),("bitNot(id)", "TRINO_FUNCTION_UNSUPPORTED"),("range(1, 10, id)", "TRINO_RANGE_STEP_UNSUPPORTED"),+ ("JSONExtractKeysAndValues(properties, 'Array(Float64)')", "TRINO_JSON_TARGET_TYPE_UNSUPPORTED"),+ ("JSONExtractKeysAndValues(properties, user_id, 'Float64')", "TRINO_JSON_DYNAMIC_PATH_UNSUPPORTED"),],)def test_rejects_unsupported_printer_function_variants(expression: str, feature_code: str) -> None:@@ -2093,7 +2095,21 @@ def test_rejects_with_fill_before_printing_invalid_trino_sql() -> None:("date_part('year', created_at)", 'EXTRACT(YEAR FROM "users"."created_at")'),("JSONExtractKeysAndValues(properties, 'Float64')",- 'map_entries(transform_values(CAST(json_extract("users"."properties", \'$\') AS MAP(VARCHAR, JSON))',+ 'ch_json_extract_keys_and_values_float64(IF(typeof("users"."properties") = \'json\', '+ 'json_format(json_extract("users"."properties", \'$\')), CAST("users"."properties" AS VARCHAR)), '+ "'[]', false)",+ ),+ # ClickHouse reads 'Float' as Float32 and 'Int' as Int32.+ ("JSONExtractKeysAndValues(properties, 'Float')", "ch_json_extract_keys_and_values_float32("),+ (+ "JSONExtractKeysAndValues(properties, 'metrics', 1, 'Int')",+ "'[\"metrics\", 1]', false, -2147483648, 2147483647) AS ARRAY(ROW(VARCHAR, INTEGER)))",+ ),+ ("JSONExtractKeysAndValues(properties, 'Int64')", "false, -9223372036854775807 - 1, 9223372036854775807)"),+ # A Nullable type keeps a JSON null as a NULL value instead of dropping the key.+ (+ "JSONExtractKeysAndValues(properties, 'Nullable(UInt8)')",+ "'[]', true, 0, 255) AS ARRAY(ROW(VARCHAR, SMALLINT)))",),("['a', 'b'][2:]", "slice(ARRAY[%(hogql_val_0)s, %(hogql_val_1)s], 2, 2147483647)"),],diff --git a/posthog/hogql/printer/trino.py b/posthog/hogql/printer/trino.pyindex 78b69cf5bd4e..0aea54e71a01 100644--- a/posthog/hogql/printer/trino.py+++ b/posthog/hogql/printer/trino.py@@ -34,6 +34,7 @@ from posthog.hogql.printer.trino_functions import (TRINO_VECTOR_REWRITES,TRINO_WINDOW_ONLY_FUNCTIONS,)+from posthog.hogql.printer.trino_json import keys_and_values, keys_and_values_path, parse_keys_and_values_typefrom posthog.hogql.printer.types import JoinExprResponsefrom posthog.hogql.transforms.trino.errors import TrinoLoweringErrorfrom posthog.hogql.transforms.trino.expressions import constant_integer@@ -1122,7 +1123,7 @@ class TrinoPrinter(PostgresPrinter):if name in {"jsonextractkeys", "jsonextractkeysandvaluesraw", "jsonhas", "jsonlength"}:return self._visit_json_metadata(node)if name == "jsonextractkeysandvalues":- return self._visit_json_metadata(node)+ return self._visit_json_extract_keys_and_values(node)if name == "jsontype":return self._visit_json_type(node)if name == "tojsonstring":@@ -2263,27 +2264,6 @@ class TrinoPrinter(PostgresPrinter):path = self._dynamic_json_key_path(key)extracted = f"json_extract({extracted}, {path})"return f"({extracted} IS NOT NULL)"- target_type: str | None = None- if name == "jsonextractkeysandvalues":- if not path_args or not isinstance(path_args[-1], ast.Constant) or not isinstance(path_args[-1].value, str):- self._invalid_function_arguments(- node, "JSONExtractKeysAndValues requires a constant target type in Trino mode."- )- target_types = {- "bool": "BOOLEAN",- "float64": "DOUBLE",- "int64": "BIGINT",- "string": "VARCHAR",- "uint64": "DECIMAL(20, 0)",- }- target_type = target_types.get(path_args[-1].value.lower())- if target_type is None:- self._unsupported(- "TRINO_JSON_TARGET_TYPE_UNSUPPORTED",- f"JSONExtractKeysAndValues target type '{path_args[-1].value}' is not supported in Trino mode.",- node,- )- path_args = path_args[:-1]path_members: list[str | int] = []for key in path_args:if not isinstance(key, ast.Constant) or not isinstance(key.value, (str, int)):@@ -2307,21 +2287,6 @@ class TrinoPrinter(PostgresPrinter):if extracted:return self._json_default(node, f"json_size({extracted}, '$')", "0")return self._json_default(node, f"json_size({source}, {path})", "0")- if name == "jsonextractkeysandvalues":- raw = (- f"CAST({extracted} AS MAP(VARCHAR, JSON))"- if extracted- else f"CAST(json_extract({source}, {path}) AS MAP(VARCHAR, JSON))"- )- value = "__hogql_json_value"- assert target_type is not None- converted = self._convert_json_scalar(value, target_type)- result = (- f"filter(map_entries(transform_values({raw}, (__hogql_json_key, {value}) -> {converted})), "- "__hogql_entry -> __hogql_entry[2] IS NOT NULL)"- )- default = f"CAST(ARRAY[] AS ARRAY(ROW(VARCHAR, {target_type})))"- return self._json_default(node, result, default)raw = (f"CAST({extracted} AS MAP(VARCHAR, JSON))"if extracted@@ -2329,6 +2294,35 @@ class TrinoPrinter(PostgresPrinter):)return self._json_default(node, f"map_keys({raw})", "CAST(ARRAY[] AS ARRAY(VARCHAR))")+ def _visit_json_extract_keys_and_values(self, node: ast.Call) -> str:+ type_arg = node.args[-1] if len(node.args) >= 2 else None+ if not isinstance(type_arg, ast.Constant) or not isinstance(type_arg.value, str):+ self._invalid_function_arguments(+ node, "JSONExtractKeysAndValues requires a constant target type in Trino mode."+ )+ value_type = parse_keys_and_values_type(type_arg.value)+ if value_type is None:+ self._unsupported(+ "TRINO_JSON_TARGET_TYPE_UNSUPPORTED",+ f"JSONExtractKeysAndValues target type '{type_arg.value}' is not supported in Trino mode.",+ node,+ )+ path: list[str | int] = []+ for key in node.args[1:-1]:+ if (+ not isinstance(key, ast.Constant)+ or isinstance(key.value, bool)+ or not isinstance(key.value, (str, int))+ ):+ self._unsupported(+ "TRINO_JSON_DYNAMIC_PATH_UNSUPPORTED",+ "JSONExtractKeysAndValues requires a constant key path in Trino mode.",+ node,+ )+ path.append(key.value)+ path_sql = escape_trino_string(keys_and_values_path(path))+ return keys_and_values(self.visit(node.args[0]), path_sql, value_type)+def _visit_json_type(self, node: ast.Call) -> str:if not 1 <= len(node.args) <= 6:self._invalid_function_arguments(node, "JSONType expects a JSON expression and up to five path items.")diff --git a/posthog/hogql/printer/trino_json.py b/posthog/hogql/printer/trino_json.pynew file mode 100644index 000000000000..7049427fd182--- /dev/null+++ b/posthog/hogql/printer/trino_json.py@@ -0,0 +1,68 @@+"""Calls to the PostHog Trino functions that reproduce ClickHouse's JSONExtractKeysAndValues."""++import json++from posthog.dataclasses import frozen+++@frozen+class KeysAndValuesType:+ # The suffix of the ch_json_extract_keys_and_values_* function for this type.+ function: str+ # The narrower Trino integer type to cast the function's BIGINT values to, if any.+ cast: str | None+ # The ClickHouse range a number must fall in, for the ch_..._int function.+ integer_range: tuple[int, int] | None+ nullable: bool+++_VALUE_TYPES: dict[str, tuple[str, str | None, tuple[int, int] | None]] = {+ "bool": ("bool", None, None),+ "float": ("float32", None, None),+ "float32": ("float32", None, None),+ "float64": ("float64", None, None),+ "int": ("int", "INTEGER", (-(2**31), 2**31 - 1)),+ "int8": ("int", "TINYINT", (-(2**7), 2**7 - 1)),+ "int16": ("int", "SMALLINT", (-(2**15), 2**15 - 1)),+ "int32": ("int", "INTEGER", (-(2**31), 2**31 - 1)),+ "int64": ("int", None, (-(2**63), 2**63 - 1)),+ "uint8": ("int", "SMALLINT", (0, 2**8 - 1)),+ "uint16": ("int", "INTEGER", (0, 2**16 - 1)),+ "uint32": ("int", None, (0, 2**32 - 1)),+ "uint64": ("uint64", None, None),+ "string": ("string", None, None),+}+++def parse_keys_and_values_type(type_name: str) -> KeysAndValuesType | None:+ name = type_name.strip()+ nullable = name.lower().startswith("nullable(") and name.endswith(")")+ if nullable:+ name = name[len("nullable(") : -1].strip()+ value_type = _VALUE_TYPES.get(name.lower())+ if value_type is None:+ return None+ function, cast, integer_range = value_type+ return KeysAndValuesType(function=function, cast=cast, integer_range=integer_range, nullable=nullable)+++def keys_and_values_path(path: list[str | int]) -> str:+ """The path argument of the ch_json_extract_keys_and_values_* functions: a JSON array of keys and indexes."""+ return json.dumps(path, ensure_ascii=False)+++def keys_and_values(source: str, path: str, value_type: KeysAndValuesType) -> str:+ # The functions read JSON text. A native JSON value cannot be cast to VARCHAR when it holds an object,+ # and the HogQL type cannot tell it from a JSON string, so choose by the runtime type.+ document = f"IF(typeof({source}) = 'json', json_format(json_extract({source}, '$')), CAST({source} AS VARCHAR))"+ arguments = f"{document}, {path}, {'true' if value_type.nullable else 'false'}"+ if value_type.integer_range is not None:+ low, high = value_type.integer_range+ # BIGINT has no literal for its minimum value.+ low_sql = f"{low + 1} - 1" if low == -(2**63) else str(low)+ arguments += f", {low_sql}, {high}"+ call = f"ch_json_extract_keys_and_values_{value_type.function}({arguments})"+ if value_type.cast is None:+ return call+ # The function already applied the type's range, so this cast cannot fail.+ return f"CAST({call} AS ARRAY(ROW(VARCHAR, {value_type.cast})))"diff --git a/posthog/hogql/test/test_type_system.py b/posthog/hogql/test/test_type_system.pyindex 45f4766fd756..4644f0c46de5 100644--- a/posthog/hogql/test/test_type_system.py+++ b/posthog/hogql/test/test_type_system.py@@ -4,6 +4,8 @@ from uuid import UUIDimport pytest+from parameterized import parameterized+from posthog.hogql import astfrom posthog.hogql.constants import HogQLDialectfrom posthog.hogql.context import HogQLContext@@ -623,6 +625,24 @@ class TestHogQLTypeSystem:ast.StringType(nullable=False),)+ @parameterized.expand(+ [+ ("Float32", ast.FloatType(nullable=False)),+ ("Nullable(UInt8)", ast.IntegerType(nullable=True)),+ ]+ )+ def test_resolver_infers_trino_json_extract_keys_and_values_clickhouse_type(+ self, type_name: str, value_type: ast.ConstantType+ ) -> None:+ self._assert_first_column_type(+ f"SELECT JSONExtractKeysAndValues('{{\"a\": 1}}', '{type_name}')",+ ast.ArrayType(+ nullable=False,+ item_type=ast.TupleType(nullable=False, item_types=[ast.StringType(nullable=False), value_type]),+ ),+ dialect="trino",+ )+def test_resolver_infers_json_helper_function_types(self) -> None:self._assert_first_column_type("SELECT JSONHas('{\"num\": 42}', 'num')", ast.IntegerType(nullable=False))self._assert_first_column_type(diff --git a/posthog/hogql/type_system.py b/posthog/hogql/type_system.pyindex 87d48083626f..f0d3a54d7d84 100644--- a/posthog/hogql/type_system.py+++ b/posthog/hogql/type_system.py@@ -2145,7 +2145,7 @@ def _infer_json_extract_keys_and_values_type(if not isinstance(type_arg, ast.Constant) or not isinstance(type_arg.value, str):return None- runtime_type = parse_sql_runtime_type(type_arg.value, dialect=dialect)+ runtime_type = parse_sql_runtime_type(type_arg.value, dialect="clickhouse" if dialect == "trino" else dialect)if runtime_type.family == "unknown":return None