:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): close six more trino lowering gaps (#124905)
Jovan Sakovic041a79b9dc59
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.pyindex 34669a281b7e..06ec7b7c94a3 100644--- a/posthog/hogql/printer/test/test_trino_printer.py+++ b/posthog/hogql/printer/test/test_trino_printer.py@@ -516,6 +516,15 @@ def test_ignores_clickhouse_cte_materialization_hint() -> None:("L2Norm([3, 4])", "sqrt(dot_product(ARRAY[3, 4], ARRAY[3, 4]))"),("toModifiedJulianDay(created_at)", "date_diff('day', DATE '1858-11-17'"),("fromModifiedJulianDay(60000)", "date_add('day', CAST(60000 AS BIGINT), DATE '1858-11-17')"),+ ("arrayFilter(x -> 0, [1, 2])", 'filter(ARRAY[1, 2], "x" -> CAST(0 AS BOOLEAN))'),+ (+ "arrayFilter((v, d) -> d >= 2, [10, 20], [1, 2])",+ 'transform(filter(zip_with(ARRAY[10, 20], ARRAY[1, 2], ("v", "d") -> ROW("v", ("d" >= 2)))',+ ),+ (+ "arrayExists((v, d) -> d, [10, 20], [0, 1])",+ 'any_match(zip_with(ARRAY[10, 20], ARRAY[0, 1], ("v", "d") -> CAST("d" AS BOOLEAN))',+ ),],)def test_prints_core_trino_expression_mappings(expression: str, expected: str) -> None:@@ -526,6 +535,33 @@ def test_prints_core_trino_expression_mappings(expression: str, expected: str) -assert expected in sql+ ("query", "expected"),+ [+ (+ "WITH a AS (SELECT arraySort([3, 1, 2]) AS xs) SELECT intDiv(length(xs) + 1, 2) FROM a",+ '(CAST((cardinality("a"."xs") + 1) AS BIGINT) / CAST(2 AS BIGINT))',+ ),+ (+ "SELECT d + 1 FROM (SELECT toDate('2024-01-01') AS d UNION ALL SELECT toDateTime('2024-01-01 00:00:00') AS d)",+ "date_add('second'",+ ),+ (+ "WITH a AS (SELECT [toDate('2024-01-01')] AS days) SELECT days[1] + 1 FROM a",+ 'date_add(\'day\', CAST(1 AS BIGINT), element_at("a"."days", 1))',+ ),+ (+ "WITH a AS (SELECT [toDate('2024-01-01')] AS days) SELECT arrayJoin(days) + 1 FROM a",+ 'date_add(\'day\', CAST(1 AS BIGINT), "__trino_array_function_0"."value_0")',+ ),+ ],+)+def test_derived_column_types_reach_type_dependent_lowerings(query: str, expected: str) -> None:+ sql, _ = prepare_and_print_ast(parse_select(query), _context_with_trino_table(), "trino")++ assert expected in sql++def test_drops_clickhouse_global_distribution_modifiers() -> None:membership_query = parse_select("SELECT user_id FROM users WHERE user_id IN ['a']")assert isinstance(membership_query, ast.SelectQuery)@@ -2780,6 +2816,16 @@ def test_lowers_unaliased_left_any_join(people_column: str, error: str | None) -"SELECT p.tiers != '[]' FROM prices LEFT ANY JOIN prices AS p ON prices.id = p.id",'("p"."tiers" != %(hogql_val_0)s)',),+ (+ "`id` String, `tiers` Nullable(String)",+ "SELECT length(t) FROM (SELECT tiers AS t FROM prices UNION ALL SELECT tiers AS t FROM prices)",+ 'SELECT length("t")',+ ),+ (+ "`id` String, `tiers` Nullable(String)",+ "WITH p AS (SELECT tiers FROM prices UNION ALL SELECT tiers FROM prices) SELECT length(tiers) FROM p",+ 'SELECT length("p"."tiers")',+ ),],)def test_curated_array_field_over_a_text_column_compares_as_text(structure: str, query: str, expected: str) -> None:@@ -2857,6 +2903,26 @@ def test_lowers_left_any_join_against_a_cte() -> None:assert '"right_rows" AS "__hogql_any_source_0"' in sql+def test_left_any_join_against_an_unaliased_cte_keeps_one_match() -> None:+ context = _context_with_trino_table()+ sql, _ = prepare_and_print_ast(+ parse_select(+ "WITH r AS (SELECT 1 AS k, 10 AS v UNION ALL SELECT 1, 20 UNION ALL SELECT 2, 30), "+ "l AS (SELECT 1 AS k UNION ALL SELECT 3) "+ "SELECT l.k, r.v FROM l LEFT ANY JOIN r ON l.k = r.k ORDER BY l.k"+ ),+ context,+ "trino",+ )++ sql, values = convert_pyformat_placeholders(sql, context.values)+ with duckdb.connect(":memory:", config={"threads": 1}) as connection:+ rows = connection.execute(sql, values).fetchall()+ assert len(rows) == 2+ assert rows[0][0] == 1 and rows[0][1] in (10, 20)+ assert rows[1] == (3, None)++@pytest.mark.parametrize(("join", "constraint", "feature_code"),[diff --git a/posthog/hogql/printer/trino.py b/posthog/hogql/printer/trino.pyindex 208deb71e08f..cbde612a7282 100644--- a/posthog/hogql/printer/trino.py+++ b/posthog/hogql/printer/trino.py@@ -38,7 +38,7 @@ from posthog.hogql.printer.types import JoinExprResponsefrom posthog.hogql.transforms.trino.errors import TrinoLoweringErrorfrom posthog.hogql.transforms.trino.expressions import constant_integerfrom posthog.hogql.transforms.trino.limits import MAX_TRINO_SQL_LENGTH-from posthog.hogql.visitor import clone_expr+from posthog.hogql.visitor import TraversingVisitor, clone_exprfrom posthog.dataclasses import frozenfrom posthog.exchange_rate_constants import EXCHANGE_RATE_DECIMAL_PRECISION@@ -264,6 +264,21 @@ class _BinaryArguments:right: str+class _JoinByTypeFinder(TraversingVisitor):+ def __init__(self, table_type: ast.ColumnAliasedTableType) -> None:+ super().__init__()+ self.table_type = table_type+ self.found: ast.JoinExpr | None = None++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ if self.found is not None:+ return+ if node.type is self.table_type:+ self.found = node+ return+ super().visit_join_expr(node)++class TrinoPrinter(PostgresPrinter):DIALECT_NAME: ClassVar[HogQLDialect] = "trino"DIALECT_LABEL: ClassVar[str] = "Trino"@@ -762,7 +777,7 @@ class TrinoPrinter(PostgresPrinter):return self._visit_lambda_array_call(node, "any_match" if name == "arrayexists" else "all_match")if name == "intdiv":binary_args = self._visit_binary_args(node)- if not all(isinstance(self._resolve_type(arg), ast.IntegerType) for arg in node.args):+ if not all(self._is_integer_operand(arg) for arg in node.args):self._unsupported("TRINO_INT_DIV_TYPE_UNSUPPORTED", "intDiv requires integer operands in Trino mode.", node)@@ -1522,7 +1537,13 @@ class TrinoPrinter(PostgresPrinter):self._is_dynamic_property(arg) for arg in node.args):return ast.BooleanType()+ if isinstance(node, ast.ArrayAccess) and isinstance(node.type, ast.UnknownType):+ # The resolver types an element read off a CTE column as unknown, though the printer can see the array.+ array_type = self._resolve_type(node.array)+ if isinstance(array_type, ast.ArrayType) and not isinstance(array_type.item_type, ast.UnknownType):+ return array_type.item_typevalue_type = node.type+ merged_type: ast.ConstantType | None = Noneseen: set[int] = set()while value_type is not None and id(value_type) not in seen:seen.add(id(value_type))@@ -1533,10 +1554,17 @@ class TrinoPrinter(PostgresPrinter):table_type = value_type.table_typeif isinstance(table_type, ast.CTETableAliasType):table_type = table_type.cte_table_type- if isinstance(table_type, ast.CTETableType) and isinstance(- table_type.select_query_type, ast.SelectQueryType- ):- column_type = table_type.select_query_type.columns.get(value_type.name)+ select_type: ast.Type | None = None+ if isinstance(table_type, (ast.CTETableType, ast.SelectQueryAliasType)):+ select_type = table_type.select_query_type+ elif isinstance(table_type, (ast.SelectQueryType, ast.SelectSetQueryType)):+ select_type = table_type+ while isinstance(select_type, ast.SelectSetQueryType) and select_type.types:+ if merged_type is None and value_type.name in select_type.columns:+ merged_type = select_type.columns[value_type.name].resolve_constant_type(self.context)+ select_type = select_type.types[0]+ if isinstance(select_type, ast.SelectQueryType):+ column_type = select_type.columns.get(value_type.name)if column_type is not None:value_type = column_typecontinue@@ -1552,6 +1580,20 @@ class TrinoPrinter(PostgresPrinter):if isinstance(value_type, ast.FieldType) and self._is_text_backed_warehouse_column(value_type):return ast.StringType(nullable=True)return ast.ArrayType(item_type=ast.StringType(), nullable=resolved.nullable)+ # A set query merges branch column types, which turns a curated string array into a plain array.+ if (+ isinstance(resolved, ast.ArrayType)+ and isinstance(value_type, ast.FieldType)+ and self._is_text_backed_warehouse_column(value_type)+ ):+ return ast.StringType(nullable=True)+ # Branches of different types are promoted, e.g. Date with DateTime, so the first branch is not the column type.+ if (+ merged_type is not None+ and not isinstance(merged_type, ast.UnknownType)+ and type(merged_type) is not type(resolved)+ ):+ return merged_typereturn resolveddef _is_text_backed_warehouse_column(self, field_type: ast.FieldType) -> bool:@@ -1582,13 +1624,15 @@ class TrinoPrinter(PostgresPrinter):table_type = current.table_typeif isinstance(table_type, ast.CTETableAliasType):table_type = table_type.cte_table_type+ select_type: ast.Type | None = table_typeif isinstance(table_type, (ast.CTETableType, ast.SelectQueryAliasType)):select_type = table_type.select_query_type- current = (- select_type.columns.get(current.name) if isinstance(select_type, ast.SelectQueryType) else None- )- continue- return current+ elif not isinstance(table_type, (ast.SelectQueryType, ast.SelectSetQueryType)):+ return current+ # A set query's columns take their names from the first branch.+ while isinstance(select_type, ast.SelectSetQueryType) and select_type.types:+ select_type = select_type.types[0]+ current = select_type.columns.get(current.name) if isinstance(select_type, ast.SelectQueryType) else Nonereturn Nonedef _unnest_column_type(self, table_type: ast.ColumnAliasedTableType, column: str) -> ast.ConstantType | None:@@ -1597,25 +1641,52 @@ class TrinoPrinter(PostgresPrinter):table_type.table_type.table, TrinoUnnestTable):return None+ join = self._unnest_join(table_type)+ if join is None or not join.table_args or not join.column_aliases or column not in join.column_aliases:+ return None+ rows_expr = join.table_args[0]+ rows = self._resolve_type(rows_expr)+ index = join.column_aliases.index(column)+ if (+ isinstance(rows, ast.ArrayType)+ and isinstance(rows.item_type, ast.TupleType)+ and index < len(rows.item_type.item_types)+ ):+ item_type = rows.item_type.item_types[index]+ if not isinstance(item_type, ast.UnknownType):+ return item_type+ # `arrayJoin(arr)` lowers to `arrayMap(v -> tuple(v), arr)`, typed before a CTE source column was known.+ if (+ index == 0+ and isinstance(rows_expr, ast.Call)+ and rows_expr.name == "arrayMap"+ and len(rows_expr.args) == 2+ and isinstance(rows_expr.args[0], ast.Lambda)+ and isinstance(rows_expr.args[0].expr, ast.Tuple)+ and len(rows_expr.args[0].expr.exprs) == 1+ and isinstance(rows_expr.args[0].expr.exprs[0], ast.Field)+ and rows_expr.args[0].expr.exprs[0].chain == list(rows_expr.args[0].args)+ ):+ source = self._resolve_type(rows_expr.args[1])+ if isinstance(source, ast.ArrayType) and not isinstance(source.item_type, ast.UnknownType):+ return source.item_type+ return None++ def _unnest_join(self, table_type: ast.ColumnAliasedTableType) -> ast.JoinExpr | None:for scope in reversed(self.stack):if not isinstance(scope, ast.SelectQuery):continuejoin = scope.select_fromwhile join is not None:if join.alias == table_type.alias and join.table_args and join.column_aliases:- if column not in join.column_aliases:- return None- rows = self._resolve_type(join.table_args[0])- index = join.column_aliases.index(column)- if (- isinstance(rows, ast.ArrayType)- and isinstance(rows.item_type, ast.TupleType)- and index < len(rows.item_type.item_types)- ):- return rows.item_type.item_types[index]- return None+ return joinjoin = join.next_join- return None+ # A column read through a CTE or subquery points at an UNNEST outside the current scope chain.+ if not self.stack:+ return None+ finder = _JoinByTypeFinder(table_type)+ finder.visit(self.stack[0])+ return finder.founddef _subquery_scalar_type(self, node: ast.SelectQuery | ast.SelectSetQuery) -> ast.ConstantType | None:query = node@@ -1648,6 +1719,22 @@ class TrinoPrinter(PostgresPrinter):node = node.exprreturn isinstance(node, ast.PropertyAccess)+ def _is_integer_operand(self, node: ast.Expr) -> bool:+ while isinstance(node, ast.Alias):+ node = node.expr+ if isinstance(self._resolve_type(node), ast.IntegerType):+ return True+ # The resolver loses array columns of derived tables, but these always return an integer in ClickHouse.+ if isinstance(node, ast.Call) and node.name.lower() in {"length", "lengthutf8", "empty", "notempty"}:+ return True+ if isinstance(node, ast.ArithmeticOperation) and node.op in {+ ast.ArithmeticOperationOp.Add,+ ast.ArithmeticOperationOp.Sub,+ ast.ArithmeticOperationOp.Mult,+ }:+ return self._is_integer_operand(node.left) and self._is_integer_operand(node.right)+ return False+def _is_numeric(self, node: ast.Expr) -> bool:return isinstance(self._resolve_type(node), (ast.IntegerType, ast.FloatType, ast.DecimalType))@@ -2411,9 +2498,45 @@ class TrinoPrinter(PostgresPrinter):f"IF(cardinality({left}) = cardinality({right}), {mapped}, ""fail('arrayMap requires equal-length arrays'))")- if len(node.args) != 2 or not isinstance(node.args[0], ast.Lambda):+ predicate = target in {"filter", "any_match", "all_match"}+ lambda_node = node.args[0] if node.args else None+ if predicate and len(node.args) == 3 and isinstance(lambda_node, ast.Lambda) and len(lambda_node.args) == 2:+ return self._visit_zipped_predicate_call(node, lambda_node, target)+ if len(node.args) != 2 or not isinstance(lambda_node, ast.Lambda):self._invalid_function_arguments(node, f"{node.name} expects a lambda and array in Trino mode.")- return f"{target}({self.visit(node.args[1])}, {self.visit(node.args[0])})"+ rendered_lambda = self._visit_predicate_lambda(lambda_node) if predicate else self.visit(lambda_node)+ return f"{target}({self.visit(node.args[1])}, {rendered_lambda})"++ def _visit_predicate_lambda(self, node: ast.Lambda) -> str:+ # ClickHouse treats any non-zero lambda result as true, while Trino's filter needs a boolean.+ identifiers = [self._print_identifier(arg) for arg in node.args]+ arguments = identifiers[0] if len(identifiers) == 1 else f"({', '.join(identifiers)})"+ return f"{arguments} -> {self._visit_predicate(self._lambda_body(node))}"++ def _visit_zipped_predicate_call(self, node: ast.Call, lambda_node: ast.Lambda, target: str) -> str:+ left = self.visit(node.args[1])+ right = self.visit(node.args[2])+ if target == "filter":+ # Keep each left item with its verdict, because the lambda sees the item pair but returns the left item.+ first = self._print_identifier(lambda_node.args[0])+ rows = f"zip_with({left}, {right}, {self._visit_zip_row_lambda(lambda_node, first)})"+ result = f"transform(filter({rows}, __hogql_row -> __hogql_row[2]), __hogql_row -> __hogql_row[1])"+ else:+ verdicts = f"zip_with({left}, {right}, {self._visit_predicate_lambda(lambda_node)})"+ result = f"{target}({verdicts}, __hogql_verdict -> __hogql_verdict)"+ return (+ f"IF(cardinality({left}) = cardinality({right}), {result}, "+ f"fail('{node.name} requires equal-length arrays'))"+ )++ def _visit_zip_row_lambda(self, node: ast.Lambda, first: str) -> str:+ identifiers = [self._print_identifier(arg) for arg in node.args]+ return f"({', '.join(identifiers)}) -> ROW({first}, {self._visit_predicate(self._lambda_body(node))})"++ def _lambda_body(self, node: ast.Lambda) -> ast.Expr:+ if isinstance(node.expr, ast.Block):+ self._unsupported("TRINO_LAMBDA_BLOCK_UNSUPPORTED", "Lambda blocks are not supported in Trino mode.", node)+ return node.exprdef _visit_array_sort_key(self, node: ast.Call) -> str:key = node.args[0]diff --git a/posthog/hogql/transforms/trino/any_join.py b/posthog/hogql/transforms/trino/any_join.pyindex 6ece2fff136f..b95ac808cf5b 100644--- a/posthog/hogql/transforms/trino/any_join.py+++ b/posthog/hogql/transforms/trino/any_join.py@@ -353,6 +353,9 @@ class TrinoAnyJoinLowerer(CloningVisitor):return field_type.nameif isinstance(table_type, ast.SelectQueryAliasType) and table_type.alias == alias:return field_type.name+ # A CTE joined without `AS` has no alias type, so its fields point at the CTE itself.+ if isinstance(table_type, ast.CTETableType) and table_type is right_table_type:+ return field_type.nameif (isinstance(table_type, ast.TableType)and isinstance(right_table_type, ast.TableType)