:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): lower trino toInt on float math, range steps and cte alias shadows (#125056)
Jovan Sakovica422e92a61db
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.pyindex 06ec7b7c94a3..a2f3996aa683 100644--- a/posthog/hogql/printer/test/test_trino_printer.py+++ b/posthog/hogql/printer/test/test_trino_printer.py@@ -640,6 +640,7 @@ def test_tracks_every_registered_function_without_a_trino_mapping(snapshot: Snap("cityHash64(user_id)", "TRINO_FUNCTION_UNSUPPORTED"),("toTypeName(user_id)", "TRINO_FUNCTION_UNSUPPORTED"),("bitNot(id)", "TRINO_FUNCTION_UNSUPPORTED"),+ ("range(1, 10, id)", "TRINO_RANGE_STEP_UNSUPPORTED"),],)def test_rejects_unsupported_printer_function_variants(expression: str, feature_code: str) -> None:@@ -729,6 +730,63 @@ def test_regex_character_classes_keep_clickhouse_meaning(pattern: str, expected:'from_unixtime((CAST(to_unixtime("users"."created_at") AS BIGINT) + 1))','CAST("created_at" AS BIGINT)',),+ (+ "WITH t AS (SELECT created_at AS period_start FROM users) "+ "SELECT toUnixTimestamp(period_start) + 1 AS period_start, fromUnixTimestamp(period_start) AS later FROM t",+ 'from_unixtime((CAST(to_unixtime("t"."period_start") AS BIGINT) + 1))',+ 'CAST("period_start" AS BIGINT)',+ ),+ # A NaN or fractional float from a rounding function becomes NULL, not a failed Trino cast.+ (+ "SELECT toInt(ceil(divide(a - n, n - a) * n + a)) FROM (SELECT sum(id) AS a, count() AS n FROM users)",+ "__hogql_accurate_cast_value",+ "CAST(ceil(",+ ),+ # Trino reads ORDER BY names as output columns first, so the qualifier `t` would read the `0 AS t` column.+ (+ "WITH t AS (SELECT 1 AS x) SELECT x AS x, 0 AS t FROM t ORDER BY x + 0",+ 'ORDER BY ("x" + 0)',+ 'ORDER BY ("t"."x"',+ ),+ # Float arithmetic over non-null operands stays non-null, so sumOrDefault keeps its 0 default.+ ("SELECT sumOrDefault(1.5 + 1) FROM users WHERE false", "DOUBLE '0'", "NULL)"),+ # nullIf can always return NULL, so sumOrDefault must not invent a 0.+ ("SELECT sumOrDefault(nullIf(true, 1)) FROM users", "NULL)", ", 0)"),+ # A window's OVER (ORDER BY ...) can't see output aliases, so the alias still expands there.+ (+ "SELECT toStartOfMonth(users.created_at) AS month, count() AS users, sum(count()) OVER (ORDER BY month) AS running FROM users GROUP BY month ORDER BY month",+ 'OVER (ORDER BY CAST(date_trunc(\'month\', "users"."created_at") AS DATE) ASC)',+ 'OVER (ORDER BY "month"',+ ),+ (+ "SELECT u.created_at AS user_id, u.user_id AS u, row_number() OVER (ORDER BY user_id) AS rn FROM users AS u ORDER BY rn",+ 'OVER (ORDER BY "u"."created_at" ASC)',+ 'OVER (ORDER BY "user_id"',+ ),+ ("SELECT arrayCumSum(x -> x * 1.0, [1, 2])", "reduce(transform(", "CASE WHEN"),+ # A boolean coalesced with a float stays a float, so toInt maps the fractional result to NULL.+ (+ "SELECT toInt(coalesce(nullIf(true, true), 1.5))",+ "__hogql_accurate_cast_value",+ "CAST(coalesce(CAST(nullif(true, true) AS INTEGER), 1.5) AS BIGINT)",+ ),+ # toInt of a NULL boolean stays NULL instead of becoming 0.+ (+ "SELECT toInt(multiSearchAnyCaseInsensitive(nullIf(user_id, user_id), ['a'])) FROM users",+ "CAST(any_match(",+ "THEN 1 ELSE 0 END",+ ),+ # Adding a float to a datetime keeps it a datetime, so toInt still reads the Unix timestamp.+ (+ "SELECT toInt(created_at + 1.0) FROM users",+ "CAST(to_unixtime(date_add(",+ "__hogql_accurate_cast_value",+ ),+ (+ "SELECT range(1, length(user_id) + 1, 3) FROM users",+ "greatest(__hogql_range[2] - 1, __hogql_range[1]), 3)",+ "greatest(__hogql_range[2] - 1, __hogql_range[1]))",+ ),("SELECT toFloat(tupleElement(kv, 2)) FROM users ARRAY JOIN JSONExtractKeysAndValuesRaw(user_id) AS kv",'TRY_CAST(("__trino_unnest_0"."kv")[2] AS DOUBLE)',@@ -750,6 +808,18 @@ def test_lowers_clickhouse_semantics_trino_rejects(query: str, required: str, foassert forbidden not in sql.replace(required, "")+def test_deeply_nested_arithmetic_prints_without_repeated_type_lookups() -> None:+ expression = "id"+ for _ in range(22):+ expression = f"coalesce({expression} + 1, 0)"++ sql, _ = prepare_and_print_ast(+ parse_select(f"SELECT {expression} FROM users"), _context_with_trino_table(), "trino"+ )++ assert sql.count("coalesce(") == 22++def test_first_regex_replacement_preserves_inline_flags() -> None:context = _context_with_trino_table()prepare_and_print_ast(parse_select("SELECT replaceRegexpOne(user_id, '(?i)a', 'b') FROM users"), context, "trino")diff --git a/posthog/hogql/printer/trino.py b/posthog/hogql/printer/trino.pyindex cbde612a7282..6f0d9e9a1881 100644--- a/posthog/hogql/printer/trino.py+++ b/posthog/hogql/printer/trino.py@@ -992,13 +992,14 @@ class TrinoPrinter(PostgresPrinter):return super().visit_call(lowered)if name in {"toint", "tointorzero", "tointordefault", "_touint64"} and node.args:arg = node.args[0]- arg_type = arg.type.resolve_constant_type(self.context) if arg.type is not None else None+ arg_type = self._resolve_type(arg)if isinstance(arg_type, ast.DateType):return f"date_diff('day', DATE '1970-01-01', {self.visit(arg)})"if isinstance(arg_type, ast.DateTimeType):return f"CAST(to_unixtime({self.visit(arg)}) AS BIGINT)"if isinstance(arg_type, ast.BooleanType):- return f"CASE WHEN {self.visit(arg)} THEN 1 ELSE 0 END"+ # A CASE would turn NULL into 0; the cast keeps it NULL like ClickHouse.+ return f"CAST({self.visit(arg)} AS BIGINT)"if name == "toint" and isinstance(arg_type, ast.StringType):return f"TRY_CAST({self.visit(arg)} AS BIGINT)"# HogQL toInt is accurateCastOrNull, which maps a fractional, NaN or out-of-range float to NULL@@ -1522,17 +1523,64 @@ class TrinoPrinter(PostgresPrinter):raise ImpossibleASTError(f"Unknown ArithmeticOperationOp {node.op}")def _resolve_type(self, node: ast.Expr) -> ast.ConstantType | None:+ # Lowerings ask for the same operand types at every nesting level, so uncached lookups grow exponentially.+ cache: dict[int, tuple[ast.Expr, ast.ConstantType | None]] = self.__dict__.setdefault("_resolved_types", {})+ cached = cache.get(id(node))+ if cached is not None and cached[0] is node:+ return cached[1]+ resolved = self._resolve_type_uncached(node)+ cache[id(node)] = (node, resolved)+ return resolved++ def _resolve_type_uncached(self, node: ast.Expr) -> ast.ConstantType | None:if isinstance(node, ast.Call) and node.name.lower() in TRINO_TUPLE_OPERATORS:if node.name.lower() == "tuplehammingdistance":return ast.IntegerType()return self._tuple_arithmetic_type(node)if isinstance(node, ast.Call) and node.name.lower() in {"jsonhas", "multisearchanycaseinsensitive"}:return ast.BooleanType()+ # Division always prints as a DOUBLE division, though the resolver can type divide() of integers as an integer.+ if (isinstance(node, ast.ArithmeticOperation) and node.op == ast.ArithmeticOperationOp.Div) or (+ isinstance(node, ast.Call) and node.name.lower() == "divide"+ ):+ division_type = node.type.resolve_constant_type(self.context) if node.type is not None else None+ return ast.FloatType(nullable=division_type.nullable if division_type is not None else True)+ if (+ isinstance(node, ast.ArithmeticOperation)+ and node.op+ in (ast.ArithmeticOperationOp.Add, ast.ArithmeticOperationOp.Sub, ast.ArithmeticOperationOp.Mult)+ and any(isinstance(self._resolve_type(side), ast.FloatType) for side in (node.left, node.right))+ # Date arithmetic prints as date_add, which keeps the date type.+ and not any(+ isinstance(self._resolve_type(side), (ast.DateType, ast.DateTimeType))+ for side in (node.left, node.right)+ )+ ):+ side_types = [self._resolve_type(side) for side in (node.left, node.right)]+ return ast.FloatType(nullable=any(side_type is None or side_type.nullable for side_type in side_types))+ # ClickHouse rounding functions return their argument's type, but the function registry declares no signature.+ if isinstance(node, ast.Call) and node.name.lower() in {"floor", "ceil", "trunc", "round"} and node.args:+ argument_type = self._resolve_type(node.args[0])+ if isinstance(argument_type, (ast.IntegerType, ast.FloatType, ast.DecimalType)):+ return argument_typeif isinstance(node, ast.Call) and node.name.lower() in {"coalesce", "ifnull", "nullif"}:if any(isinstance(self._resolve_type(arg), ast.BooleanType) for arg in node.args) and any(self._is_numeric(arg) for arg in node.args):- return ast.IntegerType()+ # Booleans print as integers here, so the other numeric arguments decide the common type.+ numeric_types = [self._resolve_type(arg) for arg in node.args]+ if node.name.lower() == "nullif":+ # nullIf returns its first argument or NULL, so it keeps that type and is always nullable.+ first_type = numeric_types[0]+ if isinstance(first_type, (ast.FloatType, ast.DecimalType, ast.IntegerType)):+ return replace(first_type, nullable=True)+ return ast.IntegerType(nullable=True)+ # coalesce is NULL only when every argument is.+ nullable = all(numeric_type is None or numeric_type.nullable for numeric_type in numeric_types)+ for numeric_type in numeric_types:+ if isinstance(numeric_type, (ast.FloatType, ast.DecimalType)):+ return replace(numeric_type, nullable=nullable)+ return ast.IntegerType(nullable=nullable)if any(isinstance(self._resolve_type(arg), ast.BooleanType) for arg in node.args) and any(self._is_dynamic_property(arg) for arg in node.args):@@ -2744,17 +2792,33 @@ class TrinoPrinter(PostgresPrinter):if len(node.args) == 1:start = "0"end = self.visit(node.args[0])- elif len(node.args) == 2:+ elif len(node.args) in {2, 3}:start = self.visit(node.args[0])end = self.visit(node.args[1])else:- self._invalid_function_arguments(node, "range expects one or two arguments in Trino mode.")+ self._invalid_function_arguments(node, "range expects one to three arguments in Trino mode.")+ step = ""+ if len(node.args) == 3:+ step_arg = node.args[2]+ # A constant positive step keeps the Trino sequence ascending, as the exclusive end filter assumes.+ if (+ not isinstance(step_arg, ast.Constant)+ or isinstance(step_arg.value, bool)+ or not isinstance(step_arg.value, int)+ or step_arg.value <= 0+ ):+ self._unsupported(+ "TRINO_RANGE_STEP_UNSUPPORTED",+ "range requires a constant positive integer step in Trino mode.",+ node,+ )+ step = f", {step_arg.value}"value = self._print_identifier("__hogql_range_value")if all(isinstance(arg, ast.Constant) for arg in node.args):- return f"filter(sequence({start}, greatest(({end}) - 1, {start})), {value} -> ({value} < {end}))"+ return f"filter(sequence({start}, greatest(({end}) - 1, {start}){step}), {value} -> ({value} < {end}))"return (f"element_at(transform(ARRAY[ROW({start}, {end})], __hogql_range -> "- "filter(sequence(__hogql_range[1], greatest(__hogql_range[2] - 1, __hogql_range[1])), "+ f"filter(sequence(__hogql_range[1], greatest(__hogql_range[2] - 1, __hogql_range[1]){step}), "f"{value} -> ({value} < __hogql_range[2]))), 1)")diff --git a/posthog/hogql/transforms/trino/normalize.py b/posthog/hogql/transforms/trino/normalize.pyindex fa3e72b38d89..96273cb37eb1 100644--- a/posthog/hogql/transforms/trino/normalize.py+++ b/posthog/hogql/transforms/trino/normalize.py@@ -259,6 +259,19 @@ class TrinoScalarCTELowerer(CloningVisitor):return super().visit_field(node)+class _QualifiedFieldFinder(TraversingVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.qualifiers: set[str] = set()++ def visit_field(self, node: ast.Field) -> None:+ if len(node.chain) > 1 and isinstance(node.chain[0], str):+ self.qualifiers.add(node.chain[0])++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ return None++class TrinoSelectAliasLowerer(CloningVisitor):def __init__(self, context: HogQLContext, budget: TrinoCompilationBudget) -> None:super().__init__(clear_types=False)@@ -267,6 +280,8 @@ class TrinoSelectAliasLowerer(CloningVisitor):self.aliases: dict[str, ast.Expr] = {}self.alias_positions: dict[str, int] = {}self.expanding: set[str] = set()+ self.in_order_by = False+ self.in_window = Falseself.expansion_budget = budgetdef visit(self, node: _T_AST) -> _T_AST:@@ -276,6 +291,10 @@ class TrinoSelectAliasLowerer(CloningVisitor):def visit_select_query(self, node: ast.SelectQuery) -> ast.SelectQuery:outer_aliases = self.aliasesouter_alias_positions = self.alias_positions+ outer_in_order_by = self.in_order_by+ outer_in_window = self.in_window+ self.in_order_by = False+ self.in_window = Falseself.aliases = {expr.alias: expr.expr for expr in node.select if isinstance(expr, ast.Alias)}self.alias_positions = {expr.alias: index for index, expr in enumerate(node.select, start=1) if isinstance(expr, ast.Alias)@@ -321,8 +340,27 @@ class TrinoSelectAliasLowerer(CloningVisitor):order.expr = ast.PositionalRef(index=self.alias_positions[order_expression.chain[-1]])self.aliases = outer_aliasesself.alias_positions = outer_alias_positions+ self.in_order_by = outer_in_order_by+ self.in_window = outer_in_windowreturn lowered+ def visit_order_expr(self, node: ast.OrderExpr) -> ast.OrderExpr:+ outer_in_order_by = self.in_order_by+ # Only a query's own ORDER BY sees output aliases; a window's OVER (ORDER BY ...) does not.+ self.in_order_by = not self.in_window+ try:+ return super().visit_order_expr(node)+ finally:+ self.in_order_by = outer_in_order_by++ def visit_window_expr(self, node: ast.WindowExpr) -> ast.WindowExpr:+ outer_in_order_by, outer_in_window = self.in_order_by, self.in_window+ self.in_order_by, self.in_window = False, True+ try:+ return super().visit_window_expr(node)+ finally:+ self.in_order_by, self.in_window = outer_in_order_by, outer_in_window+def visit_alias(self, node: ast.Alias) -> ast.Alias:if node.hidden:self.expansion_budget.visit(node)@@ -379,14 +417,24 @@ class TrinoSelectAliasLowerer(CloningVisitor):alias = node.chain[0]self.expanding.add(alias)try:- return self.visit(self.aliases[alias])+ expanded = self.visit(self.aliases[alias])finally:self.expanding.remove(alias)+ # Trino resolves ORDER BY names against output columns first, so a qualifier that is also a projection+ # alias would read that output column. The output column itself already holds this value.+ if self.in_order_by and self._has_qualifier_named_like_an_alias(expanded):+ return clone_expr(node, clear_types=False)+ return expandedqualified = self._qualified_shadowed_column(node)if qualified is not None:return qualifiedreturn super().visit_field(node)+ def _has_qualifier_named_like_an_alias(self, node: ast.Expr) -> bool:+ finder = _QualifiedFieldFinder()+ finder.visit(node)+ return any(qualifier in self.aliases for qualifier in finder.qualifiers)+def _qualified_shadowed_column(self, node: ast.Expr) -> ast.Field | None:# After alias inlining, a later resolver pass would bind a bare column named like a projection alias to the# alias. Qualifying it keeps the column the first pass chose.@@ -396,10 +444,15 @@ class TrinoSelectAliasLowerer(CloningVisitor):and isinstance(node.chain[0], str)and node.chain[0] in self.aliasesand isinstance(node.type, ast.FieldType)- and isinstance(node.type.table_type, ast.TableAliasType)):return None- qualifier = node.type.table_type.alias+ table_type = node.type.table_type+ if isinstance(table_type, (ast.TableAliasType, ast.CTETableAliasType, ast.SelectQueryAliasType)):+ qualifier = table_type.alias+ elif isinstance(table_type, ast.CTETableType):+ qualifier = table_type.name+ else:+ return Noneif self.lambda_scopes and self._lambda_reads_qualifier_as_field(qualifier):raise TrinoLoweringError("TRINO_LAMBDA_QUALIFIER_AMBIGUOUS",