Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): lower trino toInt on float math, range steps and cte alias shadows (#125056)

Jovan Sakovica422e92a61db
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.py
index 06ec7b7c94a3..a2f3996aa683 100644
--- a/posthog/hogql/printer/test/test_trino_printer.py
+++ b/posthog/hogql/printer/test/test_trino_printer.py
@@ -640,6 +640,7 @@ def test_tracks_every_registered_function_without_a_trino_mapping(snapshot: Snap
("cityHash64(user_id)", "TRINO_FUNCTION_UNSUPPORTED"),
("toTypeName(user_id)", "TRINO_FUNCTION_UNSUPPORTED"),
("bitNot(id)", "TRINO_FUNCTION_UNSUPPORTED"),
+ ("range(1, 10, id)", "TRINO_RANGE_STEP_UNSUPPORTED"),
],
)
def test_rejects_unsupported_printer_function_variants(expression: str, feature_code: str) -> None:
@@ -729,6 +730,63 @@ def test_regex_character_classes_keep_clickhouse_meaning(pattern: str, expected:
'from_unixtime((CAST(to_unixtime("users"."created_at") AS BIGINT) + 1))',
'CAST("created_at" AS BIGINT)',
),
+ (
+ "WITH t AS (SELECT created_at AS period_start FROM users) "
+ "SELECT toUnixTimestamp(period_start) + 1 AS period_start, fromUnixTimestamp(period_start) AS later FROM t",
+ 'from_unixtime((CAST(to_unixtime("t"."period_start") AS BIGINT) + 1))',
+ 'CAST("period_start" AS BIGINT)',
+ ),
+ # A NaN or fractional float from a rounding function becomes NULL, not a failed Trino cast.
+ (
+ "SELECT toInt(ceil(divide(a - n, n - a) * n + a)) FROM (SELECT sum(id) AS a, count() AS n FROM users)",
+ "__hogql_accurate_cast_value",
+ "CAST(ceil(",
+ ),
+ # Trino reads ORDER BY names as output columns first, so the qualifier `t` would read the `0 AS t` column.
+ (
+ "WITH t AS (SELECT 1 AS x) SELECT x AS x, 0 AS t FROM t ORDER BY x + 0",
+ 'ORDER BY ("x" + 0)',
+ 'ORDER BY ("t"."x"',
+ ),
+ # Float arithmetic over non-null operands stays non-null, so sumOrDefault keeps its 0 default.
+ ("SELECT sumOrDefault(1.5 + 1) FROM users WHERE false", "DOUBLE '0'", "NULL)"),
+ # nullIf can always return NULL, so sumOrDefault must not invent a 0.
+ ("SELECT sumOrDefault(nullIf(true, 1)) FROM users", "NULL)", ", 0)"),
+ # A window's OVER (ORDER BY ...) can't see output aliases, so the alias still expands there.
+ (
+ "SELECT toStartOfMonth(users.created_at) AS month, count() AS users, sum(count()) OVER (ORDER BY month) AS running FROM users GROUP BY month ORDER BY month",
+ 'OVER (ORDER BY CAST(date_trunc(\'month\', "users"."created_at") AS DATE) ASC)',
+ 'OVER (ORDER BY "month"',
+ ),
+ (
+ "SELECT u.created_at AS user_id, u.user_id AS u, row_number() OVER (ORDER BY user_id) AS rn FROM users AS u ORDER BY rn",
+ 'OVER (ORDER BY "u"."created_at" ASC)',
+ 'OVER (ORDER BY "user_id"',
+ ),
+ ("SELECT arrayCumSum(x -> x * 1.0, [1, 2])", "reduce(transform(", "CASE WHEN"),
+ # A boolean coalesced with a float stays a float, so toInt maps the fractional result to NULL.
+ (
+ "SELECT toInt(coalesce(nullIf(true, true), 1.5))",
+ "__hogql_accurate_cast_value",
+ "CAST(coalesce(CAST(nullif(true, true) AS INTEGER), 1.5) AS BIGINT)",
+ ),
+ # toInt of a NULL boolean stays NULL instead of becoming 0.
+ (
+ "SELECT toInt(multiSearchAnyCaseInsensitive(nullIf(user_id, user_id), ['a'])) FROM users",
+ "CAST(any_match(",
+ "THEN 1 ELSE 0 END",
+ ),
+ # Adding a float to a datetime keeps it a datetime, so toInt still reads the Unix timestamp.
+ (
+ "SELECT toInt(created_at + 1.0) FROM users",
+ "CAST(to_unixtime(date_add(",
+ "__hogql_accurate_cast_value",
+ ),
+ (
+ "SELECT range(1, length(user_id) + 1, 3) FROM users",
+ "greatest(__hogql_range[2] - 1, __hogql_range[1]), 3)",
+ "greatest(__hogql_range[2] - 1, __hogql_range[1]))",
+ ),
(
"SELECT toFloat(tupleElement(kv, 2)) FROM users ARRAY JOIN JSONExtractKeysAndValuesRaw(user_id) AS kv",
'TRY_CAST(("__trino_unnest_0"."kv")[2] AS DOUBLE)',
@@ -750,6 +808,18 @@ def test_lowers_clickhouse_semantics_trino_rejects(query: str, required: str, fo
assert forbidden not in sql.replace(required, "")
+def test_deeply_nested_arithmetic_prints_without_repeated_type_lookups() -> None:
+ expression = "id"
+ for _ in range(22):
+ expression = f"coalesce({expression} + 1, 0)"
+
+ sql, _ = prepare_and_print_ast(
+ parse_select(f"SELECT {expression} FROM users"), _context_with_trino_table(), "trino"
+ )
+
+ assert sql.count("coalesce(") == 22
+
+
def test_first_regex_replacement_preserves_inline_flags() -> None:
context = _context_with_trino_table()
prepare_and_print_ast(parse_select("SELECT replaceRegexpOne(user_id, '(?i)a', 'b') FROM users"), context, "trino")
diff --git a/posthog/hogql/printer/trino.py b/posthog/hogql/printer/trino.py
index cbde612a7282..6f0d9e9a1881 100644
--- a/posthog/hogql/printer/trino.py
+++ b/posthog/hogql/printer/trino.py
@@ -992,13 +992,14 @@ class TrinoPrinter(PostgresPrinter):
return super().visit_call(lowered)
if name in {"toint", "tointorzero", "tointordefault", "_touint64"} and node.args:
arg = node.args[0]
- arg_type = arg.type.resolve_constant_type(self.context) if arg.type is not None else None
+ arg_type = self._resolve_type(arg)
if isinstance(arg_type, ast.DateType):
return f"date_diff('day', DATE '1970-01-01', {self.visit(arg)})"
if isinstance(arg_type, ast.DateTimeType):
return f"CAST(to_unixtime({self.visit(arg)}) AS BIGINT)"
if isinstance(arg_type, ast.BooleanType):
- return f"CASE WHEN {self.visit(arg)} THEN 1 ELSE 0 END"
+ # A CASE would turn NULL into 0; the cast keeps it NULL like ClickHouse.
+ return f"CAST({self.visit(arg)} AS BIGINT)"
if name == "toint" and isinstance(arg_type, ast.StringType):
return f"TRY_CAST({self.visit(arg)} AS BIGINT)"
# HogQL toInt is accurateCastOrNull, which maps a fractional, NaN or out-of-range float to NULL
@@ -1522,17 +1523,64 @@ class TrinoPrinter(PostgresPrinter):
raise ImpossibleASTError(f"Unknown ArithmeticOperationOp {node.op}")
def _resolve_type(self, node: ast.Expr) -> ast.ConstantType | None:
+ # Lowerings ask for the same operand types at every nesting level, so uncached lookups grow exponentially.
+ cache: dict[int, tuple[ast.Expr, ast.ConstantType | None]] = self.__dict__.setdefault("_resolved_types", {})
+ cached = cache.get(id(node))
+ if cached is not None and cached[0] is node:
+ return cached[1]
+ resolved = self._resolve_type_uncached(node)
+ cache[id(node)] = (node, resolved)
+ return resolved
+
+ def _resolve_type_uncached(self, node: ast.Expr) -> ast.ConstantType | None:
if isinstance(node, ast.Call) and node.name.lower() in TRINO_TUPLE_OPERATORS:
if node.name.lower() == "tuplehammingdistance":
return ast.IntegerType()
return self._tuple_arithmetic_type(node)
if isinstance(node, ast.Call) and node.name.lower() in {"jsonhas", "multisearchanycaseinsensitive"}:
return ast.BooleanType()
+ # Division always prints as a DOUBLE division, though the resolver can type divide() of integers as an integer.
+ if (isinstance(node, ast.ArithmeticOperation) and node.op == ast.ArithmeticOperationOp.Div) or (
+ isinstance(node, ast.Call) and node.name.lower() == "divide"
+ ):
+ division_type = node.type.resolve_constant_type(self.context) if node.type is not None else None
+ return ast.FloatType(nullable=division_type.nullable if division_type is not None else True)
+ if (
+ isinstance(node, ast.ArithmeticOperation)
+ and node.op
+ in (ast.ArithmeticOperationOp.Add, ast.ArithmeticOperationOp.Sub, ast.ArithmeticOperationOp.Mult)
+ and any(isinstance(self._resolve_type(side), ast.FloatType) for side in (node.left, node.right))
+ # Date arithmetic prints as date_add, which keeps the date type.
+ and not any(
+ isinstance(self._resolve_type(side), (ast.DateType, ast.DateTimeType))
+ for side in (node.left, node.right)
+ )
+ ):
+ side_types = [self._resolve_type(side) for side in (node.left, node.right)]
+ return ast.FloatType(nullable=any(side_type is None or side_type.nullable for side_type in side_types))
+ # ClickHouse rounding functions return their argument's type, but the function registry declares no signature.
+ if isinstance(node, ast.Call) and node.name.lower() in {"floor", "ceil", "trunc", "round"} and node.args:
+ argument_type = self._resolve_type(node.args[0])
+ if isinstance(argument_type, (ast.IntegerType, ast.FloatType, ast.DecimalType)):
+ return argument_type
if isinstance(node, ast.Call) and node.name.lower() in {"coalesce", "ifnull", "nullif"}:
if any(isinstance(self._resolve_type(arg), ast.BooleanType) for arg in node.args) and any(
self._is_numeric(arg) for arg in node.args
):
- return ast.IntegerType()
+ # Booleans print as integers here, so the other numeric arguments decide the common type.
+ numeric_types = [self._resolve_type(arg) for arg in node.args]
+ if node.name.lower() == "nullif":
+ # nullIf returns its first argument or NULL, so it keeps that type and is always nullable.
+ first_type = numeric_types[0]
+ if isinstance(first_type, (ast.FloatType, ast.DecimalType, ast.IntegerType)):
+ return replace(first_type, nullable=True)
+ return ast.IntegerType(nullable=True)
+ # coalesce is NULL only when every argument is.
+ nullable = all(numeric_type is None or numeric_type.nullable for numeric_type in numeric_types)
+ for numeric_type in numeric_types:
+ if isinstance(numeric_type, (ast.FloatType, ast.DecimalType)):
+ return replace(numeric_type, nullable=nullable)
+ return ast.IntegerType(nullable=nullable)
if any(isinstance(self._resolve_type(arg), ast.BooleanType) for arg in node.args) and any(
self._is_dynamic_property(arg) for arg in node.args
):
@@ -2744,17 +2792,33 @@ class TrinoPrinter(PostgresPrinter):
if len(node.args) == 1:
start = "0"
end = self.visit(node.args[0])
- elif len(node.args) == 2:
+ elif len(node.args) in {2, 3}:
start = self.visit(node.args[0])
end = self.visit(node.args[1])
else:
- self._invalid_function_arguments(node, "range expects one or two arguments in Trino mode.")
+ self._invalid_function_arguments(node, "range expects one to three arguments in Trino mode.")
+ step = ""
+ if len(node.args) == 3:
+ step_arg = node.args[2]
+ # A constant positive step keeps the Trino sequence ascending, as the exclusive end filter assumes.
+ if (
+ not isinstance(step_arg, ast.Constant)
+ or isinstance(step_arg.value, bool)
+ or not isinstance(step_arg.value, int)
+ or step_arg.value <= 0
+ ):
+ self._unsupported(
+ "TRINO_RANGE_STEP_UNSUPPORTED",
+ "range requires a constant positive integer step in Trino mode.",
+ node,
+ )
+ step = f", {step_arg.value}"
value = self._print_identifier("__hogql_range_value")
if all(isinstance(arg, ast.Constant) for arg in node.args):
- return f"filter(sequence({start}, greatest(({end}) - 1, {start})), {value} -> ({value} < {end}))"
+ return f"filter(sequence({start}, greatest(({end}) - 1, {start}){step}), {value} -> ({value} < {end}))"
return (
f"element_at(transform(ARRAY[ROW({start}, {end})], __hogql_range -> "
- "filter(sequence(__hogql_range[1], greatest(__hogql_range[2] - 1, __hogql_range[1])), "
+ f"filter(sequence(__hogql_range[1], greatest(__hogql_range[2] - 1, __hogql_range[1]){step}), "
f"{value} -> ({value} < __hogql_range[2]))), 1)"
)
diff --git a/posthog/hogql/transforms/trino/normalize.py b/posthog/hogql/transforms/trino/normalize.py
index fa3e72b38d89..96273cb37eb1 100644
--- a/posthog/hogql/transforms/trino/normalize.py
+++ b/posthog/hogql/transforms/trino/normalize.py
@@ -259,6 +259,19 @@ class TrinoScalarCTELowerer(CloningVisitor):
return super().visit_field(node)
+class _QualifiedFieldFinder(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.qualifiers: set[str] = set()
+
+ def visit_field(self, node: ast.Field) -> None:
+ if len(node.chain) > 1 and isinstance(node.chain[0], str):
+ self.qualifiers.add(node.chain[0])
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ return None
+
+
class TrinoSelectAliasLowerer(CloningVisitor):
def __init__(self, context: HogQLContext, budget: TrinoCompilationBudget) -> None:
super().__init__(clear_types=False)
@@ -267,6 +280,8 @@ class TrinoSelectAliasLowerer(CloningVisitor):
self.aliases: dict[str, ast.Expr] = {}
self.alias_positions: dict[str, int] = {}
self.expanding: set[str] = set()
+ self.in_order_by = False
+ self.in_window = False
self.expansion_budget = budget
def visit(self, node: _T_AST) -> _T_AST:
@@ -276,6 +291,10 @@ class TrinoSelectAliasLowerer(CloningVisitor):
def visit_select_query(self, node: ast.SelectQuery) -> ast.SelectQuery:
outer_aliases = self.aliases
outer_alias_positions = self.alias_positions
+ outer_in_order_by = self.in_order_by
+ outer_in_window = self.in_window
+ self.in_order_by = False
+ self.in_window = False
self.aliases = {expr.alias: expr.expr for expr in node.select if isinstance(expr, ast.Alias)}
self.alias_positions = {
expr.alias: index for index, expr in enumerate(node.select, start=1) if isinstance(expr, ast.Alias)
@@ -321,8 +340,27 @@ class TrinoSelectAliasLowerer(CloningVisitor):
order.expr = ast.PositionalRef(index=self.alias_positions[order_expression.chain[-1]])
self.aliases = outer_aliases
self.alias_positions = outer_alias_positions
+ self.in_order_by = outer_in_order_by
+ self.in_window = outer_in_window
return lowered
+ def visit_order_expr(self, node: ast.OrderExpr) -> ast.OrderExpr:
+ outer_in_order_by = self.in_order_by
+ # Only a query's own ORDER BY sees output aliases; a window's OVER (ORDER BY ...) does not.
+ self.in_order_by = not self.in_window
+ try:
+ return super().visit_order_expr(node)
+ finally:
+ self.in_order_by = outer_in_order_by
+
+ def visit_window_expr(self, node: ast.WindowExpr) -> ast.WindowExpr:
+ outer_in_order_by, outer_in_window = self.in_order_by, self.in_window
+ self.in_order_by, self.in_window = False, True
+ try:
+ return super().visit_window_expr(node)
+ finally:
+ self.in_order_by, self.in_window = outer_in_order_by, outer_in_window
+
def visit_alias(self, node: ast.Alias) -> ast.Alias:
if node.hidden:
self.expansion_budget.visit(node)
@@ -379,14 +417,24 @@ class TrinoSelectAliasLowerer(CloningVisitor):
alias = node.chain[0]
self.expanding.add(alias)
try:
- return self.visit(self.aliases[alias])
+ expanded = self.visit(self.aliases[alias])
finally:
self.expanding.remove(alias)
+ # Trino resolves ORDER BY names against output columns first, so a qualifier that is also a projection
+ # alias would read that output column. The output column itself already holds this value.
+ if self.in_order_by and self._has_qualifier_named_like_an_alias(expanded):
+ return clone_expr(node, clear_types=False)
+ return expanded
qualified = self._qualified_shadowed_column(node)
if qualified is not None:
return qualified
return super().visit_field(node)
+ def _has_qualifier_named_like_an_alias(self, node: ast.Expr) -> bool:
+ finder = _QualifiedFieldFinder()
+ finder.visit(node)
+ return any(qualifier in self.aliases for qualifier in finder.qualifiers)
+
def _qualified_shadowed_column(self, node: ast.Expr) -> ast.Field | None:
# After alias inlining, a later resolver pass would bind a bare column named like a projection alias to the
# alias. Qualifying it keeps the column the first pass chose.
@@ -396,10 +444,15 @@ class TrinoSelectAliasLowerer(CloningVisitor):
and isinstance(node.chain[0], str)
and node.chain[0] in self.aliases
and isinstance(node.type, ast.FieldType)
- and isinstance(node.type.table_type, ast.TableAliasType)
):
return None
- qualifier = node.type.table_type.alias
+ table_type = node.type.table_type
+ if isinstance(table_type, (ast.TableAliasType, ast.CTETableAliasType, ast.SelectQueryAliasType)):
+ qualifier = table_type.alias
+ elif isinstance(table_type, ast.CTETableType):
+ qualifier = table_type.name
+ else:
+ return None
if self.lambda_scopes and self._lambda_reads_qualifier_as_field(qualifier):
raise TrinoLoweringError(
"TRINO_LAMBDA_QUALIFIER_AMBIGUOUS",