Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): lower clickhouse semantics trino rejects in shadow builds (#118736)

Jovan Sakovice508b4c7afff
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.py
index 4fe30f699d43..32449094822b 100644
--- a/posthog/hogql/printer/test/test_trino_printer.py
+++ b/posthog/hogql/printer/test/test_trino_printer.py
@@ -602,6 +602,61 @@ def test_first_regex_replacement_preserves_capture_indices_and_literal_dollars()
assert "__hogql_match[5]" in sql
+ "pattern, expected",
+ [
+ ("a[[]1m[]]$", r"a[\[]1m[\]]$"),
+ ("[^]x]", r"[^\]x]"),
+ (r"[a-z]\\[", r"[a-z]\["),
+ ("[[:alpha:]]+", "[[:alpha:]]+"),
+ (r"\\Q[\\E[a-z]", r"\Q[\E[a-z]"),
+ ("[[:a[[:a", r"[\[:a\[\[:a"),
+ ],
+)
+def test_regex_character_classes_keep_clickhouse_meaning(pattern: str, expected: str) -> None:
+ context = _context_with_trino_table()
+ prepare_and_print_ast(parse_select(f"SELECT match(user_id, '{pattern}') FROM users"), context, "trino")
+
+ assert list(context.values.values()) == [expected]
+
+
+ "query, required, forbidden",
+ [
+ (
+ "SELECT m['a'] FROM (SELECT JSONExtract(properties, 'Map(String, Float64)') AS m FROM users)",
+ 'element_at("m"',
+ 'json_extract_scalar("m"',
+ ),
+ (
+ "WITH b AS (SELECT toDate(now()) - 1 AS latest) "
+ "SELECT count() FROM users CROSS JOIN b WHERE toDate(created_at) > b.latest - 14",
+ 'date_add(\'day\', CAST(-(14) AS BIGINT), "b"."latest")',
+ '"b"."latest" - 14',
+ ),
+ (
+ "SELECT toDate(created_at) + late FROM users ARRAY JOIN range(1, 8) AS late",
+ 'date_add(\'day\', CAST("__trino_unnest_0"."late" AS BIGINT)',
+ '+ "__trino_unnest_0"."late"',
+ ),
+ ("SELECT user_id AS owner FROM users HAVING owner != ''", "WHERE", "HAVING"),
+ (
+ "SELECT toFloat(user_id) FROM users",
+ 'TRY_CAST("users"."user_id" AS DOUBLE)',
+ 'CAST("users"."user_id" AS DOUBLE)',
+ ),
+ ("SELECT toDate(user_id) FROM users", "TRY", 'CAST("users"."user_id" AS DATE)'),
+ # Fractional seconds just before midnight must not round up into the next day.
+ ("SELECT toDate(user_id) FROM users", "AS TIMESTAMP(12)", "AS TIMESTAMP)"),
+ ],
+)
+def test_lowers_clickhouse_semantics_trino_rejects(query: str, required: str, forbidden: str) -> None:
+ sql, _ = prepare_and_print_ast(parse_select(query), _context_with_trino_table(), "trino")
+
+ assert required in sql
+ assert forbidden not in sql.replace(required, "")
+
+
def test_first_regex_replacement_preserves_inline_flags() -> None:
context = _context_with_trino_table()
prepare_and_print_ast(parse_select("SELECT replaceRegexpOne(user_id, '(?i)a', 'b') FROM users"), context, "trino")
diff --git a/posthog/hogql/printer/trino.py b/posthog/hogql/printer/trino.py
index 5aaa9323ad75..69e3acbf73bd 100644
--- a/posthog/hogql/printer/trino.py
+++ b/posthog/hogql/printer/trino.py
@@ -178,6 +178,73 @@ _TRINO_DATE_PARSE_SPECIFIERS = frozenset(
)
+_TRINO_REGEX_PATTERN_ARGUMENTS = {
+ "countmatches": 1,
+ "countmatchescaseinsensitive": 1,
+ "extract": 1,
+ "extractall": 1,
+ "extractallgroups": 1,
+ "extractallgroupshorizontal": 1,
+ "extractallgroupsvertical": 1,
+ "extractgroups": 1,
+ "match": 1,
+ "regexpextract": 1,
+ "replaceregexpall": 1,
+ "replaceregexpone": 1,
+ "splitbyregexp": 0,
+}
+
+
+_POSIX_CLASS_PATTERN = re.compile(r"\[:\^?[a-z]+:\]")
+
+
+def _escape_regex_class_brackets(pattern: str) -> str:
+ """Keep RE2 bracket-expression meaning under Trino's Joni engine.
+
+ RE2 reads `[` inside a class, and `]` as its first member, as literal characters.
+ Joni reads the first as a nested class and the second as an empty class, so both get escaped.
+ """
+ result: list[str] = []
+ index = 0
+ in_class = False
+ while index < len(pattern):
+ char = pattern[index]
+ if char == "\\":
+ if pattern.startswith("\\Q", index):
+ quote_end = pattern.find("\\E", index + 2)
+ end = len(pattern) if quote_end == -1 else quote_end + 2
+ result.append(pattern[index:end])
+ index = end
+ continue
+ result.append(pattern[index : index + 2])
+ index += 2
+ continue
+ if not in_class:
+ result.append(char)
+ index += 1
+ if char == "[":
+ in_class = True
+ if pattern.startswith("^", index):
+ result.append("^")
+ index += 1
+ if pattern.startswith("]", index):
+ result.append("\\]")
+ index += 1
+ continue
+ posix_class = _POSIX_CLASS_PATTERN.match(pattern, index)
+ if posix_class:
+ result.append(posix_class.group())
+ index = posix_class.end()
+ continue
+ if char == "[":
+ result.append("\\[")
+ else:
+ result.append(char)
+ in_class = char != "]"
+ index += 1
+ return "".join(result)
+
+
@frozen
class _BinaryArguments:
left: str
@@ -372,6 +439,14 @@ class TrinoPrinter(PostgresPrinter):
def visit_call(self, node: ast.Call) -> str:
name = node.name.lower()
+ pattern_index = _TRINO_REGEX_PATTERN_ARGUMENTS.get(name)
+ if pattern_index is not None and pattern_index < len(node.args):
+ pattern = node.args[pattern_index]
+ if isinstance(pattern, ast.Constant) and isinstance(pattern.value, str):
+ escaped = _escape_regex_class_brackets(pattern.value)
+ if escaped != pattern.value:
+ node = clone_expr(node, clear_types=False)
+ node.args[pattern_index] = ast.Constant(value=escaped, type=pattern.type)
if name in TRINO_WINDOW_ONLY_FUNCTIONS:
self._invalid_function_arguments(node, f"{node.name} requires an OVER clause in Trino mode.")
if name in TRINO_AGGREGATE_COMBINATORS:
@@ -793,6 +868,25 @@ class TrinoPrinter(PostgresPrinter):
return f"CAST(date_diff('day', DATE '1970-01-01', {self.visit(arg)}) AS DOUBLE)"
if isinstance(arg_type, ast.DateTimeType):
return f"to_unixtime({self.visit(arg)})"
+ # HogQL defines toFloat as accurateCastOrNull, so unparsable input becomes NULL instead of an error.
+ if name == "tofloat" and len(node.args) == 1 and isinstance(arg_type, ast.StringType):
+ return f"TRY_CAST({self.visit(arg)} AS DOUBLE)"
+ if (
+ name == "todate"
+ and len(node.args) == 1
+ and isinstance(self._resolve_type(node.args[0]), ast.StringType)
+ and not (
+ isinstance(node.args[0], ast.Constant)
+ and isinstance(node.args[0].value, str)
+ and re.fullmatch(r"\d{4}-\d{2}-\d{2}", node.args[0].value)
+ )
+ ):
+ # HogQL defines toDate on a string as toDateOrNull, which also accepts a full timestamp.
+ value = self.visit(node.args[0])
+ return (
+ f"coalesce(TRY(date({value})), TRY(date(CAST({value} AS TIMESTAMP(12)))), "
+ f"TRY(date(from_iso8601_timestamp({value}))))"
+ )
if (
re.fullmatch(r"(?:add|subtract)(?:Seconds|Minutes|Hours|Days|Weeks|Months|Quarters|Years)", node.name)
and node.args
@@ -867,7 +961,7 @@ class TrinoPrinter(PostgresPrinter):
)
if name == "sum" and len(node.args) == 1 and self._is_dynamic_property(node.args[0]):
lowered = clone_expr(node)
- lowered.args = [ast.Call(name="toFloat", args=[lowered.args[0]])]
+ lowered.args = [ast.TypeCast(expr=lowered.args[0], type_name="double")]
return super().visit_call(lowered)
if name in {
"jsonextract",
@@ -1308,6 +1402,10 @@ class TrinoPrinter(PostgresPrinter):
if column_type is not None:
value_type = column_type
continue
+ if isinstance(table_type, ast.ColumnAliasedTableType):
+ unnest_type = self._unnest_column_type(table_type, value_type.name)
+ if unnest_type is not None:
+ return unnest_type
break
if isinstance(value_type, ast.CallType) and value_type.name.lower() == "jsonextractarrayraw":
return ast.ArrayType(item_type=ast.StringType())
@@ -1316,6 +1414,32 @@ class TrinoPrinter(PostgresPrinter):
return ast.ArrayType(item_type=ast.StringType(), nullable=resolved.nullable)
return resolved
+ def _unnest_column_type(self, table_type: ast.ColumnAliasedTableType, column: str) -> ast.ConstantType | None:
+ # UNNEST output columns are untyped in the schema; their type is the matching row field of the unnested array.
+ if not isinstance(table_type.table_type, ast.TableType) or not isinstance(
+ table_type.table_type.table, TrinoUnnestTable
+ ):
+ return None
+ for scope in reversed(self.stack):
+ if not isinstance(scope, ast.SelectQuery):
+ continue
+ join = scope.select_from
+ while join is not None:
+ if join.alias == table_type.alias and join.table_args and join.column_aliases:
+ if column not in join.column_aliases:
+ return None
+ rows = self._resolve_type(join.table_args[0])
+ index = join.column_aliases.index(column)
+ if (
+ isinstance(rows, ast.ArrayType)
+ and isinstance(rows.item_type, ast.TupleType)
+ and index < len(rows.item_type.item_types)
+ ):
+ return rows.item_type.item_types[index]
+ return None
+ join = join.next_join
+ return None
+
def _subquery_scalar_type(self, node: ast.SelectQuery | ast.SelectSetQuery) -> ast.ConstantType | None:
query = node
while isinstance(query, ast.SelectSetQuery):
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py
index 50bae2a91745..c7f6fa427b13 100644
--- a/posthog/hogql/resolver.py
+++ b/posthog/hogql/resolver.py
@@ -1971,6 +1971,14 @@ class Resolver(CloningVisitor):
node.type = ast.FloatType()
elif isinstance(left_type, ast.DateTimeType) or isinstance(right_type, ast.DateTimeType):
node.type = ast.DateTimeType()
+ elif (
+ self.dialect == "trino"
+ and node.op in (ast.ArithmeticOperationOp.Add, ast.ArithmeticOperationOp.Sub)
+ and isinstance(left_type, ast.DateType)
+ and isinstance(right_type, ast.IntegerType)
+ ):
+ # ClickHouse reads `Date ± Int` as a day offset; the Trino printer lowers it to date_add by this type.
+ node.type = ast.DateType()
elif isinstance(left_type, ast.DecimalType) or isinstance(right_type, ast.DecimalType):
# ClickHouse widens Decimal combined with a Float to Float; Decimal combined with a
# Decimal or Integer stays Decimal. Anything else (e.g. Decimal + String) is unknown.
diff --git a/posthog/hogql/transforms/trino/normalize.py b/posthog/hogql/transforms/trino/normalize.py
index 9eb947a7d61d..b0ab6f125985 100644
--- a/posthog/hogql/transforms/trino/normalize.py
+++ b/posthog/hogql/transforms/trino/normalize.py
@@ -10,7 +10,7 @@ from posthog.hogql.transforms.trino.asof_join import TrinoAsOfJoinLowerer
from posthog.hogql.transforms.trino.errors import TrinoLoweringError
from posthog.hogql.transforms.trino.expressions import expression_key, positional_index
from posthog.hogql.transforms.trino.limits import TrinoCompilationBudget
-from posthog.hogql.transforms.trino.query_wrappers import lower_trino_query_wrappers
+from posthog.hogql.transforms.trino.query_wrappers import _AggregateFunctionFinder, lower_trino_query_wrappers
from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr
_T_AST = TypeVar("_T_AST", bound=ast.AST | None)
@@ -447,8 +447,21 @@ class TrinoNormalizer(TraversingVisitor):
node.prewhere = None
node.settings = None
self._lower_array_join(node)
+ self._lower_row_filter_having(node)
super().visit_select_query(node)
+ def _lower_row_filter_having(self, node: ast.SelectQuery) -> None:
+ # ClickHouse reads HAVING without GROUP BY or aggregates as a row filter; Trino makes it a global aggregation.
+ if node.having is None or node.group_by:
+ return
+ finder = _AggregateFunctionFinder()
+ for expr in [*node.select, node.having, *(order.expr for order in node.order_by or [])]:
+ finder.visit(expr)
+ if finder.found:
+ return
+ node.where = ast.And(exprs=[node.where, node.having]) if node.where is not None else node.having
+ node.having = None
+
def _lower_array_join(self, node: ast.SelectQuery) -> None:
if node.array_join_op is None and not node.array_join_list:
return
diff --git a/posthog/hogql/type_system.py b/posthog/hogql/type_system.py
index 63ada840fe8f..1a43816dbc03 100644
--- a/posthog/hogql/type_system.py
+++ b/posthog/hogql/type_system.py
@@ -2086,7 +2086,8 @@ def _infer_json_extract_type(
if not isinstance(type_arg, ast.Constant) or not isinstance(type_arg.value, str):
return None
- runtime_type = parse_sql_runtime_type(type_arg.value, dialect=dialect)
+ # The Trino backend translates HogQL's ClickHouse type names, so the target is ClickHouse syntax there too.
+ runtime_type = parse_sql_runtime_type(type_arg.value, dialect="clickhouse" if dialect == "trino" else dialect)
if runtime_type.family == "unknown":
return None
result_type = constant_type_from_runtime_type(runtime_type)