:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): lower clickhouse semantics trino rejects in shadow builds (#118736)
Jovan Sakovice508b4c7afff
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.pyindex 4fe30f699d43..32449094822b 100644--- a/posthog/hogql/printer/test/test_trino_printer.py+++ b/posthog/hogql/printer/test/test_trino_printer.py@@ -602,6 +602,61 @@ def test_first_regex_replacement_preserves_capture_indices_and_literal_dollars()assert "__hogql_match[5]" in sql+ "pattern, expected",+ [+ ("a[[]1m[]]$", r"a[\[]1m[\]]$"),+ ("[^]x]", r"[^\]x]"),+ (r"[a-z]\\[", r"[a-z]\["),+ ("[[:alpha:]]+", "[[:alpha:]]+"),+ (r"\\Q[\\E[a-z]", r"\Q[\E[a-z]"),+ ("[[:a[[:a", r"[\[:a\[\[:a"),+ ],+)+def test_regex_character_classes_keep_clickhouse_meaning(pattern: str, expected: str) -> None:+ context = _context_with_trino_table()+ prepare_and_print_ast(parse_select(f"SELECT match(user_id, '{pattern}') FROM users"), context, "trino")++ assert list(context.values.values()) == [expected]+++ "query, required, forbidden",+ [+ (+ "SELECT m['a'] FROM (SELECT JSONExtract(properties, 'Map(String, Float64)') AS m FROM users)",+ 'element_at("m"',+ 'json_extract_scalar("m"',+ ),+ (+ "WITH b AS (SELECT toDate(now()) - 1 AS latest) "+ "SELECT count() FROM users CROSS JOIN b WHERE toDate(created_at) > b.latest - 14",+ 'date_add(\'day\', CAST(-(14) AS BIGINT), "b"."latest")',+ '"b"."latest" - 14',+ ),+ (+ "SELECT toDate(created_at) + late FROM users ARRAY JOIN range(1, 8) AS late",+ 'date_add(\'day\', CAST("__trino_unnest_0"."late" AS BIGINT)',+ '+ "__trino_unnest_0"."late"',+ ),+ ("SELECT user_id AS owner FROM users HAVING owner != ''", "WHERE", "HAVING"),+ (+ "SELECT toFloat(user_id) FROM users",+ 'TRY_CAST("users"."user_id" AS DOUBLE)',+ 'CAST("users"."user_id" AS DOUBLE)',+ ),+ ("SELECT toDate(user_id) FROM users", "TRY", 'CAST("users"."user_id" AS DATE)'),+ # Fractional seconds just before midnight must not round up into the next day.+ ("SELECT toDate(user_id) FROM users", "AS TIMESTAMP(12)", "AS TIMESTAMP)"),+ ],+)+def test_lowers_clickhouse_semantics_trino_rejects(query: str, required: str, forbidden: str) -> None:+ sql, _ = prepare_and_print_ast(parse_select(query), _context_with_trino_table(), "trino")++ assert required in sql+ assert forbidden not in sql.replace(required, "")++def test_first_regex_replacement_preserves_inline_flags() -> None:context = _context_with_trino_table()prepare_and_print_ast(parse_select("SELECT replaceRegexpOne(user_id, '(?i)a', 'b') FROM users"), context, "trino")diff --git a/posthog/hogql/printer/trino.py b/posthog/hogql/printer/trino.pyindex 5aaa9323ad75..69e3acbf73bd 100644--- a/posthog/hogql/printer/trino.py+++ b/posthog/hogql/printer/trino.py@@ -178,6 +178,73 @@ _TRINO_DATE_PARSE_SPECIFIERS = frozenset()+_TRINO_REGEX_PATTERN_ARGUMENTS = {+ "countmatches": 1,+ "countmatchescaseinsensitive": 1,+ "extract": 1,+ "extractall": 1,+ "extractallgroups": 1,+ "extractallgroupshorizontal": 1,+ "extractallgroupsvertical": 1,+ "extractgroups": 1,+ "match": 1,+ "regexpextract": 1,+ "replaceregexpall": 1,+ "replaceregexpone": 1,+ "splitbyregexp": 0,+}+++_POSIX_CLASS_PATTERN = re.compile(r"\[:\^?[a-z]+:\]")+++def _escape_regex_class_brackets(pattern: str) -> str:+ """Keep RE2 bracket-expression meaning under Trino's Joni engine.++ RE2 reads `[` inside a class, and `]` as its first member, as literal characters.+ Joni reads the first as a nested class and the second as an empty class, so both get escaped.+ """+ result: list[str] = []+ index = 0+ in_class = False+ while index < len(pattern):+ char = pattern[index]+ if char == "\\":+ if pattern.startswith("\\Q", index):+ quote_end = pattern.find("\\E", index + 2)+ end = len(pattern) if quote_end == -1 else quote_end + 2+ result.append(pattern[index:end])+ index = end+ continue+ result.append(pattern[index : index + 2])+ index += 2+ continue+ if not in_class:+ result.append(char)+ index += 1+ if char == "[":+ in_class = True+ if pattern.startswith("^", index):+ result.append("^")+ index += 1+ if pattern.startswith("]", index):+ result.append("\\]")+ index += 1+ continue+ posix_class = _POSIX_CLASS_PATTERN.match(pattern, index)+ if posix_class:+ result.append(posix_class.group())+ index = posix_class.end()+ continue+ if char == "[":+ result.append("\\[")+ else:+ result.append(char)+ in_class = char != "]"+ index += 1+ return "".join(result)++@frozenclass _BinaryArguments:left: str@@ -372,6 +439,14 @@ class TrinoPrinter(PostgresPrinter):def visit_call(self, node: ast.Call) -> str:name = node.name.lower()+ pattern_index = _TRINO_REGEX_PATTERN_ARGUMENTS.get(name)+ if pattern_index is not None and pattern_index < len(node.args):+ pattern = node.args[pattern_index]+ if isinstance(pattern, ast.Constant) and isinstance(pattern.value, str):+ escaped = _escape_regex_class_brackets(pattern.value)+ if escaped != pattern.value:+ node = clone_expr(node, clear_types=False)+ node.args[pattern_index] = ast.Constant(value=escaped, type=pattern.type)if name in TRINO_WINDOW_ONLY_FUNCTIONS:self._invalid_function_arguments(node, f"{node.name} requires an OVER clause in Trino mode.")if name in TRINO_AGGREGATE_COMBINATORS:@@ -793,6 +868,25 @@ class TrinoPrinter(PostgresPrinter):return f"CAST(date_diff('day', DATE '1970-01-01', {self.visit(arg)}) AS DOUBLE)"if isinstance(arg_type, ast.DateTimeType):return f"to_unixtime({self.visit(arg)})"+ # HogQL defines toFloat as accurateCastOrNull, so unparsable input becomes NULL instead of an error.+ if name == "tofloat" and len(node.args) == 1 and isinstance(arg_type, ast.StringType):+ return f"TRY_CAST({self.visit(arg)} AS DOUBLE)"+ if (+ name == "todate"+ and len(node.args) == 1+ and isinstance(self._resolve_type(node.args[0]), ast.StringType)+ and not (+ isinstance(node.args[0], ast.Constant)+ and isinstance(node.args[0].value, str)+ and re.fullmatch(r"\d{4}-\d{2}-\d{2}", node.args[0].value)+ )+ ):+ # HogQL defines toDate on a string as toDateOrNull, which also accepts a full timestamp.+ value = self.visit(node.args[0])+ return (+ f"coalesce(TRY(date({value})), TRY(date(CAST({value} AS TIMESTAMP(12)))), "+ f"TRY(date(from_iso8601_timestamp({value}))))"+ )if (re.fullmatch(r"(?:add|subtract)(?:Seconds|Minutes|Hours|Days|Weeks|Months|Quarters|Years)", node.name)and node.args@@ -867,7 +961,7 @@ class TrinoPrinter(PostgresPrinter):)if name == "sum" and len(node.args) == 1 and self._is_dynamic_property(node.args[0]):lowered = clone_expr(node)- lowered.args = [ast.Call(name="toFloat", args=[lowered.args[0]])]+ lowered.args = [ast.TypeCast(expr=lowered.args[0], type_name="double")]return super().visit_call(lowered)if name in {"jsonextract",@@ -1308,6 +1402,10 @@ class TrinoPrinter(PostgresPrinter):if column_type is not None:value_type = column_typecontinue+ if isinstance(table_type, ast.ColumnAliasedTableType):+ unnest_type = self._unnest_column_type(table_type, value_type.name)+ if unnest_type is not None:+ return unnest_typebreakif isinstance(value_type, ast.CallType) and value_type.name.lower() == "jsonextractarrayraw":return ast.ArrayType(item_type=ast.StringType())@@ -1316,6 +1414,32 @@ class TrinoPrinter(PostgresPrinter):return ast.ArrayType(item_type=ast.StringType(), nullable=resolved.nullable)return resolved+ def _unnest_column_type(self, table_type: ast.ColumnAliasedTableType, column: str) -> ast.ConstantType | None:+ # UNNEST output columns are untyped in the schema; their type is the matching row field of the unnested array.+ if not isinstance(table_type.table_type, ast.TableType) or not isinstance(+ table_type.table_type.table, TrinoUnnestTable+ ):+ return None+ for scope in reversed(self.stack):+ if not isinstance(scope, ast.SelectQuery):+ continue+ join = scope.select_from+ while join is not None:+ if join.alias == table_type.alias and join.table_args and join.column_aliases:+ if column not in join.column_aliases:+ return None+ rows = self._resolve_type(join.table_args[0])+ index = join.column_aliases.index(column)+ if (+ isinstance(rows, ast.ArrayType)+ and isinstance(rows.item_type, ast.TupleType)+ and index < len(rows.item_type.item_types)+ ):+ return rows.item_type.item_types[index]+ return None+ join = join.next_join+ return None+def _subquery_scalar_type(self, node: ast.SelectQuery | ast.SelectSetQuery) -> ast.ConstantType | None:query = nodewhile isinstance(query, ast.SelectSetQuery):diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.pyindex 50bae2a91745..c7f6fa427b13 100644--- a/posthog/hogql/resolver.py+++ b/posthog/hogql/resolver.py@@ -1971,6 +1971,14 @@ class Resolver(CloningVisitor):node.type = ast.FloatType()elif isinstance(left_type, ast.DateTimeType) or isinstance(right_type, ast.DateTimeType):node.type = ast.DateTimeType()+ elif (+ self.dialect == "trino"+ and node.op in (ast.ArithmeticOperationOp.Add, ast.ArithmeticOperationOp.Sub)+ and isinstance(left_type, ast.DateType)+ and isinstance(right_type, ast.IntegerType)+ ):+ # ClickHouse reads `Date ± Int` as a day offset; the Trino printer lowers it to date_add by this type.+ node.type = ast.DateType()elif isinstance(left_type, ast.DecimalType) or isinstance(right_type, ast.DecimalType):# ClickHouse widens Decimal combined with a Float to Float; Decimal combined with a# Decimal or Integer stays Decimal. Anything else (e.g. Decimal + String) is unknown.diff --git a/posthog/hogql/transforms/trino/normalize.py b/posthog/hogql/transforms/trino/normalize.pyindex 9eb947a7d61d..b0ab6f125985 100644--- a/posthog/hogql/transforms/trino/normalize.py+++ b/posthog/hogql/transforms/trino/normalize.py@@ -10,7 +10,7 @@ from posthog.hogql.transforms.trino.asof_join import TrinoAsOfJoinLowererfrom posthog.hogql.transforms.trino.errors import TrinoLoweringErrorfrom posthog.hogql.transforms.trino.expressions import expression_key, positional_indexfrom posthog.hogql.transforms.trino.limits import TrinoCompilationBudget-from posthog.hogql.transforms.trino.query_wrappers import lower_trino_query_wrappers+from posthog.hogql.transforms.trino.query_wrappers import _AggregateFunctionFinder, lower_trino_query_wrappersfrom posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr_T_AST = TypeVar("_T_AST", bound=ast.AST | None)@@ -447,8 +447,21 @@ class TrinoNormalizer(TraversingVisitor):node.prewhere = Nonenode.settings = Noneself._lower_array_join(node)+ self._lower_row_filter_having(node)super().visit_select_query(node)+ def _lower_row_filter_having(self, node: ast.SelectQuery) -> None:+ # ClickHouse reads HAVING without GROUP BY or aggregates as a row filter; Trino makes it a global aggregation.+ if node.having is None or node.group_by:+ return+ finder = _AggregateFunctionFinder()+ for expr in [*node.select, node.having, *(order.expr for order in node.order_by or [])]:+ finder.visit(expr)+ if finder.found:+ return+ node.where = ast.And(exprs=[node.where, node.having]) if node.where is not None else node.having+ node.having = None+def _lower_array_join(self, node: ast.SelectQuery) -> None:if node.array_join_op is None and not node.array_join_list:returndiff --git a/posthog/hogql/type_system.py b/posthog/hogql/type_system.pyindex 63ada840fe8f..1a43816dbc03 100644--- a/posthog/hogql/type_system.py+++ b/posthog/hogql/type_system.py@@ -2086,7 +2086,8 @@ def _infer_json_extract_type(if not isinstance(type_arg, ast.Constant) or not isinstance(type_arg.value, str):return None- runtime_type = parse_sql_runtime_type(type_arg.value, dialect=dialect)+ # The Trino backend translates HogQL's ClickHouse type names, so the target is ClickHouse syntax there too.+ runtime_type = parse_sql_runtime_type(type_arg.value, dialect="clickhouse" if dialect == "trino" else dialect)if runtime_type.family == "unknown":return Noneresult_type = constant_type_from_runtime_type(runtime_type)