Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): swap joined flag-call scans and read event aliases in the union (#129819)

Phil Haack4749374fa408
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
index 6e8e3ffb4e07..dedf1dc41745 100644
--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -205,8 +205,9 @@ def swap_flag_calls(
every field in it that starts with `posthog.events` reads a scan.
Otherwise it returns the resolved original. For an organization whose flag calls are only in
`flag_evaluations`, it raises a QueryError instead of returning the original. It also raises before it
- resolves anything when such an organization's query has a scan of flag calls with a shape that neither
- `flag_evaluations` nor the union can stand in for.
+ resolves anything when such an organization's query has a scan of only flag calls with a shape that
+ `flag_evaluations` cannot stand in for. When the union refuses a scan, the query raises only if that scan still
+ reads `events` after `swap_scans_found_by_type`.
While `events` still gets flag calls, the original query is the answer whenever the two tables can read a row
differently:
@@ -279,16 +280,26 @@ def _resolve_with_swaps(
)
swapper.visit(rewritten)
union = read_flag_calls_with_union(rewritten, context, compares_as_string)
- refusals = [*swapper.refusals, *union.refusals]
- if refusals:
- if only_in_flag_evaluations:
- raise flag_calls_only_in_flag_evaluations_error(refusals[0], view_name)
- logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=refusals[0])
+ if only_in_flag_evaluations and swapper.refusals:
+ raise flag_calls_only_in_flag_evaluations_error(swapper.refusals[0], view_name)
+ if not only_in_flag_evaluations and (swapper.refusals or union.refusals):
+ reason = swapper.refusals[0] if swapper.refusals else union.refusals[0].reason
+ logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=reason)
sources = JoinSources() if only_in_flag_evaluations else None
original = resolve(node, join_sources=sources)
if sources is not None:
with context.timings.measure("flag_calls_from_flag_evaluations_by_type"):
refusal = swapper.swap_scans_found_by_type(original, sources, clones)
+ # swap_scans_found_by_type can move a scan that the union refused, such as a scan joined after another table,
+ # to flag_evaluations. Each other refused scan still reads flag calls from events.
+ refusal = refusal or next(
+ (
+ union_refusal.reason
+ for union_refusal in union.refusals
+ if _table_chain(union_refusal.join) != ["posthog", "flag_evaluations"]
+ ),
+ None,
+ )
if refusal is not None:
raise flag_calls_only_in_flag_evaluations_error(refusal, view_name)
if not swapper.swapped and not union.renamed:
@@ -451,14 +462,13 @@ class EventsScan:
folded_tables=frozenset(_folded_join_names(join.next_join, database)),
flag_key_compares_as_string=flag_key_compares_as_string,
)
- collector = _AliasCollector(node)
- collector.visit(node)
+ aliases = select_aliases(node)
# The resolver tries a joined table's name, a CTE and a select alias before it reads a qualifier such as
# `posthog.events` as the name of a table.
- scan.qualifier_shadows = scan.other_tables.union(name for name, _ in collector.aliases)
+ scan.qualifier_shadows = scan.other_tables.union(name for name, _ in aliases)
# An alias such as `e.timestamp AS timestamp` reads the column it names, so it changes no read.
scan.shadowed = frozenset(
- name for name, expr in collector.aliases if name in _SCAN_COLUMNS and scan.column(expr) != [name]
+ name for name, expr in aliases if name in _SCAN_COLUMNS and scan.column(expr) != [name]
)
return scan
@@ -939,10 +949,22 @@ class SyntaxVisitor(TraversingVisitor):
pass
+def select_aliases(node: ast.SelectQuery, *, in_scope_only: bool = False) -> list[tuple[str, ast.Expr]]:
+ """Each alias that `node` defines outside its subqueries, with the expression it names.
+
+ With `in_scope_only`, the list leaves out the aliases that the resolver does not register in the scope of `node`:
+ an alias inside a lambda, and a hidden alias.
+ """
+ collector = _AliasCollector(node, in_scope_only)
+ collector.visit(node)
+ return collector.aliases
+
+
class _AliasCollector(SyntaxVisitor):
- def __init__(self, root: ast.SelectQuery) -> None:
+ def __init__(self, root: ast.SelectQuery, in_scope_only: bool) -> None:
super().__init__()
self.root = root
+ self.in_scope_only = in_scope_only
self.aliases: list[tuple[str, ast.Expr]] = []
def visit_select_query(self, node: ast.SelectQuery) -> None:
@@ -953,8 +975,14 @@ class _AliasCollector(SyntaxVisitor):
def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
pass
+ def visit_lambda(self, node: ast.Lambda) -> None:
+ # The resolver registers an alias inside a lambda in the scope of that lambda.
+ if not self.in_scope_only:
+ super().visit_lambda(node)
+
def visit_alias(self, node: ast.Alias) -> None:
- self.aliases.append((node.alias, node.expr))
+ if not (self.in_scope_only and node.hidden):
+ self.aliases.append((node.alias, node.expr))
super().visit_alias(node)
diff --git a/posthog/hogql/transforms/flag_calls_union.py b/posthog/hogql/transforms/flag_calls_union.py
index 2a3c8be98e81..fcc3d2ca9fd8 100644
--- a/posthog/hogql/transforms/flag_calls_union.py
+++ b/posthog/hogql/transforms/flag_calls_union.py
@@ -9,8 +9,10 @@ The rename runs before type resolution, so every rule here is syntactic. The uni
scan, so the rules decide which scans pay for a second table, and which shapes the union cannot read yet.
"""
+import math
+from collections import ChainMap
from collections.abc import Callable
-from functools import cache, partial
+from functools import cache, cached_property, partial
from math import floor
from typing import TYPE_CHECKING, cast
@@ -60,10 +62,12 @@ from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (
flag_key_predicate,
point_scan_at,
ratio_sample_fraction,
+ select_aliases,
team_flag_key_compares_as_string,
)
+from posthog.hogql.transforms.order_by_pushdown import unwrap_alias
from posthog.hogql.type_system import parse_sql_runtime_type
-from posthog.hogql.visitor import CloningVisitor
+from posthog.hogql.visitor import CloningVisitor, TraversingVisitor
from posthog.dataclasses import frozen
from posthog.property_columns import TableColumn
@@ -75,6 +79,10 @@ logger = structlog.get_logger(__name__)
# The joins whose resolver compares only the join's key and `extra_from_fields` on the source table. Lazy table
# resolution projects exactly those from the union, so a join that reads anything else would name a missing column.
+# The most compared values the flag-call finder reads for one scan. Past it, the finder treats the scan as one that
+# names flag calls. The union returns the right rows for any scan, so the limit costs only a second table read.
+_COMPARED_VALUES_BUDGET = 100_000
+
_UNION_JOIN_RESOLVERS = frozenset(
{
EVENTS_TO_SESSIONS_V1,
@@ -100,11 +108,17 @@ def refuse_union_table_by_name(node: ast.AST, context: HogQLContext) -> None:
raise QueryError(f"Unknown table `posthog.{EVENTS_WITH_FLAG_CALLS_TABLE}`.")
+@frozen
+class UnionRefusal:
+ join: ast.JoinExpr
+ reason: str
+
+
@frozen
class UnionRename:
renamed: bool
# Why the union cannot read each other scan that names `$feature_flag_called` with other events.
- refusals: tuple[str, ...]
+ refusals: tuple[UnionRefusal, ...]
# The (start, end) of each `$feature_flag_called` literal that a renamed scan compares with its `event`.
literal_spans: frozenset[tuple[int, int]] = frozenset()
@@ -158,7 +172,7 @@ def read_flag_calls_in_inline_cohort(
)
renamer.visit(query)
if renamer.refusals:
- raise flag_calls_only_in_flag_evaluations_error(renamer.refusals[0])
+ raise flag_calls_only_in_flag_evaluations_error(renamer.refusals[0].reason)
return query
@@ -301,7 +315,8 @@ def rename_flag_call_scans_to_union(
1. Its query compares the scan's `event` with `$feature_flag_called` by `=` or `IN`, in one of the query's own
clauses. A subquery's comparison does not count. A funnel step column such as
- `if(event = '$feature_flag_called' AND ..., 1, 0)` counts.
+ `if(event = '$feature_flag_called' AND ..., 1, 0)` counts. A name of one of the query's aliases reads the
+ expression that the alias names.
2. The scan is the first table in FROM, and every join after it is INNER, LEFT or CROSS, GLOBAL or not.
3. The scan has no FINAL, table arguments or column aliases, and its SAMPLE is 1 or a ratio between 0 and 1 without
OFFSET. Its query selects no `*` or `COLUMNS(...)`.
@@ -310,6 +325,9 @@ def rename_flag_call_scans_to_union(
more. The flag_evaluations swap swaps or refuses a first scan that reads only flag calls, so the union neither
renames nor refuses it. The swap cannot read an unqualified `event` next to a join. On mode 1 a first scan that
reads only flag calls through such an `event` stays on events. On mode 2 the union renames it.
+
+ A refusal does not decide the query on mode 2. `swap_scans_found_by_type` can still move the refused scan to
+ flag_evaluations, as it does for a scan of only flag calls joined after another table.
"""
renamer = _UnionRenamer(flag_key_compares_as_string, only_in_flag_evaluations)
renamer.visit(node)
@@ -486,10 +504,12 @@ class _FlagCallScan:
position: int
scan: EventsScan
literals: list[ast.Constant]
+ # True when the finder stopped at the budget before it found a `$feature_flag_called` literal.
+ unchecked: bool
def _flag_call_scans(
- node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]
+ node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str], aliases: "_Aliases"
) -> list[_FlagCallScan]:
"""Each `events` scan in the FROM chain of `node` whose query names `$feature_flag_called`, with its position and
the literals that name it."""
@@ -514,17 +534,95 @@ def _flag_call_scans(
flag_key_compares_as_string=flag_key_compares_as_string,
)
)
- if scan is not None and (literals := _flag_call_literals(node, scan)):
- scans.append(_FlagCallScan(position=position, scan=scan, literals=literals))
+ if scan is not None and (finder := _flag_call_finder(node, scan, aliases)).names_flag_calls:
+ scans.append(
+ _FlagCallScan(
+ position=position,
+ scan=scan,
+ literals=finder.literals,
+ unchecked=not finder.literals,
+ )
+ )
join = join.next_join
position += 1
return scans
-def _flag_call_literals(node: ast.SelectQuery, scan: EventsScan) -> list[ast.Constant]:
- finder = _FlagCallComparisonFinder(node, scan)
+def _flag_call_finder(node: ast.SelectQuery, scan: EventsScan, aliases: "_Aliases") -> "_FlagCallComparisonFinder":
+ finder = _FlagCallComparisonFinder(node, scan, aliases)
finder.visit(node)
- return finder.literals
+ return finder
+
+
+class _Aliases(CloningVisitor):
+ """Copies an unresolved expression with each name of a select alias that names a field replaced by that field.
+
+ The resolver reads a name without a table prefix as a select alias before it reads it as a column.
+ `swap_scans_found_by_type` reads a pin of `event` through aliases of a field, and through no other alias. A scan
+ that the union leaves to that swap must be one that the swap can read.
+ """
+
+ def __init__(self, node: ast.SelectQuery) -> None:
+ super().__init__()
+ self.node = node
+ self._named: dict[tuple[str, float], ast.Expr] = {}
+
+ @cached_property
+ def aliases(self) -> dict[str, tuple[int, ast.Expr]]:
+ declared: dict[str, tuple[int, ast.Expr]] = {}
+ for position, (name, expr) in enumerate(select_aliases(self.node, in_scope_only=True)):
+ declared.setdefault(name, (position, expr))
+ return declared
+
+ def target(self, expr: ast.Expr) -> ast.Expr:
+ """The expression that `expr`, without its own alias, names through a chain of select aliases."""
+ expr = unwrap_alias(expr)
+ if not (isinstance(expr, ast.Field) and len(expr.chain) == 1):
+ return expr
+ return self._follow(str(expr.chain[0])) or expr
+
+ def _follow(self, name: str) -> ast.Expr | None:
+ """The expression at the end of the chain of select aliases that starts at `name`, or None for no alias."""
+ visible: float = math.inf
+ hops: list[tuple[str, float]] = []
+ named: ast.Expr | None = None
+ # Each hop is cached, so a query that reads one long chain many times walks the chain once.
+ while (name, visible) not in self._named:
+ declared = self.aliases.get(name)
+ # The resolver binds a name in an alias's expression only to an alias declared before it. In
+ # `event AS event`, the inner `event` is therefore the column.
+ if declared is None or declared[0] >= visible:
+ break
+ hops.append((name, visible))
+ visible, aliased = declared
+ named = unwrap_alias(aliased)
+ if not (isinstance(named, ast.Field) and len(named.chain) == 1):
+ break
+ name = str(named.chain[0])
+ else:
+ named = self._named[(name, visible)]
+ for hop in hops:
+ self._named[hop] = cast(ast.Expr, named)
+ return named
+
+ def visit_field(self, node: ast.Field):
+ target = self.target(node)
+ return super().visit_field(target if isinstance(target, ast.Field) else node)
+
+ def visit_alias(self, node: ast.Alias):
+ # `swap_scans_found_by_type` reads through an alias of a field, but it does not read a comparison inside an
+ # alias.
+ inner = unwrap_alias(node)
+ if isinstance(inner, ast.Field):
+ return self.visit(inner)
+ return super().visit_alias(node)
+
+ def _keep(self, node):
+ # A subquery has its own aliases, and a lambda has its own arguments. No rule here reads a comparison inside
+ # either one.
+ return node
+
+ visit_select_query = visit_select_set_query = visit_lambda = _keep
class _ScanReadCopier(CloningVisitor):
@@ -681,16 +779,41 @@ class _UnionTableFinder(SyntaxVisitor):
class _FlagCallComparisonFinder(SyntaxVisitor):
"""Finds `event = '$feature_flag_called'` or an `IN` list with it, on the scan, in its own query's clauses."""
- def __init__(self, root: ast.SelectQuery, scan: EventsScan) -> None:
+ def __init__(self, root: ast.SelectQuery, scan: EventsScan, aliases: _Aliases) -> None:
super().__init__()
self.root = root
self.reader = scan
+ self.aliases = aliases
+ # The names that the enclosing lambdas bind, or None outside a lambda. A lambda argument maps to None. An alias
+ # inside a lambda maps to the expression that it names. Each lambda adds a layer, so entering one copies nothing.
+ self.lambda_names: ChainMap[str, ast.Expr | None] | None = None
+ self.values_left = _COMPARED_VALUES_BUDGET
self.literals: list[ast.Constant] = []
+ @property
+ def names_flag_calls(self) -> bool:
+ # Past the budget the finder cannot tell which values it compares with `event`. A query without any
+ # `$feature_flag_called` constant cannot compare `event` with one, so only a query with one counts.
+ return bool(self.literals) or (self.values_left < 0 and _has_flag_called_constant(self.root))
+
def visit_select_query(self, node: ast.SelectQuery) -> None:
if node is self.root:
super().visit_select_query(node)
+ def visit_lambda(self, node: ast.Lambda) -> None:
+ outer = self.lambda_names
+ self.lambda_names = (outer if outer is not None else ChainMap()).new_child(dict.fromkeys(node.args))
+ super().visit_lambda(node)
+ self.lambda_names = outer
+
+ def visit_alias(self, node: ast.Alias) -> None:
+ super().visit_alias(node)
+ # Inside a lambda, the resolver binds an alias's name in the lambda's scope when it visits the alias. Later
+ # uses of the name in that lambda resolve to the alias's expression, not to a select alias.
+ if self.lambda_names is not None:
+ # The write goes to this lambda's own layer, so the name stays inside this lambda.
+ self.lambda_names[node.alias] = self._resolve(node.expr)
+
def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
pass
@@ -702,11 +825,47 @@ class _FlagCallComparisonFinder(SyntaxVisitor):
self._check(node)
super().visit_call(node)
+ def _resolve(self, read: ast.Expr) -> ast.Expr:
+ read = unwrap_alias(read)
+ names = self.lambda_names
+ if names is not None and isinstance(read, ast.Field) and len(read.chain) == 1 and str(read.chain[0]) in names:
+ named = names[str(read.chain[0])]
+ return read if named is None else named
+ return self.aliases.target(read)
+
def _check(self, expr: ast.Expr) -> None:
- values = compared_values(expr, lambda read: self.reader.column(read) == ["event"])
- for value in values or []:
- if isinstance(value, ast.Constant) and value.value == FLAG_CALLED_EVENT:
- self.literals.append(value)
+ if self.values_left < 0:
+ return
+ values = compared_values(expr, lambda read: self.reader.column(self._resolve(read)) == ["event"])
+ # An alias of a list can appear many times in one comparison, so each list is read once.
+ read_lists: set[int] = set()
+ for value in map(self._resolve, values or []):
+ if isinstance(value, (ast.Tuple, ast.Array)):
+ if id(value) in read_lists:
+ continue
+ read_lists.add(id(value))
+ for item in value.exprs if isinstance(value, (ast.Tuple, ast.Array)) else [value]:
+ self.values_left -= 1
+ if self.values_left < 0:
+ return
+ named = self._resolve(item)
+ if isinstance(named, ast.Constant) and named.value == FLAG_CALLED_EVENT:
+ self.literals.append(named)
+
+
+def _has_flag_called_constant(node: ast.AST) -> bool:
+ finder = _FlagCalledConstantFinder()
+ finder.visit(node)
+ return finder.found
+
+
+class _FlagCalledConstantFinder(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.found = False
+
+ def visit_constant(self, node: ast.Constant) -> None:
+ self.found = self.found or node.value == FLAG_CALLED_EVENT
class _UnionRenamer(ScanVisitor):
@@ -722,18 +881,21 @@ class _UnionRenamer(ScanVisitor):
self.only_in_flag_evaluations = only_in_flag_evaluations
self.rename_flag_only_scans = rename_flag_only_scans
self.renamed = False
- self.refusals: list[str] = []
+ self.refusals: list[UnionRefusal] = []
self.literal_spans: set[tuple[int, int]] = set()
def visit_scan_query(self, node: ast.SelectQuery) -> None:
+ aliases = _Aliases(node)
# Only the first scan can be renamed, and it comes first, so each refusal reads the query as written.
- for found in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes):
+ for found in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes, aliases):
position, scan, literals = found.position, found.scan, found.literals
- if position == 0 and self._left_to_the_swap(node, scan):
+ if position == 0 and self._left_to_the_swap(node, scan, aliases):
continue
refusal = _union_refusal(node, scan, position)
+ if refusal is not None and found.unchecked:
+ refusal = "it compares `event` with more values than the flag-call check reads"
if refusal is not None:
- self.refusals.append(refusal)
+ self.refusals.append(UnionRefusal(join=scan.join, reason=refusal))
else:
point_scan_at(node, scan, list(EVENTS_WITH_FLAG_CALLS_CHAIN))
# A lazy table cannot be sampled. SAMPLE 1 keeps every row anyway, and the union applies a ratio
@@ -747,7 +909,7 @@ class _UnionRenamer(ScanVisitor):
if literal.start is not None and literal.end is not None
)
- def _left_to_the_swap(self, node: ast.SelectQuery, scan: EventsScan) -> bool:
+ def _left_to_the_swap(self, node: ast.SelectQuery, scan: EventsScan, aliases: _Aliases) -> bool:
# The flag_evaluations swap handles a first scan whose `event` filter it reads as only flag calls. It swaps or
# refuses that scan on mode 2. On mode 1 events holds every flag call, so the union would read the same calls
# twice there.
@@ -755,6 +917,7 @@ class _UnionRenamer(ScanVisitor):
return False
# The swap cannot read a sampled scan. On mode 2 the union reads it.
sampled = scan.join.sample is not None and not scan.join.sample.samples_all_rows()
- return scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False) and not (
- self.only_in_flag_evaluations and sampled
- )
+ conjuncts = filter_conjuncts(node)
+ if aliases.aliases:
+ conjuncts = [aliases.visit(conjunct) for conjunct in conjuncts]
+ return scan.pins_flag_calls(conjuncts, needs_flag_key=False) and not (self.only_in_flag_evaluations and sampled)
diff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.py
index 1b0501158285..4a2d55c649d9 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_union.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py
@@ -121,6 +121,71 @@ def _global_join(query: str) -> ast.SelectQuery:
return select
+def _alias_flood() -> ast.SelectQuery:
+ # Work that grows with the square of the alias count turns this case into a hang.
+ select = _select(f"SELECT 1 FROM events WHERE {MIXED}")
+ aliases = ast.Tuple(exprs=[ast.Alias(alias=f"a{n}", expr=ast.Constant(value=n)) for n in range(150_000)])
+ select.select.append(
+ ast.Call(name="arrayMap", args=[ast.Lambda(args=["x"], expr=aliases), ast.Array(exprs=[ast.Constant(value=1)])])
+ )
+ return select
+
+
+def _alias_chain_read_many_times() -> ast.SelectQuery:
+ # `event AS a0, a0 AS a1, ...` read thousands of times in one IN list.
+ select = _select("SELECT 1 FROM events")
+ select.select = [
+ ast.Alias(alias=f"a{n}", expr=ast.Field(chain=[f"a{n - 1}" if n else "event"])) for n in range(10_000)
+ ]
+ reads: list[ast.Expr] = [ast.Field(chain=["a9999"]) for _ in range(10_000)]
+ select.where = ast.CompareOperation(
+ op=ast.CompareOperationOp.In,
+ left=ast.Field(chain=["event"]),
+ right=ast.Tuple(exprs=[*reads, ast.Constant(value="$pageview"), ast.Constant(value="$feature_flag_called")]),
+ )
+ return select
+
+
+def _lambdas_inside_a_lambda_with_many_aliases() -> ast.SelectQuery:
+ select = _select(f"SELECT 1 FROM events WHERE {MIXED}")
+ body: list[ast.Expr] = [ast.Alias(alias=f"a{n}", expr=ast.Constant(value=n)) for n in range(40_000)]
+ body += [ast.Lambda(args=["y"], expr=ast.Constant(value=1)) for _ in range(40_000)]
+ select.select.append(
+ ast.Call(
+ name="arrayMap",
+ args=[ast.Lambda(args=["x"], expr=ast.Tuple(exprs=body)), ast.Array(exprs=[ast.Constant(value=1)])],
+ )
+ )
+ return select
+
+
+def _list_alias_read_many_times() -> ast.SelectQuery:
+ select = _select("SELECT 1 FROM events")
+ names = ast.Tuple(exprs=[*(ast.Constant(value=f"e{n}") for n in range(10_000)), ast.Constant(value="$pageview")])
+ select.select = [ast.Alias(alias="names", expr=names)]
+ select.where = ast.CompareOperation(
+ op=ast.CompareOperationOp.In,
+ left=ast.Field(chain=["event"]),
+ right=ast.Tuple(
+ exprs=[*(ast.Field(chain=["names"]) for _ in range(10_000)), ast.Constant(value="$feature_flag_called")]
+ ),
+ )
+ return select
+
+
+def _list_alias_in_many_comparisons() -> ast.SelectQuery:
+ select = _select(f"SELECT 1 FROM events WHERE {MIXED}")
+ names = ast.Tuple(exprs=[ast.Constant(value=f"e{n}") for n in range(10_000)])
+ comparisons: list[ast.Expr] = [
+ ast.CompareOperation(
+ op=ast.CompareOperationOp.In, left=ast.Field(chain=["event"]), right=ast.Field(chain=["names"])
+ )
+ for _ in range(10_000)
+ ]
+ select.select = [ast.Alias(alias="names", expr=names), *comparisons]
+ return select
+
+
def _set_mode(organization, mode: FlagEvaluationsMode) -> None:
OrganizationFeatureFlagsConfig.objects.update_or_create(
organization=organization, defaults={"flag_evaluations_mode": mode}
@@ -270,6 +335,143 @@ class TestRenameFlagCallScansToUnion(SimpleTestCase):
["persons", "events"],
["it joins events after another table"],
),
+ (
+ "alias_of_event_in_a_list_with_other_events",
+ "SELECT event AS name, count() FROM events "
+ "WHERE name IN ('$pageview', '$feature_flag_called') GROUP BY name",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_an_alias_of_event",
+ "SELECT event AS a, a AS b FROM events WHERE b IN ('$pageview', '$feature_flag_called')",
+ [UNION],
+ [],
+ ),
+ # Substituting each alias of a call in place would copy about 2**40 nodes, so a regression hangs here.
+ (
+ "long_chain_of_aliases_that_each_name_the_one_before_twice",
+ "SELECT event AS a0, "
+ + ", ".join(f"concat(a{n - 1}, a{n - 1}) AS a{n}" for n in range(1, 41))
+ + f" FROM events WHERE a40 = '' AND {MIXED}",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_a_comparison_with_flag_calls",
+ "SELECT event = '$feature_flag_called' AS is_flag_call, count() FROM events "
+ "WHERE is_flag_call GROUP BY is_flag_call",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_the_flag_call_event_name",
+ "SELECT '$feature_flag_called' AS name, count() FROM events WHERE event = name",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_a_comparison_in_the_filter",
+ "SELECT count() FROM events WHERE (event = '$feature_flag_called' AS is_flag_call)",
+ [UNION],
+ [],
+ ),
+ (
+ "inline_alias_of_event_pinned_to_only_flag_calls",
+ "SELECT count() FROM events WHERE (event AS name) = '$feature_flag_called'",
+ ["events"],
+ [],
+ ),
+ (
+ "alias_of_a_column_that_a_later_alias_shadows",
+ "SELECT distinct_id AS d, event AS distinct_id FROM events "
+ f"WHERE d = '$feature_flag_called' AND {MIXED}",
+ [UNION],
+ [],
+ ),
+ (
+ "lambda_argument_named_like_an_alias_of_event",
+ "SELECT e.event AS name FROM persons AS p JOIN events AS e ON p.id = e.person_id "
+ "WHERE arrayExists(name -> name = '$feature_flag_called', ['$feature_flag_called'])",
+ ["persons", "events"],
+ [],
+ ),
+ (
+ "alias_inside_a_lambda_shadowing_an_alias_of_event_after_another_table",
+ "SELECT e.event AS name, arrayMap(x -> ((x AS name), name = '$feature_flag_called'), ['$pageview']) "
+ "FROM persons AS p JOIN events AS e ON p.id = e.person_id",
+ ["persons", "events"],
+ [],
+ ),
+ ("many_aliases_inside_one_lambda", _alias_flood(), [UNION], []),
+ ("long_alias_chain_read_many_times", _alias_chain_read_many_times(), [UNION], []),
+ ("list_alias_read_many_times", _list_alias_read_many_times(), [UNION], []),
+ ("list_alias_in_many_comparisons", _list_alias_in_many_comparisons(), [UNION], []),
+ (
+ "many_lambdas_inside_a_lambda_with_many_aliases",
+ _lambdas_inside_a_lambda_with_many_aliases(),
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_event_inside_a_lambda",
+ "SELECT count() FROM events "
+ "WHERE arrayExists(x -> ((event AS ev), ev IN ('$pageview', '$feature_flag_called')).2, [1])",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_inside_a_lambda_of_an_alias_of_event",
+ "SELECT event AS e, arrayMap(x -> ((e AS ev), ev = '$feature_flag_called'), [1]) AS m FROM events",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_inside_a_lambda_named_like_an_alias_of_event",
+ "SELECT arrayMap(x -> (x AS a), [1]) AS m, event AS a FROM events "
+ "WHERE a IN ('$pageview', '$feature_flag_called')",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_event_after_an_alias_name_repeated_in_lambdas",
+ "SELECT arrayMap(x -> (x AS d), [1]) AS m, arrayMap(y -> (y AS d), [1]) AS n, event AS e FROM events "
+ "WHERE e IN ('$pageview', '$feature_flag_called')",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_event_pinned_to_only_flag_calls",
+ "SELECT event AS name FROM events WHERE name = '$feature_flag_called'",
+ ["events"],
+ [],
+ ),
+ (
+ "alias_of_event_after_another_table",
+ "SELECT e.event AS name FROM persons AS p JOIN events AS e ON p.id = e.person_id "
+ "WHERE name IN ('$pageview', '$feature_flag_called')",
+ ["persons", "events"],
+ ["it joins events after another table"],
+ ),
+ (
+ "inline_alias_of_event",
+ "SELECT count() FROM events WHERE (event AS name) IN ('$pageview', '$feature_flag_called')",
+ [UNION],
+ [],
+ ),
+ (
+ "inline_alias_of_the_flag_call_event_name",
+ "SELECT count() FROM events WHERE event IN ('$pageview', ('$feature_flag_called' AS name))",
+ [UNION],
+ [],
+ ),
+ (
+ "alias_of_a_tuple_with_flag_calls",
+ "SELECT ('$pageview', '$feature_flag_called') AS names, count() FROM events "
+ "WHERE event IN names GROUP BY names",
+ [UNION],
+ [],
+ ),
(
"cte_named_events_after_another_table",
"WITH events AS (SELECT 1 AS person_id, '$pageview' AS event) "
@@ -348,7 +550,29 @@ class TestRenameFlagCallScansToUnion(SimpleTestCase):
collector.visit(select)
assert collector.tables == expected_tables
- assert list(rename.refusals) == expected_refusals
+ assert [refusal.reason for refusal in rename.refusals] == expected_refusals
+
+ @parameterized.expand(
+ [
+ ("no_flag_call_constant", "", []),
+ (
+ "flag_call_constant_elsewhere",
+ ", '$feature_flag_called' AS name",
+ ["it compares `event` with more values than the flag-call check reads"],
+ ),
+ ]
+ )
+ def test_scan_past_the_compared_values_budget(self, _name: str, extra_select: str, expected_refusals: list[str]):
+ select = _select(
+ f"SELECT count(){extra_select} FROM persons AS p JOIN events AS e ON p.id = e.person_id "
+ "WHERE e.event IN ('$pageview', '$autocapture', '$identify')"
+ )
+ with patch("posthog.hogql.transforms.flag_calls_union._COMPARED_VALUES_BUDGET", 2):
+ rename = rename_flag_call_scans_to_union(
+ select, flag_key_compares_as_string=lambda: True, only_in_flag_evaluations=True
+ )
+
+ assert [refusal.reason for refusal in rename.refusals] == expected_refusals
def test_row_dependent_functions_cover_every_hogql_function_in_their_families(self):
families = re.compile(
@@ -443,6 +667,29 @@ class TestReadFlagCallsWithUnion(BaseTest):
mode=ONLY,
persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
),
+ param(
+ "alias_of_event_in_a_mixed_filter_on_mode_2",
+ "SELECT event AS name, count() FROM events "
+ "WHERE name IN ('$pageview', '$feature_flag_called') GROUP BY name",
+ True,
+ True,
+ mode=ONLY,
+ ),
+ param(
+ "alias_of_a_comparison_with_flag_calls_on_mode_2",
+ "SELECT event = '$feature_flag_called' AS is_flag_call, count() FROM events "
+ "WHERE is_flag_call GROUP BY is_flag_call",
+ True,
+ True,
+ mode=ONLY,
+ ),
+ param(
+ "alias_of_the_flag_call_event_name_on_mode_2",
+ "SELECT '$feature_flag_called' AS name, count() FROM events WHERE event = name",
+ True,
+ True,
+ mode=ONLY,
+ ),
# Mode 1 keeps a materialized property on events, so mode 2 shows which columns the union can read.
param(
"property_in_a_string_materialized_column_on_mode_2",
@@ -507,6 +754,18 @@ class TestReadFlagCallsWithUnion(BaseTest):
f"SELECT count() FROM events AS e RIGHT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",
"RIGHT JOIN",
),
+ (
+ "mixed_scan_after_another_table",
+ f"SELECT count() FROM persons AS p JOIN events AS e ON p.id = e.person_id WHERE {E_MIXED}",
+ "it joins events after another table",
+ ),
+ (
+ "mixed_scan_after_a_swapped_scan_of_only_flag_calls",
+ "SELECT count() FROM persons AS p JOIN events AS a ON p.id = a.person_id "
+ "JOIN events AS b ON a.uuid = b.uuid "
+ "WHERE a.event = '$feature_flag_called' AND b.event IN ('$pageview', '$feature_flag_called')",
+ "it joins events after another table",
+ ),
(
"field_flag_evaluations_lacks",
f"SELECT goe_0.properties.plan FROM events WHERE {MIXED}",
@@ -573,6 +832,43 @@ class TestReadFlagCallsWithUnion(BaseTest):
with pytest.raises(QueryError, match=reason):
self._sql(query, mode=ONLY, persons_on_events=persons_on_events, new_events_schema=new_events_schema)
+ @parameterized.expand(
+ [
+ (
+ "scan_after_another_table",
+ "SELECT count() FROM (SELECT 'x' AS id) AS d JOIN events AS e ON e.distinct_id = d.id "
+ "WHERE e.event = '$feature_flag_called'",
+ ),
+ (
+ "self_join_with_both_sides_pinned",
+ "SELECT count() FROM events AS a JOIN events AS b ON a.uuid = b.uuid "
+ "WHERE a.event = '$feature_flag_called' AND b.event = '$feature_flag_called'",
+ ),
+ (
+ "filter_on_an_alias_of_event",
+ "SELECT event AS name, count() FROM events WHERE name = '$feature_flag_called' GROUP BY name",
+ ),
+ ]
+ )
+ def test_mode_2_swaps_scans_of_only_flag_calls_as_it_does_without_the_union(self, _name: str, query: str):
+ sql = self._sql(query, mode=ONLY)
+
+ assert sql == self._sql(query, mode=ONLY, union=False)
+ assert "UNION ALL" not in sql
+ assert "flag_evaluations AS " in sql
+
+ def test_mode_2_self_join_reads_its_mixed_side_from_the_union_and_its_flag_call_side_from_flag_evaluations(
+ self,
+ ):
+ sql = self._sql(
+ "SELECT count() FROM events AS a JOIN events AS b ON a.uuid = b.uuid "
+ "WHERE a.event IN ('$pageview', '$feature_flag_called') AND b.event = '$feature_flag_called'",
+ mode=ONLY,
+ )
+
+ assert "UNION ALL" in sql
+ assert "JOIN flag_evaluations AS b" in sql
+
@parameterized.expand([("dotted", UNION), ("quoted", f"`{UNION}`")])
def test_query_cannot_name_the_union_table(self, _name: str, table: str):
with pytest.raises(QueryError, match="Unknown table `posthog.events_with_flag_calls`"):
@@ -1061,20 +1357,36 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
@parameterized.expand(
[
- (f"{mode.name.lower()}_{poe.name.lower()}", mode, poe)
- for mode in (FlagEvaluationsMode.READ_FLAG_EVALUATIONS, FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY)
- for poe in (
+ *(
+ (f"{mode.name.lower()}_{poe.name.lower()}", mode, poe)
+ for mode in (FlagEvaluationsMode.READ_FLAG_EVALUATIONS, FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY)
+ for poe in (
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ )
+ ),
+ param(
+ "filter_on_an_alias_of_event",
+ ONLY,
PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
- PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
- )
+ event="event AS name",
+ event_filter="name IN ('$pageview', '$feature_flag_called')",
+ ),
]
)
- def test_each_flag_call_comes_from_one_table(self, _name: str, mode: FlagEvaluationsMode, poe: PersonsOnEventsMode):
+ def test_each_flag_call_comes_from_one_table(
+ self,
+ _name: str,
+ mode: FlagEvaluationsMode,
+ poe: PersonsOnEventsMode,
+ event: str = "event",
+ event_filter: str = MIXED,
+ ):
response = self._run(
f"""
- SELECT distinct_id, event, properties.source, person_id, person.properties.email
+ SELECT distinct_id, {event}, properties.source, person_id, person.properties.email
FROM events
- WHERE {MIXED} AND timestamp >= now() - INTERVAL 120 DAY
+ WHERE {event_filter} AND timestamp >= now() - INTERVAL 120 DAY
ORDER BY timestamp, distinct_id
""",
mode,