Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): keep pushdowns on the flag calls union scan (#126660)

Phil Haackb6b751d1acb8
diff --git a/posthog/hogql/database/schema/util/where_clause_extractor.py b/posthog/hogql/database/schema/util/where_clause_extractor.py
index ad3cb2ec2697..c34fc93824d3 100644
--- a/posthog/hogql/database/schema/util/where_clause_extractor.py
+++ b/posthog/hogql/database/schema/util/where_clause_extractor.py
@@ -578,10 +578,8 @@ class _ReferencesLazyJoinVisitor(TraversingVisitor):
def is_events_only_field(node: ast.Field) -> bool:
- """True iff the field resolves to a direct field on the EventsTable (not a traversal to
- person/groups/sessions/etc. via a virtual/lazy join)."""
- from posthog.hogql.database.schema.events import EventsTable
-
+ """True iff the field resolves to a direct field on the EventsTable or the events_with_flag_calls union (not a
+ traversal to person/groups/sessions/etc. via a virtual/lazy join)."""
type_ = node.type
if isinstance(type_, ast.PropertyType):
type_ = type_.field_type
@@ -590,15 +588,9 @@ def is_events_only_field(node: ast.Field) -> bool:
if not isinstance(type_, ast.FieldType):
return False
- table_type = type_.table_type
- while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
- table_type = table_type.table_type
# Virtual tables (e.g. person, group) pivot to another underlying table, so they are not
# safely "events-only" even if they hang off an events row.
-
- if isinstance(table_type, ast.TableType):
- return isinstance(table_type.table, EventsTable)
- return False
+ return is_events_scan(type_.table_type)
def _unwrap_field_alias_type(expr_type: Optional[ast.Type]) -> Optional[ast.Type]:
@@ -613,6 +605,19 @@ def unwrap_table_alias_type(table_type: Optional[ast.Type]) -> Optional[ast.Type
return table_type
+def is_events_scan(table_type: Optional[ast.Type]) -> bool:
+ """True when `table_type` reads events or the events_with_flag_calls union, through table aliases."""
+ from posthog.hogql.database.schema.events import EventsTable # noqa: PLC0415 - breaks the schema import cycle
+ from posthog.hogql.database.schema.events_with_flag_calls import ( # noqa: PLC0415 - breaks the schema import cycle
+ EventsWithFlagCallsTable,
+ )
+
+ unwrapped = unwrap_table_alias_type(table_type)
+ return isinstance(unwrapped, (ast.TableType, ast.LazyTableType)) and isinstance(
+ unwrapped.table, (EventsTable, EventsWithFlagCallsTable)
+ )
+
+
def event_field_type(expr: ast.Expr) -> ast.FieldType | None:
"""The type of the `event` column that resolved `expr` reads, or None when it reads something else."""
# The resolver wraps a field without a table prefix in an alias when the query has several tables.
@@ -629,11 +634,6 @@ def event_field_type(expr: ast.Expr) -> ast.FieldType | None:
def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableType]:
"""If the expression resolves to $session_id on the events table, return its (possibly aliased) table type."""
- from posthog.hogql.database.schema.events import EventsTable # noqa: PLC0415 - breaks the schema import cycle
- from posthog.hogql.database.schema.events_with_flag_calls import ( # noqa: PLC0415 - breaks the schema import cycle
- EventsWithFlagCallsTable,
- )
-
expr_type = _unwrap_field_alias_type(node.type)
if isinstance(expr_type, ast.FieldType) and expr_type.name == "$session_id":
@@ -656,10 +656,7 @@ def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableTy
else:
return None
- unwrapped = unwrap_table_alias_type(table_type)
- if isinstance(unwrapped, (ast.TableType, ast.LazyTableType)) and isinstance(
- unwrapped.table, (EventsTable, EventsWithFlagCallsTable)
- ):
+ if is_events_scan(table_type):
return cast(ast.BaseTableType, table_type)
return None
@@ -754,7 +751,7 @@ def build_session_id_v7_pushdown_predicate(
Returns None if the outer query shape or WHERE can't be safely pushed down.
"""
- events_join = _find_join_for_alias(outer_node.select_from, join_to_add.from_table)
+ events_join = _events_scan_for_join(outer_node, join_to_add)
if events_join is None or not isinstance(events_join.table, ast.Field):
return None
diff --git a/posthog/hogql/transforms/events_predicate_pushdown.py b/posthog/hogql/transforms/events_predicate_pushdown.py
index 93a644f2c726..9fe90771344e 100644
--- a/posthog/hogql/transforms/events_predicate_pushdown.py
+++ b/posthog/hogql/transforms/events_predicate_pushdown.py
@@ -43,7 +43,13 @@ from posthog.hogql.constants import LimitContext, get_max_limit_for_context
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import DatabaseField
from posthog.hogql.database.schema.events import EventsTable
-from posthog.hogql.database.schema.util.where_clause_extractor import EventsPredicatePushdownExtractor
+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable
+from posthog.hogql.database.schema.util.where_clause_extractor import (
+ EventsPredicatePushdownExtractor,
+ contains_in_cohort,
+ contains_row_multiplying_function,
+ is_nondeterministic_function,
+)
from posthog.hogql.functions.mapping import find_hogql_aggregation
from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr
@@ -352,12 +358,60 @@ class _ShortCircuitBlockerFinder(TraversingVisitor):
super().visit_call(node)
+class _UnionScanWhereMover(CloningVisitor):
+ """Copies the WHERE of a query that reads a union scan into the scan itself.
+
+ Each read becomes the scan's own expression for that column. Sets `failed` for anything else, so the caller leaves
+ the WHERE where it is.
+ """
+
+ def __init__(self, scan: ast.SelectQuery, scan_alias: str):
+ super().__init__(clear_types=False)
+ self.columns = {column.alias: column.expr for column in scan.select if isinstance(column, ast.Alias)}
+ self.scan_alias = scan_alias
+ self.failed = False
+
+ def visit_field(self, node: ast.Field):
+ field_type = node.type
+ while isinstance(field_type, ast.FieldAliasType):
+ field_type = field_type.type
+ if (
+ isinstance(field_type, ast.FieldType)
+ and isinstance(field_type.table_type, ast.SelectQueryAliasType)
+ and field_type.table_type.alias == self.scan_alias
+ and field_type.name in self.columns
+ ):
+ return clone_expr(self.columns[field_type.name], clear_types=False)
+ self.failed = True
+ return node
+
+ def visit_alias(self, node: ast.Alias):
+ # The resolver wraps a bare column in a hidden alias.
+ if not node.hidden:
+ self.failed = True
+ return self.visit(node.expr)
+
+ def visit_call(self, node: ast.Call):
+ if is_nondeterministic_function(node.name) or find_hogql_aggregation(node.name) is not None:
+ self.failed = True
+ return super().visit_call(node)
+
+ def _refuse(self, node):
+ self.failed = True
+ return node
+
+ visit_select_query = visit_select_set_query = visit_lambda = visit_property_access = _refuse
+
+
class EventsPredicatePushdownTransform(TraversingVisitor):
"""Pushes events WHERE/PREWHERE predicates into a pre-filtering subquery:
`FROM events` -> `FROM (SELECT <needed cols> FROM events WHERE <predicates>) AS events`.
Runs after logical lowering (so property reads are PropertyAccess) and before the ClickHouse physical passes,
- applying bottom-up so nested `FROM events` subqueries benefit too."""
+ applying bottom-up so nested `FROM events` subqueries benefit too.
+
+ A FROM of the expanded `events_with_flag_calls` union already is a subquery, so the WHERE and the LIMIT move into
+ it instead."""
def __init__(
self,
@@ -399,6 +453,10 @@ class EventsPredicatePushdownTransform(TraversingVisitor):
if not joined_aliases:
return "no_safe_joined_aliases"
+ union_scan = self._union_scan(node.select_from)
+ if union_scan is not None:
+ return self._push_into_union_scan(node, union_scan)
+
events_table_type = node.select_from.type
if not isinstance(events_table_type, (ast.TableType, ast.TableAliasType)):
return "from_not_table_type"
@@ -491,14 +549,14 @@ class EventsPredicatePushdownTransform(TraversingVisitor):
return None
def _should_apply_pushdown(self, node: ast.SelectQuery) -> bool:
- """Eligible when the query selects FROM events directly (no SAMPLE), has a WHERE/PREWHERE, has joins,
- has an effective LIMIT, and does not aggregate / DISTINCT / window (those read the whole set regardless
- of the LIMIT, so the pre-filter would be pure overhead)."""
+ """Eligible when the query selects FROM events or FROM the expanded events_with_flag_calls union (no SAMPLE),
+ has a WHERE/PREWHERE, has joins, has an effective LIMIT, and does not aggregate / DISTINCT / window (those read
+ the whole set regardless of the LIMIT, so the pre-filter would be pure overhead)."""
return (
events_pushdown_enabled(self.context.modifiers)
and node.select_from is not None
and node.select_from.sample is None
- and self._from_is_events_table(node.select_from)
+ and (self._from_is_events_table(node.select_from) or self._union_scan(node.select_from) is not None)
and (node.where is not None or node.prewhere is not None)
and node.select_from.next_join is not None
and self._has_effective_limit(node)
@@ -591,6 +649,88 @@ class EventsPredicatePushdownTransform(TraversingVisitor):
table_type = table_type.table_type
return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)
+ def _union_scan(self, join_expr: ast.JoinExpr) -> ast.SelectQuery | None:
+ """The select that stands for `FROM posthog.events_with_flag_calls`, or None for any other FROM.
+
+ Lazy table resolution expands that table to `SELECT <columns> FROM (<events> UNION ALL ... <flag_evaluations>)`
+ and the query reads the result under the scan's alias. The select has no filter or limit of its own, so a
+ filter and a limit that the query would apply to its rows can apply to the select instead.
+ """
+ scan = join_expr.table
+ if not isinstance(join_expr.type, ast.SelectQueryAliasType) or not isinstance(scan, ast.SelectQuery):
+ return None
+ source = scan.select_from
+ union = source.table if source is not None else None
+ if (
+ source is None
+ or not isinstance(union, ast.SelectSetQuery)
+ or source.next_join is not None
+ or any(
+ value is not None
+ for value in (
+ scan.where,
+ scan.prewhere,
+ scan.having,
+ scan.qualify,
+ scan.group_by,
+ scan.order_by,
+ scan.limit,
+ scan.offset,
+ scan.limit_by,
+ scan.array_join_op,
+ scan.distinct,
+ scan.ctes,
+ scan.window_exprs,
+ )
+ )
+ or not all(isinstance(column, ast.Alias) and isinstance(column.expr, ast.Field) for column in scan.select)
+ or any(node.set_operator != "UNION ALL" for node in union.subsequent_select_queries)
+ ):
+ return None
+ for branch in union.select_queries():
+ table_type = branch.select_from.type if isinstance(branch, ast.SelectQuery) and branch.select_from else None
+ while isinstance(table_type, ast.TableAliasType):
+ table_type = table_type.table_type
+ if not (
+ isinstance(table_type, ast.TableType)
+ and isinstance(table_type.table, (EventsTable, FlagEvaluationsTable))
+ ):
+ return None
+ return scan
+
+ def _push_into_union_scan(self, node: ast.SelectQuery, scan: ast.SelectQuery) -> str | None:
+ """Moves the WHERE and the LIMIT of `node` into the union scan, so its rows are capped before the joins.
+
+ Applies when `_safe_inner_limit` allows an inner LIMIT and the whole WHERE reads only the scan's columns, so
+ no residual predicate is left on the joined rows.
+ """
+ assert node.select_from is not None and isinstance(node.select_from.type, ast.SelectQueryAliasType)
+ if node.where is None or node.prewhere is not None:
+ return "no_pushable_predicates"
+ if contains_in_cohort(node.where) or contains_row_multiplying_function(node.where):
+ return "where_has_cohort_or_row_multiplying_call"
+ # An arrayJoin outside the scan drops each scanned row whose array is empty, so the outer query could
+ # return fewer rows than the LIMIT.
+ join_constraints: list[ast.Expr] = []
+ join = node.select_from.next_join
+ while join is not None:
+ if join.constraint is not None:
+ join_constraints.append(join.constraint.expr)
+ join = join.next_join
+ if any(contains_row_multiplying_function(expr) for expr in (*node.select, *join_constraints)):
+ return "row_multiplying_outer_expression"
+ inner_limit = self._safe_inner_limit(node, None, None)
+ if inner_limit is None:
+ return "no_short_circuitable_limit"
+ mover = _UnionScanWhereMover(scan, node.select_from.type.alias)
+ where = mover.visit(node.where)
+ if mover.failed:
+ return "where_not_on_union_scan_columns"
+ scan.where = cast(ast.Expr, where)
+ scan.limit = inner_limit
+ node.where = None
+ return None
+
def _collect_joined_aliases(self, node: ast.SelectQuery) -> set[str]:
"""Aliases from the JOIN chain, or an empty set if any join uses an unsafe type (causing the caller to bail)."""
aliases: set[str] = set()
diff --git a/posthog/hogql/transforms/negated_person_filters.py b/posthog/hogql/transforms/negated_person_filters.py
index 2709d79c1d48..8f4e1369acf0 100644
--- a/posthog/hogql/transforms/negated_person_filters.py
+++ b/posthog/hogql/transforms/negated_person_filters.py
@@ -29,6 +29,7 @@ from posthog.hogql.base import _T_AST
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.lazy_join_tags import PERSONS
from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.database.schema.events_with_flag_calls import EventsWithFlagCallsTable
from posthog.hogql.database.schema.persons import persons_join_is_inner
from posthog.hogql.database.schema.util.where_clause_extractor import top_level_conjuncts
from posthog.hogql.resolver_utils import get_long_table_name
@@ -149,10 +150,13 @@ def _is_string_column(column: "MaterializedColumn | None") -> bool:
return column is not None and column.type in ("String", "Nullable(String)")
-def _is_events(table_type: ast.Type | None) -> TypeGuard[ast.TableType | ast.TableAliasType]:
+def _is_events(table_type: ast.Type | None) -> TypeGuard[ast.TableType | ast.LazyTableType | ast.TableAliasType]:
if isinstance(table_type, ast.TableAliasType):
table_type = table_type.table_type
- return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)
+ # The union of events and flag_evaluations has the events fields, and its `person_id` is the join key.
+ return isinstance(table_type, (ast.TableType, ast.LazyTableType)) and isinstance(
+ table_type.table, (EventsTable, EventsWithFlagCallsTable)
+ )
def _exclusion(person_key: list[str | int], filters: list[ast.Expr]) -> ast.Expr:
diff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.py
index bb5cb6af9402..69eca14b26bb 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_union.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py
@@ -1,7 +1,7 @@
import re
import uuid
from datetime import UTC, datetime, timedelta
-from typing import cast
+from typing import Any, cast
import pytest
from posthog.test.base import (
@@ -42,6 +42,7 @@ from posthog.test.persons import create_group_type_mapping
from products.access_control.backend.models.property_access_control import PropertyAccessControl
from products.access_control.backend.property_access_control import PropertyAccessLevel
+from products.cohorts.backend.models.cohort import Cohort
from products.data_tools.backend.models.join import DataWarehouseJoin
from products.event_definitions.backend.models.property_definition import PropertyDefinition
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
@@ -55,6 +56,7 @@ E_MIXED = "e.event IN ('$pageview', '$feature_flag_called')"
UNION = "posthog.events_with_flag_calls"
READ = FlagEvaluationsMode.READ_FLAG_EVALUATIONS
ONLY = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY
+UNION_EXPANSION = "FROM flag_evaluations"
class _TableCollector(TraversingVisitor):
@@ -636,6 +638,143 @@ class TestReadFlagCallsWithUnion(BaseTest):
assert "GLOBAL IN" in sql or "globalIn" in sql
assert "probe-distinct-id" in persons
+ @parameterized.expand(
+ [
+ (
+ "session_id_pushdown_selects_the_scans_sessions",
+ f"SELECT session.$session_duration FROM events WHERE {MIXED}",
+ PersonsOnEventsMode.DISABLED,
+ {"sessionIdPushdown": True},
+ "globalIn(raw_sessions.session_id_v7, (SELECT DISTINCT",
+ None,
+ None,
+ ),
+ (
+ "negated_person_filter_becomes_an_exclusion",
+ f"SELECT count() FROM events WHERE {MIXED} AND person.properties.mykey != 'x'",
+ PersonsOnEventsMode.DISABLED,
+ {"negatedPersonFiltersNotIn": True},
+ "notIn(events__pdi.person_id, (SELECT persons.id",
+ "ifNull(notEquals(events__pdi__person.",
+ None,
+ ),
+ (
+ "limit_caps_the_scan_before_the_left_join",
+ f"SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} "
+ "LIMIT 10",
+ PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ {},
+ "LIMIT 10) AS e LEFT JOIN",
+ "WHERE in(e.event",
+ "WHERE in(event, tuple('$pageview', '$feature_flag_called')) LIMIT 10) AS e LEFT JOIN",
+ ),
+ (
+ "json_extract_of_a_materialized_key_reads_its_column",
+ f"SELECT JSONExtractString(properties, 'mykey') FROM events WHERE {MIXED}",
+ PersonsOnEventsMode.DISABLED,
+ {},
+ "events.mat_mykey",
+ "events.properties, 'mykey'",
+ None,
+ ),
+ ]
+ )
+ def test_union_scan_keeps_the_pushdown_that_events_gets(
+ self,
+ _name: str,
+ query: str,
+ persons_on_events: PersonsOnEventsMode,
+ modifiers: dict[str, Any],
+ pushed: str,
+ not_pushed: str | None,
+ union_pushed: str | None,
+ ):
+ self.addCleanup(cleanup_materialized_columns)
+ materialize("events", "mykey")
+ materialize("person", "mykey")
+
+ for union in (False, True):
+ sql = self._sql(
+ query, union=union, persons_on_events=persons_on_events, modifiers=HogQLQueryModifiers(**modifiers)
+ )
+
+ assert (UNION_EXPANSION in sql) == union
+ assert pushed in sql
+ assert not_pushed is None or not_pushed not in sql
+ assert not union or union_pushed is None or union_pushed in sql
+ if union and "globalIn(raw_sessions.session_id_v7" in pushed:
+ sessions_subquery = sql[
+ sql.index("globalIn(raw_sessions.session_id_v7") : sql.index("AS events__session")
+ ]
+ assert UNION_EXPANSION in sessions_subquery
+
+ @parameterized.expand(
+ [
+ (
+ "order_by",
+ f"SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} ORDER BY e.timestamp LIMIT 10",
+ ),
+ (
+ "inner_join",
+ f"SELECT e.event, p.id FROM events AS e INNER JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} LIMIT 10",
+ ),
+ (
+ "filter_on_a_joined_table",
+ f"SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} AND p.id != e.person_id LIMIT 10",
+ ),
+ (
+ "filter_on_a_property",
+ f"SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} AND e.properties.mykey = 'a' LIMIT 10",
+ ),
+ (
+ "row_multiplying_filter",
+ f"SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} AND arrayJoin(['a', 'b']) = 'a' LIMIT 10",
+ ),
+ (
+ "nondeterministic_filter",
+ f"SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} AND rand() % 2 = 0 LIMIT 10",
+ ),
+ (
+ "subquery_filter",
+ f"SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} AND e.person_id IN (SELECT id FROM persons) LIMIT 10",
+ ),
+ (
+ "row_multiplying_select",
+ f"SELECT e.event, arrayJoin(JSONExtractArrayRaw(e.properties, 'items')) AS item, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED} LIMIT 10",
+ ),
+ ]
+ )
+ def test_union_scan_keeps_its_filter_after_the_limit_when_the_limit_would_change_rows(self, _name: str, query: str):
+ sql = self._sql(query, persons_on_events=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS)
+
+ assert UNION_EXPANSION in sql
+ assert "LIMIT 10) AS e" not in sql
+
+ def test_union_scan_keeps_a_cohort_filter_outside_the_scan(self):
+ cohort = Cohort.objects.create(team=self.team, name="static", is_static=True)
+ query = (
+ "SELECT e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id "
+ f"WHERE {E_MIXED} AND e.person_id IN COHORT {cohort.pk} LIMIT 10"
+ )
+
+ sql = self._sql(
+ query,
+ persons_on_events=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ modifiers=HogQLQueryModifiers(inCohortVia="leftjoin"),
+ )
+
+ assert UNION_EXPANSION in sql
+ assert "LIMIT 10) AS e" not in sql
+
+ def test_union_scan_pushdown_ignores_a_user_written_aggregating_subquery(self):
+ sql = self._sql(
+ "SELECT s.n FROM (SELECT count() AS n FROM (SELECT event FROM events UNION ALL SELECT event FROM events)) "
+ "AS s LEFT JOIN persons AS p ON p.id = s.n WHERE s.n > 0 LIMIT 10",
+ persons_on_events=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ )
+
+ assert "WHERE greater(s.n, 0) LIMIT 10" in sql
+
def test_persons_join_runs_one_override_join(self):
sql = self._sql(
f"SELECT person_id, person.properties.email FROM events WHERE {MIXED}",
@@ -727,6 +866,7 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
query: str | ast.SelectQuery,
mode: FlagEvaluationsMode,
persons_on_events: PersonsOnEventsMode | None = None,
+ **modifiers: Any,
):
_set_mode(self.organization, mode)
with _enabled_flags(union=True):
@@ -734,7 +874,9 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
query,
team=self.team,
modifiers=HogQLQueryModifiers(
- personsOnEventsMode=persons_on_events, useNewEventsSchema=_events_schema_for(mode, None)
+ personsOnEventsMode=persons_on_events,
+ useNewEventsSchema=_events_schema_for(mode, None),
+ **modifiers,
),
)
@@ -779,6 +921,54 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
assert rows[:4] == [*history, *recent]
assert [row[:3] for row in rows[4:]] == keyless
+ @parameterized.expand(
+ [
+ (
+ "person_id_pushdown",
+ f"SELECT distinct_id, event, person.properties.email FROM events WHERE {MIXED} ORDER BY timestamp, distinct_id",
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ "personIdPushdown",
+ ),
+ (
+ "negated_person_filters",
+ f"SELECT distinct_id, event FROM events WHERE {MIXED} AND person.properties.email != '[email protected]' "
+ "ORDER BY timestamp, distinct_id",
+ PersonsOnEventsMode.DISABLED,
+ "negatedPersonFiltersNotIn",
+ ),
+ (
+ "negated_person_filters_on_the_override_join",
+ f"SELECT distinct_id, event FROM events WHERE {MIXED} AND person.properties.email != '[email protected]' "
+ "ORDER BY timestamp, distinct_id",
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ "negatedPersonFiltersNotIn",
+ ),
+ (
+ "session_id_pushdown",
+ f"SELECT distinct_id, event, session.$session_duration FROM events WHERE {MIXED} "
+ "ORDER BY timestamp, distinct_id",
+ PersonsOnEventsMode.DISABLED,
+ "sessionIdPushdown",
+ ),
+ (
+ "limit_before_the_left_join",
+ f"SELECT e.distinct_id, e.event, p.id FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id "
+ f"WHERE {E_MIXED} AND e.timestamp >= now() - INTERVAL 120 DAY LIMIT 100",
+ PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ "pushDownPredicates",
+ ),
+ ]
+ )
+ def test_pushdown_does_not_change_the_rows(self, _name: str, query: str, poe: PersonsOnEventsMode, modifier: str):
+ self.addCleanup(cleanup_materialized_columns)
+ materialize("person", "email")
+
+ pushed = self._run(query, READ, poe, **{modifier: True}).results
+ flat = self._run(query, READ, poe, **{modifier: False}).results
+
+ assert len(pushed) >= 2
+ assert sorted(map(str, pushed)) == sorted(map(str, flat))
+
@parameterized.expand(
[
(f"{kind.name.lower()}_{mode.name.lower()}", kind, mode)