:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): swap or refuse flag call scans the mode 2 swap missed (#125649)
Phil Haack0ba4754809c2
diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.pyindex 805be073f770..2f9abbc33bc5 100644--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py@@ -1,5 +1,6 @@import jsonimport uuid+from collections.abc import Callablefrom datetime import UTC, datetime, timedeltafrom posthog.test.base import (@@ -26,6 +27,7 @@ from posthog.schema import (HogQLPropertyFilter,HogQLQueryModifiers,HogQLQueryResponse,+ InlineCohortCalculation,PersonPropertyFilter,PersonsOnEventsMode,PropertyGroupsMode,@@ -39,14 +41,16 @@ from posthog.hogql.database.database import Databasefrom posthog.hogql.database.schema.flag_evaluations import add_events_list_fields_to_flag_evaluationsfrom posthog.hogql.database.schema.util.where_clause_extractor import SESSION_BUFFER_DAYSfrom posthog.hogql.errors import QueryError+from posthog.hogql.parser import parse_selectfrom posthog.hogql.property import property_to_exprfrom posthog.hogql.query import execute_hogql_queryfrom posthog.hogql.utils import is_json_verbatim+from posthog.hogql.visitor import clear_locationsfrom posthog.clickhouse.client import sync_executefrom posthog.constants import AvailableFeaturefrom posthog.hogql_queries.events_query_runner import SELECT_STAR_FROM_EVENTS_FIELDS-from posthog.models import PropertyDefinition+from posthog.models import PropertyDefinition, Teamfrom posthog.models.group.util import create_groupfrom posthog.models.property import TableColumnfrom posthog.models.utils import uuid7@@ -57,6 +61,7 @@ from products.access_control.backend.property_access_control import PropertyAccefrom products.cohorts.backend.models.cohort import Cohortfrom products.cohorts.backend.models.util import insert_static_cohortfrom products.data_modeling.backend.facade.models import DataWarehouseSavedQuery+from products.data_tools.backend.models.expression import DataWarehouseExpressionfrom products.feature_flags.backend.facade.enums import FlagEvaluationsModefrom products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig@@ -72,6 +77,40 @@ FLAG_CALL_CONJUNCTS = ()FLAG_CALLS_ON_EVENTS = f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS}"PAGEVIEW_CONJUNCTS = "event = '$pageview' AND timestamp >= now() - INTERVAL 7 DAY"+PROBE_DISTINCT_IDS = "(SELECT 'probe-distinct-id' AS id) AS d"+UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN = (+ f"SELECT count() FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id WHERE {FLAG_CALL_CONJUNCTS}"+)+++def _create_flag_calls_expression(team: Team) -> str:+ DataWarehouseExpression.objects.for_team(team.pk).create(+ team=team, table_name="events", field_name="flag_calls", expression=f"({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN})"+ )+ # The transform runs only when the query text names $feature_flag_called. The expression body is not part of that+ # text. Without this filter, the transform skips the query. The test would then pass even if record_added stopped+ # working.+ return "SELECT flag_calls FROM events WHERE event != '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY"+++def _create_flag_calls_cohort(team: Team) -> str:+ performed_flag_call = {+ "key": "$feature_flag_called",+ "type": "behavioral",+ "value": "performed_event",+ "negation": False,+ "event_type": "events",+ "time_value": 30,+ "time_interval": "day",+ }+ cohort = Cohort.objects.create(+ team=team, filters={"properties": {"type": "OR", "values": [{"type": "AND", "values": [performed_flag_call]}]}}+ )+ return (+ "SELECT count() FROM events WHERE event IN ('$feature_flag_called', '$pageview') "+ f"AND person_id IN COHORT {cohort.pk} AND timestamp >= now() - INTERVAL 7 DAY"+ )+# Compares greater than 9 as a number and less than it as a string, so the assertion tells the two# apart rather than passing under either.@@ -214,7 +253,7 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):def _query_events(self,- query: str,+ query: str | ast.SelectQuery | ast.SelectSetQuery,*,rewrite: bool = True,filters: HogQLFilters | None = None,@@ -728,6 +767,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):f"UNION ALL SELECT 'pageviews', count() FROM events WHERE {PAGEVIEW_CONJUNCTS}",[("calls", 1), ("pageviews", 1)],),+ (+ "type_cast",+ f"SELECT CAST(({FLAG_CALLS_ON_EVENTS}) AS String)",+ [("1",)],+ ),("filters_placeholder","SELECT count() FROM events WHERE event = '$feature_flag_called' "@@ -745,6 +789,85 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):assert sorted(self._query_events(query, filters=filters).results) == expected+ @parameterized.expand(+ [+ ("unqualified_filters_next_to_a_join", UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN, [(1,)]),+ (+ "query_built_without_source_positions",+ clear_locations(parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN)),+ [(1,)],+ ),+ (+ "union_of_one_select_object_twice",+ ast.SelectSetQuery.create_from_queries(+ [parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN)] * 2, "UNION ALL"+ ),+ [(1,), (1,)],+ ),+ (+ "union_of_one_parsed_template_twice",+ ast.SelectSetQuery.create_from_queries(+ [+ parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN),+ parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN),+ ],+ "UNION ALL",+ ),+ [(1,), (1,)],+ ),+ (+ "events_after_another_table",+ f"SELECT count() FROM {PROBE_DISTINCT_IDS} JOIN events AS e ON e.distinct_id = d.id "+ "WHERE e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'probe-flag' "+ "AND e.timestamp >= now() - INTERVAL 7 DAY",+ [(1,)],+ ),+ (+ "self_join_with_both_sides_pinned",+ "SELECT count() FROM events AS a JOIN events AS b ON a.uuid = b.uuid "+ "WHERE a.event = '$feature_flag_called' AND b.event = '$feature_flag_called' "+ "AND a.timestamp >= now() - INTERVAL 7 DAY",+ [(1,)],+ ),+ (+ "unqualified_prewhere_next_to_a_join",+ f"SELECT count() FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "+ "PREWHERE event = '$feature_flag_called' WHERE timestamp >= now() - INTERVAL 7 DAY",+ [(1,)],+ ),+ (+ "scan_in_a_subquery",+ f"SELECT * FROM ({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN})",+ [(1,)],+ ),+ (+ "scan_in_a_type_cast",+ f"SELECT ({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN})::String",+ [("1",)],+ ),+ (+ "scan_in_a_cte_body",+ f"WITH calls AS ({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN}) SELECT * FROM calls",+ [(1,)],+ ),+ (+ "filter_on_an_alias_of_event",+ "SELECT event AS name, count() FROM events "+ "WHERE name = '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY GROUP BY name",+ [("$feature_flag_called", 1)],+ ),+ ]+ )+ def test_scan_that_only_resolution_shows_reads_flag_calls_swaps_once_events_gets_no_flag_calls(+ self, _name: str, query: str | ast.SelectQuery | ast.SelectSetQuery, expected: list[tuple]+ ):+ mode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS+ assert (+ self._query_events(query, mode=mode).clickhouse+ == self._query_events(query, rewrite=False, mode=mode).clickhouse+ )+ assert self._query_events(query).results == expected+@parameterized.expand([("window_id", "$window_id"),@@ -799,6 +922,18 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):"a name in it would read a different column",PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,),+ (+ "select_star_next_to_a_join",+ f"SELECT e.* FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "+ f"WHERE {FLAG_CALL_CONJUNCTS}",+ "selects \\* or COLUMNS",+ ),+ (+ "posthog_events_next_to_a_join",+ f"SELECT count() FROM posthog.events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "+ f"WHERE {FLAG_CALL_CONJUNCTS}",+ "it reads events in a shape that can't be moved",+ ),])def test_query_flag_evaluations_cannot_answer_reads_events_until_events_gets_no_flag_calls(@@ -849,6 +984,17 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):"SELECT * FROM events WHERE event IN ('$feature_flag_called', '$pageview') ""AND timestamp >= now() - INTERVAL 7 DAY",),+ (+ "flag_call_filter_on_a_joined_table",+ "SELECT count() FROM events AS e "+ "JOIN (SELECT '$feature_flag_called' AS event, 'probe-distinct-id' AS id) AS d "+ "ON e.distinct_id = d.id WHERE d.event = '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY",+ ),+ (+ "flag_calls_and_other_events_next_to_a_join",+ f"SELECT count() FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "+ "WHERE event IN ('$feature_flag_called', '$pageview') AND timestamp >= now() - INTERVAL 7 DAY",+ ),])def test_query_that_does_not_read_only_flag_calls_from_events_is_unchanged_once_events_gets_no_flag_calls(@@ -856,6 +1002,23 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):):assert self._query_events(query).clickhouse == self._query_events(query, rewrite=False).clickhouse+ @parameterized.expand(+ [+ ("saved_expression", _create_flag_calls_expression),+ ("inline_cohort", _create_flag_calls_cohort),+ ]+ )+ def test_scan_that_the_resolver_adds_reads_events_once_events_gets_no_flag_calls(+ self, _name: str, create_query: Callable[[Team], str]+ ):+ query = create_query(self.team)+ modifiers = HogQLQueryModifiers(inlineCohortCalculation=InlineCohortCalculation.ALWAYS)++ assert (+ self._query_events(query, modifiers=modifiers).clickhouse+ == self._query_events(query, rewrite=False, modifiers=modifiers).clickhouse+ )+@parameterized.expand([("no_lower_bound", "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"),diff --git a/posthog/hogql/database/schema/util/where_clause_extractor.py b/posthog/hogql/database/schema/util/where_clause_extractor.pyindex 510e048815b1..56f14d9a4cab 100644--- a/posthog/hogql/database/schema/util/where_clause_extractor.py+++ b/posthog/hogql/database/schema/util/where_clause_extractor.py@@ -607,12 +607,26 @@ def _unwrap_field_alias_type(expr_type: Optional[ast.Type]) -> Optional[ast.Typereturn expr_type-def _unwrap_table_alias_type(table_type: Optional[ast.Type]) -> Optional[ast.Type]:+def unwrap_table_alias_type(table_type: Optional[ast.Type]) -> Optional[ast.Type]:while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):table_type = table_type.table_typereturn table_type+def event_field_type(expr: ast.Expr) -> ast.FieldType | None:+ """The type of the `event` column that resolved `expr` reads, or None when it reads something else."""+ # The resolver wraps a field without a table prefix in an alias when the query has several tables.+ while isinstance(expr, ast.Alias):+ expr = expr.expr+ if not isinstance(expr, ast.Field):+ return None+ # A select alias used in WHERE wraps the column in one FieldAliasType per alias.+ field_type = expr.type+ while isinstance(field_type, ast.FieldAliasType):+ field_type = field_type.type+ return field_type if isinstance(field_type, ast.FieldType) and field_type.name == "event" else None++def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableType]:"""If the expression resolves to $session_id on the events table, return its (possibly aliased) table type."""from posthog.hogql.database.schema.events import EventsTable # noqa: PLC0415 - breaks the schema import cycle@@ -639,7 +653,7 @@ def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableTyelse:return None- unwrapped = _unwrap_table_alias_type(table_type)+ unwrapped = unwrap_table_alias_type(table_type)if isinstance(unwrapped, ast.TableType) and isinstance(unwrapped.table, EventsTable):return cast(ast.BaseTableType, table_type)return None@@ -683,7 +697,7 @@ def build_session_id_literal_pushdown_predicate(if events_join is None or not isinstance(events_join.table, ast.Field):return None- events_table_type = _unwrap_table_alias_type(events_join.table.type)+ events_table_type = unwrap_table_alias_type(events_join.table.type)if events_table_type is None:return None@@ -697,7 +711,7 @@ def build_session_id_literal_pushdown_predicate(continue# only push filters owned by the events occurrence this lazy join hangs offowner = get_events_session_id_table_type(term.left)- if owner is None or _unwrap_table_alias_type(owner) is not events_table_type:+ if owner is None or unwrap_table_alias_type(owner) is not events_table_type:continueconstants = extract_uuid_constants(term.right)if not constants or len(constants) > SESSION_ID_LITERAL_PUSHDOWN_MAX_IDS:diff --git a/posthog/hogql/flag_called_warnings.py b/posthog/hogql/flag_called_warnings.pyindex d1970a4462bb..ae5b47404c6e 100644--- a/posthog/hogql/flag_called_warnings.py+++ b/posthog/hogql/flag_called_warnings.py@@ -6,6 +6,7 @@ from posthog.hogql.context import HogQLContextfrom posthog.hogql.database.models import Tablefrom posthog.hogql.database.schema.events import EventsTablefrom posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable+from posthog.hogql.database.schema.util.where_clause_extractor import event_field_type, unwrap_table_alias_typefrom posthog.hogql.resolver import resolve_typesfrom posthog.hogql.visitor import TraversingVisitor, clone_expr@@ -56,19 +57,10 @@ class _FlagCalledLiteralFinder(TraversingVisitor):def _is_event_field_of(expr: ast.Expr, table_class: type[Table]) -> bool:- while isinstance(expr, ast.Alias):- expr = expr.expr- if not isinstance(expr, ast.Field):+ field_type = event_field_type(expr)+ if field_type is None:return False- # A select alias used in WHERE wraps the column in one FieldAliasType per alias.- field_type = expr.type- while isinstance(field_type, ast.FieldAliasType):- field_type = field_type.type- if not isinstance(field_type, ast.FieldType) or field_type.name != "event":- return False- table_type = field_type.table_type- while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):- table_type = table_type.table_type+ table_type = unwrap_table_alias_type(field_type.table_type)return isinstance(table_type, ast.TableType) and isinstance(table_type.table, table_class)diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.pyindex 8647fbc7e201..9ec6f502e2c5 100644--- a/posthog/hogql/resolver.py+++ b/posthog/hogql/resolver.py@@ -269,6 +269,59 @@ def resolve_types_from_table(return resolve_types(expr, context, dialect, [scope])+class JoinSources:+ """The `JoinExpr` of the input tree that each resolved `JoinExpr` came from, and the joins that the resolver+ added itself."""++ def __init__(self) -> None:+ # Each entry keeps its key node alive, so no other node can take its id.+ self._sources: dict[int, tuple[ast.JoinExpr, ast.JoinExpr]] = {}+ self._added: dict[int, ast.JoinExpr] = {}++ def record(self, node: ast.JoinExpr, source: ast.JoinExpr) -> None:+ """Records that `node`, a resolved join or a copy that the resolver made, came from `source`."""+ self._sources[id(node)] = (node, self.source_of(source) or source)++ def record_copied_tree(self, copy: ast.AST, source: ast.AST) -> None:+ """Records that each join in `copy`, a copy of `source`, came from the join at the same place in `source`."""+ for copied, original in zip(_all_joins(copy), _all_joins(source), strict=True):+ self.record(copied, original)++ def record_added(self, tree: ast.AST) -> None:+ """Records that the resolver added each join in `tree`, such as one in a saved view's body."""+ for join in _all_joins(tree):+ self._added[id(join)] = join++ def source_of(self, node: ast.JoinExpr) -> ast.JoinExpr | None:+ entry = self._sources.get(id(node))+ return entry[1] if entry is not None else None++ def is_added(self, node: ast.JoinExpr) -> bool:+ """Whether the resolver added the join that `node` came from."""+ source = self.source_of(node)+ return source is not None and id(source) in self._added+++def _all_joins(tree: ast.AST) -> list[ast.JoinExpr]:+ collector = _JoinExprCollector()+ collector.visit(tree)+ return collector.joins+++class _JoinExprCollector(TraversingVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.joins: list[ast.JoinExpr] = []++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ self.joins.append(node)+ super().visit_join_expr(node)++ def visit_field(self, node: ast.Field) -> None:+ # The inherited visit follows the field's type. A field holds no join.+ pass++ResolverFactory = Callable[[HogQLContext, HogQLDialect, Optional[list["ast.SelectQueryType"]]],"Resolver",@@ -290,11 +343,16 @@ def resolve_types(dialect: HogQLDialect,scopes: Optional[list[ast.SelectQueryType]] = None,resolver_factory: ResolverFactory | None = None,+ join_sources: JoinSources | None = None,) -> _T_AST:+ """`node` with every type resolved. With `join_sources`, this records the `JoinExpr` that each resolved+ `JoinExpr` came from, and which joins the resolver added itself, such as one in a saved view's body or an inline+ cohort."""if resolver_factory is None:resolver = Resolver(scopes=scopes, context=context, dialect=dialect)else:resolver = resolver_factory(context, dialect, scopes)+ resolver.join_sources = join_sourcesreturn resolver.visit(node)@@ -449,6 +507,7 @@ class Resolver(CloningVisitor):self._pending_saved_view: str | None = Noneself.context = contextself.dialect = dialect+ self.join_sources: JoinSources | None = Noneself.database = context.databaseself.cte_counter = 0self._scope_table_names: dict[int, dict[str, str]] = {}@@ -473,7 +532,10 @@ class Resolver(CloningVisitor):raise ResolutionError(f"Type already resolved for {type(node).__name__} ({type(node.type).__name__}). Can't run again.")- return super().visit(node)+ resolved = super().visit(node)+ if self.join_sources is not None and isinstance(node, ast.JoinExpr):+ self.join_sources.record(resolved, node)+ return resolveddef visit_select_set_query(self, node: ast.SelectSetQuery):parent_ctes = self.ctes@@ -593,7 +655,7 @@ class Resolver(CloningVisitor):if self.dialect not in _PIVOT_DIALECTS and self.dialect != "trino":raise QueryError(f"UNPIVOT is not allowed in {self.dialect} dialect")- node = cast(ast.UnpivotExpr, clone_expr(node))+ node = self._copy(node)# Resolve the source table in an isolated scope so we can use its columns.temp_scope = ast.SelectQueryType()@@ -628,8 +690,8 @@ class Resolver(CloningVisitor):resolved_values = [resolve_unpivot_value(val) for val in col.unpivot_values]resolved_columns.append(ast.UnpivotColumn(- value_columns=clone_expr(col.value_columns),- name_columns=clone_expr(col.name_columns),+ value_columns=self._copy(col.value_columns),+ name_columns=self._copy(col.name_columns),unpivot_values=resolved_values,))@@ -780,7 +842,7 @@ class Resolver(CloningVisitor):if self.dialect not in _PIVOT_DIALECTS and self.dialect != "trino":raise QueryError(f"PIVOT is not allowed in {self.dialect} dialect")- node = cast(ast.PivotExpr, clone_expr(node))+ node = self._copy(node)# Resolve the source table in an isolated scope so we can use its columns.temp_scope = ast.SelectQueryType()@@ -860,12 +922,12 @@ class Resolver(CloningVisitor):def visit_cte(self, node: ast.CTE):self.cte_counter += 1- cte_expr = clone_expr(node.expr)+ cte_expr = self._copy(node.expr)if node.recursive and isinstance(cte_expr, ast.SelectSetQuery):# For recursive CTEs, resolve the base case first to determine column types,# then register the CTE so the recursive branch can self-reference it.- base_select = clone_expr(cte_expr.initial_select_query)+ base_select = self._copy(cte_expr.initial_select_query)base_select = self.visit(base_select)placeholder = ast.CTE(@@ -1331,7 +1393,7 @@ class Resolver(CloningVisitor):for tbl_alias, tbl_field in table_fields:field_name = str(tbl_field.chain[-1]) if isinstance(tbl_field, ast.Field) else Noneif field_name is not None and field_name in node.replace:- replacement = clone_expr(node.replace[field_name])+ replacement = self._copy(node.replace[field_name])replace_match_counts[field_name] += 1replaced_fields.append((tbl_alias, ast.Alias(expr=replacement, alias=field_name)))else:@@ -1396,6 +1458,14 @@ class Resolver(CloningVisitor):return aliased_fields+ def _copy(self, node: _T_AST, clear_types: bool = False) -> _T_AST:+ # :TRICKY: Copy input subtrees with this, not clone_expr. A join copied without it has no source in+ # join_sources, and the flag call transform then refuses the query on mode 2.+ copy = clone_expr(node, clear_types=clear_types)+ if self.join_sources is not None:+ self.join_sources.record_copied_tree(copy, node)+ return copy+def visit_join_expr(self, node: ast.JoinExpr):"""Visit each FROM and JOIN table or subquery."""@@ -1437,7 +1507,7 @@ class Resolver(CloningVisitor):# Use CTETableAliasType for aliased CTEs (e.g., FROM my_cte AS alias)node_type = ast.CTETableAliasType(alias=table_alias, cte_table_type=cte_table_type)- node = cast(ast.JoinExpr, clone_expr(node))+ node = self._copy(node)if node.constraint and node.constraint.constraint_type == "USING":# visit USING constraint before adding the table to avoid ambiguous namesnode.constraint = self.visit_join_constraint(node.constraint)@@ -1499,6 +1569,8 @@ class Resolver(CloningVisitor):self._compilation_budget.expand_view(view_query)node.table = parse_select(view_query)_mark_view_body(node.table, database_table.name)+ if self.join_sources is not None:+ self.join_sources.record_added(node.table)node.alias = table_alias or database_table.nameprevious_pending = self._pending_saved_viewself._pending_saved_view = database_table.name@@ -1553,7 +1625,7 @@ class Resolver(CloningVisitor):else:node_type = node_table_type- node = cast(ast.JoinExpr, clone_expr(node))+ node = self._copy(node)if node.constraint and node.constraint.constraint_type == "USING":# visit USING constraint before adding the table to avoid ambiguous namesnode.constraint = self.visit_join_constraint(node.constraint)@@ -1621,7 +1693,7 @@ class Resolver(CloningVisitor):elif isinstance(node.table, ast.SelectQuery) or isinstance(node.table, ast.SelectSetQuery):view_name = self._pending_saved_viewself._pending_saved_view = None- node = cast(ast.JoinExpr, clone_expr(node))+ node = self._copy(node)if node.constraint and node.constraint.constraint_type == "USING":# visit USING constraint before adding the table to avoid ambiguous namesnode.constraint = self.visit_join_constraint(node.constraint)@@ -1730,7 +1802,7 @@ class Resolver(CloningVisitor):return nodeelif isinstance(node.table, ast.ValuesQuery):- node = cast(ast.JoinExpr, clone_expr(node))+ node = self._copy(node)if node.constraint and node.constraint.constraint_type == "USING":# visit USING constraint before adding the table to avoid ambiguous namesnode.constraint = self.visit_join_constraint(node.constraint)@@ -1776,7 +1848,7 @@ class Resolver(CloningVisitor):return nodeelif isinstance(node.table, ast.UnpivotExpr):- node = cast(ast.JoinExpr, clone_expr(node))+ node = self._copy(node)if node.constraint and node.constraint.constraint_type == "USING":node.constraint = self.visit_join_constraint(node.constraint)if node.alias is None and self._join_chain_has_using(node):@@ -1803,7 +1875,7 @@ class Resolver(CloningVisitor):return nodeelif isinstance(node.table, ast.PivotExpr):- node = cast(ast.JoinExpr, clone_expr(node))+ node = self._copy(node)if node.constraint and node.constraint.constraint_type == "USING":node.constraint = self.visit_join_constraint(node.constraint)if node.alias is None and self._join_chain_has_using(node):@@ -2067,7 +2139,7 @@ class Resolver(CloningVisitor):if is_decision_call(node.name):spec = PromptJevCall.parse(node)- node = clone_expr(node, clear_types=True)+ node = self._copy(node, clear_types=True)node.args[0] = self.visit(spec.input)node.type = ast.CallType(name=node.name.lower(),@@ -2077,7 +2149,7 @@ class Resolver(CloningVisitor):return nodeif self.dialect == "trino" and node.name.lower() == "date":- node = clone_expr(node, clear_types=False)+ node = self._copy(node)node.name = "toDate"# Expand *COLUMNS(...) in function arguments@@ -2340,7 +2412,7 @@ class Resolver(CloningVisitor):self.scopes.append(node_type)- new_node = cast(ast.Lambda, clone_expr(node))+ new_node = self._copy(node)new_node.type = node_typenew_node.expr = self.visit(new_node.expr)@@ -2365,7 +2437,7 @@ class Resolver(CloningVisitor):self.scopes.append(node_type)- new_node = cast(ast.Lambda, clone_expr(node))+ new_node = self._copy(node)new_node.type = node_typenew_node.expr = self.visit(new_node.expr)@@ -2391,13 +2463,13 @@ class Resolver(CloningVisitor):def visit_try_cast(self, node: ast.TryCast):if self.dialect not in _POSTGRES_FAMILY and self.dialect != "trino":raise QueryError(f"TRY_CAST is not allowed in {self.dialect} dialect")- node = cast(ast.TryCast, clone_expr(node))+ node = self._copy(node)node.expr = self.visit(node.expr)node.type = infer_try_cast_constant_type(node.type_name, self.dialect)return nodedef visit_type_cast(self, node: ast.TypeCast):- node = cast(ast.TypeCast, clone_expr(node))+ node = self._copy(node)node.expr = self.visit(node.expr)input_type = (node.expr.type or ast.UnknownType()).resolve_constant_type(self.context)node.type = infer_cast_constant_type(node.type_name, input_type, self.dialect)@@ -2406,14 +2478,14 @@ class Resolver(CloningVisitor):def visit_positional_ref(self, node: ast.PositionalRef):if self.dialect not in _POSTGRES_FAMILY and self.dialect != "trino":raise QueryError(f"Positional references are not allowed in {self.dialect} dialect")- node = cast(ast.PositionalRef, clone_expr(node))+ node = self._copy(node)node.type = ast.UnknownType()return nodedef visit_array_slice(self, node: ast.ArraySlice):if self.dialect not in _POSTGRES_FAMILY and self.dialect not in {"clickhouse", "trino"}:raise QueryError(f"Array slices are not allowed in {self.dialect} dialect")- node = cast(ast.ArraySlice, clone_expr(node))+ node = self._copy(node)node.array = self.visit(node.array)if node.start_expr is not None:node.start_expr = self.visit(node.start_expr)@@ -2643,6 +2715,8 @@ class Resolver(CloningVisitor):# the expression so its child fields resolve before the target printer sees them.if self.dialect != "hogql":new_expr = clone_expr(node.type.expr)+ if self.join_sources is not None:+ self.join_sources.record_added(new_expr)new_node: ast.Expr = ast.Alias(alias=node.type.name, expr=new_expr, hidden=True)if node.type.isolate_scope:@@ -2795,6 +2869,12 @@ class Resolver(CloningVisitor):)return node+ def _cohort_query(self, cohort: ast.Expr) -> ast.Expr:+ query = cohort_query_node(cohort, context=self.context)+ if self.join_sources is not None:+ self.join_sources.record_added(query)+ return query+def visit_compare_operation(self, node: ast.CompareOperation):if self.context.modifiers.inCohortVia == "subquery":if node.op == ast.CompareOperationOp.InCohort:@@ -2802,7 +2882,7 @@ class Resolver(CloningVisitor):ast.CompareOperation(op=ast.CompareOperationOp.In,left=node.left,- right=cohort_query_node(node.right, context=self.context),+ right=self._cohort_query(node.right),))elif node.op == ast.CompareOperationOp.NotInCohort:@@ -2810,7 +2890,7 @@ class Resolver(CloningVisitor):ast.CompareOperation(op=ast.CompareOperationOp.NotIn,left=node.left,- right=cohort_query_node(node.right, context=self.context),+ right=self._cohort_query(node.right),))diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.pyindex fe069ec493ed..1222f3a5d277 100644--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py@@ -5,7 +5,7 @@ query's `events` scan for `posthog.flag_evaluations` when `flag_evaluations` holsaved queries keep returning flag calls without a change.The swap runs before type resolution, so every rule here is syntactic. A shape the rules cannot classify keeps-reading `events`.+reading `events`. On mode 2, a second pass finds the scans those rules miss in the resolved query."""import json@@ -13,7 +13,7 @@ from collections import Counterfrom collections.abc import Callablefrom datetime import UTC, date, datetime, time, timedelta, tzinfofrom functools import cache, partial-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Protocolfrom zoneinfo import ZoneInfo, ZoneInfoNotFoundErrorimport structlog@@ -25,10 +25,15 @@ from posthog.hogql.context import HogQLContextfrom posthog.hogql.database.models import DatabaseFieldfrom posthog.hogql.database.schema.events import EventsTablefrom posthog.hogql.database.schema.flag_evaluations import ELEMENT_STAND_INS, FlagEvaluationsTable-from posthog.hogql.database.schema.util.where_clause_extractor import top_level_conjuncts+from posthog.hogql.database.schema.util.where_clause_extractor import (+ event_field_type,+ top_level_conjuncts,+ unwrap_table_alias_type,+)from posthog.hogql.errors import BaseHogQLError, QueryErrorfrom posthog.hogql.functions.mapping import HOGQL_COMPARISON_MAPPING, find_hogql_functionfrom posthog.hogql.property_metadata import load_property_metadata+from posthog.hogql.resolver import JoinSourcesfrom posthog.hogql.utils import is_json_verbatimfrom posthog.hogql.visitor import TraversingVisitor, clone_expr@@ -50,6 +55,11 @@ _FLAG_KEY_PROPERTY = "$feature_flag"_Predicate = Callable[[ast.Expr], bool]++class _Resolve(Protocol):+ def __call__(self, node: _T_AST, *, join_sources: JoinSources | None = None) -> _T_AST: ...++_MAX_UTC_OFFSET = timedelta(hours=14)# How far before UTC midnight a date can start in some zone. A zone behind UTC can still be on the previous date, and# a zone 14 hours ahead of UTC starts that date 14 hours before UTC does.@@ -89,6 +99,8 @@ _START_OF_UNITS = {# behind UTC, that midnight comes before the local midnight that the rounding chose._DATE_UNITS = frozenset({"Week", "Month", "Quarter", "Year"})+_UNMOVABLE_SHAPE = "it reads events in a shape that can't be moved to posthog.flag_evaluations yet"+# An alias with one of these names can shadow the column in an unqualified read._SCAN_COLUMNS = frozenset({"event", "timestamp", "properties"})@@ -135,9 +147,7 @@ def can_read_flag_evaluations()-def resolve_flag_calls_from_flag_evaluations(- node: _T_AST, context: HogQLContext, resolve: Callable[[_T_AST], _T_AST]-) -> _T_AST:+def resolve_flag_calls_from_flag_evaluations(node: _T_AST, context: HogQLContext, resolve: _Resolve) -> _T_AST:"""`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading`posthog.flag_evaluations` instead.@@ -156,8 +166,12 @@ def resolve_flag_calls_from_flag_evaluations(- A query keeps reading `events` when it reads a property from a different kind of column there, such as amaterialized column. See `_swap_reads_properties_alike`.- The resolver inlines a saved view's body, so the swap never sees a scan inside one. Once `events` gets no flag- calls, a query that reads a view whose body reads only flag calls from `events` raises.+ For an organization whose flag calls are only in `flag_evaluations`, a scan that the syntactic rules miss swaps+ too when the resolved original shows that its filters pin `event` to flag calls in its WHERE or PREWHERE. The+ query raises when such a scan cannot swap.++ A scan that the resolver adds, such as one in a saved view's body or an inline cohort, keeps reading `events`.+ Once `events` gets no flag calls, a query that reads a view whose body reads only flag calls from `events` raises."""database = context.databaseif (@@ -180,7 +194,7 @@ def resolve_flag_calls_from_flag_evaluations(def _resolve_with_swaps(node: _T_AST,context: HogQLContext,- resolve: Callable[[_T_AST], _T_AST],+ resolve: _Resolve,database: "Database",only_in_flag_evaluations: bool,) -> _T_AST:@@ -190,6 +204,7 @@ def _resolve_with_swaps(return resolve(node)with context.timings.measure("flag_calls_from_flag_evaluations"):rewritten = clone_expr(node)+ clones = _join_clones(node, rewritten) if only_in_flag_evaluations else {}swapper = _EventsScanSwapper(now=datetime.now(UTC),timezone=ZoneInfo(database.get_timezone()),@@ -202,7 +217,13 @@ def _resolve_with_swaps(if only_in_flag_evaluations:raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0])logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=swapper.refusals[0])- original = resolve(node)+ sources = JoinSources() if only_in_flag_evaluations else None+ original = resolve(node, join_sources=sources)+ if sources is not None:+ with context.timings.measure("flag_calls_from_flag_evaluations_by_type"):+ refusal = swapper.swap_scans_found_by_type(original, sources, clones)+ if refusal is not None:+ raise _flag_calls_only_in_flag_evaluations_error(refusal)if not swapper.swapped:return originaltry:@@ -323,17 +344,8 @@ class EventsScan:def pins_flag_calls(self, conjuncts: list[ast.Expr], needs_flag_key: bool) -> bool:"""Whether `conjuncts` pin `event` to flag calls. With `needs_flag_key`, a pin of `event` counts only where- these conjuncts also pin `$feature_flag`. The recursion into `OR` branches passes False once an enclosing- conjunct pins `$feature_flag`."""- needs_flag_key = needs_flag_key and not self.flag_keys(conjuncts)- return any(self._pins_flag_calls_in(conjunct, needs_flag_key) for conjunct in conjuncts)-- def _pins_flag_calls_in(self, expr: ast.Expr, needs_flag_key: bool) -> bool:- if _pinned_values(expr, self._reads_event, _is_flag_called) is not None:- return not needs_flag_key- if branches := boolean_operands(expr, ast.Or):- return all(self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in branches)- return False+ these conjuncts also pin `$feature_flag`."""+ return _pins_flag_calls(conjuncts, self._reads_event, self.flag_keys if needs_flag_key else None)def _flag_keys_in(self, expr: ast.Expr) -> list[str] | None:"""The keys `expr` pins `$feature_flag` to, or None when it does not pin it."""@@ -403,6 +415,28 @@ class EventsScan:return self.column(expr) == ["timestamp"]+def _pins_flag_calls(+ conjuncts: list[ast.Expr], reads_event: _Predicate, flag_keys: Callable[[list[ast.Expr]], list[str]] | None+) -> bool:+ """Whether `conjuncts` pin what `reads_event` matches to flag calls: by `=` or `IN` of only+ `$feature_flag_called`, or by an `OR` whose every branch pins. With `flag_keys`, a pin counts only where these+ conjuncts also pin `$feature_flag`. The recursion into `OR` branches drops `flag_keys` once an enclosing+ conjunct pins `$feature_flag`."""+ if flag_keys is not None and flag_keys(conjuncts):+ flag_keys = None+ return any(_pins_flag_calls_in(conjunct, reads_event, flag_keys) for conjunct in conjuncts)+++def _pins_flag_calls_in(+ expr: ast.Expr, reads_event: _Predicate, flag_keys: Callable[[list[ast.Expr]], list[str]] | None+) -> bool:+ if _pinned_values(expr, reads_event, _is_flag_called) is not None:+ return flag_keys is None+ if branches := boolean_operands(expr, ast.Or):+ return all(_pins_flag_calls(_conjuncts(branch), reads_event, flag_keys) for branch in branches)+ return False++def comparison_of(expr: ast.Expr) -> ast.CompareOperation | None:"""`expr` as a comparison, whether it is written `a >= b` or `greaterOrEquals(a, b)`."""if isinstance(expr, ast.CompareOperation):@@ -791,6 +825,106 @@ class _EventsScanSwapper(ScanVisitor):point_scan_at(node, scan, ["posthog", "flag_evaluations"], flag_key)self.swapped = True+ def swap_scans_found_by_type(self, resolved: ast.AST, sources: JoinSources, clones: "_JoinClones") -> str | None:+ """Swap each events scan in the clone whose counterpart in `resolved`, the resolved original, has a WHERE or+ PREWHERE that pins `event` to flag calls, such as a scan joined after another table. `sources` holds the+ original join of each resolved join, and `clones` holds the clones of each original join. Returns why one of+ them cannot swap, or None.++ A scan that the resolver adds, such as one in a saved view's body or an inline cohort, keeps reading events.+ Any other scan without a clone cannot swap.+ """+ for select, join in _joins(resolved):+ if not _reads_events_table(join) or sources.is_added(join):+ continue+ if not _pins_flag_calls(filter_conjuncts(select), partial(_reads_event_of, join.type), None):+ continue+ source = sources.source_of(join)+ source_clones = clones.get(id(source), []) if source is not None else []+ # The pin check above reads the filters of one SELECT. A join that several SELECTs share cannot swap.+ if not source_clones or len({id(clone.original_select) for clone in source_clones}) > 1:+ return _UNMOVABLE_SHAPE+ # A query can hold one node in several places, such as one SELECT in two branches of a UNION. Cloning+ # copies each place apart, so one original join can have several clones, and each one has to swap.+ for clone in source_clones:+ refusal = self._swap_clone(clone)+ if refusal is not None:+ return refusal+ return None++ def _swap_clone(self, clone: "_JoinClone") -> str | None:+ chain = _table_chain(clone.join)+ if chain == ["posthog", "flag_evaluations"]:+ # The syntactic rules, or an earlier resolved join from the same source, already swapped it.+ return None+ if chain != ["events"]:+ return _UNMOVABLE_SHAPE+ events_scan = EventsScan(+ join=clone.join,+ alias=clone.join.alias or "events",+ # The clone is unresolved, so only a name qualified with this scan's alias can count as a read of it.+ has_other_sources=True,+ flag_key_compares_as_string=self.flag_key_compares_as_string,+ )+ refusal = events_scan.shape_refusal(clone.select)+ if refusal is not None:+ return refusal+ self._swap(clone.select, events_scan, filter_conjuncts(clone.select))+ return None+++@frozen+class _JoinClone:+ # Keeps the original join alive, so no other node can take the id that keys it.+ original: ast.JoinExpr+ original_select: ast.SelectQuery+ select: ast.SelectQuery+ join: ast.JoinExpr+++_JoinClones = dict[int, list[_JoinClone]]+++def _join_clones(original: ast.AST, clone: ast.AST) -> _JoinClones:+ """Each join of `original`, with the `SelectQuery` and `JoinExpr` in each of its places in `clone`, a copy of it."""+ clones: _JoinClones = {}+ for (original_select, join), (clone_select, clone_join) in zip(_joins(original), _joins(clone), strict=True):+ clones.setdefault(id(join), []).append(+ _JoinClone(original=join, original_select=original_select, select=clone_select, join=clone_join)+ )+ return clones+++def _table_chain(join: ast.JoinExpr) -> list[str | int] | None:+ return join.table.chain if isinstance(join.table, ast.Field) else None+++def _joins(node: ast.AST) -> list[tuple[ast.SelectQuery, ast.JoinExpr]]:+ collector = _JoinCollector()+ collector.visit(node)+ return collector.joins+++class _JoinCollector(SyntaxVisitor):+ """Each `JoinExpr` in a FROM chain, with the `SelectQuery` that owns it."""++ def __init__(self) -> None:+ super().__init__()+ self.joins: list[tuple[ast.SelectQuery, ast.JoinExpr]] = []++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ join = node.select_from+ while join is not None:+ self.joins.append((node, join))+ join = join.next_join+ super().visit_select_query(node)+++def _reads_event_of(table_type: ast.Type | None, expr: ast.Expr) -> bool:+ """Whether resolved `expr` reads the `event` column of the scan whose type is `table_type`."""+ field_type = event_field_type(expr)+ return field_type is not None and field_type.table_type is table_type+def _swap_keeps_bindings(original: ast.AST, swapped: ast.AST, flag_key_predicates: int) -> bool:"""Whether every field and table in the resolved swapped query binds as it does in the resolved original.@@ -1015,7 +1149,5 @@ def _reads_only_flag_calls_from_events(node: ast.SelectQuery) -> bool:def _reads_events_table(join: ast.JoinExpr) -> bool:"""Whether a resolved FROM reads the events table, rather than a CTE named `events`."""- table_type = join.type- while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):- table_type = table_type.table_type+ table_type = unwrap_table_alias_type(join.type)return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)diff --git a/posthog/models/test/test_data_deletion_request.py b/posthog/models/test/test_data_deletion_request.pyindex a39db70a252d..67116ebcae9b 100644--- a/posthog/models/test/test_data_deletion_request.py+++ b/posthog/models/test/test_data_deletion_request.py@@ -207,17 +207,22 @@ def test_hogql_predicate_subquery_is_team_scoped(db):assert f"equals(team_id, {team_a.id})" not in sql_b-def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team):+ "predicate",+ [+ "distinct_id IN (SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY)",+ "distinct_id IN (SELECT e.distinct_id FROM events AS e JOIN (SELECT 'probe' AS id) AS d "+ "ON e.distinct_id = d.id WHERE event = '$feature_flag_called')",+ ],+)+def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, predicate):from posthog.models.data_deletion_request import compile_hogql_predicateOrganizationFeatureFlagsConfig.objects.update_or_create(organization=team.organization,defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},)- predicate = (- "distinct_id IN (SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "- "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY)"- )request = DataDeletionRequest(**_base_kwargs(team_id=team.id, events=["$pageview"], hogql_predicate=predicate))with patch(