Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): swap or refuse flag call scans the mode 2 swap missed (#125649)

Phil Haack0ba4754809c2
diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.py
index 805be073f770..2f9abbc33bc5 100644
--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py
+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py
@@ -1,5 +1,6 @@
import json
import uuid
+from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from posthog.test.base import (
@@ -26,6 +27,7 @@ from posthog.schema import (
HogQLPropertyFilter,
HogQLQueryModifiers,
HogQLQueryResponse,
+ InlineCohortCalculation,
PersonPropertyFilter,
PersonsOnEventsMode,
PropertyGroupsMode,
@@ -39,14 +41,16 @@ from posthog.hogql.database.database import Database
from posthog.hogql.database.schema.flag_evaluations import add_events_list_fields_to_flag_evaluations
from posthog.hogql.database.schema.util.where_clause_extractor import SESSION_BUFFER_DAYS
from posthog.hogql.errors import QueryError
+from posthog.hogql.parser import parse_select
from posthog.hogql.property import property_to_expr
from posthog.hogql.query import execute_hogql_query
from posthog.hogql.utils import is_json_verbatim
+from posthog.hogql.visitor import clear_locations
from posthog.clickhouse.client import sync_execute
from posthog.constants import AvailableFeature
from posthog.hogql_queries.events_query_runner import SELECT_STAR_FROM_EVENTS_FIELDS
-from posthog.models import PropertyDefinition
+from posthog.models import PropertyDefinition, Team
from posthog.models.group.util import create_group
from posthog.models.property import TableColumn
from posthog.models.utils import uuid7
@@ -57,6 +61,7 @@ from products.access_control.backend.property_access_control import PropertyAcce
from products.cohorts.backend.models.cohort import Cohort
from products.cohorts.backend.models.util import insert_static_cohort
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.data_tools.backend.models.expression import DataWarehouseExpression
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
@@ -72,6 +77,40 @@ FLAG_CALL_CONJUNCTS = (
)
FLAG_CALLS_ON_EVENTS = f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS}"
PAGEVIEW_CONJUNCTS = "event = '$pageview' AND timestamp >= now() - INTERVAL 7 DAY"
+PROBE_DISTINCT_IDS = "(SELECT 'probe-distinct-id' AS id) AS d"
+UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN = (
+ f"SELECT count() FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id WHERE {FLAG_CALL_CONJUNCTS}"
+)
+
+
+def _create_flag_calls_expression(team: Team) -> str:
+ DataWarehouseExpression.objects.for_team(team.pk).create(
+ team=team, table_name="events", field_name="flag_calls", expression=f"({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN})"
+ )
+ # The transform runs only when the query text names $feature_flag_called. The expression body is not part of that
+ # text. Without this filter, the transform skips the query. The test would then pass even if record_added stopped
+ # working.
+ return "SELECT flag_calls FROM events WHERE event != '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY"
+
+
+def _create_flag_calls_cohort(team: Team) -> str:
+ performed_flag_call = {
+ "key": "$feature_flag_called",
+ "type": "behavioral",
+ "value": "performed_event",
+ "negation": False,
+ "event_type": "events",
+ "time_value": 30,
+ "time_interval": "day",
+ }
+ cohort = Cohort.objects.create(
+ team=team, filters={"properties": {"type": "OR", "values": [{"type": "AND", "values": [performed_flag_call]}]}}
+ )
+ return (
+ "SELECT count() FROM events WHERE event IN ('$feature_flag_called', '$pageview') "
+ f"AND person_id IN COHORT {cohort.pk} AND timestamp >= now() - INTERVAL 7 DAY"
+ )
+
# Compares greater than 9 as a number and less than it as a string, so the assertion tells the two
# apart rather than passing under either.
@@ -214,7 +253,7 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
def _query_events(
self,
- query: str,
+ query: str | ast.SelectQuery | ast.SelectSetQuery,
*,
rewrite: bool = True,
filters: HogQLFilters | None = None,
@@ -728,6 +767,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
f"UNION ALL SELECT 'pageviews', count() FROM events WHERE {PAGEVIEW_CONJUNCTS}",
[("calls", 1), ("pageviews", 1)],
),
+ (
+ "type_cast",
+ f"SELECT CAST(({FLAG_CALLS_ON_EVENTS}) AS String)",
+ [("1",)],
+ ),
(
"filters_placeholder",
"SELECT count() FROM events WHERE event = '$feature_flag_called' "
@@ -745,6 +789,85 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
assert sorted(self._query_events(query, filters=filters).results) == expected
+ @parameterized.expand(
+ [
+ ("unqualified_filters_next_to_a_join", UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN, [(1,)]),
+ (
+ "query_built_without_source_positions",
+ clear_locations(parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN)),
+ [(1,)],
+ ),
+ (
+ "union_of_one_select_object_twice",
+ ast.SelectSetQuery.create_from_queries(
+ [parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN)] * 2, "UNION ALL"
+ ),
+ [(1,), (1,)],
+ ),
+ (
+ "union_of_one_parsed_template_twice",
+ ast.SelectSetQuery.create_from_queries(
+ [
+ parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN),
+ parse_select(UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN),
+ ],
+ "UNION ALL",
+ ),
+ [(1,), (1,)],
+ ),
+ (
+ "events_after_another_table",
+ f"SELECT count() FROM {PROBE_DISTINCT_IDS} JOIN events AS e ON e.distinct_id = d.id "
+ "WHERE e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'probe-flag' "
+ "AND e.timestamp >= now() - INTERVAL 7 DAY",
+ [(1,)],
+ ),
+ (
+ "self_join_with_both_sides_pinned",
+ "SELECT count() FROM events AS a JOIN events AS b ON a.uuid = b.uuid "
+ "WHERE a.event = '$feature_flag_called' AND b.event = '$feature_flag_called' "
+ "AND a.timestamp >= now() - INTERVAL 7 DAY",
+ [(1,)],
+ ),
+ (
+ "unqualified_prewhere_next_to_a_join",
+ f"SELECT count() FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "
+ "PREWHERE event = '$feature_flag_called' WHERE timestamp >= now() - INTERVAL 7 DAY",
+ [(1,)],
+ ),
+ (
+ "scan_in_a_subquery",
+ f"SELECT * FROM ({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN})",
+ [(1,)],
+ ),
+ (
+ "scan_in_a_type_cast",
+ f"SELECT ({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN})::String",
+ [("1",)],
+ ),
+ (
+ "scan_in_a_cte_body",
+ f"WITH calls AS ({UNQUALIFIED_FLAG_CALLS_NEXT_TO_A_JOIN}) SELECT * FROM calls",
+ [(1,)],
+ ),
+ (
+ "filter_on_an_alias_of_event",
+ "SELECT event AS name, count() FROM events "
+ "WHERE name = '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY GROUP BY name",
+ [("$feature_flag_called", 1)],
+ ),
+ ]
+ )
+ def test_scan_that_only_resolution_shows_reads_flag_calls_swaps_once_events_gets_no_flag_calls(
+ self, _name: str, query: str | ast.SelectQuery | ast.SelectSetQuery, expected: list[tuple]
+ ):
+ mode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS
+ assert (
+ self._query_events(query, mode=mode).clickhouse
+ == self._query_events(query, rewrite=False, mode=mode).clickhouse
+ )
+ assert self._query_events(query).results == expected
+
@parameterized.expand(
[
("window_id", "$window_id"),
@@ -799,6 +922,18 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
"a name in it would read a different column",
PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
),
+ (
+ "select_star_next_to_a_join",
+ f"SELECT e.* FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "
+ f"WHERE {FLAG_CALL_CONJUNCTS}",
+ "selects \\* or COLUMNS",
+ ),
+ (
+ "posthog_events_next_to_a_join",
+ f"SELECT count() FROM posthog.events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "
+ f"WHERE {FLAG_CALL_CONJUNCTS}",
+ "it reads events in a shape that can't be moved",
+ ),
]
)
def test_query_flag_evaluations_cannot_answer_reads_events_until_events_gets_no_flag_calls(
@@ -849,6 +984,17 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
"SELECT * FROM events WHERE event IN ('$feature_flag_called', '$pageview') "
"AND timestamp >= now() - INTERVAL 7 DAY",
),
+ (
+ "flag_call_filter_on_a_joined_table",
+ "SELECT count() FROM events AS e "
+ "JOIN (SELECT '$feature_flag_called' AS event, 'probe-distinct-id' AS id) AS d "
+ "ON e.distinct_id = d.id WHERE d.event = '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ (
+ "flag_calls_and_other_events_next_to_a_join",
+ f"SELECT count() FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "
+ "WHERE event IN ('$feature_flag_called', '$pageview') AND timestamp >= now() - INTERVAL 7 DAY",
+ ),
]
)
def test_query_that_does_not_read_only_flag_calls_from_events_is_unchanged_once_events_gets_no_flag_calls(
@@ -856,6 +1002,23 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
):
assert self._query_events(query).clickhouse == self._query_events(query, rewrite=False).clickhouse
+ @parameterized.expand(
+ [
+ ("saved_expression", _create_flag_calls_expression),
+ ("inline_cohort", _create_flag_calls_cohort),
+ ]
+ )
+ def test_scan_that_the_resolver_adds_reads_events_once_events_gets_no_flag_calls(
+ self, _name: str, create_query: Callable[[Team], str]
+ ):
+ query = create_query(self.team)
+ modifiers = HogQLQueryModifiers(inlineCohortCalculation=InlineCohortCalculation.ALWAYS)
+
+ assert (
+ self._query_events(query, modifiers=modifiers).clickhouse
+ == self._query_events(query, rewrite=False, modifiers=modifiers).clickhouse
+ )
+
@parameterized.expand(
[
("no_lower_bound", "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"),
diff --git a/posthog/hogql/database/schema/util/where_clause_extractor.py b/posthog/hogql/database/schema/util/where_clause_extractor.py
index 510e048815b1..56f14d9a4cab 100644
--- a/posthog/hogql/database/schema/util/where_clause_extractor.py
+++ b/posthog/hogql/database/schema/util/where_clause_extractor.py
@@ -607,12 +607,26 @@ def _unwrap_field_alias_type(expr_type: Optional[ast.Type]) -> Optional[ast.Type
return expr_type
-def _unwrap_table_alias_type(table_type: Optional[ast.Type]) -> Optional[ast.Type]:
+def unwrap_table_alias_type(table_type: Optional[ast.Type]) -> Optional[ast.Type]:
while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
table_type = table_type.table_type
return table_type
+def event_field_type(expr: ast.Expr) -> ast.FieldType | None:
+ """The type of the `event` column that resolved `expr` reads, or None when it reads something else."""
+ # The resolver wraps a field without a table prefix in an alias when the query has several tables.
+ while isinstance(expr, ast.Alias):
+ expr = expr.expr
+ if not isinstance(expr, ast.Field):
+ return None
+ # A select alias used in WHERE wraps the column in one FieldAliasType per alias.
+ field_type = expr.type
+ while isinstance(field_type, ast.FieldAliasType):
+ field_type = field_type.type
+ return field_type if isinstance(field_type, ast.FieldType) and field_type.name == "event" else None
+
+
def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableType]:
"""If the expression resolves to $session_id on the events table, return its (possibly aliased) table type."""
from posthog.hogql.database.schema.events import EventsTable # noqa: PLC0415 - breaks the schema import cycle
@@ -639,7 +653,7 @@ def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableTy
else:
return None
- unwrapped = _unwrap_table_alias_type(table_type)
+ unwrapped = unwrap_table_alias_type(table_type)
if isinstance(unwrapped, ast.TableType) and isinstance(unwrapped.table, EventsTable):
return cast(ast.BaseTableType, table_type)
return None
@@ -683,7 +697,7 @@ def build_session_id_literal_pushdown_predicate(
if events_join is None or not isinstance(events_join.table, ast.Field):
return None
- events_table_type = _unwrap_table_alias_type(events_join.table.type)
+ events_table_type = unwrap_table_alias_type(events_join.table.type)
if events_table_type is None:
return None
@@ -697,7 +711,7 @@ def build_session_id_literal_pushdown_predicate(
continue
# only push filters owned by the events occurrence this lazy join hangs off
owner = get_events_session_id_table_type(term.left)
- if owner is None or _unwrap_table_alias_type(owner) is not events_table_type:
+ if owner is None or unwrap_table_alias_type(owner) is not events_table_type:
continue
constants = extract_uuid_constants(term.right)
if not constants or len(constants) > SESSION_ID_LITERAL_PUSHDOWN_MAX_IDS:
diff --git a/posthog/hogql/flag_called_warnings.py b/posthog/hogql/flag_called_warnings.py
index d1970a4462bb..ae5b47404c6e 100644
--- a/posthog/hogql/flag_called_warnings.py
+++ b/posthog/hogql/flag_called_warnings.py
@@ -6,6 +6,7 @@ from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import Table
from posthog.hogql.database.schema.events import EventsTable
from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable
+from posthog.hogql.database.schema.util.where_clause_extractor import event_field_type, unwrap_table_alias_type
from posthog.hogql.resolver import resolve_types
from posthog.hogql.visitor import TraversingVisitor, clone_expr
@@ -56,19 +57,10 @@ class _FlagCalledLiteralFinder(TraversingVisitor):
def _is_event_field_of(expr: ast.Expr, table_class: type[Table]) -> bool:
- while isinstance(expr, ast.Alias):
- expr = expr.expr
- if not isinstance(expr, ast.Field):
+ field_type = event_field_type(expr)
+ if field_type is None:
return False
- # A select alias used in WHERE wraps the column in one FieldAliasType per alias.
- field_type = expr.type
- while isinstance(field_type, ast.FieldAliasType):
- field_type = field_type.type
- if not isinstance(field_type, ast.FieldType) or field_type.name != "event":
- return False
- table_type = field_type.table_type
- while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
- table_type = table_type.table_type
+ table_type = unwrap_table_alias_type(field_type.table_type)
return isinstance(table_type, ast.TableType) and isinstance(table_type.table, table_class)
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py
index 8647fbc7e201..9ec6f502e2c5 100644
--- a/posthog/hogql/resolver.py
+++ b/posthog/hogql/resolver.py
@@ -269,6 +269,59 @@ def resolve_types_from_table(
return resolve_types(expr, context, dialect, [scope])
+class JoinSources:
+ """The `JoinExpr` of the input tree that each resolved `JoinExpr` came from, and the joins that the resolver
+ added itself."""
+
+ def __init__(self) -> None:
+ # Each entry keeps its key node alive, so no other node can take its id.
+ self._sources: dict[int, tuple[ast.JoinExpr, ast.JoinExpr]] = {}
+ self._added: dict[int, ast.JoinExpr] = {}
+
+ def record(self, node: ast.JoinExpr, source: ast.JoinExpr) -> None:
+ """Records that `node`, a resolved join or a copy that the resolver made, came from `source`."""
+ self._sources[id(node)] = (node, self.source_of(source) or source)
+
+ def record_copied_tree(self, copy: ast.AST, source: ast.AST) -> None:
+ """Records that each join in `copy`, a copy of `source`, came from the join at the same place in `source`."""
+ for copied, original in zip(_all_joins(copy), _all_joins(source), strict=True):
+ self.record(copied, original)
+
+ def record_added(self, tree: ast.AST) -> None:
+ """Records that the resolver added each join in `tree`, such as one in a saved view's body."""
+ for join in _all_joins(tree):
+ self._added[id(join)] = join
+
+ def source_of(self, node: ast.JoinExpr) -> ast.JoinExpr | None:
+ entry = self._sources.get(id(node))
+ return entry[1] if entry is not None else None
+
+ def is_added(self, node: ast.JoinExpr) -> bool:
+ """Whether the resolver added the join that `node` came from."""
+ source = self.source_of(node)
+ return source is not None and id(source) in self._added
+
+
+def _all_joins(tree: ast.AST) -> list[ast.JoinExpr]:
+ collector = _JoinExprCollector()
+ collector.visit(tree)
+ return collector.joins
+
+
+class _JoinExprCollector(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.joins: list[ast.JoinExpr] = []
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ self.joins.append(node)
+ super().visit_join_expr(node)
+
+ def visit_field(self, node: ast.Field) -> None:
+ # The inherited visit follows the field's type. A field holds no join.
+ pass
+
+
ResolverFactory = Callable[
[HogQLContext, HogQLDialect, Optional[list["ast.SelectQueryType"]]],
"Resolver",
@@ -290,11 +343,16 @@ def resolve_types(
dialect: HogQLDialect,
scopes: Optional[list[ast.SelectQueryType]] = None,
resolver_factory: ResolverFactory | None = None,
+ join_sources: JoinSources | None = None,
) -> _T_AST:
+ """`node` with every type resolved. With `join_sources`, this records the `JoinExpr` that each resolved
+ `JoinExpr` came from, and which joins the resolver added itself, such as one in a saved view's body or an inline
+ cohort."""
if resolver_factory is None:
resolver = Resolver(scopes=scopes, context=context, dialect=dialect)
else:
resolver = resolver_factory(context, dialect, scopes)
+ resolver.join_sources = join_sources
return resolver.visit(node)
@@ -449,6 +507,7 @@ class Resolver(CloningVisitor):
self._pending_saved_view: str | None = None
self.context = context
self.dialect = dialect
+ self.join_sources: JoinSources | None = None
self.database = context.database
self.cte_counter = 0
self._scope_table_names: dict[int, dict[str, str]] = {}
@@ -473,7 +532,10 @@ class Resolver(CloningVisitor):
raise ResolutionError(
f"Type already resolved for {type(node).__name__} ({type(node.type).__name__}). Can't run again."
)
- return super().visit(node)
+ resolved = super().visit(node)
+ if self.join_sources is not None and isinstance(node, ast.JoinExpr):
+ self.join_sources.record(resolved, node)
+ return resolved
def visit_select_set_query(self, node: ast.SelectSetQuery):
parent_ctes = self.ctes
@@ -593,7 +655,7 @@ class Resolver(CloningVisitor):
if self.dialect not in _PIVOT_DIALECTS and self.dialect != "trino":
raise QueryError(f"UNPIVOT is not allowed in {self.dialect} dialect")
- node = cast(ast.UnpivotExpr, clone_expr(node))
+ node = self._copy(node)
# Resolve the source table in an isolated scope so we can use its columns.
temp_scope = ast.SelectQueryType()
@@ -628,8 +690,8 @@ class Resolver(CloningVisitor):
resolved_values = [resolve_unpivot_value(val) for val in col.unpivot_values]
resolved_columns.append(
ast.UnpivotColumn(
- value_columns=clone_expr(col.value_columns),
- name_columns=clone_expr(col.name_columns),
+ value_columns=self._copy(col.value_columns),
+ name_columns=self._copy(col.name_columns),
unpivot_values=resolved_values,
)
)
@@ -780,7 +842,7 @@ class Resolver(CloningVisitor):
if self.dialect not in _PIVOT_DIALECTS and self.dialect != "trino":
raise QueryError(f"PIVOT is not allowed in {self.dialect} dialect")
- node = cast(ast.PivotExpr, clone_expr(node))
+ node = self._copy(node)
# Resolve the source table in an isolated scope so we can use its columns.
temp_scope = ast.SelectQueryType()
@@ -860,12 +922,12 @@ class Resolver(CloningVisitor):
def visit_cte(self, node: ast.CTE):
self.cte_counter += 1
- cte_expr = clone_expr(node.expr)
+ cte_expr = self._copy(node.expr)
if node.recursive and isinstance(cte_expr, ast.SelectSetQuery):
# For recursive CTEs, resolve the base case first to determine column types,
# then register the CTE so the recursive branch can self-reference it.
- base_select = clone_expr(cte_expr.initial_select_query)
+ base_select = self._copy(cte_expr.initial_select_query)
base_select = self.visit(base_select)
placeholder = ast.CTE(
@@ -1331,7 +1393,7 @@ class Resolver(CloningVisitor):
for tbl_alias, tbl_field in table_fields:
field_name = str(tbl_field.chain[-1]) if isinstance(tbl_field, ast.Field) else None
if field_name is not None and field_name in node.replace:
- replacement = clone_expr(node.replace[field_name])
+ replacement = self._copy(node.replace[field_name])
replace_match_counts[field_name] += 1
replaced_fields.append((tbl_alias, ast.Alias(expr=replacement, alias=field_name)))
else:
@@ -1396,6 +1458,14 @@ class Resolver(CloningVisitor):
return aliased_fields
+ def _copy(self, node: _T_AST, clear_types: bool = False) -> _T_AST:
+ # :TRICKY: Copy input subtrees with this, not clone_expr. A join copied without it has no source in
+ # join_sources, and the flag call transform then refuses the query on mode 2.
+ copy = clone_expr(node, clear_types=clear_types)
+ if self.join_sources is not None:
+ self.join_sources.record_copied_tree(copy, node)
+ return copy
+
def visit_join_expr(self, node: ast.JoinExpr):
"""Visit each FROM and JOIN table or subquery."""
@@ -1437,7 +1507,7 @@ class Resolver(CloningVisitor):
# Use CTETableAliasType for aliased CTEs (e.g., FROM my_cte AS alias)
node_type = ast.CTETableAliasType(alias=table_alias, cte_table_type=cte_table_type)
- node = cast(ast.JoinExpr, clone_expr(node))
+ node = self._copy(node)
if node.constraint and node.constraint.constraint_type == "USING":
# visit USING constraint before adding the table to avoid ambiguous names
node.constraint = self.visit_join_constraint(node.constraint)
@@ -1499,6 +1569,8 @@ class Resolver(CloningVisitor):
self._compilation_budget.expand_view(view_query)
node.table = parse_select(view_query)
_mark_view_body(node.table, database_table.name)
+ if self.join_sources is not None:
+ self.join_sources.record_added(node.table)
node.alias = table_alias or database_table.name
previous_pending = self._pending_saved_view
self._pending_saved_view = database_table.name
@@ -1553,7 +1625,7 @@ class Resolver(CloningVisitor):
else:
node_type = node_table_type
- node = cast(ast.JoinExpr, clone_expr(node))
+ node = self._copy(node)
if node.constraint and node.constraint.constraint_type == "USING":
# visit USING constraint before adding the table to avoid ambiguous names
node.constraint = self.visit_join_constraint(node.constraint)
@@ -1621,7 +1693,7 @@ class Resolver(CloningVisitor):
elif isinstance(node.table, ast.SelectQuery) or isinstance(node.table, ast.SelectSetQuery):
view_name = self._pending_saved_view
self._pending_saved_view = None
- node = cast(ast.JoinExpr, clone_expr(node))
+ node = self._copy(node)
if node.constraint and node.constraint.constraint_type == "USING":
# visit USING constraint before adding the table to avoid ambiguous names
node.constraint = self.visit_join_constraint(node.constraint)
@@ -1730,7 +1802,7 @@ class Resolver(CloningVisitor):
return node
elif isinstance(node.table, ast.ValuesQuery):
- node = cast(ast.JoinExpr, clone_expr(node))
+ node = self._copy(node)
if node.constraint and node.constraint.constraint_type == "USING":
# visit USING constraint before adding the table to avoid ambiguous names
node.constraint = self.visit_join_constraint(node.constraint)
@@ -1776,7 +1848,7 @@ class Resolver(CloningVisitor):
return node
elif isinstance(node.table, ast.UnpivotExpr):
- node = cast(ast.JoinExpr, clone_expr(node))
+ node = self._copy(node)
if node.constraint and node.constraint.constraint_type == "USING":
node.constraint = self.visit_join_constraint(node.constraint)
if node.alias is None and self._join_chain_has_using(node):
@@ -1803,7 +1875,7 @@ class Resolver(CloningVisitor):
return node
elif isinstance(node.table, ast.PivotExpr):
- node = cast(ast.JoinExpr, clone_expr(node))
+ node = self._copy(node)
if node.constraint and node.constraint.constraint_type == "USING":
node.constraint = self.visit_join_constraint(node.constraint)
if node.alias is None and self._join_chain_has_using(node):
@@ -2067,7 +2139,7 @@ class Resolver(CloningVisitor):
if is_decision_call(node.name):
spec = PromptJevCall.parse(node)
- node = clone_expr(node, clear_types=True)
+ node = self._copy(node, clear_types=True)
node.args[0] = self.visit(spec.input)
node.type = ast.CallType(
name=node.name.lower(),
@@ -2077,7 +2149,7 @@ class Resolver(CloningVisitor):
return node
if self.dialect == "trino" and node.name.lower() == "date":
- node = clone_expr(node, clear_types=False)
+ node = self._copy(node)
node.name = "toDate"
# Expand *COLUMNS(...) in function arguments
@@ -2340,7 +2412,7 @@ class Resolver(CloningVisitor):
self.scopes.append(node_type)
- new_node = cast(ast.Lambda, clone_expr(node))
+ new_node = self._copy(node)
new_node.type = node_type
new_node.expr = self.visit(new_node.expr)
@@ -2365,7 +2437,7 @@ class Resolver(CloningVisitor):
self.scopes.append(node_type)
- new_node = cast(ast.Lambda, clone_expr(node))
+ new_node = self._copy(node)
new_node.type = node_type
new_node.expr = self.visit(new_node.expr)
@@ -2391,13 +2463,13 @@ class Resolver(CloningVisitor):
def visit_try_cast(self, node: ast.TryCast):
if self.dialect not in _POSTGRES_FAMILY and self.dialect != "trino":
raise QueryError(f"TRY_CAST is not allowed in {self.dialect} dialect")
- node = cast(ast.TryCast, clone_expr(node))
+ node = self._copy(node)
node.expr = self.visit(node.expr)
node.type = infer_try_cast_constant_type(node.type_name, self.dialect)
return node
def visit_type_cast(self, node: ast.TypeCast):
- node = cast(ast.TypeCast, clone_expr(node))
+ node = self._copy(node)
node.expr = self.visit(node.expr)
input_type = (node.expr.type or ast.UnknownType()).resolve_constant_type(self.context)
node.type = infer_cast_constant_type(node.type_name, input_type, self.dialect)
@@ -2406,14 +2478,14 @@ class Resolver(CloningVisitor):
def visit_positional_ref(self, node: ast.PositionalRef):
if self.dialect not in _POSTGRES_FAMILY and self.dialect != "trino":
raise QueryError(f"Positional references are not allowed in {self.dialect} dialect")
- node = cast(ast.PositionalRef, clone_expr(node))
+ node = self._copy(node)
node.type = ast.UnknownType()
return node
def visit_array_slice(self, node: ast.ArraySlice):
if self.dialect not in _POSTGRES_FAMILY and self.dialect not in {"clickhouse", "trino"}:
raise QueryError(f"Array slices are not allowed in {self.dialect} dialect")
- node = cast(ast.ArraySlice, clone_expr(node))
+ node = self._copy(node)
node.array = self.visit(node.array)
if node.start_expr is not None:
node.start_expr = self.visit(node.start_expr)
@@ -2643,6 +2715,8 @@ class Resolver(CloningVisitor):
# the expression so its child fields resolve before the target printer sees them.
if self.dialect != "hogql":
new_expr = clone_expr(node.type.expr)
+ if self.join_sources is not None:
+ self.join_sources.record_added(new_expr)
new_node: ast.Expr = ast.Alias(alias=node.type.name, expr=new_expr, hidden=True)
if node.type.isolate_scope:
@@ -2795,6 +2869,12 @@ class Resolver(CloningVisitor):
)
return node
+ def _cohort_query(self, cohort: ast.Expr) -> ast.Expr:
+ query = cohort_query_node(cohort, context=self.context)
+ if self.join_sources is not None:
+ self.join_sources.record_added(query)
+ return query
+
def visit_compare_operation(self, node: ast.CompareOperation):
if self.context.modifiers.inCohortVia == "subquery":
if node.op == ast.CompareOperationOp.InCohort:
@@ -2802,7 +2882,7 @@ class Resolver(CloningVisitor):
ast.CompareOperation(
op=ast.CompareOperationOp.In,
left=node.left,
- right=cohort_query_node(node.right, context=self.context),
+ right=self._cohort_query(node.right),
)
)
elif node.op == ast.CompareOperationOp.NotInCohort:
@@ -2810,7 +2890,7 @@ class Resolver(CloningVisitor):
ast.CompareOperation(
op=ast.CompareOperationOp.NotIn,
left=node.left,
- right=cohort_query_node(node.right, context=self.context),
+ right=self._cohort_query(node.right),
)
)
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
index fe069ec493ed..1222f3a5d277 100644
--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -5,7 +5,7 @@ query's `events` scan for `posthog.flag_evaluations` when `flag_evaluations` hol
saved queries keep returning flag calls without a change.
The swap runs before type resolution, so every rule here is syntactic. A shape the rules cannot classify keeps
-reading `events`.
+reading `events`. On mode 2, a second pass finds the scans those rules miss in the resolved query.
"""
import json
@@ -13,7 +13,7 @@ from collections import Counter
from collections.abc import Callable
from datetime import UTC, date, datetime, time, timedelta, tzinfo
from functools import cache, partial
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Protocol
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import structlog
@@ -25,10 +25,15 @@ from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import DatabaseField
from posthog.hogql.database.schema.events import EventsTable
from posthog.hogql.database.schema.flag_evaluations import ELEMENT_STAND_INS, FlagEvaluationsTable
-from posthog.hogql.database.schema.util.where_clause_extractor import top_level_conjuncts
+from posthog.hogql.database.schema.util.where_clause_extractor import (
+ event_field_type,
+ top_level_conjuncts,
+ unwrap_table_alias_type,
+)
from posthog.hogql.errors import BaseHogQLError, QueryError
from posthog.hogql.functions.mapping import HOGQL_COMPARISON_MAPPING, find_hogql_function
from posthog.hogql.property_metadata import load_property_metadata
+from posthog.hogql.resolver import JoinSources
from posthog.hogql.utils import is_json_verbatim
from posthog.hogql.visitor import TraversingVisitor, clone_expr
@@ -50,6 +55,11 @@ _FLAG_KEY_PROPERTY = "$feature_flag"
_Predicate = Callable[[ast.Expr], bool]
+
+class _Resolve(Protocol):
+ def __call__(self, node: _T_AST, *, join_sources: JoinSources | None = None) -> _T_AST: ...
+
+
_MAX_UTC_OFFSET = timedelta(hours=14)
# How far before UTC midnight a date can start in some zone. A zone behind UTC can still be on the previous date, and
# a zone 14 hours ahead of UTC starts that date 14 hours before UTC does.
@@ -89,6 +99,8 @@ _START_OF_UNITS = {
# behind UTC, that midnight comes before the local midnight that the rounding chose.
_DATE_UNITS = frozenset({"Week", "Month", "Quarter", "Year"})
+_UNMOVABLE_SHAPE = "it reads events in a shape that can't be moved to posthog.flag_evaluations yet"
+
# An alias with one of these names can shadow the column in an unqualified read.
_SCAN_COLUMNS = frozenset({"event", "timestamp", "properties"})
@@ -135,9 +147,7 @@ def can_read_flag_evaluations(
)
-def resolve_flag_calls_from_flag_evaluations(
- node: _T_AST, context: HogQLContext, resolve: Callable[[_T_AST], _T_AST]
-) -> _T_AST:
+def resolve_flag_calls_from_flag_evaluations(node: _T_AST, context: HogQLContext, resolve: _Resolve) -> _T_AST:
"""`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading
`posthog.flag_evaluations` instead.
@@ -156,8 +166,12 @@ def resolve_flag_calls_from_flag_evaluations(
- A query keeps reading `events` when it reads a property from a different kind of column there, such as a
materialized column. See `_swap_reads_properties_alike`.
- The resolver inlines a saved view's body, so the swap never sees a scan inside one. Once `events` gets no flag
- calls, a query that reads a view whose body reads only flag calls from `events` raises.
+ For an organization whose flag calls are only in `flag_evaluations`, a scan that the syntactic rules miss swaps
+ too when the resolved original shows that its filters pin `event` to flag calls in its WHERE or PREWHERE. The
+ query raises when such a scan cannot swap.
+
+ A scan that the resolver adds, such as one in a saved view's body or an inline cohort, keeps reading `events`.
+ Once `events` gets no flag calls, a query that reads a view whose body reads only flag calls from `events` raises.
"""
database = context.database
if (
@@ -180,7 +194,7 @@ def resolve_flag_calls_from_flag_evaluations(
def _resolve_with_swaps(
node: _T_AST,
context: HogQLContext,
- resolve: Callable[[_T_AST], _T_AST],
+ resolve: _Resolve,
database: "Database",
only_in_flag_evaluations: bool,
) -> _T_AST:
@@ -190,6 +204,7 @@ def _resolve_with_swaps(
return resolve(node)
with context.timings.measure("flag_calls_from_flag_evaluations"):
rewritten = clone_expr(node)
+ clones = _join_clones(node, rewritten) if only_in_flag_evaluations else {}
swapper = _EventsScanSwapper(
now=datetime.now(UTC),
timezone=ZoneInfo(database.get_timezone()),
@@ -202,7 +217,13 @@ def _resolve_with_swaps(
if only_in_flag_evaluations:
raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0])
logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=swapper.refusals[0])
- original = resolve(node)
+ sources = JoinSources() if only_in_flag_evaluations else None
+ original = resolve(node, join_sources=sources)
+ if sources is not None:
+ with context.timings.measure("flag_calls_from_flag_evaluations_by_type"):
+ refusal = swapper.swap_scans_found_by_type(original, sources, clones)
+ if refusal is not None:
+ raise _flag_calls_only_in_flag_evaluations_error(refusal)
if not swapper.swapped:
return original
try:
@@ -323,17 +344,8 @@ class EventsScan:
def pins_flag_calls(self, conjuncts: list[ast.Expr], needs_flag_key: bool) -> bool:
"""Whether `conjuncts` pin `event` to flag calls. With `needs_flag_key`, a pin of `event` counts only where
- these conjuncts also pin `$feature_flag`. The recursion into `OR` branches passes False once an enclosing
- conjunct pins `$feature_flag`."""
- needs_flag_key = needs_flag_key and not self.flag_keys(conjuncts)
- return any(self._pins_flag_calls_in(conjunct, needs_flag_key) for conjunct in conjuncts)
-
- def _pins_flag_calls_in(self, expr: ast.Expr, needs_flag_key: bool) -> bool:
- if _pinned_values(expr, self._reads_event, _is_flag_called) is not None:
- return not needs_flag_key
- if branches := boolean_operands(expr, ast.Or):
- return all(self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in branches)
- return False
+ these conjuncts also pin `$feature_flag`."""
+ return _pins_flag_calls(conjuncts, self._reads_event, self.flag_keys if needs_flag_key else None)
def _flag_keys_in(self, expr: ast.Expr) -> list[str] | None:
"""The keys `expr` pins `$feature_flag` to, or None when it does not pin it."""
@@ -403,6 +415,28 @@ class EventsScan:
return self.column(expr) == ["timestamp"]
+def _pins_flag_calls(
+ conjuncts: list[ast.Expr], reads_event: _Predicate, flag_keys: Callable[[list[ast.Expr]], list[str]] | None
+) -> bool:
+ """Whether `conjuncts` pin what `reads_event` matches to flag calls: by `=` or `IN` of only
+ `$feature_flag_called`, or by an `OR` whose every branch pins. With `flag_keys`, a pin counts only where these
+ conjuncts also pin `$feature_flag`. The recursion into `OR` branches drops `flag_keys` once an enclosing
+ conjunct pins `$feature_flag`."""
+ if flag_keys is not None and flag_keys(conjuncts):
+ flag_keys = None
+ return any(_pins_flag_calls_in(conjunct, reads_event, flag_keys) for conjunct in conjuncts)
+
+
+def _pins_flag_calls_in(
+ expr: ast.Expr, reads_event: _Predicate, flag_keys: Callable[[list[ast.Expr]], list[str]] | None
+) -> bool:
+ if _pinned_values(expr, reads_event, _is_flag_called) is not None:
+ return flag_keys is None
+ if branches := boolean_operands(expr, ast.Or):
+ return all(_pins_flag_calls(_conjuncts(branch), reads_event, flag_keys) for branch in branches)
+ return False
+
+
def comparison_of(expr: ast.Expr) -> ast.CompareOperation | None:
"""`expr` as a comparison, whether it is written `a >= b` or `greaterOrEquals(a, b)`."""
if isinstance(expr, ast.CompareOperation):
@@ -791,6 +825,106 @@ class _EventsScanSwapper(ScanVisitor):
point_scan_at(node, scan, ["posthog", "flag_evaluations"], flag_key)
self.swapped = True
+ def swap_scans_found_by_type(self, resolved: ast.AST, sources: JoinSources, clones: "_JoinClones") -> str | None:
+ """Swap each events scan in the clone whose counterpart in `resolved`, the resolved original, has a WHERE or
+ PREWHERE that pins `event` to flag calls, such as a scan joined after another table. `sources` holds the
+ original join of each resolved join, and `clones` holds the clones of each original join. Returns why one of
+ them cannot swap, or None.
+
+ A scan that the resolver adds, such as one in a saved view's body or an inline cohort, keeps reading events.
+ Any other scan without a clone cannot swap.
+ """
+ for select, join in _joins(resolved):
+ if not _reads_events_table(join) or sources.is_added(join):
+ continue
+ if not _pins_flag_calls(filter_conjuncts(select), partial(_reads_event_of, join.type), None):
+ continue
+ source = sources.source_of(join)
+ source_clones = clones.get(id(source), []) if source is not None else []
+ # The pin check above reads the filters of one SELECT. A join that several SELECTs share cannot swap.
+ if not source_clones or len({id(clone.original_select) for clone in source_clones}) > 1:
+ return _UNMOVABLE_SHAPE
+ # A query can hold one node in several places, such as one SELECT in two branches of a UNION. Cloning
+ # copies each place apart, so one original join can have several clones, and each one has to swap.
+ for clone in source_clones:
+ refusal = self._swap_clone(clone)
+ if refusal is not None:
+ return refusal
+ return None
+
+ def _swap_clone(self, clone: "_JoinClone") -> str | None:
+ chain = _table_chain(clone.join)
+ if chain == ["posthog", "flag_evaluations"]:
+ # The syntactic rules, or an earlier resolved join from the same source, already swapped it.
+ return None
+ if chain != ["events"]:
+ return _UNMOVABLE_SHAPE
+ events_scan = EventsScan(
+ join=clone.join,
+ alias=clone.join.alias or "events",
+ # The clone is unresolved, so only a name qualified with this scan's alias can count as a read of it.
+ has_other_sources=True,
+ flag_key_compares_as_string=self.flag_key_compares_as_string,
+ )
+ refusal = events_scan.shape_refusal(clone.select)
+ if refusal is not None:
+ return refusal
+ self._swap(clone.select, events_scan, filter_conjuncts(clone.select))
+ return None
+
+
+@frozen
+class _JoinClone:
+ # Keeps the original join alive, so no other node can take the id that keys it.
+ original: ast.JoinExpr
+ original_select: ast.SelectQuery
+ select: ast.SelectQuery
+ join: ast.JoinExpr
+
+
+_JoinClones = dict[int, list[_JoinClone]]
+
+
+def _join_clones(original: ast.AST, clone: ast.AST) -> _JoinClones:
+ """Each join of `original`, with the `SelectQuery` and `JoinExpr` in each of its places in `clone`, a copy of it."""
+ clones: _JoinClones = {}
+ for (original_select, join), (clone_select, clone_join) in zip(_joins(original), _joins(clone), strict=True):
+ clones.setdefault(id(join), []).append(
+ _JoinClone(original=join, original_select=original_select, select=clone_select, join=clone_join)
+ )
+ return clones
+
+
+def _table_chain(join: ast.JoinExpr) -> list[str | int] | None:
+ return join.table.chain if isinstance(join.table, ast.Field) else None
+
+
+def _joins(node: ast.AST) -> list[tuple[ast.SelectQuery, ast.JoinExpr]]:
+ collector = _JoinCollector()
+ collector.visit(node)
+ return collector.joins
+
+
+class _JoinCollector(SyntaxVisitor):
+ """Each `JoinExpr` in a FROM chain, with the `SelectQuery` that owns it."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.joins: list[tuple[ast.SelectQuery, ast.JoinExpr]] = []
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ join = node.select_from
+ while join is not None:
+ self.joins.append((node, join))
+ join = join.next_join
+ super().visit_select_query(node)
+
+
+def _reads_event_of(table_type: ast.Type | None, expr: ast.Expr) -> bool:
+ """Whether resolved `expr` reads the `event` column of the scan whose type is `table_type`."""
+ field_type = event_field_type(expr)
+ return field_type is not None and field_type.table_type is table_type
+
def _swap_keeps_bindings(original: ast.AST, swapped: ast.AST, flag_key_predicates: int) -> bool:
"""Whether every field and table in the resolved swapped query binds as it does in the resolved original.
@@ -1015,7 +1149,5 @@ def _reads_only_flag_calls_from_events(node: ast.SelectQuery) -> bool:
def _reads_events_table(join: ast.JoinExpr) -> bool:
"""Whether a resolved FROM reads the events table, rather than a CTE named `events`."""
- table_type = join.type
- while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
- table_type = table_type.table_type
+ table_type = unwrap_table_alias_type(join.type)
return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)
diff --git a/posthog/models/test/test_data_deletion_request.py b/posthog/models/test/test_data_deletion_request.py
index a39db70a252d..67116ebcae9b 100644
--- a/posthog/models/test/test_data_deletion_request.py
+++ b/posthog/models/test/test_data_deletion_request.py
@@ -207,17 +207,22 @@ def test_hogql_predicate_subquery_is_team_scoped(db):
assert f"equals(team_id, {team_a.id})" not in sql_b
-def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team):
+ "predicate",
+ [
+ "distinct_id IN (SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY)",
+ "distinct_id IN (SELECT e.distinct_id FROM events AS e JOIN (SELECT 'probe' AS id) AS d "
+ "ON e.distinct_id = d.id WHERE event = '$feature_flag_called')",
+ ],
+)
+def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, predicate):
from posthog.models.data_deletion_request import compile_hogql_predicate
OrganizationFeatureFlagsConfig.objects.update_or_create(
organization=team.organization,
defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},
)
- predicate = (
- "distinct_id IN (SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
- "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY)"
- )
request = DataDeletionRequest(**_base_kwargs(team_id=team.id, events=["$pageview"], hogql_predicate=predicate))
with patch(