:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): read flag calls from both tables in mixed queries (#126045)
Phil Haack6a16b4778a96
diff --git a/posthog/api/cohort.py b/posthog/api/cohort.pyindex 7ef6628023d5..055e6bf9b9f3 100644--- a/posthog/api/cohort.py+++ b/posthog/api/cohort.py@@ -38,10 +38,10 @@ from posthog.schema import ActorsQuery, HogQLQuery, ProductKeyfrom posthog.hogql.compiler.bytecode import create_bytecodefrom posthog.hogql.constants import CSV_EXPORT_LIMIT, LimitContextfrom posthog.hogql.context import HogQLContext+from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYSfrom posthog.hogql.errors import TableAccessDeniedErrorfrom posthog.hogql.printer import prepare_ast_for_printingfrom posthog.hogql.property import PERSON_METADATA_FIELDS, property_to_expr-from posthog.hogql.transforms.flag_calls_from_flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYSfrom posthog.api.forbid_destroy_model import ForbidDestroyModelfrom posthog.api.routing import TeamAndOrgViewSetMixindiff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.mdindex 5bd904e984ec..15e0a0a33ec5 100644--- a/posthog/hogql/ACCESS_CONTROL.md+++ b/posthog/hogql/ACCESS_CONTROL.md@@ -222,6 +222,7 @@ Because each class restricts its own key, a table dispatched as the wrong class,The exemptions are name collisions: `accounts.properties` and `pg_embeddings.properties` do not contain event, person, or group properties.`ai_events.properties` uses the same event-property rules as `events.properties`.+The hidden `events_with_flag_calls` table, which the flag-call union reads in place of `events`, uses them too.Covering a table means moving it out of the exemptions and into the expected mapping, so neither list can keep a stale entry.### Typed columns that mirror a propertydiff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.pyindex 3ebea3d5fd81..5c84e1313988 100644--- a/posthog/hogql/database/database.py+++ b/posthog/hogql/database/database.py@@ -88,6 +88,7 @@ from posthog.hogql.database.schema.error_tracking_issue_fingerprint_overrides im)from posthog.hogql.database.schema.error_tracking_recent_issue_state import ErrorTrackingRecentIssueStateTablefrom posthog.hogql.database.schema.events import EventsTable+from posthog.hogql.database.schema.events_with_flag_calls import EVENTS_WITH_FLAG_CALLS_TABLE, EventsWithFlagCallsTablefrom posthog.hogql.database.schema.exchange_rate import ExchangeRateTablefrom posthog.hogql.database.schema.experiment_exposures_preaggregated import ExperimentExposuresPreaggregatedTablefrom posthog.hogql.database.schema.experiment_metric_events_preaggregated import (@@ -255,6 +256,8 @@ class HogQLDatabaseSources:read_flag_calls_from_flag_evaluations: bool# The organization reads flag calls only from posthog.flag_evaluations, and ingestion no longer writes them to events.flag_calls_only_in_flag_evaluations: bool+ # Flag calls that a mixed query reads come from events and flag_evaluations together.+ read_flag_calls_with_union: bool# Userless internal contexts that must resolve every warehouse table/view; skips access controlbypass_warehouse_access_control: booldirect_connection_metadata: dict[str, Any] | None@@ -458,7 +461,7 @@ _DATABASE_ROOT_NODE_BLOBS_LOCK = threading.Lock()# LRU, and fresh entries survive the sweep. Reads are lock-free (dict.get is atomic under the# GIL); mutations and the sweep hold the lock, because the sweep iterates the dict and concurrent# inserts would raise RuntimeError mid-iteration.-_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets"})+_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets", "hogql-flag-calls-union"})_TEAM_FLAG_CACHE: dict[tuple[str, str], tuple[float, bool]] = {} # key -> (evaluated_at, value)_TEAM_FLAG_CACHE_LOCK = threading.Lock()_TEAM_FLAG_CACHE_MAX_ENTRIES = 50_000@@ -514,6 +517,21 @@ def _cached_team_flag(flag_key: str, team: Team, evaluate: Callable[[], bool]) -return value+def _cached_organization_flag(flag_key: str, team: Team) -> bool:+ """`flag_key` for the team's organization, through the team flag cache."""+ return _cached_team_flag(+ flag_key,+ team,+ lambda: feature_enabled_or_false(+ flag_key,+ str(team.uuid),+ groups={"organization": str(team.organization_id)},+ group_properties={"organization": {"id": str(team.organization_id)}},+ send_feature_flag_events=False,+ ),+ )++# We only ever load our own freshly-built blob, but restrict the unpickler anyway as defense in depth:# it can reconstruct only the classes the catalog is built from, so even a future change that fed it# untrusted bytes couldn't instantiate arbitrary code (os.system, etc.). Every catalog class lives@@ -835,6 +853,7 @@ class Database(BaseModel):_connection_id: str | None = None_read_flag_calls_from_flag_evaluations: bool = False_flag_calls_only_in_flag_evaluations: bool = False+ _read_flag_calls_with_union: bool = False_direct_connection_metadata: dict[str, Any] | None = None_direct_access_warehouse_table_names: set[str] = set()# Warnings about data warehouse tables (failed/paused/billing-limited/stale syncs),@@ -903,6 +922,7 @@ class Database(BaseModel):self._connection_id = Noneself._read_flag_calls_from_flag_evaluations = Falseself._flag_calls_only_in_flag_evaluations = False+ self._read_flag_calls_with_union = Falseself._direct_connection_metadata = Noneself._direct_access_warehouse_table_names = set()self._data_warehouse_sync_warnings = {}@@ -1307,6 +1327,10 @@ class Database(BaseModel):def flag_calls_only_in_flag_evaluations(self) -> bool:return self._flag_calls_only_in_flag_evaluations and self.read_flag_calls_from_flag_evaluations+ @property+ def read_flag_calls_with_union(self) -> bool:+ return self._read_flag_calls_with_union and self.read_flag_calls_from_flag_evaluations+@staticmethoddef _is_helper_function_table(table: object) -> bool:return isinstance(table, FunctionCallTable) and not isinstance(table, (DirectSQLTable, PostgresTable, S3Table))@@ -1504,8 +1528,13 @@ class Database(BaseModel):if self._is_direct_query()else self.get_posthog_table_names(include_hidden=include_hidden_posthog_tables))+ posthog_node = self.tables.children.get("posthog")+ # A hidden table stays readable by name, but the schema leaves it out.+ hidden_table_names = {+ f"posthog.{name}" for name, node in (posthog_node.children.items() if posthog_node else ()) if node.hidden+ }for table_name in posthog_table_names:- if include_only and table_name not in include_only:+ if (include_only and table_name not in include_only) or table_name in hidden_table_names:continuefields_dict: dict[str, DatabaseSchemaField] = {}@@ -1987,6 +2016,7 @@ class Database(BaseModel):include_flag_evaluations_table=False,read_flag_calls_from_flag_evaluations=False,flag_calls_only_in_flag_evaluations=False,+ read_flag_calls_with_union=False,group_types=[],saved_queries=[],endpoint_saved_queries=[],@@ -2097,6 +2127,9 @@ class Database(BaseModel):read_mode = get_flag_evaluations_read_mode(team.organization_id)read_flag_calls_from_flag_evaluations = read_mode != FlagEvaluationsMode.EVENTSflag_calls_only_in_flag_evaluations = read_mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY+ read_flag_calls_with_union = read_flag_calls_from_flag_evaluations and _cached_organization_flag(+ "hogql-flag-calls-union", team+ )with timings.measure("database", emit_span=True):# Function-local: keeps the direct-SQL driver imports off the django.setup() path.@@ -2365,6 +2398,7 @@ class Database(BaseModel):include_flag_evaluations_table=include_flag_evaluations_table,read_flag_calls_from_flag_evaluations=read_flag_calls_from_flag_evaluations,flag_calls_only_in_flag_evaluations=flag_calls_only_in_flag_evaluations,+ read_flag_calls_with_union=read_flag_calls_with_union,# Managed warehouse is a built-in project datastore and has no warehouse-object ACL surface.# Principals that skip warehouse access control by design:# - synthetic users (project-wide service tokens, bypass object-level RBAC)@@ -2420,6 +2454,7 @@ class Database(BaseModel):)database._read_flag_calls_from_flag_evaluations = sources.read_flag_calls_from_flag_evaluationsdatabase._flag_calls_only_in_flag_evaluations = sources.flag_calls_only_in_flag_evaluations+ database._read_flag_calls_with_union = sources.read_flag_calls_with_unionif sources.connection_id is not None:database._connection_id = sources.connection_idif sources.direct_connection_metadata is not None:@@ -2439,6 +2474,14 @@ class Database(BaseModel):if not sources.include_flag_evaluations_table:if posthog_node := database.tables.children.get("posthog"):posthog_node.children.pop("flag_evaluations", None)+ elif sources.read_flag_calls_with_union and (posthog_node := database.tables.children.get("posthog")):+ union_table = EventsWithFlagCallsTable()+ # Assigned after construction, because pydantic copies a dict passed to the constructor. Sharing the+ # dict makes every later change to the events fields, such as the persons-on-events mode, apply here too.+ union_table.fields = database.get_table("events").fields+ posthog_node.children[EVENTS_WITH_FLAG_CALLS_TABLE] = TableNode(+ name=EVENTS_WITH_FLAG_CALLS_TABLE, table=union_table, hidden=True+ )with timings.measure("modifiers", emit_span=True):if not database._is_direct_query():diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.pyindex c9969c639608..35ac972150eb 100644--- a/posthog/hogql/database/models.py+++ b/posthog/hogql/database/models.py@@ -17,7 +17,7 @@ from posthog.hogql.errors import NotImplementedError, ResolutionErrorfrom posthog.clickhouse.workload import Workloadif TYPE_CHECKING:- from posthog.hogql.ast import JoinExpr, LazyJoinType, SelectQuery+ from posthog.hogql.ast import JoinExpr, LazyJoinType, SelectQuery, Typefrom posthog.hogql.base import ConstantTypefrom posthog.hogql.context import HogQLContext@@ -534,19 +534,33 @@ class LazyTable(Table):raise NotImplementedError("LazyTable.lazy_select not overridden")-@dataclass+@dataclass(frozen=False)class LazyTableToAdd:lazy_table: LazyTablefields_accessed: dict[str, list[str | int]] = field(default_factory=dict)-@dataclass+@dataclass(frozen=True)+class UnexpandedScan:+ """The FROM entry of a lazy table before lazy table resolution replaced it with a subquery.++ A join that hangs off the table copies the query's filters into a subquery that reads the table again, and this+ holds what that subquery needs. `types` are the table types that the query's fields name for the scan.+ """++ join: "JoinExpr"+ types: tuple["Type", ...]+++@dataclass(frozen=False)class LazyJoinToAdd:from_table: strto_table: strlazy_join: LazyJoinlazy_join_type: "LazyJoinType"fields_accessed: dict[str, list[str | int]] = field(default_factory=dict)+ # Set when the table that the join starts from is a lazy table whose scan the join can read again.+ from_scan: Optional[UnexpandedScan] = Noneclass VirtualTable(Table):diff --git a/posthog/hogql/database/schema/events.py b/posthog/hogql/database/schema/events.pyindex 4c61e7527a61..a44ad1ccdd38 100644--- a/posthog/hogql/database/schema/events.py+++ b/posthog/hogql/database/schema/events.py@@ -30,6 +30,17 @@ def events_table_clickhouse_table_ref(context) -> str:return DISTRIBUTED_EVENTS_JSON_TABLE if context.uses_new_events_schema() else "events"+EVENTS_AVOID_ASTERISK_FIELDS = (+ "$session_id_uuid",+ "$virt_is_bot",+ "$virt_traffic_type",+ "$virt_traffic_category",+ "$virt_bot_name",+ "$virt_bot_operator",+ "$virt_mcp_harness",+)++class EventsPersonSubTable(VirtualTable):fields: dict[str, FieldOrTable] = {"id": UUIDDatabaseField(name="person_id", nullable=False),@@ -212,15 +223,7 @@ class EventsTable(Table):return "events"def avoid_asterisk_fields(self) -> list[str]:- return [- "$session_id_uuid",- "$virt_is_bot",- "$virt_traffic_type",- "$virt_traffic_category",- "$virt_bot_name",- "$virt_bot_operator",- "$virt_mcp_harness",- ]+ return list(EVENTS_AVOID_ASTERISK_FIELDS)# All table types that represent the events table (including virtual subtables like poe/goe).diff --git a/posthog/hogql/database/schema/events_with_flag_calls.py b/posthog/hogql/database/schema/events_with_flag_calls.pynew file mode 100644index 000000000000..0439bc8ff088--- /dev/null+++ b/posthog/hogql/database/schema/events_with_flag_calls.py@@ -0,0 +1,200 @@+from datetime import UTC, datetime+from functools import partial++from posthog.hogql import ast+from posthog.hogql.constants import FLAG_CALLED_EVENT+from posthog.hogql.database.models import (+ DatabaseField,+ ExpressionField,+ FieldOrTable,+ LazyJoin,+ LazyTable,+ LazyTableToAdd,+ Table,+ VirtualTable,+)+from posthog.hogql.database.schema.events import EVENTS_AVOID_ASTERISK_FIELDS, EventsTable+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable, flag_evaluations_coverage_start+from posthog.hogql.errors import QueryError+from posthog.hogql.parser import parse_expr+from posthog.hogql.visitor import GetFieldsTraverser++EVENTS_WITH_FLAG_CALLS_TABLE = "events_with_flag_calls"+EVENTS_WITH_FLAG_CALLS_CHAIN: list[str] = ["posthog", EVENTS_WITH_FLAG_CALLS_TABLE]++# Ingestion writes a flag call to flag_evaluations only when `$feature_flag` is a non-empty JSON string. The raw JSON+# text of a string value starts with a quote, and a number, boolean or object does not. On the native JSON events+# table the printer converts a typed key before this test reads it, so mode 1 queries there keep reading events. The+# test reads only the raw JSON text, because a materialized `$feature_flag` column reads an empty string as missing.+_HAS_FLAG_KEY = (+ "startsWith(JSONExtractRaw(events.properties, '$feature_flag'), '\"') "+ "AND JSONExtractRaw(events.properties, '$feature_flag') != '\"\"'"+)+++def _field(table: Table, chain: list[str | int]) -> tuple[FieldOrTable | None, list[str | int]]:+ """The field that `chain` names on `table`, through a virtual table, and the rest of the chain below it."""+ field = table.fields.get(str(chain[0]))+ if isinstance(field, VirtualTable) and len(chain) > 1:+ return field.fields.get(str(chain[1])), chain[2:]+ return field, chain[1:]+++def _column_name(field: FieldOrTable | None) -> str | None:+ # An ExpressionField is a DatabaseField too, but it names no column.+ if isinstance(field, DatabaseField) and not isinstance(field, ExpressionField):+ return field.name+ return None+++def _is_expression(table: Table, chain: list[str | int]) -> bool:+ return isinstance(_field(table, chain)[0], ExpressionField)+++def column_chains(table: Table, chain: list[str | int]) -> list[list[str | int]]:+ """The chains of stored columns that `chain` reads on `table`, through the expression fields and the keys of the+ joins that it reads."""+ field, _ = _field(table, chain)+ if not isinstance(field, ExpressionField):+ return [chain]+ chains: list[list[str | int]] = []+ for read in GetFieldsTraverser(field.expr).fields:+ joined = table.fields.get(str(read.chain[0]))+ if isinstance(joined, LazyJoin) and len(read.chain) > 1:+ chains.extend(column_chains(table, list(joined.from_field)))+ else:+ chains.extend(column_chains(table, list(read.chain)))+ return chains+++class FlagEvaluationsColumns:+ """Maps a chain on the events fields to the chain on `posthog.flag_evaluations` that holds the same value."""++ def __init__(self, events: Table, flag_evaluations: Table) -> None:+ self.events = events+ self.flag_evaluations = flag_evaluations+ self._chains_by_column: dict[str, list[str | int]] = {}+ for name, field in flag_evaluations.fields.items():+ if (column := _column_name(field)) is not None:+ self._chains_by_column.setdefault(column, [name])+ for name, field in flag_evaluations.fields.items():+ if isinstance(field, VirtualTable):+ for inner_name, inner in field.fields.items():+ if (column := _column_name(inner)) is not None:+ self._chains_by_column.setdefault(column, [name, inner_name])++ def chain(self, chain: list[str | int]) -> list[str | int] | None:+ """The chain on `flag_evaluations` that holds what `chain` reads on `events`, or None when it has none.++ A chain that reads an events column maps to the `flag_evaluations` field on the same column, such as+ `person_id` for `event_person_id`. Otherwise it maps to the `flag_evaluations` field of the same name, which+ that table defines for events names whose columns differ, such as `$session_id`. An expression field maps when+ each column it reads does.+ """+ field, rest = _field(self.events, chain)+ if isinstance(field, ExpressionField):+ reads = column_chains(self.events, chain)+ return chain if all(self.chain(read) is not None for read in reads) else None+ column = _column_name(field)+ if column is not None and column in self._chains_by_column:+ return [*self._chains_by_column[column], *rest]+ same_name = self.flag_evaluations.fields.get(str(chain[0]))+ if isinstance(same_name, VirtualTable):+ return chain if len(chain) > 1 and str(chain[1]) in same_name.fields else None+ return chain if same_name is not None else None+++class EventsWithFlagCallsTable(LazyTable):+ """`events` with its flag calls since the coverage start read from `posthog.flag_evaluations` instead.++ Its `fields` is the events table's own dict, so every name binds as it does on `events`.+ """++ fields: dict[str, FieldOrTable] = {}++ def avoid_asterisk_fields(self) -> list[str]:+ return list(EVENTS_AVOID_ASTERISK_FIELDS)++ def lazy_select(self, table_to_add: LazyTableToAdd, context, node):+ # Deferred: the transforms import this table.+ from posthog.hogql.transforms.flag_calls_from_flag_evaluations import ( # noqa: PLC0415+ and_conditions,+ flag_key_typed_as_string,+ )+ from posthog.hogql.transforms.flag_calls_union import union_branch_filters, union_events_filter # noqa: PLC0415++ assert context.database is not None+ # `SELECT count()` reads no field, and a subquery needs at least one column.+ accessed = dict(table_to_add.fields_accessed) or {"timestamp": ["timestamp"]}+ events_columns: list[ast.Expr] = [+ ast.Alias(alias=name, expr=ast.Field(chain=["events", *chain])) for name, chain in accessed.items()+ ]+ if any(_is_expression(self, chain) for chain in accessed.values()):+ # An expression field reaches here as the key of a join, such as `person_id` for `person`. Resolving it on+ # a second scan of this table puts its own joins, such as `override`, above the branches, so each runs+ # once. The second scan asks only for columns, and it reads the same filter from this query's WHERE.+ return ast.SelectQuery(+ select=events_columns,+ select_from=ast.JoinExpr(table=ast.Field(chain=list(EVENTS_WITH_FLAG_CALLS_CHAIN)), alias="events"),+ where=union_events_filter(node, self),+ )++ columns = FlagEvaluationsColumns(self, context.database.get_table(["posthog", "flag_evaluations"]))+ only_in_flag_evaluations = context.database.flag_calls_only_in_flag_evaluations+ cutoff = ast.Constant(value=flag_evaluations_coverage_start(datetime.now(UTC)))+ # Property type resolution ran before lazy tables, so the metadata holds the type of each property this query+ # reads, `$feature_flag` among them.+ filters = union_branch_filters(+ node, self, columns, partial(flag_key_typed_as_string, context.property_metadata)+ )++ def flag_evaluations_field(chain: list[str | int]) -> ast.Field:+ flag_chain = columns.chain(chain)+ if flag_chain is None:+ # union_unreadable_reason refuses such a query before lazy tables resolve.+ raise QueryError(f"posthog.flag_evaluations has no column for {'.'.join(str(part) for part in chain)}")+ return ast.Field(chain=["flag_evaluations", *flag_chain])++ def events_branch(*conditions: ast.Expr | None) -> ast.SelectQuery:+ return ast.SelectQuery(+ select=events_columns,+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),+ where=and_conditions([*conditions, filters.events]),+ )++ flag_call = parse_expr("events.event = {event}", {"event": ast.Constant(value=FLAG_CALLED_EVENT)})+ flag_call_since_cutoff = ast.And(+ exprs=[flag_call, parse_expr("events.timestamp >= {cutoff}", {"cutoff": cutoff})]+ )+ # On mode 2 every flag call comes from flag_evaluations, as the flag_evaluations swap reads it. That leaves out+ # the calls without a key, which ingestion keeps writing to events.+ moved = flag_call if only_in_flag_evaluations else flag_call_since_cutoff+ branches = [events_branch(ast.Not(expr=moved))]+ if not only_in_flag_evaluations and filters.keyless_flag_calls_can_match:+ branches.append(events_branch(flag_call_since_cutoff, ast.Not(expr=parse_expr(_HAS_FLAG_KEY))))+ branches.append(+ ast.SelectQuery(+ select=[ast.Alias(alias=name, expr=flag_evaluations_field(chain)) for name, chain in accessed.items()],+ select_from=ast.JoinExpr(+ table=ast.Field(chain=["posthog", "flag_evaluations"]), alias="flag_evaluations"+ ),+ where=and_conditions(+ [parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff}), filters.flag_evaluations]+ ),+ )+ )+ return ast.SelectQuery(+ select=[ast.Field(chain=[name]) for name in accessed],+ select_from=ast.JoinExpr(table=ast.SelectSetQuery.create_from_queries(branches, "UNION ALL")),+ )++ def to_printed_clickhouse(self, context):+ return "events"++ def to_printed_hogql(self):+ return EVENTS_WITH_FLAG_CALLS_TABLE+++# Tables whose rows are events with the events columns. Property types, access control and time bounds treat their reads+# as reads of events.+EVENT_ROW_TABLES = (EventsTable, FlagEvaluationsTable, EventsWithFlagCallsTable)diff --git a/posthog/hogql/database/schema/flag_evaluations.py b/posthog/hogql/database/schema/flag_evaluations.pyindex 174d2aa087fe..766e28f22700 100644--- a/posthog/hogql/database/schema/flag_evaluations.py+++ b/posthog/hogql/database/schema/flag_evaluations.py@@ -1,3 +1,4 @@+from datetime import UTC, datetime, timedeltafrom typing import TYPE_CHECKINGfrom posthog.hogql import ast@@ -22,6 +23,7 @@ from posthog.hogql.database.schema.util.uuid import uuid_string_expr_to_uint128_from posthog.hogql.parser import parse_exprfrom posthog.constants import GROUP_TYPES_LIMIT+from posthog.date_util import start_of_dayif TYPE_CHECKING:from posthog.hogql.database.database import Database@@ -32,6 +34,21 @@ if TYPE_CHECKING:# name is spelled out here rather than imported.FLAG_EVALUATIONS_CLICKHOUSE_TABLE = "flag_evaluations"+# At most FLAG_EVALUATIONS_TTL_DAYS - 1, because the TTL can already have dropped the day FLAG_EVALUATIONS_TTL_DAYS+# back. It stays lower while the table still lacks the oldest days that a shorter TTL dropped. The TTL constant+# lives in posthog/models/flag_evaluations/sql.py.+FLAG_EVALUATIONS_COMPLETE_DAYS = 89+++def utc_midnight(now: datetime) -> datetime:+ return start_of_day(now.astimezone(UTC))+++def flag_evaluations_coverage_start(now: datetime) -> datetime:+ """The start of the oldest UTC day whose flag calls are all in flag_evaluations at `now`."""+ return utc_midnight(now) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)++# Keep it equal to FLAG_EVALUATIONS_TABLE in frontend/src/scenes/feature-flags/flagEvaluationsTable.ts.FLAG_EVALUATIONS_HOGQL_TABLE = "posthog.flag_evaluations"diff --git a/posthog/hogql/database/schema/persons.py b/posthog/hogql/database/schema/persons.pyindex 7b99834ba573..c3387be6529c 100644--- a/posthog/hogql/database/schema/persons.py+++ b/posthog/hogql/database/schema/persons.py@@ -24,7 +24,11 @@ from posthog.hogql.database.models import ()from posthog.hogql.database.schema.persons_pdi import PersonsPDITablefrom posthog.hogql.database.schema.persons_revenue_analytics import PersonsRevenueAnalyticsTable-from posthog.hogql.database.schema.util.where_clause_extractor import WhereClauseExtractor, top_level_conjuncts+from posthog.hogql.database.schema.util.where_clause_extractor import (+ WhereClauseExtractor,+ is_nondeterministic_function,+ top_level_conjuncts,+)from posthog.hogql.errors import ResolutionErrorfrom posthog.hogql.parser import parse_selectfrom posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr@@ -267,20 +271,16 @@ def select_from_persons_table(return select-# These return different values in the subquery and in the outer query.-_NONDETERMINISTIC_FUNCTIONS = frozenset({"rand", "rownumberinblock", "rownumberinallblocks", "nowinblock"})--class _CopyableConjunctVisitor(TraversingVisitor):- def __init__(self, table_type: ast.Type):+ def __init__(self, table_types: tuple[ast.Type, ...]):super().__init__()- self.table_type = table_type+ self.table_types = table_typesself.reads_table = Falseself.copyable = Truedef visit_field(self, node: ast.Field):field_type = node.type.field_type if isinstance(node.type, ast.PropertyType) else node.type- if isinstance(field_type, ast.FieldType) and field_type.table_type is self.table_type:+ if isinstance(field_type, ast.FieldType) and any(field_type.table_type is t for t in self.table_types):self.reads_table = Trueelse:self.copyable = False@@ -293,7 +293,7 @@ class _CopyableConjunctVisitor(TraversingVisitor):self.copyable = Falsedef visit_call(self, node: ast.Call):- if node.name.lower() in _NONDETERMINISTIC_FUNCTIONS:+ if is_nondeterministic_function(node.name):self.copyable = Falsesuper().visit_call(node)@@ -327,12 +327,20 @@ def build_person_id_pushdown_predicate(join_to_add: LazyJoinToAdd, node: SelectQits person columns.Returns None unless the left table is the query's own FROM table and at least one term qualifies."""- left = node.select_from+ # A lazy left table, such as the union of events and flag_evaluations, is already a subquery in `node`. Its recorded+ # scan holds the FROM entry as the query wrote it.+ scan = join_to_add.from_scan+ left = scan.join if scan is not None else node.select_from+ left_types: tuple[ast.Type, ...] = (+ scan.types if scan is not None else (left.type,) if left is not None and left.type is not None else ()+ )if (left is Noneor not isinstance(left.table, ast.Field)or left.table_args- or left.type is not join_to_add.lazy_join_type.table_type+ or not left_types+ or left_types[0] is not join_to_add.lazy_join_type.table_type+ or (scan is not None and (node.select_from is None or node.select_from.alias != join_to_add.from_table))):return None@@ -342,7 +350,7 @@ def build_person_id_pushdown_predicate(join_to_add: LazyJoinToAdd, node: SelectQif where is None:continuefor conjunct in top_level_conjuncts(where):- visitor = _CopyableConjunctVisitor(left.type)+ visitor = _CopyableConjunctVisitor(left_types)visitor.visit(conjunct)if visitor.copyable and visitor.reads_table:conjuncts.append(cloner.visit(conjunct))diff --git a/posthog/hogql/database/schema/util/where_clause_extractor.py b/posthog/hogql/database/schema/util/where_clause_extractor.pyindex 56f14d9a4cab..ad3cb2ec2697 100644--- a/posthog/hogql/database/schema/util/where_clause_extractor.py+++ b/posthog/hogql/database/schema/util/where_clause_extractor.py@@ -630,6 +630,9 @@ def event_field_type(expr: ast.Expr) -> ast.FieldType | None:def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableType]:"""If the expression resolves to $session_id on the events table, return its (possibly aliased) table type."""from posthog.hogql.database.schema.events import EventsTable # noqa: PLC0415 - breaks the schema import cycle+ from posthog.hogql.database.schema.events_with_flag_calls import ( # noqa: PLC0415 - breaks the schema import cycle+ EventsWithFlagCallsTable,+ )expr_type = _unwrap_field_alias_type(node.type)@@ -654,7 +657,9 @@ def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableTyreturn Noneunwrapped = unwrap_table_alias_type(table_type)- if isinstance(unwrapped, ast.TableType) and isinstance(unwrapped.table, EventsTable):+ if isinstance(unwrapped, (ast.TableType, ast.LazyTableType)) and isinstance(+ unwrapped.table, (EventsTable, EventsWithFlagCallsTable)+ ):return cast(ast.BaseTableType, table_type)return None@@ -693,7 +698,7 @@ def build_session_id_literal_pushdown_predicate("""if outer_node.where is None:return None- events_join = _find_join_for_alias(outer_node.select_from, join_to_add.from_table)+ events_join = _events_scan_for_join(outer_node, join_to_add)if events_join is None or not isinstance(events_join.table, ast.Field):return None@@ -845,6 +850,17 @@ class _CollectTopLevelAliasesVisitor(TraversingVisitor):self.names.add(str(node.chain[0]))+def _events_scan_for_join(outer_node: ast.SelectQuery, join_to_add: LazyJoinToAdd) -> Optional[ast.JoinExpr]:+ """The FROM entry that the lazy join starts from, as the query wrote it.++ Lazy table resolution replaces the scan of a lazy table, such as the union of events and flag_evaluations, with a+ subquery, so the entry it recorded stands in for it.+ """+ if join_to_add.from_scan is not None:+ return join_to_add.from_scan.join+ return _find_join_for_alias(outer_node.select_from, join_to_add.from_table)++def _find_join_for_alias(select_from: Optional[ast.JoinExpr], alias: str) -> Optional[ast.JoinExpr]:ptr = select_fromwhile ptr:@@ -1047,6 +1063,14 @@ class _FieldNodeCollector(TraversingVisitor):# any future array-expanding function HogQL lowers into WHERE here._ROW_MULTIPLYING_FUNCTIONS = {"arrayJoin"}+# These return a different value on each read, so a copy of a filter that calls one selects other rows than the+# original. Lowercase, because HogQL accepts some of them in any case, such as RAND().+_NONDETERMINISTIC_FUNCTIONS = frozenset({"rand", "rownumberinblock", "rownumberinallblocks", "nowinblock"})+++def is_nondeterministic_function(name: str) -> bool:+ return name.lower() in _NONDETERMINISTIC_FUNCTIONS+class _RowMultiplyingFunctionFinder(TraversingVisitor):"""Detects any row-multiplying function call (e.g. arrayJoin) anywhere in an expression."""diff --git a/posthog/hogql/database/test/test_database.py b/posthog/hogql/database/test/test_database.pyindex 06e0295fb741..2297fb02eca9 100644--- a/posthog/hogql/database/test/test_database.py+++ b/posthog/hogql/database/test/test_database.py@@ -493,6 +493,12 @@ class TestDatabase(BaseTest, QueryMatchingTest):include_flag_evaluations_table=True,),("read_mode", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, True),+ param(+ "read_mode_with_the_union_flag_off",+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ True,+ union_flag_enabled=False,+ ),param("read_mode_with_reads_forced_to_events",FlagEvaluationsMode.READ_FLAG_EVALUATIONS,@@ -515,7 +521,7 @@ class TestDatabase(BaseTest, QueryMatchingTest):),])- def test_read_flag_calls_from_flag_evaluations_follows_the_read_mode(+ def test_flag_call_reads_follow_the_read_mode_and_the_union_flag(self,_name: str,mode: FlagEvaluationsMode,@@ -523,7 +529,9 @@ class TestDatabase(BaseTest, QueryMatchingTest):reads_forced_to_events: bool = False,debug: bool = False,include_flag_evaluations_table: bool | None = None,+ union_flag_enabled: bool = True,) -> None:+ enabled_flags = {"hogql-flag-calls-union"} if union_flag_enabled else set()OrganizationFeatureFlagsConfig.objects.update_or_create(organization=self.organization, defaults={"flag_evaluations_mode": mode})@@ -531,12 +539,21 @@ class TestDatabase(BaseTest, QueryMatchingTest):with (override_settings(DEBUG=debug),override_instance_config("FLAG_EVALUATIONS_READS_FORCE_EVENTS", reads_forced_to_events),+ patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: key in enabled_flags,+ ),):database = Database.create_for(team=self.team, include_flag_evaluations_table=include_flag_evaluations_table)assert database.read_flag_calls_from_flag_evaluations is expected+ union_expected = expected and union_flag_enabled+ assert database.read_flag_calls_with_union is union_expected+ assert database.has_table(["posthog", "events_with_flag_calls"]) is union_expected+ context = HogQLContext(team_id=self.team.pk, database=database)+ assert "posthog.events_with_flag_calls" not in database.serialize(context, include_hidden_posthog_tables=True)def test_serialize_database_without_fields_matches_full_metadata(self):credential = DataWarehouseCredential.objects.create(access_key="blah", access_secret="blah", team=self.team)diff --git a/posthog/hogql/helpers/timestamp_visitor.py b/posthog/hogql/helpers/timestamp_visitor.pyindex e9674115f504..554c11fa9adb 100644--- a/posthog/hogql/helpers/timestamp_visitor.py+++ b/posthog/hogql/helpers/timestamp_visitor.py@@ -132,8 +132,7 @@ class IsSimpleTimestampFieldExpressionVisitor(Visitor[bool]):raise Exception()def visit_alias(self, node: ast.Alias) -> bool:- from posthog.hogql.database.schema.events import EventsTable- from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable+ from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLESfrom posthog.hogql.database.schema.session_replay_events import RawSessionReplayEventsTablefrom posthog.hogql.database.schema.sessions_v1 import SessionsTableV1from posthog.hogql.database.schema.sessions_v2 import SessionsTableV2@@ -152,8 +151,8 @@ class IsSimpleTimestampFieldExpressionVisitor(Visitor[bool]):table_type = table_type.table_typereturn ((- isinstance(table_type, ast.TableType)- and isinstance(table_type.table, EventsTable | FlagEvaluationsTable)+ isinstance(table_type, ast.TableType | ast.LazyTableType)+ and isinstance(table_type.table, EVENT_ROW_TABLES)and resolved_field.name == "timestamp")or (diff --git a/posthog/hogql/property_planner.py b/posthog/hogql/property_planner.pyindex 70405f87b3ec..c87eada0bac8 100644--- a/posthog/hogql/property_planner.py+++ b/posthog/hogql/property_planner.py@@ -6,8 +6,9 @@ from typing import Literal, Optionalfrom posthog.hogql import astfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.database.models import DatabaseField-from posthog.hogql.database.schema.events import EventsPersonSubTable, EventsTable-from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable, FlagEvaluationsTable+from posthog.hogql.database.schema.events import EventsPersonSubTable+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTablefrom posthog.hogql.database.schema.groups import GroupsTablefrom posthog.hogql.database.schema.persons import PersonsTable, RawPersonsTablefrom posthog.hogql.errors import (@@ -497,7 +498,7 @@ def _property_scope(property_type: ast.PropertyType, context: HogQLContext) -> Pexcept (HogQLNotImplementedError, QueryError, ResolutionError):return PropertyScope.UNKNOWN- if isinstance(resolved_table, EventsTable | FlagEvaluationsTable):+ if isinstance(resolved_table, EVENT_ROW_TABLES):return PropertyScope.EVENTif isinstance(resolved_table, EventsPersonSubTable | FlagEvaluationsPersonSubTable | PersonsTable | RawPersonsTablediff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.pyindex 8f44e67b1995..c294f9172559 100644--- a/posthog/hogql/resolver.py+++ b/posthog/hogql/resolver.py@@ -24,6 +24,7 @@ from posthog.hogql.database.schema.duckdb_table_functions import (is_dangerous_table_function,)from posthog.hogql.database.schema.events import EventsTable+from posthog.hogql.database.schema.events_with_flag_calls import EventsWithFlagCallsTablefrom posthog.hogql.database.schema.log_entries import (BatchExportLogEntriesTable,LogEntriesTable,@@ -138,7 +139,11 @@ def _string_constants(node: ast.Expr) -> list[ast.Constant]:# cluster (log_entries is a Distributed over the aux cluster, so a plain IN makes every shard# of the outer query issue its own remote read against aux)._GLOBAL_IN_TABLES: tuple[type, ...] = (EventsTable, LogEntriesTable)-_GLOBAL_IN_LAZY_TABLES: tuple[type, ...] = (ReplayConsoleLogsLogEntriesTable, BatchExportLogEntriesTable)+_GLOBAL_IN_LAZY_TABLES: tuple[type, ...] = (+ ReplayConsoleLogsLogEntriesTable,+ BatchExportLogEntriesTable,+ EventsWithFlagCallsTable,+)class _ShardedTableFinder(TraversingVisitor):diff --git a/posthog/hogql/restricted_properties.py b/posthog/hogql/restricted_properties.pyindex 4c119ff0f1a6..e9e83366dcab 100644--- a/posthog/hogql/restricted_properties.py+++ b/posthog/hogql/restricted_properties.py@@ -8,6 +8,7 @@ from posthog.hogql.context import HogQLContextfrom posthog.hogql.database.postgres_table import PostgresTablefrom posthog.hogql.database.schema.ai_events import AI_PROPERTY_TO_COLUMN, AI_SUBCOLUMN_TO_SOURCE, AiEventsTablefrom posthog.hogql.database.schema.events import EventsGroupSubTable, EventsPersonSubTable, EventsTable+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLESfrom posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable, FlagEvaluationsTablefrom posthog.hogql.database.schema.groups import GroupsTable, RawGroupsTablefrom posthog.hogql.database.schema.persons import PersonsTable, RawPersonsTable@@ -121,7 +122,7 @@ def restricted_property_keys_for_table_type(elif isinstance(table, EventsGroupSubTable):prop_def_type = PropertyDefinition.Type.GROUPgroup_type_index = table.group_index- elif isinstance(table, EventsTable | FlagEvaluationsTable | AiEventsTable):+ elif isinstance(table, (*EVENT_ROW_TABLES, AiEventsTable)):prop_def_type = PropertyDefinition.Type.EVENTelif isinstance(table, (PersonsTable, RawPersonsTable)):prop_def_type = PropertyDefinition.Type.PERSONdiff --git a/posthog/hogql/transforms/events_predicate_pushdown.py b/posthog/hogql/transforms/events_predicate_pushdown.pyindex 43f6ee117a70..93a644f2c726 100644--- a/posthog/hogql/transforms/events_predicate_pushdown.py+++ b/posthog/hogql/transforms/events_predicate_pushdown.py@@ -80,6 +80,13 @@ def apply_events_predicate_pushdown(return node+# Join types across which moving an events PREDICATE is result-safe (they preserve the events/left side). This is+# broader than the all-rows-preserved check (`_all_joins_preserve_every_row`, LEFT only): INNER / CROSS are+# predicate-safe but can drop an events row, so they pass here yet are rejected when deciding whether to also push+# the LIMIT (`_safe_inner_limit`).+PREDICATE_SAFE_JOIN_TYPES = frozenset({"JOIN", "INNER JOIN", "LEFT JOIN", "LEFT OUTER JOIN", "CROSS JOIN"})++class SelectAliasInliner(CloningVisitor):"""Inlines SELECT-alias references in a pushed predicate so the full aliased expression is carried into thesubquery, not the bare name (which would re-bind to the raw column and drop the wrapper)."""@@ -352,12 +359,6 @@ class EventsPredicatePushdownTransform(TraversingVisitor):Runs after logical lowering (so property reads are PropertyAccess) and before the ClickHouse physical passes,applying bottom-up so nested `FROM events` subqueries benefit too."""- # Join types across which moving an events PREDICATE is result-safe (they preserve the events/left side). This is- # broader than the all-rows-preserved check (`_all_joins_preserve_every_row`, LEFT only): INNER / CROSS are- # predicate-safe but can drop an events row, so they pass here yet are rejected when deciding whether to also push- # the LIMIT (`_safe_inner_limit`).- _SAFE_JOIN_TYPES = {"JOIN", "INNER JOIN", "LEFT JOIN", "LEFT OUTER JOIN", "CROSS JOIN"}-def __init__(self,context: HogQLContext,@@ -596,7 +597,7 @@ class EventsPredicatePushdownTransform(TraversingVisitor):join = node.select_from.next_join if node.select_from else Nonewhile join is not None:# A missing join_type is unreachable today but treated as unsafe rather than safe.- if join.join_type is None or join.join_type not in self._SAFE_JOIN_TYPES:+ if join.join_type is None or join.join_type not in PREDICATE_SAFE_JOIN_TYPES:return set()if join.alias:aliases.add(join.alias)diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.pyindex 6dd60d1986be..125eb627c34e 100644--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py@@ -25,7 +25,13 @@ from posthog.hogql.constants import FLAG_CALLED_EVENTfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.database.models import DatabaseFieldfrom posthog.hogql.database.schema.events import EventsTable-from posthog.hogql.database.schema.flag_evaluations import ELEMENT_STAND_INS, FlagEvaluationsTable+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES+from posthog.hogql.database.schema.flag_evaluations import (+ ELEMENT_STAND_INS,+ FlagEvaluationsTable,+ flag_evaluations_coverage_start,+ utc_midnight,+)from posthog.hogql.database.schema.util.where_clause_extractor import (event_field_type,top_level_conjuncts,@@ -46,12 +52,6 @@ if TYPE_CHECKING:logger = structlog.get_logger(__name__)-# At most FLAG_EVALUATIONS_TTL_DAYS - 1, because the TTL can already have dropped the day FLAG_EVALUATIONS_TTL_DAYS-# back. It stays lower while the table still lacks the oldest days that a shorter TTL dropped. The TTL constant-# lives in posthog/models/flag_evaluations/sql.py, and importing it loads the posthog.models package, which the-# printer's import path avoids.-FLAG_EVALUATIONS_COMPLETE_DAYS = 89-_FLAG_KEY_PROPERTY = "$feature_flag"_Predicate = Callable[[ast.Expr], bool]@@ -112,7 +112,7 @@ _UNMOVABLE_SHAPE = "it reads events in a shape that can't be moved to posthog.fl# An alias with one of these names can shadow the column in an unqualified read._SCAN_COLUMNS = frozenset({"event", "timestamp", "properties"})-_EVENTS_NAMES = frozenset({"events", "posthog.events"})+EVENTS_NAMES = frozenset({"events", "posthog.events"})def can_read_flag_evaluations(@@ -194,14 +194,15 @@ def swap_flag_calls(node: _T_AST, context: HogQLContext, resolve: _Resolve[_T_AST], *, view_name: str | None = None) -> FlagCallSwap[_T_AST]:"""`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading- `posthog.flag_evaluations` instead.+ `posthog.flag_evaluations` instead. When the flag-call union is on, each other scan that reads flag calls with+ other events reads `posthog.events_with_flag_calls`, as `read_flag_calls_with_union` decides.- This returns the swapped query only when it resolves, every name in it binds as it does in the original, and+ This returns the changed query only when it resolves, every name in it binds as it does in the original, andevery field in it that starts with `posthog.events` reads a scan.Otherwise it returns the resolved original. For an organization whose flag calls are only in`flag_evaluations`, it raises a QueryError instead of returning the original. It also raises before it- resolves anything when such an organization's query has a scan of only flag calls with a shape that- `flag_evaluations` cannot stand in for.+ resolves anything when such an organization's query has a scan of flag calls with a shape that neither+ `flag_evaluations` nor the union can stand in for.While `events` still gets flag calls, the original query is the answer whenever the two tables can read a rowdifferently:@@ -222,6 +223,10 @@ def swap_flag_calls(that body already checked them. A top-level query raises when it reads a view whose resolved body still readsonly flag calls from `events`, such as a body that filters with `matchesAction`."""+ # Deferred: flag_calls_union imports this module.+ from posthog.hogql.transforms.flag_calls_union import refuse_union_table_by_name # noqa: PLC0415++ refuse_union_table_by_name(node, context)database = _swap_database(context)if database is None:return FlagCallSwap(source=node, resolved=resolve(node))@@ -244,21 +249,30 @@ def _resolve_with_swaps(node: _T_AST, context: HogQLContext, resolve: _Resolve[_T_AST], database: "Database", view_name: str | None) -> FlagCallSwap[_T_AST]:only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations+ # Deferred: flag_calls_union imports this module.+ from posthog.hogql.transforms.flag_calls_union import ( # noqa: PLC0415+ read_flag_calls_with_union,+ union_unreadable_reason,+ )+with context.timings.measure("flag_calls_from_flag_evaluations"):+ # The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.+ compares_as_string = partial(_flag_key_compares_as_string, context)rewritten = clone_expr(node)clones = _join_clones(node, rewritten) if only_in_flag_evaluations else {}swapper = _EventsScanSwapper(now=datetime.now(UTC),timezone=ZoneInfo(database.get_timezone()),only_in_flag_evaluations=only_in_flag_evaluations,- # The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.- flag_key_compares_as_string=partial(_flag_key_compares_as_string, context),+ flag_key_compares_as_string=compares_as_string,)swapper.visit(rewritten)- if swapper.refusals:+ union = read_flag_calls_with_union(rewritten, context, compares_as_string)+ refusals = [*swapper.refusals, *union.refusals]+ if refusals:if only_in_flag_evaluations:- raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0], view_name)- logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=swapper.refusals[0])+ raise _flag_calls_only_in_flag_evaluations_error(refusals[0], view_name)+ logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=refusals[0])sources = JoinSources() if only_in_flag_evaluations else Noneoriginal = resolve(node, join_sources=sources)if sources is not None:@@ -266,7 +280,7 @@ def _resolve_with_swaps(refusal = swapper.swap_scans_found_by_type(original, sources, clones)if refusal is not None:raise _flag_calls_only_in_flag_evaluations_error(refusal, view_name)- if not swapper.swapped:+ if not swapper.swapped and not union.renamed:return FlagCallSwap(source=node, resolved=original)try:swapped = resolve(rewritten)@@ -274,7 +288,10 @@ def _resolve_with_swaps(# Expected for a query that reads a field flag_evaluations lacks, such as goe_0.properties.reason = f"it reads something that table doesn't have: {error}"else:- if not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):+ unreadable = union_unreadable_reason(swapped, context) if union.renamed else None+ if unreadable is not None:+ reason = unreadable+ elif not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):reason = "a name in it would read a different column there"elif not _posthog_events_qualifiers_read_scans(swapped):reason = "a name in it that starts with posthog.events reads another table"@@ -290,12 +307,16 @@ def _resolve_with_swaps(def _flag_key_compares_as_string(context: HogQLContext) -> bool:- """Whether `properties.$feature_flag` compares as a string. The printer converts a property that the team's- property definitions type as Boolean, Numeric or DateTime before it compares the value."""if context.team_id is None:return True- metadata = _event_property_metadata(context, {_FLAG_KEY_PROPERTY})- return metadata.event_properties.get(_FLAG_KEY_PROPERTY, {}).get("type") in (None, "String")+ return flag_key_typed_as_string(_event_property_metadata(context, {_FLAG_KEY_PROPERTY}))+++def flag_key_typed_as_string(metadata: PropertyMetadata | None) -> bool:+ """Whether `properties.$feature_flag` compares as a string under the property types in `metadata`. The printer+ converts a property that the team's property definitions type as Boolean, Numeric or DateTime before it compares+ the value."""+ return metadata is None or metadata.event_properties.get(_FLAG_KEY_PROPERTY, {}).get("type") in (None, "String")@frozen@@ -366,6 +387,16 @@ class EventsScan:self.shadowed: frozenset[str] = frozenset()self.qualifier_shadows: frozenset[str] = frozenset()+ @classmethod+ def of_scan_chains(cls, flag_key_compares_as_string: Callable[[], bool]) -> "EventsScan":+ """A scan whose reads are unqualified chains, for a predicate already rewritten relative to one scan."""+ return cls(+ ast.JoinExpr(table=ast.Field(chain=["events"])),+ alias="",+ other_tables=frozenset(),+ flag_key_compares_as_string=flag_key_compares_as_string,+ )+@classmethoddef of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]@@ -377,7 +408,7 @@ class EventsScan:return Noneparts = [str(part) for part in join.table.chain]table_name = ".".join(parts)- if table_name not in _EVENTS_NAMES or table_name in ctes:+ if table_name not in EVENTS_NAMES or table_name in ctes:return Nonescan = cls(join=join,@@ -430,7 +461,7 @@ class EventsScan:return _below_posthog_events(chain)def shape_refusal(self, node: ast.SelectQuery) -> str | None:- """Why `flag_evaluations` cannot stand in for this scan whatever its filters say, or None."""+ """Why another table cannot stand in for this scan whatever its filters say, or None."""if self.join.table_args is not None or self.join.table_final:return "it uses FINAL or table arguments on events"if self.join.column_aliases:@@ -483,7 +514,7 @@ class EventsScan:return self.column(expr) == ["properties", _FLAG_KEY_PROPERTY]def bounds_timestamp(self, conjuncts: list[ast.Expr], now: datetime, timezone: ZoneInfo) -> bool:- coverage_start = _utc_midnight(now) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)+ coverage_start = flag_evaluations_coverage_start(now)for conjunct in conjuncts:bound = self._lower_bound(conjunct)if bound is None:@@ -609,7 +640,8 @@ def point_scan_at("""Points `scan` at the table `chain` names, and moves the PREWHERE of `node`, with `condition`, into its WHERE.The new table can read person columns through a join that the printer adds later, and ClickHouse rejects a- PREWHERE that reads a joined column. WHERE selects the same rows.+ PREWHERE that reads a joined column. A query that reads the union reads a subquery, which cannot have a PREWHERE.+ WHERE selects the same rows."""# The old alias keeps qualified reads such as `events.timestamp` resolving.scan.join.alias = scan.alias@@ -653,10 +685,6 @@ def filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:return _conjuncts(node.where) + _conjuncts(node.prewhere)-def _utc_midnight(now: datetime) -> datetime:- return start_of_day(now.astimezone(UTC))--def names_flag_called_event(node: ast.AST) -> bool:"""Whether `$feature_flag_called` appears as a constant anywhere in `node`."""finder = _FlagCalledConstantFinder()@@ -688,7 +716,7 @@ def _earliest_instant(expr: ast.Expr, now: datetime, timezone: ZoneInfo) -> dateif name in ("now", "now64"):return nowif name == "today" and not args:- return _utc_midnight(now) - _DAY_IN_ANY_TIMEZONE+ return utc_midnight(now) - _DAY_IN_ANY_TIMEZONEif name == "assumeNotNull" and len(args) == 1:return _earliest_instant(args[0], now, timezone)if name in ("toDateTime", "toDateTime64") and args:@@ -1205,10 +1233,10 @@ def _target(current: ast.Type | None) -> tuple[str, ...]:json.dumps(join.resolver_params, sort_keys=True, default=str),".".join(str(part) for part in join.from_field),)- elif isinstance(current, ast.TableType):+ elif isinstance(current, (ast.TableType, ast.LazyTableType)):return (("scan",)- if isinstance(current.table, (EventsTable, FlagEvaluationsTable))+ if isinstance(current.table, EVENT_ROW_TABLES)else ("table", current.table.__class__.__name__))elif isinstance(current, ast.CTETableType):diff --git a/posthog/hogql/transforms/flag_calls_union.py b/posthog/hogql/transforms/flag_calls_union.pynew file mode 100644index 000000000000..d4ac41265349--- /dev/null+++ b/posthog/hogql/transforms/flag_calls_union.py@@ -0,0 +1,663 @@+"""Read flag calls from `events` and `posthog.flag_evaluations` together when a query reads them with other events.++The flag_evaluations swap in `flag_calls_from_flag_evaluations` swaps a scan that reads only flag calls to+`posthog.flag_evaluations`. This transform renames each remaining `events` scan that names+`$feature_flag_called` to `posthog.events_with_flag_calls`, a lazy table with the events fields. Lazy table+resolution replaces it with a union of the two tables, split at the coverage start.++The rename runs before type resolution, so every rule here is syntactic. The union returns the right rows for any+scan, so the rules decide which scans pay for a second table, and which shapes the union cannot read yet.+"""++from collections.abc import Callable+from typing import TYPE_CHECKING++from posthog.hogql import ast+from posthog.hogql.constants import FLAG_CALLED_EVENT+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.lazy_join_tags import (+ EVENTS_TO_SESSIONS_V1,+ EVENTS_TO_SESSIONS_V2,+ EVENTS_TO_SESSIONS_V3,+ FOREIGN_KEY,+ GROUP_N,+ PERSON_DISTINCT_ID_OVERRIDES,+ PERSON_DISTINCT_IDS,+ PERSONS,+)+from posthog.hogql.database.schema.events_with_flag_calls import (+ EVENTS_WITH_FLAG_CALLS_CHAIN,+ EVENTS_WITH_FLAG_CALLS_TABLE,+ EventsWithFlagCallsTable,+ FlagEvaluationsColumns,+)+from posthog.hogql.database.schema.util.where_clause_extractor import (+ contains_in_cohort,+ contains_row_multiplying_function,+ is_nondeterministic_function,+)+from posthog.hogql.errors import QueryError, ResolutionError+from posthog.hogql.property_metadata import load_property_metadata+from posthog.hogql.restricted_properties import key_read_overlaps_restriction, restricted_property_keys_for_table_type+from posthog.hogql.transforms.events_predicate_pushdown import PREDICATE_SAFE_JOIN_TYPES, SelectAliasInliner+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (+ EVENTS_NAMES,+ EventsScan,+ ScanVisitor,+ SyntaxVisitor,+ and_conditions,+ boolean_operands,+ compared_values,+ comparison_of,+ filter_conjuncts,+ flag_key_predicate,+ point_scan_at,+)+from posthog.hogql.transforms.property_types import STRING_INPUT_CONVERSIONS+from posthog.hogql.visitor import CloningVisitor++from posthog.dataclasses import frozen++if TYPE_CHECKING:+ from posthog.hogql.database.database import Database++# The joins whose resolver compares only the join's key and `extra_from_fields` on the source table. Lazy table+# resolution projects exactly those from the union, so a join that reads anything else would name a missing column.+_UNION_JOIN_RESOLVERS = frozenset(+ {+ EVENTS_TO_SESSIONS_V1,+ EVENTS_TO_SESSIONS_V2,+ EVENTS_TO_SESSIONS_V3,+ FOREIGN_KEY,+ GROUP_N,+ PERSON_DISTINCT_ID_OVERRIDES,+ PERSON_DISTINCT_IDS,+ PERSONS,+ }+)+++def refuse_union_table_by_name(node: ast.AST, context: HogQLContext) -> None:+ """Raises when `node` names the union table. The table exists only for the rename to point at, so a query that+ names it would depend on an internal table."""+ if context.database is None or not context.database.has_table(EVENTS_WITH_FLAG_CALLS_CHAIN):+ return+ finder = _UnionTableFinder(context.database)+ finder.visit(node)+ if finder.found:+ raise QueryError(f"Unknown table `posthog.{EVENTS_WITH_FLAG_CALLS_TABLE}`.")+++@frozen+class UnionRename:+ renamed: bool+ # Why the union cannot read each other scan that names `$feature_flag_called` with other events.+ refusals: tuple[str, ...]+++def read_flag_calls_with_union(+ node: ast.AST, context: HogQLContext, flag_key_compares_as_string: Callable[[], bool]+) -> UnionRename:+ """Points each mixed flag-call scan in `node` at the union, in place, when the union reads this query's scans.++ An organization whose flag calls are only in `flag_evaluations` cannot read a refused scan at all. Every other+ organization keeps reading `events` for it.+ """+ database = _union_database(context)+ if database is None:+ return UnionRename(renamed=False, refusals=())+ only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations+ if not only_in_flag_evaluations and _flag_key_is_restricted(database, context):+ # The union tells the flag calls that only events holds by their raw `$feature_flag`, with `_HAS_FLAG_KEY`, and+ # a restricted property reads as missing. Each flag call since the cutoff would then come from both tables.+ return UnionRename(renamed=False, refusals=())+ return rename_flag_call_scans_to_union(+ node, flag_key_compares_as_string, only_in_flag_evaluations=only_in_flag_evaluations+ )+++def union_unreadable_reason(node: ast.AST, context: HogQLContext) -> str | None:+ """Why the union cannot read a scan in the resolved `node`, or None when it can read them all.++ Lazy table resolution builds the union after this check, from these same fields, the keys of the joins that start+ from the scan, and the other fields those joins declare that they read.+ """+ database = _union_database(context)+ if database is None:+ return None+ collector = _UnionChainCollector()+ collector.visit(node)+ if collector.unreadable_joins:+ return f"it reads {collector.unreadable_joins[0]}, a join that the union can't read yet"+ columns = FlagEvaluationsColumns(+ database.get_table(EVENTS_WITH_FLAG_CALLS_CHAIN), database.get_table(["posthog", "flag_evaluations"])+ )+ if collector.export_collision is not None:+ first, second = (".".join(str(part) for part in chain) for chain in collector.export_collision)+ return f"it reads {first} and {second}, which the union would return as one column"+ for chain in collector.chains:+ if columns.chain(chain) is None:+ return f"it reads {'.'.join(str(part) for part in chain)}, which that table doesn't have"+ return _typed_property_parsed_as_string(collector.string_parsed, context)+++def _typed_property_parsed_as_string(reads: list[tuple[str, str]], context: HogQLContext) -> str | None:+ # Each branch converts a typed property before the union returns it, so a call that parses the raw string, such+ # as toFloatOrZero, would get the converted value.+ if not reads or context.team_id is None:+ return None+ metadata = load_property_metadata(+ context,+ event_property_names={name for kind, name in reads if kind == "event"},+ person_property_names={name for kind, name in reads if kind == "person"},+ group_property_names={},+ )+ for kind, name in reads:+ types = metadata.event_properties if kind == "event" else metadata.person_properties+ property_type = types.get(name, {}).get("type")+ if property_type not in (None, "String"):+ return f"it parses the {property_type} {kind} property {name} as a string"+ return None+++@frozen+class UnionBranchFilters:+ """The predicates that a union scan's branches copy from the query that reads the scan.++ Each is a set of conjuncts from that query's WHERE and PREWHERE, so every row the query keeps passes it.+ """++ events: ast.Expr | None+ # For the branch that reads flag_evaluations. A read of `event` is `$feature_flag_called` there.+ flag_evaluations: ast.Expr | None+ # False when the query pins `$feature_flag` to keys that only a call with a non-empty string key matches.+ keyless_flag_calls_can_match: bool+++def union_branch_filters(+ node: ast.SelectQuery,+ union: EventsWithFlagCallsTable,+ columns: FlagEvaluationsColumns,+ flag_key_compares_as_string: Callable[[], bool],+) -> UnionBranchFilters:+ """The filters for the union scan that `node` reads, from the resolved WHERE and PREWHERE of `node`."""+ extracted = _scan_conjuncts(node, union)+ flag_call_conjuncts = [_fold(_EventAsFlagCall().visit(conjunct)) for conjunct in extracted]+ # The flag_evaluations swap accepts a key only when it is a non-empty string that no other JSON value prints as.+ # A flag call that ingestion left in events, with a missing, empty or non-string key, cannot match one.+ keys = EventsScan.of_scan_chains(flag_key_compares_as_string).flag_keys(+ [_StepTests().visit(conjunct) for conjunct in flag_call_conjuncts]+ )+ flag_evaluations_conjuncts = [+ mapped+ for conjunct in flag_call_conjuncts+ if _truth(conjunct) is not True and (mapped := _FlagEvaluationsReads(columns).copy(conjunct)) is not None+ ]+ return UnionBranchFilters(+ events=_events_filter(extracted),+ flag_evaluations=and_conditions([*flag_evaluations_conjuncts, flag_key_predicate("flag_evaluations", keys)]),+ keyless_flag_calls_can_match=not keys,+ )+++def union_events_filter(node: ast.SelectQuery, union: EventsWithFlagCallsTable) -> ast.Expr | None:+ """The filter that a scan of `events` copies from `node`, as `union_branch_filters` builds it."""+ return _events_filter(_scan_conjuncts(node, union))+++def rename_flag_call_scans_to_union(+ node: ast.AST,+ flag_key_compares_as_string: Callable[[], bool],+ *,+ only_in_flag_evaluations: bool,+) -> UnionRename:+ """Point each `events` scan that the union can read, and whose query names `$feature_flag_called`, at the union.++ A scan qualifies when all of these hold:++ 1. Its query compares the scan's `event` with `$feature_flag_called` by `=` or `IN`, in one of the query's own+ clauses. A subquery's comparison does not count. A funnel step column such as+ `if(event = '$feature_flag_called' AND ..., 1, 0)` counts.+ 2. The scan is the first table in FROM, and every join after it is INNER, LEFT or CROSS, GLOBAL or not.+ 3. The scan has no FINAL, table arguments, column aliases or SAMPLE other than 1, and its query selects no `*`+ or `COLUMNS(...)`.++ A scan that the flag_evaluations swap changed reads `posthog.flag_evaluations` and is not an `events` scan any+ more. The flag_evaluations swap swaps or refuses a first scan that reads only flag calls, so the union neither+ renames nor refuses it. The swap cannot read an unqualified `event` next to a join. On mode 1 a first scan that+ reads only flag calls through such an `event` stays on events. On mode 2 the union renames it.+ """+ renamer = _UnionRenamer(flag_key_compares_as_string, only_in_flag_evaluations)+ renamer.visit(node)+ return UnionRename(renamed=renamer.renamed, refusals=tuple(renamer.refusals))+++def _union_database(context: HogQLContext) -> "Database | None":+ """The database when the union reads this query's mixed flag-call scans, and None otherwise."""+ if (+ not context.rewrite_flag_calls_to_flag_evaluations+ or context.database is None+ or not context.database.read_flag_calls_with_union+ ):+ return None+ return context.database+++def _flag_key_is_restricted(database: "Database", context: HogQLContext) -> bool:+ restricted = restricted_property_keys_for_table_type(ast.TableType(table=database.get_table("events")), context)+ return key_read_overlaps_restriction("$feature_flag", restricted)+++def _union_table_of(table_type: ast.Type | None) -> EventsWithFlagCallsTable | None:+ while isinstance(table_type, (ast.TableAliasType, ast.VirtualTableType)):+ table_type = table_type.table_type+ if isinstance(table_type, ast.LazyTableType) and isinstance(table_type.table, EventsWithFlagCallsTable):+ return table_type.table+ return None+++def _union_chain(field_type: ast.Type | None) -> tuple[EventsWithFlagCallsTable, list[str | int]] | None:+ """The union scan that a resolved field reads directly, and its chain there, or None."""+ property_chain: list[str | int] = []+ if isinstance(field_type, ast.PropertyType):+ property_chain = list(field_type.chain)+ field_type = field_type.field_type+ if not isinstance(field_type, ast.FieldType):+ return None+ chain: list[str | int] = [field_type.name, *property_chain]+ if isinstance(field_type.table_type, ast.VirtualTableType):+ chain = [field_type.table_type.field, *chain]+ union = _union_table_of(field_type.table_type)+ return (union, chain) if union is not None else None+++def _scan_conjuncts(node: ast.SelectQuery, union: EventsWithFlagCallsTable) -> list[ast.Expr]:+ """The parts of the WHERE and PREWHERE of `node` that read only the union scan, with reads as chains on it."""+ inliner = SelectAliasInliner({expr.alias: expr.expr for expr in node.select if isinstance(expr, ast.Alias)})+ extracted = [_scan_predicate(inliner.visit(conjunct), union) for conjunct in filter_conjuncts(node)]+ return [conjunct for conjunct in extracted if conjunct is not None]+++def _scan_predicate(expr: ast.Expr, union: EventsWithFlagCallsTable) -> ast.Expr | None:+ """`expr` with each read as a chain on the scan, or the conjuncts of it that read only the scan, or None.++ An OR counts only when every branch of it does, so every row that `expr` accepts passes the result.+ """+ if (parts := boolean_operands(expr, ast.And)) is not None:+ return and_conditions([_scan_predicate(part, union) for part in parts])+ if (branches := boolean_operands(expr, ast.Or)) is not None:+ extracted = [_scan_predicate(branch, union) for branch in branches]+ kept = [branch for branch in extracted if branch is not None]+ return ast.Or(exprs=kept) if kept and len(kept) == len(extracted) else None+ # A copy in each branch would multiply rows, or would move a cohort join that the printer adds later.+ if contains_row_multiplying_function(expr) or contains_in_cohort(expr):+ return None+ copier = _ScanReadCopier(union)+ copied = copier.visit(expr)+ return None if copier.failed else copied+++def _events_filter(scan_conjuncts: list[ast.Expr]) -> ast.Expr | None:+ return and_conditions([_Qualified("events").visit(conjunct) for conjunct in scan_conjuncts])+++def _truth(expr: ast.Expr) -> bool | None:+ """The truth value of a constant false, true, 0 or 1, or None for any other expression."""+ if isinstance(expr, ast.Constant) and expr.value in (0, 1):+ return bool(expr.value)+ return None+++# Whether a comparison of constants passes when the left value is among the right values.+_MATCH_PASSES = {+ ast.CompareOperationOp.Eq: True,+ ast.CompareOperationOp.In: True,+ ast.CompareOperationOp.NotEq: False,+ ast.CompareOperationOp.NotIn: False,+}+++def _compares_like_clickhouse(values: list[object]) -> bool:+ """Whether Python and ClickHouse agree on how these constants compare.++ ClickHouse converts mixed types before it compares (`1 IN ('1')` is true), which Python does not.+ NULL and floats are neither str nor int, so they never fold.+ """+ return any(all(isinstance(value, kind) for value in values) for kind in (str, int))+++def _fold(expr: ast.Expr) -> ast.Expr:+ """`expr` with comparisons of constants, and the AND, OR, NOT and `if` they decide, replaced by their value."""+ if (parts := boolean_operands(expr, ast.And)) is not None:+ folded = [_fold(part) for part in parts]+ if any(_truth(part) is False for part in folded):+ return ast.Constant(value=False)+ return and_conditions([part for part in folded if _truth(part) is not True]) or ast.Constant(value=True)+ if (parts := boolean_operands(expr, ast.Or)) is not None:+ folded = [_fold(part) for part in parts]+ if any(_truth(part) is True for part in folded):+ return ast.Constant(value=True)+ kept = [part for part in folded if _truth(part) is not False]+ if not kept:+ return ast.Constant(value=False)+ return kept[0] if len(kept) == 1 else ast.Or(exprs=kept)+ if isinstance(expr, ast.Not) or (isinstance(expr, ast.Call) and expr.name == "not" and len(expr.args) == 1):+ inner = _fold(expr.expr if isinstance(expr, ast.Not) else expr.args[0])+ truth = _truth(inner)+ return ast.Constant(value=not truth) if truth is not None else ast.Not(expr=inner)+ if isinstance(expr, ast.Call) and expr.name == "if" and len(expr.args) == 3:+ args = [_fold(arg) for arg in expr.args]+ truth = _truth(args[0])+ if truth is not None:+ return args[1] if truth else args[2]+ return ast.Call(name="if", args=args)+ compared = comparison_of(expr)+ if compared is None:+ return expr+ op, left, right = compared.op, _fold(compared.left), _fold(compared.right)+ values = right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]+ constants = [value.value for value in values if isinstance(value, ast.Constant)]+ if (+ op in _MATCH_PASSES+ and isinstance(left, ast.Constant)+ and len(constants) == len(values)+ and _compares_like_clickhouse([left.value, *constants])+ and (len(values) == 1 or op in (ast.CompareOperationOp.In, ast.CompareOperationOp.NotIn))+ ):+ return ast.Constant(value=(left.value in constants) == _MATCH_PASSES[op])+ return ast.CompareOperation(op=op, left=left, right=right)+++def _step_condition(expr: ast.Expr) -> ast.Expr | None:+ """`c` for a funnel step test such as `if(c, 1, 0) = 1`, or None for any other expression."""+ compared = comparison_of(expr)+ if compared is None or compared.op != ast.CompareOperationOp.Eq:+ return None+ for step, value in ((compared.left, compared.right), (compared.right, compared.left)):+ if (+ isinstance(step, ast.Call)+ and step.name == "if"+ and len(step.args) == 3+ and _truth(step.args[1]) is True+ and _truth(step.args[2]) is False+ and _truth(value) is True+ ):+ return step.args[0]+ return None+++def _union_refusal(node: ast.SelectQuery, scan: EventsScan, position: int) -> str | None:+ """Why the union cannot stand in for `scan`, the scan at `position` in the FROM chain of `node`, or None."""+ if position > 0:+ return "it joins events after another table"+ refusal = scan.shape_refusal(node)+ if refusal is not None:+ return refusal+ join = scan.join.next_join+ while join is not None:+ # A predicate moves into the union's branches only across these joins. GLOBAL changes only where ClickHouse+ # builds the right side, not which rows the join keeps.+ if join.join_type is None or join.join_type.removeprefix("GLOBAL ") not in PREDICATE_SAFE_JOIN_TYPES:+ return f"it joins another table to events with {join.join_type or 'an unknown join type'}"+ join = join.next_join+ return None+++def _flag_call_scans(+ node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]+) -> list[tuple[int, EventsScan]]:+ """Each `events` scan in the FROM chain of `node` whose query names `$feature_flag_called`, with its position."""+ scans: list[tuple[int, EventsScan]] = []+ join = node.select_from+ position = 0+ while join is not None:+ # The rename runs before type resolution, so the chain is all it can read. EventsScan.of matches the same+ # names, and the binding check rejects a rename that changes what a name binds to.+ # nosemgrep: hogql-no-string-table-chain+ name = ".".join(str(part) for part in join.table.chain) if isinstance(join.table, ast.Field) else None+ if name is not None and name in EVENTS_NAMES and name not in ctes:+ # Only the first scan can read unqualified names, and EventsScan.of decides when it can.+ scan = (+ EventsScan.of(node, flag_key_compares_as_string, ctes)+ if position == 0+ else EventsScan(+ join,+ join.alias or name.replace(".", "__"),+ other_tables=frozenset(),+ qualified_only=True,+ flag_key_compares_as_string=flag_key_compares_as_string,+ )+ )+ if scan is not None and _names_flag_calls(node, scan):+ scans.append((position, scan))+ join = join.next_join+ position += 1+ return scans+++def _names_flag_calls(node: ast.SelectQuery, scan: EventsScan) -> bool:+ finder = _FlagCallComparisonFinder(node, scan)+ finder.visit(node)+ return finder.found+++class _ScanReadCopier(CloningVisitor):+ """Copies an expression that reads only one union scan, with each read as its chain on the scan."""++ def __init__(self, union: EventsWithFlagCallsTable) -> None:+ super().__init__(clear_types=True, clear_locations=True)+ self.union = union+ self.failed = False++ def visit_field(self, node: ast.Field):+ read = _union_chain(node.type)+ if read is None or read[0] is not self.union:+ self.failed = True+ return super().visit_field(node)+ return ast.Field(chain=read[1])++ def visit_alias(self, node: ast.Alias):+ # A hidden alias wraps an expression field, such as `person_id`, that the resolver expanded in place.+ if not node.hidden:+ self.failed = True+ return self.visit(node.expr)++ def visit_call(self, node: ast.Call):+ # Each branch would call the function again and filter on a different value.+ if is_nondeterministic_function(node.name):+ self.failed = True+ return super().visit_call(node)++ def _refuse(self, node):+ self.failed = True+ return node++ visit_select_query = visit_select_set_query = visit_lambda = _refuse+++class _EventAsFlagCall(CloningVisitor):+ def visit_field(self, node: ast.Field):+ if node.chain == ["event"]:+ return ast.Constant(value=FLAG_CALLED_EVENT)+ return super().visit_field(node)+++class _StepTests(CloningVisitor):+ """Reads each funnel step test such as `if(c, 1, 0) = 1` as `c`, to find the keys a predicate pins.++ The copy is not a filter: for a NULL `c` the test is false, and `c` is NULL.+ """++ def visit_compare_operation(self, node: ast.CompareOperation):+ condition = _step_condition(node)+ return self.visit(condition) if condition is not None else super().visit_compare_operation(node)++ def visit_call(self, node: ast.Call):+ condition = _step_condition(node)+ return self.visit(condition) if condition is not None else super().visit_call(node)+++class _Qualified(CloningVisitor):+ def __init__(self, table: str) -> None:+ super().__init__()+ self.table = table++ def visit_field(self, node: ast.Field):+ return ast.Field(chain=[self.table, *node.chain])+++class _FlagEvaluationsReads(CloningVisitor):+ def __init__(self, columns: FlagEvaluationsColumns) -> None:+ super().__init__()+ self.columns = columns+ self.failed = False++ def copy(self, expr: ast.Expr) -> ast.Expr | None:+ copied = self.visit(expr)+ return None if self.failed else copied++ def visit_field(self, node: ast.Field):+ chain = self.columns.chain(node.chain)+ if chain is None:+ self.failed = True+ return super().visit_field(node)+ return ast.Field(chain=["flag_evaluations", *chain])+++class _UnionChainCollector(SyntaxVisitor):+ """Collects the chain each resolved field reads on a union scan, as lazy table resolution will request it."""++ def __init__(self) -> None:+ super().__init__()+ self.chains: list[list[str | int]] = []+ self.unreadable_joins: list[str] = []+ self.exports: dict[tuple[int, str], list[str | int]] = {}+ self._select: ast.SelectQuery | None = None+ self.export_collision: tuple[list[str | int], list[str | int]] | None = None+ # The event and person properties that a string-parsing call such as toFloatOrZero reads from the scan.+ self.string_parsed: list[tuple[str, str]] = []+ self._in_string_input = False++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ outer = self._select+ self._select = node+ super().visit_select_query(node)+ self._select = outer++ def visit_call(self, node: ast.Call) -> None:+ # The property type swapper keeps a property raw only as the direct argument of such a call.+ outer = self._in_string_input+ self._in_string_input = node.name in STRING_INPUT_CONVERSIONS+ super().visit_call(node)+ self._in_string_input = outer++ def visit_field(self, node: ast.Field) -> None:+ read = _union_chain(node.type)+ if read is not None:+ chain = read[1]+ self.chains.append(chain)+ # Lazy table resolution names a whole-field read after the field, whichever table the field is on.+ # It names a property read by joining the path with `___`, so `a.b` and `a___b` share a name.+ name = node.type.name if isinstance(node.type, ast.FieldType) else "___".join(str(link) for link in chain)+ first = self.exports.setdefault((id(self._select), name), chain)+ if first != chain and self.export_collision is None:+ self.export_collision = (first, chain)+ if self._in_string_input and chain[:-1] in (["properties"], ["poe", "properties"]):+ self.string_parsed.append(("event" if chain[0] == "properties" else "person", str(chain[-1])))+ return+ field_type = node.type.field_type if isinstance(node.type, ast.PropertyType) else node.type+ if not isinstance(field_type, ast.FieldType):+ return+ table_type: ast.Type = field_type.table_type+ while isinstance(table_type, ast.VirtualTableType):+ table_type = table_type.table_type+ # A join that starts from the scan, such as `override` or `person`, reads its key from the union.+ while isinstance(table_type, ast.LazyJoinType):+ source = table_type.table_type+ while isinstance(source, ast.TableAliasType):+ source = source.table_type+ if _union_table_of(source) is not None:+ lazy_join = table_type.lazy_join+ if isinstance(source, ast.VirtualTableType) or lazy_join.resolver not in _UNION_JOIN_RESOLVERS:+ self.unreadable_joins.append(table_type.field)+ else:+ self.chains.extend([list(lazy_join.from_field), *lazy_join.extra_from_fields])+ return+ table_type = table_type.table_type+++class _UnionTableFinder(SyntaxVisitor):+ def __init__(self, database: "Database") -> None:+ super().__init__()+ self.database = database+ self.found = False++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ if isinstance(node.table, ast.Field):+ try:+ # Unlike get_table, a miss here builds no table name suggestions.+ table = self.database.get_table_node([str(part) for part in node.table.chain]).table+ except ResolutionError:+ # A CTE or a subquery alias, which only the resolver can tell apart.+ table = None+ self.found = self.found or isinstance(table, EventsWithFlagCallsTable)+ super().visit_join_expr(node)+++class _FlagCallComparisonFinder(SyntaxVisitor):+ """Finds `event = '$feature_flag_called'` or an `IN` list with it, on the scan, in its own query's clauses."""++ def __init__(self, root: ast.SelectQuery, scan: EventsScan) -> None:+ super().__init__()+ self.root = root+ self.reader = scan+ self.found = False++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ if node is self.root:+ super().visit_select_query(node)++ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:+ pass++ def visit_compare_operation(self, node: ast.CompareOperation) -> None:+ self._check(node)+ super().visit_compare_operation(node)++ def visit_call(self, node: ast.Call) -> None:+ self._check(node)+ super().visit_call(node)++ def _check(self, expr: ast.Expr) -> None:+ values = compared_values(expr, lambda read: self.reader.column(read) == ["event"])+ if values and any(isinstance(value, ast.Constant) and value.value == FLAG_CALLED_EVENT for value in values):+ self.found = True+++class _UnionRenamer(ScanVisitor):+ def __init__(self, flag_key_compares_as_string: Callable[[], bool], only_in_flag_evaluations: bool) -> None:+ super().__init__()+ self.flag_key_compares_as_string = flag_key_compares_as_string+ self.only_in_flag_evaluations = only_in_flag_evaluations+ self.renamed = False+ self.refusals: list[str] = []++ def visit_scan_query(self, node: ast.SelectQuery) -> None:+ # Only the first scan can be renamed, and it comes first, so each refusal reads the query as written.+ for position, scan in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes):+ if position == 0 and self._reads_only_flag_calls(node, scan):+ continue+ refusal = _union_refusal(node, scan, position)+ if refusal is not None:+ self.refusals.append(refusal)+ else:+ point_scan_at(node, scan, list(EVENTS_WITH_FLAG_CALLS_CHAIN))+ # A lazy table cannot be sampled, and SAMPLE 1 keeps every row anyway.+ scan.join.sample = None+ self.renamed = True++ def _reads_only_flag_calls(self, node: ast.SelectQuery, scan: EventsScan) -> bool:+ # The flag_evaluations swap handles a first scan whose `event` filter it reads as only flag calls. It swaps or+ # refuses that scan on mode 2. On mode 1 events holds every flag call, so the union would read the same calls+ # twice there.+ return scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False)diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.pyindex 3c7734af706f..71c54da57083 100644--- a/posthog/hogql/transforms/lazy_tables.py+++ b/posthog/hogql/transforms/lazy_tables.py@@ -5,7 +5,8 @@ from posthog.hogql import astfrom posthog.hogql.base import _T_ASTfrom posthog.hogql.constants import HogQLDialectfrom posthog.hogql.context import HogQLContext-from posthog.hogql.database.models import LazyJoinToAdd, LazyTableToAdd+from posthog.hogql.database.models import LazyJoinToAdd, LazyTableToAdd, UnexpandedScan+from posthog.hogql.database.schema.events_with_flag_calls import EventsWithFlagCallsTablefrom posthog.hogql.errors import ResolutionErrorfrom posthog.hogql.resolver import ResolverFactory, resolve_typesfrom posthog.hogql.resolver_utils import get_long_table_name@@ -22,6 +23,23 @@ from posthog.hogql.visitor import TraversingVisitor, clone_exprfrom posthog.dataclasses import frozen+def _lazy_table_of(table_type: ast.Type) -> object:+ while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):+ table_type = table_type.table_type+ return table_type.table if isinstance(table_type, ast.LazyTableType) else None+++def _unexpanded_scan(join: ast.JoinExpr, table_type: ast.Type) -> UnexpandedScan:+ """A copy of `join` as the query wrote it, with the table types that name its scan."""+ types: list[ast.Type] = [table_type]+ while isinstance(types[-1], (ast.TableAliasType, ast.ColumnAliasedTableType)):+ types.append(types[-1].table_type)+ scan = ast.JoinExpr(+ table=join.table, alias=join.alias, sample=join.sample, table_args=join.table_args, type=join.type+ )+ return UnexpandedScan(scan, tuple(types))++# This mutates the nodesdef resolve_lazy_tables(node: _T_AST,@@ -52,6 +70,13 @@ class FieldChainReplacer(TraversingVisitor):if node.chain == constraint.chain_to_replace:node.chain = [constraint.table_name, constraint.alias]+ # A subquery has its own scope. Its `events` is the table it reads, not the outer scan with the override column.+ def visit_select_query(self, node: ast.SelectQuery):+ pass++ def visit_select_set_query(self, node: ast.SelectSetQuery):+ pass+class FieldFinder(TraversingVisitor):field_chains: list[list[str | int]] = []@@ -309,13 +334,18 @@ class LazyTableResolver(TraversingVisitor):def _wrap_table_for_lazy_resolution(self, node: ast.SelectQuery, table_alias: str, select_type: ast.SelectQueryType- ) -> None:+ ) -> UnexpandedScan | None:# The scope key of a schema-qualified table (`posthog__events`) is not its written name, so match the binding.original_table_type = select_type.tables[table_alias]join_ptr = node.select_fromwhile join_ptr is not None and join_ptr.type is not original_table_type:join_ptr = join_ptr.next_joinassert join_ptr is not None and isinstance(join_ptr.table, ast.Field), f"Table {table_alias} not found"+ scan = (+ _unexpanded_scan(join_ptr, original_table_type)+ if isinstance(_lazy_table_of(original_table_type), EventsWithFlagCallsTable)+ else None+ )# Collect while the outer reads still identify the table they belong to.collector = WrappedTableReferenceCollector(original_table_type, self.context)@@ -363,6 +393,8 @@ class LazyTableResolver(TraversingVisitor):join_ptr.sample = Nonejoin_ptr.type = ast.SelectQueryAliasType(alias=table_alias, select_query_type=subquery.type)select_type.tables[table_alias] = join_ptr.type+ if scan is not None:+ scan = UnexpandedScan(scan.join, (*scan.types, join_ptr.type))for property_type, export_name, through_lazy_join in collector.properties.values():if id(property_type.field_type) in collector.fields:@@ -380,6 +412,7 @@ class LazyTableResolver(TraversingVisitor):for field_type, export_name in collector.fields.values():field_type.name = export_namefield_type.table_type = join_ptr.type+ return scandef _read_property_path(self, subquery_type: ast.SelectQueryType, column: ast.Alias, property_path: tuple[str | int, ...]@@ -693,8 +726,11 @@ class LazyTableResolver(TraversingVisitor):# This must happen BEFORE we add lazy joins to avoid forward reference issues.tables_to_wrap = self._find_tables_needing_lazy_preresolution(node, select_type)wrapped_table_types = [select_type.tables[table_alias] for table_alias in tables_to_wrap]+ unexpanded_scans: dict[str, UnexpandedScan] = {}for table_alias in tables_to_wrap:- self._wrap_table_for_lazy_resolution(node, table_alias, select_type)+ wrapped_scan = self._wrap_table_for_lazy_resolution(node, table_alias, select_type)+ if wrapped_scan is not None:+ unexpanded_scans[table_alias] = wrapped_scan# Remove lazy joins that are now resolved inside wrapped subqueriesjoins_to_add = {@@ -734,6 +770,8 @@ class LazyTableResolver(TraversingVisitor):and join_ptr.table.type == old_table_type.table_type)):+ if isinstance(table_to_add.lazy_table, EventsWithFlagCallsTable):+ unexpanded_scans[table_name] = _unexpanded_scan(join_ptr, old_table_type)join_ptr.table = subqueryjoin_ptr.type = select_type.tables[table_name]join_ptr.alias = table_name@@ -752,6 +790,7 @@ class LazyTableResolver(TraversingVisitor):# For all the collected joins, create the join subqueries, and add them to the table.for to_table, join_scope in joins_to_add.items():+ join_scope.from_scan = unexpanded_scans.get(join_scope.from_table)join_to_add: ast.JoinExpr = join_scope.lazy_join.resolve_join_to_add(join_scope,self.context,diff --git a/posthog/hogql/transforms/property_types.py b/posthog/hogql/transforms/property_types.pyindex 73b255f405c8..447fb1be1e85 100644--- a/posthog/hogql/transforms/property_types.py+++ b/posthog/hogql/transforms/property_types.py@@ -6,13 +6,9 @@ from posthog.hogql.constants import EXCEPTION_STRING_ARRAY_PROPERTIES, is_virtuafrom posthog.hogql.context import HogQLContextfrom posthog.hogql.database.models import BooleanDatabaseField, DateTimeDatabaseField, StringJSONDatabaseFieldfrom posthog.hogql.database.s3_table import S3Table-from posthog.hogql.database.schema.events import (- EVENTS_TABLE_TYPES,- EventsGroupSubTable,- EventsPersonSubTable,- EventsTable,-)-from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable, FlagEvaluationsTable+from posthog.hogql.database.schema.events import EVENTS_TABLE_TYPES, EventsGroupSubTable, EventsPersonSubTable+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTablefrom posthog.hogql.database.schema.groups import GroupsTablefrom posthog.hogql.database.schema.persons import PersonsTable, RawPersonsTablefrom posthog.hogql.escape_sql import escape_hogql_identifier@@ -118,7 +114,7 @@ class PropertyFinder(TraversingVisitor):self.person_properties.add(property_name)elif isinstance(resolved_table, EventsGroupSubTable):pass # group properties are handled above via GroupsTable- elif isinstance(resolved_table, EventsTable | FlagEvaluationsTable):+ elif isinstance(resolved_table, EVENT_ROW_TABLES):self.event_properties.add(property_name)def visit_field(self, node: ast.Field):@@ -137,6 +133,14 @@ class ToTimeZoneParts:swapped: bool+# ClickHouse string-parsing conversions (toFloat64OrZero, toInt64OrZero,+# toFloat64OrDefault, toInt64OrDefault) require a String first argument and raise+# ILLEGAL_TYPE_OF_ARGUMENT on numeric input. When a user explicitly wraps a+# Numeric-typed property in one of these, we must not auto-convert the property+# to Float, the raw String value has to flow through for the parser to work.+STRING_INPUT_CONVERSIONS = frozenset({"toFloatOrZero", "toIntOrZero", "toFloatOrDefault", "toIntOrDefault"})++class PropertySwapper(CloningVisitor):_RANGE_OPS: set[str] = {ast.CompareOperationOp.Gt,@@ -145,13 +149,6 @@ class PropertySwapper(CloningVisitor):ast.CompareOperationOp.LtEq,}- # ClickHouse string-parsing conversions (toFloat64OrZero, toInt64OrZero,- # toFloat64OrDefault, toInt64OrDefault) require a String first argument and raise- # ILLEGAL_TYPE_OF_ARGUMENT on numeric input. When a user explicitly wraps a- # Numeric-typed property in one of these, we must not auto-convert the property- # to Float, the raw String value has to flow through for the parser to work.- _STRING_INPUT_CONVERSIONS: set[str] = {"toFloatOrZero", "toIntOrZero", "toFloatOrDefault", "toIntOrDefault"}-# ClickHouse array-membership functions whose first argument must be an array. Users write these# against exception properties (e.g. hasAny(properties.$exception_values, [...])), but those# properties are stored as a raw JSON String once materialized, so the bare column read raises@@ -248,7 +245,7 @@ class PropertySwapper(CloningVisitor):# string. Re-evaluated per call, so nested non-parsing calls (e.g.# toFloatOrZero(toString(prop))) correctly reset the flag.saved_suppress = self._suppress_numeric_conversion- self._suppress_numeric_conversion = node.name in self._STRING_INPUT_CONVERSIONS+ self._suppress_numeric_conversion = node.name in STRING_INPUT_CONVERSIONSself._inside_call_depth += 1try:@@ -287,7 +284,7 @@ class PropertySwapper(CloningVisitor):table_type = type.field_type.table_typeif not isinstance(table_type, ast.BaseTableType):return False- return isinstance(table_type.resolve_database_table(self.context), EventsTable | FlagEvaluationsTable)+ return isinstance(table_type.resolve_database_table(self.context), EVENT_ROW_TABLES)@staticmethoddef _extract_string_array(field: ast.Expr) -> ast.Call:@@ -738,7 +735,7 @@ class PropertySwapper(CloningVisitor):if isinstance(resolved_table, EventsPersonSubTable | FlagEvaluationsPersonSubTable):if property_name in self.person_properties:return ("person", property_name)- elif isinstance(resolved_table, EventsTable | FlagEvaluationsTable):+ elif isinstance(resolved_table, EVENT_ROW_TABLES):if property_name in self.event_properties:return ("event", property_name)if type.field_type.name == "person_properties" and len(type.chain) == 1:diff --git a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.pyindex abf7589fabe0..9f10ae474b25 100644--- a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py+++ b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py@@ -9,9 +9,9 @@ from django.test import SimpleTestCasefrom parameterized import parameterizedfrom posthog.hogql import ast+from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYSfrom posthog.hogql.parser import parse_selectfrom posthog.hogql.transforms.flag_calls_from_flag_evaluations import (- FLAG_EVALUATIONS_COMPLETE_DAYS,EventsScan,can_read_flag_evaluations,filter_conjuncts,diff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.pynew file mode 100644index 000000000000..66963d46b273--- /dev/null+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py@@ -0,0 +1,918 @@+import re+import uuid+from datetime import UTC, datetime, timedelta+from typing import cast++import pytest+from posthog.test.base import (+ APIBaseTest,+ BaseTest,+ ClickhouseTestMixin,+ _create_event,+ _create_flag_evaluations,+ _create_person,+ cleanup_materialized_columns,+ flush_persons_and_events,+)+from unittest.mock import patch++from django.test import SimpleTestCase++from parameterized import param, parameterized++from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode++from posthog.hogql import ast+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.database.models import DatabaseField, FieldTraverser, LazyJoin, StringJSONDatabaseField, VirtualTable+from posthog.hogql.database.schema.util.where_clause_extractor import is_nondeterministic_function+from posthog.hogql.errors import QueryError+from posthog.hogql.functions.mapping import HOGQL_CLICKHOUSE_FUNCTIONS+from posthog.hogql.parser import parse_select+from posthog.hogql.printer import prepare_and_print_ast+from posthog.hogql.query import HogQLQueryExecutor, execute_hogql_query+from posthog.hogql.transforms.flag_calls_union import rename_flag_call_scans_to_union+from posthog.hogql.visitor import TraversingVisitor, clone_expr++from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.escape import substitute_params+from posthog.constants import AvailableFeature+from posthog.test.persons import create_group_type_mapping++from products.access_control.backend.models.property_access_control import PropertyAccessControl+from products.access_control.backend.property_access_control import PropertyAccessLevel+from products.data_tools.backend.models.join import DataWarehouseJoin+from products.event_definitions.backend.models.property_definition import PropertyDefinition+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig++from ee.clickhouse.materialized_columns.columns import materialize++MIXED = "event IN ('$pageview', '$feature_flag_called')"+SESSION_ID = "0190a8d1-3f7a-7b00-8000-000000000001"+E_MIXED = "e.event IN ('$pageview', '$feature_flag_called')"+UNION = "posthog.events_with_flag_calls"+READ = FlagEvaluationsMode.READ_FLAG_EVALUATIONS+ONLY = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY+++class _TableCollector(TraversingVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.tables: list[str] = []++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ if isinstance(node.table, ast.Field):+ self.tables.append(".".join(str(part) for part in node.table.chain))+ super().visit_join_expr(node)+++def _select(query: str | ast.SelectQuery) -> ast.SelectQuery:+ return cast(ast.SelectQuery, parse_select(query) if isinstance(query, str) else clone_expr(query))+++def _global_join(query: str) -> ast.SelectQuery:+ # HogQL has no GLOBAL syntax. The recordings list builds a GLOBAL join in its AST.+ select = _select(query)+ assert select.select_from is not None and select.select_from.next_join is not None+ select.select_from.next_join.join_type = "GLOBAL INNER JOIN"+ return select+++def _set_mode(organization, mode: FlagEvaluationsMode) -> None:+ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=organization, defaults={"flag_evaluations_mode": mode}+ )+++def _enabled_flags(union: bool):+ enabled = {"hogql-flag-calls-union"} if union else set()+ return patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: key in enabled,+ )+++def _restrict(organization, team, name: str) -> None:+ organization.available_product_features = [+ {"name": AvailableFeature.PROPERTY_ACCESS_CONTROL, "key": AvailableFeature.PROPERTY_ACCESS_CONTROL}+ ]+ organization.save()+ definition = PropertyDefinition.objects.create(+ team=team, name=name, property_type="String", type=PropertyDefinition.Type.EVENT+ )+ PropertyAccessControl.objects.create(+ team=team, property_definition=definition, access_level=PropertyAccessLevel.NONE.value+ )+++class TestRenameFlagCallScansToUnion(SimpleTestCase):+ @parameterized.expand(+ [+ ("event_in_a_list_with_other_events", f"SELECT count() FROM events WHERE {MIXED}", [UNION], []),+ ("paths_v2_step_sources", f"SELECT e.event FROM events AS e WHERE {E_MIXED}", [UNION], []),+ (+ "funnel_step_column",+ "SELECT if(event = '$feature_flag_called' AND properties.$feature_flag = 'f', 1, 0) AS step_1 "+ "FROM events",+ [UNION],+ [],+ ),+ (+ "or_of_event_comparisons",+ "SELECT count() FROM events WHERE event = '$pageview' OR equals(event, '$feature_flag_called')",+ [UNION],+ [],+ ),+ # The flag_evaluations swap handles this scan. Events holds every flag call on mode 1. The swap swaps or+ # refuses the scan on mode 2.+ ("only_flag_calls", "SELECT count() FROM events WHERE event = '$feature_flag_called'", ["events"], []),+ (+ "only_flag_calls_with_sample",+ "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'",+ ["events"],+ [],+ ),+ (+ "left_join_after_the_scan",+ f"SELECT count() FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",+ [UNION, "persons"],+ [],+ ),+ (+ "left_join_with_an_unqualified_event",+ f"SELECT count() FROM events LEFT JOIN persons AS p ON events.person_id = p.id WHERE {MIXED}",+ [UNION, "persons"],+ [],+ ),+ (+ "left_join_with_an_unqualified_event_aliased_to_itself",+ "SELECT event AS event, count() FROM events LEFT JOIN persons AS p ON events.person_id = p.id "+ f"WHERE {MIXED} GROUP BY event",+ [UNION, "persons"],+ [],+ ),+ (+ "only_flag_calls_with_a_join_and_an_unqualified_event",+ "SELECT count() FROM events LEFT JOIN persons AS p ON events.person_id = p.id "+ "WHERE event = '$feature_flag_called'",+ ["events", "persons"],+ [],+ ),+ (+ "array_join_with_an_unqualified_event",+ f"SELECT count() FROM events ARRAY JOIN [1, 2] AS x WHERE {MIXED}",+ [UNION],+ [],+ ),+ (+ "inner_join_after_the_scan",+ f"SELECT count() FROM events AS e INNER JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",+ [UNION, "persons"],+ [],+ ),+ (+ "global_join_after_the_scan",+ _global_join(+ f"SELECT count() FROM events AS e INNER JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}"+ ),+ [UNION, "persons"],+ [],+ ),+ ("qualified_events_name", f"SELECT count() FROM posthog.events WHERE {MIXED}", [UNION], []),+ ("sample_one", f"SELECT count() FROM events SAMPLE 1 WHERE {MIXED}", [UNION], []),+ ("scan_in_a_subquery", f"SELECT count() FROM (SELECT event FROM events WHERE {MIXED})", [UNION], []),+ (+ "comparison_only_in_a_subquery",+ f"SELECT count() FROM events WHERE distinct_id IN (SELECT distinct_id FROM events WHERE {MIXED})",+ ["events", UNION],+ [],+ ),+ (+ "no_flag_call_comparison",+ "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$pageview'",+ ["events"],+ [],+ ),+ ("not_equals", "SELECT count() FROM events WHERE event != '$feature_flag_called'", ["events"], []),+ (+ "constant_on_another_column",+ "SELECT count() FROM events WHERE distinct_id = '$feature_flag_called'",+ ["events"],+ [],+ ),+ (+ "scan_after_another_table",+ f"SELECT count() FROM persons AS p JOIN events AS e ON p.id = e.person_id WHERE {E_MIXED}",+ ["persons", "events"],+ ["it joins events after another table"],+ ),+ (+ "scan_of_only_flag_calls_after_another_table",+ "SELECT count() FROM persons AS p JOIN events AS e ON p.id = e.person_id "+ "WHERE e.event = '$feature_flag_called'",+ ["persons", "events"],+ ["it joins events after another table"],+ ),+ (+ "cte_named_events_after_another_table",+ "WITH events AS (SELECT 1 AS person_id, '$pageview' AS event) "+ f"SELECT count() FROM persons AS p JOIN events AS e ON p.id = e.person_id WHERE {E_MIXED}",+ ["persons", "events"],+ [],+ ),+ (+ "right_join_after_the_scan",+ f"SELECT count() FROM events AS e RIGHT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",+ ["events", "persons"],+ ["it joins another table to events with RIGHT JOIN"],+ ),+ (+ "final",+ f"SELECT count() FROM events FINAL WHERE {MIXED}",+ ["events"],+ ["it uses FINAL or table arguments on events"],+ ),+ (+ "column_aliases",+ f"SELECT count() FROM events AS e (uuid, event, properties, timestamp, distinct_id, x) WHERE {E_MIXED}",+ ["events"],+ ["it names the events columns with column aliases"],+ ),+ ("sample_ratio", f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}", ["events"], ["it uses SAMPLE"]),+ ("select_star", f"SELECT * FROM events WHERE {MIXED}", ["events"], ["it selects * or COLUMNS(...)"]),+ (+ "cte_named_events",+ f"WITH events AS (SELECT '$pageview' AS event) SELECT count() FROM events WHERE {MIXED}",+ ["events"],+ [],+ ),+ ]+ )+ def test_renames_the_scans_the_union_reads_and_names_why_it_cannot_read_the_rest(+ self,+ _name: str,+ query: str | ast.SelectQuery,+ expected_tables: list[str],+ expected_refusals: list[str],+ only_in_flag_evaluations: bool = True,+ ):+ select = _select(query)+ rename = rename_flag_call_scans_to_union(+ select, flag_key_compares_as_string=lambda: True, only_in_flag_evaluations=only_in_flag_evaluations+ )+ collector = _TableCollector()+ collector.visit(select)++ assert collector.tables == expected_tables+ assert list(rename.refusals) == expected_refusals++ def test_row_dependent_functions_cover_every_hogql_function_in_their_families(self):+ families = re.compile(+ r"^(rand|random|rowNumber|blockNumber|nowInBlock|generateUUID|generateULID|runningDifference|neighbor)"+ )++ assert [+ name+ for name in HOGQL_CLICKHOUSE_FUNCTIONS+ if families.match(name) and not is_nondeterministic_function(name)+ ] == []+++class TestReadFlagCallsWithUnion(BaseTest):+ def _sql(+ self,+ query: str | ast.SelectQuery,+ *,+ mode: FlagEvaluationsMode = READ,+ union: bool = True,+ persons_on_events: PersonsOnEventsMode | None = None,+ new_events_schema: bool | None = None,+ person_id_pushdown: bool | None = None,+ ) -> str:+ _set_mode(self.organization, mode)+ with _enabled_flags(union=union):+ sql, context = HogQLQueryExecutor(+ query=query,+ team=self.team,+ modifiers=HogQLQueryModifiers(+ personsOnEventsMode=persons_on_events,+ personIdPushdown=person_id_pushdown,+ useNewEventsSchema=_events_schema_for(mode, new_events_schema),+ ),+ pretty=False,+ ).generate_clickhouse_sql()+ return substitute_params(sql, context.values)++ @staticmethod+ def _branches(sql: str) -> list[str]:+ start = sql.index("UNION ALL")+ union_start = sql.rindex("(SELECT", 0, sql.rindex("FROM events", 0, start))+ union_end = sql.index(") AS ", sql.rindex("FROM flag_evaluations"))+ return sql[union_start:union_end].split("UNION ALL")++ @parameterized.expand(+ [+ ("mixed_query", f"SELECT count() FROM events WHERE {MIXED}", True, True),+ ("mixed_query_with_the_flag_off", f"SELECT count() FROM events WHERE {MIXED}", False, False),+ (+ "query_phase_1_swaps",+ "SELECT count() FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'f' AND timestamp >= now() - INTERVAL 7 DAY",+ True,+ False,+ ),+ # The CTE shadows events, so the query reads the CTE on every mode.+ (+ "cte_named_events",+ f"WITH events AS (SELECT '$pageview' AS event) SELECT count() FROM events WHERE {MIXED}",+ True,+ False,+ ),+ # Each branch reads the raw string of a property without a type, as events does.+ (+ "string_parser_on_an_untyped_property",+ f"SELECT toFloatOrZero(properties.probe) FROM events WHERE {MIXED}",+ True,+ True,+ ),+ param(+ "cte_named_events_on_mode_2",+ f"WITH events AS (SELECT '$pageview' AS event) SELECT count() FROM events WHERE {MIXED}",+ True,+ False,+ mode=ONLY,+ ),+ # Each scan exports one created_at, so the two fields never share a column.+ param(+ "fields_of_one_name_in_separate_scans",+ f"SELECT created_at FROM events WHERE {MIXED} UNION ALL "+ f"SELECT person.created_at FROM events WHERE {MIXED}",+ True,+ True,+ mode=ONLY,+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,+ ),+ ]+ )+ def test_union_reads_the_scans_phase_1_leaves_on_events(+ self,+ _name: str,+ query: str,+ union: bool,+ expected_union: bool,+ mode: FlagEvaluationsMode = READ,+ persons_on_events: PersonsOnEventsMode | None = None,+ ):+ sql = self._sql(query, union=union, mode=mode, persons_on_events=persons_on_events)++ assert ("UNION ALL" in sql) is expected_union+ if expected_union:+ assert re.search(r"FROM flag_evaluations\b", sql) is not None++ @parameterized.expand(+ [+ (+ "right_join",+ f"SELECT count() FROM events AS e RIGHT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",+ "RIGHT JOIN",+ ),+ (+ "field_flag_evaluations_lacks",+ f"SELECT goe_0.properties.plan FROM events WHERE {MIXED}",+ "goe_0.properties",+ ),+ param(+ "join_whose_on_clause_reads_another_field",+ f"SELECT probe_join.key FROM events WHERE {MIXED}",+ "probe_join",+ warehouse_join=True,+ ),+ (+ "field_and_virtual_field_of_one_name",+ f"SELECT created_at, person.created_at FROM events WHERE {MIXED}",+ "created_at and poe.created_at",+ ),+ (+ "properties_whose_paths_join_to_one_name",+ f"SELECT properties.a.b, properties.a___b FROM events WHERE {MIXED}",+ "properties.a.b and properties.a___b",+ ),+ (+ "properties_and_virtual_properties",+ f"SELECT properties, person.properties FROM events WHERE {MIXED}",+ "properties and poe.properties",+ ),+ param(+ "string_parser_on_a_typed_event_property",+ f"SELECT toFloatOrZero(properties.probe) FROM events WHERE {MIXED}",+ "Numeric event property probe",+ typed=PropertyDefinition.Type.EVENT,+ ),+ param(+ "string_parser_on_a_typed_person_property",+ f"SELECT toIntOrZero(person.properties.probe) FROM events WHERE {MIXED}",+ "Numeric person property probe",+ typed=PropertyDefinition.Type.PERSON,+ ),+ ]+ )+ def test_mode_1_reads_events_and_mode_2_raises_when_the_union_cannot_read_a_mixed_scan(+ self,+ _name: str,+ query: str,+ reason: str,+ warehouse_join: bool = False,+ typed: PropertyDefinition.Type | None = None,+ ):+ if typed is not None:+ PropertyDefinition.objects.create(team=self.team, name="probe", property_type="Numeric", type=typed)+ if warehouse_join:+ # Joined on persons, it reaches events as a join that compares `person.properties.email`, not its key.+ DataWarehouseJoin.objects.create(+ team=self.team,+ source_table_name="persons",+ source_table_key="properties.email",+ joining_table_name="groups",+ joining_table_key="key",+ field_name="probe_join",+ )+ persons_on_events = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS++ assert "UNION ALL" not in self._sql(query, persons_on_events=persons_on_events)+ with pytest.raises(QueryError, match=reason):+ self._sql(query, mode=ONLY, persons_on_events=persons_on_events)++ @parameterized.expand([("dotted", UNION), ("quoted", f"`{UNION}`")])+ def test_query_cannot_name_the_union_table(self, _name: str, table: str):+ with pytest.raises(QueryError, match="Unknown table `posthog.events_with_flag_calls`"):+ self._sql(f"SELECT count() FROM {table}")++ def test_in_subquery_over_the_union_runs_once_with_global_in(self):+ sql = self._sql(+ f"SELECT count() FROM events WHERE distinct_id IN (SELECT distinct_id FROM events WHERE {MIXED})"+ )++ assert "globalIn(events.distinct_id" in sql++ @parameterized.expand(+ [+ ("restricted_on_mode_1", READ, None, False, False),+ # Mode 2 reads every flag call from flag_evaluations, so it never asks events for the flag key.+ ("restricted_on_mode_2", ONLY, None, False, True),+ ("boolean_on_native_json_on_mode_1", READ, "Boolean", True, False),+ ("numeric_on_native_json_on_mode_1", READ, "Numeric", True, False),+ ("boolean_on_native_json_on_mode_2", ONLY, "Boolean", True, True),+ ]+ )+ def test_flag_key_that_events_cannot_read_raw_keeps_mode_1_on_events(+ self, _name: str, mode: FlagEvaluationsMode, flag_key_type: str | None, new_events_schema: bool, expected: bool+ ):+ if flag_key_type is None:+ _restrict(self.organization, self.team, "$feature_flag")+ else:+ PropertyDefinition.objects.create(+ team=self.team, name="$feature_flag", property_type=flag_key_type, type=PropertyDefinition.Type.EVENT+ )++ sql = self._sql(f"SELECT count() FROM events WHERE {MIXED}", mode=mode, new_events_schema=new_events_schema)++ assert ("UNION ALL" in sql) is expected++ @parameterized.expand(+ [+ (+ "one_key",+ "if(event = '$feature_flag_called' AND properties.$feature_flag = 'probe-flag', 1, 0) AS step_1",+ "step_1 = 1",+ "tuple('probe-flag')",+ ),+ (+ "a_key_per_step_with_other_conditions",+ "if(event = '$feature_flag_called' AND properties.$feature_flag = 'a' AND properties.x = '1', 1, 0) "+ "AS step_1, "+ "if(event = '$feature_flag_called' AND properties.$feature_flag = 'b' AND properties.x = '2', 1, 0) "+ "AS step_2",+ "step_1 = 1 OR step_2 = 1",+ "tuple('a', 'b')",+ ),+ ]+ )+ def test_funnel_steps_on_flag_keys_read_only_those_keys_and_skip_keyless_calls(+ self, _name: str, flag_steps: str, flag_step_filter: str, keys: str+ ):+ # The shape a funnel runner builds: one column per step, and a filter that keeps a row matching any step.+ sql = self._sql(+ "SELECT * FROM ("+ f"SELECT if(event = '$pageview', 1, 0) AS step_0, {flag_steps} "+ "FROM events WHERE timestamp >= toDateTime('2026-10-01') AND timestamp <= toDateTime('2026-10-20') "+ f"AND event IN ('$feature_flag_called', '$pageview') AND (step_0 = 1 OR {flag_step_filter}))"+ )++ events, flag_evaluations = self._branches(sql)+ for branch in (events, flag_evaluations):+ assert "2026-10-01" in branch and "2026-10-20" in branch+ assert f"in(flag_evaluations.flag_key, {keys})" in flag_evaluations+ assert "in(events.event, tuple('$feature_flag_called', '$pageview'))" in events++ @parameterized.expand(+ [+ ("no_flag_key_filter", "", None),+ # A Boolean type converts the key before the comparison, so 'TRUE' also matches the JSON value true.+ ("boolean_flag_key", " AND properties.$feature_flag = 'TRUE'", "Boolean"),+ ]+ )+ def test_query_without_a_string_flag_key_reads_keyless_calls_from_events(+ self, _name: str, flag_key_filter: str, flag_key_type: str | None+ ):+ if flag_key_type is not None:+ PropertyDefinition.objects.create(+ team=self.team, name="$feature_flag", property_type=flag_key_type, type=PropertyDefinition.Type.EVENT+ )++ events, keyless, flag_evaluations = self._branches(+ self._sql(f"SELECT count() FROM events WHERE {MIXED} AND distinct_id = 'probe'{flag_key_filter}")+ )++ for branch in (events, keyless):+ assert "equals(events.distinct_id, 'probe')" in branch+ assert "equals(flag_evaluations.distinct_id, 'probe')" in flag_evaluations+ assert "flag_key" not in flag_evaluations++ @parameterized.expand(+ [+ ("array_join", "arrayJoin(['a', 'b']) = 'a'", "arrayJoin"),+ ("random", "rand() % 10 = 0", "rand"),+ ("row_number", "rowNumberInBlock() < 5", "rowNumberInBlock"),+ ("random_in_upper_case", "RAND() % 10 = 0", "rand"),+ ("now_in_block", "timestamp < nowInBlock('UTC')", "nowInBlock"),+ ]+ )+ def test_filter_that_depends_on_each_read_stays_out_of_the_branches(self, _name: str, condition: str, name: str):+ sql = self._sql(f"SELECT count() FROM events WHERE {MIXED} AND {condition}")++ assert all(name not in branch for branch in self._branches(sql))+ assert name in sql++ def test_sample_one_adds_no_warning(self):+ _set_mode(self.organization, READ)+ with _enabled_flags(union=True):+ _, context = HogQLQueryExecutor(+ query=f"SELECT count() FROM events SAMPLE 1 WHERE {MIXED}", team=self.team+ ).generate_clickhouse_sql()++ assert context.warnings == []++ def test_sessions_join_keeps_its_time_bound(self):+ sql = self._sql(+ f"SELECT session.$session_duration FROM events WHERE {MIXED} "+ "AND timestamp >= toDateTime('2026-10-01') AND timestamp < toDateTime('2026-10-08')"+ )++ sessions = sql[sql.index("FROM raw_sessions") : sql.index("AS events__session")]+ assert "toDateTime('2026-10-01', 'UTC')" in sessions and "toDateTime('2026-10-08', 'UTC')" in sessions++ def test_sessions_join_keeps_a_literal_session_id_filter(self):+ sql = self._sql(+ f"SELECT session.$session_duration FROM events WHERE {MIXED} AND $session_id IN ('{SESSION_ID}')"+ )++ sessions = sql[sql.index("FROM raw_sessions") : sql.index("AS events__session")]+ assert (+ f"in(raw_sessions.session_id_v7, tuple(toUInt128(accurateCastOrNull('{SESSION_ID}', 'UUID'))))" in sessions+ )++ def test_persons_join_keeps_the_person_id_pushdown(self):+ sql = self._sql(+ f"SELECT person.properties.email FROM events WHERE {MIXED} AND distinct_id = 'probe-distinct-id'",+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ person_id_pushdown=True,+ )++ persons = sql[sql.index("FROM person_distinct_id_overrides") :]+ assert "GLOBAL IN" in sql or "globalIn" in sql+ assert "probe-distinct-id" in persons++ def test_persons_join_runs_one_override_join(self):+ sql = self._sql(+ f"SELECT person_id, person.properties.email FROM events WHERE {MIXED}",+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ )++ assert sql.count("FROM person_distinct_id_overrides") == 2 # one inner, one outer for `person_id`+++class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):+ def setUp(self):+ super().setUp()+ now = datetime.now(UTC)+ self.recent = _create_person(distinct_ids=["recent"], team_id=self.team.pk, properties={"email": "[email protected]"})+ self.old = _create_person(distinct_ids=["old"], team_id=self.team.pk, properties={"email": "[email protected]"})+ for distinct_id, person, days_ago in (("recent", self.recent, 3), ("old", self.old, 100)):+ _create_event(+ event="$pageview",+ distinct_id=distinct_id,+ person_id=person.uuid,+ person_properties={"email": person.properties["email"]},+ team=self.team,+ timestamp=now - timedelta(days=days_ago),+ )+ # Ingestion writes a keyed flag call to both tables on mode 1. Only flag_evaluations has the stale person,+ # which an override corrects.+ flag_call_uuid = uuid.uuid4()+ _create_event(+ event_uuid=flag_call_uuid,+ event="$feature_flag_called",+ distinct_id="recent",+ person_id=self.recent.uuid,+ person_properties={"email": "[email protected]"},+ team=self.team,+ timestamp=now - timedelta(days=2),+ properties={"$feature_flag": "probe-flag", "source": "events"},+ )+ _create_flag_evaluations(+ self.team.pk,+ "probe-flag",+ timestamp=now - timedelta(days=2),+ distinct_id="recent",+ person_id=uuid.uuid4(),+ person_properties={"email": "[email protected]"},+ properties={"source": "flag_evaluations"},+ )+ sync_execute(+ "INSERT INTO person_distinct_id_overrides (team_id, distinct_id, person_id, version) VALUES",+ [(self.team.pk, "recent", str(self.recent.uuid), 1)],+ )+ _create_event(+ event="$feature_flag_called",+ distinct_id="old",+ person_id=self.old.uuid,+ team=self.team,+ timestamp=now - timedelta(days=99),+ properties={"$feature_flag": "probe-flag", "source": "events"},+ )+ for distinct_id, flag_key in (("missing", None), ("empty", ""), ("number", 100)):+ _create_event(+ event="$feature_flag_called",+ distinct_id=distinct_id,+ team=self.team,+ timestamp=now - timedelta(days=1),+ properties={"source": "events", **({} if flag_key is None else {"$feature_flag": flag_key})},+ )+ flush_persons_and_events()++ def _run(+ self,+ query: str | ast.SelectQuery,+ mode: FlagEvaluationsMode,+ persons_on_events: PersonsOnEventsMode | None = None,+ ):+ _set_mode(self.organization, mode)+ with _enabled_flags(union=True):+ return execute_hogql_query(+ query,+ team=self.team,+ modifiers=HogQLQueryModifiers(+ personsOnEventsMode=persons_on_events, useNewEventsSchema=_events_schema_for(mode, None)+ ),+ )++ @parameterized.expand(+ [+ (f"{mode.name.lower()}_{poe.name.lower()}", mode, poe)+ for mode in (FlagEvaluationsMode.READ_FLAG_EVALUATIONS, FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY)+ for poe in (+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ )+ ]+ )+ def test_each_flag_call_comes_from_one_table(self, _name: str, mode: FlagEvaluationsMode, poe: PersonsOnEventsMode):+ response = self._run(+ f"""+ SELECT distinct_id, event, properties.source, person_id, person.properties.email+ FROM events+ WHERE {MIXED} AND timestamp >= now() - INTERVAL 120 DAY+ ORDER BY timestamp, distinct_id+ """,+ mode,+ poe,+ )++ recent_id, old_id = str(self.recent.uuid), str(self.old.uuid)+ history = [+ ("old", "$pageview", None, old_id, "[email protected]"),+ ("old", "$feature_flag_called", "events", old_id, "[email protected]"),+ ]+ recent = [+ ("recent", "$pageview", None, recent_id, "[email protected]"),+ ("recent", "$feature_flag_called", "flag_evaluations", recent_id, "[email protected]"),+ ]+ keyless = [(distinct_id, "$feature_flag_called", "events") for distinct_id in ("empty", "missing", "number")]+ rows = [+ tuple(str(value) if isinstance(value, uuid.UUID) else value for value in row) for row in response.results+ ]+ if mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY:+ assert rows == [history[0], *recent]+ else:+ assert rows[:4] == [*history, *recent]+ assert [row[:3] for row in rows[4:]] == keyless++ def test_prewhere_reads_both_tables(self):+ response = self._run(+ f"SELECT count() FROM events PREWHERE {MIXED} WHERE timestamp >= now() - INTERVAL 120 DAY",+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ )++ assert response.results == [(7,)]++ @parameterized.expand([("json",), ("materialized",)])+ def test_keyless_flag_calls_come_from_events(self, read: str):+ # A materialized $feature_flag column reads an empty string as missing.+ if read == "materialized":+ self.addCleanup(cleanup_materialized_columns)+ materialize("events", "$feature_flag")+ response = self._run(+ f"SELECT distinct_id FROM events WHERE {MIXED} AND event = '$feature_flag_called' "+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY distinct_id",+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ )++ assert response.results == [("empty",), ("missing",), ("number",), ("recent",)]++ @parameterized.expand(+ [+ (+ "pin_of_one_flag_key",+ "SELECT distinct_id, event, properties.source FROM events WHERE event = '$pageview' "+ "OR (event = '$feature_flag_called' AND properties.$feature_flag = 'probe-flag') ORDER BY timestamp",+ [+ ("old", "$pageview", None),+ ("old", "$feature_flag_called", "events"),+ ("recent", "$pageview", None),+ ("recent", "$feature_flag_called", "flag_evaluations"),+ ],+ ),+ # The OR reads a person property, so no branch can copy it.+ (+ "or_with_a_person_property",+ f"SELECT distinct_id, event, properties.source FROM events WHERE {MIXED} "+ "AND (event = '$pageview' OR person.properties.email = '[email protected]') "+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY timestamp",+ [("recent", "$pageview", None), ("recent", "$feature_flag_called", "flag_evaluations")],+ ),+ (+ "mixed_type_in_that_clickhouse_converts",+ f"SELECT distinct_id, event, properties.source FROM events WHERE {MIXED} AND 1 IN ('1', '2') "+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY timestamp, distinct_id",+ [+ ("recent", "$pageview", None),+ ("recent", "$feature_flag_called", "flag_evaluations"),+ ("empty", "$feature_flag_called", "events"),+ ("missing", "$feature_flag_called", "events"),+ ("number", "$feature_flag_called", "events"),+ ],+ ),+ (+ "global_join",+ _global_join(+ "SELECT e.distinct_id, e.event, e.properties.source FROM events AS e "+ "INNER JOIN (SELECT 'recent' AS distinct_id) AS r ON e.distinct_id = r.distinct_id "+ f"WHERE {E_MIXED} AND e.timestamp >= now() - INTERVAL 5 DAY ORDER BY e.timestamp"+ ),+ [("recent", "$pageview", None), ("recent", "$feature_flag_called", "flag_evaluations")],+ ),+ ]+ )+ def test_branch_filters_keep_every_row_the_query_keeps(+ self, _name: str, query: str | ast.SelectQuery, expected: list[tuple]+ ):+ response = self._run(+ query,+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ )++ assert response.results == expected++ def test_exception_values_read_as_an_array(self):+ query = f"SELECT count() FROM events WHERE {MIXED} AND has(properties.$exception_values, 'probe')"++ assert self._run(query, FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results == [(0,)]++ def test_restricted_property_is_hidden_in_every_branch(self):+ _restrict(self.organization, self.team, "source")+ query = f"SELECT properties.source FROM events WHERE {MIXED}"++ assert {row[0] for row in self._run(query, FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results} == {None}+ assert (+ self._run(f"{query} AND properties.source = 'events'", FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results+ == []+ )++ @parameterized.expand(+ [+ (f"{poe.name.lower()}_{'events_json' if new_schema else 'events'}", poe, new_schema)+ for poe in (+ PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ )+ for new_schema in (False, True)+ ]+ )+ def test_every_events_field_reads_the_same_type_or_keeps_events(+ self, _name: str, poe: PersonsOnEventsMode, new_schema: bool+ ):+ # Mode 1 keeps reading events on the native JSON table, so only mode 2 reads the union there.+ _set_mode(self.organization, ONLY if new_schema else READ)+ create_group_type_mapping(team=self.team, group_type="organization", group_type_index=0)+ # A typed property reads as a number, which the union keeps only when it types each branch.+ for property_type in (PropertyDefinition.Type.EVENT, PropertyDefinition.Type.PERSON):+ PropertyDefinition.objects.create(team=self.team, name="probe", property_type="Numeric", type=property_type)+ modifiers = HogQLQueryModifiers(personsOnEventsMode=poe, useNewEventsSchema=new_schema)+ with _enabled_flags(union=True):+ union_database = Database.create_for(team=self.team, modifiers=modifiers)+ with _enabled_flags(union=False):+ events_database = Database.create_for(team=self.team, modifiers=modifiers)++ def sql(query: str, database: Database) -> str:+ context = HogQLContext(+ team_id=self.team.pk, team=self.team, database=database, modifiers=modifiers, enable_select_queries=True+ )+ return prepare_and_print_ast(parse_select(query), context, "clickhouse")[0]++ accepted = []+ for chain in _readable_chains(union_database):+ name = ".".join(f"`{part}`" for part in chain)+ try:+ sql(f"SELECT {name} FROM events", events_database)+ except QueryError:+ continue+ # A field the union cannot read keeps the query on events on mode 1 and fails it on mode 2.+ try:+ union_sql = sql(f"SELECT {name} FROM events WHERE {MIXED}", union_database)+ except QueryError:+ continue+ if "UNION ALL" in union_sql:+ accepted.append(name)++ assert {+ "`timestamp`",+ "`properties`.`probe`",+ "`person_id`",+ "`person`.`properties`.`probe`",+ "`$session_id`",+ "`elements_chain_texts`",+ "`$group_0`",+ # The sessions join reads `$session_id_uuid`, which is not its key.+ "`session`.`id`",+ } <= set(accepted)+ types: dict[bool, dict[str, str]] = {False: {}, True: {}}+ # One query per few fields keeps each union within the memory of a local ClickHouse.+ # Two fields of one name, such as goe_0.key and goe_1.key, would return one column, so they go in different queries.+ batches: list[list[str]] = []+ for name in accepted:+ batch = next(+ (b for b in batches if len(b) < 8 and all(n.rsplit(".", 1)[-1] != name.rsplit(".", 1)[-1] for n in b)),+ None,+ )+ if batch is None:+ batches.append(batch := [])+ batch.append(name)+ for names in batches:+ types_query = f"SELECT {', '.join(f'toTypeName(any({name}))' for name in names)} FROM events WHERE {MIXED}"+ for union in (False, True):+ with _enabled_flags(union=union):+ response = execute_hogql_query(types_query, team=self.team, modifiers=modifiers)+ types[union].update(zip(names, response.results[0]))++ # flag_evaluations has no elements, and its empty Array(String) widens the union of the events Enum8 array.+ # Comparisons with string literals behave the same on both types.+ widened = {"`elements_chain_elements`": "Array(String)"}+ assert types[True] == {**types[False], **widened}+++def _events_schema_for(mode: FlagEvaluationsMode, new_events_schema: bool | None) -> bool | None:+ # Mode 1 keeps reading events on the native JSON table, so a mode 1 case reads the legacy table unless it asks.+ if new_events_schema is None and mode == READ:+ return False+ return new_events_schema+++def _readable_chains(database: Database) -> list[list[str | int]]:+ """A chain to select for each events field, and for one field behind each of its joins."""+ events = database.get_table("events")+ chains: list[list[str | int]] = []+ for name, field in events.fields.items():+ if isinstance(field, (LazyJoin, FieldTraverser)):+ chains.extend([[name, "id"], [name, "properties", "probe"], [name, "person_id"]])+ continue+ inner_fields = field.fields.items() if isinstance(field, VirtualTable) else [(None, field)]+ for inner_name, inner in inner_fields:+ chain: list[str | int] = [name] if inner_name is None else [name, inner_name]+ if isinstance(inner, StringJSONDatabaseField):+ chains.append([*chain, "probe"])+ elif isinstance(inner, DatabaseField):+ chains.append(chain)+ return chainsdiff --git a/posthog/models/test/test_data_deletion_request.py b/posthog/models/test/test_data_deletion_request.pyindex 9541685ef21a..f035490dfa0f 100644--- a/posthog/models/test/test_data_deletion_request.py+++ b/posthog/models/test/test_data_deletion_request.py@@ -213,9 +213,12 @@ def test_hogql_predicate_subquery_is_team_scoped(db):["SELECT distinct_id FROM events WHERE event = '$feature_flag_called' ""AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY",+ "SELECT distinct_id FROM events WHERE event IN ('$pageview', '$feature_flag_called') "+ "AND timestamp >= now() - INTERVAL 7 DAY","SELECT e.distinct_id FROM events AS e JOIN (SELECT 'probe' AS id) AS d ""ON e.distinct_id = d.id WHERE event = '$feature_flag_called'",],+ ids=["only_flag_calls", "flag_calls_with_other_events", "flag_calls_behind_a_join"],)@pytest.mark.parametrize("through_a_view", [False, True])def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, flag_calls, through_a_view):@@ -236,7 +239,11 @@ def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, flagpredicate = f"distinct_id IN ({flag_calls})"request = DataDeletionRequest(**_base_kwargs(team_id=team.id, events=["$pageview"], hogql_predicate=predicate))- sql, _ = compile_hogql_predicate(request)+ with patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-union",+ ):+ sql, _ = compile_hogql_predicate(request)assert "flag_evaluations" not in sqldiff --git a/products/cohorts/backend/backfill/flag_calls_table.py b/products/cohorts/backend/backfill/flag_calls_table.pyindex 3cc2fca2fb62..ed3333366315 100644--- a/products/cohorts/backend/backfill/flag_calls_table.py+++ b/products/cohorts/backend/backfill/flag_calls_table.py@@ -4,7 +4,7 @@ from datetime import UTC, datetimefrom enum import StrEnumfrom typing import Any-from posthog.hogql.transforms.flag_calls_from_flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYS+from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYSfrom posthog.exceptions_capture import capture_exceptionfrom posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_SOURCE_EVENT, FLAG_EVALUATIONS_TABLE