Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): read flag calls from both tables in mixed queries (#126045)

Phil Haack6a16b4778a96
diff --git a/posthog/api/cohort.py b/posthog/api/cohort.py
index 7ef6628023d5..055e6bf9b9f3 100644
--- a/posthog/api/cohort.py
+++ b/posthog/api/cohort.py
@@ -38,10 +38,10 @@ from posthog.schema import ActorsQuery, HogQLQuery, ProductKey
from posthog.hogql.compiler.bytecode import create_bytecode
from posthog.hogql.constants import CSV_EXPORT_LIMIT, LimitContext
from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYS
from posthog.hogql.errors import TableAccessDeniedError
from posthog.hogql.printer import prepare_ast_for_printing
from posthog.hogql.property import PERSON_METADATA_FIELDS, property_to_expr
-from posthog.hogql.transforms.flag_calls_from_flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYS
from posthog.api.forbid_destroy_model import ForbidDestroyModel
from posthog.api.routing import TeamAndOrgViewSetMixin
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.md
index 5bd904e984ec..15e0a0a33ec5 100644
--- a/posthog/hogql/ACCESS_CONTROL.md
+++ b/posthog/hogql/ACCESS_CONTROL.md
@@ -222,6 +222,7 @@ Because each class restricts its own key, a table dispatched as the wrong class,
The exemptions are name collisions: `accounts.properties` and `pg_embeddings.properties` do not contain event, person, or group properties.
`ai_events.properties` uses the same event-property rules as `events.properties`.
+The hidden `events_with_flag_calls` table, which the flag-call union reads in place of `events`, uses them too.
Covering a table means moving it out of the exemptions and into the expected mapping, so neither list can keep a stale entry.
### Typed columns that mirror a property
diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py
index 3ebea3d5fd81..5c84e1313988 100644
--- a/posthog/hogql/database/database.py
+++ b/posthog/hogql/database/database.py
@@ -88,6 +88,7 @@ from posthog.hogql.database.schema.error_tracking_issue_fingerprint_overrides im
)
from posthog.hogql.database.schema.error_tracking_recent_issue_state import ErrorTrackingRecentIssueStateTable
from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.database.schema.events_with_flag_calls import EVENTS_WITH_FLAG_CALLS_TABLE, EventsWithFlagCallsTable
from posthog.hogql.database.schema.exchange_rate import ExchangeRateTable
from posthog.hogql.database.schema.experiment_exposures_preaggregated import ExperimentExposuresPreaggregatedTable
from posthog.hogql.database.schema.experiment_metric_events_preaggregated import (
@@ -255,6 +256,8 @@ class HogQLDatabaseSources:
read_flag_calls_from_flag_evaluations: bool
# The organization reads flag calls only from posthog.flag_evaluations, and ingestion no longer writes them to events.
flag_calls_only_in_flag_evaluations: bool
+ # Flag calls that a mixed query reads come from events and flag_evaluations together.
+ read_flag_calls_with_union: bool
# Userless internal contexts that must resolve every warehouse table/view; skips access control
bypass_warehouse_access_control: bool
direct_connection_metadata: dict[str, Any] | None
@@ -458,7 +461,7 @@ _DATABASE_ROOT_NODE_BLOBS_LOCK = threading.Lock()
# LRU, and fresh entries survive the sweep. Reads are lock-free (dict.get is atomic under the
# GIL); mutations and the sweep hold the lock, because the sweep iterates the dict and concurrent
# inserts would raise RuntimeError mid-iteration.
-_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets"})
+_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets", "hogql-flag-calls-union"})
_TEAM_FLAG_CACHE: dict[tuple[str, str], tuple[float, bool]] = {} # key -> (evaluated_at, value)
_TEAM_FLAG_CACHE_LOCK = threading.Lock()
_TEAM_FLAG_CACHE_MAX_ENTRIES = 50_000
@@ -514,6 +517,21 @@ def _cached_team_flag(flag_key: str, team: Team, evaluate: Callable[[], bool]) -
return value
+def _cached_organization_flag(flag_key: str, team: Team) -> bool:
+ """`flag_key` for the team's organization, through the team flag cache."""
+ return _cached_team_flag(
+ flag_key,
+ team,
+ lambda: feature_enabled_or_false(
+ flag_key,
+ str(team.uuid),
+ groups={"organization": str(team.organization_id)},
+ group_properties={"organization": {"id": str(team.organization_id)}},
+ send_feature_flag_events=False,
+ ),
+ )
+
+
# We only ever load our own freshly-built blob, but restrict the unpickler anyway as defense in depth:
# it can reconstruct only the classes the catalog is built from, so even a future change that fed it
# untrusted bytes couldn't instantiate arbitrary code (os.system, etc.). Every catalog class lives
@@ -835,6 +853,7 @@ class Database(BaseModel):
_connection_id: str | None = None
_read_flag_calls_from_flag_evaluations: bool = False
_flag_calls_only_in_flag_evaluations: bool = False
+ _read_flag_calls_with_union: bool = False
_direct_connection_metadata: dict[str, Any] | None = None
_direct_access_warehouse_table_names: set[str] = set()
# Warnings about data warehouse tables (failed/paused/billing-limited/stale syncs),
@@ -903,6 +922,7 @@ class Database(BaseModel):
self._connection_id = None
self._read_flag_calls_from_flag_evaluations = False
self._flag_calls_only_in_flag_evaluations = False
+ self._read_flag_calls_with_union = False
self._direct_connection_metadata = None
self._direct_access_warehouse_table_names = set()
self._data_warehouse_sync_warnings = {}
@@ -1307,6 +1327,10 @@ class Database(BaseModel):
def flag_calls_only_in_flag_evaluations(self) -> bool:
return self._flag_calls_only_in_flag_evaluations and self.read_flag_calls_from_flag_evaluations
+ @property
+ def read_flag_calls_with_union(self) -> bool:
+ return self._read_flag_calls_with_union and self.read_flag_calls_from_flag_evaluations
+
@staticmethod
def _is_helper_function_table(table: object) -> bool:
return isinstance(table, FunctionCallTable) and not isinstance(table, (DirectSQLTable, PostgresTable, S3Table))
@@ -1504,8 +1528,13 @@ class Database(BaseModel):
if self._is_direct_query()
else self.get_posthog_table_names(include_hidden=include_hidden_posthog_tables)
)
+ posthog_node = self.tables.children.get("posthog")
+ # A hidden table stays readable by name, but the schema leaves it out.
+ hidden_table_names = {
+ f"posthog.{name}" for name, node in (posthog_node.children.items() if posthog_node else ()) if node.hidden
+ }
for table_name in posthog_table_names:
- if include_only and table_name not in include_only:
+ if (include_only and table_name not in include_only) or table_name in hidden_table_names:
continue
fields_dict: dict[str, DatabaseSchemaField] = {}
@@ -1987,6 +2016,7 @@ class Database(BaseModel):
include_flag_evaluations_table=False,
read_flag_calls_from_flag_evaluations=False,
flag_calls_only_in_flag_evaluations=False,
+ read_flag_calls_with_union=False,
group_types=[],
saved_queries=[],
endpoint_saved_queries=[],
@@ -2097,6 +2127,9 @@ class Database(BaseModel):
read_mode = get_flag_evaluations_read_mode(team.organization_id)
read_flag_calls_from_flag_evaluations = read_mode != FlagEvaluationsMode.EVENTS
flag_calls_only_in_flag_evaluations = read_mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY
+ read_flag_calls_with_union = read_flag_calls_from_flag_evaluations and _cached_organization_flag(
+ "hogql-flag-calls-union", team
+ )
with timings.measure("database", emit_span=True):
# Function-local: keeps the direct-SQL driver imports off the django.setup() path.
@@ -2365,6 +2398,7 @@ class Database(BaseModel):
include_flag_evaluations_table=include_flag_evaluations_table,
read_flag_calls_from_flag_evaluations=read_flag_calls_from_flag_evaluations,
flag_calls_only_in_flag_evaluations=flag_calls_only_in_flag_evaluations,
+ read_flag_calls_with_union=read_flag_calls_with_union,
# Managed warehouse is a built-in project datastore and has no warehouse-object ACL surface.
# Principals that skip warehouse access control by design:
# - synthetic users (project-wide service tokens, bypass object-level RBAC)
@@ -2420,6 +2454,7 @@ class Database(BaseModel):
)
database._read_flag_calls_from_flag_evaluations = sources.read_flag_calls_from_flag_evaluations
database._flag_calls_only_in_flag_evaluations = sources.flag_calls_only_in_flag_evaluations
+ database._read_flag_calls_with_union = sources.read_flag_calls_with_union
if sources.connection_id is not None:
database._connection_id = sources.connection_id
if sources.direct_connection_metadata is not None:
@@ -2439,6 +2474,14 @@ class Database(BaseModel):
if not sources.include_flag_evaluations_table:
if posthog_node := database.tables.children.get("posthog"):
posthog_node.children.pop("flag_evaluations", None)
+ elif sources.read_flag_calls_with_union and (posthog_node := database.tables.children.get("posthog")):
+ union_table = EventsWithFlagCallsTable()
+ # Assigned after construction, because pydantic copies a dict passed to the constructor. Sharing the
+ # dict makes every later change to the events fields, such as the persons-on-events mode, apply here too.
+ union_table.fields = database.get_table("events").fields
+ posthog_node.children[EVENTS_WITH_FLAG_CALLS_TABLE] = TableNode(
+ name=EVENTS_WITH_FLAG_CALLS_TABLE, table=union_table, hidden=True
+ )
with timings.measure("modifiers", emit_span=True):
if not database._is_direct_query():
diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.py
index c9969c639608..35ac972150eb 100644
--- a/posthog/hogql/database/models.py
+++ b/posthog/hogql/database/models.py
@@ -17,7 +17,7 @@ from posthog.hogql.errors import NotImplementedError, ResolutionError
from posthog.clickhouse.workload import Workload
if TYPE_CHECKING:
- from posthog.hogql.ast import JoinExpr, LazyJoinType, SelectQuery
+ from posthog.hogql.ast import JoinExpr, LazyJoinType, SelectQuery, Type
from posthog.hogql.base import ConstantType
from posthog.hogql.context import HogQLContext
@@ -534,19 +534,33 @@ class LazyTable(Table):
raise NotImplementedError("LazyTable.lazy_select not overridden")
-@dataclass
+@dataclass(frozen=False)
class LazyTableToAdd:
lazy_table: LazyTable
fields_accessed: dict[str, list[str | int]] = field(default_factory=dict)
-@dataclass
+@dataclass(frozen=True)
+class UnexpandedScan:
+ """The FROM entry of a lazy table before lazy table resolution replaced it with a subquery.
+
+ A join that hangs off the table copies the query's filters into a subquery that reads the table again, and this
+ holds what that subquery needs. `types` are the table types that the query's fields name for the scan.
+ """
+
+ join: "JoinExpr"
+ types: tuple["Type", ...]
+
+
+@dataclass(frozen=False)
class LazyJoinToAdd:
from_table: str
to_table: str
lazy_join: LazyJoin
lazy_join_type: "LazyJoinType"
fields_accessed: dict[str, list[str | int]] = field(default_factory=dict)
+ # Set when the table that the join starts from is a lazy table whose scan the join can read again.
+ from_scan: Optional[UnexpandedScan] = None
class VirtualTable(Table):
diff --git a/posthog/hogql/database/schema/events.py b/posthog/hogql/database/schema/events.py
index 4c61e7527a61..a44ad1ccdd38 100644
--- a/posthog/hogql/database/schema/events.py
+++ b/posthog/hogql/database/schema/events.py
@@ -30,6 +30,17 @@ def events_table_clickhouse_table_ref(context) -> str:
return DISTRIBUTED_EVENTS_JSON_TABLE if context.uses_new_events_schema() else "events"
+EVENTS_AVOID_ASTERISK_FIELDS = (
+ "$session_id_uuid",
+ "$virt_is_bot",
+ "$virt_traffic_type",
+ "$virt_traffic_category",
+ "$virt_bot_name",
+ "$virt_bot_operator",
+ "$virt_mcp_harness",
+)
+
+
class EventsPersonSubTable(VirtualTable):
fields: dict[str, FieldOrTable] = {
"id": UUIDDatabaseField(name="person_id", nullable=False),
@@ -212,15 +223,7 @@ class EventsTable(Table):
return "events"
def avoid_asterisk_fields(self) -> list[str]:
- return [
- "$session_id_uuid",
- "$virt_is_bot",
- "$virt_traffic_type",
- "$virt_traffic_category",
- "$virt_bot_name",
- "$virt_bot_operator",
- "$virt_mcp_harness",
- ]
+ return list(EVENTS_AVOID_ASTERISK_FIELDS)
# All table types that represent the events table (including virtual subtables like poe/goe).
diff --git a/posthog/hogql/database/schema/events_with_flag_calls.py b/posthog/hogql/database/schema/events_with_flag_calls.py
new file mode 100644
index 000000000000..0439bc8ff088
--- /dev/null
+++ b/posthog/hogql/database/schema/events_with_flag_calls.py
@@ -0,0 +1,200 @@
+from datetime import UTC, datetime
+from functools import partial
+
+from posthog.hogql import ast
+from posthog.hogql.constants import FLAG_CALLED_EVENT
+from posthog.hogql.database.models import (
+ DatabaseField,
+ ExpressionField,
+ FieldOrTable,
+ LazyJoin,
+ LazyTable,
+ LazyTableToAdd,
+ Table,
+ VirtualTable,
+)
+from posthog.hogql.database.schema.events import EVENTS_AVOID_ASTERISK_FIELDS, EventsTable
+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable, flag_evaluations_coverage_start
+from posthog.hogql.errors import QueryError
+from posthog.hogql.parser import parse_expr
+from posthog.hogql.visitor import GetFieldsTraverser
+
+EVENTS_WITH_FLAG_CALLS_TABLE = "events_with_flag_calls"
+EVENTS_WITH_FLAG_CALLS_CHAIN: list[str] = ["posthog", EVENTS_WITH_FLAG_CALLS_TABLE]
+
+# Ingestion writes a flag call to flag_evaluations only when `$feature_flag` is a non-empty JSON string. The raw JSON
+# text of a string value starts with a quote, and a number, boolean or object does not. On the native JSON events
+# table the printer converts a typed key before this test reads it, so mode 1 queries there keep reading events. The
+# test reads only the raw JSON text, because a materialized `$feature_flag` column reads an empty string as missing.
+_HAS_FLAG_KEY = (
+ "startsWith(JSONExtractRaw(events.properties, '$feature_flag'), '\"') "
+ "AND JSONExtractRaw(events.properties, '$feature_flag') != '\"\"'"
+)
+
+
+def _field(table: Table, chain: list[str | int]) -> tuple[FieldOrTable | None, list[str | int]]:
+ """The field that `chain` names on `table`, through a virtual table, and the rest of the chain below it."""
+ field = table.fields.get(str(chain[0]))
+ if isinstance(field, VirtualTable) and len(chain) > 1:
+ return field.fields.get(str(chain[1])), chain[2:]
+ return field, chain[1:]
+
+
+def _column_name(field: FieldOrTable | None) -> str | None:
+ # An ExpressionField is a DatabaseField too, but it names no column.
+ if isinstance(field, DatabaseField) and not isinstance(field, ExpressionField):
+ return field.name
+ return None
+
+
+def _is_expression(table: Table, chain: list[str | int]) -> bool:
+ return isinstance(_field(table, chain)[0], ExpressionField)
+
+
+def column_chains(table: Table, chain: list[str | int]) -> list[list[str | int]]:
+ """The chains of stored columns that `chain` reads on `table`, through the expression fields and the keys of the
+ joins that it reads."""
+ field, _ = _field(table, chain)
+ if not isinstance(field, ExpressionField):
+ return [chain]
+ chains: list[list[str | int]] = []
+ for read in GetFieldsTraverser(field.expr).fields:
+ joined = table.fields.get(str(read.chain[0]))
+ if isinstance(joined, LazyJoin) and len(read.chain) > 1:
+ chains.extend(column_chains(table, list(joined.from_field)))
+ else:
+ chains.extend(column_chains(table, list(read.chain)))
+ return chains
+
+
+class FlagEvaluationsColumns:
+ """Maps a chain on the events fields to the chain on `posthog.flag_evaluations` that holds the same value."""
+
+ def __init__(self, events: Table, flag_evaluations: Table) -> None:
+ self.events = events
+ self.flag_evaluations = flag_evaluations
+ self._chains_by_column: dict[str, list[str | int]] = {}
+ for name, field in flag_evaluations.fields.items():
+ if (column := _column_name(field)) is not None:
+ self._chains_by_column.setdefault(column, [name])
+ for name, field in flag_evaluations.fields.items():
+ if isinstance(field, VirtualTable):
+ for inner_name, inner in field.fields.items():
+ if (column := _column_name(inner)) is not None:
+ self._chains_by_column.setdefault(column, [name, inner_name])
+
+ def chain(self, chain: list[str | int]) -> list[str | int] | None:
+ """The chain on `flag_evaluations` that holds what `chain` reads on `events`, or None when it has none.
+
+ A chain that reads an events column maps to the `flag_evaluations` field on the same column, such as
+ `person_id` for `event_person_id`. Otherwise it maps to the `flag_evaluations` field of the same name, which
+ that table defines for events names whose columns differ, such as `$session_id`. An expression field maps when
+ each column it reads does.
+ """
+ field, rest = _field(self.events, chain)
+ if isinstance(field, ExpressionField):
+ reads = column_chains(self.events, chain)
+ return chain if all(self.chain(read) is not None for read in reads) else None
+ column = _column_name(field)
+ if column is not None and column in self._chains_by_column:
+ return [*self._chains_by_column[column], *rest]
+ same_name = self.flag_evaluations.fields.get(str(chain[0]))
+ if isinstance(same_name, VirtualTable):
+ return chain if len(chain) > 1 and str(chain[1]) in same_name.fields else None
+ return chain if same_name is not None else None
+
+
+class EventsWithFlagCallsTable(LazyTable):
+ """`events` with its flag calls since the coverage start read from `posthog.flag_evaluations` instead.
+
+ Its `fields` is the events table's own dict, so every name binds as it does on `events`.
+ """
+
+ fields: dict[str, FieldOrTable] = {}
+
+ def avoid_asterisk_fields(self) -> list[str]:
+ return list(EVENTS_AVOID_ASTERISK_FIELDS)
+
+ def lazy_select(self, table_to_add: LazyTableToAdd, context, node):
+ # Deferred: the transforms import this table.
+ from posthog.hogql.transforms.flag_calls_from_flag_evaluations import ( # noqa: PLC0415
+ and_conditions,
+ flag_key_typed_as_string,
+ )
+ from posthog.hogql.transforms.flag_calls_union import union_branch_filters, union_events_filter # noqa: PLC0415
+
+ assert context.database is not None
+ # `SELECT count()` reads no field, and a subquery needs at least one column.
+ accessed = dict(table_to_add.fields_accessed) or {"timestamp": ["timestamp"]}
+ events_columns: list[ast.Expr] = [
+ ast.Alias(alias=name, expr=ast.Field(chain=["events", *chain])) for name, chain in accessed.items()
+ ]
+ if any(_is_expression(self, chain) for chain in accessed.values()):
+ # An expression field reaches here as the key of a join, such as `person_id` for `person`. Resolving it on
+ # a second scan of this table puts its own joins, such as `override`, above the branches, so each runs
+ # once. The second scan asks only for columns, and it reads the same filter from this query's WHERE.
+ return ast.SelectQuery(
+ select=events_columns,
+ select_from=ast.JoinExpr(table=ast.Field(chain=list(EVENTS_WITH_FLAG_CALLS_CHAIN)), alias="events"),
+ where=union_events_filter(node, self),
+ )
+
+ columns = FlagEvaluationsColumns(self, context.database.get_table(["posthog", "flag_evaluations"]))
+ only_in_flag_evaluations = context.database.flag_calls_only_in_flag_evaluations
+ cutoff = ast.Constant(value=flag_evaluations_coverage_start(datetime.now(UTC)))
+ # Property type resolution ran before lazy tables, so the metadata holds the type of each property this query
+ # reads, `$feature_flag` among them.
+ filters = union_branch_filters(
+ node, self, columns, partial(flag_key_typed_as_string, context.property_metadata)
+ )
+
+ def flag_evaluations_field(chain: list[str | int]) -> ast.Field:
+ flag_chain = columns.chain(chain)
+ if flag_chain is None:
+ # union_unreadable_reason refuses such a query before lazy tables resolve.
+ raise QueryError(f"posthog.flag_evaluations has no column for {'.'.join(str(part) for part in chain)}")
+ return ast.Field(chain=["flag_evaluations", *flag_chain])
+
+ def events_branch(*conditions: ast.Expr | None) -> ast.SelectQuery:
+ return ast.SelectQuery(
+ select=events_columns,
+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),
+ where=and_conditions([*conditions, filters.events]),
+ )
+
+ flag_call = parse_expr("events.event = {event}", {"event": ast.Constant(value=FLAG_CALLED_EVENT)})
+ flag_call_since_cutoff = ast.And(
+ exprs=[flag_call, parse_expr("events.timestamp >= {cutoff}", {"cutoff": cutoff})]
+ )
+ # On mode 2 every flag call comes from flag_evaluations, as the flag_evaluations swap reads it. That leaves out
+ # the calls without a key, which ingestion keeps writing to events.
+ moved = flag_call if only_in_flag_evaluations else flag_call_since_cutoff
+ branches = [events_branch(ast.Not(expr=moved))]
+ if not only_in_flag_evaluations and filters.keyless_flag_calls_can_match:
+ branches.append(events_branch(flag_call_since_cutoff, ast.Not(expr=parse_expr(_HAS_FLAG_KEY))))
+ branches.append(
+ ast.SelectQuery(
+ select=[ast.Alias(alias=name, expr=flag_evaluations_field(chain)) for name, chain in accessed.items()],
+ select_from=ast.JoinExpr(
+ table=ast.Field(chain=["posthog", "flag_evaluations"]), alias="flag_evaluations"
+ ),
+ where=and_conditions(
+ [parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff}), filters.flag_evaluations]
+ ),
+ )
+ )
+ return ast.SelectQuery(
+ select=[ast.Field(chain=[name]) for name in accessed],
+ select_from=ast.JoinExpr(table=ast.SelectSetQuery.create_from_queries(branches, "UNION ALL")),
+ )
+
+ def to_printed_clickhouse(self, context):
+ return "events"
+
+ def to_printed_hogql(self):
+ return EVENTS_WITH_FLAG_CALLS_TABLE
+
+
+# Tables whose rows are events with the events columns. Property types, access control and time bounds treat their reads
+# as reads of events.
+EVENT_ROW_TABLES = (EventsTable, FlagEvaluationsTable, EventsWithFlagCallsTable)
diff --git a/posthog/hogql/database/schema/flag_evaluations.py b/posthog/hogql/database/schema/flag_evaluations.py
index 174d2aa087fe..766e28f22700 100644
--- a/posthog/hogql/database/schema/flag_evaluations.py
+++ b/posthog/hogql/database/schema/flag_evaluations.py
@@ -1,3 +1,4 @@
+from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING
from posthog.hogql import ast
@@ -22,6 +23,7 @@ from posthog.hogql.database.schema.util.uuid import uuid_string_expr_to_uint128_
from posthog.hogql.parser import parse_expr
from posthog.constants import GROUP_TYPES_LIMIT
+from posthog.date_util import start_of_day
if TYPE_CHECKING:
from posthog.hogql.database.database import Database
@@ -32,6 +34,21 @@ if TYPE_CHECKING:
# name is spelled out here rather than imported.
FLAG_EVALUATIONS_CLICKHOUSE_TABLE = "flag_evaluations"
+# At most FLAG_EVALUATIONS_TTL_DAYS - 1, because the TTL can already have dropped the day FLAG_EVALUATIONS_TTL_DAYS
+# back. It stays lower while the table still lacks the oldest days that a shorter TTL dropped. The TTL constant
+# lives in posthog/models/flag_evaluations/sql.py.
+FLAG_EVALUATIONS_COMPLETE_DAYS = 89
+
+
+def utc_midnight(now: datetime) -> datetime:
+ return start_of_day(now.astimezone(UTC))
+
+
+def flag_evaluations_coverage_start(now: datetime) -> datetime:
+ """The start of the oldest UTC day whose flag calls are all in flag_evaluations at `now`."""
+ return utc_midnight(now) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)
+
+
# Keep it equal to FLAG_EVALUATIONS_TABLE in frontend/src/scenes/feature-flags/flagEvaluationsTable.ts.
FLAG_EVALUATIONS_HOGQL_TABLE = "posthog.flag_evaluations"
diff --git a/posthog/hogql/database/schema/persons.py b/posthog/hogql/database/schema/persons.py
index 7b99834ba573..c3387be6529c 100644
--- a/posthog/hogql/database/schema/persons.py
+++ b/posthog/hogql/database/schema/persons.py
@@ -24,7 +24,11 @@ from posthog.hogql.database.models import (
)
from posthog.hogql.database.schema.persons_pdi import PersonsPDITable
from posthog.hogql.database.schema.persons_revenue_analytics import PersonsRevenueAnalyticsTable
-from posthog.hogql.database.schema.util.where_clause_extractor import WhereClauseExtractor, top_level_conjuncts
+from posthog.hogql.database.schema.util.where_clause_extractor import (
+ WhereClauseExtractor,
+ is_nondeterministic_function,
+ top_level_conjuncts,
+)
from posthog.hogql.errors import ResolutionError
from posthog.hogql.parser import parse_select
from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr
@@ -267,20 +271,16 @@ def select_from_persons_table(
return select
-# These return different values in the subquery and in the outer query.
-_NONDETERMINISTIC_FUNCTIONS = frozenset({"rand", "rownumberinblock", "rownumberinallblocks", "nowinblock"})
-
-
class _CopyableConjunctVisitor(TraversingVisitor):
- def __init__(self, table_type: ast.Type):
+ def __init__(self, table_types: tuple[ast.Type, ...]):
super().__init__()
- self.table_type = table_type
+ self.table_types = table_types
self.reads_table = False
self.copyable = True
def visit_field(self, node: ast.Field):
field_type = node.type.field_type if isinstance(node.type, ast.PropertyType) else node.type
- if isinstance(field_type, ast.FieldType) and field_type.table_type is self.table_type:
+ if isinstance(field_type, ast.FieldType) and any(field_type.table_type is t for t in self.table_types):
self.reads_table = True
else:
self.copyable = False
@@ -293,7 +293,7 @@ class _CopyableConjunctVisitor(TraversingVisitor):
self.copyable = False
def visit_call(self, node: ast.Call):
- if node.name.lower() in _NONDETERMINISTIC_FUNCTIONS:
+ if is_nondeterministic_function(node.name):
self.copyable = False
super().visit_call(node)
@@ -327,12 +327,20 @@ def build_person_id_pushdown_predicate(join_to_add: LazyJoinToAdd, node: SelectQ
its person columns.
Returns None unless the left table is the query's own FROM table and at least one term qualifies.
"""
- left = node.select_from
+ # A lazy left table, such as the union of events and flag_evaluations, is already a subquery in `node`. Its recorded
+ # scan holds the FROM entry as the query wrote it.
+ scan = join_to_add.from_scan
+ left = scan.join if scan is not None else node.select_from
+ left_types: tuple[ast.Type, ...] = (
+ scan.types if scan is not None else (left.type,) if left is not None and left.type is not None else ()
+ )
if (
left is None
or not isinstance(left.table, ast.Field)
or left.table_args
- or left.type is not join_to_add.lazy_join_type.table_type
+ or not left_types
+ or left_types[0] is not join_to_add.lazy_join_type.table_type
+ or (scan is not None and (node.select_from is None or node.select_from.alias != join_to_add.from_table))
):
return None
@@ -342,7 +350,7 @@ def build_person_id_pushdown_predicate(join_to_add: LazyJoinToAdd, node: SelectQ
if where is None:
continue
for conjunct in top_level_conjuncts(where):
- visitor = _CopyableConjunctVisitor(left.type)
+ visitor = _CopyableConjunctVisitor(left_types)
visitor.visit(conjunct)
if visitor.copyable and visitor.reads_table:
conjuncts.append(cloner.visit(conjunct))
diff --git a/posthog/hogql/database/schema/util/where_clause_extractor.py b/posthog/hogql/database/schema/util/where_clause_extractor.py
index 56f14d9a4cab..ad3cb2ec2697 100644
--- a/posthog/hogql/database/schema/util/where_clause_extractor.py
+++ b/posthog/hogql/database/schema/util/where_clause_extractor.py
@@ -630,6 +630,9 @@ def event_field_type(expr: ast.Expr) -> ast.FieldType | None:
def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableType]:
"""If the expression resolves to $session_id on the events table, return its (possibly aliased) table type."""
from posthog.hogql.database.schema.events import EventsTable # noqa: PLC0415 - breaks the schema import cycle
+ from posthog.hogql.database.schema.events_with_flag_calls import ( # noqa: PLC0415 - breaks the schema import cycle
+ EventsWithFlagCallsTable,
+ )
expr_type = _unwrap_field_alias_type(node.type)
@@ -654,7 +657,9 @@ def get_events_session_id_table_type(node: ast.Expr) -> Optional[ast.BaseTableTy
return None
unwrapped = unwrap_table_alias_type(table_type)
- if isinstance(unwrapped, ast.TableType) and isinstance(unwrapped.table, EventsTable):
+ if isinstance(unwrapped, (ast.TableType, ast.LazyTableType)) and isinstance(
+ unwrapped.table, (EventsTable, EventsWithFlagCallsTable)
+ ):
return cast(ast.BaseTableType, table_type)
return None
@@ -693,7 +698,7 @@ def build_session_id_literal_pushdown_predicate(
"""
if outer_node.where is None:
return None
- events_join = _find_join_for_alias(outer_node.select_from, join_to_add.from_table)
+ events_join = _events_scan_for_join(outer_node, join_to_add)
if events_join is None or not isinstance(events_join.table, ast.Field):
return None
@@ -845,6 +850,17 @@ class _CollectTopLevelAliasesVisitor(TraversingVisitor):
self.names.add(str(node.chain[0]))
+def _events_scan_for_join(outer_node: ast.SelectQuery, join_to_add: LazyJoinToAdd) -> Optional[ast.JoinExpr]:
+ """The FROM entry that the lazy join starts from, as the query wrote it.
+
+ Lazy table resolution replaces the scan of a lazy table, such as the union of events and flag_evaluations, with a
+ subquery, so the entry it recorded stands in for it.
+ """
+ if join_to_add.from_scan is not None:
+ return join_to_add.from_scan.join
+ return _find_join_for_alias(outer_node.select_from, join_to_add.from_table)
+
+
def _find_join_for_alias(select_from: Optional[ast.JoinExpr], alias: str) -> Optional[ast.JoinExpr]:
ptr = select_from
while ptr:
@@ -1047,6 +1063,14 @@ class _FieldNodeCollector(TraversingVisitor):
# any future array-expanding function HogQL lowers into WHERE here.
_ROW_MULTIPLYING_FUNCTIONS = {"arrayJoin"}
+# These return a different value on each read, so a copy of a filter that calls one selects other rows than the
+# original. Lowercase, because HogQL accepts some of them in any case, such as RAND().
+_NONDETERMINISTIC_FUNCTIONS = frozenset({"rand", "rownumberinblock", "rownumberinallblocks", "nowinblock"})
+
+
+def is_nondeterministic_function(name: str) -> bool:
+ return name.lower() in _NONDETERMINISTIC_FUNCTIONS
+
class _RowMultiplyingFunctionFinder(TraversingVisitor):
"""Detects any row-multiplying function call (e.g. arrayJoin) anywhere in an expression."""
diff --git a/posthog/hogql/database/test/test_database.py b/posthog/hogql/database/test/test_database.py
index 06e0295fb741..2297fb02eca9 100644
--- a/posthog/hogql/database/test/test_database.py
+++ b/posthog/hogql/database/test/test_database.py
@@ -493,6 +493,12 @@ class TestDatabase(BaseTest, QueryMatchingTest):
include_flag_evaluations_table=True,
),
("read_mode", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, True),
+ param(
+ "read_mode_with_the_union_flag_off",
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ True,
+ union_flag_enabled=False,
+ ),
param(
"read_mode_with_reads_forced_to_events",
FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
@@ -515,7 +521,7 @@ class TestDatabase(BaseTest, QueryMatchingTest):
),
]
)
- def test_read_flag_calls_from_flag_evaluations_follows_the_read_mode(
+ def test_flag_call_reads_follow_the_read_mode_and_the_union_flag(
self,
_name: str,
mode: FlagEvaluationsMode,
@@ -523,7 +529,9 @@ class TestDatabase(BaseTest, QueryMatchingTest):
reads_forced_to_events: bool = False,
debug: bool = False,
include_flag_evaluations_table: bool | None = None,
+ union_flag_enabled: bool = True,
) -> None:
+ enabled_flags = {"hogql-flag-calls-union"} if union_flag_enabled else set()
OrganizationFeatureFlagsConfig.objects.update_or_create(
organization=self.organization, defaults={"flag_evaluations_mode": mode}
)
@@ -531,12 +539,21 @@ class TestDatabase(BaseTest, QueryMatchingTest):
with (
override_settings(DEBUG=debug),
override_instance_config("FLAG_EVALUATIONS_READS_FORCE_EVENTS", reads_forced_to_events),
+ patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key in enabled_flags,
+ ),
):
database = Database.create_for(
team=self.team, include_flag_evaluations_table=include_flag_evaluations_table
)
assert database.read_flag_calls_from_flag_evaluations is expected
+ union_expected = expected and union_flag_enabled
+ assert database.read_flag_calls_with_union is union_expected
+ assert database.has_table(["posthog", "events_with_flag_calls"]) is union_expected
+ context = HogQLContext(team_id=self.team.pk, database=database)
+ assert "posthog.events_with_flag_calls" not in database.serialize(context, include_hidden_posthog_tables=True)
def test_serialize_database_without_fields_matches_full_metadata(self):
credential = DataWarehouseCredential.objects.create(access_key="blah", access_secret="blah", team=self.team)
diff --git a/posthog/hogql/helpers/timestamp_visitor.py b/posthog/hogql/helpers/timestamp_visitor.py
index e9674115f504..554c11fa9adb 100644
--- a/posthog/hogql/helpers/timestamp_visitor.py
+++ b/posthog/hogql/helpers/timestamp_visitor.py
@@ -132,8 +132,7 @@ class IsSimpleTimestampFieldExpressionVisitor(Visitor[bool]):
raise Exception()
def visit_alias(self, node: ast.Alias) -> bool:
- from posthog.hogql.database.schema.events import EventsTable
- from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable
+ from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES
from posthog.hogql.database.schema.session_replay_events import RawSessionReplayEventsTable
from posthog.hogql.database.schema.sessions_v1 import SessionsTableV1
from posthog.hogql.database.schema.sessions_v2 import SessionsTableV2
@@ -152,8 +151,8 @@ class IsSimpleTimestampFieldExpressionVisitor(Visitor[bool]):
table_type = table_type.table_type
return (
(
- isinstance(table_type, ast.TableType)
- and isinstance(table_type.table, EventsTable | FlagEvaluationsTable)
+ isinstance(table_type, ast.TableType | ast.LazyTableType)
+ and isinstance(table_type.table, EVENT_ROW_TABLES)
and resolved_field.name == "timestamp"
)
or (
diff --git a/posthog/hogql/property_planner.py b/posthog/hogql/property_planner.py
index 70405f87b3ec..c87eada0bac8 100644
--- a/posthog/hogql/property_planner.py
+++ b/posthog/hogql/property_planner.py
@@ -6,8 +6,9 @@ from typing import Literal, Optional
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import DatabaseField
-from posthog.hogql.database.schema.events import EventsPersonSubTable, EventsTable
-from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable, FlagEvaluationsTable
+from posthog.hogql.database.schema.events import EventsPersonSubTable
+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES
+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable
from posthog.hogql.database.schema.groups import GroupsTable
from posthog.hogql.database.schema.persons import PersonsTable, RawPersonsTable
from posthog.hogql.errors import (
@@ -497,7 +498,7 @@ def _property_scope(property_type: ast.PropertyType, context: HogQLContext) -> P
except (HogQLNotImplementedError, QueryError, ResolutionError):
return PropertyScope.UNKNOWN
- if isinstance(resolved_table, EventsTable | FlagEvaluationsTable):
+ if isinstance(resolved_table, EVENT_ROW_TABLES):
return PropertyScope.EVENT
if isinstance(
resolved_table, EventsPersonSubTable | FlagEvaluationsPersonSubTable | PersonsTable | RawPersonsTable
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py
index 8f44e67b1995..c294f9172559 100644
--- a/posthog/hogql/resolver.py
+++ b/posthog/hogql/resolver.py
@@ -24,6 +24,7 @@ from posthog.hogql.database.schema.duckdb_table_functions import (
is_dangerous_table_function,
)
from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.database.schema.events_with_flag_calls import EventsWithFlagCallsTable
from posthog.hogql.database.schema.log_entries import (
BatchExportLogEntriesTable,
LogEntriesTable,
@@ -138,7 +139,11 @@ def _string_constants(node: ast.Expr) -> list[ast.Constant]:
# cluster (log_entries is a Distributed over the aux cluster, so a plain IN makes every shard
# of the outer query issue its own remote read against aux).
_GLOBAL_IN_TABLES: tuple[type, ...] = (EventsTable, LogEntriesTable)
-_GLOBAL_IN_LAZY_TABLES: tuple[type, ...] = (ReplayConsoleLogsLogEntriesTable, BatchExportLogEntriesTable)
+_GLOBAL_IN_LAZY_TABLES: tuple[type, ...] = (
+ ReplayConsoleLogsLogEntriesTable,
+ BatchExportLogEntriesTable,
+ EventsWithFlagCallsTable,
+)
class _ShardedTableFinder(TraversingVisitor):
diff --git a/posthog/hogql/restricted_properties.py b/posthog/hogql/restricted_properties.py
index 4c119ff0f1a6..e9e83366dcab 100644
--- a/posthog/hogql/restricted_properties.py
+++ b/posthog/hogql/restricted_properties.py
@@ -8,6 +8,7 @@ from posthog.hogql.context import HogQLContext
from posthog.hogql.database.postgres_table import PostgresTable
from posthog.hogql.database.schema.ai_events import AI_PROPERTY_TO_COLUMN, AI_SUBCOLUMN_TO_SOURCE, AiEventsTable
from posthog.hogql.database.schema.events import EventsGroupSubTable, EventsPersonSubTable, EventsTable
+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES
from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable, FlagEvaluationsTable
from posthog.hogql.database.schema.groups import GroupsTable, RawGroupsTable
from posthog.hogql.database.schema.persons import PersonsTable, RawPersonsTable
@@ -121,7 +122,7 @@ def restricted_property_keys_for_table_type(
elif isinstance(table, EventsGroupSubTable):
prop_def_type = PropertyDefinition.Type.GROUP
group_type_index = table.group_index
- elif isinstance(table, EventsTable | FlagEvaluationsTable | AiEventsTable):
+ elif isinstance(table, (*EVENT_ROW_TABLES, AiEventsTable)):
prop_def_type = PropertyDefinition.Type.EVENT
elif isinstance(table, (PersonsTable, RawPersonsTable)):
prop_def_type = PropertyDefinition.Type.PERSON
diff --git a/posthog/hogql/transforms/events_predicate_pushdown.py b/posthog/hogql/transforms/events_predicate_pushdown.py
index 43f6ee117a70..93a644f2c726 100644
--- a/posthog/hogql/transforms/events_predicate_pushdown.py
+++ b/posthog/hogql/transforms/events_predicate_pushdown.py
@@ -80,6 +80,13 @@ def apply_events_predicate_pushdown(
return node
+# Join types across which moving an events PREDICATE is result-safe (they preserve the events/left side). This is
+# broader than the all-rows-preserved check (`_all_joins_preserve_every_row`, LEFT only): INNER / CROSS are
+# predicate-safe but can drop an events row, so they pass here yet are rejected when deciding whether to also push
+# the LIMIT (`_safe_inner_limit`).
+PREDICATE_SAFE_JOIN_TYPES = frozenset({"JOIN", "INNER JOIN", "LEFT JOIN", "LEFT OUTER JOIN", "CROSS JOIN"})
+
+
class SelectAliasInliner(CloningVisitor):
"""Inlines SELECT-alias references in a pushed predicate so the full aliased expression is carried into the
subquery, not the bare name (which would re-bind to the raw column and drop the wrapper)."""
@@ -352,12 +359,6 @@ class EventsPredicatePushdownTransform(TraversingVisitor):
Runs after logical lowering (so property reads are PropertyAccess) and before the ClickHouse physical passes,
applying bottom-up so nested `FROM events` subqueries benefit too."""
- # Join types across which moving an events PREDICATE is result-safe (they preserve the events/left side). This is
- # broader than the all-rows-preserved check (`_all_joins_preserve_every_row`, LEFT only): INNER / CROSS are
- # predicate-safe but can drop an events row, so they pass here yet are rejected when deciding whether to also push
- # the LIMIT (`_safe_inner_limit`).
- _SAFE_JOIN_TYPES = {"JOIN", "INNER JOIN", "LEFT JOIN", "LEFT OUTER JOIN", "CROSS JOIN"}
-
def __init__(
self,
context: HogQLContext,
@@ -596,7 +597,7 @@ class EventsPredicatePushdownTransform(TraversingVisitor):
join = node.select_from.next_join if node.select_from else None
while join is not None:
# A missing join_type is unreachable today but treated as unsafe rather than safe.
- if join.join_type is None or join.join_type not in self._SAFE_JOIN_TYPES:
+ if join.join_type is None or join.join_type not in PREDICATE_SAFE_JOIN_TYPES:
return set()
if join.alias:
aliases.add(join.alias)
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
index 6dd60d1986be..125eb627c34e 100644
--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -25,7 +25,13 @@ from posthog.hogql.constants import FLAG_CALLED_EVENT
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import DatabaseField
from posthog.hogql.database.schema.events import EventsTable
-from posthog.hogql.database.schema.flag_evaluations import ELEMENT_STAND_INS, FlagEvaluationsTable
+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES
+from posthog.hogql.database.schema.flag_evaluations import (
+ ELEMENT_STAND_INS,
+ FlagEvaluationsTable,
+ flag_evaluations_coverage_start,
+ utc_midnight,
+)
from posthog.hogql.database.schema.util.where_clause_extractor import (
event_field_type,
top_level_conjuncts,
@@ -46,12 +52,6 @@ if TYPE_CHECKING:
logger = structlog.get_logger(__name__)
-# At most FLAG_EVALUATIONS_TTL_DAYS - 1, because the TTL can already have dropped the day FLAG_EVALUATIONS_TTL_DAYS
-# back. It stays lower while the table still lacks the oldest days that a shorter TTL dropped. The TTL constant
-# lives in posthog/models/flag_evaluations/sql.py, and importing it loads the posthog.models package, which the
-# printer's import path avoids.
-FLAG_EVALUATIONS_COMPLETE_DAYS = 89
-
_FLAG_KEY_PROPERTY = "$feature_flag"
_Predicate = Callable[[ast.Expr], bool]
@@ -112,7 +112,7 @@ _UNMOVABLE_SHAPE = "it reads events in a shape that can't be moved to posthog.fl
# An alias with one of these names can shadow the column in an unqualified read.
_SCAN_COLUMNS = frozenset({"event", "timestamp", "properties"})
-_EVENTS_NAMES = frozenset({"events", "posthog.events"})
+EVENTS_NAMES = frozenset({"events", "posthog.events"})
def can_read_flag_evaluations(
@@ -194,14 +194,15 @@ def swap_flag_calls(
node: _T_AST, context: HogQLContext, resolve: _Resolve[_T_AST], *, view_name: str | None = None
) -> FlagCallSwap[_T_AST]:
"""`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading
- `posthog.flag_evaluations` instead.
+ `posthog.flag_evaluations` instead. When the flag-call union is on, each other scan that reads flag calls with
+ other events reads `posthog.events_with_flag_calls`, as `read_flag_calls_with_union` decides.
- This returns the swapped query only when it resolves, every name in it binds as it does in the original, and
+ This returns the changed query only when it resolves, every name in it binds as it does in the original, and
every field in it that starts with `posthog.events` reads a scan.
Otherwise it returns the resolved original. For an organization whose flag calls are only in
`flag_evaluations`, it raises a QueryError instead of returning the original. It also raises before it
- resolves anything when such an organization's query has a scan of only flag calls with a shape that
- `flag_evaluations` cannot stand in for.
+ resolves anything when such an organization's query has a scan of flag calls with a shape that neither
+ `flag_evaluations` nor the union can stand in for.
While `events` still gets flag calls, the original query is the answer whenever the two tables can read a row
differently:
@@ -222,6 +223,10 @@ def swap_flag_calls(
that body already checked them. A top-level query raises when it reads a view whose resolved body still reads
only flag calls from `events`, such as a body that filters with `matchesAction`.
"""
+ # Deferred: flag_calls_union imports this module.
+ from posthog.hogql.transforms.flag_calls_union import refuse_union_table_by_name # noqa: PLC0415
+
+ refuse_union_table_by_name(node, context)
database = _swap_database(context)
if database is None:
return FlagCallSwap(source=node, resolved=resolve(node))
@@ -244,21 +249,30 @@ def _resolve_with_swaps(
node: _T_AST, context: HogQLContext, resolve: _Resolve[_T_AST], database: "Database", view_name: str | None
) -> FlagCallSwap[_T_AST]:
only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations
+ # Deferred: flag_calls_union imports this module.
+ from posthog.hogql.transforms.flag_calls_union import ( # noqa: PLC0415
+ read_flag_calls_with_union,
+ union_unreadable_reason,
+ )
+
with context.timings.measure("flag_calls_from_flag_evaluations"):
+ # The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.
+ compares_as_string = partial(_flag_key_compares_as_string, context)
rewritten = clone_expr(node)
clones = _join_clones(node, rewritten) if only_in_flag_evaluations else {}
swapper = _EventsScanSwapper(
now=datetime.now(UTC),
timezone=ZoneInfo(database.get_timezone()),
only_in_flag_evaluations=only_in_flag_evaluations,
- # The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.
- flag_key_compares_as_string=partial(_flag_key_compares_as_string, context),
+ flag_key_compares_as_string=compares_as_string,
)
swapper.visit(rewritten)
- if swapper.refusals:
+ union = read_flag_calls_with_union(rewritten, context, compares_as_string)
+ refusals = [*swapper.refusals, *union.refusals]
+ if refusals:
if only_in_flag_evaluations:
- raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0], view_name)
- logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=swapper.refusals[0])
+ raise _flag_calls_only_in_flag_evaluations_error(refusals[0], view_name)
+ logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=refusals[0])
sources = JoinSources() if only_in_flag_evaluations else None
original = resolve(node, join_sources=sources)
if sources is not None:
@@ -266,7 +280,7 @@ def _resolve_with_swaps(
refusal = swapper.swap_scans_found_by_type(original, sources, clones)
if refusal is not None:
raise _flag_calls_only_in_flag_evaluations_error(refusal, view_name)
- if not swapper.swapped:
+ if not swapper.swapped and not union.renamed:
return FlagCallSwap(source=node, resolved=original)
try:
swapped = resolve(rewritten)
@@ -274,7 +288,10 @@ def _resolve_with_swaps(
# Expected for a query that reads a field flag_evaluations lacks, such as goe_0.properties.
reason = f"it reads something that table doesn't have: {error}"
else:
- if not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):
+ unreadable = union_unreadable_reason(swapped, context) if union.renamed else None
+ if unreadable is not None:
+ reason = unreadable
+ elif not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):
reason = "a name in it would read a different column there"
elif not _posthog_events_qualifiers_read_scans(swapped):
reason = "a name in it that starts with posthog.events reads another table"
@@ -290,12 +307,16 @@ def _resolve_with_swaps(
def _flag_key_compares_as_string(context: HogQLContext) -> bool:
- """Whether `properties.$feature_flag` compares as a string. The printer converts a property that the team's
- property definitions type as Boolean, Numeric or DateTime before it compares the value."""
if context.team_id is None:
return True
- metadata = _event_property_metadata(context, {_FLAG_KEY_PROPERTY})
- return metadata.event_properties.get(_FLAG_KEY_PROPERTY, {}).get("type") in (None, "String")
+ return flag_key_typed_as_string(_event_property_metadata(context, {_FLAG_KEY_PROPERTY}))
+
+
+def flag_key_typed_as_string(metadata: PropertyMetadata | None) -> bool:
+ """Whether `properties.$feature_flag` compares as a string under the property types in `metadata`. The printer
+ converts a property that the team's property definitions type as Boolean, Numeric or DateTime before it compares
+ the value."""
+ return metadata is None or metadata.event_properties.get(_FLAG_KEY_PROPERTY, {}).get("type") in (None, "String")
@frozen
@@ -366,6 +387,16 @@ class EventsScan:
self.shadowed: frozenset[str] = frozenset()
self.qualifier_shadows: frozenset[str] = frozenset()
+ @classmethod
+ def of_scan_chains(cls, flag_key_compares_as_string: Callable[[], bool]) -> "EventsScan":
+ """A scan whose reads are unqualified chains, for a predicate already rewritten relative to one scan."""
+ return cls(
+ ast.JoinExpr(table=ast.Field(chain=["events"])),
+ alias="",
+ other_tables=frozenset(),
+ flag_key_compares_as_string=flag_key_compares_as_string,
+ )
+
@classmethod
def of(
cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]
@@ -377,7 +408,7 @@ class EventsScan:
return None
parts = [str(part) for part in join.table.chain]
table_name = ".".join(parts)
- if table_name not in _EVENTS_NAMES or table_name in ctes:
+ if table_name not in EVENTS_NAMES or table_name in ctes:
return None
scan = cls(
join=join,
@@ -430,7 +461,7 @@ class EventsScan:
return _below_posthog_events(chain)
def shape_refusal(self, node: ast.SelectQuery) -> str | None:
- """Why `flag_evaluations` cannot stand in for this scan whatever its filters say, or None."""
+ """Why another table cannot stand in for this scan whatever its filters say, or None."""
if self.join.table_args is not None or self.join.table_final:
return "it uses FINAL or table arguments on events"
if self.join.column_aliases:
@@ -483,7 +514,7 @@ class EventsScan:
return self.column(expr) == ["properties", _FLAG_KEY_PROPERTY]
def bounds_timestamp(self, conjuncts: list[ast.Expr], now: datetime, timezone: ZoneInfo) -> bool:
- coverage_start = _utc_midnight(now) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)
+ coverage_start = flag_evaluations_coverage_start(now)
for conjunct in conjuncts:
bound = self._lower_bound(conjunct)
if bound is None:
@@ -609,7 +640,8 @@ def point_scan_at(
"""Points `scan` at the table `chain` names, and moves the PREWHERE of `node`, with `condition`, into its WHERE.
The new table can read person columns through a join that the printer adds later, and ClickHouse rejects a
- PREWHERE that reads a joined column. WHERE selects the same rows.
+ PREWHERE that reads a joined column. A query that reads the union reads a subquery, which cannot have a PREWHERE.
+ WHERE selects the same rows.
"""
# The old alias keeps qualified reads such as `events.timestamp` resolving.
scan.join.alias = scan.alias
@@ -653,10 +685,6 @@ def filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:
return _conjuncts(node.where) + _conjuncts(node.prewhere)
-def _utc_midnight(now: datetime) -> datetime:
- return start_of_day(now.astimezone(UTC))
-
-
def names_flag_called_event(node: ast.AST) -> bool:
"""Whether `$feature_flag_called` appears as a constant anywhere in `node`."""
finder = _FlagCalledConstantFinder()
@@ -688,7 +716,7 @@ def _earliest_instant(expr: ast.Expr, now: datetime, timezone: ZoneInfo) -> date
if name in ("now", "now64"):
return now
if name == "today" and not args:
- return _utc_midnight(now) - _DAY_IN_ANY_TIMEZONE
+ return utc_midnight(now) - _DAY_IN_ANY_TIMEZONE
if name == "assumeNotNull" and len(args) == 1:
return _earliest_instant(args[0], now, timezone)
if name in ("toDateTime", "toDateTime64") and args:
@@ -1205,10 +1233,10 @@ def _target(current: ast.Type | None) -> tuple[str, ...]:
json.dumps(join.resolver_params, sort_keys=True, default=str),
".".join(str(part) for part in join.from_field),
)
- elif isinstance(current, ast.TableType):
+ elif isinstance(current, (ast.TableType, ast.LazyTableType)):
return (
("scan",)
- if isinstance(current.table, (EventsTable, FlagEvaluationsTable))
+ if isinstance(current.table, EVENT_ROW_TABLES)
else ("table", current.table.__class__.__name__)
)
elif isinstance(current, ast.CTETableType):
diff --git a/posthog/hogql/transforms/flag_calls_union.py b/posthog/hogql/transforms/flag_calls_union.py
new file mode 100644
index 000000000000..d4ac41265349
--- /dev/null
+++ b/posthog/hogql/transforms/flag_calls_union.py
@@ -0,0 +1,663 @@
+"""Read flag calls from `events` and `posthog.flag_evaluations` together when a query reads them with other events.
+
+The flag_evaluations swap in `flag_calls_from_flag_evaluations` swaps a scan that reads only flag calls to
+`posthog.flag_evaluations`. This transform renames each remaining `events` scan that names
+`$feature_flag_called` to `posthog.events_with_flag_calls`, a lazy table with the events fields. Lazy table
+resolution replaces it with a union of the two tables, split at the coverage start.
+
+The rename runs before type resolution, so every rule here is syntactic. The union returns the right rows for any
+scan, so the rules decide which scans pay for a second table, and which shapes the union cannot read yet.
+"""
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+from posthog.hogql import ast
+from posthog.hogql.constants import FLAG_CALLED_EVENT
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.lazy_join_tags import (
+ EVENTS_TO_SESSIONS_V1,
+ EVENTS_TO_SESSIONS_V2,
+ EVENTS_TO_SESSIONS_V3,
+ FOREIGN_KEY,
+ GROUP_N,
+ PERSON_DISTINCT_ID_OVERRIDES,
+ PERSON_DISTINCT_IDS,
+ PERSONS,
+)
+from posthog.hogql.database.schema.events_with_flag_calls import (
+ EVENTS_WITH_FLAG_CALLS_CHAIN,
+ EVENTS_WITH_FLAG_CALLS_TABLE,
+ EventsWithFlagCallsTable,
+ FlagEvaluationsColumns,
+)
+from posthog.hogql.database.schema.util.where_clause_extractor import (
+ contains_in_cohort,
+ contains_row_multiplying_function,
+ is_nondeterministic_function,
+)
+from posthog.hogql.errors import QueryError, ResolutionError
+from posthog.hogql.property_metadata import load_property_metadata
+from posthog.hogql.restricted_properties import key_read_overlaps_restriction, restricted_property_keys_for_table_type
+from posthog.hogql.transforms.events_predicate_pushdown import PREDICATE_SAFE_JOIN_TYPES, SelectAliasInliner
+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (
+ EVENTS_NAMES,
+ EventsScan,
+ ScanVisitor,
+ SyntaxVisitor,
+ and_conditions,
+ boolean_operands,
+ compared_values,
+ comparison_of,
+ filter_conjuncts,
+ flag_key_predicate,
+ point_scan_at,
+)
+from posthog.hogql.transforms.property_types import STRING_INPUT_CONVERSIONS
+from posthog.hogql.visitor import CloningVisitor
+
+from posthog.dataclasses import frozen
+
+if TYPE_CHECKING:
+ from posthog.hogql.database.database import Database
+
+# The joins whose resolver compares only the join's key and `extra_from_fields` on the source table. Lazy table
+# resolution projects exactly those from the union, so a join that reads anything else would name a missing column.
+_UNION_JOIN_RESOLVERS = frozenset(
+ {
+ EVENTS_TO_SESSIONS_V1,
+ EVENTS_TO_SESSIONS_V2,
+ EVENTS_TO_SESSIONS_V3,
+ FOREIGN_KEY,
+ GROUP_N,
+ PERSON_DISTINCT_ID_OVERRIDES,
+ PERSON_DISTINCT_IDS,
+ PERSONS,
+ }
+)
+
+
+def refuse_union_table_by_name(node: ast.AST, context: HogQLContext) -> None:
+ """Raises when `node` names the union table. The table exists only for the rename to point at, so a query that
+ names it would depend on an internal table."""
+ if context.database is None or not context.database.has_table(EVENTS_WITH_FLAG_CALLS_CHAIN):
+ return
+ finder = _UnionTableFinder(context.database)
+ finder.visit(node)
+ if finder.found:
+ raise QueryError(f"Unknown table `posthog.{EVENTS_WITH_FLAG_CALLS_TABLE}`.")
+
+
+@frozen
+class UnionRename:
+ renamed: bool
+ # Why the union cannot read each other scan that names `$feature_flag_called` with other events.
+ refusals: tuple[str, ...]
+
+
+def read_flag_calls_with_union(
+ node: ast.AST, context: HogQLContext, flag_key_compares_as_string: Callable[[], bool]
+) -> UnionRename:
+ """Points each mixed flag-call scan in `node` at the union, in place, when the union reads this query's scans.
+
+ An organization whose flag calls are only in `flag_evaluations` cannot read a refused scan at all. Every other
+ organization keeps reading `events` for it.
+ """
+ database = _union_database(context)
+ if database is None:
+ return UnionRename(renamed=False, refusals=())
+ only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations
+ if not only_in_flag_evaluations and _flag_key_is_restricted(database, context):
+ # The union tells the flag calls that only events holds by their raw `$feature_flag`, with `_HAS_FLAG_KEY`, and
+ # a restricted property reads as missing. Each flag call since the cutoff would then come from both tables.
+ return UnionRename(renamed=False, refusals=())
+ return rename_flag_call_scans_to_union(
+ node, flag_key_compares_as_string, only_in_flag_evaluations=only_in_flag_evaluations
+ )
+
+
+def union_unreadable_reason(node: ast.AST, context: HogQLContext) -> str | None:
+ """Why the union cannot read a scan in the resolved `node`, or None when it can read them all.
+
+ Lazy table resolution builds the union after this check, from these same fields, the keys of the joins that start
+ from the scan, and the other fields those joins declare that they read.
+ """
+ database = _union_database(context)
+ if database is None:
+ return None
+ collector = _UnionChainCollector()
+ collector.visit(node)
+ if collector.unreadable_joins:
+ return f"it reads {collector.unreadable_joins[0]}, a join that the union can't read yet"
+ columns = FlagEvaluationsColumns(
+ database.get_table(EVENTS_WITH_FLAG_CALLS_CHAIN), database.get_table(["posthog", "flag_evaluations"])
+ )
+ if collector.export_collision is not None:
+ first, second = (".".join(str(part) for part in chain) for chain in collector.export_collision)
+ return f"it reads {first} and {second}, which the union would return as one column"
+ for chain in collector.chains:
+ if columns.chain(chain) is None:
+ return f"it reads {'.'.join(str(part) for part in chain)}, which that table doesn't have"
+ return _typed_property_parsed_as_string(collector.string_parsed, context)
+
+
+def _typed_property_parsed_as_string(reads: list[tuple[str, str]], context: HogQLContext) -> str | None:
+ # Each branch converts a typed property before the union returns it, so a call that parses the raw string, such
+ # as toFloatOrZero, would get the converted value.
+ if not reads or context.team_id is None:
+ return None
+ metadata = load_property_metadata(
+ context,
+ event_property_names={name for kind, name in reads if kind == "event"},
+ person_property_names={name for kind, name in reads if kind == "person"},
+ group_property_names={},
+ )
+ for kind, name in reads:
+ types = metadata.event_properties if kind == "event" else metadata.person_properties
+ property_type = types.get(name, {}).get("type")
+ if property_type not in (None, "String"):
+ return f"it parses the {property_type} {kind} property {name} as a string"
+ return None
+
+
+@frozen
+class UnionBranchFilters:
+ """The predicates that a union scan's branches copy from the query that reads the scan.
+
+ Each is a set of conjuncts from that query's WHERE and PREWHERE, so every row the query keeps passes it.
+ """
+
+ events: ast.Expr | None
+ # For the branch that reads flag_evaluations. A read of `event` is `$feature_flag_called` there.
+ flag_evaluations: ast.Expr | None
+ # False when the query pins `$feature_flag` to keys that only a call with a non-empty string key matches.
+ keyless_flag_calls_can_match: bool
+
+
+def union_branch_filters(
+ node: ast.SelectQuery,
+ union: EventsWithFlagCallsTable,
+ columns: FlagEvaluationsColumns,
+ flag_key_compares_as_string: Callable[[], bool],
+) -> UnionBranchFilters:
+ """The filters for the union scan that `node` reads, from the resolved WHERE and PREWHERE of `node`."""
+ extracted = _scan_conjuncts(node, union)
+ flag_call_conjuncts = [_fold(_EventAsFlagCall().visit(conjunct)) for conjunct in extracted]
+ # The flag_evaluations swap accepts a key only when it is a non-empty string that no other JSON value prints as.
+ # A flag call that ingestion left in events, with a missing, empty or non-string key, cannot match one.
+ keys = EventsScan.of_scan_chains(flag_key_compares_as_string).flag_keys(
+ [_StepTests().visit(conjunct) for conjunct in flag_call_conjuncts]
+ )
+ flag_evaluations_conjuncts = [
+ mapped
+ for conjunct in flag_call_conjuncts
+ if _truth(conjunct) is not True and (mapped := _FlagEvaluationsReads(columns).copy(conjunct)) is not None
+ ]
+ return UnionBranchFilters(
+ events=_events_filter(extracted),
+ flag_evaluations=and_conditions([*flag_evaluations_conjuncts, flag_key_predicate("flag_evaluations", keys)]),
+ keyless_flag_calls_can_match=not keys,
+ )
+
+
+def union_events_filter(node: ast.SelectQuery, union: EventsWithFlagCallsTable) -> ast.Expr | None:
+ """The filter that a scan of `events` copies from `node`, as `union_branch_filters` builds it."""
+ return _events_filter(_scan_conjuncts(node, union))
+
+
+def rename_flag_call_scans_to_union(
+ node: ast.AST,
+ flag_key_compares_as_string: Callable[[], bool],
+ *,
+ only_in_flag_evaluations: bool,
+) -> UnionRename:
+ """Point each `events` scan that the union can read, and whose query names `$feature_flag_called`, at the union.
+
+ A scan qualifies when all of these hold:
+
+ 1. Its query compares the scan's `event` with `$feature_flag_called` by `=` or `IN`, in one of the query's own
+ clauses. A subquery's comparison does not count. A funnel step column such as
+ `if(event = '$feature_flag_called' AND ..., 1, 0)` counts.
+ 2. The scan is the first table in FROM, and every join after it is INNER, LEFT or CROSS, GLOBAL or not.
+ 3. The scan has no FINAL, table arguments, column aliases or SAMPLE other than 1, and its query selects no `*`
+ or `COLUMNS(...)`.
+
+ A scan that the flag_evaluations swap changed reads `posthog.flag_evaluations` and is not an `events` scan any
+ more. The flag_evaluations swap swaps or refuses a first scan that reads only flag calls, so the union neither
+ renames nor refuses it. The swap cannot read an unqualified `event` next to a join. On mode 1 a first scan that
+ reads only flag calls through such an `event` stays on events. On mode 2 the union renames it.
+ """
+ renamer = _UnionRenamer(flag_key_compares_as_string, only_in_flag_evaluations)
+ renamer.visit(node)
+ return UnionRename(renamed=renamer.renamed, refusals=tuple(renamer.refusals))
+
+
+def _union_database(context: HogQLContext) -> "Database | None":
+ """The database when the union reads this query's mixed flag-call scans, and None otherwise."""
+ if (
+ not context.rewrite_flag_calls_to_flag_evaluations
+ or context.database is None
+ or not context.database.read_flag_calls_with_union
+ ):
+ return None
+ return context.database
+
+
+def _flag_key_is_restricted(database: "Database", context: HogQLContext) -> bool:
+ restricted = restricted_property_keys_for_table_type(ast.TableType(table=database.get_table("events")), context)
+ return key_read_overlaps_restriction("$feature_flag", restricted)
+
+
+def _union_table_of(table_type: ast.Type | None) -> EventsWithFlagCallsTable | None:
+ while isinstance(table_type, (ast.TableAliasType, ast.VirtualTableType)):
+ table_type = table_type.table_type
+ if isinstance(table_type, ast.LazyTableType) and isinstance(table_type.table, EventsWithFlagCallsTable):
+ return table_type.table
+ return None
+
+
+def _union_chain(field_type: ast.Type | None) -> tuple[EventsWithFlagCallsTable, list[str | int]] | None:
+ """The union scan that a resolved field reads directly, and its chain there, or None."""
+ property_chain: list[str | int] = []
+ if isinstance(field_type, ast.PropertyType):
+ property_chain = list(field_type.chain)
+ field_type = field_type.field_type
+ if not isinstance(field_type, ast.FieldType):
+ return None
+ chain: list[str | int] = [field_type.name, *property_chain]
+ if isinstance(field_type.table_type, ast.VirtualTableType):
+ chain = [field_type.table_type.field, *chain]
+ union = _union_table_of(field_type.table_type)
+ return (union, chain) if union is not None else None
+
+
+def _scan_conjuncts(node: ast.SelectQuery, union: EventsWithFlagCallsTable) -> list[ast.Expr]:
+ """The parts of the WHERE and PREWHERE of `node` that read only the union scan, with reads as chains on it."""
+ inliner = SelectAliasInliner({expr.alias: expr.expr for expr in node.select if isinstance(expr, ast.Alias)})
+ extracted = [_scan_predicate(inliner.visit(conjunct), union) for conjunct in filter_conjuncts(node)]
+ return [conjunct for conjunct in extracted if conjunct is not None]
+
+
+def _scan_predicate(expr: ast.Expr, union: EventsWithFlagCallsTable) -> ast.Expr | None:
+ """`expr` with each read as a chain on the scan, or the conjuncts of it that read only the scan, or None.
+
+ An OR counts only when every branch of it does, so every row that `expr` accepts passes the result.
+ """
+ if (parts := boolean_operands(expr, ast.And)) is not None:
+ return and_conditions([_scan_predicate(part, union) for part in parts])
+ if (branches := boolean_operands(expr, ast.Or)) is not None:
+ extracted = [_scan_predicate(branch, union) for branch in branches]
+ kept = [branch for branch in extracted if branch is not None]
+ return ast.Or(exprs=kept) if kept and len(kept) == len(extracted) else None
+ # A copy in each branch would multiply rows, or would move a cohort join that the printer adds later.
+ if contains_row_multiplying_function(expr) or contains_in_cohort(expr):
+ return None
+ copier = _ScanReadCopier(union)
+ copied = copier.visit(expr)
+ return None if copier.failed else copied
+
+
+def _events_filter(scan_conjuncts: list[ast.Expr]) -> ast.Expr | None:
+ return and_conditions([_Qualified("events").visit(conjunct) for conjunct in scan_conjuncts])
+
+
+def _truth(expr: ast.Expr) -> bool | None:
+ """The truth value of a constant false, true, 0 or 1, or None for any other expression."""
+ if isinstance(expr, ast.Constant) and expr.value in (0, 1):
+ return bool(expr.value)
+ return None
+
+
+# Whether a comparison of constants passes when the left value is among the right values.
+_MATCH_PASSES = {
+ ast.CompareOperationOp.Eq: True,
+ ast.CompareOperationOp.In: True,
+ ast.CompareOperationOp.NotEq: False,
+ ast.CompareOperationOp.NotIn: False,
+}
+
+
+def _compares_like_clickhouse(values: list[object]) -> bool:
+ """Whether Python and ClickHouse agree on how these constants compare.
+
+ ClickHouse converts mixed types before it compares (`1 IN ('1')` is true), which Python does not.
+ NULL and floats are neither str nor int, so they never fold.
+ """
+ return any(all(isinstance(value, kind) for value in values) for kind in (str, int))
+
+
+def _fold(expr: ast.Expr) -> ast.Expr:
+ """`expr` with comparisons of constants, and the AND, OR, NOT and `if` they decide, replaced by their value."""
+ if (parts := boolean_operands(expr, ast.And)) is not None:
+ folded = [_fold(part) for part in parts]
+ if any(_truth(part) is False for part in folded):
+ return ast.Constant(value=False)
+ return and_conditions([part for part in folded if _truth(part) is not True]) or ast.Constant(value=True)
+ if (parts := boolean_operands(expr, ast.Or)) is not None:
+ folded = [_fold(part) for part in parts]
+ if any(_truth(part) is True for part in folded):
+ return ast.Constant(value=True)
+ kept = [part for part in folded if _truth(part) is not False]
+ if not kept:
+ return ast.Constant(value=False)
+ return kept[0] if len(kept) == 1 else ast.Or(exprs=kept)
+ if isinstance(expr, ast.Not) or (isinstance(expr, ast.Call) and expr.name == "not" and len(expr.args) == 1):
+ inner = _fold(expr.expr if isinstance(expr, ast.Not) else expr.args[0])
+ truth = _truth(inner)
+ return ast.Constant(value=not truth) if truth is not None else ast.Not(expr=inner)
+ if isinstance(expr, ast.Call) and expr.name == "if" and len(expr.args) == 3:
+ args = [_fold(arg) for arg in expr.args]
+ truth = _truth(args[0])
+ if truth is not None:
+ return args[1] if truth else args[2]
+ return ast.Call(name="if", args=args)
+ compared = comparison_of(expr)
+ if compared is None:
+ return expr
+ op, left, right = compared.op, _fold(compared.left), _fold(compared.right)
+ values = right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]
+ constants = [value.value for value in values if isinstance(value, ast.Constant)]
+ if (
+ op in _MATCH_PASSES
+ and isinstance(left, ast.Constant)
+ and len(constants) == len(values)
+ and _compares_like_clickhouse([left.value, *constants])
+ and (len(values) == 1 or op in (ast.CompareOperationOp.In, ast.CompareOperationOp.NotIn))
+ ):
+ return ast.Constant(value=(left.value in constants) == _MATCH_PASSES[op])
+ return ast.CompareOperation(op=op, left=left, right=right)
+
+
+def _step_condition(expr: ast.Expr) -> ast.Expr | None:
+ """`c` for a funnel step test such as `if(c, 1, 0) = 1`, or None for any other expression."""
+ compared = comparison_of(expr)
+ if compared is None or compared.op != ast.CompareOperationOp.Eq:
+ return None
+ for step, value in ((compared.left, compared.right), (compared.right, compared.left)):
+ if (
+ isinstance(step, ast.Call)
+ and step.name == "if"
+ and len(step.args) == 3
+ and _truth(step.args[1]) is True
+ and _truth(step.args[2]) is False
+ and _truth(value) is True
+ ):
+ return step.args[0]
+ return None
+
+
+def _union_refusal(node: ast.SelectQuery, scan: EventsScan, position: int) -> str | None:
+ """Why the union cannot stand in for `scan`, the scan at `position` in the FROM chain of `node`, or None."""
+ if position > 0:
+ return "it joins events after another table"
+ refusal = scan.shape_refusal(node)
+ if refusal is not None:
+ return refusal
+ join = scan.join.next_join
+ while join is not None:
+ # A predicate moves into the union's branches only across these joins. GLOBAL changes only where ClickHouse
+ # builds the right side, not which rows the join keeps.
+ if join.join_type is None or join.join_type.removeprefix("GLOBAL ") not in PREDICATE_SAFE_JOIN_TYPES:
+ return f"it joins another table to events with {join.join_type or 'an unknown join type'}"
+ join = join.next_join
+ return None
+
+
+def _flag_call_scans(
+ node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]
+) -> list[tuple[int, EventsScan]]:
+ """Each `events` scan in the FROM chain of `node` whose query names `$feature_flag_called`, with its position."""
+ scans: list[tuple[int, EventsScan]] = []
+ join = node.select_from
+ position = 0
+ while join is not None:
+ # The rename runs before type resolution, so the chain is all it can read. EventsScan.of matches the same
+ # names, and the binding check rejects a rename that changes what a name binds to.
+ # nosemgrep: hogql-no-string-table-chain
+ name = ".".join(str(part) for part in join.table.chain) if isinstance(join.table, ast.Field) else None
+ if name is not None and name in EVENTS_NAMES and name not in ctes:
+ # Only the first scan can read unqualified names, and EventsScan.of decides when it can.
+ scan = (
+ EventsScan.of(node, flag_key_compares_as_string, ctes)
+ if position == 0
+ else EventsScan(
+ join,
+ join.alias or name.replace(".", "__"),
+ other_tables=frozenset(),
+ qualified_only=True,
+ flag_key_compares_as_string=flag_key_compares_as_string,
+ )
+ )
+ if scan is not None and _names_flag_calls(node, scan):
+ scans.append((position, scan))
+ join = join.next_join
+ position += 1
+ return scans
+
+
+def _names_flag_calls(node: ast.SelectQuery, scan: EventsScan) -> bool:
+ finder = _FlagCallComparisonFinder(node, scan)
+ finder.visit(node)
+ return finder.found
+
+
+class _ScanReadCopier(CloningVisitor):
+ """Copies an expression that reads only one union scan, with each read as its chain on the scan."""
+
+ def __init__(self, union: EventsWithFlagCallsTable) -> None:
+ super().__init__(clear_types=True, clear_locations=True)
+ self.union = union
+ self.failed = False
+
+ def visit_field(self, node: ast.Field):
+ read = _union_chain(node.type)
+ if read is None or read[0] is not self.union:
+ self.failed = True
+ return super().visit_field(node)
+ return ast.Field(chain=read[1])
+
+ def visit_alias(self, node: ast.Alias):
+ # A hidden alias wraps an expression field, such as `person_id`, that the resolver expanded in place.
+ if not node.hidden:
+ self.failed = True
+ return self.visit(node.expr)
+
+ def visit_call(self, node: ast.Call):
+ # Each branch would call the function again and filter on a different value.
+ if is_nondeterministic_function(node.name):
+ self.failed = True
+ return super().visit_call(node)
+
+ def _refuse(self, node):
+ self.failed = True
+ return node
+
+ visit_select_query = visit_select_set_query = visit_lambda = _refuse
+
+
+class _EventAsFlagCall(CloningVisitor):
+ def visit_field(self, node: ast.Field):
+ if node.chain == ["event"]:
+ return ast.Constant(value=FLAG_CALLED_EVENT)
+ return super().visit_field(node)
+
+
+class _StepTests(CloningVisitor):
+ """Reads each funnel step test such as `if(c, 1, 0) = 1` as `c`, to find the keys a predicate pins.
+
+ The copy is not a filter: for a NULL `c` the test is false, and `c` is NULL.
+ """
+
+ def visit_compare_operation(self, node: ast.CompareOperation):
+ condition = _step_condition(node)
+ return self.visit(condition) if condition is not None else super().visit_compare_operation(node)
+
+ def visit_call(self, node: ast.Call):
+ condition = _step_condition(node)
+ return self.visit(condition) if condition is not None else super().visit_call(node)
+
+
+class _Qualified(CloningVisitor):
+ def __init__(self, table: str) -> None:
+ super().__init__()
+ self.table = table
+
+ def visit_field(self, node: ast.Field):
+ return ast.Field(chain=[self.table, *node.chain])
+
+
+class _FlagEvaluationsReads(CloningVisitor):
+ def __init__(self, columns: FlagEvaluationsColumns) -> None:
+ super().__init__()
+ self.columns = columns
+ self.failed = False
+
+ def copy(self, expr: ast.Expr) -> ast.Expr | None:
+ copied = self.visit(expr)
+ return None if self.failed else copied
+
+ def visit_field(self, node: ast.Field):
+ chain = self.columns.chain(node.chain)
+ if chain is None:
+ self.failed = True
+ return super().visit_field(node)
+ return ast.Field(chain=["flag_evaluations", *chain])
+
+
+class _UnionChainCollector(SyntaxVisitor):
+ """Collects the chain each resolved field reads on a union scan, as lazy table resolution will request it."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.chains: list[list[str | int]] = []
+ self.unreadable_joins: list[str] = []
+ self.exports: dict[tuple[int, str], list[str | int]] = {}
+ self._select: ast.SelectQuery | None = None
+ self.export_collision: tuple[list[str | int], list[str | int]] | None = None
+ # The event and person properties that a string-parsing call such as toFloatOrZero reads from the scan.
+ self.string_parsed: list[tuple[str, str]] = []
+ self._in_string_input = False
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ outer = self._select
+ self._select = node
+ super().visit_select_query(node)
+ self._select = outer
+
+ def visit_call(self, node: ast.Call) -> None:
+ # The property type swapper keeps a property raw only as the direct argument of such a call.
+ outer = self._in_string_input
+ self._in_string_input = node.name in STRING_INPUT_CONVERSIONS
+ super().visit_call(node)
+ self._in_string_input = outer
+
+ def visit_field(self, node: ast.Field) -> None:
+ read = _union_chain(node.type)
+ if read is not None:
+ chain = read[1]
+ self.chains.append(chain)
+ # Lazy table resolution names a whole-field read after the field, whichever table the field is on.
+ # It names a property read by joining the path with `___`, so `a.b` and `a___b` share a name.
+ name = node.type.name if isinstance(node.type, ast.FieldType) else "___".join(str(link) for link in chain)
+ first = self.exports.setdefault((id(self._select), name), chain)
+ if first != chain and self.export_collision is None:
+ self.export_collision = (first, chain)
+ if self._in_string_input and chain[:-1] in (["properties"], ["poe", "properties"]):
+ self.string_parsed.append(("event" if chain[0] == "properties" else "person", str(chain[-1])))
+ return
+ field_type = node.type.field_type if isinstance(node.type, ast.PropertyType) else node.type
+ if not isinstance(field_type, ast.FieldType):
+ return
+ table_type: ast.Type = field_type.table_type
+ while isinstance(table_type, ast.VirtualTableType):
+ table_type = table_type.table_type
+ # A join that starts from the scan, such as `override` or `person`, reads its key from the union.
+ while isinstance(table_type, ast.LazyJoinType):
+ source = table_type.table_type
+ while isinstance(source, ast.TableAliasType):
+ source = source.table_type
+ if _union_table_of(source) is not None:
+ lazy_join = table_type.lazy_join
+ if isinstance(source, ast.VirtualTableType) or lazy_join.resolver not in _UNION_JOIN_RESOLVERS:
+ self.unreadable_joins.append(table_type.field)
+ else:
+ self.chains.extend([list(lazy_join.from_field), *lazy_join.extra_from_fields])
+ return
+ table_type = table_type.table_type
+
+
+class _UnionTableFinder(SyntaxVisitor):
+ def __init__(self, database: "Database") -> None:
+ super().__init__()
+ self.database = database
+ self.found = False
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ if isinstance(node.table, ast.Field):
+ try:
+ # Unlike get_table, a miss here builds no table name suggestions.
+ table = self.database.get_table_node([str(part) for part in node.table.chain]).table
+ except ResolutionError:
+ # A CTE or a subquery alias, which only the resolver can tell apart.
+ table = None
+ self.found = self.found or isinstance(table, EventsWithFlagCallsTable)
+ super().visit_join_expr(node)
+
+
+class _FlagCallComparisonFinder(SyntaxVisitor):
+ """Finds `event = '$feature_flag_called'` or an `IN` list with it, on the scan, in its own query's clauses."""
+
+ def __init__(self, root: ast.SelectQuery, scan: EventsScan) -> None:
+ super().__init__()
+ self.root = root
+ self.reader = scan
+ self.found = False
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ if node is self.root:
+ super().visit_select_query(node)
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
+ pass
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> None:
+ self._check(node)
+ super().visit_compare_operation(node)
+
+ def visit_call(self, node: ast.Call) -> None:
+ self._check(node)
+ super().visit_call(node)
+
+ def _check(self, expr: ast.Expr) -> None:
+ values = compared_values(expr, lambda read: self.reader.column(read) == ["event"])
+ if values and any(isinstance(value, ast.Constant) and value.value == FLAG_CALLED_EVENT for value in values):
+ self.found = True
+
+
+class _UnionRenamer(ScanVisitor):
+ def __init__(self, flag_key_compares_as_string: Callable[[], bool], only_in_flag_evaluations: bool) -> None:
+ super().__init__()
+ self.flag_key_compares_as_string = flag_key_compares_as_string
+ self.only_in_flag_evaluations = only_in_flag_evaluations
+ self.renamed = False
+ self.refusals: list[str] = []
+
+ def visit_scan_query(self, node: ast.SelectQuery) -> None:
+ # Only the first scan can be renamed, and it comes first, so each refusal reads the query as written.
+ for position, scan in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes):
+ if position == 0 and self._reads_only_flag_calls(node, scan):
+ continue
+ refusal = _union_refusal(node, scan, position)
+ if refusal is not None:
+ self.refusals.append(refusal)
+ else:
+ point_scan_at(node, scan, list(EVENTS_WITH_FLAG_CALLS_CHAIN))
+ # A lazy table cannot be sampled, and SAMPLE 1 keeps every row anyway.
+ scan.join.sample = None
+ self.renamed = True
+
+ def _reads_only_flag_calls(self, node: ast.SelectQuery, scan: EventsScan) -> bool:
+ # The flag_evaluations swap handles a first scan whose `event` filter it reads as only flag calls. It swaps or
+ # refuses that scan on mode 2. On mode 1 events holds every flag call, so the union would read the same calls
+ # twice there.
+ return scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False)
diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.py
index 3c7734af706f..71c54da57083 100644
--- a/posthog/hogql/transforms/lazy_tables.py
+++ b/posthog/hogql/transforms/lazy_tables.py
@@ -5,7 +5,8 @@ from posthog.hogql import ast
from posthog.hogql.base import _T_AST
from posthog.hogql.constants import HogQLDialect
from posthog.hogql.context import HogQLContext
-from posthog.hogql.database.models import LazyJoinToAdd, LazyTableToAdd
+from posthog.hogql.database.models import LazyJoinToAdd, LazyTableToAdd, UnexpandedScan
+from posthog.hogql.database.schema.events_with_flag_calls import EventsWithFlagCallsTable
from posthog.hogql.errors import ResolutionError
from posthog.hogql.resolver import ResolverFactory, resolve_types
from posthog.hogql.resolver_utils import get_long_table_name
@@ -22,6 +23,23 @@ from posthog.hogql.visitor import TraversingVisitor, clone_expr
from posthog.dataclasses import frozen
+def _lazy_table_of(table_type: ast.Type) -> object:
+ while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
+ table_type = table_type.table_type
+ return table_type.table if isinstance(table_type, ast.LazyTableType) else None
+
+
+def _unexpanded_scan(join: ast.JoinExpr, table_type: ast.Type) -> UnexpandedScan:
+ """A copy of `join` as the query wrote it, with the table types that name its scan."""
+ types: list[ast.Type] = [table_type]
+ while isinstance(types[-1], (ast.TableAliasType, ast.ColumnAliasedTableType)):
+ types.append(types[-1].table_type)
+ scan = ast.JoinExpr(
+ table=join.table, alias=join.alias, sample=join.sample, table_args=join.table_args, type=join.type
+ )
+ return UnexpandedScan(scan, tuple(types))
+
+
# This mutates the nodes
def resolve_lazy_tables(
node: _T_AST,
@@ -52,6 +70,13 @@ class FieldChainReplacer(TraversingVisitor):
if node.chain == constraint.chain_to_replace:
node.chain = [constraint.table_name, constraint.alias]
+ # A subquery has its own scope. Its `events` is the table it reads, not the outer scan with the override column.
+ def visit_select_query(self, node: ast.SelectQuery):
+ pass
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery):
+ pass
+
class FieldFinder(TraversingVisitor):
field_chains: list[list[str | int]] = []
@@ -309,13 +334,18 @@ class LazyTableResolver(TraversingVisitor):
def _wrap_table_for_lazy_resolution(
self, node: ast.SelectQuery, table_alias: str, select_type: ast.SelectQueryType
- ) -> None:
+ ) -> UnexpandedScan | None:
# The scope key of a schema-qualified table (`posthog__events`) is not its written name, so match the binding.
original_table_type = select_type.tables[table_alias]
join_ptr = node.select_from
while join_ptr is not None and join_ptr.type is not original_table_type:
join_ptr = join_ptr.next_join
assert join_ptr is not None and isinstance(join_ptr.table, ast.Field), f"Table {table_alias} not found"
+ scan = (
+ _unexpanded_scan(join_ptr, original_table_type)
+ if isinstance(_lazy_table_of(original_table_type), EventsWithFlagCallsTable)
+ else None
+ )
# Collect while the outer reads still identify the table they belong to.
collector = WrappedTableReferenceCollector(original_table_type, self.context)
@@ -363,6 +393,8 @@ class LazyTableResolver(TraversingVisitor):
join_ptr.sample = None
join_ptr.type = ast.SelectQueryAliasType(alias=table_alias, select_query_type=subquery.type)
select_type.tables[table_alias] = join_ptr.type
+ if scan is not None:
+ scan = UnexpandedScan(scan.join, (*scan.types, join_ptr.type))
for property_type, export_name, through_lazy_join in collector.properties.values():
if id(property_type.field_type) in collector.fields:
@@ -380,6 +412,7 @@ class LazyTableResolver(TraversingVisitor):
for field_type, export_name in collector.fields.values():
field_type.name = export_name
field_type.table_type = join_ptr.type
+ return scan
def _read_property_path(
self, subquery_type: ast.SelectQueryType, column: ast.Alias, property_path: tuple[str | int, ...]
@@ -693,8 +726,11 @@ class LazyTableResolver(TraversingVisitor):
# This must happen BEFORE we add lazy joins to avoid forward reference issues.
tables_to_wrap = self._find_tables_needing_lazy_preresolution(node, select_type)
wrapped_table_types = [select_type.tables[table_alias] for table_alias in tables_to_wrap]
+ unexpanded_scans: dict[str, UnexpandedScan] = {}
for table_alias in tables_to_wrap:
- self._wrap_table_for_lazy_resolution(node, table_alias, select_type)
+ wrapped_scan = self._wrap_table_for_lazy_resolution(node, table_alias, select_type)
+ if wrapped_scan is not None:
+ unexpanded_scans[table_alias] = wrapped_scan
# Remove lazy joins that are now resolved inside wrapped subqueries
joins_to_add = {
@@ -734,6 +770,8 @@ class LazyTableResolver(TraversingVisitor):
and join_ptr.table.type == old_table_type.table_type
)
):
+ if isinstance(table_to_add.lazy_table, EventsWithFlagCallsTable):
+ unexpanded_scans[table_name] = _unexpanded_scan(join_ptr, old_table_type)
join_ptr.table = subquery
join_ptr.type = select_type.tables[table_name]
join_ptr.alias = table_name
@@ -752,6 +790,7 @@ class LazyTableResolver(TraversingVisitor):
# For all the collected joins, create the join subqueries, and add them to the table.
for to_table, join_scope in joins_to_add.items():
+ join_scope.from_scan = unexpanded_scans.get(join_scope.from_table)
join_to_add: ast.JoinExpr = join_scope.lazy_join.resolve_join_to_add(
join_scope,
self.context,
diff --git a/posthog/hogql/transforms/property_types.py b/posthog/hogql/transforms/property_types.py
index 73b255f405c8..447fb1be1e85 100644
--- a/posthog/hogql/transforms/property_types.py
+++ b/posthog/hogql/transforms/property_types.py
@@ -6,13 +6,9 @@ from posthog.hogql.constants import EXCEPTION_STRING_ARRAY_PROPERTIES, is_virtua
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import BooleanDatabaseField, DateTimeDatabaseField, StringJSONDatabaseField
from posthog.hogql.database.s3_table import S3Table
-from posthog.hogql.database.schema.events import (
- EVENTS_TABLE_TYPES,
- EventsGroupSubTable,
- EventsPersonSubTable,
- EventsTable,
-)
-from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable, FlagEvaluationsTable
+from posthog.hogql.database.schema.events import EVENTS_TABLE_TYPES, EventsGroupSubTable, EventsPersonSubTable
+from posthog.hogql.database.schema.events_with_flag_calls import EVENT_ROW_TABLES
+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsPersonSubTable
from posthog.hogql.database.schema.groups import GroupsTable
from posthog.hogql.database.schema.persons import PersonsTable, RawPersonsTable
from posthog.hogql.escape_sql import escape_hogql_identifier
@@ -118,7 +114,7 @@ class PropertyFinder(TraversingVisitor):
self.person_properties.add(property_name)
elif isinstance(resolved_table, EventsGroupSubTable):
pass # group properties are handled above via GroupsTable
- elif isinstance(resolved_table, EventsTable | FlagEvaluationsTable):
+ elif isinstance(resolved_table, EVENT_ROW_TABLES):
self.event_properties.add(property_name)
def visit_field(self, node: ast.Field):
@@ -137,6 +133,14 @@ class ToTimeZoneParts:
swapped: bool
+# ClickHouse string-parsing conversions (toFloat64OrZero, toInt64OrZero,
+# toFloat64OrDefault, toInt64OrDefault) require a String first argument and raise
+# ILLEGAL_TYPE_OF_ARGUMENT on numeric input. When a user explicitly wraps a
+# Numeric-typed property in one of these, we must not auto-convert the property
+# to Float, the raw String value has to flow through for the parser to work.
+STRING_INPUT_CONVERSIONS = frozenset({"toFloatOrZero", "toIntOrZero", "toFloatOrDefault", "toIntOrDefault"})
+
+
class PropertySwapper(CloningVisitor):
_RANGE_OPS: set[str] = {
ast.CompareOperationOp.Gt,
@@ -145,13 +149,6 @@ class PropertySwapper(CloningVisitor):
ast.CompareOperationOp.LtEq,
}
- # ClickHouse string-parsing conversions (toFloat64OrZero, toInt64OrZero,
- # toFloat64OrDefault, toInt64OrDefault) require a String first argument and raise
- # ILLEGAL_TYPE_OF_ARGUMENT on numeric input. When a user explicitly wraps a
- # Numeric-typed property in one of these, we must not auto-convert the property
- # to Float, the raw String value has to flow through for the parser to work.
- _STRING_INPUT_CONVERSIONS: set[str] = {"toFloatOrZero", "toIntOrZero", "toFloatOrDefault", "toIntOrDefault"}
-
# ClickHouse array-membership functions whose first argument must be an array. Users write these
# against exception properties (e.g. hasAny(properties.$exception_values, [...])), but those
# properties are stored as a raw JSON String once materialized, so the bare column read raises
@@ -248,7 +245,7 @@ class PropertySwapper(CloningVisitor):
# string. Re-evaluated per call, so nested non-parsing calls (e.g.
# toFloatOrZero(toString(prop))) correctly reset the flag.
saved_suppress = self._suppress_numeric_conversion
- self._suppress_numeric_conversion = node.name in self._STRING_INPUT_CONVERSIONS
+ self._suppress_numeric_conversion = node.name in STRING_INPUT_CONVERSIONS
self._inside_call_depth += 1
try:
@@ -287,7 +284,7 @@ class PropertySwapper(CloningVisitor):
table_type = type.field_type.table_type
if not isinstance(table_type, ast.BaseTableType):
return False
- return isinstance(table_type.resolve_database_table(self.context), EventsTable | FlagEvaluationsTable)
+ return isinstance(table_type.resolve_database_table(self.context), EVENT_ROW_TABLES)
@staticmethod
def _extract_string_array(field: ast.Expr) -> ast.Call:
@@ -738,7 +735,7 @@ class PropertySwapper(CloningVisitor):
if isinstance(resolved_table, EventsPersonSubTable | FlagEvaluationsPersonSubTable):
if property_name in self.person_properties:
return ("person", property_name)
- elif isinstance(resolved_table, EventsTable | FlagEvaluationsTable):
+ elif isinstance(resolved_table, EVENT_ROW_TABLES):
if property_name in self.event_properties:
return ("event", property_name)
if type.field_type.name == "person_properties" and len(type.chain) == 1:
diff --git a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
index abf7589fabe0..9f10ae474b25 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
@@ -9,9 +9,9 @@ from django.test import SimpleTestCase
from parameterized import parameterized
from posthog.hogql import ast
+from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYS
from posthog.hogql.parser import parse_select
from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (
- FLAG_EVALUATIONS_COMPLETE_DAYS,
EventsScan,
can_read_flag_evaluations,
filter_conjuncts,
diff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.py
new file mode 100644
index 000000000000..66963d46b273
--- /dev/null
+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py
@@ -0,0 +1,918 @@
+import re
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import cast
+
+import pytest
+from posthog.test.base import (
+ APIBaseTest,
+ BaseTest,
+ ClickhouseTestMixin,
+ _create_event,
+ _create_flag_evaluations,
+ _create_person,
+ cleanup_materialized_columns,
+ flush_persons_and_events,
+)
+from unittest.mock import patch
+
+from django.test import SimpleTestCase
+
+from parameterized import param, parameterized
+
+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.database.models import DatabaseField, FieldTraverser, LazyJoin, StringJSONDatabaseField, VirtualTable
+from posthog.hogql.database.schema.util.where_clause_extractor import is_nondeterministic_function
+from posthog.hogql.errors import QueryError
+from posthog.hogql.functions.mapping import HOGQL_CLICKHOUSE_FUNCTIONS
+from posthog.hogql.parser import parse_select
+from posthog.hogql.printer import prepare_and_print_ast
+from posthog.hogql.query import HogQLQueryExecutor, execute_hogql_query
+from posthog.hogql.transforms.flag_calls_union import rename_flag_call_scans_to_union
+from posthog.hogql.visitor import TraversingVisitor, clone_expr
+
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.escape import substitute_params
+from posthog.constants import AvailableFeature
+from posthog.test.persons import create_group_type_mapping
+
+from products.access_control.backend.models.property_access_control import PropertyAccessControl
+from products.access_control.backend.property_access_control import PropertyAccessLevel
+from products.data_tools.backend.models.join import DataWarehouseJoin
+from products.event_definitions.backend.models.property_definition import PropertyDefinition
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
+
+from ee.clickhouse.materialized_columns.columns import materialize
+
+MIXED = "event IN ('$pageview', '$feature_flag_called')"
+SESSION_ID = "0190a8d1-3f7a-7b00-8000-000000000001"
+E_MIXED = "e.event IN ('$pageview', '$feature_flag_called')"
+UNION = "posthog.events_with_flag_calls"
+READ = FlagEvaluationsMode.READ_FLAG_EVALUATIONS
+ONLY = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY
+
+
+class _TableCollector(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.tables: list[str] = []
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ if isinstance(node.table, ast.Field):
+ self.tables.append(".".join(str(part) for part in node.table.chain))
+ super().visit_join_expr(node)
+
+
+def _select(query: str | ast.SelectQuery) -> ast.SelectQuery:
+ return cast(ast.SelectQuery, parse_select(query) if isinstance(query, str) else clone_expr(query))
+
+
+def _global_join(query: str) -> ast.SelectQuery:
+ # HogQL has no GLOBAL syntax. The recordings list builds a GLOBAL join in its AST.
+ select = _select(query)
+ assert select.select_from is not None and select.select_from.next_join is not None
+ select.select_from.next_join.join_type = "GLOBAL INNER JOIN"
+ return select
+
+
+def _set_mode(organization, mode: FlagEvaluationsMode) -> None:
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=organization, defaults={"flag_evaluations_mode": mode}
+ )
+
+
+def _enabled_flags(union: bool):
+ enabled = {"hogql-flag-calls-union"} if union else set()
+ return patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key in enabled,
+ )
+
+
+def _restrict(organization, team, name: str) -> None:
+ organization.available_product_features = [
+ {"name": AvailableFeature.PROPERTY_ACCESS_CONTROL, "key": AvailableFeature.PROPERTY_ACCESS_CONTROL}
+ ]
+ organization.save()
+ definition = PropertyDefinition.objects.create(
+ team=team, name=name, property_type="String", type=PropertyDefinition.Type.EVENT
+ )
+ PropertyAccessControl.objects.create(
+ team=team, property_definition=definition, access_level=PropertyAccessLevel.NONE.value
+ )
+
+
+class TestRenameFlagCallScansToUnion(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("event_in_a_list_with_other_events", f"SELECT count() FROM events WHERE {MIXED}", [UNION], []),
+ ("paths_v2_step_sources", f"SELECT e.event FROM events AS e WHERE {E_MIXED}", [UNION], []),
+ (
+ "funnel_step_column",
+ "SELECT if(event = '$feature_flag_called' AND properties.$feature_flag = 'f', 1, 0) AS step_1 "
+ "FROM events",
+ [UNION],
+ [],
+ ),
+ (
+ "or_of_event_comparisons",
+ "SELECT count() FROM events WHERE event = '$pageview' OR equals(event, '$feature_flag_called')",
+ [UNION],
+ [],
+ ),
+ # The flag_evaluations swap handles this scan. Events holds every flag call on mode 1. The swap swaps or
+ # refuses the scan on mode 2.
+ ("only_flag_calls", "SELECT count() FROM events WHERE event = '$feature_flag_called'", ["events"], []),
+ (
+ "only_flag_calls_with_sample",
+ "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'",
+ ["events"],
+ [],
+ ),
+ (
+ "left_join_after_the_scan",
+ f"SELECT count() FROM events AS e LEFT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",
+ [UNION, "persons"],
+ [],
+ ),
+ (
+ "left_join_with_an_unqualified_event",
+ f"SELECT count() FROM events LEFT JOIN persons AS p ON events.person_id = p.id WHERE {MIXED}",
+ [UNION, "persons"],
+ [],
+ ),
+ (
+ "left_join_with_an_unqualified_event_aliased_to_itself",
+ "SELECT event AS event, count() FROM events LEFT JOIN persons AS p ON events.person_id = p.id "
+ f"WHERE {MIXED} GROUP BY event",
+ [UNION, "persons"],
+ [],
+ ),
+ (
+ "only_flag_calls_with_a_join_and_an_unqualified_event",
+ "SELECT count() FROM events LEFT JOIN persons AS p ON events.person_id = p.id "
+ "WHERE event = '$feature_flag_called'",
+ ["events", "persons"],
+ [],
+ ),
+ (
+ "array_join_with_an_unqualified_event",
+ f"SELECT count() FROM events ARRAY JOIN [1, 2] AS x WHERE {MIXED}",
+ [UNION],
+ [],
+ ),
+ (
+ "inner_join_after_the_scan",
+ f"SELECT count() FROM events AS e INNER JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",
+ [UNION, "persons"],
+ [],
+ ),
+ (
+ "global_join_after_the_scan",
+ _global_join(
+ f"SELECT count() FROM events AS e INNER JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}"
+ ),
+ [UNION, "persons"],
+ [],
+ ),
+ ("qualified_events_name", f"SELECT count() FROM posthog.events WHERE {MIXED}", [UNION], []),
+ ("sample_one", f"SELECT count() FROM events SAMPLE 1 WHERE {MIXED}", [UNION], []),
+ ("scan_in_a_subquery", f"SELECT count() FROM (SELECT event FROM events WHERE {MIXED})", [UNION], []),
+ (
+ "comparison_only_in_a_subquery",
+ f"SELECT count() FROM events WHERE distinct_id IN (SELECT distinct_id FROM events WHERE {MIXED})",
+ ["events", UNION],
+ [],
+ ),
+ (
+ "no_flag_call_comparison",
+ "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$pageview'",
+ ["events"],
+ [],
+ ),
+ ("not_equals", "SELECT count() FROM events WHERE event != '$feature_flag_called'", ["events"], []),
+ (
+ "constant_on_another_column",
+ "SELECT count() FROM events WHERE distinct_id = '$feature_flag_called'",
+ ["events"],
+ [],
+ ),
+ (
+ "scan_after_another_table",
+ f"SELECT count() FROM persons AS p JOIN events AS e ON p.id = e.person_id WHERE {E_MIXED}",
+ ["persons", "events"],
+ ["it joins events after another table"],
+ ),
+ (
+ "scan_of_only_flag_calls_after_another_table",
+ "SELECT count() FROM persons AS p JOIN events AS e ON p.id = e.person_id "
+ "WHERE e.event = '$feature_flag_called'",
+ ["persons", "events"],
+ ["it joins events after another table"],
+ ),
+ (
+ "cte_named_events_after_another_table",
+ "WITH events AS (SELECT 1 AS person_id, '$pageview' AS event) "
+ f"SELECT count() FROM persons AS p JOIN events AS e ON p.id = e.person_id WHERE {E_MIXED}",
+ ["persons", "events"],
+ [],
+ ),
+ (
+ "right_join_after_the_scan",
+ f"SELECT count() FROM events AS e RIGHT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",
+ ["events", "persons"],
+ ["it joins another table to events with RIGHT JOIN"],
+ ),
+ (
+ "final",
+ f"SELECT count() FROM events FINAL WHERE {MIXED}",
+ ["events"],
+ ["it uses FINAL or table arguments on events"],
+ ),
+ (
+ "column_aliases",
+ f"SELECT count() FROM events AS e (uuid, event, properties, timestamp, distinct_id, x) WHERE {E_MIXED}",
+ ["events"],
+ ["it names the events columns with column aliases"],
+ ),
+ ("sample_ratio", f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}", ["events"], ["it uses SAMPLE"]),
+ ("select_star", f"SELECT * FROM events WHERE {MIXED}", ["events"], ["it selects * or COLUMNS(...)"]),
+ (
+ "cte_named_events",
+ f"WITH events AS (SELECT '$pageview' AS event) SELECT count() FROM events WHERE {MIXED}",
+ ["events"],
+ [],
+ ),
+ ]
+ )
+ def test_renames_the_scans_the_union_reads_and_names_why_it_cannot_read_the_rest(
+ self,
+ _name: str,
+ query: str | ast.SelectQuery,
+ expected_tables: list[str],
+ expected_refusals: list[str],
+ only_in_flag_evaluations: bool = True,
+ ):
+ select = _select(query)
+ rename = rename_flag_call_scans_to_union(
+ select, flag_key_compares_as_string=lambda: True, only_in_flag_evaluations=only_in_flag_evaluations
+ )
+ collector = _TableCollector()
+ collector.visit(select)
+
+ assert collector.tables == expected_tables
+ assert list(rename.refusals) == expected_refusals
+
+ def test_row_dependent_functions_cover_every_hogql_function_in_their_families(self):
+ families = re.compile(
+ r"^(rand|random|rowNumber|blockNumber|nowInBlock|generateUUID|generateULID|runningDifference|neighbor)"
+ )
+
+ assert [
+ name
+ for name in HOGQL_CLICKHOUSE_FUNCTIONS
+ if families.match(name) and not is_nondeterministic_function(name)
+ ] == []
+
+
+class TestReadFlagCallsWithUnion(BaseTest):
+ def _sql(
+ self,
+ query: str | ast.SelectQuery,
+ *,
+ mode: FlagEvaluationsMode = READ,
+ union: bool = True,
+ persons_on_events: PersonsOnEventsMode | None = None,
+ new_events_schema: bool | None = None,
+ person_id_pushdown: bool | None = None,
+ ) -> str:
+ _set_mode(self.organization, mode)
+ with _enabled_flags(union=union):
+ sql, context = HogQLQueryExecutor(
+ query=query,
+ team=self.team,
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=persons_on_events,
+ personIdPushdown=person_id_pushdown,
+ useNewEventsSchema=_events_schema_for(mode, new_events_schema),
+ ),
+ pretty=False,
+ ).generate_clickhouse_sql()
+ return substitute_params(sql, context.values)
+
+ @staticmethod
+ def _branches(sql: str) -> list[str]:
+ start = sql.index("UNION ALL")
+ union_start = sql.rindex("(SELECT", 0, sql.rindex("FROM events", 0, start))
+ union_end = sql.index(") AS ", sql.rindex("FROM flag_evaluations"))
+ return sql[union_start:union_end].split("UNION ALL")
+
+ @parameterized.expand(
+ [
+ ("mixed_query", f"SELECT count() FROM events WHERE {MIXED}", True, True),
+ ("mixed_query_with_the_flag_off", f"SELECT count() FROM events WHERE {MIXED}", False, False),
+ (
+ "query_phase_1_swaps",
+ "SELECT count() FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'f' AND timestamp >= now() - INTERVAL 7 DAY",
+ True,
+ False,
+ ),
+ # The CTE shadows events, so the query reads the CTE on every mode.
+ (
+ "cte_named_events",
+ f"WITH events AS (SELECT '$pageview' AS event) SELECT count() FROM events WHERE {MIXED}",
+ True,
+ False,
+ ),
+ # Each branch reads the raw string of a property without a type, as events does.
+ (
+ "string_parser_on_an_untyped_property",
+ f"SELECT toFloatOrZero(properties.probe) FROM events WHERE {MIXED}",
+ True,
+ True,
+ ),
+ param(
+ "cte_named_events_on_mode_2",
+ f"WITH events AS (SELECT '$pageview' AS event) SELECT count() FROM events WHERE {MIXED}",
+ True,
+ False,
+ mode=ONLY,
+ ),
+ # Each scan exports one created_at, so the two fields never share a column.
+ param(
+ "fields_of_one_name_in_separate_scans",
+ f"SELECT created_at FROM events WHERE {MIXED} UNION ALL "
+ f"SELECT person.created_at FROM events WHERE {MIXED}",
+ True,
+ True,
+ mode=ONLY,
+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
+ ),
+ ]
+ )
+ def test_union_reads_the_scans_phase_1_leaves_on_events(
+ self,
+ _name: str,
+ query: str,
+ union: bool,
+ expected_union: bool,
+ mode: FlagEvaluationsMode = READ,
+ persons_on_events: PersonsOnEventsMode | None = None,
+ ):
+ sql = self._sql(query, union=union, mode=mode, persons_on_events=persons_on_events)
+
+ assert ("UNION ALL" in sql) is expected_union
+ if expected_union:
+ assert re.search(r"FROM flag_evaluations\b", sql) is not None
+
+ @parameterized.expand(
+ [
+ (
+ "right_join",
+ f"SELECT count() FROM events AS e RIGHT JOIN persons AS p ON e.person_id = p.id WHERE {E_MIXED}",
+ "RIGHT JOIN",
+ ),
+ (
+ "field_flag_evaluations_lacks",
+ f"SELECT goe_0.properties.plan FROM events WHERE {MIXED}",
+ "goe_0.properties",
+ ),
+ param(
+ "join_whose_on_clause_reads_another_field",
+ f"SELECT probe_join.key FROM events WHERE {MIXED}",
+ "probe_join",
+ warehouse_join=True,
+ ),
+ (
+ "field_and_virtual_field_of_one_name",
+ f"SELECT created_at, person.created_at FROM events WHERE {MIXED}",
+ "created_at and poe.created_at",
+ ),
+ (
+ "properties_whose_paths_join_to_one_name",
+ f"SELECT properties.a.b, properties.a___b FROM events WHERE {MIXED}",
+ "properties.a.b and properties.a___b",
+ ),
+ (
+ "properties_and_virtual_properties",
+ f"SELECT properties, person.properties FROM events WHERE {MIXED}",
+ "properties and poe.properties",
+ ),
+ param(
+ "string_parser_on_a_typed_event_property",
+ f"SELECT toFloatOrZero(properties.probe) FROM events WHERE {MIXED}",
+ "Numeric event property probe",
+ typed=PropertyDefinition.Type.EVENT,
+ ),
+ param(
+ "string_parser_on_a_typed_person_property",
+ f"SELECT toIntOrZero(person.properties.probe) FROM events WHERE {MIXED}",
+ "Numeric person property probe",
+ typed=PropertyDefinition.Type.PERSON,
+ ),
+ ]
+ )
+ def test_mode_1_reads_events_and_mode_2_raises_when_the_union_cannot_read_a_mixed_scan(
+ self,
+ _name: str,
+ query: str,
+ reason: str,
+ warehouse_join: bool = False,
+ typed: PropertyDefinition.Type | None = None,
+ ):
+ if typed is not None:
+ PropertyDefinition.objects.create(team=self.team, name="probe", property_type="Numeric", type=typed)
+ if warehouse_join:
+ # Joined on persons, it reaches events as a join that compares `person.properties.email`, not its key.
+ DataWarehouseJoin.objects.create(
+ team=self.team,
+ source_table_name="persons",
+ source_table_key="properties.email",
+ joining_table_name="groups",
+ joining_table_key="key",
+ field_name="probe_join",
+ )
+ persons_on_events = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+
+ assert "UNION ALL" not in self._sql(query, persons_on_events=persons_on_events)
+ with pytest.raises(QueryError, match=reason):
+ self._sql(query, mode=ONLY, persons_on_events=persons_on_events)
+
+ @parameterized.expand([("dotted", UNION), ("quoted", f"`{UNION}`")])
+ def test_query_cannot_name_the_union_table(self, _name: str, table: str):
+ with pytest.raises(QueryError, match="Unknown table `posthog.events_with_flag_calls`"):
+ self._sql(f"SELECT count() FROM {table}")
+
+ def test_in_subquery_over_the_union_runs_once_with_global_in(self):
+ sql = self._sql(
+ f"SELECT count() FROM events WHERE distinct_id IN (SELECT distinct_id FROM events WHERE {MIXED})"
+ )
+
+ assert "globalIn(events.distinct_id" in sql
+
+ @parameterized.expand(
+ [
+ ("restricted_on_mode_1", READ, None, False, False),
+ # Mode 2 reads every flag call from flag_evaluations, so it never asks events for the flag key.
+ ("restricted_on_mode_2", ONLY, None, False, True),
+ ("boolean_on_native_json_on_mode_1", READ, "Boolean", True, False),
+ ("numeric_on_native_json_on_mode_1", READ, "Numeric", True, False),
+ ("boolean_on_native_json_on_mode_2", ONLY, "Boolean", True, True),
+ ]
+ )
+ def test_flag_key_that_events_cannot_read_raw_keeps_mode_1_on_events(
+ self, _name: str, mode: FlagEvaluationsMode, flag_key_type: str | None, new_events_schema: bool, expected: bool
+ ):
+ if flag_key_type is None:
+ _restrict(self.organization, self.team, "$feature_flag")
+ else:
+ PropertyDefinition.objects.create(
+ team=self.team, name="$feature_flag", property_type=flag_key_type, type=PropertyDefinition.Type.EVENT
+ )
+
+ sql = self._sql(f"SELECT count() FROM events WHERE {MIXED}", mode=mode, new_events_schema=new_events_schema)
+
+ assert ("UNION ALL" in sql) is expected
+
+ @parameterized.expand(
+ [
+ (
+ "one_key",
+ "if(event = '$feature_flag_called' AND properties.$feature_flag = 'probe-flag', 1, 0) AS step_1",
+ "step_1 = 1",
+ "tuple('probe-flag')",
+ ),
+ (
+ "a_key_per_step_with_other_conditions",
+ "if(event = '$feature_flag_called' AND properties.$feature_flag = 'a' AND properties.x = '1', 1, 0) "
+ "AS step_1, "
+ "if(event = '$feature_flag_called' AND properties.$feature_flag = 'b' AND properties.x = '2', 1, 0) "
+ "AS step_2",
+ "step_1 = 1 OR step_2 = 1",
+ "tuple('a', 'b')",
+ ),
+ ]
+ )
+ def test_funnel_steps_on_flag_keys_read_only_those_keys_and_skip_keyless_calls(
+ self, _name: str, flag_steps: str, flag_step_filter: str, keys: str
+ ):
+ # The shape a funnel runner builds: one column per step, and a filter that keeps a row matching any step.
+ sql = self._sql(
+ "SELECT * FROM ("
+ f"SELECT if(event = '$pageview', 1, 0) AS step_0, {flag_steps} "
+ "FROM events WHERE timestamp >= toDateTime('2026-10-01') AND timestamp <= toDateTime('2026-10-20') "
+ f"AND event IN ('$feature_flag_called', '$pageview') AND (step_0 = 1 OR {flag_step_filter}))"
+ )
+
+ events, flag_evaluations = self._branches(sql)
+ for branch in (events, flag_evaluations):
+ assert "2026-10-01" in branch and "2026-10-20" in branch
+ assert f"in(flag_evaluations.flag_key, {keys})" in flag_evaluations
+ assert "in(events.event, tuple('$feature_flag_called', '$pageview'))" in events
+
+ @parameterized.expand(
+ [
+ ("no_flag_key_filter", "", None),
+ # A Boolean type converts the key before the comparison, so 'TRUE' also matches the JSON value true.
+ ("boolean_flag_key", " AND properties.$feature_flag = 'TRUE'", "Boolean"),
+ ]
+ )
+ def test_query_without_a_string_flag_key_reads_keyless_calls_from_events(
+ self, _name: str, flag_key_filter: str, flag_key_type: str | None
+ ):
+ if flag_key_type is not None:
+ PropertyDefinition.objects.create(
+ team=self.team, name="$feature_flag", property_type=flag_key_type, type=PropertyDefinition.Type.EVENT
+ )
+
+ events, keyless, flag_evaluations = self._branches(
+ self._sql(f"SELECT count() FROM events WHERE {MIXED} AND distinct_id = 'probe'{flag_key_filter}")
+ )
+
+ for branch in (events, keyless):
+ assert "equals(events.distinct_id, 'probe')" in branch
+ assert "equals(flag_evaluations.distinct_id, 'probe')" in flag_evaluations
+ assert "flag_key" not in flag_evaluations
+
+ @parameterized.expand(
+ [
+ ("array_join", "arrayJoin(['a', 'b']) = 'a'", "arrayJoin"),
+ ("random", "rand() % 10 = 0", "rand"),
+ ("row_number", "rowNumberInBlock() < 5", "rowNumberInBlock"),
+ ("random_in_upper_case", "RAND() % 10 = 0", "rand"),
+ ("now_in_block", "timestamp < nowInBlock('UTC')", "nowInBlock"),
+ ]
+ )
+ def test_filter_that_depends_on_each_read_stays_out_of_the_branches(self, _name: str, condition: str, name: str):
+ sql = self._sql(f"SELECT count() FROM events WHERE {MIXED} AND {condition}")
+
+ assert all(name not in branch for branch in self._branches(sql))
+ assert name in sql
+
+ def test_sample_one_adds_no_warning(self):
+ _set_mode(self.organization, READ)
+ with _enabled_flags(union=True):
+ _, context = HogQLQueryExecutor(
+ query=f"SELECT count() FROM events SAMPLE 1 WHERE {MIXED}", team=self.team
+ ).generate_clickhouse_sql()
+
+ assert context.warnings == []
+
+ def test_sessions_join_keeps_its_time_bound(self):
+ sql = self._sql(
+ f"SELECT session.$session_duration FROM events WHERE {MIXED} "
+ "AND timestamp >= toDateTime('2026-10-01') AND timestamp < toDateTime('2026-10-08')"
+ )
+
+ sessions = sql[sql.index("FROM raw_sessions") : sql.index("AS events__session")]
+ assert "toDateTime('2026-10-01', 'UTC')" in sessions and "toDateTime('2026-10-08', 'UTC')" in sessions
+
+ def test_sessions_join_keeps_a_literal_session_id_filter(self):
+ sql = self._sql(
+ f"SELECT session.$session_duration FROM events WHERE {MIXED} AND $session_id IN ('{SESSION_ID}')"
+ )
+
+ sessions = sql[sql.index("FROM raw_sessions") : sql.index("AS events__session")]
+ assert (
+ f"in(raw_sessions.session_id_v7, tuple(toUInt128(accurateCastOrNull('{SESSION_ID}', 'UUID'))))" in sessions
+ )
+
+ def test_persons_join_keeps_the_person_id_pushdown(self):
+ sql = self._sql(
+ f"SELECT person.properties.email FROM events WHERE {MIXED} AND distinct_id = 'probe-distinct-id'",
+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ person_id_pushdown=True,
+ )
+
+ persons = sql[sql.index("FROM person_distinct_id_overrides") :]
+ assert "GLOBAL IN" in sql or "globalIn" in sql
+ assert "probe-distinct-id" in persons
+
+ def test_persons_join_runs_one_override_join(self):
+ sql = self._sql(
+ f"SELECT person_id, person.properties.email FROM events WHERE {MIXED}",
+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ )
+
+ assert sql.count("FROM person_distinct_id_overrides") == 2 # one inner, one outer for `person_id`
+
+
+class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
+ def setUp(self):
+ super().setUp()
+ now = datetime.now(UTC)
+ self.recent = _create_person(distinct_ids=["recent"], team_id=self.team.pk, properties={"email": "[email protected]"})
+ self.old = _create_person(distinct_ids=["old"], team_id=self.team.pk, properties={"email": "[email protected]"})
+ for distinct_id, person, days_ago in (("recent", self.recent, 3), ("old", self.old, 100)):
+ _create_event(
+ event="$pageview",
+ distinct_id=distinct_id,
+ person_id=person.uuid,
+ person_properties={"email": person.properties["email"]},
+ team=self.team,
+ timestamp=now - timedelta(days=days_ago),
+ )
+ # Ingestion writes a keyed flag call to both tables on mode 1. Only flag_evaluations has the stale person,
+ # which an override corrects.
+ flag_call_uuid = uuid.uuid4()
+ _create_event(
+ event_uuid=flag_call_uuid,
+ event="$feature_flag_called",
+ distinct_id="recent",
+ person_id=self.recent.uuid,
+ person_properties={"email": "[email protected]"},
+ team=self.team,
+ timestamp=now - timedelta(days=2),
+ properties={"$feature_flag": "probe-flag", "source": "events"},
+ )
+ _create_flag_evaluations(
+ self.team.pk,
+ "probe-flag",
+ timestamp=now - timedelta(days=2),
+ distinct_id="recent",
+ person_id=uuid.uuid4(),
+ person_properties={"email": "[email protected]"},
+ properties={"source": "flag_evaluations"},
+ )
+ sync_execute(
+ "INSERT INTO person_distinct_id_overrides (team_id, distinct_id, person_id, version) VALUES",
+ [(self.team.pk, "recent", str(self.recent.uuid), 1)],
+ )
+ _create_event(
+ event="$feature_flag_called",
+ distinct_id="old",
+ person_id=self.old.uuid,
+ team=self.team,
+ timestamp=now - timedelta(days=99),
+ properties={"$feature_flag": "probe-flag", "source": "events"},
+ )
+ for distinct_id, flag_key in (("missing", None), ("empty", ""), ("number", 100)):
+ _create_event(
+ event="$feature_flag_called",
+ distinct_id=distinct_id,
+ team=self.team,
+ timestamp=now - timedelta(days=1),
+ properties={"source": "events", **({} if flag_key is None else {"$feature_flag": flag_key})},
+ )
+ flush_persons_and_events()
+
+ def _run(
+ self,
+ query: str | ast.SelectQuery,
+ mode: FlagEvaluationsMode,
+ persons_on_events: PersonsOnEventsMode | None = None,
+ ):
+ _set_mode(self.organization, mode)
+ with _enabled_flags(union=True):
+ return execute_hogql_query(
+ query,
+ team=self.team,
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=persons_on_events, useNewEventsSchema=_events_schema_for(mode, None)
+ ),
+ )
+
+ @parameterized.expand(
+ [
+ (f"{mode.name.lower()}_{poe.name.lower()}", mode, poe)
+ for mode in (FlagEvaluationsMode.READ_FLAG_EVALUATIONS, FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY)
+ for poe in (
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ )
+ ]
+ )
+ def test_each_flag_call_comes_from_one_table(self, _name: str, mode: FlagEvaluationsMode, poe: PersonsOnEventsMode):
+ response = self._run(
+ f"""
+ SELECT distinct_id, event, properties.source, person_id, person.properties.email
+ FROM events
+ WHERE {MIXED} AND timestamp >= now() - INTERVAL 120 DAY
+ ORDER BY timestamp, distinct_id
+ """,
+ mode,
+ poe,
+ )
+
+ recent_id, old_id = str(self.recent.uuid), str(self.old.uuid)
+ history = [
+ ("old", "$pageview", None, old_id, "[email protected]"),
+ ("old", "$feature_flag_called", "events", old_id, "[email protected]"),
+ ]
+ recent = [
+ ("recent", "$pageview", None, recent_id, "[email protected]"),
+ ("recent", "$feature_flag_called", "flag_evaluations", recent_id, "[email protected]"),
+ ]
+ keyless = [(distinct_id, "$feature_flag_called", "events") for distinct_id in ("empty", "missing", "number")]
+ rows = [
+ tuple(str(value) if isinstance(value, uuid.UUID) else value for value in row) for row in response.results
+ ]
+ if mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY:
+ assert rows == [history[0], *recent]
+ else:
+ assert rows[:4] == [*history, *recent]
+ assert [row[:3] for row in rows[4:]] == keyless
+
+ def test_prewhere_reads_both_tables(self):
+ response = self._run(
+ f"SELECT count() FROM events PREWHERE {MIXED} WHERE timestamp >= now() - INTERVAL 120 DAY",
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ )
+
+ assert response.results == [(7,)]
+
+ @parameterized.expand([("json",), ("materialized",)])
+ def test_keyless_flag_calls_come_from_events(self, read: str):
+ # A materialized $feature_flag column reads an empty string as missing.
+ if read == "materialized":
+ self.addCleanup(cleanup_materialized_columns)
+ materialize("events", "$feature_flag")
+ response = self._run(
+ f"SELECT distinct_id FROM events WHERE {MIXED} AND event = '$feature_flag_called' "
+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY distinct_id",
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ )
+
+ assert response.results == [("empty",), ("missing",), ("number",), ("recent",)]
+
+ @parameterized.expand(
+ [
+ (
+ "pin_of_one_flag_key",
+ "SELECT distinct_id, event, properties.source FROM events WHERE event = '$pageview' "
+ "OR (event = '$feature_flag_called' AND properties.$feature_flag = 'probe-flag') ORDER BY timestamp",
+ [
+ ("old", "$pageview", None),
+ ("old", "$feature_flag_called", "events"),
+ ("recent", "$pageview", None),
+ ("recent", "$feature_flag_called", "flag_evaluations"),
+ ],
+ ),
+ # The OR reads a person property, so no branch can copy it.
+ (
+ "or_with_a_person_property",
+ f"SELECT distinct_id, event, properties.source FROM events WHERE {MIXED} "
+ "AND (event = '$pageview' OR person.properties.email = '[email protected]') "
+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY timestamp",
+ [("recent", "$pageview", None), ("recent", "$feature_flag_called", "flag_evaluations")],
+ ),
+ (
+ "mixed_type_in_that_clickhouse_converts",
+ f"SELECT distinct_id, event, properties.source FROM events WHERE {MIXED} AND 1 IN ('1', '2') "
+ "AND timestamp >= now() - INTERVAL 5 DAY ORDER BY timestamp, distinct_id",
+ [
+ ("recent", "$pageview", None),
+ ("recent", "$feature_flag_called", "flag_evaluations"),
+ ("empty", "$feature_flag_called", "events"),
+ ("missing", "$feature_flag_called", "events"),
+ ("number", "$feature_flag_called", "events"),
+ ],
+ ),
+ (
+ "global_join",
+ _global_join(
+ "SELECT e.distinct_id, e.event, e.properties.source FROM events AS e "
+ "INNER JOIN (SELECT 'recent' AS distinct_id) AS r ON e.distinct_id = r.distinct_id "
+ f"WHERE {E_MIXED} AND e.timestamp >= now() - INTERVAL 5 DAY ORDER BY e.timestamp"
+ ),
+ [("recent", "$pageview", None), ("recent", "$feature_flag_called", "flag_evaluations")],
+ ),
+ ]
+ )
+ def test_branch_filters_keep_every_row_the_query_keeps(
+ self, _name: str, query: str | ast.SelectQuery, expected: list[tuple]
+ ):
+ response = self._run(
+ query,
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ )
+
+ assert response.results == expected
+
+ def test_exception_values_read_as_an_array(self):
+ query = f"SELECT count() FROM events WHERE {MIXED} AND has(properties.$exception_values, 'probe')"
+
+ assert self._run(query, FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results == [(0,)]
+
+ def test_restricted_property_is_hidden_in_every_branch(self):
+ _restrict(self.organization, self.team, "source")
+ query = f"SELECT properties.source FROM events WHERE {MIXED}"
+
+ assert {row[0] for row in self._run(query, FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results} == {None}
+ assert (
+ self._run(f"{query} AND properties.source = 'events'", FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results
+ == []
+ )
+
+ @parameterized.expand(
+ [
+ (f"{poe.name.lower()}_{'events_json' if new_schema else 'events'}", poe, new_schema)
+ for poe in (
+ PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ )
+ for new_schema in (False, True)
+ ]
+ )
+ def test_every_events_field_reads_the_same_type_or_keeps_events(
+ self, _name: str, poe: PersonsOnEventsMode, new_schema: bool
+ ):
+ # Mode 1 keeps reading events on the native JSON table, so only mode 2 reads the union there.
+ _set_mode(self.organization, ONLY if new_schema else READ)
+ create_group_type_mapping(team=self.team, group_type="organization", group_type_index=0)
+ # A typed property reads as a number, which the union keeps only when it types each branch.
+ for property_type in (PropertyDefinition.Type.EVENT, PropertyDefinition.Type.PERSON):
+ PropertyDefinition.objects.create(team=self.team, name="probe", property_type="Numeric", type=property_type)
+ modifiers = HogQLQueryModifiers(personsOnEventsMode=poe, useNewEventsSchema=new_schema)
+ with _enabled_flags(union=True):
+ union_database = Database.create_for(team=self.team, modifiers=modifiers)
+ with _enabled_flags(union=False):
+ events_database = Database.create_for(team=self.team, modifiers=modifiers)
+
+ def sql(query: str, database: Database) -> str:
+ context = HogQLContext(
+ team_id=self.team.pk, team=self.team, database=database, modifiers=modifiers, enable_select_queries=True
+ )
+ return prepare_and_print_ast(parse_select(query), context, "clickhouse")[0]
+
+ accepted = []
+ for chain in _readable_chains(union_database):
+ name = ".".join(f"`{part}`" for part in chain)
+ try:
+ sql(f"SELECT {name} FROM events", events_database)
+ except QueryError:
+ continue
+ # A field the union cannot read keeps the query on events on mode 1 and fails it on mode 2.
+ try:
+ union_sql = sql(f"SELECT {name} FROM events WHERE {MIXED}", union_database)
+ except QueryError:
+ continue
+ if "UNION ALL" in union_sql:
+ accepted.append(name)
+
+ assert {
+ "`timestamp`",
+ "`properties`.`probe`",
+ "`person_id`",
+ "`person`.`properties`.`probe`",
+ "`$session_id`",
+ "`elements_chain_texts`",
+ "`$group_0`",
+ # The sessions join reads `$session_id_uuid`, which is not its key.
+ "`session`.`id`",
+ } <= set(accepted)
+ types: dict[bool, dict[str, str]] = {False: {}, True: {}}
+ # One query per few fields keeps each union within the memory of a local ClickHouse.
+ # Two fields of one name, such as goe_0.key and goe_1.key, would return one column, so they go in different queries.
+ batches: list[list[str]] = []
+ for name in accepted:
+ batch = next(
+ (b for b in batches if len(b) < 8 and all(n.rsplit(".", 1)[-1] != name.rsplit(".", 1)[-1] for n in b)),
+ None,
+ )
+ if batch is None:
+ batches.append(batch := [])
+ batch.append(name)
+ for names in batches:
+ types_query = f"SELECT {', '.join(f'toTypeName(any({name}))' for name in names)} FROM events WHERE {MIXED}"
+ for union in (False, True):
+ with _enabled_flags(union=union):
+ response = execute_hogql_query(types_query, team=self.team, modifiers=modifiers)
+ types[union].update(zip(names, response.results[0]))
+
+ # flag_evaluations has no elements, and its empty Array(String) widens the union of the events Enum8 array.
+ # Comparisons with string literals behave the same on both types.
+ widened = {"`elements_chain_elements`": "Array(String)"}
+ assert types[True] == {**types[False], **widened}
+
+
+def _events_schema_for(mode: FlagEvaluationsMode, new_events_schema: bool | None) -> bool | None:
+ # Mode 1 keeps reading events on the native JSON table, so a mode 1 case reads the legacy table unless it asks.
+ if new_events_schema is None and mode == READ:
+ return False
+ return new_events_schema
+
+
+def _readable_chains(database: Database) -> list[list[str | int]]:
+ """A chain to select for each events field, and for one field behind each of its joins."""
+ events = database.get_table("events")
+ chains: list[list[str | int]] = []
+ for name, field in events.fields.items():
+ if isinstance(field, (LazyJoin, FieldTraverser)):
+ chains.extend([[name, "id"], [name, "properties", "probe"], [name, "person_id"]])
+ continue
+ inner_fields = field.fields.items() if isinstance(field, VirtualTable) else [(None, field)]
+ for inner_name, inner in inner_fields:
+ chain: list[str | int] = [name] if inner_name is None else [name, inner_name]
+ if isinstance(inner, StringJSONDatabaseField):
+ chains.append([*chain, "probe"])
+ elif isinstance(inner, DatabaseField):
+ chains.append(chain)
+ return chains
diff --git a/posthog/models/test/test_data_deletion_request.py b/posthog/models/test/test_data_deletion_request.py
index 9541685ef21a..f035490dfa0f 100644
--- a/posthog/models/test/test_data_deletion_request.py
+++ b/posthog/models/test/test_data_deletion_request.py
@@ -213,9 +213,12 @@ def test_hogql_predicate_subquery_is_team_scoped(db):
[
"SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
"AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY",
+ "SELECT distinct_id FROM events WHERE event IN ('$pageview', '$feature_flag_called') "
+ "AND timestamp >= now() - INTERVAL 7 DAY",
"SELECT e.distinct_id FROM events AS e JOIN (SELECT 'probe' AS id) AS d "
"ON e.distinct_id = d.id WHERE event = '$feature_flag_called'",
],
+ ids=["only_flag_calls", "flag_calls_with_other_events", "flag_calls_behind_a_join"],
)
@pytest.mark.parametrize("through_a_view", [False, True])
def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, flag_calls, through_a_view):
@@ -236,7 +239,11 @@ def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, flag
predicate = f"distinct_id IN ({flag_calls})"
request = DataDeletionRequest(**_base_kwargs(team_id=team.id, events=["$pageview"], hogql_predicate=predicate))
- sql, _ = compile_hogql_predicate(request)
+ with patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-union",
+ ):
+ sql, _ = compile_hogql_predicate(request)
assert "flag_evaluations" not in sql
diff --git a/products/cohorts/backend/backfill/flag_calls_table.py b/products/cohorts/backend/backfill/flag_calls_table.py
index 3cc2fca2fb62..ed3333366315 100644
--- a/products/cohorts/backend/backfill/flag_calls_table.py
+++ b/products/cohorts/backend/backfill/flag_calls_table.py
@@ -4,7 +4,7 @@ from datetime import UTC, datetime
from enum import StrEnum
from typing import Any
-from posthog.hogql.transforms.flag_calls_from_flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYS
+from posthog.hogql.database.schema.flag_evaluations import FLAG_EVALUATIONS_COMPLETE_DAYS
from posthog.exceptions_capture import capture_exception
from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_SOURCE_EVENT, FLAG_EVALUATIONS_TABLE