:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): fix flag_evaluations array reads and lazy session joins (#125070)
Phil Haackc49e6e6459bf
diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.pyindex 29f93cc1650e..51802cbfe1e6 100644--- a/posthog/hogql/database/database.py+++ b/posthog/hogql/database/database.py@@ -176,6 +176,7 @@ from posthog.schema_enums import (DatabaseSerializedFieldType,DataWarehouseSavedQueryOrigin,PersonsOnEventsMode,+ SessionsV2JoinMode,SessionTableVersion,)from posthog.scopes import APIScopeObject@@ -2463,6 +2464,9 @@ class Database(BaseModel):events_table = database.get_table("events")events_table.fields["session"] = LazyJoin(from_field=["$session_id"],+ extra_from_fields=[["$session_id_uuid"]]+ if modifiers.sessionsV2JoinMode == SessionsV2JoinMode.UUID+ else [],join_table=sessions,resolver=EVENTS_TO_SESSIONS_V2,)@@ -2489,6 +2493,7 @@ class Database(BaseModel):events_table = database.get_table("events")events_table.fields["session"] = LazyJoin(from_field=["$session_id"],+ extra_from_fields=[["$session_id_uuid"]],join_table=sessions,resolver=EVENTS_TO_SESSIONS_V3,)diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.pyindex 408e65494d22..c9969c639608 100644--- a/posthog/hogql/database/models.py+++ b/posthog/hogql/database/models.py@@ -497,6 +497,9 @@ class LazyJoin(FieldOrTable):join_table: Table | strfrom_field: list[str | int]to_field: Optional[list[str | int]] = None+ # The other fields of the source table that the resolver's ON clause reads. A lazy source table returns only the+ # fields a query asks for, so lazy table resolution asks for these too.+ extra_from_fields: list[list[str | int]] = PydanticField(default_factory=list)def resolve_table(self, context: "HogQLContext") -> Table:if isinstance(self.join_table, Table):diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.pyindex 937f82deeb6b..805be073f770 100644--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py@@ -982,6 +982,13 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):assert response.results == [(1,)]+ def test_swapped_scan_reads_exception_values_as_an_array(self):+ results = self._query_events(+ f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS} AND NOT has(properties.$exception_values, 'x')"+ ).results++ assert results == [(1,)]+@parameterized.expand([# The events-only call has the JSON value true as its key. A typed comparison matches it on events.diff --git a/posthog/hogql/test/__snapshots__/test_resolver.ambr b/posthog/hogql/test/__snapshots__/test_resolver.ambrindex 685e5ab15697..8826c47033e2 100644--- a/posthog/hogql/test/__snapshots__/test_resolver.ambr+++ b/posthog/hogql/test/__snapshots__/test_resolver.ambr@@ -382,6 +382,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -459,6 +460,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -536,6 +538,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -613,6 +616,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -690,6 +694,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -767,6 +772,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -844,6 +850,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -921,6 +928,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -1744,6 +1752,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -1831,6 +1840,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -2736,6 +2746,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -2826,6 +2837,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -3163,6 +3175,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -3259,6 +3272,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -3355,6 +3369,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -3451,6 +3466,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -3547,6 +3563,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -3643,6 +3660,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -3739,6 +3757,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -3835,6 +3854,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -5660,6 +5680,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -5737,6 +5758,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -5814,6 +5836,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -5891,6 +5914,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -5968,6 +5992,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -6045,6 +6070,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -6122,6 +6148,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -6199,6 +6226,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -6695,6 +6723,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -6772,6 +6801,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -6849,6 +6879,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -6926,6 +6957,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -7003,6 +7035,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -7080,6 +7113,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -7157,6 +7191,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -7234,6 +7269,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -8060,6 +8096,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -8147,6 +8184,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -8708,6 +8746,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -8795,6 +8834,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -9931,6 +9971,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -10021,6 +10062,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -10358,6 +10400,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -10454,6 +10497,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -10550,6 +10594,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -10646,6 +10691,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -10742,6 +10788,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -10838,6 +10885,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -10934,6 +10982,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -11030,6 +11079,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -11618,6 +11668,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -11708,6 +11759,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -12045,6 +12097,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -12141,6 +12194,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -12237,6 +12291,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -12333,6 +12388,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -12429,6 +12485,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -12525,6 +12582,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -12621,6 +12679,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -12717,6 +12776,7 @@field: "fingerprint_issue_state"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["fingerprint"],@@ -15547,6 +15607,7 @@field: "person"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["person_id"],@@ -15560,6 +15621,7 @@field: "pdi"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -15910,6 +15972,7 @@field: "person"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["person_id"],@@ -15923,6 +15986,7 @@field: "pdi"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -16269,6 +16333,7 @@field: "pdi"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -16617,6 +16682,7 @@field: "pdi"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -16754,6 +16820,7 @@field: "person"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["person_id"],@@ -17708,6 +17775,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],@@ -17798,6 +17866,7 @@field: "override"lazy_join: {description: None,+ extra_from_fields: [],from_field: ["distinct_id"],diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.pyindex b0b71984e84d..fe069ec493ed 100644--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py@@ -129,9 +129,9 @@ def can_read_flag_evaluations(`resolve_flag_calls_from_flag_evaluations` checks the rest after resolution: every name in the swapped querybinds as it did before."""- scan = _EventsScan.of(node, lambda: flag_key_compares_as_string)+ scan = EventsScan.of(node, lambda: flag_key_compares_as_string)return scan is not None and _qualifies(- scan, node, _filter_conjuncts(node), now, timezone, only_in_flag_evaluations=only_in_flag_evaluations+ scan, node, filter_conjuncts(node), now, timezone, only_in_flag_evaluations=only_in_flag_evaluations)@@ -184,9 +184,7 @@ def _resolve_with_swaps(database: "Database",only_in_flag_evaluations: bool,) -> _T_AST:- finder = _FlagCalledConstantFinder()- finder.visit(node)- if not finder.found:+ if not names_flag_called_event(node):return resolve(node)if not only_in_flag_evaluations and context.uses_new_events_schema():return resolve(node)@@ -246,7 +244,7 @@ def _flag_calls_only_in_flag_evaluations_error(reason: str) -> QueryError:def _qualifies(- scan: "_EventsScan",+ scan: "EventsScan",node: ast.SelectQuery,conjuncts: list[ast.Expr],now: datetime,@@ -261,7 +259,7 @@ def _qualifies(return scan.pins_flag_calls(conjuncts, needs_flag_key=True) and scan.bounds_timestamp(conjuncts, now, timezone)-class _EventsScan:+class EventsScan:def __init__(self,join: ast.JoinExpr,@@ -276,7 +274,7 @@ class _EventsScan:self.shadowed: frozenset[str] = frozenset()@classmethod- def of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool]) -> "_EventsScan | None":+ def of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool]) -> "EventsScan | None":join = node.select_fromif join is None or not isinstance(join.table, ast.Field) or join.table.chain != ["events"]:return None@@ -333,10 +331,8 @@ class _EventsScan:def _pins_flag_calls_in(self, expr: ast.Expr, needs_flag_key: bool) -> bool:if _pinned_values(expr, self._reads_event, _is_flag_called) is not None:return not needs_flag_key- if isinstance(expr, ast.Or):- return bool(expr.exprs) and all(- self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in expr.exprs- )+ if branches := boolean_operands(expr, ast.Or):+ return all(self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in branches)return Falsedef _flag_keys_in(self, expr: ast.Expr) -> list[str] | None:@@ -344,10 +340,14 @@ class _EventsScan:values = _pinned_values(expr, self._reads_flag_key, _is_flag_key)if values is not None:return [value.value for value in values if isinstance(value, ast.Constant)]- if isinstance(expr, ast.Or) and expr.exprs:- branches = [self._flag_keys_in(branch) for branch in expr.exprs]- if all(keys is not None for keys in branches):- return [key for keys in branches if keys is not None for key in keys]+ if branches := boolean_operands(expr, ast.Or):+ branch_keys = [self._flag_keys_in(branch) for branch in branches]+ if all(keys is not None for keys in branch_keys):+ return [key for keys in branch_keys if keys is not None for key in keys]+ for conjunct in boolean_operands(expr, ast.And) or []:+ keys = self._flag_keys_in(conjunct)+ if keys is not None:+ return keysreturn Nonedef flag_keys(self, conjuncts: list[ast.Expr]) -> list[str]:@@ -386,7 +386,7 @@ class _EventsScan:def _lower_bound(self, expr: ast.Expr) -> ast.Expr | None:if isinstance(expr, ast.BetweenExpr):return expr.low if not expr.negated and self._reads_timestamp(expr.expr) else None- comparison = _comparison(expr)+ comparison = comparison_of(expr)if comparison is None:return Noneif comparison.op in (ast.CompareOperationOp.GtEq, ast.CompareOperationOp.Gt) and self._reads_timestamp(@@ -403,7 +403,7 @@ class _EventsScan:return self.column(expr) == ["timestamp"]-def _comparison(expr: ast.Expr) -> ast.CompareOperation | None:+def comparison_of(expr: ast.Expr) -> ast.CompareOperation | None:"""`expr` as a comparison, whether it is written `a >= b` or `greaterOrEquals(a, b)`."""if isinstance(expr, ast.CompareOperation):return expr@@ -412,24 +412,64 @@ def _comparison(expr: ast.Expr) -> ast.CompareOperation | None:return None-def _pinned_values(expr: ast.Expr, reads: _Predicate, accepts: _Predicate) -> list[ast.Expr] | None:- """The values `expr` limits what `reads` matches to, or None when they are not all values `accepts` matches."""- comparison = _comparison(expr)+def compared_values(expr: ast.Expr, reads: _Predicate) -> list[ast.Expr] | None:+ """The values that `expr` compares a read matching `reads` with by `=` or `IN`, or None."""+ comparison = comparison_of(expr)if comparison is None:return Noneif comparison.op == ast.CompareOperationOp.Eq:- if reads(comparison.left) and accepts(comparison.right):+ if reads(comparison.left):return [comparison.right]- if reads(comparison.right) and accepts(comparison.left):- return [comparison.left]- return None- if comparison.op in (ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn):+ return [comparison.left] if reads(comparison.right) else None+ if comparison.op in (ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn) and reads(comparison.left):right = comparison.right- values = right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]- return values if reads(comparison.left) and values and all(accepts(value) for value in values) else None+ return right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]return None+def _pinned_values(expr: ast.Expr, reads: _Predicate, accepts: _Predicate) -> list[ast.Expr] | None:+ """The values `expr` limits what `reads` matches to, or None when they are not all values `accepts` matches."""+ values = compared_values(expr, reads)+ return values if values and all(accepts(value) for value in values) else None+++def flag_key_predicate(table: str, keys: list[str]) -> ast.Expr | None:+ """`flag_key IN keys` on `table`, or None when a pin of `$feature_flag` to `keys` does not imply it.++ flag_key holds the key's JSON text, and a JSONExtractString pin compares the decoded key. The pin implies this+ predicate only when the JSON text of each key is the key itself.+ """+ if not keys or not all(is_json_verbatim(key) for key in keys):+ return None+ return ast.CompareOperation(+ op=ast.CompareOperationOp.In,+ left=ast.Field(chain=[table, "flag_key"]),+ right=ast.Tuple(exprs=[ast.Constant(value=key) for key in keys]),+ )+++def point_scan_at(+ node: ast.SelectQuery, scan: EventsScan, chain: list[str | int], condition: ast.Expr | None = None+) -> None:+ """Points `scan` at the table `chain` names, and moves the PREWHERE of `node`, with `condition`, into its WHERE.++ The new table can read person columns through a join that the printer adds later, and ClickHouse rejects a+ PREWHERE that reads a joined column. WHERE selects the same rows.+ """+ # The old alias keeps qualified reads such as `events.timestamp` resolving.+ scan.join.alias = scan.alias+ scan.join.table = ast.Field(chain=chain)+ node.where = and_conditions([node.prewhere, node.where, condition])+ node.prewhere = None+++def and_conditions(conditions: list[ast.Expr | None]) -> ast.Expr | None:+ present = [condition for condition in conditions if condition is not None]+ if not present:+ return None+ return present[0] if len(present) == 1 else ast.And(exprs=present)++def _is_constant(expr: ast.Expr, value: str) -> bool:return isinstance(expr, ast.Constant) and expr.value == value@@ -453,7 +493,8 @@ def _conjuncts(expr: ast.Expr | None) -> list[ast.Expr]:return top_level_conjuncts(expr) if expr is not None else []-def _filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:+def filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:+ """The top-level conjuncts of the WHERE and PREWHERE of `node`."""return _conjuncts(node.where) + _conjuncts(node.prewhere)@@ -461,6 +502,22 @@ def _utc_midnight(now: datetime) -> datetime:return start_of_day(now.astimezone(UTC))+def names_flag_called_event(node: ast.AST) -> bool:+ """Whether `$feature_flag_called` appears as a constant anywhere in `node`."""+ finder = _FlagCalledConstantFinder()+ finder.visit(node)+ return finder.found+++def boolean_operands(expr: ast.Expr, operator: type[ast.And] | type[ast.Or]) -> list[ast.Expr] | None:+ """The operands of an AND or an OR, in its node or its call form, or None for any other expression."""+ if isinstance(expr, operator):+ return expr.exprs+ if isinstance(expr, ast.Call) and expr.name == ("and" if operator is ast.And else "or"):+ return expr.args+ return None++def _earliest_instant(expr: ast.Expr, now: datetime, timezone: ZoneInfo) -> datetime | None:"""An instant no later than the value of `expr`, or None when `expr` has a shape this cannot evaluate."""if isinstance(expr, ast.Constant):@@ -620,14 +677,14 @@ def _constant_instant(value: object, zones: tuple[tzinfo, ...]) -> datetime | Noreturn min(value.replace(tzinfo=zone) for zone in zones)-class _SyntaxVisitor(TraversingVisitor):+class SyntaxVisitor(TraversingVisitor):def visit_field(self, node: ast.Field) -> None:# The inherited visit follows the field's type into shared CTE types, which takes exponential time. An# earlier step can resolve parts of the tree in place.pass-class _AliasCollector(_SyntaxVisitor):+class _AliasCollector(SyntaxVisitor):def __init__(self, root: ast.SelectQuery) -> None:super().__init__()self.root = root@@ -646,7 +703,7 @@ class _AliasCollector(_SyntaxVisitor):super().visit_alias(node)-class _FlagCalledConstantFinder(_SyntaxVisitor):+class _FlagCalledConstantFinder(SyntaxVisitor):def __init__(self) -> None:super().__init__()self.found = False@@ -656,25 +713,13 @@ class _FlagCalledConstantFinder(_SyntaxVisitor):self.found = True-class _EventsScanSwapper(_SyntaxVisitor):- def __init__(- self,- now: datetime,- timezone: ZoneInfo,- only_in_flag_evaluations: bool,- flag_key_compares_as_string: Callable[[], bool],- ) -> None:+class ScanVisitor(SyntaxVisitor):+ """Calls `visit_scan_query` for each SELECT whose `FROM events` can read the events table, not a CTE."""++ def __init__(self) -> None:super().__init__()- self.now = now- self.timezone = timezone- self.only_in_flag_evaluations = only_in_flag_evaluations- self.flag_key_compares_as_string = flag_key_compares_as_string- self.swapped = False- self.flag_key_predicates = 0- # Why each scan that reads only flag calls has a shape that flag_evaluations cannot stand in for.- self.refusals: list[str] = []# The CTE names in scope, under the resolver's rules. A `FROM events` reads a CTE when one is named `events`.- # A name missing here lets the swapper try the scan, and the binding check then rejects the swap.+ # A name missing here lets a subclass change the scan, and the binding check then rejects the change.self.ctes: frozenset[str] = frozenset()def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:@@ -697,42 +742,53 @@ class _EventsScanSwapper(_SyntaxVisitor):for cte in (node.ctes or {}).values():self.visit(cte.expr)self.ctes = self.ctes.union([cte.name])- scan = None if "events" in self.ctes else _EventsScan.of(node, self.flag_key_compares_as_string)- if scan is not None:- conjuncts = _filter_conjuncts(node)- if _qualifies(- scan, node, conjuncts, self.now, self.timezone, only_in_flag_evaluations=self.only_in_flag_evaluations- ):- self._swap(node, scan, conjuncts)- else:- refusal = scan.shape_refusal(node)- if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):- self.refusals.append(refusal)+ if "events" not in self.ctes:+ self.visit_scan_query(node)super().visit_select_query(node)self.ctes = outer- def _swap(self, node: ast.SelectQuery, scan: _EventsScan, conjuncts: list[ast.Expr]) -> None:- keys = scan.flag_keys(conjuncts)- # The old alias keeps qualified reads such as `events.timestamp` resolving.- scan.join.alias = scan.alias- scan.join.table = ast.Field(chain=["posthog", "flag_evaluations"])- # A swapped PREWHERE can read person columns through a join that the printer adds later. ClickHouse- # rejects a PREWHERE that reads a joined column, and WHERE selects the same rows.- conditions = [condition for condition in (node.prewhere, node.where) if condition is not None]- # flag_key sorts the table, so this predicate lets ClickHouse skip other flags. flag_key holds the key's JSON- # text, and a JSONExtractString pin compares the decoded key. The pin implies this predicate only when the- # JSON text of each key is the key itself.- if keys and all(is_json_verbatim(key) for key in keys):- conditions.append(- ast.CompareOperation(- op=ast.CompareOperationOp.In,- left=ast.Field(chain=[scan.alias, "flag_key"]),- right=ast.Tuple(exprs=[ast.Constant(value=key) for key in keys]),- )- )+ def visit_scan_query(self, node: ast.SelectQuery) -> None:+ raise NotImplementedError+++class _EventsScanSwapper(ScanVisitor):+ def __init__(+ self,+ now: datetime,+ timezone: ZoneInfo,+ only_in_flag_evaluations: bool,+ flag_key_compares_as_string: Callable[[], bool],+ ) -> None:+ super().__init__()+ self.now = now+ self.timezone = timezone+ self.only_in_flag_evaluations = only_in_flag_evaluations+ self.flag_key_compares_as_string = flag_key_compares_as_string+ self.swapped = False+ self.flag_key_predicates = 0+ # Why each scan that reads only flag calls has a shape that flag_evaluations cannot stand in for.+ self.refusals: list[str] = []++ def visit_scan_query(self, node: ast.SelectQuery) -> None:+ scan = EventsScan.of(node, self.flag_key_compares_as_string)+ if scan is None:+ return+ conjuncts = filter_conjuncts(node)+ if _qualifies(+ scan, node, conjuncts, self.now, self.timezone, only_in_flag_evaluations=self.only_in_flag_evaluations+ ):+ self._swap(node, scan, conjuncts)+ else:+ refusal = scan.shape_refusal(node)+ if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):+ self.refusals.append(refusal)++ def _swap(self, node: ast.SelectQuery, scan: EventsScan, conjuncts: list[ast.Expr]) -> None:+ # flag_key sorts the table, so this predicate lets ClickHouse skip other flags.+ flag_key = flag_key_predicate(scan.alias, scan.flag_keys(conjuncts))+ if flag_key is not None:self.flag_key_predicates += 1- node.prewhere = None- node.where = conditions[0] if len(conditions) == 1 else ast.And(exprs=conditions)+ point_scan_at(node, scan, ["posthog", "flag_evaluations"], flag_key)self.swapped = True@@ -764,7 +820,7 @@ def _bindings(node: ast.AST) -> list[_Binding]:return collector.bindings-class _BindingCollector(_SyntaxVisitor):+class _BindingCollector(SyntaxVisitor):def __init__(self) -> None:super().__init__()self.bindings: list[_Binding] = []@@ -907,7 +963,7 @@ def _property_sources(reads: list[_PropertyRead], context: HogQLContext) -> listreturn sources-class _PropertyReadCollector(_SyntaxVisitor):+class _PropertyReadCollector(SyntaxVisitor):def __init__(self, context: HogQLContext) -> None:super().__init__()self.context = context@@ -932,7 +988,7 @@ def _view_reading_flag_calls_from_events(node: ast.AST) -> str | None:return finder.found-class _FlagCallViewFinder(_SyntaxVisitor):+class _FlagCallViewFinder(SyntaxVisitor):def __init__(self) -> None:super().__init__()self.view: str | None = None@@ -949,11 +1005,11 @@ class _FlagCallViewFinder(_SyntaxVisitor):def _reads_only_flag_calls_from_events(node: ast.SelectQuery) -> bool:- scan = _EventsScan.of(node, lambda: True)+ scan = EventsScan.of(node, lambda: True)return (scan is not Noneand _reads_events_table(scan.join)- and scan.pins_flag_calls(_filter_conjuncts(node), needs_flag_key=False)+ and scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False))diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.pyindex e04b2405a596..d3bad3766aee 100644--- a/posthog/hogql/transforms/lazy_tables.py+++ b/posthog/hogql/transforms/lazy_tables.py@@ -614,6 +614,8 @@ class LazyTableResolver(TraversingVisitor):for new_join in joins_to_add.values():if new_join.from_table in joins_to_add or new_join.from_table in tables_to_add:create_override(new_join.from_table, new_join.lazy_join.from_field)+ for extra_from_field in new_join.lazy_join.extra_from_fields:+ create_override(new_join.from_table, extra_from_field)if new_join.lazy_join.to_field is not None and (new_join.to_table in joins_to_add or new_join.to_table in tables_to_add):diff --git a/posthog/hogql/transforms/property_types.py b/posthog/hogql/transforms/property_types.pyindex 50cd2a732e05..9c3d992ebc28 100644--- a/posthog/hogql/transforms/property_types.py+++ b/posthog/hogql/transforms/property_types.py@@ -286,7 +286,7 @@ class PropertySwapper(CloningVisitor):table_type = type.field_type.table_typeif not isinstance(table_type, ast.BaseTableType):return False- return isinstance(table_type.resolve_database_table(self.context), EventsTable)+ return isinstance(table_type.resolve_database_table(self.context), EventsTable | FlagEvaluationsTable)@staticmethoddef _extract_string_array(field: ast.Expr) -> ast.Call:diff --git a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.pyindex 9c0bb3c73321..8b1fa5c9d957 100644--- a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py+++ b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py@@ -12,7 +12,9 @@ from posthog.hogql import astfrom posthog.hogql.parser import parse_selectfrom posthog.hogql.transforms.flag_calls_from_flag_evaluations import (FLAG_EVALUATIONS_COMPLETE_DAYS,+ EventsScan,can_read_flag_evaluations,+ filter_conjuncts,)from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_TTL_DAYS@@ -24,6 +26,12 @@ IN_COVERAGE = COVERAGE_START + timedelta(days=30)FLAG_CALLED = "event = '$feature_flag_called'"FLAG_KEY = "properties.$feature_flag = 'my-flag'"BOUND = "timestamp >= now() - INTERVAL 7 DAY"+OR_OF_AND_BRANCHES = (+ "((properties.$feature_flag = 'a' AND distinct_id = 'x') OR (properties.$feature_flag = 'b' AND distinct_id = 'y'))"+)+OR_CALL_OF_AND_CALLS = (+ "or(and(properties.$feature_flag = 'a', distinct_id = 'x'), and(properties.$feature_flag = 'b', distinct_id = 'y'))"+)def _text(instant: datetime) -> str:@@ -79,6 +87,19 @@ class TestCanReadFlagEvaluations(SimpleTestCase):select, datetime.now(UTC), ZoneInfo(timezone), only_in_flag_evaluations=only_in_flag_evaluations)+ @parameterized.expand(+ [+ ("or_of_and_branches", _where(FLAG_CALLED, OR_OF_AND_BRANCHES)),+ ("or_call_of_and_calls", _where(FLAG_CALLED, OR_CALL_OF_AND_CALLS)),+ ]+ )+ def test_flag_keys_cover_every_branch_of_an_or(self, _name: str, query: str):+ select = parse_select(query)+ assert isinstance(select, ast.SelectQuery)+ scan = EventsScan.of(select, lambda: True)+ assert scan is not None+ assert scan.flag_keys(filter_conjuncts(select)) == ["a", "b"]+def test_complete_days_leave_out_the_day_the_ttl_may_have_dropped(self):assert FLAG_EVALUATIONS_COMPLETE_DAYS < FLAG_EVALUATIONS_TTL_DAYS@@ -95,6 +116,19 @@ class TestCanReadFlagEvaluations(SimpleTestCase):BOUND,),),+ ("or_of_flag_keys_with_other_conditions", _where(FLAG_CALLED, OR_OF_AND_BRANCHES, BOUND)),+ (+ "or_call_of_flag_keys",+ _where(FLAG_CALLED, "or(properties.$feature_flag = 'a', properties.$feature_flag = 'b')", BOUND),+ ),+ (+ "or_call_of_pinned_branches_with_their_own_flag_keys",+ _where(+ "or(and(event = '$feature_flag_called', properties.$feature_flag = 'a'), "+ "and(event = '$feature_flag_called', properties.$feature_flag = 'b'))",+ BOUND,+ ),+ ),("pin_in_prewhere", f"SELECT count() FROM events PREWHERE {FLAG_CALLED} WHERE {FLAG_KEY} AND {BOUND}"),("conjuncts_in_nested_ands", _where(FLAG_CALLED, f"({FLAG_KEY} AND ({BOUND} AND distinct_id = 'd'))")),(diff --git a/posthog/hogql/transforms/test/test_lazy_tables.py b/posthog/hogql/transforms/test/test_lazy_tables.pyindex 3d2d274e3c92..164e08256d26 100644--- a/posthog/hogql/transforms/test/test_lazy_tables.py+++ b/posthog/hogql/transforms/test/test_lazy_tables.py@@ -18,7 +18,7 @@ from django.test import SimpleTestCase, override_settingsfrom parameterized import parameterized-from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode, SessionsV2JoinMode, SessionTableVersionfrom posthog.hogql import astfrom posthog.hogql.context import HogQLContext@@ -573,6 +573,24 @@ class TestLazyJoins(BaseTest):'SAMPLE clause ignored: the "sessions" table is computed on the fly and cannot be sampled']+ @parameterized.expand(+ [+ ("v3", HogQLQueryModifiers(sessionTableVersion=SessionTableVersion.V3)),+ (+ "v2_uuid_join",+ HogQLQueryModifiers(+ sessionTableVersion=SessionTableVersion.V2, sessionsV2JoinMode=SessionsV2JoinMode.UUID+ ),+ ),+ ]+ )+ def test_join_from_a_lazy_join_reads_the_extra_source_field_it_declares(+ self, _name: str, modifiers: HogQLQueryModifiers+ ):+ printed = self._print_select("SELECT events.session.$session_duration FROM session_replay_events", modifiers)++ assert "$session_id_uuid" in printed+def _print_select(self, select: str, modifiers: HogQLQueryModifiers | None = None):expr = parse_select(select)query, _ = prepare_and_print_ast(