Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): fix flag_evaluations array reads and lazy session joins (#125070)

Phil Haackc49e6e6459bf
diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py
index 29f93cc1650e..51802cbfe1e6 100644
--- a/posthog/hogql/database/database.py
+++ b/posthog/hogql/database/database.py
@@ -176,6 +176,7 @@ from posthog.schema_enums import (
DatabaseSerializedFieldType,
DataWarehouseSavedQueryOrigin,
PersonsOnEventsMode,
+ SessionsV2JoinMode,
SessionTableVersion,
)
from posthog.scopes import APIScopeObject
@@ -2463,6 +2464,9 @@ class Database(BaseModel):
events_table = database.get_table("events")
events_table.fields["session"] = LazyJoin(
from_field=["$session_id"],
+ extra_from_fields=[["$session_id_uuid"]]
+ if modifiers.sessionsV2JoinMode == SessionsV2JoinMode.UUID
+ else [],
join_table=sessions,
resolver=EVENTS_TO_SESSIONS_V2,
)
@@ -2489,6 +2493,7 @@ class Database(BaseModel):
events_table = database.get_table("events")
events_table.fields["session"] = LazyJoin(
from_field=["$session_id"],
+ extra_from_fields=[["$session_id_uuid"]],
join_table=sessions,
resolver=EVENTS_TO_SESSIONS_V3,
)
diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.py
index 408e65494d22..c9969c639608 100644
--- a/posthog/hogql/database/models.py
+++ b/posthog/hogql/database/models.py
@@ -497,6 +497,9 @@ class LazyJoin(FieldOrTable):
join_table: Table | str
from_field: list[str | int]
to_field: Optional[list[str | int]] = None
+ # The other fields of the source table that the resolver's ON clause reads. A lazy source table returns only the
+ # fields a query asks for, so lazy table resolution asks for these too.
+ extra_from_fields: list[list[str | int]] = PydanticField(default_factory=list)
def resolve_table(self, context: "HogQLContext") -> Table:
if isinstance(self.join_table, Table):
diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.py
index 937f82deeb6b..805be073f770 100644
--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py
+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py
@@ -982,6 +982,13 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
assert response.results == [(1,)]
+ def test_swapped_scan_reads_exception_values_as_an_array(self):
+ results = self._query_events(
+ f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS} AND NOT has(properties.$exception_values, 'x')"
+ ).results
+
+ assert results == [(1,)]
+
@parameterized.expand(
[
# The events-only call has the JSON value true as its key. A typed comparison matches it on events.
diff --git a/posthog/hogql/test/__snapshots__/test_resolver.ambr b/posthog/hogql/test/__snapshots__/test_resolver.ambr
index 685e5ab15697..8826c47033e2 100644
--- a/posthog/hogql/test/__snapshots__/test_resolver.ambr
+++ b/posthog/hogql/test/__snapshots__/test_resolver.ambr
@@ -382,6 +382,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -459,6 +460,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -536,6 +538,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -613,6 +616,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -690,6 +694,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -767,6 +772,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -844,6 +850,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -921,6 +928,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -1744,6 +1752,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -1831,6 +1840,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -2736,6 +2746,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -2826,6 +2837,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -3163,6 +3175,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -3259,6 +3272,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -3355,6 +3369,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -3451,6 +3466,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -3547,6 +3563,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -3643,6 +3660,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -3739,6 +3757,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -3835,6 +3854,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -5660,6 +5680,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -5737,6 +5758,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -5814,6 +5836,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -5891,6 +5914,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -5968,6 +5992,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -6045,6 +6070,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -6122,6 +6148,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -6199,6 +6226,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -6695,6 +6723,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -6772,6 +6801,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -6849,6 +6879,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -6926,6 +6957,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -7003,6 +7035,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -7080,6 +7113,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -7157,6 +7191,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -7234,6 +7269,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -8060,6 +8096,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -8147,6 +8184,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -8708,6 +8746,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -8795,6 +8834,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -9931,6 +9971,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -10021,6 +10062,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -10358,6 +10400,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -10454,6 +10497,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -10550,6 +10594,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -10646,6 +10691,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -10742,6 +10788,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -10838,6 +10885,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -10934,6 +10982,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -11030,6 +11079,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -11618,6 +11668,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -11708,6 +11759,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -12045,6 +12097,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -12141,6 +12194,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -12237,6 +12291,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -12333,6 +12388,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -12429,6 +12485,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -12525,6 +12582,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -12621,6 +12679,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -12717,6 +12776,7 @@
field: "fingerprint_issue_state"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"fingerprint"
],
@@ -15547,6 +15607,7 @@
field: "person"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"person_id"
],
@@ -15560,6 +15621,7 @@
field: "pdi"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -15910,6 +15972,7 @@
field: "person"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"person_id"
],
@@ -15923,6 +15986,7 @@
field: "pdi"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -16269,6 +16333,7 @@
field: "pdi"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -16617,6 +16682,7 @@
field: "pdi"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -16754,6 +16820,7 @@
field: "person"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"person_id"
],
@@ -17708,6 +17775,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
@@ -17798,6 +17866,7 @@
field: "override"
lazy_join: {
description: None,
+ extra_from_fields: [],
from_field: [
"distinct_id"
],
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
index b0b71984e84d..fe069ec493ed 100644
--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -129,9 +129,9 @@ def can_read_flag_evaluations(
`resolve_flag_calls_from_flag_evaluations` checks the rest after resolution: every name in the swapped query
binds as it did before.
"""
- scan = _EventsScan.of(node, lambda: flag_key_compares_as_string)
+ scan = EventsScan.of(node, lambda: flag_key_compares_as_string)
return scan is not None and _qualifies(
- scan, node, _filter_conjuncts(node), now, timezone, only_in_flag_evaluations=only_in_flag_evaluations
+ scan, node, filter_conjuncts(node), now, timezone, only_in_flag_evaluations=only_in_flag_evaluations
)
@@ -184,9 +184,7 @@ def _resolve_with_swaps(
database: "Database",
only_in_flag_evaluations: bool,
) -> _T_AST:
- finder = _FlagCalledConstantFinder()
- finder.visit(node)
- if not finder.found:
+ if not names_flag_called_event(node):
return resolve(node)
if not only_in_flag_evaluations and context.uses_new_events_schema():
return resolve(node)
@@ -246,7 +244,7 @@ def _flag_calls_only_in_flag_evaluations_error(reason: str) -> QueryError:
def _qualifies(
- scan: "_EventsScan",
+ scan: "EventsScan",
node: ast.SelectQuery,
conjuncts: list[ast.Expr],
now: datetime,
@@ -261,7 +259,7 @@ def _qualifies(
return scan.pins_flag_calls(conjuncts, needs_flag_key=True) and scan.bounds_timestamp(conjuncts, now, timezone)
-class _EventsScan:
+class EventsScan:
def __init__(
self,
join: ast.JoinExpr,
@@ -276,7 +274,7 @@ class _EventsScan:
self.shadowed: frozenset[str] = frozenset()
@classmethod
- def of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool]) -> "_EventsScan | None":
+ def of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool]) -> "EventsScan | None":
join = node.select_from
if join is None or not isinstance(join.table, ast.Field) or join.table.chain != ["events"]:
return None
@@ -333,10 +331,8 @@ class _EventsScan:
def _pins_flag_calls_in(self, expr: ast.Expr, needs_flag_key: bool) -> bool:
if _pinned_values(expr, self._reads_event, _is_flag_called) is not None:
return not needs_flag_key
- if isinstance(expr, ast.Or):
- return bool(expr.exprs) and all(
- self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in expr.exprs
- )
+ if branches := boolean_operands(expr, ast.Or):
+ return all(self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in branches)
return False
def _flag_keys_in(self, expr: ast.Expr) -> list[str] | None:
@@ -344,10 +340,14 @@ class _EventsScan:
values = _pinned_values(expr, self._reads_flag_key, _is_flag_key)
if values is not None:
return [value.value for value in values if isinstance(value, ast.Constant)]
- if isinstance(expr, ast.Or) and expr.exprs:
- branches = [self._flag_keys_in(branch) for branch in expr.exprs]
- if all(keys is not None for keys in branches):
- return [key for keys in branches if keys is not None for key in keys]
+ if branches := boolean_operands(expr, ast.Or):
+ branch_keys = [self._flag_keys_in(branch) for branch in branches]
+ if all(keys is not None for keys in branch_keys):
+ return [key for keys in branch_keys if keys is not None for key in keys]
+ for conjunct in boolean_operands(expr, ast.And) or []:
+ keys = self._flag_keys_in(conjunct)
+ if keys is not None:
+ return keys
return None
def flag_keys(self, conjuncts: list[ast.Expr]) -> list[str]:
@@ -386,7 +386,7 @@ class _EventsScan:
def _lower_bound(self, expr: ast.Expr) -> ast.Expr | None:
if isinstance(expr, ast.BetweenExpr):
return expr.low if not expr.negated and self._reads_timestamp(expr.expr) else None
- comparison = _comparison(expr)
+ comparison = comparison_of(expr)
if comparison is None:
return None
if comparison.op in (ast.CompareOperationOp.GtEq, ast.CompareOperationOp.Gt) and self._reads_timestamp(
@@ -403,7 +403,7 @@ class _EventsScan:
return self.column(expr) == ["timestamp"]
-def _comparison(expr: ast.Expr) -> ast.CompareOperation | None:
+def comparison_of(expr: ast.Expr) -> ast.CompareOperation | None:
"""`expr` as a comparison, whether it is written `a >= b` or `greaterOrEquals(a, b)`."""
if isinstance(expr, ast.CompareOperation):
return expr
@@ -412,24 +412,64 @@ def _comparison(expr: ast.Expr) -> ast.CompareOperation | None:
return None
-def _pinned_values(expr: ast.Expr, reads: _Predicate, accepts: _Predicate) -> list[ast.Expr] | None:
- """The values `expr` limits what `reads` matches to, or None when they are not all values `accepts` matches."""
- comparison = _comparison(expr)
+def compared_values(expr: ast.Expr, reads: _Predicate) -> list[ast.Expr] | None:
+ """The values that `expr` compares a read matching `reads` with by `=` or `IN`, or None."""
+ comparison = comparison_of(expr)
if comparison is None:
return None
if comparison.op == ast.CompareOperationOp.Eq:
- if reads(comparison.left) and accepts(comparison.right):
+ if reads(comparison.left):
return [comparison.right]
- if reads(comparison.right) and accepts(comparison.left):
- return [comparison.left]
- return None
- if comparison.op in (ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn):
+ return [comparison.left] if reads(comparison.right) else None
+ if comparison.op in (ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn) and reads(comparison.left):
right = comparison.right
- values = right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]
- return values if reads(comparison.left) and values and all(accepts(value) for value in values) else None
+ return right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]
return None
+def _pinned_values(expr: ast.Expr, reads: _Predicate, accepts: _Predicate) -> list[ast.Expr] | None:
+ """The values `expr` limits what `reads` matches to, or None when they are not all values `accepts` matches."""
+ values = compared_values(expr, reads)
+ return values if values and all(accepts(value) for value in values) else None
+
+
+def flag_key_predicate(table: str, keys: list[str]) -> ast.Expr | None:
+ """`flag_key IN keys` on `table`, or None when a pin of `$feature_flag` to `keys` does not imply it.
+
+ flag_key holds the key's JSON text, and a JSONExtractString pin compares the decoded key. The pin implies this
+ predicate only when the JSON text of each key is the key itself.
+ """
+ if not keys or not all(is_json_verbatim(key) for key in keys):
+ return None
+ return ast.CompareOperation(
+ op=ast.CompareOperationOp.In,
+ left=ast.Field(chain=[table, "flag_key"]),
+ right=ast.Tuple(exprs=[ast.Constant(value=key) for key in keys]),
+ )
+
+
+def point_scan_at(
+ node: ast.SelectQuery, scan: EventsScan, chain: list[str | int], condition: ast.Expr | None = None
+) -> None:
+ """Points `scan` at the table `chain` names, and moves the PREWHERE of `node`, with `condition`, into its WHERE.
+
+ The new table can read person columns through a join that the printer adds later, and ClickHouse rejects a
+ PREWHERE that reads a joined column. WHERE selects the same rows.
+ """
+ # The old alias keeps qualified reads such as `events.timestamp` resolving.
+ scan.join.alias = scan.alias
+ scan.join.table = ast.Field(chain=chain)
+ node.where = and_conditions([node.prewhere, node.where, condition])
+ node.prewhere = None
+
+
+def and_conditions(conditions: list[ast.Expr | None]) -> ast.Expr | None:
+ present = [condition for condition in conditions if condition is not None]
+ if not present:
+ return None
+ return present[0] if len(present) == 1 else ast.And(exprs=present)
+
+
def _is_constant(expr: ast.Expr, value: str) -> bool:
return isinstance(expr, ast.Constant) and expr.value == value
@@ -453,7 +493,8 @@ def _conjuncts(expr: ast.Expr | None) -> list[ast.Expr]:
return top_level_conjuncts(expr) if expr is not None else []
-def _filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:
+def filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:
+ """The top-level conjuncts of the WHERE and PREWHERE of `node`."""
return _conjuncts(node.where) + _conjuncts(node.prewhere)
@@ -461,6 +502,22 @@ def _utc_midnight(now: datetime) -> datetime:
return start_of_day(now.astimezone(UTC))
+def names_flag_called_event(node: ast.AST) -> bool:
+ """Whether `$feature_flag_called` appears as a constant anywhere in `node`."""
+ finder = _FlagCalledConstantFinder()
+ finder.visit(node)
+ return finder.found
+
+
+def boolean_operands(expr: ast.Expr, operator: type[ast.And] | type[ast.Or]) -> list[ast.Expr] | None:
+ """The operands of an AND or an OR, in its node or its call form, or None for any other expression."""
+ if isinstance(expr, operator):
+ return expr.exprs
+ if isinstance(expr, ast.Call) and expr.name == ("and" if operator is ast.And else "or"):
+ return expr.args
+ return None
+
+
def _earliest_instant(expr: ast.Expr, now: datetime, timezone: ZoneInfo) -> datetime | None:
"""An instant no later than the value of `expr`, or None when `expr` has a shape this cannot evaluate."""
if isinstance(expr, ast.Constant):
@@ -620,14 +677,14 @@ def _constant_instant(value: object, zones: tuple[tzinfo, ...]) -> datetime | No
return min(value.replace(tzinfo=zone) for zone in zones)
-class _SyntaxVisitor(TraversingVisitor):
+class SyntaxVisitor(TraversingVisitor):
def visit_field(self, node: ast.Field) -> None:
# The inherited visit follows the field's type into shared CTE types, which takes exponential time. An
# earlier step can resolve parts of the tree in place.
pass
-class _AliasCollector(_SyntaxVisitor):
+class _AliasCollector(SyntaxVisitor):
def __init__(self, root: ast.SelectQuery) -> None:
super().__init__()
self.root = root
@@ -646,7 +703,7 @@ class _AliasCollector(_SyntaxVisitor):
super().visit_alias(node)
-class _FlagCalledConstantFinder(_SyntaxVisitor):
+class _FlagCalledConstantFinder(SyntaxVisitor):
def __init__(self) -> None:
super().__init__()
self.found = False
@@ -656,25 +713,13 @@ class _FlagCalledConstantFinder(_SyntaxVisitor):
self.found = True
-class _EventsScanSwapper(_SyntaxVisitor):
- def __init__(
- self,
- now: datetime,
- timezone: ZoneInfo,
- only_in_flag_evaluations: bool,
- flag_key_compares_as_string: Callable[[], bool],
- ) -> None:
+class ScanVisitor(SyntaxVisitor):
+ """Calls `visit_scan_query` for each SELECT whose `FROM events` can read the events table, not a CTE."""
+
+ def __init__(self) -> None:
super().__init__()
- self.now = now
- self.timezone = timezone
- self.only_in_flag_evaluations = only_in_flag_evaluations
- self.flag_key_compares_as_string = flag_key_compares_as_string
- self.swapped = False
- self.flag_key_predicates = 0
- # Why each scan that reads only flag calls has a shape that flag_evaluations cannot stand in for.
- self.refusals: list[str] = []
# The CTE names in scope, under the resolver's rules. A `FROM events` reads a CTE when one is named `events`.
- # A name missing here lets the swapper try the scan, and the binding check then rejects the swap.
+ # A name missing here lets a subclass change the scan, and the binding check then rejects the change.
self.ctes: frozenset[str] = frozenset()
def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
@@ -697,42 +742,53 @@ class _EventsScanSwapper(_SyntaxVisitor):
for cte in (node.ctes or {}).values():
self.visit(cte.expr)
self.ctes = self.ctes.union([cte.name])
- scan = None if "events" in self.ctes else _EventsScan.of(node, self.flag_key_compares_as_string)
- if scan is not None:
- conjuncts = _filter_conjuncts(node)
- if _qualifies(
- scan, node, conjuncts, self.now, self.timezone, only_in_flag_evaluations=self.only_in_flag_evaluations
- ):
- self._swap(node, scan, conjuncts)
- else:
- refusal = scan.shape_refusal(node)
- if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):
- self.refusals.append(refusal)
+ if "events" not in self.ctes:
+ self.visit_scan_query(node)
super().visit_select_query(node)
self.ctes = outer
- def _swap(self, node: ast.SelectQuery, scan: _EventsScan, conjuncts: list[ast.Expr]) -> None:
- keys = scan.flag_keys(conjuncts)
- # The old alias keeps qualified reads such as `events.timestamp` resolving.
- scan.join.alias = scan.alias
- scan.join.table = ast.Field(chain=["posthog", "flag_evaluations"])
- # A swapped PREWHERE can read person columns through a join that the printer adds later. ClickHouse
- # rejects a PREWHERE that reads a joined column, and WHERE selects the same rows.
- conditions = [condition for condition in (node.prewhere, node.where) if condition is not None]
- # flag_key sorts the table, so this predicate lets ClickHouse skip other flags. flag_key holds the key's JSON
- # text, and a JSONExtractString pin compares the decoded key. The pin implies this predicate only when the
- # JSON text of each key is the key itself.
- if keys and all(is_json_verbatim(key) for key in keys):
- conditions.append(
- ast.CompareOperation(
- op=ast.CompareOperationOp.In,
- left=ast.Field(chain=[scan.alias, "flag_key"]),
- right=ast.Tuple(exprs=[ast.Constant(value=key) for key in keys]),
- )
- )
+ def visit_scan_query(self, node: ast.SelectQuery) -> None:
+ raise NotImplementedError
+
+
+class _EventsScanSwapper(ScanVisitor):
+ def __init__(
+ self,
+ now: datetime,
+ timezone: ZoneInfo,
+ only_in_flag_evaluations: bool,
+ flag_key_compares_as_string: Callable[[], bool],
+ ) -> None:
+ super().__init__()
+ self.now = now
+ self.timezone = timezone
+ self.only_in_flag_evaluations = only_in_flag_evaluations
+ self.flag_key_compares_as_string = flag_key_compares_as_string
+ self.swapped = False
+ self.flag_key_predicates = 0
+ # Why each scan that reads only flag calls has a shape that flag_evaluations cannot stand in for.
+ self.refusals: list[str] = []
+
+ def visit_scan_query(self, node: ast.SelectQuery) -> None:
+ scan = EventsScan.of(node, self.flag_key_compares_as_string)
+ if scan is None:
+ return
+ conjuncts = filter_conjuncts(node)
+ if _qualifies(
+ scan, node, conjuncts, self.now, self.timezone, only_in_flag_evaluations=self.only_in_flag_evaluations
+ ):
+ self._swap(node, scan, conjuncts)
+ else:
+ refusal = scan.shape_refusal(node)
+ if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):
+ self.refusals.append(refusal)
+
+ def _swap(self, node: ast.SelectQuery, scan: EventsScan, conjuncts: list[ast.Expr]) -> None:
+ # flag_key sorts the table, so this predicate lets ClickHouse skip other flags.
+ flag_key = flag_key_predicate(scan.alias, scan.flag_keys(conjuncts))
+ if flag_key is not None:
self.flag_key_predicates += 1
- node.prewhere = None
- node.where = conditions[0] if len(conditions) == 1 else ast.And(exprs=conditions)
+ point_scan_at(node, scan, ["posthog", "flag_evaluations"], flag_key)
self.swapped = True
@@ -764,7 +820,7 @@ def _bindings(node: ast.AST) -> list[_Binding]:
return collector.bindings
-class _BindingCollector(_SyntaxVisitor):
+class _BindingCollector(SyntaxVisitor):
def __init__(self) -> None:
super().__init__()
self.bindings: list[_Binding] = []
@@ -907,7 +963,7 @@ def _property_sources(reads: list[_PropertyRead], context: HogQLContext) -> list
return sources
-class _PropertyReadCollector(_SyntaxVisitor):
+class _PropertyReadCollector(SyntaxVisitor):
def __init__(self, context: HogQLContext) -> None:
super().__init__()
self.context = context
@@ -932,7 +988,7 @@ def _view_reading_flag_calls_from_events(node: ast.AST) -> str | None:
return finder.found
-class _FlagCallViewFinder(_SyntaxVisitor):
+class _FlagCallViewFinder(SyntaxVisitor):
def __init__(self) -> None:
super().__init__()
self.view: str | None = None
@@ -949,11 +1005,11 @@ class _FlagCallViewFinder(_SyntaxVisitor):
def _reads_only_flag_calls_from_events(node: ast.SelectQuery) -> bool:
- scan = _EventsScan.of(node, lambda: True)
+ scan = EventsScan.of(node, lambda: True)
return (
scan is not None
and _reads_events_table(scan.join)
- and scan.pins_flag_calls(_filter_conjuncts(node), needs_flag_key=False)
+ and scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False)
)
diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.py
index e04b2405a596..d3bad3766aee 100644
--- a/posthog/hogql/transforms/lazy_tables.py
+++ b/posthog/hogql/transforms/lazy_tables.py
@@ -614,6 +614,8 @@ class LazyTableResolver(TraversingVisitor):
for new_join in joins_to_add.values():
if new_join.from_table in joins_to_add or new_join.from_table in tables_to_add:
create_override(new_join.from_table, new_join.lazy_join.from_field)
+ for extra_from_field in new_join.lazy_join.extra_from_fields:
+ create_override(new_join.from_table, extra_from_field)
if new_join.lazy_join.to_field is not None and (
new_join.to_table in joins_to_add or new_join.to_table in tables_to_add
):
diff --git a/posthog/hogql/transforms/property_types.py b/posthog/hogql/transforms/property_types.py
index 50cd2a732e05..9c3d992ebc28 100644
--- a/posthog/hogql/transforms/property_types.py
+++ b/posthog/hogql/transforms/property_types.py
@@ -286,7 +286,7 @@ class PropertySwapper(CloningVisitor):
table_type = type.field_type.table_type
if not isinstance(table_type, ast.BaseTableType):
return False
- return isinstance(table_type.resolve_database_table(self.context), EventsTable)
+ return isinstance(table_type.resolve_database_table(self.context), EventsTable | FlagEvaluationsTable)
@staticmethod
def _extract_string_array(field: ast.Expr) -> ast.Call:
diff --git a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
index 9c0bb3c73321..8b1fa5c9d957 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
@@ -12,7 +12,9 @@ from posthog.hogql import ast
from posthog.hogql.parser import parse_select
from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (
FLAG_EVALUATIONS_COMPLETE_DAYS,
+ EventsScan,
can_read_flag_evaluations,
+ filter_conjuncts,
)
from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_TTL_DAYS
@@ -24,6 +26,12 @@ IN_COVERAGE = COVERAGE_START + timedelta(days=30)
FLAG_CALLED = "event = '$feature_flag_called'"
FLAG_KEY = "properties.$feature_flag = 'my-flag'"
BOUND = "timestamp >= now() - INTERVAL 7 DAY"
+OR_OF_AND_BRANCHES = (
+ "((properties.$feature_flag = 'a' AND distinct_id = 'x') OR (properties.$feature_flag = 'b' AND distinct_id = 'y'))"
+)
+OR_CALL_OF_AND_CALLS = (
+ "or(and(properties.$feature_flag = 'a', distinct_id = 'x'), and(properties.$feature_flag = 'b', distinct_id = 'y'))"
+)
def _text(instant: datetime) -> str:
@@ -79,6 +87,19 @@ class TestCanReadFlagEvaluations(SimpleTestCase):
select, datetime.now(UTC), ZoneInfo(timezone), only_in_flag_evaluations=only_in_flag_evaluations
)
+ @parameterized.expand(
+ [
+ ("or_of_and_branches", _where(FLAG_CALLED, OR_OF_AND_BRANCHES)),
+ ("or_call_of_and_calls", _where(FLAG_CALLED, OR_CALL_OF_AND_CALLS)),
+ ]
+ )
+ def test_flag_keys_cover_every_branch_of_an_or(self, _name: str, query: str):
+ select = parse_select(query)
+ assert isinstance(select, ast.SelectQuery)
+ scan = EventsScan.of(select, lambda: True)
+ assert scan is not None
+ assert scan.flag_keys(filter_conjuncts(select)) == ["a", "b"]
+
def test_complete_days_leave_out_the_day_the_ttl_may_have_dropped(self):
assert FLAG_EVALUATIONS_COMPLETE_DAYS < FLAG_EVALUATIONS_TTL_DAYS
@@ -95,6 +116,19 @@ class TestCanReadFlagEvaluations(SimpleTestCase):
BOUND,
),
),
+ ("or_of_flag_keys_with_other_conditions", _where(FLAG_CALLED, OR_OF_AND_BRANCHES, BOUND)),
+ (
+ "or_call_of_flag_keys",
+ _where(FLAG_CALLED, "or(properties.$feature_flag = 'a', properties.$feature_flag = 'b')", BOUND),
+ ),
+ (
+ "or_call_of_pinned_branches_with_their_own_flag_keys",
+ _where(
+ "or(and(event = '$feature_flag_called', properties.$feature_flag = 'a'), "
+ "and(event = '$feature_flag_called', properties.$feature_flag = 'b'))",
+ BOUND,
+ ),
+ ),
("pin_in_prewhere", f"SELECT count() FROM events PREWHERE {FLAG_CALLED} WHERE {FLAG_KEY} AND {BOUND}"),
("conjuncts_in_nested_ands", _where(FLAG_CALLED, f"({FLAG_KEY} AND ({BOUND} AND distinct_id = 'd'))")),
(
diff --git a/posthog/hogql/transforms/test/test_lazy_tables.py b/posthog/hogql/transforms/test/test_lazy_tables.py
index 3d2d274e3c92..164e08256d26 100644
--- a/posthog/hogql/transforms/test/test_lazy_tables.py
+++ b/posthog/hogql/transforms/test/test_lazy_tables.py
@@ -18,7 +18,7 @@ from django.test import SimpleTestCase, override_settings
from parameterized import parameterized
-from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode, SessionsV2JoinMode, SessionTableVersion
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
@@ -573,6 +573,24 @@ class TestLazyJoins(BaseTest):
'SAMPLE clause ignored: the "sessions" table is computed on the fly and cannot be sampled'
]
+ @parameterized.expand(
+ [
+ ("v3", HogQLQueryModifiers(sessionTableVersion=SessionTableVersion.V3)),
+ (
+ "v2_uuid_join",
+ HogQLQueryModifiers(
+ sessionTableVersion=SessionTableVersion.V2, sessionsV2JoinMode=SessionsV2JoinMode.UUID
+ ),
+ ),
+ ]
+ )
+ def test_join_from_a_lazy_join_reads_the_extra_source_field_it_declares(
+ self, _name: str, modifiers: HogQLQueryModifiers
+ ):
+ printed = self._print_select("SELECT events.session.$session_duration FROM session_replay_events", modifiers)
+
+ assert "$session_id_uuid" in printed
+
def _print_select(self, select: str, modifiers: HogQLQueryModifiers | None = None):
expr = parse_select(select)
query, _ = prepare_and_print_ast(