Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): swap flag call scans spelled posthog.events (#125115)

Phil Haackca0d46f33a54
diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.py
index 2f9abbc33bc5..45d35c093a0a 100644
--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py
+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py
@@ -737,6 +737,27 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
"AND e.properties.$feature_flag = 'probe-flag' AND e.timestamp >= now() - INTERVAL 7 DAY",
[(1,)],
),
+ (
+ "posthog_events_qualified_without_an_alias",
+ "SELECT count(posthog.events.uuid) FROM posthog.events "
+ "WHERE posthog.events.event = '$feature_flag_called' "
+ "AND posthog.events.properties.$feature_flag = 'probe-flag' "
+ "AND posthog.events.timestamp >= now() - INTERVAL 7 DAY",
+ [(1,)],
+ ),
+ (
+ "posthog_events_as_one_quoted_name",
+ f"SELECT count(`posthog.events`.uuid) FROM `posthog.events` WHERE {FLAG_CALL_CONJUNCTS}",
+ [(1,)],
+ ),
+ (
+ "posthog_events_qualified_with_a_join",
+ "SELECT count() FROM posthog.events JOIN (SELECT 'probe-distinct-id' AS d) AS s "
+ "ON posthog.events.distinct_id = s.d WHERE posthog.events.event = '$feature_flag_called' "
+ "AND posthog.events.properties.$feature_flag = 'probe-flag' "
+ "AND posthog.events.timestamp >= now() - INTERVAL 7 DAY",
+ [(1,)],
+ ),
(
"alias_named_flag_key_read_back",
"SELECT properties.$feature_flag AS flag_key, count() FROM events "
@@ -745,6 +766,13 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
),
("element_column", f"SELECT elements_chain FROM events WHERE {FLAG_CALL_CONJUNCTS}", [("",)]),
("subquery_in_from", f"SELECT count() FROM (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS})", [(1,)]),
+ (
+ "beside_a_qualified_read_of_a_cte_named_posthog_events",
+ f"WITH `posthog.events` AS (SELECT uuid FROM events WHERE {PAGEVIEW_CONJUNCTS}) "
+ f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS} "
+ "AND uuid NOT IN (SELECT posthog.events.uuid FROM posthog.events)",
+ [(1,)],
+ ),
(
"cte",
f"WITH calls AS (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}) SELECT count() FROM calls",
@@ -916,6 +944,20 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
"a name in it would read a different column",
PersonsOnEventsMode.DISABLED,
),
+ (
+ "posthog_events_person_id_through_distinct_ids",
+ f"SELECT person_id FROM posthog.events WHERE {FLAG_CALL_CONJUNCTS}",
+ "a name in it would read a different column",
+ ),
+ (
+ "posthog_events_qualifier_that_a_joined_column_takes",
+ "SELECT count() FROM posthog.events "
+ "JOIN (SELECT distinct_id AS d, properties AS posthog FROM events) AS s ON distinct_id = s.d "
+ "WHERE posthog.events.event = '$feature_flag_called' "
+ "AND posthog.events.properties.$feature_flag = 'probe-flag' "
+ "AND posthog__events.timestamp >= now() - INTERVAL 7 DAY",
+ "starts with posthog.events reads another table",
+ ),
(
"current_person_properties",
f"SELECT person.properties.email FROM events WHERE {FLAG_CALL_CONJUNCTS}",
@@ -995,6 +1037,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
f"SELECT count() FROM events AS e JOIN {PROBE_DISTINCT_IDS} ON e.distinct_id = d.id "
"WHERE event IN ('$feature_flag_called', '$pageview') AND timestamp >= now() - INTERVAL 7 DAY",
),
+ (
+ "cte_named_posthog_events",
+ "WITH `posthog.events` AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "
+ f"SELECT uuid FROM posthog.events WHERE {FLAG_CALL_CONJUNCTS}",
+ ),
]
)
def test_query_that_does_not_read_only_flag_calls_from_events_is_unchanged_once_events_gets_no_flag_calls(
@@ -1281,6 +1328,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
@parameterized.expand(
[
("flag_calls", f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}", "saved_calls"),
+ (
+ "posthog_events_flag_calls",
+ f"SELECT uuid FROM posthog.events WHERE {FLAG_CALL_CONJUNCTS}",
+ "saved_calls",
+ ),
("view_of_a_flag_call_view", "SELECT uuid FROM inner_calls", "inner_calls"),
("pageviews", f"SELECT uuid FROM events WHERE {PAGEVIEW_CONJUNCTS}", None),
(
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
index 1222f3a5d277..2daa80dc458b 100644
--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -2,7 +2,8 @@
On flag_evaluations mode 2, ingestion stops writing `$feature_flag_called` to `events`. This transform swaps a
query's `events` scan for `posthog.flag_evaluations` when `flag_evaluations` holds every row the scan selects, so
-saved queries keep returning flag calls without a change.
+saved queries keep returning flag calls without a change. A scan spelled `posthog.events` reads the same table and
+swaps the same way.
The swap runs before type resolution, so every rule here is syntactic. A shape the rules cannot classify keeps
reading `events`. On mode 2, a second pass finds the scans those rules miss in the resolved query.
@@ -10,7 +11,7 @@ reading `events`. On mode 2, a second pass finds the scans those rules miss in t
import json
from collections import Counter
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime, time, timedelta, tzinfo
from functools import cache, partial
from typing import TYPE_CHECKING, Protocol
@@ -104,6 +105,8 @@ _UNMOVABLE_SHAPE = "it reads events in a shape that can't be moved to posthog.fl
# An alias with one of these names can shadow the column in an unqualified read.
_SCAN_COLUMNS = frozenset({"event", "timestamp", "properties"})
+_EVENTS_NAMES = frozenset({"events", "posthog.events"})
+
def can_read_flag_evaluations(
node: ast.SelectQuery,
@@ -139,9 +142,9 @@ def can_read_flag_evaluations(
without a flag key.
`resolve_flag_calls_from_flag_evaluations` checks the rest after resolution: every name in the swapped query
- binds as it did before.
+ binds as it did before, and every field that starts with `posthog.events` reads the scan.
"""
- scan = EventsScan.of(node, lambda: flag_key_compares_as_string)
+ scan = EventsScan.of(node, lambda: flag_key_compares_as_string, frozenset(node.ctes or ()))
return scan is not None and _qualifies(
scan, node, filter_conjuncts(node), now, timezone, only_in_flag_evaluations=only_in_flag_evaluations
)
@@ -151,7 +154,8 @@ def resolve_flag_calls_from_flag_evaluations(node: _T_AST, context: HogQLContext
"""`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading
`posthog.flag_evaluations` instead.
- This returns the swapped query only when it resolves and every name in it binds as it does in the original.
+ This returns the swapped query only when it resolves, every name in it binds as it does in the original, and
+ every field in it that starts with `posthog.events` reads a scan.
Otherwise it returns the resolved original. For an organization whose flag calls are only in
`flag_evaluations`, it raises a QueryError instead of returning the original. It also raises before it
resolves anything when such an organization's query has a scan of only flag calls with a shape that
@@ -234,6 +238,8 @@ def _resolve_with_swaps(
else:
if not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):
reason = "a name in it would read a different column there"
+ elif not _posthog_events_qualifiers_read_scans(swapped):
+ reason = "a name in it that starts with posthog.events reads another table"
elif not only_in_flag_evaluations and not _swap_reads_properties_alike(original, swapped, context):
reason = "it reads a property from a different kind of column there"
else:
@@ -293,21 +299,33 @@ class EventsScan:
self.has_other_sources = has_other_sources
self.flag_key_compares_as_string = flag_key_compares_as_string
self.shadowed: frozenset[str] = frozenset()
+ self.qualifier_shadows: frozenset[str] = frozenset()
@classmethod
- def of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool]) -> "EventsScan | None":
+ def of(
+ cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]
+ ) -> "EventsScan | None":
+ """The scan of `node`, or None when `node` does not read `events`. `ctes` holds the CTE names in scope. The
+ resolver reads a CTE when one has the table's dotted name."""
join = node.select_from
- if join is None or not isinstance(join.table, ast.Field) or join.table.chain != ["events"]:
+ if join is None or not isinstance(join.table, ast.Field):
+ return None
+ parts = [str(part) for part in join.table.chain]
+ table_name = ".".join(parts)
+ if table_name not in _EVENTS_NAMES or table_name in ctes:
return None
scan = cls(
join=join,
- alias=join.alias or "events",
+ alias=join.alias or "__".join(parts),
# A field without a table prefix can bind to a joined table or an ARRAY JOIN alias instead.
has_other_sources=join.next_join is not None or bool(node.array_join_list),
flag_key_compares_as_string=flag_key_compares_as_string,
)
collector = _AliasCollector(node)
collector.visit(node)
+ # The resolver tries a joined table's name, a CTE and a select alias before it reads a qualifier such as
+ # `posthog.events` as the name of a table.
+ scan.qualifier_shadows = ctes.union(_join_names(join.next_join), (name for name, _ in collector.aliases))
# An alias such as `e.timestamp AS timestamp` reads the column it names, so it changes no read.
scan.shadowed = frozenset(
name for name, expr in collector.aliases if name in _SCAN_COLUMNS and scan.column(expr) != [name]
@@ -323,10 +341,21 @@ class EventsScan:
return None
if expr.chain[0] == self.alias and len(expr.chain) > 1:
return expr.chain[1:]
+ unqualified = self._without_dotted_qualifier(expr.chain)
+ if unqualified is not None:
+ return unqualified
if self.has_other_sources or expr.chain[0] in self.shadowed:
return None
return expr.chain
+ def _without_dotted_qualifier(self, chain: list[str | int]) -> list[str | int] | None:
+ """`chain` without a leading `posthog.events`, or None when it has none. Without an alias, `FROM posthog.events`
+ puts the table in scope as `posthog__events`, and a read can spell that name as `posthog.events` or
+ `` `posthog.events` ``."""
+ if self.alias != "posthog__events" or str(chain[0]) in self.qualifier_shadows:
+ return None
+ return _below_posthog_events(chain)
+
def shape_refusal(self, node: ast.SelectQuery) -> str | None:
"""Why `flag_evaluations` cannot stand in for this scan whatever its filters say, or None."""
if self.join.table_args is not None or self.join.table_final:
@@ -437,6 +466,25 @@ def _pins_flag_calls_in(
return False
+def _join_names(join: ast.JoinExpr | None) -> Iterator[str]:
+ """The name each join in the chain from `join` puts in scope. A joined subquery without an alias puts none."""
+ while join is not None:
+ if join.alias is not None:
+ yield join.alias
+ elif isinstance(join.table, ast.Field):
+ yield "__".join(str(part) for part in join.table.chain)
+ join = join.next_join
+
+
+def _below_posthog_events(chain: list[str | int]) -> list[str | int] | None:
+ """`chain` without a leading `posthog.events` or `` `posthog.events` ``, or None when it has neither."""
+ if chain[0] == "posthog.events" and len(chain) > 1:
+ return chain[1:]
+ if chain[:2] == ["posthog", "events"] and len(chain) > 2:
+ return chain[2:]
+ return None
+
+
def comparison_of(expr: ast.Expr) -> ast.CompareOperation | None:
"""`expr` as a comparison, whether it is written `a >= b` or `greaterOrEquals(a, b)`."""
if isinstance(expr, ast.CompareOperation):
@@ -748,7 +796,7 @@ class _FlagCalledConstantFinder(SyntaxVisitor):
class ScanVisitor(SyntaxVisitor):
- """Calls `visit_scan_query` for each SELECT whose `FROM events` can read the events table, not a CTE."""
+ """Calls `visit_scan_query` for each SELECT. `ctes` then holds the CTE names in scope, which `EventsScan.of` needs."""
def __init__(self) -> None:
super().__init__()
@@ -776,8 +824,7 @@ class ScanVisitor(SyntaxVisitor):
for cte in (node.ctes or {}).values():
self.visit(cte.expr)
self.ctes = self.ctes.union([cte.name])
- if "events" not in self.ctes:
- self.visit_scan_query(node)
+ self.visit_scan_query(node)
super().visit_select_query(node)
self.ctes = outer
@@ -804,7 +851,7 @@ class _EventsScanSwapper(ScanVisitor):
self.refusals: list[str] = []
def visit_scan_query(self, node: ast.SelectQuery) -> None:
- scan = EventsScan.of(node, self.flag_key_compares_as_string)
+ scan = EventsScan.of(node, self.flag_key_compares_as_string, self.ctes)
if scan is None:
return
conjuncts = filter_conjuncts(node)
@@ -940,6 +987,72 @@ def _swap_keeps_bindings(original: ast.AST, swapped: ast.AST, flag_key_predicate
return Counter(_bindings(swapped)) == expected
+def _posthog_events_qualifiers_read_scans(node: ast.AST) -> bool:
+ """Whether each field that starts with `posthog.events` in the SELECT of a swapped `posthog.events` scan reads an
+ events or flag_evaluations scan.
+
+ The swap counts such a field as a read of the scan. A joined table can still take that name, for example with a
+ column named `posthog`. The field then reads that table in both queries, so `_swap_keeps_bindings` does not see it.
+ A field in another SELECT reads whatever it names there, and the swap does not count it.
+ """
+ finder = _SwappedPosthogEventsScopeFinder()
+ finder.visit(node)
+ return all(_reads_scan(field.type) for field in finder.fields)
+
+
+def _reads_scan(current: ast.Type | None) -> bool:
+ while isinstance(
+ current,
+ (
+ ast.PropertyType,
+ ast.FieldType,
+ ast.FieldTraverserType,
+ ast.LazyJoinType,
+ ast.TableAliasType,
+ ast.VirtualTableType,
+ ),
+ ):
+ current = current.field_type if isinstance(current, ast.PropertyType) else current.table_type
+ return isinstance(current, ast.TableType) and isinstance(current.table, (EventsTable, FlagEvaluationsTable))
+
+
+class _SwappedPosthogEventsScopeFinder(SyntaxVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.fields: list[ast.Field] = []
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ join = node.select_from
+ if (
+ join is not None
+ and isinstance(join.type, ast.TableAliasType)
+ and join.type.alias == "posthog__events"
+ and isinstance(join.type.table_type.table, FlagEvaluationsTable)
+ ):
+ collector = _PosthogEventsQualifiedFieldCollector(node)
+ collector.visit(node)
+ self.fields.extend(collector.fields)
+ super().visit_select_query(node)
+
+
+class _PosthogEventsQualifiedFieldCollector(SyntaxVisitor):
+ def __init__(self, root: ast.SelectQuery) -> None:
+ super().__init__()
+ self.root = root
+ self.fields: list[ast.Field] = []
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ if node is self.root:
+ super().visit_select_query(node)
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
+ pass
+
+ def visit_field(self, node: ast.Field) -> None:
+ if node.chain and _below_posthog_events(node.chain) is not None:
+ self.fields.append(node)
+
+
# A field records the last part of its chain, so `e.person_id` and `person_id` match.
_Binding = tuple[str, str | int, tuple[str, ...]]
@@ -1139,7 +1252,8 @@ class _FlagCallViewFinder(SyntaxVisitor):
def _reads_only_flag_calls_from_events(node: ast.SelectQuery) -> bool:
- scan = EventsScan.of(node, lambda: True)
+ # `_reads_events_table` tells a CTE from the table by its resolved type.
+ scan = EventsScan.of(node, lambda: True, frozenset())
return (
scan is not None
and _reads_events_table(scan.join)
diff --git a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
index 8b1fa5c9d957..26410787b74e 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
@@ -52,6 +52,16 @@ def _where(*conjuncts: str) -> str:
return f"SELECT count() FROM events WHERE {' AND '.join(conjuncts)}"
+def _on_posthog_events(where: str, select: str = "count()") -> str:
+ return f"SELECT {select} FROM posthog.events WHERE {where}"
+
+
+POSTHOG_EVENTS_QUALIFIED = (
+ "posthog.events.event = '$feature_flag_called' AND posthog.events.properties.$feature_flag = 'my-flag' "
+ "AND posthog.events.timestamp >= now() - INTERVAL 7 DAY"
+)
+
+
def _bounded_by(lower_bound: str) -> str:
return _where(FLAG_CALLED, FLAG_KEY, f"timestamp >= {lower_bound}")
@@ -96,7 +106,7 @@ class TestCanReadFlagEvaluations(SimpleTestCase):
def test_flag_keys_cover_every_branch_of_an_or(self, _name: str, query: str):
select = parse_select(query)
assert isinstance(select, ast.SelectQuery)
- scan = EventsScan.of(select, lambda: True)
+ scan = EventsScan.of(select, lambda: True, frozenset())
assert scan is not None
assert scan.flag_keys(filter_conjuncts(select)) == ["a", "b"]
@@ -216,6 +226,25 @@ class TestCanReadFlagEvaluations(SimpleTestCase):
),
# The printer drops SAMPLE 1. The trends query builders set it on every events read.
("sample_of_one", f"SELECT count() FROM events SAMPLE 1 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ ("posthog_events", _on_posthog_events(f"{FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}")),
+ ("posthog_events_qualified_without_an_alias", _on_posthog_events(POSTHOG_EVENTS_QUALIFIED)),
+ (
+ "posthog_events_as_one_quoted_name",
+ f"SELECT count() FROM `posthog.events` WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
+ ),
+ (
+ "posthog_events_qualified_by_its_scope_name",
+ _on_posthog_events(POSTHOG_EVENTS_QUALIFIED.replace("posthog.events.", "posthog__events.")),
+ ),
+ (
+ "posthog_events_qualified_by_one_quoted_name",
+ _on_posthog_events(POSTHOG_EVENTS_QUALIFIED.replace("posthog.events.", "`posthog.events`.")),
+ ),
+ (
+ "posthog_events_qualifier_with_a_join",
+ "SELECT count() FROM posthog.events JOIN persons AS p ON posthog.events.person_id = p.id "
+ f"WHERE {POSTHOG_EVENTS_QUALIFIED}",
+ ),
(
"alias_that_renames_its_own_column",
f"SELECT e.timestamp AS timestamp FROM events e WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
@@ -393,6 +422,35 @@ class TestCanReadFlagEvaluations(SimpleTestCase):
"SELECT * FROM events ARRAY JOIN [1] AS n WHERE events.event = '$feature_flag_called' "
"AND events.properties.$feature_flag = 'my-flag' AND events.timestamp >= now() - INTERVAL 7 DAY",
),
+ (
+ "posthog_events_qualifier_that_a_select_alias_shadows",
+ _on_posthog_events(POSTHOG_EVENTS_QUALIFIED, select="properties AS posthog"),
+ ),
+ (
+ "posthog_events_quoted_qualifier_that_a_select_alias_shadows",
+ _on_posthog_events(
+ POSTHOG_EVENTS_QUALIFIED.replace("posthog.events.", "`posthog.events`."),
+ select="properties AS `posthog.events`",
+ ),
+ ),
+ (
+ "posthog_events_qualifier_that_a_cte_shadows",
+ f"WITH posthog AS (SELECT 1) {_on_posthog_events(POSTHOG_EVENTS_QUALIFIED)}",
+ ),
+ (
+ "posthog_events_qualifier_that_a_join_alias_shadows",
+ "SELECT count() FROM posthog.events JOIN persons AS posthog ON posthog.events.person_id = posthog.id "
+ f"WHERE {POSTHOG_EVENTS_QUALIFIED}",
+ ),
+ (
+ "posthog_events_qualifier_under_another_alias",
+ f"SELECT count() FROM posthog.events AS e WHERE {POSTHOG_EVENTS_QUALIFIED}",
+ ),
+ (
+ "posthog_events_read_from_a_cte_of_that_name",
+ "WITH `posthog.events` AS (SELECT event, properties, timestamp FROM posthog.flag_evaluations) "
+ + _on_posthog_events(f"{FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ ),
("bound_near_year_one", _bounded_by("'0001-01-01'")),
("interval_past_datetime_range", _bounded_by("now() - INTERVAL 999999999 YEAR")),
# Each level used to evaluate its value twice, so 40 levels took 2^40 steps.