Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): read flag calls from flag_evaluations inside saved views (#125750)

Phil Haackd5feff74a78d
diff --git a/posthog/hogql/context.py b/posthog/hogql/context.py
index 81ea1073531b..f3e8bc63bab6 100644
--- a/posthog/hogql/context.py
+++ b/posthog/hogql/context.py
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
from posthog.hogql.database.models import Table
from posthog.hogql.observability import HogQLTypeObservability
from posthog.hogql.property_metadata import PropertyMetadata
+ from posthog.hogql.transforms.flag_calls_from_flag_evaluations import FlagCallPropertyMetadata
from posthog.hogql.transforms.property_types import PropertySwapper
from posthog.clickhouse.client.execute import ClickHouseExternalTable
@@ -175,6 +176,9 @@ class HogQLContext:
# Backend-only switch for transforms/flag_calls_from_flag_evaluations.py. The data-deletion path sets it False, so a
# flag-call subquery in a deletion predicate keeps reading `events` whatever the organization's mode.
rewrite_flag_calls_to_flag_evaluations: bool = True
+ # Per-query event property definitions that transform loads. The transform runs once for the query and once for
+ # each saved view the query inlines, so this loads each property's definition once per query.
+ flag_call_property_metadata: Optional["FlagCallPropertyMetadata"] = field(default=None, compare=False, repr=False)
# Backend-only opt-in for transforms/events_read_in_order.py. The prefix is slower when the range is short or
# the filter is selective, so set it only for a query shape that was measured to be faster with it.
diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.py
index 45d35c093a0a..61aefb207137 100644
--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py
+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py
@@ -1,5 +1,6 @@
import json
import uuid
+from collections import Counter
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
@@ -42,8 +43,11 @@ from posthog.hogql.database.schema.flag_evaluations import add_events_list_field
from posthog.hogql.database.schema.util.where_clause_extractor import SESSION_BUFFER_DAYS
from posthog.hogql.errors import QueryError
from posthog.hogql.parser import parse_select
+from posthog.hogql.printer.utils import prepare_and_print_ast
from posthog.hogql.property import property_to_expr
+from posthog.hogql.property_metadata import load_property_metadata
from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import swap_flag_calls
from posthog.hogql.utils import is_json_verbatim
from posthog.hogql.visitor import clear_locations
@@ -58,9 +62,11 @@ from posthog.test.persons import create_group_type_mapping, create_person
from products.access_control.backend.models.property_access_control import PropertyAccessControl
from products.access_control.backend.property_access_control import PropertyAccessLevel
+from products.actions.backend.models.action import Action
from products.cohorts.backend.models.cohort import Cohort
from products.cohorts.backend.models.util import insert_static_cohort
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.data_tools.backend.facade.models import DataWarehouseJoin
from products.data_tools.backend.models.expression import DataWarehouseExpression
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
@@ -1096,6 +1102,272 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
create_person(team=self.team, distinct_ids=["events-distinct-id"])
flush_persons_and_events()
+ def _create_view(self, name: str, query: str, columns: tuple[str, ...] = ("distinct_id",)) -> None:
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name=name,
+ query={"kind": "HogQLQuery", "query": query},
+ columns=dict.fromkeys(columns, "String"),
+ )
+
+ @parameterized.expand(
+ [
+ (
+ "mode_1_inside_coverage",
+ {"calls": f"SELECT distinct_id FROM events WHERE {FLAG_CALL_CONJUNCTS}"},
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ ["probe-distinct-id"],
+ ),
+ (
+ "mode_1_without_coverage",
+ {"calls": "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"},
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ ["events-distinct-id"],
+ ),
+ (
+ "mode_1_swapped_body_fails_to_resolve",
+ {
+ "calls": "SELECT distinct_id, goe_0.properties AS group_properties FROM events "
+ f"WHERE {FLAG_CALL_CONJUNCTS}"
+ },
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ ["events-distinct-id"],
+ ),
+ (
+ "mode_2_without_coverage",
+ {"calls": "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"},
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["probe-distinct-id"],
+ ),
+ (
+ "mode_2_posthog_events",
+ {"calls": "SELECT distinct_id FROM posthog.events WHERE event = '$feature_flag_called'"},
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["probe-distinct-id"],
+ ),
+ (
+ "mode_2_view_of_a_view",
+ {
+ "calls": "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'",
+ "calls_of_calls": "SELECT distinct_id FROM calls",
+ },
+ "SELECT distinct_id FROM calls_of_calls",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["probe-distinct-id"],
+ ),
+ (
+ "mode_2_union_body",
+ {
+ "calls": "SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
+ "UNION ALL SELECT 'other-distinct-id'"
+ },
+ "SELECT distinct_id FROM calls ORDER BY distinct_id",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["other-distinct-id", "probe-distinct-id"],
+ ),
+ (
+ "mode_2_events_after_another_table",
+ {
+ "calls": f"SELECT e.distinct_id FROM {PROBE_DISTINCT_IDS} JOIN events AS e ON e.distinct_id = d.id "
+ "WHERE e.event = '$feature_flag_called'"
+ },
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["probe-distinct-id"],
+ ),
+ (
+ "mode_2_flag_calls_and_other_events",
+ {
+ "calls": "SELECT distinct_id FROM events WHERE event IN ('$feature_flag_called', '$pageview') "
+ "AND timestamp >= now() - INTERVAL 7 DAY"
+ },
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["events-distinct-id"],
+ ),
+ (
+ "mode_2_body_with_a_cte_named_events",
+ {
+ "calls": "WITH events AS (SELECT distinct_id, event FROM posthog.flag_evaluations) "
+ "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"
+ },
+ "SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["probe-distinct-id"],
+ ),
+ (
+ "mode_2_under_an_enclosing_cte_named_events",
+ {"calls": "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"},
+ "WITH events AS (SELECT distinct_id, event FROM posthog.flag_evaluations) SELECT distinct_id FROM calls",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ["probe-distinct-id"],
+ ),
+ ]
+ )
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
+ def test_view_reads_its_flag_calls_as_a_query_of_its_own_would(
+ self, _name: str, views: dict[str, str], query: str, mode: FlagEvaluationsMode, expected: list[str]
+ ):
+ self._create_events_only_flag_call()
+ for name, body in views.items():
+ self._create_view(name, body)
+
+ assert [row[0] for row in self._query_events(query, mode=mode).results] == expected
+
+ def test_view_joined_through_a_warehouse_join_reads_its_flag_calls_from_flag_evaluations(self):
+ self._create_events_only_flag_call()
+ _create_event(team=self.team, event="$pageview", distinct_id="probe-distinct-id", timestamp=self.occurred_at)
+ flush_persons_and_events()
+ self._create_view("calls", "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'")
+ DataWarehouseJoin.objects.create(
+ team=self.team,
+ source_table_name="events",
+ source_table_key="distinct_id",
+ joining_table_name="calls",
+ joining_table_key="distinct_id",
+ field_name="calls",
+ )
+
+ results = self._query_events("SELECT calls.distinct_id FROM events WHERE event = '$pageview'").results
+
+ assert [row[0] for row in results] == ["probe-distinct-id"]
+
+ @parameterized.expand(
+ [
+ ("select_star", "SELECT * FROM events WHERE event = '$feature_flag_called'", "it selects \\* or COLUMNS"),
+ (
+ "event_time_group_properties",
+ f"SELECT distinct_id, goe_0.properties FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ "it reads something that table doesn't have",
+ ),
+ ]
+ )
+ def test_view_flag_evaluations_cannot_answer_raises_an_error_that_names_the_view(
+ self, _name: str, body: str, reason: str
+ ):
+ self._create_view("calls", body)
+
+ with self.assertRaisesRegex(QueryError, f"the view calls can't read them there because {reason}"):
+ self._query_events("SELECT distinct_id FROM calls")
+
+ def test_view_that_names_flag_calls_only_through_an_action_raises(self):
+ action = Action.objects.create(team=self.team, steps_json=[{"event": "$feature_flag_called"}])
+ self._create_view("calls", f"SELECT distinct_id FROM events WHERE matchesAction({action.pk})")
+
+ with self.assertRaisesRegex(QueryError, "it reads the saved view calls, which reads them from events"):
+ self._query_events("SELECT distinct_id FROM calls")
+
+ def test_view_read_under_a_cte_with_the_same_name_and_another_body_decides_its_swap_again(self):
+ self._create_view(
+ "calls",
+ "SELECT distinct_id, flag_key AS fk FROM events WHERE event = '$feature_flag_called'",
+ ("distinct_id", "fk"),
+ )
+ # The first branch's CTE reads flag_evaluations, so the swap keeps the binding of flag_key. The second
+ # branch's CTE is a constant, so the swap there would read the flag_key column instead.
+ query = (
+ "WITH (SELECT flag_key FROM posthog.flag_evaluations LIMIT 1) AS flag_key SELECT fk FROM calls "
+ "UNION ALL WITH 'sentinel' AS flag_key SELECT fk FROM calls"
+ )
+
+ with self.assertRaisesRegex(QueryError, "the view calls can't read them there because a name in it"):
+ self._query_events(query)
+
+ def test_chain_of_flag_call_views_runs_the_swap_once_per_view(self):
+ self._create_events_only_flag_call()
+ self._create_view(
+ "calls_0",
+ "SELECT distinct_id, event FROM events WHERE event = '$feature_flag_called'",
+ ("distinct_id", "event"),
+ )
+ for level in range(1, 4):
+ self._create_view(
+ f"calls_{level}",
+ # The resolver builds a new node for a body's CTE on each visit of that body.
+ f"WITH one AS (SELECT 1 AS one) "
+ f"SELECT distinct_id, event FROM calls_{level - 1} WHERE event = '$feature_flag_called'",
+ ("distinct_id", "event"),
+ )
+
+ with patch(
+ "posthog.hogql.resolver.swap_flag_calls",
+ wraps=swap_flag_calls,
+ ) as swap:
+ results = self._query_events("SELECT distinct_id FROM calls_3").results
+
+ assert [row[0] for row in results] == ["probe-distinct-id"]
+ # Each extra resolve of a view body would otherwise rerun the swap on every view below it.
+ assert sorted(call.kwargs["view_name"] for call in swap.call_args_list) == [
+ f"calls_{level}" for level in range(4)
+ ]
+
+ @parameterized.expand(
+ [
+ ("direct", {"calls_a": "calls_a"}),
+ ("indirect", {"calls_a": "calls_b", "calls_b": "calls_a"}),
+ ]
+ )
+ def test_flag_call_views_that_read_themselves_raise_a_cycle_error(self, _name: str, reads: dict[str, str]):
+ for name, other in reads.items():
+ self._create_view(
+ name,
+ f"SELECT distinct_id FROM events WHERE {FLAG_CALL_CONJUNCTS} UNION ALL SELECT distinct_id FROM {other}",
+ )
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization,
+ defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},
+ )
+
+ # `execute_hogql_query` resolves for HogQL first, and that resolve raises before any swap runs. A ClickHouse
+ # compile on its own reaches the swap. The low expansion limit makes a regression that recurses through the
+ # swap fail with the wrong error within seconds.
+ with (
+ patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-from-flag-evaluations",
+ ),
+ patch("posthog.hogql.compilation.MAX_VIEW_EXPANSIONS", 20),
+ self.assertRaisesRegex(QueryError, "Circular dependency detected in view 'calls_a'"),
+ ):
+ prepare_and_print_ast(
+ parse_select("SELECT distinct_id FROM calls_a"),
+ HogQLContext(team_id=self.team.pk, enable_select_queries=True),
+ "clickhouse",
+ )
+
+ @parameterized.expand(
+ [
+ ("mode_1", FlagEvaluationsMode.READ_FLAG_EVALUATIONS),
+ ("mode_2", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY),
+ ]
+ )
+ # Mode 1 keeps a project on the native-JSON events table reading events, so no swap would load a definition.
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
+ def test_flag_call_views_load_each_property_definition_once_per_query(self, _name: str, mode: FlagEvaluationsMode):
+ for name in ("calls_a", "calls_b"):
+ self._create_view(
+ name,
+ f"SELECT distinct_id, properties.$browser AS browser FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ ("distinct_id", "browser"),
+ )
+
+ with patch(
+ "posthog.hogql.transforms.flag_calls_from_flag_evaluations.load_property_metadata",
+ wraps=load_property_metadata,
+ ) as lookup:
+ results = self._query_events(
+ "SELECT distinct_id FROM calls_a UNION ALL SELECT distinct_id FROM calls_b", mode=mode
+ )
+
+ assert [row[0] for row in results.results] == ["probe-distinct-id", "probe-distinct-id"]
+ loaded = Counter(name for call in lookup.call_args_list for name in call.kwargs["event_property_names"])
+ assert loaded and max(loaded.values()) == 1
+
@parameterized.expand(
[
("string", "variant-a"),
@@ -1325,45 +1597,5 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
assert rewritten.clickhouse is not None
assert ("FROM flag_evaluations" in rewritten.clickhouse) is swaps
- @parameterized.expand(
- [
- ("flag_calls", f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}", "saved_calls"),
- (
- "posthog_events_flag_calls",
- f"SELECT uuid FROM posthog.events WHERE {FLAG_CALL_CONJUNCTS}",
- "saved_calls",
- ),
- ("view_of_a_flag_call_view", "SELECT uuid FROM inner_calls", "inner_calls"),
- ("pageviews", f"SELECT uuid FROM events WHERE {PAGEVIEW_CONJUNCTS}", None),
- (
- "cte_named_events",
- "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "
- f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}",
- None,
- ),
- ]
- )
- def test_view_that_reads_flag_calls_from_events_raises_once_events_gets_no_flag_calls(
- self, _name: str, view_query: str, failing_view: str | None
- ):
- DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="inner_calls",
- query={"query": f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}"},
- columns={"uuid": "String"},
- )
- DataWarehouseSavedQuery.objects.create(
- team=self.team, name="saved_calls", query={"query": view_query}, columns={"uuid": "String"}
- )
- query = "SELECT count() FROM saved_calls"
-
- # events still gets flag calls on mode 1, so the view's read of events is complete.
- self._query_events(query, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS)
- if failing_view is None:
- self._query_events(query)
- return
- with self.assertRaisesRegex(QueryError, f"reads the saved view {failing_view}, which reads them from events"):
- self._query_events(query)
-
def test_flag_call_query_on_events_prints_a_read_of_flag_evaluations(self):
self.assertQueryMatchesSnapshot(self._query_events(FLAG_CALLS_ON_EVENTS).clickhouse)
diff --git a/posthog/hogql/join_sources.py b/posthog/hogql/join_sources.py
new file mode 100644
index 000000000000..2287db12a696
--- /dev/null
+++ b/posthog/hogql/join_sources.py
@@ -0,0 +1,55 @@
+from posthog.hogql import ast
+from posthog.hogql.visitor import TraversingVisitor
+
+
+class JoinSources:
+ """The `JoinExpr` of the input tree that each resolved `JoinExpr` came from, and the joins that the resolver
+ added itself."""
+
+ def __init__(self) -> None:
+ # Each entry keeps its key node alive, so no other node can take its id.
+ self._sources: dict[int, tuple[ast.JoinExpr, ast.JoinExpr]] = {}
+ self._added: dict[int, ast.JoinExpr] = {}
+
+ def record(self, node: ast.JoinExpr, source: ast.JoinExpr) -> None:
+ """Records that `node`, a resolved join or a copy that the resolver made, came from `source`."""
+ self._sources[id(node)] = (node, self.source_of(source) or source)
+
+ def record_copied_tree(self, copy: ast.AST, source: ast.AST) -> None:
+ """Records that each join in `copy`, a copy of `source`, came from the join at the same place in `source`."""
+ for copied, original in zip(_all_joins(copy), _all_joins(source), strict=True):
+ self.record(copied, original)
+
+ def record_added(self, tree: ast.AST) -> None:
+ """Records that the resolver added each join in `tree`, such as one in a saved view's body."""
+ for join in _all_joins(tree):
+ self._added[id(join)] = join
+
+ def source_of(self, node: ast.JoinExpr) -> ast.JoinExpr | None:
+ entry = self._sources.get(id(node))
+ return entry[1] if entry is not None else None
+
+ def is_added(self, node: ast.JoinExpr) -> bool:
+ """Whether the resolver added the join that `node` came from."""
+ source = self.source_of(node)
+ return source is not None and id(source) in self._added
+
+
+def _all_joins(tree: ast.AST) -> list[ast.JoinExpr]:
+ collector = _JoinExprCollector()
+ collector.visit(tree)
+ return collector.joins
+
+
+class _JoinExprCollector(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.joins: list[ast.JoinExpr] = []
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ self.joins.append(node)
+ super().visit_join_expr(node)
+
+ def visit_field(self, node: ast.Field) -> None:
+ # The inherited visit follows the field's type. A field holds no join.
+ pass
diff --git a/posthog/hogql/printer/utils.py b/posthog/hogql/printer/utils.py
index a56f22104729..579f964fa97c 100644
--- a/posthog/hogql/printer/utils.py
+++ b/posthog/hogql/printer/utils.py
@@ -31,7 +31,7 @@ from posthog.hogql.printer.mysql import MySQLPrinter
from posthog.hogql.printer.postgres import PostgresPrinter
from posthog.hogql.printer.redshift import RedshiftPrinter
from posthog.hogql.printer.snowflake import SnowflakePrinter
-from posthog.hogql.resolver import ResolverFactory, resolve_types
+from posthog.hogql.resolver import ResolverFactory, resolve_types, swapping_flag_calls_in_views
from posthog.hogql.transforms.events_predicate_pushdown import apply_events_predicate_pushdown, events_pushdown_enabled
from posthog.hogql.transforms.events_read_in_order import order_events_reads_by_sort_key
from posthog.hogql.transforms.flag_calls_from_flag_evaluations import resolve_flag_calls_from_flag_evaluations
@@ -212,6 +212,10 @@ def prepare_ast_for_printing(
context.direct_postgres_connection_metadata = getattr(context.database, "_direct_connection_metadata", None)
context.modifiers = set_default_in_cohort_via(context.modifiers)
+ if dialect == "clickhouse":
+ # The resolver inlines a saved view's body during type resolution and lazy-table resolution, after the swap of
+ # the top-level query runs.
+ resolver_factory = swapping_flag_calls_in_views(resolver_factory)
if dialect == "trino":
from posthog.hogql.transforms.trino.normalize import ( # noqa: PLC0415 -- load the optional backend only for Trino compilation
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py
index 9ec6f502e2c5..8090d2f73d14 100644
--- a/posthog/hogql/resolver.py
+++ b/posthog/hogql/resolver.py
@@ -60,6 +60,7 @@ from posthog.hogql.functions.traffic_type import (
)
from posthog.hogql.functions.with_icon import with_icon
from posthog.hogql.hogqlx import HOGQLX_COMPONENTS, HOGQLX_TAGS, convert_to_hx
+from posthog.hogql.join_sources import JoinSources
from posthog.hogql.parser import parse_select
from posthog.hogql.resolver_utils import (
available_fields_suffix,
@@ -70,6 +71,7 @@ from posthog.hogql.resolver_utils import (
suggest_field_names,
suggested_field_fix,
)
+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import swap_flag_calls, swaps_flag_calls
from posthog.hogql.transforms.trino.persons import (
lower_trino_table,
resolve_internal_trino_logical_table,
@@ -269,59 +271,6 @@ def resolve_types_from_table(
return resolve_types(expr, context, dialect, [scope])
-class JoinSources:
- """The `JoinExpr` of the input tree that each resolved `JoinExpr` came from, and the joins that the resolver
- added itself."""
-
- def __init__(self) -> None:
- # Each entry keeps its key node alive, so no other node can take its id.
- self._sources: dict[int, tuple[ast.JoinExpr, ast.JoinExpr]] = {}
- self._added: dict[int, ast.JoinExpr] = {}
-
- def record(self, node: ast.JoinExpr, source: ast.JoinExpr) -> None:
- """Records that `node`, a resolved join or a copy that the resolver made, came from `source`."""
- self._sources[id(node)] = (node, self.source_of(source) or source)
-
- def record_copied_tree(self, copy: ast.AST, source: ast.AST) -> None:
- """Records that each join in `copy`, a copy of `source`, came from the join at the same place in `source`."""
- for copied, original in zip(_all_joins(copy), _all_joins(source), strict=True):
- self.record(copied, original)
-
- def record_added(self, tree: ast.AST) -> None:
- """Records that the resolver added each join in `tree`, such as one in a saved view's body."""
- for join in _all_joins(tree):
- self._added[id(join)] = join
-
- def source_of(self, node: ast.JoinExpr) -> ast.JoinExpr | None:
- entry = self._sources.get(id(node))
- return entry[1] if entry is not None else None
-
- def is_added(self, node: ast.JoinExpr) -> bool:
- """Whether the resolver added the join that `node` came from."""
- source = self.source_of(node)
- return source is not None and id(source) in self._added
-
-
-def _all_joins(tree: ast.AST) -> list[ast.JoinExpr]:
- collector = _JoinExprCollector()
- collector.visit(tree)
- return collector.joins
-
-
-class _JoinExprCollector(TraversingVisitor):
- def __init__(self) -> None:
- super().__init__()
- self.joins: list[ast.JoinExpr] = []
-
- def visit_join_expr(self, node: ast.JoinExpr) -> None:
- self.joins.append(node)
- super().visit_join_expr(node)
-
- def visit_field(self, node: ast.Field) -> None:
- # The inherited visit follows the field's type. A field holds no join.
- pass
-
-
ResolverFactory = Callable[
[HogQLContext, HogQLDialect, Optional[list["ast.SelectQueryType"]]],
"Resolver",
@@ -356,6 +305,23 @@ def resolve_types(
return resolver.visit(node)
+def swapping_flag_calls_in_views(resolver_factory: ResolverFactory | None) -> ResolverFactory:
+ """A factory whose resolvers run `swap_flag_calls` on the body of each saved view they
+ inline. Each resolver is otherwise the one `resolver_factory` builds."""
+
+ def factory(
+ context: HogQLContext, dialect: HogQLDialect, scopes: Optional[list[ast.SelectQueryType]]
+ ) -> "Resolver":
+ if resolver_factory is None:
+ resolver = Resolver(scopes=scopes, context=context, dialect=dialect)
+ else:
+ resolver = resolver_factory(context, dialect, scopes)
+ resolver.swap_flag_calls_in_views = True
+ return resolver
+
+ return factory
+
+
def _select_type_columns(
select_type: ast.SelectQueryType | ast.SelectSetQueryType,
) -> list[tuple[str, ast.Type]]:
@@ -505,6 +471,13 @@ class Resolver(CloningVisitor):
self._compilation_budget = compilation_budget.get() or CompilationBudget()
self._resolving_saved_views: set[str] = set()
self._pending_saved_view: str | None = None
+ self.swap_flag_calls_in_views = False
+ # The unresolved body that the flag call swap chose for each view, by the CTEs in scope where it was read. A
+ # CTE's body can change the swap's decision, so the key holds each CTE's text. The text stays the same when
+ # the resolver visits the same CTE again and builds a new node for it.
+ self._flag_call_view_bodies: dict[
+ tuple[str, frozenset[tuple[str, str]]], ast.SelectQuery | ast.SelectSetQuery
+ ] = {}
self.context = context
self.dialect = dialect
self.join_sources: JoinSources | None = None
@@ -1466,6 +1439,53 @@ class Resolver(CloningVisitor):
self.join_sources.record_copied_tree(copy, node)
return copy
+ def _swap_flag_calls_in_view_body(
+ self, body: ast.SelectQuery | ast.SelectSetQuery, view_name: str
+ ) -> ast.SelectQuery | ast.SelectSetQuery:
+ """`body`, unresolved, with each `events` scan of flag calls that the swap accepts reading
+ `posthog.flag_evaluations`. The swap checks the body as a query of its own."""
+ # The enclosing query's CTEs are in scope in the body. The swap reads `FROM events` as the table.
+ if not self.swap_flag_calls_in_views or "events" in self.ctes or not swaps_flag_calls(body, self.context):
+ return body
+ # Each resolve below visits the views that this body reads. Without the cache, each of those views would rerun
+ # its own swap. A chain of views would then cost twice as much at each level.
+ key = (view_name, frozenset((name, cte.expr.to_hogql()) for name, cte in self.ctes.items()))
+ cached = self._flag_call_view_bodies.get(key)
+ if cached is not None:
+ return clone_expr(cached)
+
+ def resolve(
+ query: ast.SelectQuery | ast.SelectSetQuery, *, join_sources: JoinSources | None = None
+ ) -> ast.SelectQuery | ast.SelectSetQuery:
+ # The swap catches a resolution error and keeps the original body. A visit that raises leaves its scope
+ # and CTEs on this resolver, so this function restores them.
+ scopes, ctes, view_depth, cte_counter, outer_join_sources = (
+ list(self.scopes),
+ self.ctes,
+ self.current_view_depth,
+ self.cte_counter,
+ self.join_sources,
+ )
+ # This visit skips the join that marks the view as being resolved, so it marks the view itself. Without
+ # the mark, a view that reads itself would start a new swap of its own body at each level.
+ self._resolving_saved_views.add(view_name)
+ self.current_view_depth += 1
+ # The swap's check reads `join_sources` for the joins of this body. A trial tree is not part of the
+ # enclosing query, so the enclosing query's record does not get its joins.
+ self.join_sources = join_sources
+ try:
+ resolved = self.visit(self._copy(query))
+ finally:
+ self._resolving_saved_views.remove(view_name)
+ self.scopes[:] = scopes
+ self.ctes, self.current_view_depth, self.cte_counter = ctes, view_depth, cte_counter
+ self.join_sources = outer_join_sources
+ return resolved
+
+ chosen_body = swap_flag_calls(body, self.context, resolve, view_name=view_name).source
+ self._flag_call_view_bodies[key] = chosen_body
+ return clone_expr(chosen_body)
+
def visit_join_expr(self, node: ast.JoinExpr):
"""Visit each FROM and JOIN table or subquery."""
@@ -1567,8 +1587,9 @@ class Resolver(CloningVisitor):
)
view_query = str(database_table.query)
self._compilation_budget.expand_view(view_query)
- node.table = parse_select(view_query)
- _mark_view_body(node.table, database_table.name)
+ view_body = parse_select(view_query)
+ _mark_view_body(view_body, database_table.name)
+ node.table = self._swap_flag_calls_in_view_body(view_body, database_table.name)
if self.join_sources is not None:
self.join_sources.record_added(node.table)
node.alias = table_alias or database_table.name
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
index 2daa80dc458b..7542b3fef234 100644
--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -13,8 +13,8 @@ import json
from collections import Counter
from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime, time, timedelta, tzinfo
-from functools import cache, partial
-from typing import TYPE_CHECKING, Protocol
+from functools import partial
+from typing import TYPE_CHECKING, Generic, Protocol
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import structlog
@@ -33,8 +33,8 @@ from posthog.hogql.database.schema.util.where_clause_extractor import (
)
from posthog.hogql.errors import BaseHogQLError, QueryError
from posthog.hogql.functions.mapping import HOGQL_COMPARISON_MAPPING, find_hogql_function
-from posthog.hogql.property_metadata import load_property_metadata
-from posthog.hogql.resolver import JoinSources
+from posthog.hogql.join_sources import JoinSources
+from posthog.hogql.property_metadata import PropertyMetadata, load_property_metadata
from posthog.hogql.utils import is_json_verbatim
from posthog.hogql.visitor import TraversingVisitor, clone_expr
@@ -57,8 +57,15 @@ _FLAG_KEY_PROPERTY = "$feature_flag"
_Predicate = Callable[[ast.Expr], bool]
-class _Resolve(Protocol):
- def __call__(self, node: _T_AST, *, join_sources: JoinSources | None = None) -> _T_AST: ...
+class _Resolve(Protocol[_T_AST]):
+ def __call__(self, node: _T_AST, /, *, join_sources: JoinSources | None = None) -> _T_AST: ...
+
+
+@frozen
+class FlagCallSwap(Generic[_T_AST]):
+ # The unresolved tree that `resolved` came from: the input, or its copy with the swapped scans.
+ source: _T_AST
+ resolved: _T_AST
_MAX_UTC_OFFSET = timedelta(hours=14)
@@ -150,7 +157,40 @@ def can_read_flag_evaluations(
)
-def resolve_flag_calls_from_flag_evaluations(node: _T_AST, context: HogQLContext, resolve: _Resolve) -> _T_AST:
+def swaps_flag_calls(node: ast.AST, context: HogQLContext) -> bool:
+ """Whether `resolve_flag_calls_from_flag_evaluations` can change where `node` reads flag calls from. It reads
+ only the context and the text of `node`, and it resolves nothing."""
+ database = _swap_database(context)
+ return database is not None and _swap_applies(node, context, database)
+
+
+def _swap_database(context: HogQLContext) -> "Database | None":
+ """The database when the context allows the swap, or None."""
+ database = context.database
+ if (
+ not context.rewrite_flag_calls_to_flag_evaluations
+ or database is None
+ or not database.read_flag_calls_from_flag_evaluations
+ ):
+ return None
+ return database
+
+
+def _swap_applies(node: ast.AST, context: HogQLContext, database: "Database") -> bool:
+ if not names_flag_called_event(node):
+ return False
+ return database.flag_calls_only_in_flag_evaluations or not context.uses_new_events_schema()
+
+
+def resolve_flag_calls_from_flag_evaluations(node: _T_AST, context: HogQLContext, resolve: _Resolve[_T_AST]) -> _T_AST:
+ """`node` resolved with `resolve`, with each `events` scan that `swap_flag_calls` accepts reading
+ `posthog.flag_evaluations` instead."""
+ return swap_flag_calls(node, context, resolve).resolved
+
+
+def swap_flag_calls(
+ node: _T_AST, context: HogQLContext, resolve: _Resolve[_T_AST], *, view_name: str | None = None
+) -> FlagCallSwap[_T_AST]:
"""`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading
`posthog.flag_evaluations` instead.
@@ -172,40 +212,36 @@ def resolve_flag_calls_from_flag_evaluations(node: _T_AST, context: HogQLContext
For an organization whose flag calls are only in `flag_evaluations`, a scan that the syntactic rules miss swaps
too when the resolved original shows that its filters pin `event` to flag calls in its WHERE or PREWHERE. The
- query raises when such a scan cannot swap.
+ query raises when such a scan cannot swap. A scan that the resolver adds, such as one in an inline cohort, keeps
+ reading `events`.
- A scan that the resolver adds, such as one in a saved view's body or an inline cohort, keeps reading `events`.
- Once `events` gets no flag calls, a query that reads a view whose body reads only flag calls from `events` raises.
+ The resolver runs this on the body of each saved view that it inlines, and `view_name` names that view for the
+ error. The check of resolved filters skips the joins of a view's body in the enclosing query, because the swap of
+ that body already checked them. A top-level query raises when it reads a view whose resolved body still reads
+ only flag calls from `events`, such as a body that filters with `matchesAction`.
"""
- database = context.database
- if (
- not context.rewrite_flag_calls_to_flag_evaluations
- or database is None
- or not database.read_flag_calls_from_flag_evaluations
- ):
- return resolve(node)
- only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations
- resolved = _resolve_with_swaps(node, context, resolve, database, only_in_flag_evaluations)
- if only_in_flag_evaluations:
- view_name = _view_reading_flag_calls_from_events(resolved)
- if view_name is not None:
+ database = _swap_database(context)
+ if database is None:
+ return FlagCallSwap(source=node, resolved=resolve(node))
+ if _swap_applies(node, context, database):
+ swap = _resolve_with_swaps(node, context, resolve, database, view_name)
+ else:
+ swap = FlagCallSwap(source=node, resolved=resolve(node))
+ # A view's body names flag calls only after resolution when it filters with `matchesAction`, so its swap never
+ # runs. The check reads the whole resolved query, so it runs once, for the top-level query.
+ if database.flag_calls_only_in_flag_evaluations and view_name is None:
+ unswapped_view = _view_reading_flag_calls_from_events(swap.resolved)
+ if unswapped_view is not None:
raise _flag_calls_only_in_flag_evaluations_error(
- f"it reads the saved view {view_name}, which reads them from events"
+ f"it reads the saved view {unswapped_view}, which reads them from events", None
)
- return resolved
+ return swap
def _resolve_with_swaps(
- node: _T_AST,
- context: HogQLContext,
- resolve: _Resolve,
- database: "Database",
- only_in_flag_evaluations: bool,
-) -> _T_AST:
- if not names_flag_called_event(node):
- return resolve(node)
- if not only_in_flag_evaluations and context.uses_new_events_schema():
- return resolve(node)
+ node: _T_AST, context: HogQLContext, resolve: _Resolve[_T_AST], database: "Database", view_name: str | None
+) -> FlagCallSwap[_T_AST]:
+ only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations
with context.timings.measure("flag_calls_from_flag_evaluations"):
rewritten = clone_expr(node)
clones = _join_clones(node, rewritten) if only_in_flag_evaluations else {}
@@ -214,12 +250,12 @@ def _resolve_with_swaps(
timezone=ZoneInfo(database.get_timezone()),
only_in_flag_evaluations=only_in_flag_evaluations,
# The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.
- flag_key_compares_as_string=cache(partial(_flag_key_compares_as_string, context)),
+ flag_key_compares_as_string=partial(_flag_key_compares_as_string, context),
)
swapper.visit(rewritten)
if swapper.refusals:
if only_in_flag_evaluations:
- raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0])
+ raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0], view_name)
logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=swapper.refusals[0])
sources = JoinSources() if only_in_flag_evaluations else None
original = resolve(node, join_sources=sources)
@@ -227,9 +263,9 @@ def _resolve_with_swaps(
with context.timings.measure("flag_calls_from_flag_evaluations_by_type"):
refusal = swapper.swap_scans_found_by_type(original, sources, clones)
if refusal is not None:
- raise _flag_calls_only_in_flag_evaluations_error(refusal)
+ raise _flag_calls_only_in_flag_evaluations_error(refusal, view_name)
if not swapper.swapped:
- return original
+ return FlagCallSwap(source=node, resolved=original)
try:
swapped = resolve(rewritten)
except BaseHogQLError as error:
@@ -243,12 +279,12 @@ def _resolve_with_swaps(
elif not only_in_flag_evaluations and not _swap_reads_properties_alike(original, swapped, context):
reason = "it reads a property from a different kind of column there"
else:
- return swapped
+ return FlagCallSwap(source=rewritten, resolved=swapped)
if only_in_flag_evaluations:
# events gets no new flag calls, so the original query would return only calls from before the move.
- raise _flag_calls_only_in_flag_evaluations_error(reason)
+ raise _flag_calls_only_in_flag_evaluations_error(reason, view_name)
logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=reason)
- return original
+ return FlagCallSwap(source=node, resolved=original)
def _flag_key_compares_as_string(context: HogQLContext) -> bool:
@@ -256,16 +292,41 @@ def _flag_key_compares_as_string(context: HogQLContext) -> bool:
property definitions type as Boolean, Numeric or DateTime before it compares the value."""
if context.team_id is None:
return True
- metadata = load_property_metadata(
- context, event_property_names={_FLAG_KEY_PROPERTY}, person_property_names=set(), group_property_names={}
- )
+ metadata = _event_property_metadata(context, {_FLAG_KEY_PROPERTY})
return metadata.event_properties.get(_FLAG_KEY_PROPERTY, {}).get("type") in (None, "String")
-def _flag_calls_only_in_flag_evaluations_error(reason: str) -> QueryError:
+@frozen
+class FlagCallPropertyMetadata:
+ names: frozenset[str]
+ metadata: PropertyMetadata
+
+
+def _event_property_metadata(context: HogQLContext, names: set[str]) -> PropertyMetadata:
+ """The definitions of event properties `names`, and of every event property this query loaded before them."""
+ cached = context.flag_call_property_metadata
+ loaded_names = cached.names if cached is not None else frozenset()
+ missing = names - loaded_names
+ if cached is not None and not missing:
+ return cached.metadata
+ loaded = load_property_metadata(
+ context, event_property_names=missing, person_property_names=set(), group_property_names={}
+ )
+ if cached is not None:
+ loaded = PropertyMetadata(
+ event_properties={**cached.metadata.event_properties, **loaded.event_properties},
+ # The registry loader caches its fetch, so keeping the first one fetches the registry once per query.
+ materialized_columns=cached.metadata.materialized_columns,
+ )
+ context.flag_call_property_metadata = FlagCallPropertyMetadata(names=loaded_names | missing, metadata=loaded)
+ return loaded
+
+
+def _flag_calls_only_in_flag_evaluations_error(reason: str, view_name: str | None) -> QueryError:
+ subject, part = ("this query", "query") if view_name is None else (f"the view {view_name}", "view")
return QueryError(
- f"Your organization's {FLAG_CALLED_EVENT} events are only in posthog.flag_evaluations, and this query "
- f"can't read them there because {reason}. Change that part of the query, or query "
+ f"Your organization's {FLAG_CALLED_EVENT} events are only in posthog.flag_evaluations, and {subject} "
+ f"can't read them there because {reason}. Change that part of the {part}, or query "
"posthog.flag_evaluations directly."
)
@@ -1165,11 +1226,8 @@ def _swap_reads_properties_alike(original: ast.AST, swapped: ast.AST, context: H
return False
previous_metadata = context.property_metadata
# The dmat slots come from the team's property definitions, which the printer loads only after this transform.
- context.property_metadata = load_property_metadata(
- context,
- event_property_names={name for column, name, _ in original_reads if column == "properties"},
- person_property_names=set(),
- group_property_names={},
+ context.property_metadata = _event_property_metadata(
+ context, {name for column, name, _ in original_reads if column == "properties"}
)
try:
return Counter(_property_sources(original_reads, context)) == Counter(_property_sources(swapped_reads, context))
diff --git a/posthog/models/test/test_data_deletion_request.py b/posthog/models/test/test_data_deletion_request.py
index 67116ebcae9b..11a01be655cd 100644
--- a/posthog/models/test/test_data_deletion_request.py
+++ b/posthog/models/test/test_data_deletion_request.py
@@ -29,6 +29,7 @@ from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_SOURCE_EVENT
from posthog.models.organization import Organization
from posthog.models.team import Team
+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.event_definitions.backend.models import PropertyDefinition
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
@@ -208,21 +209,31 @@ def test_hogql_predicate_subquery_is_team_scoped(db):
@pytest.mark.parametrize(
- "predicate",
+ "flag_calls",
[
- "distinct_id IN (SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
- "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY)",
- "distinct_id IN (SELECT e.distinct_id FROM events AS e JOIN (SELECT 'probe' AS id) AS d "
- "ON e.distinct_id = d.id WHERE event = '$feature_flag_called')",
+ "SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY",
+ "SELECT e.distinct_id FROM events AS e JOIN (SELECT 'probe' AS id) AS d "
+ "ON e.distinct_id = d.id WHERE event = '$feature_flag_called'",
],
)
-def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, predicate):
[email protected]("through_a_view", [False, True])
+def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team, flag_calls, through_a_view):
from posthog.models.data_deletion_request import compile_hogql_predicate
OrganizationFeatureFlagsConfig.objects.update_or_create(
organization=team.organization,
defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},
)
+ if through_a_view:
+ DataWarehouseSavedQuery.objects.create(
+ team=team,
+ name="flag_calls",
+ query={"kind": "HogQLQuery", "query": flag_calls},
+ columns={"distinct_id": "String"},
+ )
+ flag_calls = "SELECT distinct_id FROM flag_calls"
+ predicate = f"distinct_id IN ({flag_calls})"
request = DataDeletionRequest(**_base_kwargs(team_id=team.id, events=["$pageview"], hogql_predicate=predicate))
with patch(