:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): read flag calls from flag_evaluations when a query asks events for them (#124397)
Phil Haack8e9e167cd35d
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.mdindex 7c8d4f3fdd3f..a0f93d8a0a91 100644--- a/posthog/hogql/ACCESS_CONTROL.md+++ b/posthog/hogql/ACCESS_CONTROL.md@@ -230,7 +230,7 @@ Several catalog tables expose such columns because reading them scans far less d`events`' mirror columns are not masked: restricting the property they mirror masks the blob read and leaves the column readable.`flag_evaluations` and `ai_events` mirror columns are masked through `mirrored_property_for_column()` in `posthog/hogql/restricted_properties.py`, consulted by `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.-The mapping is keyed by the stored column name, not the HogQL field name, so an expression field that reads a mirror column masks too: `flag_evaluations.response` reads the stored `response` column through `nullIf`, and the hidden `flag_evaluation_response` reads it bare, and both mask.+The mapping is keyed by the stored column name, not the HogQL field name, so an expression field that reads a mirror column masks too: `flag_evaluations.response` reads the stored `response` column through `nullIf`, and the hidden `flag_evaluation_response` reads it bare, and both mask. The hidden `goe_N.key` on `flag_evaluations` reads the stored `$group_N` column through an expression field, and it masks too.For AI columns, `AI_PROPERTY_TO_COLUMN` in `posthog/hogql/database/schema/ai_events.py` is shared with the query rewriter.One rule for `$ai_input` masks both `events.properties.$ai_input` and `ai_events.input`, including nested reads such as `input[1].content`.The same rule applies when a filter or an aggregate reads the column.diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.pyindex 45f0355e9bf4..fde4e3546b44 100644--- a/posthog/hogql/ast.py+++ b/posthog/hogql/ast.py@@ -1391,12 +1391,16 @@ class RatioExpr(Expr):right: Optional[Constant] = None-@dataclass(kw_only=True, slots=True)+@dataclass(kw_only=True, slots=True, frozen=False)class SampleExpr(Expr):# k or nsample_value: RatioExproffset_value: Optional[RatioExpr] = None+ def samples_all_rows(self) -> bool:+ """`SAMPLE 1` with no denominator and no offset reads every row. The printers drop it."""+ return self.sample_value.left.value == 1 and self.sample_value.right is None and self.offset_value is None+@dataclass(kw_only=True, slots=True)class HogQLXAttribute(AST):diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.pyindex f2e2b99d57be..9e158f2b03b9 100644--- a/posthog/hogql/constants.py+++ b/posthog/hogql/constants.py@@ -30,6 +30,10 @@ INACTIVE_FEATURE_FLAG_VALUES: tuple[str, ...] = ("", "false")FEATURE_FLAG_PROPERTY_PREFIX = "$feature/"+# posthog/models/flag_evaluations/sql.py defines the same value as FLAG_EVALUATIONS_SOURCE_EVENT. That module imports+# django.conf, which HogQL keeps off its import path, so the value is spelled out here.+FLAG_CALLED_EVENT = "$feature_flag_called"+def is_virtual_feature_flag_key(key: str) -> bool:"""Whether an events_json property is rebuilt from the `$feature_flags` map instead of read under its own name."""diff --git a/posthog/hogql/context.py b/posthog/hogql/context.pyindex a5c84c42b99d..81ea1073531b 100644--- a/posthog/hogql/context.py+++ b/posthog/hogql/context.py@@ -172,6 +172,9 @@ class HogQLContext:# regardless of retention — notably the GDPR data-deletion mutation path — set this False. Deliberately NOT a# HogQLQueryModifier, so a query can't disable enforcement.apply_events_retention_floor: bool = True+ # Backend-only switch for transforms/flag_calls_from_flag_evaluations.py. The data-deletion path sets it False, so a+ # flag-call subquery in a deletion predicate keeps reading `events` whatever the organization's mode.+ rewrite_flag_calls_to_flag_evaluations: bool = True# Backend-only opt-in for transforms/events_read_in_order.py. The prefix is slower when the range is short or# the filter is selective, so set it only for a query shape that was measured to be faster with it.diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.pyindex d65f4a15146e..29f93cc1650e 100644--- a/posthog/hogql/database/database.py+++ b/posthog/hogql/database/database.py@@ -251,6 +251,9 @@ class HogQLDatabaseSources:is_billing_usage_records_enabled: bool# Not the flag's value: a direct-connection catalog excludes the table whatever the org has.include_flag_evaluations_table: bool+ read_flag_calls_from_flag_evaluations: bool+ # The organization reads flag calls only from posthog.flag_evaluations, and ingestion no longer writes them to events.+ flag_calls_only_in_flag_evaluations: bool# Userless internal contexts that must resolve every warehouse table/view; skips access controlbypass_warehouse_access_control: booldirect_connection_metadata: dict[str, Any] | None@@ -454,7 +457,7 @@ _DATABASE_ROOT_NODE_BLOBS_LOCK = threading.Lock()# LRU, and fresh entries survive the sweep. Reads are lock-free (dict.get is atomic under the# GIL); mutations and the sweep hold the lock, because the sweep iterates the dict and concurrent# inserts would raise RuntimeError mid-iteration.-_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets"})+_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets", "hogql-flag-calls-from-flag-evaluations"})_TEAM_FLAG_CACHE: dict[tuple[str, str], tuple[float, bool]] = {} # key -> (evaluated_at, value)_TEAM_FLAG_CACHE_LOCK = threading.Lock()_TEAM_FLAG_CACHE_MAX_ENTRIES = 50_000@@ -476,6 +479,16 @@ def _evaluate_warehouse_access_control_flag(team: Team) -> bool:)+def _evaluate_flag_calls_rewrite_flag(team: Team) -> bool:+ return feature_enabled_or_false(+ "hogql-flag-calls-from-flag-evaluations",+ str(team.uuid),+ groups={"organization": str(team.organization_id)},+ group_properties={"organization": {"id": str(team.organization_id)}},+ send_feature_flag_events=False,+ )++def _cached_team_flag(flag_key: str, team: Team, evaluate: Callable[[], bool]) -> bool:# Not cache_for/CachedFunction: its TTL is fixed at decoration time, and this TTL is a live# instance setting so ops can retune or disable the cache without a redeploy.@@ -829,6 +842,8 @@ class Database(BaseModel):# because a warehouse table merged into the root first can hold the bare stored name._models_alias_nodes: dict[str, TableNode] = {}_connection_id: str | None = None+ _read_flag_calls_from_flag_evaluations: bool = False+ _flag_calls_only_in_flag_evaluations: bool = False_direct_connection_metadata: dict[str, Any] | None = None_direct_access_warehouse_table_names: set[str] = set()# Warnings about data warehouse tables (failed/paused/billing-limited/stale syncs),@@ -890,6 +905,8 @@ class Database(BaseModel):self._models_aliases = {}self._models_alias_nodes = {}self._connection_id = None+ self._read_flag_calls_from_flag_evaluations = False+ self._flag_calls_only_in_flag_evaluations = Falseself._direct_connection_metadata = Noneself._direct_access_warehouse_table_names = set()self._data_warehouse_sync_warnings = {}@@ -1259,6 +1276,15 @@ class Database(BaseModel):def _is_direct_query(self) -> bool:return self._connection_id is not None+ @property+ def read_flag_calls_from_flag_evaluations(self) -> bool:+ # create_for(include_flag_evaluations_table=False) removes the table after _fetch_sources chose to read from it.+ return self._read_flag_calls_from_flag_evaluations and self.has_table(["posthog", "flag_evaluations"])++ @property+ def flag_calls_only_in_flag_evaluations(self) -> bool:+ return self._flag_calls_only_in_flag_evaluations and self.read_flag_calls_from_flag_evaluations+@staticmethoddef _is_helper_function_table(table: object) -> bool:return isinstance(table, FunctionCallTable) and not isinstance(table, (DirectSQLTable, PostgresTable, S3Table))@@ -1937,6 +1963,8 @@ class Database(BaseModel):# Resolving the org gate needs a Postgres read. This path exists to avoid that I/O,# so it prunes the table as it prunes the other gated tables.include_flag_evaluations_table=False,+ read_flag_calls_from_flag_evaluations=False,+ flag_calls_only_in_flag_evaluations=False,group_types=[],saved_queries=[],endpoint_saved_queries=[],@@ -2032,10 +2060,23 @@ class Database(BaseModel):)# Function-local + facade-only: keeps the products off the django.setup() path.- from products.feature_flags.backend.facade.flags import is_flag_evaluations_table_enabled # noqa: PLC0415+ from products.feature_flags.backend.facade.enums import FlagEvaluationsMode # noqa: PLC0415+ from products.feature_flags.backend.facade.flags import ( # noqa: PLC0415+ get_flag_evaluations_read_mode,+ is_flag_evaluations_table_enabled,+ )# A direct-connection catalog has no "posthog" node to hold the table.include_flag_evaluations_table = not is_direct_query and is_flag_evaluations_table_enabled(team)+ # This checks the read mode, not is_flag_evaluations_table_enabled. That one is true under DEBUG for an+ # organization still on EVENTS. The flag goes first because it reads no row.+ read_mode: int = FlagEvaluationsMode.EVENTS+ if include_flag_evaluations_table and _cached_team_flag(+ "hogql-flag-calls-from-flag-evaluations", team, lambda: _evaluate_flag_calls_rewrite_flag(team)+ ):+ read_mode = get_flag_evaluations_read_mode(team.organization_id)+ read_flag_calls_from_flag_evaluations = read_mode != FlagEvaluationsMode.EVENTS+ flag_calls_only_in_flag_evaluations = read_mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLYwith timings.measure("database", emit_span=True):# Function-local: keeps the direct-SQL driver imports off the django.setup() path.@@ -2300,6 +2341,8 @@ class Database(BaseModel):is_billing_usage_records_enabled="*" in settings.BILLING_USAGE_RECORDS_HOGQL_ORGANIZATION_IDSor team.organization_id in settings.BILLING_USAGE_RECORDS_HOGQL_ORGANIZATION_IDS,include_flag_evaluations_table=include_flag_evaluations_table,+ read_flag_calls_from_flag_evaluations=read_flag_calls_from_flag_evaluations,+ flag_calls_only_in_flag_evaluations=flag_calls_only_in_flag_evaluations,# Managed warehouse is a built-in project datastore and has no warehouse-object ACL surface.# Principals that skip warehouse access control by design:# - synthetic users (project-wide service tokens, bypass object-level RBAC)@@ -2353,6 +2396,8 @@ class Database(BaseModel):week_start_day=team.week_start_day,include_posthog_tables=sources.connection_id is None,)+ database._read_flag_calls_from_flag_evaluations = sources.read_flag_calls_from_flag_evaluations+ database._flag_calls_only_in_flag_evaluations = sources.flag_calls_only_in_flag_evaluationsif sources.connection_id is not None:database._connection_id = sources.connection_idif sources.direct_connection_metadata is not None:@@ -3151,7 +3196,10 @@ def _setup_group_key_fields(database: Database, group_types: list[dict[str, Any]- if(timestamp < mapping.created_at, '', $group_N) if GroupTypeMapping exists"""group_mappings = {mapping["group_type_index"]: mapping for mapping in group_types}- table = database.get_table("events")+ tables = [database.get_table("events")]+ # A flag call query on events can read flag_evaluations instead, so its group keys take the same cutoff.+ if database.has_table(["posthog", "flag_evaluations"]):+ tables.append(database.get_table(["posthog", "flag_evaluations"]))for group_index in range(5):field_name = f"$group_{group_index}"@@ -3159,11 +3207,6 @@ def _setup_group_key_fields(database: Database, group_types: list[dict[str, Any]group_mapping = group_mappings.get(group_index, None)# If no mapping exists or the mapping predated this feature, leave the original field unchangedif group_mapping and group_mapping["created_at"]:- # Store the original field as a "raw" version before replacing- original_field = table.fields[field_name]- raw_field_name = f"_{field_name}_raw"- table.fields[raw_field_name] = original_field.model_copy(update={"hidden": True})-# Must stay a datetime constant: a naive string literal would be parsed by ClickHouse in the# project's timezone (the comparison is against toTimeZone(timestamp, <project tz>)), shifting# the cutoff by the project's UTC offset.@@ -3171,22 +3214,29 @@ def _setup_group_key_fields(database: Database, group_types: list[dict[str, Any]if created_at.tzinfo is None:created_at = created_at.replace(tzinfo=UTC)- table.fields[field_name] = ExpressionField(- name=field_name,- expr=ast.Call(- name="if",- args=[- ast.CompareOperation(- left=ast.Field(chain=["timestamp"]),- op=ast.CompareOperationOp.Lt,- right=ast.Constant(value=created_at),- ),- ast.Constant(value=""),- ast.Field(chain=[raw_field_name]),- ],- ),- isolate_scope=True,- )+ for table in tables:+ # Store the original field as a "raw" version before replacing+ original_field = table.fields[field_name]+ raw_field_name = f"_{field_name}_raw"+ table.fields[raw_field_name] = original_field.model_copy(update={"hidden": True})++ table.fields[field_name] = ExpressionField(+ name=field_name,+ expr=ast.Call(+ name="if",+ args=[+ ast.CompareOperation(+ left=ast.Field(chain=["timestamp"]),+ op=ast.CompareOperationOp.Lt,+ right=ast.Constant(value=created_at),+ ),+ ast.Constant(value=""),+ ast.Field(chain=[raw_field_name]),+ ],+ ),+ isolate_scope=True,+ description=original_field.description,+ )def _use_virtual_fields(database: Database, modifiers: HogQLQueryModifiers, timings: HogQLTimings) -> None:diff --git a/posthog/hogql/database/schema/flag_evaluations.py b/posthog/hogql/database/schema/flag_evaluations.pyindex 0d574ef171c4..174d2aa087fe 100644--- a/posthog/hogql/database/schema/flag_evaluations.py+++ b/posthog/hogql/database/schema/flag_evaluations.py@@ -1,7 +1,7 @@from typing import TYPE_CHECKINGfrom posthog.hogql import ast-from posthog.hogql.database.lazy_join_tags import GROUP_N, PERSON_DISTINCT_ID_OVERRIDES, PERSONS+from posthog.hogql.database.lazy_join_tags import GROUP_N, PERSON_DISTINCT_ID_OVERRIDES, PERSON_DISTINCT_IDS, PERSONSfrom posthog.hogql.database.models import (DateTimeDatabaseField,ExpressionField,@@ -16,6 +16,7 @@ from posthog.hogql.database.models import (VirtualTable,)from posthog.hogql.database.schema.person_distinct_id_overrides import PersonDistinctIdOverridesTable+from posthog.hogql.database.schema.person_distinct_ids import PersonDistinctIdsTablefrom posthog.hogql.database.schema.util.null_sentinels import null_if_sentinelsfrom posthog.hogql.database.schema.util.uuid import uuid_string_expr_to_uint128_exprfrom posthog.hogql.parser import parse_expr@@ -77,6 +78,32 @@ class FlagEvaluationsPersonSubTable(VirtualTable):return FLAG_EVALUATIONS_CLICKHOUSE_TABLE+class FlagEvaluationsGroupKeySubTable(VirtualTable):+ """`goe_N` on events, limited to the group key. The row stores no group properties or creation time at+ evaluation time, so `goe_N.properties` and `goe_N.created_at` do not resolve here."""++ def __init__(self, group_index: int):+ super().__init__(+ fields={+ "key": ExpressionField(+ name="key", expr=ast.Field(chain=[f"_$group_{group_index}_raw"]), isolate_scope=True+ )+ }+ )++ def to_printed_clickhouse(self, context):+ return FLAG_EVALUATIONS_CLICKHOUSE_TABLE++ def to_printed_hogql(self):+ return FLAG_EVALUATIONS_CLICKHOUSE_TABLE+++# The events element columns that this table defines as constants, grouped by the type of the constant.+_ELEMENT_STRING_STAND_INS = ("elements_chain", "elements_chain_href")+_ELEMENT_ARRAY_STAND_INS = ("elements_chain_texts", "elements_chain_ids", "elements_chain_elements")+ELEMENT_STAND_INS = frozenset(_ELEMENT_STRING_STAND_INS + _ELEMENT_ARRAY_STAND_INS)++class FlagEvaluationsTable(Table):description: str = ("One row per feature flag evaluation, from the `$feature_flag_called` event. Rows are kept for 90 days. "@@ -198,6 +225,12 @@ class FlagEvaluationsTable(Table):"$group_2": StringDatabaseField(name="$group_2", nullable=False, description="Key of the type-2 group."),"$group_3": StringDatabaseField(name="$group_3", nullable=False, description="Key of the type-3 group."),"$group_4": StringDatabaseField(name="$group_4", nullable=False, description="Key of the type-4 group."),+ # The stored keys, which goe_N.key reads, as events reads goe_N.key from its stored column. A team's group+ # type mappings can replace $group_N with a key that is empty for calls from before the mapping existed.+ **{+ f"_$group_{index}_raw": StringDatabaseField(name=f"$group_{index}", nullable=False, hidden=True)+ for index in range(GROUP_TYPES_LIMIT)+ },# A group property filter, such as a team's test account filter, compiles to `group_N.properties`.# These joins let it resolve here as it does on events.**{@@ -210,11 +243,26 @@ class FlagEvaluationsTable(Table):)for index in range(GROUP_TYPES_LIMIT)},+ # The events names below let a query on events that reads flag calls resolve here when HogQL swaps the table.+ "$window_id": ExpressionField(+ name="$window_id",+ expr=ast.Call(name="ifNull", args=[ast.Field(chain=["properties", "$window_id"]), ast.Constant(value="")]),+ isolate_scope=True,+ hidden=True,+ ),+ "event_person_id": UUIDDatabaseField(name="person_id", hidden=True),+ "pdi": LazyJoin(+ from_field=["distinct_id"],+ join_table=PersonDistinctIdsTable(),+ resolver=PERSON_DISTINCT_IDS,+ hidden=True,+ ),+ **{f"goe_{index}": FlagEvaluationsGroupKeySubTable(group_index=index) for index in range(GROUP_TYPES_LIMIT)},# The table stores no elements, and a flag call on events has none either. An element filter therefore# resolves here and matches nothing, as it does on events. The empty array keeps its String element type.**{name: ExpressionField(name=name, expr=ast.Constant(value=""), hidden=True)- for name in ("elements_chain", "elements_chain_href")+ for name in _ELEMENT_STRING_STAND_INS},**{name: ExpressionField(@@ -224,7 +272,7 @@ class FlagEvaluationsTable(Table):),hidden=True,)- for name in ("elements_chain_texts", "elements_chain_ids", "elements_chain_elements")+ for name in _ELEMENT_ARRAY_STAND_INS},}diff --git a/posthog/hogql/database/schema/groups.py b/posthog/hogql/database/schema/groups.pyindex 9802d3d2129b..bea2bfac4478 100644--- a/posthog/hogql/database/schema/groups.py+++ b/posthog/hogql/database/schema/groups.py@@ -211,9 +211,12 @@ LAZY_JOIN_ALIASES = frozenset()-# The outer FROM tables whose WHERE the groups subquery can reuse. The last part of a chain is the only alias the-# outer FROM may carry.-_PREFILTER_SOURCE_CHAINS: frozenset[tuple[str, ...]] = frozenset({("events",), ("posthog", "flag_evaluations")})+# The outer FROM tables whose WHERE the groups subquery can reuse, with the aliases each may carry besides the last+# part of its chain. The flag call rewrite reads flag_evaluations under the alias `events`.+_PREFILTER_SOURCE_ALIASES: dict[tuple[str, ...], frozenset[str]] = {+ ("events",): frozenset(),+ ("posthog", "flag_evaluations"): frozenset({"events"}),+}def _guarded_aliases(context: HogQLContext, source_chain: list[str]) -> frozenset[str]:@@ -238,9 +241,9 @@ def _outer_prefilter(Extract a clone of the outer query's WHERE that we can safely embed inside the groupsjoin subquery, together with the FROM that the subquery reads. We only return it when:- 1. The outer's `FROM` is a table in `_PREFILTER_SOURCE_CHAINS`, either unaliased or aliased- by the last part of its chain. Anything else (custom alias like `FROM events AS e` in funnels,- or a subquery) means the WHERE may reference symbols+ 1. The outer's `FROM` is a table in `_PREFILTER_SOURCE_ALIASES`, either unaliased or aliased+ by the last part of its chain or by an alias listed for it. Anything else (custom alias like+ `FROM events AS e` in funnels, or a subquery) means the WHERE may reference symbols(`e.timestamp`, …) that don't exist in our inner `SELECT $group_N FROM <source table>`subquery — cloning a WHERE that names them would raise `Unable to resolve field` when theresolver walks the clone.@@ -275,17 +278,20 @@ def _outer_prefilter(if not isinstance(table, ast.Field):return Nonesource_chain = tuple(str(part) for part in table.chain)- if source_chain not in _PREFILTER_SOURCE_CHAINS:+ if source_chain not in _PREFILTER_SOURCE_ALIASES:return None- if select_from.alias is not None and select_from.alias != source_chain[-1]:+ alias = select_from.alias or source_chain[-1]+ if alias != source_chain[-1] and alias not in _PREFILTER_SOURCE_ALIASES[source_chain]:return None- if get_table_alias_for_lazy_join(join_to_add.lazy_join_type) != source_chain[-1]:+ if get_table_alias_for_lazy_join(join_to_add.lazy_join_type) != alias:return Noneif not _references_timestamp(where):return Noneguarded_aliases = _guarded_aliases(context, list(source_chain))- if any(_chain_hits_guarded_alias(chain, guarded_aliases, source_chain) for chain in find_field_chains(where)):+ if any(+ _chain_hits_guarded_alias(chain, guarded_aliases, source_chain, alias) for chain in find_field_chains(where)+ ):return Noneif select_from.type is None or _references_another_table(where, select_from.type):return None@@ -293,11 +299,11 @@ def _outer_prefilter(return clone_expr(where), ast.JoinExpr(table=ast.Field(chain=list(table.chain)), alias=select_from.alias)-def _chain_hits_guarded_alias(chain: list, guarded: frozenset[str], source_chain: tuple[str, ...]) -> bool:+def _chain_hits_guarded_alias(chain: list, guarded: frozenset[str], source_chain: tuple[str, ...], alias: str) -> bool:# Condition #1 guarantees the outer FROM's qualifier is the source table, so a leading qualifier is# transparent: `events.organization.x` names the same lazy join as `organization.x`. Peel it before matching,- # or the qualified form slips past the guard. The qualifier is the full chain or its last part.- for qualifier in (list(source_chain), [source_chain[-1]]):+ # or the qualified form slips past the guard. The qualifier is the full chain, its last part, or the FROM alias.+ for qualifier in (list(source_chain), [source_chain[-1]], [alias]):if len(chain) > len(qualifier) and chain[: len(qualifier)] == qualifier:chain = chain[len(qualifier) :]breakdiff --git a/posthog/hogql/database/schema/test/__snapshots__/test_flag_evaluations.ambr b/posthog/hogql/database/schema/test/__snapshots__/test_flag_evaluations.ambrnew file mode 100644index 000000000000..1088fe553e84--- /dev/null+++ b/posthog/hogql/database/schema/test/__snapshots__/test_flag_evaluations.ambr@@ -0,0 +1,20 @@+# serializer version: 1+# name: TestFlagEvaluationsTable.test_flag_call_query_on_events_prints_a_read_of_flag_evaluations+ '''+ SELECT count() AS `count()`+ FROM flag_evaluations AS events+ WHERE and(equals(events.team_id, 99999), and(equals(events.event, %(hogql_val_0)s), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0), greaterOrEquals(events.timestamp, toDateTime64(minus(now64(6, %(hogql_val_3)s), toIntervalDay(7)), 6, %(hogql_val_4)s))), in(events.flag_key, tuple(%(hogql_val_5)s)))+ LIMIT 100 SETTINGS readonly=2,+ max_execution_time=60,+ allow_experimental_object_type=1,+ max_ast_elements=4000000,+ max_expanded_ast_elements=4000000,+ max_bytes_before_external_group_by=0,+ transform_null_in=1,+ optimize_min_equality_disjunction_chain_length=4294967295,+ optimize_rewrite_aggregate_function_with_if=0,+ optimize_min_inequality_conjunction_chain_length=4294967295,+ allow_experimental_join_condition=1,+ use_hive_partitioning=0+ '''+# ---diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.pyindex 412836b156d4..937f82deeb6b 100644--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py@@ -2,19 +2,32 @@ import jsonimport uuidfrom datetime import UTC, datetime, timedelta-from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events+from posthog.test.base import (+ BaseTest,+ ClickhouseTestMixin,+ _create_event,+ cleanup_materialized_columns,+ flush_persons_and_events,+)from unittest.mock import patch+from django.conf import settings+from django.test import override_settings+from parameterized import parameterizedfrom posthog.schema import (DataWarehousePropertyFilter,+ DateRange,ElementPropertyFilter,EventPropertyFilter,GroupPropertyFilter,+ HogQLFilters,HogQLPropertyFilter,HogQLQueryModifiers,+ HogQLQueryResponse,PersonPropertyFilter,+ PersonsOnEventsMode,PropertyGroupsMode,PropertyOperator,SessionPropertyFilter,@@ -25,14 +38,17 @@ from posthog.hogql.context import HogQLContextfrom posthog.hogql.database.database import Databasefrom posthog.hogql.database.schema.flag_evaluations import add_events_list_fields_to_flag_evaluationsfrom posthog.hogql.database.schema.util.where_clause_extractor import SESSION_BUFFER_DAYS+from posthog.hogql.errors import QueryErrorfrom posthog.hogql.property import property_to_exprfrom posthog.hogql.query import execute_hogql_query+from posthog.hogql.utils import is_json_verbatimfrom posthog.clickhouse.client import sync_executefrom posthog.constants import AvailableFeaturefrom posthog.hogql_queries.events_query_runner import SELECT_STAR_FROM_EVENTS_FIELDSfrom posthog.models import PropertyDefinitionfrom posthog.models.group.util import create_group+from posthog.models.property import TableColumnfrom posthog.models.utils import uuid7from posthog.test.persons import create_group_type_mapping, create_person@@ -40,12 +56,23 @@ from products.access_control.backend.models.property_access_control import Propefrom products.access_control.backend.property_access_control import PropertyAccessLevelfrom products.cohorts.backend.models.cohort import Cohortfrom products.cohorts.backend.models.util import insert_static_cohort+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig++from ee.clickhouse.materialized_columns.columns import materializeEVENT_PROBE = "from-event-properties"# The sessions tables join only on a UUIDv7 session id.SESSION_ID = str(uuid7())+FLAG_CALL_CONJUNCTS = (+ "event = '$feature_flag_called' AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY"+)+FLAG_CALLS_ON_EVENTS = f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS}"+PAGEVIEW_CONJUNCTS = "event = '$pageview' AND timestamp >= now() - INTERVAL 7 DAY"+# Compares greater than 9 as a number and less than it as a string, so the assertion tells the two# apart rather than passing under either.NUMERIC_PROBE = 10@@ -109,12 +136,18 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):"$session_id": SESSION_ID,"$feature_flag_request_id": "req-456","$group_0": "acme",+ "$window_id": "probe-window","probe": EVENT_PROBE,"count": NUMERIC_PROBE,},)- def _insert_row(self, row_uuid: uuid.UUID, properties: dict[str, str | int | None]) -> None:+ def _insert_row(+ self,+ row_uuid: uuid.UUID,+ properties: dict[str, str | int | None],+ person_properties: dict[str, str | int] | None = None,+ ) -> None:# The nine typed columns are DEFAULT-computed from `properties` on the sharded table, so a# producer cannot write them and this insert must not name them.sync_execute(@@ -133,7 +166,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):"probe-distinct-id",self.occurred_at,str(self.person_id),- json.dumps({"email": "[email protected]", "plan": "free", "age": NUMERIC_PROBE}),+ json.dumps(+ person_properties+ if person_properties is not None+ else {"email": "[email protected]", "plan": "free", "age": NUMERIC_PROBE}+ ),)],)@@ -175,6 +212,24 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):):return self._execute(columns, where, row_uuids=row_uuids, placeholders=placeholders).results+ def _query_events(+ self,+ query: str,+ *,+ rewrite: bool = True,+ filters: HogQLFilters | None = None,+ modifiers: HogQLQueryModifiers | None = None,+ mode: FlagEvaluationsMode = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,+ ) -> HogQLQueryResponse:+ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=self.organization, defaults={"flag_evaluations_mode": mode}+ )+ with patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: rewrite and key == "hogql-flag-calls-from-flag-evaluations",+ ):+ return execute_hogql_query(query, team=self.team, filters=filters, modifiers=modifiers, pretty=False)+def _override(self, person_id: uuid.UUID, version: int, is_deleted: bool = False) -> None:sync_execute("INSERT INTO person_distinct_id_overrides (team_id, distinct_id, person_id, version, is_deleted) VALUES",@@ -201,7 +256,8 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):def test_every_field_reads_its_own_physical_column(self):results = self._select(- "flag_key, response, session_id, request_id, `$group_0`, person_id, person.id, person_mode, properties.probe"+ "flag_key, response, session_id, request_id, `$group_0`, goe_0.key, `$window_id`, person_id, person.id, "+ "person_mode, properties.probe")assert results == [@@ -211,6 +267,8 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):SESSION_ID,"req-456","acme",+ "acme",+ "probe-window",self.person_id,self.person_id,"full",@@ -298,7 +356,9 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):merged_person_id = uuid.uuid4()self._override(merged_person_id, version=1)- assert self._select("person_id, person.id") == [(merged_person_id, merged_person_id)]+ assert self._select("person_id, person.id, event_person_id") == [+ (merged_person_id, merged_person_id, self.person_id)+ ]def test_person_id_follows_the_latest_override_version(self):# A distinct_id collects one override row per merge, so the read has to pick the highest version.@@ -390,6 +450,7 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):("session_id", "session_id", "$session_id", SESSION_ID),("request_id", "request_id", "$feature_flag_request_id", "req-456"),("group_key", "`$group_0`", "$group_0", "acme"),+ ("group_key_through_goe", "goe_0.key", "$group_0", "acme"),])def test_column_copying_a_restricted_property_reads_and_filters_as_null(@@ -458,7 +519,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):# there too, or every flag calls insight that filters out test accounts fails.create_person(team=self.team, uuid=str(self.person_id), properties={"email": "[email protected]"})create_group_type_mapping(- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0+ team=self.team,+ project_id=self.team.project_id,+ group_type="organization",+ group_type_index=0,+ created_at=self.occurred_at - timedelta(days=1),)create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})_create_event(@@ -519,7 +584,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):):# Without the key bound, the groups subquery reads every group of the type that the team has.create_group_type_mapping(- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0+ team=self.team,+ project_id=self.team.project_id,+ group_type="organization",+ group_type_index=0,+ created_at=self.occurred_at - timedelta(days=1),)create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})@@ -530,6 +599,23 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):assert ("in(key," in response.clickhouse) == key_boundassert response.results == [("software",)]+ def test_group_join_of_a_swapped_events_scan_reads_only_the_groups_of_the_matched_rows(self):+ create_group_type_mapping(+ team=self.team,+ project_id=self.team.project_id,+ group_type="organization",+ group_type_index=0,+ created_at=self.occurred_at - timedelta(days=1),+ )+ create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})++ response = self._query_events(f"SELECT group_0.properties.industry FROM events WHERE {FLAG_CALL_CONJUNCTS}")++ assert response.clickhouse is not None+ assert "FROM flag_evaluations" in response.clickhouse+ assert "in(key," in response.clickhouse+ assert response.results == [("software",)]+@parameterized.expand([(@@ -561,7 +647,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):# The outer WHERE bounds only the outer table's rows. It can also name the joined table. A key bound is# correct only for a group of the outer table and a WHERE that names no other table.create_group_type_mapping(- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0+ team=self.team,+ project_id=self.team.project_id,+ group_type="organization",+ group_type_index=0,+ created_at=self.occurred_at - timedelta(days=1),)create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})_create_event(team=self.team, event="$pageview", distinct_id="probe-distinct-id", timestamp=self.occurred_at)@@ -589,3 +679,469 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):).clickhouseassert f"toIntervalDay({SESSION_BUFFER_DAYS})" in clickhouse++ def test_flag_call_query_on_events_reads_no_flag_evaluations_with_the_flag_off(self):+ assert self._query_events(FLAG_CALLS_ON_EVENTS, rewrite=False).results == [(0,)]++ @parameterized.expand(+ [+ ("unaliased", FLAG_CALLS_ON_EVENTS, [(1,)]),+ (+ "qualified_without_an_alias",+ "SELECT count() FROM events WHERE events.event = '$feature_flag_called' "+ "AND events.properties.$feature_flag = 'probe-flag' AND events.timestamp >= now() - INTERVAL 7 DAY",+ [(1,)],+ ),+ (+ "aliased",+ "SELECT count() FROM events AS e WHERE e.event = '$feature_flag_called' "+ "AND e.properties.$feature_flag = 'probe-flag' AND e.timestamp >= now() - INTERVAL 7 DAY",+ [(1,)],+ ),+ (+ "alias_named_flag_key_read_back",+ "SELECT properties.$feature_flag AS flag_key, count() FROM events "+ f"WHERE {FLAG_CALL_CONJUNCTS} GROUP BY flag_key",+ [("probe-flag", 1)],+ ),+ ("element_column", f"SELECT elements_chain FROM events WHERE {FLAG_CALL_CONJUNCTS}", [("",)]),+ ("subquery_in_from", f"SELECT count() FROM (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS})", [(1,)]),+ (+ "cte",+ f"WITH calls AS (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}) SELECT count() FROM calls",+ [(1,)],+ ),+ (+ "body_of_a_cte_named_events",+ f"WITH events AS (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}) SELECT count() FROM events",+ [(1,)],+ ),+ (+ "subquery_in_an_expression",+ f"SELECT count() FROM events WHERE {PAGEVIEW_CONJUNCTS} "+ f"AND distinct_id IN (SELECT distinct_id FROM events WHERE {FLAG_CALL_CONJUNCTS})",+ [(1,)],+ ),+ (+ "union_members",+ f"SELECT 'calls', count() FROM events WHERE {FLAG_CALL_CONJUNCTS} "+ f"UNION ALL SELECT 'pageviews', count() FROM events WHERE {PAGEVIEW_CONJUNCTS}",+ [("calls", 1), ("pageviews", 1)],+ ),+ (+ "filters_placeholder",+ "SELECT count() FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'probe-flag' AND {filters}",+ [(1,)],+ HogQLFilters(dateRange=DateRange(date_from="-7d")),+ ),+ ]+ )+ def test_each_select_on_events_reads_flag_evaluations_on_its_own(+ self, _name: str, query: str, expected: list[tuple], filters: HogQLFilters | None = None+ ):+ _create_event(team=self.team, event="$pageview", distinct_id="probe-distinct-id", timestamp=self.occurred_at)+ flush_persons_and_events()++ assert sorted(self._query_events(query, filters=filters).results) == expected++ @parameterized.expand(+ [+ ("window_id", "$window_id"),+ ("event_person_id", "event_person_id"),+ ("pdi", "pdi.person_id"),+ ("group_key", "goe_0.key"),+ ("group_properties", "group_0.properties.industry"),+ ]+ )+ def test_query_reading_an_events_only_name_reads_flag_evaluations(self, _name: str, field: str):+ # pdi is an inner join on distinct_id, as on events.+ create_person(team=self.team, distinct_ids=["probe-distinct-id"])+ self._create_events_only_flag_call()++ results = self._query_events(f"SELECT distinct_id, {field} FROM events WHERE {FLAG_CALL_CONJUNCTS}").results++ assert [row[0] for row in results] == ["probe-distinct-id"]++ @parameterized.expand(+ [+ (+ "event_time_group_properties",+ f"SELECT distinct_id, goe_0.properties FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ "reads something that table doesn't have",+ ),+ ("select_star", f"SELECT * FROM events WHERE {FLAG_CALL_CONJUNCTS}", "selects \\* or COLUMNS"),+ (+ "with_name_that_a_flag_evaluations_column_shadows",+ f"WITH 'all calls' AS flag_key SELECT flag_key FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ "a name in it would read a different column",+ ),+ (+ "columns_inside_an_expression",+ f"SELECT tuple(*COLUMNS('^(created|inserted)_at$')) FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ "a name in it would read a different column",+ ),+ (+ "stored_person_id",+ f"SELECT person_id FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ "a name in it would read a different column",+ PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,+ ),+ (+ "person_id_through_distinct_ids",+ f"SELECT person_id FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ "a name in it would read a different column",+ PersonsOnEventsMode.DISABLED,+ ),+ (+ "current_person_properties",+ f"SELECT person.properties.email FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ "a name in it would read a different column",+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ ),+ ]+ )+ def test_query_flag_evaluations_cannot_answer_reads_events_until_events_gets_no_flag_calls(+ self,+ _name: str,+ query: str,+ error: str,+ persons_on_events_mode: PersonsOnEventsMode = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,+ ):+ modifiers = HogQLQueryModifiers(personsOnEventsMode=persons_on_events_mode)+ mode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS+ assert (+ self._query_events(query, modifiers=modifiers, mode=mode).clickhouse+ == self._query_events(query, rewrite=False, modifiers=modifiers, mode=mode).clickhouse+ )+ with self.assertRaisesRegex(QueryError, error):+ self._query_events(query, modifiers=modifiers)++ @parameterized.expand(+ [+ (+ "select_star",+ "WITH events AS (SELECT event, distinct_id FROM posthog.flag_evaluations) "+ "SELECT * FROM events WHERE event = '$feature_flag_called'",+ ),+ (+ "flag_call_filters",+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "+ f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ ),+ (+ "root_cte_in_a_union_branch",+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "+ f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS} "+ f"UNION ALL SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ ),+ (+ "subquery",+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "+ f"SELECT count() FROM (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS})",+ ),+ (+ "select_star_excluding_flag_calls",+ "SELECT * FROM events WHERE event != '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY",+ ),+ (+ "select_star_of_flag_calls_and_other_events",+ "SELECT * FROM events WHERE event IN ('$feature_flag_called', '$pageview') "+ "AND timestamp >= now() - INTERVAL 7 DAY",+ ),+ ]+ )+ def test_query_that_does_not_read_only_flag_calls_from_events_is_unchanged_once_events_gets_no_flag_calls(+ self, _name: str, query: str+ ):+ assert self._query_events(query).clickhouse == self._query_events(query, rewrite=False).clickhouse++ @parameterized.expand(+ [+ ("no_lower_bound", "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"),+ (+ "bound_before_coverage",+ "SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 180 DAY",+ ),+ ]+ )+ def test_query_reads_flag_evaluations_without_coverage_once_events_gets_no_flag_calls(self, _name: str, query: str):+ self._create_events_only_flag_call()++ on_events = self._query_events(query, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results+ on_flag_evaluations = self._query_events(query).results++ assert [row[0] for row in on_events] == ["events-distinct-id"]+ assert [row[0] for row in on_flag_evaluations] == ["probe-distinct-id"]++ def _create_events_only_flag_call(self) -> None:+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id="events-distinct-id",+ timestamp=self.occurred_at,+ properties={"$feature_flag": "probe-flag", "$window_id": "probe-window"},+ )+ create_person(team=self.team, distinct_ids=["events-distinct-id"])+ flush_persons_and_events()++ @parameterized.expand(+ [+ ("string", "variant-a"),+ ("escaped_characters", 'say "hi" \\ bye'),+ ("number_as_string", "100"),+ ("number", 100),+ ("boolean", True),+ (+ "quoted_flag_key_read_through_json_extract_string",+ "variant-a",+ 'say "hi"',+ "JSONExtractString(properties, '$feature_flag')",+ ),+ ]+ )+ def test_swapped_query_returns_what_events_returns_for_the_same_flag_calls(+ self,+ _name: str,+ response: str | int,+ flag_key: str = "parity-flag",+ flag_key_read: str = "properties.$feature_flag",+ ):+ # Mode 1 writes each flag call to both tables, so the two reads must agree row for row.+ row_uuid = uuid.uuid4()+ properties: dict[str, str | int | None] = {"$feature_flag": flag_key, "$feature_flag_response": response}+ self._insert_row(row_uuid, properties)+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id="probe-distinct-id",+ timestamp=self.occurred_at,+ properties=properties,+ event_uuid=str(row_uuid),+ person_id=self.person_id,+ )+ # The override is a merge after the call, so each table must correct the person it stored.+ merged_person_id = uuid.uuid4()+ self._override(merged_person_id, version=1)+ flush_persons_and_events()+ query = (+ "SELECT uuid, distinct_id, person_id, properties.$feature_flag_response FROM events "+ f"WHERE event = '$feature_flag_called' AND {flag_key_read} = '{flag_key}' "+ "AND timestamp >= now() - INTERVAL 7 DAY"+ )+ modifiers = HogQLQueryModifiers(personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS)++ from_events = self._query_events(query, rewrite=False, modifiers=modifiers).results+ swapped = self._query_events(query, modifiers=modifiers)++ assert len(from_events) == 1+ assert from_events[0][2] == merged_person_id+ assert swapped.clickhouse is not None+ assert "FROM flag_evaluations" in swapped.clickhouse+ if (+ settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA+ and isinstance(response, str)+ and not is_json_verbatim(response)+ ):+ # The native-JSON events table decodes JSON escapes in a property value. flag_evaluations returns the+ # escaped text, as the legacy events table does.+ assert [row[:3] for row in swapped.results] == [row[:3] for row in from_events]+ assert from_events[0][3] == response+ assert swapped.results[0][3] == json.dumps(response)[1:-1]+ return+ assert swapped.results == from_events++ def test_group_key_of_a_call_from_before_its_group_type_existed_reads_as_on_events(self):+ create_group_type_mapping(+ team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0+ )+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id="probe-distinct-id",+ timestamp=self.occurred_at,+ properties={"$feature_flag": "probe-flag", "$group_0": "acme"},+ event_uuid=str(self.row_uuid),+ )+ flush_persons_and_events()+ query = f"SELECT `$group_0`, goe_0.key FROM events WHERE {FLAG_CALL_CONJUNCTS}"++ from_events = self._query_events(query, rewrite=False).results++ assert from_events == [("", "acme")]+ assert self._query_events(query).results == from_events++ def test_swapped_prewhere_that_reads_an_overridden_person_id_runs(self):+ response = self._query_events(+ f"SELECT count() FROM events PREWHERE person_id IS NOT NULL WHERE {FLAG_CALL_CONJUNCTS}",+ modifiers=HogQLQueryModifiers(+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS+ ),+ )++ assert response.results == [(1,)]++ @parameterized.expand(+ [+ # The events-only call has the JSON value true as its key. A typed comparison matches it on events.+ ("mode_1", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, True),+ ("mode_2", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY, False),+ ]+ )+ def test_flag_key_typed_as_boolean_selects_the_same_calls_with_and_without_the_rewrite(+ self, _name: str, mode: FlagEvaluationsMode, with_events_only_call: bool+ ):+ PropertyDefinition.objects.create(+ team=self.team, name="$feature_flag", property_type="Boolean", type=PropertyDefinition.Type.EVENT+ )+ row_uuid = uuid.uuid4()+ self._insert_row(row_uuid, {"$feature_flag": "true"})+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id="probe-distinct-id",+ timestamp=self.occurred_at,+ properties={"$feature_flag": "true"},+ event_uuid=str(row_uuid),+ )+ if with_events_only_call:+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id="events-distinct-id",+ timestamp=self.occurred_at,+ properties={"$feature_flag": True},+ )+ flush_persons_and_events()+ query = (+ "SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'TRUE' AND timestamp >= now() - INTERVAL 7 DAY"+ )++ from_events = self._query_events(query, rewrite=False, mode=mode).results++ swapped = self._query_events(query, mode=mode)++ assert len(from_events) == (2 if with_events_only_call else 1)+ assert sorted(swapped.results) == sorted(from_events)+ if mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY:+ assert swapped.clickhouse is not None+ assert "FROM flag_evaluations" in swapped.clickhouse++ @parameterized.expand(+ [+ # The legacy events table returns the escaped text, as flag_evaluations does, so neither read matches.+ ("legacy_events", False, 0),+ ("native_json_events", True, 1),+ ]+ )+ def test_filter_on_an_escaped_property_selects_the_same_calls_with_and_without_the_rewrite(+ self, _name: str, use_new_events_schema: bool, expected_count: int+ ):+ row_uuid = uuid.uuid4()+ properties: dict[str, str | int | None] = {"$feature_flag": "probe-flag", "message": 'say "hi"'}+ self._insert_row(row_uuid, properties)+ query = f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS} AND properties.message = 'say \"hi\"'"++ with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=use_new_events_schema):+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id="probe-distinct-id",+ timestamp=self.occurred_at,+ properties=properties,+ event_uuid=str(row_uuid),+ )+ flush_persons_and_events()+ from_events = self._query_events(query, rewrite=False, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS)+ rewritten = self._query_events(query, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS)++ assert from_events.results == [(expected_count,)]+ assert rewritten.results == from_events.results+ assert rewritten.clickhouse is not None+ assert ("FROM flag_evaluations" in rewritten.clickhouse) is not use_new_events_schema++ @parameterized.expand(+ [+ ("no_materialized_column", None, "isNull(properties.$current_url)", True),+ ("materialized_event_property", ("properties", "$current_url"), "isNull(properties.$current_url)", False),+ ("materialized_person_property", ("person_properties", "email"), "isNull(person.properties.email)", False),+ # The pin keeps the flag key away from "" and "null", the values a materialized column reads differently.+ ("materialized_flag_key", ("properties", "$feature_flag"), "isNull(properties.$current_url)", True),+ ]+ )+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)+ def test_property_read_from_a_materialized_column_on_events_keeps_reading_events_in_mode_1(+ self, _name: str, materialized_property: tuple[TableColumn, str] | None, condition: str, swaps: bool+ ):+ if materialized_property is not None:+ table_column, property_name = materialized_property+ materialize("events", property_name, table_column=table_column)+ self.addCleanup(cleanup_materialized_columns)+ for value in ("", "null"):+ row_uuid = uuid.uuid4()+ properties: dict[str, str | int | None] = {"$feature_flag": "parity-flag", "$current_url": value}+ self._insert_row(row_uuid, properties, person_properties={"email": value})+ _create_event(+ team=self.team,+ event="$feature_flag_called",+ distinct_id="probe-distinct-id",+ timestamp=self.occurred_at,+ properties=properties,+ person_properties={"email": value},+ person_id=self.person_id,+ event_uuid=str(row_uuid),+ )+ flush_persons_and_events()+ query = (+ "SELECT count() FROM events WHERE event = '$feature_flag_called' "+ f"AND properties.$feature_flag = 'parity-flag' AND timestamp >= now() - INTERVAL 7 DAY AND {condition}"+ )+ modifiers = HogQLQueryModifiers(personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS)+ mode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS++ from_events = self._query_events(query, rewrite=False, modifiers=modifiers, mode=mode)+ rewritten = self._query_events(query, modifiers=modifiers, mode=mode)++ assert rewritten.results == from_events.results+ assert rewritten.clickhouse is not None+ assert ("FROM flag_evaluations" in rewritten.clickhouse) is swaps++ @parameterized.expand(+ [+ ("flag_calls", f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}", "saved_calls"),+ ("view_of_a_flag_call_view", "SELECT uuid FROM inner_calls", "inner_calls"),+ ("pageviews", f"SELECT uuid FROM events WHERE {PAGEVIEW_CONJUNCTS}", None),+ (+ "cte_named_events",+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "+ f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}",+ None,+ ),+ ]+ )+ def test_view_that_reads_flag_calls_from_events_raises_once_events_gets_no_flag_calls(+ self, _name: str, view_query: str, failing_view: str | None+ ):+ DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="inner_calls",+ query={"query": f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}"},+ columns={"uuid": "String"},+ )+ DataWarehouseSavedQuery.objects.create(+ team=self.team, name="saved_calls", query={"query": view_query}, columns={"uuid": "String"}+ )+ query = "SELECT count() FROM saved_calls"++ # events still gets flag calls on mode 1, so the view's read of events is complete.+ self._query_events(query, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS)+ if failing_view is None:+ self._query_events(query)+ return+ with self.assertRaisesRegex(QueryError, f"reads the saved view {failing_view}, which reads them from events"):+ self._query_events(query)++ def test_flag_call_query_on_events_prints_a_read_of_flag_evaluations(self):+ self.assertQueryMatchesSnapshot(self._query_events(FLAG_CALLS_ON_EVENTS).clickhouse)diff --git a/posthog/hogql/database/test/test_database.py b/posthog/hogql/database/test/test_database.pyindex 70fd037cfe55..b0f9a5ce6e1c 100644--- a/posthog/hogql/database/test/test_database.py+++ b/posthog/hogql/database/test/test_database.py@@ -19,7 +19,7 @@ from django.db import connectionfrom django.test import override_settingsfrom django.test.utils import CaptureQueriesContext-from parameterized import parameterized+from parameterized import param, parameterizedfrom pydantic import BaseModelfrom posthog.schema import (@@ -101,6 +101,8 @@ from products.access_control.backend.models.access_control import AccessControlfrom products.data_modeling.backend.facade.models import DataWarehouseSavedQueryfrom products.data_tools.backend.models.expression import DataWarehouseExpressionfrom products.data_tools.backend.models.join import DataWarehouseJoin+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfigfrom products.revenue_analytics.backend.views import RevenueAnalyticsChargeViewfrom products.revenue_analytics.backend.views.core import SourceHandlefrom products.revenue_analytics.backend.views.orchestrator import build_revenue_views_for_handles@@ -479,6 +481,76 @@ class TestDatabase(BaseTest, QueryMatchingTest):with pytest.raises(QueryError, match="Unknown table"):execute_hogql_query(query, team=self.team, pretty=False)+ @parameterized.expand(+ [+ ("events_mode", FlagEvaluationsMode.EVENTS, True, False),+ # DEBUG puts the table in the catalog on EVENTS, but events still holds every flag call.+ param("events_mode_under_debug", FlagEvaluationsMode.EVENTS, True, False, debug=True),+ param(+ "events_mode_with_the_table_forced_in",+ FlagEvaluationsMode.EVENTS,+ True,+ False,+ include_flag_evaluations_table=True,+ ),+ ("read_mode", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, True, True),+ ("read_mode_with_the_flag_off", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, False, False),+ param(+ "read_mode_with_reads_forced_to_events",+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ True,+ False,+ reads_forced_to_events=True,+ ),+ ("only_mode", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY, True, True),+ ("only_mode_with_the_flag_off", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY, False, False),+ # The switch skips FLAG_EVALUATIONS_ONLY, whose flag calls are no longer in events.+ param(+ "only_mode_with_reads_forced_to_events",+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,+ True,+ True,+ reads_forced_to_events=True,+ ),+ param(+ "only_mode_without_the_table",+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,+ True,+ False,+ include_flag_evaluations_table=False,+ ),+ ]+ )+ def test_read_flag_calls_from_flag_evaluations_follows_the_read_mode_and_the_flag(+ self,+ _name: str,+ mode: FlagEvaluationsMode,+ flag_enabled: bool,+ expected: bool,+ reads_forced_to_events: bool = False,+ debug: bool = False,+ include_flag_evaluations_table: bool | None = None,+ ) -> None:+ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=self.organization, defaults={"flag_evaluations_mode": mode}+ )++ with (+ override_settings(DEBUG=debug),+ override_instance_config("FLAG_EVALUATIONS_READS_FORCE_EVENTS", reads_forced_to_events),+ patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: (+ flag_enabled and key == "hogql-flag-calls-from-flag-evaluations"+ ),+ ),+ ):+ database = Database.create_for(+ team=self.team, include_flag_evaluations_table=include_flag_evaluations_table+ )++ assert database.read_flag_calls_from_flag_evaluations is expected+def test_serialize_database_without_fields_matches_full_metadata(self):credential = DataWarehouseCredential.objects.create(access_key="blah", access_secret="blah", team=self.team)warehouse_table = DataWarehouseTable.objects.create(diff --git a/posthog/hogql/flag_called_warnings.py b/posthog/hogql/flag_called_warnings.pyindex c0026a48a604..d1970a4462bb 100644--- a/posthog/hogql/flag_called_warnings.py+++ b/posthog/hogql/flag_called_warnings.py@@ -1,14 +1,15 @@from posthog.schema import HogQLNoticefrom posthog.hogql import ast+from posthog.hogql.constants import FLAG_CALLED_EVENTfrom posthog.hogql.context import HogQLContext+from posthog.hogql.database.models import Tablefrom posthog.hogql.database.schema.events import EventsTable+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTablefrom posthog.hogql.resolver import resolve_typesfrom posthog.hogql.visitor import TraversingVisitor, clone_expr-# posthog/models/flag_evaluations/sql.py defines this name too. It imports django.conf, which this module keeps-# off its import path, so the name is spelled out here.-FLAG_CALLED_EVENT = "$feature_flag_called"+from posthog.dataclasses import frozen# Orgs that can't query posthog.flag_evaluations yet get this warning too, so the copy speaks of the move in the future.FLAG_CALLED_ON_EVENTS_WARNING = (@@ -17,10 +18,16 @@ FLAG_CALLED_ON_EVENTS_WARNING = ("posthog.flag_evaluations instead.")+FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE = (+ f"This query reads {FLAG_CALLED_EVENT} events from posthog.flag_evaluations, which keeps the last 90 days "+ "of flag calls."+)+-class _FlagCalledOnEventsFinder(TraversingVisitor):- def __init__(self) -> None:+class _FlagCalledLiteralFinder(TraversingVisitor):+ def __init__(self, table_class: type[Table]) -> None:super().__init__()+ self.table_class = table_classself.literals: list[ast.Constant] = []def visit_select_query(self, node: ast.SelectQuery) -> None:@@ -43,12 +50,12 @@ class _FlagCalledOnEventsFinder(TraversingVisitor):def visit_compare_operation(self, node: ast.CompareOperation) -> None:if node.op in (ast.CompareOperationOp.Eq, ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn):for field_side, value_side in ((node.left, node.right), (node.right, node.left)):- if _is_events_event_field(field_side):+ if _is_event_field_of(field_side, self.table_class):self.literals.extend(_flag_called_constants(value_side))super().visit_compare_operation(node)-def _is_events_event_field(expr: ast.Expr) -> bool:+def _is_event_field_of(expr: ast.Expr, table_class: type[Table]) -> bool:while isinstance(expr, ast.Alias):expr = expr.exprif not isinstance(expr, ast.Field):@@ -62,7 +69,7 @@ def _is_events_event_field(expr: ast.Expr) -> bool:table_type = field_type.table_typewhile isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):table_type = table_type.table_type- return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, table_class)def _flag_called_constants(expr: ast.Expr) -> list[ast.Constant]:@@ -84,7 +91,7 @@ def flag_called_on_events_warnings(# A caller that passes an already printed AST never builds the database, and its AST may be rewritten.if context.database is None:return []- finder = _FlagCalledOnEventsFinder()+ finder = _FlagCalledLiteralFinder(EventsTable)with context.timings.measure("flag_called_warnings"):finder.visit(resolve_types(clone_expr(node), context, dialect="clickhouse"))# A literal that a variable supplies has no span, and a notice without one marks the whole query.@@ -93,3 +100,16 @@ def flag_called_on_events_warnings(for literal in finder.literalsif literal.start is not None and literal.end is not None]+++@frozen+class LiteralSpan:+ start: int | None+ end: int | None+++def flag_called_spans_read_from_flag_evaluations(prepared: ast.AST) -> set[LiteralSpan]:+ """The spans of the `$feature_flag_called` literals that a prepared AST compares to `flag_evaluations.event`."""+ finder = _FlagCalledLiteralFinder(FlagEvaluationsTable)+ finder.visit(prepared)+ return {LiteralSpan(start=literal.start, end=literal.end) for literal in finder.literals}diff --git a/posthog/hogql/metadata.py b/posthog/hogql/metadata.pyindex db4988f367ac..0e09d279aa9d 100644--- a/posthog/hogql/metadata.py+++ b/posthog/hogql/metadata.py@@ -25,7 +25,7 @@ from posthog.schema import (from posthog.hogql import astfrom posthog.hogql.base import ASTfrom posthog.hogql.compiler.bytecode import create_bytecode-from posthog.hogql.constants import HogQLDialect+from posthog.hogql.constants import FLAG_CALLED_EVENT, HogQLDialectfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.database.database import Databasefrom posthog.hogql.direct_connection import INVALID_CONNECTION_ID_ERROR, get_direct_connection_source@@ -35,7 +35,12 @@ from posthog.hogql.errors import (NotImplementedError as HogQLNotImplementedError,)from posthog.hogql.filters import replace_filters-from posthog.hogql.flag_called_warnings import FLAG_CALLED_EVENT, flag_called_on_events_warnings+from posthog.hogql.flag_called_warnings import (+ FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE,+ LiteralSpan,+ flag_called_on_events_warnings,+ flag_called_spans_read_from_flag_evaluations,+)from posthog.hogql.index_eligibility import IndexEligibilityReport, build_index_eligibility_reportfrom posthog.hogql.metadata_heuristics import run_metadata_heuristicsfrom posthog.hogql.modifiers import create_default_modifiers_for_team@@ -53,6 +58,7 @@ from posthog.hogql.type_system import runtime_type_from_constant_typefrom posthog.hogql.variables import replace_variablesfrom posthog.hogql.visitor import TraversingVisitor, clone_expr+from posthog.dataclasses import frozenfrom posthog.hogql_queries.query_runner import get_query_runnerfrom posthog.models import Teamfrom posthog.models.user import User@@ -217,7 +223,9 @@ def get_hogql_metadata(response.ch_table_names = get_table_names(prepared_ast)if source is None and FLAG_CALLED_EVENT in query.query:- heuristic_warnings.extend(_flag_called_on_events_warnings(hogql_ast, context, team))+ flag_called_notices = _flag_called_notices(hogql_ast, prepared_ast, context, team)+ heuristic_warnings.extend(flag_called_notices.move_warnings)+ context.notices.extend(flag_called_notices.flag_evaluations_notices)if source is None and query.indexUsage and _index_usage_enabled(team):_attach_index_usage(response, hogql_ast, context)@@ -325,23 +333,47 @@ def _flag_called_announcement_url(payload: object) -> str | None:return None-def _flag_called_on_events_warnings(- hogql_ast: Union[ast.SelectQuery, ast.SelectSetQuery], context: HogQLContext, team: Team-) -> list[HogQLNotice]:+@frozen+class _FlagCalledNotices:+ move_warnings: tuple[HogQLNotice, ...] = ()+ flag_evaluations_notices: tuple[HogQLNotice, ...] = ()+++def _flag_called_notices(+ hogql_ast: Union[ast.SelectQuery, ast.SelectSetQuery],+ prepared_ast: ast.AST | None,+ context: HogQLContext,+ team: Team,+) -> _FlagCalledNotices:+ """The move warning on each flag-call literal that reads events, and a notice on each literal whose query+ reads posthog.flag_evaluations instead. `prepared_ast` is what the printer ran, so it says which scans it+ swapped."""try:move_notices = _flag_called_move_notices(team)if move_notices is None or not move_notices.enabled:- return []+ return _FlagCalledNotices()warnings = flag_called_on_events_warnings(hogql_ast, context)if not warnings:- return []+ return _FlagCalledNotices()url = _flag_called_announcement_url(move_notices.payload)- return [warning.model_copy(update={"url": url}) for warning in warnings]+ swapped = flag_called_spans_read_from_flag_evaluations(prepared_ast) if prepared_ast is not None else set()+ move_warnings: list[HogQLNotice] = []+ flag_evaluations_notices: list[HogQLNotice] = []+ for warning in warnings:+ if LiteralSpan(start=warning.start, end=warning.end) in swapped:+ flag_evaluations_notices.append(+ warning.model_copy(update={"message": FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, "url": url})+ )+ else:+ move_warnings.append(warning.model_copy(update={"url": url}))+ return _FlagCalledNotices(+ move_warnings=tuple(move_warnings), flag_evaluations_notices=tuple(flag_evaluations_notices)+ )except Exception:# The warning is advisory. A query that compiles must not be reported as invalid because this# check failed, and the caller turns any exception here into an invalid query.logger.exception("hogql_flag_called_warning_failed", team_id=context.team_id)- return []+ return _FlagCalledNotices()def _attach_index_usage(diff --git a/posthog/hogql/printer/base.py b/posthog/hogql/printer/base.pyindex 44be235b7668..a7500df893a7 100644--- a/posthog/hogql/printer/base.py+++ b/posthog/hogql/printer/base.py@@ -1356,8 +1356,7 @@ class BasePrinter(Visitor[str]):return self._unsafe_json_extract_trim_quotes(self.visit(node.expr), self._json_property_args(node.keys))def visit_sample_expr(self, node: ast.SampleExpr) -> Optional[str]:- # SAMPLE 1 means no sampling, skip it entirely- if node.sample_value.left.value == 1 and node.sample_value.right is None and node.offset_value is None:+ if node.samples_all_rows():return Nonesample_value = self.visit_ratio_expr(node.sample_value)diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.pyindex 93a312db3bed..1600924c6e2a 100644--- a/posthog/hogql/printer/clickhouse.py+++ b/posthog/hogql/printer/clickhouse.py@@ -1242,11 +1242,13 @@ class ClickHousePrinter(BasePrinter):node_type: ast.TableOrSelectType | None,) -> ast.Expr | None:from posthog.hogql.database.schema.events import EventsTable+ from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTablemonths = self.context.events_retention_monthsif months is None or node_type is None or not isinstance(table_type, ast.TableType):return None- if not isinstance(table_type.table, EventsTable):+ # flag_evaluations holds flag calls that are events too, and a query on events can read them from there.+ if not isinstance(table_type.table, (EventsTable, FlagEvaluationsTable)):return Nonereturn retention_floor_for_table(node_type, months)diff --git a/posthog/hogql/printer/test/test_printer.py b/posthog/hogql/printer/test/test_printer.pyindex b63317264bc7..63fd13f4814f 100644--- a/posthog/hogql/printer/test/test_printer.py+++ b/posthog/hogql/printer/test/test_printer.py@@ -227,14 +227,21 @@ class TestPrinter(BaseTest):)return printed- @parameterized.expand([(12,), (84,)])+ @parameterized.expand(+ [+ (12, "events", "events"),+ (84, "events", "events"),+ (12, "posthog.flag_evaluations", "flag_evaluations"),+ ]+ )@override_settings(EVENTS_DATA_RETENTION_ENFORCED=True)- def test_events_retention_floor_applied(self, months: int):+ def test_events_retention_floor_applied(self, months: int, table: str, printed_table: str):self.team.event_retention_months = monthscontext = HogQLContext(team_id=self.team.pk, team=self.team, enable_select_queries=True)- sql = self._select("select count() from events", context=context)+ with patch("products.feature_flags.backend.facade.flags.is_flag_evaluations_table_enabled", return_value=True):+ sql = self._select(f"select count() from {table}", context=context)assert f"toIntervalMonth({months})" in sql- assert "greater(events.timestamp, minus(now" in sql+ assert f"greater({printed_table}.timestamp, minus(now" in sql@override_settings(EVENTS_DATA_RETENTION_ENFORCED=False)def test_events_retention_floor_not_applied_when_disabled(self):diff --git a/posthog/hogql/printer/utils.py b/posthog/hogql/printer/utils.pyindex 8b4ee4776c15..1fbabcce1517 100644--- a/posthog/hogql/printer/utils.py+++ b/posthog/hogql/printer/utils.py@@ -1,4 +1,5 @@from collections.abc import Callable+from functools import partialfrom typing import TYPE_CHECKING, castfrom posthog.schema_enums import InCohortVia, PropertyGroupsMode@@ -32,6 +33,7 @@ from posthog.hogql.printer.snowflake import SnowflakePrinterfrom posthog.hogql.resolver import ResolverFactory, resolve_typesfrom posthog.hogql.transforms.events_predicate_pushdown import apply_events_predicate_pushdown, events_pushdown_enabledfrom posthog.hogql.transforms.events_read_in_order import order_events_reads_by_sort_key+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import resolve_flag_calls_from_flag_evaluationsfrom posthog.hogql.transforms.in_cohort import resolve_in_cohorts, resolve_in_cohorts_conjoinedfrom posthog.hogql.transforms.json_property_pushdown import (has_rewritable_json_extract,@@ -260,13 +262,17 @@ def prepare_ast_for_printing(resolve_in_cohorts_conjoined(node, dialect, context, stack, resolver_factory=resolver_factory)with context.timings.measure("resolve_types"):- node = resolve_types(- node,- context,+ resolve = partial(+ resolve_types,+ context=context,dialect=dialect,- scopes=[node.type for node in stack if node.type is not None] if stack else None,+ scopes=[scope.type for scope in stack if scope.type is not None] if stack else None,resolver_factory=resolver_factory,)+ if dialect == "clickhouse":+ node = resolve_flag_calls_from_flag_evaluations(node, context, resolve)+ else:+ node = resolve(node)# Project constant-key JSONExtractString on argMax lazy tables (groups/persons) into the# aggregate, so it does not materialize the whole JSON blob per row. Rewrites the call to adiff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.pyindex 72b96860b895..b64b35bce570 100644--- a/posthog/hogql/test/test_metadata.py+++ b/posthog/hogql/test/test_metadata.py@@ -8,7 +8,7 @@ from django.db import DatabaseError, connectionfrom django.test import override_settingsfrom django.test.utils import CaptureQueriesContext-from parameterized import parameterized+from parameterized import param, parameterizedfrom posthoganalytics import FeatureFlagResultfrom posthog.schema import (@@ -23,7 +23,7 @@ from posthog.schema import ()from posthog.hogql.direct_connection import INVALID_CONNECTION_ID_ERROR-from posthog.hogql.flag_called_warnings import FLAG_CALLED_ON_EVENTS_WARNING+from posthog.hogql.flag_called_warnings import FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, FLAG_CALLED_ON_EVENTS_WARNINGfrom posthog.hogql.metadata import get_hogql_metadatafrom posthog.hogql.parser import parse_selectfrom posthog.hogql.taxonomy_validation import MAX_SUGGESTED_NAMES@@ -36,11 +36,17 @@ from posthog.taxonomy.dynamic_properties import DYNAMIC_PROPERTY_PATTERNSfrom products.cohorts.backend.models.cohort import Cohortfrom products.data_modeling.backend.facade.models import DataWarehouseSavedQueryfrom products.data_tools.backend.models.expression import DataWarehouseExpression+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfigfrom products.product_analytics.backend.facade.models import InsightVariablefrom products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema, ExternalDataSourcefrom products.warehouse_sources.backend.facade.types import ExternalDataSourceType_FLAG_CALLED_QUERY = "SELECT count() FROM events WHERE event = '$feature_flag_called'"+_FLAG_CALLS_FROM_FLAG_EVALUATIONS_QUERY = (+ "SELECT count() FROM events WHERE event = '$feature_flag_called' AND properties.$feature_flag = 'my-flag' "+ "AND timestamp >= now() - INTERVAL 7 DAY"+)def _move_notices(url: str | None = None, enabled: bool = True) -> FeatureFlagResult:@@ -431,8 +437,47 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):_move_notices("command:editor.action.deleteLines"),),("malformed_announcement_url", _FLAG_CALLED_QUERY, True, 1, _move_notices("https://[broken")),+ ("read_from_flag_evaluations_while_the_rewrite_is_off", _FLAG_CALLS_FROM_FLAG_EVALUATIONS_QUERY, True, 1),+ param(+ "read_from_flag_evaluations",+ _FLAG_CALLS_FROM_FLAG_EVALUATIONS_QUERY,+ True,+ 0,+ _move_notices("https://example.com/announcement"),+ "https://example.com/announcement",+ rewrite_enabled=True,+ expected_notices=1,+ ),+ param("refused_by_the_rewrite", _FLAG_CALLED_QUERY, True, 1, rewrite_enabled=True),+ param(+ "read_from_flag_evaluations_without_coverage_once_events_gets_no_flag_calls",+ _FLAG_CALLED_QUERY,+ True,+ 0,+ rewrite_enabled=True,+ expected_notices=1,+ mode=FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,+ ),+ param(+ "falls_back_for_a_field_flag_evaluations_lacks",+ "SELECT goe_0.properties FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'my-flag' AND timestamp >= now() - INTERVAL 7 DAY",+ True,+ 1,+ rewrite_enabled=True,+ ),+ param(+ "falls_back_when_a_name_binds_to_a_flag_evaluations_column",+ "WITH 'all calls' AS flag_key SELECT flag_key FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'my-flag' AND timestamp >= now() - INTERVAL 7 DAY",+ True,+ 1,+ rewrite_enabled=True,+ ),])+ # Mode 1 keeps reading events on the native-JSON events table.+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)def test_metadata_warns_for_flag_called_read_from_events(self,_name: str,@@ -441,7 +486,13 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):expected: int,move_notices: FeatureFlagResult | None = _MOVE_NOTICES_WITHOUT_URL,expected_url: str | None = None,+ rewrite_enabled: bool = False,+ expected_notices: int = 0,+ mode: FlagEvaluationsMode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS,) -> None:+ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=self.organization, defaults={"flag_evaluations_mode": mode}+ )DataWarehouseSavedQuery.objects.create(team=self.team,name="flag_calls_view",@@ -462,15 +513,26 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):return_value=flag_evaluations_enabled,),patch("posthoganalytics.get_feature_flag_result", return_value=move_notices),+ patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: (+ rewrite_enabled and key == "hogql-flag-calls-from-flag-evaluations"+ ),+ ),):metadata = self._select(query)self.assertTrue(metadata.isValid, metadata.errors)warnings = [w for w in metadata.warnings if w.message == FLAG_CALLED_ON_EVENTS_WARNING]+ notices = [n for n in metadata.notices if n.message == FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE]literal_start = query.rindex("'$feature_flag_called'")+ literal_end = literal_start + len("'$feature_flag_called'")self.assertEqual([(w.start, w.end, w.fix, w.url) for w in warnings],- [(literal_start, literal_start + len("'$feature_flag_called'"), None, expected_url)] * expected,+ [(literal_start, literal_end, None, expected_url)] * expected,+ )+ self.assertEqual(+ [(n.start, n.end, n.url) for n in notices], [(literal_start, literal_end, expected_url)] * expected_notices)def test_metadata_does_not_log_invalid_announcement_url_without_a_flag_called_literal(self) -> None:diff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.pyindex b048c321e014..e0473e3781bd 100644--- a/posthog/hogql/transforms/clickhouse_property_resolution.py+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py@@ -57,7 +57,7 @@ from posthog.hogql.type_system import (parse_sql_runtime_type,runtime_type_from_constant_type,)-from posthog.hogql.utils import ilike_matches, like_matches+from posthog.hogql.utils import ilike_matches, is_json_verbatim, like_matchesfrom posthog.hogql.visitor import CloningVisitor, clone_exprfrom posthog.clickhouse.events_json import (@@ -256,11 +256,6 @@ def resolve_property_group_source(# `node.keys` is the key path (keys[0] is the property name; deeper keys index into the extracted value).-def _is_json_verbatim(value: str) -> bool:- """True when JSON text stores the string unchanged: non-empty printable ASCII with no quote or backslash."""- return bool(value) and all(" " <= char <= "~" and char not in '"\\' for char in value)--def _blob_field_type_of(node: ast.PropertyAccess) -> ast.FieldType | None:"""The source blob column's `FieldType` (`node.expr.type`), the input to `resolve_materialized_property_source`."""expr_type = node.expr.type@@ -1986,7 +1981,7 @@ class ClickHousePropertyResolver(CloningVisitor):values = self._extract_string_constants(node.right)if not values or len(values) > PERSON_JSON_PREFILTER_MAX_NEEDLES:return None- if not all(_is_json_verbatim(value) for value in values):+ if not all(is_json_verbatim(value) for value in values):return Nonereturn _call("multiSearchAny", [self.visit(access.expr), ast.Array(exprs=[_const(v) for v in values])])diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.pynew file mode 100644index 000000000000..b0b71984e84d--- /dev/null+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py@@ -0,0 +1,965 @@+"""Read flag calls from `posthog.flag_evaluations` when a query asks `events` for them.++On flag_evaluations mode 2, ingestion stops writing `$feature_flag_called` to `events`. This transform swaps a+query's `events` scan for `posthog.flag_evaluations` when `flag_evaluations` holds every row the scan selects, so+saved queries keep returning flag calls without a change.++The swap runs before type resolution, so every rule here is syntactic. A shape the rules cannot classify keeps+reading `events`.+"""++import json+from collections import Counter+from collections.abc import Callable+from datetime import UTC, date, datetime, time, timedelta, tzinfo+from functools import cache, partial+from typing import TYPE_CHECKING+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError++import structlog++from posthog.hogql import ast+from posthog.hogql.base import _T_AST+from posthog.hogql.constants import FLAG_CALLED_EVENT+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.models import DatabaseField+from posthog.hogql.database.schema.events import EventsTable+from posthog.hogql.database.schema.flag_evaluations import ELEMENT_STAND_INS, FlagEvaluationsTable+from posthog.hogql.database.schema.util.where_clause_extractor import top_level_conjuncts+from posthog.hogql.errors import BaseHogQLError, QueryError+from posthog.hogql.functions.mapping import HOGQL_COMPARISON_MAPPING, find_hogql_function+from posthog.hogql.property_metadata import load_property_metadata+from posthog.hogql.utils import is_json_verbatim+from posthog.hogql.visitor import TraversingVisitor, clone_expr++from posthog.dataclasses import frozen+from posthog.date_util import start_of_day++if TYPE_CHECKING:+ from posthog.hogql.database.database import Database++logger = structlog.get_logger(__name__)++# At most FLAG_EVALUATIONS_TTL_DAYS - 1, because the TTL can already have dropped the day FLAG_EVALUATIONS_TTL_DAYS+# back. It stays lower while the table still lacks the oldest days that a shorter TTL dropped. The TTL constant+# lives in posthog/models/flag_evaluations/sql.py, and importing it loads the posthog.models package, which the+# printer's import path avoids.+FLAG_EVALUATIONS_COMPLETE_DAYS = 89++_FLAG_KEY_PROPERTY = "$feature_flag"++_Predicate = Callable[[ast.Expr], bool]++_MAX_UTC_OFFSET = timedelta(hours=14)+# How far before UTC midnight a date can start in some zone. A zone behind UTC can still be on the previous date, and+# a zone 14 hours ahead of UTC starts that date 14 hours before UTC does.+_DAY_IN_ANY_TIMEZONE = timedelta(days=1) + _MAX_UTC_OFFSET+# A daylight-saving change moves calendar arithmetic in a timezone by up to two hours. Antarctica/Troll changes by+# two hours.+_DST_MARGIN = timedelta(hours=2)++_UNIT_LENGTHS: dict[str, tuple[timedelta, timedelta]] = {+ "Second": (timedelta(seconds=1), timedelta(seconds=1)),+ "Minute": (timedelta(minutes=1), timedelta(minutes=1)),+ "Hour": (timedelta(hours=1), timedelta(hours=1)),+ "Day": (timedelta(days=1), timedelta(days=1)),+ "Week": (timedelta(days=7), timedelta(days=7)),+ "Month": (timedelta(days=28), timedelta(days=31)),+ "Quarter": (timedelta(days=89), timedelta(days=92)),+ "Year": (timedelta(days=365), timedelta(days=366)),+}+_SHIFT_FUNCTIONS: dict[str, tuple[int, str]] = {+ **{f"add{unit}s": (1, unit) for unit in _UNIT_LENGTHS},+ **{f"subtract{unit}s": (-1, unit) for unit in _UNIT_LENGTHS},+}+_START_OF_UNITS = {+ "toStartOfMinute": "Minute",+ "toStartOfFiveMinutes": "Hour",+ "toStartOfTenMinutes": "Hour",+ "toStartOfFifteenMinutes": "Hour",+ "toStartOfHour": "Hour",+ "toStartOfDay": "Day",+ "toMonday": "Week",+ "toStartOfWeek": "Week",+ "toStartOfMonth": "Month",+ "toStartOfQuarter": "Quarter",+ "toStartOfYear": "Year",+}+# Rounding to a week or longer returns a Date. A comparison with a DateTime reads a Date at midnight UTC. In a zone+# behind UTC, that midnight comes before the local midnight that the rounding chose.+_DATE_UNITS = frozenset({"Week", "Month", "Quarter", "Year"})++# An alias with one of these names can shadow the column in an unqualified read.+_SCAN_COLUMNS = frozenset({"event", "timestamp", "properties"})+++def can_read_flag_evaluations(+ node: ast.SelectQuery,+ now: datetime,+ timezone: ZoneInfo,+ *,+ only_in_flag_evaluations: bool = False,+ flag_key_compares_as_string: bool = True,+) -> bool:+ """Whether this query's `events` scan selects only rows that `flag_evaluations` holds.++ It holds when all of these hold:++ 1. A top-level conjunct pins the scan's `event` to `$feature_flag_called`.+ 2. A top-level conjunct bounds the scan's `timestamp` from below at or after the coverage start. A time with no+ zone reads in `timezone`, the team's zone, wherever the printer reads it that way.+ 3. The scan is not sampled, because `flag_evaluations` has no sampling key. Insight runners attach `SAMPLE 1`+ to every events read, and the printer drops it, so it counts as no sampling.+ 4. The scan's `$feature_flag` property is pinned to non-empty strings that no other JSON value prints as. A+ top-level conjunct can pin it for the whole scan. Otherwise, each `OR` branch that pins `event` must pin it+ too, as an action with several steps does. Ingestion writes a call to `flag_evaluations` only when its+ `$feature_flag` is a non-empty JSON string, and leaves every other call in `events` on every+ flag_evaluations mode. A literal such as `'100'` also matches the number 100, which stays in `events`.+ Without `flag_key_compares_as_string`, nothing pins `$feature_flag`: the team's property definition gives it+ a type, and a comparison converts the value first. Under a Boolean type, `'TRUE'` matches the key "true" and+ the JSON value true.+ 5. The select list has no `*` or `COLUMNS(...)`, and the scan has no column aliases. `flag_evaluations` has+ other columns, so either one would change what the query reads.+ 6. The scan has no `FINAL` and no table arguments.++ With `only_in_flag_evaluations`, conditions 2 and 4 do not apply. Ingestion writes no new flag calls to events+ then, so the scan reads `flag_evaluations` even when that leaves out calls older than its coverage or calls+ without a flag key.++ `resolve_flag_calls_from_flag_evaluations` checks the rest after resolution: every name in the swapped query+ binds as it did before.+ """+ scan = _EventsScan.of(node, lambda: flag_key_compares_as_string)+ return scan is not None and _qualifies(+ scan, node, _filter_conjuncts(node), now, timezone, only_in_flag_evaluations=only_in_flag_evaluations+ )+++def resolve_flag_calls_from_flag_evaluations(+ node: _T_AST, context: HogQLContext, resolve: Callable[[_T_AST], _T_AST]+) -> _T_AST:+ """`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading+ `posthog.flag_evaluations` instead.++ This returns the swapped query only when it resolves and every name in it binds as it does in the original.+ Otherwise it returns the resolved original. For an organization whose flag calls are only in+ `flag_evaluations`, it raises a QueryError instead of returning the original. It also raises before it+ resolves anything when such an organization's query has a scan of only flag calls with a shape that+ `flag_evaluations` cannot stand in for.++ While `events` still gets flag calls, the original query is the answer whenever the two tables can read a row+ differently:++ - A project that reads the native-JSON events table keeps reading `events`. That table decodes JSON escapes in a+ property value. `flag_evaluations` returns the escaped text, so a filter on such a value selects different+ rows in the two tables.+ - A query keeps reading `events` when it reads a property from a different kind of column there, such as a+ materialized column. See `_swap_reads_properties_alike`.++ The resolver inlines a saved view's body, so the swap never sees a scan inside one. Once `events` gets no flag+ calls, a query that reads a view whose body reads only flag calls from `events` raises.+ """+ database = context.database+ if (+ not context.rewrite_flag_calls_to_flag_evaluations+ or database is None+ or not database.read_flag_calls_from_flag_evaluations+ ):+ return resolve(node)+ only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations+ resolved = _resolve_with_swaps(node, context, resolve, database, only_in_flag_evaluations)+ if only_in_flag_evaluations:+ view_name = _view_reading_flag_calls_from_events(resolved)+ if view_name is not None:+ raise _flag_calls_only_in_flag_evaluations_error(+ f"it reads the saved view {view_name}, which reads them from events"+ )+ return resolved+++def _resolve_with_swaps(+ node: _T_AST,+ context: HogQLContext,+ resolve: Callable[[_T_AST], _T_AST],+ database: "Database",+ only_in_flag_evaluations: bool,+) -> _T_AST:+ finder = _FlagCalledConstantFinder()+ finder.visit(node)+ if not finder.found:+ return resolve(node)+ if not only_in_flag_evaluations and context.uses_new_events_schema():+ return resolve(node)+ with context.timings.measure("flag_calls_from_flag_evaluations"):+ rewritten = clone_expr(node)+ swapper = _EventsScanSwapper(+ now=datetime.now(UTC),+ timezone=ZoneInfo(database.get_timezone()),+ only_in_flag_evaluations=only_in_flag_evaluations,+ # The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.+ flag_key_compares_as_string=cache(partial(_flag_key_compares_as_string, context)),+ )+ swapper.visit(rewritten)+ if swapper.refusals:+ if only_in_flag_evaluations:+ raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0])+ logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=swapper.refusals[0])+ original = resolve(node)+ if not swapper.swapped:+ return original+ try:+ swapped = resolve(rewritten)+ except BaseHogQLError as error:+ # Expected for a query that reads a field flag_evaluations lacks, such as goe_0.properties.+ reason = f"it reads something that table doesn't have: {error}"+ else:+ if not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):+ reason = "a name in it would read a different column there"+ elif not only_in_flag_evaluations and not _swap_reads_properties_alike(original, swapped, context):+ reason = "it reads a property from a different kind of column there"+ else:+ return swapped+ if only_in_flag_evaluations:+ # events gets no new flag calls, so the original query would return only calls from before the move.+ raise _flag_calls_only_in_flag_evaluations_error(reason)+ logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=reason)+ return original+++def _flag_key_compares_as_string(context: HogQLContext) -> bool:+ """Whether `properties.$feature_flag` compares as a string. The printer converts a property that the team's+ property definitions type as Boolean, Numeric or DateTime before it compares the value."""+ if context.team_id is None:+ return True+ metadata = load_property_metadata(+ context, event_property_names={_FLAG_KEY_PROPERTY}, person_property_names=set(), group_property_names={}+ )+ return metadata.event_properties.get(_FLAG_KEY_PROPERTY, {}).get("type") in (None, "String")+++def _flag_calls_only_in_flag_evaluations_error(reason: str) -> QueryError:+ return QueryError(+ f"Your organization's {FLAG_CALLED_EVENT} events are only in posthog.flag_evaluations, and this query "+ f"can't read them there because {reason}. Change that part of the query, or query "+ "posthog.flag_evaluations directly."+ )+++def _qualifies(+ scan: "_EventsScan",+ node: ast.SelectQuery,+ conjuncts: list[ast.Expr],+ now: datetime,+ timezone: ZoneInfo,+ *,+ only_in_flag_evaluations: bool,+) -> bool:+ if scan.shape_refusal(node) is not None:+ return False+ if only_in_flag_evaluations:+ return scan.pins_flag_calls(conjuncts, needs_flag_key=False)+ return scan.pins_flag_calls(conjuncts, needs_flag_key=True) and scan.bounds_timestamp(conjuncts, now, timezone)+++class _EventsScan:+ def __init__(+ self,+ join: ast.JoinExpr,+ alias: str,+ has_other_sources: bool,+ flag_key_compares_as_string: Callable[[], bool],+ ):+ self.join = join+ self.alias = alias+ self.has_other_sources = has_other_sources+ self.flag_key_compares_as_string = flag_key_compares_as_string+ self.shadowed: frozenset[str] = frozenset()++ @classmethod+ def of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool]) -> "_EventsScan | None":+ join = node.select_from+ if join is None or not isinstance(join.table, ast.Field) or join.table.chain != ["events"]:+ return None+ scan = cls(+ join=join,+ alias=join.alias or "events",+ # A field without a table prefix can bind to a joined table or an ARRAY JOIN alias instead.+ has_other_sources=join.next_join is not None or bool(node.array_join_list),+ flag_key_compares_as_string=flag_key_compares_as_string,+ )+ collector = _AliasCollector(node)+ collector.visit(node)+ # An alias such as `e.timestamp AS timestamp` reads the column it names, so it changes no read.+ scan.shadowed = frozenset(+ name for name, expr in collector.aliases if name in _SCAN_COLUMNS and scan.column(expr) != [name]+ )+ return scan++ def column(self, expr: ast.Expr) -> list[str | int] | None:+ """The chain below the scan when `expr` reads it, such as `["event"]` or `["properties", "$feature_flag"]`."""+ # The resolver wraps each field it resolves in a hidden alias, which the check of a resolved view body sees.+ while isinstance(expr, ast.Alias) and expr.hidden:+ expr = expr.expr+ if not isinstance(expr, ast.Field) or not expr.chain:+ return None+ if expr.chain[0] == self.alias and len(expr.chain) > 1:+ return expr.chain[1:]+ if self.has_other_sources or expr.chain[0] in self.shadowed:+ return None+ return expr.chain++ def shape_refusal(self, node: ast.SelectQuery) -> str | None:+ """Why `flag_evaluations` cannot stand in for this scan whatever its filters say, or None."""+ if self.join.table_args is not None or self.join.table_final:+ return "it uses FINAL or table arguments on events"+ if self.join.column_aliases:+ return "it names the events columns with column aliases"+ if self.join.sample is not None and not self.join.sample.samples_all_rows():+ return "it uses SAMPLE"+ for item in node.select:+ while isinstance(item, ast.Alias):+ item = item.expr+ if isinstance(item, ast.ColumnsExpr) or (isinstance(item, ast.Field) and item.chain[-1:] == ["*"]):+ return "it selects * or COLUMNS(...)"+ return None++ def pins_flag_calls(self, conjuncts: list[ast.Expr], needs_flag_key: bool) -> bool:+ """Whether `conjuncts` pin `event` to flag calls. With `needs_flag_key`, a pin of `event` counts only where+ these conjuncts also pin `$feature_flag`. The recursion into `OR` branches passes False once an enclosing+ conjunct pins `$feature_flag`."""+ needs_flag_key = needs_flag_key and not self.flag_keys(conjuncts)+ return any(self._pins_flag_calls_in(conjunct, needs_flag_key) for conjunct in conjuncts)++ def _pins_flag_calls_in(self, expr: ast.Expr, needs_flag_key: bool) -> bool:+ if _pinned_values(expr, self._reads_event, _is_flag_called) is not None:+ return not needs_flag_key+ if isinstance(expr, ast.Or):+ return bool(expr.exprs) and all(+ self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in expr.exprs+ )+ return False++ def _flag_keys_in(self, expr: ast.Expr) -> list[str] | None:+ """The keys `expr` pins `$feature_flag` to, or None when it does not pin it."""+ values = _pinned_values(expr, self._reads_flag_key, _is_flag_key)+ if values is not None:+ return [value.value for value in values if isinstance(value, ast.Constant)]+ if isinstance(expr, ast.Or) and expr.exprs:+ branches = [self._flag_keys_in(branch) for branch in expr.exprs]+ if all(keys is not None for keys in branches):+ return [key for keys in branches if keys is not None for key in keys]+ return None++ def flag_keys(self, conjuncts: list[ast.Expr]) -> list[str]:+ """The keys a top-level conjunct pins `$feature_flag` to, or an empty list when no top-level conjunct does."""+ for conjunct in conjuncts:+ keys = self._flag_keys_in(conjunct)+ if keys is not None:+ return keys if self.flag_key_compares_as_string() else []+ return []++ def _reads_event(self, expr: ast.Expr) -> bool:+ return self.column(expr) == ["event"]++ def _reads_flag_key(self, expr: ast.Expr) -> bool:+ if isinstance(expr, ast.ArrayAccess):+ return self.column(expr.array) == ["properties"] and _is_constant(expr.property, _FLAG_KEY_PROPERTY)+ if isinstance(expr, ast.Call) and expr.name == "JSONExtractString" and len(expr.args) == 2:+ return self.column(expr.args[0]) == ["properties"] and _is_constant(expr.args[1], _FLAG_KEY_PROPERTY)+ return self.column(expr) == ["properties", _FLAG_KEY_PROPERTY]++ def bounds_timestamp(self, conjuncts: list[ast.Expr], now: datetime, timezone: ZoneInfo) -> bool:+ coverage_start = _utc_midnight(now) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)+ for conjunct in conjuncts:+ bound = self._lower_bound(conjunct)+ if bound is None:+ continue+ try:+ value = _earliest_instant(bound, now, timezone)+ except OverflowError:+ # A date near year 1 or 9999, or a huge interval, leaves datetime's range.+ continue+ if value is not None and value >= coverage_start:+ return True+ return False++ def _lower_bound(self, expr: ast.Expr) -> ast.Expr | None:+ if isinstance(expr, ast.BetweenExpr):+ return expr.low if not expr.negated and self._reads_timestamp(expr.expr) else None+ comparison = _comparison(expr)+ if comparison is None:+ return None+ if comparison.op in (ast.CompareOperationOp.GtEq, ast.CompareOperationOp.Gt) and self._reads_timestamp(+ comparison.left+ ):+ return comparison.right+ if comparison.op in (ast.CompareOperationOp.LtEq, ast.CompareOperationOp.Lt) and self._reads_timestamp(+ comparison.right+ ):+ return comparison.left+ return None++ def _reads_timestamp(self, expr: ast.Expr) -> bool:+ return self.column(expr) == ["timestamp"]+++def _comparison(expr: ast.Expr) -> ast.CompareOperation | None:+ """`expr` as a comparison, whether it is written `a >= b` or `greaterOrEquals(a, b)`."""+ if isinstance(expr, ast.CompareOperation):+ return expr+ if isinstance(expr, ast.Call) and expr.name in HOGQL_COMPARISON_MAPPING and len(expr.args) == 2:+ return ast.CompareOperation(op=HOGQL_COMPARISON_MAPPING[expr.name], left=expr.args[0], right=expr.args[1])+ return None+++def _pinned_values(expr: ast.Expr, reads: _Predicate, accepts: _Predicate) -> list[ast.Expr] | None:+ """The values `expr` limits what `reads` matches to, or None when they are not all values `accepts` matches."""+ comparison = _comparison(expr)+ if comparison is None:+ return None+ if comparison.op == ast.CompareOperationOp.Eq:+ if reads(comparison.left) and accepts(comparison.right):+ return [comparison.right]+ if reads(comparison.right) and accepts(comparison.left):+ return [comparison.left]+ return None+ if comparison.op in (ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn):+ right = comparison.right+ values = right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]+ return values if reads(comparison.left) and values and all(accepts(value) for value in values) else None+ return None+++def _is_constant(expr: ast.Expr, value: str) -> bool:+ return isinstance(expr, ast.Constant) and expr.value == value+++def _is_flag_called(expr: ast.Expr) -> bool:+ return _is_constant(expr, FLAG_CALLED_EVENT)+++def _is_flag_key(expr: ast.Expr) -> bool:+ if not isinstance(expr, ast.Constant) or not isinstance(expr.value, str) or expr.value == "":+ return False+ # A property read compares the value's JSON text, so a literal that parses as JSON can match a non-string value.+ try:+ json.loads(expr.value)+ except ValueError:+ return True+ return False+++def _conjuncts(expr: ast.Expr | None) -> list[ast.Expr]:+ return top_level_conjuncts(expr) if expr is not None else []+++def _filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:+ return _conjuncts(node.where) + _conjuncts(node.prewhere)+++def _utc_midnight(now: datetime) -> datetime:+ return start_of_day(now.astimezone(UTC))+++def _earliest_instant(expr: ast.Expr, now: datetime, timezone: ZoneInfo) -> datetime | None:+ """An instant no later than the value of `expr`, or None when `expr` has a shape this cannot evaluate."""+ if isinstance(expr, ast.Constant):+ # The printer prints a naive datetime in the server's zone, which is UTC. A bare string compares in UTC or+ # in the team's zone.+ return _constant_instant(expr.value, (timezone, UTC))+ shift = _shift(expr)+ if shift is not None:+ return _shifted(_earliest_instant(shift.base, now, timezone), shift.interval, shift.sign)+ if not isinstance(expr, ast.Call):+ return None+ name, args = expr.name, expr.args+ if name in ("now", "now64"):+ return now+ if name == "today" and not args:+ return _utc_midnight(now) - _DAY_IN_ANY_TIMEZONE+ if name == "assumeNotNull" and len(args) == 1:+ return _earliest_instant(args[0], now, timezone)+ if name in ("toDateTime", "toDateTime64") and args:+ zones = _zones_of(expr, timezone)+ if isinstance(args[0], ast.Constant):+ return _constant_instant(args[0].value, zones) if zones else None+ return _earliest_instant(args[0], now, timezone)+ if name == "toDate" and args:+ instant = _earliest_instant(args[0], now, timezone)+ # toDate truncates to a day, and the comparison can read that day in a zone the caller does not name.+ return instant - _DAY_IN_ANY_TIMEZONE if instant is not None else None+ if (name in _START_OF_UNITS and args) or (name == "toStartOfInterval" and len(args) >= 2):+ # Evaluating the value once keeps nested rounding linear. Each rounding call would otherwise evaluate its+ # value twice, and the work would double at each level.+ instant = _earliest_instant(args[0], now, timezone)+ if instant is None:+ return None+ day_start = _start_of_day(expr, instant, timezone)+ if day_start is not None:+ return day_start+ return _rounded_down(instant, (1, _START_OF_UNITS[name]) if name in _START_OF_UNITS else _interval(args[1]))+ return None+++def _start_of_day(expr: ast.Call, instant: datetime, timezone: ZoneInfo) -> datetime | None:+ """The exact start of the day `toStartOfDay(x)` or `toStartOfInterval(x, toIntervalDay(1))` rounds to, given+ `instant`, the earliest value of `x`. None for any other call or when the zone it rounds in is unknown."""+ if expr.name == "toStartOfDay" and expr.args:+ zone_index = 1+ elif expr.name == "toStartOfInterval" and len(expr.args) >= 2 and _interval(expr.args[1]) == (1, "Day"):+ zone_index = 2+ else:+ return None+ value = expr.args[0]+ # ClickHouse rounds in the zone argument, or else in the zone of the value.+ zones = _zone(expr.args[zone_index]) if len(expr.args) > zone_index else _zones_of(value, timezone)+ if not zones:+ return None+ return min(start_of_day(instant.astimezone(zone)) for zone in zones)+++def _zones_of(expr: ast.Expr, timezone: ZoneInfo) -> tuple[tzinfo, ...]:+ """The zones a time value can carry. Empty when it names a zone that does not parse."""+ while True:+ shift = _shift(expr)+ if shift is not None:+ expr = shift.base+ elif isinstance(expr, ast.Call) and expr.name == "assumeNotNull" and len(expr.args) == 1:+ expr = expr.args[0]+ else:+ break+ if isinstance(expr, ast.Call):+ meta = find_hogql_function(expr.name)+ if meta is not None and meta.tz_aware and meta.max_args:+ # The printer adds the team's zone as the last argument to each timezone-aware call that names none.+ zone_index = meta.max_args - 1+ return _zone(expr.args[zone_index]) if len(expr.args) > zone_index else (timezone,)+ return (timezone, UTC)+++def _zone(expr: ast.Expr) -> tuple[tzinfo, ...]:+ if isinstance(expr, ast.Constant) and isinstance(expr.value, str):+ try:+ return (ZoneInfo(expr.value),)+ except (ZoneInfoNotFoundError, ValueError):+ pass+ return ()+++@frozen+class _Shift:+ base: ast.Expr+ interval: tuple[int, str] | None+ sign: int+++def _shift(expr: ast.Expr) -> _Shift | None:+ """`expr` as a base, an interval and a sign when it adds an interval to a time or subtracts one."""+ if isinstance(expr, ast.ArithmeticOperation) and expr.op in (+ ast.ArithmeticOperationOp.Add,+ ast.ArithmeticOperationOp.Sub,+ ):+ sign = 1 if expr.op == ast.ArithmeticOperationOp.Add else -1+ return _Shift(base=expr.left, interval=_interval(expr.right), sign=sign)+ if not isinstance(expr, ast.Call) or len(expr.args) != 2:+ return None+ if expr.name in ("plus", "minus"):+ return _Shift(base=expr.args[0], interval=_interval(expr.args[1]), sign=1 if expr.name == "plus" else -1)+ if expr.name in _SHIFT_FUNCTIONS and (count := _integer(expr.args[1])) is not None:+ sign, unit = _SHIFT_FUNCTIONS[expr.name]+ return _Shift(base=expr.args[0], interval=(count, unit), sign=sign)+ return None+++def _interval(expr: ast.Expr) -> tuple[int, str] | None:+ if isinstance(expr, ast.Call) and expr.name.startswith("toInterval") and len(expr.args) == 1:+ unit = expr.name.removeprefix("toInterval")+ count = _integer(expr.args[0])+ if unit in _UNIT_LENGTHS and count is not None:+ return count, unit+ return None+++def _shifted(instant: datetime | None, interval: tuple[int, str] | None, sign: int) -> datetime | None:+ """The earliest instant `instant` plus `sign` times `interval` can be."""+ if instant is None or interval is None:+ return None+ count, unit = interval+ shortest, longest = _UNIT_LENGTHS[unit]+ total = sign * count+ return instant + total * (shortest if total >= 0 else longest) - _DST_MARGIN+++def _rounded_down(instant: datetime | None, interval: tuple[int, str] | None) -> datetime | None:+ """The earliest instant that rounding `instant` down to a multiple of `interval` can give."""+ rounded = _shifted(instant, interval, -1)+ if rounded is not None and interval is not None and interval[1] in _DATE_UNITS:+ return rounded - _MAX_UTC_OFFSET+ return rounded+++def _integer(expr: ast.Expr) -> int | None:+ if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and not isinstance(expr.value, bool):+ return expr.value+ return None+++def _constant_instant(value: object, zones: tuple[tzinfo, ...]) -> datetime | None:+ """`value` as an instant, reading a naive value in whichever of `zones` gives the earliest one."""+ if isinstance(value, str):+ try:+ value = datetime.fromisoformat(value.strip())+ except ValueError:+ return None+ if isinstance(value, date) and not isinstance(value, datetime):+ value = datetime.combine(value, time.min)+ if not isinstance(value, datetime):+ return None+ if value.tzinfo is not None:+ return value+ return min(value.replace(tzinfo=zone) for zone in zones)+++class _SyntaxVisitor(TraversingVisitor):+ def visit_field(self, node: ast.Field) -> None:+ # The inherited visit follows the field's type into shared CTE types, which takes exponential time. An+ # earlier step can resolve parts of the tree in place.+ pass+++class _AliasCollector(_SyntaxVisitor):+ def __init__(self, root: ast.SelectQuery) -> None:+ super().__init__()+ self.root = root+ self.aliases: list[tuple[str, ast.Expr]] = []++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ # A subquery's aliases do not reach the outer scope.+ if node is self.root:+ super().visit_select_query(node)++ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:+ pass++ def visit_alias(self, node: ast.Alias) -> None:+ self.aliases.append((node.alias, node.expr))+ super().visit_alias(node)+++class _FlagCalledConstantFinder(_SyntaxVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.found = False++ def visit_constant(self, node: ast.Constant) -> None:+ if node.value == FLAG_CALLED_EVENT:+ self.found = True+++class _EventsScanSwapper(_SyntaxVisitor):+ def __init__(+ self,+ now: datetime,+ timezone: ZoneInfo,+ only_in_flag_evaluations: bool,+ flag_key_compares_as_string: Callable[[], bool],+ ) -> None:+ super().__init__()+ self.now = now+ self.timezone = timezone+ self.only_in_flag_evaluations = only_in_flag_evaluations+ self.flag_key_compares_as_string = flag_key_compares_as_string+ self.swapped = False+ self.flag_key_predicates = 0+ # Why each scan that reads only flag calls has a shape that flag_evaluations cannot stand in for.+ self.refusals: list[str] = []+ # The CTE names in scope, under the resolver's rules. A `FROM events` reads a CTE when one is named `events`.+ # A name missing here lets the swapper try the scan, and the binding check then rejects the swap.+ self.ctes: frozenset[str] = frozenset()++ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:+ outer = self.ctes+ initial = node.initial_select_query+ self.visit(initial)+ # The resolver gives every later branch the first branch's WITH.+ if isinstance(initial, ast.SelectQuery) and initial.ctes:+ self.ctes = self.ctes.union(initial.ctes)+ for subsequent in node.subsequent_select_queries:+ self.visit(subsequent.select_query)+ self.ctes = outer++ def visit_cte(self, node: ast.CTE) -> None:+ # visit_select_query visits each CTE body while only the earlier CTE names are in scope.+ pass++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ outer = self.ctes+ for cte in (node.ctes or {}).values():+ self.visit(cte.expr)+ self.ctes = self.ctes.union([cte.name])+ scan = None if "events" in self.ctes else _EventsScan.of(node, self.flag_key_compares_as_string)+ if scan is not None:+ conjuncts = _filter_conjuncts(node)+ if _qualifies(+ scan, node, conjuncts, self.now, self.timezone, only_in_flag_evaluations=self.only_in_flag_evaluations+ ):+ self._swap(node, scan, conjuncts)+ else:+ refusal = scan.shape_refusal(node)+ if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):+ self.refusals.append(refusal)+ super().visit_select_query(node)+ self.ctes = outer++ def _swap(self, node: ast.SelectQuery, scan: _EventsScan, conjuncts: list[ast.Expr]) -> None:+ keys = scan.flag_keys(conjuncts)+ # The old alias keeps qualified reads such as `events.timestamp` resolving.+ scan.join.alias = scan.alias+ scan.join.table = ast.Field(chain=["posthog", "flag_evaluations"])+ # A swapped PREWHERE can read person columns through a join that the printer adds later. ClickHouse+ # rejects a PREWHERE that reads a joined column, and WHERE selects the same rows.+ conditions = [condition for condition in (node.prewhere, node.where) if condition is not None]+ # flag_key sorts the table, so this predicate lets ClickHouse skip other flags. flag_key holds the key's JSON+ # text, and a JSONExtractString pin compares the decoded key. The pin implies this predicate only when the+ # JSON text of each key is the key itself.+ if keys and all(is_json_verbatim(key) for key in keys):+ conditions.append(+ ast.CompareOperation(+ op=ast.CompareOperationOp.In,+ left=ast.Field(chain=[scan.alias, "flag_key"]),+ right=ast.Tuple(exprs=[ast.Constant(value=key) for key in keys]),+ )+ )+ self.flag_key_predicates += 1+ node.prewhere = None+ node.where = conditions[0] if len(conditions) == 1 else ast.And(exprs=conditions)+ self.swapped = True+++def _swap_keeps_bindings(original: ast.AST, swapped: ast.AST, flag_key_predicates: int) -> bool:+ """Whether every field and table in the resolved swapped query binds as it does in the resolved original.++ The swap may change only the table each swapped scan reads. A name that binds elsewhere changes the result+ with no resolution error: a `WITH` name that a `flag_evaluations` column now shadows, a `COLUMNS(...)` that+ expands to other columns, or a `person_id` that read the stored events column and now reads the person overrides+ join. The persons-on-events mode decides how `events` reads person fields. The comparison ignores order,+ because the swap moves `PREWHERE` into `WHERE`. The swapped query also reads flag_key once for each of the+ `flag_key_predicates` that the swap adds.+ """+ expected = Counter(_bindings(original)) + Counter({_ADDED_FLAG_KEY_READ: flag_key_predicates})+ return Counter(_bindings(swapped)) == expected+++# A field records the last part of its chain, so `e.person_id` and `person_id` match.+_Binding = tuple[str, str | int, tuple[str, ...]]++# events has no flag_key column, so the original query records this binding only for a select alias named flag_key+# that reads the scan.+_ADDED_FLAG_KEY_READ: _Binding = ("field", "flag_key", ("scan",))+++def _bindings(node: ast.AST) -> list[_Binding]:+ collector = _BindingCollector()+ collector.visit(node)+ return collector.bindings+++class _BindingCollector(_SyntaxVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.bindings: list[_Binding] = []++ def visit_field(self, node: ast.Field) -> None:+ self.bindings.append(("field", node.chain[-1] if node.chain else "", _target(node.type)))++ def visit_alias(self, node: ast.Alias) -> None:+ if not node.hidden:+ super().visit_alias(node)+ return+ # The resolver replaces an expression field, such as flag_evaluations.person_id, with a hidden alias named+ # after the field. The same field can be a plain column on events, so only the alias name and what the+ # expansion reads count. An element stand-in reads no column. It returns what the events column holds for a+ # flag call, so it counts as a read of the scan.+ targets = sorted({target for _, _, target in _bindings(node.expr)})+ if targets == [("scan",)] or (not targets and node.alias in ELEMENT_STAND_INS):+ target: tuple[str, ...] = ("scan",)+ else:+ target = ("expression", *("/".join(read) for read in targets))+ self.bindings.append(("field", node.alias, target))++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ self.bindings.append(("table", "", _target(node.type)))+ if not isinstance(node.table, ast.Field):+ self.visit(node.table)+ for arg in node.table_args or []:+ self.visit(arg)+ self.visit(node.constraint)+ self.visit(node.next_join)+++def _target(current: ast.Type | None) -> tuple[str, ...]:+ """What a resolved field or table reads from: `("scan",)` for an events or flag_evaluations scan."""+ while True:+ if isinstance(current, ast.FieldAliasType):+ current = current.type+ elif isinstance(current, ast.PropertyType):+ current = current.field_type+ elif isinstance(+ current,+ (+ ast.FieldType,+ ast.ExpressionFieldType,+ ast.FieldTraverserType,+ ast.TableAliasType,+ ast.VirtualTableType,+ ),+ ):+ current = current.table_type+ elif isinstance(current, ast.LazyJoinType):+ # A join reads another table. The tables can define a join of the same name differently, so the+ # binding names the join recipe and what it joins on. Under PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ # events.person joins persons, while flag_evaluations.person reads the person properties stored with the+ # call. join_table can name the same table as an instance or as a string, so the binding leaves it out.+ join = current.lazy_join+ return (+ "join",+ current.field,+ join.resolver,+ json.dumps(join.resolver_params, sort_keys=True, default=str),+ ".".join(str(part) for part in join.from_field),+ )+ elif isinstance(current, ast.TableType):+ return (+ ("scan",)+ if isinstance(current.table, (EventsTable, FlagEvaluationsTable))+ else ("table", current.table.__class__.__name__)+ )+ elif isinstance(current, ast.CTETableType):+ return ("cte", current.name)+ elif isinstance(current, ast.CTETableAliasType):+ return ("cte", current.alias)+ elif isinstance(current, ast.SelectQueryAliasType):+ return ("subquery", current.alias)+ elif isinstance(current, ast.SelectViewType):+ return ("view", current.view_name)+ else:+ return (current.__class__.__name__,)+++def _swap_reads_properties_alike(original: ast.AST, swapped: ast.AST, context: HogQLContext) -> bool:+ """Whether each property that the swapped query reads comes from the same kind of column as in the original.++ A materialized or dmat column on `events` reads the strings "" and "null" as NULL. `flag_evaluations` reads the+ same property from the JSON blob, which returns those strings. A property-group map has the same definition on+ both tables, so reads from it agree. The swap pins `$feature_flag` to keys that are never "" or "null", so a+ read of it agrees whatever column backs it.+ """+ original_reads = _property_reads(original, context)+ swapped_reads = _property_reads(swapped, context)+ if not original_reads and not swapped_reads:+ return True+ if context.team_id is None:+ return False+ previous_metadata = context.property_metadata+ # The dmat slots come from the team's property definitions, which the printer loads only after this transform.+ context.property_metadata = load_property_metadata(+ context,+ event_property_names={name for column, name, _ in original_reads if column == "properties"},+ person_property_names=set(),+ group_property_names={},+ )+ try:+ return Counter(_property_sources(original_reads, context)) == Counter(_property_sources(swapped_reads, context))+ finally:+ context.property_metadata = previous_metadata+++# The blob column, the property, and the field type that reads it.+_PropertyRead = tuple[str, str, ast.FieldType]+++def _property_reads(node: ast.AST, context: HogQLContext) -> list[_PropertyRead]:+ collector = _PropertyReadCollector(context)+ collector.visit(node)+ return collector.reads+++@frozen+class _PropertySource:+ column: str+ name: str+ # The column the printer reads the property from, or None for the JSON blob.+ materialized_column: str | None+++def _property_sources(reads: list[_PropertyRead], context: HogQLContext) -> list[_PropertySource]:+ # Deferred: clickhouse_property_resolution imports the printer package, which imports this module.+ from posthog.hogql.transforms.clickhouse_property_resolution import ( # noqa: PLC0415+ resolve_materialized_property_source,+ )++ sources: list[_PropertySource] = []+ for column, name, field_type in reads:+ source = resolve_materialized_property_source(field_type, name, context)+ sources.append(+ _PropertySource(column=column, name=name, materialized_column=source.column if source is not None else None)+ )+ return sources+++class _PropertyReadCollector(_SyntaxVisitor):+ def __init__(self, context: HogQLContext) -> None:+ super().__init__()+ self.context = context+ self.reads: list[_PropertyRead] = []++ def visit_field(self, node: ast.Field) -> None:+ if not isinstance(node.type, ast.PropertyType) or not node.type.chain:+ return+ field_type = node.type.field_type+ database_field = field_type.resolve_database_field(self.context)+ column = database_field.name if isinstance(database_field, DatabaseField) else field_type.name+ name = str(node.type.chain[0])+ if column == "properties" and name == _FLAG_KEY_PROPERTY:+ return+ self.reads.append((column, name, field_type))+++def _view_reading_flag_calls_from_events(node: ast.AST) -> str | None:+ """The name of a saved view whose inlined body reads only flag calls from `events`, or None."""+ finder = _FlagCallViewFinder()+ finder.visit(node)+ return finder.found+++class _FlagCallViewFinder(_SyntaxVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.view: str | None = None+ self.found: str | None = None++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ outer = self.view+ # The resolver names the view on the body's top SELECT only, so a SELECT nested in the body inherits it.+ self.view = node.view_name or outer+ if self.view is not None and self.found is None and _reads_only_flag_calls_from_events(node):+ self.found = self.view+ super().visit_select_query(node)+ self.view = outer+++def _reads_only_flag_calls_from_events(node: ast.SelectQuery) -> bool:+ scan = _EventsScan.of(node, lambda: True)+ return (+ scan is not None+ and _reads_events_table(scan.join)+ and scan.pins_flag_calls(_filter_conjuncts(node), needs_flag_key=False)+ )+++def _reads_events_table(join: ast.JoinExpr) -> bool:+ """Whether a resolved FROM reads the events table, rather than a CTE named `events`."""+ table_type = join.type+ while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):+ table_type = table_type.table_type+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)diff --git a/posthog/hogql/transforms/test/test_clickhouse_property_resolution.py b/posthog/hogql/transforms/test/test_clickhouse_property_resolution.pyindex fe87922f56cf..9a06cec2b0f7 100644--- a/posthog/hogql/transforms/test/test_clickhouse_property_resolution.py+++ b/posthog/hogql/transforms/test/test_clickhouse_property_resolution.py@@ -23,10 +23,10 @@ from posthog.hogql.property_access_types import RestrictedPropertyfrom posthog.hogql.property_metadata import PropertyMetadatafrom posthog.hogql.transforms.clickhouse_property_resolution import (MAX_MATERIALIZED_LIKE_PATTERN_LENGTH,- _is_json_verbatim,clickhouse_property_resolution,)from posthog.hogql.transforms.property_types import PropertySwapper+from posthog.hogql.utils import is_json_verbatimfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.events_json import TEMPORARY_PROPERTIES_JSON_TYPE@@ -53,7 +53,7 @@ class TestJSONVerbatimValues(SimpleTestCase):@example("plain_value-1")@hypothesis_settings(max_examples=100, deadline=None)def test_accepted_values_survive_json_encoding(self, value: str) -> None:- if not _is_json_verbatim(value):+ if not is_json_verbatim(value):returnassert value in json.dumps({"key": value})assert value in json.dumps({"key": value}, ensure_ascii=False)diff --git a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.pynew file mode 100644index 000000000000..9c0bb3c73321--- /dev/null+++ b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py@@ -0,0 +1,456 @@+from datetime import UTC, datetime, time, timedelta+from zoneinfo import ZoneInfo++import pytest+import time_machine++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.hogql import ast+from posthog.hogql.parser import parse_select+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (+ FLAG_EVALUATIONS_COMPLETE_DAYS,+ can_read_flag_evaluations,+)++from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_TTL_DAYS++NOW = datetime(2026, 10, 27, 12, 0, tzinfo=UTC)+COVERAGE_START = NOW.replace(hour=0) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)+IN_COVERAGE = COVERAGE_START + timedelta(days=30)++FLAG_CALLED = "event = '$feature_flag_called'"+FLAG_KEY = "properties.$feature_flag = 'my-flag'"+BOUND = "timestamp >= now() - INTERVAL 7 DAY"+++def _text(instant: datetime) -> str:+ return instant.strftime("%Y-%m-%d %H:%M:%S")+++def _date(instant: datetime) -> str:+ return instant.strftime("%Y-%m-%d")+++def _last_days_bound(timezone: str, days: int) -> str:+ local_today = NOW.astimezone(ZoneInfo(timezone)).date()+ date_from = datetime.combine(local_today - timedelta(days=days), time.min)+ return f"toStartOfInterval(assumeNotNull(toDateTime('{_text(date_from)}')), toIntervalDay(1))"+++def _where(*conjuncts: str) -> str:+ return f"SELECT count() FROM events WHERE {' AND '.join(conjuncts)}"+++def _bounded_by(lower_bound: str) -> str:+ return _where(FLAG_CALLED, FLAG_KEY, f"timestamp >= {lower_bound}")+++def _nested(template: str, depth: int) -> str:+ expr = "properties.since"+ for _ in range(depth):+ expr = template.format(expr)+ return expr+++def _joined(where: str) -> str:+ return f"SELECT count() FROM events AS e JOIN persons AS p ON e.person_id = p.id WHERE {where}"+++def _self_joined(where: str) -> str:+ return f"SELECT count() FROM events AS e JOIN events AS f ON e.uuid = f.uuid WHERE {where}"+++@time_machine.travel(NOW, tick=False)+class TestCanReadFlagEvaluations(SimpleTestCase):+ def _can_read(+ self,+ query: str,+ placeholders: dict[str, ast.Expr] | None = None,+ timezone: str = "UTC",+ *,+ only_in_flag_evaluations: bool = False,+ ) -> bool:+ select = parse_select(query, placeholders=placeholders)+ assert isinstance(select, ast.SelectQuery)+ return can_read_flag_evaluations(+ select, datetime.now(UTC), ZoneInfo(timezone), only_in_flag_evaluations=only_in_flag_evaluations+ )++ def test_complete_days_leave_out_the_day_the_ttl_may_have_dropped(self):+ assert FLAG_EVALUATIONS_COMPLETE_DAYS < FLAG_EVALUATIONS_TTL_DAYS++ @parameterized.expand(+ [+ ("event_equals", _where(FLAG_CALLED, FLAG_KEY, BOUND)),+ ("event_equals_call", _where("equals(event, '$feature_flag_called')", FLAG_KEY, BOUND)),+ ("event_in_one_value", _where("event IN ('$feature_flag_called')", FLAG_KEY, BOUND)),+ (+ "or_of_pinned_branches_with_their_own_flag_keys",+ _where(+ "((event = '$feature_flag_called' AND properties.$feature_flag = 'a') "+ "OR (event = '$feature_flag_called' AND properties.$feature_flag = 'b'))",+ BOUND,+ ),+ ),+ ("pin_in_prewhere", f"SELECT count() FROM events PREWHERE {FLAG_CALLED} WHERE {FLAG_KEY} AND {BOUND}"),+ ("conjuncts_in_nested_ands", _where(FLAG_CALLED, f"({FLAG_KEY} AND ({BOUND} AND distinct_id = 'd'))")),+ (+ "qualified_without_an_alias",+ _where(+ "events.event = '$feature_flag_called'",+ "events.properties.$feature_flag = 'my-flag'",+ "events.timestamp >= now() - INTERVAL 7 DAY",+ ),+ ),+ (+ "qualified_by_the_alias",+ "SELECT count() FROM events AS e WHERE e.event = '$feature_flag_called' "+ "AND e.properties.$feature_flag = 'my-flag' AND e.timestamp >= now() - INTERVAL 7 DAY",+ ),+ (+ "unqualified_under_an_alias_without_a_join",+ f"SELECT count() FROM events AS e WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",+ ),+ (+ "qualified_by_the_alias_with_a_join",+ _joined(+ "e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "+ "AND e.timestamp >= now() - INTERVAL 7 DAY"+ ),+ ),+ ("flag_key_array_access", _where(FLAG_CALLED, "properties['$feature_flag'] = 'my-flag'", BOUND)),+ (+ "flag_key_array_access_by_the_alias",+ "SELECT count() FROM events AS e WHERE e.event = '$feature_flag_called' "+ "AND e.properties['$feature_flag'] = 'my-flag' AND e.timestamp >= now() - INTERVAL 7 DAY",+ ),+ (+ "flag_key_json_extract",+ _where(FLAG_CALLED, "JSONExtractString(properties, '$feature_flag') = 'my-flag'", BOUND),+ ),+ ("flag_key_equals_call", _where(FLAG_CALLED, "equals(properties.$feature_flag, 'my-flag')", BOUND)),+ ("flag_key_in_list", _where(FLAG_CALLED, "properties.$feature_flag IN ('a', 'b')", BOUND)),+ (+ "flag_key_or",+ _where(FLAG_CALLED, "(properties.$feature_flag = 'a' OR properties.$feature_flag = 'b')", BOUND),+ ),+ ("bound_greater", _where(FLAG_CALLED, FLAG_KEY, "timestamp > now() - INTERVAL 7 DAY")),+ ("bound_flipped_less_or_equals", _where(FLAG_CALLED, FLAG_KEY, "now() - INTERVAL 7 DAY <= timestamp")),+ ("bound_flipped_less", _where(FLAG_CALLED, FLAG_KEY, "now() - INTERVAL 7 DAY < timestamp")),+ ("bound_between", _where(FLAG_CALLED, FLAG_KEY, "timestamp BETWEEN now() - INTERVAL 7 DAY AND now()")),+ (+ "bound_greater_or_equals_call",+ _where(FLAG_CALLED, FLAG_KEY, "greaterOrEquals(timestamp, now() - INTERVAL 7 DAY)"),+ ),+ ("bound_greater_call", _where(FLAG_CALLED, FLAG_KEY, "greater(timestamp, now() - INTERVAL 7 DAY)")),+ (+ "latest_of_several_bounds",+ _where(FLAG_CALLED, FLAG_KEY, "timestamp >= now() - INTERVAL 365 DAY", BOUND),+ ),+ ("now64", _bounded_by("now64() - INTERVAL 7 DAY")),+ ("today", _bounded_by("today() - INTERVAL 7 DAY")),+ ("interval_function", _bounded_by("now() - toIntervalDay(7)")),+ ("minus_call", _bounded_by("minus(now(), toIntervalDay(7))")),+ ("subtract_days", _bounded_by("subtractDays(now(), 7)")),+ ("weeks", _bounded_by("now() - INTERVAL 12 WEEK")),+ ("months", _bounded_by("now() - INTERVAL 2 MONTH")),+ ("hours", _bounded_by("now() - INTERVAL 2000 HOUR")),+ ("plus_interval", _bounded_by("now() + INTERVAL 1 DAY")),+ ("last_covered_day", _bounded_by(f"now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS} DAY")),+ ("assume_not_null", _bounded_by("assumeNotNull(now() - INTERVAL 7 DAY)")),+ ("start_of_day", _bounded_by("toStartOfDay(now() - INTERVAL 7 DAY)")),+ ("start_of_week_with_a_mode", _bounded_by("toStartOfWeek(now() - INTERVAL 7 DAY, 1)")),+ ("start_of_month", _bounded_by("toStartOfMonth(now() - INTERVAL 7 DAY)")),+ ("start_of_interval", _bounded_by("toStartOfInterval(now() - INTERVAL 7 DAY, INTERVAL 1 DAY)")),+ (+ "trends_date_from",+ _bounded_by(f"toStartOfInterval(assumeNotNull(toDateTime('{_text(IN_COVERAGE)}')), toIntervalDay(1))"),+ ),+ ("to_date_time", _bounded_by(f"toDateTime('{_text(IN_COVERAGE)}')")),+ ("to_date_time_with_a_timezone", _bounded_by(f"toDateTime('{_text(IN_COVERAGE)}', 'UTC')")),+ ("to_date_time64", _bounded_by(f"toDateTime64('{_text(IN_COVERAGE)}', 6)")),+ ("to_date_time64_with_a_timezone", _bounded_by(f"toDateTime64('{_text(IN_COVERAGE)}', 6, 'UTC')")),+ ("to_date", _bounded_by(f"toDate('{_date(IN_COVERAGE)}')")),+ ("date_string", _bounded_by(f"'{_date(IN_COVERAGE)}'")),+ ("date_time_string", _bounded_by(f"'{_text(IN_COVERAGE)}'")),+ (+ "first_covered_day_in_utc",+ _bounded_by(f"toDateTime('{_text(COVERAGE_START + timedelta(hours=2))}', 'UTC')"),+ ),+ # The printer drops SAMPLE 1. The trends query builders set it on every events read.+ ("sample_of_one", f"SELECT count() FROM events SAMPLE 1 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),+ (+ "alias_that_renames_its_own_column",+ f"SELECT e.timestamp AS timestamp FROM events e WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",+ ),+ ]+ )+ def test_accepts_a_flag_call_query_inside_coverage(self, _name: str, query: str):+ assert self._can_read(query) is True++ @parameterized.expand(+ [+ ("aware_datetime", ast.Constant(value=COVERAGE_START), True),+ ("date", ast.Constant(value=(COVERAGE_START + timedelta(days=1)).date()), True),+ ("aware_datetime_before_coverage", ast.Constant(value=COVERAGE_START - timedelta(seconds=1)), False),+ ]+ )+ def test_reads_a_placeholder_bound_against_coverage(self, _name: str, bound: ast.Expr, expected: bool):+ assert self._can_read(_bounded_by("{bound}"), {"bound": bound}) is expected++ @parameterized.expand(+ [+ ("no_event_pin", _where(FLAG_KEY, BOUND)),+ ("mixed_in", _where("event IN ('$feature_flag_called', '$pageview')", FLAG_KEY, BOUND)),+ (+ "or_with_an_unpinned_branch",+ _where(+ "((event = '$feature_flag_called' AND properties.$feature_flag = 'a') "+ "OR (event = '$pageview' AND properties.$feature_flag = 'a'))",+ BOUND,+ ),+ ),+ ("or_with_a_branch_on_another_column", _where(f"({FLAG_CALLED} OR distinct_id = 'd')", FLAG_KEY, BOUND)),+ ("negated_pin", _where(f"NOT ({FLAG_CALLED})", FLAG_KEY, BOUND)),+ (+ "pin_only_in_a_subquery",+ _where(FLAG_KEY, BOUND, f"uuid IN (SELECT uuid FROM events WHERE {FLAG_CALLED})"),+ ),+ (+ "unqualified_event_with_a_join",+ _joined(+ f"{FLAG_CALLED} AND e.properties.$feature_flag = 'my-flag' AND e.timestamp >= now() - INTERVAL 7 DAY"+ ),+ ),+ (+ "pin_on_a_joined_alias",+ _self_joined(+ "f.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "+ "AND e.timestamp >= now() - INTERVAL 7 DAY"+ ),+ ),+ ("no_bound", _where(FLAG_CALLED, FLAG_KEY)),+ ("upper_bound_only", _where(FLAG_CALLED, FLAG_KEY, "timestamp <= now()")),+ ("flipped_upper_bound", _where(FLAG_CALLED, FLAG_KEY, "now() - INTERVAL 7 DAY >= timestamp")),+ (+ "greater_or_equals_call_with_flipped_arguments",+ _where(FLAG_CALLED, FLAG_KEY, "greaterOrEquals(now() - INTERVAL 7 DAY, timestamp)"),+ ),+ ("not_between", _where(FLAG_CALLED, FLAG_KEY, "timestamp NOT BETWEEN now() - INTERVAL 7 DAY AND now()")),+ ("bound_inside_an_or", _where(FLAG_CALLED, FLAG_KEY, f"({BOUND} OR distinct_id = 'd')")),+ ("bound_on_another_column", _where(FLAG_CALLED, FLAG_KEY, "created_at >= now() - INTERVAL 7 DAY")),+ (+ "unqualified_bound_with_a_join",+ _joined(f"e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' AND {BOUND}"),+ ),+ (+ "bound_on_a_joined_alias",+ _self_joined(+ "e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "+ "AND f.timestamp >= now() - INTERVAL 7 DAY"+ ),+ ),+ ("one_day_past_coverage", _bounded_by(f"now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS + 1} DAY")),+ ("months_past_coverage", _bounded_by("now() - INTERVAL 3 MONTH")),+ (+ "absolute_date_before_coverage",+ _bounded_by(f"toDateTime('{_text(COVERAGE_START - timedelta(seconds=1))}', 'UTC')"),+ ),+ ("to_date_on_the_first_covered_day", _bounded_by(f"toDate('{_date(COVERAGE_START)}')")),+ (+ "string_with_an_offset_before_coverage",+ _bounded_by(f"toDateTime('{_text(COVERAGE_START + timedelta(hours=6))}+14:00')"),+ ),+ ("negative_interval_added", _bounded_by("now() + INTERVAL -100 DAY")),+ ("subtract_days_past_coverage", _bounded_by(f"subtractDays(now(), {FLAG_EVALUATIONS_COMPLETE_DAYS + 1})")),+ (+ "start_of_month_before_coverage",+ _bounded_by(f"toStartOfMonth(now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS - 1} DAY)"),+ ),+ ("start_of_year", _bounded_by("toStartOfYear(now())")),+ (+ "start_of_interval_before_coverage",+ _bounded_by(+ f"toStartOfInterval(now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS - 1} DAY, INTERVAL 1 MONTH)"+ ),+ ),+ ("bound_read_from_a_field", _bounded_by("toDateTime(properties.since)")),+ ("bound_from_a_subquery", _bounded_by("(SELECT min(timestamp) FROM events)")),+ ("interval_read_from_a_field", _bounded_by("now() - toIntervalDay(properties.days)")),+ ("sample", f"SELECT count() FROM events SAMPLE 0.1 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),+ ("sample_ratio", f"SELECT count() FROM events SAMPLE 1/10 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),+ (+ "sample_of_one_with_an_offset",+ f"SELECT count() FROM events SAMPLE 1 OFFSET 1/2 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",+ ),+ (+ "sample_of_rows",+ f"SELECT count() FROM events SAMPLE 10000 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",+ ),+ ("no_flag_key_pin", _where(FLAG_CALLED, BOUND)),+ (+ "flag_key_in_list_with_an_empty_string",+ _where(FLAG_CALLED, "properties.$feature_flag IN ('my-flag', '')", BOUND),+ ),+ ("flag_key_empty_string", _where(FLAG_CALLED, "properties.$feature_flag = ''", BOUND)),+ ("flag_key_not_empty", _where(FLAG_CALLED, "properties.$feature_flag != ''", BOUND)),+ ("flag_key_number", _where(FLAG_CALLED, "properties.$feature_flag = 1", BOUND)),+ (+ "flag_key_or_with_an_empty_string",+ _where(FLAG_CALLED, "(properties.$feature_flag = 'my-flag' OR properties.$feature_flag = '')", BOUND),+ ),+ (+ "flag_key_or_with_a_branch_on_another_column",+ _where(FLAG_CALLED, "(properties.$feature_flag = 'my-flag' OR distinct_id = 'd')", BOUND),+ ),+ (+ "flag_key_on_person_properties",+ _where(FLAG_CALLED, "person.properties.$feature_flag = 'my-flag'", BOUND),+ ),+ ("pin_on_another_property", _where(FLAG_CALLED, "properties.$feature_flag_response = 'my-flag'", BOUND)),+ (+ "unqualified_flag_key_with_a_join",+ _joined(f"e.event = '$feature_flag_called' AND {FLAG_KEY} AND e.timestamp >= now() - INTERVAL 7 DAY"),+ ),+ (+ "flag_key_on_a_joined_alias",+ _self_joined(+ "e.event = '$feature_flag_called' AND f.properties.$feature_flag = 'my-flag' "+ "AND e.timestamp >= now() - INTERVAL 7 DAY"+ ),+ ),+ (+ "or_pin_with_a_branch_without_its_flag_key",+ _where(+ f"((event = '$feature_flag_called' AND properties.$feature_flag = 'a') OR {FLAG_CALLED})",+ BOUND,+ ),+ ),+ (+ "bound_on_an_alias_that_shadows_timestamp",+ f"SELECT now() AS timestamp FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",+ ),+ (+ "pin_on_an_alias_that_shadows_event",+ f"SELECT '$feature_flag_called' AS event FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",+ ),+ ("select_star", f"SELECT * FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),+ ("qualified_select_star", f"SELECT e.* FROM events e WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),+ ("columns_expression", f"SELECT COLUMNS('^p') FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),+ ("final", f"SELECT count() FROM events FINAL WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),+ ("flag_key_that_reads_as_a_number", _where(FLAG_CALLED, "properties.$feature_flag = '100'", BOUND)),+ ("flag_key_that_reads_as_a_boolean", _where(FLAG_CALLED, "properties.$feature_flag = 'true'", BOUND)),+ ("flag_key_that_reads_as_json_null", _where(FLAG_CALLED, "properties.$feature_flag = 'null'", BOUND)),+ (+ "flag_key_list_with_an_object",+ _where(FLAG_CALLED, "properties.$feature_flag IN ('my-flag', '{\"a\": 1}')", BOUND),+ ),+ (+ "column_aliases",+ "SELECT count() FROM events AS e (uuid, event, properties, timestamp, distinct_id, x) "+ "WHERE e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "+ "AND e.timestamp >= now() - INTERVAL 7 DAY",+ ),+ (+ "bare_star_with_an_array_join",+ "SELECT * FROM events ARRAY JOIN [1] AS n WHERE events.event = '$feature_flag_called' "+ "AND events.properties.$feature_flag = 'my-flag' AND events.timestamp >= now() - INTERVAL 7 DAY",+ ),+ ("bound_near_year_one", _bounded_by("'0001-01-01'")),+ ("interval_past_datetime_range", _bounded_by("now() - INTERVAL 999999999 YEAR")),+ # Each level used to evaluate its value twice, so 40 levels took 2^40 steps.+ ("deeply_nested_start_of_day_of_a_column", _bounded_by(_nested("toStartOfDay({})", 40))),+ (+ "deeply_nested_day_interval_of_a_column",+ _bounded_by(_nested("toStartOfInterval({}, toIntervalDay(1))", 40)),+ ),+ ]+ )+ @pytest.mark.timeout(5, func_only=True)+ def test_refuses_a_query_that_breaks_a_condition(self, _name: str, query: str):+ assert self._can_read(query) is False++ @parameterized.expand(+ [+ ("no_lower_bound", _where(FLAG_CALLED, FLAG_KEY), True),+ ("bound_before_coverage", _bounded_by("now() - INTERVAL 180 DAY"), True),+ ("no_flag_key", _where(FLAG_CALLED, BOUND), True),+ ("no_event_pin", _where(FLAG_KEY, BOUND), False),+ ("mixed_in", _where("event IN ('$feature_flag_called', '$pageview')", FLAG_KEY, BOUND), False),+ ("sample", f"SELECT count() FROM events SAMPLE 0.1 WHERE {FLAG_CALLED} AND {FLAG_KEY}", False),+ ("select_star", f"SELECT * FROM events WHERE {FLAG_CALLED}", False),+ ]+ )+ def test_skips_coverage_and_flag_key_when_only_flag_evaluations_has_flag_calls(+ self, _name: str, query: str, expected: bool+ ):+ assert self._can_read(query, only_in_flag_evaluations=True) is expected++ @parameterized.expand(+ [+ # The trends runner's date range for "Last N days" in a day-interval insight.+ *[+ (f"last_{label}_days_in_{zone}", zone, _last_days_bound(zone, days), expected)+ for zone in ("UTC", "Pacific/Kiritimati", "Etc/GMT+12")+ for label, days, expected in (+ ("complete", FLAG_EVALUATIONS_COMPLETE_DAYS - 1, True),+ ("too_many", FLAG_EVALUATIONS_COMPLETE_DAYS + 1, False),+ )+ ],+ # Pacific/Kiritimati is UTC+14, so its midnight on the first covered day is 10:00 the day before in UTC.+ ("to_date_time_on_the_first_covered_day_in_utc", "UTC", f"toDateTime('{_text(COVERAGE_START)}')", True),+ (+ "to_date_time_on_the_first_covered_day_in_kiritimati",+ "Pacific/Kiritimati",+ f"toDateTime('{_text(COVERAGE_START)}')",+ False,+ ),+ ("date_string_on_the_first_covered_day_in_utc", "UTC", f"'{_date(COVERAGE_START)}'", True),+ (+ "date_string_on_the_first_covered_day_in_kiritimati",+ "Pacific/Kiritimati",+ f"'{_date(COVERAGE_START)}'",+ False,+ ),+ (+ "naive_placeholder_on_the_first_covered_day_in_kiritimati",+ "Pacific/Kiritimati",+ "{bound}",+ False,+ ),+ ("naive_placeholder_on_the_first_covered_day_in_utc", "UTC", "{bound}", True),+ ]+ )+ def test_reads_a_time_without_a_zone_in_the_team_timezone(+ self, _name: str, timezone: str, bound: str, expected: bool+ ):+ placeholders: dict[str, ast.Expr] = {"bound": ast.Constant(value=COVERAGE_START.replace(tzinfo=None))}++ assert self._can_read(_bounded_by(bound), placeholders, timezone) is expected++ @parameterized.expand(+ [+ (+ "date_rounding_shifted_in_a_zone_behind_utc",+ datetime(2026, 10, 24, 12, 0, tzinfo=UTC),+ "America/Los_Angeles",+ "toMonday(toDateTime('2026-08-02 22:00:00')) - INTERVAL 1 HOUR",+ ),+ # Troll is on UTC+0 now and was on UTC+2 89 days back, so subtracting 89 days lands two hours earlier+ # in UTC than 89 times 24 hours does.+ (+ "day_rounding_across_a_two_hour_dst_change",+ datetime(2026, 10, 28, 23, 30, tzinfo=UTC),+ "Antarctica/Troll",+ f"toStartOfDay(now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS} DAY)",+ ),+ ]+ )+ def test_refuses_a_bound_that_calendar_arithmetic_moves_before_coverage(+ self, _name: str, now: datetime, timezone: str, bound: str+ ):+ with time_machine.travel(now, tick=False):+ assert self._can_read(_bounded_by(bound), timezone=timezone) is Falsediff --git a/posthog/hogql/transforms/trino/validate.py b/posthog/hogql/transforms/trino/validate.pyindex 38c7210f8b3e..d8af44dc55f8 100644--- a/posthog/hogql/transforms/trino/validate.py+++ b/posthog/hogql/transforms/trino/validate.py@@ -277,11 +277,7 @@ class TrinoReadyValidator(TraversingVisitor):self._fail("TRINO_GLOBAL_JOIN_UNSUPPORTED", join_type, node)if node.table_final:self._fail("TRINO_FINAL_UNSUPPORTED", "FINAL", node)- if node.sample is not None and not (- node.sample.sample_value.left.value == 1- and node.sample.sample_value.right is None- and node.sample.offset_value is None- ):+ if node.sample is not None and not node.sample.samples_all_rows():self._fail("TRINO_SAMPLE_UNSUPPORTED", "SAMPLE", node)super().visit_join_expr(node)diff --git a/posthog/hogql/utils.py b/posthog/hogql/utils.pyindex c5567f2de380..2031320c8bbe 100644--- a/posthog/hogql/utils.py+++ b/posthog/hogql/utils.py@@ -71,6 +71,11 @@ def ilike_matches(pattern: str, text: str) -> bool:return like_matches(pattern.lower(), text.lower())+def is_json_verbatim(value: str) -> bool:+ """True when JSON text stores the string unchanged: non-empty printable ASCII with no quote or backslash."""+ return bool(value) and all(" " <= char <= "~" and char not in '"\\' for char in value)++def unwrap_optional(t):if get_origin(t) is Union and type(None) in get_args(t):# Return the first argument, which is the actual type in Optional[type]diff --git a/posthog/hogql_queries/test/test_events_query_runner.py b/posthog/hogql_queries/test/test_events_query_runner.pyindex 52d0781a404a..668d79b60cde 100644--- a/posthog/hogql_queries/test/test_events_query_runner.py+++ b/posthog/hogql_queries/test/test_events_query_runner.py@@ -1340,7 +1340,11 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):properties={"email": FLAG_EVALUATIONS_EMAIL},)create_group_type_mapping(- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0+ team=self.team,+ project_id=self.team.project_id,+ group_type="organization",+ group_type_index=0,+ created_at=FLAG_CALL_TIMESTAMP - timedelta(days=1),)create_group(team_id=self.team.pk,diff --git a/posthog/hogql_queries/test/test_hogql_cohort_query.py b/posthog/hogql_queries/test/test_hogql_cohort_query.pyindex 84c38a2e4047..fb1c9ea7cff3 100644--- a/posthog/hogql_queries/test/test_hogql_cohort_query.py+++ b/posthog/hogql_queries/test/test_hogql_cohort_query.py@@ -3,7 +3,13 @@ from typing import castfrom zoneinfo import ZoneInfoimport time_machine-from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_person, flush_persons_and_events+from posthog.test.base import (+ APIBaseTest,+ ClickhouseTestMixin,+ _create_flag_evaluations,+ _create_person,+ flush_persons_and_events,+)from unittest.mock import MagicMock, patchfrom parameterized import parameterized@@ -14,6 +20,8 @@ from posthog.hogql_queries.hogql_cohort_query import HogQLCohortQueryfrom products.cohorts.backend.models.cohort import Cohortfrom products.event_definitions.backend.models.property_definition import PropertyDefinition, PropertyType+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfigclass TestHogQLCohortQuery(ClickhouseTestMixin, APIBaseTest):@@ -614,6 +622,57 @@ class TestHogQLCohortQuery(ClickhouseTestMixin, APIBaseTest):)return {str(row[0]) for row in rows}+ def test_performed_flag_call_criterion_reads_flag_evaluations_when_the_rewrite_is_on(self) -> None:+ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=self.organization,+ defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},+ )+ caller = _create_person(team=self.team, distinct_ids=["caller"], immediate=True)+ other_caller = _create_person(team=self.team, distinct_ids=["other-caller"], immediate=True)+ flush_persons_and_events()+ _create_flag_evaluations(self.team.pk, "probe-flag", distinct_id="caller", person_id=caller.uuid)+ _create_flag_evaluations(self.team.pk, "other-flag", distinct_id="other-caller", person_id=other_caller.uuid)+ cohort = Cohort.objects.create(+ team=self.team,+ name="called probe-flag",+ filters={+ "properties": {+ "type": "AND",+ "values": [+ {+ "type": "AND",+ "values": [+ {+ "key": "$feature_flag_called",+ "type": "behavioral",+ "value": "performed_event",+ "event_type": "events",+ "time_value": 30,+ "time_interval": "day",+ "event_filters": [+ {+ "key": "$feature_flag",+ "type": "event",+ "value": "probe-flag",+ "operator": "exact",+ }+ ],+ }+ ],+ }+ ],+ }+ },+ )++ with patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-from-flag-evaluations",+ ):+ cohort.calculate_people_ch(pending_version=0)++ self.assertEqual(self._cohort_member_ids(cohort, 0), {str(caller.uuid)})+def test_filter_test_accounts_excludes_person_matches_and_tracks_team_settings(self) -> None:internal = _create_person(team=self.team,diff --git a/posthog/models/data_deletion_request.py b/posthog/models/data_deletion_request.pyindex ca751c18ecb2..498fe1b41e09 100644--- a/posthog/models/data_deletion_request.py+++ b/posthog/models/data_deletion_request.py@@ -127,6 +127,7 @@ def compile_hogql_predicate(obj, use_new_events_schema: bool = False, hogql_tabl# A deletion predicate must match rows regardless of retention; the events-retention floor would otherwise# narrow an events sub-query here and leave events older than the window un-deleted.apply_events_retention_floor=False,+ rewrite_flag_calls_to_flag_evaluations=False,)try:sql = translate_hogql(diff --git a/posthog/models/test/test_data_deletion_request.py b/posthog/models/test/test_data_deletion_request.pyindex 46e4a75e0e65..a39db70a252d 100644--- a/posthog/models/test/test_data_deletion_request.py+++ b/posthog/models/test/test_data_deletion_request.py@@ -30,6 +30,8 @@ from posthog.models.organization import Organizationfrom posthog.models.team import Teamfrom products.event_definitions.backend.models import PropertyDefinition+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfigLOCMEM_CACHE = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "ddr-test"}}@@ -205,6 +207,28 @@ def test_hogql_predicate_subquery_is_team_scoped(db):assert f"equals(team_id, {team_a.id})" not in sql_b+def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team):+ from posthog.models.data_deletion_request import compile_hogql_predicate++ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=team.organization,+ defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},+ )+ predicate = (+ "distinct_id IN (SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "+ "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY)"+ )+ request = DataDeletionRequest(**_base_kwargs(team_id=team.id, events=["$pageview"], hogql_predicate=predicate))++ with patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-from-flag-evaluations",+ ):+ sql, _ = compile_hogql_predicate(request)++ assert "flag_evaluations" not in sql++def test_compile_hogql_predicate_returns_sql_and_values(team):from posthog.models.data_deletion_request import compile_hogql_predicatediff --git a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.pyindex a8e46c46fb42..025ccc363081 100644--- a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py+++ b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py@@ -1,6 +1,7 @@import reimport zoneinfoimport itertools+from collections.abc import Callablefrom dataclasses import dataclassfrom datetime import datetime, timedeltafrom itertools import groupby@@ -12,6 +13,7 @@ from posthog.test.base import (APIBaseTest,ClickhouseTestMixin,_create_event,+ _create_flag_evaluations,_create_person,also_test_with_materialized_columns,flush_persons_and_events,@@ -83,9 +85,33 @@ from posthog.test.test_utils import create_group_type_mapping_without_created_atfrom products.actions.backend.models.action import Actionfrom products.cohorts.backend.models.cohort import Cohortfrom products.event_definitions.backend.models.property_definition import PropertyDefinition+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfigfrom products.product_analytics.backend.hogql_queries.trends.trends_query_runner import TrendsQueryRunner+def _flag_call_event_series(team: Team, flag_keys: list[str]) -> EventsNode:+ return EventsNode(+ event="$feature_flag_called",+ properties=[EventPropertyFilter(key="$feature_flag", value=flag_keys, operator=PropertyOperator.EXACT)],+ )+++def _flag_call_action_series(team: Team, flag_keys: list[str]) -> ActionsNode:+ action = Action.objects.create(+ team=team,+ name="called a flag",+ steps_json=[+ {+ "event": "$feature_flag_called",+ "properties": [{"key": "$feature_flag", "type": "event", "value": flag_key, "operator": "exact"}],+ }+ for flag_key in flag_keys+ ],+ )+ return ActionsNode(id=action.id)++@dataclassclass Series:event: str@@ -6629,6 +6655,83 @@ class TestTrendsQueryRunner(ClickhouseTestMixin, APIBaseTest):assert response.results[0]["count"] == 2assert response.results[0]["data"] == [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]+ @parameterized.expand(+ [+ ("event_series", _flag_call_event_series, ["probe-flag"], 2),+ ("event_series_with_two_flag_keys", _flag_call_event_series, ["probe-flag", "other-flag"], 3),+ ("action_series", _flag_call_action_series, ["probe-flag"], 2),+ ("action_with_two_flag_call_steps", _flag_call_action_series, ["probe-flag", "other-flag"], 3),+ ("daily_active_users", _flag_call_event_series, ["probe-flag"], 2, BaseMathType.DAU),+ (+ "breakdown_by_response",+ _flag_call_event_series,+ ["probe-flag"],+ 2,+ None,+ BreakdownFilter(breakdown="$feature_flag_response", breakdown_type=BreakdownType.EVENT),+ ),+ ("test_account_filters", _flag_call_event_series, ["probe-flag"], 2, None, None, True),+ (+ "mode_1",+ _flag_call_event_series,+ ["probe-flag"],+ 2,+ None,+ None,+ False,+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ ),+ ]+ )+ def test_flag_call_series_reads_flag_evaluations_when_the_rewrite_is_on(+ self,+ _name: str,+ series: Callable[[Team, list[str]], EventsNode | ActionsNode],+ flag_keys: list[str],+ expected_count: int,+ math: BaseMathType | None = None,+ breakdown: BreakdownFilter | None = None,+ filter_test_accounts: bool = False,+ mode: FlagEvaluationsMode = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,+ ):+ OrganizationFeatureFlagsConfig.objects.update_or_create(+ organization=self.organization, defaults={"flag_evaluations_mode": mode}+ )+ if filter_test_accounts:+ self.team.test_account_filters = [+ {"key": "email", "value": "@posthog.com", "operator": "not_icontains", "type": "person"}+ ]+ self.team.save()+ _create_flag_evaluations(self.team.pk, "probe-flag", count=2)+ _create_flag_evaluations(self.team.pk, "other-flag")+ _create_flag_evaluations(self.team.pk, "unselected-flag")+ node = series(self.team, flag_keys)+ if math is not None:+ node = node.model_copy(update={"math": math})++ with patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-from-flag-evaluations",+ ):+ # Mode 2 raises when the swapped query reads people differently, so the persons-on-events mode is fixed.+ # Mode 1 reads events on the native-JSON events table, so the events table is fixed too.+ response = self._run_trends_query(+ "-7d",+ None,+ IntervalType.DAY,+ [node],+ breakdown=breakdown,+ filter_test_accounts=filter_test_accounts,+ hogql_modifiers=HogQLQueryModifiers(+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,+ useNewEventsSchema=False,+ ),+ )++ assert response.results[0]["count"] == expected_count+ if breakdown is not None:+ assert response.results[0]["breakdown_value"] == "true"+def test_multiple_breakdowns_work_with_formula(self):self._create_test_events()flush_persons_and_events()