Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): read flag calls from flag_evaluations when a query asks events for them (#124397)

Phil Haack8e9e167cd35d
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.md
index 7c8d4f3fdd3f..a0f93d8a0a91 100644
--- a/posthog/hogql/ACCESS_CONTROL.md
+++ b/posthog/hogql/ACCESS_CONTROL.md
@@ -230,7 +230,7 @@ Several catalog tables expose such columns because reading them scans far less d
`events`' mirror columns are not masked: restricting the property they mirror masks the blob read and leaves the column readable.
`flag_evaluations` and `ai_events` mirror columns are masked through `mirrored_property_for_column()` in `posthog/hogql/restricted_properties.py`, consulted by `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.
-The mapping is keyed by the stored column name, not the HogQL field name, so an expression field that reads a mirror column masks too: `flag_evaluations.response` reads the stored `response` column through `nullIf`, and the hidden `flag_evaluation_response` reads it bare, and both mask.
+The mapping is keyed by the stored column name, not the HogQL field name, so an expression field that reads a mirror column masks too: `flag_evaluations.response` reads the stored `response` column through `nullIf`, and the hidden `flag_evaluation_response` reads it bare, and both mask. The hidden `goe_N.key` on `flag_evaluations` reads the stored `$group_N` column through an expression field, and it masks too.
For AI columns, `AI_PROPERTY_TO_COLUMN` in `posthog/hogql/database/schema/ai_events.py` is shared with the query rewriter.
One rule for `$ai_input` masks both `events.properties.$ai_input` and `ai_events.input`, including nested reads such as `input[1].content`.
The same rule applies when a filter or an aggregate reads the column.
diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py
index 45f0355e9bf4..fde4e3546b44 100644
--- a/posthog/hogql/ast.py
+++ b/posthog/hogql/ast.py
@@ -1391,12 +1391,16 @@ class RatioExpr(Expr):
right: Optional[Constant] = None
-@dataclass(kw_only=True, slots=True)
+@dataclass(kw_only=True, slots=True, frozen=False)
class SampleExpr(Expr):
# k or n
sample_value: RatioExpr
offset_value: Optional[RatioExpr] = None
+ def samples_all_rows(self) -> bool:
+ """`SAMPLE 1` with no denominator and no offset reads every row. The printers drop it."""
+ return self.sample_value.left.value == 1 and self.sample_value.right is None and self.offset_value is None
+
@dataclass(kw_only=True, slots=True)
class HogQLXAttribute(AST):
diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py
index f2e2b99d57be..9e158f2b03b9 100644
--- a/posthog/hogql/constants.py
+++ b/posthog/hogql/constants.py
@@ -30,6 +30,10 @@ INACTIVE_FEATURE_FLAG_VALUES: tuple[str, ...] = ("", "false")
FEATURE_FLAG_PROPERTY_PREFIX = "$feature/"
+# posthog/models/flag_evaluations/sql.py defines the same value as FLAG_EVALUATIONS_SOURCE_EVENT. That module imports
+# django.conf, which HogQL keeps off its import path, so the value is spelled out here.
+FLAG_CALLED_EVENT = "$feature_flag_called"
+
def is_virtual_feature_flag_key(key: str) -> bool:
"""Whether an events_json property is rebuilt from the `$feature_flags` map instead of read under its own name."""
diff --git a/posthog/hogql/context.py b/posthog/hogql/context.py
index a5c84c42b99d..81ea1073531b 100644
--- a/posthog/hogql/context.py
+++ b/posthog/hogql/context.py
@@ -172,6 +172,9 @@ class HogQLContext:
# regardless of retention — notably the GDPR data-deletion mutation path — set this False. Deliberately NOT a
# HogQLQueryModifier, so a query can't disable enforcement.
apply_events_retention_floor: bool = True
+ # Backend-only switch for transforms/flag_calls_from_flag_evaluations.py. The data-deletion path sets it False, so a
+ # flag-call subquery in a deletion predicate keeps reading `events` whatever the organization's mode.
+ rewrite_flag_calls_to_flag_evaluations: bool = True
# Backend-only opt-in for transforms/events_read_in_order.py. The prefix is slower when the range is short or
# the filter is selective, so set it only for a query shape that was measured to be faster with it.
diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py
index d65f4a15146e..29f93cc1650e 100644
--- a/posthog/hogql/database/database.py
+++ b/posthog/hogql/database/database.py
@@ -251,6 +251,9 @@ class HogQLDatabaseSources:
is_billing_usage_records_enabled: bool
# Not the flag's value: a direct-connection catalog excludes the table whatever the org has.
include_flag_evaluations_table: bool
+ read_flag_calls_from_flag_evaluations: bool
+ # The organization reads flag calls only from posthog.flag_evaluations, and ingestion no longer writes them to events.
+ flag_calls_only_in_flag_evaluations: bool
# Userless internal contexts that must resolve every warehouse table/view; skips access control
bypass_warehouse_access_control: bool
direct_connection_metadata: dict[str, Any] | None
@@ -454,7 +457,7 @@ _DATABASE_ROOT_NODE_BLOBS_LOCK = threading.Lock()
# LRU, and fresh entries survive the sweep. Reads are lock-free (dict.get is atomic under the
# GIL); mutations and the sweep hold the lock, because the sweep iterates the dict and concurrent
# inserts would raise RuntimeError mid-iteration.
-_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets"})
+_CACHEABLE_TEAM_FLAGS = frozenset({"managed-viewsets", "hogql-flag-calls-from-flag-evaluations"})
_TEAM_FLAG_CACHE: dict[tuple[str, str], tuple[float, bool]] = {} # key -> (evaluated_at, value)
_TEAM_FLAG_CACHE_LOCK = threading.Lock()
_TEAM_FLAG_CACHE_MAX_ENTRIES = 50_000
@@ -476,6 +479,16 @@ def _evaluate_warehouse_access_control_flag(team: Team) -> bool:
)
+def _evaluate_flag_calls_rewrite_flag(team: Team) -> bool:
+ return feature_enabled_or_false(
+ "hogql-flag-calls-from-flag-evaluations",
+ str(team.uuid),
+ groups={"organization": str(team.organization_id)},
+ group_properties={"organization": {"id": str(team.organization_id)}},
+ send_feature_flag_events=False,
+ )
+
+
def _cached_team_flag(flag_key: str, team: Team, evaluate: Callable[[], bool]) -> bool:
# Not cache_for/CachedFunction: its TTL is fixed at decoration time, and this TTL is a live
# instance setting so ops can retune or disable the cache without a redeploy.
@@ -829,6 +842,8 @@ class Database(BaseModel):
# because a warehouse table merged into the root first can hold the bare stored name.
_models_alias_nodes: dict[str, TableNode] = {}
_connection_id: str | None = None
+ _read_flag_calls_from_flag_evaluations: bool = False
+ _flag_calls_only_in_flag_evaluations: bool = False
_direct_connection_metadata: dict[str, Any] | None = None
_direct_access_warehouse_table_names: set[str] = set()
# Warnings about data warehouse tables (failed/paused/billing-limited/stale syncs),
@@ -890,6 +905,8 @@ class Database(BaseModel):
self._models_aliases = {}
self._models_alias_nodes = {}
self._connection_id = None
+ self._read_flag_calls_from_flag_evaluations = False
+ self._flag_calls_only_in_flag_evaluations = False
self._direct_connection_metadata = None
self._direct_access_warehouse_table_names = set()
self._data_warehouse_sync_warnings = {}
@@ -1259,6 +1276,15 @@ class Database(BaseModel):
def _is_direct_query(self) -> bool:
return self._connection_id is not None
+ @property
+ def read_flag_calls_from_flag_evaluations(self) -> bool:
+ # create_for(include_flag_evaluations_table=False) removes the table after _fetch_sources chose to read from it.
+ return self._read_flag_calls_from_flag_evaluations and self.has_table(["posthog", "flag_evaluations"])
+
+ @property
+ def flag_calls_only_in_flag_evaluations(self) -> bool:
+ return self._flag_calls_only_in_flag_evaluations and self.read_flag_calls_from_flag_evaluations
+
@staticmethod
def _is_helper_function_table(table: object) -> bool:
return isinstance(table, FunctionCallTable) and not isinstance(table, (DirectSQLTable, PostgresTable, S3Table))
@@ -1937,6 +1963,8 @@ class Database(BaseModel):
# Resolving the org gate needs a Postgres read. This path exists to avoid that I/O,
# so it prunes the table as it prunes the other gated tables.
include_flag_evaluations_table=False,
+ read_flag_calls_from_flag_evaluations=False,
+ flag_calls_only_in_flag_evaluations=False,
group_types=[],
saved_queries=[],
endpoint_saved_queries=[],
@@ -2032,10 +2060,23 @@ class Database(BaseModel):
)
# Function-local + facade-only: keeps the products off the django.setup() path.
- from products.feature_flags.backend.facade.flags import is_flag_evaluations_table_enabled # noqa: PLC0415
+ from products.feature_flags.backend.facade.enums import FlagEvaluationsMode # noqa: PLC0415
+ from products.feature_flags.backend.facade.flags import ( # noqa: PLC0415
+ get_flag_evaluations_read_mode,
+ is_flag_evaluations_table_enabled,
+ )
# A direct-connection catalog has no "posthog" node to hold the table.
include_flag_evaluations_table = not is_direct_query and is_flag_evaluations_table_enabled(team)
+ # This checks the read mode, not is_flag_evaluations_table_enabled. That one is true under DEBUG for an
+ # organization still on EVENTS. The flag goes first because it reads no row.
+ read_mode: int = FlagEvaluationsMode.EVENTS
+ if include_flag_evaluations_table and _cached_team_flag(
+ "hogql-flag-calls-from-flag-evaluations", team, lambda: _evaluate_flag_calls_rewrite_flag(team)
+ ):
+ read_mode = get_flag_evaluations_read_mode(team.organization_id)
+ read_flag_calls_from_flag_evaluations = read_mode != FlagEvaluationsMode.EVENTS
+ flag_calls_only_in_flag_evaluations = read_mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY
with timings.measure("database", emit_span=True):
# Function-local: keeps the direct-SQL driver imports off the django.setup() path.
@@ -2300,6 +2341,8 @@ class Database(BaseModel):
is_billing_usage_records_enabled="*" in settings.BILLING_USAGE_RECORDS_HOGQL_ORGANIZATION_IDS
or team.organization_id in settings.BILLING_USAGE_RECORDS_HOGQL_ORGANIZATION_IDS,
include_flag_evaluations_table=include_flag_evaluations_table,
+ read_flag_calls_from_flag_evaluations=read_flag_calls_from_flag_evaluations,
+ flag_calls_only_in_flag_evaluations=flag_calls_only_in_flag_evaluations,
# Managed warehouse is a built-in project datastore and has no warehouse-object ACL surface.
# Principals that skip warehouse access control by design:
# - synthetic users (project-wide service tokens, bypass object-level RBAC)
@@ -2353,6 +2396,8 @@ class Database(BaseModel):
week_start_day=team.week_start_day,
include_posthog_tables=sources.connection_id is None,
)
+ database._read_flag_calls_from_flag_evaluations = sources.read_flag_calls_from_flag_evaluations
+ database._flag_calls_only_in_flag_evaluations = sources.flag_calls_only_in_flag_evaluations
if sources.connection_id is not None:
database._connection_id = sources.connection_id
if sources.direct_connection_metadata is not None:
@@ -3151,7 +3196,10 @@ def _setup_group_key_fields(database: Database, group_types: list[dict[str, Any]
- if(timestamp < mapping.created_at, '', $group_N) if GroupTypeMapping exists
"""
group_mappings = {mapping["group_type_index"]: mapping for mapping in group_types}
- table = database.get_table("events")
+ tables = [database.get_table("events")]
+ # A flag call query on events can read flag_evaluations instead, so its group keys take the same cutoff.
+ if database.has_table(["posthog", "flag_evaluations"]):
+ tables.append(database.get_table(["posthog", "flag_evaluations"]))
for group_index in range(5):
field_name = f"$group_{group_index}"
@@ -3159,11 +3207,6 @@ def _setup_group_key_fields(database: Database, group_types: list[dict[str, Any]
group_mapping = group_mappings.get(group_index, None)
# If no mapping exists or the mapping predated this feature, leave the original field unchanged
if group_mapping and group_mapping["created_at"]:
- # Store the original field as a "raw" version before replacing
- original_field = table.fields[field_name]
- raw_field_name = f"_{field_name}_raw"
- table.fields[raw_field_name] = original_field.model_copy(update={"hidden": True})
-
# Must stay a datetime constant: a naive string literal would be parsed by ClickHouse in the
# project's timezone (the comparison is against toTimeZone(timestamp, <project tz>)), shifting
# the cutoff by the project's UTC offset.
@@ -3171,22 +3214,29 @@ def _setup_group_key_fields(database: Database, group_types: list[dict[str, Any]
if created_at.tzinfo is None:
created_at = created_at.replace(tzinfo=UTC)
- table.fields[field_name] = ExpressionField(
- name=field_name,
- expr=ast.Call(
- name="if",
- args=[
- ast.CompareOperation(
- left=ast.Field(chain=["timestamp"]),
- op=ast.CompareOperationOp.Lt,
- right=ast.Constant(value=created_at),
- ),
- ast.Constant(value=""),
- ast.Field(chain=[raw_field_name]),
- ],
- ),
- isolate_scope=True,
- )
+ for table in tables:
+ # Store the original field as a "raw" version before replacing
+ original_field = table.fields[field_name]
+ raw_field_name = f"_{field_name}_raw"
+ table.fields[raw_field_name] = original_field.model_copy(update={"hidden": True})
+
+ table.fields[field_name] = ExpressionField(
+ name=field_name,
+ expr=ast.Call(
+ name="if",
+ args=[
+ ast.CompareOperation(
+ left=ast.Field(chain=["timestamp"]),
+ op=ast.CompareOperationOp.Lt,
+ right=ast.Constant(value=created_at),
+ ),
+ ast.Constant(value=""),
+ ast.Field(chain=[raw_field_name]),
+ ],
+ ),
+ isolate_scope=True,
+ description=original_field.description,
+ )
def _use_virtual_fields(database: Database, modifiers: HogQLQueryModifiers, timings: HogQLTimings) -> None:
diff --git a/posthog/hogql/database/schema/flag_evaluations.py b/posthog/hogql/database/schema/flag_evaluations.py
index 0d574ef171c4..174d2aa087fe 100644
--- a/posthog/hogql/database/schema/flag_evaluations.py
+++ b/posthog/hogql/database/schema/flag_evaluations.py
@@ -1,7 +1,7 @@
from typing import TYPE_CHECKING
from posthog.hogql import ast
-from posthog.hogql.database.lazy_join_tags import GROUP_N, PERSON_DISTINCT_ID_OVERRIDES, PERSONS
+from posthog.hogql.database.lazy_join_tags import GROUP_N, PERSON_DISTINCT_ID_OVERRIDES, PERSON_DISTINCT_IDS, PERSONS
from posthog.hogql.database.models import (
DateTimeDatabaseField,
ExpressionField,
@@ -16,6 +16,7 @@ from posthog.hogql.database.models import (
VirtualTable,
)
from posthog.hogql.database.schema.person_distinct_id_overrides import PersonDistinctIdOverridesTable
+from posthog.hogql.database.schema.person_distinct_ids import PersonDistinctIdsTable
from posthog.hogql.database.schema.util.null_sentinels import null_if_sentinels
from posthog.hogql.database.schema.util.uuid import uuid_string_expr_to_uint128_expr
from posthog.hogql.parser import parse_expr
@@ -77,6 +78,32 @@ class FlagEvaluationsPersonSubTable(VirtualTable):
return FLAG_EVALUATIONS_CLICKHOUSE_TABLE
+class FlagEvaluationsGroupKeySubTable(VirtualTable):
+ """`goe_N` on events, limited to the group key. The row stores no group properties or creation time at
+ evaluation time, so `goe_N.properties` and `goe_N.created_at` do not resolve here."""
+
+ def __init__(self, group_index: int):
+ super().__init__(
+ fields={
+ "key": ExpressionField(
+ name="key", expr=ast.Field(chain=[f"_$group_{group_index}_raw"]), isolate_scope=True
+ )
+ }
+ )
+
+ def to_printed_clickhouse(self, context):
+ return FLAG_EVALUATIONS_CLICKHOUSE_TABLE
+
+ def to_printed_hogql(self):
+ return FLAG_EVALUATIONS_CLICKHOUSE_TABLE
+
+
+# The events element columns that this table defines as constants, grouped by the type of the constant.
+_ELEMENT_STRING_STAND_INS = ("elements_chain", "elements_chain_href")
+_ELEMENT_ARRAY_STAND_INS = ("elements_chain_texts", "elements_chain_ids", "elements_chain_elements")
+ELEMENT_STAND_INS = frozenset(_ELEMENT_STRING_STAND_INS + _ELEMENT_ARRAY_STAND_INS)
+
+
class FlagEvaluationsTable(Table):
description: str = (
"One row per feature flag evaluation, from the `$feature_flag_called` event. Rows are kept for 90 days. "
@@ -198,6 +225,12 @@ class FlagEvaluationsTable(Table):
"$group_2": StringDatabaseField(name="$group_2", nullable=False, description="Key of the type-2 group."),
"$group_3": StringDatabaseField(name="$group_3", nullable=False, description="Key of the type-3 group."),
"$group_4": StringDatabaseField(name="$group_4", nullable=False, description="Key of the type-4 group."),
+ # The stored keys, which goe_N.key reads, as events reads goe_N.key from its stored column. A team's group
+ # type mappings can replace $group_N with a key that is empty for calls from before the mapping existed.
+ **{
+ f"_$group_{index}_raw": StringDatabaseField(name=f"$group_{index}", nullable=False, hidden=True)
+ for index in range(GROUP_TYPES_LIMIT)
+ },
# A group property filter, such as a team's test account filter, compiles to `group_N.properties`.
# These joins let it resolve here as it does on events.
**{
@@ -210,11 +243,26 @@ class FlagEvaluationsTable(Table):
)
for index in range(GROUP_TYPES_LIMIT)
},
+ # The events names below let a query on events that reads flag calls resolve here when HogQL swaps the table.
+ "$window_id": ExpressionField(
+ name="$window_id",
+ expr=ast.Call(name="ifNull", args=[ast.Field(chain=["properties", "$window_id"]), ast.Constant(value="")]),
+ isolate_scope=True,
+ hidden=True,
+ ),
+ "event_person_id": UUIDDatabaseField(name="person_id", hidden=True),
+ "pdi": LazyJoin(
+ from_field=["distinct_id"],
+ join_table=PersonDistinctIdsTable(),
+ resolver=PERSON_DISTINCT_IDS,
+ hidden=True,
+ ),
+ **{f"goe_{index}": FlagEvaluationsGroupKeySubTable(group_index=index) for index in range(GROUP_TYPES_LIMIT)},
# The table stores no elements, and a flag call on events has none either. An element filter therefore
# resolves here and matches nothing, as it does on events. The empty array keeps its String element type.
**{
name: ExpressionField(name=name, expr=ast.Constant(value=""), hidden=True)
- for name in ("elements_chain", "elements_chain_href")
+ for name in _ELEMENT_STRING_STAND_INS
},
**{
name: ExpressionField(
@@ -224,7 +272,7 @@ class FlagEvaluationsTable(Table):
),
hidden=True,
)
- for name in ("elements_chain_texts", "elements_chain_ids", "elements_chain_elements")
+ for name in _ELEMENT_ARRAY_STAND_INS
},
}
diff --git a/posthog/hogql/database/schema/groups.py b/posthog/hogql/database/schema/groups.py
index 9802d3d2129b..bea2bfac4478 100644
--- a/posthog/hogql/database/schema/groups.py
+++ b/posthog/hogql/database/schema/groups.py
@@ -211,9 +211,12 @@ LAZY_JOIN_ALIASES = frozenset(
)
-# The outer FROM tables whose WHERE the groups subquery can reuse. The last part of a chain is the only alias the
-# outer FROM may carry.
-_PREFILTER_SOURCE_CHAINS: frozenset[tuple[str, ...]] = frozenset({("events",), ("posthog", "flag_evaluations")})
+# The outer FROM tables whose WHERE the groups subquery can reuse, with the aliases each may carry besides the last
+# part of its chain. The flag call rewrite reads flag_evaluations under the alias `events`.
+_PREFILTER_SOURCE_ALIASES: dict[tuple[str, ...], frozenset[str]] = {
+ ("events",): frozenset(),
+ ("posthog", "flag_evaluations"): frozenset({"events"}),
+}
def _guarded_aliases(context: HogQLContext, source_chain: list[str]) -> frozenset[str]:
@@ -238,9 +241,9 @@ def _outer_prefilter(
Extract a clone of the outer query's WHERE that we can safely embed inside the groups
join subquery, together with the FROM that the subquery reads. We only return it when:
- 1. The outer's `FROM` is a table in `_PREFILTER_SOURCE_CHAINS`, either unaliased or aliased
- by the last part of its chain. Anything else (custom alias like `FROM events AS e` in funnels,
- or a subquery) means the WHERE may reference symbols
+ 1. The outer's `FROM` is a table in `_PREFILTER_SOURCE_ALIASES`, either unaliased or aliased
+ by the last part of its chain or by an alias listed for it. Anything else (custom alias like
+ `FROM events AS e` in funnels, or a subquery) means the WHERE may reference symbols
(`e.timestamp`, …) that don't exist in our inner `SELECT $group_N FROM <source table>`
subquery — cloning a WHERE that names them would raise `Unable to resolve field` when the
resolver walks the clone.
@@ -275,17 +278,20 @@ def _outer_prefilter(
if not isinstance(table, ast.Field):
return None
source_chain = tuple(str(part) for part in table.chain)
- if source_chain not in _PREFILTER_SOURCE_CHAINS:
+ if source_chain not in _PREFILTER_SOURCE_ALIASES:
return None
- if select_from.alias is not None and select_from.alias != source_chain[-1]:
+ alias = select_from.alias or source_chain[-1]
+ if alias != source_chain[-1] and alias not in _PREFILTER_SOURCE_ALIASES[source_chain]:
return None
- if get_table_alias_for_lazy_join(join_to_add.lazy_join_type) != source_chain[-1]:
+ if get_table_alias_for_lazy_join(join_to_add.lazy_join_type) != alias:
return None
if not _references_timestamp(where):
return None
guarded_aliases = _guarded_aliases(context, list(source_chain))
- if any(_chain_hits_guarded_alias(chain, guarded_aliases, source_chain) for chain in find_field_chains(where)):
+ if any(
+ _chain_hits_guarded_alias(chain, guarded_aliases, source_chain, alias) for chain in find_field_chains(where)
+ ):
return None
if select_from.type is None or _references_another_table(where, select_from.type):
return None
@@ -293,11 +299,11 @@ def _outer_prefilter(
return clone_expr(where), ast.JoinExpr(table=ast.Field(chain=list(table.chain)), alias=select_from.alias)
-def _chain_hits_guarded_alias(chain: list, guarded: frozenset[str], source_chain: tuple[str, ...]) -> bool:
+def _chain_hits_guarded_alias(chain: list, guarded: frozenset[str], source_chain: tuple[str, ...], alias: str) -> bool:
# Condition #1 guarantees the outer FROM's qualifier is the source table, so a leading qualifier is
# transparent: `events.organization.x` names the same lazy join as `organization.x`. Peel it before matching,
- # or the qualified form slips past the guard. The qualifier is the full chain or its last part.
- for qualifier in (list(source_chain), [source_chain[-1]]):
+ # or the qualified form slips past the guard. The qualifier is the full chain, its last part, or the FROM alias.
+ for qualifier in (list(source_chain), [source_chain[-1]], [alias]):
if len(chain) > len(qualifier) and chain[: len(qualifier)] == qualifier:
chain = chain[len(qualifier) :]
break
diff --git a/posthog/hogql/database/schema/test/__snapshots__/test_flag_evaluations.ambr b/posthog/hogql/database/schema/test/__snapshots__/test_flag_evaluations.ambr
new file mode 100644
index 000000000000..1088fe553e84
--- /dev/null
+++ b/posthog/hogql/database/schema/test/__snapshots__/test_flag_evaluations.ambr
@@ -0,0 +1,20 @@
+# serializer version: 1
+# name: TestFlagEvaluationsTable.test_flag_call_query_on_events_prints_a_read_of_flag_evaluations
+ '''
+ SELECT count() AS `count()`
+ FROM flag_evaluations AS events
+ WHERE and(equals(events.team_id, 99999), and(equals(events.event, %(hogql_val_0)s), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0), greaterOrEquals(events.timestamp, toDateTime64(minus(now64(6, %(hogql_val_3)s), toIntervalDay(7)), 6, %(hogql_val_4)s))), in(events.flag_key, tuple(%(hogql_val_5)s)))
+ LIMIT 100 SETTINGS readonly=2,
+ max_execution_time=60,
+ allow_experimental_object_type=1,
+ max_ast_elements=4000000,
+ max_expanded_ast_elements=4000000,
+ max_bytes_before_external_group_by=0,
+ transform_null_in=1,
+ optimize_min_equality_disjunction_chain_length=4294967295,
+ optimize_rewrite_aggregate_function_with_if=0,
+ optimize_min_inequality_conjunction_chain_length=4294967295,
+ allow_experimental_join_condition=1,
+ use_hive_partitioning=0
+ '''
+# ---
diff --git a/posthog/hogql/database/schema/test/test_flag_evaluations.py b/posthog/hogql/database/schema/test/test_flag_evaluations.py
index 412836b156d4..937f82deeb6b 100644
--- a/posthog/hogql/database/schema/test/test_flag_evaluations.py
+++ b/posthog/hogql/database/schema/test/test_flag_evaluations.py
@@ -2,19 +2,32 @@ import json
import uuid
from datetime import UTC, datetime, timedelta
-from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+from posthog.test.base import (
+ BaseTest,
+ ClickhouseTestMixin,
+ _create_event,
+ cleanup_materialized_columns,
+ flush_persons_and_events,
+)
from unittest.mock import patch
+from django.conf import settings
+from django.test import override_settings
+
from parameterized import parameterized
from posthog.schema import (
DataWarehousePropertyFilter,
+ DateRange,
ElementPropertyFilter,
EventPropertyFilter,
GroupPropertyFilter,
+ HogQLFilters,
HogQLPropertyFilter,
HogQLQueryModifiers,
+ HogQLQueryResponse,
PersonPropertyFilter,
+ PersonsOnEventsMode,
PropertyGroupsMode,
PropertyOperator,
SessionPropertyFilter,
@@ -25,14 +38,17 @@ from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
from posthog.hogql.database.schema.flag_evaluations import add_events_list_fields_to_flag_evaluations
from posthog.hogql.database.schema.util.where_clause_extractor import SESSION_BUFFER_DAYS
+from posthog.hogql.errors import QueryError
from posthog.hogql.property import property_to_expr
from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.utils import is_json_verbatim
from posthog.clickhouse.client import sync_execute
from posthog.constants import AvailableFeature
from posthog.hogql_queries.events_query_runner import SELECT_STAR_FROM_EVENTS_FIELDS
from posthog.models import PropertyDefinition
from posthog.models.group.util import create_group
+from posthog.models.property import TableColumn
from posthog.models.utils import uuid7
from posthog.test.persons import create_group_type_mapping, create_person
@@ -40,12 +56,23 @@ from products.access_control.backend.models.property_access_control import Prope
from products.access_control.backend.property_access_control import PropertyAccessLevel
from products.cohorts.backend.models.cohort import Cohort
from products.cohorts.backend.models.util import insert_static_cohort
+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
+
+from ee.clickhouse.materialized_columns.columns import materialize
EVENT_PROBE = "from-event-properties"
# The sessions tables join only on a UUIDv7 session id.
SESSION_ID = str(uuid7())
+FLAG_CALL_CONJUNCTS = (
+ "event = '$feature_flag_called' AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY"
+)
+FLAG_CALLS_ON_EVENTS = f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS}"
+PAGEVIEW_CONJUNCTS = "event = '$pageview' AND timestamp >= now() - INTERVAL 7 DAY"
+
# Compares greater than 9 as a number and less than it as a string, so the assertion tells the two
# apart rather than passing under either.
NUMERIC_PROBE = 10
@@ -109,12 +136,18 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
"$session_id": SESSION_ID,
"$feature_flag_request_id": "req-456",
"$group_0": "acme",
+ "$window_id": "probe-window",
"probe": EVENT_PROBE,
"count": NUMERIC_PROBE,
},
)
- def _insert_row(self, row_uuid: uuid.UUID, properties: dict[str, str | int | None]) -> None:
+ def _insert_row(
+ self,
+ row_uuid: uuid.UUID,
+ properties: dict[str, str | int | None],
+ person_properties: dict[str, str | int] | None = None,
+ ) -> None:
# The nine typed columns are DEFAULT-computed from `properties` on the sharded table, so a
# producer cannot write them and this insert must not name them.
sync_execute(
@@ -133,7 +166,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
"probe-distinct-id",
self.occurred_at,
str(self.person_id),
- json.dumps({"email": "[email protected]", "plan": "free", "age": NUMERIC_PROBE}),
+ json.dumps(
+ person_properties
+ if person_properties is not None
+ else {"email": "[email protected]", "plan": "free", "age": NUMERIC_PROBE}
+ ),
)
],
)
@@ -175,6 +212,24 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
):
return self._execute(columns, where, row_uuids=row_uuids, placeholders=placeholders).results
+ def _query_events(
+ self,
+ query: str,
+ *,
+ rewrite: bool = True,
+ filters: HogQLFilters | None = None,
+ modifiers: HogQLQueryModifiers | None = None,
+ mode: FlagEvaluationsMode = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ) -> HogQLQueryResponse:
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization, defaults={"flag_evaluations_mode": mode}
+ )
+ with patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: rewrite and key == "hogql-flag-calls-from-flag-evaluations",
+ ):
+ return execute_hogql_query(query, team=self.team, filters=filters, modifiers=modifiers, pretty=False)
+
def _override(self, person_id: uuid.UUID, version: int, is_deleted: bool = False) -> None:
sync_execute(
"INSERT INTO person_distinct_id_overrides (team_id, distinct_id, person_id, version, is_deleted) VALUES",
@@ -201,7 +256,8 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
def test_every_field_reads_its_own_physical_column(self):
results = self._select(
- "flag_key, response, session_id, request_id, `$group_0`, person_id, person.id, person_mode, properties.probe"
+ "flag_key, response, session_id, request_id, `$group_0`, goe_0.key, `$window_id`, person_id, person.id, "
+ "person_mode, properties.probe"
)
assert results == [
@@ -211,6 +267,8 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
SESSION_ID,
"req-456",
"acme",
+ "acme",
+ "probe-window",
self.person_id,
self.person_id,
"full",
@@ -298,7 +356,9 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
merged_person_id = uuid.uuid4()
self._override(merged_person_id, version=1)
- assert self._select("person_id, person.id") == [(merged_person_id, merged_person_id)]
+ assert self._select("person_id, person.id, event_person_id") == [
+ (merged_person_id, merged_person_id, self.person_id)
+ ]
def test_person_id_follows_the_latest_override_version(self):
# A distinct_id collects one override row per merge, so the read has to pick the highest version.
@@ -390,6 +450,7 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
("session_id", "session_id", "$session_id", SESSION_ID),
("request_id", "request_id", "$feature_flag_request_id", "req-456"),
("group_key", "`$group_0`", "$group_0", "acme"),
+ ("group_key_through_goe", "goe_0.key", "$group_0", "acme"),
]
)
def test_column_copying_a_restricted_property_reads_and_filters_as_null(
@@ -458,7 +519,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
# there too, or every flag calls insight that filters out test accounts fails.
create_person(team=self.team, uuid=str(self.person_id), properties={"email": "[email protected]"})
create_group_type_mapping(
- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0
+ team=self.team,
+ project_id=self.team.project_id,
+ group_type="organization",
+ group_type_index=0,
+ created_at=self.occurred_at - timedelta(days=1),
)
create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})
_create_event(
@@ -519,7 +584,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
):
# Without the key bound, the groups subquery reads every group of the type that the team has.
create_group_type_mapping(
- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0
+ team=self.team,
+ project_id=self.team.project_id,
+ group_type="organization",
+ group_type_index=0,
+ created_at=self.occurred_at - timedelta(days=1),
)
create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})
@@ -530,6 +599,23 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
assert ("in(key," in response.clickhouse) == key_bound
assert response.results == [("software",)]
+ def test_group_join_of_a_swapped_events_scan_reads_only_the_groups_of_the_matched_rows(self):
+ create_group_type_mapping(
+ team=self.team,
+ project_id=self.team.project_id,
+ group_type="organization",
+ group_type_index=0,
+ created_at=self.occurred_at - timedelta(days=1),
+ )
+ create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})
+
+ response = self._query_events(f"SELECT group_0.properties.industry FROM events WHERE {FLAG_CALL_CONJUNCTS}")
+
+ assert response.clickhouse is not None
+ assert "FROM flag_evaluations" in response.clickhouse
+ assert "in(key," in response.clickhouse
+ assert response.results == [("software",)]
+
@parameterized.expand(
[
(
@@ -561,7 +647,11 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
# The outer WHERE bounds only the outer table's rows. It can also name the joined table. A key bound is
# correct only for a group of the outer table and a WHERE that names no other table.
create_group_type_mapping(
- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0
+ team=self.team,
+ project_id=self.team.project_id,
+ group_type="organization",
+ group_type_index=0,
+ created_at=self.occurred_at - timedelta(days=1),
)
create_group(team_id=self.team.pk, group_type_index=0, group_key="acme", properties={"industry": "software"})
_create_event(team=self.team, event="$pageview", distinct_id="probe-distinct-id", timestamp=self.occurred_at)
@@ -589,3 +679,469 @@ class TestFlagEvaluationsTable(ClickhouseTestMixin, BaseTest):
).clickhouse
assert f"toIntervalDay({SESSION_BUFFER_DAYS})" in clickhouse
+
+ def test_flag_call_query_on_events_reads_no_flag_evaluations_with_the_flag_off(self):
+ assert self._query_events(FLAG_CALLS_ON_EVENTS, rewrite=False).results == [(0,)]
+
+ @parameterized.expand(
+ [
+ ("unaliased", FLAG_CALLS_ON_EVENTS, [(1,)]),
+ (
+ "qualified_without_an_alias",
+ "SELECT count() FROM events WHERE events.event = '$feature_flag_called' "
+ "AND events.properties.$feature_flag = 'probe-flag' AND events.timestamp >= now() - INTERVAL 7 DAY",
+ [(1,)],
+ ),
+ (
+ "aliased",
+ "SELECT count() FROM events AS e WHERE e.event = '$feature_flag_called' "
+ "AND e.properties.$feature_flag = 'probe-flag' AND e.timestamp >= now() - INTERVAL 7 DAY",
+ [(1,)],
+ ),
+ (
+ "alias_named_flag_key_read_back",
+ "SELECT properties.$feature_flag AS flag_key, count() FROM events "
+ f"WHERE {FLAG_CALL_CONJUNCTS} GROUP BY flag_key",
+ [("probe-flag", 1)],
+ ),
+ ("element_column", f"SELECT elements_chain FROM events WHERE {FLAG_CALL_CONJUNCTS}", [("",)]),
+ ("subquery_in_from", f"SELECT count() FROM (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS})", [(1,)]),
+ (
+ "cte",
+ f"WITH calls AS (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}) SELECT count() FROM calls",
+ [(1,)],
+ ),
+ (
+ "body_of_a_cte_named_events",
+ f"WITH events AS (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}) SELECT count() FROM events",
+ [(1,)],
+ ),
+ (
+ "subquery_in_an_expression",
+ f"SELECT count() FROM events WHERE {PAGEVIEW_CONJUNCTS} "
+ f"AND distinct_id IN (SELECT distinct_id FROM events WHERE {FLAG_CALL_CONJUNCTS})",
+ [(1,)],
+ ),
+ (
+ "union_members",
+ f"SELECT 'calls', count() FROM events WHERE {FLAG_CALL_CONJUNCTS} "
+ f"UNION ALL SELECT 'pageviews', count() FROM events WHERE {PAGEVIEW_CONJUNCTS}",
+ [("calls", 1), ("pageviews", 1)],
+ ),
+ (
+ "filters_placeholder",
+ "SELECT count() FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'probe-flag' AND {filters}",
+ [(1,)],
+ HogQLFilters(dateRange=DateRange(date_from="-7d")),
+ ),
+ ]
+ )
+ def test_each_select_on_events_reads_flag_evaluations_on_its_own(
+ self, _name: str, query: str, expected: list[tuple], filters: HogQLFilters | None = None
+ ):
+ _create_event(team=self.team, event="$pageview", distinct_id="probe-distinct-id", timestamp=self.occurred_at)
+ flush_persons_and_events()
+
+ assert sorted(self._query_events(query, filters=filters).results) == expected
+
+ @parameterized.expand(
+ [
+ ("window_id", "$window_id"),
+ ("event_person_id", "event_person_id"),
+ ("pdi", "pdi.person_id"),
+ ("group_key", "goe_0.key"),
+ ("group_properties", "group_0.properties.industry"),
+ ]
+ )
+ def test_query_reading_an_events_only_name_reads_flag_evaluations(self, _name: str, field: str):
+ # pdi is an inner join on distinct_id, as on events.
+ create_person(team=self.team, distinct_ids=["probe-distinct-id"])
+ self._create_events_only_flag_call()
+
+ results = self._query_events(f"SELECT distinct_id, {field} FROM events WHERE {FLAG_CALL_CONJUNCTS}").results
+
+ assert [row[0] for row in results] == ["probe-distinct-id"]
+
+ @parameterized.expand(
+ [
+ (
+ "event_time_group_properties",
+ f"SELECT distinct_id, goe_0.properties FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ "reads something that table doesn't have",
+ ),
+ ("select_star", f"SELECT * FROM events WHERE {FLAG_CALL_CONJUNCTS}", "selects \\* or COLUMNS"),
+ (
+ "with_name_that_a_flag_evaluations_column_shadows",
+ f"WITH 'all calls' AS flag_key SELECT flag_key FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ "a name in it would read a different column",
+ ),
+ (
+ "columns_inside_an_expression",
+ f"SELECT tuple(*COLUMNS('^(created|inserted)_at$')) FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ "a name in it would read a different column",
+ ),
+ (
+ "stored_person_id",
+ f"SELECT person_id FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ "a name in it would read a different column",
+ PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
+ ),
+ (
+ "person_id_through_distinct_ids",
+ f"SELECT person_id FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ "a name in it would read a different column",
+ PersonsOnEventsMode.DISABLED,
+ ),
+ (
+ "current_person_properties",
+ f"SELECT person.properties.email FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ "a name in it would read a different column",
+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ ),
+ ]
+ )
+ def test_query_flag_evaluations_cannot_answer_reads_events_until_events_gets_no_flag_calls(
+ self,
+ _name: str,
+ query: str,
+ error: str,
+ persons_on_events_mode: PersonsOnEventsMode = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
+ ):
+ modifiers = HogQLQueryModifiers(personsOnEventsMode=persons_on_events_mode)
+ mode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS
+ assert (
+ self._query_events(query, modifiers=modifiers, mode=mode).clickhouse
+ == self._query_events(query, rewrite=False, modifiers=modifiers, mode=mode).clickhouse
+ )
+ with self.assertRaisesRegex(QueryError, error):
+ self._query_events(query, modifiers=modifiers)
+
+ @parameterized.expand(
+ [
+ (
+ "select_star",
+ "WITH events AS (SELECT event, distinct_id FROM posthog.flag_evaluations) "
+ "SELECT * FROM events WHERE event = '$feature_flag_called'",
+ ),
+ (
+ "flag_call_filters",
+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "
+ f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ ),
+ (
+ "root_cte_in_a_union_branch",
+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "
+ f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS} "
+ f"UNION ALL SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ ),
+ (
+ "subquery",
+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "
+ f"SELECT count() FROM (SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS})",
+ ),
+ (
+ "select_star_excluding_flag_calls",
+ "SELECT * FROM events WHERE event != '$feature_flag_called' AND timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ (
+ "select_star_of_flag_calls_and_other_events",
+ "SELECT * FROM events WHERE event IN ('$feature_flag_called', '$pageview') "
+ "AND timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ ]
+ )
+ def test_query_that_does_not_read_only_flag_calls_from_events_is_unchanged_once_events_gets_no_flag_calls(
+ self, _name: str, query: str
+ ):
+ assert self._query_events(query).clickhouse == self._query_events(query, rewrite=False).clickhouse
+
+ @parameterized.expand(
+ [
+ ("no_lower_bound", "SELECT distinct_id FROM events WHERE event = '$feature_flag_called'"),
+ (
+ "bound_before_coverage",
+ "SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 180 DAY",
+ ),
+ ]
+ )
+ def test_query_reads_flag_evaluations_without_coverage_once_events_gets_no_flag_calls(self, _name: str, query: str):
+ self._create_events_only_flag_call()
+
+ on_events = self._query_events(query, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS).results
+ on_flag_evaluations = self._query_events(query).results
+
+ assert [row[0] for row in on_events] == ["events-distinct-id"]
+ assert [row[0] for row in on_flag_evaluations] == ["probe-distinct-id"]
+
+ def _create_events_only_flag_call(self) -> None:
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id="events-distinct-id",
+ timestamp=self.occurred_at,
+ properties={"$feature_flag": "probe-flag", "$window_id": "probe-window"},
+ )
+ create_person(team=self.team, distinct_ids=["events-distinct-id"])
+ flush_persons_and_events()
+
+ @parameterized.expand(
+ [
+ ("string", "variant-a"),
+ ("escaped_characters", 'say "hi" \\ bye'),
+ ("number_as_string", "100"),
+ ("number", 100),
+ ("boolean", True),
+ (
+ "quoted_flag_key_read_through_json_extract_string",
+ "variant-a",
+ 'say "hi"',
+ "JSONExtractString(properties, '$feature_flag')",
+ ),
+ ]
+ )
+ def test_swapped_query_returns_what_events_returns_for_the_same_flag_calls(
+ self,
+ _name: str,
+ response: str | int,
+ flag_key: str = "parity-flag",
+ flag_key_read: str = "properties.$feature_flag",
+ ):
+ # Mode 1 writes each flag call to both tables, so the two reads must agree row for row.
+ row_uuid = uuid.uuid4()
+ properties: dict[str, str | int | None] = {"$feature_flag": flag_key, "$feature_flag_response": response}
+ self._insert_row(row_uuid, properties)
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id="probe-distinct-id",
+ timestamp=self.occurred_at,
+ properties=properties,
+ event_uuid=str(row_uuid),
+ person_id=self.person_id,
+ )
+ # The override is a merge after the call, so each table must correct the person it stored.
+ merged_person_id = uuid.uuid4()
+ self._override(merged_person_id, version=1)
+ flush_persons_and_events()
+ query = (
+ "SELECT uuid, distinct_id, person_id, properties.$feature_flag_response FROM events "
+ f"WHERE event = '$feature_flag_called' AND {flag_key_read} = '{flag_key}' "
+ "AND timestamp >= now() - INTERVAL 7 DAY"
+ )
+ modifiers = HogQLQueryModifiers(personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS)
+
+ from_events = self._query_events(query, rewrite=False, modifiers=modifiers).results
+ swapped = self._query_events(query, modifiers=modifiers)
+
+ assert len(from_events) == 1
+ assert from_events[0][2] == merged_person_id
+ assert swapped.clickhouse is not None
+ assert "FROM flag_evaluations" in swapped.clickhouse
+ if (
+ settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA
+ and isinstance(response, str)
+ and not is_json_verbatim(response)
+ ):
+ # The native-JSON events table decodes JSON escapes in a property value. flag_evaluations returns the
+ # escaped text, as the legacy events table does.
+ assert [row[:3] for row in swapped.results] == [row[:3] for row in from_events]
+ assert from_events[0][3] == response
+ assert swapped.results[0][3] == json.dumps(response)[1:-1]
+ return
+ assert swapped.results == from_events
+
+ def test_group_key_of_a_call_from_before_its_group_type_existed_reads_as_on_events(self):
+ create_group_type_mapping(
+ team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0
+ )
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id="probe-distinct-id",
+ timestamp=self.occurred_at,
+ properties={"$feature_flag": "probe-flag", "$group_0": "acme"},
+ event_uuid=str(self.row_uuid),
+ )
+ flush_persons_and_events()
+ query = f"SELECT `$group_0`, goe_0.key FROM events WHERE {FLAG_CALL_CONJUNCTS}"
+
+ from_events = self._query_events(query, rewrite=False).results
+
+ assert from_events == [("", "acme")]
+ assert self._query_events(query).results == from_events
+
+ def test_swapped_prewhere_that_reads_an_overridden_person_id_runs(self):
+ response = self._query_events(
+ f"SELECT count() FROM events PREWHERE person_id IS NOT NULL WHERE {FLAG_CALL_CONJUNCTS}",
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ )
+
+ assert response.results == [(1,)]
+
+ @parameterized.expand(
+ [
+ # The events-only call has the JSON value true as its key. A typed comparison matches it on events.
+ ("mode_1", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, True),
+ ("mode_2", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY, False),
+ ]
+ )
+ def test_flag_key_typed_as_boolean_selects_the_same_calls_with_and_without_the_rewrite(
+ self, _name: str, mode: FlagEvaluationsMode, with_events_only_call: bool
+ ):
+ PropertyDefinition.objects.create(
+ team=self.team, name="$feature_flag", property_type="Boolean", type=PropertyDefinition.Type.EVENT
+ )
+ row_uuid = uuid.uuid4()
+ self._insert_row(row_uuid, {"$feature_flag": "true"})
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id="probe-distinct-id",
+ timestamp=self.occurred_at,
+ properties={"$feature_flag": "true"},
+ event_uuid=str(row_uuid),
+ )
+ if with_events_only_call:
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id="events-distinct-id",
+ timestamp=self.occurred_at,
+ properties={"$feature_flag": True},
+ )
+ flush_persons_and_events()
+ query = (
+ "SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'TRUE' AND timestamp >= now() - INTERVAL 7 DAY"
+ )
+
+ from_events = self._query_events(query, rewrite=False, mode=mode).results
+
+ swapped = self._query_events(query, mode=mode)
+
+ assert len(from_events) == (2 if with_events_only_call else 1)
+ assert sorted(swapped.results) == sorted(from_events)
+ if mode == FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY:
+ assert swapped.clickhouse is not None
+ assert "FROM flag_evaluations" in swapped.clickhouse
+
+ @parameterized.expand(
+ [
+ # The legacy events table returns the escaped text, as flag_evaluations does, so neither read matches.
+ ("legacy_events", False, 0),
+ ("native_json_events", True, 1),
+ ]
+ )
+ def test_filter_on_an_escaped_property_selects_the_same_calls_with_and_without_the_rewrite(
+ self, _name: str, use_new_events_schema: bool, expected_count: int
+ ):
+ row_uuid = uuid.uuid4()
+ properties: dict[str, str | int | None] = {"$feature_flag": "probe-flag", "message": 'say "hi"'}
+ self._insert_row(row_uuid, properties)
+ query = f"SELECT count() FROM events WHERE {FLAG_CALL_CONJUNCTS} AND properties.message = 'say \"hi\"'"
+
+ with override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=use_new_events_schema):
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id="probe-distinct-id",
+ timestamp=self.occurred_at,
+ properties=properties,
+ event_uuid=str(row_uuid),
+ )
+ flush_persons_and_events()
+ from_events = self._query_events(query, rewrite=False, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS)
+ rewritten = self._query_events(query, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS)
+
+ assert from_events.results == [(expected_count,)]
+ assert rewritten.results == from_events.results
+ assert rewritten.clickhouse is not None
+ assert ("FROM flag_evaluations" in rewritten.clickhouse) is not use_new_events_schema
+
+ @parameterized.expand(
+ [
+ ("no_materialized_column", None, "isNull(properties.$current_url)", True),
+ ("materialized_event_property", ("properties", "$current_url"), "isNull(properties.$current_url)", False),
+ ("materialized_person_property", ("person_properties", "email"), "isNull(person.properties.email)", False),
+ # The pin keeps the flag key away from "" and "null", the values a materialized column reads differently.
+ ("materialized_flag_key", ("properties", "$feature_flag"), "isNull(properties.$current_url)", True),
+ ]
+ )
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
+ def test_property_read_from_a_materialized_column_on_events_keeps_reading_events_in_mode_1(
+ self, _name: str, materialized_property: tuple[TableColumn, str] | None, condition: str, swaps: bool
+ ):
+ if materialized_property is not None:
+ table_column, property_name = materialized_property
+ materialize("events", property_name, table_column=table_column)
+ self.addCleanup(cleanup_materialized_columns)
+ for value in ("", "null"):
+ row_uuid = uuid.uuid4()
+ properties: dict[str, str | int | None] = {"$feature_flag": "parity-flag", "$current_url": value}
+ self._insert_row(row_uuid, properties, person_properties={"email": value})
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id="probe-distinct-id",
+ timestamp=self.occurred_at,
+ properties=properties,
+ person_properties={"email": value},
+ person_id=self.person_id,
+ event_uuid=str(row_uuid),
+ )
+ flush_persons_and_events()
+ query = (
+ "SELECT count() FROM events WHERE event = '$feature_flag_called' "
+ f"AND properties.$feature_flag = 'parity-flag' AND timestamp >= now() - INTERVAL 7 DAY AND {condition}"
+ )
+ modifiers = HogQLQueryModifiers(personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS)
+ mode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS
+
+ from_events = self._query_events(query, rewrite=False, modifiers=modifiers, mode=mode)
+ rewritten = self._query_events(query, modifiers=modifiers, mode=mode)
+
+ assert rewritten.results == from_events.results
+ assert rewritten.clickhouse is not None
+ assert ("FROM flag_evaluations" in rewritten.clickhouse) is swaps
+
+ @parameterized.expand(
+ [
+ ("flag_calls", f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}", "saved_calls"),
+ ("view_of_a_flag_call_view", "SELECT uuid FROM inner_calls", "inner_calls"),
+ ("pageviews", f"SELECT uuid FROM events WHERE {PAGEVIEW_CONJUNCTS}", None),
+ (
+ "cte_named_events",
+ "WITH events AS (SELECT uuid, event, properties, timestamp FROM posthog.flag_evaluations) "
+ f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}",
+ None,
+ ),
+ ]
+ )
+ def test_view_that_reads_flag_calls_from_events_raises_once_events_gets_no_flag_calls(
+ self, _name: str, view_query: str, failing_view: str | None
+ ):
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="inner_calls",
+ query={"query": f"SELECT uuid FROM events WHERE {FLAG_CALL_CONJUNCTS}"},
+ columns={"uuid": "String"},
+ )
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team, name="saved_calls", query={"query": view_query}, columns={"uuid": "String"}
+ )
+ query = "SELECT count() FROM saved_calls"
+
+ # events still gets flag calls on mode 1, so the view's read of events is complete.
+ self._query_events(query, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS)
+ if failing_view is None:
+ self._query_events(query)
+ return
+ with self.assertRaisesRegex(QueryError, f"reads the saved view {failing_view}, which reads them from events"):
+ self._query_events(query)
+
+ def test_flag_call_query_on_events_prints_a_read_of_flag_evaluations(self):
+ self.assertQueryMatchesSnapshot(self._query_events(FLAG_CALLS_ON_EVENTS).clickhouse)
diff --git a/posthog/hogql/database/test/test_database.py b/posthog/hogql/database/test/test_database.py
index 70fd037cfe55..b0f9a5ce6e1c 100644
--- a/posthog/hogql/database/test/test_database.py
+++ b/posthog/hogql/database/test/test_database.py
@@ -19,7 +19,7 @@ from django.db import connection
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
-from parameterized import parameterized
+from parameterized import param, parameterized
from pydantic import BaseModel
from posthog.schema import (
@@ -101,6 +101,8 @@ from products.access_control.backend.models.access_control import AccessControl
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.data_tools.backend.models.expression import DataWarehouseExpression
from products.data_tools.backend.models.join import DataWarehouseJoin
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
from products.revenue_analytics.backend.views import RevenueAnalyticsChargeView
from products.revenue_analytics.backend.views.core import SourceHandle
from products.revenue_analytics.backend.views.orchestrator import build_revenue_views_for_handles
@@ -479,6 +481,76 @@ class TestDatabase(BaseTest, QueryMatchingTest):
with pytest.raises(QueryError, match="Unknown table"):
execute_hogql_query(query, team=self.team, pretty=False)
+ @parameterized.expand(
+ [
+ ("events_mode", FlagEvaluationsMode.EVENTS, True, False),
+ # DEBUG puts the table in the catalog on EVENTS, but events still holds every flag call.
+ param("events_mode_under_debug", FlagEvaluationsMode.EVENTS, True, False, debug=True),
+ param(
+ "events_mode_with_the_table_forced_in",
+ FlagEvaluationsMode.EVENTS,
+ True,
+ False,
+ include_flag_evaluations_table=True,
+ ),
+ ("read_mode", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, True, True),
+ ("read_mode_with_the_flag_off", FlagEvaluationsMode.READ_FLAG_EVALUATIONS, False, False),
+ param(
+ "read_mode_with_reads_forced_to_events",
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ True,
+ False,
+ reads_forced_to_events=True,
+ ),
+ ("only_mode", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY, True, True),
+ ("only_mode_with_the_flag_off", FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY, False, False),
+ # The switch skips FLAG_EVALUATIONS_ONLY, whose flag calls are no longer in events.
+ param(
+ "only_mode_with_reads_forced_to_events",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ True,
+ True,
+ reads_forced_to_events=True,
+ ),
+ param(
+ "only_mode_without_the_table",
+ FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ True,
+ False,
+ include_flag_evaluations_table=False,
+ ),
+ ]
+ )
+ def test_read_flag_calls_from_flag_evaluations_follows_the_read_mode_and_the_flag(
+ self,
+ _name: str,
+ mode: FlagEvaluationsMode,
+ flag_enabled: bool,
+ expected: bool,
+ reads_forced_to_events: bool = False,
+ debug: bool = False,
+ include_flag_evaluations_table: bool | None = None,
+ ) -> None:
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization, defaults={"flag_evaluations_mode": mode}
+ )
+
+ with (
+ override_settings(DEBUG=debug),
+ override_instance_config("FLAG_EVALUATIONS_READS_FORCE_EVENTS", reads_forced_to_events),
+ patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: (
+ flag_enabled and key == "hogql-flag-calls-from-flag-evaluations"
+ ),
+ ),
+ ):
+ database = Database.create_for(
+ team=self.team, include_flag_evaluations_table=include_flag_evaluations_table
+ )
+
+ assert database.read_flag_calls_from_flag_evaluations is expected
+
def test_serialize_database_without_fields_matches_full_metadata(self):
credential = DataWarehouseCredential.objects.create(access_key="blah", access_secret="blah", team=self.team)
warehouse_table = DataWarehouseTable.objects.create(
diff --git a/posthog/hogql/flag_called_warnings.py b/posthog/hogql/flag_called_warnings.py
index c0026a48a604..d1970a4462bb 100644
--- a/posthog/hogql/flag_called_warnings.py
+++ b/posthog/hogql/flag_called_warnings.py
@@ -1,14 +1,15 @@
from posthog.schema import HogQLNotice
from posthog.hogql import ast
+from posthog.hogql.constants import FLAG_CALLED_EVENT
from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import Table
from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable
from posthog.hogql.resolver import resolve_types
from posthog.hogql.visitor import TraversingVisitor, clone_expr
-# posthog/models/flag_evaluations/sql.py defines this name too. It imports django.conf, which this module keeps
-# off its import path, so the name is spelled out here.
-FLAG_CALLED_EVENT = "$feature_flag_called"
+from posthog.dataclasses import frozen
# Orgs that can't query posthog.flag_evaluations yet get this warning too, so the copy speaks of the move in the future.
FLAG_CALLED_ON_EVENTS_WARNING = (
@@ -17,10 +18,16 @@ FLAG_CALLED_ON_EVENTS_WARNING = (
"posthog.flag_evaluations instead."
)
+FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE = (
+ f"This query reads {FLAG_CALLED_EVENT} events from posthog.flag_evaluations, which keeps the last 90 days "
+ "of flag calls."
+)
+
-class _FlagCalledOnEventsFinder(TraversingVisitor):
- def __init__(self) -> None:
+class _FlagCalledLiteralFinder(TraversingVisitor):
+ def __init__(self, table_class: type[Table]) -> None:
super().__init__()
+ self.table_class = table_class
self.literals: list[ast.Constant] = []
def visit_select_query(self, node: ast.SelectQuery) -> None:
@@ -43,12 +50,12 @@ class _FlagCalledOnEventsFinder(TraversingVisitor):
def visit_compare_operation(self, node: ast.CompareOperation) -> None:
if node.op in (ast.CompareOperationOp.Eq, ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn):
for field_side, value_side in ((node.left, node.right), (node.right, node.left)):
- if _is_events_event_field(field_side):
+ if _is_event_field_of(field_side, self.table_class):
self.literals.extend(_flag_called_constants(value_side))
super().visit_compare_operation(node)
-def _is_events_event_field(expr: ast.Expr) -> bool:
+def _is_event_field_of(expr: ast.Expr, table_class: type[Table]) -> bool:
while isinstance(expr, ast.Alias):
expr = expr.expr
if not isinstance(expr, ast.Field):
@@ -62,7 +69,7 @@ def _is_events_event_field(expr: ast.Expr) -> bool:
table_type = field_type.table_type
while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
table_type = table_type.table_type
- return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)
+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, table_class)
def _flag_called_constants(expr: ast.Expr) -> list[ast.Constant]:
@@ -84,7 +91,7 @@ def flag_called_on_events_warnings(
# A caller that passes an already printed AST never builds the database, and its AST may be rewritten.
if context.database is None:
return []
- finder = _FlagCalledOnEventsFinder()
+ finder = _FlagCalledLiteralFinder(EventsTable)
with context.timings.measure("flag_called_warnings"):
finder.visit(resolve_types(clone_expr(node), context, dialect="clickhouse"))
# A literal that a variable supplies has no span, and a notice without one marks the whole query.
@@ -93,3 +100,16 @@ def flag_called_on_events_warnings(
for literal in finder.literals
if literal.start is not None and literal.end is not None
]
+
+
+@frozen
+class LiteralSpan:
+ start: int | None
+ end: int | None
+
+
+def flag_called_spans_read_from_flag_evaluations(prepared: ast.AST) -> set[LiteralSpan]:
+ """The spans of the `$feature_flag_called` literals that a prepared AST compares to `flag_evaluations.event`."""
+ finder = _FlagCalledLiteralFinder(FlagEvaluationsTable)
+ finder.visit(prepared)
+ return {LiteralSpan(start=literal.start, end=literal.end) for literal in finder.literals}
diff --git a/posthog/hogql/metadata.py b/posthog/hogql/metadata.py
index db4988f367ac..0e09d279aa9d 100644
--- a/posthog/hogql/metadata.py
+++ b/posthog/hogql/metadata.py
@@ -25,7 +25,7 @@ from posthog.schema import (
from posthog.hogql import ast
from posthog.hogql.base import AST
from posthog.hogql.compiler.bytecode import create_bytecode
-from posthog.hogql.constants import HogQLDialect
+from posthog.hogql.constants import FLAG_CALLED_EVENT, HogQLDialect
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
from posthog.hogql.direct_connection import INVALID_CONNECTION_ID_ERROR, get_direct_connection_source
@@ -35,7 +35,12 @@ from posthog.hogql.errors import (
NotImplementedError as HogQLNotImplementedError,
)
from posthog.hogql.filters import replace_filters
-from posthog.hogql.flag_called_warnings import FLAG_CALLED_EVENT, flag_called_on_events_warnings
+from posthog.hogql.flag_called_warnings import (
+ FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE,
+ LiteralSpan,
+ flag_called_on_events_warnings,
+ flag_called_spans_read_from_flag_evaluations,
+)
from posthog.hogql.index_eligibility import IndexEligibilityReport, build_index_eligibility_report
from posthog.hogql.metadata_heuristics import run_metadata_heuristics
from posthog.hogql.modifiers import create_default_modifiers_for_team
@@ -53,6 +58,7 @@ from posthog.hogql.type_system import runtime_type_from_constant_type
from posthog.hogql.variables import replace_variables
from posthog.hogql.visitor import TraversingVisitor, clone_expr
+from posthog.dataclasses import frozen
from posthog.hogql_queries.query_runner import get_query_runner
from posthog.models import Team
from posthog.models.user import User
@@ -217,7 +223,9 @@ def get_hogql_metadata(
response.ch_table_names = get_table_names(prepared_ast)
if source is None and FLAG_CALLED_EVENT in query.query:
- heuristic_warnings.extend(_flag_called_on_events_warnings(hogql_ast, context, team))
+ flag_called_notices = _flag_called_notices(hogql_ast, prepared_ast, context, team)
+ heuristic_warnings.extend(flag_called_notices.move_warnings)
+ context.notices.extend(flag_called_notices.flag_evaluations_notices)
if source is None and query.indexUsage and _index_usage_enabled(team):
_attach_index_usage(response, hogql_ast, context)
@@ -325,23 +333,47 @@ def _flag_called_announcement_url(payload: object) -> str | None:
return None
-def _flag_called_on_events_warnings(
- hogql_ast: Union[ast.SelectQuery, ast.SelectSetQuery], context: HogQLContext, team: Team
-) -> list[HogQLNotice]:
+@frozen
+class _FlagCalledNotices:
+ move_warnings: tuple[HogQLNotice, ...] = ()
+ flag_evaluations_notices: tuple[HogQLNotice, ...] = ()
+
+
+def _flag_called_notices(
+ hogql_ast: Union[ast.SelectQuery, ast.SelectSetQuery],
+ prepared_ast: ast.AST | None,
+ context: HogQLContext,
+ team: Team,
+) -> _FlagCalledNotices:
+ """The move warning on each flag-call literal that reads events, and a notice on each literal whose query
+ reads posthog.flag_evaluations instead. `prepared_ast` is what the printer ran, so it says which scans it
+ swapped."""
try:
move_notices = _flag_called_move_notices(team)
if move_notices is None or not move_notices.enabled:
- return []
+ return _FlagCalledNotices()
warnings = flag_called_on_events_warnings(hogql_ast, context)
if not warnings:
- return []
+ return _FlagCalledNotices()
url = _flag_called_announcement_url(move_notices.payload)
- return [warning.model_copy(update={"url": url}) for warning in warnings]
+ swapped = flag_called_spans_read_from_flag_evaluations(prepared_ast) if prepared_ast is not None else set()
+ move_warnings: list[HogQLNotice] = []
+ flag_evaluations_notices: list[HogQLNotice] = []
+ for warning in warnings:
+ if LiteralSpan(start=warning.start, end=warning.end) in swapped:
+ flag_evaluations_notices.append(
+ warning.model_copy(update={"message": FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, "url": url})
+ )
+ else:
+ move_warnings.append(warning.model_copy(update={"url": url}))
+ return _FlagCalledNotices(
+ move_warnings=tuple(move_warnings), flag_evaluations_notices=tuple(flag_evaluations_notices)
+ )
except Exception:
# The warning is advisory. A query that compiles must not be reported as invalid because this
# check failed, and the caller turns any exception here into an invalid query.
logger.exception("hogql_flag_called_warning_failed", team_id=context.team_id)
- return []
+ return _FlagCalledNotices()
def _attach_index_usage(
diff --git a/posthog/hogql/printer/base.py b/posthog/hogql/printer/base.py
index 44be235b7668..a7500df893a7 100644
--- a/posthog/hogql/printer/base.py
+++ b/posthog/hogql/printer/base.py
@@ -1356,8 +1356,7 @@ class BasePrinter(Visitor[str]):
return self._unsafe_json_extract_trim_quotes(self.visit(node.expr), self._json_property_args(node.keys))
def visit_sample_expr(self, node: ast.SampleExpr) -> Optional[str]:
- # SAMPLE 1 means no sampling, skip it entirely
- if node.sample_value.left.value == 1 and node.sample_value.right is None and node.offset_value is None:
+ if node.samples_all_rows():
return None
sample_value = self.visit_ratio_expr(node.sample_value)
diff --git a/posthog/hogql/printer/clickhouse.py b/posthog/hogql/printer/clickhouse.py
index 93a312db3bed..1600924c6e2a 100644
--- a/posthog/hogql/printer/clickhouse.py
+++ b/posthog/hogql/printer/clickhouse.py
@@ -1242,11 +1242,13 @@ class ClickHousePrinter(BasePrinter):
node_type: ast.TableOrSelectType | None,
) -> ast.Expr | None:
from posthog.hogql.database.schema.events import EventsTable
+ from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable
months = self.context.events_retention_months
if months is None or node_type is None or not isinstance(table_type, ast.TableType):
return None
- if not isinstance(table_type.table, EventsTable):
+ # flag_evaluations holds flag calls that are events too, and a query on events can read them from there.
+ if not isinstance(table_type.table, (EventsTable, FlagEvaluationsTable)):
return None
return retention_floor_for_table(node_type, months)
diff --git a/posthog/hogql/printer/test/test_printer.py b/posthog/hogql/printer/test/test_printer.py
index b63317264bc7..63fd13f4814f 100644
--- a/posthog/hogql/printer/test/test_printer.py
+++ b/posthog/hogql/printer/test/test_printer.py
@@ -227,14 +227,21 @@ class TestPrinter(BaseTest):
)
return printed
- @parameterized.expand([(12,), (84,)])
+ @parameterized.expand(
+ [
+ (12, "events", "events"),
+ (84, "events", "events"),
+ (12, "posthog.flag_evaluations", "flag_evaluations"),
+ ]
+ )
@override_settings(EVENTS_DATA_RETENTION_ENFORCED=True)
- def test_events_retention_floor_applied(self, months: int):
+ def test_events_retention_floor_applied(self, months: int, table: str, printed_table: str):
self.team.event_retention_months = months
context = HogQLContext(team_id=self.team.pk, team=self.team, enable_select_queries=True)
- sql = self._select("select count() from events", context=context)
+ with patch("products.feature_flags.backend.facade.flags.is_flag_evaluations_table_enabled", return_value=True):
+ sql = self._select(f"select count() from {table}", context=context)
assert f"toIntervalMonth({months})" in sql
- assert "greater(events.timestamp, minus(now" in sql
+ assert f"greater({printed_table}.timestamp, minus(now" in sql
@override_settings(EVENTS_DATA_RETENTION_ENFORCED=False)
def test_events_retention_floor_not_applied_when_disabled(self):
diff --git a/posthog/hogql/printer/utils.py b/posthog/hogql/printer/utils.py
index 8b4ee4776c15..1fbabcce1517 100644
--- a/posthog/hogql/printer/utils.py
+++ b/posthog/hogql/printer/utils.py
@@ -1,4 +1,5 @@
from collections.abc import Callable
+from functools import partial
from typing import TYPE_CHECKING, cast
from posthog.schema_enums import InCohortVia, PropertyGroupsMode
@@ -32,6 +33,7 @@ from posthog.hogql.printer.snowflake import SnowflakePrinter
from posthog.hogql.resolver import ResolverFactory, resolve_types
from posthog.hogql.transforms.events_predicate_pushdown import apply_events_predicate_pushdown, events_pushdown_enabled
from posthog.hogql.transforms.events_read_in_order import order_events_reads_by_sort_key
+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import resolve_flag_calls_from_flag_evaluations
from posthog.hogql.transforms.in_cohort import resolve_in_cohorts, resolve_in_cohorts_conjoined
from posthog.hogql.transforms.json_property_pushdown import (
has_rewritable_json_extract,
@@ -260,13 +262,17 @@ def prepare_ast_for_printing(
resolve_in_cohorts_conjoined(node, dialect, context, stack, resolver_factory=resolver_factory)
with context.timings.measure("resolve_types"):
- node = resolve_types(
- node,
- context,
+ resolve = partial(
+ resolve_types,
+ context=context,
dialect=dialect,
- scopes=[node.type for node in stack if node.type is not None] if stack else None,
+ scopes=[scope.type for scope in stack if scope.type is not None] if stack else None,
resolver_factory=resolver_factory,
)
+ if dialect == "clickhouse":
+ node = resolve_flag_calls_from_flag_evaluations(node, context, resolve)
+ else:
+ node = resolve(node)
# Project constant-key JSONExtractString on argMax lazy tables (groups/persons) into the
# aggregate, so it does not materialize the whole JSON blob per row. Rewrites the call to a
diff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.py
index 72b96860b895..b64b35bce570 100644
--- a/posthog/hogql/test/test_metadata.py
+++ b/posthog/hogql/test/test_metadata.py
@@ -8,7 +8,7 @@ from django.db import DatabaseError, connection
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
-from parameterized import parameterized
+from parameterized import param, parameterized
from posthoganalytics import FeatureFlagResult
from posthog.schema import (
@@ -23,7 +23,7 @@ from posthog.schema import (
)
from posthog.hogql.direct_connection import INVALID_CONNECTION_ID_ERROR
-from posthog.hogql.flag_called_warnings import FLAG_CALLED_ON_EVENTS_WARNING
+from posthog.hogql.flag_called_warnings import FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, FLAG_CALLED_ON_EVENTS_WARNING
from posthog.hogql.metadata import get_hogql_metadata
from posthog.hogql.parser import parse_select
from posthog.hogql.taxonomy_validation import MAX_SUGGESTED_NAMES
@@ -36,11 +36,17 @@ from posthog.taxonomy.dynamic_properties import DYNAMIC_PROPERTY_PATTERNS
from products.cohorts.backend.models.cohort import Cohort
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.data_tools.backend.models.expression import DataWarehouseExpression
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
from products.product_analytics.backend.facade.models import InsightVariable
from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema, ExternalDataSource
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
_FLAG_CALLED_QUERY = "SELECT count() FROM events WHERE event = '$feature_flag_called'"
+_FLAG_CALLS_FROM_FLAG_EVALUATIONS_QUERY = (
+ "SELECT count() FROM events WHERE event = '$feature_flag_called' AND properties.$feature_flag = 'my-flag' "
+ "AND timestamp >= now() - INTERVAL 7 DAY"
+)
def _move_notices(url: str | None = None, enabled: bool = True) -> FeatureFlagResult:
@@ -431,8 +437,47 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
_move_notices("command:editor.action.deleteLines"),
),
("malformed_announcement_url", _FLAG_CALLED_QUERY, True, 1, _move_notices("https://[broken")),
+ ("read_from_flag_evaluations_while_the_rewrite_is_off", _FLAG_CALLS_FROM_FLAG_EVALUATIONS_QUERY, True, 1),
+ param(
+ "read_from_flag_evaluations",
+ _FLAG_CALLS_FROM_FLAG_EVALUATIONS_QUERY,
+ True,
+ 0,
+ _move_notices("https://example.com/announcement"),
+ "https://example.com/announcement",
+ rewrite_enabled=True,
+ expected_notices=1,
+ ),
+ param("refused_by_the_rewrite", _FLAG_CALLED_QUERY, True, 1, rewrite_enabled=True),
+ param(
+ "read_from_flag_evaluations_without_coverage_once_events_gets_no_flag_calls",
+ _FLAG_CALLED_QUERY,
+ True,
+ 0,
+ rewrite_enabled=True,
+ expected_notices=1,
+ mode=FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ),
+ param(
+ "falls_back_for_a_field_flag_evaluations_lacks",
+ "SELECT goe_0.properties FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'my-flag' AND timestamp >= now() - INTERVAL 7 DAY",
+ True,
+ 1,
+ rewrite_enabled=True,
+ ),
+ param(
+ "falls_back_when_a_name_binds_to_a_flag_evaluations_column",
+ "WITH 'all calls' AS flag_key SELECT flag_key FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'my-flag' AND timestamp >= now() - INTERVAL 7 DAY",
+ True,
+ 1,
+ rewrite_enabled=True,
+ ),
]
)
+ # Mode 1 keeps reading events on the native-JSON events table.
+ @override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False)
def test_metadata_warns_for_flag_called_read_from_events(
self,
_name: str,
@@ -441,7 +486,13 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
expected: int,
move_notices: FeatureFlagResult | None = _MOVE_NOTICES_WITHOUT_URL,
expected_url: str | None = None,
+ rewrite_enabled: bool = False,
+ expected_notices: int = 0,
+ mode: FlagEvaluationsMode = FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
) -> None:
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization, defaults={"flag_evaluations_mode": mode}
+ )
DataWarehouseSavedQuery.objects.create(
team=self.team,
name="flag_calls_view",
@@ -462,15 +513,26 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
return_value=flag_evaluations_enabled,
),
patch("posthoganalytics.get_feature_flag_result", return_value=move_notices),
+ patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: (
+ rewrite_enabled and key == "hogql-flag-calls-from-flag-evaluations"
+ ),
+ ),
):
metadata = self._select(query)
self.assertTrue(metadata.isValid, metadata.errors)
warnings = [w for w in metadata.warnings if w.message == FLAG_CALLED_ON_EVENTS_WARNING]
+ notices = [n for n in metadata.notices if n.message == FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE]
literal_start = query.rindex("'$feature_flag_called'")
+ literal_end = literal_start + len("'$feature_flag_called'")
self.assertEqual(
[(w.start, w.end, w.fix, w.url) for w in warnings],
- [(literal_start, literal_start + len("'$feature_flag_called'"), None, expected_url)] * expected,
+ [(literal_start, literal_end, None, expected_url)] * expected,
+ )
+ self.assertEqual(
+ [(n.start, n.end, n.url) for n in notices], [(literal_start, literal_end, expected_url)] * expected_notices
)
def test_metadata_does_not_log_invalid_announcement_url_without_a_flag_called_literal(self) -> None:
diff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.py
index b048c321e014..e0473e3781bd 100644
--- a/posthog/hogql/transforms/clickhouse_property_resolution.py
+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py
@@ -57,7 +57,7 @@ from posthog.hogql.type_system import (
parse_sql_runtime_type,
runtime_type_from_constant_type,
)
-from posthog.hogql.utils import ilike_matches, like_matches
+from posthog.hogql.utils import ilike_matches, is_json_verbatim, like_matches
from posthog.hogql.visitor import CloningVisitor, clone_expr
from posthog.clickhouse.events_json import (
@@ -256,11 +256,6 @@ def resolve_property_group_source(
# `node.keys` is the key path (keys[0] is the property name; deeper keys index into the extracted value).
-def _is_json_verbatim(value: str) -> bool:
- """True when JSON text stores the string unchanged: non-empty printable ASCII with no quote or backslash."""
- return bool(value) and all(" " <= char <= "~" and char not in '"\\' for char in value)
-
-
def _blob_field_type_of(node: ast.PropertyAccess) -> ast.FieldType | None:
"""The source blob column's `FieldType` (`node.expr.type`), the input to `resolve_materialized_property_source`."""
expr_type = node.expr.type
@@ -1986,7 +1981,7 @@ class ClickHousePropertyResolver(CloningVisitor):
values = self._extract_string_constants(node.right)
if not values or len(values) > PERSON_JSON_PREFILTER_MAX_NEEDLES:
return None
- if not all(_is_json_verbatim(value) for value in values):
+ if not all(is_json_verbatim(value) for value in values):
return None
return _call("multiSearchAny", [self.visit(access.expr), ast.Array(exprs=[_const(v) for v in values])])
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
new file mode 100644
index 000000000000..b0b71984e84d
--- /dev/null
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -0,0 +1,965 @@
+"""Read flag calls from `posthog.flag_evaluations` when a query asks `events` for them.
+
+On flag_evaluations mode 2, ingestion stops writing `$feature_flag_called` to `events`. This transform swaps a
+query's `events` scan for `posthog.flag_evaluations` when `flag_evaluations` holds every row the scan selects, so
+saved queries keep returning flag calls without a change.
+
+The swap runs before type resolution, so every rule here is syntactic. A shape the rules cannot classify keeps
+reading `events`.
+"""
+
+import json
+from collections import Counter
+from collections.abc import Callable
+from datetime import UTC, date, datetime, time, timedelta, tzinfo
+from functools import cache, partial
+from typing import TYPE_CHECKING
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+import structlog
+
+from posthog.hogql import ast
+from posthog.hogql.base import _T_AST
+from posthog.hogql.constants import FLAG_CALLED_EVENT
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import DatabaseField
+from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.database.schema.flag_evaluations import ELEMENT_STAND_INS, FlagEvaluationsTable
+from posthog.hogql.database.schema.util.where_clause_extractor import top_level_conjuncts
+from posthog.hogql.errors import BaseHogQLError, QueryError
+from posthog.hogql.functions.mapping import HOGQL_COMPARISON_MAPPING, find_hogql_function
+from posthog.hogql.property_metadata import load_property_metadata
+from posthog.hogql.utils import is_json_verbatim
+from posthog.hogql.visitor import TraversingVisitor, clone_expr
+
+from posthog.dataclasses import frozen
+from posthog.date_util import start_of_day
+
+if TYPE_CHECKING:
+ from posthog.hogql.database.database import Database
+
+logger = structlog.get_logger(__name__)
+
+# At most FLAG_EVALUATIONS_TTL_DAYS - 1, because the TTL can already have dropped the day FLAG_EVALUATIONS_TTL_DAYS
+# back. It stays lower while the table still lacks the oldest days that a shorter TTL dropped. The TTL constant
+# lives in posthog/models/flag_evaluations/sql.py, and importing it loads the posthog.models package, which the
+# printer's import path avoids.
+FLAG_EVALUATIONS_COMPLETE_DAYS = 89
+
+_FLAG_KEY_PROPERTY = "$feature_flag"
+
+_Predicate = Callable[[ast.Expr], bool]
+
+_MAX_UTC_OFFSET = timedelta(hours=14)
+# How far before UTC midnight a date can start in some zone. A zone behind UTC can still be on the previous date, and
+# a zone 14 hours ahead of UTC starts that date 14 hours before UTC does.
+_DAY_IN_ANY_TIMEZONE = timedelta(days=1) + _MAX_UTC_OFFSET
+# A daylight-saving change moves calendar arithmetic in a timezone by up to two hours. Antarctica/Troll changes by
+# two hours.
+_DST_MARGIN = timedelta(hours=2)
+
+_UNIT_LENGTHS: dict[str, tuple[timedelta, timedelta]] = {
+ "Second": (timedelta(seconds=1), timedelta(seconds=1)),
+ "Minute": (timedelta(minutes=1), timedelta(minutes=1)),
+ "Hour": (timedelta(hours=1), timedelta(hours=1)),
+ "Day": (timedelta(days=1), timedelta(days=1)),
+ "Week": (timedelta(days=7), timedelta(days=7)),
+ "Month": (timedelta(days=28), timedelta(days=31)),
+ "Quarter": (timedelta(days=89), timedelta(days=92)),
+ "Year": (timedelta(days=365), timedelta(days=366)),
+}
+_SHIFT_FUNCTIONS: dict[str, tuple[int, str]] = {
+ **{f"add{unit}s": (1, unit) for unit in _UNIT_LENGTHS},
+ **{f"subtract{unit}s": (-1, unit) for unit in _UNIT_LENGTHS},
+}
+_START_OF_UNITS = {
+ "toStartOfMinute": "Minute",
+ "toStartOfFiveMinutes": "Hour",
+ "toStartOfTenMinutes": "Hour",
+ "toStartOfFifteenMinutes": "Hour",
+ "toStartOfHour": "Hour",
+ "toStartOfDay": "Day",
+ "toMonday": "Week",
+ "toStartOfWeek": "Week",
+ "toStartOfMonth": "Month",
+ "toStartOfQuarter": "Quarter",
+ "toStartOfYear": "Year",
+}
+# Rounding to a week or longer returns a Date. A comparison with a DateTime reads a Date at midnight UTC. In a zone
+# behind UTC, that midnight comes before the local midnight that the rounding chose.
+_DATE_UNITS = frozenset({"Week", "Month", "Quarter", "Year"})
+
+# An alias with one of these names can shadow the column in an unqualified read.
+_SCAN_COLUMNS = frozenset({"event", "timestamp", "properties"})
+
+
+def can_read_flag_evaluations(
+ node: ast.SelectQuery,
+ now: datetime,
+ timezone: ZoneInfo,
+ *,
+ only_in_flag_evaluations: bool = False,
+ flag_key_compares_as_string: bool = True,
+) -> bool:
+ """Whether this query's `events` scan selects only rows that `flag_evaluations` holds.
+
+ It holds when all of these hold:
+
+ 1. A top-level conjunct pins the scan's `event` to `$feature_flag_called`.
+ 2. A top-level conjunct bounds the scan's `timestamp` from below at or after the coverage start. A time with no
+ zone reads in `timezone`, the team's zone, wherever the printer reads it that way.
+ 3. The scan is not sampled, because `flag_evaluations` has no sampling key. Insight runners attach `SAMPLE 1`
+ to every events read, and the printer drops it, so it counts as no sampling.
+ 4. The scan's `$feature_flag` property is pinned to non-empty strings that no other JSON value prints as. A
+ top-level conjunct can pin it for the whole scan. Otherwise, each `OR` branch that pins `event` must pin it
+ too, as an action with several steps does. Ingestion writes a call to `flag_evaluations` only when its
+ `$feature_flag` is a non-empty JSON string, and leaves every other call in `events` on every
+ flag_evaluations mode. A literal such as `'100'` also matches the number 100, which stays in `events`.
+ Without `flag_key_compares_as_string`, nothing pins `$feature_flag`: the team's property definition gives it
+ a type, and a comparison converts the value first. Under a Boolean type, `'TRUE'` matches the key "true" and
+ the JSON value true.
+ 5. The select list has no `*` or `COLUMNS(...)`, and the scan has no column aliases. `flag_evaluations` has
+ other columns, so either one would change what the query reads.
+ 6. The scan has no `FINAL` and no table arguments.
+
+ With `only_in_flag_evaluations`, conditions 2 and 4 do not apply. Ingestion writes no new flag calls to events
+ then, so the scan reads `flag_evaluations` even when that leaves out calls older than its coverage or calls
+ without a flag key.
+
+ `resolve_flag_calls_from_flag_evaluations` checks the rest after resolution: every name in the swapped query
+ binds as it did before.
+ """
+ scan = _EventsScan.of(node, lambda: flag_key_compares_as_string)
+ return scan is not None and _qualifies(
+ scan, node, _filter_conjuncts(node), now, timezone, only_in_flag_evaluations=only_in_flag_evaluations
+ )
+
+
+def resolve_flag_calls_from_flag_evaluations(
+ node: _T_AST, context: HogQLContext, resolve: Callable[[_T_AST], _T_AST]
+) -> _T_AST:
+ """`node` resolved with `resolve`, with each `events` scan that `can_read_flag_evaluations` accepts reading
+ `posthog.flag_evaluations` instead.
+
+ This returns the swapped query only when it resolves and every name in it binds as it does in the original.
+ Otherwise it returns the resolved original. For an organization whose flag calls are only in
+ `flag_evaluations`, it raises a QueryError instead of returning the original. It also raises before it
+ resolves anything when such an organization's query has a scan of only flag calls with a shape that
+ `flag_evaluations` cannot stand in for.
+
+ While `events` still gets flag calls, the original query is the answer whenever the two tables can read a row
+ differently:
+
+ - A project that reads the native-JSON events table keeps reading `events`. That table decodes JSON escapes in a
+ property value. `flag_evaluations` returns the escaped text, so a filter on such a value selects different
+ rows in the two tables.
+ - A query keeps reading `events` when it reads a property from a different kind of column there, such as a
+ materialized column. See `_swap_reads_properties_alike`.
+
+ The resolver inlines a saved view's body, so the swap never sees a scan inside one. Once `events` gets no flag
+ calls, a query that reads a view whose body reads only flag calls from `events` raises.
+ """
+ database = context.database
+ if (
+ not context.rewrite_flag_calls_to_flag_evaluations
+ or database is None
+ or not database.read_flag_calls_from_flag_evaluations
+ ):
+ return resolve(node)
+ only_in_flag_evaluations = database.flag_calls_only_in_flag_evaluations
+ resolved = _resolve_with_swaps(node, context, resolve, database, only_in_flag_evaluations)
+ if only_in_flag_evaluations:
+ view_name = _view_reading_flag_calls_from_events(resolved)
+ if view_name is not None:
+ raise _flag_calls_only_in_flag_evaluations_error(
+ f"it reads the saved view {view_name}, which reads them from events"
+ )
+ return resolved
+
+
+def _resolve_with_swaps(
+ node: _T_AST,
+ context: HogQLContext,
+ resolve: Callable[[_T_AST], _T_AST],
+ database: "Database",
+ only_in_flag_evaluations: bool,
+) -> _T_AST:
+ finder = _FlagCalledConstantFinder()
+ finder.visit(node)
+ if not finder.found:
+ return resolve(node)
+ if not only_in_flag_evaluations and context.uses_new_events_schema():
+ return resolve(node)
+ with context.timings.measure("flag_calls_from_flag_evaluations"):
+ rewritten = clone_expr(node)
+ swapper = _EventsScanSwapper(
+ now=datetime.now(UTC),
+ timezone=ZoneInfo(database.get_timezone()),
+ only_in_flag_evaluations=only_in_flag_evaluations,
+ # The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.
+ flag_key_compares_as_string=cache(partial(_flag_key_compares_as_string, context)),
+ )
+ swapper.visit(rewritten)
+ if swapper.refusals:
+ if only_in_flag_evaluations:
+ raise _flag_calls_only_in_flag_evaluations_error(swapper.refusals[0])
+ logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=swapper.refusals[0])
+ original = resolve(node)
+ if not swapper.swapped:
+ return original
+ try:
+ swapped = resolve(rewritten)
+ except BaseHogQLError as error:
+ # Expected for a query that reads a field flag_evaluations lacks, such as goe_0.properties.
+ reason = f"it reads something that table doesn't have: {error}"
+ else:
+ if not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):
+ reason = "a name in it would read a different column there"
+ elif not only_in_flag_evaluations and not _swap_reads_properties_alike(original, swapped, context):
+ reason = "it reads a property from a different kind of column there"
+ else:
+ return swapped
+ if only_in_flag_evaluations:
+ # events gets no new flag calls, so the original query would return only calls from before the move.
+ raise _flag_calls_only_in_flag_evaluations_error(reason)
+ logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=reason)
+ return original
+
+
+def _flag_key_compares_as_string(context: HogQLContext) -> bool:
+ """Whether `properties.$feature_flag` compares as a string. The printer converts a property that the team's
+ property definitions type as Boolean, Numeric or DateTime before it compares the value."""
+ if context.team_id is None:
+ return True
+ metadata = load_property_metadata(
+ context, event_property_names={_FLAG_KEY_PROPERTY}, person_property_names=set(), group_property_names={}
+ )
+ return metadata.event_properties.get(_FLAG_KEY_PROPERTY, {}).get("type") in (None, "String")
+
+
+def _flag_calls_only_in_flag_evaluations_error(reason: str) -> QueryError:
+ return QueryError(
+ f"Your organization's {FLAG_CALLED_EVENT} events are only in posthog.flag_evaluations, and this query "
+ f"can't read them there because {reason}. Change that part of the query, or query "
+ "posthog.flag_evaluations directly."
+ )
+
+
+def _qualifies(
+ scan: "_EventsScan",
+ node: ast.SelectQuery,
+ conjuncts: list[ast.Expr],
+ now: datetime,
+ timezone: ZoneInfo,
+ *,
+ only_in_flag_evaluations: bool,
+) -> bool:
+ if scan.shape_refusal(node) is not None:
+ return False
+ if only_in_flag_evaluations:
+ return scan.pins_flag_calls(conjuncts, needs_flag_key=False)
+ return scan.pins_flag_calls(conjuncts, needs_flag_key=True) and scan.bounds_timestamp(conjuncts, now, timezone)
+
+
+class _EventsScan:
+ def __init__(
+ self,
+ join: ast.JoinExpr,
+ alias: str,
+ has_other_sources: bool,
+ flag_key_compares_as_string: Callable[[], bool],
+ ):
+ self.join = join
+ self.alias = alias
+ self.has_other_sources = has_other_sources
+ self.flag_key_compares_as_string = flag_key_compares_as_string
+ self.shadowed: frozenset[str] = frozenset()
+
+ @classmethod
+ def of(cls, node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool]) -> "_EventsScan | None":
+ join = node.select_from
+ if join is None or not isinstance(join.table, ast.Field) or join.table.chain != ["events"]:
+ return None
+ scan = cls(
+ join=join,
+ alias=join.alias or "events",
+ # A field without a table prefix can bind to a joined table or an ARRAY JOIN alias instead.
+ has_other_sources=join.next_join is not None or bool(node.array_join_list),
+ flag_key_compares_as_string=flag_key_compares_as_string,
+ )
+ collector = _AliasCollector(node)
+ collector.visit(node)
+ # An alias such as `e.timestamp AS timestamp` reads the column it names, so it changes no read.
+ scan.shadowed = frozenset(
+ name for name, expr in collector.aliases if name in _SCAN_COLUMNS and scan.column(expr) != [name]
+ )
+ return scan
+
+ def column(self, expr: ast.Expr) -> list[str | int] | None:
+ """The chain below the scan when `expr` reads it, such as `["event"]` or `["properties", "$feature_flag"]`."""
+ # The resolver wraps each field it resolves in a hidden alias, which the check of a resolved view body sees.
+ while isinstance(expr, ast.Alias) and expr.hidden:
+ expr = expr.expr
+ if not isinstance(expr, ast.Field) or not expr.chain:
+ return None
+ if expr.chain[0] == self.alias and len(expr.chain) > 1:
+ return expr.chain[1:]
+ if self.has_other_sources or expr.chain[0] in self.shadowed:
+ return None
+ return expr.chain
+
+ def shape_refusal(self, node: ast.SelectQuery) -> str | None:
+ """Why `flag_evaluations` cannot stand in for this scan whatever its filters say, or None."""
+ if self.join.table_args is not None or self.join.table_final:
+ return "it uses FINAL or table arguments on events"
+ if self.join.column_aliases:
+ return "it names the events columns with column aliases"
+ if self.join.sample is not None and not self.join.sample.samples_all_rows():
+ return "it uses SAMPLE"
+ for item in node.select:
+ while isinstance(item, ast.Alias):
+ item = item.expr
+ if isinstance(item, ast.ColumnsExpr) or (isinstance(item, ast.Field) and item.chain[-1:] == ["*"]):
+ return "it selects * or COLUMNS(...)"
+ return None
+
+ def pins_flag_calls(self, conjuncts: list[ast.Expr], needs_flag_key: bool) -> bool:
+ """Whether `conjuncts` pin `event` to flag calls. With `needs_flag_key`, a pin of `event` counts only where
+ these conjuncts also pin `$feature_flag`. The recursion into `OR` branches passes False once an enclosing
+ conjunct pins `$feature_flag`."""
+ needs_flag_key = needs_flag_key and not self.flag_keys(conjuncts)
+ return any(self._pins_flag_calls_in(conjunct, needs_flag_key) for conjunct in conjuncts)
+
+ def _pins_flag_calls_in(self, expr: ast.Expr, needs_flag_key: bool) -> bool:
+ if _pinned_values(expr, self._reads_event, _is_flag_called) is not None:
+ return not needs_flag_key
+ if isinstance(expr, ast.Or):
+ return bool(expr.exprs) and all(
+ self.pins_flag_calls(_conjuncts(branch), needs_flag_key) for branch in expr.exprs
+ )
+ return False
+
+ def _flag_keys_in(self, expr: ast.Expr) -> list[str] | None:
+ """The keys `expr` pins `$feature_flag` to, or None when it does not pin it."""
+ values = _pinned_values(expr, self._reads_flag_key, _is_flag_key)
+ if values is not None:
+ return [value.value for value in values if isinstance(value, ast.Constant)]
+ if isinstance(expr, ast.Or) and expr.exprs:
+ branches = [self._flag_keys_in(branch) for branch in expr.exprs]
+ if all(keys is not None for keys in branches):
+ return [key for keys in branches if keys is not None for key in keys]
+ return None
+
+ def flag_keys(self, conjuncts: list[ast.Expr]) -> list[str]:
+ """The keys a top-level conjunct pins `$feature_flag` to, or an empty list when no top-level conjunct does."""
+ for conjunct in conjuncts:
+ keys = self._flag_keys_in(conjunct)
+ if keys is not None:
+ return keys if self.flag_key_compares_as_string() else []
+ return []
+
+ def _reads_event(self, expr: ast.Expr) -> bool:
+ return self.column(expr) == ["event"]
+
+ def _reads_flag_key(self, expr: ast.Expr) -> bool:
+ if isinstance(expr, ast.ArrayAccess):
+ return self.column(expr.array) == ["properties"] and _is_constant(expr.property, _FLAG_KEY_PROPERTY)
+ if isinstance(expr, ast.Call) and expr.name == "JSONExtractString" and len(expr.args) == 2:
+ return self.column(expr.args[0]) == ["properties"] and _is_constant(expr.args[1], _FLAG_KEY_PROPERTY)
+ return self.column(expr) == ["properties", _FLAG_KEY_PROPERTY]
+
+ def bounds_timestamp(self, conjuncts: list[ast.Expr], now: datetime, timezone: ZoneInfo) -> bool:
+ coverage_start = _utc_midnight(now) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)
+ for conjunct in conjuncts:
+ bound = self._lower_bound(conjunct)
+ if bound is None:
+ continue
+ try:
+ value = _earliest_instant(bound, now, timezone)
+ except OverflowError:
+ # A date near year 1 or 9999, or a huge interval, leaves datetime's range.
+ continue
+ if value is not None and value >= coverage_start:
+ return True
+ return False
+
+ def _lower_bound(self, expr: ast.Expr) -> ast.Expr | None:
+ if isinstance(expr, ast.BetweenExpr):
+ return expr.low if not expr.negated and self._reads_timestamp(expr.expr) else None
+ comparison = _comparison(expr)
+ if comparison is None:
+ return None
+ if comparison.op in (ast.CompareOperationOp.GtEq, ast.CompareOperationOp.Gt) and self._reads_timestamp(
+ comparison.left
+ ):
+ return comparison.right
+ if comparison.op in (ast.CompareOperationOp.LtEq, ast.CompareOperationOp.Lt) and self._reads_timestamp(
+ comparison.right
+ ):
+ return comparison.left
+ return None
+
+ def _reads_timestamp(self, expr: ast.Expr) -> bool:
+ return self.column(expr) == ["timestamp"]
+
+
+def _comparison(expr: ast.Expr) -> ast.CompareOperation | None:
+ """`expr` as a comparison, whether it is written `a >= b` or `greaterOrEquals(a, b)`."""
+ if isinstance(expr, ast.CompareOperation):
+ return expr
+ if isinstance(expr, ast.Call) and expr.name in HOGQL_COMPARISON_MAPPING and len(expr.args) == 2:
+ return ast.CompareOperation(op=HOGQL_COMPARISON_MAPPING[expr.name], left=expr.args[0], right=expr.args[1])
+ return None
+
+
+def _pinned_values(expr: ast.Expr, reads: _Predicate, accepts: _Predicate) -> list[ast.Expr] | None:
+ """The values `expr` limits what `reads` matches to, or None when they are not all values `accepts` matches."""
+ comparison = _comparison(expr)
+ if comparison is None:
+ return None
+ if comparison.op == ast.CompareOperationOp.Eq:
+ if reads(comparison.left) and accepts(comparison.right):
+ return [comparison.right]
+ if reads(comparison.right) and accepts(comparison.left):
+ return [comparison.left]
+ return None
+ if comparison.op in (ast.CompareOperationOp.In, ast.CompareOperationOp.GlobalIn):
+ right = comparison.right
+ values = right.exprs if isinstance(right, (ast.Tuple, ast.Array)) else [right]
+ return values if reads(comparison.left) and values and all(accepts(value) for value in values) else None
+ return None
+
+
+def _is_constant(expr: ast.Expr, value: str) -> bool:
+ return isinstance(expr, ast.Constant) and expr.value == value
+
+
+def _is_flag_called(expr: ast.Expr) -> bool:
+ return _is_constant(expr, FLAG_CALLED_EVENT)
+
+
+def _is_flag_key(expr: ast.Expr) -> bool:
+ if not isinstance(expr, ast.Constant) or not isinstance(expr.value, str) or expr.value == "":
+ return False
+ # A property read compares the value's JSON text, so a literal that parses as JSON can match a non-string value.
+ try:
+ json.loads(expr.value)
+ except ValueError:
+ return True
+ return False
+
+
+def _conjuncts(expr: ast.Expr | None) -> list[ast.Expr]:
+ return top_level_conjuncts(expr) if expr is not None else []
+
+
+def _filter_conjuncts(node: ast.SelectQuery) -> list[ast.Expr]:
+ return _conjuncts(node.where) + _conjuncts(node.prewhere)
+
+
+def _utc_midnight(now: datetime) -> datetime:
+ return start_of_day(now.astimezone(UTC))
+
+
+def _earliest_instant(expr: ast.Expr, now: datetime, timezone: ZoneInfo) -> datetime | None:
+ """An instant no later than the value of `expr`, or None when `expr` has a shape this cannot evaluate."""
+ if isinstance(expr, ast.Constant):
+ # The printer prints a naive datetime in the server's zone, which is UTC. A bare string compares in UTC or
+ # in the team's zone.
+ return _constant_instant(expr.value, (timezone, UTC))
+ shift = _shift(expr)
+ if shift is not None:
+ return _shifted(_earliest_instant(shift.base, now, timezone), shift.interval, shift.sign)
+ if not isinstance(expr, ast.Call):
+ return None
+ name, args = expr.name, expr.args
+ if name in ("now", "now64"):
+ return now
+ if name == "today" and not args:
+ return _utc_midnight(now) - _DAY_IN_ANY_TIMEZONE
+ if name == "assumeNotNull" and len(args) == 1:
+ return _earliest_instant(args[0], now, timezone)
+ if name in ("toDateTime", "toDateTime64") and args:
+ zones = _zones_of(expr, timezone)
+ if isinstance(args[0], ast.Constant):
+ return _constant_instant(args[0].value, zones) if zones else None
+ return _earliest_instant(args[0], now, timezone)
+ if name == "toDate" and args:
+ instant = _earliest_instant(args[0], now, timezone)
+ # toDate truncates to a day, and the comparison can read that day in a zone the caller does not name.
+ return instant - _DAY_IN_ANY_TIMEZONE if instant is not None else None
+ if (name in _START_OF_UNITS and args) or (name == "toStartOfInterval" and len(args) >= 2):
+ # Evaluating the value once keeps nested rounding linear. Each rounding call would otherwise evaluate its
+ # value twice, and the work would double at each level.
+ instant = _earliest_instant(args[0], now, timezone)
+ if instant is None:
+ return None
+ day_start = _start_of_day(expr, instant, timezone)
+ if day_start is not None:
+ return day_start
+ return _rounded_down(instant, (1, _START_OF_UNITS[name]) if name in _START_OF_UNITS else _interval(args[1]))
+ return None
+
+
+def _start_of_day(expr: ast.Call, instant: datetime, timezone: ZoneInfo) -> datetime | None:
+ """The exact start of the day `toStartOfDay(x)` or `toStartOfInterval(x, toIntervalDay(1))` rounds to, given
+ `instant`, the earliest value of `x`. None for any other call or when the zone it rounds in is unknown."""
+ if expr.name == "toStartOfDay" and expr.args:
+ zone_index = 1
+ elif expr.name == "toStartOfInterval" and len(expr.args) >= 2 and _interval(expr.args[1]) == (1, "Day"):
+ zone_index = 2
+ else:
+ return None
+ value = expr.args[0]
+ # ClickHouse rounds in the zone argument, or else in the zone of the value.
+ zones = _zone(expr.args[zone_index]) if len(expr.args) > zone_index else _zones_of(value, timezone)
+ if not zones:
+ return None
+ return min(start_of_day(instant.astimezone(zone)) for zone in zones)
+
+
+def _zones_of(expr: ast.Expr, timezone: ZoneInfo) -> tuple[tzinfo, ...]:
+ """The zones a time value can carry. Empty when it names a zone that does not parse."""
+ while True:
+ shift = _shift(expr)
+ if shift is not None:
+ expr = shift.base
+ elif isinstance(expr, ast.Call) and expr.name == "assumeNotNull" and len(expr.args) == 1:
+ expr = expr.args[0]
+ else:
+ break
+ if isinstance(expr, ast.Call):
+ meta = find_hogql_function(expr.name)
+ if meta is not None and meta.tz_aware and meta.max_args:
+ # The printer adds the team's zone as the last argument to each timezone-aware call that names none.
+ zone_index = meta.max_args - 1
+ return _zone(expr.args[zone_index]) if len(expr.args) > zone_index else (timezone,)
+ return (timezone, UTC)
+
+
+def _zone(expr: ast.Expr) -> tuple[tzinfo, ...]:
+ if isinstance(expr, ast.Constant) and isinstance(expr.value, str):
+ try:
+ return (ZoneInfo(expr.value),)
+ except (ZoneInfoNotFoundError, ValueError):
+ pass
+ return ()
+
+
+@frozen
+class _Shift:
+ base: ast.Expr
+ interval: tuple[int, str] | None
+ sign: int
+
+
+def _shift(expr: ast.Expr) -> _Shift | None:
+ """`expr` as a base, an interval and a sign when it adds an interval to a time or subtracts one."""
+ if isinstance(expr, ast.ArithmeticOperation) and expr.op in (
+ ast.ArithmeticOperationOp.Add,
+ ast.ArithmeticOperationOp.Sub,
+ ):
+ sign = 1 if expr.op == ast.ArithmeticOperationOp.Add else -1
+ return _Shift(base=expr.left, interval=_interval(expr.right), sign=sign)
+ if not isinstance(expr, ast.Call) or len(expr.args) != 2:
+ return None
+ if expr.name in ("plus", "minus"):
+ return _Shift(base=expr.args[0], interval=_interval(expr.args[1]), sign=1 if expr.name == "plus" else -1)
+ if expr.name in _SHIFT_FUNCTIONS and (count := _integer(expr.args[1])) is not None:
+ sign, unit = _SHIFT_FUNCTIONS[expr.name]
+ return _Shift(base=expr.args[0], interval=(count, unit), sign=sign)
+ return None
+
+
+def _interval(expr: ast.Expr) -> tuple[int, str] | None:
+ if isinstance(expr, ast.Call) and expr.name.startswith("toInterval") and len(expr.args) == 1:
+ unit = expr.name.removeprefix("toInterval")
+ count = _integer(expr.args[0])
+ if unit in _UNIT_LENGTHS and count is not None:
+ return count, unit
+ return None
+
+
+def _shifted(instant: datetime | None, interval: tuple[int, str] | None, sign: int) -> datetime | None:
+ """The earliest instant `instant` plus `sign` times `interval` can be."""
+ if instant is None or interval is None:
+ return None
+ count, unit = interval
+ shortest, longest = _UNIT_LENGTHS[unit]
+ total = sign * count
+ return instant + total * (shortest if total >= 0 else longest) - _DST_MARGIN
+
+
+def _rounded_down(instant: datetime | None, interval: tuple[int, str] | None) -> datetime | None:
+ """The earliest instant that rounding `instant` down to a multiple of `interval` can give."""
+ rounded = _shifted(instant, interval, -1)
+ if rounded is not None and interval is not None and interval[1] in _DATE_UNITS:
+ return rounded - _MAX_UTC_OFFSET
+ return rounded
+
+
+def _integer(expr: ast.Expr) -> int | None:
+ if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and not isinstance(expr.value, bool):
+ return expr.value
+ return None
+
+
+def _constant_instant(value: object, zones: tuple[tzinfo, ...]) -> datetime | None:
+ """`value` as an instant, reading a naive value in whichever of `zones` gives the earliest one."""
+ if isinstance(value, str):
+ try:
+ value = datetime.fromisoformat(value.strip())
+ except ValueError:
+ return None
+ if isinstance(value, date) and not isinstance(value, datetime):
+ value = datetime.combine(value, time.min)
+ if not isinstance(value, datetime):
+ return None
+ if value.tzinfo is not None:
+ return value
+ return min(value.replace(tzinfo=zone) for zone in zones)
+
+
+class _SyntaxVisitor(TraversingVisitor):
+ def visit_field(self, node: ast.Field) -> None:
+ # The inherited visit follows the field's type into shared CTE types, which takes exponential time. An
+ # earlier step can resolve parts of the tree in place.
+ pass
+
+
+class _AliasCollector(_SyntaxVisitor):
+ def __init__(self, root: ast.SelectQuery) -> None:
+ super().__init__()
+ self.root = root
+ self.aliases: list[tuple[str, ast.Expr]] = []
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ # A subquery's aliases do not reach the outer scope.
+ if node is self.root:
+ super().visit_select_query(node)
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
+ pass
+
+ def visit_alias(self, node: ast.Alias) -> None:
+ self.aliases.append((node.alias, node.expr))
+ super().visit_alias(node)
+
+
+class _FlagCalledConstantFinder(_SyntaxVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.found = False
+
+ def visit_constant(self, node: ast.Constant) -> None:
+ if node.value == FLAG_CALLED_EVENT:
+ self.found = True
+
+
+class _EventsScanSwapper(_SyntaxVisitor):
+ def __init__(
+ self,
+ now: datetime,
+ timezone: ZoneInfo,
+ only_in_flag_evaluations: bool,
+ flag_key_compares_as_string: Callable[[], bool],
+ ) -> None:
+ super().__init__()
+ self.now = now
+ self.timezone = timezone
+ self.only_in_flag_evaluations = only_in_flag_evaluations
+ self.flag_key_compares_as_string = flag_key_compares_as_string
+ self.swapped = False
+ self.flag_key_predicates = 0
+ # Why each scan that reads only flag calls has a shape that flag_evaluations cannot stand in for.
+ self.refusals: list[str] = []
+ # The CTE names in scope, under the resolver's rules. A `FROM events` reads a CTE when one is named `events`.
+ # A name missing here lets the swapper try the scan, and the binding check then rejects the swap.
+ self.ctes: frozenset[str] = frozenset()
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
+ outer = self.ctes
+ initial = node.initial_select_query
+ self.visit(initial)
+ # The resolver gives every later branch the first branch's WITH.
+ if isinstance(initial, ast.SelectQuery) and initial.ctes:
+ self.ctes = self.ctes.union(initial.ctes)
+ for subsequent in node.subsequent_select_queries:
+ self.visit(subsequent.select_query)
+ self.ctes = outer
+
+ def visit_cte(self, node: ast.CTE) -> None:
+ # visit_select_query visits each CTE body while only the earlier CTE names are in scope.
+ pass
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ outer = self.ctes
+ for cte in (node.ctes or {}).values():
+ self.visit(cte.expr)
+ self.ctes = self.ctes.union([cte.name])
+ scan = None if "events" in self.ctes else _EventsScan.of(node, self.flag_key_compares_as_string)
+ if scan is not None:
+ conjuncts = _filter_conjuncts(node)
+ if _qualifies(
+ scan, node, conjuncts, self.now, self.timezone, only_in_flag_evaluations=self.only_in_flag_evaluations
+ ):
+ self._swap(node, scan, conjuncts)
+ else:
+ refusal = scan.shape_refusal(node)
+ if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):
+ self.refusals.append(refusal)
+ super().visit_select_query(node)
+ self.ctes = outer
+
+ def _swap(self, node: ast.SelectQuery, scan: _EventsScan, conjuncts: list[ast.Expr]) -> None:
+ keys = scan.flag_keys(conjuncts)
+ # The old alias keeps qualified reads such as `events.timestamp` resolving.
+ scan.join.alias = scan.alias
+ scan.join.table = ast.Field(chain=["posthog", "flag_evaluations"])
+ # A swapped PREWHERE can read person columns through a join that the printer adds later. ClickHouse
+ # rejects a PREWHERE that reads a joined column, and WHERE selects the same rows.
+ conditions = [condition for condition in (node.prewhere, node.where) if condition is not None]
+ # flag_key sorts the table, so this predicate lets ClickHouse skip other flags. flag_key holds the key's JSON
+ # text, and a JSONExtractString pin compares the decoded key. The pin implies this predicate only when the
+ # JSON text of each key is the key itself.
+ if keys and all(is_json_verbatim(key) for key in keys):
+ conditions.append(
+ ast.CompareOperation(
+ op=ast.CompareOperationOp.In,
+ left=ast.Field(chain=[scan.alias, "flag_key"]),
+ right=ast.Tuple(exprs=[ast.Constant(value=key) for key in keys]),
+ )
+ )
+ self.flag_key_predicates += 1
+ node.prewhere = None
+ node.where = conditions[0] if len(conditions) == 1 else ast.And(exprs=conditions)
+ self.swapped = True
+
+
+def _swap_keeps_bindings(original: ast.AST, swapped: ast.AST, flag_key_predicates: int) -> bool:
+ """Whether every field and table in the resolved swapped query binds as it does in the resolved original.
+
+ The swap may change only the table each swapped scan reads. A name that binds elsewhere changes the result
+ with no resolution error: a `WITH` name that a `flag_evaluations` column now shadows, a `COLUMNS(...)` that
+ expands to other columns, or a `person_id` that read the stored events column and now reads the person overrides
+ join. The persons-on-events mode decides how `events` reads person fields. The comparison ignores order,
+ because the swap moves `PREWHERE` into `WHERE`. The swapped query also reads flag_key once for each of the
+ `flag_key_predicates` that the swap adds.
+ """
+ expected = Counter(_bindings(original)) + Counter({_ADDED_FLAG_KEY_READ: flag_key_predicates})
+ return Counter(_bindings(swapped)) == expected
+
+
+# A field records the last part of its chain, so `e.person_id` and `person_id` match.
+_Binding = tuple[str, str | int, tuple[str, ...]]
+
+# events has no flag_key column, so the original query records this binding only for a select alias named flag_key
+# that reads the scan.
+_ADDED_FLAG_KEY_READ: _Binding = ("field", "flag_key", ("scan",))
+
+
+def _bindings(node: ast.AST) -> list[_Binding]:
+ collector = _BindingCollector()
+ collector.visit(node)
+ return collector.bindings
+
+
+class _BindingCollector(_SyntaxVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.bindings: list[_Binding] = []
+
+ def visit_field(self, node: ast.Field) -> None:
+ self.bindings.append(("field", node.chain[-1] if node.chain else "", _target(node.type)))
+
+ def visit_alias(self, node: ast.Alias) -> None:
+ if not node.hidden:
+ super().visit_alias(node)
+ return
+ # The resolver replaces an expression field, such as flag_evaluations.person_id, with a hidden alias named
+ # after the field. The same field can be a plain column on events, so only the alias name and what the
+ # expansion reads count. An element stand-in reads no column. It returns what the events column holds for a
+ # flag call, so it counts as a read of the scan.
+ targets = sorted({target for _, _, target in _bindings(node.expr)})
+ if targets == [("scan",)] or (not targets and node.alias in ELEMENT_STAND_INS):
+ target: tuple[str, ...] = ("scan",)
+ else:
+ target = ("expression", *("/".join(read) for read in targets))
+ self.bindings.append(("field", node.alias, target))
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ self.bindings.append(("table", "", _target(node.type)))
+ if not isinstance(node.table, ast.Field):
+ self.visit(node.table)
+ for arg in node.table_args or []:
+ self.visit(arg)
+ self.visit(node.constraint)
+ self.visit(node.next_join)
+
+
+def _target(current: ast.Type | None) -> tuple[str, ...]:
+ """What a resolved field or table reads from: `("scan",)` for an events or flag_evaluations scan."""
+ while True:
+ if isinstance(current, ast.FieldAliasType):
+ current = current.type
+ elif isinstance(current, ast.PropertyType):
+ current = current.field_type
+ elif isinstance(
+ current,
+ (
+ ast.FieldType,
+ ast.ExpressionFieldType,
+ ast.FieldTraverserType,
+ ast.TableAliasType,
+ ast.VirtualTableType,
+ ),
+ ):
+ current = current.table_type
+ elif isinstance(current, ast.LazyJoinType):
+ # A join reads another table. The tables can define a join of the same name differently, so the
+ # binding names the join recipe and what it joins on. Under PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ # events.person joins persons, while flag_evaluations.person reads the person properties stored with the
+ # call. join_table can name the same table as an instance or as a string, so the binding leaves it out.
+ join = current.lazy_join
+ return (
+ "join",
+ current.field,
+ join.resolver,
+ json.dumps(join.resolver_params, sort_keys=True, default=str),
+ ".".join(str(part) for part in join.from_field),
+ )
+ elif isinstance(current, ast.TableType):
+ return (
+ ("scan",)
+ if isinstance(current.table, (EventsTable, FlagEvaluationsTable))
+ else ("table", current.table.__class__.__name__)
+ )
+ elif isinstance(current, ast.CTETableType):
+ return ("cte", current.name)
+ elif isinstance(current, ast.CTETableAliasType):
+ return ("cte", current.alias)
+ elif isinstance(current, ast.SelectQueryAliasType):
+ return ("subquery", current.alias)
+ elif isinstance(current, ast.SelectViewType):
+ return ("view", current.view_name)
+ else:
+ return (current.__class__.__name__,)
+
+
+def _swap_reads_properties_alike(original: ast.AST, swapped: ast.AST, context: HogQLContext) -> bool:
+ """Whether each property that the swapped query reads comes from the same kind of column as in the original.
+
+ A materialized or dmat column on `events` reads the strings "" and "null" as NULL. `flag_evaluations` reads the
+ same property from the JSON blob, which returns those strings. A property-group map has the same definition on
+ both tables, so reads from it agree. The swap pins `$feature_flag` to keys that are never "" or "null", so a
+ read of it agrees whatever column backs it.
+ """
+ original_reads = _property_reads(original, context)
+ swapped_reads = _property_reads(swapped, context)
+ if not original_reads and not swapped_reads:
+ return True
+ if context.team_id is None:
+ return False
+ previous_metadata = context.property_metadata
+ # The dmat slots come from the team's property definitions, which the printer loads only after this transform.
+ context.property_metadata = load_property_metadata(
+ context,
+ event_property_names={name for column, name, _ in original_reads if column == "properties"},
+ person_property_names=set(),
+ group_property_names={},
+ )
+ try:
+ return Counter(_property_sources(original_reads, context)) == Counter(_property_sources(swapped_reads, context))
+ finally:
+ context.property_metadata = previous_metadata
+
+
+# The blob column, the property, and the field type that reads it.
+_PropertyRead = tuple[str, str, ast.FieldType]
+
+
+def _property_reads(node: ast.AST, context: HogQLContext) -> list[_PropertyRead]:
+ collector = _PropertyReadCollector(context)
+ collector.visit(node)
+ return collector.reads
+
+
+@frozen
+class _PropertySource:
+ column: str
+ name: str
+ # The column the printer reads the property from, or None for the JSON blob.
+ materialized_column: str | None
+
+
+def _property_sources(reads: list[_PropertyRead], context: HogQLContext) -> list[_PropertySource]:
+ # Deferred: clickhouse_property_resolution imports the printer package, which imports this module.
+ from posthog.hogql.transforms.clickhouse_property_resolution import ( # noqa: PLC0415
+ resolve_materialized_property_source,
+ )
+
+ sources: list[_PropertySource] = []
+ for column, name, field_type in reads:
+ source = resolve_materialized_property_source(field_type, name, context)
+ sources.append(
+ _PropertySource(column=column, name=name, materialized_column=source.column if source is not None else None)
+ )
+ return sources
+
+
+class _PropertyReadCollector(_SyntaxVisitor):
+ def __init__(self, context: HogQLContext) -> None:
+ super().__init__()
+ self.context = context
+ self.reads: list[_PropertyRead] = []
+
+ def visit_field(self, node: ast.Field) -> None:
+ if not isinstance(node.type, ast.PropertyType) or not node.type.chain:
+ return
+ field_type = node.type.field_type
+ database_field = field_type.resolve_database_field(self.context)
+ column = database_field.name if isinstance(database_field, DatabaseField) else field_type.name
+ name = str(node.type.chain[0])
+ if column == "properties" and name == _FLAG_KEY_PROPERTY:
+ return
+ self.reads.append((column, name, field_type))
+
+
+def _view_reading_flag_calls_from_events(node: ast.AST) -> str | None:
+ """The name of a saved view whose inlined body reads only flag calls from `events`, or None."""
+ finder = _FlagCallViewFinder()
+ finder.visit(node)
+ return finder.found
+
+
+class _FlagCallViewFinder(_SyntaxVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.view: str | None = None
+ self.found: str | None = None
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ outer = self.view
+ # The resolver names the view on the body's top SELECT only, so a SELECT nested in the body inherits it.
+ self.view = node.view_name or outer
+ if self.view is not None and self.found is None and _reads_only_flag_calls_from_events(node):
+ self.found = self.view
+ super().visit_select_query(node)
+ self.view = outer
+
+
+def _reads_only_flag_calls_from_events(node: ast.SelectQuery) -> bool:
+ scan = _EventsScan.of(node, lambda: True)
+ return (
+ scan is not None
+ and _reads_events_table(scan.join)
+ and scan.pins_flag_calls(_filter_conjuncts(node), needs_flag_key=False)
+ )
+
+
+def _reads_events_table(join: ast.JoinExpr) -> bool:
+ """Whether a resolved FROM reads the events table, rather than a CTE named `events`."""
+ table_type = join.type
+ while isinstance(table_type, (ast.TableAliasType, ast.ColumnAliasedTableType)):
+ table_type = table_type.table_type
+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)
diff --git a/posthog/hogql/transforms/test/test_clickhouse_property_resolution.py b/posthog/hogql/transforms/test/test_clickhouse_property_resolution.py
index fe87922f56cf..9a06cec2b0f7 100644
--- a/posthog/hogql/transforms/test/test_clickhouse_property_resolution.py
+++ b/posthog/hogql/transforms/test/test_clickhouse_property_resolution.py
@@ -23,10 +23,10 @@ from posthog.hogql.property_access_types import RestrictedProperty
from posthog.hogql.property_metadata import PropertyMetadata
from posthog.hogql.transforms.clickhouse_property_resolution import (
MAX_MATERIALIZED_LIKE_PATTERN_LENGTH,
- _is_json_verbatim,
clickhouse_property_resolution,
)
from posthog.hogql.transforms.property_types import PropertySwapper
+from posthog.hogql.utils import is_json_verbatim
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.events_json import TEMPORARY_PROPERTIES_JSON_TYPE
@@ -53,7 +53,7 @@ class TestJSONVerbatimValues(SimpleTestCase):
@example("plain_value-1")
@hypothesis_settings(max_examples=100, deadline=None)
def test_accepted_values_survive_json_encoding(self, value: str) -> None:
- if not _is_json_verbatim(value):
+ if not is_json_verbatim(value):
return
assert value in json.dumps({"key": value})
assert value in json.dumps({"key": value}, ensure_ascii=False)
diff --git a/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
new file mode 100644
index 000000000000..9c0bb3c73321
--- /dev/null
+++ b/posthog/hogql/transforms/test/test_flag_calls_from_flag_evaluations.py
@@ -0,0 +1,456 @@
+from datetime import UTC, datetime, time, timedelta
+from zoneinfo import ZoneInfo
+
+import pytest
+import time_machine
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.hogql import ast
+from posthog.hogql.parser import parse_select
+from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (
+ FLAG_EVALUATIONS_COMPLETE_DAYS,
+ can_read_flag_evaluations,
+)
+
+from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_TTL_DAYS
+
+NOW = datetime(2026, 10, 27, 12, 0, tzinfo=UTC)
+COVERAGE_START = NOW.replace(hour=0) - timedelta(days=FLAG_EVALUATIONS_COMPLETE_DAYS)
+IN_COVERAGE = COVERAGE_START + timedelta(days=30)
+
+FLAG_CALLED = "event = '$feature_flag_called'"
+FLAG_KEY = "properties.$feature_flag = 'my-flag'"
+BOUND = "timestamp >= now() - INTERVAL 7 DAY"
+
+
+def _text(instant: datetime) -> str:
+ return instant.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def _date(instant: datetime) -> str:
+ return instant.strftime("%Y-%m-%d")
+
+
+def _last_days_bound(timezone: str, days: int) -> str:
+ local_today = NOW.astimezone(ZoneInfo(timezone)).date()
+ date_from = datetime.combine(local_today - timedelta(days=days), time.min)
+ return f"toStartOfInterval(assumeNotNull(toDateTime('{_text(date_from)}')), toIntervalDay(1))"
+
+
+def _where(*conjuncts: str) -> str:
+ return f"SELECT count() FROM events WHERE {' AND '.join(conjuncts)}"
+
+
+def _bounded_by(lower_bound: str) -> str:
+ return _where(FLAG_CALLED, FLAG_KEY, f"timestamp >= {lower_bound}")
+
+
+def _nested(template: str, depth: int) -> str:
+ expr = "properties.since"
+ for _ in range(depth):
+ expr = template.format(expr)
+ return expr
+
+
+def _joined(where: str) -> str:
+ return f"SELECT count() FROM events AS e JOIN persons AS p ON e.person_id = p.id WHERE {where}"
+
+
+def _self_joined(where: str) -> str:
+ return f"SELECT count() FROM events AS e JOIN events AS f ON e.uuid = f.uuid WHERE {where}"
+
+
+@time_machine.travel(NOW, tick=False)
+class TestCanReadFlagEvaluations(SimpleTestCase):
+ def _can_read(
+ self,
+ query: str,
+ placeholders: dict[str, ast.Expr] | None = None,
+ timezone: str = "UTC",
+ *,
+ only_in_flag_evaluations: bool = False,
+ ) -> bool:
+ select = parse_select(query, placeholders=placeholders)
+ assert isinstance(select, ast.SelectQuery)
+ return can_read_flag_evaluations(
+ select, datetime.now(UTC), ZoneInfo(timezone), only_in_flag_evaluations=only_in_flag_evaluations
+ )
+
+ def test_complete_days_leave_out_the_day_the_ttl_may_have_dropped(self):
+ assert FLAG_EVALUATIONS_COMPLETE_DAYS < FLAG_EVALUATIONS_TTL_DAYS
+
+ @parameterized.expand(
+ [
+ ("event_equals", _where(FLAG_CALLED, FLAG_KEY, BOUND)),
+ ("event_equals_call", _where("equals(event, '$feature_flag_called')", FLAG_KEY, BOUND)),
+ ("event_in_one_value", _where("event IN ('$feature_flag_called')", FLAG_KEY, BOUND)),
+ (
+ "or_of_pinned_branches_with_their_own_flag_keys",
+ _where(
+ "((event = '$feature_flag_called' AND properties.$feature_flag = 'a') "
+ "OR (event = '$feature_flag_called' AND properties.$feature_flag = 'b'))",
+ BOUND,
+ ),
+ ),
+ ("pin_in_prewhere", f"SELECT count() FROM events PREWHERE {FLAG_CALLED} WHERE {FLAG_KEY} AND {BOUND}"),
+ ("conjuncts_in_nested_ands", _where(FLAG_CALLED, f"({FLAG_KEY} AND ({BOUND} AND distinct_id = 'd'))")),
+ (
+ "qualified_without_an_alias",
+ _where(
+ "events.event = '$feature_flag_called'",
+ "events.properties.$feature_flag = 'my-flag'",
+ "events.timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ ),
+ (
+ "qualified_by_the_alias",
+ "SELECT count() FROM events AS e WHERE e.event = '$feature_flag_called' "
+ "AND e.properties.$feature_flag = 'my-flag' AND e.timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ (
+ "unqualified_under_an_alias_without_a_join",
+ f"SELECT count() FROM events AS e WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
+ ),
+ (
+ "qualified_by_the_alias_with_a_join",
+ _joined(
+ "e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "
+ "AND e.timestamp >= now() - INTERVAL 7 DAY"
+ ),
+ ),
+ ("flag_key_array_access", _where(FLAG_CALLED, "properties['$feature_flag'] = 'my-flag'", BOUND)),
+ (
+ "flag_key_array_access_by_the_alias",
+ "SELECT count() FROM events AS e WHERE e.event = '$feature_flag_called' "
+ "AND e.properties['$feature_flag'] = 'my-flag' AND e.timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ (
+ "flag_key_json_extract",
+ _where(FLAG_CALLED, "JSONExtractString(properties, '$feature_flag') = 'my-flag'", BOUND),
+ ),
+ ("flag_key_equals_call", _where(FLAG_CALLED, "equals(properties.$feature_flag, 'my-flag')", BOUND)),
+ ("flag_key_in_list", _where(FLAG_CALLED, "properties.$feature_flag IN ('a', 'b')", BOUND)),
+ (
+ "flag_key_or",
+ _where(FLAG_CALLED, "(properties.$feature_flag = 'a' OR properties.$feature_flag = 'b')", BOUND),
+ ),
+ ("bound_greater", _where(FLAG_CALLED, FLAG_KEY, "timestamp > now() - INTERVAL 7 DAY")),
+ ("bound_flipped_less_or_equals", _where(FLAG_CALLED, FLAG_KEY, "now() - INTERVAL 7 DAY <= timestamp")),
+ ("bound_flipped_less", _where(FLAG_CALLED, FLAG_KEY, "now() - INTERVAL 7 DAY < timestamp")),
+ ("bound_between", _where(FLAG_CALLED, FLAG_KEY, "timestamp BETWEEN now() - INTERVAL 7 DAY AND now()")),
+ (
+ "bound_greater_or_equals_call",
+ _where(FLAG_CALLED, FLAG_KEY, "greaterOrEquals(timestamp, now() - INTERVAL 7 DAY)"),
+ ),
+ ("bound_greater_call", _where(FLAG_CALLED, FLAG_KEY, "greater(timestamp, now() - INTERVAL 7 DAY)")),
+ (
+ "latest_of_several_bounds",
+ _where(FLAG_CALLED, FLAG_KEY, "timestamp >= now() - INTERVAL 365 DAY", BOUND),
+ ),
+ ("now64", _bounded_by("now64() - INTERVAL 7 DAY")),
+ ("today", _bounded_by("today() - INTERVAL 7 DAY")),
+ ("interval_function", _bounded_by("now() - toIntervalDay(7)")),
+ ("minus_call", _bounded_by("minus(now(), toIntervalDay(7))")),
+ ("subtract_days", _bounded_by("subtractDays(now(), 7)")),
+ ("weeks", _bounded_by("now() - INTERVAL 12 WEEK")),
+ ("months", _bounded_by("now() - INTERVAL 2 MONTH")),
+ ("hours", _bounded_by("now() - INTERVAL 2000 HOUR")),
+ ("plus_interval", _bounded_by("now() + INTERVAL 1 DAY")),
+ ("last_covered_day", _bounded_by(f"now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS} DAY")),
+ ("assume_not_null", _bounded_by("assumeNotNull(now() - INTERVAL 7 DAY)")),
+ ("start_of_day", _bounded_by("toStartOfDay(now() - INTERVAL 7 DAY)")),
+ ("start_of_week_with_a_mode", _bounded_by("toStartOfWeek(now() - INTERVAL 7 DAY, 1)")),
+ ("start_of_month", _bounded_by("toStartOfMonth(now() - INTERVAL 7 DAY)")),
+ ("start_of_interval", _bounded_by("toStartOfInterval(now() - INTERVAL 7 DAY, INTERVAL 1 DAY)")),
+ (
+ "trends_date_from",
+ _bounded_by(f"toStartOfInterval(assumeNotNull(toDateTime('{_text(IN_COVERAGE)}')), toIntervalDay(1))"),
+ ),
+ ("to_date_time", _bounded_by(f"toDateTime('{_text(IN_COVERAGE)}')")),
+ ("to_date_time_with_a_timezone", _bounded_by(f"toDateTime('{_text(IN_COVERAGE)}', 'UTC')")),
+ ("to_date_time64", _bounded_by(f"toDateTime64('{_text(IN_COVERAGE)}', 6)")),
+ ("to_date_time64_with_a_timezone", _bounded_by(f"toDateTime64('{_text(IN_COVERAGE)}', 6, 'UTC')")),
+ ("to_date", _bounded_by(f"toDate('{_date(IN_COVERAGE)}')")),
+ ("date_string", _bounded_by(f"'{_date(IN_COVERAGE)}'")),
+ ("date_time_string", _bounded_by(f"'{_text(IN_COVERAGE)}'")),
+ (
+ "first_covered_day_in_utc",
+ _bounded_by(f"toDateTime('{_text(COVERAGE_START + timedelta(hours=2))}', 'UTC')"),
+ ),
+ # The printer drops SAMPLE 1. The trends query builders set it on every events read.
+ ("sample_of_one", f"SELECT count() FROM events SAMPLE 1 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ (
+ "alias_that_renames_its_own_column",
+ f"SELECT e.timestamp AS timestamp FROM events e WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
+ ),
+ ]
+ )
+ def test_accepts_a_flag_call_query_inside_coverage(self, _name: str, query: str):
+ assert self._can_read(query) is True
+
+ @parameterized.expand(
+ [
+ ("aware_datetime", ast.Constant(value=COVERAGE_START), True),
+ ("date", ast.Constant(value=(COVERAGE_START + timedelta(days=1)).date()), True),
+ ("aware_datetime_before_coverage", ast.Constant(value=COVERAGE_START - timedelta(seconds=1)), False),
+ ]
+ )
+ def test_reads_a_placeholder_bound_against_coverage(self, _name: str, bound: ast.Expr, expected: bool):
+ assert self._can_read(_bounded_by("{bound}"), {"bound": bound}) is expected
+
+ @parameterized.expand(
+ [
+ ("no_event_pin", _where(FLAG_KEY, BOUND)),
+ ("mixed_in", _where("event IN ('$feature_flag_called', '$pageview')", FLAG_KEY, BOUND)),
+ (
+ "or_with_an_unpinned_branch",
+ _where(
+ "((event = '$feature_flag_called' AND properties.$feature_flag = 'a') "
+ "OR (event = '$pageview' AND properties.$feature_flag = 'a'))",
+ BOUND,
+ ),
+ ),
+ ("or_with_a_branch_on_another_column", _where(f"({FLAG_CALLED} OR distinct_id = 'd')", FLAG_KEY, BOUND)),
+ ("negated_pin", _where(f"NOT ({FLAG_CALLED})", FLAG_KEY, BOUND)),
+ (
+ "pin_only_in_a_subquery",
+ _where(FLAG_KEY, BOUND, f"uuid IN (SELECT uuid FROM events WHERE {FLAG_CALLED})"),
+ ),
+ (
+ "unqualified_event_with_a_join",
+ _joined(
+ f"{FLAG_CALLED} AND e.properties.$feature_flag = 'my-flag' AND e.timestamp >= now() - INTERVAL 7 DAY"
+ ),
+ ),
+ (
+ "pin_on_a_joined_alias",
+ _self_joined(
+ "f.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "
+ "AND e.timestamp >= now() - INTERVAL 7 DAY"
+ ),
+ ),
+ ("no_bound", _where(FLAG_CALLED, FLAG_KEY)),
+ ("upper_bound_only", _where(FLAG_CALLED, FLAG_KEY, "timestamp <= now()")),
+ ("flipped_upper_bound", _where(FLAG_CALLED, FLAG_KEY, "now() - INTERVAL 7 DAY >= timestamp")),
+ (
+ "greater_or_equals_call_with_flipped_arguments",
+ _where(FLAG_CALLED, FLAG_KEY, "greaterOrEquals(now() - INTERVAL 7 DAY, timestamp)"),
+ ),
+ ("not_between", _where(FLAG_CALLED, FLAG_KEY, "timestamp NOT BETWEEN now() - INTERVAL 7 DAY AND now()")),
+ ("bound_inside_an_or", _where(FLAG_CALLED, FLAG_KEY, f"({BOUND} OR distinct_id = 'd')")),
+ ("bound_on_another_column", _where(FLAG_CALLED, FLAG_KEY, "created_at >= now() - INTERVAL 7 DAY")),
+ (
+ "unqualified_bound_with_a_join",
+ _joined(f"e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' AND {BOUND}"),
+ ),
+ (
+ "bound_on_a_joined_alias",
+ _self_joined(
+ "e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "
+ "AND f.timestamp >= now() - INTERVAL 7 DAY"
+ ),
+ ),
+ ("one_day_past_coverage", _bounded_by(f"now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS + 1} DAY")),
+ ("months_past_coverage", _bounded_by("now() - INTERVAL 3 MONTH")),
+ (
+ "absolute_date_before_coverage",
+ _bounded_by(f"toDateTime('{_text(COVERAGE_START - timedelta(seconds=1))}', 'UTC')"),
+ ),
+ ("to_date_on_the_first_covered_day", _bounded_by(f"toDate('{_date(COVERAGE_START)}')")),
+ (
+ "string_with_an_offset_before_coverage",
+ _bounded_by(f"toDateTime('{_text(COVERAGE_START + timedelta(hours=6))}+14:00')"),
+ ),
+ ("negative_interval_added", _bounded_by("now() + INTERVAL -100 DAY")),
+ ("subtract_days_past_coverage", _bounded_by(f"subtractDays(now(), {FLAG_EVALUATIONS_COMPLETE_DAYS + 1})")),
+ (
+ "start_of_month_before_coverage",
+ _bounded_by(f"toStartOfMonth(now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS - 1} DAY)"),
+ ),
+ ("start_of_year", _bounded_by("toStartOfYear(now())")),
+ (
+ "start_of_interval_before_coverage",
+ _bounded_by(
+ f"toStartOfInterval(now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS - 1} DAY, INTERVAL 1 MONTH)"
+ ),
+ ),
+ ("bound_read_from_a_field", _bounded_by("toDateTime(properties.since)")),
+ ("bound_from_a_subquery", _bounded_by("(SELECT min(timestamp) FROM events)")),
+ ("interval_read_from_a_field", _bounded_by("now() - toIntervalDay(properties.days)")),
+ ("sample", f"SELECT count() FROM events SAMPLE 0.1 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ ("sample_ratio", f"SELECT count() FROM events SAMPLE 1/10 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ (
+ "sample_of_one_with_an_offset",
+ f"SELECT count() FROM events SAMPLE 1 OFFSET 1/2 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
+ ),
+ (
+ "sample_of_rows",
+ f"SELECT count() FROM events SAMPLE 10000 WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
+ ),
+ ("no_flag_key_pin", _where(FLAG_CALLED, BOUND)),
+ (
+ "flag_key_in_list_with_an_empty_string",
+ _where(FLAG_CALLED, "properties.$feature_flag IN ('my-flag', '')", BOUND),
+ ),
+ ("flag_key_empty_string", _where(FLAG_CALLED, "properties.$feature_flag = ''", BOUND)),
+ ("flag_key_not_empty", _where(FLAG_CALLED, "properties.$feature_flag != ''", BOUND)),
+ ("flag_key_number", _where(FLAG_CALLED, "properties.$feature_flag = 1", BOUND)),
+ (
+ "flag_key_or_with_an_empty_string",
+ _where(FLAG_CALLED, "(properties.$feature_flag = 'my-flag' OR properties.$feature_flag = '')", BOUND),
+ ),
+ (
+ "flag_key_or_with_a_branch_on_another_column",
+ _where(FLAG_CALLED, "(properties.$feature_flag = 'my-flag' OR distinct_id = 'd')", BOUND),
+ ),
+ (
+ "flag_key_on_person_properties",
+ _where(FLAG_CALLED, "person.properties.$feature_flag = 'my-flag'", BOUND),
+ ),
+ ("pin_on_another_property", _where(FLAG_CALLED, "properties.$feature_flag_response = 'my-flag'", BOUND)),
+ (
+ "unqualified_flag_key_with_a_join",
+ _joined(f"e.event = '$feature_flag_called' AND {FLAG_KEY} AND e.timestamp >= now() - INTERVAL 7 DAY"),
+ ),
+ (
+ "flag_key_on_a_joined_alias",
+ _self_joined(
+ "e.event = '$feature_flag_called' AND f.properties.$feature_flag = 'my-flag' "
+ "AND e.timestamp >= now() - INTERVAL 7 DAY"
+ ),
+ ),
+ (
+ "or_pin_with_a_branch_without_its_flag_key",
+ _where(
+ f"((event = '$feature_flag_called' AND properties.$feature_flag = 'a') OR {FLAG_CALLED})",
+ BOUND,
+ ),
+ ),
+ (
+ "bound_on_an_alias_that_shadows_timestamp",
+ f"SELECT now() AS timestamp FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
+ ),
+ (
+ "pin_on_an_alias_that_shadows_event",
+ f"SELECT '$feature_flag_called' AS event FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}",
+ ),
+ ("select_star", f"SELECT * FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ ("qualified_select_star", f"SELECT e.* FROM events e WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ ("columns_expression", f"SELECT COLUMNS('^p') FROM events WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ ("final", f"SELECT count() FROM events FINAL WHERE {FLAG_CALLED} AND {FLAG_KEY} AND {BOUND}"),
+ ("flag_key_that_reads_as_a_number", _where(FLAG_CALLED, "properties.$feature_flag = '100'", BOUND)),
+ ("flag_key_that_reads_as_a_boolean", _where(FLAG_CALLED, "properties.$feature_flag = 'true'", BOUND)),
+ ("flag_key_that_reads_as_json_null", _where(FLAG_CALLED, "properties.$feature_flag = 'null'", BOUND)),
+ (
+ "flag_key_list_with_an_object",
+ _where(FLAG_CALLED, "properties.$feature_flag IN ('my-flag', '{\"a\": 1}')", BOUND),
+ ),
+ (
+ "column_aliases",
+ "SELECT count() FROM events AS e (uuid, event, properties, timestamp, distinct_id, x) "
+ "WHERE e.event = '$feature_flag_called' AND e.properties.$feature_flag = 'my-flag' "
+ "AND e.timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ (
+ "bare_star_with_an_array_join",
+ "SELECT * FROM events ARRAY JOIN [1] AS n WHERE events.event = '$feature_flag_called' "
+ "AND events.properties.$feature_flag = 'my-flag' AND events.timestamp >= now() - INTERVAL 7 DAY",
+ ),
+ ("bound_near_year_one", _bounded_by("'0001-01-01'")),
+ ("interval_past_datetime_range", _bounded_by("now() - INTERVAL 999999999 YEAR")),
+ # Each level used to evaluate its value twice, so 40 levels took 2^40 steps.
+ ("deeply_nested_start_of_day_of_a_column", _bounded_by(_nested("toStartOfDay({})", 40))),
+ (
+ "deeply_nested_day_interval_of_a_column",
+ _bounded_by(_nested("toStartOfInterval({}, toIntervalDay(1))", 40)),
+ ),
+ ]
+ )
+ @pytest.mark.timeout(5, func_only=True)
+ def test_refuses_a_query_that_breaks_a_condition(self, _name: str, query: str):
+ assert self._can_read(query) is False
+
+ @parameterized.expand(
+ [
+ ("no_lower_bound", _where(FLAG_CALLED, FLAG_KEY), True),
+ ("bound_before_coverage", _bounded_by("now() - INTERVAL 180 DAY"), True),
+ ("no_flag_key", _where(FLAG_CALLED, BOUND), True),
+ ("no_event_pin", _where(FLAG_KEY, BOUND), False),
+ ("mixed_in", _where("event IN ('$feature_flag_called', '$pageview')", FLAG_KEY, BOUND), False),
+ ("sample", f"SELECT count() FROM events SAMPLE 0.1 WHERE {FLAG_CALLED} AND {FLAG_KEY}", False),
+ ("select_star", f"SELECT * FROM events WHERE {FLAG_CALLED}", False),
+ ]
+ )
+ def test_skips_coverage_and_flag_key_when_only_flag_evaluations_has_flag_calls(
+ self, _name: str, query: str, expected: bool
+ ):
+ assert self._can_read(query, only_in_flag_evaluations=True) is expected
+
+ @parameterized.expand(
+ [
+ # The trends runner's date range for "Last N days" in a day-interval insight.
+ *[
+ (f"last_{label}_days_in_{zone}", zone, _last_days_bound(zone, days), expected)
+ for zone in ("UTC", "Pacific/Kiritimati", "Etc/GMT+12")
+ for label, days, expected in (
+ ("complete", FLAG_EVALUATIONS_COMPLETE_DAYS - 1, True),
+ ("too_many", FLAG_EVALUATIONS_COMPLETE_DAYS + 1, False),
+ )
+ ],
+ # Pacific/Kiritimati is UTC+14, so its midnight on the first covered day is 10:00 the day before in UTC.
+ ("to_date_time_on_the_first_covered_day_in_utc", "UTC", f"toDateTime('{_text(COVERAGE_START)}')", True),
+ (
+ "to_date_time_on_the_first_covered_day_in_kiritimati",
+ "Pacific/Kiritimati",
+ f"toDateTime('{_text(COVERAGE_START)}')",
+ False,
+ ),
+ ("date_string_on_the_first_covered_day_in_utc", "UTC", f"'{_date(COVERAGE_START)}'", True),
+ (
+ "date_string_on_the_first_covered_day_in_kiritimati",
+ "Pacific/Kiritimati",
+ f"'{_date(COVERAGE_START)}'",
+ False,
+ ),
+ (
+ "naive_placeholder_on_the_first_covered_day_in_kiritimati",
+ "Pacific/Kiritimati",
+ "{bound}",
+ False,
+ ),
+ ("naive_placeholder_on_the_first_covered_day_in_utc", "UTC", "{bound}", True),
+ ]
+ )
+ def test_reads_a_time_without_a_zone_in_the_team_timezone(
+ self, _name: str, timezone: str, bound: str, expected: bool
+ ):
+ placeholders: dict[str, ast.Expr] = {"bound": ast.Constant(value=COVERAGE_START.replace(tzinfo=None))}
+
+ assert self._can_read(_bounded_by(bound), placeholders, timezone) is expected
+
+ @parameterized.expand(
+ [
+ (
+ "date_rounding_shifted_in_a_zone_behind_utc",
+ datetime(2026, 10, 24, 12, 0, tzinfo=UTC),
+ "America/Los_Angeles",
+ "toMonday(toDateTime('2026-08-02 22:00:00')) - INTERVAL 1 HOUR",
+ ),
+ # Troll is on UTC+0 now and was on UTC+2 89 days back, so subtracting 89 days lands two hours earlier
+ # in UTC than 89 times 24 hours does.
+ (
+ "day_rounding_across_a_two_hour_dst_change",
+ datetime(2026, 10, 28, 23, 30, tzinfo=UTC),
+ "Antarctica/Troll",
+ f"toStartOfDay(now() - INTERVAL {FLAG_EVALUATIONS_COMPLETE_DAYS} DAY)",
+ ),
+ ]
+ )
+ def test_refuses_a_bound_that_calendar_arithmetic_moves_before_coverage(
+ self, _name: str, now: datetime, timezone: str, bound: str
+ ):
+ with time_machine.travel(now, tick=False):
+ assert self._can_read(_bounded_by(bound), timezone=timezone) is False
diff --git a/posthog/hogql/transforms/trino/validate.py b/posthog/hogql/transforms/trino/validate.py
index 38c7210f8b3e..d8af44dc55f8 100644
--- a/posthog/hogql/transforms/trino/validate.py
+++ b/posthog/hogql/transforms/trino/validate.py
@@ -277,11 +277,7 @@ class TrinoReadyValidator(TraversingVisitor):
self._fail("TRINO_GLOBAL_JOIN_UNSUPPORTED", join_type, node)
if node.table_final:
self._fail("TRINO_FINAL_UNSUPPORTED", "FINAL", node)
- if node.sample is not None and not (
- node.sample.sample_value.left.value == 1
- and node.sample.sample_value.right is None
- and node.sample.offset_value is None
- ):
+ if node.sample is not None and not node.sample.samples_all_rows():
self._fail("TRINO_SAMPLE_UNSUPPORTED", "SAMPLE", node)
super().visit_join_expr(node)
diff --git a/posthog/hogql/utils.py b/posthog/hogql/utils.py
index c5567f2de380..2031320c8bbe 100644
--- a/posthog/hogql/utils.py
+++ b/posthog/hogql/utils.py
@@ -71,6 +71,11 @@ def ilike_matches(pattern: str, text: str) -> bool:
return like_matches(pattern.lower(), text.lower())
+def is_json_verbatim(value: str) -> bool:
+ """True when JSON text stores the string unchanged: non-empty printable ASCII with no quote or backslash."""
+ return bool(value) and all(" " <= char <= "~" and char not in '"\\' for char in value)
+
+
def unwrap_optional(t):
if get_origin(t) is Union and type(None) in get_args(t):
# Return the first argument, which is the actual type in Optional[type]
diff --git a/posthog/hogql_queries/test/test_events_query_runner.py b/posthog/hogql_queries/test/test_events_query_runner.py
index 52d0781a404a..668d79b60cde 100644
--- a/posthog/hogql_queries/test/test_events_query_runner.py
+++ b/posthog/hogql_queries/test/test_events_query_runner.py
@@ -1340,7 +1340,11 @@ class TestEventsQueryRunner(ClickhouseTestMixin, APIBaseTest):
properties={"email": FLAG_EVALUATIONS_EMAIL},
)
create_group_type_mapping(
- team=self.team, project_id=self.team.project_id, group_type="organization", group_type_index=0
+ team=self.team,
+ project_id=self.team.project_id,
+ group_type="organization",
+ group_type_index=0,
+ created_at=FLAG_CALL_TIMESTAMP - timedelta(days=1),
)
create_group(
team_id=self.team.pk,
diff --git a/posthog/hogql_queries/test/test_hogql_cohort_query.py b/posthog/hogql_queries/test/test_hogql_cohort_query.py
index 84c38a2e4047..fb1c9ea7cff3 100644
--- a/posthog/hogql_queries/test/test_hogql_cohort_query.py
+++ b/posthog/hogql_queries/test/test_hogql_cohort_query.py
@@ -3,7 +3,13 @@ from typing import cast
from zoneinfo import ZoneInfo
import time_machine
-from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_person, flush_persons_and_events
+from posthog.test.base import (
+ APIBaseTest,
+ ClickhouseTestMixin,
+ _create_flag_evaluations,
+ _create_person,
+ flush_persons_and_events,
+)
from unittest.mock import MagicMock, patch
from parameterized import parameterized
@@ -14,6 +20,8 @@ from posthog.hogql_queries.hogql_cohort_query import HogQLCohortQuery
from products.cohorts.backend.models.cohort import Cohort
from products.event_definitions.backend.models.property_definition import PropertyDefinition, PropertyType
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
class TestHogQLCohortQuery(ClickhouseTestMixin, APIBaseTest):
@@ -614,6 +622,57 @@ class TestHogQLCohortQuery(ClickhouseTestMixin, APIBaseTest):
)
return {str(row[0]) for row in rows}
+ def test_performed_flag_call_criterion_reads_flag_evaluations_when_the_rewrite_is_on(self) -> None:
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization,
+ defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},
+ )
+ caller = _create_person(team=self.team, distinct_ids=["caller"], immediate=True)
+ other_caller = _create_person(team=self.team, distinct_ids=["other-caller"], immediate=True)
+ flush_persons_and_events()
+ _create_flag_evaluations(self.team.pk, "probe-flag", distinct_id="caller", person_id=caller.uuid)
+ _create_flag_evaluations(self.team.pk, "other-flag", distinct_id="other-caller", person_id=other_caller.uuid)
+ cohort = Cohort.objects.create(
+ team=self.team,
+ name="called probe-flag",
+ filters={
+ "properties": {
+ "type": "AND",
+ "values": [
+ {
+ "type": "AND",
+ "values": [
+ {
+ "key": "$feature_flag_called",
+ "type": "behavioral",
+ "value": "performed_event",
+ "event_type": "events",
+ "time_value": 30,
+ "time_interval": "day",
+ "event_filters": [
+ {
+ "key": "$feature_flag",
+ "type": "event",
+ "value": "probe-flag",
+ "operator": "exact",
+ }
+ ],
+ }
+ ],
+ }
+ ],
+ }
+ },
+ )
+
+ with patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-from-flag-evaluations",
+ ):
+ cohort.calculate_people_ch(pending_version=0)
+
+ self.assertEqual(self._cohort_member_ids(cohort, 0), {str(caller.uuid)})
+
def test_filter_test_accounts_excludes_person_matches_and_tracks_team_settings(self) -> None:
internal = _create_person(
team=self.team,
diff --git a/posthog/models/data_deletion_request.py b/posthog/models/data_deletion_request.py
index ca751c18ecb2..498fe1b41e09 100644
--- a/posthog/models/data_deletion_request.py
+++ b/posthog/models/data_deletion_request.py
@@ -127,6 +127,7 @@ def compile_hogql_predicate(obj, use_new_events_schema: bool = False, hogql_tabl
# A deletion predicate must match rows regardless of retention; the events-retention floor would otherwise
# narrow an events sub-query here and leave events older than the window un-deleted.
apply_events_retention_floor=False,
+ rewrite_flag_calls_to_flag_evaluations=False,
)
try:
sql = translate_hogql(
diff --git a/posthog/models/test/test_data_deletion_request.py b/posthog/models/test/test_data_deletion_request.py
index 46e4a75e0e65..a39db70a252d 100644
--- a/posthog/models/test/test_data_deletion_request.py
+++ b/posthog/models/test/test_data_deletion_request.py
@@ -30,6 +30,8 @@ from posthog.models.organization import Organization
from posthog.models.team import Team
from products.event_definitions.backend.models import PropertyDefinition
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
LOCMEM_CACHE = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "ddr-test"}}
@@ -205,6 +207,28 @@ def test_hogql_predicate_subquery_is_team_scoped(db):
assert f"equals(team_id, {team_a.id})" not in sql_b
+def test_compile_hogql_predicate_keeps_a_flag_call_subquery_on_events(team):
+ from posthog.models.data_deletion_request import compile_hogql_predicate
+
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=team.organization,
+ defaults={"flag_evaluations_mode": FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY},
+ )
+ predicate = (
+ "distinct_id IN (SELECT distinct_id FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.$feature_flag = 'probe-flag' AND timestamp >= now() - INTERVAL 7 DAY)"
+ )
+ request = DataDeletionRequest(**_base_kwargs(team_id=team.id, events=["$pageview"], hogql_predicate=predicate))
+
+ with patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-from-flag-evaluations",
+ ):
+ sql, _ = compile_hogql_predicate(request)
+
+ assert "flag_evaluations" not in sql
+
+
def test_compile_hogql_predicate_returns_sql_and_values(team):
from posthog.models.data_deletion_request import compile_hogql_predicate
diff --git a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
index a8e46c46fb42..025ccc363081 100644
--- a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
+++ b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
@@ -1,6 +1,7 @@
import re
import zoneinfo
import itertools
+from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timedelta
from itertools import groupby
@@ -12,6 +13,7 @@ from posthog.test.base import (
APIBaseTest,
ClickhouseTestMixin,
_create_event,
+ _create_flag_evaluations,
_create_person,
also_test_with_materialized_columns,
flush_persons_and_events,
@@ -83,9 +85,33 @@ from posthog.test.test_utils import create_group_type_mapping_without_created_at
from products.actions.backend.models.action import Action
from products.cohorts.backend.models.cohort import Cohort
from products.event_definitions.backend.models.property_definition import PropertyDefinition
+from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
+from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
from products.product_analytics.backend.hogql_queries.trends.trends_query_runner import TrendsQueryRunner
+def _flag_call_event_series(team: Team, flag_keys: list[str]) -> EventsNode:
+ return EventsNode(
+ event="$feature_flag_called",
+ properties=[EventPropertyFilter(key="$feature_flag", value=flag_keys, operator=PropertyOperator.EXACT)],
+ )
+
+
+def _flag_call_action_series(team: Team, flag_keys: list[str]) -> ActionsNode:
+ action = Action.objects.create(
+ team=team,
+ name="called a flag",
+ steps_json=[
+ {
+ "event": "$feature_flag_called",
+ "properties": [{"key": "$feature_flag", "type": "event", "value": flag_key, "operator": "exact"}],
+ }
+ for flag_key in flag_keys
+ ],
+ )
+ return ActionsNode(id=action.id)
+
+
@dataclass
class Series:
event: str
@@ -6629,6 +6655,83 @@ class TestTrendsQueryRunner(ClickhouseTestMixin, APIBaseTest):
assert response.results[0]["count"] == 2
assert response.results[0]["data"] == [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+ @parameterized.expand(
+ [
+ ("event_series", _flag_call_event_series, ["probe-flag"], 2),
+ ("event_series_with_two_flag_keys", _flag_call_event_series, ["probe-flag", "other-flag"], 3),
+ ("action_series", _flag_call_action_series, ["probe-flag"], 2),
+ ("action_with_two_flag_call_steps", _flag_call_action_series, ["probe-flag", "other-flag"], 3),
+ ("daily_active_users", _flag_call_event_series, ["probe-flag"], 2, BaseMathType.DAU),
+ (
+ "breakdown_by_response",
+ _flag_call_event_series,
+ ["probe-flag"],
+ 2,
+ None,
+ BreakdownFilter(breakdown="$feature_flag_response", breakdown_type=BreakdownType.EVENT),
+ ),
+ ("test_account_filters", _flag_call_event_series, ["probe-flag"], 2, None, None, True),
+ (
+ "mode_1",
+ _flag_call_event_series,
+ ["probe-flag"],
+ 2,
+ None,
+ None,
+ False,
+ FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ ),
+ ]
+ )
+ def test_flag_call_series_reads_flag_evaluations_when_the_rewrite_is_on(
+ self,
+ _name: str,
+ series: Callable[[Team, list[str]], EventsNode | ActionsNode],
+ flag_keys: list[str],
+ expected_count: int,
+ math: BaseMathType | None = None,
+ breakdown: BreakdownFilter | None = None,
+ filter_test_accounts: bool = False,
+ mode: FlagEvaluationsMode = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ ):
+ OrganizationFeatureFlagsConfig.objects.update_or_create(
+ organization=self.organization, defaults={"flag_evaluations_mode": mode}
+ )
+ if filter_test_accounts:
+ self.team.test_account_filters = [
+ {"key": "email", "value": "@posthog.com", "operator": "not_icontains", "type": "person"}
+ ]
+ self.team.save()
+ _create_flag_evaluations(self.team.pk, "probe-flag", count=2)
+ _create_flag_evaluations(self.team.pk, "other-flag")
+ _create_flag_evaluations(self.team.pk, "unselected-flag")
+ node = series(self.team, flag_keys)
+ if math is not None:
+ node = node.model_copy(update={"math": math})
+
+ with patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key == "hogql-flag-calls-from-flag-evaluations",
+ ):
+ # Mode 2 raises when the swapped query reads people differently, so the persons-on-events mode is fixed.
+ # Mode 1 reads events on the native-JSON events table, so the events table is fixed too.
+ response = self._run_trends_query(
+ "-7d",
+ None,
+ IntervalType.DAY,
+ [node],
+ breakdown=breakdown,
+ filter_test_accounts=filter_test_accounts,
+ hogql_modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,
+ useNewEventsSchema=False,
+ ),
+ )
+
+ assert response.results[0]["count"] == expected_count
+ if breakdown is not None:
+ assert response.results[0]["breakdown_value"] == "true"
+
def test_multiple_breakdowns_work_with_formula(self):
self._create_test_events()
flush_persons_and_events()