:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): read inline cohorts and sample ratios through the flag calls union (#126959)
Phil Haack0fabca8b9664
diff --git a/posthog/hogql/context.py b/posthog/hogql/context.pyindex f3e8bc63bab6..c4caade32785 100644--- a/posthog/hogql/context.py+++ b/posthog/hogql/context.py@@ -179,6 +179,10 @@ class HogQLContext:# Per-query event property definitions that transform loads. The transform runs once for the query and once for# each saved view the query inlines, so this loads each property's definition once per query.flag_call_property_metadata: Optional["FlagCallPropertyMetadata"] = field(default=None, compare=False, repr=False)+ # The (start, end) of each `$feature_flag_called` literal in the query text that the flag-call union reads.+ # `_resolve_with_swaps` in transforms/flag_calls_from_flag_evaluations.py fills it when the swap succeeds, and the+ # SQL editor's notices read it.+ flag_called_union_spans: set[tuple[int, int]] = field(default_factory=set)# Backend-only opt-in for transforms/events_read_in_order.py. The prefix is slower when the range is short or# the filter is selective, so set it only for a query shape that was measured to be faster with it.diff --git a/posthog/hogql/database/schema/events_with_flag_calls.py b/posthog/hogql/database/schema/events_with_flag_calls.pyindex a5a8452d7ae6..fbebff4c70cb 100644--- a/posthog/hogql/database/schema/events_with_flag_calls.py+++ b/posthog/hogql/database/schema/events_with_flag_calls.py@@ -18,7 +18,7 @@ from posthog.hogql.database.schema.events import EVENTS_AVOID_ASTERISK_FIELDS, Efrom posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable, flag_evaluations_coverage_startfrom posthog.hogql.errors import QueryErrorfrom posthog.hogql.parser import parse_expr-from posthog.hogql.visitor import GetFieldsTraverser+from posthog.hogql.visitor import GetFieldsTraverser, clone_exprfrom posthog.dataclasses import frozen@@ -141,9 +141,24 @@ class EventsWithFlagCallsTable(LazyTable):and_conditions,flag_key_typed_as_string,)- from posthog.hogql.transforms.flag_calls_union import union_branch_filters, union_events_filter # noqa: PLC0415+ from posthog.hogql.transforms.flag_calls_union import ( # noqa: PLC0415+ sample_flag_evaluations_filter,+ union_branch_filters,+ union_events_filter,+ union_scan_sample,+ )assert context.database is not None+ # Lazy table resolution drops the scan's SAMPLE, so each branch applies the ratio itself.+ sample = union_scan_sample(node, self)++ def sampled_scan(table: ast.Field, alias: str | None = None) -> ast.JoinExpr:+ return ast.JoinExpr(+ sample=clone_expr(sample, clear_types=True, clear_locations=True) if sample is not None else None,+ table=table,+ alias=alias,+ )+# `SELECT count()` reads no field, and a subquery needs at least one column.accessed = dict(table_to_add.fields_accessed) or {"timestamp": ["timestamp"]}events_columns: list[ast.Expr] = [@@ -155,7 +170,7 @@ class EventsWithFlagCallsTable(LazyTable):# once. The second scan asks only for columns, and it reads the same filter from this query's WHERE.return ast.SelectQuery(select=events_columns,- select_from=ast.JoinExpr(table=ast.Field(chain=list(EVENTS_WITH_FLAG_CALLS_CHAIN)), alias="events"),+ select_from=sampled_scan(table=ast.Field(chain=list(EVENTS_WITH_FLAG_CALLS_CHAIN)), alias="events"),where=union_events_filter(node, self),)@@ -178,7 +193,7 @@ class EventsWithFlagCallsTable(LazyTable):def events_branch(*conditions: ast.Expr | None) -> ast.SelectQuery:return ast.SelectQuery(select=events_columns,- select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),+ select_from=sampled_scan(table=ast.Field(chain=["events"])),where=and_conditions([*conditions, filters.events]),)@@ -199,7 +214,13 @@ class EventsWithFlagCallsTable(LazyTable):table=ast.Field(chain=["posthog", "flag_evaluations"]), alias="flag_evaluations"),where=and_conditions(- [parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff}), filters.flag_evaluations]+ [+ parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff}),+ filters.flag_evaluations,+ sample_flag_evaluations_filter(sample, flag_evaluations_field(["distinct_id"]))+ if sample is not None+ else None,+ ]),))diff --git a/posthog/hogql/flag_called_warnings.py b/posthog/hogql/flag_called_warnings.pyindex 6b1c5b9fee9a..382fcd2a2f06 100644--- a/posthog/hogql/flag_called_warnings.py+++ b/posthog/hogql/flag_called_warnings.py@@ -24,6 +24,17 @@ FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE = ("of flag calls.")+FLAG_CALLED_FROM_BOTH_TABLES_NOTICE = (+ f"This query reads {FLAG_CALLED_EVENT} events from posthog.flag_evaluations, which keeps the last 90 days "+ "of flag calls, and other events from events."+)++# While events still holds every flag call, the union reads only the recent ones from posthog.flag_evaluations.+FLAG_CALLED_SPLIT_BY_AGE_NOTICE = (+ f"This query reads {FLAG_CALLED_EVENT} events from the last 90 days from posthog.flag_evaluations. "+ "It reads older ones and all other events from events."+)+class _FlagCalledLiteralFinder(TraversingVisitor):def __init__(self, table_class: type[Table]) -> None:@@ -115,3 +126,8 @@ def flag_called_spans_read_from_flag_evaluations(prepared: ast.AST) -> set[Literfinder = _FlagCalledLiteralFinder(FlagEvaluationsTable)finder.visit(prepared)return {LiteralSpan(start=literal.start, end=literal.end) for literal in finder.literals}+++def flag_called_spans_read_from_union(context: HogQLContext) -> set[LiteralSpan]:+ """The spans of the `$feature_flag_called` literals whose scan reads `events` and `flag_evaluations` together."""+ return {LiteralSpan(start=start, end=end) for start, end in context.flag_called_union_spans}diff --git a/posthog/hogql/functions/cohort.py b/posthog/hogql/functions/cohort.pyindex e03b300a1402..29141a4a5627 100644--- a/posthog/hogql/functions/cohort.py+++ b/posthog/hogql/functions/cohort.py@@ -70,6 +70,8 @@ def inline_cohort_query(is_static: bool,version: Optional[int],context: HogQLContext,+ *,+ read_flag_calls_from_union: bool = True,) -> Optional[ast.SelectQuery | ast.SelectSetQuery]:from posthog.hogql_queries.hogql_cohort_query import HogQLCohortQuery@@ -95,7 +97,13 @@ def inline_cohort_query(context.team = Team.objects.get(id=context.team_id)- return HogQLCohortQuery(cohort=cohort, team=context.team).get_query()+ query = HogQLCohortQuery(cohort=cohort, team=context.team).get_query()+ if not read_flag_calls_from_union:+ return query+ # Deferred: the transform imports the resolver, which imports this module.+ from posthog.hogql.transforms.flag_calls_union import read_flag_calls_in_inline_cohort # noqa: PLC0415++ return read_flag_calls_in_inline_cohort(query, context)def cohort_subquery(cohort_id, is_static, version: Optional[int] = None) -> ast.Expr:diff --git a/posthog/hogql/metadata.py b/posthog/hogql/metadata.pyindex 0e09d279aa9d..6ba1ed5845c1 100644--- a/posthog/hogql/metadata.py+++ b/posthog/hogql/metadata.py@@ -36,10 +36,13 @@ from posthog.hogql.errors import ()from posthog.hogql.filters import replace_filtersfrom posthog.hogql.flag_called_warnings import (+ FLAG_CALLED_FROM_BOTH_TABLES_NOTICE,FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE,+ FLAG_CALLED_SPLIT_BY_AGE_NOTICE,LiteralSpan,flag_called_on_events_warnings,flag_called_spans_read_from_flag_evaluations,+ flag_called_spans_read_from_union,)from posthog.hogql.index_eligibility import IndexEligibilityReport, build_index_eligibility_reportfrom posthog.hogql.metadata_heuristics import run_metadata_heuristics@@ -226,6 +229,7 @@ def get_hogql_metadata(flag_called_notices = _flag_called_notices(hogql_ast, prepared_ast, context, team)heuristic_warnings.extend(flag_called_notices.move_warnings)context.notices.extend(flag_called_notices.flag_evaluations_notices)+ context.notices.extend(flag_called_notices.both_tables_notices)if source is None and query.indexUsage and _index_usage_enabled(team):_attach_index_usage(response, hogql_ast, context)@@ -337,6 +341,7 @@ def _flag_called_announcement_url(payload: object) -> str | None:class _FlagCalledNotices:move_warnings: tuple[HogQLNotice, ...] = ()flag_evaluations_notices: tuple[HogQLNotice, ...] = ()+ both_tables_notices: tuple[HogQLNotice, ...] = ()def _flag_called_notices(@@ -345,9 +350,9 @@ def _flag_called_notices(context: HogQLContext,team: Team,) -> _FlagCalledNotices:- """The move warning on each flag-call literal that reads events, and a notice on each literal whose query- reads posthog.flag_evaluations instead. `prepared_ast` is what the printer ran, so it says which scans it- swapped."""+ """The move warning on each flag-call literal that reads events, a notice on each literal whose query reads+ posthog.flag_evaluations instead, and a notice on each literal whose scan reads events and+ posthog.flag_evaluations together. `prepared_ast` is what the printer ran, so it says which scans it swapped."""try:move_notices = _flag_called_move_notices(team)if move_notices is None or not move_notices.enabled:@@ -359,15 +364,27 @@ def _flag_called_notices(swapped = flag_called_spans_read_from_flag_evaluations(prepared_ast) if prepared_ast is not None else set()move_warnings: list[HogQLNotice] = []flag_evaluations_notices: list[HogQLNotice] = []+ both_tables_notices: list[HogQLNotice] = []+ read_from_both_tables = flag_called_spans_read_from_union(context)+ both_tables_message = (+ FLAG_CALLED_FROM_BOTH_TABLES_NOTICE+ if context.database is not None and context.database.flag_calls_only_in_flag_evaluations+ else FLAG_CALLED_SPLIT_BY_AGE_NOTICE+ )for warning in warnings:- if LiteralSpan(start=warning.start, end=warning.end) in swapped:+ span = LiteralSpan(start=warning.start, end=warning.end)+ if span in swapped:flag_evaluations_notices.append(warning.model_copy(update={"message": FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, "url": url}))+ elif span in read_from_both_tables:+ both_tables_notices.append(warning.model_copy(update={"message": both_tables_message, "url": url}))else:move_warnings.append(warning.model_copy(update={"url": url}))return _FlagCalledNotices(- move_warnings=tuple(move_warnings), flag_evaluations_notices=tuple(flag_evaluations_notices)+ move_warnings=tuple(move_warnings),+ flag_evaluations_notices=tuple(flag_evaluations_notices),+ both_tables_notices=tuple(both_tables_notices),)except Exception:# The warning is advisory. A query that compiles must not be reported as invalid because thisdiff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.pyindex 02cfec4ae349..a26eb0fdfd43 100644--- a/posthog/hogql/test/test_metadata.py+++ b/posthog/hogql/test/test_metadata.py@@ -23,7 +23,12 @@ from posthog.schema import ()from posthog.hogql.direct_connection import INVALID_CONNECTION_ID_ERROR-from posthog.hogql.flag_called_warnings import FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, FLAG_CALLED_ON_EVENTS_WARNING+from posthog.hogql.flag_called_warnings import (+ FLAG_CALLED_FROM_BOTH_TABLES_NOTICE,+ FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE,+ FLAG_CALLED_ON_EVENTS_WARNING,+ FLAG_CALLED_SPLIT_BY_AGE_NOTICE,+)from posthog.hogql.metadata import get_hogql_metadatafrom posthog.hogql.parser import parse_selectfrom posthog.hogql.taxonomy_validation import MAX_SUGGESTED_NAMES@@ -449,6 +454,44 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):expected_notices=1,),param("refused_by_the_swap", _FLAG_CALLED_QUERY, True, 1, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS),+ param(+ "mixed_scan_read_from_both_tables",+ "SELECT count() FROM events WHERE event IN ('$pageview', '$feature_flag_called')",+ True,+ 0,+ _move_notices("https://example.com/announcement"),+ "https://example.com/announcement",+ mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ union_enabled=True,+ expected_union_notices=1,+ ),+ param(+ "mixed_scan_read_from_both_tables_once_events_gets_no_flag_calls",+ "SELECT count() FROM events WHERE event IN ('$pageview', '$feature_flag_called')",+ True,+ 0,+ _move_notices("https://example.com/announcement"),+ "https://example.com/announcement",+ mode=FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,+ union_enabled=True,+ expected_union_notices=1,+ union_notice=FLAG_CALLED_FROM_BOTH_TABLES_NOTICE,+ ),+ param(+ "mixed_scan_the_union_declines",+ "SELECT goe_0.properties FROM events WHERE event IN ('$pageview', '$feature_flag_called')",+ True,+ 1,+ mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ union_enabled=True,+ ),+ param(+ "mixed_scan_with_the_union_off",+ "SELECT count() FROM events WHERE event IN ('$pageview', '$feature_flag_called')",+ True,+ 1,+ mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS,+ ),param("read_from_flag_evaluations_without_coverage_once_events_gets_no_flag_calls",_FLAG_CALLED_QUERY,@@ -487,6 +530,9 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):expected_url: str | None = None,expected_notices: int = 0,mode: FlagEvaluationsMode = FlagEvaluationsMode.EVENTS,+ union_enabled: bool = False,+ expected_union_notices: int = 0,+ union_notice: str = FLAG_CALLED_SPLIT_BY_AGE_NOTICE,) -> None:OrganizationFeatureFlagsConfig.objects.update_or_create(organization=self.organization,@@ -512,12 +558,17 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):return_value=flag_evaluations_enabled,),patch("posthoganalytics.get_feature_flag_result", return_value=move_notices),+ patch(+ "posthog.hogql.database.database.feature_enabled_or_false",+ side_effect=lambda key, *args, **kwargs: union_enabled and key == "hogql-flag-calls-union",+ ),):metadata = self._select(query)self.assertTrue(metadata.isValid, metadata.errors)warnings = [w for w in metadata.warnings if w.message == FLAG_CALLED_ON_EVENTS_WARNING]notices = [n for n in metadata.notices if n.message == FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE]+ union_notices = [n for n in metadata.notices if n.message == union_notice]literal_start = query.rindex("'$feature_flag_called'")literal_end = literal_start + len("'$feature_flag_called'")self.assertEqual(@@ -527,6 +578,10 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):self.assertEqual([(n.start, n.end, n.url) for n in notices], [(literal_start, literal_end, expected_url)] * expected_notices)+ self.assertEqual(+ [(n.start, n.end, n.url) for n in union_notices],+ [(literal_start, literal_end, expected_url)] * expected_union_notices,+ )def test_metadata_does_not_log_invalid_announcement_url_without_a_flag_called_literal(self) -> None:with (diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.pyindex 1a2ac95408e9..6e8e3ffb4e07 100644--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py@@ -13,7 +13,9 @@ import jsonfrom collections import Counterfrom collections.abc import Callable, Iteratorfrom datetime import UTC, date, datetime, time, timedelta, tzinfo+from fractions import Fractionfrom functools import partial+from math import isfinitefrom typing import TYPE_CHECKING, Generic, Protocolfrom zoneinfo import ZoneInfo, ZoneInfoNotFoundError@@ -217,8 +219,9 @@ def swap_flag_calls(For an organization whose flag calls are only in `flag_evaluations`, a scan that the syntactic rules miss swapstoo when the resolved original shows that its filters pin `event` to flag calls in its WHERE or PREWHERE. The- query raises when such a scan cannot swap. A scan that the resolver adds, such as one in an inline cohort, keeps- reading `events`.+ query raises when such a scan cannot swap. A scan that the resolver adds, such as one in an inline cohort, is not+ swapped here. On mode 2 with the union enabled, `read_flag_calls_in_inline_cohort` points an inline cohort's scans+ at the union. Otherwise they read `events`.The resolver runs this on the body of each saved view that it inlines, and `view_name` names that view for theerror. The check of resolved filters skips the joins of a view's body in the enclosing query, because the swap of@@ -235,13 +238,16 @@ def swap_flag_calls(if _swap_applies(node, context, database):swap = _resolve_with_swaps(node, context, resolve, database, view_name)else:- swap = FlagCallSwap(source=node, resolved=resolve(node))+ swap = FlagCallSwap(+ source=node,+ resolved=_without_unreadable_inline_cohort_scans(resolve(node), context, view_name),+ )# A view's body names flag calls only after resolution when it filters with `matchesAction`, so its swap never# runs. The check reads the whole resolved query, so it runs once, for the top-level query.if database.flag_calls_only_in_flag_evaluations and view_name is None:unswapped_view = _view_reading_flag_calls_from_events(swap.resolved)if unswapped_view is not None:- raise _flag_calls_only_in_flag_evaluations_error(+ raise flag_calls_only_in_flag_evaluations_error(f"it reads the saved view {unswapped_view}, which reads them from events", None)return swap@@ -254,12 +260,13 @@ def _resolve_with_swaps(# Deferred: flag_calls_union imports this module.from posthog.hogql.transforms.flag_calls_union import ( # noqa: PLC0415read_flag_calls_with_union,+ union_applies,union_unreadable_reason,)with context.timings.measure("flag_calls_from_flag_evaluations"):# The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.- compares_as_string = partial(_flag_key_compares_as_string, context)+ compares_as_string = partial(team_flag_key_compares_as_string, context)rewritten = clone_expr(node)clones = _join_clones(node, rewritten) if only_in_flag_evaluations else {}swapper = _EventsScanSwapper(@@ -267,6 +274,7 @@ def _resolve_with_swaps(timezone=ZoneInfo(database.get_timezone()),only_in_flag_evaluations=only_in_flag_evaluations,flag_key_compares_as_string=compares_as_string,+ union_reads_ratio_samples=only_in_flag_evaluations and union_applies(context),database=database,)swapper.visit(rewritten)@@ -274,7 +282,7 @@ def _resolve_with_swaps(refusals = [*swapper.refusals, *union.refusals]if refusals:if only_in_flag_evaluations:- raise _flag_calls_only_in_flag_evaluations_error(refusals[0], view_name)+ raise flag_calls_only_in_flag_evaluations_error(refusals[0], view_name)logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=refusals[0])sources = JoinSources() if only_in_flag_evaluations else Noneoriginal = resolve(node, join_sources=sources)@@ -282,16 +290,20 @@ def _resolve_with_swaps(with context.timings.measure("flag_calls_from_flag_evaluations_by_type"):refusal = swapper.swap_scans_found_by_type(original, sources, clones)if refusal is not None:- raise _flag_calls_only_in_flag_evaluations_error(refusal, view_name)+ raise flag_calls_only_in_flag_evaluations_error(refusal, view_name)if not swapper.swapped and not union.renamed:- return FlagCallSwap(source=node, resolved=original)+ return FlagCallSwap(+ source=node,+ resolved=_without_unreadable_inline_cohort_scans(original, context, view_name),+ )try:swapped = resolve(rewritten)except BaseHogQLError as error:# Expected for a query that reads a field flag_evaluations lacks, such as goe_0.properties.reason = f"it reads something that table doesn't have: {error}"else:- unreadable = union_unreadable_reason(swapped, context) if union.renamed else None+ # An inline cohort renames its own scans during resolution, so the outer rename does not show them.+ unreadable = union_unreadable_reason(swapped, context) if union.renamed or only_in_flag_evaluations else Noneif unreadable is not None:reason = unreadableelif not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):@@ -301,15 +313,26 @@ def _resolve_with_swaps(elif not only_in_flag_evaluations and not _swap_reads_properties_alike(original, swapped, context):reason = "it reads a property from a different kind of column there"else:+ # A view body's offsets come from the view's own text, not from the query the editor shows.+ if view_name is None:+ context.flag_called_union_spans.update(union.literal_spans)return FlagCallSwap(source=rewritten, resolved=swapped)if only_in_flag_evaluations:# events gets no new flag calls, so the original query would return only calls from before the move.- raise _flag_calls_only_in_flag_evaluations_error(reason, view_name)+ raise flag_calls_only_in_flag_evaluations_error(reason, view_name)logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=reason)return FlagCallSwap(source=node, resolved=original)-def _flag_key_compares_as_string(context: HogQLContext) -> bool:+def _without_unreadable_inline_cohort_scans(resolved: _T_AST, context: HogQLContext, view_name: str | None) -> _T_AST:+ # Deferred: flag_calls_union imports this module.+ from posthog.hogql.transforms.flag_calls_union import require_union_readable # noqa: PLC0415++ require_union_readable(resolved, context, view_name)+ return resolved+++def team_flag_key_compares_as_string(context: HogQLContext) -> bool:if context.team_id is None:return Truereturn flag_key_typed_as_string(_event_property_metadata(context, {_FLAG_KEY_PROPERTY}))@@ -348,7 +371,7 @@ def _event_property_metadata(context: HogQLContext, names: set[str]) -> Propertyreturn loaded-def _flag_calls_only_in_flag_evaluations_error(reason: str, view_name: str | None) -> QueryError:+def flag_calls_only_in_flag_evaluations_error(reason: str, view_name: str | None = None) -> QueryError:subject, part = ("this query", "query") if view_name is None else (f"the view {view_name}", "view")return QueryError(f"Your organization's {FLAG_CALLED_EVENT} events are only in posthog.flag_evaluations, and {subject} "@@ -470,14 +493,17 @@ class EventsScan:return Nonereturn _below_posthog_events(chain)- def shape_refusal(self, node: ast.SelectQuery) -> str | None:- """Why another table cannot stand in for this scan whatever its filters say, or None."""+ def shape_refusal(self, node: ast.SelectQuery, *, ratio_sample_ok: bool = False) -> str | None:+ """Why another table cannot stand in for this scan whatever its filters say, or None. With `ratio_sample_ok`,+ a SAMPLE that `ratio_sample_fraction` reads does not count, for a table that applies the ratio itself."""if self.join.table_args is not None or self.join.table_final:return "it uses FINAL or table arguments on events"if self.join.column_aliases:return "it names the events columns with column aliases"- if self.join.sample is not None and not self.join.sample.samples_all_rows():- return "it uses SAMPLE"+ sample = self.join.sample+ if sample is not None and not sample.samples_all_rows():+ if not ratio_sample_ok or ratio_sample_fraction(sample) is None:+ return "it uses SAMPLE"for item in node.select:while isinstance(item, ast.Alias):item = item.expr@@ -658,6 +684,28 @@ def flag_key_predicate(table: str, keys: list[str]) -> ast.Expr | None:)+def ratio_sample_fraction(sample: ast.SampleExpr) -> Fraction | None:+ """The share of distinct ids that `SAMPLE k` or `SAMPLE a/b` keeps, or None for any other form.++ ClickHouse reads `SAMPLE n` with n above 1 as a row count, and `OFFSET` moves the sampled range. Neither has a+ ratio that another table could apply.+ """+ if sample.offset_value is not None:+ return None+ ratio = sample.sample_value+ constants = [ratio.left, ratio.right or ast.Constant(value=1)]+ if not all(+ isinstance(c.value, int | float)+ and not isinstance(c.value, bool)+ and (isinstance(c.value, int) or isfinite(c.value))+ for c in constants+ ):+ return None+ numerator, denominator = (Fraction(c.value if isinstance(c.value, int) else str(c.value)) for c in constants)+ fraction = numerator / denominator if denominator else None+ return fraction if fraction is not None and 0 < fraction < 1 else None++def point_scan_at(node: ast.SelectQuery, scan: EventsScan, chain: list[str | int], condition: ast.Expr | None = None) -> None:@@ -965,8 +1013,10 @@ class _EventsScanSwapper(ScanVisitor):only_in_flag_evaluations: bool,flag_key_compares_as_string: Callable[[], bool],database: "Database",+ union_reads_ratio_samples: bool = False,) -> None:super().__init__()+ self.union_reads_ratio_samples = union_reads_ratio_samplesself.database = databaseself.now = nowself.timezone = timezone@@ -987,7 +1037,8 @@ class _EventsScanSwapper(ScanVisitor):):self._swap(node, scan, conjuncts)else:- refusal = scan.shape_refusal(node)+ # On mode 2 the union reads a scan with a ratio SAMPLE, so this refuses only the other shapes.+ refusal = scan.shape_refusal(node, ratio_sample_ok=self.union_reads_ratio_samples)if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):self.refusals.append(refusal)@@ -1005,7 +1056,8 @@ class _EventsScanSwapper(ScanVisitor):original join of each resolved join, and `clones` holds the clones of each original join. Returns why one ofthem cannot swap, or None.- A scan that the resolver adds, such as one in a saved view's body or an inline cohort, keeps reading events.+ A scan that the resolver adds, such as one in a saved view's body or an inline cohort, is not swapped here. A+ view's scan keeps reading events. On mode 2 with the union enabled, an inline cohort's scan reads the union.Any other scan without a clone cannot swap."""for select, join in _joins(resolved):@@ -1028,8 +1080,8 @@ class _EventsScanSwapper(ScanVisitor):def _swap_clone(self, clone: "_JoinClone") -> str | None:chain = _table_chain(clone.join)- if chain == ["posthog", "flag_evaluations"]:- # The syntactic rules, or an earlier resolved join from the same source, already swapped it.+ if chain in (["posthog", "flag_evaluations"], ["posthog", "events_with_flag_calls"]):+ # The syntactic rules, the union rename, or an earlier resolved join from the same source already moved it.return Noneif chain != ["events"]:return _UNMOVABLE_SHAPEdiff --git a/posthog/hogql/transforms/flag_calls_union.py b/posthog/hogql/transforms/flag_calls_union.pyindex 8489556d4d76..2a3c8be98e81 100644--- a/posthog/hogql/transforms/flag_calls_union.py+++ b/posthog/hogql/transforms/flag_calls_union.py@@ -10,8 +10,12 @@ scan, so the rules decide which scans pay for a second table, and which shapes t"""from collections.abc import Callable+from functools import cache, partial+from math import floorfrom typing import TYPE_CHECKING, cast+import structlog+from posthog.hogql import astfrom posthog.hogql.constants import FLAG_CALLED_EVENTfrom posthog.hogql.context import HogQLContext@@ -52,8 +56,11 @@ from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (compared_values,comparison_of,filter_conjuncts,+ flag_calls_only_in_flag_evaluations_error,flag_key_predicate,point_scan_at,+ ratio_sample_fraction,+ team_flag_key_compares_as_string,)from posthog.hogql.type_system import parse_sql_runtime_typefrom posthog.hogql.visitor import CloningVisitor@@ -64,6 +71,8 @@ from posthog.property_columns import TableColumnif TYPE_CHECKING:from posthog.hogql.database.database import Database+logger = structlog.get_logger(__name__)+# The joins whose resolver compares only the join's key and `extra_from_fields` on the source table. Lazy table# resolution projects exactly those from the union, so a join that reads anything else would name a missing column._UNION_JOIN_RESOLVERS = frozenset(@@ -96,6 +105,8 @@ class UnionRename:renamed: bool# Why the union cannot read each other scan that names `$feature_flag_called` with other events.refusals: tuple[str, ...]+ # The (start, end) of each `$feature_flag_called` literal that a renamed scan compares with its `event`.+ literal_spans: frozenset[tuple[int, int]] = frozenset()def read_flag_calls_with_union(@@ -119,6 +130,51 @@ def read_flag_calls_with_union()+def union_applies(context: HogQLContext) -> bool:+ """Whether the union reads this query's mixed flag-call scans."""+ return _union_database(context) is not None+++def read_flag_calls_in_inline_cohort(+ query: ast.SelectQuery | ast.SelectSetQuery, context: HogQLContext+) -> ast.SelectQuery | ast.SelectSetQuery:+ """Points each flag-call scan in an inline cohort `query` at the union, in place, and returns `query`.++ The resolver and the subquery and LEFTJOIN paths of `IN COHORT` splice an inline cohort in after the swap in+ `resolve_flag_calls_from_flag_evaluations` has run, so that swap never sees its scans. Only the rename applies+ here, because the swap's binding check needs the whole query.++ Only an organization whose flag calls are only in `flag_evaluations` reads the union here. While `events` holds+ every flag call, an inline cohort reads them from `events` as before. `resolve_in_cohorts_conjoined` splices its+ cohorts in before the swap runs, so its caller passes `read_flag_calls_from_union=False`: the swap sees those scans.+ """+ database = _union_database(context)+ if database is None or not database.flag_calls_only_in_flag_evaluations:+ return query+ renamer = _UnionRenamer(+ cache(partial(team_flag_key_compares_as_string, context)),+ only_in_flag_evaluations=True,+ rename_flag_only_scans=True,+ )+ renamer.visit(query)+ if renamer.refusals:+ raise flag_calls_only_in_flag_evaluations_error(renamer.refusals[0])+ return query+++def require_union_readable(resolved: ast.AST, context: HogQLContext, view_name: str | None = None) -> None:+ """Raises a QueryError when an organization with flag calls only in `flag_evaluations` has a union scan in+ `resolved` that reads a field the union cannot supply. An inline cohort is spliced in after the rename, so only+ the resolved query shows its scans. `IN COHORT` resolves its join before lazy table resolution builds the+ union, so that call is the last point where ClickHouse has not yet rejected the query."""+ database = _union_database(context)+ if database is None or not database.flag_calls_only_in_flag_evaluations:+ return+ unreadable = union_unreadable_reason(resolved, context)+ if unreadable is not None:+ raise flag_calls_only_in_flag_evaluations_error(unreadable, view_name)++def union_unreadable_reason(node: ast.AST, context: HogQLContext) -> str | None:"""Why the union cannot read a scan in the resolved `node`, or None when it can read them all.@@ -208,6 +264,31 @@ def union_events_filter(node: ast.SelectQuery, union: EventsWithFlagCallsTable)return _events_filter(_scan_conjuncts(node, union))+def union_scan_sample(node: ast.SelectQuery, union: EventsWithFlagCallsTable) -> ast.SampleExpr | None:+ """The SAMPLE that `node` puts on its scan of the union, or None."""+ join = node.select_from+ while join is not None:+ if join.sample is not None and union_table_of(join.table.type if join.table else join.type) is union:+ return join.sample+ join = join.next_join+ return None+++def sample_flag_evaluations_filter(sample: ast.SampleExpr, distinct_id: ast.Field) -> ast.Expr:+ """The predicate that keeps the `flag_evaluations` rows whose `distinct_id` ClickHouse's `SAMPLE` keeps on `events`.++ `events` samples by `cityHash64(distinct_id)`, and a ratio keeps the hashes below that ratio of the UInt64 range.+ """+ fraction = ratio_sample_fraction(sample)+ if fraction is None:+ raise QueryError("The flag call union reads only a SAMPLE ratio between 0 and 1.")+ return ast.CompareOperation(+ op=ast.CompareOperationOp.Lt,+ left=ast.Call(name="cityHash64", args=[distinct_id]),+ right=ast.Constant(value=floor(fraction * 2**64)),+ )++def rename_flag_call_scans_to_union(node: ast.AST,flag_key_compares_as_string: Callable[[], bool],@@ -222,8 +303,8 @@ def rename_flag_call_scans_to_union(clauses. A subquery's comparison does not count. A funnel step column such as`if(event = '$feature_flag_called' AND ..., 1, 0)` counts.2. The scan is the first table in FROM, and every join after it is INNER, LEFT or CROSS, GLOBAL or not.- 3. The scan has no FINAL, table arguments, column aliases or SAMPLE other than 1, and its query selects no `*`- or `COLUMNS(...)`.+ 3. The scan has no FINAL, table arguments or column aliases, and its SAMPLE is 1 or a ratio between 0 and 1 without+ OFFSET. Its query selects no `*` or `COLUMNS(...)`.A scan that the flag_evaluations swap changed reads `posthog.flag_evaluations` and is not an `events` scan anymore. The flag_evaluations swap swaps or refuses a first scan that reads only flag calls, so the union neither@@ -232,7 +313,9 @@ def rename_flag_call_scans_to_union("""renamer = _UnionRenamer(flag_key_compares_as_string, only_in_flag_evaluations)renamer.visit(node)- return UnionRename(renamed=renamer.renamed, refusals=tuple(renamer.refusals))+ return UnionRename(+ renamed=renamer.renamed, refusals=tuple(renamer.refusals), literal_spans=frozenset(renamer.literal_spans)+ )def _union_database(context: HogQLContext) -> "Database | None":@@ -385,7 +468,7 @@ def _union_refusal(node: ast.SelectQuery, scan: EventsScan, position: int) -> st"""Why the union cannot stand in for `scan`, the scan at `position` in the FROM chain of `node`, or None."""if position > 0:return "it joins events after another table"- refusal = scan.shape_refusal(node)+ refusal = scan.shape_refusal(node, ratio_sample_ok=True)if refusal is not None:return refusaljoin = scan.join.next_join@@ -398,11 +481,19 @@ def _union_refusal(node: ast.SelectQuery, scan: EventsScan, position: int) -> streturn None+@frozen+class _FlagCallScan:+ position: int+ scan: EventsScan+ literals: list[ast.Constant]++def _flag_call_scans(node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]-) -> list[tuple[int, EventsScan]]:- """Each `events` scan in the FROM chain of `node` whose query names `$feature_flag_called`, with its position."""- scans: list[tuple[int, EventsScan]] = []+) -> list[_FlagCallScan]:+ """Each `events` scan in the FROM chain of `node` whose query names `$feature_flag_called`, with its position and+ the literals that name it."""+ scans: list[_FlagCallScan] = []join = node.select_fromposition = 0while join is not None:@@ -423,17 +514,17 @@ def _flag_call_scans(flag_key_compares_as_string=flag_key_compares_as_string,))- if scan is not None and _names_flag_calls(node, scan):- scans.append((position, scan))+ if scan is not None and (literals := _flag_call_literals(node, scan)):+ scans.append(_FlagCallScan(position=position, scan=scan, literals=literals))join = join.next_joinposition += 1return scans-def _names_flag_calls(node: ast.SelectQuery, scan: EventsScan) -> bool:+def _flag_call_literals(node: ast.SelectQuery, scan: EventsScan) -> list[ast.Constant]:finder = _FlagCallComparisonFinder(node, scan)finder.visit(node)- return finder.found+ return finder.literalsclass _ScanReadCopier(CloningVisitor):@@ -594,7 +685,7 @@ class _FlagCallComparisonFinder(SyntaxVisitor):super().__init__()self.root = rootself.reader = scan- self.found = False+ self.literals: list[ast.Constant] = []def visit_select_query(self, node: ast.SelectQuery) -> None:if node is self.root:@@ -613,34 +704,57 @@ class _FlagCallComparisonFinder(SyntaxVisitor):def _check(self, expr: ast.Expr) -> None:values = compared_values(expr, lambda read: self.reader.column(read) == ["event"])- if values and any(isinstance(value, ast.Constant) and value.value == FLAG_CALLED_EVENT for value in values):- self.found = True+ for value in values or []:+ if isinstance(value, ast.Constant) and value.value == FLAG_CALLED_EVENT:+ self.literals.append(value)class _UnionRenamer(ScanVisitor):- def __init__(self, flag_key_compares_as_string: Callable[[], bool], only_in_flag_evaluations: bool) -> None:+ def __init__(+ self,+ flag_key_compares_as_string: Callable[[], bool],+ only_in_flag_evaluations: bool,+ *,+ rename_flag_only_scans: bool = False,+ ) -> None:super().__init__()self.flag_key_compares_as_string = flag_key_compares_as_stringself.only_in_flag_evaluations = only_in_flag_evaluations+ self.rename_flag_only_scans = rename_flag_only_scansself.renamed = Falseself.refusals: list[str] = []+ self.literal_spans: set[tuple[int, int]] = set()def visit_scan_query(self, node: ast.SelectQuery) -> None:# Only the first scan can be renamed, and it comes first, so each refusal reads the query as written.- for position, scan in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes):- if position == 0 and self._reads_only_flag_calls(node, scan):+ for found in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes):+ position, scan, literals = found.position, found.scan, found.literals+ if position == 0 and self._left_to_the_swap(node, scan):continuerefusal = _union_refusal(node, scan, position)if refusal is not None:self.refusals.append(refusal)else:point_scan_at(node, scan, list(EVENTS_WITH_FLAG_CALLS_CHAIN))- # A lazy table cannot be sampled, and SAMPLE 1 keeps every row anyway.- scan.join.sample = None+ # A lazy table cannot be sampled. SAMPLE 1 keeps every row anyway, and the union applies a ratio+ # sample in its branches, where lazy table resolution reads it from this scan.+ if scan.join.sample is not None and scan.join.sample.samples_all_rows():+ scan.join.sample = Noneself.renamed = True+ self.literal_spans.update(+ (literal.start, literal.end)+ for literal in literals+ if literal.start is not None and literal.end is not None+ )- def _reads_only_flag_calls(self, node: ast.SelectQuery, scan: EventsScan) -> bool:+ def _left_to_the_swap(self, node: ast.SelectQuery, scan: EventsScan) -> bool:# The flag_evaluations swap handles a first scan whose `event` filter it reads as only flag calls. It swaps or# refuses that scan on mode 2. On mode 1 events holds every flag call, so the union would read the same calls# twice there.- return scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False)+ if self.rename_flag_only_scans:+ return False+ # The swap cannot read a sampled scan. On mode 2 the union reads it.+ sampled = scan.join.sample is not None and not scan.join.sample.samples_all_rows()+ return scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False) and not (+ self.only_in_flag_evaluations and sampled+ )diff --git a/posthog/hogql/transforms/in_cohort.py b/posthog/hogql/transforms/in_cohort.pyindex 1bf7ab8a8281..a9c90ab3892b 100644--- a/posthog/hogql/transforms/in_cohort.py+++ b/posthog/hogql/transforms/in_cohort.py@@ -172,7 +172,10 @@ class MultipleInCohortResolver(TraversingVisitor):inlineable_dynamic: list[tuple[int, ast.SelectQuery | ast.SelectSetQuery]] = []non_inlineable_dynamic: list[tuple[int, StaticOrDynamic, int]] = []for cohort_id, cohort_type, version in dynamic_cohorts:- inline_ast = inline_cohort_query(cohort_id, False, version, self.context)+ # This runs before the flag-call swap, which then sees the cohort's scans.+ inline_ast = inline_cohort_query(+ cohort_id, False, version, self.context, read_flag_calls_from_union=False+ )if inline_ast is not None:inlineable_dynamic.append((cohort_id, inline_ast))else:@@ -373,6 +376,7 @@ class InCohortResolver(TraversingVisitor):negative: bool,):from posthog.hogql.functions.cohort import inline_cohort_query+ from posthog.hogql.transforms.flag_calls_union import require_union_readablefrom posthog.hogql.transforms.lazy_tables import resolve_lazy_tablesassert self.context is not None@@ -440,6 +444,7 @@ class InCohortResolver(TraversingVisitor):),)if inline_ast is not None:+ require_union_readable(new_join, self.context)resolve_lazy_tables(new_join, self.dialect, [self.stack[-1]], self.context, resolver_factory=self.resolver_factory)diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.pyindex a115d8332f37..f7ebf6d84ced 100644--- a/posthog/hogql/transforms/lazy_tables.py+++ b/posthog/hogql/transforms/lazy_tables.py@@ -814,13 +814,17 @@ class LazyTableResolver(TraversingVisitor):# A lazy table expands into an aggregating subquery. ClickHouse rejects a SAMPLE# modifier on a subquery, and sampling a pre-aggregated table is meaningless, so# drop the modifier rather than emitting invalid SQL.- if join_ptr.sample is not None:+ # EventsWithFlagCallsTable.lazy_select, called above, already copied the SAMPLE into each union+ # branch, so it needs no warning. This clear must stay after that call.+ if join_ptr.sample is not None and not isinstance(+ table_to_add.lazy_table, EventsWithFlagCallsTable+ ):self.context.add_warning(f'SAMPLE clause ignored: the "{table_name}" table is computed on the fly and cannot be sampled',start=join_ptr.sample.start,end=join_ptr.sample.end,)- join_ptr.sample = None+ join_ptr.sample = Nonebreakjoin_ptr = join_ptr.next_joindiff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.pyindex 69eca14b26bb..f99587ac7be2 100644--- a/posthog/hogql/transforms/test/test_flag_calls_union.py+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py@@ -20,7 +20,13 @@ from django.test import SimpleTestCasefrom parameterized import param, parameterized-from posthog.schema import HogQLQueryModifiers, MaterializationMode, PersonsOnEventsMode+from posthog.schema import (+ HogQLQueryModifiers,+ InCohortVia,+ InlineCohortCalculation,+ MaterializationMode,+ PersonsOnEventsMode,+)from posthog.hogql import astfrom posthog.hogql.context import HogQLContext@@ -56,9 +62,41 @@ E_MIXED = "e.event IN ('$pageview', '$feature_flag_called')"UNION = "posthog.events_with_flag_calls"READ = FlagEvaluationsMode.READ_FLAG_EVALUATIONSONLY = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY++UNION_EXPANSION = "FROM flag_evaluations"+def _performed_flag_call_cohort(team, flag_key: str) -> Cohort:+ return Cohort.objects.create(+ team=team,+ name=f"called {flag_key}",+ filters={+ "properties": {+ "type": "AND",+ "values": [+ {+ "type": "AND",+ "values": [+ {+ "key": "$feature_flag_called",+ "type": "behavioral",+ "value": "performed_event",+ "event_type": "events",+ "time_value": 30,+ "time_interval": "day",+ "event_filters": [+ {"key": "$feature_flag", "type": "event", "value": flag_key, "operator": "exact"}+ ],+ }+ ],+ }+ ],+ }+ },+ )++class _TableCollector(TraversingVisitor):def __init__(self) -> None:super().__init__()@@ -130,11 +168,25 @@ class TestRenameFlagCallScansToUnion(SimpleTestCase):# The flag_evaluations swap handles this scan. Events holds every flag call on mode 1. The swap swaps or# refuses the scan on mode 2.("only_flag_calls", "SELECT count() FROM events WHERE event = '$feature_flag_called'", ["events"], []),- (- "only_flag_calls_with_sample",+ param(+ "only_flag_calls_with_sample_on_mode_1","SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'",["events"],[],+ False,+ ),+ # Phase 1 cannot swap a sampled scan, and the union applies the ratio.+ (+ "only_flag_calls_with_sample_on_mode_2",+ "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'",+ [UNION],+ [],+ ),+ (+ "only_flag_calls_with_sample_on_posthog_events_on_mode_2",+ "SELECT count() FROM posthog.events SAMPLE 0.1 WHERE event = '$feature_flag_called'",+ [UNION],+ [],),("left_join_after_the_scan",@@ -242,7 +294,34 @@ class TestRenameFlagCallScansToUnion(SimpleTestCase):["events"],["it names the events columns with column aliases"],),- ("sample_ratio", f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}", ["events"], ["it uses SAMPLE"]),+ ("sample_ratio", f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}", [UNION], []),+ ("sample_fraction", f"SELECT count() FROM events SAMPLE 1/10 WHERE {MIXED}", [UNION], []),+ (+ "sample_row_count",+ f"SELECT count() FROM events SAMPLE 10 WHERE {MIXED}",+ ["events"],+ ["it uses SAMPLE"],+ ),+ *(+ (+ f"sample_{name}",+ f"SELECT count() FROM events SAMPLE {value} WHERE {MIXED}",+ ["events"],+ ["it uses SAMPLE"],+ )+ for name, value in (+ ("inf", "inf"),+ ("nan", "nan"),+ ("oversized_integer", "9" * 400),+ ("oversized_hex_integer", "0x" + "f" * 5000),+ )+ ),+ (+ "sample_with_offset",+ f"SELECT count() FROM events SAMPLE 0.1 OFFSET 0.5 WHERE {MIXED}",+ ["events"],+ ["it uses SAMPLE"],+ ),("select_star", f"SELECT * FROM events WHERE {MIXED}", ["events"], ["it selects * or COLUMNS(...)"]),("cte_named_events",@@ -293,6 +372,8 @@ class TestReadFlagCallsWithUnion(BaseTest):new_events_schema: bool | None = None,person_id_pushdown: bool | None = None,modifiers: HogQLQueryModifiers | None = None,+ inline_cohorts: bool = False,+ in_cohort_via: InCohortVia | None = None,) -> str:_set_mode(self.organization, mode)with _enabled_flags(union=union):@@ -304,6 +385,8 @@ class TestReadFlagCallsWithUnion(BaseTest):"personIdPushdown": person_id_pushdown,"personsOnEventsMode": persons_on_events,"useNewEventsSchema": _events_schema_for(mode, new_events_schema),+ **({"inlineCohortCalculation": InlineCohortCalculation.ALWAYS} if inline_cohorts else {}),+ **({"inCohortVia": in_cohort_via} if in_cohort_via else {}),},),pretty=False,@@ -428,6 +511,7 @@ class TestReadFlagCallsWithUnion(BaseTest):f"SELECT goe_0.properties.plan FROM events WHERE {MIXED}","goe_0.properties",),+ ("sample_row_count", f"SELECT count() FROM events SAMPLE 10 WHERE {MIXED}", "it uses SAMPLE"),param("join_whose_on_clause_reads_another_field",f"SELECT probe_join.key FROM events WHERE {MIXED}",@@ -608,6 +692,109 @@ class TestReadFlagCallsWithUnion(BaseTest):assert context.warnings == []+ def test_ratio_sample_applies_in_every_branch_and_warns_of_nothing(self):+ _set_mode(self.organization, READ)+ query = f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}"+ with _enabled_flags(union=True):+ sql, context = HogQLQueryExecutor(+ query=query, team=self.team, modifiers=HogQLQueryModifiers(useNewEventsSchema=False), pretty=False+ ).generate_clickhouse_sql()+ branches = self._branches(substitute_params(sql, context.values))++ assert context.warnings == []+ assert [("SAMPLE 0.1" in branch) for branch in branches] == [True, True, False]+ assert "less(cityHash64(flag_evaluations.distinct_id), 1844674407370955161)" in branches[-1]++ def test_ratio_sample_applies_when_a_joined_person_field_reads_a_second_union_scan(self):+ sql = self._sql(+ f"SELECT person.properties.email, count() FROM events SAMPLE 0.1 WHERE {MIXED} GROUP BY 1",+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,+ )++ assert sql.count("SAMPLE 0.1") == 2+ assert "less(cityHash64(flag_evaluations.distinct_id), 1844674407370955161)" in sql++ @parameterized.expand(+ [+ ("subquery_on_mode_1", READ, InCohortVia.SUBQUERY, False),+ ("subquery_on_mode_2", ONLY, InCohortVia.SUBQUERY, True),+ ("leftjoin_on_mode_2", ONLY, InCohortVia.LEFTJOIN, True),+ # IN COHORT splices the cohort in before the swap, which then reads its scans on either mode.+ ("conjoined_on_mode_1", READ, InCohortVia.LEFTJOIN_CONJOINED, True),+ ("conjoined_on_mode_2", ONLY, InCohortVia.LEFTJOIN_CONJOINED, True),+ ]+ )+ def test_inline_cohort_reads_flag_calls_from_flag_evaluations_only_on_mode_2(+ self, _name: str, mode: FlagEvaluationsMode, via: InCohortVia, expected_flag_evaluations: bool+ ):+ cohort = _performed_flag_call_cohort(self.team, "probe-flag")++ sql = self._sql(+ f"SELECT count() FROM events WHERE person_id IN COHORT {cohort.pk}",+ mode=mode,+ inline_cohorts=True,+ in_cohort_via=via,+ )++ assert (re.search(r"FROM flag_evaluations\b", sql) is not None) is expected_flag_evaluations++ @parameterized.expand(+ [+ ("outer_query_names_no_flag_calls", "SELECT count() FROM events WHERE person_id IN COHORT {cohort}"),+ (+ "outer_query_excludes_flag_calls",+ "SELECT count() FROM events WHERE event != '$feature_flag_called' AND person_id IN COHORT {cohort}",+ ),+ (+ "outer_query_swapped_by_phase_1",+ "SELECT count() FROM events WHERE event = '$feature_flag_called' AND person_id IN COHORT {cohort}",+ ),+ (+ "left_join_cohort",+ "SELECT count() FROM events WHERE person_id IN COHORT {cohort}",+ InCohortVia.LEFTJOIN,+ ),+ ]+ )+ def test_mode_2_raises_when_an_inline_cohort_reads_a_field_the_union_cannot_supply(+ self, _name: str, outer_query: str, via: InCohortVia | None = None+ ):+ self.addCleanup(cleanup_materialized_columns)+ materialize("events", "probe", is_nullable=True, column_type="Nullable(Float64)")+ cohort = _performed_flag_call_cohort(self.team, "probe-flag")+ cohort_query = parse_select(+ "SELECT person_id AS id FROM events WHERE event = '$feature_flag_called' "+ "AND properties.probe > 1 GROUP BY person_id"+ )++ with (+ patch("posthog.hogql_queries.hogql_cohort_query.HogQLCohortQuery.get_query", return_value=cohort_query),+ pytest.raises(QueryError, match="probe, which events stores in a Nullable"),+ ):+ self._sql(+ outer_query.format(cohort=cohort.pk),+ mode=ONLY,+ inline_cohorts=True,+ in_cohort_via=via,+ new_events_schema=False,+ )++ @parameterized.expand(+ [+ ("flag_calls_only", "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'"),+ (+ "flag_calls_only_next_to_a_join",+ "SELECT count() FROM events AS e SAMPLE 0.1 LEFT JOIN persons AS p ON e.person_id = p.id "+ "WHERE e.event = '$feature_flag_called'",+ ),+ ]+ )+ def test_mode_2_reads_a_sampled_flag_call_scan_from_the_union(self, _name: str, query: str):+ sql = self._sql(query, mode=ONLY)++ assert re.search(r"FROM flag_evaluations\b", sql) is not None+ assert "less(cityHash64(flag_evaluations.distinct_id), 1844674407370955161)" in sql+def test_sessions_join_keeps_its_time_bound(self):sql = self._sql(f"SELECT session.$session_duration FROM events WHERE {MIXED} "@@ -921,6 +1108,47 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):assert rows[:4] == [*history, *recent]assert [row[:3] for row in rows[4:]] == keyless+ def test_inline_cohort_counts_flag_calls_that_only_flag_evaluations_holds(self):+ fresh = _create_person(distinct_ids=["fresh"], team_id=self.team.pk)+ _create_flag_evaluations(+ self.team.pk,+ "fresh-flag",+ timestamp=datetime.now(UTC) - timedelta(days=2),+ distinct_id="fresh",+ person_id=fresh.uuid,+ )+ flush_persons_and_events()+ cohort = _performed_flag_call_cohort(self.team, "fresh-flag")+ _set_mode(self.organization, ONLY)+ with _enabled_flags(union=True):+ response = execute_hogql_query(+ f"SELECT id FROM persons WHERE id IN COHORT {cohort.pk}",+ team=self.team,+ modifiers=HogQLQueryModifiers(inlineCohortCalculation=InlineCohortCalculation.ALWAYS),+ )++ assert [str(row[0]) for row in response.results] == [str(fresh.uuid)]++ def test_ratio_sample_keeps_the_same_distinct_ids_in_both_tables(self):+ now = datetime.now(UTC)+ for index in range(100):+ distinct_id = f"sampled-{index}"+ _create_event(event="$pageview", distinct_id=distinct_id, team=self.team, timestamp=now)+ _create_flag_evaluations(+ self.team.pk, "sample-flag", timestamp=now, distinct_id=distinct_id, person_id=uuid.uuid4()+ )+ flush_persons_and_events()++ response = self._run(+ f"SELECT event, groupUniqArray(distinct_id) FROM events SAMPLE 0.3 "+ f"WHERE {MIXED} AND startsWith(distinct_id, 'sampled-') GROUP BY event",+ ONLY,+ )++ sampled = {event: set(ids) for event, ids in response.results}+ assert 0 < len(sampled["$pageview"]) < 100+ assert sampled["$feature_flag_called"] == sampled["$pageview"]+@parameterized.expand([(