Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): read inline cohorts and sample ratios through the flag calls union (#126959)

Phil Haack0fabca8b9664
diff --git a/posthog/hogql/context.py b/posthog/hogql/context.py
index f3e8bc63bab6..c4caade32785 100644
--- a/posthog/hogql/context.py
+++ b/posthog/hogql/context.py
@@ -179,6 +179,10 @@ class HogQLContext:
# Per-query event property definitions that transform loads. The transform runs once for the query and once for
# each saved view the query inlines, so this loads each property's definition once per query.
flag_call_property_metadata: Optional["FlagCallPropertyMetadata"] = field(default=None, compare=False, repr=False)
+ # The (start, end) of each `$feature_flag_called` literal in the query text that the flag-call union reads.
+ # `_resolve_with_swaps` in transforms/flag_calls_from_flag_evaluations.py fills it when the swap succeeds, and the
+ # SQL editor's notices read it.
+ flag_called_union_spans: set[tuple[int, int]] = field(default_factory=set)
# Backend-only opt-in for transforms/events_read_in_order.py. The prefix is slower when the range is short or
# the filter is selective, so set it only for a query shape that was measured to be faster with it.
diff --git a/posthog/hogql/database/schema/events_with_flag_calls.py b/posthog/hogql/database/schema/events_with_flag_calls.py
index a5a8452d7ae6..fbebff4c70cb 100644
--- a/posthog/hogql/database/schema/events_with_flag_calls.py
+++ b/posthog/hogql/database/schema/events_with_flag_calls.py
@@ -18,7 +18,7 @@ from posthog.hogql.database.schema.events import EVENTS_AVOID_ASTERISK_FIELDS, E
from posthog.hogql.database.schema.flag_evaluations import FlagEvaluationsTable, flag_evaluations_coverage_start
from posthog.hogql.errors import QueryError
from posthog.hogql.parser import parse_expr
-from posthog.hogql.visitor import GetFieldsTraverser
+from posthog.hogql.visitor import GetFieldsTraverser, clone_expr
from posthog.dataclasses import frozen
@@ -141,9 +141,24 @@ class EventsWithFlagCallsTable(LazyTable):
and_conditions,
flag_key_typed_as_string,
)
- from posthog.hogql.transforms.flag_calls_union import union_branch_filters, union_events_filter # noqa: PLC0415
+ from posthog.hogql.transforms.flag_calls_union import ( # noqa: PLC0415
+ sample_flag_evaluations_filter,
+ union_branch_filters,
+ union_events_filter,
+ union_scan_sample,
+ )
assert context.database is not None
+ # Lazy table resolution drops the scan's SAMPLE, so each branch applies the ratio itself.
+ sample = union_scan_sample(node, self)
+
+ def sampled_scan(table: ast.Field, alias: str | None = None) -> ast.JoinExpr:
+ return ast.JoinExpr(
+ sample=clone_expr(sample, clear_types=True, clear_locations=True) if sample is not None else None,
+ table=table,
+ alias=alias,
+ )
+
# `SELECT count()` reads no field, and a subquery needs at least one column.
accessed = dict(table_to_add.fields_accessed) or {"timestamp": ["timestamp"]}
events_columns: list[ast.Expr] = [
@@ -155,7 +170,7 @@ class EventsWithFlagCallsTable(LazyTable):
# once. The second scan asks only for columns, and it reads the same filter from this query's WHERE.
return ast.SelectQuery(
select=events_columns,
- select_from=ast.JoinExpr(table=ast.Field(chain=list(EVENTS_WITH_FLAG_CALLS_CHAIN)), alias="events"),
+ select_from=sampled_scan(table=ast.Field(chain=list(EVENTS_WITH_FLAG_CALLS_CHAIN)), alias="events"),
where=union_events_filter(node, self),
)
@@ -178,7 +193,7 @@ class EventsWithFlagCallsTable(LazyTable):
def events_branch(*conditions: ast.Expr | None) -> ast.SelectQuery:
return ast.SelectQuery(
select=events_columns,
- select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),
+ select_from=sampled_scan(table=ast.Field(chain=["events"])),
where=and_conditions([*conditions, filters.events]),
)
@@ -199,7 +214,13 @@ class EventsWithFlagCallsTable(LazyTable):
table=ast.Field(chain=["posthog", "flag_evaluations"]), alias="flag_evaluations"
),
where=and_conditions(
- [parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff}), filters.flag_evaluations]
+ [
+ parse_expr("flag_evaluations.timestamp >= {cutoff}", {"cutoff": cutoff}),
+ filters.flag_evaluations,
+ sample_flag_evaluations_filter(sample, flag_evaluations_field(["distinct_id"]))
+ if sample is not None
+ else None,
+ ]
),
)
)
diff --git a/posthog/hogql/flag_called_warnings.py b/posthog/hogql/flag_called_warnings.py
index 6b1c5b9fee9a..382fcd2a2f06 100644
--- a/posthog/hogql/flag_called_warnings.py
+++ b/posthog/hogql/flag_called_warnings.py
@@ -24,6 +24,17 @@ FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE = (
"of flag calls."
)
+FLAG_CALLED_FROM_BOTH_TABLES_NOTICE = (
+ f"This query reads {FLAG_CALLED_EVENT} events from posthog.flag_evaluations, which keeps the last 90 days "
+ "of flag calls, and other events from events."
+)
+
+# While events still holds every flag call, the union reads only the recent ones from posthog.flag_evaluations.
+FLAG_CALLED_SPLIT_BY_AGE_NOTICE = (
+ f"This query reads {FLAG_CALLED_EVENT} events from the last 90 days from posthog.flag_evaluations. "
+ "It reads older ones and all other events from events."
+)
+
class _FlagCalledLiteralFinder(TraversingVisitor):
def __init__(self, table_class: type[Table]) -> None:
@@ -115,3 +126,8 @@ def flag_called_spans_read_from_flag_evaluations(prepared: ast.AST) -> set[Liter
finder = _FlagCalledLiteralFinder(FlagEvaluationsTable)
finder.visit(prepared)
return {LiteralSpan(start=literal.start, end=literal.end) for literal in finder.literals}
+
+
+def flag_called_spans_read_from_union(context: HogQLContext) -> set[LiteralSpan]:
+ """The spans of the `$feature_flag_called` literals whose scan reads `events` and `flag_evaluations` together."""
+ return {LiteralSpan(start=start, end=end) for start, end in context.flag_called_union_spans}
diff --git a/posthog/hogql/functions/cohort.py b/posthog/hogql/functions/cohort.py
index e03b300a1402..29141a4a5627 100644
--- a/posthog/hogql/functions/cohort.py
+++ b/posthog/hogql/functions/cohort.py
@@ -70,6 +70,8 @@ def inline_cohort_query(
is_static: bool,
version: Optional[int],
context: HogQLContext,
+ *,
+ read_flag_calls_from_union: bool = True,
) -> Optional[ast.SelectQuery | ast.SelectSetQuery]:
from posthog.hogql_queries.hogql_cohort_query import HogQLCohortQuery
@@ -95,7 +97,13 @@ def inline_cohort_query(
context.team = Team.objects.get(id=context.team_id)
- return HogQLCohortQuery(cohort=cohort, team=context.team).get_query()
+ query = HogQLCohortQuery(cohort=cohort, team=context.team).get_query()
+ if not read_flag_calls_from_union:
+ return query
+ # Deferred: the transform imports the resolver, which imports this module.
+ from posthog.hogql.transforms.flag_calls_union import read_flag_calls_in_inline_cohort # noqa: PLC0415
+
+ return read_flag_calls_in_inline_cohort(query, context)
def cohort_subquery(cohort_id, is_static, version: Optional[int] = None) -> ast.Expr:
diff --git a/posthog/hogql/metadata.py b/posthog/hogql/metadata.py
index 0e09d279aa9d..6ba1ed5845c1 100644
--- a/posthog/hogql/metadata.py
+++ b/posthog/hogql/metadata.py
@@ -36,10 +36,13 @@ from posthog.hogql.errors import (
)
from posthog.hogql.filters import replace_filters
from posthog.hogql.flag_called_warnings import (
+ FLAG_CALLED_FROM_BOTH_TABLES_NOTICE,
FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE,
+ FLAG_CALLED_SPLIT_BY_AGE_NOTICE,
LiteralSpan,
flag_called_on_events_warnings,
flag_called_spans_read_from_flag_evaluations,
+ flag_called_spans_read_from_union,
)
from posthog.hogql.index_eligibility import IndexEligibilityReport, build_index_eligibility_report
from posthog.hogql.metadata_heuristics import run_metadata_heuristics
@@ -226,6 +229,7 @@ def get_hogql_metadata(
flag_called_notices = _flag_called_notices(hogql_ast, prepared_ast, context, team)
heuristic_warnings.extend(flag_called_notices.move_warnings)
context.notices.extend(flag_called_notices.flag_evaluations_notices)
+ context.notices.extend(flag_called_notices.both_tables_notices)
if source is None and query.indexUsage and _index_usage_enabled(team):
_attach_index_usage(response, hogql_ast, context)
@@ -337,6 +341,7 @@ def _flag_called_announcement_url(payload: object) -> str | None:
class _FlagCalledNotices:
move_warnings: tuple[HogQLNotice, ...] = ()
flag_evaluations_notices: tuple[HogQLNotice, ...] = ()
+ both_tables_notices: tuple[HogQLNotice, ...] = ()
def _flag_called_notices(
@@ -345,9 +350,9 @@ def _flag_called_notices(
context: HogQLContext,
team: Team,
) -> _FlagCalledNotices:
- """The move warning on each flag-call literal that reads events, and a notice on each literal whose query
- reads posthog.flag_evaluations instead. `prepared_ast` is what the printer ran, so it says which scans it
- swapped."""
+ """The move warning on each flag-call literal that reads events, a notice on each literal whose query reads
+ posthog.flag_evaluations instead, and a notice on each literal whose scan reads events and
+ posthog.flag_evaluations together. `prepared_ast` is what the printer ran, so it says which scans it swapped."""
try:
move_notices = _flag_called_move_notices(team)
if move_notices is None or not move_notices.enabled:
@@ -359,15 +364,27 @@ def _flag_called_notices(
swapped = flag_called_spans_read_from_flag_evaluations(prepared_ast) if prepared_ast is not None else set()
move_warnings: list[HogQLNotice] = []
flag_evaluations_notices: list[HogQLNotice] = []
+ both_tables_notices: list[HogQLNotice] = []
+ read_from_both_tables = flag_called_spans_read_from_union(context)
+ both_tables_message = (
+ FLAG_CALLED_FROM_BOTH_TABLES_NOTICE
+ if context.database is not None and context.database.flag_calls_only_in_flag_evaluations
+ else FLAG_CALLED_SPLIT_BY_AGE_NOTICE
+ )
for warning in warnings:
- if LiteralSpan(start=warning.start, end=warning.end) in swapped:
+ span = LiteralSpan(start=warning.start, end=warning.end)
+ if span in swapped:
flag_evaluations_notices.append(
warning.model_copy(update={"message": FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, "url": url})
)
+ elif span in read_from_both_tables:
+ both_tables_notices.append(warning.model_copy(update={"message": both_tables_message, "url": url}))
else:
move_warnings.append(warning.model_copy(update={"url": url}))
return _FlagCalledNotices(
- move_warnings=tuple(move_warnings), flag_evaluations_notices=tuple(flag_evaluations_notices)
+ move_warnings=tuple(move_warnings),
+ flag_evaluations_notices=tuple(flag_evaluations_notices),
+ both_tables_notices=tuple(both_tables_notices),
)
except Exception:
# The warning is advisory. A query that compiles must not be reported as invalid because this
diff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.py
index 02cfec4ae349..a26eb0fdfd43 100644
--- a/posthog/hogql/test/test_metadata.py
+++ b/posthog/hogql/test/test_metadata.py
@@ -23,7 +23,12 @@ from posthog.schema import (
)
from posthog.hogql.direct_connection import INVALID_CONNECTION_ID_ERROR
-from posthog.hogql.flag_called_warnings import FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE, FLAG_CALLED_ON_EVENTS_WARNING
+from posthog.hogql.flag_called_warnings import (
+ FLAG_CALLED_FROM_BOTH_TABLES_NOTICE,
+ FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE,
+ FLAG_CALLED_ON_EVENTS_WARNING,
+ FLAG_CALLED_SPLIT_BY_AGE_NOTICE,
+)
from posthog.hogql.metadata import get_hogql_metadata
from posthog.hogql.parser import parse_select
from posthog.hogql.taxonomy_validation import MAX_SUGGESTED_NAMES
@@ -449,6 +454,44 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
expected_notices=1,
),
param("refused_by_the_swap", _FLAG_CALLED_QUERY, True, 1, mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS),
+ param(
+ "mixed_scan_read_from_both_tables",
+ "SELECT count() FROM events WHERE event IN ('$pageview', '$feature_flag_called')",
+ True,
+ 0,
+ _move_notices("https://example.com/announcement"),
+ "https://example.com/announcement",
+ mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ union_enabled=True,
+ expected_union_notices=1,
+ ),
+ param(
+ "mixed_scan_read_from_both_tables_once_events_gets_no_flag_calls",
+ "SELECT count() FROM events WHERE event IN ('$pageview', '$feature_flag_called')",
+ True,
+ 0,
+ _move_notices("https://example.com/announcement"),
+ "https://example.com/announcement",
+ mode=FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY,
+ union_enabled=True,
+ expected_union_notices=1,
+ union_notice=FLAG_CALLED_FROM_BOTH_TABLES_NOTICE,
+ ),
+ param(
+ "mixed_scan_the_union_declines",
+ "SELECT goe_0.properties FROM events WHERE event IN ('$pageview', '$feature_flag_called')",
+ True,
+ 1,
+ mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ union_enabled=True,
+ ),
+ param(
+ "mixed_scan_with_the_union_off",
+ "SELECT count() FROM events WHERE event IN ('$pageview', '$feature_flag_called')",
+ True,
+ 1,
+ mode=FlagEvaluationsMode.READ_FLAG_EVALUATIONS,
+ ),
param(
"read_from_flag_evaluations_without_coverage_once_events_gets_no_flag_calls",
_FLAG_CALLED_QUERY,
@@ -487,6 +530,9 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
expected_url: str | None = None,
expected_notices: int = 0,
mode: FlagEvaluationsMode = FlagEvaluationsMode.EVENTS,
+ union_enabled: bool = False,
+ expected_union_notices: int = 0,
+ union_notice: str = FLAG_CALLED_SPLIT_BY_AGE_NOTICE,
) -> None:
OrganizationFeatureFlagsConfig.objects.update_or_create(
organization=self.organization,
@@ -512,12 +558,17 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
return_value=flag_evaluations_enabled,
),
patch("posthoganalytics.get_feature_flag_result", return_value=move_notices),
+ patch(
+ "posthog.hogql.database.database.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: union_enabled and key == "hogql-flag-calls-union",
+ ),
):
metadata = self._select(query)
self.assertTrue(metadata.isValid, metadata.errors)
warnings = [w for w in metadata.warnings if w.message == FLAG_CALLED_ON_EVENTS_WARNING]
notices = [n for n in metadata.notices if n.message == FLAG_CALLED_FROM_FLAG_EVALUATIONS_NOTICE]
+ union_notices = [n for n in metadata.notices if n.message == union_notice]
literal_start = query.rindex("'$feature_flag_called'")
literal_end = literal_start + len("'$feature_flag_called'")
self.assertEqual(
@@ -527,6 +578,10 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
self.assertEqual(
[(n.start, n.end, n.url) for n in notices], [(literal_start, literal_end, expected_url)] * expected_notices
)
+ self.assertEqual(
+ [(n.start, n.end, n.url) for n in union_notices],
+ [(literal_start, literal_end, expected_url)] * expected_union_notices,
+ )
def test_metadata_does_not_log_invalid_announcement_url_without_a_flag_called_literal(self) -> None:
with (
diff --git a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
index 1a2ac95408e9..6e8e3ffb4e07 100644
--- a/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
+++ b/posthog/hogql/transforms/flag_calls_from_flag_evaluations.py
@@ -13,7 +13,9 @@ import json
from collections import Counter
from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime, time, timedelta, tzinfo
+from fractions import Fraction
from functools import partial
+from math import isfinite
from typing import TYPE_CHECKING, Generic, Protocol
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
@@ -217,8 +219,9 @@ def swap_flag_calls(
For an organization whose flag calls are only in `flag_evaluations`, a scan that the syntactic rules miss swaps
too when the resolved original shows that its filters pin `event` to flag calls in its WHERE or PREWHERE. The
- query raises when such a scan cannot swap. A scan that the resolver adds, such as one in an inline cohort, keeps
- reading `events`.
+ query raises when such a scan cannot swap. A scan that the resolver adds, such as one in an inline cohort, is not
+ swapped here. On mode 2 with the union enabled, `read_flag_calls_in_inline_cohort` points an inline cohort's scans
+ at the union. Otherwise they read `events`.
The resolver runs this on the body of each saved view that it inlines, and `view_name` names that view for the
error. The check of resolved filters skips the joins of a view's body in the enclosing query, because the swap of
@@ -235,13 +238,16 @@ def swap_flag_calls(
if _swap_applies(node, context, database):
swap = _resolve_with_swaps(node, context, resolve, database, view_name)
else:
- swap = FlagCallSwap(source=node, resolved=resolve(node))
+ swap = FlagCallSwap(
+ source=node,
+ resolved=_without_unreadable_inline_cohort_scans(resolve(node), context, view_name),
+ )
# A view's body names flag calls only after resolution when it filters with `matchesAction`, so its swap never
# runs. The check reads the whole resolved query, so it runs once, for the top-level query.
if database.flag_calls_only_in_flag_evaluations and view_name is None:
unswapped_view = _view_reading_flag_calls_from_events(swap.resolved)
if unswapped_view is not None:
- raise _flag_calls_only_in_flag_evaluations_error(
+ raise flag_calls_only_in_flag_evaluations_error(
f"it reads the saved view {unswapped_view}, which reads them from events", None
)
return swap
@@ -254,12 +260,13 @@ def _resolve_with_swaps(
# Deferred: flag_calls_union imports this module.
from posthog.hogql.transforms.flag_calls_union import ( # noqa: PLC0415
read_flag_calls_with_union,
+ union_applies,
union_unreadable_reason,
)
with context.timings.measure("flag_calls_from_flag_evaluations"):
# The lookup reads Postgres, so it runs only when a scan pins `$feature_flag`.
- compares_as_string = partial(_flag_key_compares_as_string, context)
+ compares_as_string = partial(team_flag_key_compares_as_string, context)
rewritten = clone_expr(node)
clones = _join_clones(node, rewritten) if only_in_flag_evaluations else {}
swapper = _EventsScanSwapper(
@@ -267,6 +274,7 @@ def _resolve_with_swaps(
timezone=ZoneInfo(database.get_timezone()),
only_in_flag_evaluations=only_in_flag_evaluations,
flag_key_compares_as_string=compares_as_string,
+ union_reads_ratio_samples=only_in_flag_evaluations and union_applies(context),
database=database,
)
swapper.visit(rewritten)
@@ -274,7 +282,7 @@ def _resolve_with_swaps(
refusals = [*swapper.refusals, *union.refusals]
if refusals:
if only_in_flag_evaluations:
- raise _flag_calls_only_in_flag_evaluations_error(refusals[0], view_name)
+ raise flag_calls_only_in_flag_evaluations_error(refusals[0], view_name)
logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=refusals[0])
sources = JoinSources() if only_in_flag_evaluations else None
original = resolve(node, join_sources=sources)
@@ -282,16 +290,20 @@ def _resolve_with_swaps(
with context.timings.measure("flag_calls_from_flag_evaluations_by_type"):
refusal = swapper.swap_scans_found_by_type(original, sources, clones)
if refusal is not None:
- raise _flag_calls_only_in_flag_evaluations_error(refusal, view_name)
+ raise flag_calls_only_in_flag_evaluations_error(refusal, view_name)
if not swapper.swapped and not union.renamed:
- return FlagCallSwap(source=node, resolved=original)
+ return FlagCallSwap(
+ source=node,
+ resolved=_without_unreadable_inline_cohort_scans(original, context, view_name),
+ )
try:
swapped = resolve(rewritten)
except BaseHogQLError as error:
# Expected for a query that reads a field flag_evaluations lacks, such as goe_0.properties.
reason = f"it reads something that table doesn't have: {error}"
else:
- unreadable = union_unreadable_reason(swapped, context) if union.renamed else None
+ # An inline cohort renames its own scans during resolution, so the outer rename does not show them.
+ unreadable = union_unreadable_reason(swapped, context) if union.renamed or only_in_flag_evaluations else None
if unreadable is not None:
reason = unreadable
elif not _swap_keeps_bindings(original, swapped, swapper.flag_key_predicates):
@@ -301,15 +313,26 @@ def _resolve_with_swaps(
elif not only_in_flag_evaluations and not _swap_reads_properties_alike(original, swapped, context):
reason = "it reads a property from a different kind of column there"
else:
+ # A view body's offsets come from the view's own text, not from the query the editor shows.
+ if view_name is None:
+ context.flag_called_union_spans.update(union.literal_spans)
return FlagCallSwap(source=rewritten, resolved=swapped)
if only_in_flag_evaluations:
# events gets no new flag calls, so the original query would return only calls from before the move.
- raise _flag_calls_only_in_flag_evaluations_error(reason, view_name)
+ raise flag_calls_only_in_flag_evaluations_error(reason, view_name)
logger.info("flag_calls_swap_declined", team_id=context.team_id, reason=reason)
return FlagCallSwap(source=node, resolved=original)
-def _flag_key_compares_as_string(context: HogQLContext) -> bool:
+def _without_unreadable_inline_cohort_scans(resolved: _T_AST, context: HogQLContext, view_name: str | None) -> _T_AST:
+ # Deferred: flag_calls_union imports this module.
+ from posthog.hogql.transforms.flag_calls_union import require_union_readable # noqa: PLC0415
+
+ require_union_readable(resolved, context, view_name)
+ return resolved
+
+
+def team_flag_key_compares_as_string(context: HogQLContext) -> bool:
if context.team_id is None:
return True
return flag_key_typed_as_string(_event_property_metadata(context, {_FLAG_KEY_PROPERTY}))
@@ -348,7 +371,7 @@ def _event_property_metadata(context: HogQLContext, names: set[str]) -> Property
return loaded
-def _flag_calls_only_in_flag_evaluations_error(reason: str, view_name: str | None) -> QueryError:
+def flag_calls_only_in_flag_evaluations_error(reason: str, view_name: str | None = None) -> QueryError:
subject, part = ("this query", "query") if view_name is None else (f"the view {view_name}", "view")
return QueryError(
f"Your organization's {FLAG_CALLED_EVENT} events are only in posthog.flag_evaluations, and {subject} "
@@ -470,14 +493,17 @@ class EventsScan:
return None
return _below_posthog_events(chain)
- def shape_refusal(self, node: ast.SelectQuery) -> str | None:
- """Why another table cannot stand in for this scan whatever its filters say, or None."""
+ def shape_refusal(self, node: ast.SelectQuery, *, ratio_sample_ok: bool = False) -> str | None:
+ """Why another table cannot stand in for this scan whatever its filters say, or None. With `ratio_sample_ok`,
+ a SAMPLE that `ratio_sample_fraction` reads does not count, for a table that applies the ratio itself."""
if self.join.table_args is not None or self.join.table_final:
return "it uses FINAL or table arguments on events"
if self.join.column_aliases:
return "it names the events columns with column aliases"
- if self.join.sample is not None and not self.join.sample.samples_all_rows():
- return "it uses SAMPLE"
+ sample = self.join.sample
+ if sample is not None and not sample.samples_all_rows():
+ if not ratio_sample_ok or ratio_sample_fraction(sample) is None:
+ return "it uses SAMPLE"
for item in node.select:
while isinstance(item, ast.Alias):
item = item.expr
@@ -658,6 +684,28 @@ def flag_key_predicate(table: str, keys: list[str]) -> ast.Expr | None:
)
+def ratio_sample_fraction(sample: ast.SampleExpr) -> Fraction | None:
+ """The share of distinct ids that `SAMPLE k` or `SAMPLE a/b` keeps, or None for any other form.
+
+ ClickHouse reads `SAMPLE n` with n above 1 as a row count, and `OFFSET` moves the sampled range. Neither has a
+ ratio that another table could apply.
+ """
+ if sample.offset_value is not None:
+ return None
+ ratio = sample.sample_value
+ constants = [ratio.left, ratio.right or ast.Constant(value=1)]
+ if not all(
+ isinstance(c.value, int | float)
+ and not isinstance(c.value, bool)
+ and (isinstance(c.value, int) or isfinite(c.value))
+ for c in constants
+ ):
+ return None
+ numerator, denominator = (Fraction(c.value if isinstance(c.value, int) else str(c.value)) for c in constants)
+ fraction = numerator / denominator if denominator else None
+ return fraction if fraction is not None and 0 < fraction < 1 else None
+
+
def point_scan_at(
node: ast.SelectQuery, scan: EventsScan, chain: list[str | int], condition: ast.Expr | None = None
) -> None:
@@ -965,8 +1013,10 @@ class _EventsScanSwapper(ScanVisitor):
only_in_flag_evaluations: bool,
flag_key_compares_as_string: Callable[[], bool],
database: "Database",
+ union_reads_ratio_samples: bool = False,
) -> None:
super().__init__()
+ self.union_reads_ratio_samples = union_reads_ratio_samples
self.database = database
self.now = now
self.timezone = timezone
@@ -987,7 +1037,8 @@ class _EventsScanSwapper(ScanVisitor):
):
self._swap(node, scan, conjuncts)
else:
- refusal = scan.shape_refusal(node)
+ # On mode 2 the union reads a scan with a ratio SAMPLE, so this refuses only the other shapes.
+ refusal = scan.shape_refusal(node, ratio_sample_ok=self.union_reads_ratio_samples)
if refusal is not None and scan.pins_flag_calls(conjuncts, needs_flag_key=False):
self.refusals.append(refusal)
@@ -1005,7 +1056,8 @@ class _EventsScanSwapper(ScanVisitor):
original join of each resolved join, and `clones` holds the clones of each original join. Returns why one of
them cannot swap, or None.
- A scan that the resolver adds, such as one in a saved view's body or an inline cohort, keeps reading events.
+ A scan that the resolver adds, such as one in a saved view's body or an inline cohort, is not swapped here. A
+ view's scan keeps reading events. On mode 2 with the union enabled, an inline cohort's scan reads the union.
Any other scan without a clone cannot swap.
"""
for select, join in _joins(resolved):
@@ -1028,8 +1080,8 @@ class _EventsScanSwapper(ScanVisitor):
def _swap_clone(self, clone: "_JoinClone") -> str | None:
chain = _table_chain(clone.join)
- if chain == ["posthog", "flag_evaluations"]:
- # The syntactic rules, or an earlier resolved join from the same source, already swapped it.
+ if chain in (["posthog", "flag_evaluations"], ["posthog", "events_with_flag_calls"]):
+ # The syntactic rules, the union rename, or an earlier resolved join from the same source already moved it.
return None
if chain != ["events"]:
return _UNMOVABLE_SHAPE
diff --git a/posthog/hogql/transforms/flag_calls_union.py b/posthog/hogql/transforms/flag_calls_union.py
index 8489556d4d76..2a3c8be98e81 100644
--- a/posthog/hogql/transforms/flag_calls_union.py
+++ b/posthog/hogql/transforms/flag_calls_union.py
@@ -10,8 +10,12 @@ scan, so the rules decide which scans pay for a second table, and which shapes t
"""
from collections.abc import Callable
+from functools import cache, partial
+from math import floor
from typing import TYPE_CHECKING, cast
+import structlog
+
from posthog.hogql import ast
from posthog.hogql.constants import FLAG_CALLED_EVENT
from posthog.hogql.context import HogQLContext
@@ -52,8 +56,11 @@ from posthog.hogql.transforms.flag_calls_from_flag_evaluations import (
compared_values,
comparison_of,
filter_conjuncts,
+ flag_calls_only_in_flag_evaluations_error,
flag_key_predicate,
point_scan_at,
+ ratio_sample_fraction,
+ team_flag_key_compares_as_string,
)
from posthog.hogql.type_system import parse_sql_runtime_type
from posthog.hogql.visitor import CloningVisitor
@@ -64,6 +71,8 @@ from posthog.property_columns import TableColumn
if TYPE_CHECKING:
from posthog.hogql.database.database import Database
+logger = structlog.get_logger(__name__)
+
# The joins whose resolver compares only the join's key and `extra_from_fields` on the source table. Lazy table
# resolution projects exactly those from the union, so a join that reads anything else would name a missing column.
_UNION_JOIN_RESOLVERS = frozenset(
@@ -96,6 +105,8 @@ class UnionRename:
renamed: bool
# Why the union cannot read each other scan that names `$feature_flag_called` with other events.
refusals: tuple[str, ...]
+ # The (start, end) of each `$feature_flag_called` literal that a renamed scan compares with its `event`.
+ literal_spans: frozenset[tuple[int, int]] = frozenset()
def read_flag_calls_with_union(
@@ -119,6 +130,51 @@ def read_flag_calls_with_union(
)
+def union_applies(context: HogQLContext) -> bool:
+ """Whether the union reads this query's mixed flag-call scans."""
+ return _union_database(context) is not None
+
+
+def read_flag_calls_in_inline_cohort(
+ query: ast.SelectQuery | ast.SelectSetQuery, context: HogQLContext
+) -> ast.SelectQuery | ast.SelectSetQuery:
+ """Points each flag-call scan in an inline cohort `query` at the union, in place, and returns `query`.
+
+ The resolver and the subquery and LEFTJOIN paths of `IN COHORT` splice an inline cohort in after the swap in
+ `resolve_flag_calls_from_flag_evaluations` has run, so that swap never sees its scans. Only the rename applies
+ here, because the swap's binding check needs the whole query.
+
+ Only an organization whose flag calls are only in `flag_evaluations` reads the union here. While `events` holds
+ every flag call, an inline cohort reads them from `events` as before. `resolve_in_cohorts_conjoined` splices its
+ cohorts in before the swap runs, so its caller passes `read_flag_calls_from_union=False`: the swap sees those scans.
+ """
+ database = _union_database(context)
+ if database is None or not database.flag_calls_only_in_flag_evaluations:
+ return query
+ renamer = _UnionRenamer(
+ cache(partial(team_flag_key_compares_as_string, context)),
+ only_in_flag_evaluations=True,
+ rename_flag_only_scans=True,
+ )
+ renamer.visit(query)
+ if renamer.refusals:
+ raise flag_calls_only_in_flag_evaluations_error(renamer.refusals[0])
+ return query
+
+
+def require_union_readable(resolved: ast.AST, context: HogQLContext, view_name: str | None = None) -> None:
+ """Raises a QueryError when an organization with flag calls only in `flag_evaluations` has a union scan in
+ `resolved` that reads a field the union cannot supply. An inline cohort is spliced in after the rename, so only
+ the resolved query shows its scans. `IN COHORT` resolves its join before lazy table resolution builds the
+ union, so that call is the last point where ClickHouse has not yet rejected the query."""
+ database = _union_database(context)
+ if database is None or not database.flag_calls_only_in_flag_evaluations:
+ return
+ unreadable = union_unreadable_reason(resolved, context)
+ if unreadable is not None:
+ raise flag_calls_only_in_flag_evaluations_error(unreadable, view_name)
+
+
def union_unreadable_reason(node: ast.AST, context: HogQLContext) -> str | None:
"""Why the union cannot read a scan in the resolved `node`, or None when it can read them all.
@@ -208,6 +264,31 @@ def union_events_filter(node: ast.SelectQuery, union: EventsWithFlagCallsTable)
return _events_filter(_scan_conjuncts(node, union))
+def union_scan_sample(node: ast.SelectQuery, union: EventsWithFlagCallsTable) -> ast.SampleExpr | None:
+ """The SAMPLE that `node` puts on its scan of the union, or None."""
+ join = node.select_from
+ while join is not None:
+ if join.sample is not None and union_table_of(join.table.type if join.table else join.type) is union:
+ return join.sample
+ join = join.next_join
+ return None
+
+
+def sample_flag_evaluations_filter(sample: ast.SampleExpr, distinct_id: ast.Field) -> ast.Expr:
+ """The predicate that keeps the `flag_evaluations` rows whose `distinct_id` ClickHouse's `SAMPLE` keeps on `events`.
+
+ `events` samples by `cityHash64(distinct_id)`, and a ratio keeps the hashes below that ratio of the UInt64 range.
+ """
+ fraction = ratio_sample_fraction(sample)
+ if fraction is None:
+ raise QueryError("The flag call union reads only a SAMPLE ratio between 0 and 1.")
+ return ast.CompareOperation(
+ op=ast.CompareOperationOp.Lt,
+ left=ast.Call(name="cityHash64", args=[distinct_id]),
+ right=ast.Constant(value=floor(fraction * 2**64)),
+ )
+
+
def rename_flag_call_scans_to_union(
node: ast.AST,
flag_key_compares_as_string: Callable[[], bool],
@@ -222,8 +303,8 @@ def rename_flag_call_scans_to_union(
clauses. A subquery's comparison does not count. A funnel step column such as
`if(event = '$feature_flag_called' AND ..., 1, 0)` counts.
2. The scan is the first table in FROM, and every join after it is INNER, LEFT or CROSS, GLOBAL or not.
- 3. The scan has no FINAL, table arguments, column aliases or SAMPLE other than 1, and its query selects no `*`
- or `COLUMNS(...)`.
+ 3. The scan has no FINAL, table arguments or column aliases, and its SAMPLE is 1 or a ratio between 0 and 1 without
+ OFFSET. Its query selects no `*` or `COLUMNS(...)`.
A scan that the flag_evaluations swap changed reads `posthog.flag_evaluations` and is not an `events` scan any
more. The flag_evaluations swap swaps or refuses a first scan that reads only flag calls, so the union neither
@@ -232,7 +313,9 @@ def rename_flag_call_scans_to_union(
"""
renamer = _UnionRenamer(flag_key_compares_as_string, only_in_flag_evaluations)
renamer.visit(node)
- return UnionRename(renamed=renamer.renamed, refusals=tuple(renamer.refusals))
+ return UnionRename(
+ renamed=renamer.renamed, refusals=tuple(renamer.refusals), literal_spans=frozenset(renamer.literal_spans)
+ )
def _union_database(context: HogQLContext) -> "Database | None":
@@ -385,7 +468,7 @@ def _union_refusal(node: ast.SelectQuery, scan: EventsScan, position: int) -> st
"""Why the union cannot stand in for `scan`, the scan at `position` in the FROM chain of `node`, or None."""
if position > 0:
return "it joins events after another table"
- refusal = scan.shape_refusal(node)
+ refusal = scan.shape_refusal(node, ratio_sample_ok=True)
if refusal is not None:
return refusal
join = scan.join.next_join
@@ -398,11 +481,19 @@ def _union_refusal(node: ast.SelectQuery, scan: EventsScan, position: int) -> st
return None
+@frozen
+class _FlagCallScan:
+ position: int
+ scan: EventsScan
+ literals: list[ast.Constant]
+
+
def _flag_call_scans(
node: ast.SelectQuery, flag_key_compares_as_string: Callable[[], bool], ctes: frozenset[str]
-) -> list[tuple[int, EventsScan]]:
- """Each `events` scan in the FROM chain of `node` whose query names `$feature_flag_called`, with its position."""
- scans: list[tuple[int, EventsScan]] = []
+) -> list[_FlagCallScan]:
+ """Each `events` scan in the FROM chain of `node` whose query names `$feature_flag_called`, with its position and
+ the literals that name it."""
+ scans: list[_FlagCallScan] = []
join = node.select_from
position = 0
while join is not None:
@@ -423,17 +514,17 @@ def _flag_call_scans(
flag_key_compares_as_string=flag_key_compares_as_string,
)
)
- if scan is not None and _names_flag_calls(node, scan):
- scans.append((position, scan))
+ if scan is not None and (literals := _flag_call_literals(node, scan)):
+ scans.append(_FlagCallScan(position=position, scan=scan, literals=literals))
join = join.next_join
position += 1
return scans
-def _names_flag_calls(node: ast.SelectQuery, scan: EventsScan) -> bool:
+def _flag_call_literals(node: ast.SelectQuery, scan: EventsScan) -> list[ast.Constant]:
finder = _FlagCallComparisonFinder(node, scan)
finder.visit(node)
- return finder.found
+ return finder.literals
class _ScanReadCopier(CloningVisitor):
@@ -594,7 +685,7 @@ class _FlagCallComparisonFinder(SyntaxVisitor):
super().__init__()
self.root = root
self.reader = scan
- self.found = False
+ self.literals: list[ast.Constant] = []
def visit_select_query(self, node: ast.SelectQuery) -> None:
if node is self.root:
@@ -613,34 +704,57 @@ class _FlagCallComparisonFinder(SyntaxVisitor):
def _check(self, expr: ast.Expr) -> None:
values = compared_values(expr, lambda read: self.reader.column(read) == ["event"])
- if values and any(isinstance(value, ast.Constant) and value.value == FLAG_CALLED_EVENT for value in values):
- self.found = True
+ for value in values or []:
+ if isinstance(value, ast.Constant) and value.value == FLAG_CALLED_EVENT:
+ self.literals.append(value)
class _UnionRenamer(ScanVisitor):
- def __init__(self, flag_key_compares_as_string: Callable[[], bool], only_in_flag_evaluations: bool) -> None:
+ def __init__(
+ self,
+ flag_key_compares_as_string: Callable[[], bool],
+ only_in_flag_evaluations: bool,
+ *,
+ rename_flag_only_scans: bool = False,
+ ) -> None:
super().__init__()
self.flag_key_compares_as_string = flag_key_compares_as_string
self.only_in_flag_evaluations = only_in_flag_evaluations
+ self.rename_flag_only_scans = rename_flag_only_scans
self.renamed = False
self.refusals: list[str] = []
+ self.literal_spans: set[tuple[int, int]] = set()
def visit_scan_query(self, node: ast.SelectQuery) -> None:
# Only the first scan can be renamed, and it comes first, so each refusal reads the query as written.
- for position, scan in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes):
- if position == 0 and self._reads_only_flag_calls(node, scan):
+ for found in _flag_call_scans(node, self.flag_key_compares_as_string, self.ctes):
+ position, scan, literals = found.position, found.scan, found.literals
+ if position == 0 and self._left_to_the_swap(node, scan):
continue
refusal = _union_refusal(node, scan, position)
if refusal is not None:
self.refusals.append(refusal)
else:
point_scan_at(node, scan, list(EVENTS_WITH_FLAG_CALLS_CHAIN))
- # A lazy table cannot be sampled, and SAMPLE 1 keeps every row anyway.
- scan.join.sample = None
+ # A lazy table cannot be sampled. SAMPLE 1 keeps every row anyway, and the union applies a ratio
+ # sample in its branches, where lazy table resolution reads it from this scan.
+ if scan.join.sample is not None and scan.join.sample.samples_all_rows():
+ scan.join.sample = None
self.renamed = True
+ self.literal_spans.update(
+ (literal.start, literal.end)
+ for literal in literals
+ if literal.start is not None and literal.end is not None
+ )
- def _reads_only_flag_calls(self, node: ast.SelectQuery, scan: EventsScan) -> bool:
+ def _left_to_the_swap(self, node: ast.SelectQuery, scan: EventsScan) -> bool:
# The flag_evaluations swap handles a first scan whose `event` filter it reads as only flag calls. It swaps or
# refuses that scan on mode 2. On mode 1 events holds every flag call, so the union would read the same calls
# twice there.
- return scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False)
+ if self.rename_flag_only_scans:
+ return False
+ # The swap cannot read a sampled scan. On mode 2 the union reads it.
+ sampled = scan.join.sample is not None and not scan.join.sample.samples_all_rows()
+ return scan.pins_flag_calls(filter_conjuncts(node), needs_flag_key=False) and not (
+ self.only_in_flag_evaluations and sampled
+ )
diff --git a/posthog/hogql/transforms/in_cohort.py b/posthog/hogql/transforms/in_cohort.py
index 1bf7ab8a8281..a9c90ab3892b 100644
--- a/posthog/hogql/transforms/in_cohort.py
+++ b/posthog/hogql/transforms/in_cohort.py
@@ -172,7 +172,10 @@ class MultipleInCohortResolver(TraversingVisitor):
inlineable_dynamic: list[tuple[int, ast.SelectQuery | ast.SelectSetQuery]] = []
non_inlineable_dynamic: list[tuple[int, StaticOrDynamic, int]] = []
for cohort_id, cohort_type, version in dynamic_cohorts:
- inline_ast = inline_cohort_query(cohort_id, False, version, self.context)
+ # This runs before the flag-call swap, which then sees the cohort's scans.
+ inline_ast = inline_cohort_query(
+ cohort_id, False, version, self.context, read_flag_calls_from_union=False
+ )
if inline_ast is not None:
inlineable_dynamic.append((cohort_id, inline_ast))
else:
@@ -373,6 +376,7 @@ class InCohortResolver(TraversingVisitor):
negative: bool,
):
from posthog.hogql.functions.cohort import inline_cohort_query
+ from posthog.hogql.transforms.flag_calls_union import require_union_readable
from posthog.hogql.transforms.lazy_tables import resolve_lazy_tables
assert self.context is not None
@@ -440,6 +444,7 @@ class InCohortResolver(TraversingVisitor):
),
)
if inline_ast is not None:
+ require_union_readable(new_join, self.context)
resolve_lazy_tables(
new_join, self.dialect, [self.stack[-1]], self.context, resolver_factory=self.resolver_factory
)
diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.py
index a115d8332f37..f7ebf6d84ced 100644
--- a/posthog/hogql/transforms/lazy_tables.py
+++ b/posthog/hogql/transforms/lazy_tables.py
@@ -814,13 +814,17 @@ class LazyTableResolver(TraversingVisitor):
# A lazy table expands into an aggregating subquery. ClickHouse rejects a SAMPLE
# modifier on a subquery, and sampling a pre-aggregated table is meaningless, so
# drop the modifier rather than emitting invalid SQL.
- if join_ptr.sample is not None:
+ # EventsWithFlagCallsTable.lazy_select, called above, already copied the SAMPLE into each union
+ # branch, so it needs no warning. This clear must stay after that call.
+ if join_ptr.sample is not None and not isinstance(
+ table_to_add.lazy_table, EventsWithFlagCallsTable
+ ):
self.context.add_warning(
f'SAMPLE clause ignored: the "{table_name}" table is computed on the fly and cannot be sampled',
start=join_ptr.sample.start,
end=join_ptr.sample.end,
)
- join_ptr.sample = None
+ join_ptr.sample = None
break
join_ptr = join_ptr.next_join
diff --git a/posthog/hogql/transforms/test/test_flag_calls_union.py b/posthog/hogql/transforms/test/test_flag_calls_union.py
index 69eca14b26bb..f99587ac7be2 100644
--- a/posthog/hogql/transforms/test/test_flag_calls_union.py
+++ b/posthog/hogql/transforms/test/test_flag_calls_union.py
@@ -20,7 +20,13 @@ from django.test import SimpleTestCase
from parameterized import param, parameterized
-from posthog.schema import HogQLQueryModifiers, MaterializationMode, PersonsOnEventsMode
+from posthog.schema import (
+ HogQLQueryModifiers,
+ InCohortVia,
+ InlineCohortCalculation,
+ MaterializationMode,
+ PersonsOnEventsMode,
+)
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
@@ -56,9 +62,41 @@ E_MIXED = "e.event IN ('$pageview', '$feature_flag_called')"
UNION = "posthog.events_with_flag_calls"
READ = FlagEvaluationsMode.READ_FLAG_EVALUATIONS
ONLY = FlagEvaluationsMode.FLAG_EVALUATIONS_ONLY
+
+
UNION_EXPANSION = "FROM flag_evaluations"
+def _performed_flag_call_cohort(team, flag_key: str) -> Cohort:
+ return Cohort.objects.create(
+ team=team,
+ name=f"called {flag_key}",
+ filters={
+ "properties": {
+ "type": "AND",
+ "values": [
+ {
+ "type": "AND",
+ "values": [
+ {
+ "key": "$feature_flag_called",
+ "type": "behavioral",
+ "value": "performed_event",
+ "event_type": "events",
+ "time_value": 30,
+ "time_interval": "day",
+ "event_filters": [
+ {"key": "$feature_flag", "type": "event", "value": flag_key, "operator": "exact"}
+ ],
+ }
+ ],
+ }
+ ],
+ }
+ },
+ )
+
+
class _TableCollector(TraversingVisitor):
def __init__(self) -> None:
super().__init__()
@@ -130,11 +168,25 @@ class TestRenameFlagCallScansToUnion(SimpleTestCase):
# The flag_evaluations swap handles this scan. Events holds every flag call on mode 1. The swap swaps or
# refuses the scan on mode 2.
("only_flag_calls", "SELECT count() FROM events WHERE event = '$feature_flag_called'", ["events"], []),
- (
- "only_flag_calls_with_sample",
+ param(
+ "only_flag_calls_with_sample_on_mode_1",
"SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'",
["events"],
[],
+ False,
+ ),
+ # Phase 1 cannot swap a sampled scan, and the union applies the ratio.
+ (
+ "only_flag_calls_with_sample_on_mode_2",
+ "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'",
+ [UNION],
+ [],
+ ),
+ (
+ "only_flag_calls_with_sample_on_posthog_events_on_mode_2",
+ "SELECT count() FROM posthog.events SAMPLE 0.1 WHERE event = '$feature_flag_called'",
+ [UNION],
+ [],
),
(
"left_join_after_the_scan",
@@ -242,7 +294,34 @@ class TestRenameFlagCallScansToUnion(SimpleTestCase):
["events"],
["it names the events columns with column aliases"],
),
- ("sample_ratio", f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}", ["events"], ["it uses SAMPLE"]),
+ ("sample_ratio", f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}", [UNION], []),
+ ("sample_fraction", f"SELECT count() FROM events SAMPLE 1/10 WHERE {MIXED}", [UNION], []),
+ (
+ "sample_row_count",
+ f"SELECT count() FROM events SAMPLE 10 WHERE {MIXED}",
+ ["events"],
+ ["it uses SAMPLE"],
+ ),
+ *(
+ (
+ f"sample_{name}",
+ f"SELECT count() FROM events SAMPLE {value} WHERE {MIXED}",
+ ["events"],
+ ["it uses SAMPLE"],
+ )
+ for name, value in (
+ ("inf", "inf"),
+ ("nan", "nan"),
+ ("oversized_integer", "9" * 400),
+ ("oversized_hex_integer", "0x" + "f" * 5000),
+ )
+ ),
+ (
+ "sample_with_offset",
+ f"SELECT count() FROM events SAMPLE 0.1 OFFSET 0.5 WHERE {MIXED}",
+ ["events"],
+ ["it uses SAMPLE"],
+ ),
("select_star", f"SELECT * FROM events WHERE {MIXED}", ["events"], ["it selects * or COLUMNS(...)"]),
(
"cte_named_events",
@@ -293,6 +372,8 @@ class TestReadFlagCallsWithUnion(BaseTest):
new_events_schema: bool | None = None,
person_id_pushdown: bool | None = None,
modifiers: HogQLQueryModifiers | None = None,
+ inline_cohorts: bool = False,
+ in_cohort_via: InCohortVia | None = None,
) -> str:
_set_mode(self.organization, mode)
with _enabled_flags(union=union):
@@ -304,6 +385,8 @@ class TestReadFlagCallsWithUnion(BaseTest):
"personIdPushdown": person_id_pushdown,
"personsOnEventsMode": persons_on_events,
"useNewEventsSchema": _events_schema_for(mode, new_events_schema),
+ **({"inlineCohortCalculation": InlineCohortCalculation.ALWAYS} if inline_cohorts else {}),
+ **({"inCohortVia": in_cohort_via} if in_cohort_via else {}),
},
),
pretty=False,
@@ -428,6 +511,7 @@ class TestReadFlagCallsWithUnion(BaseTest):
f"SELECT goe_0.properties.plan FROM events WHERE {MIXED}",
"goe_0.properties",
),
+ ("sample_row_count", f"SELECT count() FROM events SAMPLE 10 WHERE {MIXED}", "it uses SAMPLE"),
param(
"join_whose_on_clause_reads_another_field",
f"SELECT probe_join.key FROM events WHERE {MIXED}",
@@ -608,6 +692,109 @@ class TestReadFlagCallsWithUnion(BaseTest):
assert context.warnings == []
+ def test_ratio_sample_applies_in_every_branch_and_warns_of_nothing(self):
+ _set_mode(self.organization, READ)
+ query = f"SELECT count() FROM events SAMPLE 0.1 WHERE {MIXED}"
+ with _enabled_flags(union=True):
+ sql, context = HogQLQueryExecutor(
+ query=query, team=self.team, modifiers=HogQLQueryModifiers(useNewEventsSchema=False), pretty=False
+ ).generate_clickhouse_sql()
+ branches = self._branches(substitute_params(sql, context.values))
+
+ assert context.warnings == []
+ assert [("SAMPLE 0.1" in branch) for branch in branches] == [True, True, False]
+ assert "less(cityHash64(flag_evaluations.distinct_id), 1844674407370955161)" in branches[-1]
+
+ def test_ratio_sample_applies_when_a_joined_person_field_reads_a_second_union_scan(self):
+ sql = self._sql(
+ f"SELECT person.properties.email, count() FROM events SAMPLE 0.1 WHERE {MIXED} GROUP BY 1",
+ persons_on_events=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ )
+
+ assert sql.count("SAMPLE 0.1") == 2
+ assert "less(cityHash64(flag_evaluations.distinct_id), 1844674407370955161)" in sql
+
+ @parameterized.expand(
+ [
+ ("subquery_on_mode_1", READ, InCohortVia.SUBQUERY, False),
+ ("subquery_on_mode_2", ONLY, InCohortVia.SUBQUERY, True),
+ ("leftjoin_on_mode_2", ONLY, InCohortVia.LEFTJOIN, True),
+ # IN COHORT splices the cohort in before the swap, which then reads its scans on either mode.
+ ("conjoined_on_mode_1", READ, InCohortVia.LEFTJOIN_CONJOINED, True),
+ ("conjoined_on_mode_2", ONLY, InCohortVia.LEFTJOIN_CONJOINED, True),
+ ]
+ )
+ def test_inline_cohort_reads_flag_calls_from_flag_evaluations_only_on_mode_2(
+ self, _name: str, mode: FlagEvaluationsMode, via: InCohortVia, expected_flag_evaluations: bool
+ ):
+ cohort = _performed_flag_call_cohort(self.team, "probe-flag")
+
+ sql = self._sql(
+ f"SELECT count() FROM events WHERE person_id IN COHORT {cohort.pk}",
+ mode=mode,
+ inline_cohorts=True,
+ in_cohort_via=via,
+ )
+
+ assert (re.search(r"FROM flag_evaluations\b", sql) is not None) is expected_flag_evaluations
+
+ @parameterized.expand(
+ [
+ ("outer_query_names_no_flag_calls", "SELECT count() FROM events WHERE person_id IN COHORT {cohort}"),
+ (
+ "outer_query_excludes_flag_calls",
+ "SELECT count() FROM events WHERE event != '$feature_flag_called' AND person_id IN COHORT {cohort}",
+ ),
+ (
+ "outer_query_swapped_by_phase_1",
+ "SELECT count() FROM events WHERE event = '$feature_flag_called' AND person_id IN COHORT {cohort}",
+ ),
+ (
+ "left_join_cohort",
+ "SELECT count() FROM events WHERE person_id IN COHORT {cohort}",
+ InCohortVia.LEFTJOIN,
+ ),
+ ]
+ )
+ def test_mode_2_raises_when_an_inline_cohort_reads_a_field_the_union_cannot_supply(
+ self, _name: str, outer_query: str, via: InCohortVia | None = None
+ ):
+ self.addCleanup(cleanup_materialized_columns)
+ materialize("events", "probe", is_nullable=True, column_type="Nullable(Float64)")
+ cohort = _performed_flag_call_cohort(self.team, "probe-flag")
+ cohort_query = parse_select(
+ "SELECT person_id AS id FROM events WHERE event = '$feature_flag_called' "
+ "AND properties.probe > 1 GROUP BY person_id"
+ )
+
+ with (
+ patch("posthog.hogql_queries.hogql_cohort_query.HogQLCohortQuery.get_query", return_value=cohort_query),
+ pytest.raises(QueryError, match="probe, which events stores in a Nullable"),
+ ):
+ self._sql(
+ outer_query.format(cohort=cohort.pk),
+ mode=ONLY,
+ inline_cohorts=True,
+ in_cohort_via=via,
+ new_events_schema=False,
+ )
+
+ @parameterized.expand(
+ [
+ ("flag_calls_only", "SELECT count() FROM events SAMPLE 0.1 WHERE event = '$feature_flag_called'"),
+ (
+ "flag_calls_only_next_to_a_join",
+ "SELECT count() FROM events AS e SAMPLE 0.1 LEFT JOIN persons AS p ON e.person_id = p.id "
+ "WHERE e.event = '$feature_flag_called'",
+ ),
+ ]
+ )
+ def test_mode_2_reads_a_sampled_flag_call_scan_from_the_union(self, _name: str, query: str):
+ sql = self._sql(query, mode=ONLY)
+
+ assert re.search(r"FROM flag_evaluations\b", sql) is not None
+ assert "less(cityHash64(flag_evaluations.distinct_id), 1844674407370955161)" in sql
+
def test_sessions_join_keeps_its_time_bound(self):
sql = self._sql(
f"SELECT session.$session_duration FROM events WHERE {MIXED} "
@@ -921,6 +1108,47 @@ class TestFlagCallsUnionRows(ClickhouseTestMixin, APIBaseTest):
assert rows[:4] == [*history, *recent]
assert [row[:3] for row in rows[4:]] == keyless
+ def test_inline_cohort_counts_flag_calls_that_only_flag_evaluations_holds(self):
+ fresh = _create_person(distinct_ids=["fresh"], team_id=self.team.pk)
+ _create_flag_evaluations(
+ self.team.pk,
+ "fresh-flag",
+ timestamp=datetime.now(UTC) - timedelta(days=2),
+ distinct_id="fresh",
+ person_id=fresh.uuid,
+ )
+ flush_persons_and_events()
+ cohort = _performed_flag_call_cohort(self.team, "fresh-flag")
+ _set_mode(self.organization, ONLY)
+ with _enabled_flags(union=True):
+ response = execute_hogql_query(
+ f"SELECT id FROM persons WHERE id IN COHORT {cohort.pk}",
+ team=self.team,
+ modifiers=HogQLQueryModifiers(inlineCohortCalculation=InlineCohortCalculation.ALWAYS),
+ )
+
+ assert [str(row[0]) for row in response.results] == [str(fresh.uuid)]
+
+ def test_ratio_sample_keeps_the_same_distinct_ids_in_both_tables(self):
+ now = datetime.now(UTC)
+ for index in range(100):
+ distinct_id = f"sampled-{index}"
+ _create_event(event="$pageview", distinct_id=distinct_id, team=self.team, timestamp=now)
+ _create_flag_evaluations(
+ self.team.pk, "sample-flag", timestamp=now, distinct_id=distinct_id, person_id=uuid.uuid4()
+ )
+ flush_persons_and_events()
+
+ response = self._run(
+ f"SELECT event, groupUniqArray(distinct_id) FROM events SAMPLE 0.3 "
+ f"WHERE {MIXED} AND startsWith(distinct_id, 'sampled-') GROUP BY event",
+ ONLY,
+ )
+
+ sampled = {event: set(ids) for event, ids in response.results}
+ assert 0 < len(sampled["$pageview"]) < 100
+ assert sampled["$feature_flag_called"] == sampled["$pageview"]
+
@parameterized.expand(
[
(