Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): count the exposure of watched flag edits per time bucket (#125728)

Gustavo H. Strassburger1e997a3d2b8a
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txt
index 23dcad17a61b..02e629057478 100644
--- a/.github/new-events-schema-targets.txt
+++ b/.github/new-events-schema-targets.txt
@@ -74,6 +74,7 @@ products/batch_exports/backend/tests/temporal/test_filters.py
products/batch_exports/backend/tests/temporal/test_record_batch_model.py
products/batch_exports/backend/tests/test_custom_events_schema.py
products/batch_exports/backend/tests/test_hogql_source.py
+products/feature_flags/backend/test/test_flag_windows_exposure.py
products/feature_flags/backend/test/test_request_usage.py
# Other modules that own a *.new_events_schema.ambr snapshot, so the snapshot bot keeps it current
diff --git a/products/feature_flags/backend/flag_windows/__init__.py b/products/feature_flags/backend/flag_windows/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/feature_flags/backend/flag_windows/exposure.py b/products/feature_flags/backend/flag_windows/exposure.py
new file mode 100644
index 000000000000..5aa11375ae06
--- /dev/null
+++ b/products/feature_flags/backend/flag_windows/exposure.py
@@ -0,0 +1,248 @@
+"""The exposure curve of a flag edit: in each time bucket, how many people carry the flag, and how many of them get the
+value that the edit started serving.
+
+Flag rollback suggestions read the curve as the clock of an observation window. The new value reaches people only when
+their app reloads flags, so the share of people with the new value rises over time and then stabilizes. The window
+compares errors against this curve, not against the edit time.
+
+- Only people whose events carry `$feature/<key>` count, because the value of the others is unknown. An edit that
+ enables the flag also counts the people whose events carry `$active_feature_flags`: a disabled flag is not in the
+ `/flags` response, so no event carried `$feature/<key>` before the edit. The new events schema drops
+ `$active_feature_flags` and checks for any `$feature/*` key instead, false values included, so there an app that
+ loaded no flags at all does not count. The legacy events table counts it when the app sent an empty
+ `$active_feature_flags`.
+- A group flag counts groups, because it assigns whole groups at once. One large customer would otherwise meet the
+ minimum volume by itself.
+- `$feature/<key>` holds the variant key on a multivariate flag, so the exposed people there are the people with any
+ value other than false.
+- Each bucket also holds the counts since the edit. Per-bucket counts can't be added up, because a person can be in many
+ buckets. The query keeps a `uniqState` per bucket and merges the states of the buckets so far, in the same scan.
+- `uniq` is exact up to about 65,536 people per count and estimates above that. Each count is a separate estimate, so
+ on a large flag `exposed` can be a little above `with_flag`. `uniqExact` would keep every person id of every count of
+ every bucket in memory, in a query that reads every event of the team.
+
+One query per team reads every watched edit, because `$feature/<key>` is on every event: a query per edit would read
+the same events again.
+"""
+
+import math
+from collections.abc import Mapping, Sequence
+from datetime import datetime, timedelta
+
+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
+
+from posthog.hogql import ast
+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, HogQLGlobalSettings
+from posthog.hogql.parser import parse_expr
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.query_tagging import Feature, Product, tags_context
+from posthog.dataclasses import frozen
+from posthog.models import Team
+
+# The health check query settings (posthog/temporal/health_checks/query.py). Nobody waits for the result, so the scan of
+# every event of a team leaves CPU to the queries people wait for.
+QUERY_TIMEOUT_SECONDS = 30
+QUERY_MAX_THREADS = 2
+
+IN_BUCKET = "_finalizeAggregation({state})"
+SINCE_EDIT = "uniqMerge({state}) OVER (ORDER BY bucket ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)"
+
+
+@frozen
+class WatchedEdit:
+ """A flag edit that raised exposure, as the exposure query reads it."""
+
+ # Names the edit in the result, for example the observation window or the activity log entry.
+ id: str
+ flag_key: str
+ # Counts start here, because events before the edit carry the old value.
+ edited_at: datetime
+ # The group type that the flag aggregates by, or None for a person flag.
+ group_type_index: int | None
+ # The variants that the flag serves after the edit (`ExposureChange.served_variants`). Empty for a boolean flag.
+ variants: tuple[str, ...]
+ # The edit enabled the flag, so no event carried the flag before it.
+ enables: bool
+
+
+@frozen
+class ExposureCounts:
+ # The people (or groups) whose events carry the flag, or that loaded flags when the edit enables the flag.
+ with_flag: int
+ # The people (or groups) with a value other than false.
+ exposed: int
+ # The people (or groups) with each variant that the flag serves. Empty for a boolean flag.
+ by_variant: Mapping[str, int]
+
+
+@frozen
+class ExposureBucket:
+ start: datetime
+ in_bucket: ExposureCounts
+ # From the edit, or from the range start when it is later, to the end of this bucket. Each person counts once.
+ since_edit: ExposureCounts
+
+
+def exposure_by_bucket(
+ team: Team, edits: Sequence[WatchedEdit], start: datetime, end: datetime, bucket: timedelta
+) -> dict[str, list[ExposureBucket]]:
+ """The exposure of each edit in each bucket of `[start, end)`, by edit id, oldest bucket first. A bucket in which
+ nobody carried the flag after the edit has no entry. A person whose value changed counts under both values."""
+ seconds = bucket.total_seconds()
+ if seconds < 1 or seconds != int(seconds):
+ raise ValueError(f"A bucket must be a whole number of seconds, got {bucket}")
+ if end <= start:
+ raise ValueError(f"The range must end after it starts, got {start} to {end}")
+ buckets = math.ceil((end - start) / bucket)
+ # HogQL caps the rows of a query, and a capped result would drop the newest buckets without an error.
+ if buckets > MAX_SELECT_RETURNED_ROWS:
+ raise ValueError(f"The range holds {buckets} buckets, more than {MAX_SELECT_RETURNED_ROWS}")
+ if len({edit.id for edit in edits}) != len(edits):
+ raise ValueError("Each watched edit needs its own id")
+ if not edits:
+ return {}
+ # Buckets start at `start`, not at a multiple of the bucket since the epoch, so a range that starts at an edit
+ # time does not begin with a partial bucket.
+ bucket_start = parse_expr(
+ "toStartOfInterval(timestamp, toIntervalSecond({seconds}), {start})",
+ placeholders={"seconds": ast.Constant(value=int(seconds)), "start": ast.Constant(value=start)},
+ )
+ states_by_edit = [_edit_states(index, edit) for index, edit in enumerate(edits)]
+ states = [state for edit_states in states_by_edit for state in edit_states]
+ per_bucket = ast.SelectQuery(
+ select=[ast.Alias(alias="bucket", expr=bucket_start), *states],
+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),
+ where=parse_expr(
+ "timestamp >= {start} AND timestamp < {end}",
+ placeholders={"start": ast.Constant(value=start), "end": ast.Constant(value=end)},
+ ),
+ group_by=[ast.Field(chain=["bucket"])],
+ )
+ query = ast.SelectQuery(
+ select=[
+ ast.Field(chain=["bucket"]),
+ *[
+ parse_expr(template, placeholders={"state": ast.Field(chain=[state.alias])})
+ for state in states
+ for template in (IN_BUCKET, SINCE_EDIT)
+ ],
+ ],
+ select_from=ast.JoinExpr(table=per_bucket),
+ order_by=[ast.OrderExpr(expr=ast.Field(chain=["bucket"]), order="ASC")],
+ # One row per bucket. HogQL adds LIMIT 100 to a query without one.
+ limit=ast.Constant(value=buckets),
+ )
+ with tags_context(
+ product=Product.FEATURE_FLAGS, feature=Feature.HEALTH_CHECK, team_id=team.pk, org_id=team.organization_id
+ ):
+ response = execute_hogql_query(
+ query,
+ team=team,
+ workload=Workload.OFFLINE,
+ # The join that applies person merges costs seconds on large teams and moves the counts by a tiny
+ # fraction, because a merge moves few people.
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ # A query that breaks on timeout returns partial counts, which would read as a lower exposure.
+ settings=HogQLGlobalSettings(
+ max_execution_time=QUERY_TIMEOUT_SECONDS,
+ max_threads=QUERY_MAX_THREADS,
+ timeout_overflow_mode="throw",
+ ),
+ )
+
+ result: dict[str, list[ExposureBucket]] = {edit.id: [] for edit in edits}
+ for row in response.results or []:
+ values = iter(row[1:])
+ for edit, edit_states in zip(edits, states_by_edit):
+ # Each count gives the bucket's own value, then the value since the edit.
+ pairs = [(int(next(values)), int(next(values))) for _ in edit_states]
+ in_bucket = _counts(edit, [pair[0] for pair in pairs])
+ since_edit = _counts(edit, [pair[1] for pair in pairs])
+ if in_bucket.with_flag:
+ result[edit.id].append(ExposureBucket(start=row[0], in_bucket=in_bucket, since_edit=since_edit))
+ return result
+
+
+def _counts(edit: WatchedEdit, values: list[int]) -> ExposureCounts:
+ with_flag, exposed, *by_variant = values
+ return ExposureCounts(with_flag=with_flag, exposed=exposed, by_variant=dict(zip(edit.variants, by_variant)))
+
+
+def _edit_states(index: int, edit: WatchedEdit) -> list[ast.Alias]:
+ """The `uniqState` of each of the edit's counts: with the flag, exposed, then one per variant in the order of
+ `edit.variants`."""
+ if edit.group_type_index is None:
+ actor = ast.Field(chain=["person_id"])
+ has_actor: ast.Expr = ast.Constant(value=True)
+ else:
+ actor = ast.Field(chain=[f"$group_{edit.group_type_index}"])
+ has_actor = parse_expr("notEmpty({actor})", placeholders={"actor": actor})
+ # A flag key can hold any character, so it goes in as an AST node, never as HogQL text.
+ property_name = f"$feature/{edit.flag_key}"
+ flag_value = ast.Call(name="toString", args=[ast.Field(chain=["properties", property_name])])
+ carries_flag = parse_expr("isNotNull({flag_value})", placeholders={"flag_value": flag_value})
+ if edit.enables:
+ # JSONHas is the presence check that both events schemas translate. Server SDKs send `$feature/<key>` without
+ # `$active_feature_flags`, so the people who carry the flag count too.
+ with_flag = parse_expr(
+ "JSONHas(properties, '$active_feature_flags') OR {carries_flag}",
+ placeholders={"carries_flag": carries_flag},
+ )
+ else:
+ with_flag = carries_flag
+ # A split under 100% serves true to a person who matches the condition but draws no variant, so on a multivariate
+ # flag every value but false counts as exposed.
+ variant_key: ast.Expr = flag_value
+ if {"true", "false"} & set(edit.variants):
+ # The plain property read gives a variant named "true" or "false" the same value as a boolean flag. The JSON
+ # functions keep the type on both events schemas, so a variant is a JSON string. On the legacy events table
+ # they parse the whole `properties` column, so only these edits pay for them.
+ variant_key = parse_expr(
+ "if(JSONType(properties, {name}) = 'String', JSONExtractString(properties, {name}), '')",
+ placeholders={"name": ast.Constant(value=property_name)},
+ )
+ exposed = parse_expr(
+ "{variant_key} != '' OR {flag_value} = 'true'",
+ placeholders={"variant_key": variant_key, "flag_value": flag_value},
+ )
+ elif edit.variants:
+ # isNotNull, because HogQL reads a missing flag as not equal to 'false'.
+ exposed = parse_expr(
+ "{carries_flag} AND {flag_value} != 'false'",
+ placeholders={"carries_flag": carries_flag, "flag_value": flag_value},
+ )
+ else:
+ exposed = parse_expr("{flag_value} = 'true'", placeholders={"flag_value": flag_value})
+
+ def count(alias: str, condition: ast.Expr) -> ast.Alias:
+ return ast.Alias(
+ alias=f"e{index}_{alias}",
+ expr=parse_expr(
+ "uniqState(if({has_actor} AND timestamp >= {edited_at} AND {condition}, {actor}, NULL))",
+ placeholders={
+ "actor": actor,
+ "has_actor": has_actor,
+ "edited_at": ast.Constant(value=edit.edited_at),
+ "condition": condition,
+ },
+ ),
+ )
+
+ return [
+ count("with_flag", with_flag),
+ count("exposed", exposed),
+ *[
+ count(
+ f"variant_{position}",
+ parse_expr(
+ "{variant_key} = {variant}",
+ placeholders={"variant_key": variant_key, "variant": ast.Constant(value=variant)},
+ ),
+ )
+ for position, variant in enumerate(edit.variants)
+ ],
+ ]
diff --git a/products/feature_flags/backend/test/test_flag_windows_exposure.py b/products/feature_flags/backend/test/test_flag_windows_exposure.py
new file mode 100644
index 000000000000..c8bcd9696b10
--- /dev/null
+++ b/products/feature_flags/backend/test/test_flag_windows_exposure.py
@@ -0,0 +1,272 @@
+import dataclasses
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS
+
+from posthog.models import Team
+from posthog.test.persons import create_person
+
+from products.feature_flags.backend.flag_windows.exposure import (
+ ExposureBucket,
+ ExposureCounts,
+ WatchedEdit,
+ exposure_by_bucket,
+)
+
+START = datetime(2026, 10, 1, 12, 0, tzinfo=UTC)
+BUCKET = timedelta(minutes=10)
+
+BOOLEAN = WatchedEdit(
+ id="boolean", flag_key="new-checkout", edited_at=START, group_type_index=None, variants=(), enables=False
+)
+MULTIVARIATE = WatchedEdit(
+ id="multivariate",
+ flag_key="pricing-test",
+ edited_at=START,
+ group_type_index=None,
+ variants=("control", "test"),
+ enables=False,
+)
+# A variant may be named "false", and only its JSON type tells it apart from a flag that evaluated to false.
+FALSE_VARIANT = WatchedEdit(
+ id="false-variant",
+ flag_key="pricing-test",
+ edited_at=START,
+ group_type_index=None,
+ variants=("control", "false"),
+ enables=False,
+)
+GROUP = WatchedEdit(id="group", flag_key="org-beta", edited_at=START, group_type_index=0, variants=(), enables=False)
+ENABLED = WatchedEdit(
+ id="enabled", flag_key="new-checkout", edited_at=START, group_type_index=None, variants=(), enables=True
+)
+
+# (distinct id, minutes after START, event properties)
+Event = tuple[str, int, dict[str, Any]]
+
+
+class TestExposureByBucket(ClickhouseTestMixin, BaseTest):
+ def _send(self, events: list[Event]) -> None:
+ for distinct_id in dict.fromkeys(distinct_id for distinct_id, _, _ in events):
+ create_person(team=self.team, distinct_ids=[distinct_id])
+ for distinct_id, minutes, properties in events:
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id=distinct_id,
+ timestamp=START + timedelta(minutes=minutes),
+ properties=properties,
+ )
+ flush_persons_and_events()
+
+ @parameterized.expand(
+ [
+ (
+ "boolean_flag_counts_people_with_true_among_people_with_the_flag",
+ BOOLEAN,
+ [
+ ("person-1", 1, {"$feature/new-checkout": True}),
+ ("person-1", 2, {"$feature/new-checkout": True}),
+ ("person-2", 3, {"$feature/new-checkout": False}),
+ ("person-3", 4, {"$feature/other-flag": True}),
+ ],
+ [(0, 2, 1, {})],
+ ),
+ (
+ "multivariate_flag_counts_any_variant_and_each_variant",
+ MULTIVARIATE,
+ [
+ ("person-1", 1, {"$feature/pricing-test": "control"}),
+ ("person-2", 2, {"$feature/pricing-test": "test"}),
+ ("person-3", 3, {"$feature/pricing-test": "test"}),
+ ("person-4", 4, {"$feature/pricing-test": False}),
+ # A split under 100% serves true to a person who matches but draws no variant.
+ ("person-5", 5, {"$feature/pricing-test": True}),
+ ],
+ [(0, 5, 4, {"control": 1, "test": 2})],
+ ),
+ (
+ "a_variant_named_false_counts_apart_from_a_flag_that_evaluated_to_false",
+ FALSE_VARIANT,
+ [
+ ("person-1", 1, {"$feature/pricing-test": "control"}),
+ ("person-2", 2, {"$feature/pricing-test": "false"}),
+ ("person-3", 3, {"$feature/pricing-test": False}),
+ ("person-4", 4, {"$feature/pricing-test": True}),
+ ],
+ [(0, 4, 3, {"control": 1, "false": 1})],
+ ),
+ (
+ "group_flag_counts_groups_not_people",
+ GROUP,
+ [
+ ("person-1", 1, {"$feature/org-beta": True, "$group_0": "org-a"}),
+ ("person-2", 2, {"$feature/org-beta": True, "$group_0": "org-a"}),
+ ("person-3", 3, {"$feature/org-beta": False, "$group_0": "org-b"}),
+ ("person-4", 4, {"$feature/org-beta": True}),
+ ],
+ [(0, 2, 1, {})],
+ ),
+ (
+ "enabling_edit_counts_people_that_loaded_flags",
+ ENABLED,
+ [
+ ("person-1", 1, {"$active_feature_flags": ["new-checkout"], "$feature/new-checkout": True}),
+ ("person-2", 2, {"$active_feature_flags": ["other-flag"], "$feature/other-flag": True}),
+ ("person-3", 3, {}),
+ # An app whose flags all evaluated to false still loaded flags.
+ ("person-4", 4, {"$active_feature_flags": [], "$feature/other-flag": False}),
+ # Server SDKs send the flag without `$active_feature_flags`.
+ ("person-5", 5, {"$feature/new-checkout": True}),
+ ],
+ [(0, 4, 2, {})],
+ ),
+ (
+ "each_bucket_counts_its_own_events_and_a_bucket_without_the_flag_has_no_entry",
+ BOOLEAN,
+ [
+ ("person-1", 1, {"$feature/new-checkout": False}),
+ ("person-2", 2, {"$feature/new-checkout": False}),
+ ("person-1", 21, {"$feature/new-checkout": True}),
+ ("person-3", 15, {}),
+ ],
+ [(0, 2, 0, {}), (20, 1, 1, {})],
+ ),
+ (
+ "buckets_start_at_the_window_start_so_the_last_bucket_is_kept",
+ BOOLEAN,
+ [
+ ("person-1", 4, {"$feature/new-checkout": False}),
+ ("person-1", 14, {"$feature/new-checkout": False}),
+ ("person-1", 24, {"$feature/new-checkout": False}),
+ ("person-2", 34, {"$feature/new-checkout": True}),
+ ],
+ [(3, 1, 0, {}), (13, 1, 0, {}), (23, 1, 0, {}), (33, 1, 1, {})],
+ (3, 35),
+ ),
+ ]
+ )
+ def test_exposure_by_bucket(
+ self,
+ _name: str,
+ edit: WatchedEdit,
+ events: list[Event],
+ expected: list[tuple[int, int, int, dict[str, int]]],
+ window: tuple[int, int] = (0, 30),
+ ) -> None:
+ self._send(events)
+
+ exposure = exposure_by_bucket(
+ self.team, [edit], START + timedelta(minutes=window[0]), START + timedelta(minutes=window[1]), BUCKET
+ )
+
+ self.assertEqual(
+ {
+ edit_id: [(bucket.start, bucket.in_bucket) for bucket in buckets]
+ for edit_id, buckets in exposure.items()
+ },
+ {
+ edit.id: [
+ (
+ START + timedelta(minutes=minutes),
+ ExposureCounts(with_flag=with_flag, exposed=exposed, by_variant=by_variant),
+ )
+ for minutes, with_flag, exposed, by_variant in expected
+ ]
+ },
+ )
+
+ def test_counts_since_the_edit_hold_each_person_once_and_skip_events_before_it(self) -> None:
+ self._send(
+ [
+ ("person-1", 5, {"$feature/new-checkout": True}),
+ ("person-1", 12, {"$feature/new-checkout": True}),
+ ("person-2", 15, {"$feature/new-checkout": False}),
+ ("person-1", 22, {"$feature/new-checkout": True}),
+ ("person-3", 25, {"$feature/new-checkout": True}),
+ ]
+ )
+ edit = dataclasses.replace(BOOLEAN, edited_at=START + timedelta(minutes=10))
+
+ exposure = exposure_by_bucket(self.team, [edit], START, START + timedelta(minutes=30), BUCKET)
+
+ self.assertEqual(
+ exposure,
+ {
+ edit.id: [
+ ExposureBucket(
+ start=START + timedelta(minutes=10),
+ in_bucket=ExposureCounts(with_flag=2, exposed=1, by_variant={}),
+ since_edit=ExposureCounts(with_flag=2, exposed=1, by_variant={}),
+ ),
+ ExposureBucket(
+ start=START + timedelta(minutes=20),
+ in_bucket=ExposureCounts(with_flag=2, exposed=2, by_variant={}),
+ since_edit=ExposureCounts(with_flag=3, exposed=2, by_variant={}),
+ ),
+ ]
+ },
+ )
+
+ def test_one_query_keeps_the_counts_of_each_edit_apart(self) -> None:
+ self._send(
+ [
+ ("person-1", 1, {"$feature/new-checkout": True, "$feature/pricing-test": "control"}),
+ ("person-2", 2, {"$feature/new-checkout": False}),
+ ("person-3", 3, {"$feature/pricing-test": "test"}),
+ ("person-4", 15, {"$feature/new-checkout": True}),
+ ("person-5", 40, {"$feature/new-checkout": True}),
+ ]
+ )
+
+ boolean = dataclasses.replace(BOOLEAN, edited_at=START + timedelta(minutes=10))
+
+ exposure = exposure_by_bucket(self.team, [MULTIVARIATE, boolean], START, START + timedelta(minutes=30), BUCKET)
+
+ self.assertEqual(
+ exposure,
+ {
+ "boolean": [
+ ExposureBucket(
+ start=START + timedelta(minutes=10),
+ in_bucket=ExposureCounts(with_flag=1, exposed=1, by_variant={}),
+ since_edit=ExposureCounts(with_flag=1, exposed=1, by_variant={}),
+ ),
+ ],
+ "multivariate": [
+ ExposureBucket(
+ start=START,
+ in_bucket=ExposureCounts(with_flag=2, exposed=2, by_variant={"control": 1, "test": 1}),
+ since_edit=ExposureCounts(with_flag=2, exposed=2, by_variant={"control": 1, "test": 1}),
+ )
+ ],
+ },
+ )
+
+
+class TestExposureByBucketInput(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("fractional_bucket", START + timedelta(minutes=30), timedelta(seconds=1.5), [BOOLEAN]),
+ (
+ "more_buckets_than_hogql_returns",
+ START + timedelta(seconds=MAX_SELECT_RETURNED_ROWS + 1),
+ timedelta(seconds=1),
+ [BOOLEAN],
+ ),
+ ("duplicate_edit_ids", START + timedelta(minutes=30), BUCKET, [BOOLEAN, BOOLEAN]),
+ ("end_not_after_start", START, BUCKET, [BOOLEAN]),
+ ]
+ )
+ def test_refuses_input_that_would_lose_or_mix_counts(
+ self, _name: str, end: datetime, bucket: timedelta, edits: list[WatchedEdit]
+ ) -> None:
+ with self.assertRaises(ValueError):
+ exposure_by_bucket(Team(), edits, START, end, bucket)