:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): count the exposure of watched flag edits per time bucket (#125728)
Gustavo H. Strassburger1e997a3d2b8a
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txtindex 23dcad17a61b..02e629057478 100644--- a/.github/new-events-schema-targets.txt+++ b/.github/new-events-schema-targets.txt@@ -74,6 +74,7 @@ products/batch_exports/backend/tests/temporal/test_filters.pyproducts/batch_exports/backend/tests/temporal/test_record_batch_model.pyproducts/batch_exports/backend/tests/test_custom_events_schema.pyproducts/batch_exports/backend/tests/test_hogql_source.py+products/feature_flags/backend/test/test_flag_windows_exposure.pyproducts/feature_flags/backend/test/test_request_usage.py# Other modules that own a *.new_events_schema.ambr snapshot, so the snapshot bot keeps it currentdiff --git a/products/feature_flags/backend/flag_windows/__init__.py b/products/feature_flags/backend/flag_windows/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/feature_flags/backend/flag_windows/exposure.py b/products/feature_flags/backend/flag_windows/exposure.pynew file mode 100644index 000000000000..5aa11375ae06--- /dev/null+++ b/products/feature_flags/backend/flag_windows/exposure.py@@ -0,0 +1,248 @@+"""The exposure curve of a flag edit: in each time bucket, how many people carry the flag, and how many of them get the+value that the edit started serving.++Flag rollback suggestions read the curve as the clock of an observation window. The new value reaches people only when+their app reloads flags, so the share of people with the new value rises over time and then stabilizes. The window+compares errors against this curve, not against the edit time.++- Only people whose events carry `$feature/<key>` count, because the value of the others is unknown. An edit that+ enables the flag also counts the people whose events carry `$active_feature_flags`: a disabled flag is not in the+ `/flags` response, so no event carried `$feature/<key>` before the edit. The new events schema drops+ `$active_feature_flags` and checks for any `$feature/*` key instead, false values included, so there an app that+ loaded no flags at all does not count. The legacy events table counts it when the app sent an empty+ `$active_feature_flags`.+- A group flag counts groups, because it assigns whole groups at once. One large customer would otherwise meet the+ minimum volume by itself.+- `$feature/<key>` holds the variant key on a multivariate flag, so the exposed people there are the people with any+ value other than false.+- Each bucket also holds the counts since the edit. Per-bucket counts can't be added up, because a person can be in many+ buckets. The query keeps a `uniqState` per bucket and merges the states of the buckets so far, in the same scan.+- `uniq` is exact up to about 65,536 people per count and estimates above that. Each count is a separate estimate, so+ on a large flag `exposed` can be a little above `with_flag`. `uniqExact` would keep every person id of every count of+ every bucket in memory, in a query that reads every event of the team.++One query per team reads every watched edit, because `$feature/<key>` is on every event: a query per edit would read+the same events again.+"""++import math+from collections.abc import Mapping, Sequence+from datetime import datetime, timedelta++from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode++from posthog.hogql import ast+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, HogQLGlobalSettings+from posthog.hogql.parser import parse_expr+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.client.connection import Workload+from posthog.clickhouse.query_tagging import Feature, Product, tags_context+from posthog.dataclasses import frozen+from posthog.models import Team++# The health check query settings (posthog/temporal/health_checks/query.py). Nobody waits for the result, so the scan of+# every event of a team leaves CPU to the queries people wait for.+QUERY_TIMEOUT_SECONDS = 30+QUERY_MAX_THREADS = 2++IN_BUCKET = "_finalizeAggregation({state})"+SINCE_EDIT = "uniqMerge({state}) OVER (ORDER BY bucket ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)"+++@frozen+class WatchedEdit:+ """A flag edit that raised exposure, as the exposure query reads it."""++ # Names the edit in the result, for example the observation window or the activity log entry.+ id: str+ flag_key: str+ # Counts start here, because events before the edit carry the old value.+ edited_at: datetime+ # The group type that the flag aggregates by, or None for a person flag.+ group_type_index: int | None+ # The variants that the flag serves after the edit (`ExposureChange.served_variants`). Empty for a boolean flag.+ variants: tuple[str, ...]+ # The edit enabled the flag, so no event carried the flag before it.+ enables: bool+++@frozen+class ExposureCounts:+ # The people (or groups) whose events carry the flag, or that loaded flags when the edit enables the flag.+ with_flag: int+ # The people (or groups) with a value other than false.+ exposed: int+ # The people (or groups) with each variant that the flag serves. Empty for a boolean flag.+ by_variant: Mapping[str, int]+++@frozen+class ExposureBucket:+ start: datetime+ in_bucket: ExposureCounts+ # From the edit, or from the range start when it is later, to the end of this bucket. Each person counts once.+ since_edit: ExposureCounts+++def exposure_by_bucket(+ team: Team, edits: Sequence[WatchedEdit], start: datetime, end: datetime, bucket: timedelta+) -> dict[str, list[ExposureBucket]]:+ """The exposure of each edit in each bucket of `[start, end)`, by edit id, oldest bucket first. A bucket in which+ nobody carried the flag after the edit has no entry. A person whose value changed counts under both values."""+ seconds = bucket.total_seconds()+ if seconds < 1 or seconds != int(seconds):+ raise ValueError(f"A bucket must be a whole number of seconds, got {bucket}")+ if end <= start:+ raise ValueError(f"The range must end after it starts, got {start} to {end}")+ buckets = math.ceil((end - start) / bucket)+ # HogQL caps the rows of a query, and a capped result would drop the newest buckets without an error.+ if buckets > MAX_SELECT_RETURNED_ROWS:+ raise ValueError(f"The range holds {buckets} buckets, more than {MAX_SELECT_RETURNED_ROWS}")+ if len({edit.id for edit in edits}) != len(edits):+ raise ValueError("Each watched edit needs its own id")+ if not edits:+ return {}+ # Buckets start at `start`, not at a multiple of the bucket since the epoch, so a range that starts at an edit+ # time does not begin with a partial bucket.+ bucket_start = parse_expr(+ "toStartOfInterval(timestamp, toIntervalSecond({seconds}), {start})",+ placeholders={"seconds": ast.Constant(value=int(seconds)), "start": ast.Constant(value=start)},+ )+ states_by_edit = [_edit_states(index, edit) for index, edit in enumerate(edits)]+ states = [state for edit_states in states_by_edit for state in edit_states]+ per_bucket = ast.SelectQuery(+ select=[ast.Alias(alias="bucket", expr=bucket_start), *states],+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),+ where=parse_expr(+ "timestamp >= {start} AND timestamp < {end}",+ placeholders={"start": ast.Constant(value=start), "end": ast.Constant(value=end)},+ ),+ group_by=[ast.Field(chain=["bucket"])],+ )+ query = ast.SelectQuery(+ select=[+ ast.Field(chain=["bucket"]),+ *[+ parse_expr(template, placeholders={"state": ast.Field(chain=[state.alias])})+ for state in states+ for template in (IN_BUCKET, SINCE_EDIT)+ ],+ ],+ select_from=ast.JoinExpr(table=per_bucket),+ order_by=[ast.OrderExpr(expr=ast.Field(chain=["bucket"]), order="ASC")],+ # One row per bucket. HogQL adds LIMIT 100 to a query without one.+ limit=ast.Constant(value=buckets),+ )+ with tags_context(+ product=Product.FEATURE_FLAGS, feature=Feature.HEALTH_CHECK, team_id=team.pk, org_id=team.organization_id+ ):+ response = execute_hogql_query(+ query,+ team=team,+ workload=Workload.OFFLINE,+ # The join that applies person merges costs seconds on large teams and moves the counts by a tiny+ # fraction, because a merge moves few people.+ modifiers=HogQLQueryModifiers(+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS+ ),+ # A query that breaks on timeout returns partial counts, which would read as a lower exposure.+ settings=HogQLGlobalSettings(+ max_execution_time=QUERY_TIMEOUT_SECONDS,+ max_threads=QUERY_MAX_THREADS,+ timeout_overflow_mode="throw",+ ),+ )++ result: dict[str, list[ExposureBucket]] = {edit.id: [] for edit in edits}+ for row in response.results or []:+ values = iter(row[1:])+ for edit, edit_states in zip(edits, states_by_edit):+ # Each count gives the bucket's own value, then the value since the edit.+ pairs = [(int(next(values)), int(next(values))) for _ in edit_states]+ in_bucket = _counts(edit, [pair[0] for pair in pairs])+ since_edit = _counts(edit, [pair[1] for pair in pairs])+ if in_bucket.with_flag:+ result[edit.id].append(ExposureBucket(start=row[0], in_bucket=in_bucket, since_edit=since_edit))+ return result+++def _counts(edit: WatchedEdit, values: list[int]) -> ExposureCounts:+ with_flag, exposed, *by_variant = values+ return ExposureCounts(with_flag=with_flag, exposed=exposed, by_variant=dict(zip(edit.variants, by_variant)))+++def _edit_states(index: int, edit: WatchedEdit) -> list[ast.Alias]:+ """The `uniqState` of each of the edit's counts: with the flag, exposed, then one per variant in the order of+ `edit.variants`."""+ if edit.group_type_index is None:+ actor = ast.Field(chain=["person_id"])+ has_actor: ast.Expr = ast.Constant(value=True)+ else:+ actor = ast.Field(chain=[f"$group_{edit.group_type_index}"])+ has_actor = parse_expr("notEmpty({actor})", placeholders={"actor": actor})+ # A flag key can hold any character, so it goes in as an AST node, never as HogQL text.+ property_name = f"$feature/{edit.flag_key}"+ flag_value = ast.Call(name="toString", args=[ast.Field(chain=["properties", property_name])])+ carries_flag = parse_expr("isNotNull({flag_value})", placeholders={"flag_value": flag_value})+ if edit.enables:+ # JSONHas is the presence check that both events schemas translate. Server SDKs send `$feature/<key>` without+ # `$active_feature_flags`, so the people who carry the flag count too.+ with_flag = parse_expr(+ "JSONHas(properties, '$active_feature_flags') OR {carries_flag}",+ placeholders={"carries_flag": carries_flag},+ )+ else:+ with_flag = carries_flag+ # A split under 100% serves true to a person who matches the condition but draws no variant, so on a multivariate+ # flag every value but false counts as exposed.+ variant_key: ast.Expr = flag_value+ if {"true", "false"} & set(edit.variants):+ # The plain property read gives a variant named "true" or "false" the same value as a boolean flag. The JSON+ # functions keep the type on both events schemas, so a variant is a JSON string. On the legacy events table+ # they parse the whole `properties` column, so only these edits pay for them.+ variant_key = parse_expr(+ "if(JSONType(properties, {name}) = 'String', JSONExtractString(properties, {name}), '')",+ placeholders={"name": ast.Constant(value=property_name)},+ )+ exposed = parse_expr(+ "{variant_key} != '' OR {flag_value} = 'true'",+ placeholders={"variant_key": variant_key, "flag_value": flag_value},+ )+ elif edit.variants:+ # isNotNull, because HogQL reads a missing flag as not equal to 'false'.+ exposed = parse_expr(+ "{carries_flag} AND {flag_value} != 'false'",+ placeholders={"carries_flag": carries_flag, "flag_value": flag_value},+ )+ else:+ exposed = parse_expr("{flag_value} = 'true'", placeholders={"flag_value": flag_value})++ def count(alias: str, condition: ast.Expr) -> ast.Alias:+ return ast.Alias(+ alias=f"e{index}_{alias}",+ expr=parse_expr(+ "uniqState(if({has_actor} AND timestamp >= {edited_at} AND {condition}, {actor}, NULL))",+ placeholders={+ "actor": actor,+ "has_actor": has_actor,+ "edited_at": ast.Constant(value=edit.edited_at),+ "condition": condition,+ },+ ),+ )++ return [+ count("with_flag", with_flag),+ count("exposed", exposed),+ *[+ count(+ f"variant_{position}",+ parse_expr(+ "{variant_key} = {variant}",+ placeholders={"variant_key": variant_key, "variant": ast.Constant(value=variant)},+ ),+ )+ for position, variant in enumerate(edit.variants)+ ],+ ]diff --git a/products/feature_flags/backend/test/test_flag_windows_exposure.py b/products/feature_flags/backend/test/test_flag_windows_exposure.pynew file mode 100644index 000000000000..c8bcd9696b10--- /dev/null+++ b/products/feature_flags/backend/test/test_flag_windows_exposure.py@@ -0,0 +1,272 @@+import dataclasses+from datetime import UTC, datetime, timedelta+from typing import Any++from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS++from posthog.models import Team+from posthog.test.persons import create_person++from products.feature_flags.backend.flag_windows.exposure import (+ ExposureBucket,+ ExposureCounts,+ WatchedEdit,+ exposure_by_bucket,+)++START = datetime(2026, 10, 1, 12, 0, tzinfo=UTC)+BUCKET = timedelta(minutes=10)++BOOLEAN = WatchedEdit(+ id="boolean", flag_key="new-checkout", edited_at=START, group_type_index=None, variants=(), enables=False+)+MULTIVARIATE = WatchedEdit(+ id="multivariate",+ flag_key="pricing-test",+ edited_at=START,+ group_type_index=None,+ variants=("control", "test"),+ enables=False,+)+# A variant may be named "false", and only its JSON type tells it apart from a flag that evaluated to false.+FALSE_VARIANT = WatchedEdit(+ id="false-variant",+ flag_key="pricing-test",+ edited_at=START,+ group_type_index=None,+ variants=("control", "false"),+ enables=False,+)+GROUP = WatchedEdit(id="group", flag_key="org-beta", edited_at=START, group_type_index=0, variants=(), enables=False)+ENABLED = WatchedEdit(+ id="enabled", flag_key="new-checkout", edited_at=START, group_type_index=None, variants=(), enables=True+)++# (distinct id, minutes after START, event properties)+Event = tuple[str, int, dict[str, Any]]+++class TestExposureByBucket(ClickhouseTestMixin, BaseTest):+ def _send(self, events: list[Event]) -> None:+ for distinct_id in dict.fromkeys(distinct_id for distinct_id, _, _ in events):+ create_person(team=self.team, distinct_ids=[distinct_id])+ for distinct_id, minutes, properties in events:+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id=distinct_id,+ timestamp=START + timedelta(minutes=minutes),+ properties=properties,+ )+ flush_persons_and_events()++ @parameterized.expand(+ [+ (+ "boolean_flag_counts_people_with_true_among_people_with_the_flag",+ BOOLEAN,+ [+ ("person-1", 1, {"$feature/new-checkout": True}),+ ("person-1", 2, {"$feature/new-checkout": True}),+ ("person-2", 3, {"$feature/new-checkout": False}),+ ("person-3", 4, {"$feature/other-flag": True}),+ ],+ [(0, 2, 1, {})],+ ),+ (+ "multivariate_flag_counts_any_variant_and_each_variant",+ MULTIVARIATE,+ [+ ("person-1", 1, {"$feature/pricing-test": "control"}),+ ("person-2", 2, {"$feature/pricing-test": "test"}),+ ("person-3", 3, {"$feature/pricing-test": "test"}),+ ("person-4", 4, {"$feature/pricing-test": False}),+ # A split under 100% serves true to a person who matches but draws no variant.+ ("person-5", 5, {"$feature/pricing-test": True}),+ ],+ [(0, 5, 4, {"control": 1, "test": 2})],+ ),+ (+ "a_variant_named_false_counts_apart_from_a_flag_that_evaluated_to_false",+ FALSE_VARIANT,+ [+ ("person-1", 1, {"$feature/pricing-test": "control"}),+ ("person-2", 2, {"$feature/pricing-test": "false"}),+ ("person-3", 3, {"$feature/pricing-test": False}),+ ("person-4", 4, {"$feature/pricing-test": True}),+ ],+ [(0, 4, 3, {"control": 1, "false": 1})],+ ),+ (+ "group_flag_counts_groups_not_people",+ GROUP,+ [+ ("person-1", 1, {"$feature/org-beta": True, "$group_0": "org-a"}),+ ("person-2", 2, {"$feature/org-beta": True, "$group_0": "org-a"}),+ ("person-3", 3, {"$feature/org-beta": False, "$group_0": "org-b"}),+ ("person-4", 4, {"$feature/org-beta": True}),+ ],+ [(0, 2, 1, {})],+ ),+ (+ "enabling_edit_counts_people_that_loaded_flags",+ ENABLED,+ [+ ("person-1", 1, {"$active_feature_flags": ["new-checkout"], "$feature/new-checkout": True}),+ ("person-2", 2, {"$active_feature_flags": ["other-flag"], "$feature/other-flag": True}),+ ("person-3", 3, {}),+ # An app whose flags all evaluated to false still loaded flags.+ ("person-4", 4, {"$active_feature_flags": [], "$feature/other-flag": False}),+ # Server SDKs send the flag without `$active_feature_flags`.+ ("person-5", 5, {"$feature/new-checkout": True}),+ ],+ [(0, 4, 2, {})],+ ),+ (+ "each_bucket_counts_its_own_events_and_a_bucket_without_the_flag_has_no_entry",+ BOOLEAN,+ [+ ("person-1", 1, {"$feature/new-checkout": False}),+ ("person-2", 2, {"$feature/new-checkout": False}),+ ("person-1", 21, {"$feature/new-checkout": True}),+ ("person-3", 15, {}),+ ],+ [(0, 2, 0, {}), (20, 1, 1, {})],+ ),+ (+ "buckets_start_at_the_window_start_so_the_last_bucket_is_kept",+ BOOLEAN,+ [+ ("person-1", 4, {"$feature/new-checkout": False}),+ ("person-1", 14, {"$feature/new-checkout": False}),+ ("person-1", 24, {"$feature/new-checkout": False}),+ ("person-2", 34, {"$feature/new-checkout": True}),+ ],+ [(3, 1, 0, {}), (13, 1, 0, {}), (23, 1, 0, {}), (33, 1, 1, {})],+ (3, 35),+ ),+ ]+ )+ def test_exposure_by_bucket(+ self,+ _name: str,+ edit: WatchedEdit,+ events: list[Event],+ expected: list[tuple[int, int, int, dict[str, int]]],+ window: tuple[int, int] = (0, 30),+ ) -> None:+ self._send(events)++ exposure = exposure_by_bucket(+ self.team, [edit], START + timedelta(minutes=window[0]), START + timedelta(minutes=window[1]), BUCKET+ )++ self.assertEqual(+ {+ edit_id: [(bucket.start, bucket.in_bucket) for bucket in buckets]+ for edit_id, buckets in exposure.items()+ },+ {+ edit.id: [+ (+ START + timedelta(minutes=minutes),+ ExposureCounts(with_flag=with_flag, exposed=exposed, by_variant=by_variant),+ )+ for minutes, with_flag, exposed, by_variant in expected+ ]+ },+ )++ def test_counts_since_the_edit_hold_each_person_once_and_skip_events_before_it(self) -> None:+ self._send(+ [+ ("person-1", 5, {"$feature/new-checkout": True}),+ ("person-1", 12, {"$feature/new-checkout": True}),+ ("person-2", 15, {"$feature/new-checkout": False}),+ ("person-1", 22, {"$feature/new-checkout": True}),+ ("person-3", 25, {"$feature/new-checkout": True}),+ ]+ )+ edit = dataclasses.replace(BOOLEAN, edited_at=START + timedelta(minutes=10))++ exposure = exposure_by_bucket(self.team, [edit], START, START + timedelta(minutes=30), BUCKET)++ self.assertEqual(+ exposure,+ {+ edit.id: [+ ExposureBucket(+ start=START + timedelta(minutes=10),+ in_bucket=ExposureCounts(with_flag=2, exposed=1, by_variant={}),+ since_edit=ExposureCounts(with_flag=2, exposed=1, by_variant={}),+ ),+ ExposureBucket(+ start=START + timedelta(minutes=20),+ in_bucket=ExposureCounts(with_flag=2, exposed=2, by_variant={}),+ since_edit=ExposureCounts(with_flag=3, exposed=2, by_variant={}),+ ),+ ]+ },+ )++ def test_one_query_keeps_the_counts_of_each_edit_apart(self) -> None:+ self._send(+ [+ ("person-1", 1, {"$feature/new-checkout": True, "$feature/pricing-test": "control"}),+ ("person-2", 2, {"$feature/new-checkout": False}),+ ("person-3", 3, {"$feature/pricing-test": "test"}),+ ("person-4", 15, {"$feature/new-checkout": True}),+ ("person-5", 40, {"$feature/new-checkout": True}),+ ]+ )++ boolean = dataclasses.replace(BOOLEAN, edited_at=START + timedelta(minutes=10))++ exposure = exposure_by_bucket(self.team, [MULTIVARIATE, boolean], START, START + timedelta(minutes=30), BUCKET)++ self.assertEqual(+ exposure,+ {+ "boolean": [+ ExposureBucket(+ start=START + timedelta(minutes=10),+ in_bucket=ExposureCounts(with_flag=1, exposed=1, by_variant={}),+ since_edit=ExposureCounts(with_flag=1, exposed=1, by_variant={}),+ ),+ ],+ "multivariate": [+ ExposureBucket(+ start=START,+ in_bucket=ExposureCounts(with_flag=2, exposed=2, by_variant={"control": 1, "test": 1}),+ since_edit=ExposureCounts(with_flag=2, exposed=2, by_variant={"control": 1, "test": 1}),+ )+ ],+ },+ )+++class TestExposureByBucketInput(SimpleTestCase):+ @parameterized.expand(+ [+ ("fractional_bucket", START + timedelta(minutes=30), timedelta(seconds=1.5), [BOOLEAN]),+ (+ "more_buckets_than_hogql_returns",+ START + timedelta(seconds=MAX_SELECT_RETURNED_ROWS + 1),+ timedelta(seconds=1),+ [BOOLEAN],+ ),+ ("duplicate_edit_ids", START + timedelta(minutes=30), BUCKET, [BOOLEAN, BOOLEAN]),+ ("end_not_after_start", START, BUCKET, [BOOLEAN]),+ ]+ )+ def test_refuses_input_that_would_lose_or_mix_counts(+ self, _name: str, end: datetime, bucket: timedelta, edits: list[WatchedEdit]+ ) -> None:+ with self.assertRaises(ValueError):+ exposure_by_bucket(Team(), edits, START, end, bucket)