Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): count errors after watched flag edits per time bucket (#125919)

Gustavo H. Strassburger88168a05e13c
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txt
index 02e629057478..40ee94b70df3 100644
--- a/.github/new-events-schema-targets.txt
+++ b/.github/new-events-schema-targets.txt
@@ -74,6 +74,7 @@ products/batch_exports/backend/tests/temporal/test_filters.py
products/batch_exports/backend/tests/temporal/test_record_batch_model.py
products/batch_exports/backend/tests/test_custom_events_schema.py
products/batch_exports/backend/tests/test_hogql_source.py
+products/feature_flags/backend/test/test_flag_windows_errors.py
products/feature_flags/backend/test/test_flag_windows_exposure.py
products/feature_flags/backend/test/test_request_usage.py
diff --git a/products/feature_flags/backend/flag_windows/errors.py b/products/feature_flags/backend/flag_windows/errors.py
new file mode 100644
index 000000000000..3f563f491bbd
--- /dev/null
+++ b/products/feature_flags/backend/flag_windows/errors.py
@@ -0,0 +1,130 @@
+"""The errors after a flag edit: in each time bucket, how many people hit an error, and how the people whose error event
+carries the flag split between the value that the edit started serving and the other values.
+
+Flag rollback suggestions attribute errors to an edit from this split and from the exposure curve:
+
+- Level 1 compares the error rate of the exposed people with the rest of the people with the flag. Only an error event
+ that carries `$feature/<key>` tells which value its person had, and client SDKs attach the flags to `$exception`,
+ but server SDKs mostly don't.
+- Levels 4 and 5 follow all errors over time. An error of an issue first seen after the edit is stronger evidence than
+ an older error that became more frequent.
+
+The query reads only `$exception` events, so it costs far less than the exposure query. `flag_windows/query.py` holds
+the counting rules.
+"""
+
+from collections.abc import Mapping, Sequence
+from datetime import datetime, timedelta
+
+from posthog.hogql import ast
+from posthog.hogql.parser import parse_expr, parse_select
+
+from posthog.dataclasses import frozen
+from posthog.models import Team
+
+from products.feature_flags.backend.flag_windows.query import FlagRead, WatchedEdit, counts_by_bucket
+
+
+@frozen
+class ErrorCounts:
+ # The people (or groups) with an error, whether or not the error event carries the flag.
+ with_errors: int
+ # The people (or groups) with an error of an issue first seen after the edit.
+ with_new_errors: int
+ # The people (or groups) whose error event carries a value other than false.
+ exposed_with_errors: int
+ # The people (or groups) whose error event carries the flag with the value false, or no value when the edit
+ # enables the flag.
+ unexposed_with_errors: int
+ # The people (or groups) whose error event carries each variant that the flag serves. Empty for a boolean flag.
+ by_variant: Mapping[str, int]
+
+
+@frozen
+class ErrorBucket:
+ start: datetime
+ in_bucket: ErrorCounts
+ # From the edit, or from the range start when it is later, to the end of this bucket. Each person counts once.
+ since_edit: ErrorCounts
+
+
+def errors_by_bucket(
+ team: Team, edits: Sequence[WatchedEdit], start: datetime, end: datetime, bucket: timedelta
+) -> dict[str, list[ErrorBucket]]:
+ """The errors after each edit in each bucket of `[start, end)`, by edit id, oldest bucket first. A bucket without
+ errors after the edit has no entry."""
+ counts = counts_by_bucket(
+ team,
+ edits,
+ _conditions,
+ start,
+ end,
+ bucket,
+ query_type="flag_errors_by_bucket",
+ event="$exception",
+ join=_issue_first_seen_join(),
+ )
+ return {
+ edit.id: [
+ ErrorBucket(
+ start=row.start,
+ in_bucket=_errors(edit, row.in_bucket),
+ since_edit=_errors(edit, row.since_edit),
+ )
+ for row in counts[edit.id]
+ if _errors(edit, row.in_bucket).with_errors
+ ]
+ for edit in edits
+ }
+
+
+def _issue_first_seen_join() -> ast.JoinExpr:
+ # Error tracking stores the first seen time per fingerprint, and a merged issue has many fingerprints. The join
+ # keeps the earliest one, so a new fingerprint does not make an old issue look new. One join serves every edit,
+ # because a subquery per edit would read all fingerprints of the team again for each edit.
+ return ast.JoinExpr(
+ join_type="LEFT JOIN",
+ table=parse_select(
+ """
+ SELECT issue_id, min(first_seen) AS first_seen
+ FROM posthog.error_tracking_fingerprint_issue_state
+ GROUP BY issue_id
+ """
+ ),
+ alias="earliest_issue_seen",
+ constraint=ast.JoinConstraint(
+ expr=parse_expr("earliest_issue_seen.issue_id = events.issue_id"), constraint_type="ON"
+ ),
+ )
+
+
+def _conditions(edit: WatchedEdit, read: FlagRead) -> list[ast.Expr]:
+ return [
+ ast.Constant(value=True),
+ # A fingerprint that isn't synced to ClickHouse yet has no issue, and counts as new. An event without a
+ # fingerprint has no issue either, for example when cymbal can't process the exception, so it doesn't count.
+ parse_expr(
+ """
+ (isNull(events.issue_id) AND isNotNull(properties.$exception_fingerprint))
+ OR earliest_issue_seen.first_seen >= {edited_at}
+ """,
+ placeholders={"edited_at": ast.Constant(value=edit.edited_at)},
+ ),
+ read.exposed,
+ parse_expr(
+ "{with_flag} AND NOT {exposed}",
+ placeholders={"with_flag": read.with_flag, "exposed": read.exposed},
+ ),
+ *[read.has_variant(variant) for variant in edit.variants],
+ ]
+
+
+def _errors(edit: WatchedEdit, values: tuple[int, ...]) -> ErrorCounts:
+ with_errors, with_new_errors, exposed_with_errors, unexposed_with_errors, *by_variant = values
+ return ErrorCounts(
+ with_errors=with_errors,
+ with_new_errors=with_new_errors,
+ exposed_with_errors=exposed_with_errors,
+ unexposed_with_errors=unexposed_with_errors,
+ by_variant=dict(zip(edit.variants, by_variant, strict=True)),
+ )
diff --git a/products/feature_flags/backend/flag_windows/exposure.py b/products/feature_flags/backend/flag_windows/exposure.py
index 5aa11375ae06..7a11808b01e8 100644
--- a/products/feature_flags/backend/flag_windows/exposure.py
+++ b/products/feature_flags/backend/flag_windows/exposure.py
@@ -5,66 +5,19 @@ Flag rollback suggestions read the curve as the clock of an observation window.
their app reloads flags, so the share of people with the new value rises over time and then stabilizes. The window
compares errors against this curve, not against the edit time.
-- Only people whose events carry `$feature/<key>` count, because the value of the others is unknown. An edit that
- enables the flag also counts the people whose events carry `$active_feature_flags`: a disabled flag is not in the
- `/flags` response, so no event carried `$feature/<key>` before the edit. The new events schema drops
- `$active_feature_flags` and checks for any `$feature/*` key instead, false values included, so there an app that
- loaded no flags at all does not count. The legacy events table counts it when the app sent an empty
- `$active_feature_flags`.
-- A group flag counts groups, because it assigns whole groups at once. One large customer would otherwise meet the
- minimum volume by itself.
-- `$feature/<key>` holds the variant key on a multivariate flag, so the exposed people there are the people with any
- value other than false.
-- Each bucket also holds the counts since the edit. Per-bucket counts can't be added up, because a person can be in many
- buckets. The query keeps a `uniqState` per bucket and merges the states of the buckets so far, in the same scan.
-- `uniq` is exact up to about 65,536 people per count and estimates above that. Each count is a separate estimate, so
- on a large flag `exposed` can be a little above `with_flag`. `uniqExact` would keep every person id of every count of
- every bucket in memory, in a query that reads every event of the team.
-
-One query per team reads every watched edit, because `$feature/<key>` is on every event: a query per edit would read
-the same events again.
+The query reads every event, because `$feature/<key>` is on every event. `flag_windows/query.py` holds the counting
+rules.
"""
-import math
from collections.abc import Mapping, Sequence
from datetime import datetime, timedelta
-from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
-
from posthog.hogql import ast
-from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, HogQLGlobalSettings
-from posthog.hogql.parser import parse_expr
-from posthog.hogql.query import execute_hogql_query
-from posthog.clickhouse.client.connection import Workload
-from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.dataclasses import frozen
from posthog.models import Team
-# The health check query settings (posthog/temporal/health_checks/query.py). Nobody waits for the result, so the scan of
-# every event of a team leaves CPU to the queries people wait for.
-QUERY_TIMEOUT_SECONDS = 30
-QUERY_MAX_THREADS = 2
-
-IN_BUCKET = "_finalizeAggregation({state})"
-SINCE_EDIT = "uniqMerge({state}) OVER (ORDER BY bucket ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)"
-
-
-@frozen
-class WatchedEdit:
- """A flag edit that raised exposure, as the exposure query reads it."""
-
- # Names the edit in the result, for example the observation window or the activity log entry.
- id: str
- flag_key: str
- # Counts start here, because events before the edit carry the old value.
- edited_at: datetime
- # The group type that the flag aggregates by, or None for a person flag.
- group_type_index: int | None
- # The variants that the flag serves after the edit (`ExposureChange.served_variants`). Empty for a boolean flag.
- variants: tuple[str, ...]
- # The edit enabled the flag, so no event carried the flag before it.
- enables: bool
+from products.feature_flags.backend.flag_windows.query import FlagRead, WatchedEdit, counts_by_bucket
@frozen
@@ -90,159 +43,27 @@ def exposure_by_bucket(
) -> dict[str, list[ExposureBucket]]:
"""The exposure of each edit in each bucket of `[start, end)`, by edit id, oldest bucket first. A bucket in which
nobody carried the flag after the edit has no entry. A person whose value changed counts under both values."""
- seconds = bucket.total_seconds()
- if seconds < 1 or seconds != int(seconds):
- raise ValueError(f"A bucket must be a whole number of seconds, got {bucket}")
- if end <= start:
- raise ValueError(f"The range must end after it starts, got {start} to {end}")
- buckets = math.ceil((end - start) / bucket)
- # HogQL caps the rows of a query, and a capped result would drop the newest buckets without an error.
- if buckets > MAX_SELECT_RETURNED_ROWS:
- raise ValueError(f"The range holds {buckets} buckets, more than {MAX_SELECT_RETURNED_ROWS}")
- if len({edit.id for edit in edits}) != len(edits):
- raise ValueError("Each watched edit needs its own id")
- if not edits:
- return {}
- # Buckets start at `start`, not at a multiple of the bucket since the epoch, so a range that starts at an edit
- # time does not begin with a partial bucket.
- bucket_start = parse_expr(
- "toStartOfInterval(timestamp, toIntervalSecond({seconds}), {start})",
- placeholders={"seconds": ast.Constant(value=int(seconds)), "start": ast.Constant(value=start)},
- )
- states_by_edit = [_edit_states(index, edit) for index, edit in enumerate(edits)]
- states = [state for edit_states in states_by_edit for state in edit_states]
- per_bucket = ast.SelectQuery(
- select=[ast.Alias(alias="bucket", expr=bucket_start), *states],
- select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),
- where=parse_expr(
- "timestamp >= {start} AND timestamp < {end}",
- placeholders={"start": ast.Constant(value=start), "end": ast.Constant(value=end)},
- ),
- group_by=[ast.Field(chain=["bucket"])],
- )
- query = ast.SelectQuery(
- select=[
- ast.Field(chain=["bucket"]),
- *[
- parse_expr(template, placeholders={"state": ast.Field(chain=[state.alias])})
- for state in states
- for template in (IN_BUCKET, SINCE_EDIT)
- ],
- ],
- select_from=ast.JoinExpr(table=per_bucket),
- order_by=[ast.OrderExpr(expr=ast.Field(chain=["bucket"]), order="ASC")],
- # One row per bucket. HogQL adds LIMIT 100 to a query without one.
- limit=ast.Constant(value=buckets),
- )
- with tags_context(
- product=Product.FEATURE_FLAGS, feature=Feature.HEALTH_CHECK, team_id=team.pk, org_id=team.organization_id
- ):
- response = execute_hogql_query(
- query,
- team=team,
- workload=Workload.OFFLINE,
- # The join that applies person merges costs seconds on large teams and moves the counts by a tiny
- # fraction, because a merge moves few people.
- modifiers=HogQLQueryModifiers(
- personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS
- ),
- # A query that breaks on timeout returns partial counts, which would read as a lower exposure.
- settings=HogQLGlobalSettings(
- max_execution_time=QUERY_TIMEOUT_SECONDS,
- max_threads=QUERY_MAX_THREADS,
- timeout_overflow_mode="throw",
- ),
- )
-
- result: dict[str, list[ExposureBucket]] = {edit.id: [] for edit in edits}
- for row in response.results or []:
- values = iter(row[1:])
- for edit, edit_states in zip(edits, states_by_edit):
- # Each count gives the bucket's own value, then the value since the edit.
- pairs = [(int(next(values)), int(next(values))) for _ in edit_states]
- in_bucket = _counts(edit, [pair[0] for pair in pairs])
- since_edit = _counts(edit, [pair[1] for pair in pairs])
- if in_bucket.with_flag:
- result[edit.id].append(ExposureBucket(start=row[0], in_bucket=in_bucket, since_edit=since_edit))
- return result
-
-
-def _counts(edit: WatchedEdit, values: list[int]) -> ExposureCounts:
- with_flag, exposed, *by_variant = values
- return ExposureCounts(with_flag=with_flag, exposed=exposed, by_variant=dict(zip(edit.variants, by_variant)))
-
-
-def _edit_states(index: int, edit: WatchedEdit) -> list[ast.Alias]:
- """The `uniqState` of each of the edit's counts: with the flag, exposed, then one per variant in the order of
- `edit.variants`."""
- if edit.group_type_index is None:
- actor = ast.Field(chain=["person_id"])
- has_actor: ast.Expr = ast.Constant(value=True)
- else:
- actor = ast.Field(chain=[f"$group_{edit.group_type_index}"])
- has_actor = parse_expr("notEmpty({actor})", placeholders={"actor": actor})
- # A flag key can hold any character, so it goes in as an AST node, never as HogQL text.
- property_name = f"$feature/{edit.flag_key}"
- flag_value = ast.Call(name="toString", args=[ast.Field(chain=["properties", property_name])])
- carries_flag = parse_expr("isNotNull({flag_value})", placeholders={"flag_value": flag_value})
- if edit.enables:
- # JSONHas is the presence check that both events schemas translate. Server SDKs send `$feature/<key>` without
- # `$active_feature_flags`, so the people who carry the flag count too.
- with_flag = parse_expr(
- "JSONHas(properties, '$active_feature_flags') OR {carries_flag}",
- placeholders={"carries_flag": carries_flag},
- )
- else:
- with_flag = carries_flag
- # A split under 100% serves true to a person who matches the condition but draws no variant, so on a multivariate
- # flag every value but false counts as exposed.
- variant_key: ast.Expr = flag_value
- if {"true", "false"} & set(edit.variants):
- # The plain property read gives a variant named "true" or "false" the same value as a boolean flag. The JSON
- # functions keep the type on both events schemas, so a variant is a JSON string. On the legacy events table
- # they parse the whole `properties` column, so only these edits pay for them.
- variant_key = parse_expr(
- "if(JSONType(properties, {name}) = 'String', JSONExtractString(properties, {name}), '')",
- placeholders={"name": ast.Constant(value=property_name)},
- )
- exposed = parse_expr(
- "{variant_key} != '' OR {flag_value} = 'true'",
- placeholders={"variant_key": variant_key, "flag_value": flag_value},
- )
- elif edit.variants:
- # isNotNull, because HogQL reads a missing flag as not equal to 'false'.
- exposed = parse_expr(
- "{carries_flag} AND {flag_value} != 'false'",
- placeholders={"carries_flag": carries_flag, "flag_value": flag_value},
- )
- else:
- exposed = parse_expr("{flag_value} = 'true'", placeholders={"flag_value": flag_value})
-
- def count(alias: str, condition: ast.Expr) -> ast.Alias:
- return ast.Alias(
- alias=f"e{index}_{alias}",
- expr=parse_expr(
- "uniqState(if({has_actor} AND timestamp >= {edited_at} AND {condition}, {actor}, NULL))",
- placeholders={
- "actor": actor,
- "has_actor": has_actor,
- "edited_at": ast.Constant(value=edit.edited_at),
- "condition": condition,
- },
- ),
- )
-
- return [
- count("with_flag", with_flag),
- count("exposed", exposed),
- *[
- count(
- f"variant_{position}",
- parse_expr(
- "{variant_key} = {variant}",
- placeholders={"variant_key": variant_key, "variant": ast.Constant(value=variant)},
- ),
+ counts = counts_by_bucket(team, edits, _conditions, start, end, bucket, query_type="flag_exposure_by_bucket")
+ return {
+ edit.id: [
+ ExposureBucket(
+ start=row.start,
+ in_bucket=_exposure(edit, row.in_bucket),
+ since_edit=_exposure(edit, row.since_edit),
)
- for position, variant in enumerate(edit.variants)
- ],
- ]
+ for row in counts[edit.id]
+ if _exposure(edit, row.in_bucket).with_flag
+ ]
+ for edit in edits
+ }
+
+
+def _conditions(edit: WatchedEdit, read: FlagRead) -> list[ast.Expr]:
+ return [read.with_flag, read.exposed, *[read.has_variant(variant) for variant in edit.variants]]
+
+
+def _exposure(edit: WatchedEdit, values: tuple[int, ...]) -> ExposureCounts:
+ with_flag, exposed, *by_variant = values
+ return ExposureCounts(
+ with_flag=with_flag, exposed=exposed, by_variant=dict(zip(edit.variants, by_variant, strict=True))
+ )
diff --git a/products/feature_flags/backend/flag_windows/query.py b/products/feature_flags/backend/flag_windows/query.py
new file mode 100644
index 000000000000..f0fa37174a78
--- /dev/null
+++ b/products/feature_flags/backend/flag_windows/query.py
@@ -0,0 +1,258 @@
+"""What the observation window queries share: the watched edit, how an event reads its flag, and the query that counts
+people per bucket and since the edit.
+
+- Only people whose events carry `$feature/<key>` count, because the value of the others is unknown. An edit that
+ enables the flag also counts the people whose events carry `$active_feature_flags`: a disabled flag is not in the
+ `/flags` response, so no event carried `$feature/<key>` before the edit. The new events schema drops
+ `$active_feature_flags` and checks for any `$feature/*` key instead, false values included, so there an app that
+ loaded no flags at all does not count. The legacy events table counts it when the app sent an empty
+ `$active_feature_flags`.
+- A group flag counts groups, because it assigns whole groups at once. One large customer would otherwise meet the
+ minimum volume by itself.
+- `$feature/<key>` holds the variant key on a multivariate flag, so the exposed people there are the people with any
+ value other than false.
+- Each bucket also holds the counts since the edit. Per-bucket counts can't be added up, because a person can be in many
+ buckets. The query keeps a `uniqState` per bucket and merges the states of the buckets so far, in the same scan.
+- `uniq` is exact up to about 65,536 people per count and estimates above that. Each count is a separate estimate, so
+ on a large flag a count of a subset can come out a little above the count of the whole, for example `exposed` above
+ `with_flag`. `uniqExact` would keep every person id of every count of every bucket in memory, in a query that reads
+ every event of the team.
+
+One query per team reads every watched edit: a query per edit would read the same events again.
+"""
+
+import math
+from collections.abc import Callable, Sequence
+from datetime import datetime, timedelta
+
+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
+
+from posthog.hogql import ast
+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, HogQLGlobalSettings
+from posthog.hogql.parser import parse_expr
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.query_tagging import Feature, Product, tags_context
+from posthog.dataclasses import frozen
+from posthog.models import Team
+
+# The health check query settings (posthog/temporal/health_checks/query.py). Nobody waits for the result, so the scan of
+# every event of a team leaves CPU to the queries people wait for.
+QUERY_TIMEOUT_SECONDS = 30
+QUERY_MAX_THREADS = 2
+
+IN_BUCKET = "_finalizeAggregation({state})"
+SINCE_EDIT = "uniqMerge({state}) OVER (ORDER BY bucket ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)"
+
+
+@frozen
+class WatchedEdit:
+ """A flag edit that raised exposure, as the window queries read it."""
+
+ # Names the edit in the result, for example the observation window or the activity log entry.
+ id: str
+ flag_key: str
+ # Counts start here, because events before the edit carry the old value.
+ edited_at: datetime
+ # The group type that the flag aggregates by, or None for a person flag.
+ group_type_index: int | None
+ # The variants that the flag serves after the edit (`ExposureChange.served_variants`). Empty for a boolean flag.
+ variants: tuple[str, ...]
+ # The edit enabled the flag, so no event carried the flag before it.
+ enables: bool
+
+
+@frozen
+class FlagRead:
+ """How an event reads the flag of a watched edit, as HogQL expressions."""
+
+ # The person, or the group for a group flag.
+ actor: ast.Expr
+ # A group flag needs the group on the event.
+ has_actor: ast.Expr
+ # The event carries the flag, or loaded flags when the edit enables the flag.
+ with_flag: ast.Expr
+ # The event carries a value other than false.
+ exposed: ast.Expr
+ # The variant key on the event. Compare it with a served variant.
+ variant_key: ast.Expr
+
+ def has_variant(self, variant: str) -> ast.Expr:
+ return parse_expr(
+ "{variant_key} = {variant}",
+ placeholders={"variant_key": self.variant_key, "variant": ast.Constant(value=variant)},
+ )
+
+
+@frozen
+class BucketCounts:
+ start: datetime
+ # One value per condition, in the order the caller gave them.
+ in_bucket: tuple[int, ...]
+ # From the edit, or from the range start when it is later, to the end of this bucket. Each person counts once.
+ since_edit: tuple[int, ...]
+
+
+def read_flag(edit: WatchedEdit) -> FlagRead:
+ if edit.group_type_index is None:
+ actor = ast.Field(chain=["person_id"])
+ has_actor: ast.Expr = ast.Constant(value=True)
+ else:
+ actor = ast.Field(chain=[f"$group_{edit.group_type_index}"])
+ has_actor = parse_expr("notEmpty({actor})", placeholders={"actor": actor})
+ # A flag key can hold any character, so it goes in as an AST node, never as HogQL text.
+ property_name = f"$feature/{edit.flag_key}"
+ flag_value = ast.Call(name="toString", args=[ast.Field(chain=["properties", property_name])])
+ carries_flag = parse_expr("isNotNull({flag_value})", placeholders={"flag_value": flag_value})
+ if edit.enables:
+ # JSONHas is the presence check that both events schemas translate. Server SDKs send `$feature/<key>` without
+ # `$active_feature_flags`, so the people who carry the flag count too.
+ with_flag = parse_expr(
+ "JSONHas(properties, '$active_feature_flags') OR {carries_flag}",
+ placeholders={"carries_flag": carries_flag},
+ )
+ else:
+ with_flag = carries_flag
+ # A split under 100% serves true to a person who matches the condition but draws no variant, so on a multivariate
+ # flag every value but false counts as exposed.
+ variant_key: ast.Expr = flag_value
+ if {"true", "false"} & set(edit.variants):
+ # The plain property read gives a variant named "true" or "false" the same value as a boolean flag. The JSON
+ # functions keep the type on both events schemas, so a variant is a JSON string. On the legacy events table
+ # they parse the whole `properties` column, so only these edits pay for them.
+ variant_key = parse_expr(
+ "if(JSONType(properties, {name}) = 'String', JSONExtractString(properties, {name}), '')",
+ placeholders={"name": ast.Constant(value=property_name)},
+ )
+ exposed = parse_expr(
+ "{variant_key} != '' OR {flag_value} = 'true'",
+ placeholders={"variant_key": variant_key, "flag_value": flag_value},
+ )
+ elif edit.variants:
+ # isNotNull, because HogQL reads a missing flag as not equal to 'false'.
+ exposed = parse_expr(
+ "{carries_flag} AND {flag_value} != 'false'",
+ placeholders={"carries_flag": carries_flag, "flag_value": flag_value},
+ )
+ else:
+ exposed = parse_expr("{flag_value} = 'true'", placeholders={"flag_value": flag_value})
+ return FlagRead(actor=actor, has_actor=has_actor, with_flag=with_flag, exposed=exposed, variant_key=variant_key)
+
+
+def counts_by_bucket(
+ team: Team,
+ edits: Sequence[WatchedEdit],
+ conditions: Callable[[WatchedEdit, FlagRead], Sequence[ast.Expr]],
+ start: datetime,
+ end: datetime,
+ bucket: timedelta,
+ query_type: str,
+ event: str | None = None,
+ join: ast.JoinExpr | None = None,
+) -> dict[str, list[BucketCounts]]:
+ """For each edit, the people (or groups) that meet each of its conditions after the edit, in each bucket of
+ `[start, end)` that has events, oldest bucket first. `query_type` names the query in the query log. `event` limits
+ the scan to one event name. `join` joins a table to the events, so that the conditions of every edit can read it."""
+ seconds = bucket.total_seconds()
+ if seconds < 1 or seconds != int(seconds):
+ raise ValueError(f"A bucket must be a whole number of seconds, got {bucket}")
+ if end <= start:
+ raise ValueError(f"The range must end after it starts, got {start} to {end}")
+ buckets = math.ceil((end - start) / bucket)
+ # HogQL caps the rows of a query, and a capped result would drop the newest buckets without an error.
+ if buckets > MAX_SELECT_RETURNED_ROWS:
+ raise ValueError(f"The range holds {buckets} buckets, more than {MAX_SELECT_RETURNED_ROWS}")
+ if len({edit.id for edit in edits}) != len(edits):
+ raise ValueError("Each watched edit needs its own id")
+ if not edits:
+ return {}
+
+ states_by_edit = [_states(index, edit, conditions(edit, read_flag(edit))) for index, edit in enumerate(edits)]
+ states = [state for edit_states in states_by_edit for state in edit_states]
+ # Buckets start at `start`, not at a multiple of the bucket since the epoch, so a range that starts at an edit
+ # time does not begin with a partial bucket.
+ bucket_start = parse_expr(
+ "toStartOfInterval(timestamp, toIntervalSecond({seconds}), {start})",
+ placeholders={"seconds": ast.Constant(value=int(seconds)), "start": ast.Constant(value=start)},
+ )
+ where = parse_expr(
+ "timestamp >= {start} AND timestamp < {end}",
+ placeholders={"start": ast.Constant(value=start), "end": ast.Constant(value=end)},
+ )
+ if event is not None:
+ where = ast.And(exprs=[where, parse_expr("event = {event}", placeholders={"event": ast.Constant(value=event)})])
+ per_bucket = ast.SelectQuery(
+ select=[ast.Alias(alias="bucket", expr=bucket_start), *states],
+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"]), next_join=join),
+ where=where,
+ group_by=[ast.Field(chain=["bucket"])],
+ )
+ query = ast.SelectQuery(
+ select=[
+ ast.Field(chain=["bucket"]),
+ *[
+ parse_expr(template, placeholders={"state": ast.Field(chain=[state.alias])})
+ for state in states
+ for template in (IN_BUCKET, SINCE_EDIT)
+ ],
+ ],
+ select_from=ast.JoinExpr(table=per_bucket),
+ order_by=[ast.OrderExpr(expr=ast.Field(chain=["bucket"]), order="ASC")],
+ # One row per bucket. HogQL adds LIMIT 100 to a query without one.
+ limit=ast.Constant(value=buckets),
+ )
+ with tags_context(
+ product=Product.FEATURE_FLAGS, feature=Feature.HEALTH_CHECK, team_id=team.pk, org_id=team.organization_id
+ ):
+ response = execute_hogql_query(
+ query,
+ team=team,
+ query_type=query_type,
+ workload=Workload.OFFLINE,
+ # The join that applies person merges costs seconds on large teams and moves the counts by a tiny
+ # fraction, because a merge moves few people.
+ modifiers=HogQLQueryModifiers(
+ personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS
+ ),
+ # A query that breaks on timeout returns partial counts, which would read as lower counts.
+ settings=HogQLGlobalSettings(
+ max_execution_time=QUERY_TIMEOUT_SECONDS,
+ max_threads=QUERY_MAX_THREADS,
+ timeout_overflow_mode="throw",
+ ),
+ )
+
+ result: dict[str, list[BucketCounts]] = {edit.id: [] for edit in edits}
+ for row in response.results or []:
+ values = iter(row[1:])
+ for edit, edit_states in zip(edits, states_by_edit):
+ # Each state gives the bucket's own value, then the value since the edit.
+ pairs = [(int(next(values)), int(next(values))) for _ in edit_states]
+ result[edit.id].append(
+ BucketCounts(
+ start=row[0],
+ in_bucket=tuple(pair[0] for pair in pairs),
+ since_edit=tuple(pair[1] for pair in pairs),
+ )
+ )
+ return result
+
+
+def _states(index: int, edit: WatchedEdit, conditions: Sequence[ast.Expr]) -> list[ast.Alias]:
+ read = read_flag(edit)
+ return [
+ ast.Alias(
+ alias=f"e{index}_{position}",
+ expr=parse_expr(
+ "uniqState(if({has_actor} AND timestamp >= {edited_at} AND {condition}, {actor}, NULL))",
+ placeholders={
+ "actor": read.actor,
+ "has_actor": read.has_actor,
+ "edited_at": ast.Constant(value=edit.edited_at),
+ "condition": condition,
+ },
+ ),
+ )
+ for position, condition in enumerate(conditions)
+ ]
diff --git a/products/feature_flags/backend/test/test_flag_windows_errors.py b/products/feature_flags/backend/test/test_flag_windows_errors.py
new file mode 100644
index 000000000000..af0dd2ba9819
--- /dev/null
+++ b/products/feature_flags/backend/test/test_flag_windows_errors.py
@@ -0,0 +1,154 @@
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+
+from parameterized import parameterized
+
+from posthog.test.persons import create_person
+
+from products.error_tracking.backend.facade.testing import (
+ create_issue,
+ create_issue_fingerprint,
+ sync_issue_to_clickhouse,
+)
+from products.feature_flags.backend.flag_windows.errors import ErrorCounts, errors_by_bucket
+from products.feature_flags.backend.flag_windows.query import WatchedEdit
+
+EDITED_AT = datetime(2026, 10, 1, 12, 0, tzinfo=UTC)
+WINDOW = timedelta(minutes=30)
+
+BOOLEAN = WatchedEdit(
+ id="boolean", flag_key="new-checkout", edited_at=EDITED_AT, group_type_index=None, variants=(), enables=False
+)
+MULTIVARIATE = WatchedEdit(
+ id="multivariate",
+ flag_key="pricing-test",
+ edited_at=EDITED_AT,
+ group_type_index=None,
+ variants=("control", "test"),
+ enables=False,
+)
+ENABLED = WatchedEdit(
+ id="enabled", flag_key="new-checkout", edited_at=EDITED_AT, group_type_index=None, variants=(), enables=True
+)
+
+# (distinct id, event, minutes after the edit, event properties)
+Event = tuple[str, str, int, dict[str, Any]]
+
+
+class TestErrorsByBucket(ClickhouseTestMixin, BaseTest):
+ def _send(self, events: list[Event]) -> None:
+ for distinct_id in dict.fromkeys(distinct_id for distinct_id, _, _, _ in events):
+ create_person(team=self.team, distinct_ids=[distinct_id])
+ for distinct_id, event, minutes, properties in events:
+ _create_event(
+ team=self.team,
+ event=event,
+ distinct_id=distinct_id,
+ timestamp=EDITED_AT + timedelta(minutes=minutes),
+ properties=properties,
+ )
+ flush_persons_and_events()
+
+ def _since_edit(self, edit: WatchedEdit) -> ErrorCounts:
+ errors = errors_by_bucket(self.team, [edit], EDITED_AT, EDITED_AT + WINDOW, WINDOW)
+ [bucket] = errors[edit.id]
+ return bucket.since_edit
+
+ @parameterized.expand(
+ [
+ (
+ "error_events_split_by_the_flag_value_they_carry",
+ BOOLEAN,
+ [
+ ("person-1", "$exception", 1, {"$feature/new-checkout": True}),
+ ("person-1", "$exception", 2, {"$feature/new-checkout": True}),
+ ("person-2", "$exception", 3, {"$feature/new-checkout": False}),
+ # A server error without the flag counts only among all errors.
+ ("person-3", "$exception", 4, {}),
+ ("person-4", "$pageview", 5, {"$feature/new-checkout": True}),
+ # Errors before the edit belong to the old value.
+ ("person-5", "$exception", -5, {"$feature/new-checkout": True}),
+ ],
+ ErrorCounts(
+ with_errors=3, with_new_errors=0, exposed_with_errors=1, unexposed_with_errors=1, by_variant={}
+ ),
+ ),
+ (
+ "multivariate_errors_count_per_variant",
+ MULTIVARIATE,
+ [
+ ("person-1", "$exception", 1, {"$feature/pricing-test": "control"}),
+ ("person-2", "$exception", 2, {"$feature/pricing-test": "test"}),
+ ("person-3", "$exception", 3, {"$feature/pricing-test": False}),
+ ],
+ ErrorCounts(
+ with_errors=3,
+ with_new_errors=0,
+ exposed_with_errors=2,
+ unexposed_with_errors=1,
+ by_variant={"control": 1, "test": 1},
+ ),
+ ),
+ (
+ "enabling_edit_counts_errors_of_apps_that_loaded_flags_without_it_as_unexposed",
+ ENABLED,
+ [
+ ("person-1", "$exception", 1, {"$feature/new-checkout": True}),
+ (
+ "person-2",
+ "$exception",
+ 2,
+ {"$active_feature_flags": ["other-flag"], "$feature/other-flag": True},
+ ),
+ ],
+ ErrorCounts(
+ with_errors=2, with_new_errors=0, exposed_with_errors=1, unexposed_with_errors=1, by_variant={}
+ ),
+ ),
+ ]
+ )
+ def test_errors_since_the_edit(
+ self, _name: str, edit: WatchedEdit, events: list[Event], expected: ErrorCounts
+ ) -> None:
+ self._send(events)
+
+ self.assertEqual(self._since_edit(edit), expected)
+
+ def test_an_error_is_new_only_when_its_issue_was_first_seen_after_that_edit(self) -> None:
+ issue_id = create_issue(team_id=self.team.pk, name="TypeError")
+ create_issue_fingerprint(
+ team_id=self.team.pk, issue_id=issue_id, fingerprint="old", first_seen=EDITED_AT - timedelta(days=3)
+ )
+ # An old issue can gain a fingerprint after the edit, for example by a merge.
+ create_issue_fingerprint(
+ team_id=self.team.pk, issue_id=issue_id, fingerprint="added", first_seen=EDITED_AT + timedelta(minutes=1)
+ )
+ sync_issue_to_clickhouse(team_id=self.team.pk, issue_id=issue_id)
+ self._send(
+ [
+ ("person-1", "$exception", 1, {"$exception_fingerprint": "old"}),
+ ("person-2", "$exception", 2, {"$exception_fingerprint": "added"}),
+ # A fingerprint that error tracking hasn't synced yet is new.
+ ("person-3", "$exception", 3, {"$exception_fingerprint": "unsynced"}),
+ # An exception that error tracking couldn't fingerprint has no issue to be new.
+ ("person-4", "$exception", 4, {}),
+ ]
+ )
+
+ # The issue is old for BOOLEAN and new for an edit before its first fingerprint, in the same query.
+ earlier = WatchedEdit(
+ id="earlier",
+ flag_key="new-checkout",
+ edited_at=EDITED_AT - timedelta(days=4),
+ group_type_index=None,
+ variants=(),
+ enables=False,
+ )
+ errors = errors_by_bucket(self.team, [BOOLEAN, earlier], EDITED_AT, EDITED_AT + WINDOW, WINDOW)
+
+ self.assertEqual(
+ {edit_id: buckets[0].since_edit.with_new_errors for edit_id, buckets in errors.items()},
+ {"boolean": 1, "earlier": 3},
+ )
diff --git a/products/feature_flags/backend/test/test_flag_windows_exposure.py b/products/feature_flags/backend/test/test_flag_windows_exposure.py
index c8bcd9696b10..162717afe2cb 100644
--- a/products/feature_flags/backend/test/test_flag_windows_exposure.py
+++ b/products/feature_flags/backend/test/test_flag_windows_exposure.py
@@ -13,12 +13,8 @@ from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS
from posthog.models import Team
from posthog.test.persons import create_person
-from products.feature_flags.backend.flag_windows.exposure import (
- ExposureBucket,
- ExposureCounts,
- WatchedEdit,
- exposure_by_bucket,
-)
+from products.feature_flags.backend.flag_windows.exposure import ExposureBucket, ExposureCounts, exposure_by_bucket
+from products.feature_flags.backend.flag_windows.query import WatchedEdit
START = datetime(2026, 10, 1, 12, 0, tzinfo=UTC)
BUCKET = timedelta(minutes=10)