:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): health checks for exposure criteria filters (#124632)
Marcel Poelker52d5f50da81e
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex c2d18bc2665d..ae69fef4f87d 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -11171,9 +11171,9 @@ snapshots:scenes-app-experiments--experiment-frequentist-two-variants--light:hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmIIscenes-app-experiments--experiment-health-debug-tab--dark:- hash: v1.k794b7964.87bfeebb075566d42d1fc0dc9c82a152fb4d9c90fdaa6c94234eae55240d5eeb.jb3eznlVBtPsqVv0VJkgM6Qx7cnQxoVLo0u8tPvZ2PA+ hash: v1.k794b7964.07a80c2033b866a07cebe59831d8db3bb9d3ac9495c547023dd3a4f23102cf76.5EvtNcty-1BR6fQcptymV7gZMB3hGt9Z44k-kLns33gscenes-app-experiments--experiment-health-debug-tab--light:- hash: v1.k794b7964.7990ff112e1fb79448cbb6fe0791813e4cb4196db7a9892927acfb2276cffb22.9m5Oh5rh43QLj42kZh4a20jt7ydQyTlgj3J0E8Q0pCQ+ hash: v1.k794b7964.f0ea28894142a01a25cf3ea19ae46943fb58a3ee6ace693ebd4fd7a9dfea1d28.vCFW8ATe64fjDJDITqHBkQJveadbaLPrD_4v2m_sXv4scenes-app-experiments--experiment-recordings-dropped-metric--dark:hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiwscenes-app-experiments--experiment-recordings-dropped-metric--light:diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex d193252cd596..4065613725f0 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -9869,6 +9869,8 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort+ * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -9891,6 +9893,8 @@ export namespace Schemas {| "metric_cannot_compile"| "metric_calculation_failed"| "exposure_criteria_cannot_match"+ | "exposure_filter_on_dynamic_cohort"+ | "exposure_filter_excludes_events"| "srm"| "zero_exposures"| "forced_variant_release_condition"@@ -9943,6 +9947,8 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort+ * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -10001,6 +10007,7 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm@@ -10021,6 +10028,7 @@ export namespace Schemas {| "metric_cannot_compile"| "metric_calculation_failed"| "exposure_criteria_cannot_match"+ | "exposure_filters"| "early_significance"| "zero_exposures"| "srm"@@ -10054,6 +10062,7 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srmdiff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex 2cce87d6804e..ae1ce8c0ffa3 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -130,6 +130,8 @@ class ExperimentHealthFindingCode(LabeledStrEnum):METRIC_CANNOT_COMPILE = "metric_cannot_compile"METRIC_CALCULATION_FAILED = "metric_calculation_failed"EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"+ EXPOSURE_FILTER_ON_DYNAMIC_COHORT = "exposure_filter_on_dynamic_cohort"+ EXPOSURE_FILTER_EXCLUDES_EVENTS = "exposure_filter_excludes_events"SRM = "srm", "Sample Ratio Mismatch"ZERO_EXPOSURES = "zero_exposures"FORCED_VARIANT_RELEASE_CONDITION = "forced_variant_release_condition"@@ -234,6 +236,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):METRIC_CANNOT_COMPILE = "metric_cannot_compile"METRIC_CALCULATION_FAILED = "metric_calculation_failed"EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"+ EXPOSURE_FILTERS = "exposure_filters"EARLY_SIGNIFICANCE = "early_significance"ZERO_EXPOSURES = "zero_exposures"SRM = "srm"diff --git a/products/experiments/backend/health/checks/exposure_criteria.py b/products/experiments/backend/health/checks/exposure_criteria.pyindex f08606d1a8d6..50e682937b9c 100644--- a/products/experiments/backend/health/checks/exposure_criteria.py+++ b/products/experiments/backend/health/checks/exposure_criteria.py@@ -2,25 +2,32 @@Exposure criteria that no event can match, read from the criteria alone. Pure functions, no I/O.The zero exposures check reports an experiment that counted nobody. This check names the cause when the criteria show-it: a custom exposure event that the project never received, or a filter on the flag call events that requires-another flag than the one the exposure query requires.+it: a custom exposure event that the project never received, a filter on the flag call events that requires another+flag than the one the exposure query requires, or a filter that requires a property the project never received."""from posthog.schema import PropertyOperatorfrom products.experiments.backend.facade.contracts import (+ ExperimentHealthCheckStatus,ExperimentHealthFinding,ExperimentHealthFindingActionKind,ExperimentHealthFindingCode,ExperimentHealthFindingSeverity,)-from products.experiments.backend.health.checks.events_received import not_received_severity, quoted-from products.experiments.backend.health.context import FlagKeyFilter, HealthContext+from products.experiments.backend.health.checks.events_received import joined, not_received_severity, quoted+from products.experiments.backend.health.context import (+ ExposureEvents,+ ExposurePropertyFilter,+ FlagKeyFilter,+ HealthContext,+)from products.experiments.backend.health.skips import (NO_FLAG,NOT_AFTER_END,NOT_ON_ARCHIVED,CheckOutcome,+ CheckSkip,not_applicable,not_checked,)@@ -28,6 +35,7 @@ from products.experiments.backend.health.skips import (# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.EXPOSURE_EVENT_NOT_INGESTED = "exposure_event_not_ingested"FILTER_ON_ANOTHER_FLAG = "filter_on_another_flag"+FILTER_ON_PROPERTY_NOT_INGESTED = "filter_on_property_not_ingested"def exposure_criteria_cannot_match(ctx: HealthContext) -> CheckOutcome:@@ -36,6 +44,20 @@ def exposure_criteria_cannot_match(ctx: HealthContext) -> CheckOutcome:events = ctx.exposure_eventsif events is None:return not_checked("Couldn't check because the exposure criteria can't be read")+ event_outcome = _exposure_event_cannot_match(ctx, events)+ if isinstance(event_outcome, ExperimentHealthFinding):+ return event_outcome+ property_outcome = _property_filter_cannot_match(ctx, events)+ if isinstance(property_outcome, ExperimentHealthFinding):+ return property_outcome+ # A part that couldn't read its input leaves a problem possible, and a part that looked makes the check look.+ for outcome in (event_outcome, property_outcome):+ if isinstance(outcome, CheckSkip) and outcome.status == ExperimentHealthCheckStatus.NOT_CHECKED:+ return outcome+ return None if event_outcome is None or property_outcome is None else event_outcome+++def _exposure_event_cannot_match(ctx: HealthContext, events: ExposureEvents) -> CheckOutcome:event = events.custom_eventif event is not None:if ctx.has_ended:@@ -51,6 +73,19 @@ def exposure_criteria_cannot_match(ctx: HealthContext) -> CheckOutcome:return None if other is None else _filter_on_another_flag(flag_key, other)+def _property_filter_cannot_match(ctx: HealthContext, events: ExposureEvents) -> CheckOutcome:+ filters = events.required_properties+ if not filters:+ return not_applicable("The exposure criteria have no filter that needs a property")+ received = ctx.exposure_properties_received+ missing = [f for f in filters if f.definition_recorded and received.get((f.type, f.key)) is False]+ if missing:+ return _filter_on_property_not_ingested(ctx, missing)+ if any(not f.definition_recorded or (f.type, f.key) not in received for f in filters):+ return not_checked("Couldn't check because PostHog can't tell whether the project received a filter's property")+ return None++def _requires_another_flag(flag_filter: FlagKeyFilter, flag_key: str) -> bool:# `exact` compares the property with each value as a case-sensitive string. A filter without values matches every# event, and the query converts a number to a string, so only string values prove that the filter excludes the key.@@ -96,6 +131,42 @@ def _exposure_event_not_ingested(ctx: HealthContext, event: str) -> ExperimentHe)+def _filter_on_property_not_ingested(+ ctx: HealthContext, filters: list[ExposurePropertyFilter]+) -> ExperimentHealthFinding:+ properties = list(dict.fromkeys(f'the {f.type} property "{f.key}"' for f in filters))+ one = len(properties) == 1+ subject = (+ f"{'A filter' if one else 'Filters'} in the exposure criteria {'needs' if one else 'need'} {joined(properties)}"+ )+ the_filters = "the filter" if one else "the filters"+ if ctx.has_ended:+ detail = (+ f"{subject}, which this project hasn't received. No event passed {the_filters}, so nobody was counted as "+ f"exposed. Correct the {'name' if one else 'names'} or remove {the_filters} to count the exposures up to "+ "the end."+ )+ else:+ advice = (+ f"Check the {'name' if one else 'names'} for a typo, or launch the experiment after your code starts "+ f"sending {'it' if one else 'them'}."+ if not ctx.is_launched+ else f"Check that the {'name matches' if one else 'names match'} what your code sends, or remove "+ f"{the_filters}."+ )+ detail = (+ f"{subject}, which this project hasn't received yet. Until {'it arrives' if one else 'they arrive'}, no "+ f"event passes {the_filters}, so nobody is counted as exposed. {advice}"+ )+ return _finding(+ subcode=FILTER_ON_PROPERTY_NOT_INGESTED,+ severity=not_received_severity(ctx, affects_results=True),+ title="An exposure filter needs a property this project hasn't received",+ detail=detail,+ evidence={"properties": ", ".join(dict.fromkeys(f.key for f in filters))},+ )++def _finding(*, subcode: str, severity: ExperimentHealthFindingSeverity, title: str, detail: str, evidence: dict[str, str]) -> ExperimentHealthFinding:diff --git a/products/experiments/backend/health/checks/exposure_filters.py b/products/experiments/backend/health/checks/exposure_filters.pynew file mode 100644index 000000000000..10f94cbaefa2--- /dev/null+++ b/products/experiments/backend/health/checks/exposure_filters.py@@ -0,0 +1,159 @@+"""+Exposure criteria filters that count other users than they read like. Pure functions, no I/O.++The exposure query applies each filter to the events that it reads, and reads a cohort's members when it runs. So a+dynamic cohort counts the users who are in it at the calculation, not at their exposure, and a filter that leaves out+events with a property value keeps a user who sent another exposure event.+"""++from posthog.hogql.property import operator_is_negative++from products.experiments.backend.facade.contracts import (+ ExperimentHealthFinding,+ ExperimentHealthFindingActionKind,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+)+from products.experiments.backend.health.checks.events_received import quoted+from products.experiments.backend.health.context import (+ ExposureCohort,+ ExposureEvents,+ ExposurePropertyFilter,+ HealthContext,+)+from products.experiments.backend.health.skips import NOT_ON_ARCHIVED, CheckOutcome, not_applicable, not_checked++# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.+COHORT_FORCES_VARIANT = "cohort_forces_variant"+COHORT_IN_RELEASE_CONDITIONS = "cohort_in_release_conditions"+++def exposure_filters(ctx: HealthContext) -> CheckOutcome:+ if ctx.archived:+ return not_applicable(NOT_ON_ARCHIVED)+ events = ctx.exposure_events+ if events is None:+ return not_checked("Couldn't check because the exposure criteria can't be read")+ exclusions = [f for f in events.property_filters if _excludes_events(f)]+ if not events.cohort_ids and not exclusions:+ return not_applicable("The exposure criteria have no cohort filter and no negated event property filter")+ dynamic = [cohort for cohort in ctx.exposure_cohorts if not cohort.is_static]+ if dynamic:+ return _filter_on_dynamic_cohort(ctx, events, dynamic)+ if exclusions:+ return _filter_excludes_events(ctx, exclusions)+ if len(ctx.exposure_cohorts) < len(events.cohort_ids):+ return not_checked("Couldn't check because a cohort in the exposure criteria is deleted or missing")+ return None+++def _excludes_events(property_filter: ExposurePropertyFilter) -> bool:+ # A person property has one value per user at a time, so a filter on it reads as leaving out the user.+ return (+ property_filter.type == "event"+ and property_filter.key != "$feature_flag"+ and operator_is_negative(property_filter.operator)+ )+++def _filter_on_dynamic_cohort(+ ctx: HealthContext, events: ExposureEvents, cohorts: list[ExposureCohort]+) -> ExperimentHealthFinding:+ conditions = ctx.flag.reachable_conditions if ctx.flag else ()+ # The criteria and a release condition read the same users only when both keep the cohort's members.+ in_flag = [+ cohort+ for cohort in cohorts+ if cohort.id in events.member_cohort_ids and any(cohort.id in c.member_cohort_ids for c in conditions)+ ]+ pinning = [+ c for c in conditions if c.pinned_variant and any(cohort.id in c.member_cohort_ids for cohort in in_flag)+ ]+ forced = [cohort for cohort in in_flag if any(cohort.id in c.member_cohort_ids for c in pinning)]+ many = len(cohorts) > 1+ lead = (+ f"The exposure criteria filter on the dynamic {'cohorts' if many else 'cohort'} {_names(cohorts)}. The results "+ f"read {'their' if many else 'its'} members at the time of the calculation, not at the time of each user's "+ "exposure."+ )+ subcode: str | None+ if forced:+ subcode = COHORT_FORCES_VARIANT+ pinned = (+ "Release conditions of the feature flag on {names} give one variant to all the users they match"+ if len(pinning) > 1+ else "A release condition of the feature flag on {names} gives one variant to all the users it matches"+ ).format(names=_names(forced))+ effect = (+ f"{pinned}, and the flag checks who is in {_them(forced)} on each call. Users who joined or left since the "+ "calculation are counted differently from how the flag assigned them, which can make the split of users "+ f"across variants uneven. {_static_in_both_places(ctx)}"+ )+ elif in_flag:+ subcode = COHORT_IN_RELEASE_CONDITIONS+ effect = (+ f"The feature flag's release conditions use {_names(in_flag)} too, and the flag checks who is in "+ f"{_them(in_flag)} on each call, so users the flag served since the calculation can be missing from the "+ f"results. If the release conditions already limit the experiment to these users, remove {_names(in_flag)} "+ f"from the exposure criteria. {_static_in_both_places(ctx, otherwise=True)}"+ )+ else:+ subcode = None+ effect = (+ "Users who join or leave after their exposure move in or out of the results, so if the variant changes who "+ "joins, the results are biased. Use a static cohort instead, which keeps the same members."+ )+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.EXPOSURE_FILTER_ON_DYNAMIC_COHORT,+ subcode=subcode,+ severity=ExperimentHealthFindingSeverity.WARNING,+ title="The exposure criteria filter on a dynamic cohort",+ detail=f"{lead} {effect}",+ evidence={"cohort_ids": ", ".join(str(cohort.id) for cohort in cohorts)},+ actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),+ diagnostic_ref=None,+ )+++def _static_in_both_places(ctx: HealthContext, *, otherwise: bool = False) -> str:+ use = "Otherwise, use" if otherwise else "Use"+ if not ctx.is_launched:+ return f"{use} a static cohort in both places, so the flag and the results agree on who is in it."+ if ctx.has_ended:+ return f"{'Otherwise, in' if otherwise else 'In'} your next experiment, use a static cohort in both places."+ return f"{use} a static cohort in both places, then reset the experiment, because the change moves users."+++def _names(cohorts: list[ExposureCohort]) -> str:+ return quoted([cohort.name for cohort in cohorts])+++def _them(cohorts: list[ExposureCohort]) -> str:+ return "them" if len(cohorts) > 1 else "it"+++def _filter_excludes_events(ctx: HealthContext, filters: list[ExposurePropertyFilter]) -> ExperimentHealthFinding:+ keys = list(dict.fromkeys(f.key for f in filters))+ subject = (+ f"The filter on the event property {quoted(keys)} in the exposure criteria leaves"+ if len(keys) == 1+ else f"The filters on the event properties {quoted(keys)} in the exposure criteria leave"+ )+ advice = (+ " To keep users out of the experiment, leave them out in the feature flag's release conditions."+ if not ctx.is_launched+ else ""+ )+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.EXPOSURE_FILTER_EXCLUDES_EVENTS,+ subcode=None,+ severity=ExperimentHealthFindingSeverity.INFO,+ title="An exposure filter leaves out events, not users",+ detail=(+ f"{subject} out single events, not the users who sent them. A user with a left-out event is still counted "+ f"when another of their events passes.{advice}"+ ),+ evidence={"properties": ", ".join(keys)},+ actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),+ diagnostic_ref=None,+ )diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.pyindex 10c7ef6ad929..7a2ffb6e25c6 100644--- a/products/experiments/backend/health/checks/zero_exposures.py+++ b/products/experiments/backend/health/checks/zero_exposures.py@@ -22,12 +22,12 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:return not_applicable(RUNS_AFTER_LAUNCH)if exposures.hours_since_launch < ZERO_EXPOSURES_GRACE_HOURS:return not_applicable(f"Runs {ZERO_EXPOSURES_GRACE_HOURS} hours after launch")+ if sum(exposures.total_exposures.values()) > 0:+ return None# Criteria that no event can match leave nobody exposed, and the exposure criteria check names that cause. This# check calls it rather than repeating its rules, so exactly one of the two reports the experiment.if isinstance(exposure_criteria_cannot_match(ctx), ExperimentHealthFinding):return not_applicable("The exposure criteria check reports why no users are exposed")- if sum(exposures.total_exposures.values()) > 0:- return Noneevidence = {"hours_since_launch": int(exposures.hours_since_launch)}# A group experiment counts groups, by the rule the exposure query uses.unit = "group" if exposures.counts_groups else "user"diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex c554b04249b3..c2e46d27632d 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -1,35 +1,45 @@from collections.abc import Mappingfrom datetime import date, datetime, timedeltafrom enum import StrEnum-from typing import Any+from functools import reduce+from operator import or_+from typing import Any, Literalfrom django.conf import settings+from django.db.models import Q, QuerySetfrom django.utils import timezoneimport pydanticfrom posthog.schema import (ActionsNode,+ CohortPropertyFilter,EventPropertyFilter,ExperimentEventExposureConfig,MultipleVariantHandling,+ PersonPropertyFilter,PropertyOperator,)+from posthog.hogql.constants import EXCEPTION_STRING_ARRAY_PROPERTIES+from posthog.hogql.property import operator_is_negative+from posthog.dataclasses import frozenfrom posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.models.team.team import Team+from products.cohorts.backend.models.cohort import Cohort+from products.event_definitions.backend.models import EventDefinition, PropertyDefinition, effective_project_id_exprfrom products.experiments.backend.facade.contracts import (ExperimentHealthFlagChangeKind,ExperimentHealthMetricCompileErrorKind,)from products.experiments.backend.health.delays import ENDED_FLAG_QUIET_PERIOD, LAST_CALLED_AT_LAG_DURATION+from products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builderfrom products.experiments.backend.hogql_queries.exposure_query_logic import (DEFAULT_EXPOSURE_EVENT,EXPERIMENT_EXPOSURE_EVENT,get_exposure_event_and_property,- has_activation_config,normalize_to_exposure_criteria,)from products.experiments.backend.logic.result_stats import MetricResultStats@@ -161,6 +171,8 @@ class ReachableCondition:# The person or group property that the condition matches on alone, when its one value is the pinned variant's# key. The variant then comes from whatever sets that property, usually the customer's own assignment.variant_property: VariantProperty | None+ # The cohorts whose members the condition matches. A `not_in` cohort filter matches the users outside the cohort.+ member_cohort_ids: frozenset[int] = frozenset()@frozen@@ -316,6 +328,71 @@ class FlagKeyFilter:values: tuple[str | float | bool, ...]+ExposurePropertyType = Literal["event", "person"]+++@frozen+class ExposurePropertyFilter:+ """A filter of the exposure criteria on an event or a person property. The exposure query applies it to each event+ that it reads, so it leaves out events, not users."""++ type: ExposurePropertyType+ key: str+ operator: PropertyOperator+ # None when the filter has no value. A list of values compiles differently from a missing value.+ values: tuple[str | float | bool, ...] | None+ # The event that the filter applies to. None for an action, which can match several events.+ event: str | None++ @classmethod+ def of(cls, prop: EventPropertyFilter | PersonPropertyFilter, event: str | None) -> "ExposurePropertyFilter":+ return cls(+ type=prop.type,+ key=prop.key,+ # The exposure query reads a filter without an operator as `exact`, as `property_to_expr` does.+ operator=prop.operator or PropertyOperator.EXACT,+ values=None if prop.value is None else _filter_values(prop.value),+ event=event,+ )++ @property+ def fails_when_missing(self) -> bool:+ """Whether no event without the property passes the filter, as `property_to_expr` compiles it.+ `TestFailsWhenMissing` compares this with the compiled filter in ClickHouse for every operator and value shape."""+ if self.values is None:+ return self.operator == PropertyOperator.IS_SET+ if not self.values:+ # An empty list matches every event, except for `icontains`.+ return self.operator == PropertyOperator.ICONTAINS+ if self.operator == PropertyOperator.FLAG_EVALUATES_TO or (+ self.operator == PropertyOperator.BETWEEN and len(self.values) != 2+ ):+ # The filter doesn't compile, so the exposure query fails before it reads a property.+ return False+ if self.operator == PropertyOperator.SEMVER_NEQ:+ # The scalar comparison drops a missing value, which isn't a version. The negated test of an exception+ # array keeps a missing array.+ return not (self.type == "event" and self.key in EXCEPTION_STRING_ARRAY_PROPERTIES)+ return not operator_is_negative(self.operator)++ @property+ def definition_recorded(self) -> bool:+ """Whether ingestion writes a definition of the property when it arrives, so a missing one shows that the+ project never received it."""+ return _has_property_definition(self.type, self.key) and (+ self.type == "person" or (self.event is not None and _records_event_properties(self.event))+ )+++@frozen+class ExposureCohort:+ """A cohort that the exposure criteria filter on."""++ id: int+ name: str+ is_static: bool++@frozenclass ExposureEvents:"""The events that the exposure criteria count."""@@ -333,6 +410,16 @@ class ExposureEvents:# The event, or the action's name, that has to follow a user's flag call before the user counts. None without an# activation, which the exposure query reads only next to the flag call events.activation: str | None = None+ property_filters: tuple[ExposurePropertyFilter, ...] = ()+ # The filters inside an action's steps are not read.+ cohort_ids: tuple[int, ...] = ()+ # Of `cohort_ids`, the cohorts whose members the criteria keep. A `not_in` filter leaves the members out.+ member_cohort_ids: frozenset[int] = frozenset()++ @property+ def required_properties(self) -> tuple[ExposurePropertyFilter, ...]:+ """The filters that no event passes without their property. `flag_key_filters` reads the flag key filter."""+ return tuple(f for f in self.property_filters if f.key != "$feature_flag" and f.fails_when_missing)@frozen@@ -373,8 +460,14 @@ class HealthContext:# None when the stored exposure criteria don't parse.exposure_events: ExposureEvents | None# Whether the project has received each event that `metric_events` and `exposure_events` name. A name that- # ingestion never writes an event definition for is absent, because its absence proves nothing.+ # ingestion never writes an event definition for is absent, because its absence proves nothing. So is every name+ # without a definition when no event of the experiment shows that ingestion records the project.events_received: Mapping[str, bool]+ # Whether the project has received each property that `exposure_events.property_filters` name, by type and key. A+ # property that the definitions can't answer for is absent, under the same rules as `events_received`.+ exposure_properties_received: Mapping[tuple[ExposurePropertyType, str], bool]+ # The cohorts of `exposure_events.cohort_ids` that exist. The exposure query fails on a deleted or missing cohort.+ exposure_cohorts: tuple[ExposureCohort, ...]exposures: ExposureTotals | None# None when not loaded. The health endpoint loads it. The exposure answer loads it only when the sample ratio test# fails, because the load can query the exposures and `srm` is the only exposure check that reads it.@@ -420,6 +513,20 @@ def parse_flag_state()+def _member_cohort_ids(condition: Mapping[str, Any]) -> frozenset[int]:+ # The flag service reads a cohort filter without an operator as `in`, and a filter with an operator other than `in`+ # or `not_in` as no match. See `cohort_operations.rs` in rust/feature-flags.+ properties = condition.get("properties")+ return frozenset(+ int(prop["value"])+ for prop in (properties if isinstance(properties, list) else [])+ if isinstance(prop, dict)+ and prop.get("type") == "cohort"+ and prop.get("operator") in (None, PropertyOperator.IN_)+ and (isinstance(prop.get("value"), int) or (isinstance(prop.get("value"), str) and prop["value"].isdecimal()))+ )++def _reachable_condition(number: int, condition: Mapping[str, Any], variant_keys: set[str], flag_aggregation: int | None) -> ReachableCondition:@@ -428,6 +535,7 @@ def _reachable_condition(number=number,pinned_variant=pinned,variant_property=None if pinned is None else _variant_property(condition, pinned, flag_aggregation),+ member_cohort_ids=_member_cohort_ids(condition),)@@ -565,9 +673,12 @@ def _exposure_events(experiment: Experiment) -> ExposureEvents | None:flag_key or "", criteria, default_exposure_event=DEFAULT_EXPOSURE_EVENT)reads_flag_calls = event in (DEFAULT_EXPOSURE_EVENT, EXPERIMENT_EXPOSURE_EVENT)- config = criteria.exposure_config if criteria else None+ # The configs that the exposure query builds from, so the filters read here are the ones it applies.+ params = get_exposure_config_params_for_builder(criteria, experiment.team, experiment.start_date)+ config, activation = params.exposure_config, params.activation_configproperties = config.properties if isinstance(config, ExperimentEventExposureConfig) else []- activation = criteria.activation_config if criteria and has_activation_config(criteria) else None+ filters = [(prop, source) for source in (config, activation) if source for prop in source.properties or []]+ cohorts = [prop for prop, _ in filters if isinstance(prop, CohortPropertyFilter)]return ExposureEvents(custom_event=event if event is not None and not reads_flag_calls else None,required_flag_key=flag_key if reads_flag_calls else None,@@ -579,6 +690,14 @@ def _exposure_events(experiment: Experiment) -> ExposureEvents | None:if isinstance(prop, EventPropertyFilter) and prop.key == "$feature_flag"),activation=_activation_name(activation) if activation else None,+ property_filters=tuple(+ ExposurePropertyFilter.of(prop, source.event if isinstance(source, ExperimentEventExposureConfig) else None)+ for prop, source in filters+ if isinstance(prop, EventPropertyFilter | PersonPropertyFilter)+ ),+ cohort_ids=tuple(dict.fromkeys(prop.value for prop in cohorts)),+ # `property_to_expr` negates only a `not_in` cohort filter.+ member_cohort_ids=frozenset(prop.value for prop in cohorts if prop.operator != PropertyOperator.NOT_IN),)@@ -605,6 +724,11 @@ def _has_event_definition(name: str) -> bool:return name not in _EVENTS_WITHOUT_DEFINITION and len(name.encode()) <= _MAX_EVENT_DEFINITION_NAME_BYTES+def _records_event_properties(event: str) -> bool:+ # Of `$groupidentify`, ingestion writes only the group properties in `$group_set`.+ return _has_event_definition(event) and event != "$groupidentify"++def _events_received(experiment: Experiment, metric_events: tuple[MetricEvents, ...], exposure_events: ExposureEvents | None) -> dict[str, bool]:@@ -614,15 +738,127 @@ def _events_received(checked = {name for name in names if _has_event_definition(name)}project_id = experiment.team.project_idreceived = defined_event_names(project_id, checked, seen_only=True)- checked -= _deleted_definition_names(project_id, checked - received, since=_deletions_since(experiment))+ checked -= _deleted_definition_names(+ project_id, checked - received, scope="EventDefinition", since=_deletions_since(experiment)+ )return {name: name in received for name in checked}+# Ingestion writes no property definition for these names: HogQL reads them from columns, or ingestion skips them. See+# `SKIP_PROPERTIES` in rust/property-defs-rs/src/types.rs, and `property_to_expr` in posthog/hogql/property.py.+_PROPERTIES_WITHOUT_DEFINITION: dict[ExposurePropertyType, frozenset[str]] = {+ "event": frozenset(+ {"$set", "$set_once", "$unset", "$groups", *(f"$group_{index}" for index in range(5)), "$elements_chain"}+ ),+ # Ingestion sets `$creator_event_uuid` when it creates the person, outside `$set`.+ "person": frozenset({"distinct_id", "$creator_event_uuid"}),+}+++def _has_property_definition(property_type: ExposurePropertyType, key: str) -> bool:+ # HogQL computes a `$virt` property at query time, and ingestion skips a person property of a push subscription.+ # Ingestion replaces a NUL in a name, so the definition holds another name.+ return not (+ key in _PROPERTIES_WITHOUT_DEFINITION[property_type]+ or key.startswith("$virt")+ or (property_type == "person" and key.startswith("$device_push_subscription_"))+ or "\x00" in key+ or len(key.encode()) > _MAX_EVENT_DEFINITION_NAME_BYTES+ )+++def _exposure_properties_received(+ experiment: Experiment, exposure_events: ExposureEvents | None+) -> dict[tuple[ExposurePropertyType, str], bool]:+ filters = exposure_events.required_properties if exposure_events else ()+ checked = {(f.type, f.key) for f in filters if f.definition_recorded}+ if not checked:+ return {}+ project_id = experiment.team.project_id+ received: set[tuple[ExposurePropertyType, str]] = set()+ for definition_type, name in _property_definitions(project_id, name__in={key for _, key in checked}).values_list(+ "type", "name"+ ):+ if definition_type == PropertyDefinition.Type.EVENT:+ received.add(("event", name))+ elif definition_type == PropertyDefinition.Type.PERSON:+ received.add(("person", name))+ missing = checked - received+ # Ingestion writes a deleted definition again only once the property drops out of its cache, which no time bounds,+ # so a deletion at any time leaves the answer open.+ deleted = _deleted_definition_names(project_id, {key for _, key in missing}, scope="PropertyDefinition", since=None)+ return {+ (property_type, key): (property_type, key) in received+ for property_type, key in checked+ if (property_type, key) in received or key not in deleted+ }+++def _property_definitions(project_id: int, **filters: Any) -> QuerySet[PropertyDefinition]:+ return PropertyDefinition.objects.alias(effective_project_id=effective_project_id_expr()).filter(+ effective_project_id=project_id, **filters+ )+++def _exposure_cohorts(experiment: Experiment, exposure_events: ExposureEvents | None) -> tuple[ExposureCohort, ...]:+ cohort_ids = exposure_events.cohort_ids if exposure_events else ()+ if not cohort_ids:+ return ()+ cohorts = {+ cohort_id: (name, is_static)+ for cohort_id, name, is_static in Cohort.objects.filter(+ team__project_id=experiment.team.project_id, pk__in=cohort_ids, deleted=False+ ).values_list("id", "name", "is_static")+ }+ return tuple(+ ExposureCohort(+ id=cohort_id, name=cohorts[cohort_id][0] or f"Cohort {cohort_id}", is_static=cohorts[cohort_id][1]+ )+ for cohort_id in cohort_ids+ if cohort_id in cohorts+ )++# Ingestion writes a deleted definition again when it next sees the event, which takes up to about an hour for an# event that keeps arriving. A margin of a day also covers events that arrive a few times a day._DEFINITION_RECREATION_MARGIN = timedelta(days=1)+def _received(+ experiment: Experiment, metric_events: tuple[MetricEvents, ...], exposure_events: ExposureEvents | None+) -> tuple[dict[str, bool], dict[tuple[ExposurePropertyType, str], bool]]:+ events = _events_received(experiment, metric_events, exposure_events)+ properties = _exposure_properties_received(experiment, exposure_events)+ if False not in (*events.values(), *properties.values()) or _ingestion_records_project(+ experiment, metric_events, exposure_events+ ):+ return events, properties+ return {name: True for name, seen in events.items() if seen}, {+ key: True for key, seen in properties.items() if seen+ }+++def _ingestion_records_project(+ experiment: Experiment, metric_events: tuple[MetricEvents, ...], exposure_events: ExposureEvents | None+) -> bool:+ """Whether ingestion writes the project's definitions now, which a missing definition needs to prove anything.+ Ingestion can be set to skip a team, and then writes no definition at all. It refreshes `last_seen_at` of an event+ that it sees about once an hour. So a flag call or an event of the experiment seen within the margin shows that it+ records the project."""+ names = {DEFAULT_EXPOSURE_EVENT, EXPERIMENT_EXPOSURE_EVENT, *(event for m in metric_events for event in m.events)}+ if exposure_events is not None and exposure_events.custom_event is not None:+ names.add(exposure_events.custom_event)+ return (+ EventDefinition.objects.alias(effective_project_id=effective_project_id_expr())+ .filter(+ effective_project_id=experiment.team.project_id,+ name__in=names,+ last_seen_at__gte=timezone.now() - _DEFINITION_RECREATION_MARGIN,+ )+ .exists()+ )++def _deletions_since(experiment: Experiment) -> datetime:# An earlier deletion matters only if the event stopped arriving: an event that arrived during the run, or# within the margin, has a new definition. So a deletion before then leaves the finding correct.@@ -630,21 +866,31 @@ def _deletions_since(experiment: Experiment) -> datetime:return min(experiment.start_date, recent) if experiment.start_date is not None else recent-def _deleted_definition_names(project_id: int, names: set[str], *, since: datetime) -> set[str]:+def _deleted_definition_names(+ project_id: int,+ names: set[str],+ *,+ scope: Literal["EventDefinition", "PropertyDefinition"],+ since: datetime | None,+) -> set[str]:# Until ingestion writes a deleted definition again, the project can hold the event without a definition.if not names:return set()+ deletions = ActivityLog.objects.filter(+ team_id__in=Team.objects.filter(project_id=project_id).values("id"),+ scope=scope,+ activity="deleted",+ )+ if since is not None:+ deletions = deletions.filter(created_at__gte=since)return set(- ActivityLog.objects.filter(- team_id__in=Team.objects.filter(project_id=project_id).values("id"),- scope="EventDefinition",- activity="deleted",- created_at__gte=since,+ deletions.filter(+ # A containment test per name can use `idx_alog_detail_gin_path_ops`, which `detail__name__in` can't.+ reduce(or_, (Q(detail__contains={"name": name}) for name in names)),# These match the condition of `idx_alog_team_act_scope_usr`, so the lookup reads only the deletions. A# deletion made while impersonating a user is not read.was_impersonated=False,is_system=False,- detail__name__in=names,).values_list("detail__name", flat=True))@@ -666,6 +912,7 @@ def load_health_context(flag_called = _flag_called(experiment)metric_events = _metric_events(experiment, links)exposure_events = _exposure_events(experiment)+ events_received, properties_received = _received(experiment, metric_events, exposure_events)return HealthContext(is_launched=experiment.is_launched,is_running=experiment.is_running,@@ -687,7 +934,9 @@ def load_health_context(hours_since_launch=experiment.hours_since_launch,metric_events=metric_events,exposure_events=exposure_events,- events_received=_events_received(experiment, metric_events, exposure_events),+ events_received=events_received,+ exposure_properties_received=properties_received,+ exposure_cohorts=_exposure_cohorts(experiment, exposure_events),exposures=exposures,flag_history=flag_history,condition_set_traffic=None,diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex 8f514a8bbd84..7269c9d8be5d 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -16,6 +16,7 @@ from products.experiments.backend.health.checks.bucketing_changed import bucketifrom products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_runfrom products.experiments.backend.health.checks.early_significance import early_significancefrom products.experiments.backend.health.checks.exposure_criteria import exposure_criteria_cannot_match+from products.experiments.backend.health.checks.exposure_filters import exposure_filtersfrom products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readablefrom products.experiments.backend.health.checks.flag_state import flag_statefrom products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch@@ -106,6 +107,11 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (name="Exposure criteria",run=exposure_criteria_cannot_match,),+ HealthCheck(+ key=ExperimentHealthCheckKey.EXPOSURE_FILTERS,+ name="Cohorts and event filters in the exposure criteria",+ run=exposure_filters,+ ),HealthCheck(key=ExperimentHealthCheckKey.EARLY_SIGNIFICANCE,name="Sample size behind significant results",diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.pyindex d21ad4655f11..c55fdb4a5e1f 100644--- a/products/experiments/backend/health/test/factories.py+++ b/products/experiments/backend/health/test/factories.py@@ -37,6 +37,8 @@ RUNNING_CONTEXT = HealthContext(custom_event=None, required_flag_key="flag", variant_property="$feature_flag_response", flag_key_filters=()),events_received={},+ exposure_properties_received={},+ exposure_cohorts=(),exposures=None,flag_history=None,condition_set_traffic=None,diff --git a/products/experiments/backend/health/test/test_events_received.py b/products/experiments/backend/health/test/test_events_received.pyindex 74db6bc4da1e..fa637c20a4cd 100644--- a/products/experiments/backend/health/test/test_events_received.py+++ b/products/experiments/backend/health/test/test_events_received.py@@ -15,6 +15,8 @@ from posthog.models.team.team import Teamfrom products.event_definitions.backend.models import EventDefinitionfrom products.experiments.backend.facade.api import get_experiment_healthfrom products.experiments.backend.facade.contracts import (+ ExperimentHealthCheckKey,+ ExperimentHealthCheckStatus,ExperimentHealthFindingCode,ExperimentHealthFindingSeverity,ExperimentHealthReaderAccess,@@ -26,6 +28,8 @@ NO_QUERIES = ExperimentHealthReaderAccess(can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset())Code = ExperimentHealthFindingCode+Key = ExperimentHealthCheckKey+Status = ExperimentHealthCheckStatusWARNING = ExperimentHealthFindingSeverity.WARNING@@ -122,6 +126,37 @@ class TestEventsReceived(BaseTest):self.assertEqual((exposure.subcode, dict(exposure.evidence)), expected_exposure)self.assertEqual(exposure.severity, WARNING)+ def test_events_without_a_definition_go_unchecked_when_no_event_of_the_experiment_was_seen_recently(self) -> None:+ EventDefinition.objects.create(+ team=self.team,+ project=self.project,+ name="$feature_flag_called",+ last_seen_at=timezone.now() - timedelta(days=2),+ )+ flag = FeatureFlag.objects.create(team=self.team, key="checkout-test", created_by=self.user)+ experiment = Experiment.objects.create(+ team=self.team,+ name="Checkout",+ feature_flag=flag,+ start_date=timezone.now() - timedelta(days=3),+ metrics=[_mean("signup", uuid="signups")],+ exposure_criteria={+ "exposure_config": {+ "kind": "ExperimentEventExposureConfig",+ "event": "checkout_viewed",+ "properties": [],+ }+ },+ )++ health = get_experiment_health(+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user+ )++ statuses = {check.key: check.status for check in health.checks}+ self.assertEqual(statuses[Key.METRIC_EVENT_NOT_INGESTED], Status.NOT_CHECKED)+ self.assertEqual(statuses[Key.EXPOSURE_CRITERIA_CANNOT_MATCH], Status.NOT_CHECKED)+def test_a_filter_on_a_deleted_flags_own_key_can_match(self) -> None:flag = FeatureFlag.objects.create(team=self.team, key="checkout-test", created_by=self.user)flag.key, flag.deleted = flag.tombstoned_key(), Truediff --git a/products/experiments/backend/health/test/test_exposure_criteria.py b/products/experiments/backend/health/test/test_exposure_criteria.pyindex 669b289c0b04..258a5e8232e7 100644--- a/products/experiments/backend/health/test/test_exposure_criteria.py+++ b/products/experiments/backend/health/test/test_exposure_criteria.py@@ -1,19 +1,33 @@from dataclasses import replacefrom typing import Any+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_eventsfrom unittest import TestCasefrom parameterized import parameterized-from posthog.schema import PropertyOperator+from posthog.schema import EventPropertyFilter, PersonPropertyFilter, PropertyOperator++from posthog.hogql import ast+from posthog.hogql.errors import BaseHogQLError+from posthog.hogql.property import property_to_expr+from posthog.hogql.query import execute_hogql_queryfrom products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingSeverityfrom products.experiments.backend.health.checks.exposure_criteria import (EXPOSURE_EVENT_NOT_INGESTED,FILTER_ON_ANOTHER_FLAG,+ FILTER_ON_PROPERTY_NOT_INGESTED,exposure_criteria_cannot_match,)-from products.experiments.backend.health.context import ExposureEvents, FlagKeyFilter, HealthContext+from products.experiments.backend.health.context import (+ ExposureEvents,+ ExposurePropertyFilter,+ ExposurePropertyType,+ FlagKeyFilter,+ FlagState,+ HealthContext,+)from products.experiments.backend.health.skips import (NOT_AFTER_END,NOT_ON_ARCHIVED,@@ -47,6 +61,100 @@ def _custom_event(received: dict[str, bool], **changes: Any) -> HealthContext:return replace(RUNNING_CONTEXT, exposure_events=events, events_received=received, **changes)+FLAG = FlagState(active=True, deleted=False, release_groups=(), variants=(), reachable_conditions=())+++def _property_filter(+ received: bool | None,+ operator: PropertyOperator = EXACT,+ values: tuple[str | float | bool, ...] | None = ("pro",),+ *,+ property_type: ExposurePropertyType = "person",+ custom_event: str | None = None,+ action: bool = False,+ **changes: Any,+) -> HealthContext:+ events = ExposureEvents(+ custom_event=custom_event,+ required_flag_key=None if custom_event or action else "checkout-test",+ variant_property="$feature/checkout-test" if custom_event or action else "$feature_flag_response",+ flag_key_filters=(),+ property_filters=(+ ExposurePropertyFilter(+ type=property_type,+ key="plan",+ operator=operator,+ values=values,+ event=None if action else custom_event or "$feature_flag_called",+ ),+ ),+ )+ return replace(+ RUNNING_CONTEXT,+ flag=FLAG,+ exposure_events=events,+ exposure_properties_received={} if received is None else {(property_type, "plan"): received},+ **changes,+ )+++NOT_CHECKED_PROPERTY = not_checked(+ "Couldn't check because PostHog can't tell whether the project received a filter's property"+)+++def _values(operator: PropertyOperator) -> tuple[str | float, str | float, str | float]:+ if operator.value.startswith("semver"):+ return ("1.2.*", "2.*", "3.*") if operator == PropertyOperator.SEMVER_WILDCARD else ("1.2.3", "2.0.0", "3.0.0")+ if operator.value.startswith("is_date"):+ return ("2024-01-01", "2024-02-01", "2024-03-01")+ if operator in (PropertyOperator.REGEX, PropertyOperator.NOT_REGEX):+ return ("^pro", "^team", "^free")+ if operator.value in ("gt", "gte", "lt", "lte", "min", "max", "between", "not_between"):+ return (1, 5, 9)+ return ("pro", "team", "free")+++class TestFailsWhenMissing(ClickhouseTestMixin, BaseTest):+ def test_matches_the_compiled_filter_on_an_event_without_the_property(self) -> None:+ _create_person(team=self.team, distinct_ids=["user"], properties={"name": "Ada"})+ _create_event(team=self.team, event="checkout_viewed", distinct_id="user", properties={"step": 1})+ flush_persons_and_events()+ compiled: dict[str, ast.Expr] = {}+ expected: dict[str, bool] = {}+ actual: dict[str, bool] = {}+ for filter_class, key in (+ (EventPropertyFilter, "plan"),+ (EventPropertyFilter, "$exception_types"),+ (PersonPropertyFilter, "plan"),+ (PersonPropertyFilter, "$exception_types"),+ ):+ for operator in PropertyOperator:+ first, second, third = _values(operator)+ for value in (None, [], first, [first, second], [first, second, third]):+ prop = filter_class(key=key, operator=operator, value=value)+ label = f"{prop.type} {key} {operator.value} {value!r}"+ actual[label] = ExposurePropertyFilter.of(prop, "checkout_viewed").fails_when_missing+ try:+ compiled[label] = property_to_expr(prop, self.team)+ except BaseHogQLError:+ # The exposure query fails before it reads a property, so no filter fails on its account.+ expected[label] = False+ query = ast.SelectQuery(+ select=[ast.Call(name="countIf", args=[expr]) for expr in compiled.values()],+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),+ where=ast.CompareOperation(+ op=ast.CompareOperationOp.Eq,+ left=ast.Field(chain=["event"]),+ right=ast.Constant(value="checkout_viewed"),+ ),+ )+ counts = execute_hogql_query(query, self.team).results[0]+ expected.update({label: count == 0 for label, count in zip(compiled, counts)})++ self.assertEqual(actual, expected)++class TestExposureCriteriaCannotMatch(TestCase):@parameterized.expand([@@ -93,6 +201,58 @@ class TestExposureCriteriaCannotMatch(TestCase):_custom_event({"checkout_viewed": False}, **ENDED),not_applicable(NOT_AFTER_END),),+ ("property_not_received", _property_filter(False), (FILTER_ON_PROPERTY_NOT_INGESTED, Severity.WARNING)),+ (+ "property_not_received_in_a_draft",+ _property_filter(False, **DRAFT),+ (FILTER_ON_PROPERTY_NOT_INGESTED, Severity.INFO),+ ),+ (+ "property_not_received_after_the_end",+ _property_filter(False, **ENDED),+ (FILTER_ON_PROPERTY_NOT_INGESTED, Severity.WARNING),+ ),+ (+ "property_not_received_after_the_end_of_a_custom_event",+ _property_filter(False, custom_event="checkout_viewed", **ENDED),+ (FILTER_ON_PROPERTY_NOT_INGESTED, Severity.WARNING),+ ),+ (+ "person_property_not_received_on_an_action",+ _property_filter(False, action=True),+ (FILTER_ON_PROPERTY_NOT_INGESTED, Severity.WARNING),+ ),+ (+ "event_property_on_an_action",+ _property_filter(False, property_type="event", action=True),+ NOT_CHECKED_PROPERTY,+ ),+ (+ "event_property_of_a_group_identify_event",+ _property_filter(+ False,+ property_type="event",+ custom_event="$groupidentify",+ events_received={"$groupidentify": True},+ ),+ NOT_CHECKED_PROPERTY,+ ),+ (+ "is_set_without_a_value",+ _property_filter(False, PropertyOperator.IS_SET, None),+ (FILTER_ON_PROPERTY_NOT_INGESTED, Severity.WARNING),+ ),+ ("property_received", _property_filter(True), None),+ ("a_missing_property_passes_a_negated_filter", _property_filter(False, PropertyOperator.IS_NOT), None),+ ("an_empty_list_matches_every_event", _property_filter(False, values=()), None),+ ("is_set_with_an_empty_list", _property_filter(False, PropertyOperator.IS_SET, ()), None),+ ("exact_without_a_value_matches_a_missing_property", _property_filter(False, values=None), None),+ ("property_without_a_definition", _property_filter(None), NOT_CHECKED_PROPERTY),+ (+ "custom_event_after_the_end_with_a_received_property",+ _property_filter(True, custom_event="checkout_viewed", **ENDED),+ None,+ ),])def test_exposure_criteria_cannot_match(diff --git a/products/experiments/backend/health/test/test_exposure_filters.py b/products/experiments/backend/health/test/test_exposure_filters.pynew file mode 100644index 000000000000..66c477030df5--- /dev/null+++ b/products/experiments/backend/health/test/test_exposure_filters.py@@ -0,0 +1,285 @@+from dataclasses import replace+from datetime import timedelta+from typing import Any++from posthog.test.base import BaseTest+from unittest import TestCase++from django.utils import timezone++from parameterized import parameterized++from posthog.schema import PropertyOperator++from posthog.models.activity_logging.activity_log import Detail, log_activity+from posthog.models.team.team import Team++from products.cohorts.backend.models.cohort import Cohort+from products.event_definitions.backend.models import EventDefinition, PropertyDefinition+from products.experiments.backend.facade.contracts import (+ ExperimentHealthFinding,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+)+from products.experiments.backend.health.checks.exposure_filters import (+ COHORT_FORCES_VARIANT,+ COHORT_IN_RELEASE_CONDITIONS,+ exposure_filters,+)+from products.experiments.backend.health.context import (+ ExposureCohort,+ ExposureEvents,+ ExposurePropertyFilter,+ ExposurePropertyType,+ FlagState,+ HealthContext,+ ReachableCondition,+ load_health_context,+)+from products.experiments.backend.health.skips import NOT_ON_ARCHIVED, CheckSkip, not_applicable, not_checked+from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT+from products.experiments.backend.models.experiment import Experiment+from products.feature_flags.backend.models.feature_flag import FeatureFlag++Code = ExperimentHealthFindingCode+Severity = ExperimentHealthFindingSeverity+DYNAMIC = ExposureCohort(id=7, name="Paying users", is_static=False)+STATIC = ExposureCohort(id=8, name="QA team", is_static=True)+NO_FILTERS = not_applicable("The exposure criteria have no cohort filter and no negated event property filter")+++def _filters(+ *property_filters: tuple[ExposurePropertyType, str, PropertyOperator],+ cohorts: tuple[ExposureCohort, ...] = (),+ cohort_ids: tuple[int, ...] | None = None,+ excluded_cohort_ids: frozenset[int] = frozenset(),+ flag_cohort_ids: frozenset[int] = frozenset(),+ forced_variant: str | None = None,+ **changes: Any,+) -> HealthContext:+ ids = tuple(cohort.id for cohort in cohorts) if cohort_ids is None else cohort_ids+ events = ExposureEvents(+ custom_event=None,+ required_flag_key="checkout-test",+ variant_property="$feature_flag_response",+ flag_key_filters=(),+ property_filters=tuple(+ ExposurePropertyFilter(+ type=property_type, key=key, operator=operator, values=("x",), event="$feature_flag_called"+ )+ for property_type, key, operator in property_filters+ ),+ cohort_ids=ids,+ member_cohort_ids=frozenset(ids) - excluded_cohort_ids,+ )+ condition = ReachableCondition(+ number=1, pinned_variant=forced_variant, variant_property=None, member_cohort_ids=flag_cohort_ids+ )+ flag = FlagState(active=True, deleted=False, release_groups=(), variants=(), reachable_conditions=(condition,))+ return replace(RUNNING_CONTEXT, exposure_events=events, exposure_cohorts=cohorts, flag=flag, **changes)+++def _filter(property_type: str, key: str) -> dict[str, Any]:+ return {"type": property_type, "key": key, "operator": "exact", "value": "x"}+++def _config(event: str, *properties: dict[str, Any]) -> dict[str, Any]:+ return {"kind": "ExperimentEventExposureConfig", "event": event, "properties": list(properties)}+++EXCLUDED_URL: tuple[ExposurePropertyType, str, PropertyOperator] = (+ "event",+ "$current_url",+ PropertyOperator.NOT_ICONTAINS,+)+COHORT_WARNING = (Code.EXPOSURE_FILTER_ON_DYNAMIC_COHORT, None, Severity.WARNING)+++class TestExposureFilters(TestCase):+ @parameterized.expand(+ [+ ("dynamic_cohort", _filters(cohorts=(DYNAMIC,)), COHORT_WARNING),+ (+ "dynamic_cohort_in_the_release_conditions",+ _filters(cohorts=(DYNAMIC,), flag_cohort_ids=frozenset({7})),+ (Code.EXPOSURE_FILTER_ON_DYNAMIC_COHORT, COHORT_IN_RELEASE_CONDITIONS, Severity.WARNING),+ ),+ (+ "dynamic_cohort_given_one_variant_in_the_release_conditions",+ _filters(cohorts=(DYNAMIC,), flag_cohort_ids=frozenset({7}), forced_variant="test"),+ (Code.EXPOSURE_FILTER_ON_DYNAMIC_COHORT, COHORT_FORCES_VARIANT, Severity.WARNING),+ ),+ (+ "dynamic_cohort_left_out_by_the_criteria_and_kept_by_the_release_conditions",+ _filters(cohorts=(DYNAMIC,), excluded_cohort_ids=frozenset({7}), flag_cohort_ids=frozenset({7})),+ COHORT_WARNING,+ ),+ ("dynamic_cohort_after_the_end", _filters(cohorts=(DYNAMIC,), **ENDED), COHORT_WARNING),+ (+ "dynamic_cohort_before_an_event_exclusion",+ _filters(EXCLUDED_URL, cohorts=(STATIC, DYNAMIC)),+ COHORT_WARNING,+ ),+ ("static_cohort", _filters(cohorts=(STATIC,)), None),+ (+ "deleted_cohort",+ _filters(cohorts=(STATIC,), cohort_ids=(8, 9)),+ not_checked("Couldn't check because a cohort in the exposure criteria is deleted or missing"),+ ),+ ("event_exclusion", _filters(EXCLUDED_URL), (Code.EXPOSURE_FILTER_EXCLUDES_EVENTS, None, Severity.INFO)),+ ("event_filter_that_selects", _filters(("event", "$current_url", PropertyOperator.ICONTAINS)), NO_FILTERS),+ ("person_exclusion", _filters(("person", "email", PropertyOperator.NOT_ICONTAINS)), NO_FILTERS),+ ("flag_key_exclusion", _filters(("event", "$feature_flag", PropertyOperator.IS_NOT)), NO_FILTERS),+ ("archived", _filters(cohorts=(DYNAMIC,), archived=True), not_applicable(NOT_ON_ARCHIVED)),+ ]+ )+ def test_exposure_filters(+ self, _name: str, ctx: HealthContext, expected: tuple[Code, str | None, Severity] | CheckSkip | None+ ) -> None:+ outcome = exposure_filters(ctx)++ actual = (+ (outcome.code, outcome.subcode, outcome.severity)+ if isinstance(outcome, ExperimentHealthFinding)+ else outcome+ )+ self.assertEqual(actual, expected)+++class TestExposureFilterInputs(BaseTest):+ @parameterized.expand(+ [+ (+ "ingestion_records_the_project",+ timedelta(hours=2),+ {+ ("event", "$current_url"): True,+ ("event", "$curent_url"): False,+ ("event", "email"): False,+ ("person", "email"): True,+ ("person", "plan"): False,+ },+ ),+ (+ "no_event_of_the_experiment_seen_recently",+ timedelta(days=2),+ {("event", "$current_url"): True, ("person", "email"): True},+ ),+ ]+ )+ def test_reads_the_properties_and_cohorts_as_the_exposure_query_does(+ self, _name: str, last_seen_ago: timedelta, expected_received: dict[tuple[str, str], bool]+ ) -> None:+ sibling = Team.objects.create(organization=self.organization, project=self.project, name="Sibling")+ EventDefinition.objects.create(+ team=sibling, project=self.project, name="$feature_flag_called", last_seen_at=timezone.now() - last_seen_ago+ )+ PropertyDefinition.objects.create(team=sibling, project=self.project, name="$current_url")+ PropertyDefinition.objects.create(+ team=self.team, project=self.project, name="email", type=PropertyDefinition.Type.PERSON+ )+ with self.captureOnCommitCallbacks(execute=True):+ log_activity(+ organization_id=self.organization.id,+ team_id=sibling.id,+ user=self.user,+ was_impersonated=False,+ item_id="deleted_prop",+ scope="PropertyDefinition",+ activity="deleted",+ detail=Detail(name="deleted_prop", changes=None),+ )+ dynamic = Cohort.objects.create(team=sibling, name="Paying users")+ static = Cohort.objects.create(team=self.team, name="QA team", is_static=True)+ deleted = Cohort.objects.create(team=self.team, name="Old", deleted=True)+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="checkout-test",+ created_by=self.user,+ filters={+ "groups": [+ {"properties": [{"type": "cohort", "key": "id", "value": dynamic.pk}], "rollout_percentage": 100},+ {+ "properties": [{"type": "cohort", "key": "id", "value": static.pk, "operator": "not_in"}],+ "rollout_percentage": 100,+ },+ ]+ },+ )+ filters = [+ {"type": "event", "key": key, "operator": "exact", "value": "x"}+ for key in ("$current_url", "$curent_url", "deleted_prop", "$virt_is_bot", "email")+ ]+ experiment = Experiment.objects.create(+ team=self.team,+ name="Checkout",+ feature_flag=flag,+ start_date=timezone.now(),+ exposure_criteria={+ "exposure_config": {+ "kind": "ExperimentEventExposureConfig",+ "event": "$feature_flag_called",+ "properties": [+ *filters,+ {"type": "person", "key": "email", "operator": "icontains", "value": "@"},+ {"type": "person", "key": "plan", "operator": "exact", "value": "pro"},+ {"type": "cohort", "key": "id", "value": dynamic.pk},+ {"type": "cohort", "key": "id", "value": static.pk, "operator": "not_in"},+ {"type": "cohort", "key": "id", "value": deleted.pk},+ ],+ }+ },+ )++ ctx = load_health_context(experiment)++ self.assertEqual(dict(ctx.exposure_properties_received), expected_received)+ self.assertEqual(+ ctx.exposure_cohorts,+ (+ ExposureCohort(id=dynamic.pk, name="Paying users", is_static=False),+ ExposureCohort(id=static.pk, name="QA team", is_static=True),+ ),+ )+ assert ctx.exposure_events is not None and ctx.flag is not None+ self.assertEqual(ctx.exposure_events.member_cohort_ids, {dynamic.pk, deleted.pk})+ self.assertEqual([c.member_cohort_ids for c in ctx.flag.reachable_conditions], [{dynamic.pk}, frozenset()])++ @parameterized.expand(+ [+ (+ "activation_next_to_the_default_exposure",+ {+ "exposure_config": _config("$feature_flag_called", _filter("person", "plan")),+ "activation_config": _config("checkout_viewed", _filter("event", "step")),+ },+ [("person", "plan", "$feature_flag_called"), ("event", "step", "checkout_viewed")],+ ),+ (+ "activation_stored_next_to_a_custom_exposure",+ {+ "exposure_config": _config("signup", _filter("person", "plan")),+ "activation_config": _config("checkout_viewed", _filter("event", "step")),+ },+ [("person", "plan", "signup")],+ ),+ (+ "action",+ {"exposure_config": {"kind": "ActionsNode", "id": 1, "properties": [_filter("event", "step")]}},+ [("event", "step", None)],+ ),+ ]+ )+ def test_reads_the_filters_the_exposure_query_applies(+ self, _name: str, criteria: dict[str, Any], expected: list[tuple[str, str, str | None]]+ ) -> None:+ flag = FeatureFlag.objects.create(team=self.team, key="checkout-test", created_by=self.user)+ experiment = Experiment.objects.create(+ team=self.team, name="Checkout", feature_flag=flag, exposure_criteria=criteria+ )++ ctx = load_health_context(experiment)++ assert ctx.exposure_events is not None+ self.assertEqual([(f.type, f.key, f.event) for f in ctx.exposure_events.property_filters], expected)+ self.assertEqual(ctx.exposure_properties_received, {})diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex 8da06659f90d..3b67670d1d3d 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -960,8 +960,9 @@ class EnterpriseExperimentsViewSet(each check, so a check that found no problem can be told from one that did not apply or couldnot read an input.- The checks here read the experiment, its flag, its linked metrics, and the changes of the- flag's variant split since launch. The checks that read exposure counts come with the+ The checks here read the experiment, its flag, its linked metrics, the project's event and+ property definitions, the cohorts of the exposure criteria, and the changes of the flag's+ variant split since launch. The checks that read exposure counts come with theexposure query. The split-change check compares the split changes with the exposures onlyfor exposure criteria that name an event without property filters. When the split changedafter launch and the criteria have filters or an action, or an API key lacks `query:read`,@@ -974,14 +975,32 @@ class EnterpriseExperimentsViewSet(definition without `last_seen_at`, such as one Wizard wrote for a planned event, counts asnot received. A name whose definition was deleted since launch, or in the last day for adraft, is not checked, because ingestion writes the definition again only when the event- arrives. `metric_event_not_ingested` names each metric that+ arrives. No missing definition is checked when ingestion saw none of the experiment's events+ in the last day, because ingestion can be set to skip a team. `metric_event_not_ingested`+ names each metric thatreads an event the project never received, in the `evidence` keys `metric_N`,`metric_N_role` and `metric_N_events`. `exposure_criteria_cannot_match` reports a custom- exposure event the project never received (`exposure_event_not_ingested`), and a+ exposure event the project never received (`exposure_event_not_ingested`), a`$feature_flag` filter on the flag call events that requires another flag- (`filter_on_another_flag`). An event not received is info before launch, in the first 24- hours after launch, and when only secondary metrics read it, and a warning otherwise.- `filter_on_another_flag` is a warning in every state, and the other findings stop at the end.+ (`filter_on_another_flag`), and a filter on an event or person property that the project has+ no definition of, with an operator that a missing property fails+ (`filter_on_property_not_ingested`, the keys in `evidence.properties`). A property whose+ definition was ever deleted is not checked, and neither is an event property filter on an+ action or on `$groupidentify`, whose properties ingestion keeps no definitions of. An event+ or property not received is info before launch, in the first 24 hours after launch, and when+ only secondary metrics read it, and a warning otherwise. `filter_on_another_flag` is a+ warning in every state, `filter_on_property_not_ingested` also reports after the end, and+ the other findings stop at the end.++ `exposure_filters` reads the cohort and event property filters of the exposure criteria and+ of the activation, not the test account filters. `exposure_filter_on_dynamic_cohort` is a+ warning in every state: the results use the members of a dynamic cohort at the calculation,+ not at exposure. Its subcode is `cohort_forces_variant` when a release condition of the flag+ that matches the cohort's members gives one variant, `cohort_in_release_conditions` when the+ release conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts.+ Both subcodes need the exposure criteria to keep the members too, not to leave them out.+ `exposure_filter_excludes_events` is info: a negated filter on an event property leaves out+ events, not users (`evidence.properties`).`metric_cannot_compile` builds and compiles the query of each metric without running it, asthe caller, so a metric that every calculation fails on shows before launch. A metric thatdiff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex 33c4b952577d..34cac444a5da 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2307,6 +2307,8 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort+ * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -2332,6 +2334,8 @@ export const ExperimentHealthFindingCodeEnumApi = {MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',+ ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',+ ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',Srm: 'srm',ZeroExposures: 'zero_exposures',ForcedVariantReleaseCondition: 'forced_variant_release_condition',@@ -2400,6 +2404,8 @@ export interface ExperimentHealthFindingApi {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort+ * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -2450,6 +2456,7 @@ export interface ExperimentHealthFindingApi {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm@@ -2473,6 +2480,7 @@ export const ExperimentHealthCheckKeyEnumApi = {MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',+ ExposureFilters: 'exposure_filters',EarlySignificance: 'early_significance',ZeroExposures: 'zero_exposures',Srm: 'srm',@@ -2512,6 +2520,7 @@ export interface ExperimentHealthCheckResultApi {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srmdiff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex 72ef229607a7..265ea3364f07 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -705,8 +705,9 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* each check, so a check that found no problem can be told from one that did not apply or could* not read an input.*- * The checks here read the experiment, its flag, its linked metrics, and the changes of the- * flag's variant split since launch. The checks that read exposure counts come with the+ * The checks here read the experiment, its flag, its linked metrics, the project's event and+ * property definitions, the cohorts of the exposure criteria, and the changes of the flag's+ * variant split since launch. The checks that read exposure counts come with the* exposure query. The split-change check compares the split changes with the exposures only* for exposure criteria that name an event without property filters. When the split changed* after launch and the criteria have filters or an action, or an API key lacks `query:read`,@@ -719,14 +720,32 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* definition without `last_seen_at`, such as one Wizard wrote for a planned event, counts as* not received. A name whose definition was deleted since launch, or in the last day for a* draft, is not checked, because ingestion writes the definition again only when the event- * arrives. `metric_event_not_ingested` names each metric that+ * arrives. No missing definition is checked when ingestion saw none of the experiment's events+ * in the last day, because ingestion can be set to skip a team. `metric_event_not_ingested`+ * names each metric that* reads an event the project never received, in the `evidence` keys `metric_N`,* `metric_N_role` and `metric_N_events`. `exposure_criteria_cannot_match` reports a custom- * exposure event the project never received (`exposure_event_not_ingested`), and a+ * exposure event the project never received (`exposure_event_not_ingested`), a* `$feature_flag` filter on the flag call events that requires another flag- * (`filter_on_another_flag`). An event not received is info before launch, in the first 24- * hours after launch, and when only secondary metrics read it, and a warning otherwise.- * `filter_on_another_flag` is a warning in every state, and the other findings stop at the end.+ * (`filter_on_another_flag`), and a filter on an event or person property that the project has+ * no definition of, with an operator that a missing property fails+ * (`filter_on_property_not_ingested`, the keys in `evidence.properties`). A property whose+ * definition was ever deleted is not checked, and neither is an event property filter on an+ * action or on `$groupidentify`, whose properties ingestion keeps no definitions of. An event+ * or property not received is info before launch, in the first 24 hours after launch, and when+ * only secondary metrics read it, and a warning otherwise. `filter_on_another_flag` is a+ * warning in every state, `filter_on_property_not_ingested` also reports after the end, and+ * the other findings stop at the end.+ *+ * `exposure_filters` reads the cohort and event property filters of the exposure criteria and+ * of the activation, not the test account filters. `exposure_filter_on_dynamic_cohort` is a+ * warning in every state: the results use the members of a dynamic cohort at the calculation,+ * not at exposure. Its subcode is `cohort_forces_variant` when a release condition of the flag+ * that matches the cohort's members gives one variant, `cohort_in_release_conditions` when the+ * release conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts.+ * Both subcodes need the exposure criteria to keep the members too, not to leave them out.+ * `exposure_filter_excludes_events` is info: a negated filter on an event property leaves out+ * events, not users (`evidence.properties`).** `metric_cannot_compile` builds and compiles the query of each metric without running it, as* the caller, so a metric that every calculation fails on shows before launch. A metric thatdiff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex fbd44cdcc86c..b9587ba017a8 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -248,9 +248,37 @@ function metricCalculationNote({ evidence }: ExperimentHealthFindingApi): string}function exposureCriteriaNote({ subcode, evidence }: ExperimentHealthFindingApi): string {- return subcode === 'filter_on_another_flag'- ? `The query requires ${evidence.flag_key}, a filter requires ${evidence.filter_flag_keys}.`- : `Not received: ${evidence.event}.`+ if (subcode === 'filter_on_another_flag') {+ return `The query requires ${evidence.flag_key}, a filter requires ${evidence.filter_flag_keys}.`+ }+ if (subcode === 'filter_on_property_not_ingested') {+ return `Property not received: ${evidence.properties}.`+ }+ return `Not received: ${evidence.event}.`+}++const EXPOSURE_FILTERS_FINDING_CODES: ReadonlySet<ExperimentHealthFindingApi['code']> = new Set([+ 'exposure_filter_on_dynamic_cohort',+ 'exposure_filter_excludes_events',+])++function exposureFiltersCheck(input: HealthDebugInput): HealthDebugCheck {+ const finding = input.health?.findings.find(({ code }) => EXPOSURE_FILTERS_FINDING_CODES.has(code)) ?? null+ return {+ check: 'exposure_filters',+ source: 'health endpoint',+ reads: ['experiment', 'flag', 'project'],+ ...serverResult('exposure_filters', input.health, (code) => EXPOSURE_FILTERS_FINDING_CODES.has(code)),+ browser: null,+ browserLabel: BROWSER_RULES,+ ...NO_TIMES,+ difference: null,+ note: !finding+ ? null+ : finding.code === 'exposure_filter_on_dynamic_cohort'+ ? `Dynamic cohorts: ${finding.evidence.cohort_ids}.`+ : `Leaves out events by: ${finding.evidence.properties}.`,+ }}function noMetricCheck(input: HealthDebugInput): HealthDebugCheck {@@ -353,6 +381,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChecserverOnlyCheck(input, 'metric_cannot_compile', metricCompileNote, ['metrics', 'experiment']),serverOnlyCheck(input, 'metric_calculation_failed', metricCalculationNote, ['metrics']),serverOnlyCheck(input, 'exposure_criteria_cannot_match', exposureCriteriaNote, ['experiment', 'project']),+ exposureFiltersCheck(input),serverOnlyCheck(input, 'early_significance', null, ['experiment', 'metrics']),...exposureChecks(input),]diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex 02fb4fb5b909..8dcf6a38595e 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -27,6 +27,8 @@ export type ExperimentHealthFindingCode =| 'metric_cannot_compile'| 'metric_calculation_failed'| 'exposure_criteria_cannot_match'+ | 'exposure_filter_on_dynamic_cohort'+ | 'exposure_filter_excludes_events'| 'forced_variant_release_condition'| 'flag_variants_mismatch'| 'split_changed_mid_run'diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex b4cab4df4bae..893386631dd9 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -47493,6 +47493,8 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort+ * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -47518,6 +47520,8 @@ export namespace Schemas {MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',+ ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',+ ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',Srm: 'srm',ZeroExposures: 'zero_exposures',ForcedVariantReleaseCondition: 'forced_variant_release_condition',@@ -47586,6 +47590,8 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort+ * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition@@ -47636,6 +47642,7 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm@@ -47659,6 +47666,7 @@ export namespace Schemas {MetricCannotCompile: 'metric_cannot_compile',MetricCalculationFailed: 'metric_calculation_failed',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',+ ExposureFilters: 'exposure_filters',EarlySignificance: 'early_significance',ZeroExposures: 'zero_exposures',Srm: 'srm',@@ -47698,6 +47706,7 @@ export namespace Schemas {* * `metric_cannot_compile` - Metric Cannot Compile* * `metric_calculation_failed` - Metric Calculation Failed* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match+ * * `exposure_filters` - Exposure Filters* * `early_significance` - Early Significance* * `zero_exposures` - Zero Exposures* * `srm` - Srm