:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): stop reporting negated exposure filters as findings (#126812)
Marcel Poelkerc3edd81da884
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 933ad1df680b..0bb6eeaba5fe 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -10072,7 +10072,6 @@ export namespace Schemas {* * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures@@ -10099,7 +10098,6 @@ export namespace Schemas {| "metric_all_zero"| "exposure_criteria_cannot_match"| "exposure_filter_on_dynamic_cohort"- | "exposure_filter_excludes_events"| "srm"| "zero_exposures"| "test_account_filter_removes_exposures"@@ -10156,7 +10154,6 @@ export namespace Schemas {* * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposuresdiff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex bc19fa497ee9..d604c05035f8 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -205,7 +205,6 @@ class ExperimentHealthFindingCode(LabeledStrEnum):METRIC_ALL_ZERO = "metric_all_zero"EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"EXPOSURE_FILTER_ON_DYNAMIC_COHORT = "exposure_filter_on_dynamic_cohort"- EXPOSURE_FILTER_EXCLUDES_EVENTS = "exposure_filter_excludes_events"SRM = "srm", "Sample Ratio Mismatch"ZERO_EXPOSURES = "zero_exposures"TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES = "test_account_filter_removes_exposures"diff --git a/products/experiments/backend/health/checks/exposure_filters.py b/products/experiments/backend/health/checks/exposure_filters.pyindex 3a22e06646d6..99677f5af911 100644--- a/products/experiments/backend/health/checks/exposure_filters.py+++ b/products/experiments/backend/health/checks/exposure_filters.py@@ -1,13 +1,10 @@"""-Exposure criteria filters that count other users than they read like. Pure functions, no I/O.+Cohort filters of the exposure criteria that count other users than they read like. Pure functions, no I/O.-The exposure query applies each filter to the events that it reads, and reads a cohort's members when it runs. So a-dynamic cohort counts the users who are in it at the calculation, not at their exposure, and a filter that leaves out-events with a property value keeps a user who sent another exposure event.+The exposure query reads a cohort's members when it runs, so a dynamic cohort counts the users who are in it at the+calculation, not at their exposure."""-from posthog.hogql.property import operator_is_negative-from products.experiments.backend.facade.contracts import (ExperimentHealthFinding,ExperimentHealthFindingActionKind,@@ -15,12 +12,7 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthFindingSeverity,)from products.experiments.backend.health.checks.events_received import quoted-from products.experiments.backend.health.context import (- ExposureCohort,- ExposureEvents,- ExposurePropertyFilter,- HealthContext,-)+from products.experiments.backend.health.context import ExposureCohort, ExposureEvents, HealthContextfrom products.experiments.backend.health.skips import NOT_ON_ARCHIVED, CheckOutcome, not_applicable, not_checked# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.@@ -34,28 +26,16 @@ def exposure_filters(ctx: HealthContext) -> CheckOutcome:events = ctx.exposure_eventsif events is None:return not_checked("Couldn't check because the exposure criteria can't be read")- exclusions = [f for f in events.property_filters if _excludes_events(f)]- if not events.cohort_ids and not exclusions:- return not_applicable("The exposure criteria have no cohort filter and no negated event property filter")+ if not events.cohort_ids:+ return not_applicable("The exposure criteria have no cohort filter")dynamic = [cohort for cohort in ctx.exposure_cohorts if not cohort.is_static]if dynamic:return _filter_on_dynamic_cohort(ctx, events, dynamic)- if exclusions:- return _filter_excludes_events(ctx, exclusions)if len(ctx.exposure_cohorts) < len(events.cohort_ids):return not_checked("Couldn't check because a cohort in the exposure criteria is deleted or missing")return None-def _excludes_events(property_filter: ExposurePropertyFilter) -> bool:- # A person property has one value per user at a time, so a filter on it reads as leaving out the user.- return (- property_filter.type == "event"- and property_filter.key != "$feature_flag"- and operator_is_negative(property_filter.operator)- )--def _filter_on_dynamic_cohort(ctx: HealthContext, events: ExposureEvents, cohorts: list[ExposureCohort]) -> ExperimentHealthFinding:@@ -130,30 +110,3 @@ def _names(cohorts: list[ExposureCohort]) -> str:def _them(cohorts: list[ExposureCohort]) -> str:return "them" if len(cohorts) > 1 else "it"---def _filter_excludes_events(ctx: HealthContext, filters: list[ExposurePropertyFilter]) -> ExperimentHealthFinding:- keys = list(dict.fromkeys(f.key for f in filters))- subject = (- f"The filter on the event property {quoted(keys)} in the exposure criteria leaves"- if len(keys) == 1- else f"The filters on the event properties {quoted(keys)} in the exposure criteria leave"- )- advice = (- " To keep users out of the experiment, leave them out in the feature flag's release conditions."- if not ctx.is_launched- else ""- )- return ExperimentHealthFinding(- code=ExperimentHealthFindingCode.EXPOSURE_FILTER_EXCLUDES_EVENTS,- subcode=None,- severity=ExperimentHealthFindingSeverity.INFO,- title="An exposure filter leaves out events, not users",- detail=(- f"{subject} out single events, not the users who sent them. A user with a left-out event is still counted "- f"when another of their events passes.{advice}"- ),- evidence={"properties": ", ".join(keys)},- actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),- diagnostic_ref=None,- )diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex dd0fe8703103..b2d1e15445fe 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -117,7 +117,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (),HealthCheck(key=ExperimentHealthCheckKey.EXPOSURE_FILTERS,- name="Exposure filters that count unexpected users",+ name="Dynamic cohorts in the exposure criteria",run=exposure_filters,),HealthCheck(diff --git a/products/experiments/backend/health/test/test_exposure_filters.py b/products/experiments/backend/health/test/test_exposure_filters.pyindex 66c477030df5..18f43755126d 100644--- a/products/experiments/backend/health/test/test_exposure_filters.py+++ b/products/experiments/backend/health/test/test_exposure_filters.py@@ -45,7 +45,7 @@ Code = ExperimentHealthFindingCodeSeverity = ExperimentHealthFindingSeverityDYNAMIC = ExposureCohort(id=7, name="Paying users", is_static=False)STATIC = ExposureCohort(id=8, name="QA team", is_static=True)-NO_FILTERS = not_applicable("The exposure criteria have no cohort filter and no negated event property filter")+NO_COHORT_FILTER = not_applicable("The exposure criteria have no cohort filter")def _filters(@@ -115,21 +115,14 @@ class TestExposureFilters(TestCase):COHORT_WARNING,),("dynamic_cohort_after_the_end", _filters(cohorts=(DYNAMIC,), **ENDED), COHORT_WARNING),- (- "dynamic_cohort_before_an_event_exclusion",- _filters(EXCLUDED_URL, cohorts=(STATIC, DYNAMIC)),- COHORT_WARNING,- ),+ ("dynamic_cohort_next_to_a_static_cohort", _filters(cohorts=(STATIC, DYNAMIC)), COHORT_WARNING),("static_cohort", _filters(cohorts=(STATIC,)), None),("deleted_cohort",_filters(cohorts=(STATIC,), cohort_ids=(8, 9)),not_checked("Couldn't check because a cohort in the exposure criteria is deleted or missing"),),- ("event_exclusion", _filters(EXCLUDED_URL), (Code.EXPOSURE_FILTER_EXCLUDES_EVENTS, None, Severity.INFO)),- ("event_filter_that_selects", _filters(("event", "$current_url", PropertyOperator.ICONTAINS)), NO_FILTERS),- ("person_exclusion", _filters(("person", "email", PropertyOperator.NOT_ICONTAINS)), NO_FILTERS),- ("flag_key_exclusion", _filters(("event", "$feature_flag", PropertyOperator.IS_NOT)), NO_FILTERS),+ ("event_exclusion_without_a_cohort", _filters(EXCLUDED_URL), NO_COHORT_FILTER),("archived", _filters(cohorts=(DYNAMIC,), archived=True), not_applicable(NOT_ON_ARCHIVED)),])diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex c53b76876d51..d918071cf58c 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -992,15 +992,13 @@ class EnterpriseExperimentsViewSet(warning in every state, `filter_on_property_not_ingested` also reports after the end, andthe other findings stop at the end.- `exposure_filters` reads the cohort and event property filters of the exposure criteria and- of the activation, not the test account filters. `exposure_filter_on_dynamic_cohort` is a- warning in every state: the results use the members of a dynamic cohort at the calculation,- not at exposure. Its subcode is `cohort_forces_variant` when a release condition of the flag- that matches the cohort's members gives one variant, `cohort_in_release_conditions` when the- release conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts.- Both subcodes need the exposure criteria to keep the members too, not to leave them out.- `exposure_filter_excludes_events` is info: a negated filter on an event property leaves out- events, not users (`evidence.properties`).+ `exposure_filters` reads the cohort filters of the exposure criteria and of the activation,+ not the test account filters. `exposure_filter_on_dynamic_cohort` is a warning in every+ state: the results use the members of a dynamic cohort at the calculation, not at exposure.+ Its subcode is `cohort_forces_variant` when a release condition of the flag that matches the+ cohort's members gives one variant, `cohort_in_release_conditions` when the release+ conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts. Both+ subcodes need the exposure criteria to keep the members too, not to leave them out.`metric_cannot_compile` builds and compiles the query of each metric without running it, asthe caller, so a metric that every calculation fails on shows before launch. A metric thatdiff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex e07654d60e25..5c200dd3a89e 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2309,7 +2309,6 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]* * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures@@ -2339,7 +2338,6 @@ export const ExperimentHealthFindingCodeEnumApi = {MetricAllZero: 'metric_all_zero',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',- ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',Srm: 'srm',ZeroExposures: 'zero_exposures',TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',@@ -2412,7 +2410,6 @@ export interface ExperimentHealthFindingApi {* * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposuresdiff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex bcddfbf29978..7c1525836fdb 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -737,15 +737,13 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* warning in every state, `filter_on_property_not_ingested` also reports after the end, and* the other findings stop at the end.*- * `exposure_filters` reads the cohort and event property filters of the exposure criteria and- * of the activation, not the test account filters. `exposure_filter_on_dynamic_cohort` is a- * warning in every state: the results use the members of a dynamic cohort at the calculation,- * not at exposure. Its subcode is `cohort_forces_variant` when a release condition of the flag- * that matches the cohort's members gives one variant, `cohort_in_release_conditions` when the- * release conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts.- * Both subcodes need the exposure criteria to keep the members too, not to leave them out.- * `exposure_filter_excludes_events` is info: a negated filter on an event property leaves out- * events, not users (`evidence.properties`).+ * `exposure_filters` reads the cohort filters of the exposure criteria and of the activation,+ * not the test account filters. `exposure_filter_on_dynamic_cohort` is a warning in every+ * state: the results use the members of a dynamic cohort at the calculation, not at exposure.+ * Its subcode is `cohort_forces_variant` when a release condition of the flag that matches the+ * cohort's members gives one variant, `cohort_in_release_conditions` when the release+ * conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts. Both+ * subcodes need the exposure criteria to keep the members too, not to leave them out.** `metric_cannot_compile` builds and compiles the query of each metric without running it, as* the caller, so a metric that every calculation fails on shows before launch. A metric thatdiff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex 31dbd6dac523..1054ec6c91a8 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -267,27 +267,18 @@ function exposureCriteriaNote({ subcode, evidence }: ExperimentHealthFindingApi)return `Not received: ${evidence.event}.`}-const EXPOSURE_FILTERS_FINDING_CODES: ReadonlySet<ExperimentHealthFindingApi['code']> = new Set([- 'exposure_filter_on_dynamic_cohort',- 'exposure_filter_excludes_events',-])-function exposureFiltersCheck(input: HealthDebugInput): HealthDebugCheck {- const finding = input.health?.findings.find(({ code }) => EXPOSURE_FILTERS_FINDING_CODES.has(code)) ?? null+ const finding = findingOf(input, 'exposure_filter_on_dynamic_cohort')return {check: 'exposure_filters',source: 'health endpoint',reads: ['experiment', 'flag', 'project'],- ...serverResult('exposure_filters', input.health, (code) => EXPOSURE_FILTERS_FINDING_CODES.has(code)),+ ...serverResult('exposure_filters', input.health, (code) => code === 'exposure_filter_on_dynamic_cohort'),browser: null,browserLabel: BROWSER_RULES,...NO_TIMES,difference: null,- note: !finding- ? null- : finding.code === 'exposure_filter_on_dynamic_cohort'- ? `Dynamic cohorts: ${finding.evidence.cohort_ids}.`- : `Leaves out events by: ${finding.evidence.properties}.`,+ note: finding ? `Dynamic cohorts: ${finding.evidence.cohort_ids}.` : null,}}diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex e7e466fef090..dd1fb0b0219a 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -34,7 +34,6 @@ export type ExperimentHealthFindingCode =| 'metric_all_zero'| 'exposure_criteria_cannot_match'| 'exposure_filter_on_dynamic_cohort'- | 'exposure_filter_excludes_events'| 'forced_variant_release_condition'| 'flag_variants_mismatch'| 'split_changed_mid_run'diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex fe407a448b8b..23866188e946 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -48139,7 +48139,6 @@ export namespace Schemas {* * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures@@ -48169,7 +48168,6 @@ export namespace Schemas {MetricAllZero: 'metric_all_zero',ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',- ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',Srm: 'srm',ZeroExposures: 'zero_exposures',TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',@@ -48242,7 +48240,6 @@ export namespace Schemas {* * `metric_all_zero` - Metric All Zero* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events* * `srm` - Sample Ratio Mismatch* * `zero_exposures` - Zero Exposures* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures