Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): stop reporting negated exposure filters as findings (#126812)

Marcel Poelkerc3edd81da884
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 933ad1df680b..0bb6eeaba5fe 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10072,7 +10072,6 @@ export namespace Schemas {
* * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
@@ -10099,7 +10098,6 @@ export namespace Schemas {
| "metric_all_zero"
| "exposure_criteria_cannot_match"
| "exposure_filter_on_dynamic_cohort"
- | "exposure_filter_excludes_events"
| "srm"
| "zero_exposures"
| "test_account_filter_removes_exposures"
@@ -10156,7 +10154,6 @@ export namespace Schemas {
* * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index bc19fa497ee9..d604c05035f8 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -205,7 +205,6 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
METRIC_ALL_ZERO = "metric_all_zero"
EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"
EXPOSURE_FILTER_ON_DYNAMIC_COHORT = "exposure_filter_on_dynamic_cohort"
- EXPOSURE_FILTER_EXCLUDES_EVENTS = "exposure_filter_excludes_events"
SRM = "srm", "Sample Ratio Mismatch"
ZERO_EXPOSURES = "zero_exposures"
TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES = "test_account_filter_removes_exposures"
diff --git a/products/experiments/backend/health/checks/exposure_filters.py b/products/experiments/backend/health/checks/exposure_filters.py
index 3a22e06646d6..99677f5af911 100644
--- a/products/experiments/backend/health/checks/exposure_filters.py
+++ b/products/experiments/backend/health/checks/exposure_filters.py
@@ -1,13 +1,10 @@
"""
-Exposure criteria filters that count other users than they read like. Pure functions, no I/O.
+Cohort filters of the exposure criteria that count other users than they read like. Pure functions, no I/O.
-The exposure query applies each filter to the events that it reads, and reads a cohort's members when it runs. So a
-dynamic cohort counts the users who are in it at the calculation, not at their exposure, and a filter that leaves out
-events with a property value keeps a user who sent another exposure event.
+The exposure query reads a cohort's members when it runs, so a dynamic cohort counts the users who are in it at the
+calculation, not at their exposure.
"""
-from posthog.hogql.property import operator_is_negative
-
from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
ExperimentHealthFindingActionKind,
@@ -15,12 +12,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.checks.events_received import quoted
-from products.experiments.backend.health.context import (
- ExposureCohort,
- ExposureEvents,
- ExposurePropertyFilter,
- HealthContext,
-)
+from products.experiments.backend.health.context import ExposureCohort, ExposureEvents, HealthContext
from products.experiments.backend.health.skips import NOT_ON_ARCHIVED, CheckOutcome, not_applicable, not_checked
# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.
@@ -34,28 +26,16 @@ def exposure_filters(ctx: HealthContext) -> CheckOutcome:
events = ctx.exposure_events
if events is None:
return not_checked("Couldn't check because the exposure criteria can't be read")
- exclusions = [f for f in events.property_filters if _excludes_events(f)]
- if not events.cohort_ids and not exclusions:
- return not_applicable("The exposure criteria have no cohort filter and no negated event property filter")
+ if not events.cohort_ids:
+ return not_applicable("The exposure criteria have no cohort filter")
dynamic = [cohort for cohort in ctx.exposure_cohorts if not cohort.is_static]
if dynamic:
return _filter_on_dynamic_cohort(ctx, events, dynamic)
- if exclusions:
- return _filter_excludes_events(ctx, exclusions)
if len(ctx.exposure_cohorts) < len(events.cohort_ids):
return not_checked("Couldn't check because a cohort in the exposure criteria is deleted or missing")
return None
-def _excludes_events(property_filter: ExposurePropertyFilter) -> bool:
- # A person property has one value per user at a time, so a filter on it reads as leaving out the user.
- return (
- property_filter.type == "event"
- and property_filter.key != "$feature_flag"
- and operator_is_negative(property_filter.operator)
- )
-
-
def _filter_on_dynamic_cohort(
ctx: HealthContext, events: ExposureEvents, cohorts: list[ExposureCohort]
) -> ExperimentHealthFinding:
@@ -130,30 +110,3 @@ def _names(cohorts: list[ExposureCohort]) -> str:
def _them(cohorts: list[ExposureCohort]) -> str:
return "them" if len(cohorts) > 1 else "it"
-
-
-def _filter_excludes_events(ctx: HealthContext, filters: list[ExposurePropertyFilter]) -> ExperimentHealthFinding:
- keys = list(dict.fromkeys(f.key for f in filters))
- subject = (
- f"The filter on the event property {quoted(keys)} in the exposure criteria leaves"
- if len(keys) == 1
- else f"The filters on the event properties {quoted(keys)} in the exposure criteria leave"
- )
- advice = (
- " To keep users out of the experiment, leave them out in the feature flag's release conditions."
- if not ctx.is_launched
- else ""
- )
- return ExperimentHealthFinding(
- code=ExperimentHealthFindingCode.EXPOSURE_FILTER_EXCLUDES_EVENTS,
- subcode=None,
- severity=ExperimentHealthFindingSeverity.INFO,
- title="An exposure filter leaves out events, not users",
- detail=(
- f"{subject} out single events, not the users who sent them. A user with a left-out event is still counted "
- f"when another of their events passes.{advice}"
- ),
- evidence={"properties": ", ".join(keys)},
- actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),
- diagnostic_ref=None,
- )
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index dd0fe8703103..b2d1e15445fe 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -117,7 +117,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_FILTERS,
- name="Exposure filters that count unexpected users",
+ name="Dynamic cohorts in the exposure criteria",
run=exposure_filters,
),
HealthCheck(
diff --git a/products/experiments/backend/health/test/test_exposure_filters.py b/products/experiments/backend/health/test/test_exposure_filters.py
index 66c477030df5..18f43755126d 100644
--- a/products/experiments/backend/health/test/test_exposure_filters.py
+++ b/products/experiments/backend/health/test/test_exposure_filters.py
@@ -45,7 +45,7 @@ Code = ExperimentHealthFindingCode
Severity = ExperimentHealthFindingSeverity
DYNAMIC = ExposureCohort(id=7, name="Paying users", is_static=False)
STATIC = ExposureCohort(id=8, name="QA team", is_static=True)
-NO_FILTERS = not_applicable("The exposure criteria have no cohort filter and no negated event property filter")
+NO_COHORT_FILTER = not_applicable("The exposure criteria have no cohort filter")
def _filters(
@@ -115,21 +115,14 @@ class TestExposureFilters(TestCase):
COHORT_WARNING,
),
("dynamic_cohort_after_the_end", _filters(cohorts=(DYNAMIC,), **ENDED), COHORT_WARNING),
- (
- "dynamic_cohort_before_an_event_exclusion",
- _filters(EXCLUDED_URL, cohorts=(STATIC, DYNAMIC)),
- COHORT_WARNING,
- ),
+ ("dynamic_cohort_next_to_a_static_cohort", _filters(cohorts=(STATIC, DYNAMIC)), COHORT_WARNING),
("static_cohort", _filters(cohorts=(STATIC,)), None),
(
"deleted_cohort",
_filters(cohorts=(STATIC,), cohort_ids=(8, 9)),
not_checked("Couldn't check because a cohort in the exposure criteria is deleted or missing"),
),
- ("event_exclusion", _filters(EXCLUDED_URL), (Code.EXPOSURE_FILTER_EXCLUDES_EVENTS, None, Severity.INFO)),
- ("event_filter_that_selects", _filters(("event", "$current_url", PropertyOperator.ICONTAINS)), NO_FILTERS),
- ("person_exclusion", _filters(("person", "email", PropertyOperator.NOT_ICONTAINS)), NO_FILTERS),
- ("flag_key_exclusion", _filters(("event", "$feature_flag", PropertyOperator.IS_NOT)), NO_FILTERS),
+ ("event_exclusion_without_a_cohort", _filters(EXCLUDED_URL), NO_COHORT_FILTER),
("archived", _filters(cohorts=(DYNAMIC,), archived=True), not_applicable(NOT_ON_ARCHIVED)),
]
)
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index c53b76876d51..d918071cf58c 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -992,15 +992,13 @@ class EnterpriseExperimentsViewSet(
warning in every state, `filter_on_property_not_ingested` also reports after the end, and
the other findings stop at the end.
- `exposure_filters` reads the cohort and event property filters of the exposure criteria and
- of the activation, not the test account filters. `exposure_filter_on_dynamic_cohort` is a
- warning in every state: the results use the members of a dynamic cohort at the calculation,
- not at exposure. Its subcode is `cohort_forces_variant` when a release condition of the flag
- that matches the cohort's members gives one variant, `cohort_in_release_conditions` when the
- release conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts.
- Both subcodes need the exposure criteria to keep the members too, not to leave them out.
- `exposure_filter_excludes_events` is info: a negated filter on an event property leaves out
- events, not users (`evidence.properties`).
+ `exposure_filters` reads the cohort filters of the exposure criteria and of the activation,
+ not the test account filters. `exposure_filter_on_dynamic_cohort` is a warning in every
+ state: the results use the members of a dynamic cohort at the calculation, not at exposure.
+ Its subcode is `cohort_forces_variant` when a release condition of the flag that matches the
+ cohort's members gives one variant, `cohort_in_release_conditions` when the release
+ conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts. Both
+ subcodes need the exposure criteria to keep the members too, not to leave them out.
`metric_cannot_compile` builds and compiles the query of each metric without running it, as
the caller, so a metric that every calculation fails on shows before launch. A metric that
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index e07654d60e25..5c200dd3a89e 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2309,7 +2309,6 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
@@ -2339,7 +2338,6 @@ export const ExperimentHealthFindingCodeEnumApi = {
MetricAllZero: 'metric_all_zero',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',
- ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',
Srm: 'srm',
ZeroExposures: 'zero_exposures',
TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
@@ -2412,7 +2410,6 @@ export interface ExperimentHealthFindingApi {
* * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index bcddfbf29978..7c1525836fdb 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -737,15 +737,13 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* warning in every state, `filter_on_property_not_ingested` also reports after the end, and
* the other findings stop at the end.
*
- * `exposure_filters` reads the cohort and event property filters of the exposure criteria and
- * of the activation, not the test account filters. `exposure_filter_on_dynamic_cohort` is a
- * warning in every state: the results use the members of a dynamic cohort at the calculation,
- * not at exposure. Its subcode is `cohort_forces_variant` when a release condition of the flag
- * that matches the cohort's members gives one variant, `cohort_in_release_conditions` when the
- * release conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts.
- * Both subcodes need the exposure criteria to keep the members too, not to leave them out.
- * `exposure_filter_excludes_events` is info: a negated filter on an event property leaves out
- * events, not users (`evidence.properties`).
+ * `exposure_filters` reads the cohort filters of the exposure criteria and of the activation,
+ * not the test account filters. `exposure_filter_on_dynamic_cohort` is a warning in every
+ * state: the results use the members of a dynamic cohort at the calculation, not at exposure.
+ * Its subcode is `cohort_forces_variant` when a release condition of the flag that matches the
+ * cohort's members gives one variant, `cohort_in_release_conditions` when the release
+ * conditions match its members otherwise, and `evidence.cohort_ids` names the cohorts. Both
+ * subcodes need the exposure criteria to keep the members too, not to leave them out.
*
* `metric_cannot_compile` builds and compiles the query of each metric without running it, as
* the caller, so a metric that every calculation fails on shows before launch. A metric that
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index 31dbd6dac523..1054ec6c91a8 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -267,27 +267,18 @@ function exposureCriteriaNote({ subcode, evidence }: ExperimentHealthFindingApi)
return `Not received: ${evidence.event}.`
}
-const EXPOSURE_FILTERS_FINDING_CODES: ReadonlySet<ExperimentHealthFindingApi['code']> = new Set([
- 'exposure_filter_on_dynamic_cohort',
- 'exposure_filter_excludes_events',
-])
-
function exposureFiltersCheck(input: HealthDebugInput): HealthDebugCheck {
- const finding = input.health?.findings.find(({ code }) => EXPOSURE_FILTERS_FINDING_CODES.has(code)) ?? null
+ const finding = findingOf(input, 'exposure_filter_on_dynamic_cohort')
return {
check: 'exposure_filters',
source: 'health endpoint',
reads: ['experiment', 'flag', 'project'],
- ...serverResult('exposure_filters', input.health, (code) => EXPOSURE_FILTERS_FINDING_CODES.has(code)),
+ ...serverResult('exposure_filters', input.health, (code) => code === 'exposure_filter_on_dynamic_cohort'),
browser: null,
browserLabel: BROWSER_RULES,
...NO_TIMES,
difference: null,
- note: !finding
- ? null
- : finding.code === 'exposure_filter_on_dynamic_cohort'
- ? `Dynamic cohorts: ${finding.evidence.cohort_ids}.`
- : `Leaves out events by: ${finding.evidence.properties}.`,
+ note: finding ? `Dynamic cohorts: ${finding.evidence.cohort_ids}.` : null,
}
}
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index e7e466fef090..dd1fb0b0219a 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -34,7 +34,6 @@ export type ExperimentHealthFindingCode =
| 'metric_all_zero'
| 'exposure_criteria_cannot_match'
| 'exposure_filter_on_dynamic_cohort'
- | 'exposure_filter_excludes_events'
| 'forced_variant_release_condition'
| 'flag_variants_mismatch'
| 'split_changed_mid_run'
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index fe407a448b8b..23866188e946 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -48139,7 +48139,6 @@ export namespace Schemas {
* * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
@@ -48169,7 +48168,6 @@ export namespace Schemas {
MetricAllZero: 'metric_all_zero',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
ExposureFilterOnDynamicCohort: 'exposure_filter_on_dynamic_cohort',
- ExposureFilterExcludesEvents: 'exposure_filter_excludes_events',
Srm: 'srm',
ZeroExposures: 'zero_exposures',
TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
@@ -48242,7 +48240,6 @@ export namespace Schemas {
* * `metric_all_zero` - Metric All Zero
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `exposure_filter_on_dynamic_cohort` - Exposure Filter On Dynamic Cohort
- * * `exposure_filter_excludes_events` - Exposure Filter Excludes Events
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
* * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures