:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): read a missing filterTestAccounts as true (#119305)
Marcel Poelker8b194d139d4f
diff --git a/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsx b/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsxindex 5596ad3c9b3d..1ee19c49f273 100644--- a/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsx+++ b/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsx@@ -20,6 +20,7 @@ import {EXPOSURE_DEFAULT_EVENT,exposureEventLabel,getActivationConfig,+ resolveFilterTestAccounts,resolvedExposureEvent,} from '../exposureContract'import { commonActionFilterProps } from '../Metrics/Selectors'@@ -229,7 +230,7 @@ function ExposureCriteriaFields({{/* Test Account Filtering */}<div><TestAccountFilterSwitch- checked={hasFilters && !!experiment.exposure_criteria?.filterTestAccounts}+ checked={hasFilters && resolveFilterTestAccounts(experiment.exposure_criteria)}onChange={(checked: boolean) => {onChange({ filterTestAccounts: checked })}}@@ -407,7 +408,7 @@ export function ExposureCriteriaPanel({ experiment, onChange, compact }: Exposur{/* As tall as the small selects above, so the three labels are evenly spaced */}<div className="flex min-h-[2.0625rem] items-center"><TestAccountFilterSwitch- checked={hasFilters && !!experiment.exposure_criteria?.filterTestAccounts}+ checked={hasFilters && resolveFilterTestAccounts(experiment.exposure_criteria)}onChange={(checked: boolean) => {onChange({ filterTestAccounts: checked })}}diff --git a/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsx b/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsxindex 4e799fb5214f..5875993455e2 100644--- a/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsx+++ b/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsx@@ -1,5 +1,6 @@import { useActions } from 'kea'+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'import { SharedMetric } from 'scenes/experiments/SharedMetrics/sharedMetricLogic'import type { ExperimentExposureCriteria, ExperimentMetric } from '~/queries/schema/schema-general'@@ -51,7 +52,7 @@ export const MetricsPanel = ({const { closeSharedMetricModal } = useActions(sharedMetricModalLogic)// we need this value to calculate the recent activity on the metrics list- const filterTestAccounts = experiment.exposure_criteria?.filterTestAccounts ?? false+ const filterTestAccounts = resolveFilterTestAccounts(experiment.exposure_criteria)const primaryMetrics = [...(experiment.metrics || []).filter(isExperimentMetric), ...sharedMetrics.primary]const secondaryMetrics = [diff --git a/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsx b/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsxindex 659a2f3dd88e..d624ce777db2 100644--- a/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsx+++ b/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsx@@ -16,7 +16,7 @@ import { ChartDisplayCategory } from '~/types'import { SelectableCard } from 'products/experiments/frontend/components/SelectableCard'import { experimentLogic } from '../experimentLogic'-import { EXPOSURE_DEFAULT_EVENT, getActivationConfig } from '../exposureContract'+import { EXPOSURE_DEFAULT_EVENT, getActivationConfig, resolveFilterTestAccounts } from '../exposureContract'import { commonActionFilterProps } from '../Metrics/Selectors'import { exposureConfigToSeries, seriesToExposureConfig } from '../utils'import { exposureCriteriaModalLogic } from './exposureCriteriaModalLogic'@@ -238,10 +238,7 @@ export function ExposureCriteriaModal({ onSave }: ExposureCriteriaModalProps): J</div></div><TestAccountFilterSwitch- checked={(() => {- const val = exposureCriteria?.filterTestAccounts- return hasFilters ? !!val : false- })()}+ checked={hasFilters && resolveFilterTestAccounts(exposureCriteria)}onChange={(checked: boolean) => {setExposureCriteria({...exposureCriteria,diff --git a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.ts b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.tsindex e86e10cb520d..2275f2fabdc6 100644--- a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.ts+++ b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.ts@@ -444,6 +444,14 @@ describe('experimentReplayTabLogic', () => {// event-shaped stands in for it in the filter tree.expect(recordingsFilters.experiment_exposure).toEqual({ experiment_id: 42 })expect(recordingsFilters.filter_group).toEqual(EMPTY_FILTER_GROUP)++ // Criteria without the key exclude test accounts from the results, so the filter bar must show them excluded.+ const withoutKey = experimentReplayTabLogic({+ experiment: { ...EXPERIMENT, id: 69, exposure_criteria: {} } as Experiment,+ })+ withoutKey.mount()+ expect(withoutKey.values.recordingsFilters.filter_test_accounts).toBe(true)+ withoutKey.unmount()})it('keeps the selected variant across remounts, in step with the playlist persisting its filters', async () => {diff --git a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.ts b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.tsindex fa813e3a7d2b..e3438d3c1fb1 100644--- a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.ts+++ b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.ts@@ -38,6 +38,7 @@ import {} from 'lib/utils/eventUsageLogic'import { objectsEqual } from 'lib/utils/objects'import { addProductIntentForCrossSell } from 'lib/utils/product-intents'+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'import { HideViewedRecordingsOptions, playerSettingsLogic } from 'scenes/session-recordings/player/playerSettingsLogic'import { playerSidebarLogic } from 'scenes/session-recordings/player/sidebar/playerSidebarLogic'import {@@ -1939,7 +1940,7 @@ export const experimentReplayTabLogic = kea<experimentReplayTabLogicType>([duration: selectedWatchCard ? [] : DEFAULT_RECORDING_FILTERS.duration,date_from: experiment.start_date ?? DEFAULT_RECORDING_FILTERS.date_from,date_to: experiment.end_date ?? null,- filter_test_accounts: experiment.exposure_criteria?.filterTestAccounts ?? false,+ filter_test_accounts: resolveFilterTestAccounts(experiment.exposure_criteria),// Resolved server-side from the experiment (same population the analysis// counts), so it works even when exposure events are fired server-side.// Deliberately not an event filter in `filter_group`. The in-sessiondiff --git a/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsx b/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsxindex b63993d51eec..13a38637a32a 100644--- a/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsx+++ b/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsx@@ -250,6 +250,55 @@ describe('experimentActivityDescriber', () => {})})+ describe('test account filter rows', () => {+ it.each([+ {+ name: 'a write that stores the default adds no line',+ before: null,+ after: { filterTestAccounts: true },+ expected: null,+ },+ {+ name: 'turning off a filter that had no stored value',+ before: {},+ after: { filterTestAccounts: false },+ expected: 'removed the test account filter',+ },+ {+ name: 'a write that drops an explicit false turns the filter back on',+ before: { filterTestAccounts: false },+ after: {},+ expected: 'added the test account filter',+ },+ ])('$name', ({ before, after, expected }) => {+ const result = experimentActivityDescriber(+ baseLogItem({+ activity: 'updated',+ detail: {+ name: 'Checkout funnel',+ changes: [+ {+ type: ActivityScope.EXPERIMENT,+ action: 'changed',+ field: 'exposure_criteria',+ before,+ after,+ },+ ],+ merge: null,+ trigger: null,+ },+ })+ )+ const text = textOf(result)+ const described =+ ['added the test account filter', 'removed the test account filter'].find((line) =>+ text.includes(line)+ ) ?? null+ expect(described).toBe(expected)+ })+ })+describe('running_time_calculation rows', () => {it('describes a running_time_calculation change', () => {const result = experimentActivityDescriber(diff --git a/frontend/src/scenes/experiments/exposureContract.ts b/frontend/src/scenes/experiments/exposureContract.tsindex 588b2c37904d..66ccb1bcba62 100644--- a/frontend/src/scenes/experiments/exposureContract.ts+++ b/frontend/src/scenes/experiments/exposureContract.ts@@ -38,6 +38,17 @@ export function resolvedExposureEvent(return experiment.resolved_exposure_event || fallbackEvent}+/**+ * Whether the experiment's analysis excludes test accounts. Mirrors the backend's+ * `resolve_filter_test_accounts`: criteria without the key mean the default, not an opt-out. An+ * experiment stored before creation set the key has none, and so does one whose criteria an+ * update replaced without it. A surface that reads absence as false counts different people than+ * the results do.+ */+export function resolveFilterTestAccounts(exposureCriteria: ExperimentExposureCriteria | null | undefined): boolean {+ return exposureCriteria?.filterTestAccounts ?? true+}+export function exposureEventLabel(event: string): string {return event === EXPERIMENT_EXPOSURE_EVENT ? 'Experiment exposure' : 'Feature flag is called'}diff --git a/frontend/src/scenes/experiments/replayVisionScanner.test.ts b/frontend/src/scenes/experiments/replayVisionScanner.test.tsindex 8cfae2813257..2ffc2ed2a154 100644--- a/frontend/src/scenes/experiments/replayVisionScanner.test.ts+++ b/frontend/src/scenes/experiments/replayVisionScanner.test.ts@@ -44,7 +44,7 @@ describe('replayVisionScanner', () => {{name: 'a default exposure event',exposure_criteria: undefined,- expectedFilterTestAccounts: false,+ expectedFilterTestAccounts: true,},{name: 'a custom exposure event',@@ -55,12 +55,12 @@ describe('replayVisionScanner', () => {properties: [],},} satisfies ExperimentExposureCriteria as ExperimentExposureCriteria,- expectedFilterTestAccounts: false,+ expectedFilterTestAccounts: true,},{- name: 'an experiment that filters test accounts',- exposure_criteria: { filterTestAccounts: true } satisfies ExperimentExposureCriteria,- expectedFilterTestAccounts: true,+ name: 'an experiment that keeps test accounts',+ exposure_criteria: { filterTestAccounts: false } satisfies ExperimentExposureCriteria,+ expectedFilterTestAccounts: false,},])('saves an experiment scanner that waits for launch, with no filters: $name',diff --git a/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsx b/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsxindex 1d4b54d5c198..8a8444a4a7ec 100644--- a/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsx+++ b/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsx@@ -6,6 +6,7 @@ import { ActivityChange } from 'lib/components/ActivityLog/humanizeActivity'import { dayjs } from 'lib/dayjs'import { LemonTag } from 'lib/lemon-ui/LemonTag'import { Link } from 'lib/lemon-ui/Link'+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'import { getExposureConfigDisplayName } from 'scenes/experiments/utils'import { urls } from 'scenes/urls'@@ -123,6 +124,22 @@ const describeMetricReorder = (before: unknown, after: unknown, description: strreturn equal(membershipOnlyRewrite, a) ? null : description}+/**+ * Compares the setting the analysis applies on each side, not the stored key. A write that adds or+ * drops the key without changing the setting gets no line, and a write that drops an explicit+ * `false` turns the filter back on.+ */+function describeTestAccountFilterChange(+ before: ExperimentExposureCriteria | null,+ after: ExperimentExposureCriteria | null+): string | null {+ const filteredAfter = resolveFilterTestAccounts(after)+ if (resolveFilterTestAccounts(before) === filteredAfter) {+ return null+ }+ return filteredAfter ? 'added the test account filter' : 'removed the test account filter'+}+export const getExperimentChangeDescription = (experimentChange: ActivityChange): string | JSX.Element | (string | JSX.Element)[] | null => {@@ -235,15 +252,8 @@ export const getExperimentChangeDescription = (const changes: (string | JSX.Element | null)[] = Object.keys(after || {}).map((key) =>match(key as keyof ExperimentExposureCriteria)- .with('filterTestAccounts', () => {- if (typedAfter?.filterTestAccounts === typedBefore?.filterTestAccounts) {- return null- }-- return typedAfter?.filterTestAccounts- ? 'added the test account filter'- : 'removed the test account filter'- })+ // Described below, because a write that drops the key can change the setting too.+ .with('filterTestAccounts', () => null).with('multiple_variant_handling', () => {if (typedAfter?.multiple_variant_handling === typedBefore?.multiple_variant_handling) {return null@@ -292,6 +302,8 @@ export const getExperimentChangeDescription = (.exhaustive())+ changes.unshift(describeTestAccountFilterChange(typedBefore, typedAfter))+// Check if exposure_config was removed (returning to default)if (typedBefore?.exposure_config && !typedAfter?.exposure_config) {changes.push(diff --git a/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsx b/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsxindex aba8234557e3..efb0d1216b13 100644--- a/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsx+++ b/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsx@@ -4,7 +4,7 @@ import { LemonButton, LemonDialog, LemonInput, LemonLabel, LemonModal } from '@pimport { ExperimentMetricForm } from 'scenes/experiments/ExperimentMetricForm'import { exposureCriteriaModalLogic } from 'scenes/experiments/ExperimentView/exposureCriteriaModalLogic'-import { resolvedExposureEvent } from 'scenes/experiments/exposureContract'+import { resolveFilterTestAccounts, resolvedExposureEvent } from 'scenes/experiments/exposureContract'import type { ExperimentExposureCriteria, ExperimentMetric } from '~/queries/schema/schema-general'import type { Experiment } from '~/types'@@ -97,7 +97,7 @@ export function ExperimentMetricModal({<ExperimentMetricFormmetric={metric}handleSetMetric={setModalMetric}- filterTestAccounts={experiment.exposure_criteria?.filterTestAccounts || false}+ filterTestAccounts={resolveFilterTestAccounts(experiment.exposure_criteria)}exposureCriteria={exposureCriteria}openExposureCriteriaModal={openExposureCriteriaModal}resolvedExposureEvent={resolvedExposureEvent(experiment)}diff --git a/products/replay_vision/frontend/replay_scanners/experimentTargeting.ts b/products/replay_vision/frontend/replay_scanners/experimentTargeting.tsindex 7b5df773c5cf..89f722aacf0c 100644--- a/products/replay_vision/frontend/replay_scanners/experimentTargeting.ts+++ b/products/replay_vision/frontend/replay_scanners/experimentTargeting.ts@@ -1,3 +1,4 @@+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'import { getExperimentVariants } from 'scenes/experiments/utils'import { NodeKind, type RecordingsQuery } from '~/queries/schema/schema-general'@@ -93,7 +94,7 @@ export function scopeVariantsLabel(scope: ScannerExperimentScope, everyVariant:export function experimentScannerQuery(experiment: Experiment): Pick<RecordingsQuery, 'kind' | 'filter_test_accounts'> {return {kind: NodeKind.RecordingsQuery,- filter_test_accounts: experiment.exposure_criteria?.filterTestAccounts ?? false,+ filter_test_accounts: resolveFilterTestAccounts(experiment.exposure_criteria),}}diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.tsindex e932c132a6c0..843e849fc16c 100644--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts@@ -469,7 +469,12 @@ describe('replayScannerLogic', () => {logic.actions.draftScannerFromGoal('where do people give up in billing')).toFinishAllListeners()- expect(logic.values.scanner).toMatchObject({ name: 'Billing drop-off: Checkout redesign', ...expected })+ // The mocked experiment stores no test-account setting, which its results read as on.+ expect(logic.values.scanner).toMatchObject({+ name: 'Billing drop-off: Checkout redesign',+ query: { kind: 'RecordingsQuery', filter_test_accounts: true },+ ...expected,+ })})diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.tsindex dc708b59d7bc..c4a966545350 100644--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts@@ -76,6 +76,7 @@ import {buildExperimentTargeting,experimentScannerConfig,experimentScannerName,+ experimentScannerQuery,parseExperimentScannerParams,prefillScannerForExperiment,ScannerExperimentScope,@@ -2396,11 +2397,7 @@ export const replayScannerLogic = kea<replayScannerLogicType>([...(draftQuery? {query: context- ? {- ...draftQuery,- filter_test_accounts:- context.experiment.exposure_criteria?.filterTestAccounts ?? false,- }+ ? { ...draftQuery, ...experimentScannerQuery(context.experiment) }: draftQuery,}: {}),