Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): read a missing filterTestAccounts as true (#119305)

Marcel Poelker8b194d139d4f
diff --git a/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsx b/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsx
index 5596ad3c9b3d..1ee19c49f273 100644
--- a/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsx
+++ b/frontend/src/scenes/experiments/ExperimentForm/ExposureCriteriaPanel.tsx
@@ -20,6 +20,7 @@ import {
EXPOSURE_DEFAULT_EVENT,
exposureEventLabel,
getActivationConfig,
+ resolveFilterTestAccounts,
resolvedExposureEvent,
} from '../exposureContract'
import { commonActionFilterProps } from '../Metrics/Selectors'
@@ -229,7 +230,7 @@ function ExposureCriteriaFields({
{/* Test Account Filtering */}
<div>
<TestAccountFilterSwitch
- checked={hasFilters && !!experiment.exposure_criteria?.filterTestAccounts}
+ checked={hasFilters && resolveFilterTestAccounts(experiment.exposure_criteria)}
onChange={(checked: boolean) => {
onChange({ filterTestAccounts: checked })
}}
@@ -407,7 +408,7 @@ export function ExposureCriteriaPanel({ experiment, onChange, compact }: Exposur
{/* As tall as the small selects above, so the three labels are evenly spaced */}
<div className="flex min-h-[2.0625rem] items-center">
<TestAccountFilterSwitch
- checked={hasFilters && !!experiment.exposure_criteria?.filterTestAccounts}
+ checked={hasFilters && resolveFilterTestAccounts(experiment.exposure_criteria)}
onChange={(checked: boolean) => {
onChange({ filterTestAccounts: checked })
}}
diff --git a/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsx b/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsx
index 4e799fb5214f..5875993455e2 100644
--- a/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsx
+++ b/frontend/src/scenes/experiments/ExperimentForm/MetricsPanel/MetricsPanel.tsx
@@ -1,5 +1,6 @@
import { useActions } from 'kea'
+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'
import { SharedMetric } from 'scenes/experiments/SharedMetrics/sharedMetricLogic'
import type { ExperimentExposureCriteria, ExperimentMetric } from '~/queries/schema/schema-general'
@@ -51,7 +52,7 @@ export const MetricsPanel = ({
const { closeSharedMetricModal } = useActions(sharedMetricModalLogic)
// we need this value to calculate the recent activity on the metrics list
- const filterTestAccounts = experiment.exposure_criteria?.filterTestAccounts ?? false
+ const filterTestAccounts = resolveFilterTestAccounts(experiment.exposure_criteria)
const primaryMetrics = [...(experiment.metrics || []).filter(isExperimentMetric), ...sharedMetrics.primary]
const secondaryMetrics = [
diff --git a/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsx b/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsx
index 659a2f3dd88e..d624ce777db2 100644
--- a/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/ExposureCriteria.tsx
@@ -16,7 +16,7 @@ import { ChartDisplayCategory } from '~/types'
import { SelectableCard } from 'products/experiments/frontend/components/SelectableCard'
import { experimentLogic } from '../experimentLogic'
-import { EXPOSURE_DEFAULT_EVENT, getActivationConfig } from '../exposureContract'
+import { EXPOSURE_DEFAULT_EVENT, getActivationConfig, resolveFilterTestAccounts } from '../exposureContract'
import { commonActionFilterProps } from '../Metrics/Selectors'
import { exposureConfigToSeries, seriesToExposureConfig } from '../utils'
import { exposureCriteriaModalLogic } from './exposureCriteriaModalLogic'
@@ -238,10 +238,7 @@ export function ExposureCriteriaModal({ onSave }: ExposureCriteriaModalProps): J
</div>
</div>
<TestAccountFilterSwitch
- checked={(() => {
- const val = exposureCriteria?.filterTestAccounts
- return hasFilters ? !!val : false
- })()}
+ checked={hasFilters && resolveFilterTestAccounts(exposureCriteria)}
onChange={(checked: boolean) => {
setExposureCriteria({
...exposureCriteria,
diff --git a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.ts b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.ts
index e86e10cb520d..2275f2fabdc6 100644
--- a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.ts
+++ b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.test.ts
@@ -444,6 +444,14 @@ describe('experimentReplayTabLogic', () => {
// event-shaped stands in for it in the filter tree.
expect(recordingsFilters.experiment_exposure).toEqual({ experiment_id: 42 })
expect(recordingsFilters.filter_group).toEqual(EMPTY_FILTER_GROUP)
+
+ // Criteria without the key exclude test accounts from the results, so the filter bar must show them excluded.
+ const withoutKey = experimentReplayTabLogic({
+ experiment: { ...EXPERIMENT, id: 69, exposure_criteria: {} } as Experiment,
+ })
+ withoutKey.mount()
+ expect(withoutKey.values.recordingsFilters.filter_test_accounts).toBe(true)
+ withoutKey.unmount()
})
it('keeps the selected variant across remounts, in step with the playlist persisting its filters', async () => {
diff --git a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.ts b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.ts
index fa813e3a7d2b..e3438d3c1fb1 100644
--- a/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.ts
+++ b/frontend/src/scenes/experiments/ExperimentView/experimentReplayTabLogic.ts
@@ -38,6 +38,7 @@ import {
} from 'lib/utils/eventUsageLogic'
import { objectsEqual } from 'lib/utils/objects'
import { addProductIntentForCrossSell } from 'lib/utils/product-intents'
+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'
import { HideViewedRecordingsOptions, playerSettingsLogic } from 'scenes/session-recordings/player/playerSettingsLogic'
import { playerSidebarLogic } from 'scenes/session-recordings/player/sidebar/playerSidebarLogic'
import {
@@ -1939,7 +1940,7 @@ export const experimentReplayTabLogic = kea<experimentReplayTabLogicType>([
duration: selectedWatchCard ? [] : DEFAULT_RECORDING_FILTERS.duration,
date_from: experiment.start_date ?? DEFAULT_RECORDING_FILTERS.date_from,
date_to: experiment.end_date ?? null,
- filter_test_accounts: experiment.exposure_criteria?.filterTestAccounts ?? false,
+ filter_test_accounts: resolveFilterTestAccounts(experiment.exposure_criteria),
// Resolved server-side from the experiment (same population the analysis
// counts), so it works even when exposure events are fired server-side.
// Deliberately not an event filter in `filter_group`. The in-session
diff --git a/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsx b/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsx
index b63993d51eec..13a38637a32a 100644
--- a/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsx
+++ b/frontend/src/scenes/experiments/experimentActivityDescriber.test.tsx
@@ -250,6 +250,55 @@ describe('experimentActivityDescriber', () => {
})
})
+ describe('test account filter rows', () => {
+ it.each([
+ {
+ name: 'a write that stores the default adds no line',
+ before: null,
+ after: { filterTestAccounts: true },
+ expected: null,
+ },
+ {
+ name: 'turning off a filter that had no stored value',
+ before: {},
+ after: { filterTestAccounts: false },
+ expected: 'removed the test account filter',
+ },
+ {
+ name: 'a write that drops an explicit false turns the filter back on',
+ before: { filterTestAccounts: false },
+ after: {},
+ expected: 'added the test account filter',
+ },
+ ])('$name', ({ before, after, expected }) => {
+ const result = experimentActivityDescriber(
+ baseLogItem({
+ activity: 'updated',
+ detail: {
+ name: 'Checkout funnel',
+ changes: [
+ {
+ type: ActivityScope.EXPERIMENT,
+ action: 'changed',
+ field: 'exposure_criteria',
+ before,
+ after,
+ },
+ ],
+ merge: null,
+ trigger: null,
+ },
+ })
+ )
+ const text = textOf(result)
+ const described =
+ ['added the test account filter', 'removed the test account filter'].find((line) =>
+ text.includes(line)
+ ) ?? null
+ expect(described).toBe(expected)
+ })
+ })
+
describe('running_time_calculation rows', () => {
it('describes a running_time_calculation change', () => {
const result = experimentActivityDescriber(
diff --git a/frontend/src/scenes/experiments/exposureContract.ts b/frontend/src/scenes/experiments/exposureContract.ts
index 588b2c37904d..66ccb1bcba62 100644
--- a/frontend/src/scenes/experiments/exposureContract.ts
+++ b/frontend/src/scenes/experiments/exposureContract.ts
@@ -38,6 +38,17 @@ export function resolvedExposureEvent(
return experiment.resolved_exposure_event || fallbackEvent
}
+/**
+ * Whether the experiment's analysis excludes test accounts. Mirrors the backend's
+ * `resolve_filter_test_accounts`: criteria without the key mean the default, not an opt-out. An
+ * experiment stored before creation set the key has none, and so does one whose criteria an
+ * update replaced without it. A surface that reads absence as false counts different people than
+ * the results do.
+ */
+export function resolveFilterTestAccounts(exposureCriteria: ExperimentExposureCriteria | null | undefined): boolean {
+ return exposureCriteria?.filterTestAccounts ?? true
+}
+
export function exposureEventLabel(event: string): string {
return event === EXPERIMENT_EXPOSURE_EVENT ? 'Experiment exposure' : 'Feature flag is called'
}
diff --git a/frontend/src/scenes/experiments/replayVisionScanner.test.ts b/frontend/src/scenes/experiments/replayVisionScanner.test.ts
index 8cfae2813257..2ffc2ed2a154 100644
--- a/frontend/src/scenes/experiments/replayVisionScanner.test.ts
+++ b/frontend/src/scenes/experiments/replayVisionScanner.test.ts
@@ -44,7 +44,7 @@ describe('replayVisionScanner', () => {
{
name: 'a default exposure event',
exposure_criteria: undefined,
- expectedFilterTestAccounts: false,
+ expectedFilterTestAccounts: true,
},
{
name: 'a custom exposure event',
@@ -55,12 +55,12 @@ describe('replayVisionScanner', () => {
properties: [],
},
} satisfies ExperimentExposureCriteria as ExperimentExposureCriteria,
- expectedFilterTestAccounts: false,
+ expectedFilterTestAccounts: true,
},
{
- name: 'an experiment that filters test accounts',
- exposure_criteria: { filterTestAccounts: true } satisfies ExperimentExposureCriteria,
- expectedFilterTestAccounts: true,
+ name: 'an experiment that keeps test accounts',
+ exposure_criteria: { filterTestAccounts: false } satisfies ExperimentExposureCriteria,
+ expectedFilterTestAccounts: false,
},
])(
'saves an experiment scanner that waits for launch, with no filters: $name',
diff --git a/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsx b/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsx
index 1d4b54d5c198..8a8444a4a7ec 100644
--- a/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsx
+++ b/products/experiments/frontend/components/activity-descriptions/experimentChangeDescription.tsx
@@ -6,6 +6,7 @@ import { ActivityChange } from 'lib/components/ActivityLog/humanizeActivity'
import { dayjs } from 'lib/dayjs'
import { LemonTag } from 'lib/lemon-ui/LemonTag'
import { Link } from 'lib/lemon-ui/Link'
+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'
import { getExposureConfigDisplayName } from 'scenes/experiments/utils'
import { urls } from 'scenes/urls'
@@ -123,6 +124,22 @@ const describeMetricReorder = (before: unknown, after: unknown, description: str
return equal(membershipOnlyRewrite, a) ? null : description
}
+/**
+ * Compares the setting the analysis applies on each side, not the stored key. A write that adds or
+ * drops the key without changing the setting gets no line, and a write that drops an explicit
+ * `false` turns the filter back on.
+ */
+function describeTestAccountFilterChange(
+ before: ExperimentExposureCriteria | null,
+ after: ExperimentExposureCriteria | null
+): string | null {
+ const filteredAfter = resolveFilterTestAccounts(after)
+ if (resolveFilterTestAccounts(before) === filteredAfter) {
+ return null
+ }
+ return filteredAfter ? 'added the test account filter' : 'removed the test account filter'
+}
+
export const getExperimentChangeDescription = (
experimentChange: ActivityChange
): string | JSX.Element | (string | JSX.Element)[] | null => {
@@ -235,15 +252,8 @@ export const getExperimentChangeDescription = (
const changes: (string | JSX.Element | null)[] = Object.keys(after || {}).map((key) =>
match(key as keyof ExperimentExposureCriteria)
- .with('filterTestAccounts', () => {
- if (typedAfter?.filterTestAccounts === typedBefore?.filterTestAccounts) {
- return null
- }
-
- return typedAfter?.filterTestAccounts
- ? 'added the test account filter'
- : 'removed the test account filter'
- })
+ // Described below, because a write that drops the key can change the setting too.
+ .with('filterTestAccounts', () => null)
.with('multiple_variant_handling', () => {
if (typedAfter?.multiple_variant_handling === typedBefore?.multiple_variant_handling) {
return null
@@ -292,6 +302,8 @@ export const getExperimentChangeDescription = (
.exhaustive()
)
+ changes.unshift(describeTestAccountFilterChange(typedBefore, typedAfter))
+
// Check if exposure_config was removed (returning to default)
if (typedBefore?.exposure_config && !typedAfter?.exposure_config) {
changes.push(
diff --git a/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsx b/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsx
index aba8234557e3..efb0d1216b13 100644
--- a/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsx
+++ b/products/experiments/frontend/modals/ExperimentMetricModal/ExperimentMetricModal.tsx
@@ -4,7 +4,7 @@ import { LemonButton, LemonDialog, LemonInput, LemonLabel, LemonModal } from '@p
import { ExperimentMetricForm } from 'scenes/experiments/ExperimentMetricForm'
import { exposureCriteriaModalLogic } from 'scenes/experiments/ExperimentView/exposureCriteriaModalLogic'
-import { resolvedExposureEvent } from 'scenes/experiments/exposureContract'
+import { resolveFilterTestAccounts, resolvedExposureEvent } from 'scenes/experiments/exposureContract'
import type { ExperimentExposureCriteria, ExperimentMetric } from '~/queries/schema/schema-general'
import type { Experiment } from '~/types'
@@ -97,7 +97,7 @@ export function ExperimentMetricModal({
<ExperimentMetricForm
metric={metric}
handleSetMetric={setModalMetric}
- filterTestAccounts={experiment.exposure_criteria?.filterTestAccounts || false}
+ filterTestAccounts={resolveFilterTestAccounts(experiment.exposure_criteria)}
exposureCriteria={exposureCriteria}
openExposureCriteriaModal={openExposureCriteriaModal}
resolvedExposureEvent={resolvedExposureEvent(experiment)}
diff --git a/products/replay_vision/frontend/replay_scanners/experimentTargeting.ts b/products/replay_vision/frontend/replay_scanners/experimentTargeting.ts
index 7b5df773c5cf..89f722aacf0c 100644
--- a/products/replay_vision/frontend/replay_scanners/experimentTargeting.ts
+++ b/products/replay_vision/frontend/replay_scanners/experimentTargeting.ts
@@ -1,3 +1,4 @@
+import { resolveFilterTestAccounts } from 'scenes/experiments/exposureContract'
import { getExperimentVariants } from 'scenes/experiments/utils'
import { NodeKind, type RecordingsQuery } from '~/queries/schema/schema-general'
@@ -93,7 +94,7 @@ export function scopeVariantsLabel(scope: ScannerExperimentScope, everyVariant:
export function experimentScannerQuery(experiment: Experiment): Pick<RecordingsQuery, 'kind' | 'filter_test_accounts'> {
return {
kind: NodeKind.RecordingsQuery,
- filter_test_accounts: experiment.exposure_criteria?.filterTestAccounts ?? false,
+ filter_test_accounts: resolveFilterTestAccounts(experiment.exposure_criteria),
}
}
diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts
index e932c132a6c0..843e849fc16c 100644
--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts
+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts
@@ -469,7 +469,12 @@ describe('replayScannerLogic', () => {
logic.actions.draftScannerFromGoal('where do people give up in billing')
).toFinishAllListeners()
- expect(logic.values.scanner).toMatchObject({ name: 'Billing drop-off: Checkout redesign', ...expected })
+ // The mocked experiment stores no test-account setting, which its results read as on.
+ expect(logic.values.scanner).toMatchObject({
+ name: 'Billing drop-off: Checkout redesign',
+ query: { kind: 'RecordingsQuery', filter_test_accounts: true },
+ ...expected,
+ })
}
)
diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts
index dc708b59d7bc..c4a966545350 100644
--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts
+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts
@@ -76,6 +76,7 @@ import {
buildExperimentTargeting,
experimentScannerConfig,
experimentScannerName,
+ experimentScannerQuery,
parseExperimentScannerParams,
prefillScannerForExperiment,
ScannerExperimentScope,
@@ -2396,11 +2397,7 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
...(draftQuery
? {
query: context
- ? {
- ...draftQuery,
- filter_test_accounts:
- context.experiment.exposure_criteria?.filterTestAccounts ?? false,
- }
+ ? { ...draftQuery, ...experimentScannerQuery(context.experiment) }
: draftQuery,
}
: {}),