:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { useActions, useValues } from 'kea'
import { LemonButton, LemonInput, LemonLabel, LemonSelect } from '@posthog/lemon-ui'
import { LemonModal } from '@posthog/lemon-ui'
import { experimentLogic } from 'scenes/experiments/experimentLogic'
import {
DEFAULT_SEQUENTIAL_TUNING_PARAMETER,
MAX_SEQUENTIAL_TUNING_PARAMETER,
SequentialSelection,
getSequentialSelection,
resolveSequentialTuningParameter,
} from 'scenes/experiments/ExperimentView/sequential'
import { modalsLogic } from 'scenes/experiments/modalsLogic'
import { experimentsConfigLogic } from 'scenes/settings/environment/experimentsConfigLogic'
import { ExperimentStatsMethod } from '~/types'
import { StatsMethodSelector } from 'products/experiments/frontend/components/StatsMethodSelector'
import { CONFIDENCE_LEVEL_OPTIONS } from 'products/experiments/frontend/constants'
export function StatsMethodModal(): JSX.Element {
const { experiment, statsMethod, experimentUpdateLoading } = useValues(experimentLogic)
const { setExperiment, restoreUnmodifiedExperiment, updateExperimentSettings } = useActions(experimentLogic)
const { closeStatsEngineModal } = useActions(modalsLogic)
const { isStatsEngineModalOpen } = useValues(modalsLogic)
const { experimentsConfig } = useValues(experimentsConfigLogic)
const savingReason = experimentUpdateLoading ? 'Saving in progress' : undefined
const onClose = (): void => {
restoreUnmodifiedExperiment()
closeStatsEngineModal()
}
const isBayesian = statsMethod === ExperimentStatsMethod.Bayesian
// For Bayesian: ci_level (default 0.95)
// For Frequentist: confidence = 1 - alpha (default alpha 0.05 = 95% confidence)
const currentConfidenceLevel = isBayesian
? (experiment.stats_config?.bayesian?.ci_level ?? 0.95)
: 1 - (experiment.stats_config?.frequentist?.alpha ?? 0.05)
const handleConfidenceLevelChange = (value: number): void => {
if (isBayesian) {
setExperiment({
stats_config: {
...experiment.stats_config,
bayesian: {
...experiment.stats_config?.bayesian,
ci_level: value,
},
},
})
} else {
setExperiment({
stats_config: {
...experiment.stats_config,
frequentist: {
...experiment.stats_config?.frequentist,
alpha: 1 - value,
},
},
})
}
}
const sequentialSelection = getSequentialSelection(experiment.stats_config?.frequentist)
const teamDefaultSequentialEnabled = experimentsConfig?.default_sequential_testing_enabled ?? false
const teamDefaultSequentialTuningParameter = experimentsConfig?.default_sequential_tuning_parameter ?? null
const sequentialTuningParameter = resolveSequentialTuningParameter(
experiment.stats_config?.frequentist,
teamDefaultSequentialTuningParameter,
DEFAULT_SEQUENTIAL_TUNING_PARAMETER
)
const updateSequentialSelection = (next: SequentialSelection): void => {
const existingFrequentist = experiment.stats_config?.frequentist ?? {}
if (next === 'default') {
// Drop the sequential keys entirely so the team default applies at evaluation time.
const {
sequential_testing_enabled: _enabled,
sequential_tuning_parameter: _param,
...restFrequentist
} = existingFrequentist
setExperiment({
stats_config: {
...experiment.stats_config,
frequentist: restFrequentist,
},
})
return
}
setExperiment({
stats_config: {
...experiment.stats_config,
frequentist: {
...existingFrequentist,
sequential_testing_enabled: next === 'enabled',
},
},
})
}
const updateSequentialTuningParameter = (sequential_tuning_parameter: number): void => {
setExperiment({
stats_config: {
...experiment.stats_config,
frequentist: {
...experiment.stats_config?.frequentist,
sequential_tuning_parameter,
},
},
})
}
return (
<LemonModal
maxWidth={600}
isOpen={isStatsEngineModalOpen}
onClose={onClose}
// Cancel does not stop a running save. The save still applies, and its response replaces any new edit.
closable={!experimentUpdateLoading}
title="Statistics configuration"
footer={
<div className="flex items-center gap-2 justify-end">
<LemonButton type="secondary" onClick={onClose} disabledReason={savingReason}>
Cancel
</LemonButton>
<LemonButton
type="primary"
loading={experimentUpdateLoading}
onClick={() =>
updateExperimentSettings({ stats_config: experiment.stats_config }, 'statsMethod')
}
>
Save
</LemonButton>
</div>
}
>
<div className="mb-4">
<StatsMethodSelector
value={statsMethod}
disabled={experimentUpdateLoading}
disabledReason={savingReason}
onChange={(newStatsMethod) => {
setExperiment({
stats_config: {
...experiment.stats_config,
method: newStatsMethod,
},
})
}}
/>
</div>
<div className="flex flex-col gap-2">
<LemonLabel>Confidence level</LemonLabel>
<LemonSelect
value={currentConfidenceLevel}
onChange={handleConfidenceLevelChange}
disabledReason={savingReason}
options={CONFIDENCE_LEVEL_OPTIONS}
className="w-24"
/>
<p className="text-xs text-secondary m-0">
{isBayesian
? `At ${currentConfidenceLevel * 100}%, we require a variant to have a ${currentConfidenceLevel * 100}% or higher chance of being better before calling it a winner.`
: `Higher confidence means we need stronger evidence before declaring a winner. ${currentConfidenceLevel * 100}% confidence requires a p-value below ${(1 - currentConfidenceLevel).toFixed(2)}.`}
</p>
</div>
{!isBayesian && (
<div className="flex flex-col gap-2 mt-4">
<LemonLabel>Sequential testing</LemonLabel>
<LemonSelect<SequentialSelection>
value={sequentialSelection}
onChange={updateSequentialSelection}
disabledReason={savingReason}
options={[
{
value: 'default',
label: `Use team default (${teamDefaultSequentialEnabled ? 'Enabled' : 'Disabled'})`,
},
{ value: 'enabled', label: 'Enabled' },
{ value: 'disabled', label: 'Disabled' },
]}
className="w-72"
/>
<p className="text-xs text-secondary m-0">
Always-valid p-values that are robust to peeking. You can check the experiment as often as you
want without inflating the false positive rate, at the cost of slightly wider confidence
intervals.
</p>
{sequentialSelection === 'enabled' && (
<div className="flex flex-col gap-1 mt-1">
<LemonLabel>Tuning parameter</LemonLabel>
<LemonInput
type="number"
min={1}
max={MAX_SEQUENTIAL_TUNING_PARAMETER}
value={sequentialTuningParameter}
disabledReason={savingReason}
onChange={(value) => {
if (
typeof value !== 'number' ||
!Number.isFinite(value) ||
value < 1 ||
value > MAX_SEQUENTIAL_TUNING_PARAMETER
) {
return
}
updateSequentialTuningParameter(Math.round(value))
}}
className="w-32"
/>
<p className="text-xs text-secondary m-0">
Roughly the sample size at which the confidence sequence is tightest. Set close to the
expected total sample size of the experiment. Default is{' '}
{DEFAULT_SEQUENTIAL_TUNING_PARAMETER}.
</p>
</div>
)}
</div>
)}
</LemonModal>
)
}