:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): size runtime estimation target like the analysis (#122625)
Marcel Poelkereb6c7a105397
diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.tsindex 8b80a806df9c..d3a87620eeec 100644--- a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.ts+++ b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.ts@@ -1,119 +1,11 @@-import { uuid } from 'lib/utils/dom'+import { CachedNewExperimentQueryResponse } from '~/queries/schema/schema-general'-import {- CachedNewExperimentQueryResponse,- ExperimentMetric,- ExperimentMetricType,- NodeKind,-} from '~/queries/schema/schema-general'-import { ExperimentMetricMathType } from '~/types'--import {- baselineStatsFromResults,- calculateCurrentExposures,- calculateDaysElapsed,- calculateExposureRate,- getCalculatorMetricType,- hasRequiredBaselineStats,-} from './calculations'+import { calculateCurrentExposures, calculateDaysElapsed, calculateExposureRate } from './calculations'// The sample-size / variance math is verified in the backend// (products/experiments/backend/test/test_running_time_calculator.py). These tests-// cover the client-side helpers that classify metrics and read live experiment state.+// cover the client-side helpers that read live experiment state.describe('running time calculations', () => {- describe('getCalculatorMetricType', () => {- const meanMetric = (math: ExperimentMetricMathType): ExperimentMetric =>- ({- uuid: uuid(),- kind: NodeKind.ExperimentMetric,- metric_type: ExperimentMetricType.MEAN,- source: { kind: NodeKind.EventsNode, event: 'evt', math },- }) as ExperimentMetric-- it('classifies mean count', () => {- expect(getCalculatorMetricType(meanMetric(ExperimentMetricMathType.TotalCount))).toBe('mean_count')- })-- it('classifies mean sum', () => {- expect(getCalculatorMetricType(meanMetric(ExperimentMetricMathType.Sum))).toBe('mean_sum_or_avg')- })-- it('classifies funnel', () => {- const metric = {- uuid: uuid(),- metric_type: ExperimentMetricType.FUNNEL,- series: [{ kind: NodeKind.EventsNode, event: 'step' }],- } as ExperimentMetric- expect(getCalculatorMetricType(metric)).toBe('funnel')- })-- it('classifies ratio', () => {- const metric = {- uuid: uuid(),- kind: NodeKind.ExperimentMetric,- metric_type: ExperimentMetricType.RATIO,- numerator: { kind: NodeKind.EventsNode, event: 'purchase' },- denominator: { kind: NodeKind.EventsNode, event: 'page_view' },- } as ExperimentMetric- expect(getCalculatorMetricType(metric)).toBe('ratio')- })-- it('classifies retention', () => {- const metric = {- uuid: uuid(),- kind: NodeKind.ExperimentMetric,- metric_type: ExperimentMetricType.RETENTION,- start_event: { kind: NodeKind.EventsNode, event: 'uploaded' },- completion_event: { kind: NodeKind.EventsNode, event: 'downloaded' },- } as ExperimentMetric- expect(getCalculatorMetricType(metric)).toBe('retention')- })- })-- describe('baselineStatsFromResults', () => {- it('maps the results baseline into the request shape', () => {- const baseline: CachedNewExperimentQueryResponse['baseline'] = {- key: 'control',- number_of_samples: 10000,- sum: 500000,- sum_squares: 30000000,- denominator_sum: 50000,- denominator_sum_squares: 300000,- numerator_denominator_sum_product: 2600000,- step_counts: [1000, 100],- }- expect(baselineStatsFromResults(baseline)).toEqual({- number_of_samples: 10000,- sum: 500000,- sum_squares: 30000000,- denominator_sum: 50000,- denominator_sum_squares: 300000,- numerator_denominator_sum_product: 2600000,- step_counts: [1000, 100],- })- })- })-- describe('hasRequiredBaselineStats', () => {- const baseline = (denominatorSum: number | undefined): CachedNewExperimentQueryResponse['baseline'] => ({- key: 'control',- number_of_samples: 10000,- sum: 500000,- sum_squares: 30000000,- denominator_sum: denominatorSum,- })-- it.each(['ratio', 'retention'] as const)('requires a positive denominator_sum for %s', (metricType) => {- expect(hasRequiredBaselineStats(metricType, baseline(50000))).toBe(true)- expect(hasRequiredBaselineStats(metricType, baseline(0))).toBe(false)- expect(hasRequiredBaselineStats(metricType, baseline(undefined))).toBe(false)- })-- it.each(['funnel', 'mean_count', 'mean_sum_or_avg'] as const)('never blocks %s', (metricType) => {- expect(hasRequiredBaselineStats(metricType, baseline(undefined))).toBe(true)- })- })-describe('calculateCurrentExposures', () => {it('sums baseline and variant sample counts', () => {const results = {diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.tsindex da1adcd27af5..829a5d07bed4 100644--- a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts+++ b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts@@ -1,83 +1,14 @@import { dayjs } from 'lib/dayjs'-import type { CachedNewExperimentQueryResponse, ExperimentMetric } from '~/queries/schema/schema-general'-import {- isExperimentFunnelMetric,- isExperimentMeanMetric,- isExperimentRatioMetric,- isExperimentRetentionMetric,-} from '~/queries/schema/schema-general'-import { ExperimentMetricMathType } from '~/types'+import type { CachedNewExperimentQueryResponse } from '~/queries/schema/schema-general'-import type { RunningTimeBaselineStatsApi } from 'products/experiments/frontend/generated/api.schemas'--// The sample-size and variance math lives in the backend running-time calculator-// (products/experiments/backend/running_time_calculator.py), reached via-// POST /experiments/calculate_running_time. This module keeps only the client-side-// helpers that classify a metric and read live experiment state to build that request.+// The target sample lives in the backend (products/experiments/backend/running_time_target.py), reached via+// GET /experiments/:id/running_time_target, and the manual preview's math via POST+// /experiments/calculate_running_time. This module keeps only the client-side helpers that read live experiment state.// Manual calculator only supports these types (ratio/retention require full baseline data).export type ManualCalculatorMetricType = 'funnel' | 'mean_count' | 'mean_sum_or_avg'-// Full calculator supports all metric types.-export type CalculatorMetricType = ManualCalculatorMetricType | 'ratio' | 'retention'--/**- * Map an ExperimentMetric to the calculator metric type expected by the backend endpoint.- */-export function getCalculatorMetricType(metric: ExperimentMetric): CalculatorMetricType {- if (isExperimentFunnelMetric(metric)) {- return 'funnel'- }- if (isExperimentRatioMetric(metric)) {- return 'ratio'- }- if (isExperimentRetentionMetric(metric)) {- return 'retention'- }- if (isExperimentMeanMetric(metric) && metric.source.math === ExperimentMetricMathType.Sum) {- return 'mean_sum_or_avg'- }- return 'mean_count'-}--/**- * Shape the raw baseline stats from experiment results into the backend request body.- * The backend derives the baseline value and (for ratio/retention) the delta-method variance from these.- */-export function baselineStatsFromResults(- baseline: CachedNewExperimentQueryResponse['baseline']-): RunningTimeBaselineStatsApi {- // Omit optional stats that are null for this metric type (e.g. step_counts on non-funnel- // metrics, denominator_* on non-ratio metrics) — the serializer rejects explicit nulls,- // and the backend already treats absent values as unset.- return {- number_of_samples: baseline.number_of_samples,- sum: baseline.sum,- sum_squares: baseline.sum_squares,- denominator_sum: baseline.denominator_sum ?? undefined,- denominator_sum_squares: baseline.denominator_sum_squares ?? undefined,- numerator_denominator_sum_product: baseline.numerator_denominator_sum_product ?? undefined,- step_counts: baseline.step_counts ?? undefined,- }-}--/**- * Whether the baseline stats are sufficient to size this metric type. Ratio and retention- * sizing needs the baseline's denominator stats; low-data experiments return- * denominator_sum as null or 0, which the backend rejects — callers should skip the- * request rather than fire a doomed one.- */-export function hasRequiredBaselineStats(- metricType: CalculatorMetricType,- baseline: CachedNewExperimentQueryResponse['baseline']-): boolean {- if (metricType === 'ratio' || metricType === 'retention') {- return (baseline.denominator_sum ?? 0) > 0- }- return true-}-export function calculateCurrentExposures(results: CachedNewExperimentQueryResponse | null): number | null {if (!results) {return null@@ -104,3 +35,16 @@ export function calculateExposureRate(currentExposures: number | null, daysElapsreturn currentExposures / daysElapsed}++/**+ * The target sample for another minimum detectable effect, by the rule the target endpoint documents: the variances come+ * from the results, so the sample scales with 1 / effect². The settings rescale locally, which lets every value of the+ * slider show its estimate at once, without a request.+ */+export function rescaleSampleSize(sampleSize: number, fromEffect: number, toEffect: number): number | null {+ if (!(sampleSize > 0) || !(fromEffect > 0) || !(toEffect > 0)) {+ return null+ }+ // Round before the ceiling, as the backend does, so that float noise doesn't turn an exact 3,200 into 3,201.+ return Math.ceil(Number((sampleSize * (fromEffect / toEffect) ** 2).toFixed(6)))+}diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.tsindex 2c736f095633..bddda9c4b730 100644--- a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.ts+++ b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.ts@@ -21,10 +21,10 @@ jest.mock('lib/lemon-ui/LemonToast/LemonToast', () => ({},}))-const calculateRunningTimeMock = jest.fn()+const runningTimeTargetMock = jest.fn()jest.mock('products/experiments/frontend/generated/api', () => ({...jest.requireActual('products/experiments/frontend/generated/api'),- experimentsCalculateRunningTimeCreate: (...args: any[]) => calculateRunningTimeMock(...args),+ experimentsRunningTimeTargetRetrieve: (...args: any[]) => runningTimeTargetMock(...args),}))const EXPERIMENT_ID = 99@@ -98,7 +98,7 @@ describe('runningTimeLogic', () => {initKeaTests()jest.spyOn(api, 'update')api.update.mockClear()- calculateRunningTimeMock.mockReset()+ runningTimeTargetMock.mockReset();(lemonToast.error as jest.Mock).mockClear()experimentLogicInstance = experimentLogic({ experimentId: EXPERIMENT_ID })@@ -120,9 +120,9 @@ describe('runningTimeLogic', () => {describe('persistRunningTimeEstimate', () => {it('auto-persists with the concurrency handshake and absorbs the response version', async () => {- calculateRunningTimeMock.mockResolvedValue({+ runningTimeTargetMock.mockResolvedValue({recommended_sample_size: 2000,- recommended_running_time_days: 20,+ minimum_detectable_effect: 30,})api.update.mockResolvedValue({...experiment,@@ -135,6 +135,9 @@ describe('runningTimeLogic', () => {await expectLogic(logic).toDispatchActions(['persistRunningTimeEstimate']).toFinishAllListeners()+ expect(runningTimeTargetMock).toHaveBeenCalledWith(expect.any(String), EXPERIMENT_ID, {+ mode: 'automatic',+ })// Without the version handshake this write bumps the server version invisibly,// making every later scalar save from any open tab a guaranteed 409.expect(api.update).toHaveBeenCalledWith(@@ -151,9 +154,9 @@ describe('runningTimeLogic', () => {it('never persists a non-positive estimate', async () => {// A bad baseline can make the backend return a negative sample size. It must not reach the record.- calculateRunningTimeMock.mockResolvedValue({+ runningTimeTargetMock.mockResolvedValue({recommended_sample_size: -2000,- recommended_running_time_days: -20,+ minimum_detectable_effect: 30,})logic = runningTimeLogic({ experiment })@@ -166,9 +169,9 @@ describe('runningTimeLogic', () => {})it('drops the estimate silently and resyncs the snapshot when it loses a concurrency race', async () => {- calculateRunningTimeMock.mockResolvedValue({+ runningTimeTargetMock.mockResolvedValue({recommended_sample_size: 2000,- recommended_running_time_days: 20,+ minimum_detectable_effect: 30,})api.update.mockRejectedValue({status: 409,@@ -192,10 +195,100 @@ describe('runningTimeLogic', () => {})})+ describe('automatic target', () => {+ it('rescales the target for an edited effect without asking the backend', async () => {+ runningTimeTargetMock.mockResolvedValue({ recommended_sample_size: 2000, minimum_detectable_effect: 30 })+ api.update.mockResolvedValue({ ...experiment, version: 4 })++ logic = runningTimeLogic({ experiment })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners()+ runningTimeTargetMock.mockClear()++ logic.actions.setConfig({ mde: 15 })+ await expectLogic(logic).toFinishAllListeners()++ expect(logic.values.targetSampleSize).toEqual(8000)+ expect(runningTimeTargetMock).not.toHaveBeenCalled()+ })++ it('counts the exposures the backend sizes the target over', async () => {+ // The results hold 1,200 exposures. The backend counts 1,100, because one variant is at 0%.+ runningTimeTargetMock.mockResolvedValue({+ recommended_sample_size: 2000,+ minimum_detectable_effect: 30,+ current_sample_size: 1100,+ })+ api.update.mockResolvedValue({ ...experiment, version: 4 })++ logic = runningTimeLogic({ experiment })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners()++ expect(logic.values.currentExposures).toEqual(1100)+ })++ it('asks again when the metric sums change without new exposures', async () => {+ runningTimeTargetMock.mockResolvedValue({ recommended_sample_size: 2000, minimum_detectable_effect: 30 })+ api.update.mockResolvedValue({ ...experiment, version: 4 })++ logic = runningTimeLogic({ experiment })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners()++ metricsLogicInstance.actions.setPrimaryMetricsResults([+ {+ ...metricResultWithBaseline,+ baseline: { ...metricResultWithBaseline.baseline, sum: 150, sum_squares: 90 },+ } as any,+ ])+ await expectLogic(logic).toDispatchActions(['loadAutomaticCalculation']).toFinishAllListeners()++ expect(runningTimeTargetMock).toHaveBeenCalledTimes(2)+ })+ })+describe('save', () => {+ it('persists the estimate the backend sizes for the saved effect', async () => {+ runningTimeTargetMock.mockResolvedValueOnce({+ recommended_sample_size: 2000,+ minimum_detectable_effect: 30,+ })+ api.update.mockImplementation(async (_url: string, payload: unknown) => ({+ ...experiment,+ ...(payload as Partial<Experiment>),+ version: 4,+ }))++ logic = runningTimeLogic({ experiment })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners()+ modalsLogic.actions.openRunningTimeConfigModal()+ logic.actions.setConfig({ mde: 15 })+ runningTimeTargetMock.mockResolvedValueOnce({+ recommended_sample_size: 8000,+ minimum_detectable_effect: 15,+ })+ api.update.mockClear()++ await expectLogic(logic, () => logic.actions.save())+ .toDispatchActions(['loadAutomaticCalculationSuccess'])+ .toFinishAllListeners()++ expect(api.update).toHaveBeenLastCalledWith(+ expect.stringContaining(`/experiments/${EXPERIMENT_ID}`),+ expect.objectContaining({+ running_time_calculation: expect.objectContaining({+ minimum_detectable_effect: 15,+ recommended_sample_size: 8000,+ }),+ })+ )+ })+it('keeps the modal open with the edited config when the experiment save fails', async () => {// A failed estimate request stops the automatic estimate from saving anything on mount.- calculateRunningTimeMock.mockRejectedValue(new Error('backend unavailable'))+ runningTimeTargetMock.mockRejectedValue(new Error('backend unavailable'))api.update.mockRejectedValueOnce(new Error('network down'))logic = runningTimeLogic({ experiment })@@ -217,7 +310,7 @@ describe('runningTimeLogic', () => {describe('isCalculating', () => {it('stays true while the automatic estimate is in flight, then settles on success', async () => {let resolveCalc: (value: unknown) => void = () => {}- calculateRunningTimeMock.mockReturnValue(+ runningTimeTargetMock.mockReturnValue(new Promise((resolve) => {resolveCalc = resolve})@@ -230,7 +323,7 @@ describe('runningTimeLogic', () => {await expectLogic(logic).toDispatchActions(['loadAutomaticCalculation'])expect(logic.values.isCalculating).toBe(true)- resolveCalc({ recommended_sample_size: 2000, recommended_running_time_days: 20 })+ resolveCalc({ recommended_sample_size: 2000, minimum_detectable_effect: 30 })await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners()expect(logic.values.isCalculating).toBe(false)})@@ -238,7 +331,7 @@ describe('runningTimeLogic', () => {it('clears calculating when the automatic estimate request fails', async () => {// A failed request leaves the result null for good, so calculating must still settle —// otherwise the meta bar hangs on "Calculating…" instead of showing a resting state.- calculateRunningTimeMock.mockRejectedValue(new Error('backend unavailable'))+ runningTimeTargetMock.mockRejectedValue(new Error('backend unavailable'))logic = runningTimeLogic({ experiment })logic.mount()@@ -248,9 +341,9 @@ describe('runningTimeLogic', () => {})it('follows the recalculation loading state', async () => {- calculateRunningTimeMock.mockResolvedValue({+ runningTimeTargetMock.mockResolvedValue({recommended_sample_size: 2000,- recommended_running_time_days: 20,+ minimum_detectable_effect: 30,})logic = runningTimeLogic({ experiment })diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.tsindex e6ab71c5e78f..1c032901e282 100644--- a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.ts+++ b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.ts@@ -10,10 +10,13 @@ import { experimentsConfigLogic } from '~/scenes/settings/environment/experimentimport { ConversionRateInputType, Experiment } from '~/types'import { isLaunched } from 'products/experiments/frontend/experimentStatus'-import { experimentsCalculateRunningTimeCreate } from 'products/experiments/frontend/generated/api'+import {+ experimentsCalculateRunningTimeCreate,+ experimentsRunningTimeTargetRetrieve,+} from 'products/experiments/frontend/generated/api'import type {RunningTimeCalculationInputApi,- RunningTimeCalculationResultApi,+ RunningTimeTargetApi,} from 'products/experiments/frontend/generated/api.schemas'import type { CachedNewExperimentQueryResponse } from '../../../queries/schema/schema-general'@@ -24,12 +27,10 @@ import { getFlagVariants, metricResults, isExperimentConflictError, toConcurrencimport type { MetricWithResult } from '../utils'import {ManualCalculatorMetricType,- baselineStatsFromResults,calculateCurrentExposures,calculateDaysElapsed,calculateExposureRate,- getCalculatorMetricType,- hasRequiredBaselineStats,+ rescaleSampleSize,} from './calculations'export interface RunningTimeLogicProps {@@ -49,6 +50,14 @@ export interface ManualPreview {runningTime: number | null}+// The backend sizes the target for the saved minimum detectable effect. The settings rescale it for an edited one, so an+// edit sends no request.+export interface AutomaticTargetRequest {+ // Never sent. A new result of the primary metric changes it, so that the backend sizes the target again. The sums+ // count too, because more metric events from the same exposed users change the variance.+ resultStats: number[]+}+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.export interface runningTimeLogicValues {currentProjectId: number | null // experimentLogic@@ -59,13 +68,14 @@ export interface runningTimeLogicValues {primaryMetricsResultsErrors: (unknown | null)[] // experimentMetricsLogicdefaultMinimumDetectableEffect: number // experimentsConfigLogicisRunningTimeConfigModalOpen: boolean // modalsLogic- automaticCalculation: RunningTimeCalculationResultApi | null- automaticCalculationInput: RunningTimeCalculationInputApi | null+ automaticCalculation: RunningTimeTargetApi | null+ automaticCalculationInput: AutomaticTargetRequest | nullautomaticCalculationLoading: booleanconfig: RunningTimeConfigconfigOverrides: Partial<RunningTimeConfig> | nullcurrentExposures: number | nulldailyExposureRate: number | null+ editedMde: number | nullinitialConfig: RunningTimeConfigisCalculating: booleanisComplete: boolean@@ -80,6 +90,7 @@ export interface runningTimeLogicValues {mode: RunningTimeConfig['mode']numberOfVariants: numberremainingDays: number | null+ targetEffect: numbertargetSampleSize: number | null}@@ -97,7 +108,7 @@ export interface runningTimeLogicActions {cancel: () => {value: true}- loadAutomaticCalculation: (input: RunningTimeCalculationInputApi) => RunningTimeCalculationInputApi+ loadAutomaticCalculation: (_: AutomaticTargetRequest) => AutomaticTargetRequestloadAutomaticCalculationFailure: (error: string,errorObject?: any@@ -106,11 +117,11 @@ export interface runningTimeLogicActions {errorObject?: any}loadAutomaticCalculationSuccess: (- automaticCalculation: RunningTimeCalculationResultApi,- payload?: RunningTimeCalculationInputApi+ automaticCalculation: RunningTimeTargetApi | null,+ payload?: AutomaticTargetRequest) => {- automaticCalculation: RunningTimeCalculationResultApi- payload?: RunningTimeCalculationInputApi+ automaticCalculation: RunningTimeTargetApi | null+ payload?: AutomaticTargetRequest}loadManualPreview: (input: RunningTimeCalculationInputApi) => RunningTimeCalculationInputApiloadManualPreviewFailure: (@@ -166,28 +177,37 @@ export interface runningTimeLogicMeta {configOverrides: Partial<RunningTimeConfig> | null) => RunningTimeConfigmde: (config: RunningTimeConfig) => number+ editedMde: (configOverrides: Partial<RunningTimeConfig> | null) => number | null+ targetEffect: (+ editedMde: number | null,+ automaticCalculation: RunningTimeTargetApi | null,+ mde: number+ ) => numbermode: (config: RunningTimeConfig) => RunningTimeConfig['mode']metricsWithResults: (experiment: Experiment,primaryMetricsResults: CachedNewExperimentQueryResponse[],primaryMetricsResultsErrors: unknown[]) => MetricWithResult[]- currentExposures: (metricsWithResults: MetricWithResult[]) => number | null+ currentExposures: (+ isManualMode: boolean,+ metricsWithResults: MetricWithResult[],+ automaticCalculation: RunningTimeTargetApi | null+ ) => number | nullautomaticCalculationInput: (mode: 'automatic' | 'manual',- metricsWithResults: MetricWithResult[],- numberOfVariants: number,- mde: number- ) => RunningTimeCalculationInputApi | null+ metricsWithResults: MetricWithResult[]+ ) => AutomaticTargetRequest | nullmanualPreviewInput: (config: RunningTimeConfig,numberOfVariants: number) => RunningTimeCalculationInputApi | nulltargetSampleSize: (isManualMode: boolean,+ targetEffect: number,experiment: Experiment,- automaticCalculationInput: RunningTimeCalculationInputApi | null,- automaticCalculation: RunningTimeCalculationResultApi | null+ automaticCalculationInput: AutomaticTargetRequest | null,+ automaticCalculation: RunningTimeTargetApi | null) => number | nulldailyExposureRate: (isManualMode: boolean,@@ -272,13 +292,22 @@ export const runningTimeLogic = kea<runningTimeLogicType>([],}),- loaders(({ values }) => ({- // Automatic mode: recommended sample size derived by the backend from live results baseline.+ loaders(({ values, props }) => ({+ // Automatic mode: the target the backend sizes from the primary metric's stored result and the flag's split.automaticCalculation: [- null as RunningTimeCalculationResultApi | null,+ null as RunningTimeTargetApi | null,{- loadAutomaticCalculation: async (input: RunningTimeCalculationInputApi, breakpoint) => {- const result = await experimentsCalculateRunningTimeCreate(String(values.currentProjectId), input)+ loadAutomaticCalculation: async (_: AutomaticTargetRequest, breakpoint) => {+ const experimentId = props.experiment.id+ if (typeof experimentId !== 'number') {+ return null+ }+ const result = await experimentsRunningTimeTargetRetrieve(+ String(values.currentProjectId),+ experimentId,+ // Sized from the results also while a manual experiment is being switched to automatic.+ { mode: 'automatic' }+ )breakpoint()return result},@@ -343,9 +372,20 @@ export const runningTimeLogic = kea<runningTimeLogicType>([configOverrides ? { ...initialConfig, ...configOverrides } : initialConfig,],- // Pulled out as primitives so automaticCalculationInput stays stable when the experiment object- // is replaced (e.g. after persisting an estimate) but the underlying values are unchanged.mde: [(s) => [s.config], (config: RunningTimeConfig): number => config.mde],+ editedMde: [+ (s) => [s.configOverrides],+ (configOverrides: Partial<RunningTimeConfig> | null): number | null => configOverrides?.mde ?? null,+ ],+ // The effect the automatic target is sized for: an edited one, else the one the backend sized for. The backend's+ // can differ from `mde` before the team default loads, or when the saved effect isn't a positive number.+ targetEffect: [+ (s) => [s.editedMde, s.automaticCalculation, s.mde],+ (editedMde: number | null, automaticCalculation: RunningTimeTargetApi | null, mde: number): number =>+ editedMde ?? automaticCalculation?.minimum_detectable_effect ?? mde,+ ],+ // A primitive, so that automaticCalculationInput stays stable when the experiment object is replaced (e.g.+ // after persisting an estimate) but the mode is unchanged.mode: [(s) => [s.config], (config: RunningTimeConfig): RunningTimeConfig['mode'] => config.mode],// Primary metrics zipped with their results, so automatic mode always has a baseline to read.@@ -359,21 +399,29 @@ export const runningTimeLogic = kea<runningTimeLogicType>([],currentExposures: [- (s) => [s.metricsWithResults],+ (s) => [s.isManualMode, s.metricsWithResults, s.automaticCalculation],(+ isManualMode: boolean,results: {displayIndex: numbererror: anymetric: import('../../../queries/schema').ExperimentMetricUnionmetricIndex: numberresult: any- }[]- ): number | null => calculateCurrentExposures(results?.[0]?.result ?? null),+ }[],+ automaticCalculation: RunningTimeTargetApi | null+ ): number | null => {+ // The backend counts the exposures the automatic target sizes, so a variant at 0% does not count.+ if (!isManualMode && automaticCalculation?.current_sample_size != null) {+ return automaticCalculation.current_sample_size+ }+ return calculateCurrentExposures(results?.[0]?.result ?? null)+ },],- // Request body for the automatic-mode calculation, or null when not applicable (manual mode / no results yet).+ // Request for the automatic-mode target, or null when not applicable (manual mode / no results yet).automaticCalculationInput: [- (s) => [s.mode, s.metricsWithResults, s.numberOfVariants, s.mde],+ (s) => [s.mode, s.metricsWithResults],(mode: RunningTimeConfig['mode'],results: {@@ -382,29 +430,24 @@ export const runningTimeLogic = kea<runningTimeLogicType>([metric: import('../../../queries/schema').ExperimentMetricUnionmetricIndex: numberresult: any- }[],- numberOfVariants: number,- mde: number- ): RunningTimeCalculationInputApi | null => {+ }[]+ ): AutomaticTargetRequest | null => {// Gate on the in-form mode rather than the persisted mode. While the form is in manual mode// (even if the experiment is still saved as automatic), the automatic estimate is neither- // shown nor persisted, so editing the MDE shouldn't fire a wasted automatic calculation.+ // shown nor persisted, so it shouldn't be requested.if (mode !== 'automatic') {return null}- const firstMetric = results?.[0]- if (!firstMetric?.metric || !firstMetric?.result?.baseline) {- return null- }- const metricType = getCalculatorMetricType(firstMetric.metric)- if (!hasRequiredBaselineStats(metricType, firstMetric.result.baseline)) {+ const result = results?.[0]?.result as CachedNewExperimentQueryResponse | undefined+ if (!result?.baseline) {return null}return {- metric_type: metricType,- minimum_detectable_effect: mde,- number_of_variants: numberOfVariants,- baseline_stats: baselineStatsFromResults(firstMetric.result.baseline),+ resultStats: [result.baseline, ...result.variant_results].flatMap((stats) => [+ stats.number_of_samples,+ stats.sum,+ stats.sum_squares,+ ]),}},],@@ -429,21 +472,26 @@ export const runningTimeLogic = kea<runningTimeLogicType>([],targetSampleSize: [- (s) => [s.isManualMode, s.experiment, s.automaticCalculationInput, s.automaticCalculation],+ (s) => [s.isManualMode, s.targetEffect, s.experiment, s.automaticCalculationInput, s.automaticCalculation],(isManualMode: boolean,+ targetEffect: number,experiment: Experiment,- automaticInput: RunningTimeCalculationInputApi | null,- automaticCalculation: RunningTimeCalculationResultApi | null+ automaticInput: AutomaticTargetRequest | null,+ automaticCalculation: RunningTimeTargetApi | null): number | null => {if (isManualMode) {// Persisted by the save listener from the same backend calculation.return experiment?.running_time_calculation?.recommended_sample_size ?? null}- if (!automaticInput) {+ if (!automaticInput || !automaticCalculation?.recommended_sample_size) {return null}- return automaticCalculation?.recommended_sample_size ?? null+ return rescaleSampleSize(+ automaticCalculation.recommended_sample_size,+ automaticCalculation.minimum_detectable_effect,+ targetEffect+ )},],@@ -520,10 +568,7 @@ export const runningTimeLogic = kea<runningTimeLogicType>([// kea-subscriptions fires on reference change. The recalc poll replaces the metric-results array every// tick, so these inputs get a fresh object reference even when their values are identical, which would// re-fire the calculation POST on every poll. Skip the dispatch unless the value actually changed.- automaticCalculationInput: (- input: RunningTimeCalculationInputApi | null,- oldInput: RunningTimeCalculationInputApi | null- ) => {+ automaticCalculationInput: (input: AutomaticTargetRequest | null, oldInput: AutomaticTargetRequest | null) => {if (input && !objectsEqual(input, oldInput)) {actions.loadAutomaticCalculation(input)}@@ -686,6 +731,11 @@ export const runningTimeLogic = kea<runningTimeLogicType>([}actions.closeRunningTimeConfigModal()actions.resetConfig()+ // The target on hand is sized for the previous effect. The backend sizes it for the saved one, and its+ // success persists the estimate.+ if (config.mode === 'automatic' && values.automaticCalculationInput) {+ actions.loadAutomaticCalculation(values.automaticCalculationInput)+ }},cancel: () => {diff --git a/frontend/src/scenes/experiments/stories/ExperimentRecordingsListEmpty.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentRecordingsListEmpty.stories.tsxindex aa85a125ba7e..755b7007d5ee 100644--- a/frontend/src/scenes/experiments/stories/ExperimentRecordingsListEmpty.stories.tsx+++ b/frontend/src/scenes/experiments/stories/ExperimentRecordingsListEmpty.stories.tsx@@ -62,11 +62,11 @@ const meta: Meta = {'/api/environments/:team_id/default_release_conditions/': [],'/api/projects/:team_id/property_definitions/seen_together': {},'/api/projects/:team_id/vision/scanners/': { count: 0, results: [] },+ // Loaded results start the running time estimate, and a failed request raises a toast.+ '/api/projects/:team_id/experiments/:id/running_time_target/': {},},post: {...RECALCULATION.post,- // Loaded results start the running time estimate, and a failed request raises a toast.- '/api/projects/:team_id/experiments/calculate_running_time/': {},'/api/environments/:team_id/query/:kind': [200, { results: [] }],},}),diff --git a/frontend/src/scenes/experiments/stories/ExperimentResultsRowRecordingLinks.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentResultsRowRecordingLinks.stories.tsxindex 981bf48ed862..7a451be1fac8 100644--- a/frontend/src/scenes/experiments/stories/ExperimentResultsRowRecordingLinks.stories.tsx+++ b/frontend/src/scenes/experiments/stories/ExperimentResultsRowRecordingLinks.stories.tsx@@ -57,12 +57,12 @@ const meta: Meta = {// The linkability check decides whether the menu items are offered or disabled, so// it is answered here rather than left to fail open on a missing handler.'/api/projects/:team_id/property_definitions/seen_together': {},+ // Answered so the scene does not raise a failure toast over the table.+ '/api/projects/:team_id/experiments/:id/running_time_target/': {},},post: {...FUNNEL_RECALCULATION.post,...MEAN_RECALCULATION.post,- // Answered so the scene does not raise a failure toast over the table.- '/api/projects/:team_id/experiments/calculate_running_time/': {},// Exposures still load through experimentLogic; metric results come from the recalculation run.'/api/environments/:team_id/query/:kind': async ({ request }) => {const body = (await request.json()) as Record<string, any>diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.pyindex 26a10ef21e2d..5721c95bf3b1 100644--- a/products/experiments/backend/facade/api.py+++ b/products/experiments/backend/facade/api.py@@ -32,14 +32,19 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (resolve_default_exposure_event,)from products.experiments.backend.models.experiment import Experiment as ExperimentModel+from products.experiments.backend.running_time_target import load_running_time_targetfrom .contracts import (CreateExperimentInput,EndedExperiment,Experiment,+ ExperimentDetectableEffectCheckpoint,ExperimentHealth,ExperimentHealthReaderAccess,+ ExperimentRunningTimeOutlook,+ ExperimentRunningTimeTarget,RunningExperiment,+ RunningTimeMode,)logger = structlog.get_logger(__name__)@@ -166,6 +171,36 @@ def get_experiment_health_inputs(return health_inputs_payload(inputs)+def get_running_time_target(+ *, team_id: int, experiment_id: int, mode: RunningTimeMode | None = None+) -> ExperimentRunningTimeTarget:+ """The target sample of the experiment's running time estimate, with what the experiment can detect over time.+ `mode` replaces the saved one, so that the page can show the automatic target while the user switches to it."""+ experiment = ExperimentModel.objects.select_related("feature_flag").get(team_id=team_id, id=experiment_id)+ target = load_running_time_target(experiment, mode=mode)+ outlook = target.outlook+ return ExperimentRunningTimeTarget(+ minimum_detectable_effect=target.minimum_detectable_effect,+ recommended_sample_size=target.recommended_sample_size,+ current_sample_size=target.current_sample_size,+ outlook=ExperimentRunningTimeOutlook(+ days_since_launch=outlook.days_since_launch,+ exposures_per_day=outlook.exposures_per_day,+ detectable_effect_now=outlook.detectable_effect_now,+ upcoming_checkpoints=tuple(+ ExperimentDetectableEffectCheckpoint(+ weeks_since_launch=checkpoint.weeks_since_launch,+ detectable_effect=checkpoint.detectable_effect,+ )+ for checkpoint in outlook.upcoming_checkpoints+ ),+ reachable_effect=outlook.reachable_effect,+ )+ if outlook+ else None,+ )++def list_ended_experiments_without_conclusion(*, team_id: int, user_id: int, since: datetime, limit: int) -> list[EndedExperiment]:diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex 65af16684b56..75bf3940cd83 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -366,6 +366,52 @@ class DailyMetricCalculation:calculation_key: str+class RunningTimeMode(LabeledStrEnum):+ # pinned: the page's `ConversionRateInputType` values in frontend/src/types.ts, stored in+ # `running_time_calculation.exposure_estimate_config.conversionRateInputType`.+ AUTOMATIC = "automatic", "Sized from the results"+ MANUAL = "manual", "Saved from a baseline the user entered"+++@frozen+class ExperimentDetectableEffectCheckpoint:+ weeks_since_launch: int+ # The smallest relative effect, as a percentage, that the exposures by then detect.+ detectable_effect: float+++@frozen+class ExperimentRunningTimeOutlook:+ """What the experiment can detect over time, at its exposure rate so far."""++ days_since_launch: float+ # The exposures that count toward the target, per day from launch to the end of the window the results cover.+ exposures_per_day: float+ # The smallest relative effect, as a percentage, that the exposures so far detect.+ detectable_effect_now: float+ # The next week marks after today.+ upcoming_checkpoints: tuple[ExperimentDetectableEffectCheckpoint, ...]+ # The smallest effect the experiment detects within four weeks from today. A target for a smaller one is out of+ # reach.+ reachable_effect: float+++@frozen+class ExperimentRunningTimeTarget:+ """The target sample of an experiment's running time estimate."""++ # The relative effect, as a percentage, that the target sizes the experiment to detect.+ minimum_detectable_effect: float+ # The exposures across the analyzed variants that the experiment needs. None when no stored result can size it.+ recommended_sample_size: int | None+ # The exposures so far that count toward the target: the control's and those of the variants that get traffic.+ # None in manual mode, and when no stored result can size the target.+ current_sample_size: int | None+ # None in manual mode, before the experiment has run long enough to project, and when no stored result can size the+ # target.+ outlook: ExperimentRunningTimeOutlook | None++@frozenclass EndedExperiment:id: intdiff --git a/products/experiments/backend/health/checks/early_significance.py b/products/experiments/backend/health/checks/early_significance.pyindex a984e17c5a82..3585c2612d3c 100644--- a/products/experiments/backend/health/checks/early_significance.py+++ b/products/experiments/backend/health/checks/early_significance.py@@ -24,7 +24,8 @@ from products.experiments.backend.health.skips import (not_checked,)from products.experiments.backend.logic.result_stats import MetricResultStats, VariantStats-from products.experiments.backend.running_time_calculator import calculate_detectable_effect, calculate_sample_size+from products.experiments.backend.running_time_calculator import calculate_detectable_effect+from products.experiments.backend.running_time_target import RunningTimeTarget# pinned: sent as `finding_variant` on the health finding events.BELOW_DETECTABLE_EFFECT = "below_detectable_effect"@@ -57,10 +58,6 @@ def early_significance(ctx: HealthContext) -> CheckOutcome:return not_checked("Couldn't read the results of the primary metric")if not metric.is_funnel:return not_applicable("Runs on funnel primary metrics only")- # The page's target in this mode comes from the user's guessed baseline, so a note about the target could say- # "wait" after the page says the target is reached.- if metric.manual_running_time:- return not_applicable("Runs with the automatic running time estimate")result = metric.resultif result is None:return not_applicable("Runs once the primary metric has results")@@ -70,17 +67,15 @@ def early_significance(ctx: HealthContext) -> CheckOutcome:control_rate = result.baseline.mean or 0if not 0 < control_rate < 1:return not_applicable("Runs once the control group's conversion rate is above 0% and below 100%")- # The running time estimate counts the variants of the flag.- number_of_variants = len(ctx.flag.variants) if ctx.flag and ctx.flag.variants else len(result.variants) + 1- planned_samples = calculate_sample_size(- "funnel", control_rate, metric.minimum_detectable_effect, number_of_variants- )+ planned_samples = metric.target.recommended_sample_size# The running time estimate's "Target reached" compares the same total, so the note and the estimate agree.+ if metric.target.current_sample_size is not None:+ samples = metric.target.current_sample_sizeif planned_samples is None or samples >= planned_samples:return Nonebelow_detectable = [comparison- for comparison in _significant_comparisons(result, control_rate)+ for comparison in _significant_comparisons(result, control_rate, metric.target)if abs(comparison.observed) < comparison.detectable]if not below_detectable:@@ -101,7 +96,7 @@ def early_significance(ctx: HealthContext) -> CheckOutcome:f"With {samples:,} users so far, the experiment can reliably detect differences of about "f"{comparison.detectable:.0f}% or more, so this difference may shrink or go away as more users arrive.",f"The running time estimate needs about {planned_samples:,} users to detect a "- f"{metric.minimum_detectable_effect:g}% difference.",+ f"{metric.target.minimum_detectable_effect:g}% difference.","Wait for that sample before you decide.",]),@@ -109,7 +104,7 @@ def early_significance(ctx: HealthContext) -> CheckOutcome:"variant": comparison.variant.key,"observed_effect": round(comparison.observed, 1),"detectable_effect": round(comparison.detectable, 1),- "minimum_detectable_effect": metric.minimum_detectable_effect,+ "minimum_detectable_effect": metric.target.minimum_detectable_effect,"samples": samples,"planned_samples": planned_samples,},@@ -118,13 +113,24 @@ def early_significance(ctx: HealthContext) -> CheckOutcome:)-def _significant_comparisons(result: MetricResultStats, control_rate: float) -> list[_Comparison]:+def _significant_comparisons(+ result: MetricResultStats, control_rate: float, target: RunningTimeTarget+) -> list[_Comparison]:comparisons: list[_Comparison] = []for variant in result.variants:if variant.significant is not True or variant.interval is None:continue+ # The variances and the level that the running time target uses, so CUPED applies here too. Without the+ # variances, p(1-p) of the control.+ variance = target.variances.get(variant.key)detectable = calculate_detectable_effect(- "funnel", control_rate, result.baseline.number_of_samples, variant.number_of_samples+ "funnel",+ control_rate,+ result.baseline.number_of_samples,+ variant.number_of_samples,+ variance=variance.control if variance else None,+ variant_variance=variance.variant if variance else None,+ confidence_level=target.confidence_level,)if detectable is not None:# The difference the results page shows, from the analysis itself, so CUPED and the Bayesian model apply.diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex d45a3d768d6e..f03148b4c4aa 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -37,6 +37,7 @@ from products.experiments.backend.metric_resolution import MetricRole, saved_metfrom products.experiments.backend.metric_utils import get_default_metric_titlefrom products.experiments.backend.metric_validation import defined_event_names, extract_entity_nodesfrom products.experiments.backend.models.experiment import Experiment, ExperimentToSavedMetric+from products.experiments.backend.running_time_target import RunningTimeTargetfrom products.feature_flags.backend.facade.edits import (BucketingChange,ConditionsChange,@@ -210,11 +211,8 @@ class PrimaryMetric:name: stris_funnel: bool- # The relative effect, as a percentage, that the experiment plans to detect.- minimum_detectable_effect: float- # Whether the running time estimate is in manual mode, where its target comes from a baseline the user entered- # instead of from the results.- manual_running_time: bool+ # The target of the running time estimate, which the header of the experiment page shows.+ target: RunningTimeTarget# The current stored result, the one the results page shows after its last calculation. None when no completed# result is stored under the metric's current calculation key. Only read for a funnel.result: MetricResultStats | Nonediff --git a/products/experiments/backend/health/primary_metric.py b/products/experiments/backend/health/primary_metric.pyindex 7608b1ba48ea..6c1556be09c0 100644--- a/products/experiments/backend/health/primary_metric.py+++ b/products/experiments/backend/health/primary_metric.py@@ -1,6 +1,7 @@from posthog.exceptions_capture import capture_exceptionfrom products.experiments.backend.health.context import PrimaryMetric+from products.experiments.backend.logic.result_stats import MetricResultStatsfrom products.experiments.backend.metric_calculation.config import (build_primary_calculation_configs,calculation_metric_name,@@ -8,8 +9,7 @@ from products.experiments.backend.metric_calculation.config import ()from products.experiments.backend.metric_calculation.results import MetricResultStorefrom products.experiments.backend.models.experiment import Experiment-from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig-from products.experiments.backend.setup_context import PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT+from products.experiments.backend.running_time_target import running_time_targetdef load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:@@ -32,31 +32,12 @@ def _load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:if calculation_config is None:return Noneis_funnel = calculation_config.definition.get("metric_type") == "funnel"+ response = MetricResultStore(experiment_id=experiment.id).current_response(calculation_config)return PrimaryMetric(name=calculation_metric_name(experiment, calculation_config),is_funnel=is_funnel,- minimum_detectable_effect=_minimum_detectable_effect(experiment, team_config),- manual_running_time=_manual_running_time(experiment),- result=MetricResultStore(experiment_id=experiment.id).current_result(calculation_config) if is_funnel else None,+ target=running_time_target(+ experiment, team_config=team_config, calculation_config=calculation_config, response=response+ ),+ result=MetricResultStats.from_response(response) if is_funnel and response is not None else None,)---def _minimum_detectable_effect(experiment: Experiment, team_config: TeamExperimentsConfig) -> float:- running_time_calculation = experiment.running_time_calculation- planned = (- running_time_calculation.get("minimum_detectable_effect")- if isinstance(running_time_calculation, dict)- else None- )- if isinstance(planned, int | float) and not isinstance(planned, bool) and planned > 0:- return float(planned)- return float(team_config.default_minimum_detectable_effect or PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT)---def _manual_running_time(experiment: Experiment) -> bool:- # pinned: the page's `ConversionRateInputType.MANUAL` in frontend/src/types.ts, read by `runningTimeLogic`.- running_time_calculation = experiment.running_time_calculation- estimate_config = (- running_time_calculation.get("exposure_estimate_config") if isinstance(running_time_calculation, dict) else None- )- return isinstance(estimate_config, dict) and estimate_config.get("conversionRateInputType") == "manual"diff --git a/products/experiments/backend/health/test/test_early_significance.py b/products/experiments/backend/health/test/test_early_significance.pyindex 65a829898529..c049d33833ee 100644--- a/products/experiments/backend/health/test/test_early_significance.py+++ b/products/experiments/backend/health/test/test_early_significance.py@@ -31,6 +31,7 @@ from products.experiments.backend.models.experiment import (ExperimentToSavedMetric,)from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig+from products.experiments.backend.running_time_target import ComparisonVariance, RunningTimeTargetfrom products.feature_flags.backend.models.feature_flag import FeatureFlagStatus = ExperimentHealthCheckStatus@@ -44,7 +45,9 @@ def _context(variants: list[tuple[str, int, float, bool | None]],*,baseline: tuple[int, int] = (1000, 100),- manual_running_time: bool = False,+ planned_samples: int | None = 3201,+ current_samples: int | None = None,+ variances: dict[str, ComparisonVariance] | None = None,**changes: Any,) -> HealthContext:control_samples, control_conversions = baseline@@ -69,8 +72,12 @@ def _context(metric = PrimaryMetric(name="Checkout",is_funnel=True,- minimum_detectable_effect=30,- manual_running_time=manual_running_time,+ target=RunningTimeTarget(+ minimum_detectable_effect=30,+ recommended_sample_size=planned_samples,+ current_sample_size=current_samples,+ variances=variances or {},+ ),result=result,)return replace(RUNNING_CONTEXT, primary_metric=metric, **changes)@@ -87,10 +94,20 @@ class TestEarlySignificance(TestCase):("significant_and_larger_than_detectable", _context([("test", 1000, -0.5, True)]), None),("not_significant", _context([("test", 1000, 0.2, False)]), None),("analysis_stopped", _context([("test", 1000, 0.2, None)]), None),- ("sample_reached_the_plan", _context([("test", 20000, 0.06, True)], baseline=(20000, 2000)), None),+ ("sample_reached_the_plan", _context([("test", 1000, 0.2, True)], planned_samples=2000), None),+ ("no_planned_sample", _context([("test", 1000, 0.2, True)], planned_samples=None), None),+ (+ "a_paused_variant_counts_toward_nothing",+ _context(+ [("test", 1000, 0.2, True), ("paused", 500, 0.0, None)], planned_samples=2400, current_samples=2000+ ),+ "test",+ ),(- "uneven_split_past_the_planned_total",- _context([("test", 500, 0.34, True)], baseline=(9000, 900)),+ "detectable_with_the_targets_variances",+ _context(+ [("test", 1000, 0.2, True)], variances={"test": ComparisonVariance(control=0.02, variant=0.02)}+ ),None,),(@@ -144,18 +161,12 @@ class TestEarlySignificance(TestCase):primary_metric=PrimaryMetric(name="Revenue",is_funnel=False,- minimum_detectable_effect=30,- manual_running_time=False,+ target=RunningTimeTarget(minimum_detectable_effect=30, recommended_sample_size=5000),result=None,),),Status.NOT_APPLICABLE,),- (- "manual_running_time",- _context([("test", 1000, 0.2, True)], manual_running_time=True),- Status.NOT_APPLICABLE,- ),("under_50_users", _context([("test", 20, 0.2, True)], baseline=(20, 2)), Status.NOT_APPLICABLE),("no_control_conversions",@@ -180,20 +191,20 @@ class TestEarlySignificanceFromStoredResults(BaseTest):{"minimum_detectable_effect": 20},15,Status.FINDING,- 20,+ (20, 7824),),- ("inline_metric_and_the_teams_default_effect", False, True, {}, 15, Status.FINDING, 15),- ("inline_metric_and_the_product_default_effect", False, True, {}, None, Status.FINDING, 30),- ("shared_metric", True, True, {}, None, Status.FINDING, 30),+ ("inline_metric_and_the_teams_default_effect", False, True, {}, 15, Status.FINDING, (15, 13910)),+ ("inline_metric_and_the_product_default_effect", False, True, {}, None, Status.FINDING, (30, 3478)),+ ("shared_metric", True, True, {}, None, Status.FINDING, (30, 3478)),("result_of_an_earlier_configuration", False, False, {}, None, Status.NOT_APPLICABLE, None),(- "manual_running_time",+ "manual_running_time_reads_the_saved_target",False,True,- {"exposure_estimate_config": {"conversionRateInputType": "manual"}},- None,- Status.NOT_APPLICABLE,+ {"exposure_estimate_config": {"conversionRateInputType": "manual"}, "recommended_sample_size": 5000},None,+ Status.FINDING,+ (30, 5000),),])@@ -205,7 +216,7 @@ class TestEarlySignificanceFromStoredResults(BaseTest):running_time_calculation: dict[str, Any],team_effect: int | None,expected: Status,- expected_effect: int | None,+ expected_plan: tuple[int, int] | None,) -> None:if team_effect is not None:TeamExperimentsConfig.objects.create(team=self.team, default_minimum_detectable_effect=team_effect)@@ -275,8 +286,13 @@ class TestEarlySignificanceFromStoredResults(BaseTest):None,)self.assertEqual(- (check.status, finding.evidence["minimum_detectable_effect"] if finding else None),- (expected, expected_effect),+ (+ check.status,+ (finding.evidence["minimum_detectable_effect"], finding.evidence["planned_samples"])+ if finding+ else None,+ ),+ (expected, expected_plan),)if finding:self.assertIn('"Checkout funnel"' if shared else '"purchase conversion"', finding.detail)diff --git a/products/experiments/backend/logic/result_stats.py b/products/experiments/backend/logic/result_stats.pyindex c0863b67bc7b..a5d758006c5c 100644--- a/products/experiments/backend/logic/result_stats.py+++ b/products/experiments/backend/logic/result_stats.py@@ -17,10 +17,12 @@ StatsMethod = Literal["bayesian", "frequentist"]_INTERVAL_KEYS = ("credible_interval", "confidence_interval")-_BASELINE_FIELDS = frozenset(ExperimentStatsBaseValidated.model_fields)+# A funnel's stored stats carry sample sessions per step, which can be large, and no reader of a stored row needs them.+_UNREAD_FIELDS = frozenset({"step_sessions"})+_BASELINE_FIELDS = frozenset(ExperimentStatsBaseValidated.model_fields) - _UNREAD_FIELDS_VARIANT_FIELDS_BY_METHOD = {- "bayesian": frozenset(ExperimentVariantResultBayesian.model_fields),- "frequentist": frozenset(ExperimentVariantResultFrequentist.model_fields),+ "bayesian": frozenset(ExperimentVariantResultBayesian.model_fields) - _UNREAD_FIELDS,+ "frequentist": frozenset(ExperimentVariantResultFrequentist.model_fields) - _UNREAD_FIELDS,}@@ -60,6 +62,28 @@ def _without_non_finite_intervals(variant: object) -> object:}+def parse_stored_response(raw: object) -> ExperimentQueryResponse | None:+ """The baseline and variant results of a stored result row, or None when the row does not parse."""+ if not isinstance(raw, Mapping):+ return None+ # A stored row is a full response dump in the schema of its write time. It can carry keys that the+ # current schema forbids: the cache fields of a cached response at the top level, and fields that a+ # later schema renamed or removed inside the baseline and the variants. The models reject unknown+ # keys, so keep only the keys that they declare. Schema drift then does not discard the row.+ baseline = _declared_fields_only(raw.get("baseline"), _BASELINE_FIELDS)+ variant_results = raw.get("variant_results")+ if isinstance(variant_results, list):+ # sanitize_non_finite stores an overflowed interval bound as null, which the schema rejects.+ # Drop such an interval so that the counts and means of the day still render.+ variant_results = [+ _without_non_finite_intervals(_declared_variant_fields_only(variant)) for variant in variant_results+ ]+ try:+ return ExperimentQueryResponse.model_validate({"baseline": baseline, "variant_results": variant_results})+ except ValidationError:+ return None++@frozenclass Interval:low: float@@ -137,27 +161,8 @@ class MetricResultStats:@classmethoddef from_stored(cls, raw: object) -> Self | None:"""Return None instead of raising when the row does not parse or has no baseline."""- if not isinstance(raw, Mapping):- return None- # A stored row is a full response dump in the schema of its write time. It can carry keys that the- # current schema forbids: the cache fields of a cached response at the top level, and fields that a- # later schema renamed or removed inside the baseline and the variants. The models reject unknown- # keys, so keep only the keys that they declare. Schema drift then does not discard the row.- baseline = _declared_fields_only(raw.get("baseline"), _BASELINE_FIELDS)- variant_results = raw.get("variant_results")- if isinstance(variant_results, list):- # sanitize_non_finite stores an overflowed interval bound as null, which the schema rejects.- # Drop such an interval so that the counts and means of the day still render.- variant_results = [- _without_non_finite_intervals(_declared_variant_fields_only(variant)) for variant in variant_results- ]- try:- response = ExperimentQueryResponse.model_validate(- {"baseline": baseline, "variant_results": variant_results}- )- except ValidationError:- return None- return cls.from_response(response)+ response = parse_stored_response(raw)+ return cls.from_response(response) if response is not None else Nonedef variant(self, key: str) -> VariantStats | None:return next((variant for variant in self.variants if variant.key == key), None)diff --git a/products/experiments/backend/metric_calculation/results.py b/products/experiments/backend/metric_calculation/results.pyindex 24f592121c18..b8b50b76e80a 100644--- a/products/experiments/backend/metric_calculation/results.py+++ b/products/experiments/backend/metric_calculation/results.py@@ -36,9 +36,11 @@ from django.utils import timezone as django_timezoneimport structlog+from posthog.schema import ExperimentQueryResponse+from posthog.dataclasses import frozen-from products.experiments.backend.logic.result_stats import MetricResultStats+from products.experiments.backend.logic.result_stats import MetricResultStats, parse_stored_responsefrom products.experiments.backend.metric_calculation.config import MetricCalculationConfig, build_calculation_configsfrom products.experiments.backend.models.experiment import (Experiment,@@ -187,8 +189,21 @@ class MetricResultStore:def current_result(self, calculation_config: MetricCalculationConfig) -> MetricResultStats | None:"""The statistics of the result that `current_outcome` picks, or None when it has none or doesn't parse."""+ response = self.current_response(calculation_config)+ return MetricResultStats.from_response(response) if response is not None else None++ def current_response(self, calculation_config: MetricCalculationConfig) -> ExperimentQueryResponse | None:+ """The baseline and variant results of the result that `current_outcome` picks, or None when it has none or+ doesn't parse."""stored = self._current_rows({self.experiment_id: calculation_config}).values_list("result", flat=True).first()- return MetricResultStats.from_stored(stored)+ return parse_stored_response(stored)++ def current_data_until(self, calculation_config: MetricCalculationConfig) -> datetime | None:+ """When the data of the result that `current_outcome` picks ends, as `current_outcomes` defines it, or None when+ it has none."""+ # F() selects the alias, which values_list can't select by name.+ rows = self._current_rows({self.experiment_id: calculation_config})+ return rows.values_list(F("data_until"), flat=True).first()def previous_completed(self, metric_uuid: str, calculation_key: str, *, before: datetimediff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.pyindex 5f51289ea922..b85e3fbe2b40 100644--- a/products/experiments/backend/presentation/serializers.py+++ b/products/experiments/backend/presentation/serializers.py@@ -6,6 +6,7 @@ All serializer classes and custom field classes live here.ViewSet remains in experiments.py."""+import mathfrom copy import deepcopyfrom typing import Annotated, Any, TypeGuard@@ -54,6 +55,7 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthInputsTrafficStatus,ExperimentHealthMetricCompileErrorKind,HealthExposureInputsStatus,+ RunningTimeMode,)from products.experiments.backend.facade.timeseries import METRIC_BUILDERS, apply_saved_metric_overridesfrom products.experiments.backend.hogql_queries.exposure_query_logic import resolve_default_exposure_event@@ -1720,6 +1722,17 @@ class ExperimentMetricsRecalculationLatestSerializer(ExperimentMetricsRecalculat)+class FiniteFloatField(serializers.FloatField):+ """A float field that rejects "nan" and "inf". `FloatField` accepts both, no estimate can use them, and JSON can't+ carry them back."""++ def to_internal_value(self, data: Any) -> float:+ value = super().to_internal_value(data)+ if not math.isfinite(value):+ self.fail("invalid")+ return value++class RunningTimeBaselineStatsSerializer(serializers.Serializer):"""Raw control-group statistics the calculator uses to derive a baseline value and variance.@@ -1730,27 +1743,32 @@ class RunningTimeBaselineStatsSerializer(serializers.Serializer):number_of_samples = serializers.IntegerField(min_value=0, help_text="Number of control-group samples (users/units) observed.")- sum = serializers.FloatField(+ sum = FiniteFloatField(help_text="Sum of the metric values across the control group (for funnels, the numerator/conversions).")- sum_squares = serializers.FloatField(- required=False, default=0.0, help_text="Sum of squared metric values. Required for ratio/retention variance."+ sum_squares = FiniteFloatField(+ required=False,+ default=0.0,+ help_text=(+ "Sum of squared metric values. Mean metrics read their variance from it, and ratio/retention metrics need "+ "it. Without it, a mean metric's variance is approximated from the baseline value."+ ),)- denominator_sum = serializers.FloatField(+ denominator_sum = FiniteFloatField(required=False,allow_null=True,help_text="Sum of the denominator values. Required for ratio/retention metrics.",)- denominator_sum_squares = serializers.FloatField(+ denominator_sum_squares = FiniteFloatField(required=False, allow_null=True, help_text="Sum of squared denominator values (ratio/retention variance).")- numerator_denominator_sum_product = serializers.FloatField(+ numerator_denominator_sum_product = FiniteFloatField(required=False,allow_null=True,help_text="Sum of numerator×denominator products, used for the delta-method covariance term.",)step_counts = serializers.ListField(- child=serializers.FloatField(),+ child=FiniteFloatField(),required=False,help_text="Per-step counts for funnel metrics; the last entry is the final-step count.",)@@ -1767,7 +1785,7 @@ class RunningTimeCalculationInputSerializer(serializers.Serializer):"(both require baseline_stats or an explicit variance)."),)- minimum_detectable_effect = serializers.FloatField(+ minimum_detectable_effect = FiniteFloatField(min_value=0,help_text="Smallest relative change to detect, as a percentage (e.g. 5 means a 5% lift). Must be > 0.",)@@ -1775,15 +1793,16 @@ class RunningTimeCalculationInputSerializer(serializers.Serializer):required=False,default=2,min_value=2,- help_text="Total number of variants including control (default 2).",+ max_value=100,+ help_text="Total number of variants including control (default 2, at most 100).",)- exposure_rate_per_day = serializers.FloatField(+ exposure_rate_per_day = FiniteFloatField(required=False,allow_null=True,min_value=0,help_text="Expected exposures per day. When provided, the response includes the recommended running time.",)- baseline_value = serializers.FloatField(+ baseline_value = FiniteFloatField(required=False,allow_null=True,help_text=(@@ -1791,12 +1810,13 @@ class RunningTimeCalculationInputSerializer(serializers.Serializer):"(ratio/retention). Provide this or baseline_stats."),)- variance = serializers.FloatField(+ variance = FiniteFloatField(required=False,allow_null=True,help_text=(- "Pre-computed variance for ratio/retention metrics. Provide this or baseline_stats when metric_type is "- "ratio/retention and baseline_value is given directly."+ "Pre-computed variance per user. Provide this or baseline_stats when metric_type is ratio/retention and "+ "baseline_value is given directly. When given, it replaces the variance the calculator derives for any "+ "metric type, including p(1-p) for funnels."),)baseline_stats = RunningTimeBaselineStatsSerializer(@@ -1851,6 +1871,85 @@ class RunningTimeCalculationResultSerializer(serializers.Serializer):)+class RunningTimeTargetQuerySerializer(serializers.Serializer):+ mode = serializers.ChoiceField(+ required=False,+ choices=RunningTimeMode.choices,+ help_text=(+ "The running time mode to size the target for. Defaults to the experiment's saved mode. 'automatic' sizes "+ "it from the results, also for an experiment saved in manual mode."+ ),+ )+++class DetectableEffectCheckpointSerializer(serializers.Serializer):+ weeks_since_launch = serializers.IntegerField(help_text="The week mark, in whole weeks since launch.")+ detectable_effect = serializers.FloatField(+ help_text="The smallest relative change, as a percentage, that the exposures by this week mark can detect."+ )+++class RunningTimeOutlookSerializer(serializers.Serializer):+ """What the experiment can detect over time, at its exposure rate so far."""++ days_since_launch = serializers.FloatField(help_text="Days since the experiment launched, with fractions.")+ exposures_per_day = serializers.FloatField(+ help_text=(+ "Exposures that count toward the target, per day from launch to the end of the window the current "+ "results cover."+ )+ )+ detectable_effect_now = serializers.FloatField(+ help_text="The smallest relative change, as a percentage, that the exposures so far can detect."+ )+ upcoming_checkpoints = DetectableEffectCheckpointSerializer(+ many=True,+ help_text=(+ "Up to three week marks after today, out of weeks 1, 2, 4, 6, 8, 12, 16, 26 and 52, with the change each "+ "can detect at the exposure rate so far."+ ),+ )+ reachable_effect = serializers.FloatField(+ help_text=(+ "The smallest relative change, as a percentage, that the experiment can detect within four weeks from "+ "today. A target for a smaller change is out of reach."+ )+ )+++class RunningTimeTargetSerializer(serializers.Serializer):+ """The target sample of the experiment's running time estimate."""++ minimum_detectable_effect = serializers.FloatField(+ help_text="The relative change, as a percentage, that the target sample is sized to detect."+ )+ recommended_sample_size = serializers.IntegerField(+ allow_null=True,+ help_text=(+ "Exposures needed across the analyzed variants, so that the comparison of each variant with the control "+ "can detect the minimum detectable effect at the analysis's confidence level and 80% power. In manual running time mode, "+ "the saved target. Otherwise sized from the first primary metric's current result, with the variance the "+ "analysis uses and the flag's traffic split. Null when no result can size it yet. The variances come from "+ "the result and don't depend on the effect, so the sample scales with 1 / effect²: the target for another "+ "effect is recommended_sample_size × (minimum_detectable_effect / effect)²."+ ),+ )+ current_sample_size = serializers.IntegerField(+ allow_null=True,+ help_text=(+ "Exposures so far that count toward the target: the control's and those of the variants that get "+ "traffic. A variant at 0% does not count. Null in manual mode, and when no result can size the target."+ ),+ )+ outlook = RunningTimeOutlookSerializer(+ allow_null=True,+ help_text=(+ "What the experiment can detect over time, on the same curve the target is sized on. Null in manual mode, "+ "before launch, when the current results cover less than 0.1 days, and when no result can size the target."+ ),+ )++class ExperimentSessionMetricSourceHitSerializer(serializers.Serializer):"""One event/action source of a metric with at least one matching event in a session recording."""diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex c88e686d87e0..93dd23b973b5 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -80,8 +80,12 @@ from products.access_control.backend.facade.user_access_control import (from products.access_control.backend.presentation.access_control import AccessControlViewSetMixinfrom products.approvals.backend.mixins import ApprovalHandlingMixinfrom products.experiments.backend.experiment_service import ExperimentService, ExperimentVersionConflict-from products.experiments.backend.facade.api import get_experiment_health, get_experiment_health_inputs-from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess+from products.experiments.backend.facade.api import (+ get_experiment_health,+ get_experiment_health_inputs,+ get_running_time_target,+)+from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess, RunningTimeModefrom products.experiments.backend.facade.legacy_migration import (LegacyMigrationError,migrate_experiment as migrate_legacy_experiment,@@ -139,6 +143,8 @@ from products.experiments.backend.presentation.serializers import (RecalculateMetricsRequestSerializer,RunningTimeCalculationInputSerializer,RunningTimeCalculationResultSerializer,+ RunningTimeTargetQuerySerializer,+ RunningTimeTargetSerializer,ShipVariantSerializer,)from products.experiments.backend.recalculation import (@@ -1691,6 +1697,27 @@ class EnterpriseExperimentsViewSet(service = ExperimentService(team=self.team, user=request.user)return Response(service.get_velocity_stats())+ @validated_request(+ query_serializer=RunningTimeTargetQuerySerializer,+ responses={200: OpenApiResponse(response=RunningTimeTargetSerializer)},+ )+ @action(methods=["GET"], detail=True, url_path="running_time_target", required_scopes=["experiment:read"])+ def running_time_target(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response:+ """The target sample of the experiment's running time estimate, which the experiment page's header compares+ with the exposures so far, and what the experiment can detect over time.++ In manual running time mode it is the saved target. Otherwise it is sized from the first primary metric's+ current result: the variance the analysis uses (with CUPED when the analysis applied it) and the flag's+ traffic split, so that the comparison of each variant with the control reaches the minimum detectable effect.+ """+ experiment: Experiment = self.get_object()+ target = get_running_time_target(+ team_id=self.team.id,+ experiment_id=experiment.id,+ mode=RunningTimeMode(mode) if (mode := request.validated_query_data.get("mode")) else None,+ )+ return Response(RunningTimeTargetSerializer(target).data)+@validated_request(request_serializer=RunningTimeCalculationInputSerializer,responses={200: OpenApiResponse(response=RunningTimeCalculationResultSerializer)},diff --git a/products/experiments/backend/running_time_calculator.py b/products/experiments/backend/running_time_calculator.pyindex 6503a847860f..a265f4301e5a 100644--- a/products/experiments/backend/running_time_calculator.py+++ b/products/experiments/backend/running_time_calculator.py@@ -1,23 +1,26 @@"""Experiment running-time calculator.Pure statistical helpers that estimate the recommended sample size and running-time for an experiment. This mirrors the frontend calculator at-``frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts`` so the-same math is available server-side (e.g. for MCP tools).+time for an experiment.The sample-size formula uses the constant 16 ≈ 4 · 1.96², the multiplier for atwo-tailed test at 95% confidence and 80% power comparing two variants."""import math+from collections.abc import Sequencefrom dataclasses import dataclass, field+from statistics import NormalDistfrom typing import Literal+from products.experiments.stats.shared.statistics import RatioStatistic, SampleMeanStatistic, StatisticError+VARIANCE_SCALING_FACTOR_TOTAL_COUNT = 2VARIANCE_SCALING_FACTOR_SUM = 0.25# 4 · 1.96² ≈ 16: critical-value multiplier for 95% confidence / 80% power, two-tailed, two variants.SAMPLE_SIZE_Z_FACTOR = 16+_POWER_Z = NormalDist().inv_cdf(0.8)# Manual calculator only supports these types (ratio/retention require full baseline data).ManualCalculatorMetricType = Literal["funnel", "mean_count", "mean_sum_or_avg"]@@ -48,9 +51,9 @@ def calculate_variance(metric_type: CalculatorMetricType, baseline_value: float):func:`calculate_variance_from_stats`)."""if metric_type == "mean_count":- return VARIANCE_SCALING_FACTOR_TOTAL_COUNT * baseline_value+ return _finite(VARIANCE_SCALING_FACTOR_TOTAL_COUNT * baseline_value)if metric_type == "mean_sum_or_avg":- return VARIANCE_SCALING_FACTOR_SUM * baseline_value**2+ return _finite(VARIANCE_SCALING_FACTOR_SUM * baseline_value * baseline_value)# funnel: variance is embedded in the p(1-p) sample-size formula; ratio/retention need stats.return None@@ -62,41 +65,36 @@ def calculate_variance_from_stats() -> float | None:"""Variance from full baseline statistics.- - mean metrics (count/sum): scaling factor on the baseline value+ - mean metrics: the sample variance of the stats, as the analysis computes it. Without usable stats, a scaling+ factor on the baseline value- funnel: ``None`` (variance is implicit in p(1-p))- - ratio and retention: delta method with covariance-- Delta method for ratio R = M/D::-- Var(R) ≈ Var(M)/D² + M²·Var(D)/D⁴ - 2·M·Cov(M,D)/D³+ - ratio and retention: the analysis's delta method with covariance (``RatioStatistic``)"""if metric_type in ("mean_count", "mean_sum_or_avg"):- return calculate_variance(metric_type, baseline_value)+ observed = _sample_variance(baseline) if baseline is not None else None+ return observed if observed is not None else calculate_variance(metric_type, baseline_value)if metric_type == "funnel":return Noneif metric_type in ("ratio", "retention"):- if baseline is None or not baseline.denominator_sum:+ if baseline is None or not baseline.denominator_sum or baseline.number_of_samples < 2:return None-n = baseline.number_of_samples- if n == 0:+ try:+ return _finite(+ RatioStatistic(+ n=n,+ m_statistic=SampleMeanStatistic(n=n, sum=baseline.sum, sum_squares=baseline.sum_squares),+ d_statistic=SampleMeanStatistic(+ n=n, sum=baseline.denominator_sum, sum_squares=baseline.denominator_sum_squares or 0+ ),+ m_d_sum_of_products=baseline.numerator_denominator_sum_product or 0,+ ).variance+ )+ except (StatisticError, ArithmeticError):return None- # Means for numerator (M) and denominator (D).- mean_m = baseline.sum / n- mean_d = baseline.denominator_sum / n-- # Variances via Var(X) = E[X²] - E[X]².- var_m = baseline.sum_squares / n - mean_m**2- var_d = (baseline.denominator_sum_squares or 0) / n - mean_d**2-- # Covariance: Cov(M,D) = E[MD] - E[M]·E[D].- cov = (baseline.numerator_denominator_sum_product or 0) / n - mean_m * mean_d-- return var_m / mean_d**2 + (mean_m**2 * var_d) / mean_d**4 - (2 * mean_m * cov) / mean_d**3-return None@@ -106,13 +104,17 @@ def calculate_sample_size(mde: float,number_of_variants: int,variance: float | None = None,+ split: Sequence[float] | None = None,) -> int | None:"""Total recommended sample size across all variants.``mde`` is a percentage (e.g. ``5`` for a 5% minimum detectable effect). Forratio/retention metrics ``variance`` must be supplied (see:func:`calculate_variance_from_stats`); for mean metrics it is derived from- ``baseline_value`` when omitted.+ ``baseline_value`` when omitted, and for funnels from p(1-p).++ ``split`` holds the traffic shares of the analyzed variants with the control first, in any unit. Without it, the+ ``number_of_variants`` variants get equal shares."""if mde == 0:return None@@ -121,18 +123,37 @@ def calculate_sample_size(if d == 0:return None- if metric_type == "funnel":- # Binomial metric: N = (16 · p · (1 - p)) / d²- sample_size_formula = (SAMPLE_SIZE_Z_FACTOR * baseline_value * (1 - baseline_value)) / d**2- else:- if variance is None:+ if variance is None:+ if metric_type == "funnel":+ variance = baseline_value * (1 - baseline_value)+ else:variance = calculate_variance(metric_type, baseline_value)- if variance is None:- return None- # Count / Sum / Ratio / Retention: N = (16 · variance) / d²- sample_size_formula = (SAMPLE_SIZE_Z_FACTOR * variance) / d**2+ if variance is None:+ return None- sample_size = math.ceil(sample_size_formula * number_of_variants)+ if split is None:+ if number_of_variants < 2:+ return None+ # Every comparison of an even split is the same, so one share stands for all of them.+ control_share, variant_shares = 1 / number_of_variants, [1 / number_of_variants]+ else:+ shares = _normalized_shares(split)+ if shares is None:+ return None+ control_share, *variant_shares = shares+ comparisons = [+ comparison_sample_size(+ d,+ control_variance=variance,+ variant_variance=variance,+ control_share=control_share,+ variant_share=variant_share,+ )+ for variant_share in variant_shares+ ]+ if any(comparison is None for comparison in comparisons):+ return None+ sample_size = round_up_samples(max(comparison for comparison in comparisons if comparison is not None))# A funnel baseline above 1 flips the (1 - p) term negative, and the delta method can return a# negative variance for ratio/retention. Both make the sample size non-positive, which has no@@ -149,6 +170,11 @@ def calculate_baseline_value(baseline: BaselineStats, metric_type: CalculatorMetReturns: avg events/user (count), avg property value/user (sum), conversionrate (funnel), or the ratio (ratio/retention)."""+ value = _baseline_value(baseline, metric_type)+ return _finite(value) if value is not None else None+++def _baseline_value(baseline: BaselineStats, metric_type: CalculatorMetricType) -> float | None:if baseline.number_of_samples == 0:return None@@ -178,18 +204,20 @@ def calculate_detectable_effect(control_samples: int,variant_samples: int,variance: float | None = None,+ variant_variance: float | None = None,+ confidence_level: float = 0.95,) -> float | None:"""The smallest relative effect, as a percentage, that the sample-size formula detects between a control and avariant of these sizes.The inverse of :func:`calculate_sample_size`: for two variants of ``n`` samples each, the sample size for thereturned effect is about ``2 · n``. The formula takes variants of equal size, so ``1/n_control + 1/n_variant``- replaces its ``2/n`` for an uneven split.+ replaces its ``2/n`` for an uneven split. `variant_variance` defaults to the control's `variance`."""if control_samples <= 0 or variant_samples <= 0 or baseline_value == 0:return None- if metric_type == "funnel":+ if metric_type == "funnel" and variance is None:# A conversion rate of 0 or 1 has no binomial variance, so no effect size follows from it.if not 0 < baseline_value < 1:return None@@ -198,13 +226,103 @@ def calculate_detectable_effect(variance = calculate_variance(metric_type, baseline_value)if variance is None or variance <= 0:return None+ if variant_variance is None:+ variant_variance = variance- d = math.sqrt(SAMPLE_SIZE_Z_FACTOR / 2 * variance * (1 / control_samples + 1 / variant_samples))- return 100 * d / abs(baseline_value)+ d = math.sqrt(z_factor(confidence_level) / 2 * (variance / control_samples + variant_variance / variant_samples))+ return _finite(100 * d / abs(baseline_value))+++def comparison_sample_size(+ difference: float,+ *,+ control_variance: float,+ variant_variance: float,+ control_share: float,+ variant_share: float,+ confidence_level: float = 0.95,+) -> float | None:+ """The exposures across all variants at which the comparison of one variant with the control detects an absolute+ `difference`, for each side's share of the exposures and its own variance.++ For two even variants with the same variance this is ``2 · 16 · variance / difference²``, twice the samples per+ variant of the classic formula. None when the inputs are too extreme to give a finite number.+ """+ denominator = difference * difference+ # A tiny difference underflows to 0 when squared.+ if denominator == 0:+ return None+ return _finite(+ z_factor(confidence_level)+ * (control_variance / control_share + variant_variance / variant_share)+ / 2+ / denominator+ )+++def comparison_detectable_difference(+ samples: float,+ *,+ control_variance: float,+ variant_variance: float,+ control_share: float,+ variant_share: float,+ confidence_level: float = 0.95,+) -> float | None:+ """The absolute difference that the comparison of one variant with the control detects at `samples` exposures+ across all variants, split by each side's share. The inverse of :func:`comparison_sample_size`, so the detectable+ difference shrinks with the square root of the samples."""+ squared = z_factor(confidence_level) * (control_variance / control_share + variant_variance / variant_share) / 2+ # The delta method can return a negative variance, and stats without a spread a zero one. Neither has a detectable+ # difference, just as neither gets a target sample.+ if samples <= 0 or not squared > 0:+ return None+ return _finite(math.sqrt(squared / samples))+++def z_factor(confidence_level: float) -> float:+ """The multiplier of the sample-size formula for a two-sided test at `confidence_level` with 80% power. It is+ `SAMPLE_SIZE_Z_FACTOR` at 95%, so that every estimate at 95% keeps its value."""+ if not 0 < confidence_level < 1:+ confidence_level = 0.95+ scale = (NormalDist().inv_cdf((1 + confidence_level) / 2) + _POWER_Z) / (NormalDist().inv_cdf(0.975) + _POWER_Z)+ return SAMPLE_SIZE_Z_FACTOR * scale**2+++def round_up_samples(samples: float) -> int:+ # Float noise would otherwise turn an exact 3,200 into 3,201.+ return math.ceil(round(samples, 6))+++def _normalized_shares(split: Sequence[float]) -> list[float] | None:+ total = sum(split)+ if len(split) < 2 or total <= 0 or any(share <= 0 for share in split):+ return None+ return [share / total for share in split]+++def _sample_variance(baseline: BaselineStats) -> float | None:+ # Stats that don't fit together, such as a sum_squares the caller left at its default of 0, carry no variance.+ if baseline.number_of_samples < 2:+ return None+ try:+ return _finite(+ SampleMeanStatistic(+ n=baseline.number_of_samples, sum=baseline.sum, sum_squares=baseline.sum_squares+ ).variance+ )+ except (StatisticError, ArithmeticError):+ return Nonedef calculate_running_time_days(sample_size: int | None, exposure_rate_per_day: float | None) -> int | None:"""Days to reach ``sample_size`` at ``exposure_rate_per_day`` exposures/day."""if not sample_size or not exposure_rate_per_day or exposure_rate_per_day <= 0:return None- return math.ceil(sample_size / exposure_rate_per_day)+ days = _finite(sample_size / exposure_rate_per_day)+ return math.ceil(days) if days is not None else None+++def _finite(value: float) -> float | None:+ # Extreme inputs overflow to inf or come out as nan. Neither is an estimate, and JSON can't carry either.+ return value if math.isfinite(value) else Nonediff --git a/products/experiments/backend/running_time_target.py b/products/experiments/backend/running_time_target.pynew file mode 100644index 000000000000..a055bc08749a--- /dev/null+++ b/products/experiments/backend/running_time_target.py@@ -0,0 +1,443 @@+"""The target sample of an experiment's running time estimate. The header of the experiment page and the health checks+read it from here, so that both show the same target.++In manual mode the target is the one the user saved from a baseline they entered. Otherwise it comes from the stored+result of the first primary metric: the variance the analysis computes from it, with CUPED when the analysis applied+CUPED, and the traffic split of the flag. The outlook projects what the experiment can detect over time on the same+curve, so that the page holds no second copy of the sizing.+"""++from collections.abc import Mapping+from dataclasses import field+from datetime import datetime++from django.utils import timezone++from pydantic import ValidationError++from posthog.schema import (+ ExperimentFunnelMetric,+ ExperimentMeanMetric,+ ExperimentQueryResponse,+ ExperimentStatsBaseValidated,+ ExperimentVariantResultBayesian,+ ExperimentVariantResultFrequentist,+)++from posthog.dataclasses import frozen++from products.experiments.backend.facade.contracts import RunningTimeMode+from products.experiments.backend.hogql_queries.utils import (+ FrequentistSettings,+ metric_variant_to_cuped_data,+ metric_variant_to_statistic,+)+from products.experiments.backend.metric_calculation.config import (+ MetricCalculationConfig,+ build_primary_calculation_configs,+ team_experiments_configs,+)+from products.experiments.backend.metric_calculation.results import MetricResultStore+from products.experiments.backend.metric_resolution import ExperimentMetric, build_metric+from products.experiments.backend.models.experiment import Experiment+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig+from products.experiments.backend.running_time_calculator import (+ comparison_detectable_difference,+ comparison_sample_size,+ round_up_samples,+)+from products.experiments.backend.setup_context import PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT+from products.experiments.stats.shared.cuped import cuped_adjust+from products.experiments.stats.shared.statistics import (+ ProportionStatistic,+ RatioStatistic,+ SampleMeanStatistic,+ StatisticError,+)+from products.feature_flags.backend.facade.edits import Split, parse_variants++VariantResult = ExperimentVariantResultBayesian | ExperimentVariantResultFrequentist++# A funnel's stored stats carry sample sessions per step, which can be large, and the target never reads them.+_STATS_FIELDS = set(ExperimentStatsBaseValidated.model_fields) - {"step_sessions"}++# Most experiments end within four weeks, so a target further away than this from today is out of reach.+OUT_OF_REACH_DAYS = 28+# The outlook projects to the next `UPCOMING_CHECKPOINTS` of these week marks.+CHECKPOINT_WEEKS = (1, 2, 4, 6, 8, 12, 16, 26, 52)+UPCOMING_CHECKPOINTS = 3+# Before this much of a day has passed, the exposure rate rests on too few minutes to project from.+MINIMUM_DAYS_TO_PROJECT = 0.1+++@frozen+class ComparisonVariance:+ """The variances of the control and of one variant in their comparison, as the analysis computes them."""++ control: float+ variant: float+++@frozen+class DetectableEffectCheckpoint:+ weeks_since_launch: int+ # The smallest relative effect, as a percentage, that the exposures by then detect.+ detectable_effect: float+++@frozen+class RunningTimeOutlook:+ """What the experiment can detect over time, at its exposure rate so far. The detectable effect shrinks with the+ square root of the exposures, on the same curve the target is sized on."""++ days_since_launch: float+ # The exposures that count toward the target, per day from launch to when the data in the results ends.+ exposures_per_day: float+ # The smallest relative effect, as a percentage, that the exposures so far detect.+ detectable_effect_now: float+ # The next week marks after today.+ upcoming_checkpoints: tuple[DetectableEffectCheckpoint, ...]+ # The smallest effect the experiment detects within `OUT_OF_REACH_DAYS` from today. A target for a smaller effect+ # is out of reach.+ reachable_effect: float+++@frozen+class RunningTimeTarget:+ # The relative effect, as a percentage, that the target sizes the experiment to detect.+ minimum_detectable_effect: float+ # The exposures across the analyzed variants that the experiment needs. None when there is no stored result yet,+ # or the result can't size a target.+ recommended_sample_size: int | None+ # The exposures so far that count toward the target: the control's and those of the variants that get traffic.+ # None in manual mode, whose saved target counts every variant, and when no stored result can size the target.+ current_sample_size: int | None = None+ # By variant key. Empty when no stored result was read, or it can't size a target.+ variances: Mapping[str, ComparisonVariance] = field(default_factory=dict)+ # The level the analysis tests at, which sizes the target.+ confidence_level: float = 0.95+ # None in manual mode, before the experiment has run long enough to project, when no stored result can size the+ # target, and when the caller doesn't say when the data in the result ends.+ outlook: RunningTimeOutlook | None = None+++@frozen+class _Comparison:+ # The variant's share of the exposures across the analyzed variants.+ share: float+ variance: ComparisonVariance+ samples: int+++@frozen+class _Sizing:+ baseline_value: float+ control_share: float+ control_samples: int+ comparisons: Mapping[str, _Comparison]+ confidence_level: float++ @property+ def samples(self) -> int:+ return self.control_samples + sum(comparison.samples for comparison in self.comparisons.values())++ def sample_size(self, minimum_detectable_effect: float) -> int | None:+ difference = minimum_detectable_effect / 100 * self.baseline_value+ if difference == 0 or not self.comparisons:+ return None+ totals = [+ comparison_sample_size(+ difference,+ control_variance=comparison.variance.control,+ variant_variance=comparison.variance.variant,+ control_share=self.control_share,+ variant_share=comparison.share,+ confidence_level=self.confidence_level,+ )+ for comparison in self.comparisons.values()+ ]+ if any(total is None for total in totals):+ return None+ samples = round_up_samples(max(total for total in totals if total is not None))+ # The delta method can return a negative variance, and a negative sample has no meaning as a count.+ return samples if samples > 0 else None++ def detectable_effect(self, samples: float) -> float | None:+ """The smallest relative effect, as a percentage, that every comparison with the control detects at `samples`+ exposures across the analyzed variants. The inverse of `sample_size`."""+ if self.baseline_value == 0 or not self.comparisons:+ return None+ differences = [+ comparison_detectable_difference(+ samples,+ control_variance=comparison.variance.control,+ variant_variance=comparison.variance.variant,+ control_share=self.control_share,+ variant_share=comparison.share,+ confidence_level=self.confidence_level,+ )+ for comparison in self.comparisons.values()+ ]+ if any(difference is None for difference in differences):+ return None+ return 100 * max(difference for difference in differences if difference is not None) / abs(self.baseline_value)++ def outlook(self, *, days_since_launch: float, days_measured: float) -> RunningTimeOutlook | None:+ """The outlook `days_since_launch` into the experiment, for exposures that the results counted over+ `days_measured` from launch."""+ samples = self.samples+ if days_measured < MINIMUM_DAYS_TO_PROJECT or samples <= 0:+ return None+ exposures_per_day = samples / days_measured+ now = self.detectable_effect(samples)+ reachable = self.detectable_effect(samples + exposures_per_day * OUT_OF_REACH_DAYS)+ upcoming_weeks = [weeks for weeks in CHECKPOINT_WEEKS if weeks * 7 > days_since_launch][:UPCOMING_CHECKPOINTS]+ checkpoints = [(weeks, self.detectable_effect(exposures_per_day * weeks * 7)) for weeks in upcoming_weeks]+ if now is None or reachable is None or any(effect is None for _, effect in checkpoints):+ return None+ return RunningTimeOutlook(+ days_since_launch=days_since_launch,+ exposures_per_day=exposures_per_day,+ detectable_effect_now=now,+ upcoming_checkpoints=tuple(+ DetectableEffectCheckpoint(weeks_since_launch=weeks, detectable_effect=effect)+ for weeks, effect in checkpoints+ if effect is not None+ ),+ reachable_effect=reachable,+ )+++def load_running_time_target(experiment: Experiment, *, mode: RunningTimeMode | None = None) -> RunningTimeTarget:+ """The target of the experiment's running time estimate. `mode` replaces the saved one, so that the page can show the+ automatic target while the user switches a manual experiment to it. The page rescales the target for an effect the+ user is still editing, because the sample scales with 1 / effect²."""+ team_config = team_experiments_configs([experiment.team_id])[experiment.team_id]+ calculation_config = next(iter(build_primary_calculation_configs(experiment, team_config=team_config)), None)+ manual = _is_manual(experiment, mode)+ store = MetricResultStore(experiment_id=experiment.id)+ response = store.current_response(calculation_config) if calculation_config is not None and not manual else None+ return running_time_target(+ experiment,+ team_config=team_config,+ calculation_config=calculation_config,+ response=response,+ mode=mode,+ measured_until=store.current_data_until(calculation_config)+ if calculation_config is not None and response is not None+ else None,+ )+++def running_time_target(+ experiment: Experiment,+ *,+ team_config: TeamExperimentsConfig,+ calculation_config: MetricCalculationConfig | None,+ response: ExperimentQueryResponse | None,+ mode: RunningTimeMode | None = None,+ measured_until: datetime | None = None,+) -> RunningTimeTarget:+ """The target from the first primary metric's `calculation_config` and its current stored `response`, for a+ caller that has loaded them already. `measured_until` is when the data in the response ends, which the outlook's+ exposure rate divides by, and there is no outlook without it. Results are calculated about once a day, so dividing+ by the days until now would understate the rate early in an experiment. The week marks start after today."""+ planned = planned_minimum_detectable_effect(experiment, team_config)+ metric = _analyzed_metric(calculation_config.definition) if calculation_config is not None else None+ confidence_level = _confidence_level(calculation_config) if calculation_config is not None else 0.95+ sizing = (+ _sizing(metric, response, parse_variants(experiment.feature_flag.variants), confidence_level)+ if metric is not None and response is not None+ else None+ )+ variances = {key: comparison.variance for key, comparison in sizing.comparisons.items()} if sizing else {}+ if _is_manual(experiment, mode):+ return RunningTimeTarget(+ minimum_detectable_effect=planned,+ recommended_sample_size=_saved_sample_size(experiment),+ variances=variances,+ confidence_level=confidence_level,+ )+ now = timezone.now()+ days_since_launch = (now - experiment.start_date).total_seconds() / 86400 if experiment.start_date else None+ days_measured = (+ (measured_until - experiment.start_date).total_seconds() / 86400+ if measured_until is not None and experiment.start_date+ else None+ )+ sample_size = sizing.sample_size(planned) if sizing else None+ return RunningTimeTarget(+ minimum_detectable_effect=planned,+ recommended_sample_size=sample_size,+ current_sample_size=sizing.samples if sizing else None,+ variances=variances,+ confidence_level=confidence_level,+ # The page places the target on the outlook, so an outlook without a target has nothing to show.+ outlook=sizing.outlook(days_since_launch=days_since_launch, days_measured=days_measured)+ if sizing and sample_size is not None and days_since_launch is not None and days_measured is not None+ else None,+ )+++def size_target(+ *,+ metric: ExperimentMetric,+ response: ExperimentQueryResponse,+ split: Split,+ minimum_detectable_effect: float,+ confidence_level: float = 0.95,+) -> int | None:+ """The exposures across the analyzed variants that give each comparison with the control the power to detect+ `minimum_detectable_effect`, at the variances the analysis computes from `response`."""+ sizing = _sizing(metric, response, split, confidence_level)+ return sizing.sample_size(minimum_detectable_effect) if sizing else None+++def project_outlook(+ *,+ metric: ExperimentMetric,+ response: ExperimentQueryResponse,+ split: Split,+ days_since_launch: float,+ days_measured: float | None = None,+ confidence_level: float = 0.95,+) -> RunningTimeOutlook | None:+ """What the exposures in `response` detect now and at the next week marks. `days_measured` is how long after launch+ the results end, and defaults to `days_since_launch`."""+ sizing = _sizing(metric, response, split, confidence_level)+ if sizing is None:+ return None+ return sizing.outlook(+ days_since_launch=days_since_launch,+ days_measured=days_since_launch if days_measured is None else days_measured,+ )+++def _sizing(+ metric: ExperimentMetric, response: ExperimentQueryResponse, split: Split, confidence_level: float+) -> _Sizing | None:+ baseline = response.baseline+ if baseline is None or baseline.number_of_samples < 2:+ return None+ variant_results = [_as_stats(variant) for variant in response.variant_results or []]+ shares = _traffic_shares(split, baseline.key, [variant.key for variant in variant_results])+ if shares is None:+ return None+ try:+ control = metric_variant_to_statistic(metric, baseline)+ comparisons = {+ variant.key: _Comparison(+ share=shares[variant.key],+ variance=_comparison_variance(metric, control, baseline, variant),+ samples=variant.number_of_samples,+ )+ for variant in variant_results+ if variant.key in shares+ }+ except (StatisticError, ArithmeticError):+ return None+ return _Sizing(+ baseline_value=_value(control),+ control_share=shares[baseline.key],+ control_samples=baseline.number_of_samples,+ comparisons=comparisons,+ confidence_level=confidence_level,+ )+++def _confidence_level(calculation_config: MetricCalculationConfig) -> float:+ stats = calculation_config.settings.stats+ return 1 - stats.alpha if isinstance(stats, FrequentistSettings) else stats.ci_level+++def planned_minimum_detectable_effect(experiment: Experiment, team_config: TeamExperimentsConfig) -> float:+ planned = _running_time_calculation(experiment).get("minimum_detectable_effect")+ if isinstance(planned, int | float) and not isinstance(planned, bool) and planned > 0:+ return float(planned)+ return float(team_config.default_minimum_detectable_effect or PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT)+++def is_manual_running_time(experiment: Experiment) -> bool:+ estimate_config = _running_time_calculation(experiment).get("exposure_estimate_config")+ return (+ isinstance(estimate_config, dict) and estimate_config.get("conversionRateInputType") == RunningTimeMode.MANUAL+ )+++def _is_manual(experiment: Experiment, mode: RunningTimeMode | None) -> bool:+ return mode == RunningTimeMode.MANUAL if mode is not None else is_manual_running_time(experiment)+++def _saved_sample_size(experiment: Experiment) -> int | None:+ saved = _running_time_calculation(experiment).get("recommended_sample_size")+ if isinstance(saved, int | float) and not isinstance(saved, bool) and saved > 0:+ return int(saved)+ return None+++def _running_time_calculation(experiment: Experiment) -> dict:+ return experiment.running_time_calculation if isinstance(experiment.running_time_calculation, dict) else {}+++def _comparison_variance(+ metric: ExperimentMetric,+ control: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ baseline: ExperimentStatsBaseValidated,+ variant: ExperimentStatsBaseValidated,+) -> ComparisonVariance:+ # A variant with fewer than 2 samples has no spread of its own yet, so the plan takes the control's.+ if variant.number_of_samples < 2:+ return ComparisonVariance(control=control.variance, variant=control.variance)+ treatment = metric_variant_to_statistic(metric, variant)+ # The analysis applies CUPED only to mean and funnel metrics, and the stored stats carry the covariate only when+ # it did. Its adjustment pools the control with each variant, so each comparison has its own pair of variances.+ if (+ isinstance(metric, ExperimentMeanMetric | ExperimentFunnelMetric)+ and isinstance(control, SampleMeanStatistic | ProportionStatistic)+ and isinstance(treatment, SampleMeanStatistic | ProportionStatistic)+ and baseline.covariate_sum is not None+ and variant.covariate_sum is not None+ ):+ adjusted = cuped_adjust(+ treatment, control, metric_variant_to_cuped_data(variant), metric_variant_to_cuped_data(baseline)+ )+ return ComparisonVariance(+ control=adjusted.control_adjusted.variance, variant=adjusted.treatment_adjusted.variance+ )+ return ComparisonVariance(control=control.variance, variant=treatment.variance)+++def _traffic_shares(split: Split, control_key: str, variant_keys: list[str]) -> dict[str, float] | None:+ """Each analyzed variant's share of the exposures, for the variants that get traffic, with the control first.+ Equal shares when the flag has no readable percentage for one of them. None when no comparison with the control+ can get a sample."""+ percentages = {variant.key: variant.rollout_percentage for variant in split if variant.key}+ keys = [control_key, *variant_keys]+ if any(percentages.get(key) is None for key in keys):+ return dict.fromkeys(keys, 1 / len(keys)) if variant_keys else None+ # A variant at 0% gets no exposures, so no comparison with it can reach any sample.+ served = {key: float(percentages[key] or 0) for key in keys if (percentages[key] or 0) > 0}+ if control_key not in served or len(served) < 2:+ return None+ total = sum(served.values())+ return {key: percentage / total for key, percentage in served.items()}+++def _value(statistic: SampleMeanStatistic | ProportionStatistic | RatioStatistic) -> float:+ if isinstance(statistic, SampleMeanStatistic):+ return statistic.mean+ if isinstance(statistic, ProportionStatistic):+ return statistic.proportion+ return statistic.ratio+++def _analyzed_metric(definition: dict) -> ExperimentMetric | None:+ # The parse the analysis runs, so that a metric it can't compute gets no target.+ try:+ return build_metric(definition)+ except (KeyError, TypeError, ValidationError):+ return None+++def _as_stats(variant: VariantResult) -> ExperimentStatsBaseValidated:+ return ExperimentStatsBaseValidated.model_validate(variant.model_dump(include=_STATS_FIELDS))diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.pyindex bca3d5da0019..6e44664dd35e 100644--- a/products/experiments/backend/test/test_presentation_api.py+++ b/products/experiments/backend/test/test_presentation_api.py@@ -51,7 +51,10 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (EXPERIMENT_EXPOSURE_EVENT_CUTOFF,EXPERIMENT_EXPOSURE_EVENT_FLAG,)-from products.experiments.backend.metric_calculation.config import get_metric_calculation_config+from products.experiments.backend.metric_calculation.config import (+ build_primary_calculation_configs,+ get_metric_calculation_config,+)from products.experiments.backend.models.experiment import (EXPOSURE_FROZEN_GROUP_KEY,EXPOSURE_FROZEN_GROUP_MARKER,@@ -8977,6 +8980,178 @@ class TestExperimentExcludedVariants(_HoistFlagConfigClientMixin, APILicensedTesself.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)+MANUAL = {"exposure_estimate_config": {"conversionRateInputType": "manual"}, "recommended_sample_size": 5000}+++class TestRunningTimeTargetEndpoint(APILicensedTest):+ def _experiment(+ self,+ running_time_calculation: dict[str, Any],+ *,+ with_result: bool = True,+ stats_config: dict | None = None,+ paused_variant_samples: int | None = None,+ result_days_ago: float = 0,+ ) -> Experiment:+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="revenue-flag",+ created_by=self.user,+ filters={+ "groups": [{"properties": [], "rollout_percentage": 100}],+ "multivariate": {+ "variants": [+ {"key": "control", "rollout_percentage": 80},+ {"key": "test", "rollout_percentage": 20},+ *([{"key": "paused", "rollout_percentage": 0}] if paused_variant_samples else []),+ ]+ },+ },+ )+ launched_at = timezone.now() - timedelta(days=3)+ experiment = Experiment.objects.create(+ team=self.team,+ name="revenue",+ feature_flag=flag,+ start_date=launched_at,+ metrics=[+ {+ "kind": "ExperimentMetric",+ "metric_type": "mean",+ "source": {"kind": "EventsNode", "event": "purchase", "math": "sum"},+ "uuid": "revenue",+ }+ ],+ running_time_calculation=running_time_calculation,+ stats_config=stats_config or {},+ )+ if with_result:+ spread = {"number_of_samples": 1000, "sum": 1000, "sum_squares": 26000}+ ExperimentMetricResult.objects.create(+ experiment=experiment,+ metric_uuid="revenue",+ fingerprint=build_primary_calculation_configs(experiment)[0].calculation_key(),+ query_from=launched_at,+ query_to=timezone.now() - timedelta(days=result_days_ago),+ completed_at=timezone.now(),+ status=ExperimentMetricResult.Status.COMPLETED,+ result={+ "baseline": {"key": "control", **spread},+ "variant_results": [+ {"key": "test", "method": "bayesian", **spread},+ *(+ [+ {+ "key": "paused",+ "method": "bayesian",+ "number_of_samples": paused_variant_samples,+ "sum": paused_variant_samples,+ "sum_squares": 26 * paused_variant_samples,+ }+ ]+ if paused_variant_samples+ else []+ ),+ ],+ },+ )+ return experiment++ @parameterized.expand(+ [+ ("sized_from_the_result_and_the_split", {}, {}, "", (30, 13903, 2000, True)),+ ("no_result_yet", {}, {"with_result": False}, "", (30, None, None, False)),+ (+ "a_saved_effect_outside_the_editable_range",+ {"minimum_detectable_effect": 0.5},+ {},+ "",+ (0.5, 50050051, 2000, True),+ ),+ (+ "a_paused_variant_counts_toward_nothing",+ {},+ {"paused_variant_samples": 300},+ "",+ (30, 13903, 2000, True),+ ),+ ("manual_mode_keeps_the_saved_target", MANUAL, {}, "", (30, 5000, None, False)),+ (+ "manual_experiment_sized_from_the_results_on_request",+ MANUAL,+ {},+ "?mode=automatic",+ (30, 13903, 2000, True),+ ),+ ]+ )+ def test_returns_the_target_the_header_compares(+ self,+ _name: str,+ running_time_calculation: dict[str, Any],+ setup: dict[str, Any],+ query: str,+ expected: tuple[float, int | None, int | None, bool],+ ) -> None:+ experiment = self._experiment(running_time_calculation, **setup)++ response = self.client.get(+ f"/api/projects/{self.team.id}/experiments/{experiment.id}/running_time_target/{query}"+ )++ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ effect, sample_size, current, has_outlook = expected+ self.assertEqual(+ response.json(),+ {+ "minimum_detectable_effect": effect,+ "recommended_sample_size": sample_size,+ "current_sample_size": current,+ "outlook": ANY if has_outlook else None,+ },+ )+ if has_outlook:+ assert current is not None+ outlook = response.json()["outlook"]+ # Launched three days ago, so the exposures so far set the rate and the next marks are weeks 1, 2 and 4.+ self.assertAlmostEqual(outlook["exposures_per_day"], current / 3, places=0)+ self.assertEqual([c["weeks_since_launch"] for c in outlook["upcoming_checkpoints"]], [1, 2, 4])++ @parameterized.expand(+ [+ # Launched three days ago with results up to yesterday: 2,000 exposures over two days.+ ("results_up_to_yesterday", 1, 1000),+ # A backfill stamps today with the end of the day, but its data ends when it ran: 2,000 over three days.+ ("a_backfill_of_today", -0.5, 2000 / 3),+ ]+ )+ def test_projects_the_rate_over_the_window_the_results_cover(+ self, _name: str, result_days_ago: float, exposures_per_day: float+ ) -> None:+ experiment = self._experiment({}, result_days_ago=result_days_ago)++ response = self.client.get(f"/api/projects/{self.team.id}/experiments/{experiment.id}/running_time_target/")++ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertAlmostEqual(response.json()["outlook"]["exposures_per_day"], exposures_per_day, places=0)++ @parameterized.expand(+ [+ ("bayesian_99_percent", {"method": "bayesian", "bayesian": {"ci_level": 0.99}}, 20688),+ ("frequentist_90_percent", {"method": "frequentist", "frequentist": {"alpha": 0.1}}, 10952),+ ]+ )+ def test_sizes_at_the_experiments_confidence_level(+ self, _name: str, stats_config: dict[str, Any], sample_size: int+ ) -> None:+ experiment = self._experiment({}, stats_config=stats_config)++ response = self.client.get(f"/api/projects/{self.team.id}/experiments/{experiment.id}/running_time_target/")++ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(response.json()["recommended_sample_size"], sample_size)++class TestCalculateRunningTimeEndpoint(APILicensedTest):def _calculate(self, payload: dict[str, Any]):return self.client.post(@@ -9020,6 +9195,19 @@ class TestCalculateRunningTimeEndpoint(APILicensedTest):self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())self.assertIsNone(response.json()["recommended_running_time_days"])+ def test_mean_from_baseline_stats(self):+ response = self._calculate(+ {+ "metric_type": "mean_sum_or_avg",+ "minimum_detectable_effect": 30,+ "baseline_stats": {"number_of_samples": 1000, "sum": 1000, "sum_squares": 26000},+ }+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ body = response.json()+ self.assertAlmostEqual(body["variance"], 25000 / 999, places=4)+ self.assertEqual(body["recommended_sample_size"], 8898)+def test_ratio_from_baseline_stats(self):response = self._calculate({@@ -9039,7 +9227,7 @@ class TestCalculateRunningTimeEndpoint(APILicensedTest):body = response.json()self.assertAlmostEqual(body["baseline_value"], 10, places=4)self.assertAlmostEqual(body["variance"], 32, places=1)- self.assertEqual(body["recommended_sample_size"], 1024)+ self.assertEqual(body["recommended_sample_size"], 1025)def test_funnel_from_step_counts(self):response = self._calculate(@@ -9059,6 +9247,7 @@ class TestCalculateRunningTimeEndpoint(APILicensedTest):[("missing_baseline", {"metric_type": "funnel", "minimum_detectable_effect": 5}),("zero_mde", {"metric_type": "funnel", "baseline_value": 0.1, "minimum_detectable_effect": 0}),+ ("non_finite_baseline", {"metric_type": "funnel", "baseline_value": "nan", "minimum_detectable_effect": 5}),("ratio_without_variance", {"metric_type": "ratio", "baseline_value": 10, "minimum_detectable_effect": 10}),("ratio_stats_without_denominator_sum",@@ -9074,6 +9263,22 @@ class TestCalculateRunningTimeEndpoint(APILicensedTest):response = self._calculate(payload)self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())+ @parameterized.expand(+ [+ ("effect_too_small_to_compute", {"baseline_value": 0.1, "minimum_detectable_effect": 1e-160}),+ (+ "exposure_rate_too_small_to_compute",+ {"baseline_value": 0.1, "minimum_detectable_effect": 1e-160, "exposure_rate_per_day": 1e-320},+ ),+ ]+ )+ def test_extreme_input_gives_no_estimate(self, _name: str, payload: dict):+ response = self._calculate({"metric_type": "funnel", **payload})+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(+ (response.json()["recommended_sample_size"], response.json()["recommended_running_time_days"]), (None, None)+ )+class TestExperimentSetupContextEndpoint(ClickhouseTestMixin, APILicensedTest):def setUp(self) -> None:diff --git a/products/experiments/backend/test/test_running_time_calculator.py b/products/experiments/backend/test/test_running_time_calculator.pyindex 10ff6440539d..5c8ab4c6ee03 100644--- a/products/experiments/backend/test/test_running_time_calculator.py+++ b/products/experiments/backend/test/test_running_time_calculator.py@@ -70,7 +70,7 @@ class TestCalculateVarianceFromStats:)variance = calculate_variance_from_stats(10, "ratio", baseline)assert variance is not None- assert math.isclose(variance, 32, rel_tol=1e-6)+ assert math.isclose(variance, 32 * 10000 / 9999, rel_tol=1e-6)def test_ratio_zero_covariance(self):baseline = BaselineStats(@@ -83,7 +83,7 @@ class TestCalculateVarianceFromStats:)variance = calculate_variance_from_stats(0.5, "ratio", baseline)assert variance is not None- assert math.isclose(variance, 0.0625, rel_tol=1e-6)+ assert math.isclose(variance, 0.0625 * 1000 / 999, rel_tol=1e-6)def test_ratio_high_positive_covariance_reduces_variance(self):baseline = BaselineStats(@@ -109,7 +109,7 @@ class TestCalculateVarianceFromStats:)variance = calculate_variance_from_stats(0.7, "retention", baseline)assert variance is not None- assert math.isclose(variance, 0.21, rel_tol=1e-6)+ assert math.isclose(variance, 0.21 * 10000 / 9999, rel_tol=1e-6)@parameterized.expand([@@ -132,6 +132,15 @@ class TestCalculateVarianceFromStats:assert variance is not Noneassert math.isclose(variance, 0, abs_tol=1e-9)+ @parameterized.expand(+ [+ ("observed_spread", BaselineStats(number_of_samples=1000, sum=1000, sum_squares=26000), 25000 / 999),+ ("sum_squares_left_out", BaselineStats(number_of_samples=1000, sum=1000), 0.25),+ ]+ )+ def test_mean_variance_follows_the_observed_spread(self, _name, baseline, expected):+ assert math.isclose(calculate_variance_from_stats(1, "mean_sum_or_avg", baseline) or 0, expected)+def test_returns_none_without_baseline(self):assert calculate_variance_from_stats(0.05, "ratio", None) is None@@ -139,10 +148,9 @@ class TestCalculateVarianceFromStats:baseline = BaselineStats(number_of_samples=0, sum=100, denominator_sum=1000)assert calculate_variance_from_stats(10, "ratio", baseline) is None- def test_handles_missing_optional_fields(self):+ def test_returns_none_without_the_denominators_spread(self):baseline = BaselineStats(number_of_samples=1000, sum=5000, sum_squares=30000, denominator_sum=10000)- variance = calculate_variance_from_stats(0.5, "ratio", baseline)- assert variance is not None+ assert calculate_variance_from_stats(0.5, "ratio", baseline) is Noneclass TestCalculateSampleSize:@@ -157,6 +165,46 @@ class TestCalculateSampleSize:def test_sample_size(self, metric_type, baseline_value, mde, variants, expected):assert calculate_sample_size(metric_type, baseline_value, mde, variants) == expected+ @parameterized.expand(+ [+ ("uneven_split", [80, 20], 1800),+ ("even_split_of_three", [1, 1, 1], 1728),+ ]+ )+ def test_sample_size_powers_the_comparison_with_the_smallest_variant(self, _name, split, expected):+ assert calculate_sample_size("funnel", 0.1, 50, len(split), split=split) == expected++ @parameterized.expand(+ [+ ("effect_that_underflows", lambda: calculate_sample_size("mean_sum_or_avg", 1.0, 1e-160, 2)),+ ("effect_that_overflows", lambda: calculate_sample_size("mean_sum_or_avg", 1.0, 1e160, 2)),+ ("baseline_that_overflows", lambda: calculate_sample_size("mean_sum_or_avg", 1e200, 5, 2)),+ ("variance_that_overflows", lambda: calculate_sample_size("ratio", 10, 10, 2, variance=1e308 * 10)),+ (+ "stats_that_overflow",+ lambda: calculate_variance_from_stats(+ 1e200, "mean_sum_or_avg", BaselineStats(number_of_samples=2, sum=1e200, sum_squares=1e300)+ ),+ ),+ (+ "ratio_that_overflows",+ lambda: calculate_baseline_value(+ BaselineStats(number_of_samples=10, sum=1e308, denominator_sum=1e-308), "ratio"+ ),+ ),+ ("exposure_rate_that_underflows", lambda: calculate_running_time_days(1000, 1e-320)),+ (+ "detectable_effect_that_overflows",+ lambda: calculate_detectable_effect("mean_count", 1e-320, 10, 10, 1.0),+ ),+ ]+ )+ def test_extreme_inputs_give_no_estimate(self, _name, calculate):+ assert calculate() is None++ def test_sizes_an_even_split_of_any_count_without_a_list_per_variant(self):+ assert calculate_sample_size("funnel", 0.1, 50, 10**12) == pytest.approx(576 * 10**12, rel=1e-9)+def test_returns_none_for_zero_mde(self):assert calculate_sample_size("funnel", 0.1, 0, 2) is Nonediff --git a/products/experiments/backend/test/test_running_time_target.py b/products/experiments/backend/test/test_running_time_target.pynew file mode 100644index 000000000000..12b19c37ec51--- /dev/null+++ b/products/experiments/backend/test/test_running_time_target.py@@ -0,0 +1,291 @@+import math+from typing import Any++import pytest++from parameterized import parameterized++from posthog.schema import ExperimentFunnelMetric, ExperimentMeanMetric, ExperimentQueryResponse, ExperimentRatioMetric++from products.experiments.backend.metric_resolution import ExperimentMetric+from products.experiments.backend.running_time_target import project_outlook, size_target+from products.feature_flags.backend.facade.edits import FlagVariant++REVENUE = ExperimentMeanMetric.model_validate(+ {"kind": "ExperimentMetric", "metric_type": "mean", "source": {"kind": "EventsNode", "event": "purchase"}}+)+CHECKOUT = ExperimentFunnelMetric.model_validate(+ {"kind": "ExperimentMetric", "metric_type": "funnel", "series": [{"kind": "EventsNode", "event": "purchase"}]}+)+REVENUE_PER_ORDER = ExperimentRatioMetric.model_validate(+ {+ "kind": "ExperimentMetric",+ "metric_type": "ratio",+ "numerator": {"kind": "EventsNode", "event": "purchase", "math": "sum", "math_property": "amount"},+ "denominator": {"kind": "EventsNode", "event": "purchase"},+ }+)+# A mean of 1 with a variance of about 25, so the spread of a revenue metric rather than of a count.+SPREAD = {"number_of_samples": 1000, "sum": 1000, "sum_squares": 26000}+# A covariate with the metric's own spread and a correlation of 0.5, so CUPED keeps 75% of the variance.+COVARIATE = {"covariate_sum": 1000, "covariate_sum_squares": 26000, "covariate_sum_product": 13500}+CONVERSIONS = {"number_of_samples": 1000, "sum": 100, "sum_squares": 100}+# A ratio of 10 with a delta-method variance of about 32.+ORDERS = {+ "number_of_samples": 10000,+ "sum": 500000,+ "sum_squares": 30000000,+ "denominator_sum": 50000,+ "denominator_sum_squares": 300000,+ "numerator_denominator_sum_product": 2600000,+}+EVEN = (("control", 50), ("test", 50))+++def _response(+ *variants: tuple[str, dict[str, Any]], control: dict[str, Any] = SPREAD, control_key: str = "control"+) -> ExperimentQueryResponse:+ return ExperimentQueryResponse.model_validate(+ {+ "baseline": {"key": control_key, **control},+ "variant_results": [{"key": key, "method": "bayesian", **stats} for key, stats in variants],+ }+ )+++def _split(*percentages: tuple[str, float | None]) -> tuple[FlagVariant, ...]:+ return tuple(FlagVariant(key=key, rollout_percentage=percentage) for key, percentage in percentages)+++class TestSizeTarget:+ @parameterized.expand(+ [+ ("observed_variance", REVENUE, _response(("test", SPREAD)), _split(*EVEN), 8898),+ (+ "variant_with_a_wider_spread",+ REVENUE,+ _response(("test", {**SPREAD, "sum_squares": 51000})),+ _split(*EVEN),+ 13347,+ ),+ ("uneven_split", REVENUE, _response(("test", SPREAD)), _split(("control", 80), ("test", 20)), 13903),+ (+ "smallest_of_three_variants_with_the_control_listed_second",+ REVENUE,+ _response(("a", SPREAD), ("b", SPREAD), control_key="original"),+ _split(("b", 10), ("original", 60), ("a", 30)),+ 25952,+ ),+ (+ "variant_at_zero_percent_left_out",+ REVENUE,+ _response(("test", SPREAD), ("paused", {"number_of_samples": 0, "sum": 0, "sum_squares": 0})),+ _split(*EVEN, ("paused", 0)),+ 8898,+ ),+ (+ "excluded_variant_with_traffic_left_out",+ REVENUE,+ _response(("test", SPREAD)),+ _split(("control", 34), ("test", 33), ("excluded", 33)),+ 8900,+ ),+ (+ "unreadable_percentages_as_an_even_split",+ REVENUE,+ _response(("test", SPREAD)),+ _split(("control", None), ("test", 20)),+ 8898,+ ),+ (+ "cuped_variance",+ REVENUE,+ _response(("test", {**SPREAD, **COVARIATE}), control={**SPREAD, **COVARIATE}),+ _split(*EVEN),+ 6674,+ ),+ (+ "cuped_variance_on_an_uneven_split",+ REVENUE,+ _response(+ (+ "test",+ {+ "number_of_samples": 400,+ "sum": 480,+ "sum_squares": 14940,+ "covariate_sum": 480,+ "covariate_sum_squares": 14940,+ "covariate_sum_product": 12067.2,+ },+ ),+ control={+ "number_of_samples": 1600,+ "sum": 1600,+ "sum_squares": 41575,+ "covariate_sum": 1600,+ "covariate_sum_squares": 41575,+ "covariate_sum_product": 21587.5,+ },+ ),+ _split(("control", 80), ("test", 20)),+ 8640,+ ),+ (+ "funnel_conversion_rate",+ CHECKOUT,+ _response(("test", CONVERSIONS), control=CONVERSIONS),+ _split(*EVEN),+ 3200,+ ),+ (+ "ratio_delta_method",+ REVENUE_PER_ORDER,+ _response(("test", ORDERS), control=ORDERS),+ _split(*EVEN),+ 114,+ ),+ (+ "stats_that_do_not_fit_together",+ REVENUE,+ _response(("test", SPREAD), control={**SPREAD, "sum_squares": 0}),+ _split(*EVEN),+ None,+ ),+ ]+ )+ def test_sizes_each_comparison_with_the_control(+ self,+ _name: str,+ metric: ExperimentMetric,+ response: ExperimentQueryResponse,+ split: tuple[FlagVariant, ...],+ expected: int | None,+ ) -> None:+ assert size_target(metric=metric, response=response, split=split, minimum_detectable_effect=30) == expected++ @parameterized.expand(+ [+ ("total", {}),+ ("sum", {"math": "sum", "math_property": "amount"}),+ ("avg", {"math": "avg", "math_property": "amount"}),+ ("hogql", {"math": "hogql", "math_hogql": "sum(toFloat(properties.amount))"}),+ ("unique_session", {"math": "unique_session"}),+ ("dau", {"math": "dau"}),+ ]+ )+ def test_sizes_every_mean_math_from_its_spread(self, _name: str, math: dict[str, str]) -> None:+ metric = ExperimentMeanMetric.model_validate(+ {+ "kind": "ExperimentMetric",+ "metric_type": "mean",+ "source": {"kind": "EventsNode", "event": "purchase", **math},+ }+ )++ assert (+ size_target(+ metric=metric, response=_response(("test", SPREAD)), split=_split(*EVEN), minimum_detectable_effect=30+ )+ == 8898+ )++ @parameterized.expand([("90_percent", 0.9, 7009), ("95_percent", 0.95, 8898), ("99_percent", 0.99, 13240)])+ def test_sizes_at_the_analysis_confidence_level(self, _name: str, confidence_level: float, expected: int) -> None:+ assert (+ size_target(+ metric=REVENUE,+ response=_response(("test", SPREAD)),+ split=_split(*EVEN),+ minimum_detectable_effect=30,+ confidence_level=confidence_level,+ )+ == expected+ )++ @parameterized.expand([("underflows", 1e-160), ("overflows", 1e160)])+ def test_an_effect_too_extreme_to_compute_gives_no_target(self, _name: str, effect: float) -> None:+ assert (+ size_target(+ metric=REVENUE,+ response=_response(("test", SPREAD)),+ split=_split(*EVEN),+ minimum_detectable_effect=effect,+ )+ is None+ )++ @parameterized.expand(+ [+ ("mean", REVENUE, _response(("test", SPREAD))),+ ("cuped", REVENUE, _response(("test", {**SPREAD, **COVARIATE}), control={**SPREAD, **COVARIATE})),+ ("funnel", CHECKOUT, _response(("test", CONVERSIONS), control=CONVERSIONS)),+ ("ratio", REVENUE_PER_ORDER, _response(("test", ORDERS), control=ORDERS)),+ ]+ )+ def test_sample_scales_with_the_inverse_square_of_the_effect(+ self, _name: str, metric: ExperimentMetric, response: ExperimentQueryResponse+ ) -> None:+ # The page rescales the target by this rule while the user edits the effect, without asking again.+ split = _split(("control", 80), ("test", 20))+ at_30 = size_target(metric=metric, response=response, split=split, minimum_detectable_effect=30)+ at_15 = size_target(metric=metric, response=response, split=split, minimum_detectable_effect=15)++ assert at_30 is not None and at_15 is not None+ assert at_15 == pytest.approx(4 * at_30, abs=4)+++class TestProjectOutlook:+ @parameterized.expand(+ [+ ("first_days", 3, 3, (1, 2, 4)),+ ("past_week_four", 30, 30, (6, 8, 12)),+ # Results that end before today set the rate, and the week marks still start after today.+ ("results_from_before_today", 14.2, 13.9, (4, 6, 8)),+ ("first_two_hours", 0.05, 0.05, None),+ ]+ )+ def test_projects_the_next_week_marks_on_the_target_curve(+ self, _name: str, days_since_launch: float, days_measured: float, weeks: tuple[int, ...] | None+ ) -> None:+ response = _response(("test", SPREAD))+ target = size_target(metric=REVENUE, response=response, split=_split(*EVEN), minimum_detectable_effect=30)+ assert target is not None++ outlook = project_outlook(+ metric=REVENUE,+ response=response,+ split=_split(*EVEN),+ days_since_launch=days_since_launch,+ days_measured=days_measured,+ )++ if weeks is None:+ assert outlook is None+ return+ assert outlook is not None+ rate = 2000 / days_measured++ def on_curve(samples: float) -> Any:+ return pytest.approx(30 * math.sqrt(target / samples), rel=1e-3)++ assert outlook.exposures_per_day == pytest.approx(rate)+ assert outlook.detectable_effect_now == on_curve(2000)+ assert [(c.weeks_since_launch, c.detectable_effect) for c in outlook.upcoming_checkpoints] == [+ (week, on_curve(rate * 7 * week)) for week in weeks+ ]+ assert outlook.reachable_effect == on_curve(2000 + rate * 28)++ @parameterized.expand(+ [+ # A covariance larger than the two spreads allow gives the delta method a negative variance.+ ("negative_ratio_variance", REVENUE_PER_ORDER, {**ORDERS, "numerator_denominator_sum_product": 3100000}),+ ("constant_metric", REVENUE, {"number_of_samples": 1000, "sum": 1000, "sum_squares": 1000}),+ ]+ )+ def test_projects_nothing_from_stats_without_a_spread(+ self, _name: str, metric: ExperimentMetric, stats: dict[str, Any]+ ) -> None:+ response = _response(("test", stats), control=stats)++ assert project_outlook(metric=metric, response=response, split=_split(*EVEN), days_since_launch=10) is Nonediff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex d115d264a013..8ae567f1b9d1 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -3992,6 +3992,49 @@ export interface ExperimentMetricsRecalculationLatestApi {readonly result_source: ResultSourceEnumApi}+export interface DetectableEffectCheckpointApi {+ /** The week mark, in whole weeks since launch. */+ weeks_since_launch: number+ /** The smallest relative change, as a percentage, that the exposures by this week mark can detect. */+ detectable_effect: number+}++/**+ * What the experiment can detect over time, at its exposure rate so far.+ */+export interface RunningTimeOutlookApi {+ /** Days since the experiment launched, with fractions. */+ days_since_launch: number+ /** Exposures that count toward the target, per day from launch to the end of the window the current results cover. */+ exposures_per_day: number+ /** The smallest relative change, as a percentage, that the exposures so far can detect. */+ detectable_effect_now: number+ /** Up to three week marks after today, out of weeks 1, 2, 4, 6, 8, 12, 16, 26 and 52, with the change each can detect at the exposure rate so far. */+ upcoming_checkpoints: DetectableEffectCheckpointApi[]+ /** The smallest relative change, as a percentage, that the experiment can detect within four weeks from today. A target for a smaller change is out of reach. */+ reachable_effect: number+}++/**+ * The target sample of the experiment's running time estimate.+ */+export interface RunningTimeTargetApi {+ /** The relative change, as a percentage, that the target sample is sized to detect. */+ minimum_detectable_effect: number+ /**+ * Exposures needed across the analyzed variants, so that the comparison of each variant with the control can detect the minimum detectable effect at the analysis's confidence level and 80% power. In manual running time mode, the saved target. Otherwise sized from the first primary metric's current result, with the variance the analysis uses and the flag's traffic split. Null when no result can size it yet. The variances come from the result and don't depend on the effect, so the sample scales with 1 / effect²: the target for another effect is recommended_sample_size × (minimum_detectable_effect / effect)².+ * @nullable+ */+ recommended_sample_size: number | null+ /**+ * Exposures so far that count toward the target: the control's and those of the variants that get traffic. A variant at 0% does not count. Null in manual mode, and when no result can size the target.+ * @nullable+ */+ current_sample_size: number | null+ /** What the experiment can detect over time, on the same curve the target is sized on. Null in manual mode, before launch, when the current results cover less than 0.1 days, and when no result can size the target. */+ outlook: RunningTimeOutlookApi | null+}+/*** * `fired_any` - fired_any* * `no_metric_activity` - no_metric_activity@@ -4357,7 +4400,7 @@ export interface RunningTimeBaselineStatsApi {number_of_samples: number/** Sum of the metric values across the control group (for funnels, the numerator/conversions). */sum: number- /** Sum of squared metric values. Required for ratio/retention variance. */+ /** Sum of squared metric values. Mean metrics read their variance from it, and ratio/retention metrics need it. Without it, a mean metric's variance is approximated from the baseline value. */sum_squares?: number/*** Sum of the denominator values. Required for ratio/retention metrics.@@ -4396,8 +4439,9 @@ export interface RunningTimeCalculationInputApi {*/minimum_detectable_effect: number/**- * Total number of variants including control (default 2).+ * Total number of variants including control (default 2, at most 100).* @minimum 2+ * @maximum 100*/number_of_variants?: number/**@@ -4412,7 +4456,7 @@ export interface RunningTimeCalculationInputApi {*/baseline_value?: number | null/**- * Pre-computed variance for ratio/retention metrics. Provide this or baseline_stats when metric_type is ratio/retention and baseline_value is given directly.+ * Pre-computed variance per user. Provide this or baseline_stats when metric_type is ratio/retention and baseline_value is given directly. When given, it replaces the variance the calculator derives for any metric type, including p(1-p) for funnels.* @nullable*/variance?: number | null@@ -5248,6 +5292,25 @@ export type ExperimentsActivityRetrieveParams = {page?: number}+export type ExperimentsRunningTimeTargetRetrieveParams = {+ /**+ * The running time mode to size the target for. Defaults to the experiment's saved mode. 'automatic' sizes it from the results, also for an experiment saved in manual mode.+ *+ * * `automatic` - Sized from the results+ * * `manual` - Saved from a baseline the user entered+ * @minLength 1+ */+ mode?: ExperimentsRunningTimeTargetRetrieveMode+}++export type ExperimentsRunningTimeTargetRetrieveMode =+ (typeof ExperimentsRunningTimeTargetRetrieveMode)[keyof typeof ExperimentsRunningTimeTargetRetrieveMode]++export const ExperimentsRunningTimeTargetRetrieveMode = {+ Automatic: 'automatic',+ Manual: 'manual',+} as const+export type ExperimentsTimeseriesResultsRetrieveParams = {/*** Fingerprint of the metric configuration. Available alongside metric_uuid on each metric in the experiment's metrics array.diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex daec45124e10..1f90f3afe2c0 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -43,6 +43,7 @@ import type {ExperimentsListParams,ExperimentsMatchingIdsRetrieveParams,ExperimentsPromptTemplatesRetrieve200Item,+ ExperimentsRunningTimeTargetRetrieveParams,ExperimentsSessionContextRetrieveParams,ExperimentsTimeseriesResultsRetrieveParams,PaginatedExperimentBasicListApi,@@ -54,6 +55,7 @@ import type {RecalculateMetricsRequestApi,RunningTimeCalculationInputApi,RunningTimeCalculationResultApi,+ RunningTimeTargetApi,ShipVariantApi,} from './api.schemas'@@ -1076,6 +1078,46 @@ export const experimentsResumeCreate = async (})}+export const getExperimentsRunningTimeTargetRetrieveUrl = (+ projectId: string,+ id: number,+ params?: ExperimentsRunningTimeTargetRetrieveParams+) => {+ const normalizedParams = new URLSearchParams()++ Object.entries(params || {}).forEach(([key, value]) => {+ if (value !== undefined) {+ normalizedParams.append(key, value === null ? 'null' : String(value))+ }+ })++ const stringifiedParams = normalizedParams.toString()++ return stringifiedParams.length > 0+ ? `/api/projects/${projectId}/experiments/${id}/running_time_target/?${stringifiedParams}`+ : `/api/projects/${projectId}/experiments/${id}/running_time_target/`+}++/**+ * The target sample of the experiment's running time estimate, which the experiment page's header compares+ * with the exposures so far, and what the experiment can detect over time.+ *+ * In manual running time mode it is the saved target. Otherwise it is sized from the first primary metric's+ * current result: the variance the analysis uses (with CUPED when the analysis applied it) and the flag's+ * traffic split, so that the comparison of each variant with the control reaches the minimum detectable effect.+ */+export const experimentsRunningTimeTargetRetrieve = async (+ projectId: string,+ id: number,+ params?: ExperimentsRunningTimeTargetRetrieveParams,+ options?: RequestInit+): Promise<RunningTimeTargetApi> => {+ return apiMutator<RunningTimeTargetApi>(getExperimentsRunningTimeTargetRetrieveUrl(projectId, id, params), {+ ...options,+ method: 'GET',+ })+}+export const getExperimentsSessionBucketsCreateUrl = (projectId: string, id: number) => {return `/api/projects/${projectId}/experiments/${id}/session_buckets/`}diff --git a/products/experiments/frontend/generated/api.zod.ts b/products/experiments/frontend/generated/api.zod.tsindex d1081d9a0750..df4733648267 100644--- a/products/experiments/frontend/generated/api.zod.ts+++ b/products/experiments/frontend/generated/api.zod.ts@@ -1328,6 +1328,7 @@ export const experimentsCalculateRunningTimeCreateBodyMinimumDetectableEffectMinexport const experimentsCalculateRunningTimeCreateBodyNumberOfVariantsDefault = 2export const experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMin = 2+export const experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMax = 100export const experimentsCalculateRunningTimeCreateBodyExposureRatePerDayMin = 0@@ -1352,8 +1353,9 @@ export const ExperimentsCalculateRunningTimeCreateBody = /* @__PURE__ */ zodnumber_of_variants: zod.number().min(experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMin)+ .max(experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMax).default(experimentsCalculateRunningTimeCreateBodyNumberOfVariantsDefault)- .describe('Total number of variants including control (default 2).'),+ .describe('Total number of variants including control (default 2, at most 100).'),exposure_rate_per_day: zod.number().min(experimentsCalculateRunningTimeCreateBodyExposureRatePerDayMin)@@ -1369,7 +1371,7 @@ export const ExperimentsCalculateRunningTimeCreateBody = /* @__PURE__ */ zod.number().nullish().describe(- 'Pre-computed variance for ratio\/retention metrics. Provide this or baseline_stats when metric_type is ratio\/retention and baseline_value is given directly.'+ 'Pre-computed variance per user. Provide this or baseline_stats when metric_type is ratio\/retention and baseline_value is given directly. When given, it replaces the variance the calculator derives for any metric type, including p(1-p) for funnels.'),baseline_stats: zod.union([@@ -1387,7 +1389,9 @@ export const ExperimentsCalculateRunningTimeCreateBody = /* @__PURE__ */ zodsum_squares: zod.number().default(experimentsCalculateRunningTimeCreateBodyBaselineStatsOneSumSquaresDefault)- .describe('Sum of squared metric values. Required for ratio\/retention variance.'),+ .describe(+ "Sum of squared metric values. Mean metrics read their variance from it, and ratio\/retention metrics need it. Without it, a mean metric's variance is approximated from the baseline value."+ ),denominator_sum: zod.number().nullish()diff --git a/products/experiments/skills/diagnosing-experiment-health/references/interpretation.md b/products/experiments/skills/diagnosing-experiment-health/references/interpretation.mdindex c217091901d6..a8674f3f5212 100644--- a/products/experiments/skills/diagnosing-experiment-health/references/interpretation.md+++ b/products/experiments/skills/diagnosing-experiment-health/references/interpretation.md@@ -343,16 +343,19 @@ Older versions read "Complete".On an ended experiment "Complete" is the status tag: every ended experiment carries it, and the duration display is hidden.Read `status` first when the user says "complete".-- The target is the sample needed to detect the **minimum detectable effect** with about 80% power at 95% confidence.-- The estimate uses the **first primary metric** only.+- The target is the sample needed to detect the **minimum detectable effect** with about 80% power, at the confidence level of the experiment's analysis (95% by default).+- The estimate uses the **first primary metric** only, with the variance that the analysis computes from its results (after CUPED, when the experiment uses it).+- The target is sized so that the comparison of the control with the smallest variant can detect the effect, so an uneven split needs a larger total than an even one.- The minimum detectable effect is the project's default, and 30% where none is set.- A 30% effect needs a small sample, so the target is reached within days on a busy surface.+ A 30% effect on a conversion rate needs a small sample, so the target is reached within days on a busy surface.+ A summed value such as revenue varies much more from person to person, so its target can be many times larger.+- In manual mode the target comes from the baseline the user typed, not from the results.- "Target reached" says that an effect of that size would probably have shown.It says nothing about significance, and it does not end the experiment.<!-- Source for maintainers (may rot): ExperimentRemainingTime.tsx inproducts/experiments/frontend/components/ExperimentMetaBar; runningTimeLogic.ts in-frontend/src/scenes/experiments/RunningTimeCalculator. -->+frontend/src/scenes/experiments/RunningTimeCalculator; running_time_target.py in products/experiments/backend. -->**Evidence to gather.** `running_time_calculation` in `experiment-get`:diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 8d5cfdda469f..ad504371e7c6 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -37783,6 +37783,13 @@ export namespace Schemas {steps: DestinationConnectionTestStep[];}+ export interface DetectableEffectCheckpoint {+ /** The week mark, in whole weeks since launch. */+ weeks_since_launch: number;+ /** The smallest relative change, as a percentage, that the exposures by this week mark can detect. */+ detectable_effect: number;+ }+export interface DevUserUsage {/*** PostHog user id. Null when no user has this distinct_id.@@ -105526,7 +105533,7 @@ export namespace Schemas {number_of_samples: number;/** Sum of the metric values across the control group (for funnels, the numerator/conversions). */sum: number;- /** Sum of squared metric values. Required for ratio/retention variance. */+ /** Sum of squared metric values. Mean metrics read their variance from it, and ratio/retention metrics need it. Without it, a mean metric's variance is approximated from the baseline value. */sum_squares?: number;/*** Sum of the denominator values. Required for ratio/retention metrics.@@ -105565,8 +105572,9 @@ export namespace Schemas {*/minimum_detectable_effect: number;/**- * Total number of variants including control (default 2).+ * Total number of variants including control (default 2, at most 100).* @minimum 2+ * @maximum 100*/number_of_variants?: number;/**@@ -105581,7 +105589,7 @@ export namespace Schemas {*/baseline_value?: number | null;/**- * Pre-computed variance for ratio/retention metrics. Provide this or baseline_stats when metric_type is ratio/retention and baseline_value is given directly.+ * Pre-computed variance per user. Provide this or baseline_stats when metric_type is ratio/retention and baseline_value is given directly. When given, it replaces the variance the calculator derives for any metric type, including p(1-p) for funnels.* @nullable*/variance?: number | null;@@ -105615,6 +105623,42 @@ export namespace Schemas {recommended_running_time_days: number | null;}+ /**+ * What the experiment can detect over time, at its exposure rate so far.+ */+ export interface RunningTimeOutlook {+ /** Days since the experiment launched, with fractions. */+ days_since_launch: number;+ /** Exposures that count toward the target, per day from launch to the end of the window the current results cover. */+ exposures_per_day: number;+ /** The smallest relative change, as a percentage, that the exposures so far can detect. */+ detectable_effect_now: number;+ /** Up to three week marks after today, out of weeks 1, 2, 4, 6, 8, 12, 16, 26 and 52, with the change each can detect at the exposure rate so far. */+ upcoming_checkpoints: DetectableEffectCheckpoint[];+ /** The smallest relative change, as a percentage, that the experiment can detect within four weeks from today. A target for a smaller change is out of reach. */+ reachable_effect: number;+ }++ /**+ * The target sample of the experiment's running time estimate.+ */+ export interface RunningTimeTarget {+ /** The relative change, as a percentage, that the target sample is sized to detect. */+ minimum_detectable_effect: number;+ /**+ * Exposures needed across the analyzed variants, so that the comparison of each variant with the control can detect the minimum detectable effect at the analysis's confidence level and 80% power. In manual running time mode, the saved target. Otherwise sized from the first primary metric's current result, with the variance the analysis uses and the flag's traffic split. Null when no result can size it yet. The variances come from the result and don't depend on the effect, so the sample scales with 1 / effect²: the target for another effect is recommended_sample_size × (minimum_detectable_effect / effect)².+ * @nullable+ */+ recommended_sample_size: number | null;+ /**+ * Exposures so far that count toward the target: the control's and those of the variants that get traffic. A variant at 0% does not count. Null in manual mode, and when no result can size the target.+ * @nullable+ */+ current_sample_size: number | null;+ /** What the experiment can detect over time, on the same curve the target is sized on. Null in manual mode, before launch, when the current results cover less than 0.1 days, and when no result can size the target. */+ outlook: RunningTimeOutlook | null;+ }+/*** * `S3Compatible` - S3Compatible*/@@ -133247,6 +133291,25 @@ export namespace Schemas {page?: number;};+ export type ExperimentsRunningTimeTargetRetrieveParams = {+ /**+ * The running time mode to size the target for. Defaults to the experiment's saved mode. 'automatic' sizes it from the results, also for an experiment saved in manual mode.+ *+ * * `automatic` - Sized from the results+ * * `manual` - Saved from a baseline the user entered+ * @minLength 1+ */+ mode?: ExperimentsRunningTimeTargetRetrieveMode;+ };++ export type ExperimentsRunningTimeTargetRetrieveMode = typeof ExperimentsRunningTimeTargetRetrieveMode[keyof typeof ExperimentsRunningTimeTargetRetrieveMode];+++ export const ExperimentsRunningTimeTargetRetrieveMode = {+ Automatic: 'automatic',+ Manual: 'manual',+ } as const;+export type ExperimentsTimeseriesResultsRetrieveParams = {/*** Fingerprint of the metric configuration. Available alongside metric_uuid on each metric in the experiment's metrics array.diff --git a/services/mcp/src/generated/experiments/api.ts b/services/mcp/src/generated/experiments/api.tsindex 665b5dfa4a4e..131c5975e5d0 100644--- a/services/mcp/src/generated/experiments/api.ts+++ b/services/mcp/src/generated/experiments/api.ts@@ -19953,6 +19953,7 @@ export const experimentsCalculateRunningTimeCreateBodyMinimumDetectableEffectMinexport const experimentsCalculateRunningTimeCreateBodyNumberOfVariantsDefault = 2export const experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMin = 2+export const experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMax = 100export const experimentsCalculateRunningTimeCreateBodyExposureRatePerDayMin = 0@@ -19977,8 +19978,9 @@ export const ExperimentsCalculateRunningTimeCreateBody = () => zodnumber_of_variants: zod.number().min(experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMin)+ .max(experimentsCalculateRunningTimeCreateBodyNumberOfVariantsMax).default(experimentsCalculateRunningTimeCreateBodyNumberOfVariantsDefault)- .describe('Total number of variants including control (default 2).'),+ .describe('Total number of variants including control (default 2, at most 100).'),exposure_rate_per_day: zod.number().min(experimentsCalculateRunningTimeCreateBodyExposureRatePerDayMin)@@ -19994,7 +19996,7 @@ export const ExperimentsCalculateRunningTimeCreateBody = () => zod.number().nullish().describe(- 'Pre-computed variance for ratio\/retention metrics. Provide this or baseline_stats when metric_type is ratio\/retention and baseline_value is given directly.'+ 'Pre-computed variance per user. Provide this or baseline_stats when metric_type is ratio\/retention and baseline_value is given directly. When given, it replaces the variance the calculator derives for any metric type, including p(1-p) for funnels.'),baseline_stats: zod.union([@@ -20012,7 +20014,9 @@ export const ExperimentsCalculateRunningTimeCreateBody = () => zodsum_squares: zod.number().default(experimentsCalculateRunningTimeCreateBodyBaselineStatsOneSumSquaresDefault)- .describe('Sum of squared metric values. Required for ratio\/retention variance.'),+ .describe(+ "Sum of squared metric values. Mean metrics read their variance from it, and ratio\/retention metrics need it. Without it, a mean metric's variance is approximated from the baseline value."+ ),denominator_sum: zod.number().nullish()diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/experiment-calculate-running-time.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/experiment-calculate-running-time.jsonindex fa5a2ce6867e..a611a1d095b9 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/experiment-calculate-running-time.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/experiment-calculate-running-time.json@@ -58,7 +58,7 @@},"sum_squares": {"default": 0,- "description": "Sum of squared metric values. Required for ratio/retention variance.",+ "description": "Sum of squared metric values. Mean metrics read their variance from it, and ratio/retention metrics need it. Without it, a mean metric's variance is approximated from the baseline value.","type": "number"}},@@ -106,7 +106,8 @@},"number_of_variants": {"default": 2,- "description": "Total number of variants including control (default 2).",+ "description": "Total number of variants including control (default 2, at most 100).",+ "maximum": 100,"minimum": 2,"type": "number"},@@ -119,7 +120,7 @@"type": "null"}],- "description": "Pre-computed variance for ratio/retention metrics. Provide this or baseline_stats when metric_type is ratio/retention and baseline_value is given directly."+ "description": "Pre-computed variance per user. Provide this or baseline_stats when metric_type is ratio/retention and baseline_value is given directly. When given, it replaces the variance the calculator derives for any metric type, including p(1-p) for funnels."}},"required": ["metric_type", "minimum_detectable_effect"],