Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.4 KB
Raw Blame
import type { ExperimentFunnelsQuery, ExperimentMetric, ExperimentTrendsQuery } from '~/queries/schema/schema-general'
import { ExperimentMetricType, NodeKind } from '~/queries/schema/schema-general'
import { ExperimentMetricGoal, ExperimentMetricMathType, FunnelConversionWindowTimeUnit } from '~/types'

import {
    type ExperimentVariantResult,
    formatChanceToWinForGoal,
    formatMetricValue,
    formatPValue,
    formatTickValue,
    getChanceToWin,
    getDefaultMetricTitle,
    getMetricColors,
    getMetricTag,
    isProportionMetric,
    isWinning,
} from './utils'

describe('getMetricTag', () => {
    it('handles different metric types correctly', () => {
        const experimentMetric: ExperimentMetric = {
            kind: NodeKind.ExperimentMetric,
            metric_type: ExperimentMetricType.MEAN,
            source: {
                kind: NodeKind.EventsNode,
                event: 'purchase',
                math: ExperimentMetricMathType.TotalCount,
            },
        }

        const funnelMetric: ExperimentFunnelsQuery = {
            kind: NodeKind.ExperimentFunnelsQuery,
            funnels_query: {
                kind: NodeKind.FunnelsQuery,
                series: [],
            },
        }

        const trendMetric: ExperimentTrendsQuery = {
            kind: NodeKind.ExperimentTrendsQuery,
            count_query: {
                kind: NodeKind.TrendsQuery,
                series: [],
            },
        }

        expect(getMetricTag(experimentMetric)).toBe('Mean')
        expect(getMetricTag(funnelMetric)).toBe('Funnel')
        expect(getMetricTag(trendMetric)).toBe('Trend')
    })
})

describe('getDefaultMetricTitle', () => {
    it('handles ExperimentEventMetricConfig with math and math_property', () => {
        const metric: ExperimentMetric = {
            kind: NodeKind.ExperimentMetric,
            metric_type: ExperimentMetricType.MEAN,
            source: {
                kind: NodeKind.EventsNode,
                event: 'purchase completed',
            },
        }
        expect(getDefaultMetricTitle(metric)).toBe('purchase completed')
    })

    it('returns action name for ExperimentActionMetricConfig', () => {
        const metric: ExperimentMetric = {
            kind: NodeKind.ExperimentMetric,
            metric_type: ExperimentMetricType.MEAN,
            source: {
                kind: NodeKind.ActionsNode,
                id: 1,
                name: 'purchase',
            },
        }

        expect(getDefaultMetricTitle(metric)).toBe('purchase')
    })

    it('returns table name for ExperimentDataWarehouseMetricConfig', () => {
        const metric: ExperimentMetric = {
            kind: NodeKind.ExperimentMetric,
            metric_type: ExperimentMetricType.MEAN,
            source: {
                kind: NodeKind.ExperimentDataWarehouseNode,
                table_name: 'purchase_events',
                timestamp_field: 'timestamp',
                events_join_key: 'person_id',
                data_warehouse_join_key: 'person_id',
            },
        }
        expect(getDefaultMetricTitle(metric)).toBe('purchase_events')
    })
})

describe('isWinning', () => {
    const createResult = (credible_interval: [number, number] | undefined): ExperimentVariantResult => ({
        key: 'test',
        sum: 100,
        number_of_samples: 100,
        sum_squares: 100,
        significant: true,
        method: 'bayesian',
        credible_interval,
        chance_to_win: 0.75,
    })

    it('returns true for positive delta with increase goal', () => {
        const result = createResult([0.05, 0.15])
        expect(isWinning(result, ExperimentMetricGoal.Increase)).toBe(true)
    })

    it('returns false for positive delta with decrease goal', () => {
        const result = createResult([0.05, 0.15])
        expect(isWinning(result, ExperimentMetricGoal.Decrease)).toBe(false)
    })

    it('returns false for negative delta with increase goal', () => {
        const result = createResult([-0.15, -0.05])
        expect(isWinning(result, ExperimentMetricGoal.Increase)).toBe(false)
    })

    it('returns true for negative delta with decrease goal', () => {
        const result = createResult([-0.15, -0.05])
        expect(isWinning(result, ExperimentMetricGoal.Decrease)).toBe(true)
    })

    it('returns undefined when no interval is present', () => {
        const result = createResult(undefined)
        expect(isWinning(result, ExperimentMetricGoal.Increase)).toBeUndefined()
        expect(isWinning(result, ExperimentMetricGoal.Decrease)).toBeUndefined()
    })
})

describe('getChanceToWin', () => {
    const createBayesianResult = (chance_to_win: number | undefined): ExperimentVariantResult => ({
        key: 'test',
        sum: 100,
        number_of_samples: 100,
        sum_squares: 100,
        significant: true,
        method: 'bayesian',
        credible_interval: [0.05, 0.15],
        chance_to_win,
    })

    const createFrequentistResult = (): ExperimentVariantResult => ({
        key: 'test',
        sum: 100,
        number_of_samples: 100,
        sum_squares: 100,
        significant: true,
        method: 'frequentist',
        confidence_interval: [0.05, 0.15],
        p_value: 0.03,
    })

    it('returns chance to win as-is for increase goal', () => {
        const result = createBayesianResult(0.75)
        expect(getChanceToWin(result, ExperimentMetricGoal.Increase)).toBe(0.75)
    })

    it('inverts chance to win for decrease goal', () => {
        const result = createBayesianResult(0.75)
        expect(getChanceToWin(result, ExperimentMetricGoal.Decrease)).toBe(0.25)
    })

    it('handles null chance to win', () => {
        const result = createBayesianResult(undefined)
        expect(getChanceToWin(result, ExperimentMetricGoal.Increase)).toBeUndefined()
        expect(getChanceToWin(result, ExperimentMetricGoal.Decrease)).toBeUndefined()
    })

    it('returns undefined for frequentist results', () => {
        const result = createFrequentistResult()
        expect(getChanceToWin(result, ExperimentMetricGoal.Increase)).toBeUndefined()
        expect(getChanceToWin(result, ExperimentMetricGoal.Decrease)).toBeUndefined()
    })
})

describe('formatChanceToWinForGoal', () => {
    const createResult = (chance_to_win: number): ExperimentVariantResult => ({
        key: 'test',
        sum: 100,
        number_of_samples: 100,
        sum_squares: 100,
        significant: true,
        method: 'bayesian',
        credible_interval: [0.05, 0.15],
        chance_to_win,
    })

    it('formats chance to win for increase goal', () => {
        const result = createResult(0.756)
        expect(formatChanceToWinForGoal(result, ExperimentMetricGoal.Increase)).toBe('75.6%')
    })

    it('formats inverted chance to win for decrease goal', () => {
        const result = createResult(0.756)
        expect(formatChanceToWinForGoal(result, ExperimentMetricGoal.Decrease)).toBe('24.4%')
    })

    it('handles very high chances', () => {
        const result = createResult(0.9995)
        expect(formatChanceToWinForGoal(result, ExperimentMetricGoal.Increase)).toBe('> 99.9%')
        expect(formatChanceToWinForGoal(result, ExperimentMetricGoal.Decrease)).toBe('< 0.1%')
    })

    it('handles very low chances', () => {
        const result = createResult(0.0005)
        expect(formatChanceToWinForGoal(result, ExperimentMetricGoal.Increase)).toBe('< 0.1%')
        expect(formatChanceToWinForGoal(result, ExperimentMetricGoal.Decrease)).toBe('> 99.9%')
    })
})

describe('getMetricColors', () => {
    const colors = {
        BAR_POSITIVE: '#00FF00',
        BAR_NEGATIVE: '#FF0000',
    }

    it('returns normal colors for increase goal', () => {
        const result = getMetricColors(colors, ExperimentMetricGoal.Increase)
        expect(result.positive).toBe('#00FF00')
        expect(result.negative).toBe('#FF0000')
    })

    it('swaps colors for decrease goal', () => {
        const result = getMetricColors(colors, ExperimentMetricGoal.Decrease)
        expect(result.positive).toBe('#FF0000')
        expect(result.negative).toBe('#00FF00')
    })
})

describe('formatPValue', () => {
    it.each([
        [null, '—'],
        [undefined, '—'],
    ])('returns "—" for %p', (input, expected) => {
        expect(formatPValue(input)).toBe(expected)
    })

    it.each([
        [0, '< 0.001'],
        [0.0001, '< 0.001'],
        [0.00001, '< 0.001'],
        [4.196532010780629e-11, '< 0.001'],
        [1e-100, '< 0.001'],
        [Number.MIN_VALUE, '< 0.001'],
    ])('returns "< 0.001" for very small p-value %p', (input, expected) => {
        expect(formatPValue(input)).toBe(expected)
    })

    it.each([
        [0.001, '0.0010'],
        [0.005, '0.0050'],
        [0.009, '0.0090'],
        [0.00999, '0.0100'],
    ])('returns 4 decimal places for p-value %p between 0.001 and 0.01', (input, expected) => {
        expect(formatPValue(input)).toBe(expected)
    })

    it.each([
        [0.01, '0.010'],
        [0.05, '0.050'],
        [0.1, '0.100'],
        [0.5, '0.500'],
        [0.999, '0.999'],
        [1, '1.000'],
    ])('returns 3 decimal places for p-value %p >= 0.01', (input, expected) => {
        expect(formatPValue(input)).toBe(expected)
    })
})

describe('formatTickValue', () => {
    it.each([
        [0, '0%'],
        [0.5, '50%'],
        [-0.15, '-15%'],
        [0.055, '5.50%'],
        [0.005, '0.500%'],
        [1.5, '150%'],
    ])('formatTickValue(%p) === %p', (input, expected) => {
        expect(formatTickValue(input)).toBe(expected)
    })
})

const meanMetric = (threshold?: number): ExperimentMetric => ({
    kind: NodeKind.ExperimentMetric,
    metric_type: ExperimentMetricType.MEAN,
    source: { kind: NodeKind.EventsNode, event: 'purchase', math: ExperimentMetricMathType.Sum },
    ...(threshold != null ? { threshold } : {}),
})

const funnelMetric: ExperimentMetric = {
    kind: NodeKind.ExperimentMetric,
    metric_type: ExperimentMetricType.FUNNEL,
    series: [{ kind: NodeKind.EventsNode, event: 'signup' }],
}

const retentionMetric: ExperimentMetric = {
    kind: NodeKind.ExperimentMetric,
    metric_type: ExperimentMetricType.RETENTION,
    start_event: { kind: NodeKind.EventsNode, event: 'signup' },
    completion_event: { kind: NodeKind.EventsNode, event: 'return' },
    retention_window_start: 0,
    retention_window_end: 7,
    retention_window_unit: FunnelConversionWindowTimeUnit.Day,
    start_handling: 'first_seen',
}

const ratioMetric: ExperimentMetric = {
    kind: NodeKind.ExperimentMetric,
    metric_type: ExperimentMetricType.RATIO,
    numerator: { kind: NodeKind.EventsNode, event: 'revenue' },
    denominator: { kind: NodeKind.EventsNode, event: 'sessions' },
}

describe('isProportionMetric', () => {
    const cases: { label: string; metric: ExperimentMetric; expected: boolean }[] = [
        { label: 'threshold mean', metric: meanMetric(5), expected: true },
        { label: 'plain mean', metric: meanMetric(), expected: false },
        { label: 'funnel', metric: funnelMetric, expected: true },
        { label: 'retention', metric: retentionMetric, expected: true },
        { label: 'ratio', metric: ratioMetric, expected: false },
    ]
    it.each(cases)('classifies $label metric', ({ metric, expected }) => {
        expect(isProportionMetric(metric)).toBe(expected)
    })
})

describe('formatMetricValue', () => {
    it('formats a plain mean metric as a raw number', () => {
        expect(formatMetricValue({ sum: 4, number_of_samples: 100 }, meanMetric())).toBe('0.04')
    })

    it('formats a threshold mean metric as a percentage', () => {
        expect(formatMetricValue({ sum: 40, number_of_samples: 100 }, meanMetric(5))).toBe('40.00%')
    })

    it('formats a funnel metric as a percentage', () => {
        expect(formatMetricValue({ sum: 25, number_of_samples: 100 }, funnelMetric)).toBe('25.00%')
    })

    it('formats a retention metric as a percentage', () => {
        expect(formatMetricValue({ sum: 60, number_of_samples: 100 }, retentionMetric)).toBe('60.00%')
    })

    it('formats a ratio metric as a quotient', () => {
        expect(formatMetricValue({ sum: 50, denominator_sum: 200 }, ratioMetric)).toBe('0.25')
    })

    it('returns "—" for a ratio metric with a zero denominator', () => {
        expect(formatMetricValue({ sum: 3, denominator_sum: 0 }, ratioMetric)).toBe('—')
    })

    it('returns "—" when the value is not a number', () => {
        expect(formatMetricValue({ sum: 0, number_of_samples: 0 }, meanMetric())).toBe('—')
    })
})