Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

27.9 KB
Raw Blame
import { MakeLogicType, actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'
import { loaders } from 'kea-loaders'

import api, { ApiError } from 'lib/api'
import { dayjs, Dayjs } from 'lib/dayjs'
import { ConcurrencyController } from 'lib/utils/concurrencyController'
import { projectLogic } from 'scenes/projectLogic'

import { PersonType } from '~/types'

import { featureFlagsTestEvaluationCreate } from 'products/feature_flags/frontend/generated/api'
import type {
    FeatureFlagConditionAnalysisApi,
    FeatureFlagTestEvaluationRequestApi,
    FeatureFlagTestEvaluationResponseApi,
} from 'products/feature_flags/frontend/generated/api.schemas'

import type { FeatureFlagConditionPropertyAnalysisApi } from '../../../../products/feature_flags/frontend/generated/api.schemas'

export type ConditionAnalysis = FeatureFlagConditionAnalysisApi
export type TestResult = FeatureFlagTestEvaluationResponseApi

export interface TestFormData {
    distinct_id: string
    timestamp: string
    groups: string
}

// One row of a batch evaluation — how the flag bucketed for a single one of the
// person's merged distinct IDs. `error` is set (and `result` null) when that ID's
// individual evaluation failed, so one bad ID doesn't blank the whole table.
export interface PerDistinctIdEvaluation {
    distinctId: string
    result: TestResult | null
    error: string | null
}

const EMPTY_FORM: TestFormData = { distinct_id: '', timestamp: '', groups: '' }

// Caps how many merged distinct IDs are evaluated concurrently. test_evaluation is
// ClickHouse-backed and throttle-gated, so a person with many merged IDs shouldn't
// turn one button click into an unbounded burst of concurrent requests.
const TEST_EVALUATION_CONCURRENCY_LIMIT = 6
const testEvaluationConcurrencyController = new ConcurrencyController(TEST_EVALUATION_CONCURRENCY_LIMIT)

// Build the request body for a single evaluation. Shared by the single-ID and
// batch paths so groups/timestamp are parsed and validated identically. Throws on
// invalid groups or timestamp. submitTestEvaluation checks the form with
// testFormValidationError first, so this throw does not reach the loaders.
function buildEvaluationRequest(formData: TestFormData, distinctId: string): FeatureFlagTestEvaluationRequestApi {
    const data: FeatureFlagTestEvaluationRequestApi = {}

    const trimmedDistinctId = distinctId.trim()
    if (trimmedDistinctId) {
        data.distinct_id = trimmedDistinctId
    }

    data.groups = validateAndParseGroups(formData.groups || '')

    if (formData.timestamp?.trim()) {
        data.timestamp = formData.timestamp.trim()

        // Validate ISO string format with strict parsing
        const parsedTimestamp = dayjs(data.timestamp, 'YYYY-MM-DDTHH:mm:ss.SSS[Z]', true)
        if (!parsedTimestamp.isValid()) {
            throw new Error('Invalid timestamp format')
        }
    }

    return data
}

// A loader that throws an error without an HTTP status is reported to error tracking,
// so invalid user input must stop before a loader runs.
function testFormValidationError(formData: TestFormData): string | null {
    try {
        buildEvaluationRequest(formData, '')
        return null
    } catch (e) {
        return e instanceof Error ? e.message : String(e)
    }
}

// Map an evaluation failure to the user-facing message. Shared by the single-ID
// and batch failure handlers so both surface the same friendly rewrites.
function evaluationErrorMessage(error: string, errorObject?: unknown): string {
    const apiError = errorObject as ApiError | undefined
    // ApiError.message already holds the body's `error` field, which test_evaluation uses for its own failures.
    const message = apiError?.detail || apiError?.message || error || ''
    const lowerMessage = message.toLowerCase()

    if (lowerMessage.includes('failed to build person properties at specified timestamp')) {
        return 'Unable to build person properties at the selected timestamp. This person may not have had any recorded activity at that time, or the timestamp may be too far in the past.'
    }

    if (lowerMessage.includes('person') && lowerMessage.includes('not found')) {
        return 'Person not found. This person may not have existed at the selected timestamp.'
    }

    if (lowerMessage.includes('invalid timestamp') || apiError?.attr === 'timestamp') {
        return 'Invalid timestamp. Please select a valid date and time.'
    }

    return message || 'An unexpected error occurred while testing the feature flag'
}

function validateAndParseGroups(groups: string): Record<string, any> {
    const trimmed = groups.trim()
    if (!trimmed) {
        return {}
    }

    try {
        const parsed = JSON.parse(trimmed)
        if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
            throw new Error('groups must be a JSON object')
        }
        return parsed
    } catch (e) {
        throw e instanceof SyntaxError ? new Error('Invalid JSON format for groups') : e
    }
}

export interface FeatureFlagTestingLogicProps {
    flagId: number
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface featureFlagTestingLogicValues {
    currentProjectId: number | null // projectLogic
    activeResult: TestResult | null
    allEvaluations: PerDistinctIdEvaluation[] | null
    allEvaluationsLoading: boolean
    bucketingDistinctId: string | null
    datePickerOpen: boolean
    datePickerValue: Dayjs | null
    enrichedConditions: {
        display: {
            label: string | null
            tone: 'info' | 'muted' | 'success' | 'warning'
        }
        explanation: string
        index: number
        isWinningCondition: boolean
        matched: boolean
        matchedButNotWinner: boolean | undefined
        properties: FeatureFlagConditionPropertyAnalysisApi[]
        properties_matched?: boolean | undefined
        rollout_excluded: boolean
        rollout_percentage: number
        variant: string | null
    }[]
    errorDisplay: {
        helpText: string | null
        message: string
    } | null
    hasMultipleDistinctIds: boolean
    hasValidPerson: boolean
    includeTime: boolean
    personDistinctIds: string[]
    resolvedPersonDistinctIds: string[] | null
    resolvedPersonDistinctIdsLoading: boolean
    resultsDiverge: boolean
    selectedPerson: Partial<PersonType> | null
    selectedResultDistinctId: string | null
    testError: string | null
    testEvaluation: TestResult | null
    testEvaluationLoading: boolean
    testFormData: TestFormData
    testResult: TestResult | null
    usedProperties: Set<string>
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface featureFlagTestingLogicActions {
    clearBatchEvaluations: () => {
        value: true
    }
    clearBatchEvaluationsFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    clearBatchEvaluationsSuccess: (
        allEvaluations: null,
        payload?: {
            value: true
        }
    ) => {
        allEvaluations: null
        payload?: {
            value: true
        }
    }
    clearTestForm: () => {
        value: true
    }
    clearTestFormFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    clearTestFormSuccess: (
        resolvedPersonDistinctIds: null,
        payload?: {
            value: true
        }
    ) => {
        resolvedPersonDistinctIds: null
        payload?: {
            value: true
        }
    }
    resolvePersonDistinctIds: (distinctId: string) => string
    resolvePersonDistinctIdsFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    resolvePersonDistinctIdsSuccess: (
        resolvedPersonDistinctIds: string[],
        payload?: string
    ) => {
        resolvedPersonDistinctIds: string[]
        payload?: string
    }
    setDatePickerOpen: (open: boolean) => {
        open: boolean
    }
    setDatePickerValue: (value: Dayjs | null) => {
        value: Dayjs | null
    }
    setIncludeTime: (includeTime: boolean) => {
        includeTime: boolean
    }
    setSelectedPerson: (
        person: Partial<PersonType> | null,
        distinctId?: string
    ) => {
        distinctId: string | undefined
        person: Partial<PersonType> | null
    }
    setSelectedPersonFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    setSelectedPersonSuccess: (
        resolvedPersonDistinctIds: null,
        payload?: {
            distinctId: string | undefined
            person: Partial<PersonType> | null
        }
    ) => {
        resolvedPersonDistinctIds: null
        payload?: {
            distinctId: string | undefined
            person: Partial<PersonType> | null
        }
    }
    setSelectedResultDistinctId: (distinctId: string | null) => {
        distinctId: string | null
    }
    setTestError: (error: string | null) => {
        error: string | null
    }
    setTestFormData: (formData: Partial<TestFormData>) => {
        formData: Partial<TestFormData>
    }
    submitTestEvaluation: () => {
        value: true
    }
    testAllDistinctIds: ({
        flagId,
        distinctIds,
        formData,
    }: {
        distinctIds: string[]
        flagId: number
        formData: TestFormData
    }) => {
        flagId: number
        distinctIds: string[]
        formData: TestFormData
    }
    testAllDistinctIdsFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    testAllDistinctIdsSuccess: (
        allEvaluations: PerDistinctIdEvaluation[],
        payload?: {
            flagId: number
            distinctIds: string[]
            formData: TestFormData
        }
    ) => {
        allEvaluations: PerDistinctIdEvaluation[]
        payload?: {
            flagId: number
            distinctIds: string[]
            formData: TestFormData
        }
    }
    testFlagEvaluation: ({ flagId, formData }: { flagId: number; formData: TestFormData }) => {
        flagId: number
        formData: TestFormData
    }
    testFlagEvaluationFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    testFlagEvaluationSuccess: (
        testEvaluation: FeatureFlagTestEvaluationResponseApi,
        payload?: {
            flagId: number
            formData: TestFormData
        }
    ) => {
        testEvaluation: FeatureFlagTestEvaluationResponseApi
        payload?: {
            flagId: number
            formData: TestFormData
        }
    }
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface featureFlagTestingLogicMeta {
    key: number
    __keaTypeGenInternalSelectorTypes: {
        activeResult: (
            testResult: FeatureFlagTestEvaluationResponseApi | null,
            allEvaluations: PerDistinctIdEvaluation[] | null,
            selectedResultDistinctId: string | null
        ) => TestResult | null
        resultsDiverge: (allEvaluations: PerDistinctIdEvaluation[] | null) => boolean
        usedProperties: (activeResult: FeatureFlagTestEvaluationResponseApi | null) => Set<string>
        enrichedConditions: (activeResult: FeatureFlagTestEvaluationResponseApi | null) => {
            display: {
                label: string | null
                tone: 'info' | 'muted' | 'success' | 'warning'
            }
            explanation: string
            index: number
            isWinningCondition: boolean
            matched: boolean
            matchedButNotWinner: boolean | undefined
            properties: FeatureFlagConditionPropertyAnalysisApi[]
            properties_matched?: boolean | undefined
            rollout_excluded: boolean
            rollout_percentage: number
            variant: string | null
        }[]
        hasValidPerson: (testFormData: TestFormData) => boolean
        personDistinctIds: (
            selectedPerson: Partial<PersonType> | null,
            resolvedPersonDistinctIds: string[] | null
        ) => string[]
        hasMultipleDistinctIds: (personDistinctIds: string[]) => boolean
        bucketingDistinctId: (activeResult: FeatureFlagTestEvaluationResponseApi | null) => string | null
        errorDisplay: (testError: string | null) => {
            helpText: string | null
            message: string
        } | null
    }
}

export type featureFlagTestingLogicType = MakeLogicType<
    featureFlagTestingLogicValues,
    featureFlagTestingLogicActions,
    FeatureFlagTestingLogicProps,
    featureFlagTestingLogicMeta
>

export const featureFlagTestingLogic = kea<featureFlagTestingLogicType>([
    props({} as FeatureFlagTestingLogicProps),
    key((props) => props.flagId),
    path((key) => ['scenes', 'feature-flags', 'featureFlagTestingLogic', key]),
    connect(() => ({
        values: [projectLogic, ['currentProjectId']],
    })),
    actions({
        setTestFormData: (formData: Partial<TestFormData>) => ({ formData }),
        setTestError: (error: string | null) => ({ error }),
        setDatePickerOpen: (open: boolean) => ({ open }),
        setDatePickerValue: (value: Dayjs | null) => ({ value }),
        setSelectedPerson: (person: Partial<PersonType> | null, distinctId?: string) => ({ person, distinctId }),
        setIncludeTime: (includeTime: boolean) => ({ includeTime }),
        setSelectedResultDistinctId: (distinctId: string | null) => ({ distinctId }),
        clearBatchEvaluations: true,
        clearTestForm: true,
        submitTestEvaluation: true,
    }),
    loaders(({ values }) => ({
        resolvedPersonDistinctIds: [
            null as string[] | null,
            {
                resolvePersonDistinctIds: async (distinctId: string) => {
                    const response = await api.persons.list({ distinct_id: distinctId })
                    return response.results[0]?.distinct_ids ?? []
                },
                setSelectedPerson: () => null,
                clearTestForm: () => null,
            },
        ],
        testEvaluation: [
            null as TestResult | null,
            {
                testFlagEvaluation: async ({ flagId, formData }: { flagId: number; formData: TestFormData }) => {
                    const data = buildEvaluationRequest(formData, formData.distinct_id || '')
                    return await featureFlagsTestEvaluationCreate(String(values.currentProjectId), flagId, data)
                },
            },
        ],
        // Batch counterpart to testEvaluation: evaluate every one of a person's merged
        // distinct IDs in one action so their variants can be shown side by side, instead
        // of forcing the user to re-run the single-ID path once per ID.
        allEvaluations: [
            null as PerDistinctIdEvaluation[] | null,
            {
                testAllDistinctIds: async ({
                    flagId,
                    distinctIds,
                    formData,
                }: {
                    flagId: number
                    distinctIds: string[]
                    formData: TestFormData
                }) => {
                    // Parse groups/timestamp once up front, not once per distinct ID.
                    const baseData = buildEvaluationRequest(formData, '')

                    return await Promise.all(
                        distinctIds.map(async (distinctId): Promise<PerDistinctIdEvaluation> => {
                            try {
                                const result = await testEvaluationConcurrencyController.run({
                                    debugTag: `feature-flag-test-evaluation-${distinctId}`,
                                    fn: () =>
                                        featureFlagsTestEvaluationCreate(String(values.currentProjectId), flagId, {
                                            ...baseData,
                                            distinct_id: distinctId,
                                        }),
                                })
                                return { distinctId, result, error: null }
                            } catch (e) {
                                return { distinctId, result: null, error: evaluationErrorMessage('', e) }
                            }
                        })
                    )
                },
                // Reset via a dedicated action rather than reusing other actions as keys.
                // Every key in a loader map registers a listener that dispatches
                // `${key}Success`, so listing an action another loader already owns
                // (setSelectedPerson/clearTestForm) or another loader's trigger
                // (testFlagEvaluation) would double-dispatch its success action and corrupt
                // that loader's value. A unique action keeps the reset isolated; listeners
                // dispatch it on person/form changes.
                clearBatchEvaluations: () => null,
            },
        ],
    })),
    reducers({
        testFormData: [
            EMPTY_FORM,
            {
                setTestFormData: (state, { formData }) => ({ ...state, ...formData }),
                clearTestForm: () => EMPTY_FORM,
            },
        ],
        testError: [
            null as string | null,
            {
                setTestError: (_, { error }: { error: string | null }) => error,
                testFlagEvaluation: () => null,
                testAllDistinctIds: () => null,
                clearTestForm: () => null,
                testFlagEvaluationFailure: (_, { error, errorObject }: { error: string; errorObject?: unknown }) =>
                    evaluationErrorMessage(error, errorObject),
                // The loader catches per-ID API errors and shows them in the table.
                testAllDistinctIdsFailure: (_, { error, errorObject }: { error: string; errorObject?: unknown }) =>
                    evaluationErrorMessage(error, errorObject),
            },
        ],
        testResult: [
            null as TestResult | null,
            {
                testFlagEvaluationSuccess: (_, { testEvaluation }) => testEvaluation,
                testAllDistinctIds: () => null,
                clearTestForm: () => null,
                setTestFormData: () => null,
            },
        ],
        // Which batch row's detailed condition analysis is expanded in the right panel.
        // Null falls back to the first row so a batch run always shows some detail.
        selectedResultDistinctId: [
            null as string | null,
            {
                setSelectedResultDistinctId: (_, { distinctId }) => distinctId,
                testAllDistinctIds: () => null,
                testFlagEvaluation: () => null,
                setTestFormData: () => null,
                clearTestForm: () => null,
                setSelectedPerson: () => null,
            },
        ],
        datePickerOpen: [
            false,
            {
                setDatePickerOpen: (_, { open }) => open,
            },
        ],
        datePickerValue: [
            null as Dayjs | null,
            {
                setDatePickerValue: (_, { value }) => value,
                clearTestForm: () => null,
            },
        ],
        selectedPerson: [
            null as Partial<PersonType> | null,
            {
                setSelectedPerson: (_, { person }) => person ?? null,
                clearTestForm: () => null,
            },
        ],
        includeTime: [
            true,
            {
                setIncludeTime: (_, { includeTime }) => includeTime,
            },
        ],
    }),
    listeners(({ actions, values, props }) => ({
        submitTestEvaluation: () => {
            const formData = values.testFormData
            const formError = testFormValidationError(formData)
            if (formError) {
                actions.setTestError(formError)
                return
            }
            if (values.hasMultipleDistinctIds) {
                // Evaluate every merged distinct ID in one go so their variants can be
                // compared side by side, rather than re-running the tool per ID.
                actions.testAllDistinctIds({ flagId: props.flagId, distinctIds: values.personDistinctIds, formData })
            } else {
                actions.testFlagEvaluation({ flagId: props.flagId, formData })
            }
        },
        setSelectedPerson: ({ person, distinctId }) => {
            // A different person invalidates any batch table from the previous one.
            actions.clearBatchEvaluations()
            // Persons from the recent tab arrive without distinct_ids. Fetch the full
            // person so hasMultipleDistinctIds and the bucketing picker work correctly.
            if (person && !person.distinct_ids?.length) {
                const id = distinctId ?? values.testFormData.distinct_id
                if (id) {
                    actions.resolvePersonDistinctIds(id)
                }
            }
        },
        // Editing distinct_id/timestamp/groups invalidates prior results, matching how
        // testResult resets on the same action.
        setTestFormData: () => {
            actions.clearBatchEvaluations()
        },
        clearTestForm: () => {
            actions.clearBatchEvaluations()
        },
    })),
    selectors({
        // The result currently driving the detail panel (condition analysis + person
        // properties). In batch mode it's the selected row (or the first, if none picked);
        // otherwise it's the single-ID evaluation.
        activeResult: [
            (s) => [s.testResult, s.allEvaluations, s.selectedResultDistinctId],
            (
                single: TestResult | null,
                all: PerDistinctIdEvaluation[] | null,
                selectedId: string | null
            ): TestResult | null => {
                if (all?.length) {
                    const selected = selectedId ? all.find((e) => e.distinctId === selectedId) : undefined
                    return (selected ?? all[0]).result
                }
                return single
            },
        ],
        // True when the person's merged distinct IDs don't all bucket to the same result —
        // the exact divergence that makes runtime evaluation ambiguous. Errored rows are
        // ignored so a single failed ID doesn't read as a disagreement.
        resultsDiverge: [
            (s) => [s.allEvaluations],
            (all: PerDistinctIdEvaluation[] | null): boolean => {
                if (!all?.length) {
                    return false
                }
                const outcomes = new Set(
                    all.filter((e) => e.result).map((e) => JSON.stringify(e.result?.result ?? null))
                )
                return outcomes.size > 1
            },
        ],
        // Get the set of properties used in any condition
        usedProperties: [
            (s) => [s.activeResult],
            (result: TestResult | null): Set<string> => {
                const used = new Set<string>()
                if (result?.conditions) {
                    for (const condition of result.conditions) {
                        for (const prop of condition.properties) {
                            used.add(prop.key)
                        }
                    }
                }
                return used
            },
        ],
        // Get enriched conditions with derived flags for UI rendering
        enrichedConditions: [
            (s) => [s.activeResult],
            (result: TestResult | null) => {
                if (!result?.conditions) {
                    return []
                }

                return result.conditions.map((condition: ConditionAnalysis) => {
                    // Check if this condition is the actual winner
                    // Per the API serializer, matched is the source of truth - "at most one condition per flag is True"
                    const isWinningCondition = condition.matched
                    // Determine if this condition matched but wasn't the winner
                    const matchedButNotWinner =
                        condition.properties_matched && !isWinningCondition && !condition.rollout_excluded

                    // Determine display properties
                    let tone: 'success' | 'info' | 'warning' | 'muted' = 'muted'
                    let label: string | null = null

                    if (isWinningCondition) {
                        tone = 'success'
                        label = 'MATCHED'
                    } else if (matchedButNotWinner) {
                        tone = 'info'
                        label = 'PROPERTIES MATCHED'
                    } else if (condition.rollout_excluded) {
                        tone = 'warning'
                        label = 'EXCLUDED FROM ROLLOUT'
                    }

                    return {
                        ...condition,
                        isWinningCondition,
                        matchedButNotWinner,
                        display: {
                            tone,
                            label,
                        },
                    }
                })
            },
        ],
        // Check if form has valid person selected
        hasValidPerson: [
            (s) => [s.testFormData],
            (formData: TestFormData): boolean => Boolean(formData.distinct_id?.trim()),
        ],
        // The distinct IDs known for the selected person. For persons from the Persons
        // tab distinct_ids comes directly from the picker. For partial persons (e.g. the
        // recent tab, which carries name + distinct_id only) we fall back to the async
        // lookup in resolvedPersonDistinctIds, which is null while loading and [] when the
        // person turns out to have only one ID.
        personDistinctIds: [
            (s) => [s.selectedPerson, s.resolvedPersonDistinctIds],
            (person: Partial<PersonType> | null, resolved: string[] | null): string[] =>
                person?.distinct_ids ?? resolved ?? [],
        ],
        // A person merged from multiple distinct IDs can bucket into a different
        // rollout/variant depending on which ID is hashed, so the test result may not
        // match what runtime evaluation produces for the same person.
        hasMultipleDistinctIds: [
            (s) => [s.personDistinctIds],
            (distinctIds: string[]): boolean => distinctIds.length > 1,
        ],
        // The distinct ID the backend reports it bucketed against. Null when the API
        // withholds it to avoid leaking distinct IDs to feature_flag:read-only tokens —
        // we must not fall back to the requested ID, which would mislabel the result.
        bucketingDistinctId: [
            (s) => [s.activeResult],
            (result: TestResult | null): string | null => result?.evaluation_distinct_id ?? null,
        ],
        // Get formatted error display information
        errorDisplay: [
            (s) => [s.testError],
            (error: string | null) => {
                if (!error) {
                    return null
                }

                let helpText: string | null = null

                if (error.toLowerCase().includes('build person properties')) {
                    helpText =
                        'Try a more recent timestamp when this person was active, remove the timestamp to test with current person properties, or select a different person who was active at that time.'
                } else if (error.toLowerCase().includes('person') && error.toLowerCase().includes('not found')) {
                    helpText = 'Try selecting a different person or removing the timestamp to test with current data.'
                } else if (error.toLowerCase().includes('timestamp')) {
                    helpText =
                        'When using historical timestamps, the person must have existed at that time and had the necessary properties for evaluation.'
                }

                return {
                    message: error,
                    helpText,
                }
            },
        ],
    }),
])