Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.3 KB
Raw Blame
import {
    ClaudeTaskRunCreateSchemaApi,
    CodexTaskRunCreateSchemaInitialPermissionModeEnumApi,
    InitialPermissionModeEnumApi,
    ModelChoiceApi,
    ReasoningEffortEnumApi,
    RuntimeAdapterEnumApi,
    TaskRuntimeEnumApi,
} from 'products/tasks/frontend/generated/api.schemas'
import { DEFAULT_MODEL_BY_RUNTIME_ADAPTER, MODELS } from 'products/tasks/frontend/modelCatalog.generated'

import {
    buildRunCreateRequest,
    getCapabilityLadder,
    getDefaultModelForRuntimeAdapter,
    getEffortsForModel,
    getModelCost,
    getModelLabel,
    getRuntimeAdapterForModel,
    listRuntimeAdapters,
    modelsForRuntimeAdapter,
    pickerModels,
} from './composerModels'
import { type PermissionMode } from './composerModes'

describe('composerModels', () => {
    const CATALOGUE: ModelChoiceApi[] = [
        {
            runtime_adapter: 'claude',
            model: 'claude-opus-4-8',
            display_name: 'Claude Opus 4.8',
            supported_efforts: ['low', 'medium', 'high'],
        },
        {
            runtime_adapter: 'codex',
            model: 'gpt-5.6-luna',
            display_name: 'GPT-5.6 Luna',
            supported_efforts: ['low', 'medium', 'high'],
        },
    ]

    it.each([
        [null, 'gpt-6.1-sol'],
        ['gpt-5.6-luna', 'gpt-5.6-luna'],
        ['openai/gpt-5.6-luna', 'gpt-5.6-luna'],
        ['claude-opus-4-8', 'gpt-6.1-sol'],
        ['retired-model', 'gpt-6.1-sol'],
    ])('selects the Codex default with preference %s', (preference, expected) => {
        const catalogue: ModelChoiceApi[] = [
            ...CATALOGUE,
            {
                runtime_adapter: 'codex',
                model: 'gpt-6-sol',
                display_name: 'GPT-6 Sol',
                supported_efforts: ['low', 'medium', 'high', 'xhigh', 'max'],
            },
            {
                runtime_adapter: 'codex',
                model: 'gpt-6.1-sol',
                display_name: 'GPT-6.1 Sol',
                supported_efforts: ['low', 'medium', 'high', 'xhigh', 'max'],
            },
        ]

        expect(getDefaultModelForRuntimeAdapter(catalogue, RuntimeAdapterEnumApi.Codex, preference)).toBe(expected)
    })

    it.each([RuntimeAdapterEnumApi.Claude, RuntimeAdapterEnumApi.Codex])(
        'selects the shared default for %s before the capability ladder',
        (adapter) => {
            const catalogue: ModelChoiceApi[] = MODELS.map((model) => ({
                runtime_adapter: model.runtimeAdapter as RuntimeAdapterEnumApi,
                model: model.id,
                display_name: model.label,
                supported_efforts: [...model.reasoningEfforts] as ReasoningEffortEnumApi[],
            }))
            const model = getDefaultModelForRuntimeAdapter(catalogue, adapter, null)

            expect(model).toBe(DEFAULT_MODEL_BY_RUNTIME_ADAPTER[adapter])
        }
    )

    it.each([
        [
            CATALOGUE.concat({
                runtime_adapter: RuntimeAdapterEnumApi.Codex,
                model: 'gpt-6-sol',
                display_name: 'GPT-6 Sol',
                supported_efforts: [ReasoningEffortEnumApi.High],
            }),
            RuntimeAdapterEnumApi.Codex,
            'gpt-6-sol',
        ],
        [CATALOGUE, RuntimeAdapterEnumApi.Codex, 'gpt-5.6-luna'],
        [CATALOGUE, RuntimeAdapterEnumApi.Claude, 'claude-opus-4-8'],
        [[], RuntimeAdapterEnumApi.Codex, null],
    ])('uses an available model when no preferred default is available', (catalogue, adapter, expected) => {
        expect(getDefaultModelForRuntimeAdapter(catalogue, adapter, 'retired-model')).toBe(expected)
    })

    // The runtime follows from the model, so a Codex pick must not launch on the Claude adapter — and each
    // runtime validates permission modes against its own vocabulary, so `bypassPermissions` has to become
    // `full-access` on the way out. Getting either wrong is a request the backend rejects.
    it.each([
        ['claude-opus-4-8', 'claude', InitialPermissionModeEnumApi.BypassPermissions],
        ['gpt-5.6-luna', 'codex', CodexTaskRunCreateSchemaInitialPermissionModeEnumApi.FullAccess],
    ])('%s launches on %s', (model, expectedAdapter, expectedMode) => {
        const request = buildRunCreateRequest(
            CATALOGUE,
            model,
            ReasoningEffortEnumApi.High,
            InitialPermissionModeEnumApi.BypassPermissions as PermissionMode,
            { branch: 'main' }
        )

        expect(request).toMatchObject({
            runtime_adapter: expectedAdapter,
            model,
            initial_permission_mode: expectedMode,
            branch: 'main',
        })
    })

    // The gateway serves some models provider-qualified, so a run or workflow config can hold either spelling.
    // The backend and the desktop app both fold them onto the bare id; without this the web alone reads a stored
    // `anthropic/...` run as an unknown model — raw id for a name, and the generic effort floor.
    it.each([
        ['anthropic/claude-opus-4-8', 'Claude Opus 4.8'],
        ['openai/gpt-5.6-luna', 'GPT-5.6 Luna'],
    ])('resolves the provider-qualified id %s like the bare one', (qualified, expectedLabel) => {
        const bare = qualified.split('/')[1]
        expect(getModelLabel(CATALOGUE, qualified)).toBe(expectedLabel)
        expect(getEffortsForModel(CATALOGUE, qualified)).toEqual(getEffortsForModel(CATALOGUE, bare))
        expect(
            buildRunCreateRequest(
                CATALOGUE,
                qualified,
                ReasoningEffortEnumApi.High,
                InitialPermissionModeEnumApi.Auto as PermissionMode,
                {}
            )
        ).toMatchObject({ runtime_adapter: getRuntimeAdapterForModel(CATALOGUE, bare) })
    })

    // A stored `anthropic/...` spelling must not lose its cost while keeping its name, which would
    // show as one row in the list silently missing a figure every other row has.
    it.each([
        ['claude-opus-5', '2.5×'],
        ['anthropic/claude-opus-5', '2.5×'],
        ['gpt-5', null],
        ['some-unreleased-model', null],
    ])('reads the cost of %s off the catalog', (model, expected) => {
        expect(getModelCost(model)?.multiplier ?? null).toBe(expected)
    })

    // A model absent from the catalogue (still loading, or retired from the gateway) must still produce a
    // sendable request rather than an undefined adapter.
    it('falls back to the claude adapter for an unknown model', () => {
        const request = buildRunCreateRequest(
            CATALOGUE,
            'some-unreleased-model',
            ReasoningEffortEnumApi.High,
            InitialPermissionModeEnumApi.Auto as PermissionMode,
            {}
        )

        expect(request).toMatchObject({ runtime_adapter: 'claude' })
    })

    // Some models take no reasoning effort at all, and the backend rejects any value for them
    // ("Supported values: none"). The composer always holds an effort for display, so the request has to drop it.
    it('omits the effort for a model that takes none', () => {
        const request = buildRunCreateRequest(
            [
                ...CATALOGUE,
                {
                    runtime_adapter: 'claude',
                    model: 'moonshotai/kimi-k3',
                    display_name: 'Moonshotai Kimi K3',
                    supported_efforts: [],
                },
            ],
            'moonshotai/kimi-k3',
            ReasoningEffortEnumApi.High,
            InitialPermissionModeEnumApi.Plan as PermissionMode,
            {}
        )

        expect((request as ClaudeTaskRunCreateSchemaApi).reasoning_effort).toBeUndefined()
    })

    // Every rung the Faster/Smarter slider offers has to be sendable. The catalog is checked in, so a model the
    // gateway has retired — or one that no longer takes the paired effort — must drop out of the stops rather
    // than become a notch whose run the backend rejects.
    it('keeps only the ladder rungs the catalogue still serves', () => {
        const catalogue: ModelChoiceApi[] = [
            {
                runtime_adapter: 'claude',
                model: 'claude-sonnet-5-5',
                display_name: 'Claude Sonnet 5.5',
                supported_efforts: ['low', 'medium'],
            },
            {
                runtime_adapter: 'claude',
                model: 'claude-opus-5-5',
                display_name: 'Claude Opus 5.5',
                supported_efforts: ['medium', 'high', 'xhigh'],
            },
        ]

        // Dropped: sonnet at `high` (unsupported effort) and fable entirely (absent from the catalogue).
        expect(getCapabilityLadder(catalogue, RuntimeAdapterEnumApi.Claude)).toEqual([
            { model: 'claude-sonnet-5-5', effort: ReasoningEffortEnumApi.Medium },
            { model: 'claude-opus-5-5', effort: ReasoningEffortEnumApi.Medium },
            { model: 'claude-opus-5-5', effort: ReasoningEffortEnumApi.Xhigh },
        ])
        expect(getCapabilityLadder(catalogue, RuntimeAdapterEnumApi.Codex)).toEqual([])
    })

    it('uses GPT-6 Astra at Max as the smartest Codex rung', () => {
        const catalogue: ModelChoiceApi[] = [
            {
                runtime_adapter: 'codex',
                model: 'gpt-5.6-sol',
                display_name: 'GPT-5.6 Sol',
                supported_efforts: ['low', 'medium', 'high', 'xhigh', 'max'],
            },
            {
                runtime_adapter: 'codex',
                model: 'gpt-6-astra',
                display_name: 'GPT-6 Astra',
                supported_efforts: ['low', 'medium', 'high', 'xhigh', 'max'],
            },
        ]

        expect(getCapabilityLadder(catalogue, RuntimeAdapterEnumApi.Codex).at(-1)).toEqual({
            model: 'gpt-6-astra',
            effort: ReasoningEffortEnumApi.Max,
        })
    })

    // The picker groups by harness and offers one row per runtime, so both have to come off the catalogue rather
    // than a hardcoded list — a runtime the gateway stops serving must stop being offered.
    it.each([
        [TaskRuntimeEnumApi.Acp, ['claude', 'codex'], ['gpt-5.6-luna']],
        [TaskRuntimeEnumApi.Pi, [], ['claude-opus-4-8', 'gpt-5.6-luna']],
    ])("derives the harness list and each harness's models from the catalogue for %s", (runtime, adapters, models) => {
        expect(listRuntimeAdapters(CATALOGUE, runtime)).toEqual(adapters)
        expect(modelsForRuntimeAdapter(CATALOGUE, RuntimeAdapterEnumApi.Codex, runtime).map((o) => o.model)).toEqual(
            models
        )
    })

    describe('pickerModels', () => {
        const RETIRED: ModelChoiceApi = {
            runtime_adapter: 'claude',
            model: 'claude-opus-4-7',
            display_name: 'Claude Opus 4.7',
            supported_efforts: ['low', 'medium', 'high', 'xhigh', 'max', 'ultracode'],
        }
        const FULL = [...CATALOGUE, RETIRED]

        it('lists the retired model this run is on, so its name and efforts survive', () => {
            expect(pickerModels(FULL, 'claude-opus-4-7')).toEqual(FULL)
        })

        it('resolves the run model through the same normalization the catalogue uses', () => {
            expect(pickerModels(FULL, 'anthropic/claude-opus-4-7')).toEqual(FULL)
        })

        it.each([null, undefined, 'claude-opus-4-8', 'model-no-catalogue-knows'])(
            'drops every retired model for %s',
            (selected: string | null | undefined) => {
                expect(pickerModels(FULL, selected)).toEqual(CATALOGUE)
            }
        )
    })
})