The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import { describe, expect, it } from 'vitest'
import { DEFAULT_ROUTING_POLICY, effectivePolicy, priorFor, routingOverrideSaveSchema, routingOverrideSchema, routingPolicySaveSchema, routingPolicySchema } from './routingPolicy.js'
import type { PriorsTable } from './benchmarkPriors/priors.js'
/** The `default default` Codex row policies saved before task #2972 hold; the catalog no longer has it, and a saved policy's Codex
* switch is still read off it (effectivePolicy). */
const SAVED_CODEX_DEFAULT = { engine: 'codex' as const, model: 'default', effort: 'default', quality: 3, cost: 2, enabled: true }
describe('routing policy', () => {
it('ships a valid default that needs no setup', () => {
expect(routingPolicySchema.parse(DEFAULT_ROUTING_POLICY)).toEqual(DEFAULT_ROUTING_POLICY)
expect(DEFAULT_ROUTING_POLICY.floor).toEqual({ low: 1, medium: 2, high: 3 })
expect(DEFAULT_ROUTING_POLICY.quotaReserve).toBe(10)
expect(DEFAULT_ROUTING_POLICY.notes).toBe('')
expect(new Set(DEFAULT_ROUTING_POLICY.models.map(m => m.engine))).toEqual(new Set(['claude', 'codex']))
})
it('offers Claude Opus at xhigh as its own row, dearer than at high for the extra thinking', () => {
expect(DEFAULT_ROUTING_POLICY.models).toContainEqual({ engine: 'claude', model: 'opus', effort: 'xhigh', quality: 3, cost: 4, enabled: true })
})
// Ken, 2026-11-02: Sonnet at high effort is back for small work (2026-10-30 had turned it off). Only the default table
// changes; an owner's own table is theirs (Lead's ruling).
it('offers Sonnet at high effort in the default table, and keeps an owner\'s saved Sonnet row and a tab override\'s as they are', () => {
const sonnet = (models: Array<{ engine: string; model: string; enabled: boolean }>) => models.filter(m => m.engine === 'claude' && m.model === 'sonnet')
expect(sonnet(DEFAULT_ROUTING_POLICY.models)).toEqual([expect.objectContaining({ effort: 'high', quality: 2, enabled: true })])
// Haiku too (review of #2314): an older CLI on the new server picked it for easy work the server refuses, on every route.
expect(DEFAULT_ROUTING_POLICY.models.filter(m => m.model === 'haiku')).toEqual([expect.objectContaining({ quality: 1, enabled: false })])
// Nothing it may pick is rated below 2 (an older CLI's own route still keeps its builds at 3).
expect(DEFAULT_ROUTING_POLICY.models.filter(m => m.enabled).every(m => m.quality >= 2)).toBe(true)
expect(sonnet(effectivePolicy(null, null).policy.models).map(m => m.enabled)).toEqual([true])
// An owner who saved the table with Sonnet off keeps it off; it is their row.
const saved = { ...DEFAULT_ROUTING_POLICY, models: DEFAULT_ROUTING_POLICY.models.map(m => m.model === 'sonnet' ? { ...m, enabled: false } : m) }
const read = effectivePolicy(saved, null)
expect(read.warnings).toEqual([])
expect(sonnet(read.policy.models).map(m => m.enabled)).toEqual([false])
expect(read.policy.models.filter(m => m.model !== 'sonnet')).toEqual(saved.models.filter(m => m.model !== 'sonnet'))
// A tab override's own models win over the default too.
expect(sonnet(effectivePolicy(null, { models: saved.models }).policy.models).map(m => m.enabled)).toEqual([false])
})
it('uses the default when the account has none, and merges a tab override field by field', () => {
expect(effectivePolicy(null, null)).toEqual({ policy: DEFAULT_ROUTING_POLICY, warnings: [], overridden: [] })
const account = { ...DEFAULT_ROUTING_POLICY, quotaReserve: 20, notes: 'Frontend prefers Claude.' }
const result = effectivePolicy(account, { notes: 'Long jobs on remote-1.' })
expect(result.policy.quotaReserve).toBe(20)
expect(result.policy.notes).toBe('Long jobs on remote-1.')
expect(result.overridden).toEqual(['notes'])
expect(result.warnings).toEqual([])
})
it('reads a policy saved at the 40-row limit with the added rows, and saves and reads it back after an edit', () => {
// Review of #2271: rows added on read must not push an owner's full policy past what saving accepts.
const owner = Array.from({ length: 40 }, (_, i) => ({ engine: 'claude' as const, model: `model-${i}`, effort: 'high', quality: 2, cost: 2, enabled: true }))
const read = effectivePolicy({ ...DEFAULT_ROUTING_POLICY, models: owner }, null).policy
expect(read.models).toHaveLength(42)
expect(read.models.slice(0, 40)).toEqual(owner)
const saved = routingPolicySchema.parse({ ...read, notes: 'Edited.' })
expect(saved.models).toEqual(read.models)
expect(effectivePolicy(saved, null).policy.models).toEqual(read.models)
})
it('keeps the owner\'s limit at exactly 40 rows of their own, whatever built-in rows are added on read', () => {
const own = (n: number) => Array.from({ length: n }, (_, i) => ({ engine: 'claude' as const, model: `model-${i}`, effort: 'high', quality: 2, cost: 2, enabled: true }))
const parses = (models: unknown[]) => routingPolicySchema.safeParse({ ...DEFAULT_ROUTING_POLICY, models }).success
expect(parses(own(40))).toBe(true)
expect(parses(own(41))).toBe(false)
const sol = DEFAULT_ROUTING_POLICY.models.find(m => m.model === 'gpt-6.1-sol')!
expect(parses([...own(40), sol])).toBe(true)
// The owner's own version of a built-in row takes its place, not one of their 40.
expect(parses([...own(40), { ...sol, enabled: false }])).toBe(true)
expect(parses([...own(41), sol])).toBe(false)
// Both built-in rows ride free of the 40, Astra's as Sol's.
const astra = DEFAULT_ROUTING_POLICY.models.find(m => m.model === 'gpt-6-astra')!
expect(parses([...own(40), sol, astra])).toBe(true)
expect(parses([...own(40), sol, { ...astra, quality: 2 }])).toBe(true)
expect(routingPolicySaveSchema.safeParse({ ...DEFAULT_ROUTING_POLICY, models: [...own(40), sol, astra] }).success).toBe(true)
expect(routingPolicySaveSchema.safeParse({ ...DEFAULT_ROUTING_POLICY, models: [...own(40), sol, astra, astra] }).success).toBe(false)
})
// Lead's ruling, 10-30: the owners' standing decision is that Builders run on Opus high or Astra xhigh; with no row Astra rated
// 1 and was never picked.
it('ships Codex GPT-6-Astra at xhigh as a strong built-in row, dearer than Opus, and keeps an owner\'s own Astra row', () => {
const astra = DEFAULT_ROUTING_POLICY.models.find(m => m.model === 'gpt-6-astra')!
expect(astra).toEqual({ engine: 'codex', model: 'gpt-6-astra', effort: 'xhigh', quality: 3, cost: 5, enabled: true })
const opusCosts = DEFAULT_ROUTING_POLICY.models.filter(m => m.model === 'opus').map(m => m.cost)
expect(astra.cost).toBeGreaterThan(Math.max(...opusCosts))
// A policy saved before the row reads it.
const before = { ...DEFAULT_ROUTING_POLICY, models: [...DEFAULT_ROUTING_POLICY.models.filter(m => m.model !== 'gpt-6-astra'), SAVED_CODEX_DEFAULT] }
expect(effectivePolicy(before, null).policy.models.filter(m => m.model === 'gpt-6-astra')).toEqual([astra])
// An owner's own Astra row wins over the built-in one, on or off, and is never doubled (review of #2271).
const owned = { ...before, models: [...before.models, { ...astra, effort: 'high', quality: 2, cost: 9, enabled: false }] }
expect(effectivePolicy(owned, null).policy.models.filter(m => m.model === 'gpt-6-astra')).toEqual([{ ...astra, effort: 'high', quality: 2, cost: 9, enabled: false }])
})
describe('a built-in row added on read (review of #2271 at 1d9499646)', () => {
const sol = DEFAULT_ROUTING_POLICY.models.find(m => m.model === 'gpt-6.1-sol')!
const codexDefault = SAVED_CODEX_DEFAULT
const claudeRows = DEFAULT_ROUTING_POLICY.models.filter(m => m.engine === 'claude')
const solOf = (models: Array<{ engine: string; model: string; enabled: boolean }>) => models.find(m => m.model === 'gpt-6.1-sol')
it('is on only while its engine\'s default row is on, so an owner who turned Codex off keeps it off', () => {
const off = effectivePolicy({ ...DEFAULT_ROUTING_POLICY, models: [...claudeRows, { ...codexDefault, enabled: false }] }, null)
expect(solOf(off.policy.models)).toEqual({ ...sol, enabled: false })
const absent = effectivePolicy({ ...DEFAULT_ROUTING_POLICY, models: claudeRows }, null)
expect(solOf(absent.policy.models)).toEqual({ ...sol, enabled: false })
const on = effectivePolicy({ ...DEFAULT_ROUTING_POLICY, models: [...claudeRows, codexDefault] }, null)
expect(solOf(on.policy.models)).toEqual(sol)
// A tab override without the row and with Codex off, over no saved account policy.
const tab = effectivePolicy(null, { models: [...claudeRows, { ...codexDefault, enabled: false }] })
expect(solOf(tab.policy.models)).toEqual({ ...sol, enabled: false })
})
it('comes from the account policy when a tab override saved without it', () => {
const account = { ...DEFAULT_ROUTING_POLICY, models: [...claudeRows, codexDefault, { ...sol, enabled: false, quality: 2 }] }
const tab = effectivePolicy(account, { models: [...claudeRows, codexDefault] })
expect(solOf(tab.policy.models)).toEqual({ ...sol, enabled: false, quality: 2 })
// The account's row on, but this tab turned Codex off: it stays off here.
const accountOn = { ...DEFAULT_ROUTING_POLICY, models: [...claudeRows, codexDefault, sol] }
const codexOff = effectivePolicy(accountOn, { models: [...claudeRows, { ...codexDefault, enabled: false }] })
expect(solOf(codexOff.policy.models)).toEqual({ ...sol, enabled: false })
expect(solOf(effectivePolicy(accountOn, { models: [...claudeRows, codexDefault] }).policy.models)).toEqual(sol)
})
})
it('counts a second row for a built-in model against the 40 when saving, and still reads an old policy saved with one', () => {
// Review of #2271: a second copy never rides free. Review of #2314: an owner with two Astra rows (two efforts, from before
// Astra was built in) must still save.
const own = (n: number) => Array.from({ length: n }, (_, i) => ({ engine: 'claude' as const, model: `model-${i}`, effort: 'high', quality: 2, cost: 2, enabled: true }))
const sol = DEFAULT_ROUTING_POLICY.models.find(m => m.model === 'gpt-6.1-sol')!
const astra = DEFAULT_ROUTING_POLICY.models.find(m => m.model === 'gpt-6-astra')!
const saves = (models: unknown[]) => routingPolicySaveSchema.safeParse({ ...DEFAULT_ROUTING_POLICY, models }).success
expect(saves([...own(10), sol, astra, { ...astra, effort: 'high', quality: 2 }])).toBe(true)
expect(saves([...own(39), sol, astra, { ...astra, effort: 'high' }])).toBe(true)
const twice = { ...DEFAULT_ROUTING_POLICY, models: [...own(40), sol, { ...sol, enabled: false }] }
const refused = routingPolicySaveSchema.safeParse(twice)
expect(refused.success).toBe(false)
expect(refused.error?.issues.map(i => i.message)).toEqual(['At most 40 rows of your own.'])
expect(routingOverrideSaveSchema.safeParse({ models: twice.models }).success).toBe(false)
expect(saves([...own(40), sol, astra])).toBe(true)
// Saved before this count: still read as saved, never as unreadable; the Astra row joins it, off with Codex off.
expect(effectivePolicy(twice, null)).toEqual({ policy: { ...twice, models: [...twice.models, { ...astra, enabled: false }] }, warnings: [], overridden: [] })
})
it('falls back to the default with a warning for a corrupt account policy or override', () => {
const corrupt = effectivePolicy({ version: 9 }, { floor: { low: 7 } })
expect(corrupt.policy).toEqual(DEFAULT_ROUTING_POLICY)
expect(corrupt.warnings).toEqual([
'The account routing policy is unreadable; the built-in default applies.',
'This Super Tab routing override is unreadable and was ignored.',
])
})
it('rejects out-of-range values and unknown engines', () => {
const bad = (patch: Record<string, unknown>) => routingPolicySchema.safeParse({ ...DEFAULT_ROUTING_POLICY, ...patch }).success
expect(bad({ quotaReserve: 91 })).toBe(false)
expect(bad({ notes: 'x'.repeat(501) })).toBe(false)
expect(bad({ floor: { low: 0, medium: 2, high: 3 } })).toBe(false)
expect(bad({ models: [{ ...DEFAULT_ROUTING_POLICY.models[0], engine: 'cursor' }] })).toBe(false)
expect(bad({ models: [{ ...DEFAULT_ROUTING_POLICY.models[0], cost: 0 }] })).toBe(false)
expect(bad({ models: [] })).toBe(false)
expect(routingOverrideSchema.safeParse({ version: 1 }).success).toBe(false)
})
// Grid ratings are the catalog's (catalog.test.ts); the CLI's route rates grid models by them (cli superTabs/route.ts
// gridStrength). The migration still reads a saved policy's own grid rows.
it('reads a policy saved before per-model grid rows with the built-in ones, and an owner\'s own rows as saved', () => {
const builtIn = DEFAULT_ROUTING_POLICY.localModels!.models
const saved = { ...DEFAULT_ROUTING_POLICY, localModels: { quality: 1, enabled: true } }
expect(effectivePolicy(saved, null).policy.localModels).toEqual({ quality: 1, enabled: true, models: builtIn })
const own = [{ model: 'qwen 3', quality: 3 }, { model: 'qwen', quality: 1 }]
expect(effectivePolicy(saved, { localModels: { quality: 2, enabled: true, models: own } }).policy.localModels).toEqual({ quality: 2, enabled: true, models: own })
expect(routingPolicySchema.safeParse({ ...DEFAULT_ROUTING_POLICY, localModels: { quality: 1, enabled: true, models: [{ model: 'x', quality: 4 }] } }).success).toBe(false)
})
describe('owner override of a benchmark prior', () => {
const src = 'https://example.org/board'
const entry = (model: string, value: number, engine: 'claude' | 'grid' = 'claude') => ({ engine, model, names: [model], benchmarkNames: [model],
scores: [{ benchmark: 'terminal-bench-4.0', score: value, harness: 'Claude Code', source: src, date: '2026-09-22', via: 'leaderboard' as const }] })
const table: PriorsTable = { refreshed: '2026-10-29', note: 'Depends on the harness.', benchmarks: [{ id: 'terminal-bench-4.0', name: 'Terminal-Bench 4.0', url: src }],
models: [entry('best', 80), entry('moonshotai/kimi-k3', 12, 'grid')] }
const withOverrides = (priorOverrides: unknown) => effectivePolicy({ ...DEFAULT_ROUTING_POLICY, priorOverrides }, null).policy
it('uses the benchmark prior when the owner sets nothing, and says where it came from', () => {
const prior = priorFor(DEFAULT_ROUTING_POLICY, { engine: 'grid', model: 'moonshotai/kimi-k3' }, 'cli', table)
expect(prior?.quality).toBe(1)
expect(prior?.source).toBe('benchmark')
})
it('lets the owner replace one prior: that model and kind only, matched by the words of its name', () => {
const policy = withOverrides([{ model: 'kimi k3', kind: 'cli', quality: 3 }])
expect(priorFor(policy, { engine: 'grid', model: 'moonshotai/kimi-k3' }, 'cli', table)).toEqual({ quality: 3, source: 'owner' })
expect(priorFor(policy, { engine: 'grid', model: 'moonshotai/kimi-k3' }, 'backend', table)?.source).toBe('benchmark')
expect(priorFor(policy, { engine: 'claude', model: 'best' }, 'cli', table)?.quality).toBe(3)
expect(priorFor(policy, { engine: 'claude', model: 'best' }, 'cli', table)?.source).toBe('benchmark')
})
it('gives a model with no data an owner prior, and keeps first match first', () => {
const policy = withOverrides([{ model: 'qwen 3', kind: 'tests', quality: 2 }, { model: 'qwen', kind: 'tests', quality: 1 }])
expect(priorFor(policy, { engine: 'grid', model: 'qwen3:4b', label: 'Qwen 3 4B' }, 'tests', table)).toEqual({ quality: 2, source: 'owner' })
expect(priorFor(policy, { engine: 'grid', model: 'qwen3:4b', label: 'Qwen 3 4B' }, 'docs', table)).toBeUndefined()
})
it('reads a policy saved before overrides as having none', () => {
expect(routingPolicySchema.safeParse(DEFAULT_ROUTING_POLICY).success).toBe(true)
expect(priorFor(effectivePolicy(null, null).policy, { engine: 'claude', model: 'best' }, 'cli', table)?.source).toBe('benchmark')
})
it('replaces the account list with the tab override, and one policy never reads another policy\'s rows', () => {
const account = { ...DEFAULT_ROUTING_POLICY, priorOverrides: [{ model: 'best', kind: 'cli', quality: 1 }] }
const tab = effectivePolicy(account, { priorOverrides: [{ model: 'best', kind: 'cli', quality: 2 }] }).policy
const other = effectivePolicy(null, null).policy
expect(priorFor(tab, { engine: 'claude', model: 'best' }, 'cli', table)?.quality).toBe(2)
expect(priorFor(effectivePolicy(account, null).policy, { engine: 'claude', model: 'best' }, 'cli', table)?.quality).toBe(1)
expect(priorFor(other, { engine: 'claude', model: 'best' }, 'cli', table)?.quality).toBe(3)
})
it('rejects a level outside 1-3, an unknown kind, an empty or oversized model name, more than 100 rows and unknown fields', () => {
const bad = (rows: unknown) => routingPolicySchema.safeParse({ ...DEFAULT_ROUTING_POLICY, priorOverrides: rows }).success
const ok = { model: 'kimi k3', kind: 'cli', quality: 2 }
expect(bad([ok])).toBe(true)
expect(bad([{ ...ok, quality: 4 }])).toBe(false)
expect(bad([{ ...ok, quality: 0 }])).toBe(false)
expect(bad([{ ...ok, kind: 'ui' }])).toBe(false)
expect(bad([{ ...ok, model: ' ' }])).toBe(false)
expect(bad([{ ...ok, model: 'x'.repeat(201) }])).toBe(false)
expect(bad(Array.from({ length: 101 }, () => ok))).toBe(false)
expect(bad([{ ...ok, note: 'because' }])).toBe(false)
expect(routingOverrideSchema.safeParse({ priorOverrides: [ok] }).success).toBe(true)
})
it('falls back to the default with a warning when the saved overrides are corrupt', () => {
const corrupt = effectivePolicy({ ...DEFAULT_ROUTING_POLICY, priorOverrides: [{ model: 'x', kind: 'cli', quality: 9 }] }, null)
expect(corrupt.policy).toEqual(DEFAULT_ROUTING_POLICY)
expect(corrupt.warnings).toEqual(['The account routing policy is unreadable; the built-in default applies.'])
})
})
})