The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
// cli/src/superTabs/routeCheapFirst.spec.ts
import { describe, expect, it, vi } from 'vitest'
import { choose, decide, recordOf, type Assessment, type Catalog, type Probe } from './route.js'
import { makePlugins, MAKER_TIMEOUT_MS, ROUTE_PLUGIN_MAKERS } from './routePlugins.js'
import { CHEAP_FIRST, CHEAP_FIRST_MAKER, CHEAP_FIRST_MAKERS, cheapFirst, cheapTaskOf, EARLIER_UNREADABLE, GRID_KINDS, GRID_KINDS_ONLY, GRID_LIMITS, GRID_LIMITS_MAKER, gridLimits, HISTORY_WAIT_MS, FIRST_ATTEMPT_ONLY, GRID_FIRST, MAX_OWNED, NEVER_REVIEWS, NEVER_SECURITY, type CheapTask } from './routeCheapFirst.js'
const NOW = 1_000_000_000, H = 3_600_000, D = 86_400_000
/** Today's catalog (backend superTabs/catalog.ts): Opus and Codex at 3, Kimi K3 and DeepSeek v4.1 at 2 on the grid. */
const catalog: Catalog = { grid: { strength: 1, models: [{ model: 'kimi k3', strength: 2 }, { model: 'deepseek v4.1', strength: 2 }] }, models: [
{ engine: 'claude', model: 'opus', effort: 'high', strength: 3, cost: 3, offered: true },
{ engine: 'codex', model: 'default', effort: 'default', strength: 3, cost: 2, offered: true },
] }
const a = (patch: Partial<Assessment> = {}): Assessment => ({ difficulty: 'low', risk: 'low', size: 'small', why: 'Fix a typo in the docs.', kind: 'docs', needs: { tools: [], devices: [] }, ...patch })
const quota = { known: true, leftPercent: 80, resetsAt: null, windows: [{ name: '5-hour', leftPercent: 80, resetsAt: NOW + 3 * H }, { name: 'weekly', leftPercent: 90, resetsAt: NOW + 5 * D }] }
const opus: Probe = { kind: 'new', machineId: 'm1', machineName: 'remote-1', engine: 'claude', model: 'opus', effort: 'high', quality: 3, cost: 3,
quota: { ...quota, account: 'acct' }, activity: 'new agent', known: true, refusals: [], notes: [] }
const codex: Probe = { ...opus, engine: 'codex', model: 'default', effort: 'default', quality: 3, cost: 2, quota: { ...quota, account: 'cx' } }
/** A new agent on a grid model, as routeCommand.ts localProbes makes one. */
const grid = (model: string, label: string, quality = 2): Probe => ({ ...opus, model: 'default', effort: 'default', quality, cost: 1,
quota: { known: false, leftPercent: null, resetsAt: null }, gridModel: model, gridName: 'swarm-grid', gridLabel: label, gridKind: 'api' })
const kimi = grid('moonshotai/kimi-k3', 'Kimi K3'), deepseek = grid('deepseek/deepseek-v4.1', 'DeepSeek v4.1'), qwen = grid('qwen3:4b', 'Qwen 3', 1)
const first: CheapTask = { review: false, owned: 1, earlier: null }
const route = (task: CheapTask, assessment: Partial<Assessment> = {}, probes: Probe[] = [opus, codex, kimi, deepseek, qwen]) =>
choose(probes, catalog, a(assessment), NOW, [], [], [], [gridLimits(task.review), cheapFirst(task)])
const ok = (r: ReturnType<typeof choose>) => { if ('error' in r) throw new Error(JSON.stringify(r)); return r }
const picks = (r: ReturnType<typeof choose>) => ok(r).shown.map(s => s.gridLabel ?? `${s.engine} ${s.model}`)
const reasons = (r: ReturnType<typeof choose>) => r.excluded.map(e => e.reason)
describe('cheap first: a small, low-risk build notes a cheap grid model as the first choice', () => {
it('notes Kimi K3 and DeepSeek as the cheap first choice beside Opus and Codex, says why on the candidate, and records it', () => {
const r = ok(route(first))
// Since task #2901 a plug-in's finding is a note, never the order.
expect(picks(r)).toEqual(picks(choose([opus, codex, kimi, deepseek, qwen], catalog, a(), NOW)))
for (const label of ['DeepSeek v4.1', 'Kimi K3']) {
const s = r.shown.find(c => c.gridLabel === label)!
expect(s.notes).toEqual([`${CHEAP_FIRST}: ${GRID_FIRST}`])
expect(s.label).toContain(GRID_FIRST)
}
expect(r.adjusted).toBe(true)
// Opus and Codex are not noted, and a grid model the catalog does not rate (Qwen, 1) is not either.
expect(r.shown.find(s => s.model === 'opus')!.notes).toEqual([])
expect(r.shown.find(s => s.engine === 'codex')!.notes).toEqual([])
expect(r.shown.find(s => s.gridLabel === 'Qwen 3')!.notes).toEqual([])
const verdict = decide(r)
expect(verdict.via).toBe('instructions')
const record = recordOf(r, verdict, a()) as { candidates: Array<{ gridModel?: string; quality: number; label: string }> }
// It is recorded as what it is: a grid model rated 2, with the reason people read.
const recorded = record.candidates.find(c => c.gridModel === 'deepseek/deepseek-v4.1')!
expect(recorded).toMatchObject({ quality: 2 })
expect(recorded.label).toContain(GRID_FIRST)
expect(record).not.toHaveProperty('pick')
})
it('says nothing unless the task is low risk, low difficulty and small in scope', () => {
const today = picks(choose([opus, codex, kimi, deepseek, qwen], catalog, a(), NOW))
for (const [task, assessment] of [
[first, { difficulty: 'medium' }], [first, { risk: 'medium' }], [{ ...first, owned: 0 }, {}], [{ ...first, owned: MAX_OWNED + 1 }, {}],
] as Array<[CheapTask, Partial<Assessment>]>) {
const r = ok(route(task, assessment))
expect(picks(r)).toEqual(today)
expect(r.adjusted).toBeUndefined()
}
expect(ok(route({ ...first, owned: MAX_OWNED })).shown.find(s => s.gridLabel === 'DeepSeek v4.1')!.notes).toEqual([`${CHEAP_FIRST}: ${GRID_FIRST}`])
})
it('never puts a review on a grid model, even one a person names or a local one asked for, and says why', () => {
for (const task of [{ ...first, review: true }, first]) {
const kind = task.review ? {} : { kind: 'review' }
const r = ok(route(task, kind))
expect(picks(r)).toEqual(['claude opus', 'codex default'])
expect(reasons(r)).toEqual([`${GRID_LIMITS}: ${NEVER_REVIEWS}`, `${GRID_LIMITS}: ${NEVER_REVIEWS}`, `${GRID_LIMITS}: ${NEVER_REVIEWS}`])
expect(route(task, { ...kind, model: 'kimi k3' })).toEqual({ error: 'NO_CANDIDATE', excluded: [{ title: expect.stringContaining('Kimi K3'), reason: `${GRID_LIMITS}: ${NEVER_REVIEWS}`, machine: 'remote-1' }] })
expect(route(task, { ...kind, local: true })).toMatchObject({ error: 'NO_CANDIDATE' })
}
})
it('never puts security-sensitive work on a grid model: a security kind, or high risk', () => {
for (const assessment of [{ kind: 'security' }, { kind: 'auth' }, { kind: 'privacy' }, { kind: 'secrets' }, { risk: 'high' as const }]) {
const r = ok(route(first, assessment))
expect(r.shown.some(s => s.gridModel)).toBe(false)
expect(new Set(reasons(r))).toEqual(new Set([`${GRID_LIMITS}: ${NEVER_SECURITY}`]))
expect(route(first, { ...assessment, model: 'deepseek' })).toMatchObject({ error: 'NO_CANDIDATE' })
}
})
it('fails closed on the kind: a task with no kind, or one not on the list, gets no grid model, even a named one', () => {
for (const kind of [undefined, 'ui', 'parser', 'Docs']) {
const r = ok(route(first, { kind }))
expect(r.shown.some(s => s.gridModel)).toBe(false)
expect(new Set(reasons(r))).toEqual(new Set([`${GRID_LIMITS}: ${GRID_KINDS_ONLY}`]))
expect(route(first, { kind, model: 'kimi' })).toMatchObject({ error: 'NO_CANDIDATE' })
expect(route(first, { kind, local: true })).toMatchObject({ error: 'NO_CANDIDATE' })
}
expect(GRID_KINDS_ONLY).toBe('A model on the Swarm\'s grid takes only work whose kind is docs, tests, fixtures, copy or refactor; this task names no such kind.')
// Every kind on the list may try the grid first.
for (const kind of GRID_KINDS) expect(ok(route(first, { kind })).shown.find(s => s.gridLabel === 'DeepSeek v4.1')!.notes).toEqual([`${CHEAP_FIRST}: ${GRID_FIRST}`])
})
it('steps up: a task with an earlier attempt routes without grid models, and says why', () => {
const earlier = { ...first, earlier: 'Its first attempt ran on Kimi K3.' }
const r = ok(route(earlier))
expect(picks(r)).toEqual(['claude opus', 'codex default'])
expect(new Set(reasons(r))).toEqual(new Set([`${CHEAP_FIRST}: ${FIRST_ATTEMPT_ONLY} Its first attempt ran on Kimi K3.`]))
// A person who names a grid model for the next attempt still gets it, told that the task stepped up.
const named = ok(route(earlier, { model: 'kimi' }))
expect(picks(named)).toEqual(['Kimi K3'])
expect(named.shown[0]!.notes).toEqual([`${CHEAP_FIRST}: ${FIRST_ATTEMPT_ONLY} Its first attempt ran on Kimi K3. A person named this model, so it is offered.`])
// A review is never offered one, named or not, whatever came before.
expect(route({ ...earlier, review: true }, { model: 'kimi' })).toMatchObject({ error: 'NO_CANDIDATE' })
})
it('adds no candidate: it only notes and leaves out the grid models the route itself found served and rated', () => {
// Grid candidates come only from routeCommand.ts localProbes: models a fresh reading says are served, rated from the catalog.
expect(cheapFirst(first).candidates).toBeUndefined()
expect(gridLimits(true).candidates).toBeUndefined()
expect(ok(route(first)).shown).toHaveLength(5)
})
it('leaves subscription candidates and members alone in every case', () => {
const member: Probe = { ...opus, kind: 'member', agentId: 'a1', name: 'Wren' }
for (const task of [first, { ...first, review: true }, { ...first, earlier: 'x' }]) {
const r = ok(route(task, {}, [opus, member]))
expect(r.excluded).toEqual([])
expect(r.adjusted).toBeUndefined()
}
})
})
describe('what the route reads of a task before it routes it', () => {
const group = (patch: Record<string, unknown> = {}) => ({ id: 'g1', task: { goal: 'Fix it', owns: ['docs/a.md'] }, checks: [{ id: 'c1' }], assignment: null, ...patch })
const state = (patch: Record<string, unknown> = {}) => ({ agents: [], starts: [], ...patch })
const history = { decisions: [], attempts: [] }
it('a first attempt: no earlier assignment, start, routed pick or step-up', () => {
expect(cheapTaskOf(group(), state(), history)).toEqual({ review: false, owned: 1, earlier: null })
// A route nobody acted on is no attempt; another group's work is not this one's.
expect(cheapTaskOf(group(), state({ starts: [{ request: { conversationId: 'g2', gridModel: 'k' } }] }),
{ decisions: [{ conversationId: 'g1', final: null }, { conversationId: 'g2', final: { kind: 'new' } }], attempts: [] }).earlier).toBeNull()
})
it('reads a review task, and counts owned code', () => {
expect(cheapTaskOf(group({ reviewOf: 'g0' }), state(), history).review).toBe(true)
expect(cheapTaskOf(group({ task: { owns: ['a', 'b', 7] } }), state(), history).owned).toBe(2)
expect(cheapTaskOf(group({ task: undefined }), state(), history).owned).toBe(0)
expect(cheapTaskOf(group({ task: { owns: 'a' } }), state(), history).owned).toBe(0)
})
it('finds any earlier attempt, naming the grid model it ran on when it was one', () => {
const on = (patch: Record<string, unknown>) => cheapTaskOf(group(patch.group as never), state(patch.state as never), (patch.history as never) ?? history).earlier
// The current or last assignment, or one the backend kept after work.define.
expect(on({ group: { assignment: { id: 'w1', agentId: 'a1', status: 'working' } } })).toBe('It had an earlier attempt.')
expect(on({ history: { decisions: [], attempts: [{ id: 'w0', agentId: 'a1' }] } })).toBe('It had an earlier attempt.')
// A lost attempt, a routed pick that started or was assigned, or a start (even one that failed) for it.
expect(on({ group: { lostAttempt: { id: 'w0', agentId: 'a9' } } })).toBe('It had an earlier attempt.')
expect(on({ history: { decisions: [{ conversationId: 'g1', final: { kind: 'member', agentId: 'a1' } }], attempts: [] } })).toBe('It had an earlier attempt.')
expect(on({ state: { starts: [{ request: { conversationId: 'g1' }, error: 'Codex is not signed in.' }] } })).toBe('It had an earlier attempt.')
// On the grid: its model is named, whether its start failed or its agent worked on it.
expect(on({ state: { starts: [{ request: { conversationId: 'g1', gridModel: 'moonshotai/kimi-k3' }, error: 'The grid host did not answer.' }] } }))
.toBe('Its earlier attempt ran on moonshotai/kimi-k3 on the Swarm\'s grid.')
expect(on({ group: { assignment: { id: 'w1', agentId: 'a1', status: 'working' } },
state: { agents: [{ id: 'a1', harnessId: 'h1' }], starts: [{ harnessId: 'h1', request: { conversationId: 'g0', gridModel: 'deepseek/deepseek-v4.1' } }] } }))
.toBe('Its earlier attempt ran on deepseek/deepseek-v4.1 on the Swarm\'s grid.')
// A step-up the backend recorded (after a grid attempt's failed first review) says why, in its words.
expect(on({ group: { stepUp: { attempt: 2, reason: 'Its first review said Ready no.' } } })).toBe('Its first review said Ready no.')
})
it('an earlier attempt it cannot rule out counts as one: unreadable history is no first attempt', () => {
expect(cheapTaskOf(group(), state(), null).earlier).toBe(EARLIER_UNREADABLE)
expect(cheapTaskOf(group(), state(), { decisions: null, attempts: [] }).earlier).toBe(EARLIER_UNREADABLE)
expect(cheapTaskOf(group(), state(), { decisions: [], attempts: null }).earlier).toBe(EARLIER_UNREADABLE)
// What the state shows still names the model.
expect(cheapTaskOf(group(), state({ starts: [{ request: { conversationId: 'g1', gridModel: 'k' } }] }), null).earlier).toContain('ran on k')
// Junk is read as nothing, never thrown on.
expect(cheapTaskOf({ id: 'g1' }, { agents: 'x', starts: [null, 3, { request: null }] }, { decisions: [null], attempts: [7] }).earlier).toBeNull()
})
})
describe('the maker', () => {
it('is in the route\'s makers list, the guard before cheap first', () => {
const at = CHEAP_FIRST_MAKERS.map(m => ROUTE_PLUGIN_MAKERS.indexOf(m))
expect(at.every(i => i >= 0)).toBe(true)
expect(at[0]).toBeLessThan(at[1]!)
})
const read = (conversations: unknown[] = [{ id: 'g1', task: { owns: ['docs/a.md'] } }]) => ({ tab: { state: { conversations, agents: [], starts: [] } } })
const make = (history: Promise<Record<string, unknown> | null>, tab = read()) =>
CHEAP_FIRST_MAKER.make({ conversationId: 'g1', assessment: a(), read: tab, catalog, history, request: async () => ({}) })
const gridOut = async (made: ReturnType<typeof make>) => {
const r = ok(choose([opus, kimi], catalog, a(), NOW, [], [], [], [(await made)!]))
return r.shown.some(s => s.gridModel) ? null : r.excluded.map(e => e.reason)
}
it('makes the plug-in from the route\'s own reads: a first attempt when the history shows none', async () => {
expect(await gridOut(make(Promise.resolve({ ok: true, value: { decisions: [], attempts: [] } })))).toBeNull()
expect((await make(Promise.resolve({ ok: true, value: { decisions: [], attempts: [] } })))!.name).toBe(CHEAP_FIRST)
})
it('leaves the grid out rather than skip the rule when the history fails, is junk, or is later than it waits', async () => {
const unread = [`${CHEAP_FIRST}: ${FIRST_ATTEMPT_ONLY} ${EARLIER_UNREADABLE}`]
expect(await gridOut(make(Promise.resolve(null)))).toEqual(unread)
expect(await gridOut(make(Promise.reject(new Error('socket closed'))))).toEqual(unread)
expect(await gridOut(make(Promise.resolve({ ok: true, value: { decisions: 'x' } })))).toEqual(unread)
expect(await gridOut(make(Promise.resolve({ error: 'BACKEND_UNAVAILABLE' })))).toEqual(unread)
vi.useFakeTimers()
try {
const made = make(new Promise(() => {}))
await vi.advanceTimersByTimeAsync(HISTORY_WAIT_MS)
vi.useRealTimers()
expect(await gridOut(made)).toEqual(unread)
} finally { vi.useRealTimers() }
})
it('reads a group the tab does not list, or a tab it cannot read, as a task with nothing owned', async () => {
const empty = { ok: true, value: { decisions: [], attempts: [] } }
for (const tab of [read([]), {}, { tab: { state: 'x' } }]) {
const r = ok(choose([opus, kimi], catalog, a(), NOW, [], [], [], [(await make(Promise.resolve(empty), tab as never))!]))
// Nothing owned is no small task: the grid model is not noted as the cheap first choice.
expect(r.shown.map(s => s.gridModel ?? s.model)).toEqual(['opus', 'moonshotai/kimi-k3'])
expect(r.shown.flatMap(s => s.notes)).toEqual([])
}
// A review the tab lists is read as one: cheap first leaves it to the guard.
const listed = read([{ id: 'g1', reviewOf: 'g0' }])
const guard = await GRID_LIMITS_MAKER.make({ conversationId: 'g1', assessment: a(), read: listed, catalog, history: Promise.resolve(empty), request: async () => ({}) })
const review = ok(choose([opus, kimi], catalog, a(), NOW, [], [], [], [guard!, (await make(Promise.resolve(empty), listed))!]))
expect(review.excluded.map(e => e.reason)).toEqual([`${GRID_LIMITS}: ${NEVER_REVIEWS}`])
})
it('a review or security route has no grid choice when the history is unreadable, slow, or cheap first is dropped', async () => {
const review = read([{ id: 'g1', reviewOf: 'g0', task: { owns: ['docs/a.md'] } }])
const gridIn = (plugins: Parameters<typeof choose>[7], assessment: Partial<Assessment> = {}) =>
ok(choose([opus, kimi], catalog, a(assessment), NOW, [], [], [], plugins)).shown.some(s => s.gridModel)
const input = (history: Promise<Record<string, unknown> | null>, tab = review) => ({ conversationId: 'g1', assessment: a(), read: tab, catalog, history, request: async () => ({}) })
// The guard reads nothing: it is made before a history that never answers has done anything.
let made = false
void GRID_LIMITS_MAKER.make(input(new Promise(() => {}))).then(() => { made = true })
await Promise.resolve(); await Promise.resolve()
expect(made).toBe(true)
vi.useFakeTimers()
try {
const hangs = () => new Promise<null>(() => {})
for (const [history, tab, assessment] of [[hangs, review, {}], [() => Promise.reject(new Error('down')), review, {}],
[hangs, read(), { kind: 'security' }], [() => Promise.resolve(null), read(), { risk: 'high' as const }]] as const) {
const making = makePlugins(CHEAP_FIRST_MAKERS, input(history() as Promise<null>, tab))
await vi.advanceTimersByTimeAsync(MAKER_TIMEOUT_MS)
const { plugins, warnings } = await making
expect(warnings).toEqual([])
expect(plugins.map(p => p.name)).toEqual([GRID_LIMITS, CHEAP_FIRST])
expect(gridIn(plugins, assessment)).toBe(false)
// With cheap first dropped, as the hook drops a late maker, the guard alone still leaves the grid out.
expect(gridIn([plugins[0]!], assessment)).toBe(false)
}
// A cheap first whose maker never answers is dropped after the hook's limit; the guard's plug-in is there all the same.
const hung = { name: CHEAP_FIRST, make: () => new Promise<never>(() => {}) }
const making = makePlugins([GRID_LIMITS_MAKER, hung], input(Promise.resolve(null)))
await vi.advanceTimersByTimeAsync(MAKER_TIMEOUT_MS)
const dropped = await making
expect(dropped.plugins.map(p => p.name)).toEqual([GRID_LIMITS])
expect(dropped.warnings).toHaveLength(1)
expect(gridIn(dropped.plugins)).toBe(false)
} finally { vi.useRealTimers() }
// A build with no such kind is not the guard's to touch.
expect(gridIn([gridLimits(false)])).toBe(true)
})
})