The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(swarm): a failed Builder start is retried once elsewhere or reported, never left (#1826)
0xken15ead2f8c44e47
diff --git a/backend/src/superTabs/commands.test.ts b/backend/src/superTabs/commands.test.tsindex 88d30b9ad..60e4dcba4 100644--- a/backend/src/superTabs/commands.test.ts+++ b/backend/src/superTabs/commands.test.ts@@ -1520,6 +1520,23 @@ it('marks every row the system writes in the owner\'s name, so a Lead cannot quoexpect(rows.filter(r => !r.source).map(r => r.text)).toEqual([])})+// The round-device group, 2026-10-27: the host failed a queued start, the notice went to nobody, and the group sat without a+// worker for an hour. The Lead is woken to start it elsewhere once or report it (cli superTabs/start.ts).+it('delivers a failed start\'s notice to the Lead of its group, and to nobody when the group has no Lead in it', () => {+ const f = fixture(); f.invite()+ const lead = f.add('lead', 'owner', 'lead', true)+ const group = f.run({ type: 'conversation.create', title: 'Drafts', brief: 'Save drafts.', checks: [{ description: 'Saved.', independent: false }] }, owner)+ .result.conversationId as string+ const start = (id: string, conversationId: string) => ({ id, actorKey: 'person:owner', contributorId: 'owner', membershipGeneration: f.state.people[0]!.generation,+ request: { conversationId, machineId: 'mac', name: 'Builder' }, error: null })+ f.state.starts = [start('s1', 'lead'), start('s2', group)] as never+ expect(f.run({ type: 'agent.startFailed', startId: 's1', error: 'Launch refused.' }, owner).messages)+ .toMatchObject([{ conversationId: 'lead', text: 'Could not start Builder: Launch refused.', recipients: [lead] }])+ // A Lead that left the group's members is not told there, as a removed machine's notice is not (removeMachine).+ Object.assign(f.state.conversations.find(c => c.id === group)!, { leadId: lead, agentIds: [] })+ expect(f.run({ type: 'agent.startFailed', startId: 's2', error: 'Launch refused.' }, owner).messages).toMatchObject([{ conversationId: group, recipients: [] }])+})+describe('the owner\'s rules, after review', () => {const set = (rule: Record<string, unknown>, text = 'Rule.') => ({ type: 'rule.set', rule, text } as Command)const request = (patch: Record<string, unknown> = {}) => ({ conversationId: 'lead', machineId: 'machine', engine: 'claude', model: 'opus', effort: 'high', name: 'Builder',diff --git a/backend/src/superTabs/commands.ts b/backend/src/superTabs/commands.tsindex 5f17aab94..cfdfd84b9 100644--- a/backend/src/superTabs/commands.ts+++ b/backend/src/superTabs/commands.ts@@ -1250,7 +1250,11 @@ export function applyCommand(original: SuperTabState, actor: Actor, input: Commaif (command.type === 'agent.startFailed') {if (start.harnessId) fail('ALREADY_STARTED', 'This agent already started.')start.error = command.error- message(start.request.conversationId, `Could not start ${start.request.name}: ${command.error}`, [], null, SYSTEM_ROW)+ // The Lead is woken, so it starts the work elsewhere once or reports it (cli superTabs/start.ts): written in its own+ // name for nobody, this notice left the round-device group without a worker for an hour (2026-10-27).+ const group = conversation(state, start.request.conversationId)+ const lead = group.leadId && group.agentIds.includes(group.leadId) ? group.leadId : null+ message(group.id, `Could not start ${start.request.name}: ${command.error}`, lead ? [lead] : [], null, SYSTEM_ROW)break}if (start.error) fail('START_FAILED', 'This start failed. Use a new start request.')diff --git a/cli/src/lib/swarmSkills.spec.ts b/cli/src/lib/swarmSkills.spec.tsindex bd0a33127..103599366 100644--- a/cli/src/lib/swarmSkills.spec.ts+++ b/cli/src/lib/swarmSkills.spec.ts@@ -73,6 +73,8 @@ describe('built-in Swarm skills', () => {// Last in the Default Lead text, which has no red flags: after a blank line, ending the text.const requestsLast = `\n${requests.trimEnd()}\n`const twoHours = '- A Builder or Reviewer that has been down for 2 hours (machine offline, session gone or stopped): cancel its task with the reason, route it, and assign it to another agent with what the first one left (branch, last revision, last report).\n'+ // A failed start is never left (the round-device group, 2026-10-27; superTabs/start.ts): the Lead's alone, in both modes.+ const failedStart = '- A failed start is retried elsewhere or reported, never left: when one fails, resume it; it starts once on the route\'s next machine, or tells the group and its requester.\n'it.each([['lead', 'b9254e4bb5c633a73e1b87e6ca6690915e1c3a7bb55db0d3d093e6710dd719db'],['dev', 'e78ce31a71f169d8956a2acd46e8bb43b79bd654b80b0fe3c33dbced9d2c3c75'],@@ -97,7 +99,8 @@ describe('built-in Swarm skills', () => {expect(text.split(mention + decision)).toHaveLength(role === 'lead' ? 2 : 1)expect(text.split(ask)).toHaveLength(role === 'dev' ? 2 : 1)expect(text.split(requests)).toHaveLength(role === 'lead' ? 2 : 1)- expect(sha(text.replace(twoHours, '').replace(terminal, '').replace(overlap, '').replace(local, '').replace(mention, '').replace(lost, '').replace(checkpoint, '')+ expect(text.split(failedStart)).toHaveLength(role === 'lead' ? 2 : 1)+ expect(sha(text.replace(twoHours, '').replace(terminal, '').replace(overlap, '').replace(local, '').replace(failedStart, '').replace(mention, '').replace(lost, '').replace(checkpoint, '').replace(limit, '').replace(digest, '').replace(decision, '').replace(ask, '').replace(requests, ''))).toBe(sha256)}})@@ -105,6 +108,15 @@ describe('built-in Swarm skills', () => {const text = readFileSync(prepareSwarmSkill(fixture(), { role: 'lead', key: 'session' }).file, 'utf8')expect(text).toContain(`reassign work that has not moved for 6 hours; report on-screen trust, sign-in or permission questions in chat, naming the machine owner who can answer.\n${twoHours}`)})+ it('adds the failed-start line right after each Lead skill\'s local models line, and to no other role', () => {+ const orchestration = readFileSync(prepareSwarmSkill(fixture(), { role: 'lead', key: 'session' }).file, 'utf8')+ expect(orchestration).toContain(`${local}${failedStart}- The brief is all a Builder gets.`)+ const plain = readFileSync(prepareSwarmSkill(fixture(), { role: 'lead', key: 'session', mode: 'default' }).file, 'utf8')+ expect(plain).toContain(`${localDefault}${failedStart}- The brief is all a Builder gets;`)+ for (const mode of ['orchestration', 'default'] as const) for (const role of ['dev', 'review'] as const) {+ expect(readFileSync(prepareSwarmSkill(fixture(), { role, key: 'session', mode }).file, 'utf8')).not.toContain(failedStart)+ }+ })it('adds the terminal line where each Lead skill keeps work moving, right after the silent-worker rule', () => {const orchestration = readFileSync(prepareSwarmSkill(fixture(), { role: 'lead', key: 'session' }).file, 'utf8')expect(orchestration).toContain(`naming the machine owner who can answer.\n${twoHours}${lost}${limit}${digest}${terminal}- Keep each worker supplied`)@@ -182,7 +194,8 @@ describe('built-in Swarm skills', () => {expect(text.split(mention + decision)).toHaveLength(role === 'lead' ? 2 : 1)expect(text.split(ask)).toHaveLength(role === 'dev' ? 2 : 1)expect(text.split(requestsLast)).toHaveLength(role === 'lead' ? 2 : 1)- expect(sha(text.replace(terminal, '').replace(overlap, '').replace(localDefault, '').replace(mention, '').replace(lost, '').replace(checkpoint, '')+ expect(text.split(failedStart)).toHaveLength(role === 'lead' ? 2 : 1)+ expect(sha(text.replace(terminal, '').replace(overlap, '').replace(localDefault, '').replace(failedStart, '').replace(mention, '').replace(lost, '').replace(checkpoint, '').replace(limit, '').replace(digest, '').replace(decision, '').replace(ask, '').replace(requestsLast, ''))).toBe(sha256)expect(prepared.instructions).toBe(`Use the swarm-${role} skill for this session: the team process; otherwise work your usual way. Before working, read ${JSON.stringify(prepared.file)} with your file or shell tool.`)})diff --git a/cli/src/lib/swarmSkills.ts b/cli/src/lib/swarmSkills.tsindex 10615ed59..f7809d7ee 100644--- a/cli/src/lib/swarmSkills.ts+++ b/cli/src/lib/swarmSkills.ts@@ -99,6 +99,7 @@ For large work, post the group plan and get the owner's ok before assigning; onc- Match strength to work: mechanical → cheaper; integration and judgment → standard; design or subtle risk → strongest. A review is at least as strong as the build. Don't go below the standard tier for Reviewers, or for Builders working from prose; the cheapest models take more turns and end up costing more.- Reuse an idle agent before starting a new one. Start only on the Swarm's listed machines, from the branch the work builds on.- Local models: when the Swarm serves one (a member's model, a shared API, a model shared with a member), the route may pick it for low-difficulty, well-specified work; it costs no subscription. Set "local":true in the route only when a person asks for a local model. When a person names a model, set "model" to their words ("deepseek 4.1 flash"): the route then offers that model alone. Start the pick with the gridModel and gridName the route gives, never a model or effort.+- A failed start is retried elsewhere or reported, never left: when one fails, resume it; it starts once on the route's next machine, or tells the group and its requester.- The brief is all a Builder gets. Never pass your chat history; put everything needed in the brief.- Before assigning, look at other projects' active work; if your task touches the same code, say so in the Leads conversation and agree an order first.@@ -386,6 +387,7 @@ Keep each Builder's list a few tasks deep; do the easy, self-contained parts fir- Route each group; reuse an idle agent before starting one; start only on the Swarm's machines.- With 3 or more groups waiting, run plan first: it places them all at once on the best machines and models. Then route each group in the plan's order and start the plan's pick, with overrideReason "scheduler plan" when the route picks otherwise. If plan cannot run here, route as usual.- Local models: the route may pick one for low-difficulty, well-specified work; set "local":true only when a person asks for one, set "model" to a model a person names, and start it with the route's gridModel and gridName.+- A failed start is retried elsewhere or reported, never left: when one fails, resume it; it starts once on the route's next machine, or tells the group and its requester.- The brief is all a Builder gets; never rely on chat history.- Before assigning, look at other projects' active work; if your task touches the same code, say so in the Leads conversation and agree an order first.diff --git a/cli/src/superTabs/service.ts b/cli/src/superTabs/service.tsindex e3db371a9..9eac00cd7 100644--- a/cli/src/superTabs/service.ts+++ b/cli/src/superTabs/service.ts@@ -378,6 +378,10 @@ export function superTabService(core: CoreApi, options: SuperTabServiceOptions =if (payload.action === 'startStatus' && identifier(payload.startId)) {return { ok: true, value: await backend(core, 'GET', `${path}/starts/${payload.startId}?${identity}`) }}+ // The route a Lead's failed start is tried again from (start.ts): the backend lets only who may read its group.+ if (payload.action === 'routeDecision' && identifier(payload.decisionId)) {+ return { ok: true, value: await backend(core, 'GET', `${path}/route-decisions/${payload.decisionId}?${identity}`) }+ }if (payload.action === 'status') return { ok: true, value: statusView(answerOf(await swarm()), agentId) }// A Lead's terminal read or typing (`screen`, `type`): the backend answers only the agent that asked.if (payload.action === 'terminalResult') {diff --git a/cli/src/superTabs/serviceReads.spec.ts b/cli/src/superTabs/serviceReads.spec.tsindex 93943a312..1c493d767 100644--- a/cli/src/superTabs/serviceReads.spec.ts+++ b/cli/src/superTabs/serviceReads.spec.ts@@ -62,6 +62,13 @@ describe('what each action reads of the Swarm', () => {expect(answer).not.toHaveProperty('error')expect(s.reads()).toEqual([])})+ it('reads a routing decision by id for its Lead, from the backend that checks who may read it', async () => {+ const s = swarm((_method, path) => path === `/api/super-tabs/${T}/route-decisions/d-1?asAgentId=${FIXTURE.leadId}`+ ? ok({ id: 'd-1', record: { pick: 'c0' } }) : undefined)+ expect(await s.ask({ action: 'routeDecision', tabId: T, agentId: FIXTURE.leadId, decisionId: 'd-1' }, owner))+ .toEqual({ ok: true, value: { id: 'd-1', record: { pick: 'c0' } } })+ expect(s.reads()).toEqual([])+ })it('reads the people only for a post that tags someone', async () => {const s = swarm()await s.ask({ action: 'command', tabId: T, agentId: FIXTURE.leadId, operationId: 'op-3',diff --git a/cli/src/superTabs/start.spec.ts b/cli/src/superTabs/start.spec.tsindex 2d3975398..d8bfafb03 100644--- a/cli/src/superTabs/start.spec.ts+++ b/cli/src/superTabs/start.spec.ts@@ -2,6 +2,8 @@ import { describe, expect, it, vi } from 'vitest'import { inspectCapacity, startAgent } from './start.js'import { buildEngineCommandArgv } from '../lib/engineLaunch.js'const where = { port: 14001, machineId: 'lead-machine', dataDir: '/fixture' }+/** A queued start is watched before the Lead moves on (startAgent); these tests read the queued answer at once. */+const noPause = async () => {}function fixture(options: { state?: Record<string, unknown> | Error; githubApp?: boolean; refuse?: string; loseCancel?: boolean; loseFailure?: boolean; launchModel?: string[]; acceptedLaunchModel?: { model: string | null; effort: string | null }; refuseModel?: string; frame?: string; guest?: boolean; identity?: { sessionId: string; createdAt: string } } = {}) {const start = { creationId: 'super-native-receipt', error: null as string | null, request: { machineId: 'target', conversationId: 'group', engine: 'claude', name: 'Builder', cwd: '/repo', worktree: true,requirements: { memoryBytes: 100, diskBytes: 200, maxCpuPercent: 80, tools: ['git'], devices: [], quotaRequired: false } }, dispatched: false }@@ -122,7 +124,7 @@ describe('Swarm permission mode', () => {})const input = { action: 'start', tabId: 'tab', operationId: 'same-op', request: f.start.request }await expect(startAgent(input, f.scoped, where, f.open)).rejects.toThrow('Accepted reply lost')- expect(await startAgent(input, f.scoped, where, f.open)).toMatchObject({ ok: true, queued: true, startId: 'start' })+ expect(await startAgent(input, f.scoped, where, f.open, noPause)).toMatchObject({ ok: true, queued: true, startId: 'start' })expect(accepted).toBe(1)expect(f.open).not.toHaveBeenCalled()})@@ -138,7 +140,7 @@ describe('Swarm permission mode', () => {const f = fixture()Object.assign(f.start, { poolGeneration: 'pool', ...(settled ? { harnessId: 'existing', poolSettled: true } : {}) })f.scoped.mockResolvedValueOnce({ error: 'OPERATION_REUSED', detail: 'Use a new operation ID for a different request.' })- const result = await startAgent({ action: 'start', tabId: 'tab', operationId: 'same-op', request: f.start.request }, f.scoped, where, f.open)+ const result = await startAgent({ action: 'start', tabId: 'tab', operationId: 'same-op', request: f.start.request }, f.scoped, where, f.open, noPause)expect(result).toMatchObject({ ok: true, ...(settled ? { harnessId: 'existing' } : { queued: true }) })expect(result).not.toHaveProperty('permissionWarning')const attempts = f.scoped.mock.calls.map(c => c[0]).filter(c => (c.command as any)?.type === 'agent.start')@@ -157,7 +159,7 @@ describe('Swarm permission mode', () => {}return scoped(payload)})- const result = await startAgent({ action: 'start', tabId: 'tab', operationId: 'same-op', request: f.start.request }, f.scoped, where, f.open)+ const result = await startAgent({ action: 'start', tabId: 'tab', operationId: 'same-op', request: f.start.request }, f.scoped, where, f.open, noPause)expect(result).toMatchObject({ ok: true, ...(settled ? { harnessId: 'existing' } : { queued: true }), permissionWarning: expect.stringContaining("machine's default") })const attempts = f.scoped.mock.calls.map(c => c[0]).filter(c => (c.command as any)?.type === 'agent.start')expect(attempts).toHaveLength(2)@@ -887,3 +889,222 @@ describe('Swarm starts for another person\'s project', () => {}})})++describe('a Lead\'s start that fails before its agent exists', () => {+ const requirements = { memoryBytes: 100, diskBytes: 200, maxCpuPercent: 80, tools: ['git'], devices: ['round'], quotaRequired: false }+ const candidate = (id: string, machineId: string, kind = 'new') => ({ id, kind, ...(kind === 'member' ? { agentId: `agent-${machineId}` } : {}), machineId,+ engine: 'claude', model: 'opus', effort: 'high', quality: 3, cost: 10, title: `Builder · Claude opus high · ${machineId}`, label: machineId })+ const decision = (candidates: ReturnType<typeof candidate>[], override: unknown = null) => ({ id: 'decision-1', conversationId: 'group', outcome: 'failed', final: null, override,+ record: { assessment: { difficulty: 'medium', risk: 'low', why: 'Queued Lead routing.', size: 'medium' }, candidates, excluded: [],+ floor: 2, notes: false, pick: candidates[0]!.id, via: 'order', p: null, belowFloor: false, why: 'Policy order.' } })+ const swarm = (rules: unknown[] = []) => ({+ conversations: [{ id: 'group', projectId: 'project', leadId: 'lead', requesterId: 'person-1' }],+ projects: [{ id: 'project', ownerId: 'owner-1', repository: { machineId: 'repo-mac', cwd: '/repo' } }],+ machines: [{ machineId: 'mac-one', name: 'Mac One', scope: 'swarm', cwd: '/one/swarm' }, { machineId: 'mac-two', name: 'Mac Two', scope: 'swarm', cwd: '/two/swarm/' }],+ rules,+ })+ const TOKEN = 'The Harness GitHub App token is not in the form GitHub gives.'+ /** A backend with contributed machines whose hosts fail a start in preparation, as the round-device group saw. */+ function backend(options: { candidates?: ReturnType<typeof candidate>[]; rules?: unknown[]; fails?: string[]; override?: unknown+ /** What happens to the retry's start: the backend refuses it, or its reply is lost after or before the backend recorded it. */+ retry?: 'refused' | 'lost' | 'lostUnrecorded'+ /** Machines off the contributed list: the Lead's own start launches there, and this one refuses for its owner's sharing. */+ direct?: string[] } = {}) {+ const starts = new Map<string, Record<string, unknown>>()+ const fails = new Set(options.fails ?? ['mac-one'])+ const commands: Record<string, unknown>[] = []+ const scoped = vi.fn(async (payload: Record<string, unknown>): Promise<Record<string, unknown>> => {+ if (payload.action === 'startStatus') {+ const start = starts.get(String(payload.startId))+ if (!start) return { error: 'NOT_FOUND', detail: 'This agent start is unavailable.' }+ // The host readies a queued start on its next pass: a machine whose app token cannot be had fails it there.+ const request = start.request as Record<string, unknown>+ if (start.read && fails.has(String(request.machineId))) start.error = TOKEN+ start.read = true+ return { ok: true, value: structuredClone(start) }+ }+ if (payload.action === 'routeDecision') return { ok: true, value: decision(options.candidates ?? [candidate('c0', 'mac-one'), candidate('c1', 'mac-one', 'member'), candidate('c2', 'mac-two')], options.override) }+ if (payload.action === 'status') return { ok: true, value: { state: swarm(options.rules) } }+ const cmd = payload.command as Record<string, unknown>+ commands.push({ operationId: payload.operationId, ...cmd })+ if (cmd.type === 'agent.start') {+ const retry = payload.operationId !== 'first-start' ? options.retry : undefined+ if (retry === 'refused') return { error: 'MACHINE_NOT_LISTED', detail: 'Add this machine to the Swarm first.' }+ if (retry === 'lostUnrecorded') { options.retry = undefined; throw new Error('The reply was lost.') }+ const machineId = String((cmd.request as Record<string, unknown>).machineId)+ starts.set(String(payload.operationId), { id: payload.operationId, creationId: `super-${String(payload.operationId)}`,+ ...(options.direct?.includes(machineId) ? {} : { poolGeneration: 'pool' }),+ routeDecisionId: cmd.routeDecisionId, request: cmd.request, harnessId: null, dispatched: false, error: null })+ if (retry === 'lost') { options.retry = undefined; throw new Error('The reply was lost.') }+ return { ok: true, value: { result: { startId: payload.operationId } } }+ }+ if (cmd.type === 'agent.startFailed') starts.get(String(cmd.startId))!.error = cmd.error+ if (cmd.type === 'route.record') return { ok: true, value: { result: { routeDecisionId: payload.operationId } } }+ return { ok: true, value: { result: { messageId: payload.operationId } } }+ })+ const daemon = { request: vi.fn(async (type: string): Promise<Record<string, unknown>> => type === 'agent_create_status'+ ? { creationId: `super-${derived}`, state: 'missing' } : type === 'super_tabs' ? { error: 'GUEST_NOT_ALLOWED' } : {}), close: vi.fn() }+ let derived = ''+ const open = vi.fn(async (_where: unknown, machineId?: string) => {+ if (!options.direct?.includes(String(machineId))) throw new Error('A queued start is dispatched by its host, not the Lead.')+ derived = [...starts.keys()].at(-1)!+ return daemon as never+ })+ const pause = vi.fn(async () => {})+ const lead = (input: Record<string, unknown>) => startAgent(input, scoped, where, open, pause)+ const first = { action: 'start', tabId: 'tab', operationId: 'first-start', request: { conversationId: 'group', machineId: 'mac-one', engine: 'claude',+ model: 'opus', effort: 'high', name: 'Builder', cwd: '/one/swarm/project', worktree: true, requirements, routeDecisionId: 'decision-1' } }+ return { starts, commands, scoped, pause, lead, first, of: (type: string) => commands.filter(c => c.type === type) }+ }++ it('starts once on the route\'s next machine when the first machine fails it in preparation', async () => {+ const b = backend()+ const result = await b.lead(b.first)+ const [, retry] = b.of('agent.start')+ const [rerouted] = b.of('route.record')+ // The retry is routed again from the stored decision: without the failed machine, picking the next new agent.+ expect(rerouted).toMatchObject({ conversationId: 'group', decision: { pick: 'c2', via: 'order', candidates: [candidate('c2', 'mac-two')],+ excluded: [{ title: 'Builder · Claude opus high · mac-one', reason: 'Its machine could not start this task.' }, { title: 'Builder · Claude opus high · mac-one', reason: 'Its machine could not start this task.' }] } })+ expect(retry).toMatchObject({ routeDecisionId: rerouted!.operationId, request: { machineId: 'mac-two', cwd: '/two/swarm/project', engine: 'claude',+ model: 'opus', effort: 'high', name: 'Builder', worktree: true, requirements, conversationId: 'group' } })+ expect(b.of('agent.start')).toHaveLength(2)+ expect(result).toMatchObject({ ok: true, queued: true, startId: retry!.operationId, retriedFrom: 'first-start' })+ expect(b.of('message.send')).toEqual([])+ })++ it('counts a Mac\'s Harness Team account as that Mac, and leaves the failed machine\'s own settings behind', async () => {+ const b = backend({ fails: ['team-one'] })+ const answer = b.scoped.getMockImplementation()!+ b.scoped.mockImplementation(async payload => {+ if (payload.action !== 'status') return answer(payload)+ const state = swarm()+ state.machines.push({ machineId: 'team-one', name: 'Mac One (Team)', scope: 'swarm', cwd: '/team/swarm', hostMachineId: 'mac-one' } as never)+ return { ok: true, value: { state } }+ })+ // The backend moved the Lead's start on Mac One to its team account (machines.ts teamStartRequest).+ const result = await b.lead({ ...b.first, request: { ...b.first.request, machineId: 'team-one', codexHome: '/one/.codex' } })+ expect(b.of('route.record')[0]).toMatchObject({ decision: { pick: 'c2', candidates: [candidate('c2', 'mac-two')] } })+ expect(b.of('agent.start')[1]!.request).toMatchObject({ machineId: 'mac-two' })+ expect(b.of('agent.start')[1]!.request).not.toHaveProperty('codexHome')+ expect(result).toMatchObject({ queued: true, retriedFrom: 'first-start' })+ })+ it('reports when the backend refuses the retry\'s start after its route was recorded', async () => {+ const b = backend({ retry: 'refused' })+ expect(await b.lead(b.first)).toMatchObject({ error: 'START_BLOCKED', startId: expect.stringMatching(/^retry-/) })+ expect(b.of('route.record')).toHaveLength(1)+ expect(b.of('agent.start')).toHaveLength(2)+ expect(b.of('message.send')).toMatchObject([{ text: expect.stringMatching(/^Builder could not start on Mac Two, and no other machine/) }])+ })+ it('reconciles the retry\'s own start when its reply is lost after the backend recorded it, never making another', async () => {+ const b = backend({ retry: 'lost' })+ expect(await b.lead(b.first)).toMatchObject({ ok: true, queued: true, startId: b.of('agent.start')[1]!.operationId, retriedFrom: 'first-start' })+ expect(b.of('agent.start')).toHaveLength(2)+ expect(b.of('message.send')).toEqual([])+ })+ it('says the retry is unconfirmed when its reply is lost before it was recorded, and a resume sends the same one', async () => {+ const b = backend({ retry: 'lostUnrecorded' })+ expect(await b.lead(b.first)).toMatchObject({ error: 'START_UNCERTAIN', retriedFrom: 'first-start' })+ expect(b.of('message.send')).toEqual([])+ expect(await b.lead({ action: 'resume', tabId: 'tab', startId: 'first-start' })).toMatchObject({ ok: true, queued: true, retriedFrom: 'first-start' })+ const retries = b.of('agent.start').slice(1)+ expect(retries).toHaveLength(2)+ expect(retries[1]!.operationId).toBe(retries[0]!.operationId)+ })+ it('fails and reports a retry its machine refuses before launching it', async () => {+ const b = backend({ direct: ['mac-two'] })+ expect(await b.lead(b.first)).toMatchObject({ error: 'START_BLOCKED' })+ const retryId = b.of('agent.start')[1]!.operationId+ expect(b.of('agent.startFailed')).toMatchObject([{ startId: retryId, error: expect.stringContaining('machine owner') }])+ expect(b.of('message.send')).toHaveLength(1)+ })+ it('never moves to the candidate a Lead\'s override moved the task away from, and keeps what it chose to run', async () => {+ const overridden = { by: 'lead', actorKey: 'agent:lead', reason: 'Spread off the Mac Studio.' }+ // The policy put the Mac Studio first; the Lead started opus on Mac One instead, which failed.+ const studio = { ...candidate('c0', 'mac-studio'), model: 'sonnet', effort: 'medium' }+ const b = backend({ override: overridden, candidates: [studio, { ...candidate('c1', 'mac-two'), model: 'sonnet', effort: 'medium' }, candidate('c2', 'mac-one'), candidate('c3', 'mac-two')] })+ await b.lead(b.first)+ expect(b.of('route.record')[0]).toMatchObject({ decision: { pick: 'c3' } })+ expect(b.of('agent.start')[1]!.request).toMatchObject({ machineId: 'mac-two', model: 'opus', effort: 'high' })+ const none = backend({ override: overridden, candidates: [studio, candidate('c2', 'mac-one')] })+ expect(await none.lead(none.first)).toMatchObject({ error: 'START_BLOCKED' })+ expect(none.of('route.record')).toEqual([])+ })+ it('reports instead of moving a task a person chose the machine for', async () => {+ const b = backend({ override: { by: 'person', actorKey: 'person:owner', candidateId: 'c0', reason: 'Mine.' } })+ expect(await b.lead(b.first)).toMatchObject({ error: 'START_BLOCKED' })+ expect(b.of('route.record')).toEqual([])+ })+ it('a resumed failed start is retried the same way, once, under the same operations', async () => {+ const b = backend()+ await b.lead(b.first)+ const before = b.commands.length+ // The Lead, woken by the failure, resumes the first start: the retry already exists, so nothing new is sent.+ expect(await b.lead({ action: 'resume', tabId: 'tab', startId: 'first-start' })).toMatchObject({ queued: true, retriedFrom: 'first-start' })+ expect(b.commands.slice(before).map(c => c.operationId)).toEqual(b.commands.slice(before - 2, before).map(c => c.operationId))+ })++ it('with no other machine, tells the group and its requester that the task has no worker, without the raw error', async () => {+ const b = backend({ candidates: [candidate('c0', 'mac-one'), candidate('c1', 'mac-one', 'member')] })+ const result = await b.lead(b.first)+ expect(b.of('agent.start')).toHaveLength(1)+ const [notice] = b.of('message.send')+ expect(notice).toMatchObject({ conversationId: 'group', mentions: ['person-1'], recipientIds: [],+ text: 'Builder could not start on Mac One, and no other machine allowed for this task could take it. This task has no worker until that machine is fixed or another one is added.' })+ expect(String(notice!.text)).not.toContain('token')+ expect(result).toMatchObject({ error: 'START_BLOCKED', startId: 'first-start', detail: expect.stringContaining(TOKEN) })+ })++ it('reports instead of trying a third machine when the retry fails as well', async () => {+ const b = backend({ fails: ['mac-one', 'mac-two'] })+ const result = await b.lead(b.first)+ expect(b.of('agent.start')).toHaveLength(2)+ expect(b.of('message.send')).toHaveLength(1)+ expect(b.of('message.send')[0]!.text).toMatch(/^Builder could not start on Mac Two, and no other machine/)+ expect(result).toMatchObject({ error: 'START_BLOCKED' })+ // Resuming the failed retry later reports the same notice again, never a further start.+ const retryId = String(b.of('agent.start')[1]!.operationId)+ expect(await b.lead({ action: 'resume', tabId: 'tab', startId: retryId })).toMatchObject({ error: 'START_BLOCKED' })+ expect(b.of('agent.start')).toHaveLength(2)+ expect(new Set(b.of('message.send').map(c => c.operationId)).size).toBe(1)+ })++ it('keeps a task bound by a hard rule this client does not know where it is, and reports it', async () => {+ const b = backend({ rules: [{ id: 'rule-1', projectId: 'project', rule: { kind: 'machines', machineIds: ['mac-one', 'mac-two'] } },+ { id: 'rule-2', projectId: 'project', rule: { kind: 'requiresMachine', machineId: 'mac-one' } }] })+ expect(await b.lead(b.first)).toMatchObject({ error: 'START_BLOCKED' })+ expect(b.of('route.record')).toEqual([])+ expect(b.of('agent.start')).toHaveLength(1)+ })++ it('a rule of another project does not hold the task back', async () => {+ const b = backend({ rules: [{ id: 'rule-2', projectId: 'other', rule: { kind: 'requiresMachine', machineId: 'mac-one' } }] })+ expect(await b.lead(b.first)).toMatchObject({ ok: true, queued: true })+ })++ it('reports when the backend refuses the new pick, for a hard rule it knows', async () => {+ const b = backend()+ const answer = b.scoped.getMockImplementation()!+ b.scoped.mockImplementation(async payload => (payload.command as Record<string, unknown> | undefined)?.type === 'route.record'+ ? { error: 'RULE_BROKEN', detail: 'Only these machines.' } : answer(payload))+ expect(await b.lead(b.first)).toMatchObject({ error: 'START_BLOCKED' })+ expect(b.of('agent.start')).toHaveLength(1)+ })++ it('returns a start still queued after the watch as before, without retrying it', async () => {+ const b = backend({ fails: [] })+ expect(await b.lead(b.first)).toMatchObject({ ok: true, queued: true, startId: 'first-start' })+ expect(b.pause).toHaveBeenCalledTimes(10)+ expect(b.of('route.record')).toEqual([])+ })++ it('never retries on the host side, nor a start that was not routed', async () => {+ const b = backend()+ await b.lead(b.first)+ const hosted = await startAgent({ action: 'resume', tabId: 'tab', startId: 'first-start', hostDispatch: true }, b.scoped, where, vi.fn(), b.pause)+ expect(hosted).toMatchObject({ error: 'START_FAILED' })+ const unrouted = backend()+ const { routeDecisionId: _id, ...request } = unrouted.first.request+ expect(await unrouted.lead({ ...unrouted.first, request })).toMatchObject({ error: 'START_FAILED', detail: TOKEN })+ expect(unrouted.of('route.record')).toEqual([])+ })+})diff --git a/cli/src/superTabs/start.ts b/cli/src/superTabs/start.tsindex 924c46df4..75b66a1f9 100644--- a/cli/src/superTabs/start.ts+++ b/cli/src/superTabs/start.ts@@ -1,6 +1,6 @@import { encodeRuntimeProfile } from '../engines/kit/runtime.js'import { parseRuntimeProfile } from '../lib/runtimeProfileWire.js'-import { randomUUID } from 'node:crypto'+import { createHash, randomUUID } from 'node:crypto'import { openDaemon, type Daemon, type Where } from '../agentClient/daemon.js'import { admission, quotaOf, type Requirements } from './capacity.js'import { groupSkillMode, identifier, object } from './host.js'@@ -28,7 +28,9 @@ const MODEL_SWITCH_WAIT_MS = 60_000const MODEL_SWITCH_RETRY_MS = 1_000const CANCELLED_START = 'The machine cancelled this request before starting an agent. Review its settings and create a new start request.'function requireSuccess(reply: Json): Json {- if (reply.error || reply.ok !== true) throw new Error(String(reply.detail ?? reply.error ?? 'This Super Tab request did not succeed.'))+ // The code says whether the backend answered (a refusal) or the reply was lost on the way (a transport code, or none).+ if (reply.error || reply.ok !== true) throw Object.assign(new Error(String(reply.detail ?? reply.error ?? 'This Super Tab request did not succeed.')),+ { code: typeof reply.error === 'string' ? reply.error : null })return object(reply.value)}export async function inspectCapacity(target: Daemon, request: Json) {@@ -86,8 +88,131 @@ const launchesWithModel = (capacity: Json, engine: unknown) => Array.isArray(cap/** Starts through the ordinary native launch protocol, with its durable receipt.* This CLI owns no machine credential and does not change legacy spawn settings.- * Every invocation resumes the backend intent before it may open a native pane. */-export async function startAgent(input: Json, scoped: ScopedRequest, where: Where, open: typeof openDaemon = openDaemon): Promise<Json> {+ * Every invocation resumes the backend intent before it may open a native pane.+ *+ * A Lead's routed start that fails before its agent exists is never left there (the round-device group, 2026-10-27: a+ * contributed host failed it in preparation, the Lead had been told only "queued", and the group sat without a worker).+ * A queued start is watched for a short while; a failed one starts once on the route's next machine, or the group and its+ * requester are told it has no worker. Resuming the failed start later does the same, under the same operations. */+export async function startAgent(input: Json, scoped: ScopedRequest, where: Where, open: typeof openDaemon = openDaemon,+ pause: (ms: number) => Promise<void> = ms => new Promise(resolve => setTimeout(resolve, ms))): Promise<Json> {+ let result = await startOnce(input, scoped, where, open)+ // The host settles its own queued starts; the Lead's route and retry are the Lead's alone.+ if (input.hostDispatch === true) return result+ if (result.queued === true && input.action === 'start') result = await watchQueued(result, scoped, pause)+ if (result.error !== 'START_FAILED' || !identifier(result.startId)) return result+ const failed = requireSuccess(await scoped({ action: 'startStatus', startId: result.startId }))+ // An agent that joined is the group's worker, and a start nobody routed has no route to try again.+ if (failed.harnessId || !failed.error || !identifier(failed.routeDecisionId)) return result+ return startElsewhere(input, failed, scoped, where, open, pause)+}++const WATCH_READS = 10, WATCH_EVERY_MS = 2_000+/** A contributed machine's host readies a queued start on its next pass (5 s apart, machines.ts): the Lead reads its+ * receipt for about 20 s, so a start its host fails in preparation is known before the Lead moves on. Later failures+ * are delivered to the Lead (backend agent.startFailed), which resumes the start. */+async function watchQueued(queued: Json, scoped: ScopedRequest, pause: (ms: number) => Promise<void>): Promise<Json> {+ for (let read = 0; read < WATCH_READS; read++) {+ await pause(WATCH_EVERY_MS)+ const start = requireSuccess(await scoped({ action: 'startStatus', startId: queued.startId }))+ if (start.error && !start.harnessId) return { error: 'START_FAILED', startId: queued.startId, detail: start.error }+ if (start.dispatched || start.harnessId) break+ }+ return queued+}++/** Hard rules this client reads before it moves a task. A rule of any other kind may bind the task to its machine (a+ * machine or device requirement added later), so the task then stays where it is and is reported instead. */+const KNOWN_RULES = new Set(['forbidModel', 'machines', 'minEffort', 'preferEngine', 'preferIdle'])+const RETRY = /^retry-[0-9a-f]{24}$/+/** Answers that are not the request's own (routeCommand.ts): the backend may still have recorded it. */+const TRANSPORT = new Set(['AGENT_UNAVAILABLE', 'SUPER_TAB_UNAVAILABLE', 'BACKEND_UNAVAILABLE', 'BACKEND_TIMEOUT', 'BACKEND_UNREACHABLE'])+/** A start in these states may still launch, or has already been settled by the start itself (superTabs/machines.ts). */+const UNSETTLED = new Set(['START_UNCERTAIN', 'START_PENDING', 'START_REFUSED'])+const derived = (prefix: string, startId: string) => `${prefix}-${createHash('sha256').update(startId).digest('hex').slice(0, 24)}`++/** One more start for a routed start that failed before its agent existed: the stored route without the failed machine,+ * recorded again so the backend checks the new pick against the owner's hard rules, then started there. A retry is never+ * retried; with no machine left, the group is told. Every operation is named after the failed start, so a Lead that+ * resumes it again replays the same ones. */+async function startElsewhere(input: Json, failed: Json, scoped: ScopedRequest, where: Where, open: typeof openDaemon,+ pause: (ms: number) => Promise<void>): Promise<Json> {+ const startId = String(failed.id ?? input.startId ?? input.operationId), request = object(failed.request)+ const state = object(object(requireSuccess(await scoped({ action: 'status' }))).state)+ const group = (Array.isArray(state.conversations) ? state.conversations.map(object) : []).find(g => g.id === request.conversationId)+ const machines = Array.isArray(state.machines) ? state.machines.map(object) : []+ const project = (Array.isArray(state.projects) ? state.projects.map(object) : []).find(p => p.id === group?.projectId)+ const machineName = (id: unknown) => text(machines.find(m => m.machineId === id)?.name) ?? 'its machine'+ const blocked = async (on: Json): Promise<Json> => {+ const requester = identifier(group?.requesterId) ? [group.requesterId] : []+ // People read why in the failure row the backend wrote; this one says only what it means, never the raw error.+ requireSuccess(await scoped({ action: 'command', operationId: derived('start-blocked', String(on.id)), command: { type: 'message.send',+ conversationId: request.conversationId, replyTo: null, recipientIds: [], ...(requester.length ? { mentions: requester } : {}),+ text: `${String(request.name)} could not start on ${machineName(object(on.request).machineId)}, and no other machine allowed for this task could take it. `+ + 'This task has no worker until that machine is fixed or another one is added.' } }))+ return { error: 'START_BLOCKED', startId: on.id, detail: `${String(on.error)} No other machine allowed for this task could start it; the group was told.` }+ }+ if (RETRY.test(startId)) return blocked(failed)+ const rules = (Array.isArray(state.rules) ? state.rules.map(object) : []).filter(r => !r.projectId || r.projectId === group?.projectId)+ if (rules.some(r => !KNOWN_RULES.has(String(object(r.rule).kind)))) return blocked(failed)+ const decision = object(requireSuccess(await scoped({ action: 'routeDecision', decisionId: failed.routeDecisionId })))+ const record = object(decision.record), repository = object(project?.repository)+ /** Where the project's folder is on a machine: as the route says it (routeCommand.ts); unknown elsewhere. */+ const folder = (machineId: unknown) => {+ if (repository.machineId === machineId && text(repository.cwd)) return text(repository.cwd)+ const pool = machines.find(m => m.machineId === machineId)+ return pool?.scope === 'swarm' && text(pool.cwd) && identifier(project?.id) ? `${String(pool.cwd).replace(/\/+$/, '')}/${project.id}` : null+ }+ // A Mac's Harness Team account is that Mac (backend machines.ts teamStartRequest): neither runs the retry.+ const failedOn = new Set([request.machineId, machines.find(m => m.machineId === request.machineId)?.hostMachineId].filter(identifier))+ const all = Array.isArray(record.candidates) ? record.candidates.map(object) : []+ const others = all.filter(c => !failedOn.has(String(c.machineId)))+ // A person who chose the machine chooses another; a Lead's override keeps holding: never the pick it moved the task away+ // from, and the engine and model it chose to run.+ const override = object(decision.override)+ if (override.by === 'person') return blocked(failed)+ const avoided = override.by === 'lead' ? all.find(c => c.id === record.pick) : undefined+ const chosen = (c: Json) => !avoided || (c.machineId !== avoided.machineId && c.engine === request.engine+ && (c.gridModel ?? null) === (request.gridModel ?? null) && (c.gridName ?? null) === (request.gridName ?? null)+ && (c.gridModel !== undefined || (c.model === (request.model ?? 'default') && c.effort === (request.effort ?? 'default'))))+ const pick = others.find(c => c.kind === 'new' && folder(c.machineId) && chosen(c))+ if (!pick) return blocked(failed)+ const reason = 'Its machine could not start this task.'+ const why = `Retry: ${String(request.name)} could not start on ${machineName(request.machineId)}.`.slice(0, 300)+ const rerouted = await scoped({ action: 'command', operationId: derived('reroute', startId), command: { type: 'route.record',+ conversationId: request.conversationId, decision: { ...record, assessment: { ...object(record.assessment), why }, candidates: others,+ excluded: [...all.filter(c => failedOn.has(String(c.machineId))).map(c => ({ title: c.title, reason })),+ ...(Array.isArray(record.excluded) ? record.excluded : [])].slice(0, 200),+ pick: pick.id, via: 'order', fallback: undefined, p: null, belowFloor: false, why } } })+ // The backend refuses a pick that breaks a hard rule or cannot take a teammate's work: then nothing else may run it.+ const routeDecisionId = rerouted.ok === true ? object(object(rerouted.value).result).routeDecisionId : null+ if (!identifier(routeDecisionId)) return blocked(failed)+ // What named the failed machine's own setup stays with it; the pick says what runs on the new one.+ const { permissionMode: _mode, model: _model, effort: _effort, gridModel: _grid, gridName: _name, codexHome: _home, ...kept } = request+ const retryId = derived('retry', startId), retryRequest = { ...kept, machineId: pick.machineId, engine: pick.engine, cwd: folder(pick.machineId),+ ...(pick.gridModel ? { gridModel: pick.gridModel, gridName: pick.gridName } : { model: pick.model, effort: pick.effort }), routeDecisionId }+ const refused = (error: unknown) => blocked({ id: retryId, request: retryRequest, error })+ const uncertain = { error: 'START_UNCERTAIN', startId: retryId, retriedFrom: startId,+ detail: 'The retry was sent but not confirmed. Resume the failed start to finish it; it sends the same retry again, never another.' }+ let retry: Json+ try { retry = await startOnce({ action: 'start', tabId: input.tabId, operationId: retryId, request: retryRequest }, scoped, where, open) } catch (error) {+ // A lost reply may follow a recorded start: its own receipt, read again, says which. Only a backend that answered with+ // a refusal and holds no such start refused it.+ const code = object(error).code, receipt = await scoped({ action: 'startStatus', startId: retryId }).catch((): Json => ({}))+ if (receipt.ok !== true) return receipt.error === 'NOT_FOUND' && typeof code === 'string' && !TRANSPORT.has(code) ? refused((error as Error).message) : uncertain+ try { retry = await startOnce({ action: 'resume', tabId: input.tabId, startId: retryId }, scoped, where, open) } catch { return uncertain }+ }+ const settled = retry.queued === true ? await watchQueued(retry, scoped, pause) : retry+ if (settled.error === undefined || UNSETTLED.has(String(settled.error))) return { ...settled, retriedFrom: startId }+ const receipt = requireSuccess(await scoped({ action: 'startStatus', startId: retryId }))+ if (receipt.harnessId) return { ...settled, retriedFrom: startId }+ // Refused before it was dispatched (capacity, the machine's sharing): fail it, so it holds no slot and its notice is written.+ if (!receipt.error) requireSuccess(await scoped({ action: 'command', operationId: derived('retry-refused', retryId),+ command: { type: 'agent.startFailed', startId: retryId, error: String(settled.detail ?? settled.error) } }))+ return refused(receipt.error ?? settled.detail ?? settled.error)+}++async function startOnce(input: Json, scoped: ScopedRequest, where: Where, open: typeof openDaemon): Promise<Json> {let startId = input.startIdlet routed: unknownlet permissionWarning: string | undefined