Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Keep Swarm route quota held through reroutes until work ends (#1610)

0xken154469ff86501f
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.ts
index 5ec4ff997..64cadc668 100644
--- a/backend/integration/superTabs.test.ts
+++ b/backend/integration/superTabs.test.ts
@@ -35,6 +35,18 @@ async function fixture() {
name: 'Lead', configuration: config, lead: true })).result.agentId as string
return { tab, lead }
}
+async function routedStart(tab: string, lead: string, role?: 'reviewer') {
+ const decision = { assessment: { difficulty: 'low', risk: 'low', why: 'Small task.' },
+ candidates: [{ id: 'c0', kind: 'new', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',
+ quality: 2, cost: 2, title: 'New agent', label: 'New agent' }],
+ excluded: [], floor: 1, notes: false, pick: 'c0', via: 'only', p: null, belowFloor: false, why: 'Only candidate.' }
+ const routeDecisionId = String((await run(tab, { type: 'route.record', conversationId: 'lead', decision } as Command, host, lead)).result.routeDecisionId)
+ const request = { conversationId: 'lead', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',
+ name: 'Builder', cwd: '/fixture/repo', worktree: false, ...(role ? { role } : {}),
+ requirements: { memoryBytes: 0, diskBytes: 0, maxCpuPercent: 100, tools: [], devices: [], quotaRequired: false } }
+ const startId = String((await run(tab, { type: 'agent.start', request, routeDecisionId } as Command, host, lead)).result.startId)
+ return { routeDecisionId, startId }
+}
const message = (text: string): Command => ({ type: 'message.send', conversationId: 'lead', text, replyTo: null, recipientIds: [] })
async function invite(tab: string, role: 'collaborator' | 'viewer' = 'collaborator') {
const invitation = await run(tab, { type: 'invite.create', email: peer.email, role, conversationId: 'lead' })
@@ -1744,6 +1756,9 @@ describe('Super Tabs on a real Mongo replica set', () => {
const request = { conversationId: 'lead', machineId: 'test-host', engine: 'claude', model: 'haiku', effort: 'auto', name: 'Builder', cwd: '/fixture/repo', worktree: false,
requirements: { memoryBytes: 0, diskBytes: 0, maxCpuPercent: 100, tools: [], devices: [], quotaRequired: false } }
const startId = (await run(tab, { type: 'agent.start', request, routeDecisionId } as Command, host, lead)).result.startId as string
+ // A queued start already holds its account's quota when the Lead routes again.
+ await run(tab, { type: 'route.record', conversationId: 'lead', decision } as Command, host, lead)
+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()
await expect(run(tab, { type: 'route.modelResult', decisionId: routeDecisionId, startId: 'other', model: 'sonnet', effort: 'auto' } as Command, host, lead))
.rejects.toMatchObject({ code: 'ROUTE_MISMATCH' })
const before = (await store.messages(owner, tab, 'lead', {})).messages.length
@@ -1754,6 +1769,53 @@ describe('Super Tabs on a real Mongo replica set', () => {
final: { kind: 'new', startId, model: 'sonnet', effort: 'auto', requested: { model: 'haiku', effort: 'auto' } } })
})
+ it('releases a queued start\'s route hold when that start fails', async () => {
+ const { tab, lead } = await fixture()
+ const { routeDecisionId, startId } = await routedStart(tab, lead)
+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()
+ await run(tab, { type: 'agent.startFailed', startId, error: 'Launch refused.' }, host, lead)
+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('failed')
+ })
+
+ it('releases every queued route hold when its machine is removed', async () => {
+ // The Lead runs elsewhere: a machine that hosts a project's Lead cannot leave the Swarm.
+ const tab = (await store.create(owner, randomUUID(), 'Pilot')).id
+ const leadMachineId = `lead-${randomUUID()}`
+ await db.machine.create({ data: { machineId: leadMachineId, userId: host.sub, apiKey: 'fixture-only', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
+ await db.machineShare.create({ data: { machineId: leadMachineId, ownerId: host.sub, autonomousEnv: 'prod', emails: [owner.email] } })
+ const lead = (await run(tab, { type: 'agent.add', conversationId: 'lead', machineId: leadMachineId, harnessId: randomUUID(),
+ name: 'Lead', configuration: config, lead: true })).result.agentId as string
+ const invitation = await run(tab, { type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })
+ await run(tab, { type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)
+ await run(tab, { type: 'machine.contribute', machineId: 'test-host', name: 'Host', cwd: '/fixture/repo', engines: ['codex'], maxAgents: 4 }, host)
+ const machine = (await store.hostPools(host, 'test-host')).find(pool => pool.tabId === tab)!.machine
+ await store.observeMachine(host, tab, 'test-host', { generation: machine.generation,
+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,
+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },
+ quotas: [], tools: {}, runningHarnessIds: [], stoppedHarnessIds: [] })
+ const starts = [await routedStart(tab, lead), await routedStart(tab, lead)]
+ for (const { routeDecisionId } of starts) expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()
+
+ await run(tab, { type: 'machine.remove', machineId: 'test-host' })
+
+ const state = (await store.read(owner, tab)).state
+ for (const { routeDecisionId, startId } of starts) {
+ expect(state.starts?.find(s => s.id === startId)?.error).toBe('Its machine was removed from the Swarm.')
+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('failed')
+ }
+ })
+
+ it('releases a started agent\'s route hold when the host stops it before assignment', async () => {
+ const { tab, lead } = await fixture()
+ const { routeDecisionId, startId } = await routedStart(tab, lead, 'reviewer')
+ const agentId = String((await run(tab, { type: 'agent.started', startId, harnessId: randomUUID(), configuration: config }, host, lead)).result.agentId)
+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()
+ await run(tab, { type: 'agent.release', agentId, decision: 'stop' }, host, lead)
+ const releaseId = (await store.read(owner, tab)).state.agents.find(agent => agent.id === agentId)!.releaseId!
+ await run(tab, { type: 'agent.remove', agentId, releaseId }, host, agentId)
+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('stopped')
+ })
+
it('names the failing field when a command is invalid', async () => {
const { tab } = await fixture()
const command = { type: 'agent.start', request: { conversationId: 'lead', machineId: 'test-host', engine: 'claude', name: 'Builder', cwd: '/fixture/repo', worktree: false } }
@@ -1774,6 +1836,8 @@ describe('Super Tabs on a real Mongo replica set', () => {
const routeDecisionId = (await run(tab, { type: 'route.record', conversationId: group, decision } as Command, host, lead)).result.routeDecisionId as string
const assignmentId = (await run(tab, { type: 'work.assign', conversationId: group, agentId: worker, routeDecisionId } as Command, host, lead)).result.assignmentId as string
expect(await store.routeDecision(owner, tab, routeDecisionId)).toMatchObject({ assignmentId, final: { kind: 'member', agentId: worker } })
+ await run(tab, { type: 'route.record', conversationId: group, decision } as Command, host, lead)
+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()
await store.stopped(host, tab, assignmentId, 'native-stop-receipt')
expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('stopped')
})
diff --git a/backend/src/superTabs/routeDecisions.test.ts b/backend/src/superTabs/routeDecisions.test.ts
new file mode 100644
index 000000000..783856557
--- /dev/null
+++ b/backend/src/superTabs/routeDecisions.test.ts
@@ -0,0 +1,73 @@
+import { describe, expect, it } from 'vitest'
+import { closeAssignment, writeRouteEffects } from './routeDecisions.js'
+
+type Row = Record<string, unknown> & { id: string; tabId: string; conversationId: string; outcome?: string | null }
+
+// Exercise the persistence effects against rows, including Mongo's distinction between null and an unset field.
+function memoryTx(rows: Row[]) {
+ const matches = (row: Row, where: Record<string, unknown>): boolean => Object.entries(where).every(([key, want]) => {
+ if (key === 'AND') return (want as Record<string, unknown>[]).every(part => matches(row, part))
+ if (key === 'OR') return (want as Record<string, unknown>[]).some(part => matches(row, part))
+ const actual = row[key]
+ if (want && typeof want === 'object') {
+ const filter = want as Record<string, unknown>
+ if ('isSet' in filter) return (key in row) === filter.isSet
+ if ('equals' in filter) return actual === filter.equals
+ if ('not' in filter) return actual !== filter.not
+ if ('in' in filter) return (filter.in as unknown[]).includes(actual)
+ }
+ return want === null ? actual == null : actual === want
+ })
+ const tx = { superTabRouteDecision: {
+ findMany: async ({ where }: { where: Record<string, unknown> }) => rows.filter(row => matches(row, where))
+ .map(row => ({ ...row, final: row.final ?? null, assignmentId: row.assignmentId ?? null })),
+ updateMany: async ({ where, data }: { where: Record<string, unknown>; data: Record<string, unknown> }) => {
+ const found = rows.filter(row => matches(row, where))
+ found.forEach(row => Object.assign(row, data))
+ return { count: found.length }
+ },
+ update: async ({ where, data }: { where: Record<string, unknown>; data: Record<string, unknown> }) => {
+ const row = rows.find(row => matches(row, where))!
+ Object.assign(row, data)
+ return row
+ },
+ create: async ({ data }: { data: Row }) => { rows.push({ ...data }); return data },
+ deleteMany: async () => ({ count: 0 }),
+ } }
+ return tx as unknown as Parameters<typeof writeRouteEffects>[0]
+}
+
+describe('route decision persistence', () => {
+ it('reroutes unused proposals while queued starts and assignments retain quota until their work ends', async () => {
+ const now = Date.now()
+ const final = { kind: 'new' as const, startId: 'start-1', machineId: 'machine', engine: 'claude', model: 'opus', effort: 'high' }
+ const rows: Row[] = [
+ { id: 'proposal', tabId: 'tab', conversationId: 'group', final: null, assignmentId: null, outcome: null },
+ { id: 'unset-proposal', tabId: 'tab', conversationId: 'group' },
+ { id: 'queued', tabId: 'tab', conversationId: 'group', final, assignmentId: null, outcome: null },
+ { id: 'assigned', tabId: 'tab', conversationId: 'group', final: { kind: 'member', agentId: 'agent', machineId: 'machine' }, assignmentId: 'assignment-1', outcome: null },
+ { id: 'assigned-without-final', tabId: 'tab', conversationId: 'group', final: null, assignmentId: 'assignment-2', outcome: null },
+ { id: 'finished', tabId: 'tab', conversationId: 'group', final, assignmentId: 'assignment-3', outcome: 'done' },
+ { id: 'other-group', tabId: 'tab', conversationId: 'other', final: null, assignmentId: null, outcome: null },
+ ]
+ const tx = memoryTx(rows)
+ await writeRouteEffects(tx, 'tab', [{ op: 'create', id: 'next', conversationId: 'group', record: {} as never }], now)
+ const outcome = (id: string) => rows.find(row => row.id === id)?.outcome ?? null
+ expect(outcome('proposal')).toBe('superseded')
+ expect(outcome('unset-proposal')).toBe('superseded')
+ expect(outcome('queued')).toBeNull()
+ expect(outcome('assigned')).toBeNull()
+ expect(outcome('assigned-without-final')).toBeNull()
+ expect(outcome('finished')).toBe('done')
+ expect(outcome('other-group')).toBeNull()
+ expect(rows.some(row => row.id === 'next')).toBe(true)
+
+ // A queued start later becomes an assignment; both it and the already assigned work close on completion.
+ await writeRouteEffects(tx, 'tab', [{ op: 'final', id: 'queued', final, assignmentId: 'assignment-queued' }], now + 1)
+ await closeAssignment(tx, 'tab', 'assignment-queued', 'done', now + 2)
+ await closeAssignment(tx, 'tab', 'assignment-1', 'done', now + 2)
+ expect(outcome('queued')).toBe('done')
+ expect(outcome('assigned')).toBe('done')
+ expect(outcome('next')).toBeNull()
+ })
+})
diff --git a/backend/src/superTabs/routeDecisions.ts b/backend/src/superTabs/routeDecisions.ts
index c633cfc9c..d009867dd 100644
--- a/backend/src/superTabs/routeDecisions.ts
+++ b/backend/src/superTabs/routeDecisions.ts
@@ -29,9 +29,13 @@ export async function readDecision(tx: Tx, tabId: string, id: string): Promise<R
export async function writeRouteEffects(tx: Tx, tabId: string, effects: RouteDecisionEffect[], now: number): Promise<void> {
for (const effect of effects) {
if (effect.op === 'create') {
- // One open decision per group: a newer route supersedes the open one.
- await tx.superTabRouteDecision.updateMany({ where: { tabId, conversationId: effect.conversationId, ...open },
- data: { outcome: 'superseded', outcomeAt: new Date(now) } })
+ // A newer route supersedes unused proposals; queued starts and assignments keep their quota until work ends.
+ // This Prisma/Mongo client cannot filter `isSet` on a Json field; reading it maps both unset and null to null.
+ const prior = await tx.superTabRouteDecision.findMany({ where: { tabId, conversationId: effect.conversationId, ...open },
+ select: { id: true, final: true, assignmentId: true } })
+ const unusedIds = prior.filter(row => row.final === null && row.assignmentId === null).map(row => row.id)
+ if (unusedIds.length) await tx.superTabRouteDecision.updateMany({ where: { tabId, conversationId: effect.conversationId,
+ id: { in: unusedIds }, ...open }, data: { outcome: 'superseded', outcomeAt: new Date(now) } })
await tx.superTabRouteDecision.create({ data: { id: effect.id, tabId, conversationId: effect.conversationId,
record: json(effect.record), createdAt: new Date(now) } })
await tx.superTabRouteDecision.deleteMany({ where: { tabId, createdAt: { lt: new Date(now - RETENTION_MS) } } })
@@ -62,3 +66,13 @@ export async function closeAssignment(tx: Tx, tabId: string, assignmentId: strin
}
await tx.superTabRouteDecision.updateMany({ where: { tabId, assignmentId, ...open }, data: { outcome, outcomeAt: new Date(now) } })
}
+
+/** A failed start or removed agent releases its hold only while the start has no linked assignment. */
+export async function closeUnassignedStart(tx: Tx, tabId: string, decisionId: string, startId: string, outcome: 'failed' | 'stopped', now: number): Promise<void> {
+ const row = await tx.superTabRouteDecision.findFirst({ where: { id: decisionId, tabId, ...open }, select: { final: true, assignmentId: true } })
+ const final = row?.final as { kind?: unknown; startId?: unknown } | null | undefined
+ if (row?.assignmentId != null || final?.kind !== 'new' || final.startId !== startId) return
+ await tx.superTabRouteDecision.updateMany({ where: { id: decisionId, tabId,
+ AND: [open, { OR: [{ assignmentId: null }, { assignmentId: { isSet: false } }] }] },
+ data: { outcome, outcomeAt: new Date(now) } })
+}
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index c2615e331..a28e0cea7 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -20,7 +20,7 @@ import {
} from './model.js'
import { holdNoticeText, type HoldReason } from './holdNotice.js'
import { mayDrive, TERMINAL_ANSWER_MS, TERMINAL_EXPIRY_MS, TERMINAL_PICKUP_MS, type TerminalEffect } from './terminal.js'
-import { closeAssignment, readDecision, recentRoutes, timedDecisionOf, writeRouteEffects } from './routeDecisions.js'
+import { closeAssignment, closeUnassignedStart, readDecision, recentRoutes, timedDecisionOf, writeRouteEffects } from './routeDecisions.js'
import { DEFAULT_ROUTING_POLICY, effectivePolicy, routingPolicySchema, type RoutingPolicy } from './routingPolicy.js'
import { authenticatedEmail, authorizeSwarmJoin, configureSharing, joinSwarmSchema, requireSharingOwner,
saveSharingSchema, sharingDetails, type JoinSwarm, type SaveSharing } from './sharing.js'
@@ -589,6 +589,23 @@ export class SuperTabStore {
revision, state: json(effects.state), ...indexes(effects.state),
} })
await writeRouteEffects(tx, tabId, effects.routeDecisions, context.now)
+ if (command.type === 'agent.startFailed') {
+ const start = original.starts?.find(s => s.id === command.startId)
+ if (start?.routeDecisionId) await closeUnassignedStart(tx, tabId, start.routeDecisionId, start.id, 'failed', context.now)
+ }
+ if (command.type === 'machine.remove') {
+ for (const start of original.starts ?? []) {
+ if (!start.routeDecisionId || start.request.machineId !== command.machineId || start.harnessId || start.error) continue
+ if (effects.state.starts?.find(s => s.id === start.id)?.error) {
+ await closeUnassignedStart(tx, tabId, start.routeDecisionId, start.id, 'failed', context.now)
+ }
+ }
+ }
+ if (command.type === 'agent.remove' && actor.kind === 'agent' && actor.agentId === command.agentId) {
+ const start = original.starts?.find(s => s.routeDecisionId && s.harnessId &&
+ stableId(s.request.machineId, s.harnessId) === command.agentId)
+ if (start?.routeDecisionId) await closeUnassignedStart(tx, tabId, start.routeDecisionId, start.id, 'stopped', context.now)
+ }
if (effects.terminal) await this.writeTerminal(tx, tabId, effects.terminal, context.now)
if (command.type === 'work.cancel') {
await closeAssignment(tx, tabId, command.assignmentId, 'cancelled', context.now)
diff --git a/cli/src/superTabs/routeCommand.spec.ts b/cli/src/superTabs/routeCommand.spec.ts
index 002484f82..36c7a28bb 100644
--- a/cli/src/superTabs/routeCommand.spec.ts
+++ b/cli/src/superTabs/routeCommand.spec.ts
@@ -333,6 +333,18 @@ describe('routeTask', () => {
expect(recorded(s).command.decision.candidates.every((c: { fit: { status: string; account: string } }) => c.fit.status === 'short' && c.fit.account === 'acct')).toBe(true)
expect(result.warnings).toEqual(['w', expect.stringMatching(/^No candidate is expected to cover this task; its 5-hour quota is expected to run out partway and resets in 3h\.$/)])
})
+ it('keeps a queued start in the reroute quota hold until its linked work finishes', async () => {
+ const queued = { id: 'queued', conversationId: 'g1', account: 'acct', spend: { '5-hour': 45 }, size: 'medium',
+ final: { kind: 'new', startId: 'start-1', engine: 'claude', model: 'opus' }, assignmentId: null,
+ outcome: null as string | null, createdAt: new Date().toISOString() }
+ const routeWith = async (decision: typeof queued) => {
+ const s = scoped({ history: { ok: true, value: { decisions: [decision], attempts: [] } } })
+ await routeTask(input, s, where as never, opener(), () => 'op')
+ return recorded(s).command.decision.candidates as Array<{ fit: { status: string } }>
+ }
+ expect((await routeWith(queued)).every(candidate => candidate.fit.status === 'short')).toBe(true)
+ expect((await routeWith({ ...queued, outcome: 'done' })).some(candidate => candidate.fit.status === 'fits')).toBe(true)
+ })
it('records again in the shape an older backend knows when it refuses the new fields as invalid', async () => {
const s = scoped(), base = s.getMockImplementation()!
let refused = 0