The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Keep Swarm route quota held through reroutes until work ends (#1610)
0xken154469ff86501f
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.tsindex 5ec4ff997..64cadc668 100644--- a/backend/integration/superTabs.test.ts+++ b/backend/integration/superTabs.test.ts@@ -35,6 +35,18 @@ async function fixture() {name: 'Lead', configuration: config, lead: true })).result.agentId as stringreturn { tab, lead }}+async function routedStart(tab: string, lead: string, role?: 'reviewer') {+ const decision = { assessment: { difficulty: 'low', risk: 'low', why: 'Small task.' },+ candidates: [{ id: 'c0', kind: 'new', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',+ quality: 2, cost: 2, title: 'New agent', label: 'New agent' }],+ excluded: [], floor: 1, notes: false, pick: 'c0', via: 'only', p: null, belowFloor: false, why: 'Only candidate.' }+ const routeDecisionId = String((await run(tab, { type: 'route.record', conversationId: 'lead', decision } as Command, host, lead)).result.routeDecisionId)+ const request = { conversationId: 'lead', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',+ name: 'Builder', cwd: '/fixture/repo', worktree: false, ...(role ? { role } : {}),+ requirements: { memoryBytes: 0, diskBytes: 0, maxCpuPercent: 100, tools: [], devices: [], quotaRequired: false } }+ const startId = String((await run(tab, { type: 'agent.start', request, routeDecisionId } as Command, host, lead)).result.startId)+ return { routeDecisionId, startId }+}const message = (text: string): Command => ({ type: 'message.send', conversationId: 'lead', text, replyTo: null, recipientIds: [] })async function invite(tab: string, role: 'collaborator' | 'viewer' = 'collaborator') {const invitation = await run(tab, { type: 'invite.create', email: peer.email, role, conversationId: 'lead' })@@ -1744,6 +1756,9 @@ describe('Super Tabs on a real Mongo replica set', () => {const request = { conversationId: 'lead', machineId: 'test-host', engine: 'claude', model: 'haiku', effort: 'auto', name: 'Builder', cwd: '/fixture/repo', worktree: false,requirements: { memoryBytes: 0, diskBytes: 0, maxCpuPercent: 100, tools: [], devices: [], quotaRequired: false } }const startId = (await run(tab, { type: 'agent.start', request, routeDecisionId } as Command, host, lead)).result.startId as string+ // A queued start already holds its account's quota when the Lead routes again.+ await run(tab, { type: 'route.record', conversationId: 'lead', decision } as Command, host, lead)+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()await expect(run(tab, { type: 'route.modelResult', decisionId: routeDecisionId, startId: 'other', model: 'sonnet', effort: 'auto' } as Command, host, lead)).rejects.toMatchObject({ code: 'ROUTE_MISMATCH' })const before = (await store.messages(owner, tab, 'lead', {})).messages.length@@ -1754,6 +1769,53 @@ describe('Super Tabs on a real Mongo replica set', () => {final: { kind: 'new', startId, model: 'sonnet', effort: 'auto', requested: { model: 'haiku', effort: 'auto' } } })})+ it('releases a queued start\'s route hold when that start fails', async () => {+ const { tab, lead } = await fixture()+ const { routeDecisionId, startId } = await routedStart(tab, lead)+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()+ await run(tab, { type: 'agent.startFailed', startId, error: 'Launch refused.' }, host, lead)+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('failed')+ })++ it('releases every queued route hold when its machine is removed', async () => {+ // The Lead runs elsewhere: a machine that hosts a project's Lead cannot leave the Swarm.+ const tab = (await store.create(owner, randomUUID(), 'Pilot')).id+ const leadMachineId = `lead-${randomUUID()}`+ await db.machine.create({ data: { machineId: leadMachineId, userId: host.sub, apiKey: 'fixture-only', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })+ await db.machineShare.create({ data: { machineId: leadMachineId, ownerId: host.sub, autonomousEnv: 'prod', emails: [owner.email] } })+ const lead = (await run(tab, { type: 'agent.add', conversationId: 'lead', machineId: leadMachineId, harnessId: randomUUID(),+ name: 'Lead', configuration: config, lead: true })).result.agentId as string+ const invitation = await run(tab, { type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })+ await run(tab, { type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)+ await run(tab, { type: 'machine.contribute', machineId: 'test-host', name: 'Host', cwd: '/fixture/repo', engines: ['codex'], maxAgents: 4 }, host)+ const machine = (await store.hostPools(host, 'test-host')).find(pool => pool.tabId === tab)!.machine+ await store.observeMachine(host, tab, 'test-host', { generation: machine.generation,+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },+ quotas: [], tools: {}, runningHarnessIds: [], stoppedHarnessIds: [] })+ const starts = [await routedStart(tab, lead), await routedStart(tab, lead)]+ for (const { routeDecisionId } of starts) expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()++ await run(tab, { type: 'machine.remove', machineId: 'test-host' })++ const state = (await store.read(owner, tab)).state+ for (const { routeDecisionId, startId } of starts) {+ expect(state.starts?.find(s => s.id === startId)?.error).toBe('Its machine was removed from the Swarm.')+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('failed')+ }+ })++ it('releases a started agent\'s route hold when the host stops it before assignment', async () => {+ const { tab, lead } = await fixture()+ const { routeDecisionId, startId } = await routedStart(tab, lead, 'reviewer')+ const agentId = String((await run(tab, { type: 'agent.started', startId, harnessId: randomUUID(), configuration: config }, host, lead)).result.agentId)+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()+ await run(tab, { type: 'agent.release', agentId, decision: 'stop' }, host, lead)+ const releaseId = (await store.read(owner, tab)).state.agents.find(agent => agent.id === agentId)!.releaseId!+ await run(tab, { type: 'agent.remove', agentId, releaseId }, host, agentId)+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('stopped')+ })+it('names the failing field when a command is invalid', async () => {const { tab } = await fixture()const command = { type: 'agent.start', request: { conversationId: 'lead', machineId: 'test-host', engine: 'claude', name: 'Builder', cwd: '/fixture/repo', worktree: false } }@@ -1774,6 +1836,8 @@ describe('Super Tabs on a real Mongo replica set', () => {const routeDecisionId = (await run(tab, { type: 'route.record', conversationId: group, decision } as Command, host, lead)).result.routeDecisionId as stringconst assignmentId = (await run(tab, { type: 'work.assign', conversationId: group, agentId: worker, routeDecisionId } as Command, host, lead)).result.assignmentId as stringexpect(await store.routeDecision(owner, tab, routeDecisionId)).toMatchObject({ assignmentId, final: { kind: 'member', agentId: worker } })+ await run(tab, { type: 'route.record', conversationId: group, decision } as Command, host, lead)+ expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBeNull()await store.stopped(host, tab, assignmentId, 'native-stop-receipt')expect((await store.routeDecision(owner, tab, routeDecisionId)).outcome).toBe('stopped')})diff --git a/backend/src/superTabs/routeDecisions.test.ts b/backend/src/superTabs/routeDecisions.test.tsnew file mode 100644index 000000000..783856557--- /dev/null+++ b/backend/src/superTabs/routeDecisions.test.ts@@ -0,0 +1,73 @@+import { describe, expect, it } from 'vitest'+import { closeAssignment, writeRouteEffects } from './routeDecisions.js'++type Row = Record<string, unknown> & { id: string; tabId: string; conversationId: string; outcome?: string | null }++// Exercise the persistence effects against rows, including Mongo's distinction between null and an unset field.+function memoryTx(rows: Row[]) {+ const matches = (row: Row, where: Record<string, unknown>): boolean => Object.entries(where).every(([key, want]) => {+ if (key === 'AND') return (want as Record<string, unknown>[]).every(part => matches(row, part))+ if (key === 'OR') return (want as Record<string, unknown>[]).some(part => matches(row, part))+ const actual = row[key]+ if (want && typeof want === 'object') {+ const filter = want as Record<string, unknown>+ if ('isSet' in filter) return (key in row) === filter.isSet+ if ('equals' in filter) return actual === filter.equals+ if ('not' in filter) return actual !== filter.not+ if ('in' in filter) return (filter.in as unknown[]).includes(actual)+ }+ return want === null ? actual == null : actual === want+ })+ const tx = { superTabRouteDecision: {+ findMany: async ({ where }: { where: Record<string, unknown> }) => rows.filter(row => matches(row, where))+ .map(row => ({ ...row, final: row.final ?? null, assignmentId: row.assignmentId ?? null })),+ updateMany: async ({ where, data }: { where: Record<string, unknown>; data: Record<string, unknown> }) => {+ const found = rows.filter(row => matches(row, where))+ found.forEach(row => Object.assign(row, data))+ return { count: found.length }+ },+ update: async ({ where, data }: { where: Record<string, unknown>; data: Record<string, unknown> }) => {+ const row = rows.find(row => matches(row, where))!+ Object.assign(row, data)+ return row+ },+ create: async ({ data }: { data: Row }) => { rows.push({ ...data }); return data },+ deleteMany: async () => ({ count: 0 }),+ } }+ return tx as unknown as Parameters<typeof writeRouteEffects>[0]+}++describe('route decision persistence', () => {+ it('reroutes unused proposals while queued starts and assignments retain quota until their work ends', async () => {+ const now = Date.now()+ const final = { kind: 'new' as const, startId: 'start-1', machineId: 'machine', engine: 'claude', model: 'opus', effort: 'high' }+ const rows: Row[] = [+ { id: 'proposal', tabId: 'tab', conversationId: 'group', final: null, assignmentId: null, outcome: null },+ { id: 'unset-proposal', tabId: 'tab', conversationId: 'group' },+ { id: 'queued', tabId: 'tab', conversationId: 'group', final, assignmentId: null, outcome: null },+ { id: 'assigned', tabId: 'tab', conversationId: 'group', final: { kind: 'member', agentId: 'agent', machineId: 'machine' }, assignmentId: 'assignment-1', outcome: null },+ { id: 'assigned-without-final', tabId: 'tab', conversationId: 'group', final: null, assignmentId: 'assignment-2', outcome: null },+ { id: 'finished', tabId: 'tab', conversationId: 'group', final, assignmentId: 'assignment-3', outcome: 'done' },+ { id: 'other-group', tabId: 'tab', conversationId: 'other', final: null, assignmentId: null, outcome: null },+ ]+ const tx = memoryTx(rows)+ await writeRouteEffects(tx, 'tab', [{ op: 'create', id: 'next', conversationId: 'group', record: {} as never }], now)+ const outcome = (id: string) => rows.find(row => row.id === id)?.outcome ?? null+ expect(outcome('proposal')).toBe('superseded')+ expect(outcome('unset-proposal')).toBe('superseded')+ expect(outcome('queued')).toBeNull()+ expect(outcome('assigned')).toBeNull()+ expect(outcome('assigned-without-final')).toBeNull()+ expect(outcome('finished')).toBe('done')+ expect(outcome('other-group')).toBeNull()+ expect(rows.some(row => row.id === 'next')).toBe(true)++ // A queued start later becomes an assignment; both it and the already assigned work close on completion.+ await writeRouteEffects(tx, 'tab', [{ op: 'final', id: 'queued', final, assignmentId: 'assignment-queued' }], now + 1)+ await closeAssignment(tx, 'tab', 'assignment-queued', 'done', now + 2)+ await closeAssignment(tx, 'tab', 'assignment-1', 'done', now + 2)+ expect(outcome('queued')).toBe('done')+ expect(outcome('assigned')).toBe('done')+ expect(outcome('next')).toBeNull()+ })+})diff --git a/backend/src/superTabs/routeDecisions.ts b/backend/src/superTabs/routeDecisions.tsindex c633cfc9c..d009867dd 100644--- a/backend/src/superTabs/routeDecisions.ts+++ b/backend/src/superTabs/routeDecisions.ts@@ -29,9 +29,13 @@ export async function readDecision(tx: Tx, tabId: string, id: string): Promise<Rexport async function writeRouteEffects(tx: Tx, tabId: string, effects: RouteDecisionEffect[], now: number): Promise<void> {for (const effect of effects) {if (effect.op === 'create') {- // One open decision per group: a newer route supersedes the open one.- await tx.superTabRouteDecision.updateMany({ where: { tabId, conversationId: effect.conversationId, ...open },- data: { outcome: 'superseded', outcomeAt: new Date(now) } })+ // A newer route supersedes unused proposals; queued starts and assignments keep their quota until work ends.+ // This Prisma/Mongo client cannot filter `isSet` on a Json field; reading it maps both unset and null to null.+ const prior = await tx.superTabRouteDecision.findMany({ where: { tabId, conversationId: effect.conversationId, ...open },+ select: { id: true, final: true, assignmentId: true } })+ const unusedIds = prior.filter(row => row.final === null && row.assignmentId === null).map(row => row.id)+ if (unusedIds.length) await tx.superTabRouteDecision.updateMany({ where: { tabId, conversationId: effect.conversationId,+ id: { in: unusedIds }, ...open }, data: { outcome: 'superseded', outcomeAt: new Date(now) } })await tx.superTabRouteDecision.create({ data: { id: effect.id, tabId, conversationId: effect.conversationId,record: json(effect.record), createdAt: new Date(now) } })await tx.superTabRouteDecision.deleteMany({ where: { tabId, createdAt: { lt: new Date(now - RETENTION_MS) } } })@@ -62,3 +66,13 @@ export async function closeAssignment(tx: Tx, tabId: string, assignmentId: strin}await tx.superTabRouteDecision.updateMany({ where: { tabId, assignmentId, ...open }, data: { outcome, outcomeAt: new Date(now) } })}++/** A failed start or removed agent releases its hold only while the start has no linked assignment. */+export async function closeUnassignedStart(tx: Tx, tabId: string, decisionId: string, startId: string, outcome: 'failed' | 'stopped', now: number): Promise<void> {+ const row = await tx.superTabRouteDecision.findFirst({ where: { id: decisionId, tabId, ...open }, select: { final: true, assignmentId: true } })+ const final = row?.final as { kind?: unknown; startId?: unknown } | null | undefined+ if (row?.assignmentId != null || final?.kind !== 'new' || final.startId !== startId) return+ await tx.superTabRouteDecision.updateMany({ where: { id: decisionId, tabId,+ AND: [open, { OR: [{ assignmentId: null }, { assignmentId: { isSet: false } }] }] },+ data: { outcome, outcomeAt: new Date(now) } })+}diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex c2615e331..a28e0cea7 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -20,7 +20,7 @@ import {} from './model.js'import { holdNoticeText, type HoldReason } from './holdNotice.js'import { mayDrive, TERMINAL_ANSWER_MS, TERMINAL_EXPIRY_MS, TERMINAL_PICKUP_MS, type TerminalEffect } from './terminal.js'-import { closeAssignment, readDecision, recentRoutes, timedDecisionOf, writeRouteEffects } from './routeDecisions.js'+import { closeAssignment, closeUnassignedStart, readDecision, recentRoutes, timedDecisionOf, writeRouteEffects } from './routeDecisions.js'import { DEFAULT_ROUTING_POLICY, effectivePolicy, routingPolicySchema, type RoutingPolicy } from './routingPolicy.js'import { authenticatedEmail, authorizeSwarmJoin, configureSharing, joinSwarmSchema, requireSharingOwner,saveSharingSchema, sharingDetails, type JoinSwarm, type SaveSharing } from './sharing.js'@@ -589,6 +589,23 @@ export class SuperTabStore {revision, state: json(effects.state), ...indexes(effects.state),} })await writeRouteEffects(tx, tabId, effects.routeDecisions, context.now)+ if (command.type === 'agent.startFailed') {+ const start = original.starts?.find(s => s.id === command.startId)+ if (start?.routeDecisionId) await closeUnassignedStart(tx, tabId, start.routeDecisionId, start.id, 'failed', context.now)+ }+ if (command.type === 'machine.remove') {+ for (const start of original.starts ?? []) {+ if (!start.routeDecisionId || start.request.machineId !== command.machineId || start.harnessId || start.error) continue+ if (effects.state.starts?.find(s => s.id === start.id)?.error) {+ await closeUnassignedStart(tx, tabId, start.routeDecisionId, start.id, 'failed', context.now)+ }+ }+ }+ if (command.type === 'agent.remove' && actor.kind === 'agent' && actor.agentId === command.agentId) {+ const start = original.starts?.find(s => s.routeDecisionId && s.harnessId &&+ stableId(s.request.machineId, s.harnessId) === command.agentId)+ if (start?.routeDecisionId) await closeUnassignedStart(tx, tabId, start.routeDecisionId, start.id, 'stopped', context.now)+ }if (effects.terminal) await this.writeTerminal(tx, tabId, effects.terminal, context.now)if (command.type === 'work.cancel') {await closeAssignment(tx, tabId, command.assignmentId, 'cancelled', context.now)diff --git a/cli/src/superTabs/routeCommand.spec.ts b/cli/src/superTabs/routeCommand.spec.tsindex 002484f82..36c7a28bb 100644--- a/cli/src/superTabs/routeCommand.spec.ts+++ b/cli/src/superTabs/routeCommand.spec.ts@@ -333,6 +333,18 @@ describe('routeTask', () => {expect(recorded(s).command.decision.candidates.every((c: { fit: { status: string; account: string } }) => c.fit.status === 'short' && c.fit.account === 'acct')).toBe(true)expect(result.warnings).toEqual(['w', expect.stringMatching(/^No candidate is expected to cover this task; its 5-hour quota is expected to run out partway and resets in 3h\.$/)])})+ it('keeps a queued start in the reroute quota hold until its linked work finishes', async () => {+ const queued = { id: 'queued', conversationId: 'g1', account: 'acct', spend: { '5-hour': 45 }, size: 'medium',+ final: { kind: 'new', startId: 'start-1', engine: 'claude', model: 'opus' }, assignmentId: null,+ outcome: null as string | null, createdAt: new Date().toISOString() }+ const routeWith = async (decision: typeof queued) => {+ const s = scoped({ history: { ok: true, value: { decisions: [decision], attempts: [] } } })+ await routeTask(input, s, where as never, opener(), () => 'op')+ return recorded(s).command.decision.candidates as Array<{ fit: { status: string } }>+ }+ expect((await routeWith(queued)).every(candidate => candidate.fit.status === 'short')).toBe(true)+ expect((await routeWith({ ...queued, outcome: 'done' })).some(candidate => candidate.fit.status === 'fits')).toBe(true)+ })it('records again in the shape an older backend knows when it refuses the new fields as invalid', async () => {const s = scoped(), base = s.getMockImplementation()!let refused = 0