The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(swarm): an agent whose task was lost takes work again once it is back (#1781)
0xken15923e7741f4f9
diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.tsindex ee142998b..b56218d03 100644--- a/backend/integration/superTabLiveness.test.ts+++ b/backend/integration/superTabLiveness.test.ts@@ -274,6 +274,142 @@ describe('work that loses its owner', () => {})})+// After a release every machine went offline for about 2 minutes while its CLI updated: 16 tasks were lost, their agents came+// back, and every assignment to the same agent was refused as busy for good, even after the Lead cancelled the lost task.+describe('an agent whose task was lost', () => {+ type Fixture = Awaited<ReturnType<typeof swarm>>+ /** The worker's machine goes silent for over 2 minutes; the loss is found by the Lead machine's report or by the Lead's poll. */+ async function loseByOffline(f: Fixture, foundBy: 'report' | 'poll' = 'report') {+ await f.beat(f.clock.now)+ f.clock.now += 2 * MINUTE + 5_000+ if (foundBy === 'report') await f.observe('lead-host', [f.harness.lead])+ else await f.leadInbox()+ expect((await f.assignment()).status).toBe('lost')+ }+ const backOnline = async (f: Fixture) => { f.clock.now += MINUTE; await f.beat(f.clock.now) }+ const runOf = (agentId: string) => db.superTabRun.findUnique({ where: { id: agentId } })+ const pending = (tabId: string, assignmentId: string) => db.superTabDelivery.findMany({ where: { tabId, assignmentId, status: { in: ['queued', 'claimed', 'uncertain'] } } })+ /** What an older server left behind when a task was lost outside a command: the agent's run record, and the lost task's+ * deliveries still pending (one claimed, the rest queued), which the loss on this server has already cancelled. */+ async function leftBehind(f: Fixture, assignmentId = f.assignmentId, tabId = f.tab) {+ await db.superTabRun.upsert({ where: { id: f.worker },+ create: { id: f.worker, tabId, assignmentId, machineId: 'test-host', harnessId: f.harness.worker }, update: { tabId, assignmentId } })+ const rows = await db.superTabDelivery.findMany({ where: { tabId, assignmentId } })+ await db.superTabDelivery.updateMany({ where: { tabId, assignmentId }, data: { status: 'queued' } })+ if (rows[0]) await db.superTabDelivery.update({ where: { id: rows[0].id }, data: { status: 'claimed' } })+ return rows.length+ }++ it.each(['report', 'poll'] as const)('takes work again once its machine is back, the loss found by a %s', async foundBy => {+ const f = await swarm()+ expect(await pending(f.tab, f.assignmentId)).not.toEqual([])+ await loseByOffline(f, foundBy)+ // The loss frees the agent and drops what was still queued for the lost task.+ expect(await runOf(f.worker)).toBeNull()+ expect(await pending(f.tab, f.assignmentId)).toEqual([])+ await backOnline(f)+ const next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).result.assignmentId)+ expect(await f.assignment()).toMatchObject({ id: next, agentId: f.worker, status: 'working' })+ expect(await runOf(f.worker)).toMatchObject({ tabId: f.tab, assignmentId: next })+ })++ it('takes the Lead\'s explicit assignment with a reason, while the mark that keeps routing away from it stays', async () => {+ const f = await swarm()+ await loseByOffline(f)+ await backOnline(f)+ const decision = { assessment: { difficulty: 'low', risk: 'low', why: 'Small task.' },+ candidates: [{ id: 'c0', kind: 'new', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',+ quality: 2, cost: 2, title: 'New agent', label: 'New agent' }],+ excluded: [], floor: 1, notes: false, pick: 'c0', via: 'only', p: null, belowFloor: false, why: 'Only candidate.' }+ const routeDecisionId = String((await f.execute({ type: 'route.record', conversationId: f.group, decision } as Command, host, f.lead)).result.routeDecisionId)+ await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker, routeDecisionId,+ overrideReason: 'Its machine is back and it has the context.' } as Command, host, f.lead)+ expect(await f.assignment()).toMatchObject({ agentId: f.worker, status: 'working' })+ expect((await f.timed.read(owner, f.tab)).state.agents.find(a => a.id === f.worker)!.lostTask).toMatchObject({ assignmentId: f.assignmentId })+ })++ it('takes work again after the Lead cancels its lost task, though an older server left it marked running', async () => {+ const f = await swarm()+ await loseByOffline(f)+ await leftBehind(f)+ await f.execute({ type: 'work.cancel', conversationId: f.group, assignmentId: f.assignmentId, reason: 'Not needed any more' }, host, f.lead)+ expect(await runOf(f.worker)).toBeNull()+ await backOnline(f)+ await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })+ expect(await f.assignment()).toMatchObject({ agentId: f.worker, status: 'working' })+ })++ it('is assigned past a record an older server left for its lost task, which is then replaced, its old deliveries dropped', async () => {+ const f = await swarm()+ await loseByOffline(f)+ expect(await leftBehind(f)).toBeGreaterThan(0)+ expect(await pending(f.tab, f.assignmentId)).not.toEqual([])+ await backOnline(f)+ const next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).result.assignmentId)+ expect(await runOf(f.worker)).toMatchObject({ tabId: f.tab, assignmentId: next })+ // The lost task's old instructions never reach the worker after its new assignment.+ expect(await pending(f.tab, f.assignmentId)).toEqual([])+ expect(await pending(f.tab, next)).not.toEqual([])+ })++ it('has records left for ended work removed on the Lead\'s poll, keeping those for active work and stopping agents', async () => {+ const f = await swarm()+ const second = String((await f.execute({ type: 'conversation.create', title: 'Write the docs', brief: 'Document it',+ checks: [{ description: 'Read', independent: false }] })).result.conversationId)+ // Its work runs on the Lead's machine, which keeps reporting, so it stays active while the worker's machine goes silent.+ const other = await f.add(second, 'lead-host', f.harness.other, 'Other')+ const active = String((await f.execute({ type: 'work.assign', conversationId: second, agentId: other })).result.assignmentId)+ await loseByOffline(f)+ await leftBehind(f)+ const stopping = await db.superTabRun.create({ data: { id: randomUUID(), tabId: f.tab, assignmentId: `release-${randomUUID()}`, machineId: 'test-host', harnessId: randomUUID() } })+ const live = await pending(f.tab, active)+ expect(live).not.toEqual([])+ await f.leadInbox()+ expect(await runOf(f.worker)).toBeNull()+ expect(await pending(f.tab, f.assignmentId)).toEqual([])+ expect(await pending(f.tab, active)).toEqual(live)+ expect(await runOf(other)).toMatchObject({ assignmentId: active })+ expect(await runOf(stopping.id)).not.toBeNull()+ // Removing them again changes nothing.+ await f.leadInbox()+ expect(await runOf(other)).toMatchObject({ assignmentId: active })+ })++ it('keeps the record of an assignment that commits between the Lead\'s poll reading the Swarm and removing stale records', async () => {+ const f = await swarm()+ await loseByOffline(f)+ await leftBehind(f)+ await backOnline(f)+ // The poll reads the old record as stale; just before it removes it, the Lead's assignment replaces it with a live one.+ let next = ''+ const racing = new SuperTabStore(beforeFirstWrite(db, async () => {+ if (next) return+ next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).result.assignmentId)+ }), () => f.clock.now)+ await racing.deliveries(host, f.tab, f.lead)+ expect(next).not.toBe('')+ expect(await runOf(f.worker)).toMatchObject({ tabId: f.tab, assignmentId: next })+ expect(await f.assignment()).toMatchObject({ id: next, status: 'working' })+ })++ it('stays busy while it works in another Swarm, whose record is never removed from this one', async () => {+ const f = await swarm()+ await loseByOffline(f)+ await backOnline(f)+ // The same session is a member of a second Swarm and holds active work there, which this Swarm cannot read.+ const elsewhere = await swarm()+ // It is that Swarm's worker: its record there is this agent's.+ await db.superTabRun.delete({ where: { id: elsewhere.worker } })+ await leftBehind(f, elsewhere.assignmentId, elsewhere.tab)+ const theirs = await pending(elsewhere.tab, elsewhere.assignmentId)+ expect(theirs).not.toEqual([])+ await expect(f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).rejects.toMatchObject({ code: 'AGENT_BUSY' })+ await f.leadInbox()+ expect(await runOf(f.worker)).toMatchObject({ tabId: elsewhere.tab })+ expect(await pending(elsewhere.tab, elsewhere.assignmentId)).toEqual(theirs)+ })+})+describe('a start that never runs', () => {it('fails 3 minutes after dispatch, naming its machine, and tells the Lead in its group', async () => {const f = await swarm()diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 3ec1885ee..18e02c707 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -851,17 +851,16 @@ export class SuperTabStore {if (command.type === 'work.assign') {const assigned = conversation(effects.state, command.conversationId).assignment!const running = await tx.superTabRun.findUnique({ where: { id: assigned.agentId } })- if (running) fail('AGENT_BUSY', 'This agent already owns active work in a Super Tab.')+ // A record of this Swarm's for work that has ended here is stale (endRuns): it goes, by its own assignment, and the agent+ // is free. A record of another Swarm's, or a stopping agent's, still means busy.+ if (running && (running.tabId !== tabId || !(await this.staleRuns(tx, tabId, original)).includes(running.assignmentId))) {+ fail('AGENT_BUSY', 'This agent already owns active work in a Super Tab.')+ }+ if (running) await this.releaseRuns(tx, tabId, [running.assignmentId])await tx.superTabRun.create({ data: { id: assigned.agentId, tabId, assignmentId: assigned.id,machineId: assigned.execution.machineId, harnessId: assigned.execution.harnessId } })}- for (const group of original.conversations.filter(activeWork)) {- const changed = conversation(effects.state, group.id)- if (activeWork(changed)) continue- await tx.superTabRun.deleteMany({ where: { tabId, assignmentId: group.assignment!.id } })- await tx.superTabDelivery.updateMany({ where: { tabId, assignmentId: group.assignment!.id,- status: { in: ['queued', 'claimed', 'uncertain'] } }, data: { status: 'cancelled', holdReason: null, holdSince: null } })- }+ await this.endRuns(tx, tabId, original, effects.state)const committed = await this.save(tx, loaded, effects.state, stamp, touchesFor(command, actor, original, effects, context))if (command.type === 'agent.add' || command.type === 'agent.started' || command.type === 'agent.configuration' || command.type === 'agent.rebind') {for (const target of effects.state.agents) {@@ -1214,6 +1213,42 @@ export class SuperTabStore {return null}+ /** Frees the agents of work that ended in this write, wherever it ended: a command, the liveness checks on a machine's report+ * or on the Lead's poll (a lost task), or a cancel of work already lost. Their run record goes, so the agent can take work+ * again, and whatever was still queued for that work is dropped. Until 2026-10-27 only a command that ended active work did+ * this, so the 16 tasks a release's update outage lost kept their agents busy for good. */+ private async endRuns(tx: Transaction, tabId: string, before: SuperTabState, after: SuperTabState) {+ const ended = new Set<string>()+ for (const group of after.conversations) {+ const now = group.assignment, was = before.conversations.find(c => c.id === group.id)?.assignment+ if (now && !activeWork(group) && (now.id !== was?.id || now.status !== was.status)) ended.add(now.id)+ }+ for (const group of before.conversations.filter(activeWork)) {+ const changed = after.conversations.find(c => c.id === group.id)+ if (!changed || !activeWork(changed) || changed.assignment!.id !== group.assignment!.id) ended.add(group.assignment!.id)+ }+ await this.releaseRuns(tx, tabId, [...ended])+ }++ /** For these ended assignments of this Swarm, by their exact ids: their run records go, and their deliveries still queued,+ * claimed or uncertain are cancelled, so a returning worker never receives a lost task's old instructions after new work+ * (review of #1781). Exact ids never match the record or deliveries of an assignment made since. */+ private async releaseRuns(tx: Transaction, tabId: string, assignmentIds: string[]) {+ if (!assignmentIds.length) return+ const assignmentId = { in: assignmentIds }+ await tx.superTabRun.deleteMany({ where: { tabId, assignmentId } })+ await tx.superTabDelivery.updateMany({ where: { tabId, assignmentId, status: { in: ['queued', 'claimed', 'uncertain'] } },+ data: { status: 'cancelled', holdReason: null, holdSince: null } })+ }++ /** Run records an older server left in this Swarm for work that is no longer active here (see endRuns). Only this Swarm's,+ * whose work it can see has ended; never one of another Swarm's, nor a stopping agent's (its record is the stop's fence). */+ private async staleRuns(tx: Transaction, tabId: string, state: SuperTabState): Promise<string[]> {+ const active = new Set(state.conversations.filter(activeWork).map(c => c.assignment!.id))+ const rows = await tx.superTabRun.findMany({ where: { tabId }, select: { assignmentId: true } })+ return rows.map(r => r.assignmentId).filter(id => !id.startsWith('release-') && !active.has(id))+ }+/** Records committed messages, each at the sequence its conversation gave it, and one delivery per recipient. */private async writeMessages(tx: Transaction, tabId: string, state: SuperTabState, messages: MessageDraft[], sequences: ReadonlyMap<string, number>) {for (const message of messages) {@@ -1395,6 +1430,7 @@ export class SuperTabStore {const saved = async (lost: MessageDraft[] = []) => {const sequences = numberMessages(state, lost, now)const touches = judgedMachines(state, lost).filter(t => t.key !== machineId)+ await this.endRuns(tx, tabId, loaded.state, state)const version = (await this.save(tx, loaded, state, stamp, touches)).versions.get(`machine:${machineId}`) ?? 0await this.writeMessages(tx, tabId, state, lost, sequences)return { version }@@ -1505,6 +1541,12 @@ export class SuperTabStore {woken = true}}+ // The poll also frees agents whose task the checks just lost, and any an older server left marked running (each poll; a+ // second finds none).+ await this.endRuns(tx, tabId, loaded.state, state)+ if (ledProject(state, agentId)) {+ await this.releaseRuns(tx, tabId, await this.staleRuns(tx, tabId, state))+ }// One write for both: the time checks' changes and the wake-up's.const sequences = numberMessages(state, written, this.now())await this.save(tx, loaded, state, clock.next(), [...judgedMachines(state, written), ...woken ? [{ kind: 'agent' as const, key: agentId }] : []])