Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(swarm): an agent whose task was lost takes work again once it is back (#1781)

0xken15923e7741f4f9
diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.ts
index ee142998b..b56218d03 100644
--- a/backend/integration/superTabLiveness.test.ts
+++ b/backend/integration/superTabLiveness.test.ts
@@ -274,6 +274,142 @@ describe('work that loses its owner', () => {
})
})
+// After a release every machine went offline for about 2 minutes while its CLI updated: 16 tasks were lost, their agents came
+// back, and every assignment to the same agent was refused as busy for good, even after the Lead cancelled the lost task.
+describe('an agent whose task was lost', () => {
+ type Fixture = Awaited<ReturnType<typeof swarm>>
+ /** The worker's machine goes silent for over 2 minutes; the loss is found by the Lead machine's report or by the Lead's poll. */
+ async function loseByOffline(f: Fixture, foundBy: 'report' | 'poll' = 'report') {
+ await f.beat(f.clock.now)
+ f.clock.now += 2 * MINUTE + 5_000
+ if (foundBy === 'report') await f.observe('lead-host', [f.harness.lead])
+ else await f.leadInbox()
+ expect((await f.assignment()).status).toBe('lost')
+ }
+ const backOnline = async (f: Fixture) => { f.clock.now += MINUTE; await f.beat(f.clock.now) }
+ const runOf = (agentId: string) => db.superTabRun.findUnique({ where: { id: agentId } })
+ const pending = (tabId: string, assignmentId: string) => db.superTabDelivery.findMany({ where: { tabId, assignmentId, status: { in: ['queued', 'claimed', 'uncertain'] } } })
+ /** What an older server left behind when a task was lost outside a command: the agent's run record, and the lost task's
+ * deliveries still pending (one claimed, the rest queued), which the loss on this server has already cancelled. */
+ async function leftBehind(f: Fixture, assignmentId = f.assignmentId, tabId = f.tab) {
+ await db.superTabRun.upsert({ where: { id: f.worker },
+ create: { id: f.worker, tabId, assignmentId, machineId: 'test-host', harnessId: f.harness.worker }, update: { tabId, assignmentId } })
+ const rows = await db.superTabDelivery.findMany({ where: { tabId, assignmentId } })
+ await db.superTabDelivery.updateMany({ where: { tabId, assignmentId }, data: { status: 'queued' } })
+ if (rows[0]) await db.superTabDelivery.update({ where: { id: rows[0].id }, data: { status: 'claimed' } })
+ return rows.length
+ }
+
+ it.each(['report', 'poll'] as const)('takes work again once its machine is back, the loss found by a %s', async foundBy => {
+ const f = await swarm()
+ expect(await pending(f.tab, f.assignmentId)).not.toEqual([])
+ await loseByOffline(f, foundBy)
+ // The loss frees the agent and drops what was still queued for the lost task.
+ expect(await runOf(f.worker)).toBeNull()
+ expect(await pending(f.tab, f.assignmentId)).toEqual([])
+ await backOnline(f)
+ const next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).result.assignmentId)
+ expect(await f.assignment()).toMatchObject({ id: next, agentId: f.worker, status: 'working' })
+ expect(await runOf(f.worker)).toMatchObject({ tabId: f.tab, assignmentId: next })
+ })
+
+ it('takes the Lead\'s explicit assignment with a reason, while the mark that keeps routing away from it stays', async () => {
+ const f = await swarm()
+ await loseByOffline(f)
+ await backOnline(f)
+ const decision = { assessment: { difficulty: 'low', risk: 'low', why: 'Small task.' },
+ candidates: [{ id: 'c0', kind: 'new', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',
+ quality: 2, cost: 2, title: 'New agent', label: 'New agent' }],
+ excluded: [], floor: 1, notes: false, pick: 'c0', via: 'only', p: null, belowFloor: false, why: 'Only candidate.' }
+ const routeDecisionId = String((await f.execute({ type: 'route.record', conversationId: f.group, decision } as Command, host, f.lead)).result.routeDecisionId)
+ await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker, routeDecisionId,
+ overrideReason: 'Its machine is back and it has the context.' } as Command, host, f.lead)
+ expect(await f.assignment()).toMatchObject({ agentId: f.worker, status: 'working' })
+ expect((await f.timed.read(owner, f.tab)).state.agents.find(a => a.id === f.worker)!.lostTask).toMatchObject({ assignmentId: f.assignmentId })
+ })
+
+ it('takes work again after the Lead cancels its lost task, though an older server left it marked running', async () => {
+ const f = await swarm()
+ await loseByOffline(f)
+ await leftBehind(f)
+ await f.execute({ type: 'work.cancel', conversationId: f.group, assignmentId: f.assignmentId, reason: 'Not needed any more' }, host, f.lead)
+ expect(await runOf(f.worker)).toBeNull()
+ await backOnline(f)
+ await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })
+ expect(await f.assignment()).toMatchObject({ agentId: f.worker, status: 'working' })
+ })
+
+ it('is assigned past a record an older server left for its lost task, which is then replaced, its old deliveries dropped', async () => {
+ const f = await swarm()
+ await loseByOffline(f)
+ expect(await leftBehind(f)).toBeGreaterThan(0)
+ expect(await pending(f.tab, f.assignmentId)).not.toEqual([])
+ await backOnline(f)
+ const next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).result.assignmentId)
+ expect(await runOf(f.worker)).toMatchObject({ tabId: f.tab, assignmentId: next })
+ // The lost task's old instructions never reach the worker after its new assignment.
+ expect(await pending(f.tab, f.assignmentId)).toEqual([])
+ expect(await pending(f.tab, next)).not.toEqual([])
+ })
+
+ it('has records left for ended work removed on the Lead\'s poll, keeping those for active work and stopping agents', async () => {
+ const f = await swarm()
+ const second = String((await f.execute({ type: 'conversation.create', title: 'Write the docs', brief: 'Document it',
+ checks: [{ description: 'Read', independent: false }] })).result.conversationId)
+ // Its work runs on the Lead's machine, which keeps reporting, so it stays active while the worker's machine goes silent.
+ const other = await f.add(second, 'lead-host', f.harness.other, 'Other')
+ const active = String((await f.execute({ type: 'work.assign', conversationId: second, agentId: other })).result.assignmentId)
+ await loseByOffline(f)
+ await leftBehind(f)
+ const stopping = await db.superTabRun.create({ data: { id: randomUUID(), tabId: f.tab, assignmentId: `release-${randomUUID()}`, machineId: 'test-host', harnessId: randomUUID() } })
+ const live = await pending(f.tab, active)
+ expect(live).not.toEqual([])
+ await f.leadInbox()
+ expect(await runOf(f.worker)).toBeNull()
+ expect(await pending(f.tab, f.assignmentId)).toEqual([])
+ expect(await pending(f.tab, active)).toEqual(live)
+ expect(await runOf(other)).toMatchObject({ assignmentId: active })
+ expect(await runOf(stopping.id)).not.toBeNull()
+ // Removing them again changes nothing.
+ await f.leadInbox()
+ expect(await runOf(other)).toMatchObject({ assignmentId: active })
+ })
+
+ it('keeps the record of an assignment that commits between the Lead\'s poll reading the Swarm and removing stale records', async () => {
+ const f = await swarm()
+ await loseByOffline(f)
+ await leftBehind(f)
+ await backOnline(f)
+ // The poll reads the old record as stale; just before it removes it, the Lead's assignment replaces it with a live one.
+ let next = ''
+ const racing = new SuperTabStore(beforeFirstWrite(db, async () => {
+ if (next) return
+ next = String((await f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).result.assignmentId)
+ }), () => f.clock.now)
+ await racing.deliveries(host, f.tab, f.lead)
+ expect(next).not.toBe('')
+ expect(await runOf(f.worker)).toMatchObject({ tabId: f.tab, assignmentId: next })
+ expect(await f.assignment()).toMatchObject({ id: next, status: 'working' })
+ })
+
+ it('stays busy while it works in another Swarm, whose record is never removed from this one', async () => {
+ const f = await swarm()
+ await loseByOffline(f)
+ await backOnline(f)
+ // The same session is a member of a second Swarm and holds active work there, which this Swarm cannot read.
+ const elsewhere = await swarm()
+ // It is that Swarm's worker: its record there is this agent's.
+ await db.superTabRun.delete({ where: { id: elsewhere.worker } })
+ await leftBehind(f, elsewhere.assignmentId, elsewhere.tab)
+ const theirs = await pending(elsewhere.tab, elsewhere.assignmentId)
+ expect(theirs).not.toEqual([])
+ await expect(f.execute({ type: 'work.assign', conversationId: f.group, agentId: f.worker })).rejects.toMatchObject({ code: 'AGENT_BUSY' })
+ await f.leadInbox()
+ expect(await runOf(f.worker)).toMatchObject({ tabId: elsewhere.tab })
+ expect(await pending(elsewhere.tab, elsewhere.assignmentId)).toEqual(theirs)
+ })
+})
+
describe('a start that never runs', () => {
it('fails 3 minutes after dispatch, naming its machine, and tells the Lead in its group', async () => {
const f = await swarm()
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 3ec1885ee..18e02c707 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -851,17 +851,16 @@ export class SuperTabStore {
if (command.type === 'work.assign') {
const assigned = conversation(effects.state, command.conversationId).assignment!
const running = await tx.superTabRun.findUnique({ where: { id: assigned.agentId } })
- if (running) fail('AGENT_BUSY', 'This agent already owns active work in a Super Tab.')
+ // A record of this Swarm's for work that has ended here is stale (endRuns): it goes, by its own assignment, and the agent
+ // is free. A record of another Swarm's, or a stopping agent's, still means busy.
+ if (running && (running.tabId !== tabId || !(await this.staleRuns(tx, tabId, original)).includes(running.assignmentId))) {
+ fail('AGENT_BUSY', 'This agent already owns active work in a Super Tab.')
+ }
+ if (running) await this.releaseRuns(tx, tabId, [running.assignmentId])
await tx.superTabRun.create({ data: { id: assigned.agentId, tabId, assignmentId: assigned.id,
machineId: assigned.execution.machineId, harnessId: assigned.execution.harnessId } })
}
- for (const group of original.conversations.filter(activeWork)) {
- const changed = conversation(effects.state, group.id)
- if (activeWork(changed)) continue
- await tx.superTabRun.deleteMany({ where: { tabId, assignmentId: group.assignment!.id } })
- await tx.superTabDelivery.updateMany({ where: { tabId, assignmentId: group.assignment!.id,
- status: { in: ['queued', 'claimed', 'uncertain'] } }, data: { status: 'cancelled', holdReason: null, holdSince: null } })
- }
+ await this.endRuns(tx, tabId, original, effects.state)
const committed = await this.save(tx, loaded, effects.state, stamp, touchesFor(command, actor, original, effects, context))
if (command.type === 'agent.add' || command.type === 'agent.started' || command.type === 'agent.configuration' || command.type === 'agent.rebind') {
for (const target of effects.state.agents) {
@@ -1214,6 +1213,42 @@ export class SuperTabStore {
return null
}
+ /** Frees the agents of work that ended in this write, wherever it ended: a command, the liveness checks on a machine's report
+ * or on the Lead's poll (a lost task), or a cancel of work already lost. Their run record goes, so the agent can take work
+ * again, and whatever was still queued for that work is dropped. Until 2026-10-27 only a command that ended active work did
+ * this, so the 16 tasks a release's update outage lost kept their agents busy for good. */
+ private async endRuns(tx: Transaction, tabId: string, before: SuperTabState, after: SuperTabState) {
+ const ended = new Set<string>()
+ for (const group of after.conversations) {
+ const now = group.assignment, was = before.conversations.find(c => c.id === group.id)?.assignment
+ if (now && !activeWork(group) && (now.id !== was?.id || now.status !== was.status)) ended.add(now.id)
+ }
+ for (const group of before.conversations.filter(activeWork)) {
+ const changed = after.conversations.find(c => c.id === group.id)
+ if (!changed || !activeWork(changed) || changed.assignment!.id !== group.assignment!.id) ended.add(group.assignment!.id)
+ }
+ await this.releaseRuns(tx, tabId, [...ended])
+ }
+
+ /** For these ended assignments of this Swarm, by their exact ids: their run records go, and their deliveries still queued,
+ * claimed or uncertain are cancelled, so a returning worker never receives a lost task's old instructions after new work
+ * (review of #1781). Exact ids never match the record or deliveries of an assignment made since. */
+ private async releaseRuns(tx: Transaction, tabId: string, assignmentIds: string[]) {
+ if (!assignmentIds.length) return
+ const assignmentId = { in: assignmentIds }
+ await tx.superTabRun.deleteMany({ where: { tabId, assignmentId } })
+ await tx.superTabDelivery.updateMany({ where: { tabId, assignmentId, status: { in: ['queued', 'claimed', 'uncertain'] } },
+ data: { status: 'cancelled', holdReason: null, holdSince: null } })
+ }
+
+ /** Run records an older server left in this Swarm for work that is no longer active here (see endRuns). Only this Swarm's,
+ * whose work it can see has ended; never one of another Swarm's, nor a stopping agent's (its record is the stop's fence). */
+ private async staleRuns(tx: Transaction, tabId: string, state: SuperTabState): Promise<string[]> {
+ const active = new Set(state.conversations.filter(activeWork).map(c => c.assignment!.id))
+ const rows = await tx.superTabRun.findMany({ where: { tabId }, select: { assignmentId: true } })
+ return rows.map(r => r.assignmentId).filter(id => !id.startsWith('release-') && !active.has(id))
+ }
+
/** Records committed messages, each at the sequence its conversation gave it, and one delivery per recipient. */
private async writeMessages(tx: Transaction, tabId: string, state: SuperTabState, messages: MessageDraft[], sequences: ReadonlyMap<string, number>) {
for (const message of messages) {
@@ -1395,6 +1430,7 @@ export class SuperTabStore {
const saved = async (lost: MessageDraft[] = []) => {
const sequences = numberMessages(state, lost, now)
const touches = judgedMachines(state, lost).filter(t => t.key !== machineId)
+ await this.endRuns(tx, tabId, loaded.state, state)
const version = (await this.save(tx, loaded, state, stamp, touches)).versions.get(`machine:${machineId}`) ?? 0
await this.writeMessages(tx, tabId, state, lost, sequences)
return { version }
@@ -1505,6 +1541,12 @@ export class SuperTabStore {
woken = true
}
}
+ // The poll also frees agents whose task the checks just lost, and any an older server left marked running (each poll; a
+ // second finds none).
+ await this.endRuns(tx, tabId, loaded.state, state)
+ if (ledProject(state, agentId)) {
+ await this.releaseRuns(tx, tabId, await this.staleRuns(tx, tabId, state))
+ }
// One write for both: the time checks' changes and the wake-up's.
const sequences = numberMessages(state, written, this.now())
await this.save(tx, loaded, state, clock.next(), [...judgedMachines(state, written), ...woken ? [{ kind: 'agent' as const, key: agentId }] : []])