Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Swarm: removing an offline machine drops it at once, cleanup waits for its host (#1682)

0xken159ae70608c768
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.ts
index e0c761256..bdcafbc7e 100644
--- a/backend/integration/superTabs.test.ts
+++ b/backend/integration/superTabs.test.ts
@@ -2537,6 +2537,12 @@ describe('machine removal', () => {
const invitation = await run(tab, { type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })
await run(tab, { type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)
await run(tab, { type: 'machine.contribute', machineId: 'test-host', name: 'Host', cwd: '/fixture/repo', engines: ['codex'], maxAgents: 4 }, host)
+ // Its host is here and reporting, so removing it waits for that host to acknowledge.
+ const machine = (await store.hostPools(host, 'test-host')).find(pool => pool.tabId === tab)!.machine
+ await store.observeMachine(host, tab, 'test-host', { generation: machine.generation,
+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,
+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },
+ quotas: [], tools: {}, runningHarnessIds: [], stoppedHarnessIds: [] })
return tab
}
const cleanups = async (tab: string, from = store) => (await from.hostTabs(host, 'test-host')).filter(r => r.tabId === tab && 'cleanup' in r)
@@ -2568,6 +2574,34 @@ describe('machine removal', () => {
expect((await store.read(owner, tab)).state.machines).toEqual([])
})
+ it('drops a machine that went silent while cleaning, and a reconnecting host gets that cleanup once before its ack', async () => {
+ const tab = await contributed()
+ const generation = (await store.hostPools(host, 'test-host')).find(pool => pool.tabId === tab)!.machine.generation
+ const removalId = (await run(tab, { type: 'machine.remove', machineId: 'test-host' }, host)).result.removalId as string
+ // Its host goes silent for longer than three beats: a read leaves the machine out, but its stored removal is still there.
+ const later = new SuperTabStore(db, () => Date.now() + 60_000)
+ const stored = async () => (await db.superTab.findUniqueOrThrow({ where: { id: tab } })).state as unknown as SuperTabState
+ expect((await later.read(owner, tab)).state.machines).toEqual([])
+ expect((await stored()).machines!.map(m => m.removal?.id)).toEqual([removalId])
+ // It reconnects and reports before acknowledging: that reading must not bring the machine back as cleaning.
+ await later.observeMachine(host, tab, 'test-host', { generation,
+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,
+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },
+ quotas: [], tools: {}, runningHarnessIds: [], stoppedHarnessIds: [] })
+ expect((await later.read(owner, tab)).state.machines).toEqual([])
+ expect((await stored()).machines).toEqual([])
+ // One cleanup, not one per poll.
+ expect(await cleanups(tab, later)).toEqual([{ tabId: tab, cleanup: { machineId: 'test-host', removalId } }])
+ expect(await cleanups(tab, later)).toEqual([{ tabId: tab, cleanup: { machineId: 'test-host', removalId } }])
+ // A delayed acknowledgement closes it; a second, late one finds nothing left to undo.
+ await later.execute(host, tab, { operationId: randomUUID(), command: { type: 'machine.cleaned', machineId: 'test-host', removalId } })
+ expect(await cleanups(tab, later)).toEqual([])
+ expect((await stored()).machineCleanups).toBeUndefined()
+ const late = await app.inject({ method: 'POST', url: `/api/super-tabs/${tab}/commands`, headers: { authorization: `Bearer ${host.sub}` },
+ payload: { operationId: randomUUID(), command: { type: 'machine.cleaned', machineId: 'test-host', removalId } } })
+ expect(late.statusCode).toBe(404)
+ })
+
it('lists a pending cleanup to its host even when the machine is in more than the 100 Swarms a host poll reads', async () => {
const tab = await contributed()
const removalId = (await run(tab, { type: 'machine.remove', machineId: 'test-host' }, host)).result.removalId as string
diff --git a/backend/src/superTabs/commands.test.ts b/backend/src/superTabs/commands.test.ts
index 170126c04..de1854d53 100644
--- a/backend/src/superTabs/commands.test.ts
+++ b/backend/src/superTabs/commands.test.ts
@@ -1505,10 +1505,15 @@ describe('the owner\'s rules, after review', () => {
})
describe('machine.remove', () => {
- const contribute = (f: ReturnType<typeof fixture>, contributorId = 'owner', machineId = 'host') => {
+ const observationAt = (receivedAt: number) => ({ generation: 'g1', receivedAt,
+ resources: { cpuPercent: null, memoryTotalBytes: null, memoryUsedBytes: null, diskFreeBytes: null, diskTotalBytes: null,
+ projectDiskFreeBytes: null, deviceType: null, chip: null, os: null, gpuPercent: null },
+ tools: {}, quotas: [], runningHarnessIds: [], stoppedHarnessIds: [] })
+ /** A contributed machine, online unless it is given no reading or an old one (the fixture's now is 1000). */
+ const contribute = (f: ReturnType<typeof fixture>, contributorId = 'owner', machineId = 'host', receivedAt: number | null = 1000) => {
const generation = f.state.people.find(p => p.id === contributorId)!.generation
f.state.machines = [...(f.state.machines ?? []), { machineId, name: 'Studio', contributorId, membershipGeneration: generation, generation: 'g1',
- cwd: '/repo', engines: ['claude'], maxAgents: 2, paused: false } as never]
+ cwd: '/repo', engines: ['claude'], maxAgents: 2, paused: false, ...(receivedAt === null ? {} : { observation: observationAt(receivedAt) }) } as never]
}
const start = (f: ReturnType<typeof fixture>, group: string, dispatched = false) => {
f.state.starts = [...(f.state.starts ?? []), { id: `start-${(f.state.starts ?? []).length}`, creationId: 'creation', actorKey: 'agent:lead', contributorId: 'owner',
@@ -1625,6 +1630,8 @@ describe('machine.remove', () => {
it('drops the machine\'s records after 24 hours without an acknowledgement, and keeps the cleanup for its host', () => {
const { f, removalId } = cleaning()
const day = 24 * 3_600_000
+ // Its host keeps reporting, so only the day-long fallback can expire it.
+ f.state.machines![0]!.observation!.receivedAt = 1000 + day - 1
f.run({ type: 'tab.rename', name: 'Still cleaning' }, owner, { now: 1000 + day - 1 })
expect(f.state.machines!.map(m => m.removal?.id)).toEqual([removalId])
expect(visibleState(f.state, owner, 1000 + day).machines).toEqual([])
@@ -1637,6 +1644,69 @@ describe('machine.remove', () => {
expect(f.state.machineCleanups).toBeUndefined()
})
+ it.each([['has no reading', null], ['was last read long ago', 1000 - 45_001]] as const)(
+ 'drops a machine that is offline (%s) at once, and keeps its cleanup for its host', (_why, receivedAt) => {
+ const f = fixture()
+ contribute(f, 'owner', 'host', receivedAt)
+ // Listed, if offline, until its owner removes it.
+ expect(visibleState(f.state, owner).machines).toHaveLength(1)
+ const effects = f.run({ type: 'machine.remove', machineId: 'host' })
+ const removalId = effects.result.removalId as string
+ expect(f.state.machines).toEqual([])
+ expect(f.state.machineCleanups).toEqual([{ machineId: 'host', removalId, machineOwnerId: 'owner' }])
+ expect(visibleState(f.state, owner).machines).toEqual([])
+ // Its host still gets the cleanup on its next connection; its later acknowledgement closes it and changes nothing else.
+ f.run(cleaned(removalId))
+ expect(f.state.machineCleanups).toBeUndefined()
+ expect(f.state.machines).toEqual([])
+ })
+
+ it('hands its cleanup to its host once when it comes back, and a second acknowledgement changes nothing', () => {
+ const f = fixture()
+ contribute(f, 'owner', 'host', null)
+ const removalId = f.run({ type: 'machine.remove', machineId: 'host' }).result.removalId as string
+ // While its host is away, more reads and changes never queue the cleanup twice.
+ const away = 1000 + 45_001
+ expect(visibleState(f.state, owner, away).machines).toEqual([])
+ f.run({ type: 'tab.rename', name: 'Still away' }, owner, { now: away })
+ f.run({ type: 'tab.rename', name: 'Away a day' }, owner, { now: 1000 + 86_400_000 })
+ expect(f.state.machineCleanups).toEqual([{ machineId: 'host', removalId, machineOwnerId: 'owner' }])
+ // It comes back, is handed that one cleanup, and acknowledges it: nothing comes back to life.
+ f.run(cleaned(removalId))
+ expect(f.state.machineCleanups).toBeUndefined()
+ expect(f.state.machines).toEqual([])
+ // A late second acknowledgement finds nothing left to undo.
+ expect(() => f.run(cleaned(removalId))).toThrow(expect.objectContaining({ code: 'NOT_FOUND' }))
+ expect(f.state.machines).toEqual([])
+ expect(f.state.machineCleanups).toBeUndefined()
+ })
+
+ it('drops a machine already cleaning once it is offline, on the next read and on the next change', () => {
+ const { f, removalId } = cleaning()
+ const later = 1000 + 45_001
+ // A read that finds it offline leaves it out without writing.
+ expect(visibleState(f.state, owner, later).machines).toEqual([])
+ expect(visibleState(f.state, owner, later).machineCleanups).toBeUndefined()
+ expect(f.state.machines).toEqual([expect.objectContaining({ machineId: 'host' })])
+ // The next change applies the same rule and keeps the cleanup for its host.
+ f.run({ type: 'tab.rename', name: 'Still cleaning' }, owner, { now: later })
+ expect(f.state.machines).toEqual([])
+ expect(f.state.machineCleanups).toEqual([{ machineId: 'host', removalId, machineOwnerId: 'owner' }])
+ f.run(cleaned(removalId))
+ expect(f.state.machineCleanups).toBeUndefined()
+ })
+
+ it('keeps a machine that is still online cleaning until its host acknowledges', () => {
+ const { f, removalId } = cleaning()
+ expect(visibleState(f.state, owner, 1000 + 45_000).machines).toEqual([
+ expect.objectContaining({ machineId: 'host', paused: true, removal: { id: removalId, state: 'cleaning', requestedAt: new Date(1000).toISOString() } })])
+ expect(f.state.machineCleanups).toBeUndefined()
+ f.run({ type: 'tab.rename', name: 'Still online' }, owner, { now: 1000 + 45_000 })
+ expect(f.state.machines!.map(m => m.removal?.id)).toEqual([removalId])
+ f.run(cleaned(removalId))
+ expect(f.state.machines).toEqual([])
+ })
+
it('takes the machine out of every project\'s machines rule, telling each project\'s chat, and removes a rule left empty', () => {
const f = fixture()
contribute(f)
diff --git a/backend/src/superTabs/commands.ts b/backend/src/superTabs/commands.ts
index cc411a4cf..75325f880 100644
--- a/backend/src/superTabs/commands.ts
+++ b/backend/src/superTabs/commands.ts
@@ -1,5 +1,5 @@
import { createHash } from 'node:crypto'
-import { admitMachineStart, expireRemovals, machineContribution, ownedMachine, type ContributedMachine } from './machines.js'
+import { admitMachineStart, expireRemoval, expireRemovals, machineContribution, machineOffline, ownedMachine, type ContributedMachine } from './machines.js'
import { assignAgentDisplayNames } from './agentNames.js'
import { z } from 'zod'
import {
@@ -466,7 +466,8 @@ export function applyCommand(original: SuperTabState, actor: Actor, input: Comma
/** Takes a machine out of the Swarm with every agent on it (machine.remove, or its owner leaving): each agent is removed
* like one agent, its pending starts fail, routing rules lose it, and a contributed machine stays paused and cleaning
- * until its host deletes the Swarm's folders there (spec 2026-10-23-remove-machine.md). */
+ * until its host deletes the Swarm's folders there — or leaves at once when that host is offline
+ * (spec 2026-10-23-remove-machine.md). */
const removeMachine = (machineId: string, pool: ContributedMachine | undefined, members: Agent[], how: (machine: string) => string) => {
const pending = state.starts?.filter(s => s.request.machineId === machineId && !s.harnessId && !s.error) ?? []
// A launch already under way would arrive with nowhere to go; it finishes in moments.
@@ -510,6 +511,9 @@ export function applyCommand(original: SuperTabState, actor: Actor, input: Comma
text: `${how(machine)}, and ${join} with it.${task ? ` Its task "${group.title}" needs to be assigned again or closed.` : ''}`,
replyTo: null, recipients: lead ? [lead] : [], source: SYSTEM_ROW })
}
+ // Its host is offline and cannot acknowledge, so the day-long fallback applies at once: the machine leaves the list
+ // now and its cleanup waits for its next connection (spec 2026-10-23-remove-machine.md).
+ if (pool?.removal && machineOffline(pool, context.now)) expireRemoval(state, pool)
return removed
}
diff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.ts
index 3af921fb6..79a7e72f2 100644
--- a/backend/src/superTabs/machines.ts
+++ b/backend/src/superTabs/machines.ts
@@ -16,8 +16,9 @@ export interface ContributedMachine {
maxAgents: number
paused: boolean
observation?: MachineObservation & { receivedAt: number }
- /** Set by machine.remove: the machine stays listed, paused, while its host deletes the Swarm's folders there, until the
- * host says it cleaned up (machine.cleaned) or REMOVAL_FALLBACK_MS passes (expireRemovals). */
+ /** Set by machine.remove: an online machine stays listed, paused, while its host deletes the Swarm's folders there, until
+ * the host says it cleaned up (machine.cleaned) or REMOVAL_FALLBACK_MS passes. An offline one leaves the list at once and
+ * its cleanup waits for its host (expireRemovals). */
removal?: { id: string; state: 'cleaning'; requestedAt: string }
}
/** A removed machine whose host never acknowledged in time: its records are gone, its host still cleans up on its next poll. */
@@ -124,11 +125,23 @@ export function removalPreview(state: SuperTabState, actor: Actor, machineId: st
}
}
-/** A removal its host has not acknowledged in a day: the machine's records go, and the cleanup waits for its host. */
+/** No reading yet, or one older than three missed beats: the machine Desktop shows as offline, and the one a start
+ * is refused on. Nothing is written while a machine is silent, so this is read from the last reading itself. */
+export function machineOffline(pool: ContributedMachine, now: number): boolean {
+ return !pool.observation || now - pool.observation.receivedAt > MACHINE_FRESH_MS
+}
+
+/** A removal its host cannot acknowledge (offline) or has not acknowledged in a day: the machine's records go, and the
+ * cleanup waits for its host on its next connection. */
+export function expireRemoval(state: SuperTabState, pool: ContributedMachine): void {
+ (state.machineCleanups ??= []).push({ machineId: pool.machineId, removalId: pool.removal!.id, machineOwnerId: pool.contributorId })
+ state.machines = state.machines!.filter(m => m !== pool)
+}
+
export function expireRemovals(state: SuperTabState, now: number): void {
- for (const pool of state.machines?.filter(m => m.removal && now - Date.parse(m.removal.requestedAt) >= REMOVAL_FALLBACK_MS) ?? []) {
- (state.machineCleanups ??= []).push({ machineId: pool.machineId, removalId: pool.removal!.id, machineOwnerId: pool.contributorId })
- state.machines = state.machines!.filter(m => m !== pool)
+ for (const pool of state.machines?.filter(m => m.removal
+ && (machineOffline(m, now) || now - Date.parse(m.removal.requestedAt) >= REMOVAL_FALLBACK_MS)) ?? []) {
+ expireRemoval(state, pool)
}
}
@@ -215,9 +228,7 @@ export function admitMachineStart(state: SuperTabState, request: AgentStartReque
if (refused.length) {
fail('OUTSIDE_CONTRIBUTION', `${pool.name} accepts only its contributed project folder, engines and default account settings, with full permissions for Swarm agents. Refused: ${refused.join('; ')}.`, 403)
}
- if (!pool.observation || now - pool.observation.receivedAt > MACHINE_FRESH_MS) {
- fail('MACHINE_OFFLINE', 'Wait for a fresh reading from the contributed machine.')
- }
+ if (machineOffline(pool, now)) fail('MACHINE_OFFLINE', 'Wait for a fresh reading from the contributed machine.')
if (!start && machineSlots(state, pool) >= pool.maxAgents) fail('MACHINE_FULL', 'This machine has reached its contributed agent limit.')
return pool
}
diff --git a/backend/src/superTabs/model.ts b/backend/src/superTabs/model.ts
index c5738f0a8..19dbec89c 100644
--- a/backend/src/superTabs/model.ts
+++ b/backend/src/superTabs/model.ts
@@ -400,8 +400,9 @@ export function visibleState(state: SuperTabState, actor: Actor, now = Date.now(
delete visible.overlapAlerts
// Only that machine's host roster lists them (store.ts hostTabs); a removed private agent's session is not every member's.
delete visible.closingSessions
- delete visible.machineCleanups
expireRemovals(visible, now)
+ // A read never hands out the cleanup records it just applied (store.ts hostTabs delivers them to the machine's host).
+ delete visible.machineCleanups
delete visible.parkedConversations
numberConversations(visible)
delete visible.retiredConversationNumber
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 69531c9fd..9812056cc 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -1,4 +1,4 @@
-import { contributedEngines, hostLocalGrid, machineContribution, machineObservationSchema, machineSlots, recordAgentPresence, removalPreview, swarmGridRow, type MachineObservation } from './machines.js'
+import { contributedEngines, expireRemovals, hostLocalGrid, machineContribution, machineObservationSchema, machineSlots, recordAgentPresence, removalPreview, swarmGridRow, type MachineObservation } from './machines.js'
import { createHash } from 'node:crypto'
import type { Prisma, PrismaClient, SuperTab, SuperTabDelivery } from '@prisma/client'
import type { AuthUser } from '../lib/ssoAuth.js'
@@ -1026,8 +1026,18 @@ export class SuperTabStore {
if (!pool || pool.contributorId !== user.sub || pool.generation !== observation.generation) fail('CONTRIBUTION_REVOKED', 'Read the current machine contribution.', 403)
const owned = await tx.machine.findFirst({ where: { machineId, userId: user.sub, autonomousEnv: user.autonomousEnv, ...machineAlive } })
if (!owned || !machineBillingAllowsDataPlane(owned)) fail('HOST_UNAVAILABLE', 'This execution host is unavailable.', 403)
- recordAgentPresence(state, machineId, observation.runningHarnessIds, this.now())
- pool.observation = { ...observation, quotas: observation.quotas.filter(q => contributedEngines(pool, observation).includes(q.engine)), receivedAt: this.now() }
+ const now = this.now()
+ // A machine that went silent while cleaning takes the fallback before this fresh reading can bring it back: its records
+ // go, its cleanup waits for this host, as the next sweep would have done (machines.ts expireRemovals).
+ expireRemovals(state, now)
+ if (!state.machines?.includes(pool)) {
+ await tx.superTab.update({ where: { id: tabId, revision: row.revision }, data: {
+ state: json(state), revision: row.revision + 1, ...indexes(state),
+ } })
+ return { revision: row.revision + 1 }
+ }
+ recordAgentPresence(state, machineId, observation.runningHarnessIds, now)
+ pool.observation = { ...observation, quotas: observation.quotas.filter(q => contributedEngines(pool, observation).includes(q.engine)), receivedAt: now }
// Current-generation physical stop proof also finishes a close whose membership was already withdrawn.
for (const proof of observation.stoppedReservations ?? []) {
if (!observation.stoppedHarnessIds.includes(proof.harnessId)) continue