The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Swarm: removing an offline machine drops it at once, cleanup waits for its host (#1682)
0xken159ae70608c768
diff --git a/backend/integration/superTabs.test.ts b/backend/integration/superTabs.test.tsindex e0c761256..bdcafbc7e 100644--- a/backend/integration/superTabs.test.ts+++ b/backend/integration/superTabs.test.ts@@ -2537,6 +2537,12 @@ describe('machine removal', () => {const invitation = await run(tab, { type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })await run(tab, { type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)await run(tab, { type: 'machine.contribute', machineId: 'test-host', name: 'Host', cwd: '/fixture/repo', engines: ['codex'], maxAgents: 4 }, host)+ // Its host is here and reporting, so removing it waits for that host to acknowledge.+ const machine = (await store.hostPools(host, 'test-host')).find(pool => pool.tabId === tab)!.machine+ await store.observeMachine(host, tab, 'test-host', { generation: machine.generation,+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },+ quotas: [], tools: {}, runningHarnessIds: [], stoppedHarnessIds: [] })return tab}const cleanups = async (tab: string, from = store) => (await from.hostTabs(host, 'test-host')).filter(r => r.tabId === tab && 'cleanup' in r)@@ -2568,6 +2574,34 @@ describe('machine removal', () => {expect((await store.read(owner, tab)).state.machines).toEqual([])})+ it('drops a machine that went silent while cleaning, and a reconnecting host gets that cleanup once before its ack', async () => {+ const tab = await contributed()+ const generation = (await store.hostPools(host, 'test-host')).find(pool => pool.tabId === tab)!.machine.generation+ const removalId = (await run(tab, { type: 'machine.remove', machineId: 'test-host' }, host)).result.removalId as string+ // Its host goes silent for longer than three beats: a read leaves the machine out, but its stored removal is still there.+ const later = new SuperTabStore(db, () => Date.now() + 60_000)+ const stored = async () => (await db.superTab.findUniqueOrThrow({ where: { id: tab } })).state as unknown as SuperTabState+ expect((await later.read(owner, tab)).state.machines).toEqual([])+ expect((await stored()).machines!.map(m => m.removal?.id)).toEqual([removalId])+ // It reconnects and reports before acknowledging: that reading must not bring the machine back as cleaning.+ await later.observeMachine(host, tab, 'test-host', { generation,+ resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },+ quotas: [], tools: {}, runningHarnessIds: [], stoppedHarnessIds: [] })+ expect((await later.read(owner, tab)).state.machines).toEqual([])+ expect((await stored()).machines).toEqual([])+ // One cleanup, not one per poll.+ expect(await cleanups(tab, later)).toEqual([{ tabId: tab, cleanup: { machineId: 'test-host', removalId } }])+ expect(await cleanups(tab, later)).toEqual([{ tabId: tab, cleanup: { machineId: 'test-host', removalId } }])+ // A delayed acknowledgement closes it; a second, late one finds nothing left to undo.+ await later.execute(host, tab, { operationId: randomUUID(), command: { type: 'machine.cleaned', machineId: 'test-host', removalId } })+ expect(await cleanups(tab, later)).toEqual([])+ expect((await stored()).machineCleanups).toBeUndefined()+ const late = await app.inject({ method: 'POST', url: `/api/super-tabs/${tab}/commands`, headers: { authorization: `Bearer ${host.sub}` },+ payload: { operationId: randomUUID(), command: { type: 'machine.cleaned', machineId: 'test-host', removalId } } })+ expect(late.statusCode).toBe(404)+ })+it('lists a pending cleanup to its host even when the machine is in more than the 100 Swarms a host poll reads', async () => {const tab = await contributed()const removalId = (await run(tab, { type: 'machine.remove', machineId: 'test-host' }, host)).result.removalId as stringdiff --git a/backend/src/superTabs/commands.test.ts b/backend/src/superTabs/commands.test.tsindex 170126c04..de1854d53 100644--- a/backend/src/superTabs/commands.test.ts+++ b/backend/src/superTabs/commands.test.ts@@ -1505,10 +1505,15 @@ describe('the owner\'s rules, after review', () => {})describe('machine.remove', () => {- const contribute = (f: ReturnType<typeof fixture>, contributorId = 'owner', machineId = 'host') => {+ const observationAt = (receivedAt: number) => ({ generation: 'g1', receivedAt,+ resources: { cpuPercent: null, memoryTotalBytes: null, memoryUsedBytes: null, diskFreeBytes: null, diskTotalBytes: null,+ projectDiskFreeBytes: null, deviceType: null, chip: null, os: null, gpuPercent: null },+ tools: {}, quotas: [], runningHarnessIds: [], stoppedHarnessIds: [] })+ /** A contributed machine, online unless it is given no reading or an old one (the fixture's now is 1000). */+ const contribute = (f: ReturnType<typeof fixture>, contributorId = 'owner', machineId = 'host', receivedAt: number | null = 1000) => {const generation = f.state.people.find(p => p.id === contributorId)!.generationf.state.machines = [...(f.state.machines ?? []), { machineId, name: 'Studio', contributorId, membershipGeneration: generation, generation: 'g1',- cwd: '/repo', engines: ['claude'], maxAgents: 2, paused: false } as never]+ cwd: '/repo', engines: ['claude'], maxAgents: 2, paused: false, ...(receivedAt === null ? {} : { observation: observationAt(receivedAt) }) } as never]}const start = (f: ReturnType<typeof fixture>, group: string, dispatched = false) => {f.state.starts = [...(f.state.starts ?? []), { id: `start-${(f.state.starts ?? []).length}`, creationId: 'creation', actorKey: 'agent:lead', contributorId: 'owner',@@ -1625,6 +1630,8 @@ describe('machine.remove', () => {it('drops the machine\'s records after 24 hours without an acknowledgement, and keeps the cleanup for its host', () => {const { f, removalId } = cleaning()const day = 24 * 3_600_000+ // Its host keeps reporting, so only the day-long fallback can expire it.+ f.state.machines![0]!.observation!.receivedAt = 1000 + day - 1f.run({ type: 'tab.rename', name: 'Still cleaning' }, owner, { now: 1000 + day - 1 })expect(f.state.machines!.map(m => m.removal?.id)).toEqual([removalId])expect(visibleState(f.state, owner, 1000 + day).machines).toEqual([])@@ -1637,6 +1644,69 @@ describe('machine.remove', () => {expect(f.state.machineCleanups).toBeUndefined()})+ it.each([['has no reading', null], ['was last read long ago', 1000 - 45_001]] as const)(+ 'drops a machine that is offline (%s) at once, and keeps its cleanup for its host', (_why, receivedAt) => {+ const f = fixture()+ contribute(f, 'owner', 'host', receivedAt)+ // Listed, if offline, until its owner removes it.+ expect(visibleState(f.state, owner).machines).toHaveLength(1)+ const effects = f.run({ type: 'machine.remove', machineId: 'host' })+ const removalId = effects.result.removalId as string+ expect(f.state.machines).toEqual([])+ expect(f.state.machineCleanups).toEqual([{ machineId: 'host', removalId, machineOwnerId: 'owner' }])+ expect(visibleState(f.state, owner).machines).toEqual([])+ // Its host still gets the cleanup on its next connection; its later acknowledgement closes it and changes nothing else.+ f.run(cleaned(removalId))+ expect(f.state.machineCleanups).toBeUndefined()+ expect(f.state.machines).toEqual([])+ })++ it('hands its cleanup to its host once when it comes back, and a second acknowledgement changes nothing', () => {+ const f = fixture()+ contribute(f, 'owner', 'host', null)+ const removalId = f.run({ type: 'machine.remove', machineId: 'host' }).result.removalId as string+ // While its host is away, more reads and changes never queue the cleanup twice.+ const away = 1000 + 45_001+ expect(visibleState(f.state, owner, away).machines).toEqual([])+ f.run({ type: 'tab.rename', name: 'Still away' }, owner, { now: away })+ f.run({ type: 'tab.rename', name: 'Away a day' }, owner, { now: 1000 + 86_400_000 })+ expect(f.state.machineCleanups).toEqual([{ machineId: 'host', removalId, machineOwnerId: 'owner' }])+ // It comes back, is handed that one cleanup, and acknowledges it: nothing comes back to life.+ f.run(cleaned(removalId))+ expect(f.state.machineCleanups).toBeUndefined()+ expect(f.state.machines).toEqual([])+ // A late second acknowledgement finds nothing left to undo.+ expect(() => f.run(cleaned(removalId))).toThrow(expect.objectContaining({ code: 'NOT_FOUND' }))+ expect(f.state.machines).toEqual([])+ expect(f.state.machineCleanups).toBeUndefined()+ })++ it('drops a machine already cleaning once it is offline, on the next read and on the next change', () => {+ const { f, removalId } = cleaning()+ const later = 1000 + 45_001+ // A read that finds it offline leaves it out without writing.+ expect(visibleState(f.state, owner, later).machines).toEqual([])+ expect(visibleState(f.state, owner, later).machineCleanups).toBeUndefined()+ expect(f.state.machines).toEqual([expect.objectContaining({ machineId: 'host' })])+ // The next change applies the same rule and keeps the cleanup for its host.+ f.run({ type: 'tab.rename', name: 'Still cleaning' }, owner, { now: later })+ expect(f.state.machines).toEqual([])+ expect(f.state.machineCleanups).toEqual([{ machineId: 'host', removalId, machineOwnerId: 'owner' }])+ f.run(cleaned(removalId))+ expect(f.state.machineCleanups).toBeUndefined()+ })++ it('keeps a machine that is still online cleaning until its host acknowledges', () => {+ const { f, removalId } = cleaning()+ expect(visibleState(f.state, owner, 1000 + 45_000).machines).toEqual([+ expect.objectContaining({ machineId: 'host', paused: true, removal: { id: removalId, state: 'cleaning', requestedAt: new Date(1000).toISOString() } })])+ expect(f.state.machineCleanups).toBeUndefined()+ f.run({ type: 'tab.rename', name: 'Still online' }, owner, { now: 1000 + 45_000 })+ expect(f.state.machines!.map(m => m.removal?.id)).toEqual([removalId])+ f.run(cleaned(removalId))+ expect(f.state.machines).toEqual([])+ })+it('takes the machine out of every project\'s machines rule, telling each project\'s chat, and removes a rule left empty', () => {const f = fixture()contribute(f)diff --git a/backend/src/superTabs/commands.ts b/backend/src/superTabs/commands.tsindex cc411a4cf..75325f880 100644--- a/backend/src/superTabs/commands.ts+++ b/backend/src/superTabs/commands.ts@@ -1,5 +1,5 @@import { createHash } from 'node:crypto'-import { admitMachineStart, expireRemovals, machineContribution, ownedMachine, type ContributedMachine } from './machines.js'+import { admitMachineStart, expireRemoval, expireRemovals, machineContribution, machineOffline, ownedMachine, type ContributedMachine } from './machines.js'import { assignAgentDisplayNames } from './agentNames.js'import { z } from 'zod'import {@@ -466,7 +466,8 @@ export function applyCommand(original: SuperTabState, actor: Actor, input: Comma/** Takes a machine out of the Swarm with every agent on it (machine.remove, or its owner leaving): each agent is removed* like one agent, its pending starts fail, routing rules lose it, and a contributed machine stays paused and cleaning- * until its host deletes the Swarm's folders there (spec 2026-10-23-remove-machine.md). */+ * until its host deletes the Swarm's folders there — or leaves at once when that host is offline+ * (spec 2026-10-23-remove-machine.md). */const removeMachine = (machineId: string, pool: ContributedMachine | undefined, members: Agent[], how: (machine: string) => string) => {const pending = state.starts?.filter(s => s.request.machineId === machineId && !s.harnessId && !s.error) ?? []// A launch already under way would arrive with nowhere to go; it finishes in moments.@@ -510,6 +511,9 @@ export function applyCommand(original: SuperTabState, actor: Actor, input: Commatext: `${how(machine)}, and ${join} with it.${task ? ` Its task "${group.title}" needs to be assigned again or closed.` : ''}`,replyTo: null, recipients: lead ? [lead] : [], source: SYSTEM_ROW })}+ // Its host is offline and cannot acknowledge, so the day-long fallback applies at once: the machine leaves the list+ // now and its cleanup waits for its next connection (spec 2026-10-23-remove-machine.md).+ if (pool?.removal && machineOffline(pool, context.now)) expireRemoval(state, pool)return removed}diff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.tsindex 3af921fb6..79a7e72f2 100644--- a/backend/src/superTabs/machines.ts+++ b/backend/src/superTabs/machines.ts@@ -16,8 +16,9 @@ export interface ContributedMachine {maxAgents: numberpaused: booleanobservation?: MachineObservation & { receivedAt: number }- /** Set by machine.remove: the machine stays listed, paused, while its host deletes the Swarm's folders there, until the- * host says it cleaned up (machine.cleaned) or REMOVAL_FALLBACK_MS passes (expireRemovals). */+ /** Set by machine.remove: an online machine stays listed, paused, while its host deletes the Swarm's folders there, until+ * the host says it cleaned up (machine.cleaned) or REMOVAL_FALLBACK_MS passes. An offline one leaves the list at once and+ * its cleanup waits for its host (expireRemovals). */removal?: { id: string; state: 'cleaning'; requestedAt: string }}/** A removed machine whose host never acknowledged in time: its records are gone, its host still cleans up on its next poll. */@@ -124,11 +125,23 @@ export function removalPreview(state: SuperTabState, actor: Actor, machineId: st}}-/** A removal its host has not acknowledged in a day: the machine's records go, and the cleanup waits for its host. */+/** No reading yet, or one older than three missed beats: the machine Desktop shows as offline, and the one a start+ * is refused on. Nothing is written while a machine is silent, so this is read from the last reading itself. */+export function machineOffline(pool: ContributedMachine, now: number): boolean {+ return !pool.observation || now - pool.observation.receivedAt > MACHINE_FRESH_MS+}++/** A removal its host cannot acknowledge (offline) or has not acknowledged in a day: the machine's records go, and the+ * cleanup waits for its host on its next connection. */+export function expireRemoval(state: SuperTabState, pool: ContributedMachine): void {+ (state.machineCleanups ??= []).push({ machineId: pool.machineId, removalId: pool.removal!.id, machineOwnerId: pool.contributorId })+ state.machines = state.machines!.filter(m => m !== pool)+}+export function expireRemovals(state: SuperTabState, now: number): void {- for (const pool of state.machines?.filter(m => m.removal && now - Date.parse(m.removal.requestedAt) >= REMOVAL_FALLBACK_MS) ?? []) {- (state.machineCleanups ??= []).push({ machineId: pool.machineId, removalId: pool.removal!.id, machineOwnerId: pool.contributorId })- state.machines = state.machines!.filter(m => m !== pool)+ for (const pool of state.machines?.filter(m => m.removal+ && (machineOffline(m, now) || now - Date.parse(m.removal.requestedAt) >= REMOVAL_FALLBACK_MS)) ?? []) {+ expireRemoval(state, pool)}}@@ -215,9 +228,7 @@ export function admitMachineStart(state: SuperTabState, request: AgentStartRequeif (refused.length) {fail('OUTSIDE_CONTRIBUTION', `${pool.name} accepts only its contributed project folder, engines and default account settings, with full permissions for Swarm agents. Refused: ${refused.join('; ')}.`, 403)}- if (!pool.observation || now - pool.observation.receivedAt > MACHINE_FRESH_MS) {- fail('MACHINE_OFFLINE', 'Wait for a fresh reading from the contributed machine.')- }+ if (machineOffline(pool, now)) fail('MACHINE_OFFLINE', 'Wait for a fresh reading from the contributed machine.')if (!start && machineSlots(state, pool) >= pool.maxAgents) fail('MACHINE_FULL', 'This machine has reached its contributed agent limit.')return pool}diff --git a/backend/src/superTabs/model.ts b/backend/src/superTabs/model.tsindex c5738f0a8..19dbec89c 100644--- a/backend/src/superTabs/model.ts+++ b/backend/src/superTabs/model.ts@@ -400,8 +400,9 @@ export function visibleState(state: SuperTabState, actor: Actor, now = Date.now(delete visible.overlapAlerts// Only that machine's host roster lists them (store.ts hostTabs); a removed private agent's session is not every member's.delete visible.closingSessions- delete visible.machineCleanupsexpireRemovals(visible, now)+ // A read never hands out the cleanup records it just applied (store.ts hostTabs delivers them to the machine's host).+ delete visible.machineCleanupsdelete visible.parkedConversationsnumberConversations(visible)delete visible.retiredConversationNumberdiff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 69531c9fd..9812056cc 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -1,4 +1,4 @@-import { contributedEngines, hostLocalGrid, machineContribution, machineObservationSchema, machineSlots, recordAgentPresence, removalPreview, swarmGridRow, type MachineObservation } from './machines.js'+import { contributedEngines, expireRemovals, hostLocalGrid, machineContribution, machineObservationSchema, machineSlots, recordAgentPresence, removalPreview, swarmGridRow, type MachineObservation } from './machines.js'import { createHash } from 'node:crypto'import type { Prisma, PrismaClient, SuperTab, SuperTabDelivery } from '@prisma/client'import type { AuthUser } from '../lib/ssoAuth.js'@@ -1026,8 +1026,18 @@ export class SuperTabStore {if (!pool || pool.contributorId !== user.sub || pool.generation !== observation.generation) fail('CONTRIBUTION_REVOKED', 'Read the current machine contribution.', 403)const owned = await tx.machine.findFirst({ where: { machineId, userId: user.sub, autonomousEnv: user.autonomousEnv, ...machineAlive } })if (!owned || !machineBillingAllowsDataPlane(owned)) fail('HOST_UNAVAILABLE', 'This execution host is unavailable.', 403)- recordAgentPresence(state, machineId, observation.runningHarnessIds, this.now())- pool.observation = { ...observation, quotas: observation.quotas.filter(q => contributedEngines(pool, observation).includes(q.engine)), receivedAt: this.now() }+ const now = this.now()+ // A machine that went silent while cleaning takes the fallback before this fresh reading can bring it back: its records+ // go, its cleanup waits for this host, as the next sweep would have done (machines.ts expireRemovals).+ expireRemovals(state, now)+ if (!state.machines?.includes(pool)) {+ await tx.superTab.update({ where: { id: tabId, revision: row.revision }, data: {+ state: json(state), revision: row.revision + 1, ...indexes(state),+ } })+ return { revision: row.revision + 1 }+ }+ recordAgentPresence(state, machineId, observation.runningHarnessIds, now)+ pool.observation = { ...observation, quotas: observation.quotas.filter(q => contributedEngines(pool, observation).includes(q.engine)), receivedAt: now }// Current-generation physical stop proof also finishes a close whose membership was already withdrawn.for (const proof of observation.stoppedReservations ?? []) {if (!observation.stoppedHarnessIds.includes(proof.harnessId)) continue