Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Swarm: a removed machine whose account is deleted takes its pending cleanups with it (#1991)

0xken15b52c8f171b62
diff --git a/backend/integration/superTabTeam.test.ts b/backend/integration/superTabTeam.test.ts
index bcef8ff62..9ca8e4064 100644
--- a/backend/integration/superTabTeam.test.ts
+++ b/backend/integration/superTabTeam.test.ts
@@ -667,4 +667,24 @@ describe('one machine: a Mac and its Harness Team account', () => {
await add()
expect((await storedState(tab)).machineCleanups).toBeUndefined()
})
+
+ // Ken's remote-deepseek and WIN-DELL, 2026-10-28: removed while offline, then their machine accounts were deleted. Their
+ // cleanups waited for a host that could never come back, and kept the Swarm in every host's cleanup lookup.
+ it('drops the cleanup of a machine whose account is deleted on the Swarm\'s next change, and keeps a live machine\'s', async () => {
+ const tab = (await store.create(owner, randomUUID(), 'Retired machines')).id
+ for (const [machineId, computerId] of [['retired-pc', 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'], ['kept-pc', 'ffffffffffffffffffffffffffffffff']]) {
+ await db.machine.create({ data: { machineId, userId: owner.sub, computerId, sessionKind: 'computer', apiKey: `fixture-${machineId}`,
+ managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
+ onTestFinished(() => db.machine.delete({ where: { machineId } }).then(() => undefined))
+ await run(tab, { type: 'machine.contribute', machineId, name: machineId, cwd: '/work', scope: 'swarm', engines: ['codex'], maxAgents: 1 })
+ // Never reported, so offline: its records go at once and its cleanup waits for its host.
+ await run(tab, { type: 'machine.remove', machineId })
+ }
+ const pending = async () => ({ cleanups: (await storedState(tab)).machineCleanups?.map(c => c.machineId).sort() ?? [],
+ indexed: (await db.superTab.findUnique({ where: { id: tab } }))!.cleanupMachineIds.sort() })
+ expect(await pending()).toEqual({ cleanups: ['kept-pc', 'retired-pc'], indexed: ['kept-pc', 'retired-pc'] })
+ await db.machine.update({ where: { machineId: 'retired-pc' }, data: { deletedAt: new Date() } })
+ await run(tab, { type: 'tab.rename', name: 'Next change' })
+ expect(await pending()).toEqual({ cleanups: ['kept-pc'], indexed: ['kept-pc'] })
+ })
})
diff --git a/backend/src/superTabs/commands.test.ts b/backend/src/superTabs/commands.test.ts
index 1857595c9..539de02a1 100644
--- a/backend/src/superTabs/commands.test.ts
+++ b/backend/src/superTabs/commands.test.ts
@@ -2124,6 +2124,40 @@ describe('machine.remove', () => {
expect(f.state.machineCleanups).toBeUndefined()
})
+ // Ken's remote-deepseek and WIN-DELL, 2026-10-28: removed, then their machine accounts deleted, so no host would ever
+ // confirm their cleanups, which stayed for good. A machine whose account is gone takes its cleanups with it.
+ it('drops the cleanups of machines whose account is deleted, on any change, and says nothing', () => {
+ const f = fixture()
+ f.state.machineCleanups = [{ machineId: 'gone', removalId: 'a', machineOwnerId: 'owner' }, { machineId: 'host', removalId: 'b', machineOwnerId: 'owner' },
+ { machineId: 'gone', removalId: 'c', machineOwnerId: 'owner' }]
+ expect(f.run({ type: 'tab.rename', name: 'Tidy' }, owner, { deletedMachineIds: ['gone'] }).messages).toEqual([])
+ expect(f.state.machineCleanups).toEqual([{ machineId: 'host', removalId: 'b', machineOwnerId: 'owner' }])
+ f.run({ type: 'tab.rename', name: 'Tidier' }, owner, { deletedMachineIds: ['host'] })
+ expect(f.state.machineCleanups).toBeUndefined()
+ })
+
+ // Review of #1986 (Minor 2, intended): the adds a host makes by itself, the owner's machine running its Lead and a Mac's
+ // Harness Team account joining where the Mac is, take an overdue cleanup's place like the owner's own add; a fresh one
+ // still holds them off without an error.
+ it('lets the automatic adds take an overdue cleanup\'s place, and leaves a fresh one waiting', () => {
+ const f = fixture()
+ const fresh = new Date(1000 + 30 * 60_000 - 1).toISOString()
+ f.state.machineCleanups = [{ machineId: 'host', removalId: 'old', machineOwnerId: 'owner', requestedAt: fresh },
+ { machineId: 'team', removalId: 'old', machineOwnerId: 'owner', requestedAt: fresh }]
+ const add = (machineId: string) => ({ type: 'machine.contribute', machineId, name: 'Studio', cwd: '/repo', scope: 'swarm', engines: ['claude'], maxAgents: 2,
+ ifMissing: true }) as Command
+ const owners = (now: number, hostMachineId?: string) => ({ contribution: { machineOwnerId: 'owner', contributorId: 'owner', ...(hostMachineId ? { hostMachineId } : {}) }, now })
+ // Fresh: the owner's host skips its automatic add and nothing changes.
+ f.run(add('host'), owner, owners(1000 + 30 * 60_000 - 1))
+ expect([f.state.machines ?? [], f.state.machineCleanups!.map(c => c.machineId)]).toEqual([[], ['host', 'team']])
+ // Overdue: the owner's host adds its machine, then its Team account joins where that Mac is.
+ const later = 1000 + 60 * 60_000
+ f.run(add('host'), owner, owners(later))
+ f.run(add('team'), owner, owners(later, 'host'))
+ expect(f.state.machines!.map(m => [m.machineId, m.hostMachineId, m.removal])).toEqual([['host', undefined, undefined], ['team', 'host', undefined]])
+ expect(f.state.machineCleanups).toBeUndefined()
+ })
+
it('tells each project\'s chat that the machine left once its host confirmed the cleanup', () => {
const { f, removalId } = cleaning()
const docs = f.run({ type: 'project.create', name: 'Docs', visibility: 'public', brief: '' }).result as { conversationId: string }
diff --git a/backend/src/superTabs/commands.ts b/backend/src/superTabs/commands.ts
index 177774c78..267de0c49 100644
--- a/backend/src/superTabs/commands.ts
+++ b/backend/src/superTabs/commands.ts
@@ -243,6 +243,9 @@ export interface CommandContext {
confidentialSession?: boolean
/** For a command that registers a session or can make one confidential: the other Swarms' registrations (confidential.ts). */
registrations?: import('./projects.js').OtherRegistrations
+ /** The machines of this Swarm's pending cleanups whose machine account is deleted, read by the store: no host will ever
+ * confirm those cleanups (Ken's remote-deepseek and WIN-DELL, 2026-10-28), so they go. */
+ deletedMachineIds?: string[]
/** Read from host ownership/access records by the store, never copied from the request. */
contribution?: { machineOwnerId: string; contributorId: string; hostGrant?: string | null
/** A Harness Team account's machine: the Mac it lives on, from its account row (machines.ts ContributedMachine). */
@@ -467,6 +470,12 @@ export function applyCommand(original: SuperTabState, actor: Actor, input: Comma
numberConversations(state)
placeRules(state)
const ended = expireRemovals(state, context.now)
+ // Their removal already said they left; nothing is said again.
+ const deleted = new Set(context.deletedMachineIds ?? [])
+ if (deleted.size && state.machineCleanups) {
+ state.machineCleanups = state.machineCleanups.filter(c => !deleted.has(c.machineId))
+ if (!state.machineCleanups.length) delete state.machineCleanups
+ }
assignAgentDisplayNames(state.agents)
const result: Record<string, unknown> = {}
const effects: CommandEffects = { state, messages: ended, cancelledAgents: [], previousAttempts: [], routeDecisions: [], result }
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index fe33b89ba..2e79bb1c6 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -351,6 +351,15 @@ export class SuperTabStore {
}
}
+ /** The machines of the Swarm's pending cleanups whose account is deleted or missing: no host of theirs can read the
+ * Swarm again to confirm (read/host.ts asks for a live account), so applyCommand drops their cleanups. */
+ private async deletedMachines(tx: Transaction, state: SuperTabState): Promise<string[]> {
+ const ids = [...new Set((state.machineCleanups ?? []).map(c => c.machineId))]
+ const live = new Set((await tx.machine.findMany({ where: { machineId: { in: ids }, autonomousEnv: state.environment, ...machineAlive },
+ select: { machineId: true } })).map(m => m.machineId))
+ return ids.filter(id => !live.has(id))
+ }
+
/** The tab's contributed machines whose account row is signed in as a Harness Team account (`Machine.sessionKind`,
* lib/teamSession.ts): one of the two signs routing reads that a machine is a Team machine (machines.ts teamWorkCheck). */
/** Shared with the read API (superTabs/read, design section 7.2). */
@@ -910,6 +919,8 @@ export class SuperTabStore {
// The native receipt remains intact; the operation records the person's resolution.
} })
}
+ // Read only while the Swarm has a cleanup pending, which is rare: one indexed read of those machines' accounts.
+ if (original.machineCleanups?.length) context.deletedMachineIds = await this.deletedMachines(tx, original)
if (command.type === 'machine.contribute' || command.type === 'agent.add' || command.type === 'tab.project') {
context.contribution = await this.contribution(tx, original, actor, command)
}
diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.ts
index e3ed8c83d..2954a208c 100644
--- a/cli/src/superTabs/machines.spec.ts
+++ b/cli/src/superTabs/machines.spec.ts
@@ -1399,6 +1399,16 @@ describe('removed machine cleanup', () => {
setLive: (sessions: RegisteredSession[]) => { live = sessions }, step: (ms: number) => { now += ms } }
}
+ /** Its first acknowledgement fails as an unreachable backend would; every read still answers. */
+ const failFirstAck = (m: ReturnType<typeof machine>) => {
+ const answer = m.backend.getMockImplementation()!
+ let failed = false
+ m.backend.mockImplementation(async (core, method, path, body) => {
+ if (method === 'POST' && !failed) { failed = true; throw new Error('Backend offline') }
+ return answer(core, method, path, body)
+ })
+ }
+
it('removes a stopped worker\'s worktree once nothing runs there and nothing is uncommitted, and its branch once merged', async () => {
const m = machine()
const host = m.open()
@@ -1474,6 +1484,38 @@ describe('removed machine cleanup', () => {
expect(m.backend.mock.calls.filter(c => c[1] === 'GET')).toHaveLength(1)
})
+ // Review of #1986 (Minor 1): the Swarm was read once, at the start. An add that lands while the cleanup runs must keep its
+ // clone and its worktree records, so the Swarm is read again before anything is deleted and before the records change.
+ const addedOnRead = (m: ReturnType<typeof machine>, read: number) => {
+ let reads = 0
+ m.backend.mockImplementation(async (_core, method, path) => {
+ if (method !== 'GET' || !path.startsWith('/api/super-tabs/tab/')) return { parts: [] }
+ if (path.includes('/parts')) reads++
+ return swarmReply(path, { starts: m.starts, ...(reads >= read ? { machines: [{ machineId: 'host', contributorId: 'person', cwd: m.newRoot }] } : {}) })
+ })
+ }
+ const records = (m: ReturnType<typeof machine>) => JSON.parse(readFileSync(swarmWorktreesFile(m.data, 'host'), 'utf8')).worktrees.map((w: { tabId: string }) => w.tabId)
+
+ it('closes and deletes nothing when this machine is added again after the cleanup started', async () => {
+ const m = machine()
+ addedOnRead(m, 2)
+ await m.open().clean('tab', 'removal')
+ for (const kept of [join(m.data, 'swarm-projects', 'tab'), m.newRoot, m.recorded, m.earlier]) expect(existsSync(kept), kept).toBe(true)
+ expect(m.core.agents.closeStarted).not.toHaveBeenCalled()
+ expect(records(m)).toEqual(['tab', 'other-tab'])
+ expect(m.acks()).toEqual([])
+ })
+
+ it('keeps the worktree records and confirms nothing when this machine is added again while its folders are deleted', async () => {
+ const m = machine()
+ addedOnRead(m, 3)
+ await m.open().clean('tab', 'removal')
+ expect(existsSync(m.newRoot)).toBe(false)
+ expect(records(m)).toEqual(['tab', 'other-tab'])
+ expect(m.acks()).toEqual([])
+ expect(m.log.mock.calls.flat().join('\n')).toContain('added to that Swarm again')
+ })
+
it('cleans up while the Swarm still lists this machine under this very removal', async () => {
const m = machine()
m.backend.mockImplementation(async (_core, method, path) => method === 'GET' && path.startsWith('/api/super-tabs/tab/')
@@ -1505,7 +1547,7 @@ describe('removed machine cleanup', () => {
it('waits longer after each failed acknowledgement instead of cleaning again every pass', async () => {
const m = machine()
const host = m.open()
- m.backend.mockImplementationOnce(async (_core, _method, path) => swarmReply(path, { starts: [] })).mockRejectedValueOnce(new Error('Backend offline'))
+ failFirstAck(m)
await expect(host.clean('tab', 'removal')).rejects.toThrow('Backend offline')
const calls = m.backend.mock.calls.length
await host.clean('tab', 'removal')
@@ -1538,7 +1580,7 @@ describe('removed machine cleanup', () => {
it('finishes a cleanup after a restart that interrupted its acknowledgement', async () => {
const m = machine()
- m.backend.mockImplementationOnce(async (_core, _method, path) => swarmReply(path, { starts: [] })).mockRejectedValueOnce(new Error('Backend offline'))
+ failFirstAck(m)
await expect(m.open().clean('tab', 'removal')).rejects.toThrow('Backend offline')
expect(existsSync(m.recorded)).toBe(false)
await m.open().clean('tab', 'removal')
diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.ts
index 606b769c4..538a5abcc 100644
--- a/cli/src/superTabs/machines.ts
+++ b/cli/src/superTabs/machines.ts
@@ -602,18 +602,24 @@ export class ContributedMachines {
// Starts from before the record: a worktree counts only when this machine's receipt for that start created the
// very agent the registry lists as agent-started in it. A start receipt read is required; a tab gone or no longer
// readable by this owner leaves only the record.
- let starts: Record<string, unknown>[] = []
- let state: Record<string, unknown> = {}
- try { state = (await this.sources.reads.read(tabId)).state; starts = list(state.starts) }
- catch (error) { if (!(error instanceof SuperTabHostError) || ![403, 404].includes(error.status)) throw error }
// A cleanup that waited past the backend's fallback gives way to its owner adding this machine again (backend machines.ts
// cleanupOverdue, 2026-10-28). The Swarm listing this machine other than under this very removal means a newer add owns
- // the folders now: delete nothing, and confirm nothing, since the backend dropped this cleanup with that add.
- const listed = list(state.machines).find(m => m.machineId === this.machineId)
- if (listed && object(listed.removal).id !== removalId) {
+ // the folders now: delete nothing, and confirm nothing, since the backend dropped this cleanup with that add. Read at
+ // the start, again before anything is closed or deleted, and again before the worktree records change (review of
+ // #1986), so an add that lands while this runs keeps what it made.
+ const read = async () => {
+ try { return (await this.sources.reads.read(tabId)).state }
+ catch (error) { if (!(error instanceof SuperTabHostError) || ![403, 404].includes(error.status)) throw error; return {} }
+ }
+ const addedAgain = (state: Record<string, unknown>) => {
+ const listed = list(state.machines).find(m => m.machineId === this.machineId)
+ if (!listed || object(listed.removal).id === removalId) return false
this.log(`Kept the Swarm folders of ${tabId}: this machine was added to that Swarm again.`)
- return
+ return true
}
+ const state = await read()
+ if (addedAgain(state)) return
+ const starts = list(state.starts)
const swarm = new Set(recorded.map(w => w.harnessId))
const candidates = recorded.map(w => ({ source: w.source, path: w.path, startId: w.startId, proven: true }))
for (const start of starts) {
@@ -632,6 +638,7 @@ export class ContributedMachines {
if (c.proven && await swarmWorktree(c.path, c.source, c.startId)) worktrees.push(c.path)
else if (existsSync(c.path)) this.log(`Kept ${c.path}: it is not provably a worktree a Swarm start made.`)
}
+ if (addedAgain(await read())) return
const targets = [...worktrees, ...swarmCloneRoots(this.core.dataDir, tabId)]
const inside = (cwd: string | null) => !!cwd && targets.some(t => within(t, cwd))
for (const agent of this.core.agents.live()) {
@@ -647,6 +654,7 @@ export class ContributedMachines {
if (source) await git(source, ['worktree', 'remove', '--force', target]).catch(() => null)
await rm(target, { recursive: true, force: true })
}
+ if (addedAgain(await read())) return
writeWorktrees(file, readWorktrees(file).filter(w => w.tabId !== tabId))
await this.deps.backend(this.core, 'POST', `/api/super-tabs/${tabId}/commands`, { operationId: `machine-cleaned-${removalId}`,
command: { type: 'machine.cleaned', machineId: this.machineId, removalId } })