The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Swarm host: no starts in a Swarm while it is cleaned up, and a fresh read before each deletion (#2004)
0xken15dbd802e65138
diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.tsindex 2954a208c..aeb80d406 100644--- a/cli/src/superTabs/machines.spec.ts+++ b/cli/src/superTabs/machines.spec.ts@@ -37,6 +37,78 @@ function fixture(env: NodeJS.ProcessEnv = {}, privacy: () => PrivacyObservationstamp: (next: string) => { stamp = next } }}+// Second review of #1991: a start in a Swarm this host is cleaning up could write into a folder the cleanup then deleted.+describe('a Swarm this host is cleaning up', () => {+ const homes: string[] = []+ afterEach(() => { vi.unstubAllEnvs(); for (const home of homes.splice(0)) rmSync(home, { recursive: true, force: true }) })+ it('starts no agent there while its cleanup runs, and starts the waiting one once the cleanup ends', async () => {+ // The cleanup deletes Swarm folders under the home folder: a throwaway one.+ const home = mkdtempSync(join(tmpdir(), 'swarm-cleaning-')); homes.push(home)+ vi.stubEnv('HOME', home); vi.stubEnv('USERPROFILE', home)+ const f = fixture()+ let release!: () => void+ const held = new Promise<void>(resolve => { release = resolve })+ const answer = f.backend.getMockImplementation()!+ f.backend.mockImplementation(async (core, method, path, body) => method === 'GET' && path.startsWith('/api/super-tabs/tab/')+ ? held.then(() => swarmReply(path, {})) : answer(core, method, path, body))+ const cleaning = f.machine.clean('tab', 'removal').catch(() => null)+ await f.machine.tick()+ expect(f.start).not.toHaveBeenCalled()+ expect(f.log.mock.calls.flat().join('\n')).toContain('finish cleaning up this Swarm')+ release()+ await cleaning+ f.advance()+ await f.machine.tick()+ expect(f.start).toHaveBeenCalledTimes(1)+ })+})++// Review of #2004: a start passed the cleaning check, waited in its folder preparation, and launched after a cleanup began.+// A start holds its Swarm from that check through its launch, and a cleanup waits for the starts in flight, for a while.+describe('a start and a cleanup of one Swarm', () => {+ const homes: string[] = []+ afterEach(() => { vi.unstubAllEnvs(); for (const home of homes.splice(0)) rmSync(home, { recursive: true, force: true }) })+ function held(cleanupWaitMs?: number) {+ const home = mkdtempSync(join(tmpdir(), 'swarm-cleaning-')); homes.push(home)+ vi.stubEnv('HOME', home); vi.stubEnv('USERPROFILE', home)+ const f = fixture()+ let release!: () => void+ f.prepare.mockImplementationOnce(() => new Promise<void>(resolve => { release = resolve }))+ const answer = f.backend.getMockImplementation()!+ f.backend.mockImplementation(async (core, method, path, body) => method === 'GET' && path.startsWith('/api/super-tabs/tab/')+ ? swarmReply(path, {}) : answer(core, method, path, body))+ const machine = cleanupWaitMs === undefined ? f.machine : new ContributedMachines(f.core, 'host', { ...f.deps, cleanupWaitMs } as never)+ const swarmRead = () => f.backend.mock.invocationCallOrder.find((_order, i) => f.backend.mock.calls[i]![1] === 'GET'+ && String(f.backend.mock.calls[i]![2]).startsWith('/api/super-tabs/tab/'))+ return { f, machine, release: () => release(), swarmRead }+ }++ it('lets a start that passed the check launch before the cleanup touches anything', async () => {+ const { f, machine, release, swarmRead } = held()+ const pass = machine.tick()+ await vi.waitFor(() => expect(f.prepare).toHaveBeenCalled())+ const cleaning = machine.clean('tab', 'removal').catch(() => null)+ // Room for a cleanup that does not wait to read the Swarm and delete: what is checked is that nothing happens.+ await new Promise(resolve => setTimeout(resolve, 50))+ release()+ await pass+ await cleaning+ expect(f.start).toHaveBeenCalledTimes(1)+ expect(swarmRead()).toBeGreaterThan(f.start.mock.invocationCallOrder[0]!)+ })++ it('gives up waiting for a start that never finishes, with a clear error, and touches nothing', async () => {+ const { f, machine, release, swarmRead } = held(20)+ const pass = machine.tick()+ await vi.waitFor(() => expect(f.prepare).toHaveBeenCalled())+ await expect(machine.clean('tab', 'removal')).rejects.toThrow('an agent start there is still running')+ expect(swarmRead()).toBeUndefined()+ release()+ await pass+ expect(f.start).toHaveBeenCalledTimes(1)+ })+})+// #1726: the pass read GET /hosts/:m/pools, up to 100 whole Swarms, every 5 s. Its pools now come with the host form.describe('the pools of a contributed machine', () => {it('come from the host form, never the old pools roster', async () => {@@ -1508,7 +1580,8 @@ describe('removed machine cleanup', () => {it('keeps the worktree records and confirms nothing when this machine is added again while its folders are deleted', async () => {const m = machine()- addedOnRead(m, 3)+ m.backend.mockImplementation(async (_core, method, path) => method === 'GET' && path.startsWith('/api/super-tabs/tab/')+ ? swarmReply(path, { starts: m.starts, ...(existsSync(m.newRoot) ? {} : { machines: [{ machineId: 'host', contributorId: 'person', cwd: m.newRoot }] }) }) : { parts: [] })await m.open().clean('tab', 'removal')expect(existsSync(m.newRoot)).toBe(false)expect(records(m)).toEqual(['tab', 'other-tab'])@@ -1516,6 +1589,28 @@ describe('removed machine cleanup', () => {expect(m.log.mock.calls.flat().join('\n')).toContain('added to that Swarm again')})+ // Second review of #1991 (the Reviewer's regression, with real folders): the cleanup passed its Swarm read, then waited+ // on closing an old session; the owner added the machine again meanwhile and a new task wrote into its clone, which was+ // then deleted. The Swarm is read again after the sessions close and before each deletion.+ it('keeps the clone a new task wrote into when this machine is added again while an old session closes', async () => {+ const m = machine()+ let added = false+ m.backend.mockImplementation(async (_core, method, path) => {+ if (method !== 'GET' || !path.startsWith('/api/super-tabs/tab/')) return { parts: [] }+ return swarmReply(path, { starts: m.starts, ...(added ? { machines: [{ machineId: 'host', contributorId: 'person', cwd: m.newRoot }] } : {}) })+ })+ vi.mocked(m.core.agents.closeStarted!).mockImplementation(async () => {+ // The close was slow; the owner re-added the machine while it was awaited and its new start reused this clone.+ added = true+ m.setLive([])+ writeFileSync(join(m.newRoot, 'new-task.txt'), 'created after re-add')+ return { closed: true } as never+ })+ await m.open().clean('tab', 'removal')+ expect(existsSync(join(m.newRoot, 'new-task.txt'))).toBe(true)+ expect(m.acks()).toEqual([])+ })+it('cleans up while the Swarm still lists this machine under this very removal', async () => {const m = machine()m.backend.mockImplementation(async (_core, method, path) => method === 'GET' && path.startsWith('/api/super-tabs/tab/')diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.tsindex 538a5abcc..f142973e5 100644--- a/cli/src/superTabs/machines.ts+++ b/cli/src/superTabs/machines.ts@@ -116,6 +116,8 @@ export interface MachineDeps {sources?: SwarmSources/** git for the readiness of the Swarm folder's checkouts ([quietGit]). */git?: GitRun+ /** How long a cleanup waits for the agent starts already running in its Swarm ([CLEANUP_WAIT_MS]). */+ cleanupWaitMs?: number/** When each of these harnesses' screens last changed, epoch ms (superTabs/terminal.ts `progressOf`): the backend* renews a task's lease from it, so work never sits with a dead owner (docs: swarm-never-stops). */progress?: (harnessIds: readonly string[]) => Record<string, number>@@ -124,6 +126,9 @@ export interface MachineDeps {relayed?: () => boolean}const MACHINE_DEPS = { backend, open: openDaemon, capabilities: readCapabilities, start: startAgent, now: Date.now }+/** A start's folder preparation can clone a whole repository; past this, its cleanup gives up and tries again later. */+const CLEANUP_WAIT_MS = 10 * 60_000+const CLEANING_WAIT = 'Waiting for this machine to finish cleaning up this Swarm.'/** Separate from chat delivery: slow resource readers and agent launches never* hold up a delivery lease. Only a tab's explicit owner grant wakes this queue. */@@ -161,6 +166,13 @@ export class ContributedMachines {/** Removals cleaned and acknowledged by this host, and the one in progress per removal. */private readonly cleaned = new Set<string>()private readonly cleaning = new Map<string, Promise<void>>()+ /** One lock per Swarm between its cleanups and its starts (second review of #1991: a start wrote into a clone the+ * cleanup then deleted; review of #2004: a start that passed the check launched after a cleanup began). A cleanup+ * marks its Swarm for its whole run, then waits for the starts already in flight there; a start checks the mark and+ * joins the starts in flight in one step, with no wait between, and stays until its launch is done. A start that finds+ * the mark waits for a later pass, so a pass is never held up and this machine keeps reporting. */+ private readonly cleaningTabs = new Map<string, number>()+ private readonly startsInFlight = new Map<string, Set<Promise<void>>>()/** Per removal whose cleanup or acknowledgement failed, how often and when it may run again. */private readonly cleanupRetries = new Map<string, { failures: number; at: number }>()/** Until when the reading leaves out its privacy field, which the backend refused. */@@ -459,60 +471,64 @@ export class ContributedMachines {for (const start of round) {if (this.stopped) returnif (!identifier(start.id) || ((row.paused || pool.paused) && !start.dispatched)) continue- const path = `/api/super-tabs/hosts/${this.machineId}/starts/${row.tabId}/${start.id}`- // The start endpoint reads this start and takes its commands, nothing else. Any other read went out as an empty- // command, which the backend refused with a 400 at every start (prod logs 2026-10-28).- const scoped: ScopedRequest = async input => input.action !== 'startStatus' && input.action !== 'command'- ? { error: 'HOST_UNSUPPORTED', detail: 'A contributed machine reads only its own starts.' }- : { ok: true, value: await this.deps.backend(this.core, input.action === 'startStatus' ? 'GET' : 'POST', path,- input.action === 'startStatus' ? undefined : { operationId: input.operationId, command: input.command }) }+ if (this.cleaningTabs.has(String(row.tabId))) { this.noteWaiting(start.id, CLEANING_WAIT); continue }+ const launched = this.startInFlight(String(row.tabId))try {- // The Harness Team account is not private (its owner's home folder opened again, say): a new start is refused- // here as well as by routing, and never moved to the owner's account instead.- if (privacy?.mode === 'account' && !privacy.passed && !start.dispatched) {- await scoped({ action: 'command', operationId: `privacy-failed-${start.id}`,- command: { type: 'agent.startFailed', startId: start.id,- error: `This computer's Harness Team account is not private (${privacy.item}): ${privacy.detail} Its owner can run \`harness team setup\` to repair it.` } })- continue- }- // Another person's project is cloned and fetched with the Harness GitHub App's token, never this machine's own- // git login; a token that cannot be had fails the start (superTabs/projectFolder.ts).- let github: { requester: string } | null = null- if (!start.dispatched) {- try {- github = (await (this.deps.prepare ?? ((core, pooled, queued) => prepareSwarmStart(core, pooled, queued,- { backend: this.deps.backend, team: (this.deps.team ?? (() => readAuthSession()?.kind === 'team'))() })))(this.core, row, start)) ?? null- } catch (error) {- await scoped({ action: 'command', operationId: `prepare-failed-${start.id}`,+ const path = `/api/super-tabs/hosts/${this.machineId}/starts/${row.tabId}/${start.id}`+ // The start endpoint reads this start and takes its commands, nothing else. Any other read went out as an empty+ // command, which the backend refused with a 400 at every start (prod logs 2026-10-28).+ const scoped: ScopedRequest = async input => input.action !== 'startStatus' && input.action !== 'command'+ ? { error: 'HOST_UNSUPPORTED', detail: 'A contributed machine reads only its own starts.' }+ : { ok: true, value: await this.deps.backend(this.core, input.action === 'startStatus' ? 'GET' : 'POST', path,+ input.action === 'startStatus' ? undefined : { operationId: input.operationId, command: input.command }) }+ try {+ // The Harness Team account is not private (its owner's home folder opened again, say): a new start is refused+ // here as well as by routing, and never moved to the owner's account instead.+ if (privacy?.mode === 'account' && !privacy.passed && !start.dispatched) {+ await scoped({ action: 'command', operationId: `privacy-failed-${start.id}`,command: { type: 'agent.startFailed', startId: start.id,- error: error instanceof Error ? error.message : 'Could not prepare this project on the machine.' } })+ error: `This computer's Harness Team account is not private (${privacy.item}): ${privacy.detail} Its owner can run \`harness team setup\` to repair it.` } })continue}- }- const result = await this.deps.start({ tabId: row.tabId, startId: start.id, hostDispatch: true, ...(github ? { github } : {}) }, scoped, where, this.deps.open)- const refused = result.ok !== true && !start.dispatched && !NOT_REFUSED.has(String(result.error))- // Launched, and its creation still under way on the daemon: anything else is not preparing.- if (result.error === 'START_PENDING' && result.state === 'pending') this.preparing.add(String(start.id))- else this.preparing.delete(String(start.id))- if (!refused) this.refusedSince.delete(start.id)- if (result.ok === true && identifier(result.harnessId)) {- this.waiting.delete(start.id)- // Before the start settles: a settled start is no longer listed, so a missed record would never be made.- this.recordWorktree(String(row.tabId), start.id, result.harnessId, object(start.request))- await scoped({ action: 'command', operationId: `settled-${start.id}`,- command: { type: 'agent.startSettled', startId: start.id, warning: result.modelWarning ?? null } })- } else if (result.ok !== true) {- const reason = String(result.detail ?? (Array.isArray(result.reasons) ? result.reasons.join(' ') : result.error))- this.noteWaiting(start.id, reason)- const since = this.refusedSince.get(start.id) ?? began- if (refused) this.refusedSince.set(start.id, since)- if (refused && began - since >= START_REFUSED_MS) {- await scoped({ action: 'command', operationId: `refused-${start.id}`, command: { type: 'agent.startFailed', startId: start.id, error: reason } })+ // Another person's project is cloned and fetched with the Harness GitHub App's token, never this machine's own+ // git login; a token that cannot be had fails the start (superTabs/projectFolder.ts).+ let github: { requester: string } | null = null+ if (!start.dispatched) {+ try {+ github = (await (this.deps.prepare ?? ((core, pooled, queued) => prepareSwarmStart(core, pooled, queued,+ { backend: this.deps.backend, team: (this.deps.team ?? (() => readAuthSession()?.kind === 'team'))() })))(this.core, row, start)) ?? null+ } catch (error) {+ await scoped({ action: 'command', operationId: `prepare-failed-${start.id}`,+ command: { type: 'agent.startFailed', startId: start.id,+ error: error instanceof Error ? error.message : 'Could not prepare this project on the machine.' } })+ continue+ }+ }+ const result = await this.deps.start({ tabId: row.tabId, startId: start.id, hostDispatch: true, ...(github ? { github } : {}) }, scoped, where, this.deps.open)+ const refused = result.ok !== true && !start.dispatched && !NOT_REFUSED.has(String(result.error))+ // Launched, and its creation still under way on the daemon: anything else is not preparing.+ if (result.error === 'START_PENDING' && result.state === 'pending') this.preparing.add(String(start.id))+ else this.preparing.delete(String(start.id))+ if (!refused) this.refusedSince.delete(start.id)+ if (result.ok === true && identifier(result.harnessId)) {+ this.waiting.delete(start.id)+ // Before the start settles: a settled start is no longer listed, so a missed record would never be made.+ this.recordWorktree(String(row.tabId), start.id, result.harnessId, object(start.request))+ await scoped({ action: 'command', operationId: `settled-${start.id}`,+ command: { type: 'agent.startSettled', startId: start.id, warning: result.modelWarning ?? null } })+ } else if (result.ok !== true) {+ const reason = String(result.detail ?? (Array.isArray(result.reasons) ? result.reasons.join(' ') : result.error))+ this.noteWaiting(start.id, reason)+ const since = this.refusedSince.get(start.id) ?? began+ if (refused) this.refusedSince.set(start.id, since)+ if (refused && began - since >= START_REFUSED_MS) {+ await scoped({ action: 'command', operationId: `refused-${start.id}`, command: { type: 'agent.startFailed', startId: start.id, error: reason } })+ }}}- }- // Keep its durable receipt; another start must not be starved by this failure.- catch (error) { this.refusedSince.delete(start.id); this.noteWaiting(start.id, error instanceof Error ? error.message : String(error)) }+ // Keep its durable receipt; another start must not be starved by this failure.+ catch (error) { this.refusedSince.delete(start.id); this.noteWaiting(start.id, error instanceof Error ? error.message : String(error)) }+ } finally { launched() }}} catch { /* Revocation or one unavailable project cannot block another contribution. */ }}@@ -585,7 +601,7 @@ export class ContributedMachines {// Both name folders and an operation: nothing else may reach a path.if (!identifier(tabId) || !identifier(removalId)) return Promise.reject(new Error('Invalid machine removal.'))if (this.cleaned.has(removalId) || this.deps.now() < (this.cleanupRetries.get(removalId)?.at ?? 0)) return Promise.resolve()- const running = this.cleaning.get(removalId) ?? this.runCleanup(tabId, removalId)+ const running = this.cleaning.get(removalId) ?? this.cleanTab(tabId, removalId).then(() => { this.cleaned.add(removalId); this.cleanupRetries.delete(removalId) }, error => {// The row asks again every poll; a backend that keeps failing is asked a minute later, doubling up to half an hour.const retry = this.cleanupRetries.get(removalId) ?? { failures: 0, at: 0 }@@ -596,6 +612,36 @@ export class ContributedMachines {this.cleaning.set(removalId, running)return running}+ /** A start in flight in this Swarm until the returned call. */+ private startInFlight(tabId: string): () => void {+ let done!: () => void+ const flight = new Promise<void>(resolve => { done = resolve })+ const flights = this.startsInFlight.get(tabId) ?? new Set()+ this.startsInFlight.set(tabId, flights.add(flight))+ return () => {+ done()+ flights.delete(flight)+ if (!flights.size && this.startsInFlight.get(tabId) === flights) this.startsInFlight.delete(tabId)+ }+ }+ /** Runs one cleanup with its Swarm marked as cleaning for its whole run, once the starts already in flight there are done. */+ private async cleanTab(tabId: string, removalId: string) {+ this.cleaningTabs.set(tabId, (this.cleaningTabs.get(tabId) ?? 0) + 1)+ try {+ const flights = [...this.startsInFlight.get(tabId) ?? []]+ if (flights.length) {+ const waitMs = this.deps.cleanupWaitMs ?? CLEANUP_WAIT_MS+ let timer: NodeJS.Timeout | undefined+ const done = await Promise.race([Promise.all(flights).then(() => true),+ new Promise<false>(resolve => { timer = setTimeout(resolve, waitMs, false) })]).finally(() => clearTimeout(timer))+ if (!done) throw new Error(`Did not clean up Swarm ${tabId} yet: an agent start there is still running after ${Math.round(waitMs / 1000)} s. It tries again later.`)+ }+ await this.runCleanup(tabId, removalId)+ } finally {+ const left = this.cleaningTabs.get(tabId)! - 1+ if (left) this.cleaningTabs.set(tabId, left); else this.cleaningTabs.delete(tabId)+ }+ }private async runCleanup(tabId: string, removalId: string) {const file = swarmWorktreesFile(this.core.dataDir, this.machineId)const recorded = readWorktrees(file).filter(w => w.tabId === tabId)@@ -605,8 +651,9 @@ export class ContributedMachines {// A cleanup that waited past the backend's fallback gives way to its owner adding this machine again (backend machines.ts// cleanupOverdue, 2026-10-28). The Swarm listing this machine other than under this very removal means a newer add owns// the folders now: delete nothing, and confirm nothing, since the backend dropped this cleanup with that add. Read at- // the start, again before anything is closed or deleted, and again before the worktree records change (review of- // #1986), so an add that lands while this runs keeps what it made.+ // the start, again before anything is closed, again before each deletion (closing a session waits, and an add can land+ // meanwhile: second review of #1991), and again before the worktree records change, so an add that lands while this+ // runs keeps what it made.const read = async () => {try { return (await this.sources.reads.read(tabId)).state }catch (error) { if (!(error instanceof SuperTabHostError) || ![403, 404].includes(error.status)) throw error; return {} }@@ -649,6 +696,7 @@ export class ContributedMachines {const open = this.core.agents.live().map(a => a.cwd).filter((cwd): cwd is string => !!cwd)for (const target of targets) {if (open.some(cwd => within(target, cwd))) { this.log(`Kept ${target}: a session still runs in it.`); continue }+ if (addedAgain(await read())) returnconst source = candidates.find(c => c.path === target)?.source// git forgets the worktree in its repository too; a folder it refuses (a submodule, a lock) is deleted as a folder.if (source) await git(source, ['worktree', 'remove', '--force', target]).catch(() => null)