Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(swarm): stop workers from pre-#1527 starts and tell the Lead about refused stops (#1543)

0xdiego2541ddcda1ab3f
diff --git a/cli/src/superTabs/host.spec.ts b/cli/src/superTabs/host.spec.ts
index 0599969ba..c6475aaad 100644
--- a/cli/src/superTabs/host.spec.ts
+++ b/cli/src/superTabs/host.spec.ts
@@ -570,15 +570,15 @@ it('retries a lost removal acknowledgement from native stopped evidence without
})
-function releaseFixture(nativeKind = 'swarm', result: Record<string, unknown> = { error: 'NOT_SWARM_STARTED' }) {
+function releaseFixture(nativeKind = 'swarm', result: Record<string, unknown> = { error: 'NOT_SWARM_STARTED' }, extra: Record<string, unknown> = {}) {
const t = setup(), host = t.open()
const identity = { creationId: 'super-original-creation', sessionId: 'native', createdAt: new Date(0).toISOString() }
- const target = { id: 'agent', harnessId: 'harness', machineId: 'machine-1', workDecision: 'stop', releaseId: 'release-1',
- ...(nativeKind === 'missing-record' ? {} : { startedSession: identity }) }
- const native = { agentId: 'harness', sessionId: nativeKind === 'changed' ? 'replacement' : 'native', registeredAt: 0,
- startedBy: nativeKind === 'personal' ? undefined : 'agent', processIdentity: { pid: 1, startMarker: 'first', executable: 'codex' } }
+ const target = { id: 'agent', harnessId: 'harness', machineId: 'machine-1', workDecision: 'stop', releaseId: 'release-1', name: 'Permission builder',
+ ...(['missing-record', 'legacy', 'legacy-personal'].includes(nativeKind) ? {} : { startedSession: identity }) }
+ const native = { agentId: 'harness', sessionId: nativeKind === 'changed' ? 'replacement' : 'native', registeredAt: 0, boundAt: 3_000 as number | null,
+ startedBy: ['personal', 'legacy-personal'].includes(nativeKind) ? undefined : 'agent', processIdentity: { pid: 1, startMarker: 'first', executable: 'codex' } }
const lookup = vi.fn(async () => ({ creationId: identity.creationId, state: 'created' as const, agentId: 'harness' }))
- const close = vi.fn(async () => result), lease = vi.fn(), remove = vi.fn()
+ const close = vi.fn(async () => result), lease = vi.fn(), remove = vi.fn(), sent = vi.fn()
t.core.agents.byAgent = vi.fn(() => native as never)
t.core.agents.creationStatus = lookup
t.core.agents.closeSwarm = close as NonNullable<CoreApi['agents']['closeSwarm']>
@@ -586,13 +586,127 @@ function releaseFixture(nativeKind = 'swarm', result: Record<string, unknown> =
const command = (body as any)?.command
if (command?.type === 'agent.stopLease') { lease(); return { status: 200, body: { data: { result: { durationMs: 120_000 } } } } }
if (command?.type === 'agent.remove') { remove(); return { status: 200, body: { data: {} } } }
+ if (command?.type === 'message.send') { sent(body); return { status: 200, body: { data: {} } } }
if (path.includes('/hosts/')) return { status: 200, body: { data: [{ tabId: 'tab', agent: target, stopRequested: true }] } }
- return { status: 200, body: { data: { state: { agents: [target], conversations: [] } } } }
+ return { status: 200, body: { data: { state: { agents: [target], conversations: [], ...extra } } } }
}
const poll = async (advance = 0) => { t.wait(advance); await host.tick(); await new Promise(resolve => setImmediate(resolve)) }
- return { ...t, host, target, native, lookup, close, lease, remove, poll }
+ return { ...t, host, target, native, lookup, close, lease, remove, sent, poll }
}
+// A Swarm start before #1527 left only its start record and the native creation receipt (super-<startId>).
+const group = { id: 'group', leadId: 'lead', agentIds: ['lead', 'agent'], assignment: { agentId: 'agent', status: 'done' } }
+const legacyStarts = { starts: [{ id: 'start-1', creationId: 'super-start-1', harnessId: 'harness', request: { machineId: 'machine-1', name: 'Permission builder' }, error: null }] }
+
+it('closes a worker a pre-#1527 Swarm start launched, proven by its start record and creation receipt', async () => {
+ const t = releaseFixture('legacy', { closed: true }, legacyStarts)
+ t.lookup.mockResolvedValue({ creationId: 'super-start-1', state: 'created', agentId: 'harness' })
+ await t.poll()
+ await vi.waitFor(() => expect(t.remove).toHaveBeenCalledOnce())
+ expect(t.lookup).toHaveBeenCalledWith('super-start-1')
+ expect(t.close).toHaveBeenCalledWith(expect.objectContaining({ agentId: 'harness', creationId: 'super-start-1', sessionId: 'native', createdAt: new Date(0).toISOString() }))
+ expect(t.sent).not.toHaveBeenCalled()
+})
+
+it.each([
+ ['a person started it', 'legacy-personal', legacyStarts],
+ ['no start record names it', 'legacy', { starts: [{ ...legacyStarts.starts[0], harnessId: 'another-harness' }] }],
+ ['its start record is for another machine', 'legacy', { starts: [{ ...legacyStarts.starts[0], request: { machineId: 'machine-2' } }] }],
+ ['its start failed', 'legacy', { starts: [{ ...legacyStarts.starts[0], error: 'Native creation failed.' }] }],
+ ['its creation id is not the start\'s', 'legacy', { starts: [{ ...legacyStarts.starts[0], creationId: 'person-made-creation' }] }],
+])('never closes a similarly named worker when %s', async (_why, kind, extra) => {
+ const t = releaseFixture(kind, { closed: true }, extra)
+ for (let i = 0; i < 10; i++) await t.poll(130_000)
+ expect(t.lease).not.toHaveBeenCalled()
+ expect(t.close).not.toHaveBeenCalled()
+ expect(t.remove).not.toHaveBeenCalled()
+})
+
+it.each([['/clear', 3_600_000], ['an unknown bind', null], ['a bind before launch', -1]])('refuses, as changed, a pre-#1527 worker whose session moved on (%s)', async (_why, boundAt) => {
+ const t = releaseFixture('legacy', { closed: true }, { ...legacyStarts, conversations: [group] })
+ t.native.boundAt = boundAt
+ for (let i = 0; i < 10; i++) await t.poll(130_000)
+ expect(t.lease).not.toHaveBeenCalled()
+ expect(t.close).not.toHaveBeenCalled()
+ expect(t.sent).toHaveBeenCalledOnce()
+ expect(t.sent).toHaveBeenCalledWith(expect.objectContaining({ command: expect.objectContaining({ text: expect.stringContaining('it now runs a different session than the one the Swarm started') }) }))
+})
+
+it('retries a pre-#1527 stop while its creation receipt is still pending, then closes it', async () => {
+ const t = releaseFixture('legacy', { closed: true }, { ...legacyStarts, conversations: [group] })
+ t.lookup.mockResolvedValue({ creationId: 'super-start-1', state: 'pending' } as never)
+ await t.poll()
+ expect(t.lookup).toHaveBeenCalledOnce()
+ expect(t.lease).not.toHaveBeenCalled()
+ t.lookup.mockResolvedValue({ creationId: 'super-start-1', state: 'created', agentId: 'harness' })
+ await t.poll(5_000)
+ await vi.waitFor(() => expect(t.remove).toHaveBeenCalledOnce())
+ expect(t.lease).toHaveBeenCalledOnce()
+ expect(t.sent).not.toHaveBeenCalled()
+})
+
+it('rejects a pre-#1527 start whose receipt names a different native harness', async () => {
+ const t = releaseFixture('legacy', { closed: true }, legacyStarts)
+ t.lookup.mockResolvedValue({ creationId: 'super-start-1', state: 'created', agentId: 'another-harness' })
+ for (let i = 0; i < 10; i++) await t.poll(130_000)
+ expect(t.lease).not.toHaveBeenCalled()
+ expect(t.close).not.toHaveBeenCalled()
+})
+
+it.each([
+ ['personal', {}, 'it was not started by the Swarm'],
+ ['changed', {}, 'it now runs a different session than the one the Swarm started'],
+ ['swarm', { error: 'NOT_SWARM_STARTED' }, 'it was not started by the Swarm'],
+ ['swarm', { error: 'AGENT_CHANGED' }, 'it now runs a different session than the one the Swarm started'],
+])('tells the Lead once when a %s stop is refused for good (%o)', async (kind, result, why) => {
+ const t = releaseFixture(kind, result, { conversations: [{ id: 'lead-only', leadId: null, agentIds: ['lead'] }, group] })
+ vi.mocked(t.core.machine.name).mockReturnValue('MacbookPro')
+ for (let i = 0; i < 10; i++) await t.poll(130_000)
+ expect(t.sent).toHaveBeenCalledOnce()
+ expect(t.sent).toHaveBeenCalledWith({ operationId: 'stop-refused-release-1', asAgentId: 'agent', command: { type: 'message.send', conversationId: 'group', replyTo: null,
+ recipientIds: ['lead'], text: `Could not stop Permission builder on MacbookPro: ${why}. Close it on that machine.` } })
+})
+
+it('tells the Lead nothing through an archived group, which would refuse the notice', async () => {
+ const t = releaseFixture('personal', {}, { conversations: [{ ...group, archived: true }] })
+ for (let i = 0; i < 3; i++) await t.poll(130_000)
+ expect(t.sent).not.toHaveBeenCalled()
+})
+
+it('says the session changed when one is back after the worker had closed', async () => {
+ const t = releaseFixture('swarm', { closed: true }, { conversations: [group] })
+ t.remove.mockImplementationOnce(() => { throw new Error('Lost acknowledgement') })
+ await t.poll()
+ t.close.mockResolvedValue({ error: 'NOT_SWARM_STARTED' })
+ await t.poll(5_000)
+ await vi.waitFor(() => expect(t.sent).toHaveBeenCalledWith(expect.objectContaining({ command: expect.objectContaining({ text: expect.stringContaining('it now runs a different session') }) })))
+})
+
+it('tells no one of a transient close failure, and logs a notice that cannot be sent', async () => {
+ const quiet = releaseFixture('swarm', { error: 'SERVICE_UNAVAILABLE' }, { conversations: [group] })
+ for (let i = 0; i < 10; i++) await quiet.poll(130_000)
+ expect(quiet.sent).not.toHaveBeenCalled()
+ const lost = releaseFixture('personal', {}, { conversations: [group] })
+ lost.sent.mockImplementation(() => { throw new Error('Backend unavailable') })
+ await lost.poll()
+ await vi.waitFor(() => expect(lost.log).toHaveBeenCalledWith('Backend unavailable'))
+})
+
+it('names the worker by its member name, else plainly, and sends nothing without a Lead in its group', async () => {
+ const named = releaseFixture('personal', {}, { conversations: [group] })
+ Object.assign(named.target, { displayName: 'Wren' })
+ await named.poll()
+ await vi.waitFor(() => expect(named.sent).toHaveBeenCalledWith(expect.objectContaining({ command: expect.objectContaining({ text: expect.stringContaining('Could not stop Wren on') }) })))
+ const unnamed = releaseFixture('personal', {}, { conversations: [group] })
+ Reflect.deleteProperty(unnamed.target, 'name')
+ await unnamed.poll()
+ await vi.waitFor(() => expect(unnamed.sent).toHaveBeenCalledWith(expect.objectContaining({ command: expect.objectContaining({ text: expect.stringContaining('Could not stop this worker on') }) })))
+ const leaderless = releaseFixture('personal', {}, { conversations: [{ ...group, leadId: null }, { ...group, id: 'gone', agentIds: ['agent'] }] })
+ for (let i = 0; i < 3; i++) await leaderless.poll(130_000)
+ expect(leaderless.sent).not.toHaveBeenCalled()
+ expect(leaderless.log).toHaveBeenCalledWith(expect.stringContaining('not the original Swarm launch'))
+})
+
it.each(['personal', 'missing-record', 'changed'])('requests no close lease for an ineligible %s runtime', async kind => {
const t = releaseFixture(kind)
for (let i = 0; i < 10; i++) await t.poll(1_000)
diff --git a/cli/src/superTabs/host.ts b/cli/src/superTabs/host.ts
index 5d10d33a4..93e606306 100644
--- a/cli/src/superTabs/host.ts
+++ b/cli/src/superTabs/host.ts
@@ -50,6 +50,8 @@ export interface SuperTabHostOptions {
}
export const hostJournalFile = (dataDir: string, machineId: string) => join(dataDir, 'super-tabs', `host-${machineId}.json`)
+/** How long after a pre-#1527 Swarm launch its engine may first bind a session and still be the Swarm's own. */
+const LEGACY_FIRST_BIND_MS = 120_000
const typedId = (entry: Entry) => `team:service:superTabs:${entry.deliveryId}`
const writeJournal = (file: string, journal: Journal) => {
mkdirSync(dirname(file), { recursive: true, mode: 0o700 })
@@ -242,15 +244,46 @@ export class SuperTabHost {
const work = object(c.assignment)
return work.agentId === agent.id && !['done', 'stopped'].includes(String(work.status))
})) return
+ // A refusal no retry can change is told to the Lead once: until 2026-10-22 it went only to this machine's log,
+ // and a stopped worker stayed open with no one told why.
+ const refuse = (error: unknown) => {
+ attempt.retryAt = Infinity
+ const why = error === 'AGENT_CHANGED' ? 'it now runs a different session than the one the Swarm started' : 'it was not started by the Swarm'
+ this.log(`This session is not the original Swarm launch (${String(error)}). Remove its stuck contribution.`)
+ const group = (current.conversations as unknown[]).map(object).find(c => identifier(c.leadId) && c.leadId !== agent.id
+ && c.archived !== true && Array.isArray(c.agentIds) && c.agentIds.includes(agent.id) && c.agentIds.includes(c.leadId))
+ if (!group) return
+ // The operation id makes a restarted host's repeat of the same refusal one message, not two.
+ void backend(this.core, 'POST', `/api/super-tabs/${tabId}/commands`, {
+ operationId: `stop-refused-${agent.releaseId ?? agent.id}`, asAgentId: agent.id,
+ command: { type: 'message.send', conversationId: group.id, replyTo: null, recipientIds: [group.leadId],
+ text: `Could not stop ${String(member.displayName ?? member.name ?? 'this worker')} on ${this.core.machine.name()}: ${why}. Close it on that machine.` },
+ }).catch(failure => this.log(failure instanceof Error ? failure.message : String(failure)))
+ }
if (attempt.closed) {
// A failed acknowledgement can outlive the native stop; a fresh core read must still rule out personal resume.
const confirmed = await this.core.agents.closeSwarm({ agentId: agent.harnessId })
if (!confirmed.closed) {
- if (['NOT_SWARM_STARTED', 'AGENT_CHANGED'].includes(String(confirmed.error))) attempt.retryAt = Infinity
+ // It closed once; a session there now was resumed or replaced after that close.
+ if (['NOT_SWARM_STARTED', 'AGENT_CHANGED'].includes(String(confirmed.error))) refuse('AGENT_CHANGED')
throw new Error(confirmed.detail ?? confirmed.error ?? 'The stopped session could not be confirmed.')
}
} else {
- const identity = object(member.startedSession)
+ let identity = object(member.startedSession)
+ if (!identity.creationId && native?.startedBy === 'agent' && !native.resumeOnly && native.sessionId) {
+ // Swarm starts before #1527 recorded no session identity, so their workers were never closed. The start record
+ // names the native creation receipt it dispatched (super-<startId>) and the harness that joined from it; the
+ // receipt is read below and again by the core. A name or folder alone is never proof.
+ const start = Array.isArray(current.starts) ? current.starts.map(object).find(s => s.harnessId === agent.harnessId
+ && object(s.request).machineId === this.machineId && !s.error && s.creationId === `super-${String(s.id)}`) : undefined
+ // Nor is the live session: the registry keeps `startedBy: agent` across /clear, /new and /resume, so a person who
+ // took over an old worker would be closed mid-turn. Only the first bind, which lands seconds after launch, is
+ // the Swarm's session; any later one leaves no identity, and the stop is refused as changed.
+ // ponytail: launch-to-first-bind window; a launch slower than this is refused and closed by hand.
+ const firstBind = native.boundAt !== null && native.boundAt >= native.registeredAt && native.boundAt - native.registeredAt <= LEGACY_FIRST_BIND_MS
+ if (start && identifier(start.creationId)) identity = { creationId: start.creationId,
+ ...(firstBind ? { sessionId: native.sessionId, createdAt: new Date(native.registeredAt).toISOString() } : {}) }
+ }
let result
if (!native) {
// The process snapshot can lag. Only the authoritative core may confirm absence; no lease can close a live row.
@@ -259,14 +292,12 @@ export class SuperTabHost {
if (native.startedBy !== 'agent' || native.resumeOnly || !identifier(identity.creationId)
|| typeof identity.sessionId !== 'string' || !identity.sessionId || typeof identity.createdAt !== 'string'
|| native.sessionId !== identity.sessionId || native.registeredAt !== Date.parse(identity.createdAt)) {
- attempt.retryAt = Infinity
- this.log('This session is not the original Swarm launch. Remove its stuck contribution.')
+ refuse(native.startedBy === 'agent' && !native.resumeOnly && identifier(identity.creationId) ? 'AGENT_CHANGED' : 'NOT_SWARM_STARTED')
return
}
const creation = await this.core.agents.creationStatus?.(identity.creationId)
if (creation?.state === 'created' && creation.agentId !== agent.harnessId) {
- attempt.retryAt = Infinity
- this.log('This session is not the original Swarm launch. Remove its stuck contribution.')
+ refuse('NOT_SWARM_STARTED')
return
}
if (creation?.state !== 'created') throw new Error('The original Swarm launch could not be confirmed. Stop will retry with backoff.')
@@ -282,7 +313,7 @@ export class SuperTabHost {
creationId: identity.creationId, sessionId: identity.sessionId, createdAt: identity.createdAt })
}
if (!result.closed) {
- if (['NOT_SWARM_STARTED', 'AGENT_CHANGED'].includes(String(result.error))) attempt.retryAt = Infinity
+ if (['NOT_SWARM_STARTED', 'AGENT_CHANGED'].includes(String(result.error))) refuse(result.error)
throw new Error(result.detail ?? result.error ?? 'Could not stop this Swarm worker.')
}
attempt.closed = true