The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Swarm host: a worker's stop keeps retrying through a short backend outage (Diego) (#2484)
0xken15f2d2f2b95937
diff --git a/cli/src/superTabs/host.spec.ts b/cli/src/superTabs/host.spec.tsindex 23fed6492..e48f4233a 100644--- a/cli/src/superTabs/host.spec.ts+++ b/cli/src/superTabs/host.spec.ts@@ -2313,8 +2313,8 @@ it.each(['NOT_SWARM_STARTED', 'AGENT_CHANGED'])('does not keep leasing a runtimeexpect(t.lease).toHaveBeenCalledTimes(1)})-it('backs off transient close failures and caps retries for the same runtime', async () => {- const t = releaseFixture('swarm', { error: 'SERVICE_UNAVAILABLE' })+it('caps retries of a close refused for good for the same runtime', async () => {+ const t = releaseFixture('swarm', { error: 'UNSUPPORTED' })await t.poll()for (let i = 0; i < 4; i++) await t.poll(1_000)expect(t.lease).toHaveBeenCalledTimes(1)@@ -2331,11 +2331,152 @@ it('backs off transient close failures and caps retries for the same runtime', aexpect(t.lease).toHaveBeenCalledTimes(6)})+// Diego, 11-01: every failure counted toward the five tries, so a few minutes of a busy Swarm, a timeout or a sign-in+// renewal ended a stop for good, and the stopped worker ran on with its slot and memory until the daemon restarted.+it('backs off a close the core cannot do yet, doubling to ten minutes, and never gives it up', async () => {+ const t = releaseFixture('swarm', { error: 'SERVICE_UNAVAILABLE' })+ await t.poll()+ for (let i = 0; i < 4; i++) await t.poll(1_000)+ expect(t.lease).toHaveBeenCalledTimes(1)+ await t.poll(1_000)+ expect(t.lease).toHaveBeenCalledTimes(2)+ await t.poll(9_000)+ expect(t.lease).toHaveBeenCalledTimes(2)+ await t.poll(1_000)+ expect(t.lease).toHaveBeenCalledTimes(3)+ for (let i = 0; i < 20; i++) await t.poll(130_000)+ expect(t.lease.mock.calls.length).toBeGreaterThan(5)+ expect(t.lease.mock.calls.length).toBeLessThan(20)+ for (let i = 0; i < 3; i++) {+ const before = t.lease.mock.calls.length+ await t.poll(10 * 60_000)+ expect(t.lease).toHaveBeenCalledTimes(before + 1)+ }+ expect(t.sent).not.toHaveBeenCalled()+})++/** Answers [command]'s requests with [reply] while [failing] says so; counts them. */+function failCommand(t: ReturnType<typeof releaseFixture>, command: string, reply: () => { status: number; body: Record<string, unknown> }) {+ const answer = t.core.account.backend+ const state = { failing: true, calls: 0 }+ t.core.account.backend = async (method, path, body) => {+ if ((body as any)?.command?.type !== command || !state.failing) return answer(method, path, body)+ state.calls+++ return reply()+ }+ return state+}+const refusal = (status: number, code: string) => () => ({ status, body: { success: false, error: { code, message: 'Refused.' } } })+const thrown = (message: string) => () => { throw new Error(message) }++// Main's waits (5, 15, 60, 120 s) spend all five tries in the first four minutes of an outage.+it('closes and removes a stopped worker once four minutes of a busy Swarm are over', async () => {+ const t = releaseFixture('swarm', { closed: true })+ const busy = failCommand(t, 'agent.stopLease', refusal(503, 'SWARM_BUSY'))+ for (let s = 0; s < 240; s += 10) await t.poll(10_000)+ expect(busy.calls).toBeGreaterThanOrEqual(5)+ expect(busy.calls).toBeLessThan(10)+ expect(t.close).not.toHaveBeenCalled()+ busy.failing = false+ for (let s = 0; s < 15 * 60 && !t.remove.mock.calls.length; s += 10) await t.poll(10_000)+ expect(t.close).toHaveBeenCalledOnce()+ expect(t.remove).toHaveBeenCalledOnce()+ expect(t.lease).toHaveBeenCalledOnce()+})++it('confirms the removal of a closed worker once an outage of its acknowledgement is over', async () => {+ const t = releaseFixture('swarm', { closed: true })+ const busy = failCommand(t, 'agent.remove', refusal(503, 'SWARM_BUSY'))+ for (let s = 0; s < 600; s += 10) await t.poll(10_000)+ expect(busy.calls).toBeGreaterThan(5)+ busy.failing = false+ for (let s = 0; s < 15 * 60 && !t.remove.mock.calls.length; s += 10) await t.poll(10_000)+ expect(t.remove).toHaveBeenCalledOnce()+ expect(t.lease).toHaveBeenCalledOnce()+ // The retries after the close re-check, with the core alone, that the session is still gone; no lease is taken again.+ expect(t.close).toHaveBeenLastCalledWith({ agentId: 'harness' })+})++it.each([+ ['a busy Swarm', refusal(503, 'SWARM_BUSY')],+ ['a Swarm moving to its new store', refusal(503, 'SWARM_MIGRATING')],+ ['a backend that is down', refusal(502, 'BACKEND_UNAVAILABLE')],+ ['a sign-in being renewed', refusal(401, 'UNAUTHORIZED')],+ ['throttling', refusal(429, 'RATE_LIMITED')],+ ['a stop request read before it changed', refusal(403, 'STALE_RELEASE')],+ ['a timeout', thrown('The request timed out.')],+ ['a sign-in that could not be renewed', thrown("Could not renew this computer's sign-in.")],+])('keeps asking for a close lease through %s, backed off', async (_why, reply) => {+ const t = releaseFixture('swarm', { closed: true })+ const failing = failCommand(t, 'agent.stopLease', reply)+ for (let i = 0; i < 40; i++) await t.poll(130_000)+ expect(failing.calls).toBeGreaterThan(5)+ expect(failing.calls).toBeLessThan(30)+ failing.failing = false+ await t.poll(10 * 60_000)+ expect(t.remove).toHaveBeenCalledOnce()+})++// Lead, 11-01: a stop that never gives up must not hammer the backend. Six hours of a busy Swarm, looked at each minute:+// waits of 5 s doubling to 10 minutes are 7 tries in the first 11 minutes, then one each 10 minutes, about 42 in all.+it('asks at most once each ten minutes through six hours of a busy Swarm', async () => {+ const t = releaseFixture('swarm', { closed: true })+ const busy = failCommand(t, 'agent.stopLease', refusal(503, 'SWARM_BUSY'))+ const reads = () => vi.mocked(t.lookup).mock.calls.length+ for (let i = 0; i < 6 * 60; i++) await t.poll(60_000)+ expect(busy.calls).toBeGreaterThanOrEqual(30)+ expect(busy.calls).toBeLessThanOrEqual(7 + 6 * 6)+ // Each try reads the launch receipt once, and no more.+ expect(reads()).toBe(busy.calls)+ const before = busy.calls+ await t.poll(9 * 60_000)+ expect(busy.calls - before).toBeLessThanOrEqual(1)+})++it.each([+ ['refused', refusal(403, 'FORBIDDEN')],+ ['gone', refusal(404, 'NOT_FOUND')],+ ['for a client to update', refusal(426, 'UPDATE_REQUIRED')],+])('gives up a close lease %s after five tries for the same runtime', async (_why, reply) => {+ const t = releaseFixture('swarm', { closed: true })+ const failing = failCommand(t, 'agent.stopLease', reply)+ for (let i = 0; i < 40; i++) await t.poll(130_000)+ expect(failing.calls).toBe(5)+ expect(t.close).not.toHaveBeenCalled()+})++it('gives up an old backend\'s close lease it cannot use after five tries', async () => {+ const t = releaseFixture('swarm', { closed: true })+ const failing = failCommand(t, 'agent.stopLease', () => ({ status: 200, body: { data: { result: {} } } }))+ for (let i = 0; i < 40; i++) await t.poll(130_000)+ expect(failing.calls).toBe(5)+ expect(t.close).not.toHaveBeenCalled()+})-it('backs off unavailable launch-receipt reads without writing any close leases', async () => {+it('keeps reading an unavailable launch receipt, backed off, without writing any close leases', async () => {const t = releaseFixture()t.lookup.mockRejectedValue(new Error('Core temporarily unavailable'))- for (let i = 0; i < 10; i++) await t.poll(130_000)+ for (let i = 0; i < 40; i++) await t.poll(130_000)+ expect(t.lookup.mock.calls.length).toBeGreaterThan(5)+ expect(t.lookup.mock.calls.length).toBeLessThan(30)+ expect(t.lease).not.toHaveBeenCalled()+})++it.each(['pending', 'unconfirmed'] as const)('keeps reading a launch receipt still %s', async state => {+ const t = releaseFixture('swarm', { closed: true })+ t.lookup.mockResolvedValue({ creationId: 'super-original-creation', state } as never)+ for (let i = 0; i < 40; i++) await t.poll(130_000)+ expect(t.lookup.mock.calls.length).toBeGreaterThan(5)+ expect(t.lease).not.toHaveBeenCalled()+ t.lookup.mockResolvedValue({ creationId: 'super-original-creation', state: 'created', agentId: 'harness' })+ await t.poll(10 * 60_000)+ expect(t.remove).toHaveBeenCalledOnce()+})++it.each(['missing', 'failed', 'cancelled'] as const)('gives up a launch receipt that is %s after five reads, writing no close lease', async state => {+ const t = releaseFixture()+ t.lookup.mockResolvedValue({ creationId: 'super-original-creation', state } as never)+ for (let i = 0; i < 40; i++) await t.poll(130_000)expect(t.lookup).toHaveBeenCalledTimes(5)expect(t.lease).not.toHaveBeenCalled()})diff --git a/cli/src/superTabs/host.ts b/cli/src/superTabs/host.tsindex 681545392..c015269d5 100644--- a/cli/src/superTabs/host.ts+++ b/cli/src/superTabs/host.ts@@ -310,6 +310,28 @@ const ackRefused = (error: unknown) => error instanceof SuperTabHostError && [40/** The first wait after a session the backend ended could not be closed or acknowledged, doubling up to ten minutes. */const END_RETRY_MS = 5_000const END_RETRY_MAX_MS = 10 * 60_000+/** A stop failure that says "not yet" rather than "no" (Diego, 11-01): every failure used to count toward the five tries,+ * so a few minutes of a busy Swarm, a timeout or a sign-in being renewed ended a stop for good, and the stopped worker ran+ * on, holding its slot and memory, until the daemon restarted. These back off as an ended session does (END_RETRY_MS+ * doubling to END_RETRY_MAX_MS) and are never given up: a sign-in (401), a timeout (408), throttling (429), a busy or+ * moving Swarm, a stop request read before it changed (the next read names the new one), work not yet finished. */+const RELEASE_RETRY_STATUSES = [401, 408, 429]+const RELEASE_RETRY_CODES = ['SWARM_BUSY', 'SWARM_MIGRATING', 'STALE_RELEASE', 'WORK_ACTIVE']+/** The core's close refusals no retry for the same runtime changes; any other (unavailable, an expired lease, data not+ * deleted yet) is tried again. NOT_SWARM_STARTED and AGENT_CHANGED are refused for good where they are answered. */+const RELEASE_FINAL_CLOSES = ['UNSUPPORTED', 'LEAD_PROTECTED', 'BUDDY_PROTECTED']+/** A launch receipt that can never come to name this harness's start. */+const RELEASE_FINAL_RECEIPTS = ['missing', 'failed', 'cancelled']+/** A failure this host knows no retry changes: it counts toward the five tries. */+class ReleaseFinal extends Error {}+/** Whether a stop failure counts toward the five tries: this host's own final answers and the backend's 4xx refusals,+ * less the ones that say "not yet" above. A transport failure, a timeout or a server failure (5xx) says nothing final. */+const releaseFinal = (error: unknown) => error instanceof ReleaseFinal || error instanceof SuperTabHostError+ && error.status >= 400 && error.status < 500 && !RELEASE_RETRY_STATUSES.includes(error.status) && !RELEASE_RETRY_CODES.includes(error.code)+const closeFailure = (result: { error?: string; detail?: string }, fallback: string) => {+ const message = result.detail ?? result.error ?? fallback+ return RELEASE_FINAL_CLOSES.includes(String(result.error)) ? new ReleaseFinal(message) : new Error(message)+}/** How long after a pre-#1527 Swarm launch its engine may first bind a session and still be the Swarm's own. */const LEGACY_FIRST_BIND_MS = 120_000const typedId = (entry: Entry) => `team:service:superTabs:${entry.deliveryId}`@@ -403,7 +425,8 @@ export class SuperTabHost {private stopsRunning: Promise<void> | null = nullprivate readonly releases = new Set<string>()/** Eligibility and failures belong to this stop and native runtime; a new runtime or stop gets a fresh decision. */- private readonly releaseAttempts = new Map<string, { signature: string; failures: number; retryAt: number; closed?: true }>()+ /** `failures` counts final answers toward the five tries; `waits` the transient ones behind the current backoff. */+ private readonly releaseAttempts = new Map<string, { signature: string; failures: number; waits: number; retryAt: number; closed?: true }>()private nextStopsAt = 0private receiptsRunning: Promise<void> | null = nullprivate nextReceiptsAt = 0@@ -669,7 +692,7 @@ export class SuperTabHost {const signature = JSON.stringify([agent.releaseId, object(agent).startedSession, native? [native.sessionId, native.registeredAt, native.startedBy, native.resumeOnly, native.processIdentity] : null])const previous = this.releaseAttempts.get(key)- const attempt = previous?.signature === signature ? previous : { signature, failures: 0, retryAt: 0, closed: undefined }+ const attempt = previous?.signature === signature ? previous : { signature, failures: 0, waits: 0, retryAt: 0, closed: undefined }if (this.now() < attempt.retryAt) returnthis.releaseAttempts.set(key, attempt)this.releases.add(key)@@ -704,7 +727,7 @@ export class SuperTabHost {if (!confirmed.closed) {// It closed once; a session there now was resumed or replaced after that close.if (['NOT_SWARM_STARTED', 'AGENT_CHANGED'].includes(String(confirmed.error))) refuse('AGENT_CHANGED')- throw new Error(confirmed.detail ?? confirmed.error ?? 'The stopped session could not be confirmed.')+ throw closeFailure(confirmed, 'The stopped session could not be confirmed.')}} else {let identity = object(member.startedSession)@@ -735,11 +758,14 @@ export class SuperTabHost {refuse(native.startedBy === 'agent' && identifier(identity.creationId) ? 'AGENT_CHANGED' : 'NOT_SWARM_STARTED')return}- const creation = await this.core.agents.creationStatus?.(identity.creationId)+ if (!this.core.agents.creationStatus) throw new ReleaseFinal('This core cannot confirm the original Swarm launch.')+ const creation = await this.core.agents.creationStatus(identity.creationId)if (creation?.state === 'created' && creation.agentId !== agent.harnessId) {refuse('NOT_SWARM_STARTED')return}+ // A receipt still pending or unconfirmed, or unreadable now, may yet confirm it; a missing, failed or cancelled one never.+ if (creation && RELEASE_FINAL_RECEIPTS.includes(creation.state)) throw new ReleaseFinal(`The original Swarm launch is ${creation.state}.`)if (creation?.state !== 'created') throw new Error('The original Swarm launch could not be confirmed. Stop will retry with backoff.')if (this.stopped) return// Start locally before asking: server clock skew cannot extend authority, and network delay consumes it.@@ -748,7 +774,7 @@ export class SuperTabHost {operationId: `close-lease-${randomUUID()}`, asAgentId: agent.id,command: { type: 'agent.stopLease', agentId: agent.id, releaseId: agent.releaseId },})).result)- if (typeof leased.durationMs !== 'number' || !Number.isFinite(leased.durationMs) || leased.durationMs <= 0 || leased.durationMs > 120_000) throw new Error('The host could not reserve this native close. Remove a stuck contribution or update its backend.')+ if (typeof leased.durationMs !== 'number' || !Number.isFinite(leased.durationMs) || leased.durationMs <= 0 || leased.durationMs > 120_000) throw new ReleaseFinal('The host could not reserve this native close. Remove a stuck contribution or update its backend.')// The idle check's stop trusted a report up to a minute old (machines.ts): an edit since, or a folder git cannot// read now, withdraws it instead (review of #1807). Read after the lease, whose answer can take minutes, so it// is the last thing before the close. A stop its Lead asked for closes as before.@@ -764,7 +790,7 @@ export class SuperTabHost {}if (!result.closed) {if (['NOT_SWARM_STARTED', 'AGENT_CHANGED'].includes(String(result.error))) refuse(result.error)- throw new Error(result.detail ?? result.error ?? 'Could not stop this Swarm worker.')+ throw closeFailure(result, 'Could not stop this Swarm worker.')}attempt.closed = true}@@ -777,10 +803,14 @@ export class SuperTabHost {await this.contributed().removeStoppedWorktree(tabId, agent.harnessId).catch(error => this.log(`Kept the worktree of a stopped worker: ${error instanceof Error ? error.message : String(error)}`))} catch (error) {- attempt.failures++- // Five attempts total; permanent ownership/identity refusals never acquire another lease for this runtime.- if (attempt.retryAt !== Infinity) attempt.retryAt = attempt.failures >= 5 ? Infinity- : this.now() + [5_000, 15_000, 60_000, 120_000][attempt.failures - 1]!+ // Five final answers in all; permanent ownership/identity refusals never acquire another lease for this runtime. A+ // transient failure waits 5 s doubling to 10 minutes, and is tried again for as long as the stop is asked.+ if (attempt.retryAt !== Infinity && releaseFinal(error)) {+ attempt.failures+++ attempt.retryAt = attempt.failures >= 5 ? Infinity : this.now() + [5_000, 15_000, 60_000, 120_000][attempt.failures - 1]!+ } else if (attempt.retryAt !== Infinity) {+ attempt.retryAt = this.now() + Math.min(END_RETRY_MAX_MS, END_RETRY_MS * 2 ** attempt.waits++)+ }this.log(error instanceof Error ? error.message : String(error))} finally { this.releases.delete(key) }}