Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(swarm): a start that is still preparing is not failed at 3 minutes (#1849)

0xken152a356e1909aa
diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.ts
index 807580a9e..124208db0 100644
--- a/backend/integration/superTabLiveness.test.ts
+++ b/backend/integration/superTabLiveness.test.ts
@@ -65,7 +65,7 @@ async function swarm({ legacy = false, client = db }: { legacy?: boolean; client
checks: [{ description: 'Verified', independent: false }] })).result.conversationId)
const worker = await add(group, 'test-host', harness.worker, 'Worker')
const assignmentId = String((await execute({ type: 'work.assign', conversationId: group, agentId: worker })).result.assignmentId)
- const observe = (machineId: string, running: string[], extra: Partial<MachineObservation> = {}) => timed.observeMachine(host, tab, machineId, {
+ const observe = (machineId: string, running: string[], extra: Partial<MachineObservation> = {}, store = timed) => store.observeMachine(host, tab, machineId, {
generation: generations.get(machineId)!, resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9,
diskTotalBytes: 500e9, projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },
quotas: [], tools: {}, runningHarnessIds: running, stoppedHarnessIds: [], ...extra })
@@ -481,9 +481,8 @@ describe('an agent whose task was lost', () => {
})
describe('a start that never runs', () => {
- it('fails 3 minutes after dispatch, naming its machine to its Lead only, and tells the Lead in its group', async () => {
- const f = await swarm()
- await f.beat(f.clock.now)
+ /** The Lead starts "Builder" on Worker Mac, and Worker Mac launches it. */
+ const launch = async (f: Awaited<ReturnType<typeof swarm>>) => {
const decision = { assessment: { difficulty: 'low', risk: 'low', why: 'Small task.' },
candidates: [{ id: 'c0', kind: 'new', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',
quality: 2, cost: 2, title: 'New agent', label: 'New agent' }],
@@ -495,6 +494,13 @@ describe('a start that never runs', () => {
// The start's own machine dispatches and confirms it.
const onMachine = (command: Command) => f.timed.execute(host, f.tab, { operationId: randomUUID(), command }, undefined, 'test-host')
await onMachine({ type: 'agent.startDispatch', startId })
+ return { startId, onMachine }
+ }
+
+ it('fails 3 minutes after dispatch, naming its machine to its Lead only, and tells the Lead in its group', async () => {
+ const f = await swarm()
+ await f.beat(f.clock.now)
+ const { startId, onMachine } = await launch(f)
const dispatched = f.clock.now
f.clock.now = dispatched + 3 * MINUTE
await f.leadInbox()
@@ -507,6 +513,62 @@ describe('a start that never runs', () => {
await expect(onMachine({ type: 'agent.started', startId, harnessId: randomUUID(), configuration: config }))
.rejects.toMatchObject({ code: 'START_FAILED' })
})
+
+ it('waits while its own machine reports it preparing, shows the Lead it is preparing, and fails it after 15 minutes', async () => {
+ const f = await swarm()
+ await f.beat(f.clock.now)
+ const { startId } = await launch(f)
+ const dispatched = f.clock.now
+ const ownerState = async () => (await f.timed.read(owner, f.tab)).state
+ // Another machine cannot keep it waiting, and a machine keeps only its own launched starts in its report.
+ await f.observe('lead-host', [f.harness.lead], { preparingStartIds: [startId] })
+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId, 'someone-elses-start'] })
+ const machines = (await ownerState()).machines!
+ expect(machines.find(m => m.machineId === 'lead-host')!.observation!.preparingStartIds).toEqual([])
+ expect(machines.find(m => m.machineId === 'test-host')!.observation!.preparingStartIds).toEqual([startId])
+ for (const minutes of [3, 8, 14]) {
+ f.clock.now = dispatched + minutes * MINUTE + 1
+ // Its machine reports first: this jump spans many of its 15-second beats.
+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })
+ await f.observe('lead-host', [f.harness.lead])
+ expect((await f.leadInbox()).filter(m => (m.source as { kind?: string }).kind === 'start-timeout')).toEqual([])
+ expect(await f.timed.start(host, f.tab, startId, f.lead)).toMatchObject({ error: null, preparing: true })
+ expect((await ownerState()).starts!.find(s => s.id === startId)).toMatchObject({ error: null, preparing: true })
+ }
+ f.clock.now = dispatched + 15 * MINUTE + 1
+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })
+ await f.observe('lead-host', [f.harness.lead])
+ const told = (await f.leadInbox()).find(m => (m.source as { kind?: string }).kind === 'start-timeout')!
+ expect(told).toMatchObject({ conversationId: 'lead', text: 'Could not start Builder: its machine was still preparing it after 15 minutes.' })
+ expect((await f.timed.start(host, f.tab, startId, f.lead)).error).toBe('Worker Mac was still preparing it after 15 minutes')
+ expect((await ownerState()).starts!.find(s => s.id === startId)!.error).toBe('Its machine was still preparing it after 15 minutes.')
+ expect((await ownerState()).starts!.find(s => s.id === startId)).not.toHaveProperty('preparing')
+ })
+
+ // The time checks run on the Lead's poll and on any other machine's report (store.ts).
+ it.each([
+ ['the Lead\'s poll', (racing: SuperTabStore, f: Awaited<ReturnType<typeof swarm>>) => racing.deliveries(host, f.tab, f.lead)],
+ ['another machine\'s report', (racing: SuperTabStore, f: Awaited<ReturnType<typeof swarm>>) => f.observe('lead-host', [f.harness.lead], {}, racing)],
+ ])('is not failed by %s that read its machine silent while that machine reported it preparing', async (_, sweep) => {
+ const f = await swarm()
+ await f.beat(f.clock.now)
+ const { startId } = await launch(f)
+ f.clock.now += 4 * MINUTE
+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })
+ await f.observe('lead-host', [f.harness.lead])
+ // Past MACHINE_FRESH_MS: the time checks read Worker Mac as silent; just before they write, it reports again.
+ f.clock.now += 46_000
+ let raced = false
+ const racing = new SuperTabStore(beforeFirstWrite(db, async () => {
+ if (raced) return
+ raced = true
+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })
+ }), () => f.clock.now)
+ await sweep(racing, f)
+ expect(raced).toBe(true)
+ expect((await f.timed.read(owner, f.tab)).state.starts!.find(s => s.id === startId)).toMatchObject({ error: null, preparing: true })
+ expect((await db.superTabMessage.findMany({ where: { tabId: f.tab } })).filter(m => (m.source as { kind?: string } | null)?.kind === 'start-timeout')).toEqual([])
+ })
})
/** Counts every Prisma operation a store makes through this client, as "Model.operation". */
diff --git a/backend/src/superTabs/commands.test.ts b/backend/src/superTabs/commands.test.ts
index 2897fc818..90f973c8d 100644
--- a/backend/src/superTabs/commands.test.ts
+++ b/backend/src/superTabs/commands.test.ts
@@ -1665,6 +1665,11 @@ describe('a failed start\'s reason', () => {
// A start its machine never ran reads as that; the server's own reasons name nothing and read as they are.
f.state.starts![0]!.error = 'Studio did not start it within 3 minutes'
expect(read(owner)).toBe('Its machine did not start it within 3 minutes.')
+ f.state.starts![0]!.error = 'Studio was still preparing it after 15 minutes'
+ expect(read(owner)).toBe('Its machine was still preparing it after 15 minutes.')
+ f.state.starts![0]!.error = 'Studio stopped reporting while preparing it'
+ expect(read(owner)).toBe('Its machine stopped reporting while preparing it.')
+ expect(read(peer)).toBe('Studio stopped reporting while preparing it')
f.state.starts![0]!.error = 'Its machine was removed from the Swarm.'
expect(read(owner)).toBe('Its machine was removed from the Swarm.')
})
diff --git a/backend/src/superTabs/liveness.test.ts b/backend/src/superTabs/liveness.test.ts
index 84c950231..b0fe6763e 100644
--- a/backend/src/superTabs/liveness.test.ts
+++ b/backend/src/superTabs/liveness.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest'
import type { Assignment, SuperTabState } from './model.js'
import { LEASE_MS, QUIET_RUNNING_MS, sweepLiveness } from './liveness.js'
+import { MACHINE_FRESH_MS, startPreparing } from './machines.js'
import { split } from './parts/index.js'
const MINUTE = 60_000
@@ -55,3 +56,68 @@ describe('a worker whose task is lost', () => {
expect(after.groups.find(g => g.key === 'g1')!.data).not.toEqual(before.groups.find(g => g.key === 'g1')!.data)
})
})
+
+describe('a start its machine has launched', () => {
+ const LAUNCHED = 1_000_000
+ type Report = { machineId?: string; receivedAt: number; preparingStartIds?: string[] }
+ /** The Lead's group, and a start for "Builder" sent to machine `m` (Worker Mac) at LAUNCHED; each report is a machine's last. */
+ const launched = (reports: Report[] = [], start: Record<string, unknown> = {}) => ({ id: 't',
+ agents: [{ id: 'L', machineId: 'lm', harnessId: 'lh', machineOwnerId: 'o', contributorId: 'p', membershipGeneration: 'g', name: 'Lead' }],
+ people: [{ id: 'p', email: '[email protected]', role: 'owner', generation: 'g' }],
+ machines: [{ machineId: 'm', name: 'Worker Mac' }, { machineId: 'x', name: 'Other Mac' }].map(machine => {
+ const report = reports.find(r => (r.machineId ?? 'm') === machine.machineId)
+ if (!report) return machine
+ const { machineId: _machine, ...reading } = report
+ return { ...machine, observation: { runningHarnessIds: [], stoppedHarnessIds: [], ...reading } }
+ }),
+ conversations: [{ id: 'lead', title: 'Lead', leadId: 'L', agentIds: ['L'], archived: false, brief: '', checks: [], assignment: null, lastMessageSequence: 0 }],
+ starts: [{ id: 's1', request: { conversationId: 'lead', machineId: 'm', name: 'Builder' }, dispatched: true, dispatchedAt: LAUNCHED,
+ harnessId: null, error: null, ...start }],
+ } as unknown as SuperTabState)
+ const sweep = (state: SuperTabState, now: number) => {
+ const told = sweepLiveness(state, now).filter(row => row.source?.kind === 'start-timeout').map(row => row.text)
+ return { error: state.starts![0]!.error, told }
+ }
+
+ it('fails 3 minutes after launch when its machine never says it is preparing, as an older machine does', () => {
+ const at = LAUNCHED + 3 * MINUTE
+ expect(sweep(launched([{ receivedAt: at }]), at)).toEqual({ error: null, told: [] })
+ expect(sweep(launched([{ receivedAt: at }]), at + 1)).toEqual({ error: 'Worker Mac did not start it within 3 minutes',
+ told: ['Could not start Builder: its machine did not start it within 3 minutes.'] })
+ expect(sweep(launched(), at + 1).error).toBe('Worker Mac did not start it within 3 minutes')
+ })
+
+ it('waits while its machine\'s reports say it is preparing, and fails it 15 minutes after launch', () => {
+ const at = LAUNCHED + 15 * MINUTE
+ expect(sweep(launched([{ receivedAt: LAUNCHED + 5 * MINUTE, preparingStartIds: ['s1'] }]), LAUNCHED + 5 * MINUTE).error).toBeNull()
+ expect(sweep(launched([{ receivedAt: at - 10_000, preparingStartIds: ['s1'] }]), at)).toEqual({ error: null, told: [] })
+ expect(sweep(launched([{ receivedAt: at, preparingStartIds: ['s1'] }]), at + 1)).toEqual({ error: 'Worker Mac was still preparing it after 15 minutes',
+ told: ['Could not start Builder: its machine was still preparing it after 15 minutes.'] })
+ })
+
+ it('fails once its machine\'s reports saying it is preparing have stopped for three beats', () => {
+ const now = LAUNCHED + 6 * MINUTE
+ expect(sweep(launched([{ receivedAt: now - MACHINE_FRESH_MS, preparingStartIds: ['s1'] }]), now).error).toBeNull()
+ expect(sweep(launched([{ receivedAt: now - MACHINE_FRESH_MS - 1, preparingStartIds: ['s1'] }]), now)).toEqual({
+ error: 'Worker Mac stopped reporting while preparing it', told: ['Could not start Builder: its machine stopped reporting while preparing it.'] })
+ // The latest report no longer lists it: preparing is over without a start, so the 3 minutes apply.
+ expect(sweep(launched([{ receivedAt: now, preparingStartIds: [] }]), now).error).toBe('Worker Mac did not start it within 3 minutes')
+ })
+
+ it('is kept waiting only by its own machine\'s report naming it', () => {
+ const now = LAUNCHED + 3 * MINUTE + 1
+ expect(sweep(launched([{ receivedAt: now, preparingStartIds: ['another-start'] }]), now).error).toBe('Worker Mac did not start it within 3 minutes')
+ expect(sweep(launched([{ machineId: 'x', receivedAt: now, preparingStartIds: ['s1'] }]), now).error).toBe('Worker Mac did not start it within 3 minutes')
+ })
+
+ it('says a start is preparing only while its own machine\'s fresh report names it and it is neither running nor failed', () => {
+ const now = LAUNCHED + 4 * MINUTE
+ const preparing = (state: SuperTabState) => startPreparing(state, state.starts![0]!, now)
+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }]))).toBe(true)
+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }], { harnessId: 'h' }))).toBe(false)
+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }], { error: 'Could not prepare' }))).toBe(false)
+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }], { dispatched: false, dispatchedAt: undefined }))).toBe(false)
+ expect(preparing(launched([{ machineId: 'x', receivedAt: now, preparingStartIds: ['s1'] }]))).toBe(false)
+ expect(preparing(launched([{ receivedAt: now - MACHINE_FRESH_MS - 1, preparingStartIds: ['s1'] }]))).toBe(false)
+ })
+})
diff --git a/backend/src/superTabs/liveness.ts b/backend/src/superTabs/liveness.ts
index 3045aa71b..86d50ff55 100644
--- a/backend/src/superTabs/liveness.ts
+++ b/backend/src/superTabs/liveness.ts
@@ -1,8 +1,9 @@
import { activeWork, stableId, type Agent, type Assignment, type Author, type Conversation, type MessageDraft,
type SuperTabState } from './model.js'
import { agentDisplayName } from './agentNames.js'
-import { failureNotice, TIMED_OUT_SUFFIX } from './startFailure.js'
+import { failureNotice, STILL_PREPARING_SUFFIX, STOPPED_REPORTING_SUFFIX, TIMED_OUT_SUFFIX } from './startFailure.js'
import { leadOf, toLead } from './terminal.js'
+import { MACHINE_FRESH_MS } from './machines.js'
// Work never sits with a dead owner (design: "A Swarm that never stops", 2026-10-26). A worker's active assignment is held
// by a lease that its progress renews. A dead, stalled or limited worker loses it within minutes: the assignment becomes
@@ -24,6 +25,10 @@ const PROGRESS_FRESH_MS = QUIET_RUNNING_MS - LEASE_MS
const OFFLINE_LOST_MS = 2 * 60_000
/** A dispatched start not running this long after has failed. */
const START_RUN_MS = 3 * 60_000
+/** A start its machine keeps reporting as preparing may take this long: the worktree checkout's own limit (10 minutes,
+ * cli lib/gitProject.ts WORKTREE_ADD_TIMEOUT) and the fetch before it. 3 minutes failed starts on a busy Mac that was
+ * still fetching and checking out a 470 MB project (2026-10-27). */
+const START_PREPARE_MS = 15 * 60_000
/** Any machine's report runs the time checks for its Swarm at most this often. */
export const SWEEP_EVERY_MS = 30_000
@@ -147,16 +152,18 @@ export function loseTasks(state: SuperTabState, now: number, which: (worker: Age
})
}
-/** The machine parts a loss was judged from (offline, stopped, its reports' renewals). The write that records the loss
- * touches them, so a fresh report committed since the read makes it retry on current data instead of fencing a live worker. */
-export function judgedMachines(state: SuperTabState, rows: MessageDraft[]): Array<{ kind: 'machine'; key: string }> {
+/** The machine parts a loss was judged from (offline, stopped, its reports' renewals), and those a start failed since
+ * `before` was judged from (its own machine's preparing report). The write touches them, so a fresh report committed since
+ * the read makes it retry on current data instead of fencing a live worker or failing a start still being prepared. */
+export function judgedMachines(before: SuperTabState, state: SuperTabState, rows: MessageDraft[]): Array<{ kind: 'machine'; key: string }> {
const lost = new Set(rows.flatMap(r => r.source?.kind === 'work-lost' ? [r.source.agentId as string] : []))
- const machines = new Set(state.agents.filter(a => lost.has(a.id)).map(a => a.machineId))
+ const failed = (state.starts ?? []).filter(s => s.error && !before.starts?.find(b => b.id === s.id)?.error).map(s => s.request.machineId)
+ const machines = new Set([...state.agents.filter(a => lost.has(a.id)).map(a => a.machineId), ...failed])
return [...machines].filter(id => state.machines?.some(m => m.machineId === id)).map(key => ({ kind: 'machine' as const, key }))
}
/** The time checks, run on the Lead's poll and on any machine's report: every worker task with a reason is lost, and
- * every start dispatched over 3 minutes ago that is not running fails. */
+ * every start dispatched over 3 minutes ago that is not running fails, unless its machine is still preparing it. */
export function sweepLiveness(state: SuperTabState, now: number): MessageDraft[] {
return [...loseTasks(state, now), ...failStalledStarts(state, now)]
}
@@ -164,7 +171,13 @@ export function sweepLiveness(state: SuperTabState, now: number): MessageDraft[]
function failStalledStarts(state: SuperTabState, now: number): MessageDraft[] {
return (state.starts ?? []).flatMap(start => {
if (!start.dispatched || start.harnessId || start.error || start.dispatchedAt === undefined || now - start.dispatchedAt <= START_RUN_MS) return []
- start.error = `${machineName(state, start.request.machineId)}${TIMED_OUT_SUFFIX}`
+ // Only its own machine's report can say it is preparing; one before preparingStartIds never does, so its starts get 3 minutes.
+ const report = state.machines?.find(m => m.machineId === start.request.machineId)?.observation
+ const named = !!report?.preparingStartIds?.includes(start.id)
+ const silent = !!report && now - report.receivedAt > MACHINE_FRESH_MS
+ if (named && !silent && now - start.dispatchedAt <= START_PREPARE_MS) return []
+ const machine = machineName(state, start.request.machineId)
+ start.error = `${machine}${!named ? TIMED_OUT_SUFFIX : silent ? STOPPED_REPORTING_SUFFIX : STILL_PREPARING_SUFFIX}`
// Never a lookup that throws: these checks run inside every machine's report.
const group = state.conversations.find(c => c.id === start.request.conversationId)
const lead = group?.leadId ? state.agents.find(a => a.id === group.leadId) : undefined
diff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.ts
index 78f97c543..85d0086ce 100644
--- a/backend/src/superTabs/machines.ts
+++ b/backend/src/superTabs/machines.ts
@@ -110,12 +110,24 @@ export const machineObservationSchema = z.object({ generation: z.string().max(16
* it only when a team session reported it (observedPrivacy). */
privacy: machinePrivacySchema.optional(),
readiness: readiness.optional(),
+ /** The starts this machine launched and is still preparing (fetching the project, checking its worktree out, starting
+ * the engine), as of this report. Such a start waits past the 3 minutes while reports keep naming it (liveness.ts);
+ * the store keeps only this machine's own launched starts. Absent from hosts before it, whose starts get 3 minutes. */
+ preparingStartIds: z.array(z.string().min(1).max(160)).max(100).optional(),
}).strict()
export type MachineObservation = z.infer<typeof machineObservationSchema>
/** A machine reports every 15 seconds (cli/src/superTabs/machines.ts); three missed beats is offline. */
export const MACHINE_FRESH_MS = 45_000
+/** Whether [start] is preparing now: launched, neither running nor failed, and named by a fresh report of its own machine.
+ * Read from that report, never written into the start, so a report a minute per start writes nothing more. */
+export function startPreparing(state: SuperTabState, start: AgentStart, now: number): boolean {
+ if (!start.dispatched || start.harnessId || start.error || start.dispatchedAt === undefined) return false
+ const report = state.machines?.find(m => m.machineId === start.request.machineId)?.observation
+ return !!report?.preparingStartIds?.includes(start.id) && now - report.receivedAt <= MACHINE_FRESH_MS
+}
+
// ── Teammates' work runs only on Team machines (docs/plans/2026-10-24-swarm-team-account.md, Task 3) ──────────────
// A start or an assignment for a project whose owner is not the person whose grant runs the machine is a teammate's
// work. It would run as that machine's owner, with their files, logins and subscriptions, so it is admitted only on a
diff --git a/backend/src/superTabs/model.ts b/backend/src/superTabs/model.ts
index 411797c0b..da93af657 100644
--- a/backend/src/superTabs/model.ts
+++ b/backend/src/superTabs/model.ts
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
import type { RouteDecisionEffect } from './routeRecord.js'
import { agentDisplayName, assignAgentDisplayNames } from './agentNames.js'
import { withAvatarView } from './profile.js'
-import { downSince, expireRemovals } from './machines.js'
+import { downSince, expireRemovals, startPreparing } from './machines.js'
import { publicReason, readsRawReason } from './startFailure.js'
import { agentProjectId, agentSetRole, canReadAgent, canReadGroup, canReadProject, groupProject, ledProject, placeRules, projectsOf, sharedWithLeads, skillSetOf, type ProjectVisibility, type SwarmProject } from './projects.js'
@@ -109,8 +109,11 @@ export interface AgentStart {
request: AgentStartRequest
harnessId: string | null
dispatched: boolean
- /** When its host was asked to launch it; a start not running 3 minutes later fails (liveness.ts). */
+ /** When its host was asked to launch it; a start not running 3 minutes later fails, or 15 minutes later while its machine
+ * reports it preparing (liveness.ts). */
dispatchedAt?: number
+ /** On reads only, never stored: its machine's fresh report says it is preparing (machines.ts startPreparing). */
+ preparing?: true
error: string | null
}
/** `needs_human`: the agent needs a person's decision (its result is the question). `blocked`: it cannot go on (its result
@@ -546,7 +549,9 @@ export function visibleState(state: SuperTabState, actor: Actor, now = Date.now(
pool.observation.stoppedHarnessIds = pool.observation.stoppedHarnessIds.filter(id => harnesses.has(id))
if (pool.observation.harnessProgress) pool.observation.harnessProgress = Object.fromEntries(Object.entries(pool.observation.harnessProgress).filter(([id]) => harnesses.has(id)))
if (pool.observation.stoppedReservations) pool.observation.stoppedReservations = pool.observation.stoppedReservations.filter(r => harnesses.has(r.harnessId))
+ if (pool.observation.preparingStartIds) pool.observation.preparingStartIds = pool.observation.preparingStartIds.filter(id => visible.starts?.some(s => s.id === id))
}
+ for (const start of visible.starts ?? []) if (startPreparing(state, start, now)) start.preparing = true
visible.invitations = visible.invitations.filter(i => visible.conversations.some(c => c.id === i.conversationId))
}
for (const project of projectsOf(visible)) for (const target of awaitingWorkers(visible, project.id)) target.workDecision = 'awaiting'
diff --git a/backend/src/superTabs/startFailure.ts b/backend/src/superTabs/startFailure.ts
index a8e05ec2c..0573ebab4 100644
--- a/backend/src/superTabs/startFailure.ts
+++ b/backend/src/superTabs/startFailure.ts
@@ -7,13 +7,20 @@ export const MACHINE_FAILED = 'Its machine could not prepare or launch it.'
export const TIMED_OUT = 'Its machine did not start it within 3 minutes.'
/** liveness.ts writes `<machine name> did not start it within 3 minutes`. */
export const TIMED_OUT_SUFFIX = ' did not start it within 3 minutes'
+/** liveness.ts writes these after a machine named the start as preparing: past its limit, or with its reports stopped. */
+export const STILL_PREPARING_SUFFIX = ' was still preparing it after 15 minutes'
+export const STOPPED_REPORTING_SUFFIX = ' stopped reporting while preparing it'
+/** Each suffix the server writes after a machine name, with what a group member reads instead. */
+const TIMEOUTS: Array<[string, string]> = [[TIMED_OUT_SUFFIX, TIMED_OUT],
+ [STILL_PREPARING_SUFFIX, 'Its machine was still preparing it after 15 minutes.'],
+ [STOPPED_REPORTING_SUFFIX, 'Its machine stopped reporting while preparing it.']]
/** Reasons the server writes itself, which name nothing (commands.ts removeMachine and machine.contribute). */
export const MACHINE_REMOVED = 'Its machine was removed from the Swarm.'
export const ABANDONED = 'Abandoned before it started.'
export function publicReason(error: string): string {
if (error === MACHINE_REMOVED || error === ABANDONED) return error
- return error.endsWith(TIMED_OUT_SUFFIX) ? TIMED_OUT : MACHINE_FAILED
+ return TIMEOUTS.find(([suffix]) => error.endsWith(suffix))?.[1] ?? MACHINE_FAILED
}
/** The group's notice of a failed start; its Lead, woken by it, reads the raw reason on the start. */
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 04bf4caf7..112c89756 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -1,4 +1,4 @@
-import { contributedEngines, expireRemovals, LASTING_TEAM_REFUSALS, machineContribution, machineObservationSchema, machineSlots, observedPrivacy, recordAgentPresence, removalPreview, swarmGridRow, teamAccountRequired, teamWorkRefusal, type MachineObservation } from './machines.js'
+import { contributedEngines, expireRemovals, LASTING_TEAM_REFUSALS, machineContribution, machineObservationSchema, machineSlots, observedPrivacy, recordAgentPresence, removalPreview, startPreparing, swarmGridRow, teamAccountRequired, teamWorkRefusal, type MachineObservation } from './machines.js'
import { createHash } from 'node:crypto'
import type { Prisma, PrismaClient, SuperTab, SuperTabDelivery, SuperTabMessage } from '@prisma/client'
import type { AuthUser } from '../lib/ssoAuth.js'
@@ -675,7 +675,8 @@ export class SuperTabStore {
if (grant.machineOwnerId !== start.machineOwnerId || (grant.hostGrant ?? null) !== start.hostGrant) {
fail('CONTRIBUTION_REVOKED', 'This start belongs to a machine grant that was revoked.', 403)
}
- return start
+ // The Lead reads that its machine is still preparing it, and waits instead of starting another (liveness.ts).
+ return startPreparing(state, start, this.now()) ? { ...start, preparing: true as const } : start
})
}
@@ -1640,7 +1641,7 @@ export class SuperTabStore {
// when a task there is lost.
const saved = async (lost: MessageDraft[] = []) => {
const sequences = numberMessages(state, lost, now)
- const touches = judgedMachines(state, lost).filter(t => t.key !== machineId)
+ const touches = judgedMachines(loaded.state, state, lost).filter(t => t.key !== machineId)
await this.endRuns(tx, tabId, loaded.state, state)
const version = (await this.save(tx, loaded, state, stamp, touches)).versions.get(`machine:${machineId}`) ?? 0
await this.writeMessages(tx, tabId, state, lost, sequences)
@@ -1651,13 +1652,17 @@ export class SuperTabStore {
const team = isTeamSession(user)
// Its privacy check counts only from the Harness Team account's own session; an ordinary sign-in's claim is dropped.
// Progress is kept for running sessions only: their leases are read from it (liveness.ts).
- const { privacy: reported, harnessProgress, ...reading } = observation
+ const { privacy: reported, harnessProgress, preparingStartIds, ...reading } = observation
const progress = harnessProgress && Object.fromEntries(Object.entries(harnessProgress).filter(([id]) => observation.runningHarnessIds.includes(id)))
+ // A machine says preparing only of the starts it launched that are neither running nor failed (liveness.ts).
+ const preparing = preparingStartIds?.filter(id => state.starts?.some(s => s.id === id && s.request.machineId === machineId
+ && s.dispatched && !s.harnessId && !s.error))
const privacy = observedPrivacy(pool.observation?.privacy, reported, team, now)
accrueLent(state, pool, now)
pool.observation = { ...reading, quotas: observation.quotas.filter(q => contributedEngines(pool, observation).includes(q.engine)), receivedAt: now,
// How the machine is signed in, from the session that reported and never from the report (lib/teamSession.ts).
- sessionKind: team ? 'team' : 'computer', ...(privacy ? { privacy } : {}), ...(progress ? { harnessProgress: progress } : {}) }
+ sessionKind: team ? 'team' : 'computer', ...(privacy ? { privacy } : {}), ...(progress ? { harnessProgress: progress } : {}),
+ ...(preparing ? { preparingStartIds: preparing } : {}) }
// Current-generation physical stop proof also finishes a close whose membership was already withdrawn.
for (const proof of observation.stoppedReservations ?? []) {
if (!observation.stoppedHarnessIds.includes(proof.harnessId)) continue
@@ -1809,8 +1814,8 @@ export class SuperTabStore {
}
// One write for all: the time checks' changes, the digest and the wake-ups'.
const sequences = numberMessages(state, written, this.now())
- // A stop was judged from its machine's report that the session is clean: a newer report makes this write retry.
- await this.save(tx, loaded, state, clock.next(), [...judgedMachines(state, written), ...stoppedMachines(state, idle),
+ // A loss, a start's failure or a stop was judged from a machine's report: a newer report makes this write retry.
+ await this.save(tx, loaded, state, clock.next(), [...judgedMachines(loaded.state, state, written), ...stoppedMachines(state, idle),
...woken ? [{ kind: 'agent' as const, key: agentId }] : []])
await this.writeMessages(tx, tabId, state, written, sequences)
const rows = await tx.superTabDelivery.findMany({ where: { tabId, agentId, status: { in: [...OPEN_DELIVERY] } },
diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.ts
index 6deb58fba..703510b01 100644
--- a/cli/src/superTabs/machines.spec.ts
+++ b/cli/src/superTabs/machines.spec.ts
@@ -522,6 +522,60 @@ describe('contributed machine executor', () => {
expect(posts().map(c => 'harnessProgress' in (c[3] as object))).toEqual([true, false, false, true, false])
})
+ describe('a start this machine launched and is still preparing', () => {
+ const machinePosts = (f: ReturnType<typeof fixture>) => f.backend.mock.calls
+ .filter(c => c[1] === 'POST' && String(c[2]).includes('/machines/')).map(c => c[3] as Record<string, unknown>)
+ const pendingCreation = { error: 'START_PENDING', startId: 'start', creationId: 'creation', state: 'pending',
+ detail: 'Keep this start ID. Run resume with it to check the same native creation; do not make another start request.' }
+
+ it('is reported as preparing on every report while its creation is under way, and no longer once it runs', async () => {
+ const f = fixture()
+ f.start.mockResolvedValue(pendingCreation as never)
+ // The pass that launches it reports first and launches after; the next pass says so at once.
+ await f.machine.tick()
+ expect(machinePosts(f).at(-1)).not.toHaveProperty('preparingStartIds')
+ f.step(5_000); await f.machine.tick()
+ expect(machinePosts(f).at(-1)!.preparingStartIds).toEqual(['start'])
+ // Unchanged, it is sent again with each beat, so the backend sees it fresh.
+ f.advance(); await f.machine.tick()
+ expect(machinePosts(f)).toHaveLength(3)
+ expect(machinePosts(f).at(-1)!.preparingStartIds).toEqual(['start'])
+ f.start.mockResolvedValue({ ok: true, harnessId: 'harness' } as never)
+ f.advance(); await f.machine.tick()
+ f.advance(); await f.machine.tick()
+ expect(machinePosts(f).at(-1)).not.toHaveProperty('preparingStartIds')
+ })
+
+ it('is not a start whose launch is uncertain, refused or failed', async () => {
+ for (const result of [{ ...pendingCreation, state: 'unconfirmed' }, { error: 'START_UNCERTAIN', startId: 'start', detail: 'Inspect the machine.' },
+ { error: 'CAPACITY_UNAVAILABLE', startId: 'start', reasons: ['Busy.'] }, { error: 'START_FAILED', startId: 'start', detail: 'Failed.' }]) {
+ const f = fixture()
+ f.start.mockResolvedValue(result as never)
+ await f.machine.tick()
+ f.advance(); await f.machine.tick()
+ expect(machinePosts(f).at(-1)).not.toHaveProperty('preparingStartIds')
+ }
+ })
+
+ it('is left out for a backend that refuses it, which then fails it at 3 minutes as before, and offered again ten minutes later', async () => {
+ const f = fixture()
+ f.start.mockResolvedValue(pendingCreation as never)
+ const original = f.backend.getMockImplementation()!
+ f.backend.mockImplementation(async (core, method, path, body) => {
+ if (method === 'POST' && (body as Record<string, unknown>)?.preparingStartIds) throw new SuperTabHostError('VALIDATION_ERROR', 'Unrecognized key: preparingStartIds', 400)
+ return original(core, method, path, body)
+ })
+ await f.machine.tick()
+ f.step(5_000); await f.machine.tick()
+ const sent = () => machinePosts(f).map(post => 'preparingStartIds' in post)
+ expect(sent()).toEqual([false, true, false])
+ f.advance(); await f.machine.tick()
+ expect(sent()).toEqual([false, true, false, false])
+ f.step(600_000); await f.machine.tick()
+ expect(sent()).toEqual([false, true, false, false, true, false])
+ })
+ })
+
it('drops readiness alone for a backend that takes progress, and both, keeping privacy, for one that takes neither', async () => {
const f = fixture({}, () => ({ mode: 'dedicated' }))
const usage = f.request.getMockImplementation()!
diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.ts
index 84926d04e..043f8711a 100644
--- a/cli/src/superTabs/machines.ts
+++ b/cli/src/superTabs/machines.ts
@@ -135,6 +135,10 @@ export class ContributedMachines {
private readonly waiting = new Map<string, string>()
/** Per waiting start, since when this machine has refused it without a break. */
private readonly refusedSince = new Map<string, number>()
+ /** The starts this machine launched whose creation the daemon is still carrying out (fetch, worktree, engine), as its
+ * last pass found them. Reported so the backend waits past its 3 minutes (backend liveness.ts): Diego's Mac fetched and
+ * checked out a 470 MB project, and the start failed while it did (2026-10-27). */
+ private readonly preparing = new Set<string>()
/** Removals cleaned and acknowledged by this host, and the one in progress per removal. */
private readonly cleaned = new Set<string>()
private readonly cleaning = new Map<string, Promise<void>>()
@@ -149,6 +153,8 @@ export class ContributedMachines {
private readonly uncommittedCache = new Map<string, { cwd: string; dirty: boolean; until: number }>()
/** Until when the reading leaves out its readiness field, which the backend refused. */
private readinessRefusedUntil = 0
+ /** Until when the reading leaves out the starts it is preparing, which the backend refused. */
+ private preparingRefusedUntil = 0
/** Whether each engine is installed and signed in, and until when that stands: one probe starts login shells and reads
* the engines' accounts, so it runs every [READINESS_MS] (a failed one again after [READINESS_RETRY_MS]). */
private engineRead: { value: ReadyEngine[] | null; until: number } | null = null
@@ -240,6 +246,7 @@ export class ContributedMachines {
const pendingIds = new Set(pools.flatMap(row => list(row.starts).map(start => start.id)))
for (const id of this.waiting.keys()) if (!pendingIds.has(id)) this.waiting.delete(id)
for (const id of this.refusedSince.keys()) if (!pendingIds.has(id)) this.refusedSince.delete(id)
+ for (const id of this.preparing) if (!pendingIds.has(id)) this.preparing.delete(id)
for (const tab of this.published.keys()) if (!pools.some(row => row.tabId === tab)) this.published.delete(tab)
const privacy = this.privacy()
// A model run or stopped for a Swarm, or an API shared with it, is sent on this pass, not on the next beat: the
@@ -265,12 +272,13 @@ export class ContributedMachines {
}
const running = ids.filter(id => !!this.core.agents.byAgent(id) && !stopped.includes(id))
const pending = list(row.starts)
+ const preparing = began >= this.preparingRefusedUntil ? pending.flatMap(start => this.preparing.has(String(start.id)) ? [String(start.id)] : []) : []
const tools = [...new Set([...COMMON_TOOLS, ...pending.flatMap(start => {
const wanted = object(object(start.request).requirements).tools
return Array.isArray(wanted) ? wanted.filter((name): name is string => typeof name === 'string' && /^[A-Za-z0-9_.+-]{1,80}$/.test(name)) : []
})])].slice(0, 20)
const signature = JSON.stringify([row.reservations, pool.generation, [...running].sort(), [...stopped].sort(), row.paused === true || pool.paused === true,
- pending.map(start => start.id).sort(),
+ pending.map(start => start.id).sort(), [...preparing].sort(),
// A privacy result that changes (a home folder opened again) is sent at once, not on the next beat.
privacy ? [privacy.mode, 'passed' in privacy ? privacy.passed : null, 'item' in privacy ? privacy.item : null] : null,
typeof row.localGrid === 'string' ? modelsStamp : null])
@@ -332,7 +340,7 @@ export class ContributedMachines {
const shown = began >= this.progressRefusedUntil ? this.deps.progress?.(running) ?? {} : {}
const progress = Object.fromEntries(running.flatMap(id => typeof shown[id] === 'number' ? [[id, shown[id]]] : []))
const uncommitted = began >= this.uncommittedRefusedUntil ? await this.uncommitted(running, began) : {}
- const observation = { generation: pool.generation, tools: reportedTools,
+ const observation = { generation: pool.generation, tools: reportedTools, ...(preparing.length ? { preparingStartIds: preparing } : {}),
...(Object.keys(uncommitted).length ? { harnessUncommitted: uncommitted } : {}),
...(Object.keys(progress).length ? { harnessProgress: progress } : {}), ...(localModels ? { localModels } : {}),
...((readyEngines || checkouts) && began >= this.readinessRefusedUntil ? { readiness } : {}),
@@ -354,10 +362,12 @@ export class ContributedMachines {
try {
const path = `/api/super-tabs/${row.tabId}/machines/${this.machineId}`
const refused = (error: unknown) => error instanceof SuperTabHostError && error.status === 400 && error.code === 'VALIDATION_ERROR'
- // Older backends reject the additive fields. The newest go alone first, readiness, progress, then privacy: a backend
- // that takes everything else keeps the local models it reads. Their boolean-only view still safely refuses an
- // unverified tool, and a reading without privacy is never one a Team machine passes on.
- const { harnessUncommitted: _uncommitted, ...known } = observation
+ // Older backends reject the additive fields. The newest go alone first (the starts it is preparing, uncommitted work,
+ // readiness, progress, then privacy): a backend that takes everything else keeps the local models it reads. Their
+ // boolean-only view still safely refuses an unverified tool, and a reading without privacy is never one a Team
+ // machine passes on.
+ const { preparingStartIds: _preparing, ...unprepared } = observation
+ const { harnessUncommitted: _uncommitted, ...known } = unprepared
const { readiness: _readiness, ...ready } = known
const { harnessProgress: _progress, ...withoutProgress } = ready
const { privacy: _privacy, ...withoutPrivacy } = withoutProgress
@@ -365,7 +375,8 @@ export class ContributedMachines {
// A backend before `starting` refuses it: it gets the models the grid lists, as before, not none at all.
const listedOnly = localModels?.some(m => m.starting)
? [{ ...withoutPrivacy, localModels: localModels.filter(m => !m.starting) }] : []
- const attempts = [...('harnessUncommitted' in observation ? [observation] : []), ...('readiness' in known ? [known] : []), ready, ...('harnessProgress' in ready ? [withoutProgress] : []),
+ const attempts = [...('preparingStartIds' in observation ? [observation] : []), ...('harnessUncommitted' in unprepared ? [unprepared] : []),
+ ...('readiness' in known ? [known] : []), ready, ...('harnessProgress' in ready ? [withoutProgress] : []),
...('privacy' in withoutProgress ? [withoutPrivacy] : []), ...listedOnly,
...('toolProbeErrors' in observation || 'localModels' in observation || 'refusedModels' in observation ? [legacy] : [])]
for (const [index, attempt] of attempts.entries()) {
@@ -374,7 +385,8 @@ export class ContributedMachines {
break
} catch (error) {
if (!refused(error) || index === attempts.length - 1) throw error
- if ('harnessUncommitted' in attempt) this.uncommittedRefusedUntil = began + FIELD_REFUSED_MS
+ if ('preparingStartIds' in attempt) this.preparingRefusedUntil = began + FIELD_REFUSED_MS
+ else if ('harnessUncommitted' in attempt) this.uncommittedRefusedUntil = began + FIELD_REFUSED_MS
else if ('readiness' in attempt) this.readinessRefusedUntil = began + FIELD_REFUSED_MS
else if ('harnessProgress' in attempt) this.progressRefusedUntil = began + FIELD_REFUSED_MS
else if ('privacy' in attempt) this.privacyRefusedUntil = began + FIELD_REFUSED_MS
@@ -423,6 +435,9 @@ export class ContributedMachines {
}
const result = await this.deps.start({ tabId: row.tabId, startId: start.id, hostDispatch: true, ...(github ? { github } : {}) }, scoped, where, this.deps.open)
const refused = result.ok !== true && !start.dispatched && !NOT_REFUSED.has(String(result.error))
+ // Launched, and its creation still under way on the daemon: anything else is not preparing.
+ if (result.error === 'START_PENDING' && result.state === 'pending') this.preparing.add(String(start.id))
+ else this.preparing.delete(String(start.id))
if (!refused) this.refusedSince.delete(start.id)
if (result.ok === true && identifier(result.harnessId)) {
this.waiting.delete(start.id)
diff --git a/cli/src/superTabs/start.spec.ts b/cli/src/superTabs/start.spec.ts
index 19752e402..29684637c 100644
--- a/cli/src/superTabs/start.spec.ts
+++ b/cli/src/superTabs/start.spec.ts
@@ -129,6 +129,19 @@ describe('Swarm permission mode', () => {
expect(f.open).not.toHaveBeenCalled()
})
+ it('tells the Lead a contributed start is still being prepared on its machine, so it waits rather than starting another', async () => {
+ const f = fixture()
+ Object.assign(f.start, { poolGeneration: 'pool', dispatched: true, preparing: true })
+ expect(await startAgent({ action: 'resume', tabId: 'tab', startId: 'start' }, f.scoped, where, f.open)).toEqual({ ok: true, queued: true,
+ preparing: true, startId: 'start', machineId: f.start.request.machineId,
+ detail: 'Its machine is still preparing it (fetching the project, checking its worktree out); that can take up to 15 minutes. Resume this start ID to read its receipt; do not start another.' })
+ // One its machine has not said it is preparing (an older machine, or not launched yet) reads as queued, as before.
+ delete (f.start as { preparing?: boolean }).preparing
+ expect(await startAgent({ action: 'resume', tabId: 'tab', startId: 'start' }, f.scoped, where, f.open)).toEqual({ ok: true, queued: true,
+ startId: 'start', machineId: f.start.request.machineId, detail: 'Queued on the contributed machine. Resume this start ID to read its receipt.' })
+ expect(f.open).not.toHaveBeenCalled()
+ })
+
it('asks the backend for full mode even when the caller asks for another mode', async () => {
const f = fixture()
await startAgent({ action: 'start', tabId: 'tab', operationId: 'op', request: { ...f.start.request, permissionMode: 'ask' } }, f.scoped, where, f.open)
diff --git a/cli/src/superTabs/start.ts b/cli/src/superTabs/start.ts
index ea3aaac75..27e0abe32 100644
--- a/cli/src/superTabs/start.ts
+++ b/cli/src/superTabs/start.ts
@@ -267,9 +267,13 @@ async function startOnce(input: Json, scoped: ScopedRequest, where: Where, open:
...(permissionWarning ? { permissionWarning } : {}) }
// Contributed capacity is executed by its owner. A collaborator receives no
// machine credentials and cannot issue arbitrary commands to that computer.
- if (start.poolGeneration && input.hostDispatch !== true) return { ok: true, queued: true, startId, machineId: request.machineId,
- ...(permissionWarning ? { permissionWarning } : {}),
- detail: 'Queued on the contributed machine. Resume this start ID to read its receipt.' }
+ // Its machine reports it preparing (backend liveness.ts): the backend waits up to 15 minutes, and so should the Lead.
+ if (start.poolGeneration && input.hostDispatch !== true) return start.preparing === true
+ ? { ok: true, queued: true, preparing: true, startId, machineId: request.machineId, ...(permissionWarning ? { permissionWarning } : {}),
+ detail: 'Its machine is still preparing it (fetching the project, checking its worktree out); that can take up to 15 minutes. Resume this start ID to read its receipt; do not start another.' }
+ : { ok: true, queued: true, startId, machineId: request.machineId,
+ ...(permissionWarning ? { permissionWarning } : {}),
+ detail: 'Queued on the contributed machine. Resume this start ID to read its receipt.' }
const target = await open(where, request.machineId)
/** What the agent runs now, as its frame says; null when the frame cannot say. */
const ranNow = async (agentId: string, engine: string) => {