The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(swarm): a start that is still preparing is not failed at 3 minutes (#1849)
0xken152a356e1909aa
diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.tsindex 807580a9e..124208db0 100644--- a/backend/integration/superTabLiveness.test.ts+++ b/backend/integration/superTabLiveness.test.ts@@ -65,7 +65,7 @@ async function swarm({ legacy = false, client = db }: { legacy?: boolean; clientchecks: [{ description: 'Verified', independent: false }] })).result.conversationId)const worker = await add(group, 'test-host', harness.worker, 'Worker')const assignmentId = String((await execute({ type: 'work.assign', conversationId: group, agentId: worker })).result.assignmentId)- const observe = (machineId: string, running: string[], extra: Partial<MachineObservation> = {}) => timed.observeMachine(host, tab, machineId, {+ const observe = (machineId: string, running: string[], extra: Partial<MachineObservation> = {}, store = timed) => store.observeMachine(host, tab, machineId, {generation: generations.get(machineId)!, resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9,diskTotalBytes: 500e9, projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os: 'macOS', gpuPercent: null },quotas: [], tools: {}, runningHarnessIds: running, stoppedHarnessIds: [], ...extra })@@ -481,9 +481,8 @@ describe('an agent whose task was lost', () => {})describe('a start that never runs', () => {- it('fails 3 minutes after dispatch, naming its machine to its Lead only, and tells the Lead in its group', async () => {- const f = await swarm()- await f.beat(f.clock.now)+ /** The Lead starts "Builder" on Worker Mac, and Worker Mac launches it. */+ const launch = async (f: Awaited<ReturnType<typeof swarm>>) => {const decision = { assessment: { difficulty: 'low', risk: 'low', why: 'Small task.' },candidates: [{ id: 'c0', kind: 'new', machineId: 'test-host', engine: 'codex', model: 'fixture-model', effort: 'high',quality: 2, cost: 2, title: 'New agent', label: 'New agent' }],@@ -495,6 +494,13 @@ describe('a start that never runs', () => {// The start's own machine dispatches and confirms it.const onMachine = (command: Command) => f.timed.execute(host, f.tab, { operationId: randomUUID(), command }, undefined, 'test-host')await onMachine({ type: 'agent.startDispatch', startId })+ return { startId, onMachine }+ }++ it('fails 3 minutes after dispatch, naming its machine to its Lead only, and tells the Lead in its group', async () => {+ const f = await swarm()+ await f.beat(f.clock.now)+ const { startId, onMachine } = await launch(f)const dispatched = f.clock.nowf.clock.now = dispatched + 3 * MINUTEawait f.leadInbox()@@ -507,6 +513,62 @@ describe('a start that never runs', () => {await expect(onMachine({ type: 'agent.started', startId, harnessId: randomUUID(), configuration: config })).rejects.toMatchObject({ code: 'START_FAILED' })})++ it('waits while its own machine reports it preparing, shows the Lead it is preparing, and fails it after 15 minutes', async () => {+ const f = await swarm()+ await f.beat(f.clock.now)+ const { startId } = await launch(f)+ const dispatched = f.clock.now+ const ownerState = async () => (await f.timed.read(owner, f.tab)).state+ // Another machine cannot keep it waiting, and a machine keeps only its own launched starts in its report.+ await f.observe('lead-host', [f.harness.lead], { preparingStartIds: [startId] })+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId, 'someone-elses-start'] })+ const machines = (await ownerState()).machines!+ expect(machines.find(m => m.machineId === 'lead-host')!.observation!.preparingStartIds).toEqual([])+ expect(machines.find(m => m.machineId === 'test-host')!.observation!.preparingStartIds).toEqual([startId])+ for (const minutes of [3, 8, 14]) {+ f.clock.now = dispatched + minutes * MINUTE + 1+ // Its machine reports first: this jump spans many of its 15-second beats.+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })+ await f.observe('lead-host', [f.harness.lead])+ expect((await f.leadInbox()).filter(m => (m.source as { kind?: string }).kind === 'start-timeout')).toEqual([])+ expect(await f.timed.start(host, f.tab, startId, f.lead)).toMatchObject({ error: null, preparing: true })+ expect((await ownerState()).starts!.find(s => s.id === startId)).toMatchObject({ error: null, preparing: true })+ }+ f.clock.now = dispatched + 15 * MINUTE + 1+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })+ await f.observe('lead-host', [f.harness.lead])+ const told = (await f.leadInbox()).find(m => (m.source as { kind?: string }).kind === 'start-timeout')!+ expect(told).toMatchObject({ conversationId: 'lead', text: 'Could not start Builder: its machine was still preparing it after 15 minutes.' })+ expect((await f.timed.start(host, f.tab, startId, f.lead)).error).toBe('Worker Mac was still preparing it after 15 minutes')+ expect((await ownerState()).starts!.find(s => s.id === startId)!.error).toBe('Its machine was still preparing it after 15 minutes.')+ expect((await ownerState()).starts!.find(s => s.id === startId)).not.toHaveProperty('preparing')+ })++ // The time checks run on the Lead's poll and on any other machine's report (store.ts).+ it.each([+ ['the Lead\'s poll', (racing: SuperTabStore, f: Awaited<ReturnType<typeof swarm>>) => racing.deliveries(host, f.tab, f.lead)],+ ['another machine\'s report', (racing: SuperTabStore, f: Awaited<ReturnType<typeof swarm>>) => f.observe('lead-host', [f.harness.lead], {}, racing)],+ ])('is not failed by %s that read its machine silent while that machine reported it preparing', async (_, sweep) => {+ const f = await swarm()+ await f.beat(f.clock.now)+ const { startId } = await launch(f)+ f.clock.now += 4 * MINUTE+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })+ await f.observe('lead-host', [f.harness.lead])+ // Past MACHINE_FRESH_MS: the time checks read Worker Mac as silent; just before they write, it reports again.+ f.clock.now += 46_000+ let raced = false+ const racing = new SuperTabStore(beforeFirstWrite(db, async () => {+ if (raced) return+ raced = true+ await f.observe('test-host', [f.harness.worker], { preparingStartIds: [startId] })+ }), () => f.clock.now)+ await sweep(racing, f)+ expect(raced).toBe(true)+ expect((await f.timed.read(owner, f.tab)).state.starts!.find(s => s.id === startId)).toMatchObject({ error: null, preparing: true })+ expect((await db.superTabMessage.findMany({ where: { tabId: f.tab } })).filter(m => (m.source as { kind?: string } | null)?.kind === 'start-timeout')).toEqual([])+ })})/** Counts every Prisma operation a store makes through this client, as "Model.operation". */diff --git a/backend/src/superTabs/commands.test.ts b/backend/src/superTabs/commands.test.tsindex 2897fc818..90f973c8d 100644--- a/backend/src/superTabs/commands.test.ts+++ b/backend/src/superTabs/commands.test.ts@@ -1665,6 +1665,11 @@ describe('a failed start\'s reason', () => {// A start its machine never ran reads as that; the server's own reasons name nothing and read as they are.f.state.starts![0]!.error = 'Studio did not start it within 3 minutes'expect(read(owner)).toBe('Its machine did not start it within 3 minutes.')+ f.state.starts![0]!.error = 'Studio was still preparing it after 15 minutes'+ expect(read(owner)).toBe('Its machine was still preparing it after 15 minutes.')+ f.state.starts![0]!.error = 'Studio stopped reporting while preparing it'+ expect(read(owner)).toBe('Its machine stopped reporting while preparing it.')+ expect(read(peer)).toBe('Studio stopped reporting while preparing it')f.state.starts![0]!.error = 'Its machine was removed from the Swarm.'expect(read(owner)).toBe('Its machine was removed from the Swarm.')})diff --git a/backend/src/superTabs/liveness.test.ts b/backend/src/superTabs/liveness.test.tsindex 84c950231..b0fe6763e 100644--- a/backend/src/superTabs/liveness.test.ts+++ b/backend/src/superTabs/liveness.test.ts@@ -1,6 +1,7 @@import { describe, expect, it } from 'vitest'import type { Assignment, SuperTabState } from './model.js'import { LEASE_MS, QUIET_RUNNING_MS, sweepLiveness } from './liveness.js'+import { MACHINE_FRESH_MS, startPreparing } from './machines.js'import { split } from './parts/index.js'const MINUTE = 60_000@@ -55,3 +56,68 @@ describe('a worker whose task is lost', () => {expect(after.groups.find(g => g.key === 'g1')!.data).not.toEqual(before.groups.find(g => g.key === 'g1')!.data)})})++describe('a start its machine has launched', () => {+ const LAUNCHED = 1_000_000+ type Report = { machineId?: string; receivedAt: number; preparingStartIds?: string[] }+ /** The Lead's group, and a start for "Builder" sent to machine `m` (Worker Mac) at LAUNCHED; each report is a machine's last. */+ const launched = (reports: Report[] = [], start: Record<string, unknown> = {}) => ({ id: 't',+ agents: [{ id: 'L', machineId: 'lm', harnessId: 'lh', machineOwnerId: 'o', contributorId: 'p', membershipGeneration: 'g', name: 'Lead' }],+ people: [{ id: 'p', email: '[email protected]', role: 'owner', generation: 'g' }],+ machines: [{ machineId: 'm', name: 'Worker Mac' }, { machineId: 'x', name: 'Other Mac' }].map(machine => {+ const report = reports.find(r => (r.machineId ?? 'm') === machine.machineId)+ if (!report) return machine+ const { machineId: _machine, ...reading } = report+ return { ...machine, observation: { runningHarnessIds: [], stoppedHarnessIds: [], ...reading } }+ }),+ conversations: [{ id: 'lead', title: 'Lead', leadId: 'L', agentIds: ['L'], archived: false, brief: '', checks: [], assignment: null, lastMessageSequence: 0 }],+ starts: [{ id: 's1', request: { conversationId: 'lead', machineId: 'm', name: 'Builder' }, dispatched: true, dispatchedAt: LAUNCHED,+ harnessId: null, error: null, ...start }],+ } as unknown as SuperTabState)+ const sweep = (state: SuperTabState, now: number) => {+ const told = sweepLiveness(state, now).filter(row => row.source?.kind === 'start-timeout').map(row => row.text)+ return { error: state.starts![0]!.error, told }+ }++ it('fails 3 minutes after launch when its machine never says it is preparing, as an older machine does', () => {+ const at = LAUNCHED + 3 * MINUTE+ expect(sweep(launched([{ receivedAt: at }]), at)).toEqual({ error: null, told: [] })+ expect(sweep(launched([{ receivedAt: at }]), at + 1)).toEqual({ error: 'Worker Mac did not start it within 3 minutes',+ told: ['Could not start Builder: its machine did not start it within 3 minutes.'] })+ expect(sweep(launched(), at + 1).error).toBe('Worker Mac did not start it within 3 minutes')+ })++ it('waits while its machine\'s reports say it is preparing, and fails it 15 minutes after launch', () => {+ const at = LAUNCHED + 15 * MINUTE+ expect(sweep(launched([{ receivedAt: LAUNCHED + 5 * MINUTE, preparingStartIds: ['s1'] }]), LAUNCHED + 5 * MINUTE).error).toBeNull()+ expect(sweep(launched([{ receivedAt: at - 10_000, preparingStartIds: ['s1'] }]), at)).toEqual({ error: null, told: [] })+ expect(sweep(launched([{ receivedAt: at, preparingStartIds: ['s1'] }]), at + 1)).toEqual({ error: 'Worker Mac was still preparing it after 15 minutes',+ told: ['Could not start Builder: its machine was still preparing it after 15 minutes.'] })+ })++ it('fails once its machine\'s reports saying it is preparing have stopped for three beats', () => {+ const now = LAUNCHED + 6 * MINUTE+ expect(sweep(launched([{ receivedAt: now - MACHINE_FRESH_MS, preparingStartIds: ['s1'] }]), now).error).toBeNull()+ expect(sweep(launched([{ receivedAt: now - MACHINE_FRESH_MS - 1, preparingStartIds: ['s1'] }]), now)).toEqual({+ error: 'Worker Mac stopped reporting while preparing it', told: ['Could not start Builder: its machine stopped reporting while preparing it.'] })+ // The latest report no longer lists it: preparing is over without a start, so the 3 minutes apply.+ expect(sweep(launched([{ receivedAt: now, preparingStartIds: [] }]), now).error).toBe('Worker Mac did not start it within 3 minutes')+ })++ it('is kept waiting only by its own machine\'s report naming it', () => {+ const now = LAUNCHED + 3 * MINUTE + 1+ expect(sweep(launched([{ receivedAt: now, preparingStartIds: ['another-start'] }]), now).error).toBe('Worker Mac did not start it within 3 minutes')+ expect(sweep(launched([{ machineId: 'x', receivedAt: now, preparingStartIds: ['s1'] }]), now).error).toBe('Worker Mac did not start it within 3 minutes')+ })++ it('says a start is preparing only while its own machine\'s fresh report names it and it is neither running nor failed', () => {+ const now = LAUNCHED + 4 * MINUTE+ const preparing = (state: SuperTabState) => startPreparing(state, state.starts![0]!, now)+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }]))).toBe(true)+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }], { harnessId: 'h' }))).toBe(false)+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }], { error: 'Could not prepare' }))).toBe(false)+ expect(preparing(launched([{ receivedAt: now, preparingStartIds: ['s1'] }], { dispatched: false, dispatchedAt: undefined }))).toBe(false)+ expect(preparing(launched([{ machineId: 'x', receivedAt: now, preparingStartIds: ['s1'] }]))).toBe(false)+ expect(preparing(launched([{ receivedAt: now - MACHINE_FRESH_MS - 1, preparingStartIds: ['s1'] }]))).toBe(false)+ })+})diff --git a/backend/src/superTabs/liveness.ts b/backend/src/superTabs/liveness.tsindex 3045aa71b..86d50ff55 100644--- a/backend/src/superTabs/liveness.ts+++ b/backend/src/superTabs/liveness.ts@@ -1,8 +1,9 @@import { activeWork, stableId, type Agent, type Assignment, type Author, type Conversation, type MessageDraft,type SuperTabState } from './model.js'import { agentDisplayName } from './agentNames.js'-import { failureNotice, TIMED_OUT_SUFFIX } from './startFailure.js'+import { failureNotice, STILL_PREPARING_SUFFIX, STOPPED_REPORTING_SUFFIX, TIMED_OUT_SUFFIX } from './startFailure.js'import { leadOf, toLead } from './terminal.js'+import { MACHINE_FRESH_MS } from './machines.js'// Work never sits with a dead owner (design: "A Swarm that never stops", 2026-10-26). A worker's active assignment is held// by a lease that its progress renews. A dead, stalled or limited worker loses it within minutes: the assignment becomes@@ -24,6 +25,10 @@ const PROGRESS_FRESH_MS = QUIET_RUNNING_MS - LEASE_MSconst OFFLINE_LOST_MS = 2 * 60_000/** A dispatched start not running this long after has failed. */const START_RUN_MS = 3 * 60_000+/** A start its machine keeps reporting as preparing may take this long: the worktree checkout's own limit (10 minutes,+ * cli lib/gitProject.ts WORKTREE_ADD_TIMEOUT) and the fetch before it. 3 minutes failed starts on a busy Mac that was+ * still fetching and checking out a 470 MB project (2026-10-27). */+const START_PREPARE_MS = 15 * 60_000/** Any machine's report runs the time checks for its Swarm at most this often. */export const SWEEP_EVERY_MS = 30_000@@ -147,16 +152,18 @@ export function loseTasks(state: SuperTabState, now: number, which: (worker: Age})}-/** The machine parts a loss was judged from (offline, stopped, its reports' renewals). The write that records the loss- * touches them, so a fresh report committed since the read makes it retry on current data instead of fencing a live worker. */-export function judgedMachines(state: SuperTabState, rows: MessageDraft[]): Array<{ kind: 'machine'; key: string }> {+/** The machine parts a loss was judged from (offline, stopped, its reports' renewals), and those a start failed since+ * `before` was judged from (its own machine's preparing report). The write touches them, so a fresh report committed since+ * the read makes it retry on current data instead of fencing a live worker or failing a start still being prepared. */+export function judgedMachines(before: SuperTabState, state: SuperTabState, rows: MessageDraft[]): Array<{ kind: 'machine'; key: string }> {const lost = new Set(rows.flatMap(r => r.source?.kind === 'work-lost' ? [r.source.agentId as string] : []))- const machines = new Set(state.agents.filter(a => lost.has(a.id)).map(a => a.machineId))+ const failed = (state.starts ?? []).filter(s => s.error && !before.starts?.find(b => b.id === s.id)?.error).map(s => s.request.machineId)+ const machines = new Set([...state.agents.filter(a => lost.has(a.id)).map(a => a.machineId), ...failed])return [...machines].filter(id => state.machines?.some(m => m.machineId === id)).map(key => ({ kind: 'machine' as const, key }))}/** The time checks, run on the Lead's poll and on any machine's report: every worker task with a reason is lost, and- * every start dispatched over 3 minutes ago that is not running fails. */+ * every start dispatched over 3 minutes ago that is not running fails, unless its machine is still preparing it. */export function sweepLiveness(state: SuperTabState, now: number): MessageDraft[] {return [...loseTasks(state, now), ...failStalledStarts(state, now)]}@@ -164,7 +171,13 @@ export function sweepLiveness(state: SuperTabState, now: number): MessageDraft[]function failStalledStarts(state: SuperTabState, now: number): MessageDraft[] {return (state.starts ?? []).flatMap(start => {if (!start.dispatched || start.harnessId || start.error || start.dispatchedAt === undefined || now - start.dispatchedAt <= START_RUN_MS) return []- start.error = `${machineName(state, start.request.machineId)}${TIMED_OUT_SUFFIX}`+ // Only its own machine's report can say it is preparing; one before preparingStartIds never does, so its starts get 3 minutes.+ const report = state.machines?.find(m => m.machineId === start.request.machineId)?.observation+ const named = !!report?.preparingStartIds?.includes(start.id)+ const silent = !!report && now - report.receivedAt > MACHINE_FRESH_MS+ if (named && !silent && now - start.dispatchedAt <= START_PREPARE_MS) return []+ const machine = machineName(state, start.request.machineId)+ start.error = `${machine}${!named ? TIMED_OUT_SUFFIX : silent ? STOPPED_REPORTING_SUFFIX : STILL_PREPARING_SUFFIX}`// Never a lookup that throws: these checks run inside every machine's report.const group = state.conversations.find(c => c.id === start.request.conversationId)const lead = group?.leadId ? state.agents.find(a => a.id === group.leadId) : undefineddiff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.tsindex 78f97c543..85d0086ce 100644--- a/backend/src/superTabs/machines.ts+++ b/backend/src/superTabs/machines.ts@@ -110,12 +110,24 @@ export const machineObservationSchema = z.object({ generation: z.string().max(16* it only when a team session reported it (observedPrivacy). */privacy: machinePrivacySchema.optional(),readiness: readiness.optional(),+ /** The starts this machine launched and is still preparing (fetching the project, checking its worktree out, starting+ * the engine), as of this report. Such a start waits past the 3 minutes while reports keep naming it (liveness.ts);+ * the store keeps only this machine's own launched starts. Absent from hosts before it, whose starts get 3 minutes. */+ preparingStartIds: z.array(z.string().min(1).max(160)).max(100).optional(),}).strict()export type MachineObservation = z.infer<typeof machineObservationSchema>/** A machine reports every 15 seconds (cli/src/superTabs/machines.ts); three missed beats is offline. */export const MACHINE_FRESH_MS = 45_000+/** Whether [start] is preparing now: launched, neither running nor failed, and named by a fresh report of its own machine.+ * Read from that report, never written into the start, so a report a minute per start writes nothing more. */+export function startPreparing(state: SuperTabState, start: AgentStart, now: number): boolean {+ if (!start.dispatched || start.harnessId || start.error || start.dispatchedAt === undefined) return false+ const report = state.machines?.find(m => m.machineId === start.request.machineId)?.observation+ return !!report?.preparingStartIds?.includes(start.id) && now - report.receivedAt <= MACHINE_FRESH_MS+}+// ── Teammates' work runs only on Team machines (docs/plans/2026-10-24-swarm-team-account.md, Task 3) ──────────────// A start or an assignment for a project whose owner is not the person whose grant runs the machine is a teammate's// work. It would run as that machine's owner, with their files, logins and subscriptions, so it is admitted only on adiff --git a/backend/src/superTabs/model.ts b/backend/src/superTabs/model.tsindex 411797c0b..da93af657 100644--- a/backend/src/superTabs/model.ts+++ b/backend/src/superTabs/model.ts@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'import type { RouteDecisionEffect } from './routeRecord.js'import { agentDisplayName, assignAgentDisplayNames } from './agentNames.js'import { withAvatarView } from './profile.js'-import { downSince, expireRemovals } from './machines.js'+import { downSince, expireRemovals, startPreparing } from './machines.js'import { publicReason, readsRawReason } from './startFailure.js'import { agentProjectId, agentSetRole, canReadAgent, canReadGroup, canReadProject, groupProject, ledProject, placeRules, projectsOf, sharedWithLeads, skillSetOf, type ProjectVisibility, type SwarmProject } from './projects.js'@@ -109,8 +109,11 @@ export interface AgentStart {request: AgentStartRequestharnessId: string | nulldispatched: boolean- /** When its host was asked to launch it; a start not running 3 minutes later fails (liveness.ts). */+ /** When its host was asked to launch it; a start not running 3 minutes later fails, or 15 minutes later while its machine+ * reports it preparing (liveness.ts). */dispatchedAt?: number+ /** On reads only, never stored: its machine's fresh report says it is preparing (machines.ts startPreparing). */+ preparing?: trueerror: string | null}/** `needs_human`: the agent needs a person's decision (its result is the question). `blocked`: it cannot go on (its result@@ -546,7 +549,9 @@ export function visibleState(state: SuperTabState, actor: Actor, now = Date.now(pool.observation.stoppedHarnessIds = pool.observation.stoppedHarnessIds.filter(id => harnesses.has(id))if (pool.observation.harnessProgress) pool.observation.harnessProgress = Object.fromEntries(Object.entries(pool.observation.harnessProgress).filter(([id]) => harnesses.has(id)))if (pool.observation.stoppedReservations) pool.observation.stoppedReservations = pool.observation.stoppedReservations.filter(r => harnesses.has(r.harnessId))+ if (pool.observation.preparingStartIds) pool.observation.preparingStartIds = pool.observation.preparingStartIds.filter(id => visible.starts?.some(s => s.id === id))}+ for (const start of visible.starts ?? []) if (startPreparing(state, start, now)) start.preparing = truevisible.invitations = visible.invitations.filter(i => visible.conversations.some(c => c.id === i.conversationId))}for (const project of projectsOf(visible)) for (const target of awaitingWorkers(visible, project.id)) target.workDecision = 'awaiting'diff --git a/backend/src/superTabs/startFailure.ts b/backend/src/superTabs/startFailure.tsindex a8e05ec2c..0573ebab4 100644--- a/backend/src/superTabs/startFailure.ts+++ b/backend/src/superTabs/startFailure.ts@@ -7,13 +7,20 @@ export const MACHINE_FAILED = 'Its machine could not prepare or launch it.'export const TIMED_OUT = 'Its machine did not start it within 3 minutes.'/** liveness.ts writes `<machine name> did not start it within 3 minutes`. */export const TIMED_OUT_SUFFIX = ' did not start it within 3 minutes'+/** liveness.ts writes these after a machine named the start as preparing: past its limit, or with its reports stopped. */+export const STILL_PREPARING_SUFFIX = ' was still preparing it after 15 minutes'+export const STOPPED_REPORTING_SUFFIX = ' stopped reporting while preparing it'+/** Each suffix the server writes after a machine name, with what a group member reads instead. */+const TIMEOUTS: Array<[string, string]> = [[TIMED_OUT_SUFFIX, TIMED_OUT],+ [STILL_PREPARING_SUFFIX, 'Its machine was still preparing it after 15 minutes.'],+ [STOPPED_REPORTING_SUFFIX, 'Its machine stopped reporting while preparing it.']]/** Reasons the server writes itself, which name nothing (commands.ts removeMachine and machine.contribute). */export const MACHINE_REMOVED = 'Its machine was removed from the Swarm.'export const ABANDONED = 'Abandoned before it started.'export function publicReason(error: string): string {if (error === MACHINE_REMOVED || error === ABANDONED) return error- return error.endsWith(TIMED_OUT_SUFFIX) ? TIMED_OUT : MACHINE_FAILED+ return TIMEOUTS.find(([suffix]) => error.endsWith(suffix))?.[1] ?? MACHINE_FAILED}/** The group's notice of a failed start; its Lead, woken by it, reads the raw reason on the start. */diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 04bf4caf7..112c89756 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -1,4 +1,4 @@-import { contributedEngines, expireRemovals, LASTING_TEAM_REFUSALS, machineContribution, machineObservationSchema, machineSlots, observedPrivacy, recordAgentPresence, removalPreview, swarmGridRow, teamAccountRequired, teamWorkRefusal, type MachineObservation } from './machines.js'+import { contributedEngines, expireRemovals, LASTING_TEAM_REFUSALS, machineContribution, machineObservationSchema, machineSlots, observedPrivacy, recordAgentPresence, removalPreview, startPreparing, swarmGridRow, teamAccountRequired, teamWorkRefusal, type MachineObservation } from './machines.js'import { createHash } from 'node:crypto'import type { Prisma, PrismaClient, SuperTab, SuperTabDelivery, SuperTabMessage } from '@prisma/client'import type { AuthUser } from '../lib/ssoAuth.js'@@ -675,7 +675,8 @@ export class SuperTabStore {if (grant.machineOwnerId !== start.machineOwnerId || (grant.hostGrant ?? null) !== start.hostGrant) {fail('CONTRIBUTION_REVOKED', 'This start belongs to a machine grant that was revoked.', 403)}- return start+ // The Lead reads that its machine is still preparing it, and waits instead of starting another (liveness.ts).+ return startPreparing(state, start, this.now()) ? { ...start, preparing: true as const } : start})}@@ -1640,7 +1641,7 @@ export class SuperTabStore {// when a task there is lost.const saved = async (lost: MessageDraft[] = []) => {const sequences = numberMessages(state, lost, now)- const touches = judgedMachines(state, lost).filter(t => t.key !== machineId)+ const touches = judgedMachines(loaded.state, state, lost).filter(t => t.key !== machineId)await this.endRuns(tx, tabId, loaded.state, state)const version = (await this.save(tx, loaded, state, stamp, touches)).versions.get(`machine:${machineId}`) ?? 0await this.writeMessages(tx, tabId, state, lost, sequences)@@ -1651,13 +1652,17 @@ export class SuperTabStore {const team = isTeamSession(user)// Its privacy check counts only from the Harness Team account's own session; an ordinary sign-in's claim is dropped.// Progress is kept for running sessions only: their leases are read from it (liveness.ts).- const { privacy: reported, harnessProgress, ...reading } = observation+ const { privacy: reported, harnessProgress, preparingStartIds, ...reading } = observationconst progress = harnessProgress && Object.fromEntries(Object.entries(harnessProgress).filter(([id]) => observation.runningHarnessIds.includes(id)))+ // A machine says preparing only of the starts it launched that are neither running nor failed (liveness.ts).+ const preparing = preparingStartIds?.filter(id => state.starts?.some(s => s.id === id && s.request.machineId === machineId+ && s.dispatched && !s.harnessId && !s.error))const privacy = observedPrivacy(pool.observation?.privacy, reported, team, now)accrueLent(state, pool, now)pool.observation = { ...reading, quotas: observation.quotas.filter(q => contributedEngines(pool, observation).includes(q.engine)), receivedAt: now,// How the machine is signed in, from the session that reported and never from the report (lib/teamSession.ts).- sessionKind: team ? 'team' : 'computer', ...(privacy ? { privacy } : {}), ...(progress ? { harnessProgress: progress } : {}) }+ sessionKind: team ? 'team' : 'computer', ...(privacy ? { privacy } : {}), ...(progress ? { harnessProgress: progress } : {}),+ ...(preparing ? { preparingStartIds: preparing } : {}) }// Current-generation physical stop proof also finishes a close whose membership was already withdrawn.for (const proof of observation.stoppedReservations ?? []) {if (!observation.stoppedHarnessIds.includes(proof.harnessId)) continue@@ -1809,8 +1814,8 @@ export class SuperTabStore {}// One write for all: the time checks' changes, the digest and the wake-ups'.const sequences = numberMessages(state, written, this.now())- // A stop was judged from its machine's report that the session is clean: a newer report makes this write retry.- await this.save(tx, loaded, state, clock.next(), [...judgedMachines(state, written), ...stoppedMachines(state, idle),+ // A loss, a start's failure or a stop was judged from a machine's report: a newer report makes this write retry.+ await this.save(tx, loaded, state, clock.next(), [...judgedMachines(loaded.state, state, written), ...stoppedMachines(state, idle),...woken ? [{ kind: 'agent' as const, key: agentId }] : []])await this.writeMessages(tx, tabId, state, written, sequences)const rows = await tx.superTabDelivery.findMany({ where: { tabId, agentId, status: { in: [...OPEN_DELIVERY] } },diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.tsindex 6deb58fba..703510b01 100644--- a/cli/src/superTabs/machines.spec.ts+++ b/cli/src/superTabs/machines.spec.ts@@ -522,6 +522,60 @@ describe('contributed machine executor', () => {expect(posts().map(c => 'harnessProgress' in (c[3] as object))).toEqual([true, false, false, true, false])})+ describe('a start this machine launched and is still preparing', () => {+ const machinePosts = (f: ReturnType<typeof fixture>) => f.backend.mock.calls+ .filter(c => c[1] === 'POST' && String(c[2]).includes('/machines/')).map(c => c[3] as Record<string, unknown>)+ const pendingCreation = { error: 'START_PENDING', startId: 'start', creationId: 'creation', state: 'pending',+ detail: 'Keep this start ID. Run resume with it to check the same native creation; do not make another start request.' }++ it('is reported as preparing on every report while its creation is under way, and no longer once it runs', async () => {+ const f = fixture()+ f.start.mockResolvedValue(pendingCreation as never)+ // The pass that launches it reports first and launches after; the next pass says so at once.+ await f.machine.tick()+ expect(machinePosts(f).at(-1)).not.toHaveProperty('preparingStartIds')+ f.step(5_000); await f.machine.tick()+ expect(machinePosts(f).at(-1)!.preparingStartIds).toEqual(['start'])+ // Unchanged, it is sent again with each beat, so the backend sees it fresh.+ f.advance(); await f.machine.tick()+ expect(machinePosts(f)).toHaveLength(3)+ expect(machinePosts(f).at(-1)!.preparingStartIds).toEqual(['start'])+ f.start.mockResolvedValue({ ok: true, harnessId: 'harness' } as never)+ f.advance(); await f.machine.tick()+ f.advance(); await f.machine.tick()+ expect(machinePosts(f).at(-1)).not.toHaveProperty('preparingStartIds')+ })++ it('is not a start whose launch is uncertain, refused or failed', async () => {+ for (const result of [{ ...pendingCreation, state: 'unconfirmed' }, { error: 'START_UNCERTAIN', startId: 'start', detail: 'Inspect the machine.' },+ { error: 'CAPACITY_UNAVAILABLE', startId: 'start', reasons: ['Busy.'] }, { error: 'START_FAILED', startId: 'start', detail: 'Failed.' }]) {+ const f = fixture()+ f.start.mockResolvedValue(result as never)+ await f.machine.tick()+ f.advance(); await f.machine.tick()+ expect(machinePosts(f).at(-1)).not.toHaveProperty('preparingStartIds')+ }+ })++ it('is left out for a backend that refuses it, which then fails it at 3 minutes as before, and offered again ten minutes later', async () => {+ const f = fixture()+ f.start.mockResolvedValue(pendingCreation as never)+ const original = f.backend.getMockImplementation()!+ f.backend.mockImplementation(async (core, method, path, body) => {+ if (method === 'POST' && (body as Record<string, unknown>)?.preparingStartIds) throw new SuperTabHostError('VALIDATION_ERROR', 'Unrecognized key: preparingStartIds', 400)+ return original(core, method, path, body)+ })+ await f.machine.tick()+ f.step(5_000); await f.machine.tick()+ const sent = () => machinePosts(f).map(post => 'preparingStartIds' in post)+ expect(sent()).toEqual([false, true, false])+ f.advance(); await f.machine.tick()+ expect(sent()).toEqual([false, true, false, false])+ f.step(600_000); await f.machine.tick()+ expect(sent()).toEqual([false, true, false, false, true, false])+ })+ })+it('drops readiness alone for a backend that takes progress, and both, keeping privacy, for one that takes neither', async () => {const f = fixture({}, () => ({ mode: 'dedicated' }))const usage = f.request.getMockImplementation()!diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.tsindex 84926d04e..043f8711a 100644--- a/cli/src/superTabs/machines.ts+++ b/cli/src/superTabs/machines.ts@@ -135,6 +135,10 @@ export class ContributedMachines {private readonly waiting = new Map<string, string>()/** Per waiting start, since when this machine has refused it without a break. */private readonly refusedSince = new Map<string, number>()+ /** The starts this machine launched whose creation the daemon is still carrying out (fetch, worktree, engine), as its+ * last pass found them. Reported so the backend waits past its 3 minutes (backend liveness.ts): Diego's Mac fetched and+ * checked out a 470 MB project, and the start failed while it did (2026-10-27). */+ private readonly preparing = new Set<string>()/** Removals cleaned and acknowledged by this host, and the one in progress per removal. */private readonly cleaned = new Set<string>()private readonly cleaning = new Map<string, Promise<void>>()@@ -149,6 +153,8 @@ export class ContributedMachines {private readonly uncommittedCache = new Map<string, { cwd: string; dirty: boolean; until: number }>()/** Until when the reading leaves out its readiness field, which the backend refused. */private readinessRefusedUntil = 0+ /** Until when the reading leaves out the starts it is preparing, which the backend refused. */+ private preparingRefusedUntil = 0/** Whether each engine is installed and signed in, and until when that stands: one probe starts login shells and reads* the engines' accounts, so it runs every [READINESS_MS] (a failed one again after [READINESS_RETRY_MS]). */private engineRead: { value: ReadyEngine[] | null; until: number } | null = null@@ -240,6 +246,7 @@ export class ContributedMachines {const pendingIds = new Set(pools.flatMap(row => list(row.starts).map(start => start.id)))for (const id of this.waiting.keys()) if (!pendingIds.has(id)) this.waiting.delete(id)for (const id of this.refusedSince.keys()) if (!pendingIds.has(id)) this.refusedSince.delete(id)+ for (const id of this.preparing) if (!pendingIds.has(id)) this.preparing.delete(id)for (const tab of this.published.keys()) if (!pools.some(row => row.tabId === tab)) this.published.delete(tab)const privacy = this.privacy()// A model run or stopped for a Swarm, or an API shared with it, is sent on this pass, not on the next beat: the@@ -265,12 +272,13 @@ export class ContributedMachines {}const running = ids.filter(id => !!this.core.agents.byAgent(id) && !stopped.includes(id))const pending = list(row.starts)+ const preparing = began >= this.preparingRefusedUntil ? pending.flatMap(start => this.preparing.has(String(start.id)) ? [String(start.id)] : []) : []const tools = [...new Set([...COMMON_TOOLS, ...pending.flatMap(start => {const wanted = object(object(start.request).requirements).toolsreturn Array.isArray(wanted) ? wanted.filter((name): name is string => typeof name === 'string' && /^[A-Za-z0-9_.+-]{1,80}$/.test(name)) : []})])].slice(0, 20)const signature = JSON.stringify([row.reservations, pool.generation, [...running].sort(), [...stopped].sort(), row.paused === true || pool.paused === true,- pending.map(start => start.id).sort(),+ pending.map(start => start.id).sort(), [...preparing].sort(),// A privacy result that changes (a home folder opened again) is sent at once, not on the next beat.privacy ? [privacy.mode, 'passed' in privacy ? privacy.passed : null, 'item' in privacy ? privacy.item : null] : null,typeof row.localGrid === 'string' ? modelsStamp : null])@@ -332,7 +340,7 @@ export class ContributedMachines {const shown = began >= this.progressRefusedUntil ? this.deps.progress?.(running) ?? {} : {}const progress = Object.fromEntries(running.flatMap(id => typeof shown[id] === 'number' ? [[id, shown[id]]] : []))const uncommitted = began >= this.uncommittedRefusedUntil ? await this.uncommitted(running, began) : {}- const observation = { generation: pool.generation, tools: reportedTools,+ const observation = { generation: pool.generation, tools: reportedTools, ...(preparing.length ? { preparingStartIds: preparing } : {}),...(Object.keys(uncommitted).length ? { harnessUncommitted: uncommitted } : {}),...(Object.keys(progress).length ? { harnessProgress: progress } : {}), ...(localModels ? { localModels } : {}),...((readyEngines || checkouts) && began >= this.readinessRefusedUntil ? { readiness } : {}),@@ -354,10 +362,12 @@ export class ContributedMachines {try {const path = `/api/super-tabs/${row.tabId}/machines/${this.machineId}`const refused = (error: unknown) => error instanceof SuperTabHostError && error.status === 400 && error.code === 'VALIDATION_ERROR'- // Older backends reject the additive fields. The newest go alone first, readiness, progress, then privacy: a backend- // that takes everything else keeps the local models it reads. Their boolean-only view still safely refuses an- // unverified tool, and a reading without privacy is never one a Team machine passes on.- const { harnessUncommitted: _uncommitted, ...known } = observation+ // Older backends reject the additive fields. The newest go alone first (the starts it is preparing, uncommitted work,+ // readiness, progress, then privacy): a backend that takes everything else keeps the local models it reads. Their+ // boolean-only view still safely refuses an unverified tool, and a reading without privacy is never one a Team+ // machine passes on.+ const { preparingStartIds: _preparing, ...unprepared } = observation+ const { harnessUncommitted: _uncommitted, ...known } = unpreparedconst { readiness: _readiness, ...ready } = knownconst { harnessProgress: _progress, ...withoutProgress } = readyconst { privacy: _privacy, ...withoutPrivacy } = withoutProgress@@ -365,7 +375,8 @@ export class ContributedMachines {// A backend before `starting` refuses it: it gets the models the grid lists, as before, not none at all.const listedOnly = localModels?.some(m => m.starting)? [{ ...withoutPrivacy, localModels: localModels.filter(m => !m.starting) }] : []- const attempts = [...('harnessUncommitted' in observation ? [observation] : []), ...('readiness' in known ? [known] : []), ready, ...('harnessProgress' in ready ? [withoutProgress] : []),+ const attempts = [...('preparingStartIds' in observation ? [observation] : []), ...('harnessUncommitted' in unprepared ? [unprepared] : []),+ ...('readiness' in known ? [known] : []), ready, ...('harnessProgress' in ready ? [withoutProgress] : []),...('privacy' in withoutProgress ? [withoutPrivacy] : []), ...listedOnly,...('toolProbeErrors' in observation || 'localModels' in observation || 'refusedModels' in observation ? [legacy] : [])]for (const [index, attempt] of attempts.entries()) {@@ -374,7 +385,8 @@ export class ContributedMachines {break} catch (error) {if (!refused(error) || index === attempts.length - 1) throw error- if ('harnessUncommitted' in attempt) this.uncommittedRefusedUntil = began + FIELD_REFUSED_MS+ if ('preparingStartIds' in attempt) this.preparingRefusedUntil = began + FIELD_REFUSED_MS+ else if ('harnessUncommitted' in attempt) this.uncommittedRefusedUntil = began + FIELD_REFUSED_MSelse if ('readiness' in attempt) this.readinessRefusedUntil = began + FIELD_REFUSED_MSelse if ('harnessProgress' in attempt) this.progressRefusedUntil = began + FIELD_REFUSED_MSelse if ('privacy' in attempt) this.privacyRefusedUntil = began + FIELD_REFUSED_MS@@ -423,6 +435,9 @@ export class ContributedMachines {}const result = await this.deps.start({ tabId: row.tabId, startId: start.id, hostDispatch: true, ...(github ? { github } : {}) }, scoped, where, this.deps.open)const refused = result.ok !== true && !start.dispatched && !NOT_REFUSED.has(String(result.error))+ // Launched, and its creation still under way on the daemon: anything else is not preparing.+ if (result.error === 'START_PENDING' && result.state === 'pending') this.preparing.add(String(start.id))+ else this.preparing.delete(String(start.id))if (!refused) this.refusedSince.delete(start.id)if (result.ok === true && identifier(result.harnessId)) {this.waiting.delete(start.id)diff --git a/cli/src/superTabs/start.spec.ts b/cli/src/superTabs/start.spec.tsindex 19752e402..29684637c 100644--- a/cli/src/superTabs/start.spec.ts+++ b/cli/src/superTabs/start.spec.ts@@ -129,6 +129,19 @@ describe('Swarm permission mode', () => {expect(f.open).not.toHaveBeenCalled()})+ it('tells the Lead a contributed start is still being prepared on its machine, so it waits rather than starting another', async () => {+ const f = fixture()+ Object.assign(f.start, { poolGeneration: 'pool', dispatched: true, preparing: true })+ expect(await startAgent({ action: 'resume', tabId: 'tab', startId: 'start' }, f.scoped, where, f.open)).toEqual({ ok: true, queued: true,+ preparing: true, startId: 'start', machineId: f.start.request.machineId,+ detail: 'Its machine is still preparing it (fetching the project, checking its worktree out); that can take up to 15 minutes. Resume this start ID to read its receipt; do not start another.' })+ // One its machine has not said it is preparing (an older machine, or not launched yet) reads as queued, as before.+ delete (f.start as { preparing?: boolean }).preparing+ expect(await startAgent({ action: 'resume', tabId: 'tab', startId: 'start' }, f.scoped, where, f.open)).toEqual({ ok: true, queued: true,+ startId: 'start', machineId: f.start.request.machineId, detail: 'Queued on the contributed machine. Resume this start ID to read its receipt.' })+ expect(f.open).not.toHaveBeenCalled()+ })+it('asks the backend for full mode even when the caller asks for another mode', async () => {const f = fixture()await startAgent({ action: 'start', tabId: 'tab', operationId: 'op', request: { ...f.start.request, permissionMode: 'ask' } }, f.scoped, where, f.open)diff --git a/cli/src/superTabs/start.ts b/cli/src/superTabs/start.tsindex ea3aaac75..27e0abe32 100644--- a/cli/src/superTabs/start.ts+++ b/cli/src/superTabs/start.ts@@ -267,9 +267,13 @@ async function startOnce(input: Json, scoped: ScopedRequest, where: Where, open:...(permissionWarning ? { permissionWarning } : {}) }// Contributed capacity is executed by its owner. A collaborator receives no// machine credentials and cannot issue arbitrary commands to that computer.- if (start.poolGeneration && input.hostDispatch !== true) return { ok: true, queued: true, startId, machineId: request.machineId,- ...(permissionWarning ? { permissionWarning } : {}),- detail: 'Queued on the contributed machine. Resume this start ID to read its receipt.' }+ // Its machine reports it preparing (backend liveness.ts): the backend waits up to 15 minutes, and so should the Lead.+ if (start.poolGeneration && input.hostDispatch !== true) return start.preparing === true+ ? { ok: true, queued: true, preparing: true, startId, machineId: request.machineId, ...(permissionWarning ? { permissionWarning } : {}),+ detail: 'Its machine is still preparing it (fetching the project, checking its worktree out); that can take up to 15 minutes. Resume this start ID to read its receipt; do not start another.' }+ : { ok: true, queued: true, startId, machineId: request.machineId,+ ...(permissionWarning ? { permissionWarning } : {}),+ detail: 'Queued on the contributed machine. Resume this start ID to read its receipt.' }const target = await open(where, request.machineId)/** What the agent runs now, as its frame says; null when the frame cannot say. */const ranNow = async (agentId: string, engine: string) => {