The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Distinguish missing and unverified tools in Swarm routing (#1606)
0xken15dac7ceeb812b
diff --git a/backend/src/superTabs/machines.test.ts b/backend/src/superTabs/machines.test.tsindex a0ea081cb..70e218f01 100644--- a/backend/src/superTabs/machines.test.ts+++ b/backend/src/superTabs/machines.test.ts@@ -1,12 +1,18 @@import { describe, expect, it } from 'vitest'import { createSuperTab, SuperTabError, type AgentStart, type AgentStartRequest, type SuperTabState } from './model.js'-import { admitMachineStart, machineSlots, type ContributedMachine } from './machines.js'+import { admitMachineStart, machineSlots, machineObservationSchema, type ContributedMachine } from './machines.js'const NOW = 1_000_000const requirements = { memoryBytes: 1, diskBytes: 1, maxCpuPercent: 50, tools: ['git'], devices: [], quotaRequired: false }const observation = { generation: 'o', receivedAt: NOW, resources: { cpuPercent: 1, memoryTotalBytes: 1, memoryUsedBytes: 1, diskFreeBytes: 1, diskTotalBytes: 1,projectDiskFreeBytes: 1, deviceType: null, chip: null, os: null, gpuPercent: null }, tools: {}, quotas: [], runningHarnessIds: [], stoppedHarnessIds: [] }+it('accepts optional tool probe failures while preserving older observations', () => {+ const { receivedAt: _receivedAt, ...sent } = observation+ expect(machineObservationSchema.safeParse(sent).success).toBe(true)+ expect(machineObservationSchema.safeParse({ ...sent, tools: { git: true }, toolProbeErrors: { flutter: 'interactive shell timed out' } }).success).toBe(true)+})+function setup(machine: Partial<ContributedMachine> = {}) {const state: SuperTabState = createSuperTab('tab', 'Team', { id: 'owner', email: '[email protected]' }, 'test')state.machines = [{ machineId: 'host', name: 'Studio Mac', contributorId: 'owner', membershipGeneration: 'tab', generation: 'g1',diff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.tsindex c4bc2a9dd..57fb48e2d 100644--- a/backend/src/superTabs/machines.ts+++ b/backend/src/superTabs/machines.ts@@ -32,6 +32,7 @@ const quota = z.object({ engine: z.string().regex(/^[a-z][a-z0-9_-]{0,79}$/), kn}).strict()export const machineObservationSchema = z.object({ generation: z.string().max(160), resources: resource,tools: z.record(z.string().max(80), z.boolean()).refine(v => Object.keys(v).length <= 20),+ toolProbeErrors: z.record(z.string().max(80), z.string().max(200)).refine(v => Object.keys(v).length <= 20).optional(),quotas: z.array(quota).max(32), runningHarnessIds: z.array(z.string().max(160)).max(1000),stoppedHarnessIds: z.array(z.string().max(160)).max(1000),stoppedReservations: z.array(z.object({ harnessId: z.string().min(1).max(160), generation: z.string().min(1).max(160) }).strict()).max(1000).optional(),diff --git a/cli/src/lib/engineLaunch.spec.ts b/cli/src/lib/engineLaunch.spec.tsindex 9d5a91869..d18d334ae 100644--- a/cli/src/lib/engineLaunch.spec.ts+++ b/cli/src/lib/engineLaunch.spec.ts@@ -1136,14 +1136,37 @@ exec ${ZSH} -c "$script" "$@"}describe('commandAvailableInInteractiveShell', () => {+ it('reports a shell failure as unverified rather than a missing command', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'harness-failed-probe-'))+ dirs.push(dir)+ const shell = join(dir, 'bash')+ writeFileSync(shell, '#!/bin/sh\nexit 9\n')+ chmodSync(shell, 0o700)+ const result = await commandAvailableInInteractiveShell('git', shell, undefined, true)+ expect(result).toMatchObject({ status: 'unverified', detail: expect.stringContaining('9') })+ })+ it('bounds a hung shell probe and reports a timeout as unverified', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'harness-slow-probe-'))+ dirs.push(dir)+ const shell = join(dir, 'bash')+ writeFileSync(shell, '#!/bin/sh\nexec /bin/sleep 30\n')+ chmodSync(shell, 0o700)+ const started = Date.now()+ const result = await commandAvailableInInteractiveShell('git', shell, undefined, true)+ expect(result).toEqual({ status: 'unverified', detail: 'Interactive shell tool probe timed out after 5 seconds.' })+ expect(Date.now() - started).toBeLessThan(7_000)+ }, 8_000)it('uses the same bash interactive PATH that launches a new engine', async () => {const binDir = mkdtempSync(join(tmpdir(), 'harness-engine-bin-'))dirs.push(binDir)executable(binDir, 'kilo')process.env.HARNESS_ENGINE_TEST_PATH = binDir- await expect(commandAvailableInInteractiveShell('kilo', bashProbeShell())).resolves.toBe(true)- await expect(commandAvailableInInteractiveShell('missing-engine', bashProbeShell())).resolves.toBe(false)+ const shell = bashProbeShell()+ await expect(commandAvailableInInteractiveShell('kilo', shell)).resolves.toBe(true)+ await expect(commandAvailableInInteractiveShell('missing-engine', shell)).resolves.toBe(false)+ await expect(commandAvailableInInteractiveShell('kilo', shell, undefined, true)).resolves.toEqual({ status: 'available' })+ await expect(commandAvailableInInteractiveShell('missing-engine', shell, undefined, true)).resolves.toEqual({ status: 'missing' })})it('recognizes an installed vendor path before the shell profile has it on PATH', async () => {diff --git a/cli/src/lib/engineLaunch.ts b/cli/src/lib/engineLaunch.tsindex 7dd59e2ed..19d5376e7 100644--- a/cli/src/lib/engineLaunch.ts+++ b/cli/src/lib/engineLaunch.ts@@ -1335,35 +1335,46 @@ function availabilityScript(recipe: EngineInstallRecipe | undefined): string {` for bin in ${names}; do [ -f "$npm_prefix/bin/$bin" ] && [ -x "$npm_prefix/bin/$bin" ] && exit 0; done`,'fi',] : []),- 'exit 1',+ // An ordinary shell/profile failure may also exit 1. Only this value proves the probe ran to completion and found nothing.+ 'exit 42',].join('\n')}+export type CommandAvailability = { status: 'available' | 'missing' | 'unverified'; detail?: string }/*** Does the same interactive shell that launches a new agent resolve this CLI?** The fallback is intentionally the daemon PATH: without a usable absolute* SHELL there is no safer context to consult and direct launch is used too.*/+export function commandAvailableInInteractiveShell(command: string, shell: string | undefined, recipe: EngineInstallRecipe | undefined, detailed: true): Promise<CommandAvailability>+export function commandAvailableInInteractiveShell(command: string, shell?: string, recipe?: EngineInstallRecipe): Promise<boolean>export async function commandAvailableInInteractiveShell(command: string,shell: string | undefined = undefined,recipe: EngineInstallRecipe | undefined = undefined,-): Promise<boolean> {+ detailed = false,+): Promise<boolean | CommandAvailability> {const interactive = interactiveEngineShell(shell)if (!interactive) {- return binaryOnPath(command)- || (recipe ? engineInstallPaths(recipe).some((candidate) => binaryOnPath(candidate)) : false)+ const found = binaryOnPath(command) || (recipe ? engineInstallPaths(recipe).some((candidate) => binaryOnPath(candidate)) : false)+ return detailed ? { status: found ? 'available' : 'missing' } : found}const [file, ...args] = engineShellArgv(interactive, [availabilityScript(recipe), 'harness-engine-probe', command])- return await new Promise((resolve) => {+ const result = await new Promise<CommandAvailability>((resolve) => {execFile(file,args,{ timeout: 5_000 },- (error) => resolve(!error),+ (error) => {+ if (!error) return resolve({ status: 'available' })+ if (Number(error.code) === 42) return resolve({ status: 'missing' })+ if (error.killed) return resolve({ status: 'unverified', detail: 'Interactive shell tool probe timed out after 5 seconds.' })+ resolve({ status: 'unverified', detail: `Interactive shell tool probe failed (${String(error.code ?? error.signal ?? 'unknown error')}).` })+ },)})+ return detailed ? result : result.status === 'available'}export type CommandFlagSupport = 'supported' | 'unsupported' | 'unknown'diff --git a/cli/src/superTabs/capacity.spec.ts b/cli/src/superTabs/capacity.spec.tsindex 3bb9f4418..c1bef3504 100644--- a/cli/src/superTabs/capacity.spec.ts+++ b/cli/src/superTabs/capacity.spec.ts@@ -3,7 +3,7 @@ import { admission, quotaOf, readCapabilities } from './capacity.js'const requirements = { memoryBytes: 100, diskBytes: 200, maxCpuPercent: 80, tools: ['git'], devices: ['/dev/test'], quotaRequired: false }it('uses the destination filesystem and reports missing tools/devices without executing commands', async () => {const deps = { resources: vi.fn(async () => ({ cpuPercent: 25, memoryTotalBytes: 1000, memoryUsedBytes: 300 })),- disk: vi.fn(async () => ({ freeBytes: 500, volume: 'volume', path: '/fixture/repo' })), tool: vi.fn(async () => true), device: vi.fn(async () => true) }+ disk: vi.fn(async () => ({ freeBytes: 500, volume: 'volume', path: '/fixture/repo' })), tool: vi.fn(async () => ({ status: 'available' as const })), device: vi.fn(async () => true) }const capacity = await readCapabilities({ cwd: '/fixture/repo', tools: ['git'], devices: ['/dev/test'] }, deps)expect(deps.disk).toHaveBeenCalledWith('/fixture/repo', false)// Which engines this daemon launches with a model: a daemon without the field gets the switch after the start.@@ -28,3 +28,18 @@ it('keeps unknown quota distinct from unused quota and honors all limiting windoexpect(admission({}, { installed: true, signedIn: false }, quotaOf({}, 'codex', false), requirements).reasons).toContain('The selected agent engine is not signed in on this machine.')expect(admission({}, {}, quotaOf({}, 'claude', false), { ...requirements, quotaRequired: true }).reasons).toContain('The requested subscription allowance could not be verified.')})+it('distinguishes a missing tool from a failed probe and keeps the failure detail', async () => {+ const deps = { resources: vi.fn(async () => ({ cpuPercent: 10, memoryTotalBytes: 1000, memoryUsedBytes: 100 })),+ disk: vi.fn(async () => ({ freeBytes: 500, volume: 'volume', path: '/fixture/repo' })),+ tool: vi.fn(async () => ({ status: 'missing' as const })), device: vi.fn(async () => true) }+ const missing = await readCapabilities({ cwd: '/fixture/repo', tools: ['git'] }, deps)+ expect(admission(missing, { installed: true }, quotaOf({}, 'codex', false), { ...requirements, devices: [] }).reasons)+ .toContain('Required tool missing: git.')++ deps.tool.mockRejectedValueOnce(new Error('interactive shell timed out'))+ const unverified = await readCapabilities({ cwd: '/fixture/repo', tools: ['git'] }, deps)+ expect(unverified.tools).toEqual({})+ expect(unverified.toolProbeErrors).toEqual({ git: 'interactive shell timed out' })+ expect(admission(unverified, { installed: true }, quotaOf({}, 'codex', false), { ...requirements, devices: [] }).reasons)+ .toEqual(expect.arrayContaining([expect.stringMatching(/Required tool not verified: git.*interactive shell timed out/)]))+})diff --git a/cli/src/superTabs/capacity.ts b/cli/src/superTabs/capacity.tsindex d6aa7a7b5..09bfd8c3c 100644--- a/cli/src/superTabs/capacity.ts+++ b/cli/src/superTabs/capacity.ts@@ -2,7 +2,7 @@ import { access } from 'node:fs/promises'import { constants } from 'node:fs'import { isAbsolute } from 'node:path'import { readMachineResources } from '../lib/machineResources.js'-import { commandAvailableInInteractiveShell } from '../lib/engineLaunch.js'+import { commandAvailableInInteractiveShell, type CommandAvailability } from '../lib/engineLaunch.js'import { launchModelEngines } from '../lib/launchModel.js'import { object } from './wire.js'import { readCapacityDisk } from './disk.js'@@ -19,12 +19,20 @@ export const startRequirements = (needs: { tools: string[]; devices: string[] })// CPU use is only a momentary reading: a busy machine would refuse every candidate (seen in the route e2e at 98.7%), so the route does not cap it.({ ...START_REQUIREMENTS, maxCpuPercent: 100, tools: [...new Set([...START_REQUIREMENTS.tools, ...needs.tools])], devices: needs.devices })+/** A missing result is a completed check; an absent value or failed check is only unverified. */+export function requiredToolReason(tool: string, reported: unknown, failure?: unknown): string | null {+ if (reported === true) return null+ if (reported === false) return `Required tool missing: ${tool}.`+ const detail = typeof failure === 'string' && failure ? `${failure} ` : ''+ return `Required tool not verified: ${tool}. ${detail}Ask the machine owner to check it and refresh the machine reading.`+}+export async function readCapabilities(input: Json, deps = {resources: readMachineResources,disk: readCapacityDisk,- tool: commandAvailableInInteractiveShell,+ tool: (name: string): Promise<CommandAvailability> => commandAvailableInInteractiveShell(name, undefined, undefined, true),device: async (path: string) => { await access(path, constants.R_OK | constants.W_OK); return true },-}) {+}): Promise<Json> {const cwd = input.cwd, tools = input.tools ?? [], devices = input.devices ?? []if (typeof cwd !== 'string' || !isAbsolute(cwd) || cwd.length > 4096 || /[\x00-\x1f\x7f]/.test(cwd)|| (input.worktree !== undefined && typeof input.worktree !== 'boolean')@@ -33,9 +41,13 @@ export async function readCapabilities(input: Json, deps = {return { error: 'INVALID_REQUEST', detail: 'Name an absolute project path and up to 20 tools and devices.' }}const probeTools = async () => {- const answers: Array<readonly [string, boolean]> = []+ const answers: Array<readonly [string, CommandAvailability]> = []for (let index = 0; index < tools.length; index += 4) {- answers.push(...await Promise.all(tools.slice(index, index + 4).map(async t => [t, await deps.tool(t).catch(() => false)] as const)))+ answers.push(...await Promise.all(tools.slice(index, index + 4).map(async t => {+ const result = await deps.tool(t).catch(error => ({ status: 'unverified' as const,+ detail: (error instanceof Error ? error.message : String(error)).slice(0, 200) }))+ return [t, result] as const+ })))}return answers}@@ -46,9 +58,12 @@ export async function readCapabilities(input: Json, deps = {probeTools(),Promise.all(devices.map(async d => [d, await deps.device(d).catch(() => false)] as const)),])+ const toolProbeErrors = Object.fromEntries(toolValues.filter(([, result]) => result.status === 'unverified')+ .map(([name, result]) => [name, result.detail ?? 'The tool probe did not complete.']))return { ok: true, sampledAt: Date.now(), resources: { ...resources, projectDiskFreeBytes: disk?.freeBytes ?? null, projectDiskTotalBytes: disk?.totalBytes ?? null,projectDiskVolume: disk?.volume ?? null, projectDiskPath: disk?.path ?? null },- tools: Object.fromEntries(toolValues), devices: Object.fromEntries(deviceValues),+ tools: Object.fromEntries(toolValues.filter(([, result]) => result.status !== 'unverified').map(([name, result]) => [name, result.status === 'available'])),+ ...(Object.keys(toolProbeErrors).length ? { toolProbeErrors } : {}), devices: Object.fromEntries(deviceValues),// A daemon from before the launch flags leaves this out, and a start then switches the model after it.launchModel: launchModelEngines() }}@@ -73,7 +88,7 @@ export function quotaOf(reply: Json, engine: string, customAccount: boolean) {}export function admission(capacity: Json, engine: Json, quota: ReturnType<typeof quotaOf>, requirements: Requirements, customAccount = false) {const reasons: string[] = [], unknown: string[] = []- const resources = object(capacity.resources), tools = object(capacity.tools), devices = object(capacity.devices)+ const resources = object(capacity.resources), tools = object(capacity.tools), devices = object(capacity.devices), toolProbeErrors = object(capacity.toolProbeErrors)if (engine.installed !== true) reasons.push('The selected agent engine is not installed on this machine.')if (engine.signedIn === false && !customAccount) reasons.push('The selected agent engine is not signed in on this machine.')const memory = finite(resources.memoryTotalBytes) && finite(resources.memoryUsedBytes)@@ -90,7 +105,10 @@ export function admission(capacity: Json, engine: Json, quota: ReturnType<typeofif (requirements.maxCpuPercent < 100) reasons.push('CPU headroom could not be verified.')} else if (resources.cpuPercent > requirements.maxCpuPercent) reasons.push(`CPU use is ${resources.cpuPercent}%; the requested maximum is ${requirements.maxCpuPercent}%.`)if (resources.memoryPressure === 'critical') reasons.push('The machine reports critical memory pressure.')- for (const t of requirements.tools) if (tools[t] !== true) reasons.push(`Required tool unavailable: ${t}.`)+ for (const t of requirements.tools) {+ const reason = requiredToolReason(t, tools[t], toolProbeErrors[t])+ if (reason) reasons.push(reason)+ }for (const d of requirements.devices) if (devices[d] !== true) reasons.push(`Required device unavailable: ${d}.`)if (!quota.known) {unknown.push('Subscription allowance is unknown for this account.')diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.tsindex 3fde72c38..fc0474dcc 100644--- a/cli/src/superTabs/machines.spec.ts+++ b/cli/src/superTabs/machines.spec.ts@@ -1,6 +1,7 @@import { describe, expect, it, vi } from 'vitest'import { fakeCore } from '../testing/fakeCore.js'import { ContributedMachines } from './machines.js'+import { SuperTabHostError } from './host.js'import { startAgent } from './start.js'function fixture() {@@ -15,7 +16,7 @@ function fixture() {const request = vi.fn(async () => ({ providers: [{ provider: 'codex', outcome: 'answered', httpStatus: 200,body: { account_email: '[email protected]', secret: 'private-token', rate_limit: { primary_window: { used_percent: 25, reset_at: 1234567890 } } } }] }))const close = vi.fn(), open = vi.fn(async () => ({ machineId: 'host', request, close }))- const capabilities = vi.fn(async () => ({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 }, tools: { git: true } }))+ const capabilities = vi.fn(async (_input: { cwd: string; tools: string[]; devices: string[] }) => ({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 }, tools: { git: true } }))const start = vi.fn(async () => ({ ok: true }))const prepare = vi.fn(async () => {})const machine = new ContributedMachines(core, 'host', { backend, open, capabilities, start, prepare, now: () => now } as never)@@ -44,7 +45,7 @@ describe('contributed machine executor', () => {const f = fixture()await f.machine.tick()expect(f.open).toHaveBeenCalledOnce()- expect(f.capabilities).toHaveBeenCalledWith({ cwd: '/repo', tools: ['git'], devices: [] })+ expect(f.capabilities).toHaveBeenCalledWith({ cwd: '/repo', tools: ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3'], devices: [] })const body = f.backend.mock.calls.find(c => c[1] === 'POST')![3]expect(body).toMatchObject({ generation: 'grant', runningHarnessIds: ['running'], stoppedHarnessIds: ['stopped'], quotas: [{ engine: 'codex', known: true }] })expect(JSON.stringify(body)).not.toMatch(/private@example|private-token|account_email/)@@ -59,6 +60,63 @@ describe('contributed machine executor', () => {f.machine.stop(); f.advance(); await f.machine.tick()expect(f.open).toHaveBeenCalledOnce()})+ it('probes common tools plus pending-start requirements and publishes failures without calling them missing', async () => {+ const f = fixture()+ Object.assign(f.pool.starts[0]!, { request: { requirements: { tools: ['docker'] } } })+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },+ tools: { git: true, node: true, docker: false }, toolProbeErrors: { flutter: 'interactive shell timed out' } } as never)+ await f.machine.tick()+ expect(f.capabilities).toHaveBeenCalledWith({ cwd: '/repo',+ tools: ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3', 'docker'], devices: [] })+ const body = f.backend.mock.calls.find(c => c[1] === 'POST')![3]+ expect(body).toMatchObject({ tools: { git: true, node: true, docker: false }, toolProbeErrors: { flutter: 'interactive shell timed out' } })+ })+ it('reuses tool readings across tabs and heartbeat passes without another shell probe', async () => {+ const f = fixture()+ const original = f.backend.getMockImplementation()!+ f.backend.mockImplementation(async (core, method, path, body) => path.endsWith('/pools')+ ? [f.pool, { ...f.pool, tabId: 'second-tab' }]+ : original(core, method, path, body))+ await f.machine.tick()+ expect(f.capabilities.mock.calls.map(([input]) => input.tools)).toEqual([+ ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3'], []])+ const observations = f.backend.mock.calls.filter(c => c[1] === 'POST' && String(c[2]).includes('/machines/'))+ expect(observations[1]?.[3]).toMatchObject({ tools: { git: true }, toolProbeErrors: { node: 'The tool probe did not complete.' } })+ f.advance()+ await f.machine.tick()+ expect(f.capabilities.mock.calls.slice(2).map(([input]) => input.tools)).toEqual([[], []])+ })+ it('probes a newly queued tool immediately and retries an unverified tool after its short cache window', async () => {+ const f = fixture()+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },+ tools: { git: true, node: true, npm: true, gh: true, cargo: true, python3: true },+ toolProbeErrors: { flutter: 'interactive shell timed out' } } as never)+ await f.machine.tick()+ Object.assign(f.pool.starts[0]!, { request: { requirements: { tools: ['docker'] } } })+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },+ tools: { docker: true } } as never)+ f.advance()+ await f.machine.tick()+ expect(f.capabilities.mock.lastCall?.[0]).toMatchObject({ tools: ['docker'] })+ f.step(61_000)+ await f.machine.tick()+ expect(f.capabilities.mock.lastCall?.[0]).toMatchObject({ tools: ['flutter'] })+ })+ it('retries a probe-failure observation without its new field for an older backend', async () => {+ const f = fixture()+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },+ tools: { git: true }, toolProbeErrors: { flutter: 'interactive shell timed out' } } as never)+ const original = f.backend.getMockImplementation()!+ f.backend.mockImplementation(async (core, method, path, body) => {+ if (method === 'POST' && (body as Record<string, unknown>)?.toolProbeErrors) throw new SuperTabHostError('VALIDATION_ERROR', 'Unrecognized key: toolProbeErrors', 400)+ return original(core, method, path, body)+ })+ await f.machine.tick()+ const posts = f.backend.mock.calls.filter(c => c[1] === 'POST')+ expect(posts).toHaveLength(2)+ expect(posts[0]![3]).toHaveProperty('toolProbeErrors')+ expect(posts[1]![3]).not.toHaveProperty('toolProbeErrors')+ })it('binds stopped native proof to the current slot reservation generation', async () => {const f = fixture()diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.tsindex 946071831..24f4af9b9 100644--- a/cli/src/superTabs/machines.ts+++ b/cli/src/superTabs/machines.ts@@ -1,6 +1,6 @@import type { CoreApi } from '../core/api.js'import { openDaemon } from '../agentClient/daemon.js'-import { backend } from './host.js'+import { backend, SuperTabHostError } from './host.js'import { identifier, object } from './wire.js'import { readCapabilities, quotaOf } from './capacity.js'import { startAgent, type ScopedRequest } from './start.js'@@ -9,6 +9,11 @@ import { createSwarmCloneTrustRepair } from './cloneTrust.js'const finite = (v: unknown) => typeof v === 'number' && Number.isFinite(v) ? v : nullconst list = (v: unknown) => Array.isArray(v) ? v.map(object) : []+// The route's requirements are not known before the Lead routes. Include common tools in each snapshot, then+// include tools from pending starts. An uncommon tool first named during routing remains unverified and is refused.+const COMMON_TOOLS = ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3']+const VERIFIED_TOOL_TTL_MS = 10 * 60_000+const UNVERIFIED_TOOL_TTL_MS = 60_000/** How often the executor asks the backend what its tabs want of it (a start waiting, a pause) and compares what it* would report with what it last did. Only this read is this frequent: the resource readers run when there is@@ -32,6 +37,8 @@ export class ContributedMachines {* running and stopped, whether new work is paused, and which starts are waiting. A change in any of them is* sent at once; otherwise the heartbeat keeps the reading fresh. */private readonly published = new Map<string, { at: number; signature: string }>()+ /** Shell tool availability belongs to this machine, not a tab. Resource readings still refresh every heartbeat. */+ private readonly toolCache = new Map<string, { value: boolean | null; detail?: string; until: number }>()constructor(private readonly core: CoreApi, private readonly machineId: string,private readonly deps: { backend: typeof backend; open: typeof openDaemon; capabilities: typeof readCapabilities;start: typeof startAgent; now: () => number; prepare?: typeof prepareSwarmProject } =@@ -67,6 +74,10 @@ export class ContributedMachines {for (const id of ids) if (await this.core.agents.stopped?.(id)) stopped.push(id)const running = ids.filter(id => !!this.core.agents.byAgent(id) && !stopped.includes(id))const pending = list(row.starts)+ const tools = [...new Set([...COMMON_TOOLS, ...pending.flatMap(start => {+ const wanted = object(object(start.request).requirements).tools+ return Array.isArray(wanted) ? wanted.filter((name): name is string => typeof name === 'string' && /^[A-Za-z0-9_.+-]{1,80}$/.test(name)) : []+ })])].slice(0, 20)const signature = JSON.stringify([row.reservations, pool.generation, [...running].sort(), [...stopped].sort(), row.paused === true || pool.paused === true,pending.map(start => start.id).sort()])const last = this.published.get(String(row.tabId))@@ -75,11 +86,27 @@ export class ContributedMachines {// The existing usage reader owns its provider cache. Raw responses and// account credentials never leave this machine.const readings = usage ??= await local.request('usage_read', {}, 20_000).catch(() => ({}))- const observed = await this.deps.capabilities({ cwd: pool.cwd, tools: ['git'], devices: [] })+ const sampledAt = this.deps.now()+ const probeTools = tools.filter(name => (this.toolCache.get(name)?.until ?? 0) <= sampledAt)+ const observed = await this.deps.capabilities({ cwd: pool.cwd, tools: probeTools, devices: [] })if (this.stopped) returnconst resources = object('resources' in observed ? observed.resources : null)const gpus = list(resources.gpus).map(g => finite(g.utilizationPercent)).filter((n): n is number => n !== null)- const observation = { generation: pool.generation, tools: 'tools' in observed ? observed.tools : {},+ const measuredTools = object('tools' in observed ? observed.tools : null)+ const measuredErrors = object('toolProbeErrors' in observed ? observed.toolProbeErrors : null)+ for (const name of probeTools) {+ const value = typeof measuredTools[name] === 'boolean' ? measuredTools[name] as boolean : null+ const detail = value === null ? String(measuredErrors[name] ?? 'The tool probe did not complete.') : undefined+ this.toolCache.set(name, { value, detail, until: sampledAt + (value === null ? UNVERIFIED_TOOL_TTL_MS : VERIFIED_TOOL_TTL_MS) })+ }+ const reportedTools: Record<string, boolean> = {}, toolProbeErrors: Record<string, string> = {}+ for (const name of tools) {+ const result = this.toolCache.get(name)+ if (result?.value !== null && result?.value !== undefined) reportedTools[name] = result.value+ else toolProbeErrors[name] = result?.detail ?? 'The tool probe did not complete.'+ }+ const observation = { generation: pool.generation, tools: reportedTools,+ ...(Object.keys(toolProbeErrors).length ? { toolProbeErrors } : {}),resources: { cpuPercent: finite(resources.cpuPercent), memoryUsedBytes: finite(resources.memoryUsedBytes), memoryTotalBytes: finite(resources.memoryTotalBytes),diskFreeBytes: finite(resources.diskFreeBytes), diskTotalBytes: finite(resources.diskTotalBytes), projectDiskFreeBytes: finite(resources.projectDiskFreeBytes), projectDiskTotalBytes: finite(resources.projectDiskTotalBytes),deviceType: typeof resources.deviceType === 'string' ? resources.deviceType : null,@@ -92,7 +119,14 @@ export class ContributedMachines {...(Array.isArray(row.reservations) ? { stoppedReservations: list(row.reservations).filter(r => stopped.includes(String(r.harnessId))&& identifier(r.harnessId) && identifier(r.generation)).map(r => ({ harnessId: r.harnessId, generation: r.generation })) } : {}) }try {- await this.deps.backend(this.core, 'POST', `/api/super-tabs/${row.tabId}/machines/${this.machineId}`, observation)+ const path = `/api/super-tabs/${row.tabId}/machines/${this.machineId}`+ try { await this.deps.backend(this.core, 'POST', path, observation) }+ catch (error) {+ if (!(error instanceof SuperTabHostError) || error.status !== 400 || error.code !== 'VALIDATION_ERROR' || !('toolProbeErrors' in observation)) throw error+ // Older backends reject the additive field. Their boolean-only view still safely refuses an unverified tool.+ const { toolProbeErrors: _probeErrors, ...legacy } = observation+ await this.deps.backend(this.core, 'POST', path, legacy)+ }this.published.set(String(row.tabId), { at: began, signature })} catch (error) { this.failed = true; throw error }}diff --git a/cli/src/superTabs/routeCommand.spec.ts b/cli/src/superTabs/routeCommand.spec.tsindex 03355c979..787d17947 100644--- a/cli/src/superTabs/routeCommand.spec.ts+++ b/cli/src/superTabs/routeCommand.spec.ts@@ -210,7 +210,7 @@ describe('routeTask', () => {expect(reasonOf('mac').some(r => r.includes('This machine does not have the project folder. Add it to the Swarm so it can clone the project.'))).toBe(true)expect(reasonOf('mac').some(r => /Project disk space/.test(r))).toBe(false)// Its other reasons stay.- expect(reasonOf('mac').some(r => /Required tool unavailable: git/.test(r))).toBe(true)+ expect(reasonOf('mac').some(r => /Required tool not verified: git/.test(r))).toBe(true)// The Lead's own machine keeps the plain reason: its folder was named by the Lead itself.expect(reasonOf('remote-1').some(r => r.includes('Project disk space could not be verified.'))).toBe(true)})@@ -391,7 +391,7 @@ describe('routeTask', () => {const result = await routeTask({ ...input, assessment: { ...input.assessment, needs } }, s, where as never, open)expect(result).toMatchObject({ error: 'NO_CANDIDATE' })const reasons = (result.excluded as Array<{ reason: string }>).map(e => e.reason)- expect(reasons).toContain('Required tool unavailable: git. Required device unavailable: /dev/ttyUSB0.')+ expect(reasons).toContain('Required tool missing: git. Required device unavailable: /dev/ttyUSB0.')const d = open.mock.results[0] && await open.mock.results[0].value as ReturnType<typeof daemon>expect(d.request).toHaveBeenCalledWith('super_tabs', expect.objectContaining({ action: 'capacity', cwd: '/repo', tools: ['git'], devices: ['/dev/ttyUSB0'] }), 20_000)})@@ -409,7 +409,7 @@ describe('routeTask', () => {const requirements = { memoryBytes: 0, diskBytes: 0, maxCpuPercent: 100, tools: ['git'], devices: ['/dev/x'], quotaRequired: false }const { reasons } = admission({ resources: {}, tools: {}, devices: {} }, { installed: true, signedIn: true }, quota, requirements)expect(reasons).toEqual([- 'Required tool unavailable: git.', 'Required device unavailable: /dev/x.',+ 'Required tool not verified: git. Ask the machine owner to check it and refresh the machine reading.', 'Required device unavailable: /dev/x.','The subscription has an exhausted usage window. Wait for a fresh allowance reading or choose another account.'])const unknown = admission({ resources: {}, tools: {}, devices: {} }, { installed: true, signedIn: true }, quotaOf({}, 'claude', false), { ...requirements, tools: [], devices: [] })expect(unknown).toMatchObject({ allowed: true, reasons: [] })@@ -653,7 +653,16 @@ describe('routing contributed capacity', () => {const { answer } = await runPool(pool, true)expect(answer).toMatchObject({ error: 'NO_CANDIDATE' })expect(answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ title: expect.stringContaining('Multiply'),- reason: expect.stringContaining('Required tool unavailable: git.') })]))+ reason: expect.stringContaining('Required tool missing: git.') })]))+ })+ it('explains a contributed member tool probe that did not complete', async () => {+ const pool = healthy()+ delete (pool.observation.tools as Record<string, unknown>).git+ Object.assign(pool.observation, { toolProbeErrors: { git: 'interactive shell timed out' } })+ const { answer } = await runPool(pool, true)+ expect(answer).toMatchObject({ error: 'NO_CANDIDATE' })+ expect(answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ title: expect.stringContaining('Multiply'),+ reason: expect.stringMatching(/Required tool not verified: git.*interactive shell timed out.*check it/) })]))})it('withholds a contributed member absent from an explicit running-terminal list', async () => {const pool = healthy(); pool.observation.runningHarnessIds = []@@ -709,4 +718,14 @@ describe('routing contributed capacity', () => {it('refuses unobserved tools and devices, without probing arbitrary paths on a teammate computer', async () => {expect((await runPool(healthy(), false, { tools: ['docker'], devices: ['/dev/fixture'] })).answer.ok).not.toBe(true)})+ it('distinguishes reported missing tools from unverified contributed probes in route reasons', async () => {+ const missing = healthy(); missing.observation.tools.git = false+ const absent = await runPool(missing)+ expect(absent.answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ reason: expect.stringContaining('Required tool missing: git.') })]))++ const unverified = healthy(); delete (unverified.observation.tools as Record<string, unknown>).git+ Object.assign(unverified.observation, { toolProbeErrors: { git: 'interactive shell timed out' } })+ const failed = await runPool(unverified)+ expect(failed.answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ reason: expect.stringMatching(/Required tool not verified: git.*interactive shell timed out/) })]))+ })})diff --git a/cli/src/superTabs/routeCommand.ts b/cli/src/superTabs/routeCommand.tsindex f9584c089..6ef077bac 100644--- a/cli/src/superTabs/routeCommand.ts+++ b/cli/src/superTabs/routeCommand.ts@@ -4,7 +4,7 @@* and records the decision in the tab. */import { randomUUID } from 'node:crypto'import { openDaemon, type Daemon, type Where } from '../agentClient/daemon.js'-import { startRequirements } from './capacity.js'+import { requiredToolReason, startRequirements } from './capacity.js'import { inspectCapacity, type ScopedRequest } from './start.js'import { ENDED_WORK, object, within } from './wire.js'import { choose, decide, parseAssessment, quotaOf2, rateFor, recordOf, repoMatch, type Assessment, type Commitment, type MemberFrame, type Policy, type Probe, type SwarmRule, type TaskFolder } from './route.js'@@ -24,10 +24,10 @@ const UNKNOWN_QUOTA: Probe['quota'] = { known: false, leftPercent: null, resetsAconst UNKNOWN_LOAD: NonNullable<Probe['load']> = { agents: null, freeMemoryBytes: null, hasFolder: null }const EXHAUSTED_WINDOW = 'The subscription has an exhausted usage window. Wait for a fresh allowance reading or choose another account.'/** A running member keeps its engine and memory, but still needs a live terminal, account allowance and the task's tools. */-function memberRefusals(running: boolean | null, quota: Probe['quota'], tools: string[], devices: string[], reportedTools: Json, reportedDevices: Json): string[] {+function memberRefusals(running: boolean | null, quota: Probe['quota'], tools: string[], devices: string[], reportedTools: Json, reportedDevices: Json, toolProbeErrors: Json): string[] {return [...(running === false ? ['Its terminal is not running.'] : []),- ...tools.filter(tool => reportedTools[tool] !== true).map(tool => `Required tool unavailable: ${tool}.`),+ ...tools.map(tool => requiredToolReason(tool, reportedTools[tool], toolProbeErrors[tool])).filter((reason): reason is string => reason !== null),...devices.filter(device => reportedDevices[device] !== true).map(device => `Required device unavailable: ${device}.`),...(quota.leftPercent === 0 ? [EXHAUSTED_WINDOW] : []),]@@ -234,12 +234,12 @@ async function probeMachine(machineId: string, { name: machineName, status, acco...(pool.availableSlots === 0 ? ['This machine has no available agent slots.'] : []),...(!finite(resources.projectDiskFreeBytes) || resources.projectDiskFreeBytes < needs.diskBytes ? ['The project has insufficient measured free disk.'] : []),...(!finite(resources.memoryTotalBytes) || !finite(resources.memoryUsedBytes) || resources.memoryTotalBytes - resources.memoryUsedBytes < needs.memoryBytes ? ['Insufficient measured free memory.'] : []),- ...needs.tools.filter(t => tools[t] !== true).map(t => `Required tool not reported: ${t}.`),+ ...needs.tools.map(t => requiredToolReason(t, tools[t], object(observed.toolProbeErrors)[t])).filter((reason): reason is string => reason !== null),...(needs.devices.length ? ['Required devices have not been reported.'] : [])]return [...members.map(m => {const allowance = quota(String(object(m.configuration).engine))const running = Array.isArray(observed.runningHarnessIds) ? observed.runningHarnessIds.includes(m.harnessId) : null- return memberProbe(m, fresh ? memberRefusals(running, allowance, needs.tools, needs.devices, tools, object(observed.devices)) : [OFFLINE],+ return memberProbe(m, fresh ? memberRefusals(running, allowance, needs.tools, needs.devices, tools, object(observed.devices), object(observed.toolProbeErrors)) : [OFFLINE],allowance, null, fresh && running === null)}),...models.map(m => newProbe(m, [...newReasons, ...(quota(m.engine).leftPercent === 0 ? ['Subscription exhausted.'] : [])], quota(m.engine), fresh && quota(m.engine).known,@@ -271,7 +271,7 @@ async function probeMachine(machineId: string, { name: machineName, status, accoconst allowance = quotaOf2(inspected.quota)const capacity = object(inspected.capacity)const refusals = memberRefusals(live.has(String(m.harnessId)), allowance, needs.tools, needs.devices,- object(capacity.tools), object(capacity.devices))+ object(capacity.tools), object(capacity.devices), object(capacity.toolProbeErrors))const frame = frames.find(f => f.id === m.harnessId)probes.push(memberProbe(m, refusals, allowance, frame ? frameOf(frame) : null))}