Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Distinguish missing and unverified tools in Swarm routing (#1606)

0xken15dac7ceeb812b
diff --git a/backend/src/superTabs/machines.test.ts b/backend/src/superTabs/machines.test.ts
index a0ea081cb..70e218f01 100644
--- a/backend/src/superTabs/machines.test.ts
+++ b/backend/src/superTabs/machines.test.ts
@@ -1,12 +1,18 @@
import { describe, expect, it } from 'vitest'
import { createSuperTab, SuperTabError, type AgentStart, type AgentStartRequest, type SuperTabState } from './model.js'
-import { admitMachineStart, machineSlots, type ContributedMachine } from './machines.js'
+import { admitMachineStart, machineSlots, machineObservationSchema, type ContributedMachine } from './machines.js'
const NOW = 1_000_000
const requirements = { memoryBytes: 1, diskBytes: 1, maxCpuPercent: 50, tools: ['git'], devices: [], quotaRequired: false }
const observation = { generation: 'o', receivedAt: NOW, resources: { cpuPercent: 1, memoryTotalBytes: 1, memoryUsedBytes: 1, diskFreeBytes: 1, diskTotalBytes: 1,
projectDiskFreeBytes: 1, deviceType: null, chip: null, os: null, gpuPercent: null }, tools: {}, quotas: [], runningHarnessIds: [], stoppedHarnessIds: [] }
+it('accepts optional tool probe failures while preserving older observations', () => {
+ const { receivedAt: _receivedAt, ...sent } = observation
+ expect(machineObservationSchema.safeParse(sent).success).toBe(true)
+ expect(machineObservationSchema.safeParse({ ...sent, tools: { git: true }, toolProbeErrors: { flutter: 'interactive shell timed out' } }).success).toBe(true)
+})
+
function setup(machine: Partial<ContributedMachine> = {}) {
const state: SuperTabState = createSuperTab('tab', 'Team', { id: 'owner', email: '[email protected]' }, 'test')
state.machines = [{ machineId: 'host', name: 'Studio Mac', contributorId: 'owner', membershipGeneration: 'tab', generation: 'g1',
diff --git a/backend/src/superTabs/machines.ts b/backend/src/superTabs/machines.ts
index c4bc2a9dd..57fb48e2d 100644
--- a/backend/src/superTabs/machines.ts
+++ b/backend/src/superTabs/machines.ts
@@ -32,6 +32,7 @@ const quota = z.object({ engine: z.string().regex(/^[a-z][a-z0-9_-]{0,79}$/), kn
}).strict()
export const machineObservationSchema = z.object({ generation: z.string().max(160), resources: resource,
tools: z.record(z.string().max(80), z.boolean()).refine(v => Object.keys(v).length <= 20),
+ toolProbeErrors: z.record(z.string().max(80), z.string().max(200)).refine(v => Object.keys(v).length <= 20).optional(),
quotas: z.array(quota).max(32), runningHarnessIds: z.array(z.string().max(160)).max(1000),
stoppedHarnessIds: z.array(z.string().max(160)).max(1000),
stoppedReservations: z.array(z.object({ harnessId: z.string().min(1).max(160), generation: z.string().min(1).max(160) }).strict()).max(1000).optional(),
diff --git a/cli/src/lib/engineLaunch.spec.ts b/cli/src/lib/engineLaunch.spec.ts
index 9d5a91869..d18d334ae 100644
--- a/cli/src/lib/engineLaunch.spec.ts
+++ b/cli/src/lib/engineLaunch.spec.ts
@@ -1136,14 +1136,37 @@ exec ${ZSH} -c "$script" "$@"
}
describe('commandAvailableInInteractiveShell', () => {
+ it('reports a shell failure as unverified rather than a missing command', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'harness-failed-probe-'))
+ dirs.push(dir)
+ const shell = join(dir, 'bash')
+ writeFileSync(shell, '#!/bin/sh\nexit 9\n')
+ chmodSync(shell, 0o700)
+ const result = await commandAvailableInInteractiveShell('git', shell, undefined, true)
+ expect(result).toMatchObject({ status: 'unverified', detail: expect.stringContaining('9') })
+ })
+ it('bounds a hung shell probe and reports a timeout as unverified', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'harness-slow-probe-'))
+ dirs.push(dir)
+ const shell = join(dir, 'bash')
+ writeFileSync(shell, '#!/bin/sh\nexec /bin/sleep 30\n')
+ chmodSync(shell, 0o700)
+ const started = Date.now()
+ const result = await commandAvailableInInteractiveShell('git', shell, undefined, true)
+ expect(result).toEqual({ status: 'unverified', detail: 'Interactive shell tool probe timed out after 5 seconds.' })
+ expect(Date.now() - started).toBeLessThan(7_000)
+ }, 8_000)
it('uses the same bash interactive PATH that launches a new engine', async () => {
const binDir = mkdtempSync(join(tmpdir(), 'harness-engine-bin-'))
dirs.push(binDir)
executable(binDir, 'kilo')
process.env.HARNESS_ENGINE_TEST_PATH = binDir
- await expect(commandAvailableInInteractiveShell('kilo', bashProbeShell())).resolves.toBe(true)
- await expect(commandAvailableInInteractiveShell('missing-engine', bashProbeShell())).resolves.toBe(false)
+ const shell = bashProbeShell()
+ await expect(commandAvailableInInteractiveShell('kilo', shell)).resolves.toBe(true)
+ await expect(commandAvailableInInteractiveShell('missing-engine', shell)).resolves.toBe(false)
+ await expect(commandAvailableInInteractiveShell('kilo', shell, undefined, true)).resolves.toEqual({ status: 'available' })
+ await expect(commandAvailableInInteractiveShell('missing-engine', shell, undefined, true)).resolves.toEqual({ status: 'missing' })
})
it('recognizes an installed vendor path before the shell profile has it on PATH', async () => {
diff --git a/cli/src/lib/engineLaunch.ts b/cli/src/lib/engineLaunch.ts
index 7dd59e2ed..19d5376e7 100644
--- a/cli/src/lib/engineLaunch.ts
+++ b/cli/src/lib/engineLaunch.ts
@@ -1335,35 +1335,46 @@ function availabilityScript(recipe: EngineInstallRecipe | undefined): string {
` for bin in ${names}; do [ -f "$npm_prefix/bin/$bin" ] && [ -x "$npm_prefix/bin/$bin" ] && exit 0; done`,
'fi',
] : []),
- 'exit 1',
+ // An ordinary shell/profile failure may also exit 1. Only this value proves the probe ran to completion and found nothing.
+ 'exit 42',
].join('\n')
}
+export type CommandAvailability = { status: 'available' | 'missing' | 'unverified'; detail?: string }
/**
* Does the same interactive shell that launches a new agent resolve this CLI?
*
* The fallback is intentionally the daemon PATH: without a usable absolute
* SHELL there is no safer context to consult and direct launch is used too.
*/
+export function commandAvailableInInteractiveShell(command: string, shell: string | undefined, recipe: EngineInstallRecipe | undefined, detailed: true): Promise<CommandAvailability>
+export function commandAvailableInInteractiveShell(command: string, shell?: string, recipe?: EngineInstallRecipe): Promise<boolean>
export async function commandAvailableInInteractiveShell(
command: string,
shell: string | undefined = undefined,
recipe: EngineInstallRecipe | undefined = undefined,
-): Promise<boolean> {
+ detailed = false,
+): Promise<boolean | CommandAvailability> {
const interactive = interactiveEngineShell(shell)
if (!interactive) {
- return binaryOnPath(command)
- || (recipe ? engineInstallPaths(recipe).some((candidate) => binaryOnPath(candidate)) : false)
+ const found = binaryOnPath(command) || (recipe ? engineInstallPaths(recipe).some((candidate) => binaryOnPath(candidate)) : false)
+ return detailed ? { status: found ? 'available' : 'missing' } : found
}
const [file, ...args] = engineShellArgv(interactive, [availabilityScript(recipe), 'harness-engine-probe', command])
- return await new Promise((resolve) => {
+ const result = await new Promise<CommandAvailability>((resolve) => {
execFile(
file,
args,
{ timeout: 5_000 },
- (error) => resolve(!error),
+ (error) => {
+ if (!error) return resolve({ status: 'available' })
+ if (Number(error.code) === 42) return resolve({ status: 'missing' })
+ if (error.killed) return resolve({ status: 'unverified', detail: 'Interactive shell tool probe timed out after 5 seconds.' })
+ resolve({ status: 'unverified', detail: `Interactive shell tool probe failed (${String(error.code ?? error.signal ?? 'unknown error')}).` })
+ },
)
})
+ return detailed ? result : result.status === 'available'
}
export type CommandFlagSupport = 'supported' | 'unsupported' | 'unknown'
diff --git a/cli/src/superTabs/capacity.spec.ts b/cli/src/superTabs/capacity.spec.ts
index 3bb9f4418..c1bef3504 100644
--- a/cli/src/superTabs/capacity.spec.ts
+++ b/cli/src/superTabs/capacity.spec.ts
@@ -3,7 +3,7 @@ import { admission, quotaOf, readCapabilities } from './capacity.js'
const requirements = { memoryBytes: 100, diskBytes: 200, maxCpuPercent: 80, tools: ['git'], devices: ['/dev/test'], quotaRequired: false }
it('uses the destination filesystem and reports missing tools/devices without executing commands', async () => {
const deps = { resources: vi.fn(async () => ({ cpuPercent: 25, memoryTotalBytes: 1000, memoryUsedBytes: 300 })),
- disk: vi.fn(async () => ({ freeBytes: 500, volume: 'volume', path: '/fixture/repo' })), tool: vi.fn(async () => true), device: vi.fn(async () => true) }
+ disk: vi.fn(async () => ({ freeBytes: 500, volume: 'volume', path: '/fixture/repo' })), tool: vi.fn(async () => ({ status: 'available' as const })), device: vi.fn(async () => true) }
const capacity = await readCapabilities({ cwd: '/fixture/repo', tools: ['git'], devices: ['/dev/test'] }, deps)
expect(deps.disk).toHaveBeenCalledWith('/fixture/repo', false)
// Which engines this daemon launches with a model: a daemon without the field gets the switch after the start.
@@ -28,3 +28,18 @@ it('keeps unknown quota distinct from unused quota and honors all limiting windo
expect(admission({}, { installed: true, signedIn: false }, quotaOf({}, 'codex', false), requirements).reasons).toContain('The selected agent engine is not signed in on this machine.')
expect(admission({}, {}, quotaOf({}, 'claude', false), { ...requirements, quotaRequired: true }).reasons).toContain('The requested subscription allowance could not be verified.')
})
+it('distinguishes a missing tool from a failed probe and keeps the failure detail', async () => {
+ const deps = { resources: vi.fn(async () => ({ cpuPercent: 10, memoryTotalBytes: 1000, memoryUsedBytes: 100 })),
+ disk: vi.fn(async () => ({ freeBytes: 500, volume: 'volume', path: '/fixture/repo' })),
+ tool: vi.fn(async () => ({ status: 'missing' as const })), device: vi.fn(async () => true) }
+ const missing = await readCapabilities({ cwd: '/fixture/repo', tools: ['git'] }, deps)
+ expect(admission(missing, { installed: true }, quotaOf({}, 'codex', false), { ...requirements, devices: [] }).reasons)
+ .toContain('Required tool missing: git.')
+
+ deps.tool.mockRejectedValueOnce(new Error('interactive shell timed out'))
+ const unverified = await readCapabilities({ cwd: '/fixture/repo', tools: ['git'] }, deps)
+ expect(unverified.tools).toEqual({})
+ expect(unverified.toolProbeErrors).toEqual({ git: 'interactive shell timed out' })
+ expect(admission(unverified, { installed: true }, quotaOf({}, 'codex', false), { ...requirements, devices: [] }).reasons)
+ .toEqual(expect.arrayContaining([expect.stringMatching(/Required tool not verified: git.*interactive shell timed out/)]))
+})
diff --git a/cli/src/superTabs/capacity.ts b/cli/src/superTabs/capacity.ts
index d6aa7a7b5..09bfd8c3c 100644
--- a/cli/src/superTabs/capacity.ts
+++ b/cli/src/superTabs/capacity.ts
@@ -2,7 +2,7 @@ import { access } from 'node:fs/promises'
import { constants } from 'node:fs'
import { isAbsolute } from 'node:path'
import { readMachineResources } from '../lib/machineResources.js'
-import { commandAvailableInInteractiveShell } from '../lib/engineLaunch.js'
+import { commandAvailableInInteractiveShell, type CommandAvailability } from '../lib/engineLaunch.js'
import { launchModelEngines } from '../lib/launchModel.js'
import { object } from './wire.js'
import { readCapacityDisk } from './disk.js'
@@ -19,12 +19,20 @@ export const startRequirements = (needs: { tools: string[]; devices: string[] })
// CPU use is only a momentary reading: a busy machine would refuse every candidate (seen in the route e2e at 98.7%), so the route does not cap it.
({ ...START_REQUIREMENTS, maxCpuPercent: 100, tools: [...new Set([...START_REQUIREMENTS.tools, ...needs.tools])], devices: needs.devices })
+/** A missing result is a completed check; an absent value or failed check is only unverified. */
+export function requiredToolReason(tool: string, reported: unknown, failure?: unknown): string | null {
+ if (reported === true) return null
+ if (reported === false) return `Required tool missing: ${tool}.`
+ const detail = typeof failure === 'string' && failure ? `${failure} ` : ''
+ return `Required tool not verified: ${tool}. ${detail}Ask the machine owner to check it and refresh the machine reading.`
+}
+
export async function readCapabilities(input: Json, deps = {
resources: readMachineResources,
disk: readCapacityDisk,
- tool: commandAvailableInInteractiveShell,
+ tool: (name: string): Promise<CommandAvailability> => commandAvailableInInteractiveShell(name, undefined, undefined, true),
device: async (path: string) => { await access(path, constants.R_OK | constants.W_OK); return true },
-}) {
+}): Promise<Json> {
const cwd = input.cwd, tools = input.tools ?? [], devices = input.devices ?? []
if (typeof cwd !== 'string' || !isAbsolute(cwd) || cwd.length > 4096 || /[\x00-\x1f\x7f]/.test(cwd)
|| (input.worktree !== undefined && typeof input.worktree !== 'boolean')
@@ -33,9 +41,13 @@ export async function readCapabilities(input: Json, deps = {
return { error: 'INVALID_REQUEST', detail: 'Name an absolute project path and up to 20 tools and devices.' }
}
const probeTools = async () => {
- const answers: Array<readonly [string, boolean]> = []
+ const answers: Array<readonly [string, CommandAvailability]> = []
for (let index = 0; index < tools.length; index += 4) {
- answers.push(...await Promise.all(tools.slice(index, index + 4).map(async t => [t, await deps.tool(t).catch(() => false)] as const)))
+ answers.push(...await Promise.all(tools.slice(index, index + 4).map(async t => {
+ const result = await deps.tool(t).catch(error => ({ status: 'unverified' as const,
+ detail: (error instanceof Error ? error.message : String(error)).slice(0, 200) }))
+ return [t, result] as const
+ })))
}
return answers
}
@@ -46,9 +58,12 @@ export async function readCapabilities(input: Json, deps = {
probeTools(),
Promise.all(devices.map(async d => [d, await deps.device(d).catch(() => false)] as const)),
])
+ const toolProbeErrors = Object.fromEntries(toolValues.filter(([, result]) => result.status === 'unverified')
+ .map(([name, result]) => [name, result.detail ?? 'The tool probe did not complete.']))
return { ok: true, sampledAt: Date.now(), resources: { ...resources, projectDiskFreeBytes: disk?.freeBytes ?? null, projectDiskTotalBytes: disk?.totalBytes ?? null,
projectDiskVolume: disk?.volume ?? null, projectDiskPath: disk?.path ?? null },
- tools: Object.fromEntries(toolValues), devices: Object.fromEntries(deviceValues),
+ tools: Object.fromEntries(toolValues.filter(([, result]) => result.status !== 'unverified').map(([name, result]) => [name, result.status === 'available'])),
+ ...(Object.keys(toolProbeErrors).length ? { toolProbeErrors } : {}), devices: Object.fromEntries(deviceValues),
// A daemon from before the launch flags leaves this out, and a start then switches the model after it.
launchModel: launchModelEngines() }
}
@@ -73,7 +88,7 @@ export function quotaOf(reply: Json, engine: string, customAccount: boolean) {
}
export function admission(capacity: Json, engine: Json, quota: ReturnType<typeof quotaOf>, requirements: Requirements, customAccount = false) {
const reasons: string[] = [], unknown: string[] = []
- const resources = object(capacity.resources), tools = object(capacity.tools), devices = object(capacity.devices)
+ const resources = object(capacity.resources), tools = object(capacity.tools), devices = object(capacity.devices), toolProbeErrors = object(capacity.toolProbeErrors)
if (engine.installed !== true) reasons.push('The selected agent engine is not installed on this machine.')
if (engine.signedIn === false && !customAccount) reasons.push('The selected agent engine is not signed in on this machine.')
const memory = finite(resources.memoryTotalBytes) && finite(resources.memoryUsedBytes)
@@ -90,7 +105,10 @@ export function admission(capacity: Json, engine: Json, quota: ReturnType<typeof
if (requirements.maxCpuPercent < 100) reasons.push('CPU headroom could not be verified.')
} else if (resources.cpuPercent > requirements.maxCpuPercent) reasons.push(`CPU use is ${resources.cpuPercent}%; the requested maximum is ${requirements.maxCpuPercent}%.`)
if (resources.memoryPressure === 'critical') reasons.push('The machine reports critical memory pressure.')
- for (const t of requirements.tools) if (tools[t] !== true) reasons.push(`Required tool unavailable: ${t}.`)
+ for (const t of requirements.tools) {
+ const reason = requiredToolReason(t, tools[t], toolProbeErrors[t])
+ if (reason) reasons.push(reason)
+ }
for (const d of requirements.devices) if (devices[d] !== true) reasons.push(`Required device unavailable: ${d}.`)
if (!quota.known) {
unknown.push('Subscription allowance is unknown for this account.')
diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.ts
index 3fde72c38..fc0474dcc 100644
--- a/cli/src/superTabs/machines.spec.ts
+++ b/cli/src/superTabs/machines.spec.ts
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from 'vitest'
import { fakeCore } from '../testing/fakeCore.js'
import { ContributedMachines } from './machines.js'
+import { SuperTabHostError } from './host.js'
import { startAgent } from './start.js'
function fixture() {
@@ -15,7 +16,7 @@ function fixture() {
const request = vi.fn(async () => ({ providers: [{ provider: 'codex', outcome: 'answered', httpStatus: 200,
body: { account_email: '[email protected]', secret: 'private-token', rate_limit: { primary_window: { used_percent: 25, reset_at: 1234567890 } } } }] }))
const close = vi.fn(), open = vi.fn(async () => ({ machineId: 'host', request, close }))
- const capabilities = vi.fn(async () => ({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 }, tools: { git: true } }))
+ const capabilities = vi.fn(async (_input: { cwd: string; tools: string[]; devices: string[] }) => ({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 }, tools: { git: true } }))
const start = vi.fn(async () => ({ ok: true }))
const prepare = vi.fn(async () => {})
const machine = new ContributedMachines(core, 'host', { backend, open, capabilities, start, prepare, now: () => now } as never)
@@ -44,7 +45,7 @@ describe('contributed machine executor', () => {
const f = fixture()
await f.machine.tick()
expect(f.open).toHaveBeenCalledOnce()
- expect(f.capabilities).toHaveBeenCalledWith({ cwd: '/repo', tools: ['git'], devices: [] })
+ expect(f.capabilities).toHaveBeenCalledWith({ cwd: '/repo', tools: ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3'], devices: [] })
const body = f.backend.mock.calls.find(c => c[1] === 'POST')![3]
expect(body).toMatchObject({ generation: 'grant', runningHarnessIds: ['running'], stoppedHarnessIds: ['stopped'], quotas: [{ engine: 'codex', known: true }] })
expect(JSON.stringify(body)).not.toMatch(/private@example|private-token|account_email/)
@@ -59,6 +60,63 @@ describe('contributed machine executor', () => {
f.machine.stop(); f.advance(); await f.machine.tick()
expect(f.open).toHaveBeenCalledOnce()
})
+ it('probes common tools plus pending-start requirements and publishes failures without calling them missing', async () => {
+ const f = fixture()
+ Object.assign(f.pool.starts[0]!, { request: { requirements: { tools: ['docker'] } } })
+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },
+ tools: { git: true, node: true, docker: false }, toolProbeErrors: { flutter: 'interactive shell timed out' } } as never)
+ await f.machine.tick()
+ expect(f.capabilities).toHaveBeenCalledWith({ cwd: '/repo',
+ tools: ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3', 'docker'], devices: [] })
+ const body = f.backend.mock.calls.find(c => c[1] === 'POST')![3]
+ expect(body).toMatchObject({ tools: { git: true, node: true, docker: false }, toolProbeErrors: { flutter: 'interactive shell timed out' } })
+ })
+ it('reuses tool readings across tabs and heartbeat passes without another shell probe', async () => {
+ const f = fixture()
+ const original = f.backend.getMockImplementation()!
+ f.backend.mockImplementation(async (core, method, path, body) => path.endsWith('/pools')
+ ? [f.pool, { ...f.pool, tabId: 'second-tab' }]
+ : original(core, method, path, body))
+ await f.machine.tick()
+ expect(f.capabilities.mock.calls.map(([input]) => input.tools)).toEqual([
+ ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3'], []])
+ const observations = f.backend.mock.calls.filter(c => c[1] === 'POST' && String(c[2]).includes('/machines/'))
+ expect(observations[1]?.[3]).toMatchObject({ tools: { git: true }, toolProbeErrors: { node: 'The tool probe did not complete.' } })
+ f.advance()
+ await f.machine.tick()
+ expect(f.capabilities.mock.calls.slice(2).map(([input]) => input.tools)).toEqual([[], []])
+ })
+ it('probes a newly queued tool immediately and retries an unverified tool after its short cache window', async () => {
+ const f = fixture()
+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },
+ tools: { git: true, node: true, npm: true, gh: true, cargo: true, python3: true },
+ toolProbeErrors: { flutter: 'interactive shell timed out' } } as never)
+ await f.machine.tick()
+ Object.assign(f.pool.starts[0]!, { request: { requirements: { tools: ['docker'] } } })
+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },
+ tools: { docker: true } } as never)
+ f.advance()
+ await f.machine.tick()
+ expect(f.capabilities.mock.lastCall?.[0]).toMatchObject({ tools: ['docker'] })
+ f.step(61_000)
+ await f.machine.tick()
+ expect(f.capabilities.mock.lastCall?.[0]).toMatchObject({ tools: ['flutter'] })
+ })
+ it('retries a probe-failure observation without its new field for an older backend', async () => {
+ const f = fixture()
+ f.capabilities.mockResolvedValueOnce({ ok: true, resources: { cpuPercent: 20, memoryTotalBytes: 1000, memoryUsedBytes: 200, projectDiskFreeBytes: 1000 },
+ tools: { git: true }, toolProbeErrors: { flutter: 'interactive shell timed out' } } as never)
+ const original = f.backend.getMockImplementation()!
+ f.backend.mockImplementation(async (core, method, path, body) => {
+ if (method === 'POST' && (body as Record<string, unknown>)?.toolProbeErrors) throw new SuperTabHostError('VALIDATION_ERROR', 'Unrecognized key: toolProbeErrors', 400)
+ return original(core, method, path, body)
+ })
+ await f.machine.tick()
+ const posts = f.backend.mock.calls.filter(c => c[1] === 'POST')
+ expect(posts).toHaveLength(2)
+ expect(posts[0]![3]).toHaveProperty('toolProbeErrors')
+ expect(posts[1]![3]).not.toHaveProperty('toolProbeErrors')
+ })
it('binds stopped native proof to the current slot reservation generation', async () => {
const f = fixture()
diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.ts
index 946071831..24f4af9b9 100644
--- a/cli/src/superTabs/machines.ts
+++ b/cli/src/superTabs/machines.ts
@@ -1,6 +1,6 @@
import type { CoreApi } from '../core/api.js'
import { openDaemon } from '../agentClient/daemon.js'
-import { backend } from './host.js'
+import { backend, SuperTabHostError } from './host.js'
import { identifier, object } from './wire.js'
import { readCapabilities, quotaOf } from './capacity.js'
import { startAgent, type ScopedRequest } from './start.js'
@@ -9,6 +9,11 @@ import { createSwarmCloneTrustRepair } from './cloneTrust.js'
const finite = (v: unknown) => typeof v === 'number' && Number.isFinite(v) ? v : null
const list = (v: unknown) => Array.isArray(v) ? v.map(object) : []
+// The route's requirements are not known before the Lead routes. Include common tools in each snapshot, then
+// include tools from pending starts. An uncommon tool first named during routing remains unverified and is refused.
+const COMMON_TOOLS = ['git', 'node', 'npm', 'flutter', 'gh', 'cargo', 'python3']
+const VERIFIED_TOOL_TTL_MS = 10 * 60_000
+const UNVERIFIED_TOOL_TTL_MS = 60_000
/** How often the executor asks the backend what its tabs want of it (a start waiting, a pause) and compares what it
* would report with what it last did. Only this read is this frequent: the resource readers run when there is
@@ -32,6 +37,8 @@ export class ContributedMachines {
* running and stopped, whether new work is paused, and which starts are waiting. A change in any of them is
* sent at once; otherwise the heartbeat keeps the reading fresh. */
private readonly published = new Map<string, { at: number; signature: string }>()
+ /** Shell tool availability belongs to this machine, not a tab. Resource readings still refresh every heartbeat. */
+ private readonly toolCache = new Map<string, { value: boolean | null; detail?: string; until: number }>()
constructor(private readonly core: CoreApi, private readonly machineId: string,
private readonly deps: { backend: typeof backend; open: typeof openDaemon; capabilities: typeof readCapabilities;
start: typeof startAgent; now: () => number; prepare?: typeof prepareSwarmProject } =
@@ -67,6 +74,10 @@ export class ContributedMachines {
for (const id of ids) if (await this.core.agents.stopped?.(id)) stopped.push(id)
const running = ids.filter(id => !!this.core.agents.byAgent(id) && !stopped.includes(id))
const pending = list(row.starts)
+ const tools = [...new Set([...COMMON_TOOLS, ...pending.flatMap(start => {
+ const wanted = object(object(start.request).requirements).tools
+ return Array.isArray(wanted) ? wanted.filter((name): name is string => typeof name === 'string' && /^[A-Za-z0-9_.+-]{1,80}$/.test(name)) : []
+ })])].slice(0, 20)
const signature = JSON.stringify([row.reservations, pool.generation, [...running].sort(), [...stopped].sort(), row.paused === true || pool.paused === true,
pending.map(start => start.id).sort()])
const last = this.published.get(String(row.tabId))
@@ -75,11 +86,27 @@ export class ContributedMachines {
// The existing usage reader owns its provider cache. Raw responses and
// account credentials never leave this machine.
const readings = usage ??= await local.request('usage_read', {}, 20_000).catch(() => ({}))
- const observed = await this.deps.capabilities({ cwd: pool.cwd, tools: ['git'], devices: [] })
+ const sampledAt = this.deps.now()
+ const probeTools = tools.filter(name => (this.toolCache.get(name)?.until ?? 0) <= sampledAt)
+ const observed = await this.deps.capabilities({ cwd: pool.cwd, tools: probeTools, devices: [] })
if (this.stopped) return
const resources = object('resources' in observed ? observed.resources : null)
const gpus = list(resources.gpus).map(g => finite(g.utilizationPercent)).filter((n): n is number => n !== null)
- const observation = { generation: pool.generation, tools: 'tools' in observed ? observed.tools : {},
+ const measuredTools = object('tools' in observed ? observed.tools : null)
+ const measuredErrors = object('toolProbeErrors' in observed ? observed.toolProbeErrors : null)
+ for (const name of probeTools) {
+ const value = typeof measuredTools[name] === 'boolean' ? measuredTools[name] as boolean : null
+ const detail = value === null ? String(measuredErrors[name] ?? 'The tool probe did not complete.') : undefined
+ this.toolCache.set(name, { value, detail, until: sampledAt + (value === null ? UNVERIFIED_TOOL_TTL_MS : VERIFIED_TOOL_TTL_MS) })
+ }
+ const reportedTools: Record<string, boolean> = {}, toolProbeErrors: Record<string, string> = {}
+ for (const name of tools) {
+ const result = this.toolCache.get(name)
+ if (result?.value !== null && result?.value !== undefined) reportedTools[name] = result.value
+ else toolProbeErrors[name] = result?.detail ?? 'The tool probe did not complete.'
+ }
+ const observation = { generation: pool.generation, tools: reportedTools,
+ ...(Object.keys(toolProbeErrors).length ? { toolProbeErrors } : {}),
resources: { cpuPercent: finite(resources.cpuPercent), memoryUsedBytes: finite(resources.memoryUsedBytes), memoryTotalBytes: finite(resources.memoryTotalBytes),
diskFreeBytes: finite(resources.diskFreeBytes), diskTotalBytes: finite(resources.diskTotalBytes), projectDiskFreeBytes: finite(resources.projectDiskFreeBytes), projectDiskTotalBytes: finite(resources.projectDiskTotalBytes),
deviceType: typeof resources.deviceType === 'string' ? resources.deviceType : null,
@@ -92,7 +119,14 @@ export class ContributedMachines {
...(Array.isArray(row.reservations) ? { stoppedReservations: list(row.reservations).filter(r => stopped.includes(String(r.harnessId))
&& identifier(r.harnessId) && identifier(r.generation)).map(r => ({ harnessId: r.harnessId, generation: r.generation })) } : {}) }
try {
- await this.deps.backend(this.core, 'POST', `/api/super-tabs/${row.tabId}/machines/${this.machineId}`, observation)
+ const path = `/api/super-tabs/${row.tabId}/machines/${this.machineId}`
+ try { await this.deps.backend(this.core, 'POST', path, observation) }
+ catch (error) {
+ if (!(error instanceof SuperTabHostError) || error.status !== 400 || error.code !== 'VALIDATION_ERROR' || !('toolProbeErrors' in observation)) throw error
+ // Older backends reject the additive field. Their boolean-only view still safely refuses an unverified tool.
+ const { toolProbeErrors: _probeErrors, ...legacy } = observation
+ await this.deps.backend(this.core, 'POST', path, legacy)
+ }
this.published.set(String(row.tabId), { at: began, signature })
} catch (error) { this.failed = true; throw error }
}
diff --git a/cli/src/superTabs/routeCommand.spec.ts b/cli/src/superTabs/routeCommand.spec.ts
index 03355c979..787d17947 100644
--- a/cli/src/superTabs/routeCommand.spec.ts
+++ b/cli/src/superTabs/routeCommand.spec.ts
@@ -210,7 +210,7 @@ describe('routeTask', () => {
expect(reasonOf('mac').some(r => r.includes('This machine does not have the project folder. Add it to the Swarm so it can clone the project.'))).toBe(true)
expect(reasonOf('mac').some(r => /Project disk space/.test(r))).toBe(false)
// Its other reasons stay.
- expect(reasonOf('mac').some(r => /Required tool unavailable: git/.test(r))).toBe(true)
+ expect(reasonOf('mac').some(r => /Required tool not verified: git/.test(r))).toBe(true)
// The Lead's own machine keeps the plain reason: its folder was named by the Lead itself.
expect(reasonOf('remote-1').some(r => r.includes('Project disk space could not be verified.'))).toBe(true)
})
@@ -391,7 +391,7 @@ describe('routeTask', () => {
const result = await routeTask({ ...input, assessment: { ...input.assessment, needs } }, s, where as never, open)
expect(result).toMatchObject({ error: 'NO_CANDIDATE' })
const reasons = (result.excluded as Array<{ reason: string }>).map(e => e.reason)
- expect(reasons).toContain('Required tool unavailable: git. Required device unavailable: /dev/ttyUSB0.')
+ expect(reasons).toContain('Required tool missing: git. Required device unavailable: /dev/ttyUSB0.')
const d = open.mock.results[0] && await open.mock.results[0].value as ReturnType<typeof daemon>
expect(d.request).toHaveBeenCalledWith('super_tabs', expect.objectContaining({ action: 'capacity', cwd: '/repo', tools: ['git'], devices: ['/dev/ttyUSB0'] }), 20_000)
})
@@ -409,7 +409,7 @@ describe('routeTask', () => {
const requirements = { memoryBytes: 0, diskBytes: 0, maxCpuPercent: 100, tools: ['git'], devices: ['/dev/x'], quotaRequired: false }
const { reasons } = admission({ resources: {}, tools: {}, devices: {} }, { installed: true, signedIn: true }, quota, requirements)
expect(reasons).toEqual([
- 'Required tool unavailable: git.', 'Required device unavailable: /dev/x.',
+ 'Required tool not verified: git. Ask the machine owner to check it and refresh the machine reading.', 'Required device unavailable: /dev/x.',
'The subscription has an exhausted usage window. Wait for a fresh allowance reading or choose another account.'])
const unknown = admission({ resources: {}, tools: {}, devices: {} }, { installed: true, signedIn: true }, quotaOf({}, 'claude', false), { ...requirements, tools: [], devices: [] })
expect(unknown).toMatchObject({ allowed: true, reasons: [] })
@@ -653,7 +653,16 @@ describe('routing contributed capacity', () => {
const { answer } = await runPool(pool, true)
expect(answer).toMatchObject({ error: 'NO_CANDIDATE' })
expect(answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ title: expect.stringContaining('Multiply'),
- reason: expect.stringContaining('Required tool unavailable: git.') })]))
+ reason: expect.stringContaining('Required tool missing: git.') })]))
+ })
+ it('explains a contributed member tool probe that did not complete', async () => {
+ const pool = healthy()
+ delete (pool.observation.tools as Record<string, unknown>).git
+ Object.assign(pool.observation, { toolProbeErrors: { git: 'interactive shell timed out' } })
+ const { answer } = await runPool(pool, true)
+ expect(answer).toMatchObject({ error: 'NO_CANDIDATE' })
+ expect(answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ title: expect.stringContaining('Multiply'),
+ reason: expect.stringMatching(/Required tool not verified: git.*interactive shell timed out.*check it/) })]))
})
it('withholds a contributed member absent from an explicit running-terminal list', async () => {
const pool = healthy(); pool.observation.runningHarnessIds = []
@@ -709,4 +718,14 @@ describe('routing contributed capacity', () => {
it('refuses unobserved tools and devices, without probing arbitrary paths on a teammate computer', async () => {
expect((await runPool(healthy(), false, { tools: ['docker'], devices: ['/dev/fixture'] })).answer.ok).not.toBe(true)
})
+ it('distinguishes reported missing tools from unverified contributed probes in route reasons', async () => {
+ const missing = healthy(); missing.observation.tools.git = false
+ const absent = await runPool(missing)
+ expect(absent.answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ reason: expect.stringContaining('Required tool missing: git.') })]))
+
+ const unverified = healthy(); delete (unverified.observation.tools as Record<string, unknown>).git
+ Object.assign(unverified.observation, { toolProbeErrors: { git: 'interactive shell timed out' } })
+ const failed = await runPool(unverified)
+ expect(failed.answer.excluded).toEqual(expect.arrayContaining([expect.objectContaining({ reason: expect.stringMatching(/Required tool not verified: git.*interactive shell timed out/) })]))
+ })
})
diff --git a/cli/src/superTabs/routeCommand.ts b/cli/src/superTabs/routeCommand.ts
index f9584c089..6ef077bac 100644
--- a/cli/src/superTabs/routeCommand.ts
+++ b/cli/src/superTabs/routeCommand.ts
@@ -4,7 +4,7 @@
* and records the decision in the tab. */
import { randomUUID } from 'node:crypto'
import { openDaemon, type Daemon, type Where } from '../agentClient/daemon.js'
-import { startRequirements } from './capacity.js'
+import { requiredToolReason, startRequirements } from './capacity.js'
import { inspectCapacity, type ScopedRequest } from './start.js'
import { ENDED_WORK, object, within } from './wire.js'
import { choose, decide, parseAssessment, quotaOf2, rateFor, recordOf, repoMatch, type Assessment, type Commitment, type MemberFrame, type Policy, type Probe, type SwarmRule, type TaskFolder } from './route.js'
@@ -24,10 +24,10 @@ const UNKNOWN_QUOTA: Probe['quota'] = { known: false, leftPercent: null, resetsA
const UNKNOWN_LOAD: NonNullable<Probe['load']> = { agents: null, freeMemoryBytes: null, hasFolder: null }
const EXHAUSTED_WINDOW = 'The subscription has an exhausted usage window. Wait for a fresh allowance reading or choose another account.'
/** A running member keeps its engine and memory, but still needs a live terminal, account allowance and the task's tools. */
-function memberRefusals(running: boolean | null, quota: Probe['quota'], tools: string[], devices: string[], reportedTools: Json, reportedDevices: Json): string[] {
+function memberRefusals(running: boolean | null, quota: Probe['quota'], tools: string[], devices: string[], reportedTools: Json, reportedDevices: Json, toolProbeErrors: Json): string[] {
return [
...(running === false ? ['Its terminal is not running.'] : []),
- ...tools.filter(tool => reportedTools[tool] !== true).map(tool => `Required tool unavailable: ${tool}.`),
+ ...tools.map(tool => requiredToolReason(tool, reportedTools[tool], toolProbeErrors[tool])).filter((reason): reason is string => reason !== null),
...devices.filter(device => reportedDevices[device] !== true).map(device => `Required device unavailable: ${device}.`),
...(quota.leftPercent === 0 ? [EXHAUSTED_WINDOW] : []),
]
@@ -234,12 +234,12 @@ async function probeMachine(machineId: string, { name: machineName, status, acco
...(pool.availableSlots === 0 ? ['This machine has no available agent slots.'] : []),
...(!finite(resources.projectDiskFreeBytes) || resources.projectDiskFreeBytes < needs.diskBytes ? ['The project has insufficient measured free disk.'] : []),
...(!finite(resources.memoryTotalBytes) || !finite(resources.memoryUsedBytes) || resources.memoryTotalBytes - resources.memoryUsedBytes < needs.memoryBytes ? ['Insufficient measured free memory.'] : []),
- ...needs.tools.filter(t => tools[t] !== true).map(t => `Required tool not reported: ${t}.`),
+ ...needs.tools.map(t => requiredToolReason(t, tools[t], object(observed.toolProbeErrors)[t])).filter((reason): reason is string => reason !== null),
...(needs.devices.length ? ['Required devices have not been reported.'] : [])]
return [...members.map(m => {
const allowance = quota(String(object(m.configuration).engine))
const running = Array.isArray(observed.runningHarnessIds) ? observed.runningHarnessIds.includes(m.harnessId) : null
- return memberProbe(m, fresh ? memberRefusals(running, allowance, needs.tools, needs.devices, tools, object(observed.devices)) : [OFFLINE],
+ return memberProbe(m, fresh ? memberRefusals(running, allowance, needs.tools, needs.devices, tools, object(observed.devices), object(observed.toolProbeErrors)) : [OFFLINE],
allowance, null, fresh && running === null)
}),
...models.map(m => newProbe(m, [...newReasons, ...(quota(m.engine).leftPercent === 0 ? ['Subscription exhausted.'] : [])], quota(m.engine), fresh && quota(m.engine).known,
@@ -271,7 +271,7 @@ async function probeMachine(machineId: string, { name: machineName, status, acco
const allowance = quotaOf2(inspected.quota)
const capacity = object(inspected.capacity)
const refusals = memberRefusals(live.has(String(m.harnessId)), allowance, needs.tools, needs.devices,
- object(capacity.tools), object(capacity.devices))
+ object(capacity.tools), object(capacity.devices), object(capacity.toolProbeErrors))
const frame = frames.find(f => f.id === m.harnessId)
probes.push(memberProbe(m, refusals, allowance, frame ? frameOf(frame) : null))
}