Multiplayer agent harness for work.
Probe the exec path in sandbox status for every sandbox (#1684)
Joshua France3ca5bfdf133b
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.tsindex e9a387791..76f1372e3 100644--- a/src/harness/agent-tools.ts+++ b/src/harness/agent-tools.ts@@ -592,7 +592,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):"the `background` tool to run it detached and poll for the result across turns. " +"Always start servers with the background tool, not shell ampersand: inherited output streams can keep execute waiting even after its shell exits. " +"If commands hang or fail with transport errors that nothing you ran explains, the computer itself may be " +- "wedged — use sandbox action=status to inspect it out-of-band and action=restart to recover it.";+ "wedged — use sandbox action=status to inspect it out-of-band and action=restart to recover it. If a restart or two does not bring the shell back, stop restarting: create a fresh sandbox, set it as the default, and retry there, recovering work from git or Files.";const executeBaseParams = {command: Type.String({ description: "The shell command to run." }),@@ -982,7 +982,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):}const verdictLine =verdict === "wedged"- ? " — WEDGED: a machine exists but its shell is not answering; the platform's health reporting goes stale in exactly this state, so trust the shell probe over any healthy/running claim and restart the computer"+ ? " — WEDGED: a machine exists but its shell is not answering; the platform's health reporting goes stale in exactly this state, so trust the shell probe over any healthy/running claim and restart the computer; if it is still wedged after a restart or two, create a fresh sandbox, set it as the default, and retry there": "";return recordResult(callId,diff --git a/src/tools/primitives.ts b/src/tools/primitives.tsindex 524f4469f..64a233808 100644--- a/src/tools/primitives.ts+++ b/src/tools/primitives.ts@@ -1,5 +1,5 @@import { MaskedExecutionError, executionSecretEnv, createExactSecretValueMasker } from "../security/secret-masking.ts";-import { withAbort } from "../util/async.ts";+import { withAbort, withTimeout } from "../util/async.ts";import type { RuntimeRequest, RuntimeResult } from "../harness/runtime-types.ts";import { readContextFile } from "../resolution/context-files.ts";import { contextMemory, type TurnContext } from "../resolution/turn-context.ts";@@ -666,25 +666,31 @@ export function createToolContext(deps: ToolContextDeps): ToolContext {}: {}),async computerStatus(sandboxId?: string): Promise<ComputerStatus> {+ const probe = async (status: ComputerStatus, provision: () => Promise<SandboxHandle>) => {+ if (!status.provisioned || status.lifecycleState === "paused") return status;+ try {+ const code = await withTimeout(+ async () =>+ (await deps.sandbox.run(await provision(), "true", { timeoutMs: COMMAND_PATH_PROBE_TIMEOUT_MS })).code,+ COMMAND_PATH_PROBE_TIMEOUT_MS * 2,+ "command probe",+ );+ return { ...status, guestResponsive: code === 0 };+ } catch (e) {+ return { ...status, guestResponsive: false, probeError: errMessage(e) };+ }+ };if (sandboxId) {const resources = deps.sandboxResources;- if (!resources) throw new Error("sandbox inventory unavailable");- await accessSandbox(sandboxId);- return resources.status(deps.createdBy, sandboxId);+ if (!resources || !deps.provisionResource) throw new Error("sandbox inventory unavailable");+ const access = await accessSandbox(sandboxId);+ return probe(await resources.status(deps.createdBy, sandboxId), () => deps.provisionResource!(access));}if (!deps.sandbox.computerStatus) {throw new CapabilityUnsupportedError(deps.sandbox.profile.backend, "reporting computer status");}if (!writableScopeId) throw new Error("this turn has no scoped computer");- const status = await deps.sandbox.computerStatus(writableScopeId);- if (!status.provisioned || ("lifecycleState" in status && status.lifecycleState === "paused")) return status;- try {- const handle = await deps.provision();- const probe = await deps.sandbox.run(handle, "true", { timeoutMs: COMMAND_PATH_PROBE_TIMEOUT_MS });- return { ...status, guestResponsive: probe.code === 0 };- } catch (e) {- return { ...status, guestResponsive: false, probeError: errMessage(e) };- }+ return probe(await deps.sandbox.computerStatus(writableScopeId), deps.provision);},async restartComputer(sandboxId?: string): Promise<void> {if (sandboxId) {@@ -1465,7 +1471,7 @@ function tryDecodeUtf8(bytes: Uint8Array): string | null {}}-const COMMAND_PATH_PROBE_TIMEOUT_MS = 15_000;+const COMMAND_PATH_PROBE_TIMEOUT_MS = 10_000;async function collectTree(sandbox: Sandbox,diff --git a/test/computer-status-probe.test.ts b/test/computer-status-probe.test.tsindex 38612ab87..dae74d15b 100644--- a/test/computer-status-probe.test.ts+++ b/test/computer-status-probe.test.ts@@ -72,3 +72,45 @@ test("computerStatus never provisions when the backend reports no machine", asynassert.deepEqual(await ctx.computerStatus(), none);assert.equal(provisions, 0, "a status check must not cold-start a box");});++test("a probe that hangs on the real exec path reports NOT answering within the probe bound", async () => {+ const ctx = ctxFor(+ {+ computerStatus: async () => healthy,+ run: () => new Promise(() => {}),+ },+ async () => handle,+ );+ const started = Date.now();+ const status = await ctx.computerStatus();+ assert.equal(status.guestResponsive, false);+ assert.match(status.probeError ?? "", /timed out/);+ assert.ok(Date.now() - started < 30_000);+});++test("status for an explicit sandbox probes its exec path instead of trusting the provider", async () => {+ const scope = scopeId("channel", "C1");+ const layers: WorkspaceLayer[] = [{ scopeId: scope, mountPath: "", mode: "rw" }];+ const access = { resource: { id: "sb1", ownerScopeId: scope }, crossScope: false } as never;+ const ctx = createToolContext({+ sandbox: {+ profile: { backend: "sprites" },+ run: () => new Promise(() => {}),+ } as unknown as Sandbox,+ provision: async () => handle,+ provisionResource: async () => handle,+ accessSandboxResource: async () => access,+ sandboxResources: { status: async () => healthy } as never,+ layers,+ commandPolicy: () => ({ mode: "denylist", rules: [] }),+ authorizeCommand: () => false,+ grantedHandles: [],+ workspace: {} as never,+ deploy: {} as never,+ acl: {} as never,+ createdBy: "U1",+ });+ const status = await ctx.computerStatus("sb1");+ assert.equal(status.guestResponsive, false);+ assert.match(status.probeError ?? "", /timed out/);+});