Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Probe the exec path in sandbox status for every sandbox (#1684)

Joshua France3ca5bfdf133b
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.ts
index e9a387791..76f1372e3 100644
--- a/src/harness/agent-tools.ts
+++ b/src/harness/agent-tools.ts
@@ -592,7 +592,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
"the `background` tool to run it detached and poll for the result across turns. " +
"Always start servers with the background tool, not shell ampersand: inherited output streams can keep execute waiting even after its shell exits. " +
"If commands hang or fail with transport errors that nothing you ran explains, the computer itself may be " +
- "wedged — use sandbox action=status to inspect it out-of-band and action=restart to recover it.";
+ "wedged — use sandbox action=status to inspect it out-of-band and action=restart to recover it. If a restart or two does not bring the shell back, stop restarting: create a fresh sandbox, set it as the default, and retry there, recovering work from git or Files.";
const executeBaseParams = {
command: Type.String({ description: "The shell command to run." }),
@@ -982,7 +982,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
}
const verdictLine =
verdict === "wedged"
- ? " — WEDGED: a machine exists but its shell is not answering; the platform's health reporting goes stale in exactly this state, so trust the shell probe over any healthy/running claim and restart the computer"
+ ? " — WEDGED: a machine exists but its shell is not answering; the platform's health reporting goes stale in exactly this state, so trust the shell probe over any healthy/running claim and restart the computer; if it is still wedged after a restart or two, create a fresh sandbox, set it as the default, and retry there"
: "";
return recordResult(
callId,
diff --git a/src/tools/primitives.ts b/src/tools/primitives.ts
index 524f4469f..64a233808 100644
--- a/src/tools/primitives.ts
+++ b/src/tools/primitives.ts
@@ -1,5 +1,5 @@
import { MaskedExecutionError, executionSecretEnv, createExactSecretValueMasker } from "../security/secret-masking.ts";
-import { withAbort } from "../util/async.ts";
+import { withAbort, withTimeout } from "../util/async.ts";
import type { RuntimeRequest, RuntimeResult } from "../harness/runtime-types.ts";
import { readContextFile } from "../resolution/context-files.ts";
import { contextMemory, type TurnContext } from "../resolution/turn-context.ts";
@@ -666,25 +666,31 @@ export function createToolContext(deps: ToolContextDeps): ToolContext {
}
: {}),
async computerStatus(sandboxId?: string): Promise<ComputerStatus> {
+ const probe = async (status: ComputerStatus, provision: () => Promise<SandboxHandle>) => {
+ if (!status.provisioned || status.lifecycleState === "paused") return status;
+ try {
+ const code = await withTimeout(
+ async () =>
+ (await deps.sandbox.run(await provision(), "true", { timeoutMs: COMMAND_PATH_PROBE_TIMEOUT_MS })).code,
+ COMMAND_PATH_PROBE_TIMEOUT_MS * 2,
+ "command probe",
+ );
+ return { ...status, guestResponsive: code === 0 };
+ } catch (e) {
+ return { ...status, guestResponsive: false, probeError: errMessage(e) };
+ }
+ };
if (sandboxId) {
const resources = deps.sandboxResources;
- if (!resources) throw new Error("sandbox inventory unavailable");
- await accessSandbox(sandboxId);
- return resources.status(deps.createdBy, sandboxId);
+ if (!resources || !deps.provisionResource) throw new Error("sandbox inventory unavailable");
+ const access = await accessSandbox(sandboxId);
+ return probe(await resources.status(deps.createdBy, sandboxId), () => deps.provisionResource!(access));
}
if (!deps.sandbox.computerStatus) {
throw new CapabilityUnsupportedError(deps.sandbox.profile.backend, "reporting computer status");
}
if (!writableScopeId) throw new Error("this turn has no scoped computer");
- const status = await deps.sandbox.computerStatus(writableScopeId);
- if (!status.provisioned || ("lifecycleState" in status && status.lifecycleState === "paused")) return status;
- try {
- const handle = await deps.provision();
- const probe = await deps.sandbox.run(handle, "true", { timeoutMs: COMMAND_PATH_PROBE_TIMEOUT_MS });
- return { ...status, guestResponsive: probe.code === 0 };
- } catch (e) {
- return { ...status, guestResponsive: false, probeError: errMessage(e) };
- }
+ return probe(await deps.sandbox.computerStatus(writableScopeId), deps.provision);
},
async restartComputer(sandboxId?: string): Promise<void> {
if (sandboxId) {
@@ -1465,7 +1471,7 @@ function tryDecodeUtf8(bytes: Uint8Array): string | null {
}
}
-const COMMAND_PATH_PROBE_TIMEOUT_MS = 15_000;
+const COMMAND_PATH_PROBE_TIMEOUT_MS = 10_000;
async function collectTree(
sandbox: Sandbox,
diff --git a/test/computer-status-probe.test.ts b/test/computer-status-probe.test.ts
index 38612ab87..dae74d15b 100644
--- a/test/computer-status-probe.test.ts
+++ b/test/computer-status-probe.test.ts
@@ -72,3 +72,45 @@ test("computerStatus never provisions when the backend reports no machine", asyn
assert.deepEqual(await ctx.computerStatus(), none);
assert.equal(provisions, 0, "a status check must not cold-start a box");
});
+
+test("a probe that hangs on the real exec path reports NOT answering within the probe bound", async () => {
+ const ctx = ctxFor(
+ {
+ computerStatus: async () => healthy,
+ run: () => new Promise(() => {}),
+ },
+ async () => handle,
+ );
+ const started = Date.now();
+ const status = await ctx.computerStatus();
+ assert.equal(status.guestResponsive, false);
+ assert.match(status.probeError ?? "", /timed out/);
+ assert.ok(Date.now() - started < 30_000);
+});
+
+test("status for an explicit sandbox probes its exec path instead of trusting the provider", async () => {
+ const scope = scopeId("channel", "C1");
+ const layers: WorkspaceLayer[] = [{ scopeId: scope, mountPath: "", mode: "rw" }];
+ const access = { resource: { id: "sb1", ownerScopeId: scope }, crossScope: false } as never;
+ const ctx = createToolContext({
+ sandbox: {
+ profile: { backend: "sprites" },
+ run: () => new Promise(() => {}),
+ } as unknown as Sandbox,
+ provision: async () => handle,
+ provisionResource: async () => handle,
+ accessSandboxResource: async () => access,
+ sandboxResources: { status: async () => healthy } as never,
+ layers,
+ commandPolicy: () => ({ mode: "denylist", rules: [] }),
+ authorizeCommand: () => false,
+ grantedHandles: [],
+ workspace: {} as never,
+ deploy: {} as never,
+ acl: {} as never,
+ createdBy: "U1",
+ });
+ const status = await ctx.computerStatus("sb1");
+ assert.equal(status.guestResponsive, false);
+ assert.match(status.probeError ?? "", /timed out/);
+});