Multiplayer agent harness for work.
sandbox: out-of-band computer status and restart via the execute tool (#469)
Regan Bell28eb08aaded3
diff --git a/package.json b/package.jsonindex 6b33db575..f7a0175d2 100644--- a/package.json+++ b/package.json@@ -98,13 +98,13 @@"@types/node": "^24.0.0","@types/pg": "^8.11.10","@types/tar-stream": "^3.1.4",+ "@yc-software/qm": "file:./cli","eslint": "^10.4.1","globals": "^17.6.0","knip": "^6.26.0","oxlint": "^1.74.0","prettier": "3.9.1","typescript": "^5.8.2",- "typescript-eslint": "^8.61.0",- "@yc-software/qm": "file:./cli"+ "typescript-eslint": "^8.61.0"}}diff --git a/src/harness/pi-tools.ts b/src/harness/pi-tools.tsindex 62beb4441..18a4699ab 100644--- a/src/harness/pi-tools.ts+++ b/src/harness/pi-tools.ts@@ -460,10 +460,23 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD"higher for builds/installs/tsc/test runs that legitimately take minutes, or lower for " +"commands you expect to be quick so a hang frees the machine fast. For work that " +`legitimately exceeds the ${execCeilingSec}s ceiling (long builds, installs, test suites, servers), use ` +- "the `background` tool to run it detached and poll for the result across turns.";+ "the `background` tool to run it detached and poll for the result across turns. " ++ "If commands hang or fail with transport errors that nothing you ran explains, the computer itself may be " ++ 'wedged — use `computer:"status"` to check it out-of-band and `computer:"restart"` to reboot it.';const executeBaseParams = {- command: Type.String({ description: "The shell command to run." }),+ command: Type.String({+ description: 'The shell command to run. With `computer`, pass "" — no command runs.',+ }),+ computer: Type.Optional(+ Type.String({+ enum: ["status", "restart"],+ description:+ "Manage the scoped computer itself instead of running a command — these act out-of-band, so they work even when the computer is unresponsive. " ++ '"status" reports the machine\'s health and whether its shell answers; "restart" reboots it (files survive; running processes don\'t, and an interrupted command may or may not have taken effect). ' ++ "Reach for these when commands hang or fail with transport errors that nothing you ran explains: check status first, restart only if the machine is up but its shell is not answering.",+ }),+ ),purpose: Type.String({description:"One short sentence on what this command accomplishes and why you're running it now — " +@@ -484,11 +497,45 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolDconst runExecute = async (callId: string,- params: { command: string; timeout_seconds?: number; purpose?: string },+ params: { command: string; computer?: string; timeout_seconds?: number; purpose?: string },route?: { scratch?: boolean; ownerAuth?: boolean; reachTarget?: string },) => {const tc = ref.current;if (!tc) return text("[error] no active tool context");+ if (params.computer) {+ await recordCall(callId, { tool: "execute", computer: params.computer });+ if (route?.scratch || route?.ownerAuth || route?.reachTarget !== undefined) {+ return recordResult(+ callId,+ { tool: "execute", computer: params.computer, invalid: "scoped_only" },+ text("[error] `computer` manages this conversation's scoped computer only — drop `scope`"),+ true,+ );+ }+ try {+ if (params.computer === "restart") {+ await tc.restartComputer();+ return recordResult(+ callId,+ { tool: "execute", computer: "restart", restarted: true },+ text("Computer restarting. Give it a moment to boot before running the next command."),+ );+ }+ const s = await tc.computerStatus();+ return recordResult(+ callId,+ { tool: "execute", computer: "status", ...s },+ text(`machine: ${s.machine}; shell: ${s.guestResponsive ? "answering" : "NOT answering"}`),+ );+ } catch (e) {+ return recordResult(+ callId,+ { tool: "execute", computer: params.computer, failed: true },+ text(`[error] ${errMessage(e)}`),+ true,+ );+ }+ }let scopeNote: { scope?: string } = {};if (route?.reachTarget !== undefined) {scopeNote = { scope: route.reachTarget };@@ -604,7 +651,14 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolDconst runScopedExecute = async (callId: string,- params: { command: string; timeout_seconds?: number; purpose?: string; scope?: string; durable?: boolean },+ params: {+ command: string;+ computer?: string;+ timeout_seconds?: number;+ purpose?: string;+ scope?: string;+ durable?: boolean;+ },) => {const scope = (params.scope ?? "scoped").trim();const keyword = scope.toLowerCase();diff --git a/src/sandbox/sandbox.ts b/src/sandbox/sandbox.tsindex 596dba04d..b010863c5 100644--- a/src/sandbox/sandbox.ts+++ b/src/sandbox/sandbox.ts@@ -123,6 +123,11 @@ export interface ProcessSession {status: ProcessState;}+export interface ComputerStatus {+ machine: string;+ guestResponsive: boolean;+}+export interface TeardownOptions {keepWarm?: boolean;destroy?: boolean;@@ -148,6 +153,8 @@ export interface Sandbox {writeStdin?(handle: SandboxHandle, processId: string, data: string): Promise<void>;signalProcess?(handle: SandboxHandle, processId: string, signal: string): Promise<void>;listProcesses?(handle: SandboxHandle): Promise<ProcessSession[]>;+ computerStatus?(scopeId: string): Promise<ComputerStatus>;+ restartComputer?(scopeId: string): Promise<void>;teardown(handle: SandboxHandle, opts?: TeardownOptions): Promise<void>;reapDeepIdle?(idleMs: number, devIdleMs?: number): Promise<{ reaped: number }>;}diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.tsindex dbfdbcdf8..9908b0399 100644--- a/src/sandbox/sprites-sandbox.ts+++ b/src/sandbox/sprites-sandbox.ts@@ -41,6 +41,9 @@ const RO_LAYERS_MANIFEST = ".ro-layers.manifest";const MISSING_RC = 44;const READ_CHUNK = 512 * 1024;const EXIT_GRACE_MS = 60_000;+const RESTART_TIMEOUT_MS = 60_000;+const CHECK_TIMEOUT_MS = 30_000;+const GUEST_PROBE_TIMEOUT_SEC = 15;const DEFAULT_SPRITES_BASE_URL = "https://api.sprites.dev";export interface SpritesClientLike {@@ -161,7 +164,10 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan}async function writeAbsBytes(name: string, absPath: string, data: Uint8Array): Promise<void> {- const script = `mkdir -p "$(dirname ${shq(absPath)})" && cat > ${shq(absPath)} && wc -c < ${shq(absPath)}`;+ const tmp = `${absPath}.part.${randomUUID()}`;+ const script =+ `mkdir -p "$(dirname ${shq(absPath)})" && cat > ${shq(tmp)} && ` ++ `mv -f ${shq(tmp)} ${shq(absPath)} && wc -c < ${shq(absPath)}`;const r = await postExec(name, ["sh", "-c", script], 120, data);const written = Number.parseInt(r.stdout.toString("utf8").trim(), 10);if (r.rc !== 0 || written !== data.length) {@@ -478,6 +484,39 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanbackupComputer: execBackup.backupComputer,+ async computerStatus(scopeId: string) {+ const name = spriteScopeName(prefix, scopeId);+ let machine: string;+ try {+ const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}/check`, {+ headers: { authorization: `Bearer ${opts.token ?? ""}` },+ signal: AbortSignal.timeout(CHECK_TIMEOUT_MS),+ });+ const body = res.ok ? ((await res.json().catch(() => null)) as { status?: string } | null) : null;+ machine = body?.status ?? `check failed: http ${res.status}`;+ } catch (e) {+ machine = `check failed: ${errMessage(e)}`;+ }+ let guestResponsive = false;+ try {+ guestResponsive = (await execRaw(name, "true", GUEST_PROBE_TIMEOUT_SEC)).code === 0;+ } catch (e) {+ void e;+ }+ return { machine, guestResponsive };+ },++ async restartComputer(scopeId: string): Promise<void> {+ const name = spriteScopeName(prefix, scopeId);+ ensured.delete(name);+ const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}/restart`, {+ method: "POST",+ headers: { authorization: `Bearer ${opts.token ?? ""}` },+ signal: AbortSignal.timeout(RESTART_TIMEOUT_MS),+ });+ if (!res.ok) throw new Error(`sprites restart ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ },+async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {if (handle.scratch) {const key = scratchKeyByName.get(handle.id);diff --git a/src/tools/primitives.ts b/src/tools/primitives.tsindex b7aa0bca4..ce18b93cb 100644--- a/src/tools/primitives.ts+++ b/src/tools/primitives.ts@@ -1,7 +1,7 @@import { join } from "node:path";import { interpolateSplitEnv } from "../deployment/deployment-layer.ts";import { BASE_RESIDENT_AUTH_PATHS, residentAuthPaths, type CredentialPathSpec } from "../credentials/resident-paths.ts";-import type { ExecResult, Sandbox, SandboxHandle } from "../sandbox/sandbox.ts";+import type { ComputerStatus, ExecResult, Sandbox, SandboxHandle } from "../sandbox/sandbox.ts";import { CapabilityUnsupportedError, hasParentPathSegment, supportsAgentComputerBackup } from "../sandbox/sandbox.ts";import type {CommandPolicy,@@ -166,6 +166,8 @@ export interface ToolContext extends SurfaceToolDeps {signal?: AbortSignal;},): Promise<ExecResult & { reached?: ReachedProvenance }>;+ computerStatus(): Promise<ComputerStatus>;+ restartComputer(): Promise<void>;read(path: string): Promise<ReadResult>;write(path: string, data?: string, share?: ShareDirective[]): Promise<WriteResult>;publish(input: PublishInput): Promise<PublishResult>;@@ -476,6 +478,20 @@ export function createToolContext(deps: ToolContextDeps): ToolContext {return {...(deps.credentialExecServices ? { credentialExecServices: deps.credentialExecServices } : {}),...(deps.credentialExec ? { credentialExec: deps.credentialExec } : {}),+ async computerStatus(): Promise<ComputerStatus> {+ if (!deps.sandbox.computerStatus) {+ throw new CapabilityUnsupportedError(deps.sandbox.profile.backend, "reporting computer status");+ }+ if (!writableScopeId) throw new Error("this turn has no scoped computer");+ return deps.sandbox.computerStatus(writableScopeId);+ },+ async restartComputer(): Promise<void> {+ if (!deps.sandbox.restartComputer) {+ throw new CapabilityUnsupportedError(deps.sandbox.profile.backend, "restarting the computer");+ }+ if (!writableScopeId) throw new Error("this turn has no scoped computer to restart");+ await deps.sandbox.restartComputer(writableScopeId);+ },async execute(command: string,execOpts?: {diff --git a/test/execute-scratch.test.ts b/test/execute-scratch.test.tsindex d34f1beba..d227a06ae 100644--- a/test/execute-scratch.test.ts+++ b/test/execute-scratch.test.ts@@ -84,7 +84,7 @@ const schemaRequired = (tool: ReturnType<typeof createPiTools>[number]): string[test("flag OFF: the execute surface is exactly the legacy one (no scope/durable, scoped box)", async () => {const { tc, seen } = sinkToolContext();const [execute] = createPiTools({ current: tc });- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds"]);+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds"]);assert.deepEqual(schemaRequired(execute!), ["command", "purpose"]);await call(execute, { command: "echo hi" });assert.deepEqual(seen, [{ command: "echo hi", opts: undefined }]);@@ -94,7 +94,7 @@ test("flag ON: scope defaults to the durable scoped box; scratch is an explicitconst { tc, seen } = sinkToolContext();const ref: ToolContextRef = { current: tc };const [execute] = createPiTools(ref, { scratchExec: true });- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds", "scope", "durable"]);+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds", "scope", "durable"]);await call(execute, { command: "echo hi" });assert.deepEqual(diff --git a/test/pi-tools.test.ts b/test/pi-tools.test.tsindex a19a5cd46..762e7dc93 100644--- a/test/pi-tools.test.ts+++ b/test/pi-tools.test.ts@@ -11,6 +11,10 @@ function fakeToolContext(sink?: { lastExecOpts?: Parameters<ToolContext["executeif (sink) sink.lastExecOpts = opts;return { stdout: `ran ${command}`, stderr: "", code: 0, timedOut: false };},+ async restartComputer() {},+ async computerStatus() {+ return { machine: "healthy", guestResponsive: true };+ },async read(path) {return path === "a.txt"? { content: "data", sourceScopeId: "personal:U1" }@@ -335,6 +339,41 @@ test("each pi tool emits a tool_call then a tool_result", async () => {);});+test("execute's computer param manages the box out-of-band instead of running a command", async () => {+ const restarted: number[] = [];+ const tc = {+ ...fakeToolContext(),+ restartComputer: async () => {+ restarted.push(1);+ },+ computerStatus: async () => ({ machine: "healthy", guestResponsive: false }),+ };+ const ref: ToolContextRef = { current: tc, emit: () => {}, scopeLabel: "personal:U1" };+ const [execute] = createPiTools(ref);++ const status = (await call(execute, { command: "", computer: "status", purpose: "p" })) as {+ content: Array<{ text?: string }>;+ };+ assert.match(status.content[0]!.text!, /machine: healthy; shell: NOT answering/);++ const restart = (await call(execute, { command: "", computer: "restart", purpose: "p" })) as {+ content: Array<{ text?: string }>;+ };+ assert.equal(restarted.length, 1);+ assert.match(restart.content[0]!.text!, /restarting/i);++ ref.current = {+ ...tc,+ restartComputer: async () => {+ throw new Error("this computer's substrate (local) does not support restarting the computer");+ },+ };+ const err = (await call(execute, { command: "", computer: "restart", purpose: "p" })) as {+ content: Array<{ text?: string }>;+ };+ assert.match(err.content[0]!.text!, /does not support restarting/);+});+test("a giant tool result is capped for the model, keeping the tail and matching the persisted replay record", async () => {const emitted: Emitted[] = [];const tc = {diff --git a/test/scope-reach.test.ts b/test/scope-reach.test.tsindex 12b0f54ff..a75cbde21 100644--- a/test/scope-reach.test.ts+++ b/test/scope-reach.test.ts@@ -242,7 +242,7 @@ test("reachExec OFF: the execute scope never accepts a room", () => {test("reachExec ON (no scratch): scope is a free string; a room routes to reachTarget; default is scoped", async () => {const { tc, seen } = sinkToolContext();const [execute] = createPiTools({ current: tc }, { reachExec: true });- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds", "scope"]);+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds", "scope"]);await call(execute, { command: "cat x", scope: "#project-alpha" });assert.deepEqual(seen.at(-1)!.opts, { reachTarget: "#project-alpha" });await call(execute, { command: "echo hi" });@@ -256,7 +256,7 @@ test("reachExec ON (no scratch): scope is a free string; a room routes to reachTtest("reachExec ON + scratchExec ON: scope accepts scoped, scratch, AND a room", async () => {const { tc, seen } = sinkToolContext();const [execute] = createPiTools({ current: tc }, { reachExec: true, scratchExec: true });- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds", "scope", "durable"]);+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds", "scope", "durable"]);await call(execute, { command: "x", scope: "scratch" });assert.deepEqual(seen.at(-1)!.opts, { scratch: true });await call(execute, { command: "x", scope: "#ops" });diff --git a/test/sprites-sandbox.test.ts b/test/sprites-sandbox.test.tsindex 77612b7bc..a617ce0eb 100644--- a/test/sprites-sandbox.test.ts+++ b/test/sprites-sandbox.test.ts@@ -256,3 +256,53 @@ test("stageIn pulls a blob into the guest atomically (temp then mv)", async () =assert.match(script, /mv -f /, "and only then moves it into place");assert.match(script, /curl -fsS/, "-f so an HTTP error fails loudly instead of writing the error body");});++test("restartComputer reboots the scope's sprite and heals a wedged exec channel", async () => {+ const h = await sandbox.provision(layers);+ fake.fail502(h.id);+ await assert.rejects(sandbox.run(h, "echo back"), /http 502/);++ await sandbox.restartComputer!(scope);+ assert.deepEqual(fake.restarts(), [h.id]);++ const after = await sandbox.run(h, "echo back");+ assert.equal(after.code, 0);+ assert.equal(after.stdout.trim(), "back");+});++test("computerStatus reports a healthy machine whose shell has stopped answering", async () => {+ const h = await sandbox.provision(layers);+ assert.deepEqual(await sandbox.computerStatus!(scope), { machine: "healthy", guestResponsive: true });++ fake.fail502(h.id);+ assert.deepEqual(await sandbox.computerStatus!(scope), { machine: "healthy", guestResponsive: false });+});++test("restartComputer surfaces a refused restart instead of swallowing it", async () => {+ const h = await sandbox.provision(layers);+ fake.refuseRestart(h.id);+ await assert.rejects(sandbox.restartComputer!(scope), /sprites restart .*: http 502/);+ assert.deepEqual(fake.restarts(), []);+});++test("a command that ran before the response was lost is never re-executed", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.run(h, ": > /home/sprite/workspace/ledger");+ fake.stallAfterRun(h.id);++ await assert.rejects(sandbox.run(h, "echo entry >> /home/sprite/workspace/ledger"));++ const ledger = await sandbox.readFile(h, "ledger");+ assert.equal(ledger, "entry\n", "the side effect must have happened exactly once");+});++test("an inline write lands atomically, so a reboot mid-write cannot truncate the target", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "cfg.txt", "value\n");+ assert.equal(await sandbox.readFile(h, "cfg.txt"), "value\n");++ const write = fake.execScripts().find((s) => s.includes("cfg.txt") && s.includes("cat >"));+ assert.ok(write, "expected an inline write script");+ assert.match(write!, /\.part\./, "the payload must land on a temp path");+ assert.match(write!, /mv -f/, "and be renamed over the target, never streamed into it");+});diff --git a/test/support/fake-sprites.ts b/test/support/fake-sprites.tsindex 64d44e81f..256f30c26 100644--- a/test/support/fake-sprites.ts+++ b/test/support/fake-sprites.ts@@ -25,6 +25,10 @@ export interface FakeSprites {names(): string[];policy(name: string): NetworkRule[] | null;execScripts(): string[];+ stallAfterRun(name: string): void;+ fail502(name: string): void;+ refuseRestart(name: string): void;+ restarts(): string[];reset(): void;cleanup(): void;}@@ -38,6 +42,10 @@ export function installFakeSprites(): FakeSprites {const policies = new Map<string, NetworkRule[]>();const execScripts: string[] = [];const calls: SpritesCall[] = [];+ const gateway502 = new Set<string>();+ const stallAfterRun = new Set<string>();+ const refusedRestart = new Set<string>();+ const restarts: string[] = [];const ensureDir = (name: string): string => {let s = sprites.get(name);@@ -97,6 +105,21 @@ export function installFakeSprites(): FakeSprites {const url = new URL(typeof input === "string" ? input : input.toString());const method = init?.method ?? "GET";calls.push({ method, path: url.pathname });+ const health = /^\/v1\/sprites\/([^/]+)\/check$/.exec(url.pathname);+ if (health) {+ const name = decodeURIComponent(health[1]!);+ if (!sprites.has(name)) return new Response("sprite not found", { status: 404 });+ return Response.json({ sprite_name: name, status: "healthy" });+ }+ const boot = /^\/v1\/sprites\/([^/]+)\/restart$/.exec(url.pathname);+ if (boot && method === "POST") {+ const name = decodeURIComponent(boot[1]!);+ if (!sprites.has(name)) return new Response("sprite not found", { status: 404 });+ if (refusedRestart.has(name)) return new Response('{"error":"upstream restart failed"}', { status: 502 });+ restarts.push(name);+ gateway502.delete(name);+ return Response.json({ sprite_name: name });+ }const sub = /\/v1\/sprites\/([^/]+)\/(exec|policy\/network)$/.exec(url.pathname);if (sub) {const name = decodeURIComponent(sub[1]!);@@ -105,6 +128,17 @@ export function installFakeSprites(): FakeSprites {const script = argv[argv.length - 1] ?? "";calls[calls.length - 1]!.script = script;const stdin = url.searchParams.get("stdin") === "true" ? toBuf(init?.body) : undefined;+ const stall = (): never => {+ throw Object.assign(new Error("The operation was aborted due to timeout"), { name: "TimeoutError" });+ };+ if (stallAfterRun.has(name)) {+ stallAfterRun.delete(name);+ runExec(name, script, stdin);+ stall();+ }+ if (gateway502.has(name)) {+ return new Response('{"error":"bad gateway"}', { status: 502 });+ }return new Response(runExec(name, script, stdin), { status: 200 });}if (method === "GET") return Response.json({ rules: policies.get(name) ?? [] });@@ -155,10 +189,24 @@ export function installFakeSprites(): FakeSprites {names: () => [...sprites.keys()],policy: (name) => policies.get(name) ?? null,execScripts: () => [...execScripts],+ stallAfterRun: (name) => {+ stallAfterRun.add(name);+ },+ fail502: (name) => {+ gateway502.add(name);+ },+ refuseRestart: (name) => {+ refusedRestart.add(name);+ },+ restarts: () => [...restarts],reset: () => {for (const name of Array.from(sprites.keys())) deleteSprite(name);execScripts.length = 0;calls.length = 0;+ stallAfterRun.clear();+ gateway502.clear();+ refusedRestart.clear();+ restarts.length = 0;},cleanup: () => rmSync(root, { recursive: true, force: true }),};