Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

sandbox: out-of-band computer status and restart via the execute tool (#469)

Regan Bell28eb08aaded3
diff --git a/package.json b/package.json
index 6b33db575..f7a0175d2 100644
--- a/package.json
+++ b/package.json
@@ -98,13 +98,13 @@
"@types/node": "^24.0.0",
"@types/pg": "^8.11.10",
"@types/tar-stream": "^3.1.4",
+ "@yc-software/qm": "file:./cli",
"eslint": "^10.4.1",
"globals": "^17.6.0",
"knip": "^6.26.0",
"oxlint": "^1.74.0",
"prettier": "3.9.1",
"typescript": "^5.8.2",
- "typescript-eslint": "^8.61.0",
- "@yc-software/qm": "file:./cli"
+ "typescript-eslint": "^8.61.0"
}
}
diff --git a/src/harness/pi-tools.ts b/src/harness/pi-tools.ts
index 62beb4441..18a4699ab 100644
--- a/src/harness/pi-tools.ts
+++ b/src/harness/pi-tools.ts
@@ -460,10 +460,23 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD
"higher for builds/installs/tsc/test runs that legitimately take minutes, or lower for " +
"commands you expect to be quick so a hang frees the machine fast. For work that " +
`legitimately exceeds the ${execCeilingSec}s ceiling (long builds, installs, test suites, servers), use ` +
- "the `background` tool to run it detached and poll for the result across turns.";
+ "the `background` tool to run it detached and poll for the result across turns. " +
+ "If commands hang or fail with transport errors that nothing you ran explains, the computer itself may be " +
+ 'wedged — use `computer:"status"` to check it out-of-band and `computer:"restart"` to reboot it.';
const executeBaseParams = {
- command: Type.String({ description: "The shell command to run." }),
+ command: Type.String({
+ description: 'The shell command to run. With `computer`, pass "" — no command runs.',
+ }),
+ computer: Type.Optional(
+ Type.String({
+ enum: ["status", "restart"],
+ description:
+ "Manage the scoped computer itself instead of running a command — these act out-of-band, so they work even when the computer is unresponsive. " +
+ '"status" reports the machine\'s health and whether its shell answers; "restart" reboots it (files survive; running processes don\'t, and an interrupted command may or may not have taken effect). ' +
+ "Reach for these when commands hang or fail with transport errors that nothing you ran explains: check status first, restart only if the machine is up but its shell is not answering.",
+ }),
+ ),
purpose: Type.String({
description:
"One short sentence on what this command accomplishes and why you're running it now — " +
@@ -484,11 +497,45 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD
const runExecute = async (
callId: string,
- params: { command: string; timeout_seconds?: number; purpose?: string },
+ params: { command: string; computer?: string; timeout_seconds?: number; purpose?: string },
route?: { scratch?: boolean; ownerAuth?: boolean; reachTarget?: string },
) => {
const tc = ref.current;
if (!tc) return text("[error] no active tool context");
+ if (params.computer) {
+ await recordCall(callId, { tool: "execute", computer: params.computer });
+ if (route?.scratch || route?.ownerAuth || route?.reachTarget !== undefined) {
+ return recordResult(
+ callId,
+ { tool: "execute", computer: params.computer, invalid: "scoped_only" },
+ text("[error] `computer` manages this conversation's scoped computer only — drop `scope`"),
+ true,
+ );
+ }
+ try {
+ if (params.computer === "restart") {
+ await tc.restartComputer();
+ return recordResult(
+ callId,
+ { tool: "execute", computer: "restart", restarted: true },
+ text("Computer restarting. Give it a moment to boot before running the next command."),
+ );
+ }
+ const s = await tc.computerStatus();
+ return recordResult(
+ callId,
+ { tool: "execute", computer: "status", ...s },
+ text(`machine: ${s.machine}; shell: ${s.guestResponsive ? "answering" : "NOT answering"}`),
+ );
+ } catch (e) {
+ return recordResult(
+ callId,
+ { tool: "execute", computer: params.computer, failed: true },
+ text(`[error] ${errMessage(e)}`),
+ true,
+ );
+ }
+ }
let scopeNote: { scope?: string } = {};
if (route?.reachTarget !== undefined) {
scopeNote = { scope: route.reachTarget };
@@ -604,7 +651,14 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD
const runScopedExecute = async (
callId: string,
- params: { command: string; timeout_seconds?: number; purpose?: string; scope?: string; durable?: boolean },
+ params: {
+ command: string;
+ computer?: string;
+ timeout_seconds?: number;
+ purpose?: string;
+ scope?: string;
+ durable?: boolean;
+ },
) => {
const scope = (params.scope ?? "scoped").trim();
const keyword = scope.toLowerCase();
diff --git a/src/sandbox/sandbox.ts b/src/sandbox/sandbox.ts
index 596dba04d..b010863c5 100644
--- a/src/sandbox/sandbox.ts
+++ b/src/sandbox/sandbox.ts
@@ -123,6 +123,11 @@ export interface ProcessSession {
status: ProcessState;
}
+export interface ComputerStatus {
+ machine: string;
+ guestResponsive: boolean;
+}
+
export interface TeardownOptions {
keepWarm?: boolean;
destroy?: boolean;
@@ -148,6 +153,8 @@ export interface Sandbox {
writeStdin?(handle: SandboxHandle, processId: string, data: string): Promise<void>;
signalProcess?(handle: SandboxHandle, processId: string, signal: string): Promise<void>;
listProcesses?(handle: SandboxHandle): Promise<ProcessSession[]>;
+ computerStatus?(scopeId: string): Promise<ComputerStatus>;
+ restartComputer?(scopeId: string): Promise<void>;
teardown(handle: SandboxHandle, opts?: TeardownOptions): Promise<void>;
reapDeepIdle?(idleMs: number, devIdleMs?: number): Promise<{ reaped: number }>;
}
diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.ts
index dbfdbcdf8..9908b0399 100644
--- a/src/sandbox/sprites-sandbox.ts
+++ b/src/sandbox/sprites-sandbox.ts
@@ -41,6 +41,9 @@ const RO_LAYERS_MANIFEST = ".ro-layers.manifest";
const MISSING_RC = 44;
const READ_CHUNK = 512 * 1024;
const EXIT_GRACE_MS = 60_000;
+const RESTART_TIMEOUT_MS = 60_000;
+const CHECK_TIMEOUT_MS = 30_000;
+const GUEST_PROBE_TIMEOUT_SEC = 15;
const DEFAULT_SPRITES_BASE_URL = "https://api.sprites.dev";
export interface SpritesClientLike {
@@ -161,7 +164,10 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
}
async function writeAbsBytes(name: string, absPath: string, data: Uint8Array): Promise<void> {
- const script = `mkdir -p "$(dirname ${shq(absPath)})" && cat > ${shq(absPath)} && wc -c < ${shq(absPath)}`;
+ const tmp = `${absPath}.part.${randomUUID()}`;
+ const script =
+ `mkdir -p "$(dirname ${shq(absPath)})" && cat > ${shq(tmp)} && ` +
+ `mv -f ${shq(tmp)} ${shq(absPath)} && wc -c < ${shq(absPath)}`;
const r = await postExec(name, ["sh", "-c", script], 120, data);
const written = Number.parseInt(r.stdout.toString("utf8").trim(), 10);
if (r.rc !== 0 || written !== data.length) {
@@ -478,6 +484,39 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
backupComputer: execBackup.backupComputer,
+ async computerStatus(scopeId: string) {
+ const name = spriteScopeName(prefix, scopeId);
+ let machine: string;
+ try {
+ const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}/check`, {
+ headers: { authorization: `Bearer ${opts.token ?? ""}` },
+ signal: AbortSignal.timeout(CHECK_TIMEOUT_MS),
+ });
+ const body = res.ok ? ((await res.json().catch(() => null)) as { status?: string } | null) : null;
+ machine = body?.status ?? `check failed: http ${res.status}`;
+ } catch (e) {
+ machine = `check failed: ${errMessage(e)}`;
+ }
+ let guestResponsive = false;
+ try {
+ guestResponsive = (await execRaw(name, "true", GUEST_PROBE_TIMEOUT_SEC)).code === 0;
+ } catch (e) {
+ void e;
+ }
+ return { machine, guestResponsive };
+ },
+
+ async restartComputer(scopeId: string): Promise<void> {
+ const name = spriteScopeName(prefix, scopeId);
+ ensured.delete(name);
+ const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}/restart`, {
+ method: "POST",
+ headers: { authorization: `Bearer ${opts.token ?? ""}` },
+ signal: AbortSignal.timeout(RESTART_TIMEOUT_MS),
+ });
+ if (!res.ok) throw new Error(`sprites restart ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ },
+
async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
if (handle.scratch) {
const key = scratchKeyByName.get(handle.id);
diff --git a/src/tools/primitives.ts b/src/tools/primitives.ts
index b7aa0bca4..ce18b93cb 100644
--- a/src/tools/primitives.ts
+++ b/src/tools/primitives.ts
@@ -1,7 +1,7 @@
import { join } from "node:path";
import { interpolateSplitEnv } from "../deployment/deployment-layer.ts";
import { BASE_RESIDENT_AUTH_PATHS, residentAuthPaths, type CredentialPathSpec } from "../credentials/resident-paths.ts";
-import type { ExecResult, Sandbox, SandboxHandle } from "../sandbox/sandbox.ts";
+import type { ComputerStatus, ExecResult, Sandbox, SandboxHandle } from "../sandbox/sandbox.ts";
import { CapabilityUnsupportedError, hasParentPathSegment, supportsAgentComputerBackup } from "../sandbox/sandbox.ts";
import type {
CommandPolicy,
@@ -166,6 +166,8 @@ export interface ToolContext extends SurfaceToolDeps {
signal?: AbortSignal;
},
): Promise<ExecResult & { reached?: ReachedProvenance }>;
+ computerStatus(): Promise<ComputerStatus>;
+ restartComputer(): Promise<void>;
read(path: string): Promise<ReadResult>;
write(path: string, data?: string, share?: ShareDirective[]): Promise<WriteResult>;
publish(input: PublishInput): Promise<PublishResult>;
@@ -476,6 +478,20 @@ export function createToolContext(deps: ToolContextDeps): ToolContext {
return {
...(deps.credentialExecServices ? { credentialExecServices: deps.credentialExecServices } : {}),
...(deps.credentialExec ? { credentialExec: deps.credentialExec } : {}),
+ async computerStatus(): Promise<ComputerStatus> {
+ if (!deps.sandbox.computerStatus) {
+ throw new CapabilityUnsupportedError(deps.sandbox.profile.backend, "reporting computer status");
+ }
+ if (!writableScopeId) throw new Error("this turn has no scoped computer");
+ return deps.sandbox.computerStatus(writableScopeId);
+ },
+ async restartComputer(): Promise<void> {
+ if (!deps.sandbox.restartComputer) {
+ throw new CapabilityUnsupportedError(deps.sandbox.profile.backend, "restarting the computer");
+ }
+ if (!writableScopeId) throw new Error("this turn has no scoped computer to restart");
+ await deps.sandbox.restartComputer(writableScopeId);
+ },
async execute(
command: string,
execOpts?: {
diff --git a/test/execute-scratch.test.ts b/test/execute-scratch.test.ts
index d34f1beba..d227a06ae 100644
--- a/test/execute-scratch.test.ts
+++ b/test/execute-scratch.test.ts
@@ -84,7 +84,7 @@ const schemaRequired = (tool: ReturnType<typeof createPiTools>[number]): string[
test("flag OFF: the execute surface is exactly the legacy one (no scope/durable, scoped box)", async () => {
const { tc, seen } = sinkToolContext();
const [execute] = createPiTools({ current: tc });
- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds"]);
+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds"]);
assert.deepEqual(schemaRequired(execute!), ["command", "purpose"]);
await call(execute, { command: "echo hi" });
assert.deepEqual(seen, [{ command: "echo hi", opts: undefined }]);
@@ -94,7 +94,7 @@ test("flag ON: scope defaults to the durable scoped box; scratch is an explicit
const { tc, seen } = sinkToolContext();
const ref: ToolContextRef = { current: tc };
const [execute] = createPiTools(ref, { scratchExec: true });
- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds", "scope", "durable"]);
+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds", "scope", "durable"]);
await call(execute, { command: "echo hi" });
assert.deepEqual(
diff --git a/test/pi-tools.test.ts b/test/pi-tools.test.ts
index a19a5cd46..762e7dc93 100644
--- a/test/pi-tools.test.ts
+++ b/test/pi-tools.test.ts
@@ -11,6 +11,10 @@ function fakeToolContext(sink?: { lastExecOpts?: Parameters<ToolContext["execute
if (sink) sink.lastExecOpts = opts;
return { stdout: `ran ${command}`, stderr: "", code: 0, timedOut: false };
},
+ async restartComputer() {},
+ async computerStatus() {
+ return { machine: "healthy", guestResponsive: true };
+ },
async read(path) {
return path === "a.txt"
? { content: "data", sourceScopeId: "personal:U1" }
@@ -335,6 +339,41 @@ test("each pi tool emits a tool_call then a tool_result", async () => {
);
});
+test("execute's computer param manages the box out-of-band instead of running a command", async () => {
+ const restarted: number[] = [];
+ const tc = {
+ ...fakeToolContext(),
+ restartComputer: async () => {
+ restarted.push(1);
+ },
+ computerStatus: async () => ({ machine: "healthy", guestResponsive: false }),
+ };
+ const ref: ToolContextRef = { current: tc, emit: () => {}, scopeLabel: "personal:U1" };
+ const [execute] = createPiTools(ref);
+
+ const status = (await call(execute, { command: "", computer: "status", purpose: "p" })) as {
+ content: Array<{ text?: string }>;
+ };
+ assert.match(status.content[0]!.text!, /machine: healthy; shell: NOT answering/);
+
+ const restart = (await call(execute, { command: "", computer: "restart", purpose: "p" })) as {
+ content: Array<{ text?: string }>;
+ };
+ assert.equal(restarted.length, 1);
+ assert.match(restart.content[0]!.text!, /restarting/i);
+
+ ref.current = {
+ ...tc,
+ restartComputer: async () => {
+ throw new Error("this computer's substrate (local) does not support restarting the computer");
+ },
+ };
+ const err = (await call(execute, { command: "", computer: "restart", purpose: "p" })) as {
+ content: Array<{ text?: string }>;
+ };
+ assert.match(err.content[0]!.text!, /does not support restarting/);
+});
+
test("a giant tool result is capped for the model, keeping the tail and matching the persisted replay record", async () => {
const emitted: Emitted[] = [];
const tc = {
diff --git a/test/scope-reach.test.ts b/test/scope-reach.test.ts
index 12b0f54ff..a75cbde21 100644
--- a/test/scope-reach.test.ts
+++ b/test/scope-reach.test.ts
@@ -242,7 +242,7 @@ test("reachExec OFF: the execute scope never accepts a room", () => {
test("reachExec ON (no scratch): scope is a free string; a room routes to reachTarget; default is scoped", async () => {
const { tc, seen } = sinkToolContext();
const [execute] = createPiTools({ current: tc }, { reachExec: true });
- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds", "scope"]);
+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds", "scope"]);
await call(execute, { command: "cat x", scope: "#project-alpha" });
assert.deepEqual(seen.at(-1)!.opts, { reachTarget: "#project-alpha" });
await call(execute, { command: "echo hi" });
@@ -256,7 +256,7 @@ test("reachExec ON (no scratch): scope is a free string; a room routes to reachT
test("reachExec ON + scratchExec ON: scope accepts scoped, scratch, AND a room", async () => {
const { tc, seen } = sinkToolContext();
const [execute] = createPiTools({ current: tc }, { reachExec: true, scratchExec: true });
- assert.deepEqual(schemaProps(execute!), ["command", "purpose", "timeout_seconds", "scope", "durable"]);
+ assert.deepEqual(schemaProps(execute!), ["command", "computer", "purpose", "timeout_seconds", "scope", "durable"]);
await call(execute, { command: "x", scope: "scratch" });
assert.deepEqual(seen.at(-1)!.opts, { scratch: true });
await call(execute, { command: "x", scope: "#ops" });
diff --git a/test/sprites-sandbox.test.ts b/test/sprites-sandbox.test.ts
index 77612b7bc..a617ce0eb 100644
--- a/test/sprites-sandbox.test.ts
+++ b/test/sprites-sandbox.test.ts
@@ -256,3 +256,53 @@ test("stageIn pulls a blob into the guest atomically (temp then mv)", async () =
assert.match(script, /mv -f /, "and only then moves it into place");
assert.match(script, /curl -fsS/, "-f so an HTTP error fails loudly instead of writing the error body");
});
+
+test("restartComputer reboots the scope's sprite and heals a wedged exec channel", async () => {
+ const h = await sandbox.provision(layers);
+ fake.fail502(h.id);
+ await assert.rejects(sandbox.run(h, "echo back"), /http 502/);
+
+ await sandbox.restartComputer!(scope);
+ assert.deepEqual(fake.restarts(), [h.id]);
+
+ const after = await sandbox.run(h, "echo back");
+ assert.equal(after.code, 0);
+ assert.equal(after.stdout.trim(), "back");
+});
+
+test("computerStatus reports a healthy machine whose shell has stopped answering", async () => {
+ const h = await sandbox.provision(layers);
+ assert.deepEqual(await sandbox.computerStatus!(scope), { machine: "healthy", guestResponsive: true });
+
+ fake.fail502(h.id);
+ assert.deepEqual(await sandbox.computerStatus!(scope), { machine: "healthy", guestResponsive: false });
+});
+
+test("restartComputer surfaces a refused restart instead of swallowing it", async () => {
+ const h = await sandbox.provision(layers);
+ fake.refuseRestart(h.id);
+ await assert.rejects(sandbox.restartComputer!(scope), /sprites restart .*: http 502/);
+ assert.deepEqual(fake.restarts(), []);
+});
+
+test("a command that ran before the response was lost is never re-executed", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.run(h, ": > /home/sprite/workspace/ledger");
+ fake.stallAfterRun(h.id);
+
+ await assert.rejects(sandbox.run(h, "echo entry >> /home/sprite/workspace/ledger"));
+
+ const ledger = await sandbox.readFile(h, "ledger");
+ assert.equal(ledger, "entry\n", "the side effect must have happened exactly once");
+});
+
+test("an inline write lands atomically, so a reboot mid-write cannot truncate the target", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.writeFile(h, "cfg.txt", "value\n");
+ assert.equal(await sandbox.readFile(h, "cfg.txt"), "value\n");
+
+ const write = fake.execScripts().find((s) => s.includes("cfg.txt") && s.includes("cat >"));
+ assert.ok(write, "expected an inline write script");
+ assert.match(write!, /\.part\./, "the payload must land on a temp path");
+ assert.match(write!, /mv -f/, "and be renamed over the target, never streamed into it");
+});
diff --git a/test/support/fake-sprites.ts b/test/support/fake-sprites.ts
index 64d44e81f..256f30c26 100644
--- a/test/support/fake-sprites.ts
+++ b/test/support/fake-sprites.ts
@@ -25,6 +25,10 @@ export interface FakeSprites {
names(): string[];
policy(name: string): NetworkRule[] | null;
execScripts(): string[];
+ stallAfterRun(name: string): void;
+ fail502(name: string): void;
+ refuseRestart(name: string): void;
+ restarts(): string[];
reset(): void;
cleanup(): void;
}
@@ -38,6 +42,10 @@ export function installFakeSprites(): FakeSprites {
const policies = new Map<string, NetworkRule[]>();
const execScripts: string[] = [];
const calls: SpritesCall[] = [];
+ const gateway502 = new Set<string>();
+ const stallAfterRun = new Set<string>();
+ const refusedRestart = new Set<string>();
+ const restarts: string[] = [];
const ensureDir = (name: string): string => {
let s = sprites.get(name);
@@ -97,6 +105,21 @@ export function installFakeSprites(): FakeSprites {
const url = new URL(typeof input === "string" ? input : input.toString());
const method = init?.method ?? "GET";
calls.push({ method, path: url.pathname });
+ const health = /^\/v1\/sprites\/([^/]+)\/check$/.exec(url.pathname);
+ if (health) {
+ const name = decodeURIComponent(health[1]!);
+ if (!sprites.has(name)) return new Response("sprite not found", { status: 404 });
+ return Response.json({ sprite_name: name, status: "healthy" });
+ }
+ const boot = /^\/v1\/sprites\/([^/]+)\/restart$/.exec(url.pathname);
+ if (boot && method === "POST") {
+ const name = decodeURIComponent(boot[1]!);
+ if (!sprites.has(name)) return new Response("sprite not found", { status: 404 });
+ if (refusedRestart.has(name)) return new Response('{"error":"upstream restart failed"}', { status: 502 });
+ restarts.push(name);
+ gateway502.delete(name);
+ return Response.json({ sprite_name: name });
+ }
const sub = /\/v1\/sprites\/([^/]+)\/(exec|policy\/network)$/.exec(url.pathname);
if (sub) {
const name = decodeURIComponent(sub[1]!);
@@ -105,6 +128,17 @@ export function installFakeSprites(): FakeSprites {
const script = argv[argv.length - 1] ?? "";
calls[calls.length - 1]!.script = script;
const stdin = url.searchParams.get("stdin") === "true" ? toBuf(init?.body) : undefined;
+ const stall = (): never => {
+ throw Object.assign(new Error("The operation was aborted due to timeout"), { name: "TimeoutError" });
+ };
+ if (stallAfterRun.has(name)) {
+ stallAfterRun.delete(name);
+ runExec(name, script, stdin);
+ stall();
+ }
+ if (gateway502.has(name)) {
+ return new Response('{"error":"bad gateway"}', { status: 502 });
+ }
return new Response(runExec(name, script, stdin), { status: 200 });
}
if (method === "GET") return Response.json({ rules: policies.get(name) ?? [] });
@@ -155,10 +189,24 @@ export function installFakeSprites(): FakeSprites {
names: () => [...sprites.keys()],
policy: (name) => policies.get(name) ?? null,
execScripts: () => [...execScripts],
+ stallAfterRun: (name) => {
+ stallAfterRun.add(name);
+ },
+ fail502: (name) => {
+ gateway502.add(name);
+ },
+ refuseRestart: (name) => {
+ refusedRestart.add(name);
+ },
+ restarts: () => [...restarts],
reset: () => {
for (const name of Array.from(sprites.keys())) deleteSprite(name);
execScripts.length = 0;
calls.length = 0;
+ stallAfterRun.clear();
+ gateway502.clear();
+ refusedRestart.clear();
+ restarts.length = 0;
},
cleanup: () => rmSync(root, { recursive: true, force: true }),
};