Multiplayer agent harness for work.
Report safe submission state for Sprites exec failures (#1708)
Joshua Francee8322b93e02c
diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.tsindex 9c60b1432..805466cc1 100644--- a/src/sandbox/sprites-sandbox.ts+++ b/src/sandbox/sprites-sandbox.ts@@ -181,6 +181,7 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanasync function spawnExec(name: string, argv: string[], stdin: Buffer, deadlineMs: number): Promise<RawExec> {const cmd = sprite(name).spawn(argv[0]!, argv.slice(1));+ let submissionStarted = false;const out: Buffer[] = [];const err: Buffer[] = [];const collect = (stream: Readable, into: Buffer[]): Promise<void> =>@@ -189,15 +190,31 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanstream.once("end", resolve);});const exited = new Promise<number>((resolve, reject) => {- cmd.on("error", (e) => reject(new Error(`sprites exec ${name}: ${errMessage(e)}`, { cause: e })));- cmd.once("exit", resolve);- cmd.once("spawn", () => cmd.stdin.end(stdin));+ const fail = (detail: string): void => {+ const phase = submissionStarted ? "script submission started; execution unknown" : "script not submitted";+ reject(new Error(`sprites exec: WebSocket error (${phase}): ${detail}`));+ };+ cmd.on("error", (e) => {+ const detail =+ /^(?:WebSocket error: )?(TypeError|ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|connection reset by peer|Received network error or non-101 status code|WebSocket closed before open)\b/.exec(+ errMessage(e),+ )?.[1] ?? "unknown transport error";+ fail(detail);+ });+ cmd.once("exit", (code) => {+ if (code < 0) fail("connection closed without an exit status");+ else resolve(code);+ });+ cmd.once("spawn", () => {+ submissionStarted = true;+ cmd.stdin.end(stdin);+ });});try {const [rc] = await withTimeout(() => Promise.all([exited, collect(cmd.stdout, out), collect(cmd.stderr, err)]),deadlineMs,- `sprites exec ${name}`,+ "sprites exec",);return { rc, stdout: Buffer.concat(out), stderr: Buffer.concat(err) };} catch (e) {diff --git a/test/sprites-sandbox.test.ts b/test/sprites-sandbox.test.tsindex 42dccf234..6eab4db5d 100644--- a/test/sprites-sandbox.test.ts+++ b/test/sprites-sandbox.test.ts@@ -6,6 +6,7 @@ import assert from "node:assert/strict";import { mkdtempSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";+import { inspect } from "node:util";import {createSpritesSandbox,processKeepaliveScript,@@ -22,7 +23,7 @@ import { scopeId } from "../src/types.ts";import { mintCapabilityToken, EGRESS_PROXY_AUD } from "../src/auth/capability-token.ts";import { installFakeSprites, FAKE_SPRITES_TOKEN, type FakeSprites } from "./support/fake-sprites.ts";import type { Sandbox } from "../src/sandbox/sandbox.ts";-import { APIError } from "@fly/sprites";+import { APIError, SpriteCommand } from "@fly/sprites";let fake: FakeSprites;let sandbox: Sandbox;@@ -397,15 +398,52 @@ test("a refused restart with no checkpoint to fall back on names all three failuawait assert.rejects(sandbox.restartComputer!(scope), /http 502.*boot loop.*no checkpoint to restore/s);});-test("a command that ran before the response was lost is never re-executed", async () => {- const h = await sandbox.provision(layers);- await sandbox.run(h, ": > /home/sprite/workspace/ledger");- fake.stallAfterRun(h.id);-- await assert.rejects(sandbox.run(h, "echo entry >> /home/sprite/workspace/ledger"));+for (const submitted of [false, true]) {+ for (const closeOnly of [false, true]) {+ test(`exec failures report submission state without replay or private details (submitted=${submitted}, closeOnly=${closeOnly})`, async (t) => {+ const h = await sandbox.provision(layers);+ if (closeOnly) {+ const dispatch = EventTarget.prototype.dispatchEvent;+ t.mock.method(EventTarget.prototype, "dispatchEvent", function (this: EventTarget, event: Event) {+ return event.type === "error" || dispatch.call(this, event);+ });+ }+ const before = fake.calls.filter((call) => call.method === "WS").length;+ if (submitted) fake.stallAfterRun(h.id);+ else fake.fail502(h.id);++ await assert.rejects(sandbox.run(h, "echo entry >> /home/sprite/workspace/ledger"), (error: Error) => {+ assert.match(+ error.message,+ submitted ? /script submission started; execution unknown/ : /script not submitted/,+ );+ for (const value of [h.id, fake.baseUrl.replace("https:", "wss:"), "cmd=", "echo entry"])+ assert.ok(!inspect(error).includes(value));+ return true;+ });++ assert.equal(fake.calls.filter((call) => call.method === "WS").length, before + 1);+ assert.equal(await sandbox.readFile(h, "ledger"), submitted ? "entry\n" : null);+ });+ }+}- const ledger = await sandbox.readFile(h, "ledger");- assert.equal(ledger, "entry\n", "the side effect must have happened exactly once");+test("exec diagnostics retain known transport reasons without raw error causes", async (t) => {+ const h = await sandbox.provision(layers);+ let reason = "";+ t.mock.method(SpriteCommand.prototype, "start", async () => {+ throw new Error(`WebSocket error: ${reason} (url: wss://example.invalid/TypeError?token=example-secret)`, {+ cause: new Error("example-secret"),+ });+ });+ for (reason of ["TypeError", "unclassified provider failure"]) {+ await assert.rejects(sandbox.run(h, "echo ignored"), (error: Error) => {+ assert.match(error.message, /script not submitted/);+ assert.ok(error.message.endsWith(reason === "TypeError" ? "TypeError" : "unknown transport error"));+ assert.doesNotMatch(inspect(error), /example-secret|example\.invalid|token=/);+ return true;+ });+ }});test("exec results carry io pressure when the guest exposes it, and omit it when it can't be read", async () => {