Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Report safe submission state for Sprites exec failures (#1708)

Joshua Francee8322b93e02c
diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.ts
index 9c60b1432..805466cc1 100644
--- a/src/sandbox/sprites-sandbox.ts
+++ b/src/sandbox/sprites-sandbox.ts
@@ -181,6 +181,7 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
async function spawnExec(name: string, argv: string[], stdin: Buffer, deadlineMs: number): Promise<RawExec> {
const cmd = sprite(name).spawn(argv[0]!, argv.slice(1));
+ let submissionStarted = false;
const out: Buffer[] = [];
const err: Buffer[] = [];
const collect = (stream: Readable, into: Buffer[]): Promise<void> =>
@@ -189,15 +190,31 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
stream.once("end", resolve);
});
const exited = new Promise<number>((resolve, reject) => {
- cmd.on("error", (e) => reject(new Error(`sprites exec ${name}: ${errMessage(e)}`, { cause: e })));
- cmd.once("exit", resolve);
- cmd.once("spawn", () => cmd.stdin.end(stdin));
+ const fail = (detail: string): void => {
+ const phase = submissionStarted ? "script submission started; execution unknown" : "script not submitted";
+ reject(new Error(`sprites exec: WebSocket error (${phase}): ${detail}`));
+ };
+ cmd.on("error", (e) => {
+ const detail =
+ /^(?:WebSocket error: )?(TypeError|ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|connection reset by peer|Received network error or non-101 status code|WebSocket closed before open)\b/.exec(
+ errMessage(e),
+ )?.[1] ?? "unknown transport error";
+ fail(detail);
+ });
+ cmd.once("exit", (code) => {
+ if (code < 0) fail("connection closed without an exit status");
+ else resolve(code);
+ });
+ cmd.once("spawn", () => {
+ submissionStarted = true;
+ cmd.stdin.end(stdin);
+ });
});
try {
const [rc] = await withTimeout(
() => Promise.all([exited, collect(cmd.stdout, out), collect(cmd.stderr, err)]),
deadlineMs,
- `sprites exec ${name}`,
+ "sprites exec",
);
return { rc, stdout: Buffer.concat(out), stderr: Buffer.concat(err) };
} catch (e) {
diff --git a/test/sprites-sandbox.test.ts b/test/sprites-sandbox.test.ts
index 42dccf234..6eab4db5d 100644
--- a/test/sprites-sandbox.test.ts
+++ b/test/sprites-sandbox.test.ts
@@ -6,6 +6,7 @@ import assert from "node:assert/strict";
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
+import { inspect } from "node:util";
import {
createSpritesSandbox,
processKeepaliveScript,
@@ -22,7 +23,7 @@ import { scopeId } from "../src/types.ts";
import { mintCapabilityToken, EGRESS_PROXY_AUD } from "../src/auth/capability-token.ts";
import { installFakeSprites, FAKE_SPRITES_TOKEN, type FakeSprites } from "./support/fake-sprites.ts";
import type { Sandbox } from "../src/sandbox/sandbox.ts";
-import { APIError } from "@fly/sprites";
+import { APIError, SpriteCommand } from "@fly/sprites";
let fake: FakeSprites;
let sandbox: Sandbox;
@@ -397,15 +398,52 @@ test("a refused restart with no checkpoint to fall back on names all three failu
await assert.rejects(sandbox.restartComputer!(scope), /http 502.*boot loop.*no checkpoint to restore/s);
});
-test("a command that ran before the response was lost is never re-executed", async () => {
- const h = await sandbox.provision(layers);
- await sandbox.run(h, ": > /home/sprite/workspace/ledger");
- fake.stallAfterRun(h.id);
-
- await assert.rejects(sandbox.run(h, "echo entry >> /home/sprite/workspace/ledger"));
+for (const submitted of [false, true]) {
+ for (const closeOnly of [false, true]) {
+ test(`exec failures report submission state without replay or private details (submitted=${submitted}, closeOnly=${closeOnly})`, async (t) => {
+ const h = await sandbox.provision(layers);
+ if (closeOnly) {
+ const dispatch = EventTarget.prototype.dispatchEvent;
+ t.mock.method(EventTarget.prototype, "dispatchEvent", function (this: EventTarget, event: Event) {
+ return event.type === "error" || dispatch.call(this, event);
+ });
+ }
+ const before = fake.calls.filter((call) => call.method === "WS").length;
+ if (submitted) fake.stallAfterRun(h.id);
+ else fake.fail502(h.id);
+
+ await assert.rejects(sandbox.run(h, "echo entry >> /home/sprite/workspace/ledger"), (error: Error) => {
+ assert.match(
+ error.message,
+ submitted ? /script submission started; execution unknown/ : /script not submitted/,
+ );
+ for (const value of [h.id, fake.baseUrl.replace("https:", "wss:"), "cmd=", "echo entry"])
+ assert.ok(!inspect(error).includes(value));
+ return true;
+ });
+
+ assert.equal(fake.calls.filter((call) => call.method === "WS").length, before + 1);
+ assert.equal(await sandbox.readFile(h, "ledger"), submitted ? "entry\n" : null);
+ });
+ }
+}
- const ledger = await sandbox.readFile(h, "ledger");
- assert.equal(ledger, "entry\n", "the side effect must have happened exactly once");
+test("exec diagnostics retain known transport reasons without raw error causes", async (t) => {
+ const h = await sandbox.provision(layers);
+ let reason = "";
+ t.mock.method(SpriteCommand.prototype, "start", async () => {
+ throw new Error(`WebSocket error: ${reason} (url: wss://example.invalid/TypeError?token=example-secret)`, {
+ cause: new Error("example-secret"),
+ });
+ });
+ for (reason of ["TypeError", "unclassified provider failure"]) {
+ await assert.rejects(sandbox.run(h, "echo ignored"), (error: Error) => {
+ assert.match(error.message, /script not submitted/);
+ assert.ok(error.message.endsWith(reason === "TypeError" ? "TypeError" : "unknown transport error"));
+ assert.doesNotMatch(inspect(error), /example-secret|example\.invalid|token=/);
+ return true;
+ });
+ }
});
test("exec results carry io pressure when the guest exposes it, and omit it when it can't be read", async () => {