Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

sandbox: classify Modal and process-session errors by type and state, not message text (#2440)

Regan Bellc7fa7e86fae5
diff --git a/src/api/app-sessions.ts b/src/api/app-sessions.ts
index fc2ee85e6..c209ecf31 100644
--- a/src/api/app-sessions.ts
+++ b/src/api/app-sessions.ts
@@ -14,7 +14,7 @@ import { cancelGoal, goalCancelledPayload, openGoal } from "../harness/goal.ts";
import { goalView } from "../runs/turn-stream.ts";
import { SEARCH_HIT_LIMIT, entrySearchText, searchSnippet, searchTerms } from "../sessions/entry-search.ts";
import { supportsProcessSessions } from "../sandbox/sandbox.ts";
-import { processIsGone } from "../sandbox/process-poll.ts";
+import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";
import { cronIsActive, cronTiedTo } from "../cron/cron-store.ts";
import { cronRef, deployRef, encodeRef, fileRef, parseRef, skillRef } from "../acl/resource-ref.ts";
import { samePerson } from "../directory/person.ts";
@@ -644,7 +644,7 @@ export function createSessionMethods(
...(read.status.state === "exited" ? { exitCode: read.status.code } : {}),
};
} catch (e) {
- if (processIsGone(e)) return { chunk: "", cursor: sinceCursor, state: "exited" };
+ if (e instanceof ProcessSessionGoneError) return { chunk: "", cursor: sinceCursor, state: "exited" };
throw e;
} finally {
await sandbox.teardown(handle, { keepWarm: true }).catch(() => {});
diff --git a/src/connectors/background-exec-broker.ts b/src/connectors/background-exec-broker.ts
index 76477d25b..bf4c28dfc 100644
--- a/src/connectors/background-exec-broker.ts
+++ b/src/connectors/background-exec-broker.ts
@@ -2,8 +2,8 @@ import type { ProcessSandbox, ProcessState, SandboxHandle } from "../sandbox/san
import type { ProcessRegistry, ProcessStatus } from "../processes/process-registry.ts";
import type { ProcessCredentialFiles } from "../credentials/execute-files.ts";
import { awaitProcessExit } from "../sandbox/await-process-exit.ts";
-import { pollProcess, processIsGone } from "../sandbox/process-poll.ts";
-import { redactCommand } from "../sandbox/exec-process-session.ts";
+import { pollProcess } from "../sandbox/process-poll.ts";
+import { ProcessSessionGoneError, redactCommand } from "../sandbox/exec-process-session.ts";
import { CONFIG_DEFAULTS } from "../config.ts";
import { createExactSecretValueMasker } from "../security/secret-masking.ts";
import { withCleanup } from "../util/errors.ts";
@@ -149,7 +149,7 @@ export function createBackgroundBroker(deps: BackgroundExecBrokerDeps): Backgrou
processId = null;
}
} catch (error) {
- if (!processIsGone(error)) throw error;
+ if (!(error instanceof ProcessSessionGoneError)) throw error;
await deps.registry.delete(existingProcessId);
processId = null;
}
diff --git a/src/monitors/monitor-poller.ts b/src/monitors/monitor-poller.ts
index c1991a325..28149639f 100644
--- a/src/monitors/monitor-poller.ts
+++ b/src/monitors/monitor-poller.ts
@@ -6,7 +6,7 @@ import type { DeliveryStore } from "../delivery/delivery-store.ts";
import type { IdempotencyStore } from "../idempotency/idempotency-store.ts";
import type { IdentityService } from "../identity/identity-service.ts";
import { supportsProcessSessions, type ProcessSandbox, type Sandbox, type SandboxHandle } from "../sandbox/sandbox.ts";
-import { processIsGone } from "../sandbox/process-poll.ts";
+import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";
import { runTrigger, type TriggerDeps, type TriggerOutcome } from "../triggers/run-trigger.ts";
import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";
import { createSweeper } from "../util/sweeper.ts";
@@ -189,7 +189,7 @@ export function createMonitorPoller(deps: MonitorPollerDeps): MonitorPoller {
waitMs: 0,
});
} catch (e) {
- if (processIsGone(e)) {
+ if (e instanceof ProcessSessionGoneError) {
await reportLost(m);
return true;
}
diff --git a/src/processes/process-reaper.ts b/src/processes/process-reaper.ts
index eb023c015..6c01a0054 100644
--- a/src/processes/process-reaper.ts
+++ b/src/processes/process-reaper.ts
@@ -2,7 +2,7 @@ import type { ProcessRecord, ProcessRegistry } from "./process-registry.ts";
import { createSweeper } from "../util/sweeper.ts";
import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";
import { awaitProcessExit } from "../sandbox/await-process-exit.ts";
-import { processIsGone } from "../sandbox/process-poll.ts";
+import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";
import type { ProcessSandbox, SandboxHandle } from "../sandbox/sandbox.ts";
import { errMessage } from "../util/errors.ts";
@@ -35,7 +35,7 @@ export function createReaperKillHook(
}
if (status.state !== "exited") throw new Error(`process survived TERM+KILL: ${rec.processId}`);
} catch (e) {
- if (!processIsGone(e)) throw e;
+ if (!(e instanceof ProcessSessionGoneError)) throw e;
}
await opts
?.onExit?.(handle, rec.processId)
diff --git a/src/sandbox/exec-process-session.ts b/src/sandbox/exec-process-session.ts
index 1b8ea6347..ccad5c7d6 100644
--- a/src/sandbox/exec-process-session.ts
+++ b/src/sandbox/exec-process-session.ts
@@ -24,6 +24,13 @@ export interface ExecProcessSessions {
listProcesses(handle: SandboxHandle): Promise<ProcessSession[]>;
}
+export class ProcessSessionGoneError extends Error {
+ constructor(processId: string) {
+ super(`no such process session: ${processId}`);
+ this.name = "ProcessSessionGoneError";
+ }
+}
+
const PROC_BASE = "${HOME:-/root}/.agent-proc";
export const processSessionDir = (processId: string): string => `${PROC_BASE}/${processId}`;
const REBOOT_RC = 137;
@@ -133,7 +140,7 @@ export function createExecProcessSessions(io: ExecProcessIo): ExecProcessSession
].join("\n");
const r = await io.run(handle, script, { timeoutMs: waitMs + 20_000 });
const out = r.stdout;
- if (/MISSING=1/.test(out)) throw new Error(`no such process session: ${processId}`);
+ if (/MISSING=1/.test(out)) throw new ProcessSessionGoneError(processId);
const cur = /CURSOR=(\d+)/.exec(out);
const st = /STATUS=(running|exited:-?\d+)/.exec(out);
const dataIdx = out.indexOf("DATA=");
diff --git a/src/sandbox/modal-client.ts b/src/sandbox/modal-client.ts
index 4be571e5f..42524b688 100644
--- a/src/sandbox/modal-client.ts
+++ b/src/sandbox/modal-client.ts
@@ -1,6 +1,7 @@
import { randomUUID } from "node:crypto";
import { isIP } from "node:net";
import { resolveModalImage } from "./modal-image.ts";
+import { swallow } from "../util/errors.ts";
export interface ModalCommandResult {
stdout: string;
@@ -79,46 +80,40 @@ function wholeSeconds(ms: number): number {
return Math.max(1000, Math.ceil(ms / 1000) * 1000);
}
-function errName(err: unknown): string {
- return String((err as Error)?.name ?? "");
-}
-
function errText(err: unknown): string {
return String((err as Error)?.message ?? err);
}
-function isFileNotFoundError(err: unknown): boolean {
- return errName(err) === "SandboxFilesystemNotFoundError" || /no such file or directory/i.test(errText(err));
-}
+type ModalSdk = typeof import("modal");
-function isFileTooLargeError(err: unknown): boolean {
- return errName(err) === "SandboxFilesystemFileTooLargeError";
-}
+const GRPC_INVALID_ARGUMENT = 3;
+const GRPC_NOT_FOUND = 5;
-function isSandboxGoneError(err: unknown): boolean {
- if (isFileNotFoundError(err)) return false;
- const name = errName(err);
- if (name === "NotFoundError" || name === "ClientClosedError") return true;
- return /(sandbox|task).*(not found|has been terminated|already (terminated|finished|completed)|does not exist|is not running)|detached sandbox/i.test(
- errText(err),
- );
+function grpcCode(err: unknown): unknown {
+ return (err as { code?: unknown } | null)?.code;
}
-const GRPC_INVALID_ARGUMENT = 3;
-
-function isSandboxIdLookupGone(err: unknown): boolean {
- if (isSandboxGoneError(err)) return true;
- if (errName(err) === "InvalidError") return true;
- if ((err as { code?: unknown }).code === GRPC_INVALID_ARGUMENT) return true;
- return /INVALID_ARGUMENT/.test(errText(err));
+function isSandboxGoneError(sdk: ModalSdk, err: unknown): boolean {
+ return err instanceof sdk.NotFoundError || err instanceof sdk.ClientClosedError || grpcCode(err) === GRPC_NOT_FOUND;
}
-function isNameConflictError(err: unknown): boolean {
- return errName(err) === "AlreadyExistsError" || /already exists|already in use/i.test(errText(err));
+function isSandboxIdLookupGone(sdk: ModalSdk, err: unknown): boolean {
+ return isSandboxGoneError(sdk, err) || err instanceof sdk.InvalidError || grpcCode(err) === GRPC_INVALID_ARGUMENT;
}
-function isDeadlineError(err: unknown): boolean {
- return /timeouterror$/i.test(errName(err)) || /deadline exceeded/i.test(errText(err));
+async function hasExited(
+ sdk: ModalSdk,
+ sbx: { sandboxId: string; poll(): Promise<number | null> },
+ err: unknown,
+): Promise<boolean> {
+ if (isSandboxGoneError(sdk, err)) return true;
+ try {
+ return (await sbx.poll()) !== null;
+ } catch (pollErr) {
+ if (isSandboxGoneError(sdk, pollErr)) return true;
+ swallow(`modal-client: poll ${sbx.sandboxId} after a failed call`, pollErr);
+ return false;
+ }
}
export function modalEgressAllowlist(
@@ -143,14 +138,14 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
throw new Error("Modal snapshot retention must be a positive finite duration");
const snapshotTtlMs = wholeSeconds(snapshotRetentionMs);
const network = opts.egressProxyUrl ? modalEgressAllowlist(opts.egressProxyUrl) : {};
- type ModalSdk = typeof import("modal");
type SdkClient = InstanceType<ModalSdk["ModalClient"]>;
type SdkApp = Awaited<ReturnType<SdkClient["apps"]["fromName"]>>;
type SdkImage = ReturnType<SdkClient["images"]["fromRegistry"]>;
type SdkSandbox = Awaited<ReturnType<SdkClient["sandboxes"]["fromId"]>>;
- let ctx: Promise<{ client: SdkClient; app: SdkApp; image: SdkImage }> | null = null;
- const buildCtx = async (): Promise<{ client: SdkClient; app: SdkApp; image: SdkImage }> => {
+ type Ctx = { sdk: ModalSdk; client: SdkClient; app: SdkApp; image: SdkImage };
+ let ctx: Promise<Ctx> | null = null;
+ const buildCtx = async (): Promise<Ctx> => {
let sdk: ModalSdk;
try {
sdk = (await import("modal")) as unknown as ModalSdk;
@@ -164,19 +159,20 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
});
const app = await client.apps.fromName(opts.appName, { createIfMissing: true });
const image = await resolveModalImage(client, opts.image);
- return { client, app, image };
+ return { sdk, client, app, image };
};
- const loadCtx = (): Promise<{ client: SdkClient; app: SdkApp; image: SdkImage }> =>
+ const loadCtx = (): Promise<Ctx> =>
(ctx ??= buildCtx().catch((e) => {
ctx = null;
throw e;
}));
- const wrap = (sbx: SdkSandbox): ModalSession => ({
+ const wrap = (sdk: ModalSdk, sbx: SdkSandbox): ModalSession => ({
sandboxId: sbx.sandboxId,
async runCommand(command, runOpts): Promise<ModalCommandResult> {
const commandMs = wholeSeconds(runOpts?.timeoutMs ?? maxCommandMs);
const timeoutMs = commandMs + MODAL_EXEC_GRACE_MS;
+ const deadlineAt = Date.now() + timeoutMs;
try {
let args = ["timeout", String(commandMs / 1000), "sh", "-c", command];
if (Buffer.byteLength(command, "utf8") > MODAL_MAX_EXEC_ARG_BYTES) {
@@ -192,8 +188,9 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
const [stdout, stderr, exitCode] = await Promise.all([p.stdout.readText(), p.stderr.readText(), p.wait()]);
return { stdout, stderr, exitCode };
} catch (err) {
- if (isDeadlineError(err)) return { stdout: "", stderr: errText(err), exitCode: 124 };
- if (isSandboxGoneError(err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));
+ if (await hasExited(sdk, sbx, err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));
+ if (err instanceof sdk.TimeoutError || Date.now() >= deadlineAt)
+ return { stdout: "", stderr: errText(err), exitCode: 124 };
throw err;
}
},
@@ -201,13 +198,13 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
try {
return await sbx.filesystem.readBytes(absPath);
} catch (err) {
- if (isSandboxGoneError(err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));
- if (isFileNotFoundError(err)) return null;
- if (isFileTooLargeError(err))
+ if (err instanceof sdk.SandboxFilesystemNotFoundError) return null;
+ if (err instanceof sdk.SandboxFilesystemFileTooLargeError)
throw new Error(
`modal read ${absPath}: the file exceeds Modal's filesystem read limit; move it with blob staging or split it`,
{ cause: err },
);
+ if (await hasExited(sdk, sbx, err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));
throw err;
}
},
@@ -215,7 +212,7 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
try {
await sbx.filesystem.writeBytes(data, absPath);
} catch (err) {
- if (isSandboxGoneError(err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));
+ if (await hasExited(sdk, sbx, err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));
throw err;
}
},
@@ -230,7 +227,7 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
},
async terminate(): Promise<void> {
await sbx.terminate().catch((err) => {
- if (!isSandboxGoneError(err)) throw err;
+ if (!isSandboxGoneError(sdk, err)) throw err;
});
},
});
@@ -239,7 +236,7 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
lifetimeMs: sandboxTimeoutMs,
nativeSnapshots: true,
async create(createOpts): Promise<ModalSession> {
- const { client, app, image } = await loadCtx();
+ const { sdk, client, app, image } = await loadCtx();
try {
const sbx = await client.sandboxes.create(app, image, {
...(createOpts.name ? { name: createOpts.name } : {}),
@@ -251,66 +248,67 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {
...(opts.regions?.length ? { regions: opts.regions } : {}),
...network,
});
- return wrap(sbx);
+ return wrap(sdk, sbx);
} catch (err) {
- if (isNameConflictError(err)) throw new ModalNameConflictError(createOpts.name ?? "(unnamed)", errText(err));
+ if (err instanceof sdk.AlreadyExistsError)
+ throw new ModalNameConflictError(createOpts.name ?? "(unnamed)", errText(err));
throw err;
}
},
async fromId(sandboxId): Promise<ModalSession> {
- const { client } = await loadCtx();
+ const { sdk, client } = await loadCtx();
try {
const sbx = await client.sandboxes.fromId(sandboxId);
const exited = await sbx.poll();
if (exited !== null) throw new ModalSandboxGoneError(sandboxId, `exited with code ${exited}`);
- return wrap(sbx);
+ return wrap(sdk, sbx);
} catch (err) {
if (err instanceof ModalSandboxGoneError) throw err;
- if (isSandboxIdLookupGone(err)) throw new ModalSandboxGoneError(sandboxId, errText(err));
+ if (isSandboxIdLookupGone(sdk, err)) throw new ModalSandboxGoneError(sandboxId, errText(err));
throw err;
}
},
async fromName(name): Promise<ModalSession | null> {
- const { client } = await loadCtx();
+ const { sdk, client } = await loadCtx();
let sbx: SdkSandbox;
try {
sbx = await client.sandboxes.fromName(opts.appName, name);
} catch (err) {
- if (isSandboxGoneError(err)) return null;
+ if (isSandboxGoneError(sdk, err)) return null;
throw err;
}
try {
const exited = await sbx.poll();
if (exited !== null) return null;
} catch (err) {
- if (isSandboxGoneError(err)) return null;
+ if (isSandboxGoneError(sdk, err)) return null;
}
- return wrap(sbx);
+ return wrap(sdk, sbx);
},
async terminate(sandboxId): Promise<void> {
- const { client } = await loadCtx();
+ const { sdk, client } = await loadCtx();
let sbx: SdkSandbox;
try {
sbx = await client.sandboxes.fromId(sandboxId);
} catch (err) {
- if (isSandboxIdLookupGone(err)) return;
+ if (isSandboxIdLookupGone(sdk, err)) return;
throw err;
}
try {
await sbx.terminate();
} catch (err) {
- if (isSandboxGoneError(err)) return;
+ if (isSandboxGoneError(sdk, err)) return;
throw err;
}
},
async *listRunning(tags): AsyncIterable<string> {
- const { client, app } = await loadCtx();
+ const { sdk, client, app } = await loadCtx();
for await (const sbx of client.sandboxes.list({ appId: app.appId, tags })) {
let exited: number | null;
try {
exited = await sbx.poll();
} catch (err) {
- if (isSandboxGoneError(err)) continue;
+ if (isSandboxGoneError(sdk, err)) continue;
throw err;
}
if (exited === null) yield sbx.sandboxId;
diff --git a/src/sandbox/process-poll.ts b/src/sandbox/process-poll.ts
index 303096ea4..0bb12d29d 100644
--- a/src/sandbox/process-poll.ts
+++ b/src/sandbox/process-poll.ts
@@ -1,9 +1,6 @@
import type { ProcessSandbox, ProcessState, ReadProcessResult, SandboxHandle } from "./sandbox.ts";
-import { errMessage } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
-export const processIsGone = (e: unknown): boolean => /^no such process/i.test(errMessage(e));
-
export interface PollProcessOptions {
deadlineMs: number;
waitMs?: number;
diff --git a/test/background-exec-broker.test.ts b/test/background-exec-broker.test.ts
index 6bdcd2344..9dc518c08 100644
--- a/test/background-exec-broker.test.ts
+++ b/test/background-exec-broker.test.ts
@@ -4,6 +4,7 @@ import { createBackgroundBroker } from "../src/connectors/background-exec-broker
import { createMemoryProcessRegistry } from "../src/processes/process-registry.ts";
import { redactCommand } from "../src/sandbox/exec-process-session.ts";
import type { ProcessSandbox, ProcessSession, SandboxHandle, StartProcessOptions } from "../src/sandbox/sandbox.ts";
+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";
const SCOPE = "personal:U1";
const handle: SandboxHandle = { id: "vm", rootDir: "/workspace", homeDir: "/root", resourceId: "sandbox-1" };
@@ -59,9 +60,9 @@ function fakeSandbox(opts?: { seedOutput?: string }) {
readErrors.delete(id);
throw error;
}
- if (missing.has(id)) throw new Error(`no such process session: ${id}`);
+ if (missing.has(id)) throw new ProcessSessionGoneError(id);
const p = procs.get(id);
- if (!p) throw new Error(`no such process session: ${id}`);
+ if (!p) throw new ProcessSessionGoneError(id);
const cur = opts?.sinceCursor ?? 0;
const maxBytes = opts?.maxBytes ?? 64 * 1024;
const end = Math.min(p.output.length, cur + maxBytes);
diff --git a/test/durable-process-sessions.test.ts b/test/durable-process-sessions.test.ts
index b40fc4d6a..e52e7cf6d 100644
--- a/test/durable-process-sessions.test.ts
+++ b/test/durable-process-sessions.test.ts
@@ -24,6 +24,7 @@ import { createDeployService } from "../src/deploy/deploy-service.ts";
import { writableMemoryScope } from "../src/memory/policy.ts";
import type { Sandbox, ProcessSession, SandboxHandle } from "../src/sandbox/sandbox.ts";
import type { Conversation, Principal } from "../src/types.ts";
+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";
const ORG = "default-org";
const actor: Principal = { id: "U1", type: "internal" };
@@ -77,7 +78,7 @@ function fakeProcessSandbox() {
},
async readProcess(_h, id, opts) {
const p = procs.get(id);
- if (!p) throw new Error(`no such process session: ${id}`);
+ if (!p) throw new ProcessSessionGoneError(id);
const cur = opts?.sinceCursor ?? 0;
return {
chunks: p.output.slice(cur),
diff --git a/test/exec-process-session.test.ts b/test/exec-process-session.test.ts
index 9aba7af7b..01366c1c6 100644
--- a/test/exec-process-session.test.ts
+++ b/test/exec-process-session.test.ts
@@ -4,7 +4,12 @@ import { spawn } from "node:child_process";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { createExecProcessSessions, type ExecProcessIo, redactCommand } from "../src/sandbox/exec-process-session.ts";
+import {
+ createExecProcessSessions,
+ type ExecProcessIo,
+ ProcessSessionGoneError,
+ redactCommand,
+} from "../src/sandbox/exec-process-session.ts";
import type { ExecResult, SandboxHandle } from "../src/sandbox/sandbox.ts";
function shellIo(home: string): ExecProcessIo {
@@ -138,7 +143,7 @@ test("signalProcess(KILL) records an exit even though the launcher trap can't ca
test("reading or writing an unknown process id is rejected", async () => {
const { proc, handle } = fixture();
const bogus = "00000000-0000-0000-0000-000000000000";
- await assert.rejects(proc.readProcess(handle, bogus), /no such process/);
+ await assert.rejects(proc.readProcess(handle, bogus), ProcessSessionGoneError);
await assert.rejects(proc.writeStdin(handle, bogus, "x"), /writeStdin failed/);
});
diff --git a/test/monitor-poller.test.ts b/test/monitor-poller.test.ts
index cbd84df5c..b9f8274da 100644
--- a/test/monitor-poller.test.ts
+++ b/test/monitor-poller.test.ts
@@ -9,6 +9,7 @@ import { createIdentityService } from "../src/identity/identity-service.ts";
import type { LeaderLease } from "../src/persistence/leader-lease.ts";
import { scopeId, type TurnRequest, type TurnResult } from "../src/types.ts";
import type { ProcessSandbox, ProcessSession, SandboxHandle } from "../src/sandbox/sandbox.ts";
+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";
const SCOPE = "personal:U1";
const handle: SandboxHandle = { id: "vm", rootDir: "/workspace", homeDir: "/root" };
@@ -56,7 +57,7 @@ function fakeSandbox() {
return { processId };
},
async readProcess(_h, id, opts) {
- if (missing.has(id) || !procs.has(id)) throw new Error(`no such process session: ${id}`);
+ if (missing.has(id) || !procs.has(id)) throw new ProcessSessionGoneError(id);
const p = procs.get(id)!;
const cur = opts?.sinceCursor ?? 0;
const end = Math.min(p.output.length, cur + (opts?.maxBytes ?? 64 * 1024));
diff --git a/test/native-provider-clients.test.ts b/test/native-provider-clients.test.ts
index 4e32a32c3..5f1bbab2a 100644
--- a/test/native-provider-clients.test.ts
+++ b/test/native-provider-clients.test.ts
@@ -9,6 +9,8 @@ import { test, mock } from "node:test";
import assert from "node:assert/strict";
import {
createSdkModalClient,
+ ModalNameConflictError,
+ ModalSandboxGoneError,
modalEgressAllowlist,
MODAL_EXEC_GRACE_MS,
MODAL_MAX_EXEC_ARG_BYTES,
@@ -26,12 +28,26 @@ const modalCreateParams: Record<string, unknown>[] = [];
const modalExecs: { args: string[]; params: Record<string, unknown> }[] = [];
const modalWrites: { path: string; bytes: number }[] = [];
const modalRunning = new Map<string, boolean>();
-class SandboxFilesystemFileTooLargeError extends Error {
- override name = "SandboxFilesystemFileTooLargeError";
-}
+class SandboxFilesystemFileTooLargeError extends Error {}
+class ModalSandboxFilesystemNotFoundError extends Error {}
+class ModalNotFoundError extends Error {}
+class ModalClientClosedError extends Error {}
+class ModalAlreadyExistsError extends Error {}
+class ModalInvalidError extends Error {}
+class ModalTimeoutError extends Error {}
+let modalFault: { exec?: Error; read?: Error; write?: Error; create?: Error; terminate?: Error } = {};
+let modalPoll: number | null | Error = null;
const modalSandbox = {
sandboxId: "sb-native",
+ async poll() {
+ if (modalPoll instanceof Error) throw modalPoll;
+ return modalPoll;
+ },
+ async terminate() {
+ if (modalFault.terminate) throw modalFault.terminate;
+ },
async exec(args: string[], params: Record<string, unknown>) {
+ if (modalFault.exec) throw modalFault.exec;
if (typeof params.timeoutMs === "number" && params.timeoutMs % 1000 !== 0)
throw new Error(`timeoutMs must be a multiple of 1000ms, got ${params.timeoutMs}`);
modalExecs.push({ args, params });
@@ -39,9 +55,11 @@ const modalSandbox = {
},
filesystem: {
async writeBytes(data: Uint8Array, path: string) {
+ if (modalFault.write) throw modalFault.write;
modalWrites.push({ path, bytes: data.byteLength });
},
async readBytes(path: string) {
+ if (modalFault.read) throw modalFault.read;
if (path.endsWith("huge.bin")) throw new SandboxFilesystemFileTooLargeError("file too large");
return new Uint8Array([1]);
},
@@ -56,14 +74,23 @@ const modalSandbox = {
};
mock.module("modal", {
namedExports: {
+ SandboxFilesystemFileTooLargeError,
+ SandboxFilesystemNotFoundError: ModalSandboxFilesystemNotFoundError,
+ NotFoundError: ModalNotFoundError,
+ ClientClosedError: ModalClientClosedError,
+ AlreadyExistsError: ModalAlreadyExistsError,
+ InvalidError: ModalInvalidError,
+ TimeoutError: ModalTimeoutError,
ModalClient: class {
apps = { fromName: async () => ({ appId: "ap-1" }) };
images = { fromRegistry: () => ({}), fromId: async (imageId: string) => ({ imageId }) };
sandboxes = {
create: async (_app: unknown, _image: unknown, params: Record<string, unknown>) => {
+ if (modalFault.create) throw modalFault.create;
modalCreateParams.push(params);
return modalSandbox;
},
+ fromId: async () => modalSandbox,
list: async function* (params: { appId?: string; tags?: Record<string, string> }) {
modalCalls.push(["list", params]);
for (const [sandboxId, running] of modalRunning) yield { sandboxId, poll: async () => (running ? null : 0) };
@@ -291,6 +318,53 @@ test("Modal lists running tagged sandboxes within the app and reports oversized
await assert.rejects(session.readFileBytes("/root/huge.bin"), /exceeds Modal's filesystem read limit/);
});
+test("Modal classifies gone sandboxes, missing files, deadlines and name conflicts by SDK error class and sandbox state", async (t) => {
+ t.after(() => {
+ modalFault = {};
+ modalPoll = null;
+ });
+ const client = createSdkModalClient({ tokenId: "id", tokenSecret: "secret", appName: "test", image: "ubuntu" });
+ const session = await client.create({});
+ const plain = new Error("unrecognized failure");
+ modalFault = { exec: plain, read: plain, write: plain };
+ await assert.rejects(session.runCommand("true"), (e) => e === plain, "a running sandbox keeps the original error");
+ await assert.rejects(session.readFileBytes("/root/a"), (e) => e === plain);
+ await assert.rejects(session.writeFileBytes("/root/a", new Uint8Array()), (e) => e === plain);
+ modalPoll = 0;
+ await assert.rejects(session.runCommand("true"), ModalSandboxGoneError, "an exited sandbox is gone");
+ await assert.rejects(session.readFileBytes("/root/a"), ModalSandboxGoneError);
+ await assert.rejects(session.writeFileBytes("/root/a", new Uint8Array()), ModalSandboxGoneError);
+ modalPoll = new ModalNotFoundError("gone");
+ await assert.rejects(session.runCommand("true"), ModalSandboxGoneError, "a sandbox that cannot be polled is gone");
+ modalPoll = null;
+ for (const err of [
+ new ModalNotFoundError("x"),
+ new ModalClientClosedError("x"),
+ Object.assign(new Error("x"), { code: 5 }),
+ ]) {
+ modalFault = { exec: err };
+ await assert.rejects(session.runCommand("true"), ModalSandboxGoneError);
+ }
+ modalFault = { exec: new ModalTimeoutError("x") };
+ assert.equal((await session.runCommand("true")).exitCode, 124);
+ modalPoll = 137;
+ await assert.rejects(
+ session.runCommand("true"),
+ ModalSandboxGoneError,
+ "a sandbox that died at the deadline is gone",
+ );
+ modalPoll = null;
+ modalFault = { read: new ModalSandboxFilesystemNotFoundError("x") };
+ assert.equal(await session.readFileBytes("/root/missing"), null);
+ modalFault = { create: new ModalAlreadyExistsError("x") };
+ await assert.rejects(client.create({ name: "taken" }), ModalNameConflictError);
+ modalFault = { terminate: Object.assign(new Error("x"), { code: 5 }) };
+ await session.terminate();
+ await client.terminate("sb-native");
+ modalPoll = new ModalInvalidError("malformed sandbox id");
+ await assert.rejects(client.fromId("not-a-sandbox"), ModalSandboxGoneError, "an unknown id is a gone sandbox");
+});
+
test("Modal rejects invalid checkpoint retention", () => {
for (const snapshotRetentionMs of [0, -1, Infinity, NaN]) {
assert.throws(
diff --git a/test/process-poll.test.ts b/test/process-poll.test.ts
index 0e3b8854e..9349f29b8 100644
--- a/test/process-poll.test.ts
+++ b/test/process-poll.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { pollProcess, processIsGone } from "../src/sandbox/process-poll.ts";
+import { pollProcess } from "../src/sandbox/process-poll.ts";
import type { ProcessSandbox, ReadProcessResult, SandboxHandle } from "../src/sandbox/sandbox.ts";
const handle: SandboxHandle = { id: "vm", rootDir: "/workspace" };
@@ -23,15 +23,6 @@ const running = (chunks: string, cursor: number): ReadProcessResult => ({
status: { state: "running" },
});
-test("processIsGone: the bare-string sentinel counts as gone", () => {
- assert.equal(processIsGone("no such process session: p-1"), true);
- assert.equal(processIsGone(new Error("no such process session: p-1")), true);
-});
-
-test("processIsGone: an embedded phrase in a wrapped error does NOT confirm a kill", () => {
- assert.equal(processIsGone(new Error("fly api 500: upstream said 'no such process' while restarting")), false);
-});
-
test("pollProcess stops when `until` fires, returning the output and cursor so far", async () => {
const { sandbox, callCount } = fakeSandbox([running("a", 1), running("b", 2), running("c", 3)]);
const res = await pollProcess(sandbox, handle, "p", {
diff --git a/test/process-reaper.test.ts b/test/process-reaper.test.ts
index 27ab8b3a6..29d645f24 100644
--- a/test/process-reaper.test.ts
+++ b/test/process-reaper.test.ts
@@ -5,6 +5,7 @@ import { createProcessReaper, createReaperKillHook } from "../src/processes/proc
import type { LeaderLease } from "../src/persistence/leader-lease.ts";
import type { ProcessSandbox, SandboxHandle, ProvisionOptions, TeardownOptions } from "../src/sandbox/sandbox.ts";
import type { WorkspaceLayer } from "../src/types.ts";
+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
@@ -12,7 +13,7 @@ const ID = "00000000-0000-0000-0000-000000000abc";
const FAST_GRACE = { termGraceMs: 20, killGraceMs: 20 };
-function killSpySandbox(opts?: { diesOn?: "TERM" | "KILL" | "never"; vanished?: boolean | "string-throw" }) {
+function killSpySandbox(opts?: { diesOn?: "TERM" | "KILL" | "never"; vanished?: boolean }) {
const diesOn = opts?.diesOn ?? "TERM";
const provisions: Array<{ layers: WorkspaceLayer[]; opts?: ProvisionOptions }> = [];
const signals: Array<{ processId: string; signal: string }> = [];
@@ -26,8 +27,7 @@ function killSpySandbox(opts?: { diesOn?: "TERM" | "KILL" | "never"; vanished?:
return handle;
},
async readProcess(_h: SandboxHandle, processId: string) {
- if (opts?.vanished === "string-throw") throw `no such process session: ${processId}`;
- if (opts?.vanished) throw new Error(`no such process session: ${processId}`);
+ if (opts?.vanished) throw new ProcessSessionGoneError(processId);
return {
chunks: "",
cursor: 0,
@@ -337,23 +337,6 @@ test("a vanished process session counts as a confirmed kill", async () => {
assert.equal(teardowns[0]!.opts?.keepWarm, true, "the box is still released keep-warm");
});
-test("a non-Error 'no such process' rejection also counts as a confirmed kill", async () => {
- const reg = createMemoryProcessRegistry();
- await reg.register({
- processId: ID,
- scopeId: "s",
- sandboxId: "sandbox-1",
- kind: "background",
- command: "bg: x",
- ttlMs: -1,
- });
- const { sandbox } = killSpySandbox({ vanished: "string-throw" });
- const reaper = createProcessReaper(reg, { intervalMs: 60_000, kill: createReaperKillHook(sandbox, FAST_GRACE) });
-
- assert.equal((await reaper.sweep()).reaped, 1);
- assert.equal((await reg.get(ID))!.status, "reaped");
-});
-
test("an expired job with no recorded computer is retired without touching the scope's current computer", async () => {
const reg = createMemoryProcessRegistry();
await reg.register({