Multiplayer agent harness for work.
sandbox: classify Modal and process-session errors by type and state, not message text (#2440)
Regan Bellc7fa7e86fae5
diff --git a/src/api/app-sessions.ts b/src/api/app-sessions.tsindex fc2ee85e6..c209ecf31 100644--- a/src/api/app-sessions.ts+++ b/src/api/app-sessions.ts@@ -14,7 +14,7 @@ import { cancelGoal, goalCancelledPayload, openGoal } from "../harness/goal.ts";import { goalView } from "../runs/turn-stream.ts";import { SEARCH_HIT_LIMIT, entrySearchText, searchSnippet, searchTerms } from "../sessions/entry-search.ts";import { supportsProcessSessions } from "../sandbox/sandbox.ts";-import { processIsGone } from "../sandbox/process-poll.ts";+import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";import { cronIsActive, cronTiedTo } from "../cron/cron-store.ts";import { cronRef, deployRef, encodeRef, fileRef, parseRef, skillRef } from "../acl/resource-ref.ts";import { samePerson } from "../directory/person.ts";@@ -644,7 +644,7 @@ export function createSessionMethods(...(read.status.state === "exited" ? { exitCode: read.status.code } : {}),};} catch (e) {- if (processIsGone(e)) return { chunk: "", cursor: sinceCursor, state: "exited" };+ if (e instanceof ProcessSessionGoneError) return { chunk: "", cursor: sinceCursor, state: "exited" };throw e;} finally {await sandbox.teardown(handle, { keepWarm: true }).catch(() => {});diff --git a/src/connectors/background-exec-broker.ts b/src/connectors/background-exec-broker.tsindex 76477d25b..bf4c28dfc 100644--- a/src/connectors/background-exec-broker.ts+++ b/src/connectors/background-exec-broker.ts@@ -2,8 +2,8 @@ import type { ProcessSandbox, ProcessState, SandboxHandle } from "../sandbox/sanimport type { ProcessRegistry, ProcessStatus } from "../processes/process-registry.ts";import type { ProcessCredentialFiles } from "../credentials/execute-files.ts";import { awaitProcessExit } from "../sandbox/await-process-exit.ts";-import { pollProcess, processIsGone } from "../sandbox/process-poll.ts";-import { redactCommand } from "../sandbox/exec-process-session.ts";+import { pollProcess } from "../sandbox/process-poll.ts";+import { ProcessSessionGoneError, redactCommand } from "../sandbox/exec-process-session.ts";import { CONFIG_DEFAULTS } from "../config.ts";import { createExactSecretValueMasker } from "../security/secret-masking.ts";import { withCleanup } from "../util/errors.ts";@@ -149,7 +149,7 @@ export function createBackgroundBroker(deps: BackgroundExecBrokerDeps): BackgrouprocessId = null;}} catch (error) {- if (!processIsGone(error)) throw error;+ if (!(error instanceof ProcessSessionGoneError)) throw error;await deps.registry.delete(existingProcessId);processId = null;}diff --git a/src/monitors/monitor-poller.ts b/src/monitors/monitor-poller.tsindex c1991a325..28149639f 100644--- a/src/monitors/monitor-poller.ts+++ b/src/monitors/monitor-poller.ts@@ -6,7 +6,7 @@ import type { DeliveryStore } from "../delivery/delivery-store.ts";import type { IdempotencyStore } from "../idempotency/idempotency-store.ts";import type { IdentityService } from "../identity/identity-service.ts";import { supportsProcessSessions, type ProcessSandbox, type Sandbox, type SandboxHandle } from "../sandbox/sandbox.ts";-import { processIsGone } from "../sandbox/process-poll.ts";+import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";import { runTrigger, type TriggerDeps, type TriggerOutcome } from "../triggers/run-trigger.ts";import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";import { createSweeper } from "../util/sweeper.ts";@@ -189,7 +189,7 @@ export function createMonitorPoller(deps: MonitorPollerDeps): MonitorPoller {waitMs: 0,});} catch (e) {- if (processIsGone(e)) {+ if (e instanceof ProcessSessionGoneError) {await reportLost(m);return true;}diff --git a/src/processes/process-reaper.ts b/src/processes/process-reaper.tsindex eb023c015..6c01a0054 100644--- a/src/processes/process-reaper.ts+++ b/src/processes/process-reaper.ts@@ -2,7 +2,7 @@ import type { ProcessRecord, ProcessRegistry } from "./process-registry.ts";import { createSweeper } from "../util/sweeper.ts";import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";import { awaitProcessExit } from "../sandbox/await-process-exit.ts";-import { processIsGone } from "../sandbox/process-poll.ts";+import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";import type { ProcessSandbox, SandboxHandle } from "../sandbox/sandbox.ts";import { errMessage } from "../util/errors.ts";@@ -35,7 +35,7 @@ export function createReaperKillHook(}if (status.state !== "exited") throw new Error(`process survived TERM+KILL: ${rec.processId}`);} catch (e) {- if (!processIsGone(e)) throw e;+ if (!(e instanceof ProcessSessionGoneError)) throw e;}await opts?.onExit?.(handle, rec.processId)diff --git a/src/sandbox/exec-process-session.ts b/src/sandbox/exec-process-session.tsindex 1b8ea6347..ccad5c7d6 100644--- a/src/sandbox/exec-process-session.ts+++ b/src/sandbox/exec-process-session.ts@@ -24,6 +24,13 @@ export interface ExecProcessSessions {listProcesses(handle: SandboxHandle): Promise<ProcessSession[]>;}+export class ProcessSessionGoneError extends Error {+ constructor(processId: string) {+ super(`no such process session: ${processId}`);+ this.name = "ProcessSessionGoneError";+ }+}+const PROC_BASE = "${HOME:-/root}/.agent-proc";export const processSessionDir = (processId: string): string => `${PROC_BASE}/${processId}`;const REBOOT_RC = 137;@@ -133,7 +140,7 @@ export function createExecProcessSessions(io: ExecProcessIo): ExecProcessSession].join("\n");const r = await io.run(handle, script, { timeoutMs: waitMs + 20_000 });const out = r.stdout;- if (/MISSING=1/.test(out)) throw new Error(`no such process session: ${processId}`);+ if (/MISSING=1/.test(out)) throw new ProcessSessionGoneError(processId);const cur = /CURSOR=(\d+)/.exec(out);const st = /STATUS=(running|exited:-?\d+)/.exec(out);const dataIdx = out.indexOf("DATA=");diff --git a/src/sandbox/modal-client.ts b/src/sandbox/modal-client.tsindex 4be571e5f..42524b688 100644--- a/src/sandbox/modal-client.ts+++ b/src/sandbox/modal-client.ts@@ -1,6 +1,7 @@import { randomUUID } from "node:crypto";import { isIP } from "node:net";import { resolveModalImage } from "./modal-image.ts";+import { swallow } from "../util/errors.ts";export interface ModalCommandResult {stdout: string;@@ -79,46 +80,40 @@ function wholeSeconds(ms: number): number {return Math.max(1000, Math.ceil(ms / 1000) * 1000);}-function errName(err: unknown): string {- return String((err as Error)?.name ?? "");-}-function errText(err: unknown): string {return String((err as Error)?.message ?? err);}-function isFileNotFoundError(err: unknown): boolean {- return errName(err) === "SandboxFilesystemNotFoundError" || /no such file or directory/i.test(errText(err));-}+type ModalSdk = typeof import("modal");-function isFileTooLargeError(err: unknown): boolean {- return errName(err) === "SandboxFilesystemFileTooLargeError";-}+const GRPC_INVALID_ARGUMENT = 3;+const GRPC_NOT_FOUND = 5;-function isSandboxGoneError(err: unknown): boolean {- if (isFileNotFoundError(err)) return false;- const name = errName(err);- if (name === "NotFoundError" || name === "ClientClosedError") return true;- return /(sandbox|task).*(not found|has been terminated|already (terminated|finished|completed)|does not exist|is not running)|detached sandbox/i.test(- errText(err),- );+function grpcCode(err: unknown): unknown {+ return (err as { code?: unknown } | null)?.code;}-const GRPC_INVALID_ARGUMENT = 3;--function isSandboxIdLookupGone(err: unknown): boolean {- if (isSandboxGoneError(err)) return true;- if (errName(err) === "InvalidError") return true;- if ((err as { code?: unknown }).code === GRPC_INVALID_ARGUMENT) return true;- return /INVALID_ARGUMENT/.test(errText(err));+function isSandboxGoneError(sdk: ModalSdk, err: unknown): boolean {+ return err instanceof sdk.NotFoundError || err instanceof sdk.ClientClosedError || grpcCode(err) === GRPC_NOT_FOUND;}-function isNameConflictError(err: unknown): boolean {- return errName(err) === "AlreadyExistsError" || /already exists|already in use/i.test(errText(err));+function isSandboxIdLookupGone(sdk: ModalSdk, err: unknown): boolean {+ return isSandboxGoneError(sdk, err) || err instanceof sdk.InvalidError || grpcCode(err) === GRPC_INVALID_ARGUMENT;}-function isDeadlineError(err: unknown): boolean {- return /timeouterror$/i.test(errName(err)) || /deadline exceeded/i.test(errText(err));+async function hasExited(+ sdk: ModalSdk,+ sbx: { sandboxId: string; poll(): Promise<number | null> },+ err: unknown,+): Promise<boolean> {+ if (isSandboxGoneError(sdk, err)) return true;+ try {+ return (await sbx.poll()) !== null;+ } catch (pollErr) {+ if (isSandboxGoneError(sdk, pollErr)) return true;+ swallow(`modal-client: poll ${sbx.sandboxId} after a failed call`, pollErr);+ return false;+ }}export function modalEgressAllowlist(@@ -143,14 +138,14 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {throw new Error("Modal snapshot retention must be a positive finite duration");const snapshotTtlMs = wholeSeconds(snapshotRetentionMs);const network = opts.egressProxyUrl ? modalEgressAllowlist(opts.egressProxyUrl) : {};- type ModalSdk = typeof import("modal");type SdkClient = InstanceType<ModalSdk["ModalClient"]>;type SdkApp = Awaited<ReturnType<SdkClient["apps"]["fromName"]>>;type SdkImage = ReturnType<SdkClient["images"]["fromRegistry"]>;type SdkSandbox = Awaited<ReturnType<SdkClient["sandboxes"]["fromId"]>>;- let ctx: Promise<{ client: SdkClient; app: SdkApp; image: SdkImage }> | null = null;- const buildCtx = async (): Promise<{ client: SdkClient; app: SdkApp; image: SdkImage }> => {+ type Ctx = { sdk: ModalSdk; client: SdkClient; app: SdkApp; image: SdkImage };+ let ctx: Promise<Ctx> | null = null;+ const buildCtx = async (): Promise<Ctx> => {let sdk: ModalSdk;try {sdk = (await import("modal")) as unknown as ModalSdk;@@ -164,19 +159,20 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {});const app = await client.apps.fromName(opts.appName, { createIfMissing: true });const image = await resolveModalImage(client, opts.image);- return { client, app, image };+ return { sdk, client, app, image };};- const loadCtx = (): Promise<{ client: SdkClient; app: SdkApp; image: SdkImage }> =>+ const loadCtx = (): Promise<Ctx> =>(ctx ??= buildCtx().catch((e) => {ctx = null;throw e;}));- const wrap = (sbx: SdkSandbox): ModalSession => ({+ const wrap = (sdk: ModalSdk, sbx: SdkSandbox): ModalSession => ({sandboxId: sbx.sandboxId,async runCommand(command, runOpts): Promise<ModalCommandResult> {const commandMs = wholeSeconds(runOpts?.timeoutMs ?? maxCommandMs);const timeoutMs = commandMs + MODAL_EXEC_GRACE_MS;+ const deadlineAt = Date.now() + timeoutMs;try {let args = ["timeout", String(commandMs / 1000), "sh", "-c", command];if (Buffer.byteLength(command, "utf8") > MODAL_MAX_EXEC_ARG_BYTES) {@@ -192,8 +188,9 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {const [stdout, stderr, exitCode] = await Promise.all([p.stdout.readText(), p.stderr.readText(), p.wait()]);return { stdout, stderr, exitCode };} catch (err) {- if (isDeadlineError(err)) return { stdout: "", stderr: errText(err), exitCode: 124 };- if (isSandboxGoneError(err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));+ if (await hasExited(sdk, sbx, err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));+ if (err instanceof sdk.TimeoutError || Date.now() >= deadlineAt)+ return { stdout: "", stderr: errText(err), exitCode: 124 };throw err;}},@@ -201,13 +198,13 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {try {return await sbx.filesystem.readBytes(absPath);} catch (err) {- if (isSandboxGoneError(err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));- if (isFileNotFoundError(err)) return null;- if (isFileTooLargeError(err))+ if (err instanceof sdk.SandboxFilesystemNotFoundError) return null;+ if (err instanceof sdk.SandboxFilesystemFileTooLargeError)throw new Error(`modal read ${absPath}: the file exceeds Modal's filesystem read limit; move it with blob staging or split it`,{ cause: err },);+ if (await hasExited(sdk, sbx, err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));throw err;}},@@ -215,7 +212,7 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {try {await sbx.filesystem.writeBytes(data, absPath);} catch (err) {- if (isSandboxGoneError(err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));+ if (await hasExited(sdk, sbx, err)) throw new ModalSandboxGoneError(sbx.sandboxId, errText(err));throw err;}},@@ -230,7 +227,7 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {},async terminate(): Promise<void> {await sbx.terminate().catch((err) => {- if (!isSandboxGoneError(err)) throw err;+ if (!isSandboxGoneError(sdk, err)) throw err;});},});@@ -239,7 +236,7 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {lifetimeMs: sandboxTimeoutMs,nativeSnapshots: true,async create(createOpts): Promise<ModalSession> {- const { client, app, image } = await loadCtx();+ const { sdk, client, app, image } = await loadCtx();try {const sbx = await client.sandboxes.create(app, image, {...(createOpts.name ? { name: createOpts.name } : {}),@@ -251,66 +248,67 @@ export function createSdkModalClient(opts: SdkModalClientOptions): ModalClient {...(opts.regions?.length ? { regions: opts.regions } : {}),...network,});- return wrap(sbx);+ return wrap(sdk, sbx);} catch (err) {- if (isNameConflictError(err)) throw new ModalNameConflictError(createOpts.name ?? "(unnamed)", errText(err));+ if (err instanceof sdk.AlreadyExistsError)+ throw new ModalNameConflictError(createOpts.name ?? "(unnamed)", errText(err));throw err;}},async fromId(sandboxId): Promise<ModalSession> {- const { client } = await loadCtx();+ const { sdk, client } = await loadCtx();try {const sbx = await client.sandboxes.fromId(sandboxId);const exited = await sbx.poll();if (exited !== null) throw new ModalSandboxGoneError(sandboxId, `exited with code ${exited}`);- return wrap(sbx);+ return wrap(sdk, sbx);} catch (err) {if (err instanceof ModalSandboxGoneError) throw err;- if (isSandboxIdLookupGone(err)) throw new ModalSandboxGoneError(sandboxId, errText(err));+ if (isSandboxIdLookupGone(sdk, err)) throw new ModalSandboxGoneError(sandboxId, errText(err));throw err;}},async fromName(name): Promise<ModalSession | null> {- const { client } = await loadCtx();+ const { sdk, client } = await loadCtx();let sbx: SdkSandbox;try {sbx = await client.sandboxes.fromName(opts.appName, name);} catch (err) {- if (isSandboxGoneError(err)) return null;+ if (isSandboxGoneError(sdk, err)) return null;throw err;}try {const exited = await sbx.poll();if (exited !== null) return null;} catch (err) {- if (isSandboxGoneError(err)) return null;+ if (isSandboxGoneError(sdk, err)) return null;}- return wrap(sbx);+ return wrap(sdk, sbx);},async terminate(sandboxId): Promise<void> {- const { client } = await loadCtx();+ const { sdk, client } = await loadCtx();let sbx: SdkSandbox;try {sbx = await client.sandboxes.fromId(sandboxId);} catch (err) {- if (isSandboxIdLookupGone(err)) return;+ if (isSandboxIdLookupGone(sdk, err)) return;throw err;}try {await sbx.terminate();} catch (err) {- if (isSandboxGoneError(err)) return;+ if (isSandboxGoneError(sdk, err)) return;throw err;}},async *listRunning(tags): AsyncIterable<string> {- const { client, app } = await loadCtx();+ const { sdk, client, app } = await loadCtx();for await (const sbx of client.sandboxes.list({ appId: app.appId, tags })) {let exited: number | null;try {exited = await sbx.poll();} catch (err) {- if (isSandboxGoneError(err)) continue;+ if (isSandboxGoneError(sdk, err)) continue;throw err;}if (exited === null) yield sbx.sandboxId;diff --git a/src/sandbox/process-poll.ts b/src/sandbox/process-poll.tsindex 303096ea4..0bb12d29d 100644--- a/src/sandbox/process-poll.ts+++ b/src/sandbox/process-poll.ts@@ -1,9 +1,6 @@import type { ProcessSandbox, ProcessState, ReadProcessResult, SandboxHandle } from "./sandbox.ts";-import { errMessage } from "../util/errors.ts";import { sleep } from "../util/async.ts";-export const processIsGone = (e: unknown): boolean => /^no such process/i.test(errMessage(e));-export interface PollProcessOptions {deadlineMs: number;waitMs?: number;diff --git a/test/background-exec-broker.test.ts b/test/background-exec-broker.test.tsindex 6bdcd2344..9dc518c08 100644--- a/test/background-exec-broker.test.ts+++ b/test/background-exec-broker.test.ts@@ -4,6 +4,7 @@ import { createBackgroundBroker } from "../src/connectors/background-exec-brokerimport { createMemoryProcessRegistry } from "../src/processes/process-registry.ts";import { redactCommand } from "../src/sandbox/exec-process-session.ts";import type { ProcessSandbox, ProcessSession, SandboxHandle, StartProcessOptions } from "../src/sandbox/sandbox.ts";+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";const SCOPE = "personal:U1";const handle: SandboxHandle = { id: "vm", rootDir: "/workspace", homeDir: "/root", resourceId: "sandbox-1" };@@ -59,9 +60,9 @@ function fakeSandbox(opts?: { seedOutput?: string }) {readErrors.delete(id);throw error;}- if (missing.has(id)) throw new Error(`no such process session: ${id}`);+ if (missing.has(id)) throw new ProcessSessionGoneError(id);const p = procs.get(id);- if (!p) throw new Error(`no such process session: ${id}`);+ if (!p) throw new ProcessSessionGoneError(id);const cur = opts?.sinceCursor ?? 0;const maxBytes = opts?.maxBytes ?? 64 * 1024;const end = Math.min(p.output.length, cur + maxBytes);diff --git a/test/durable-process-sessions.test.ts b/test/durable-process-sessions.test.tsindex b40fc4d6a..e52e7cf6d 100644--- a/test/durable-process-sessions.test.ts+++ b/test/durable-process-sessions.test.ts@@ -24,6 +24,7 @@ import { createDeployService } from "../src/deploy/deploy-service.ts";import { writableMemoryScope } from "../src/memory/policy.ts";import type { Sandbox, ProcessSession, SandboxHandle } from "../src/sandbox/sandbox.ts";import type { Conversation, Principal } from "../src/types.ts";+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";const ORG = "default-org";const actor: Principal = { id: "U1", type: "internal" };@@ -77,7 +78,7 @@ function fakeProcessSandbox() {},async readProcess(_h, id, opts) {const p = procs.get(id);- if (!p) throw new Error(`no such process session: ${id}`);+ if (!p) throw new ProcessSessionGoneError(id);const cur = opts?.sinceCursor ?? 0;return {chunks: p.output.slice(cur),diff --git a/test/exec-process-session.test.ts b/test/exec-process-session.test.tsindex 9aba7af7b..01366c1c6 100644--- a/test/exec-process-session.test.ts+++ b/test/exec-process-session.test.ts@@ -4,7 +4,12 @@ import { spawn } from "node:child_process";import { mkdtempSync, writeFileSync, rmSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import { createExecProcessSessions, type ExecProcessIo, redactCommand } from "../src/sandbox/exec-process-session.ts";+import {+ createExecProcessSessions,+ type ExecProcessIo,+ ProcessSessionGoneError,+ redactCommand,+} from "../src/sandbox/exec-process-session.ts";import type { ExecResult, SandboxHandle } from "../src/sandbox/sandbox.ts";function shellIo(home: string): ExecProcessIo {@@ -138,7 +143,7 @@ test("signalProcess(KILL) records an exit even though the launcher trap can't catest("reading or writing an unknown process id is rejected", async () => {const { proc, handle } = fixture();const bogus = "00000000-0000-0000-0000-000000000000";- await assert.rejects(proc.readProcess(handle, bogus), /no such process/);+ await assert.rejects(proc.readProcess(handle, bogus), ProcessSessionGoneError);await assert.rejects(proc.writeStdin(handle, bogus, "x"), /writeStdin failed/);});diff --git a/test/monitor-poller.test.ts b/test/monitor-poller.test.tsindex cbd84df5c..b9f8274da 100644--- a/test/monitor-poller.test.ts+++ b/test/monitor-poller.test.ts@@ -9,6 +9,7 @@ import { createIdentityService } from "../src/identity/identity-service.ts";import type { LeaderLease } from "../src/persistence/leader-lease.ts";import { scopeId, type TurnRequest, type TurnResult } from "../src/types.ts";import type { ProcessSandbox, ProcessSession, SandboxHandle } from "../src/sandbox/sandbox.ts";+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";const SCOPE = "personal:U1";const handle: SandboxHandle = { id: "vm", rootDir: "/workspace", homeDir: "/root" };@@ -56,7 +57,7 @@ function fakeSandbox() {return { processId };},async readProcess(_h, id, opts) {- if (missing.has(id) || !procs.has(id)) throw new Error(`no such process session: ${id}`);+ if (missing.has(id) || !procs.has(id)) throw new ProcessSessionGoneError(id);const p = procs.get(id)!;const cur = opts?.sinceCursor ?? 0;const end = Math.min(p.output.length, cur + (opts?.maxBytes ?? 64 * 1024));diff --git a/test/native-provider-clients.test.ts b/test/native-provider-clients.test.tsindex 4e32a32c3..5f1bbab2a 100644--- a/test/native-provider-clients.test.ts+++ b/test/native-provider-clients.test.ts@@ -9,6 +9,8 @@ import { test, mock } from "node:test";import assert from "node:assert/strict";import {createSdkModalClient,+ ModalNameConflictError,+ ModalSandboxGoneError,modalEgressAllowlist,MODAL_EXEC_GRACE_MS,MODAL_MAX_EXEC_ARG_BYTES,@@ -26,12 +28,26 @@ const modalCreateParams: Record<string, unknown>[] = [];const modalExecs: { args: string[]; params: Record<string, unknown> }[] = [];const modalWrites: { path: string; bytes: number }[] = [];const modalRunning = new Map<string, boolean>();-class SandboxFilesystemFileTooLargeError extends Error {- override name = "SandboxFilesystemFileTooLargeError";-}+class SandboxFilesystemFileTooLargeError extends Error {}+class ModalSandboxFilesystemNotFoundError extends Error {}+class ModalNotFoundError extends Error {}+class ModalClientClosedError extends Error {}+class ModalAlreadyExistsError extends Error {}+class ModalInvalidError extends Error {}+class ModalTimeoutError extends Error {}+let modalFault: { exec?: Error; read?: Error; write?: Error; create?: Error; terminate?: Error } = {};+let modalPoll: number | null | Error = null;const modalSandbox = {sandboxId: "sb-native",+ async poll() {+ if (modalPoll instanceof Error) throw modalPoll;+ return modalPoll;+ },+ async terminate() {+ if (modalFault.terminate) throw modalFault.terminate;+ },async exec(args: string[], params: Record<string, unknown>) {+ if (modalFault.exec) throw modalFault.exec;if (typeof params.timeoutMs === "number" && params.timeoutMs % 1000 !== 0)throw new Error(`timeoutMs must be a multiple of 1000ms, got ${params.timeoutMs}`);modalExecs.push({ args, params });@@ -39,9 +55,11 @@ const modalSandbox = {},filesystem: {async writeBytes(data: Uint8Array, path: string) {+ if (modalFault.write) throw modalFault.write;modalWrites.push({ path, bytes: data.byteLength });},async readBytes(path: string) {+ if (modalFault.read) throw modalFault.read;if (path.endsWith("huge.bin")) throw new SandboxFilesystemFileTooLargeError("file too large");return new Uint8Array([1]);},@@ -56,14 +74,23 @@ const modalSandbox = {};mock.module("modal", {namedExports: {+ SandboxFilesystemFileTooLargeError,+ SandboxFilesystemNotFoundError: ModalSandboxFilesystemNotFoundError,+ NotFoundError: ModalNotFoundError,+ ClientClosedError: ModalClientClosedError,+ AlreadyExistsError: ModalAlreadyExistsError,+ InvalidError: ModalInvalidError,+ TimeoutError: ModalTimeoutError,ModalClient: class {apps = { fromName: async () => ({ appId: "ap-1" }) };images = { fromRegistry: () => ({}), fromId: async (imageId: string) => ({ imageId }) };sandboxes = {create: async (_app: unknown, _image: unknown, params: Record<string, unknown>) => {+ if (modalFault.create) throw modalFault.create;modalCreateParams.push(params);return modalSandbox;},+ fromId: async () => modalSandbox,list: async function* (params: { appId?: string; tags?: Record<string, string> }) {modalCalls.push(["list", params]);for (const [sandboxId, running] of modalRunning) yield { sandboxId, poll: async () => (running ? null : 0) };@@ -291,6 +318,53 @@ test("Modal lists running tagged sandboxes within the app and reports oversizedawait assert.rejects(session.readFileBytes("/root/huge.bin"), /exceeds Modal's filesystem read limit/);});+test("Modal classifies gone sandboxes, missing files, deadlines and name conflicts by SDK error class and sandbox state", async (t) => {+ t.after(() => {+ modalFault = {};+ modalPoll = null;+ });+ const client = createSdkModalClient({ tokenId: "id", tokenSecret: "secret", appName: "test", image: "ubuntu" });+ const session = await client.create({});+ const plain = new Error("unrecognized failure");+ modalFault = { exec: plain, read: plain, write: plain };+ await assert.rejects(session.runCommand("true"), (e) => e === plain, "a running sandbox keeps the original error");+ await assert.rejects(session.readFileBytes("/root/a"), (e) => e === plain);+ await assert.rejects(session.writeFileBytes("/root/a", new Uint8Array()), (e) => e === plain);+ modalPoll = 0;+ await assert.rejects(session.runCommand("true"), ModalSandboxGoneError, "an exited sandbox is gone");+ await assert.rejects(session.readFileBytes("/root/a"), ModalSandboxGoneError);+ await assert.rejects(session.writeFileBytes("/root/a", new Uint8Array()), ModalSandboxGoneError);+ modalPoll = new ModalNotFoundError("gone");+ await assert.rejects(session.runCommand("true"), ModalSandboxGoneError, "a sandbox that cannot be polled is gone");+ modalPoll = null;+ for (const err of [+ new ModalNotFoundError("x"),+ new ModalClientClosedError("x"),+ Object.assign(new Error("x"), { code: 5 }),+ ]) {+ modalFault = { exec: err };+ await assert.rejects(session.runCommand("true"), ModalSandboxGoneError);+ }+ modalFault = { exec: new ModalTimeoutError("x") };+ assert.equal((await session.runCommand("true")).exitCode, 124);+ modalPoll = 137;+ await assert.rejects(+ session.runCommand("true"),+ ModalSandboxGoneError,+ "a sandbox that died at the deadline is gone",+ );+ modalPoll = null;+ modalFault = { read: new ModalSandboxFilesystemNotFoundError("x") };+ assert.equal(await session.readFileBytes("/root/missing"), null);+ modalFault = { create: new ModalAlreadyExistsError("x") };+ await assert.rejects(client.create({ name: "taken" }), ModalNameConflictError);+ modalFault = { terminate: Object.assign(new Error("x"), { code: 5 }) };+ await session.terminate();+ await client.terminate("sb-native");+ modalPoll = new ModalInvalidError("malformed sandbox id");+ await assert.rejects(client.fromId("not-a-sandbox"), ModalSandboxGoneError, "an unknown id is a gone sandbox");+});+test("Modal rejects invalid checkpoint retention", () => {for (const snapshotRetentionMs of [0, -1, Infinity, NaN]) {assert.throws(diff --git a/test/process-poll.test.ts b/test/process-poll.test.tsindex 0e3b8854e..9349f29b8 100644--- a/test/process-poll.test.ts+++ b/test/process-poll.test.ts@@ -1,6 +1,6 @@import { test } from "node:test";import assert from "node:assert/strict";-import { pollProcess, processIsGone } from "../src/sandbox/process-poll.ts";+import { pollProcess } from "../src/sandbox/process-poll.ts";import type { ProcessSandbox, ReadProcessResult, SandboxHandle } from "../src/sandbox/sandbox.ts";const handle: SandboxHandle = { id: "vm", rootDir: "/workspace" };@@ -23,15 +23,6 @@ const running = (chunks: string, cursor: number): ReadProcessResult => ({status: { state: "running" },});-test("processIsGone: the bare-string sentinel counts as gone", () => {- assert.equal(processIsGone("no such process session: p-1"), true);- assert.equal(processIsGone(new Error("no such process session: p-1")), true);-});--test("processIsGone: an embedded phrase in a wrapped error does NOT confirm a kill", () => {- assert.equal(processIsGone(new Error("fly api 500: upstream said 'no such process' while restarting")), false);-});-test("pollProcess stops when `until` fires, returning the output and cursor so far", async () => {const { sandbox, callCount } = fakeSandbox([running("a", 1), running("b", 2), running("c", 3)]);const res = await pollProcess(sandbox, handle, "p", {diff --git a/test/process-reaper.test.ts b/test/process-reaper.test.tsindex 27ab8b3a6..29d645f24 100644--- a/test/process-reaper.test.ts+++ b/test/process-reaper.test.ts@@ -5,6 +5,7 @@ import { createProcessReaper, createReaperKillHook } from "../src/processes/procimport type { LeaderLease } from "../src/persistence/leader-lease.ts";import type { ProcessSandbox, SandboxHandle, ProvisionOptions, TeardownOptions } from "../src/sandbox/sandbox.ts";import type { WorkspaceLayer } from "../src/types.ts";+import { ProcessSessionGoneError } from "../src/sandbox/exec-process-session.ts";const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));@@ -12,7 +13,7 @@ const ID = "00000000-0000-0000-0000-000000000abc";const FAST_GRACE = { termGraceMs: 20, killGraceMs: 20 };-function killSpySandbox(opts?: { diesOn?: "TERM" | "KILL" | "never"; vanished?: boolean | "string-throw" }) {+function killSpySandbox(opts?: { diesOn?: "TERM" | "KILL" | "never"; vanished?: boolean }) {const diesOn = opts?.diesOn ?? "TERM";const provisions: Array<{ layers: WorkspaceLayer[]; opts?: ProvisionOptions }> = [];const signals: Array<{ processId: string; signal: string }> = [];@@ -26,8 +27,7 @@ function killSpySandbox(opts?: { diesOn?: "TERM" | "KILL" | "never"; vanished?:return handle;},async readProcess(_h: SandboxHandle, processId: string) {- if (opts?.vanished === "string-throw") throw `no such process session: ${processId}`;- if (opts?.vanished) throw new Error(`no such process session: ${processId}`);+ if (opts?.vanished) throw new ProcessSessionGoneError(processId);return {chunks: "",cursor: 0,@@ -337,23 +337,6 @@ test("a vanished process session counts as a confirmed kill", async () => {assert.equal(teardowns[0]!.opts?.keepWarm, true, "the box is still released keep-warm");});-test("a non-Error 'no such process' rejection also counts as a confirmed kill", async () => {- const reg = createMemoryProcessRegistry();- await reg.register({- processId: ID,- scopeId: "s",- sandboxId: "sandbox-1",- kind: "background",- command: "bg: x",- ttlMs: -1,- });- const { sandbox } = killSpySandbox({ vanished: "string-throw" });- const reaper = createProcessReaper(reg, { intervalMs: 60_000, kill: createReaperKillHook(sandbox, FAST_GRACE) });-- assert.equal((await reaper.sweep()).reaped, 1);- assert.equal((await reg.get(ID))!.status, "reaped");-});-test("an expired job with no recorded computer is retired without touching the scope's current computer", async () => {const reg = createMemoryProcessRegistry();await reg.register({