Multiplayer agent harness for work.
import { cleanupFailedProvision } from "./sandbox.ts";
import { randomUUID } from "node:crypto";
import type { WorkspaceLayer } from "../types.ts";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
import type { AdvisoryLock } from "../persistence/advisory-lock.ts";
import type { DurableMap } from "../persistence/durable-map.ts";
import { orgId as configOrgId } from "../config.ts";
import { createKeyedQueue } from "../util/async.ts";
import { swallowAs, errMessage } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, forceThroughProxyEnv } from "./sandbox-env.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
import { materializeRoLayers } from "./ro-layers.ts";
import { withConnectorSdk, type ConnectorSdkBundle } from "./connector-sdk.ts";
import { createLayerToolInstaller } from "./layer-tool-install.ts";
import type { LayerInstallFile } from "../deployment/load-layer.ts";
import {
createBackendBlobStaging,
createExecExport,
createExecFileOps,
posixJoin,
type BlobStagingOptions,
} from "./exec-file-ops.ts";
import {
ephemeralCredLinkScript,
ephemeralCredLinkPaths,
type CredentialPathSpec,
} from "../credentials/resident-paths.ts";
import { killableScript, killScript } from "./exec-kill.ts";
import { visibleNotInstalled, visibleTools } from "./sandbox.ts";
import { sandboxScopeName } from "./exec-sandbox-base.ts";
import { ModalNameConflictError, ModalSandboxGoneError, type ModalClient, type ModalSession } from "./modal-client.ts";
import { createHomeSnapshotOps, HOME_SNAPSHOT_PRUNE, snapshotDue, type HomeSnapshotStore } from "./home-snapshot.ts";
import type {
AgentComputerProfile,
ComputerStatus,
ExecOptions,
ExecResult,
ProvisionOptions,
Sandbox,
SandboxHandle,
TeardownOptions,
} from "./sandbox.ts";
const HOME_DIR = "/root";
const WORKSPACE_BASENAME = "workspace";
const RO_LAYERS_TAR = ".ro-layers.tar";
const RO_LAYERS_MANIFEST = ".ro-layers.manifest";
const HOME_TAR = `${HOME_DIR}/.qm-home.tar`;
const HYDRATED_MARKER = `${HOME_DIR}/.qm-hydrated`;
const ACTIVITY_TOUCH_INTERVAL_MS = 10 * 60_000;
const swallowGone = (error: unknown): void => {
if (!(error instanceof ModalSandboxGoneError)) throw error;
};
const SNAPSHOT_PRUNE = ["./.qm-hydrated", ...HOME_SNAPSHOT_PRUNE];
export interface StoredModalSandbox {
sandboxId: string;
hydrationPending?: boolean;
nativeSnapshotId?: string;
nativeSnapshotExpiresAtMs?: number;
expiresAtMs?: number;
recoveryError?: string;
snapshotGeneration?: number;
createdAtMs: number;
lastSnapshotMs?: number;
lastSnapshotAttemptMs?: number;
lastActivityMs?: number;
homeDirty?: boolean;
orgId?: string;
}
export interface ModalSandboxOptions extends BlobStagingOptions {
client: ModalClient;
advisoryLock: AdvisoryLock;
namePrefix?: string;
defaultTimeoutSec?: number;
snapshotIntervalMs?: number;
nativeSnapshotsEnabled?: boolean;
nativeSnapshotIntervalMs?: number;
rotateAfterMs?: number;
reapIdleMs?: number;
egressProxyUrl?: string;
extraTools?: string[];
credentialPaths?: CredentialPathSpec[];
connectorSdk?: () => Promise<ConnectorSdkBundle>;
layerToolFiles?: () => readonly LayerInstallFile[];
fileChunkBytes?: number;
rotationHoldMs?: number;
lifetimeMarginMs?: number;
checkpointRenewMarginMs?: number;
orphanGraceMs?: number;
store: DurableMap<StoredModalSandbox>;
snapshots?: HomeSnapshotStore;
onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;
}
export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxOptions): Sandbox {
const client = opts.client;
const prefix = opts.namePrefix ?? "qm";
const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;
const usesNativeSnapshots = (stored?: StoredModalSandbox | null): boolean =>
!!stored?.nativeSnapshotId || !!(client.nativeSnapshots && opts.nativeSnapshotsEnabled);
const snapshotIntervalMs = (stored?: StoredModalSandbox | null): number =>
usesNativeSnapshots(stored) ? (opts.nativeSnapshotIntervalMs ?? 5 * 60_000) : (opts.snapshotIntervalMs ?? 0);
const lastSnapshotAttemptMs = (stored?: StoredModalSandbox | null): number =>
Math.max(stored?.lastSnapshotMs ?? 0, usesNativeSnapshots(stored) ? (stored?.lastSnapshotAttemptMs ?? 0) : 0);
const rotateAfterMs = opts.rotateAfterMs ?? 20 * 3600_000;
const reapIdleMs = opts.reapIdleMs ?? 6 * 3600_000;
const fileChunkBytes = opts.fileChunkBytes ?? 64 * 1024 * 1024;
const rotationHoldMs = opts.rotationHoldMs ?? 10 * 60_000;
const lifetimeMarginMs = opts.lifetimeMarginMs ?? 30 * 60_000;
const checkpointRenewMarginMs = opts.checkpointRenewMarginMs ?? 7 * 24 * 3600_000;
const orphanGraceMs = opts.orphanGraceMs ?? 10 * 60_000;
const rotationHoldUntil = new Map<string, number>();
const orphanFirstSeenMs = new Map<string, number>();
const tags = (kind: "scope" | "scratch"): Record<string, string> => ({
"qm-org": configOrgId(),
"qm-prefix": prefix,
"qm-kind": kind,
});
const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`;
const store = opts.store;
const localQueue = createKeyedQueue<string>();
const advisoryLock = opts.advisoryLock;
const provisionQueue = <T>(scope: string, action: () => Promise<T>): Promise<T> =>
localQueue(scope, () => advisoryLock.withLock(`modal-provision:${scope}`, action));
const sessionByName = new Map<string, ModalSession>();
const scopeByName = new Map<string, string>();
const scratchKeyByName = new Map<string, string>();
const activeScratch = new Map<string, number>();
const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {
opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });
};
const homeSnapshots = opts.snapshots
? createHomeSnapshotOps<ModalSession>({
label: "modal",
homeDir: HOME_DIR,
homeTarPath: HOME_TAR,
prunePaths: [
...SNAPSHOT_PRUNE,
...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`),
],
store: opts.snapshots,
io: {
runCommand: (session, script, timeoutMs) => session.runCommand(script, { timeoutMs }),
readFileBytes: (session, abs) => session.readFileBytes(abs),
writeFileBytes: (session, abs, data) => session.writeFileBytes(abs, data),
},
partBytes: fileChunkBytes,
})
: undefined;
async function assertHydrated(session: ModalSession): Promise<void> {
const marked = await session.runCommand(`test -f ${shq(HYDRATED_MARKER)}`, { timeoutMs: 30_000 });
if (marked.exitCode !== 0)
throw new Error(
"modal snapshot refused: this sandbox never finished hydrating, its home is not a trustworthy source",
);
}
async function snapshotHome(scope: string, session: ModalSession): Promise<void> {
await assertHydrated(session);
if (usesNativeSnapshots(await store.get(scope))) {
if (!session.snapshotHome) throw new Error("native Modal home checkpoint is not supported by this client");
if (!store.update) throw new Error("native Modal checkpoints require an atomic durable store");
const requested = await store.update(scope, (current) => {
if (current.sandboxId !== session.sandboxId) throw new Error("Modal checkpoint source has been replaced");
return {
...current,
snapshotGeneration: (current.snapshotGeneration ?? 0) + 1,
lastSnapshotAttemptMs: Date.now(),
};
});
if (!requested) throw new Error("Modal checkpoint source is no longer tracked");
const capturedAtMs = Date.now();
try {
const snapshot = await session.snapshotHome();
const committed = await store.update(scope, (current) =>
current.sandboxId === session.sandboxId && current.snapshotGeneration === requested.snapshotGeneration
? {
...current,
nativeSnapshotId: snapshot.imageId,
nativeSnapshotExpiresAtMs: snapshot.expiresAtMs,
recoveryError: undefined,
lastSnapshotAttemptMs: undefined,
lastSnapshotMs: capturedAtMs,
homeDirty: false,
}
: current,
);
if (committed?.nativeSnapshotId !== snapshot.imageId)
throw new Error("Modal checkpoint was superseded; keeping the source running");
} catch (error) {
await store.update(scope, (current) =>
current.sandboxId === session.sandboxId && current.snapshotGeneration === requested.snapshotGeneration
? { ...current, recoveryError: errMessage(error) }
: current,
);
throw error;
}
} else if (homeSnapshots) {
await homeSnapshots.snapshotHome(scope, session);
await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false });
}
}
async function createHydrated(scope: string, name: string): Promise<{ session: ModalSession; coldStart: boolean }> {
const previous = await store.get(scope);
if (previous?.nativeSnapshotExpiresAtMs && previous.nativeSnapshotExpiresAtMs <= Date.now()) {
const recoveryError =
"Modal home checkpoint has expired; explicitly import a recovery snapshot before creating a replacement";
await store.merge(scope, { recoveryError });
throw new Error(recoveryError);
}
let session: ModalSession;
try {
session = await client.create({ name, tags: tags("scope") });
} catch (err) {
if (!(err instanceof ModalNameConflictError)) throw err;
const adopted = await client.fromName(name);
if (!adopted) throw err;
if ((await store.get(scope))?.hydrationPending)
throw new Error("Modal home hydration is incomplete; refusing to adopt the replacement", { cause: err });
await assertHydrated(adopted);
sessionByName.set(name, adopted);
const merged = await store.merge(scope, { sandboxId: adopted.sandboxId, lastActivityMs: Date.now() });
if (!merged) {
await store.put(scope, {
sandboxId: adopted.sandboxId,
createdAtMs: Date.now(),
lastActivityMs: Date.now(),
orgId: configOrgId(),
});
}
return { session: adopted, coldStart: false };
}
await store.put(scope, {
...previous,
hydrationPending: true,
lastSnapshotAttemptMs: undefined,
...(client.lifetimeMs ? { expiresAtMs: Date.now() + client.lifetimeMs } : {}),
sandboxId: session.sandboxId,
createdAtMs: Date.now(),
lastActivityMs: Date.now(),
orgId: configOrgId(),
});
const terminateOrphan = (): Promise<unknown> =>
session.terminate().then(
() => store.merge(scope, { hydrationPending: false }),
(err: unknown) => reportError("sandbox_hydrate", "orphan_terminate_failed", errMessage(err), scope),
);
let hydrated: boolean;
try {
if (previous?.nativeSnapshotId) {
if (!session.restoreHome) throw new Error("native Modal home restore is not supported by this client");
await session.restoreHome(previous.nativeSnapshotId);
hydrated = true;
} else {
hydrated = homeSnapshots ? await homeSnapshots.hydrateHome(scope, session) : false;
}
} catch (e) {
await store.merge(scope, { recoveryError: errMessage(e) });
reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);
sessionByName.delete(name);
await terminateOrphan();
throw new Error(`modal provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`, {
cause: e,
});
}
const marked = await session.runCommand(`touch ${shq(HYDRATED_MARKER)}`, { timeoutMs: 30_000 });
if (marked.exitCode !== 0) {
sessionByName.delete(name);
await terminateOrphan();
throw new Error(`modal provision: could not mark the sandbox hydrated: ${marked.stderr.slice(0, 200)}`);
}
await store.merge(scope, { hydrationPending: false });
sessionByName.set(name, session);
if (hydrated && !previous?.nativeSnapshotId) await store.merge(scope, { lastSnapshotMs: Date.now() });
return { session, coldStart: !hydrated };
}
async function ensureSession(
scope: string,
name: string,
onStatus?: (text: string) => void,
allowRotation = true,
): Promise<{ session: ModalSession; coldStart: boolean }> {
return provisionQueue(scope, async () => {
const adopt = async (session: ModalSession): Promise<{ session: ModalSession; coldStart: boolean }> => {
await assertHydrated(session);
sessionByName.set(name, session);
await store.merge(scope, { lastActivityMs: Date.now() });
return { session, coldStart: false };
};
const stored = await store.get(scope);
if (stored?.hydrationPending)
throw new Error(
"Modal home hydration was interrupted; use computer restart to discard the incomplete replacement and retry the retained checkpoint",
);
const stale =
allowRotation &&
!!stored &&
(!!homeSnapshots || usesNativeSnapshots(stored)) &&
Date.now() - stored.createdAtMs > rotateAfterMs &&
Date.now() >= (rotationHoldUntil.get(scope) ?? 0);
const cached = sessionByName.get(name);
if (cached && !stale) return { session: cached, coldStart: false };
if (stored && !stale) {
try {
return await adopt(await client.fromId(stored.sandboxId));
} catch (err) {
if (!(err instanceof ModalSandboxGoneError)) throw err;
}
const found = await client.fromName(name);
if (found) {
await store.merge(scope, { sandboxId: found.sandboxId });
return adopt(found);
}
}
if (stored && stale) {
let session: ModalSession | null = null;
try {
session = cached ?? (await client.fromId(stored.sandboxId));
} catch (err) {
if (!(err instanceof ModalSandboxGoneError)) throw err;
}
if (session) {
await assertHydrated(session);
try {
await snapshotHome(scope, session);
} catch (e) {
if (!(e instanceof ModalSandboxGoneError)) {
reportError("sandbox_snapshot", "rotate_snapshot_failed", errMessage(e), scope);
rotationHoldUntil.set(scope, Date.now() + rotationHoldMs);
sessionByName.set(name, session);
await store.merge(scope, { lastActivityMs: Date.now() });
return { session, coldStart: false };
}
}
try {
await session.terminate();
} catch (e) {
if (!(e instanceof ModalSandboxGoneError)) {
reportError("sandbox_teardown", "rotate_terminate_failed", errMessage(e), scope);
rotationHoldUntil.set(scope, Date.now() + rotationHoldMs);
sessionByName.set(name, session);
await store.merge(scope, { lastActivityMs: Date.now() });
return { session, coldStart: false };
}
}
}
sessionByName.delete(name);
if (!(await store.get(scope))?.nativeSnapshotId)
await store.delete(scope).catch(swallowAs("modal-sandbox: forget retired sandbox", undefined));
}
if (!stored) {
const found = await client.fromName(name);
if (found) {
await assertHydrated(found);
await store.put(scope, {
sandboxId: found.sandboxId,
createdAtMs: Date.now(),
lastActivityMs: Date.now(),
orgId: configOrgId(),
});
sessionByName.set(name, found);
return { session: found, coldStart: false };
}
}
onStatus?.("Creating the sandbox…");
return createHydrated(scope, name);
});
}
async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> {
const name = sandboxScopeName(`${prefix}-scratch`, key);
return provisionQueue(`scratch:${key}`, async () => {
scratchKeyByName.set(name, key);
const active = activeScratch.get(name) ?? 0;
if (active === 0 && !sessionByName.has(name)) {
const session = await client.create({ tags: tags("scratch") });
sessionByName.set(name, session);
}
activeScratch.set(name, active + 1);
return { name, coldStart: active === 0 };
});
}
async function withSessionUnlocked<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {
const scratchKey = scratchKeyByName.get(name);
if (scratchKey === undefined && !scopeByName.has(name)) throw new Error("sandbox handle has been released");
const reviveScratch = async (): Promise<ModalSession> => {
const session = await client.create({ tags: tags("scratch") });
sessionByName.set(name, session);
return session;
};
const first =
scratchKey !== undefined
? { session: sessionByName.get(name) ?? (await reviveScratch()) }
: {
session:
sessionByName.get(name) ??
(await ensureSession(scopeByName.get(name) ?? "default", name, undefined, false)).session,
};
try {
return await action(first.session);
} catch (err) {
if (!(err instanceof ModalSandboxGoneError)) throw err;
sessionByName.delete(name);
const second =
scratchKey !== undefined
? { session: await reviveScratch() }
: await ensureSession(scopeByName.get(name) ?? "default", name, undefined, false);
return action(second.session);
}
}
function withSession<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {
if (scratchKeyByName.has(name)) return withSessionUnlocked(name, action);
const shared = advisoryLock.withSharedLock ?? advisoryLock.withLock;
return shared(`modal-use:${scopeByName.get(name) ?? "default"}`, () => withSessionUnlocked(name, action));
}
const lastTouchMs = new Map<string, number>();
async function touchActivity(name: string): Promise<void> {
const scope = scopeByName.get(name);
if (!scope || scratchKeyByName.has(name)) return;
const now = Date.now();
if (now - (lastTouchMs.get(scope) ?? 0) < ACTIVITY_TOUCH_INTERVAL_MS) return;
lastTouchMs.set(scope, now);
await store.merge(scope, { lastActivityMs: now }).catch(swallowAs("modal-sandbox: record activity", undefined));
}
async function runTimed(
session: ModalSession,
script: string,
timeoutSec: number,
env?: Record<string, string>,
): Promise<ExecResult> {
const r = await session.runCommand(script, {
timeoutMs: timeoutSec * 1000,
...(env ? { env } : {}),
});
return { stdout: r.stdout, stderr: r.stderr, code: r.exitCode, timedOut: r.exitCode === 124 };
}
async function execRaw(
name: string,
script: string,
timeoutSec: number,
env?: Record<string, string>,
): Promise<ExecResult> {
await touchActivity(name);
return withSession(name, (session) => runTimed(session, script, timeoutSec, env));
}
const profile: AgentComputerProfile = {
backend: "modal",
writablePersistence: usesNativeSnapshots() ? "provider_managed" : "snapshot_to_workspace",
processSessions: true,
egressEnforcement: opts.egressProxyUrl ? "domain" : "none",
spec: {
os: "Debian 12 — Modal sandbox (24h max lifetime; home checkpoints have limited retention; publish durable work to git or Files)",
runtimes: ["Node 24", "Python 3"],
get tools() {
return visibleTools(["git", "curl", "jq", "tar", "python3", ...(opts.extraTools ?? [])]);
},
get notInstalled() {
return visibleNotInstalled(["gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], opts.extraTools ?? []);
},
homeDir: HOME_DIR,
workdir: workspaceDir,
},
};
const procIo: ExecProcessIo = {
async run(handle, command, execOpts): Promise<ExecResult> {
const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
return execRaw(handle.id, command, timeoutSec);
},
};
const procSessions = createExecProcessSessions(procIo);
const directProcIo = (session: ModalSession): ExecProcessIo => ({
run(_handle, command, execOpts): Promise<ExecResult> {
const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
return runTimed(session, command, timeoutSec);
},
});
const writeAbsBytes = (name: string, absPath: string, data: Uint8Array): Promise<void> =>
withSession(name, (session) => session.writeFileBytes(absPath, data));
const readAbsBytes = (name: string, absPath: string): Promise<Uint8Array | null> =>
withSession(name, (session) => session.readFileBytes(absPath));
const installLayerTools = withConnectorSdk(
HOME_DIR,
createLayerToolInstaller(opts.layerToolFiles ?? (() => [])),
opts.connectorSdk,
);
const execFileOps = createExecFileOps({
combineRemoveAndList: true,
label: "modal",
exec: (id, script, t) => execRaw(id, script, t),
writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),
});
const execExport = createExecExport({
label: "modal",
exec: (id, script, t) => execRaw(id, script, t),
readAbsBytes,
defaultHomeDir: HOME_DIR,
ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),
});
const blobStaging = createBackendBlobStaging("modal", (id, script, t) => execRaw(id, script, t), opts);
async function destroyStoredScope(scope: string): Promise<void> {
const name = sandboxScopeName(prefix, scope);
const stored = await store.get(scope);
const cached = sessionByName.get(name);
const ids = new Set([stored?.sandboxId, cached?.sandboxId].filter((id): id is string => !!id));
for (const id of ids) {
try {
await client.terminate(id);
} catch (error) {
if (!(error instanceof ModalSandboxGoneError)) throw error;
}
}
await store.delete(scope);
sessionByName.delete(name);
scopeByName.delete(name);
rotationHoldUntil.delete(scope);
lastTouchMs.delete(scope);
}
function checkpointRenewalDue(rec: StoredModalSandbox, now: number): boolean {
if (!rec.nativeSnapshotId || !rec.nativeSnapshotExpiresAtMs) return false;
const remaining = rec.nativeSnapshotExpiresAtMs - now;
const lifetime = rec.nativeSnapshotExpiresAtMs - (rec.lastSnapshotMs ?? 0);
return remaining > 0 && remaining <= Math.min(checkpointRenewMarginMs, lifetime / 2);
}
function renewCheckpoint(scope: string, name: string, rec: StoredModalSandbox): Promise<void> {
return advisoryLock.withLock(`modal-use:${scope}`, () =>
provisionQueue(scope, async () => {
const current = await store.get(scope);
if (
!current ||
current.hydrationPending ||
current.nativeSnapshotId !== rec.nativeSnapshotId ||
!checkpointRenewalDue(current, Date.now())
)
return;
try {
const { session } = await createHydrated(scope, name);
try {
await snapshotHome(scope, session);
} finally {
sessionByName.delete(name);
await session.terminate().catch(swallowGone);
}
await store.merge(scope, { lastActivityMs: current.lastActivityMs });
} catch (e) {
if (!(e instanceof ModalSandboxGoneError))
reportError("sandbox_snapshot", "checkpoint_renewal_failed", errMessage(e), scope);
}
}),
);
}
function refreshCheckpoint(scope: string, rec: StoredModalSandbox, session: ModalSession): Promise<void> {
return provisionQueue(scope, async () => {
try {
const current = await store.get(scope);
if (
!current ||
current.sandboxId !== rec.sandboxId ||
Date.now() - lastSnapshotAttemptMs(current) <= snapshotIntervalMs(current)
)
return;
await snapshotHome(scope, session);
} catch (error) {
if (!(error instanceof ModalSandboxGoneError))
reportError("sandbox_snapshot", "periodic_snapshot_failed", errMessage(error), scope);
}
});
}
function retire(
scope: string,
name: string,
rec: StoredModalSandbox,
session: ModalSession,
idleCutoffMs?: number,
): Promise<number> {
return advisoryLock.withLock(`modal-use:${scope}`, () =>
provisionQueue(scope, async (): Promise<number> => {
const current = await store.get(scope);
if (!current || current.sandboxId !== rec.sandboxId) return 0;
if (idleCutoffMs !== undefined && (!current.lastActivityMs || current.lastActivityMs > idleCutoffMs)) return 0;
try {
await snapshotHome(scope, session);
await session.terminate();
sessionByName.delete(name);
if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope);
return 1;
} catch (e) {
if (e instanceof ModalSandboxGoneError) {
sessionByName.delete(name);
if (!(await store.get(scope))?.nativeSnapshotId)
await store.delete(scope).catch(swallowAs("modal-sandbox: forget retired sandbox", undefined));
} else {
reportError("sandbox_reap", "deep_idle_reap_failed", errMessage(e), scope);
}
return 0;
}
}),
);
}
async function sweepOrphans(now: number): Promise<number> {
if (!client.listRunning) return 0;
const tracked = async (): Promise<Set<string>> => {
const ids = new Set<string>();
for (const [, rec] of await store.entries()) ids.add(rec.sandboxId);
for (const session of sessionByName.values()) ids.add(session.sandboxId);
return ids;
};
let known = await tracked();
const listed = new Set<string>();
const candidates: string[] = [];
try {
for await (const sandboxId of client.listRunning(tags("scope"))) {
listed.add(sandboxId);
if (known.has(sandboxId)) continue;
const firstSeen = orphanFirstSeenMs.get(sandboxId) ?? now;
orphanFirstSeenMs.set(sandboxId, firstSeen);
if (now - firstSeen >= orphanGraceMs) candidates.push(sandboxId);
}
} catch (e) {
reportError("sandbox_reap", "orphan_list_failed", errMessage(e));
return 0;
}
for (const id of orphanFirstSeenMs.keys()) if (!listed.has(id) || known.has(id)) orphanFirstSeenMs.delete(id);
if (!candidates.length) return 0;
known = await tracked();
let reaped = 0;
for (const sandboxId of candidates) {
if (known.has(sandboxId)) continue;
try {
await client.terminate(sandboxId);
orphanFirstSeenMs.delete(sandboxId);
reaped++;
} catch (e) {
if (e instanceof ModalSandboxGoneError) orphanFirstSeenMs.delete(sandboxId);
else reportError("sandbox_reap", "orphan_terminate_failed", errMessage(e));
}
}
return reaped;
}
const sandbox: Sandbox = {
destroyScope(scopeId: string): Promise<void> {
return provisionQueue(scopeId, () => destroyStoredScope(scopeId));
},
profile,
startProcess: procSessions.startProcess,
readProcess: procSessions.readProcess,
writeStdin: procSessions.writeStdin,
signalProcess: procSessions.signalProcess,
listProcesses: procSessions.listProcesses,
...execFileOps,
...blobStaging,
async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise<SandboxHandle> {
const scratch = provOpts?.scratch;
const writable = layers.find((l) => l.mode === "rw") ?? layers[0];
const scope = writable?.scopeId ?? "default";
let name: string;
let coldStart: boolean;
if (scratch) {
({ name, coldStart } = await ensureScratch(scratch.key));
} else {
name = sandboxScopeName(prefix, scope);
scopeByName.set(name, scope);
({ coldStart } = await ensureSession(scope, name, provOpts?.onStatus));
}
const forceEgress = !!opts.egressProxyUrl && !!provOpts?.egressToken;
const turnEnv = Object.fromEntries(
Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),
);
const env = forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!, turnEnv) : turnEnv;
const handle: SandboxHandle = {
id: name,
rootDir: workspaceDir,
homeDir: HOME_DIR,
coldStart,
...(scratch ? { scratch: true } : {}),
...(Object.keys(env).length ? { env } : {}),
};
try {
const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(HOME_DIR, opts.credentialPaths ?? [])}`;
await installLayerTools(
{
exec: (script, t) => execRaw(name, script, t),
writeAbs: (abs, data) => writeAbsBytes(name, abs, data),
},
`mkdir -p ${shq(workspaceDir)}${credLinks}`,
);
await materializeRoLayers(
workspace,
layers,
handle,
{
readFile: (h, rel) => sandbox.readFile(h, rel),
writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data),
exec: (script, t) => execRaw(name, script, t),
},
{ manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "modal" },
);
return handle;
} catch (err) {
await cleanupFailedProvision(sandbox, handle, err);
throw err;
}
},
async run(handle, command, execOpts?: ExecOptions): Promise<ExecResult> {
const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
const script = `${nonInteractiveShellPrefix()}cd ${handle.rootDir} 2>/dev/null; ${command}`;
const signal = execOpts?.signal;
if (!signal) return execRaw(handle.id, script, timeoutSec, handle.env);
const killUid = randomUUID();
const fireKill = () => {
execRaw(handle.id, killScript(killUid), 15).catch(swallowAs("modal-sandbox: kill in-flight exec", undefined));
};
signal.throwIfAborted();
const onAbort = () => fireKill();
signal.addEventListener("abort", onAbort, { once: true });
try {
return await execRaw(handle.id, killableScript(script, killUid), timeoutSec, handle.env);
} finally {
signal.removeEventListener("abort", onAbort);
}
},
async writeFileBytes(handle, relPath, data): Promise<void> {
await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data);
},
async writeFile(handle, relPath, data): Promise<void> {
await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8"));
},
async readFileBytes(handle, relPath): Promise<Uint8Array | null> {
return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath));
},
async readFile(handle, relPath): Promise<string | null> {
const bytes = await sandbox.readFileBytes(handle, relPath);
return bytes === null ? null : Buffer.from(bytes).toString("utf8");
},
exportFiles: execExport.exportFiles,
...(homeSnapshots
? {
async persistHomeSnapshot(scopeId: string): Promise<void> {
const name = sandboxScopeName(prefix, scopeId);
scopeByName.set(name, scopeId);
const { session } = await ensureSession(scopeId, name);
await provisionQueue(scopeId, async () => {
await assertHydrated(session);
await homeSnapshots.snapshotHome(scopeId, session);
});
},
}
: {}),
async restartComputer(scopeId: string): Promise<void> {
await provisionQueue(scopeId, async () => {
const stored = await store.get(scopeId);
if (!stored) return;
const name = sandboxScopeName(prefix, scopeId);
let session: ModalSession | undefined;
try {
session = await client.fromId(stored.sandboxId);
} catch (error) {
if (!(error instanceof ModalSandboxGoneError)) throw error;
}
if (session) {
if (!stored.hydrationPending) await snapshotHome(scopeId, session);
await session.terminate();
}
sessionByName.delete(name);
await store.merge(scopeId, { hydrationPending: false, recoveryError: undefined });
});
},
async computerStatus(scopeId: string): Promise<ComputerStatus> {
const name = sandboxScopeName(prefix, scopeId);
const stored = await store.get(scopeId);
if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false };
const recovery = {
strategy: stored.nativeSnapshotId ? ("provider_snapshot" as const) : ("workspace_snapshot" as const),
...(stored.nativeSnapshotId
? { checkpointId: stored.nativeSnapshotId, checkpointExpiresAtMs: stored.nativeSnapshotExpiresAtMs }
: {}),
...(stored.lastSnapshotMs ? { checkpointAtMs: stored.lastSnapshotMs } : {}),
...(stored.recoveryError ? { error: stored.recoveryError } : {}),
};
const machine = `modal sandbox ${stored.sandboxId}`;
if (stored.hydrationPending)
return {
machine,
provisioned: true,
guestResponsive: false,
recovery: {
...recovery,
state: "failed",
error:
"Home hydration is incomplete. After confirming no core is still hydrating, use computer restart to discard the incomplete replacement and retry the retained checkpoint.",
},
};
try {
scopeByName.set(name, scopeId);
let session = sessionByName.get(name);
if (!session) {
session = await client.fromId(stored.sandboxId);
}
const r = await session.runCommand("echo responsive", { timeoutMs: 30_000 });
return {
machine,
expiresAtMs: stored.expiresAtMs,
recovery,
provisioned: true,
guestResponsive: r.exitCode === 0 && /responsive/.test(r.stdout),
};
} catch (e) {
return {
expiresAtMs: stored.expiresAtMs,
recovery,
machine: `${machine} (${errMessage(e).slice(0, 120)})`,
provisioned: !(e instanceof ModalSandboxGoneError),
guestResponsive: false,
};
}
},
async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
if (handle.scratch) {
const key = scratchKeyByName.get(handle.id);
return provisionQueue(key ? `scratch:${key}` : handle.id, async () => {
const remaining = (activeScratch.get(handle.id) ?? 1) - 1;
if (remaining > 0) {
activeScratch.set(handle.id, remaining);
return;
}
activeScratch.delete(handle.id);
const session = sessionByName.get(handle.id);
if (session) {
if (tdOpts?.destroy) await session.terminate();
else await session.terminate().catch(swallowAs("modal-sandbox: scratch terminate", undefined));
}
sessionByName.delete(handle.id);
scratchKeyByName.delete(handle.id);
});
}
if (tdOpts?.destroy && !scopeByName.has(handle.id)) return;
const scope = scopeByName.get(handle.id) ?? "default";
return provisionQueue(scope, async () => {
const session = sessionByName.get(handle.id);
if (tdOpts?.destroy) return destroyStoredScope(scope);
if (!session) return;
const stored = await store.get(scope);
if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });
if (
snapshotDue({ ...stored, lastSnapshotMs: lastSnapshotAttemptMs(stored) }, tdOpts, snapshotIntervalMs(stored))
) {
try {
await snapshotHome(scope, session);
await store.merge(scope, { lastActivityMs: Date.now() });
} catch (e) {
reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope);
await store
.merge(scope, { lastActivityMs: Date.now() })
.catch(swallowAs("modal-sandbox: record activity", undefined));
}
} else {
await store
.merge(scope, { lastActivityMs: Date.now() })
.catch(swallowAs("modal-sandbox: record activity", undefined));
}
});
},
async reapDeepIdle(idleMs): Promise<{ reaped: number }> {
if (!(idleMs > 0)) return { reaped: 0 };
const now = Date.now();
const cutoff = now - Math.min(idleMs, reapIdleMs);
let reaped = 0;
for (const [scope, rec] of await store.entries()) {
if (rec.orgId && rec.orgId !== configOrgId()) continue;
if (rec.hydrationPending) continue;
const name = sandboxScopeName(prefix, scope);
scopeByName.set(name, scope);
let session: ModalSession | undefined;
try {
session = sessionByName.get(name) ?? (await client.fromId(rec.sandboxId));
} catch (e) {
if (!(e instanceof ModalSandboxGoneError)) {
reportError("sandbox_reap", "deep_idle_probe_failed", errMessage(e), scope);
continue;
}
}
if (!session) {
sessionByName.delete(name);
if (!rec.nativeSnapshotId)
await store.delete(scope).catch(swallowAs("modal-sandbox: forget retired sandbox", undefined));
else if (checkpointRenewalDue(rec, now)) await renewCheckpoint(scope, name, rec);
continue;
}
if (rec.expiresAtMs && rec.expiresAtMs - now <= lifetimeMarginMs) {
reaped += await retire(scope, name, rec, session);
continue;
}
if (rec.lastActivityMs && rec.lastActivityMs <= cutoff) {
let busy: boolean;
try {
const handle: SandboxHandle = { id: name, rootDir: workspaceDir, homeDir: HOME_DIR, coldStart: false };
const live = await createExecProcessSessions(directProcIo(session)).listProcesses(handle);
busy = live.some((p) => p.status.state === "running");
} catch (e) {
if (e instanceof ModalSandboxGoneError) {
sessionByName.delete(name);
if (!rec.nativeSnapshotId)
await store.delete(scope).catch(swallowAs("modal-sandbox: forget retired sandbox", undefined));
} else {
reportError("sandbox_reap", "deep_idle_probe_failed", errMessage(e), scope);
}
continue;
}
if (!busy) {
reaped += await retire(scope, name, rec, session, cutoff);
continue;
}
}
if (usesNativeSnapshots(rec) && now - lastSnapshotAttemptMs(rec) > snapshotIntervalMs(rec))
await refreshCheckpoint(scope, rec, session);
}
reaped += await sweepOrphans(now);
return { reaped };
},
};
return sandbox;
}