Multiplayer agent harness for work.
import { cleanupFailedProvision } from "./sandbox.ts";
import { randomUUID } from "node:crypto";
import type { WorkspaceLayer } from "../types.ts";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
import type { DurableMap } from "../persistence/durable-map.ts";
import { createKeyedQueue } from "../util/async.ts";
import { swallowAs, errMessage } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, forceThroughProxyEnv } from "./sandbox-env.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
import { materializeRoLayers } from "./ro-layers.ts";
import { createLayerToolInstaller } from "./layer-tool-install.ts";
import type { LayerInstallFile } from "../deployment/load-layer.ts";
import {
createBackendBlobStaging,
createExecExport,
createExecFileOps,
posixJoin,
type BlobStagingOptions,
} from "./exec-file-ops.ts";
import {
ephemeralCredLinkScript,
ephemeralCredLinkPaths,
type CredentialPathSpec,
} from "../credentials/resident-paths.ts";
import { killableScript, killScript } from "./exec-kill.ts";
import { hasParentPathSegment, visibleNotInstalled, visibleTools } from "./sandbox.ts";
import { sandboxScopeName } from "./exec-sandbox-base.ts";
import {
ManagedAgentsCommandLostError,
ManagedAgentsSandboxGoneError,
type ManagedAgentsClient,
type ManagedAgentsSessionInfo,
type ManagedAgentsSession,
} from "./managed-agents-client.ts";
import { createHomeSnapshotOps, HOME_SNAPSHOT_PRUNE, snapshotDue, type HomeSnapshotStore } from "./home-snapshot.ts";
import type {
AgentComputerProfile,
ComputerStatus,
ExecOptions,
ExecResult,
ProvisionOptions,
Sandbox,
SandboxHandle,
TeardownOptions,
} from "./sandbox.ts";
const WORKSPACE_ROOT = "/workspace";
const HOME_DIR = `${WORKSPACE_ROOT}/home`;
const TRANSFER_RELAY_BASENAME = ".qm-transfer.";
const WORKSPACE_BASENAME = "workspace";
const RO_LAYERS_TAR = ".ro-layers.tar";
const RO_LAYERS_MANIFEST = ".ro-layers.manifest";
const HOME_TAR = `${HOME_DIR}/.qm-home.tar`;
const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE;
const DEFAULT_SNAPSHOT_INTERVAL_MS = 5 * 60_000;
const CHECKPOINT_LABEL = "qm teardown";
export const QM_TEMPLATE_TOOLS = ["jq", "rg", "unzip", "wget"];
function transfersDirectly(absPath: string): boolean {
return (absPath === WORKSPACE_ROOT || absPath.startsWith(`${WORKSPACE_ROOT}/`)) && !hasParentPathSegment(absPath);
}
export interface StoredManagedAgentsSandbox {
sessionId: string;
sandboxId: string;
preservationState?: "running" | "paused" | "pause_failed";
preservationError?: string;
createdAtMs: number;
lastSnapshotMs?: number;
homeDirty?: boolean;
checkpointId?: string;
checkpointAtMs?: number;
checkpointError?: string;
pendingCheckpointDeletes?: string[];
}
export interface ManagedAgentsSandboxOptions extends BlobStagingOptions {
client: ManagedAgentsClient;
namePrefix?: string;
defaultTimeoutSec?: number;
snapshotIntervalMs?: number;
egressProxyUrl?: string;
extraTools?: string[];
credentialPaths?: CredentialPathSpec[];
layerToolFiles?: () => readonly LayerInstallFile[];
store: DurableMap<StoredManagedAgentsSandbox>;
snapshots?: HomeSnapshotStore;
onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;
}
export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: ManagedAgentsSandboxOptions): Sandbox {
const client = opts.client;
const prefix = opts.namePrefix ?? "qm";
const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;
const snapshotIntervalMs = opts.snapshotIntervalMs ?? DEFAULT_SNAPSHOT_INTERVAL_MS;
const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`;
const store = opts.store;
const provisionQueue = createKeyedQueue<string>();
const sessionByName = new Map<string, ManagedAgentsSession>();
const scopeByName = new Map<string, string>();
const scratchKeyByName = new Map<string, string>();
const activeScratch = new Map<string, number>();
const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {
opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });
};
const homeSnapshots = opts.snapshots
? createHomeSnapshotOps<ManagedAgentsSession>({
label: "do-managed-agents",
homeDir: HOME_DIR,
homeTarPath: HOME_TAR,
prunePaths: [
...SNAPSHOT_PRUNE,
...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`),
],
store: opts.snapshots,
io: {
runCommand: (session, script, timeoutMs) => session.runCommand(script, { timeoutMs }),
readFileBytes: (session, abs) => session.readFileBytes(abs),
writeFileBytes: (session, abs, data) => session.writeFileBytes(abs, data),
},
})
: undefined;
async function snapshotHome(scope: string, session: ManagedAgentsSession): Promise<void> {
if (!homeSnapshots) return;
await homeSnapshots.snapshotHome(scope, session);
await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false });
}
async function pendingDeletes(scope: string): Promise<string[]> {
return (await store.get(scope))?.pendingCheckpointDeletes ?? [];
}
async function forgetCheckpoint(scope: string, session: ManagedAgentsSession, checkpointId: string): Promise<void> {
try {
await session.deleteCheckpoint(checkpointId);
} catch (e) {
reportError("sandbox_snapshot", "checkpoint_delete_failed", errMessage(e), scope);
return;
}
const pending = await pendingDeletes(scope);
await store.merge(scope, { pendingCheckpointDeletes: pending.filter((id) => id !== checkpointId) });
}
async function retryPendingCheckpointDeletes(scope: string, session: ManagedAgentsSession): Promise<void> {
const current = (await store.get(scope))?.checkpointId;
for (const checkpointId of await pendingDeletes(scope)) {
if (checkpointId === current) continue;
await forgetCheckpoint(scope, session, checkpointId);
}
}
async function captureCheckpoint(scope: string, session: ManagedAgentsSession): Promise<void> {
const stored = await store.get(scope);
const previous = stored?.checkpointId;
const captured = await session.createCheckpoint(CHECKPOINT_LABEL);
const pending = stored?.pendingCheckpointDeletes ?? [];
const queued =
previous && previous !== captured.checkpointId && !pending.includes(previous) ? [...pending, previous] : pending;
await store.merge(scope, {
checkpointId: captured.checkpointId,
checkpointAtMs: captured.createdAtMs ?? Date.now(),
checkpointError: undefined,
pendingCheckpointDeletes: queued,
});
if (previous && previous !== captured.checkpointId) await forgetCheckpoint(scope, session, previous);
}
const hydrateHome = async (scope: string, session: ManagedAgentsSession): Promise<boolean> =>
homeSnapshots ? homeSnapshots.hydrateHome(scope, session) : false;
const recordSession = (scope: string, session: ManagedAgentsSession, createdAtMs: number): Promise<void> =>
store.put(scope, {
sessionId: session.sessionId,
sandboxId: session.sandboxId,
createdAtMs,
preservationState: "running",
});
async function ensureSession(
scope: string,
name: string,
onStatus?: (text: string) => void,
): Promise<{ session: ManagedAgentsSession; coldStart: boolean }> {
return provisionQueue(scope, async () => {
const cached = sessionByName.get(name);
if (cached) return { session: cached, coldStart: false };
const adopt = (session: ManagedAgentsSession): { session: ManagedAgentsSession; coldStart: boolean } => {
sessionByName.set(name, session);
return { session, coldStart: false };
};
const stored = await store.get(scope);
if (stored) {
try {
onStatus?.("Resuming the sandbox…");
const session = await client.connect(stored.sessionId);
await store.merge(scope, { preservationState: "running", sandboxId: session.sandboxId });
return adopt(session);
} catch (err) {
if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;
}
}
for (const summary of await client.list(name)) {
try {
const session = await client.connect(summary.sessionId);
await recordSession(scope, session, Date.now());
return adopt(session);
} catch (err) {
if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;
}
}
onStatus?.("Creating the sandbox…");
const session = await client.create({ name });
sessionByName.set(name, session);
await recordSession(scope, session, Date.now());
let hydrated: boolean;
try {
hydrated = await hydrateHome(scope, session);
} catch (e) {
reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);
sessionByName.delete(name);
await client.kill(session.sessionId).catch((err: unknown) => {
reportError("sandbox_hydrate", "orphan_kill_failed", errMessage(err), scope);
});
throw new Error(
`do-managed-agents provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`,
{
cause: e,
},
);
}
return { session, coldStart: !hydrated };
});
}
async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> {
const name = sandboxScopeName(`${prefix}-scratch`, key);
return provisionQueue(`scratch:${key}`, async () => {
scratchKeyByName.set(name, key);
const active = activeScratch.get(name) ?? 0;
if (active === 0 && !sessionByName.has(name)) {
sessionByName.set(name, await client.create({ name }));
}
activeScratch.set(name, active + 1);
return { name, coldStart: active === 0 };
});
}
async function withSession<T>(name: string, action: (session: ManagedAgentsSession) => Promise<T>): Promise<T> {
const scratchKey = scratchKeyByName.get(name);
if (scratchKey === undefined && !scopeByName.has(name)) throw new Error("sandbox handle has been released");
const reviveScratch = async (): Promise<ManagedAgentsSession> => {
const session = await client.create({ name });
sessionByName.set(name, session);
return session;
};
const first =
scratchKey !== undefined
? { session: sessionByName.get(name) ?? (await reviveScratch()) }
: await ensureSession(scopeByName.get(name) ?? "default", name);
try {
return await action(first.session);
} catch (err) {
if (err instanceof ManagedAgentsCommandLostError) sessionByName.delete(name);
if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;
sessionByName.delete(name);
const second =
scratchKey !== undefined
? { session: await reviveScratch() }
: await ensureSession(scopeByName.get(name) ?? "default", name);
return action(second.session);
}
}
async function execRaw(name: string, script: string, timeoutSec: number): Promise<ExecResult> {
return withSession(name, async (session) => {
const r = await session.runCommand(`sh -c ${shq(`timeout -k 5 ${timeoutSec} sh -c ${shq(script)}; exit $?`)}`, {
timeoutMs: timeoutSec * 1000 + 30_000,
});
return {
stdout: r.stdout,
stderr: r.stderr,
code: r.exitCode,
timedOut: r.exitCode === 124 || r.exitCode === 137,
};
});
}
const profile: AgentComputerProfile = {
backend: "do-managed-agents",
writablePersistence: client.nativePause ? "provider_managed" : "snapshot_to_workspace",
processSessions: true,
parksOnTeardown: true,
egressEnforcement: opts.egressProxyUrl ? "domain" : "none",
spec: {
os: "Linux — DigitalOcean Managed Agents Firecracker microVM (provider pause preserves state; publish durable work to git or Files)",
runtimes: ["Node", "Python 3"],
get tools() {
return visibleTools(["git", "curl", "tar", "python3", ...(opts.extraTools ?? [])]);
},
get notInstalled() {
return visibleNotInstalled(
["jq", "rg", "unzip", "wget", "gh", "aws", "gcloud", "kubectl", "flyctl", "glab"],
opts.extraTools ?? [],
);
},
homeDir: HOME_DIR,
workdir: workspaceDir,
},
};
const procIo: ExecProcessIo = {
async run(handle, command, execOpts): Promise<ExecResult> {
const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
return execRaw(handle.id, command, timeoutSec);
},
};
const procSessions = createExecProcessSessions(procIo);
const relayFailure = (what: string, r: ExecResult): Error =>
new Error(`do-managed-agents ${what} failed: ${r.stderr.trim() || `exit ${r.code}`}`);
const writeAbsBytes = async (name: string, absPath: string, data: Uint8Array): Promise<void> => {
if (transfersDirectly(absPath)) return withSession(name, (session) => session.writeFileBytes(absPath, data));
const relay = `${WORKSPACE_ROOT}/${TRANSFER_RELAY_BASENAME}${randomUUID()}`;
await withSession(name, (session) => session.writeFileBytes(relay, data));
const moved = await execRaw(
name,
`find ${WORKSPACE_ROOT} -maxdepth 1 -name ${shq(`${TRANSFER_RELAY_BASENAME}*`)} -mmin +120 -delete 2>/dev/null; mkdir -p "$(dirname ${shq(absPath)})" && mv -f ${shq(relay)} ${shq(absPath)} || { rm -f ${shq(relay)}; exit 1; }`,
60,
);
if (moved.code !== 0) throw relayFailure(`write ${absPath}`, moved);
};
const readAbsBytes = async (name: string, absPath: string): Promise<Uint8Array | null> => {
if (transfersDirectly(absPath)) return withSession(name, (session) => session.readFileBytes(absPath));
const relay = `${WORKSPACE_ROOT}/${TRANSFER_RELAY_BASENAME}${randomUUID()}`;
const copied = await execRaw(name, `test -f ${shq(absPath)} || exit 3; cp ${shq(absPath)} ${shq(relay)}`, 60);
if (copied.code === 3) return null;
if (copied.code !== 0) throw relayFailure(`read ${absPath}`, copied);
try {
return await withSession(name, (session) => session.readFileBytes(relay));
} finally {
await execRaw(name, `rm -f ${shq(relay)}`, 60).catch(
swallowAs("managed-agents-sandbox: transfer relay cleanup", undefined),
);
}
};
const installLayerTools = opts.layerToolFiles ? createLayerToolInstaller(opts.layerToolFiles) : null;
const execFileOps = createExecFileOps({
label: "do-managed-agents",
exec: (id, script, t) => execRaw(id, script, t),
writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),
});
const execExport = createExecExport({
label: "do-managed-agents",
exec: (id, script, t) => execRaw(id, script, t),
readAbsBytes,
defaultHomeDir: HOME_DIR,
ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),
});
const blobStaging = createBackendBlobStaging("do-managed-agents", (id, script, t) => execRaw(id, script, t), opts);
async function destroyStoredScope(scope: string): Promise<void> {
const name = sandboxScopeName(prefix, scope);
const stored = await store.get(scope);
const cached = sessionByName.get(name);
const ids = new Set([stored?.sessionId, cached?.sessionId].filter((id): id is string => !!id));
for (const id of ids) {
try {
if (cached && id === cached.sessionId) await cached.kill();
else await client.kill(id);
} catch (error) {
if (!(error instanceof ManagedAgentsSandboxGoneError)) throw error;
}
}
await store.delete(scope);
sessionByName.delete(name);
scopeByName.delete(name);
}
const sandbox: Sandbox = {
destroyScope(scopeId: string): Promise<void> {
return provisionQueue(scopeId, () => destroyStoredScope(scopeId));
},
profile,
startProcess: procSessions.startProcess,
readProcess: procSessions.readProcess,
writeStdin: procSessions.writeStdin,
signalProcess: procSessions.signalProcess,
listProcesses: procSessions.listProcesses,
...execFileOps,
...blobStaging,
async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise<SandboxHandle> {
const scratch = provOpts?.scratch;
const writable = layers.find((l) => l.mode === "rw") ?? layers[0];
const scope = writable?.scopeId ?? "default";
let name: string;
let coldStart: boolean;
if (scratch) {
({ name, coldStart } = await ensureScratch(scratch.key));
} else {
name = sandboxScopeName(prefix, scope);
scopeByName.set(name, scope);
({ coldStart } = await ensureSession(scope, name, provOpts?.onStatus));
}
const forceEgress = !!opts.egressProxyUrl && !!provOpts?.egressToken;
const turnEnv = Object.fromEntries(
Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),
);
const env = forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!, turnEnv) : turnEnv;
const handle: SandboxHandle = {
id: name,
rootDir: workspaceDir,
homeDir: HOME_DIR,
coldStart,
...(scratch ? { scratch: true } : {}),
...(Object.keys(env).length ? { env } : {}),
};
try {
const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(HOME_DIR, opts.credentialPaths ?? [])}`;
const prep = await execRaw(name, `mkdir -p ${shq(workspaceDir)}${credLinks}`, 60);
if (prep.code !== 0)
throw new Error(`do-managed-agents provision prep failed: ${(prep.stderr || prep.stdout).slice(0, 200)}`);
await materializeRoLayers(
workspace,
layers,
handle,
{
readFile: (h, rel) => sandbox.readFile(h, rel),
writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data),
exec: (script, t) => execRaw(name, script, t),
},
{ manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "do-managed-agents" },
);
await installLayerTools?.({
exec: (script, t) => execRaw(name, script, t),
writeAbs: (abs, data) => writeAbsBytes(name, abs, data),
});
return handle;
} catch (err) {
await cleanupFailedProvision(sandbox, handle, err);
throw err;
}
},
async run(handle, command, execOpts?: ExecOptions): Promise<ExecResult> {
const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
const exports = Object.entries(handle.env ?? {})
.map(([k, v]) => `export ${k}=${shq(v)}`)
.join("; ");
const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${handle.rootDir} 2>/dev/null; ${command}`;
const signal = execOpts?.signal;
if (!signal) return execRaw(handle.id, script, timeoutSec);
const killUid = randomUUID();
const fireKill = (): void => {
execRaw(handle.id, killScript(killUid), 15).catch(
swallowAs("managed-agents-sandbox: kill in-flight exec", undefined),
);
};
signal.throwIfAborted();
const onAbort = (): void => fireKill();
signal.addEventListener("abort", onAbort, { once: true });
try {
return await execRaw(handle.id, killableScript(script, killUid), timeoutSec);
} finally {
signal.removeEventListener("abort", onAbort);
}
},
async writeFileBytes(handle, relPath, data): Promise<void> {
await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data);
},
async writeFile(handle, relPath, data): Promise<void> {
await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8"));
},
async readFileBytes(handle, relPath): Promise<Uint8Array | null> {
return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath));
},
async readFile(handle, relPath): Promise<string | null> {
const bytes = await sandbox.readFileBytes(handle, relPath);
return bytes === null ? null : Buffer.from(bytes).toString("utf8");
},
exportFiles: execExport.exportFiles,
...(homeSnapshots
? {
async persistHomeSnapshot(scopeId: string): Promise<void> {
const name = sandboxScopeName(prefix, scopeId);
scopeByName.set(name, scopeId);
const { session } = await ensureSession(scopeId, name);
await snapshotHome(scopeId, session);
},
}
: {}),
async restartComputer(scopeId: string): Promise<void> {
const name = sandboxScopeName(prefix, scopeId);
scopeByName.set(name, scopeId);
return provisionQueue(scopeId, async () => {
const stored = await store.get(scopeId);
if (!stored?.sessionId) throw new Error("do-managed-agents restart: no sandbox provisioned yet");
const checkpointId = stored.checkpointId;
if (!checkpointId) throw new Error("do-managed-agents restart: no checkpoint to restore");
const cached = sessionByName.get(name);
sessionByName.delete(name);
if (cached) await cached.close();
let restored: ManagedAgentsSessionInfo;
try {
restored = await client.rollback(stored.sessionId, checkpointId);
} catch (e) {
reportError("agent_computer", "checkpoint_rollback_failed", errMessage(e), scopeId);
throw e;
}
await store.merge(scopeId, {
sandboxId: restored.sandboxId,
preservationState: "running",
preservationError: undefined,
});
reportError(
"agent_computer",
"checkpoint_restored",
`session ${stored.sessionId}: restored checkpoint ${checkpointId} onto sandbox ${restored.sandboxId}`,
scopeId,
);
});
},
async computerStatus(scopeId: string): Promise<ComputerStatus> {
const name = sandboxScopeName(prefix, scopeId);
const stored = await store.get(scopeId);
if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false };
const machine = `do-managed-agents sandbox ${stored.sandboxId || stored.sessionId}`;
const checkpointAtMs = stored.checkpointAtMs ?? stored.lastSnapshotMs;
const recoveryError = stored.preservationError ?? stored.checkpointError;
const recovery = {
strategy: "workspace_snapshot" as const,
state: stored.preservationState,
...(recoveryError ? { error: recoveryError } : {}),
...(stored.checkpointId ? { checkpointId: stored.checkpointId } : {}),
...(checkpointAtMs ? { checkpointAtMs } : {}),
};
try {
const info: ManagedAgentsSessionInfo = await client.info(stored.sessionId);
if (info.state === "paused")
return {
machine,
listed: info.state,
lifecycleState: "paused",
provisioned: true,
guestResponsive: false,
recovery: { ...recovery, state: "paused" },
};
scopeByName.set(name, scopeId);
let session = sessionByName.get(name);
if (!session) {
session = await client.connect(stored.sessionId);
sessionByName.set(name, session);
}
const r = await session.runCommand("echo responsive", { timeoutMs: 30_000 });
return {
machine,
recovery,
provisioned: true,
guestResponsive: r.exitCode === 0 && r.stdout.trim() === "responsive",
};
} catch (e) {
return {
recovery,
machine: `${machine} (${errMessage(e).slice(0, 120)})`,
provisioned: !(e instanceof ManagedAgentsSandboxGoneError),
guestResponsive: false,
};
}
},
async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
if (handle.scratch) {
const key = scratchKeyByName.get(handle.id);
return provisionQueue(key ? `scratch:${key}` : handle.id, async () => {
const remaining = (activeScratch.get(handle.id) ?? 1) - 1;
if (remaining > 0) {
activeScratch.set(handle.id, remaining);
return;
}
activeScratch.delete(handle.id);
const session = sessionByName.get(handle.id);
if (session) {
if (tdOpts?.destroy) await session.kill();
else await session.kill().catch(swallowAs("managed-agents-sandbox: scratch kill", undefined));
}
sessionByName.delete(handle.id);
scratchKeyByName.delete(handle.id);
});
}
if (tdOpts?.destroy && !scopeByName.has(handle.id)) return;
const scope = scopeByName.get(handle.id) ?? "default";
return provisionQueue(scope, () => teardownScope(handle, scope, tdOpts));
},
};
async function teardownScope(handle: SandboxHandle, scope: string, tdOpts?: TeardownOptions): Promise<void> {
const session = sessionByName.get(handle.id);
if (tdOpts?.destroy) return destroyStoredScope(scope);
if (!session) return;
const stored = await store.get(scope);
if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });
await retryPendingCheckpointDeletes(scope, session);
const homeDirty = stored?.homeDirty;
if (snapshotDue({ lastSnapshotMs: stored?.checkpointAtMs, homeDirty }, tdOpts, snapshotIntervalMs)) {
try {
await captureCheckpoint(scope, session);
} catch (e) {
await store.merge(scope, { checkpointError: errMessage(e) });
reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), scope);
}
}
if (snapshotDue({ lastSnapshotMs: stored?.lastSnapshotMs, homeDirty }, tdOpts, snapshotIntervalMs)) {
try {
await snapshotHome(scope, session);
} catch (e) {
reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope);
}
}
if (tdOpts?.keepWarm) return;
try {
await session.pause();
await store.merge(scope, { preservationState: "paused", preservationError: undefined });
sessionByName.delete(handle.id);
} catch (error) {
await store.merge(scope, { preservationState: "pause_failed", preservationError: errMessage(error) });
reportError("sandbox_preservation", "pause_failed", errMessage(error), scope);
throw error;
}
}
return sandbox;
}