Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

24.7 KB
Raw Blame
import { cleanupFailedProvision } from "./sandbox.ts";
import { randomUUID } from "node:crypto";
import type { WorkspaceLayer } from "../types.ts";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
import type { DurableMap } from "../persistence/durable-map.ts";
import type { AdvisoryLock } from "../persistence/advisory-lock.ts";
import { createKeyedQueue } from "../util/async.ts";
import { swallowAs, errMessage } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, forceThroughProxyEnv } from "./sandbox-env.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
import { materializeRoLayers } from "./ro-layers.ts";
import { createLayerToolInstaller } from "./layer-tool-install.ts";
import type { LayerInstallFile } from "../deployment/load-layer.ts";
import {
  createBackendBlobStaging,
  createExecExport,
  createExecFileOps,
  posixJoin,
  type BlobStagingOptions,
} from "./exec-file-ops.ts";
import {
  ephemeralCredLinkScript,
  ephemeralCredLinkPaths,
  type CredentialPathSpec,
} from "../credentials/resident-paths.ts";
import { killableScript, killScript } from "./exec-kill.ts";
import { visibleNotInstalled, visibleTools } from "./sandbox.ts";
import { sandboxScopeName } from "./exec-sandbox-base.ts";
import {
  E2bCommandLostError,
  E2bSandboxGoneError,
  type E2bClient,
  type E2bSandboxInfo,
  type E2bSession,
  runE2bScript,
} from "./e2b-client.ts";
import { createHomeSnapshotOps, HOME_SNAPSHOT_PRUNE, snapshotDue, type HomeSnapshotStore } from "./home-snapshot.ts";
import type {
  AgentComputerProfile,
  ComputerStatus,
  ExecOptions,
  ExecResult,
  ProvisionOptions,
  Sandbox,
  SandboxHandle,
  TeardownOptions,
} from "./sandbox.ts";

const HOME_DIR = "/home/user";
const WORKSPACE_BASENAME = "workspace";
const RO_LAYERS_TAR = ".ro-layers.tar";
const RO_LAYERS_MANIFEST = ".ro-layers.manifest";
const HOME_TAR = `${HOME_DIR}/.qm-home.tar`;

const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE;
const DEFAULT_KEEP_WARM_SEC = 3600;
const DEFAULT_NATIVE_SNAPSHOT_INTERVAL_MS = 5 * 60_000;

export interface StoredE2bSandbox {
  sandboxId: string;
  nativePause?: boolean;
  preservationState?: "running" | "paused" | "pause_failed";
  preservationError?: string;
  recoverySnapshotId?: string;
  recoveryError?: string;
  createdAtMs: number;
  lastSnapshotMs?: number;
  homeDirty?: boolean;
}

export const E2B_TEMPLATE_TOOLS = ["wget", "unzip", "gh", "aws", "x-api"];

export interface E2bSandboxOptions extends BlobStagingOptions {
  advisoryLock: AdvisoryLock;
  client: E2bClient;
  namePrefix?: string;
  defaultTimeoutSec?: number;
  keepWarmSec?: number;

  snapshotIntervalMs?: number;
  nativeSnapshotIntervalMs?: number;
  egressProxyUrl?: string;
  extraTools?: string[];
  credentialPaths?: CredentialPathSpec[];
  layerToolFiles?: () => readonly LayerInstallFile[];
  store: DurableMap<StoredE2bSandbox>;
  snapshots?: HomeSnapshotStore;
  onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;
}

export function createE2bSandbox(workspace: WorkspaceStore, opts: E2bSandboxOptions): Sandbox {
  const client = opts.client;
  const prefix = opts.namePrefix ?? "qm";
  const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;
  const keepWarmMs = (opts.keepWarmSec ?? DEFAULT_KEEP_WARM_SEC) * 1000;
  const snapshotIntervalMs = opts.snapshotIntervalMs ?? 0;
  const nativeSnapshotIntervalMs = opts.nativeSnapshotIntervalMs ?? DEFAULT_NATIVE_SNAPSHOT_INTERVAL_MS;
  const observed: { cpus?: number; memoryMb?: number; diskGb?: number } = {};
  const noteInfo = (info: E2bSandboxInfo | undefined): void => {
    if (info?.cpuCount) observed.cpus = info.cpuCount;
    if (info?.memoryMb) observed.memoryMb = info.memoryMb;
  };
  const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`;
  const store = opts.store;
  const localQueue = createKeyedQueue<string>();
  const advisoryLock = opts.advisoryLock;
  const provisionQueue = <T>(scope: string, action: () => Promise<T>): Promise<T> =>
    localQueue(scope, () => advisoryLock.withLock(`e2b-provision:${scope}`, action));

  const sessionByName = new Map<string, E2bSession>();
  const scopeByName = new Map<string, string>();
  const scratchKeyByName = new Map<string, string>();
  const activeScratch = new Map<string, number>();

  const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {
    opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });
  };

  const homeSnapshots = opts.snapshots
    ? createHomeSnapshotOps<E2bSession>({
        label: "e2b",
        homeDir: HOME_DIR,
        homeTarPath: HOME_TAR,
        prunePaths: [
          ...SNAPSHOT_PRUNE,
          ...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`),
        ],
        store: opts.snapshots,
        io: {
          runCommand: (session, script, timeoutMs) => session.runCommand(script, { timeoutMs }),
          readFileBytes: (session, abs) => session.readFileBytes(abs),
          writeFileBytes: (session, abs, data) => session.writeFileBytes(abs, data),
        },
      })
    : undefined;

  async function snapshotHome(scope: string, session: E2bSession): Promise<void> {
    if (!homeSnapshots) return;
    await homeSnapshots.snapshotHome(scope, session);
    await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false });
  }

  const hydrateHome = async (scope: string, session: E2bSession): Promise<boolean> =>
    homeSnapshots ? homeSnapshots.hydrateHome(scope, session) : false;

  async function ensureSession(
    scope: string,
    name: string,
    onStatus?: (text: string) => void,
  ): Promise<{ session: E2bSession; coldStart: boolean }> {
    return provisionQueue(scope, async () => {
      const cached = sessionByName.get(name);
      if (cached) return { session: cached, coldStart: false };

      const adopt = (session: E2bSession): { session: E2bSession; coldStart: boolean } => {
        sessionByName.set(name, session);
        return { session, coldStart: false };
      };

      const stored = await store.get(scope);
      if (stored) {
        try {
          const session = await client.connect(stored.sandboxId);
          const info = await client.info?.(session.sandboxId);
          noteInfo(info);
          await store.merge(scope, {
            preservationState: "running",
            ...(info ? { nativePause: info.onTimeout === "pause" } : {}),
          });
          return adopt(session);
        } catch (err) {
          if (!(err instanceof E2bSandboxGoneError)) throw err;
        }
      }

      const listed = await client.list({ name });
      for (const summary of listed) {
        try {
          const session = await client.connect(summary.sandboxId);
          const info = await client.info?.(session.sandboxId);
          noteInfo(info);
          await store.put(scope, {
            ...stored,
            sandboxId: session.sandboxId,
            createdAtMs: Date.now(),
            nativePause: info?.onTimeout === "pause",
          });
          return adopt(session);
        } catch (err) {
          if (!(err instanceof E2bSandboxGoneError)) throw err;
        }
      }

      if (stored?.nativePause) {
        if (!stored.recoverySnapshotId)
          throw new Error("e2b sandbox is gone; explicitly import a recovery snapshot before replacing its home");
        onStatus?.("Restoring the sandbox from its recovery snapshot…");
        let session: E2bSession;
        try {
          session = await client.create({
            metadata: { name },
            autoPause: true,
            fromSnapshot: stored.recoverySnapshotId,
          });
        } catch (e) {
          await store.merge(scope, { recoveryError: errMessage(e) });
          reportError("sandbox_hydrate", "snapshot_restore_failed", errMessage(e), scope);
          throw new Error(`e2b provision: restore from recovery snapshot ${stored.recoverySnapshotId} failed`, {
            cause: e,
          });
        }
        await store.merge(scope, {
          sandboxId: session.sandboxId,
          createdAtMs: Date.now(),
          preservationState: "running",
          preservationError: undefined,
          recoveryError: undefined,
        });
        return adopt(session);
      }
      onStatus?.("Creating the sandbox…");
      const session = await client.create({ metadata: { name }, autoPause: true });
      sessionByName.set(name, session);
      await store.put(scope, {
        sandboxId: session.sandboxId,
        createdAtMs: Date.now(),
        nativePause: false,
        preservationState: "running",
      });
      let hydrated: boolean;
      try {
        hydrated = await hydrateHome(scope, session);
      } catch (e) {
        reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);
        sessionByName.delete(name);
        await client.kill(session.sandboxId).catch(() => undefined);
        throw new Error(`e2b provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`, {
          cause: e,
        });
      }
      await store.merge(scope, { nativePause: client.nativePause });
      noteInfo(await client.info?.(session.sandboxId).catch(() => undefined));
      return { session, coldStart: !hydrated };
    });
  }

  async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> {
    const name = sandboxScopeName(`${prefix}-scratch`, key);
    return provisionQueue(`scratch:${key}`, async () => {
      scratchKeyByName.set(name, key);
      const active = activeScratch.get(name) ?? 0;
      if (active === 0 && !sessionByName.has(name)) {
        const session = await client.create({ metadata: { name, scratch: "true" }, autoPause: false });
        sessionByName.set(name, session);
      }
      activeScratch.set(name, active + 1);
      return { name, coldStart: active === 0 };
    });
  }

  async function withSession<T>(name: string, action: (session: E2bSession) => Promise<T>): Promise<T> {
    const scratchKey = scratchKeyByName.get(name);
    if (scratchKey === undefined && !scopeByName.has(name)) throw new Error("sandbox handle has been released");

    const reviveScratch = async (): Promise<E2bSession> => {
      const session = await client.create({ metadata: { name, scratch: "true" }, autoPause: false });
      sessionByName.set(name, session);
      return session;
    };
    const first =
      scratchKey !== undefined
        ? { session: sessionByName.get(name) ?? (await reviveScratch()) }
        : await ensureSession(scopeByName.get(name) ?? "default", name);
    try {
      return await action(first.session);
    } catch (err) {
      if (err instanceof E2bCommandLostError) sessionByName.delete(name);
      if (!(err instanceof E2bSandboxGoneError)) throw err;
      sessionByName.delete(name);
      const second =
        scratchKey !== undefined
          ? { session: await reviveScratch() }
          : await ensureSession(scopeByName.get(name) ?? "default", name);
      return action(second.session);
    }
  }

  async function execRaw(name: string, script: string, timeoutSec: number): Promise<ExecResult> {
    return withSession(name, (session) => runE2bScript(session, script, timeoutSec));
  }

  const profile: AgentComputerProfile = {
    backend: "e2b",
    writablePersistence: client.nativePause ? "provider_managed" : "snapshot_to_workspace",
    processSessions: true,
    parksOnTeardown: true,
    egressEnforcement: opts.egressProxyUrl ? "domain" : "none",
    spec: {
      os: "Linux — E2B Firecracker sandbox (provider pause preserves state; publish durable work to git or Files)",
      runtimes: ["Node", "Python 3"],
      get tools() {
        return visibleTools(["git", "curl", "jq", "tar", "python3", ...(opts.extraTools ?? [])]);
      },
      get notInstalled() {
        return visibleNotInstalled(["gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], opts.extraTools ?? []);
      },
      get cpus() {
        return observed.cpus;
      },
      get memoryMb() {
        return observed.memoryMb;
      },
      get diskGb() {
        return observed.diskGb;
      },
      homeDir: HOME_DIR,
      workdir: workspaceDir,
    },
  };

  const procIo: ExecProcessIo = {
    async run(handle, command, execOpts): Promise<ExecResult> {
      const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
      return execRaw(handle.id, command, timeoutSec);
    },
  };
  const procSessions = createExecProcessSessions(procIo);

  const writeAbsBytes = (name: string, absPath: string, data: Uint8Array): Promise<void> =>
    withSession(name, (session) => session.writeFileBytes(absPath, data));
  const readAbsBytes = (name: string, absPath: string): Promise<Uint8Array | null> =>
    withSession(name, (session) => session.readFileBytes(absPath));
  const installLayerTools = opts.layerToolFiles ? createLayerToolInstaller(opts.layerToolFiles) : null;

  const execFileOps = createExecFileOps({
    label: "e2b",
    exec: (id, script, t) => execRaw(id, script, t),
    writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),
  });

  const execExport = createExecExport({
    label: "e2b",
    exec: (id, script, t) => execRaw(id, script, t),
    readAbsBytes,
    defaultHomeDir: HOME_DIR,
    ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),
  });

  const blobStaging = createBackendBlobStaging("e2b", (id, script, t) => execRaw(id, script, t), opts);

  async function destroyStoredScope(scope: string): Promise<void> {
    const name = sandboxScopeName(prefix, scope);
    const stored = await store.get(scope);
    const cached = sessionByName.get(name);
    const ids = new Set([stored?.sandboxId, cached?.sandboxId].filter((id): id is string => !!id));
    for (const id of ids) {
      try {
        await client.kill(id);
      } catch (error) {
        if (!(error instanceof E2bSandboxGoneError)) throw error;
      }
    }
    await store.delete(scope);
    sessionByName.delete(name);
    scopeByName.delete(name);
    await forgetSnapshot(stored?.recoverySnapshotId);
  }

  const forgetSnapshot = (snapshotId: string | undefined): Promise<void> =>
    snapshotId
      ? client.deleteSnapshot(snapshotId).catch(swallowAs("e2b-sandbox: recovery snapshot delete", undefined))
      : Promise.resolve();

  async function captureRecoverySnapshot(scope: string, session: E2bSession, previous?: string): Promise<void> {
    try {
      const { snapshotId } = await session.createSnapshot();
      await store.merge(scope, {
        recoverySnapshotId: snapshotId,
        recoveryError: undefined,
        lastSnapshotMs: Date.now(),
        homeDirty: false,
      });
      if (previous !== snapshotId) await forgetSnapshot(previous);
    } catch (e) {
      await store.merge(scope, { recoveryError: errMessage(e) });
      reportError("sandbox_snapshot", "recovery_snapshot_failed", errMessage(e), scope);
    }
  }

  const sandbox: Sandbox = {
    destroyScope(scopeId: string): Promise<void> {
      return provisionQueue(scopeId, () => destroyStoredScope(scopeId));
    },

    profile,
    startProcess: procSessions.startProcess,
    readProcess: procSessions.readProcess,
    writeStdin: procSessions.writeStdin,
    signalProcess: procSessions.signalProcess,
    listProcesses: procSessions.listProcesses,
    ...execFileOps,
    ...blobStaging,

    async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise<SandboxHandle> {
      const scratch = provOpts?.scratch;
      const writable = layers.find((l) => l.mode === "rw") ?? layers[0];
      const scope = writable?.scopeId ?? "default";
      let name: string;
      let coldStart: boolean;
      if (scratch) {
        ({ name, coldStart } = await ensureScratch(scratch.key));
      } else {
        name = sandboxScopeName(prefix, scope);
        scopeByName.set(name, scope);
        ({ coldStart } = await ensureSession(scope, name, provOpts?.onStatus));
      }

      const forceEgress = !!opts.egressProxyUrl && !!provOpts?.egressToken;
      const turnEnv = Object.fromEntries(
        Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),
      );
      const env = forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!, turnEnv) : turnEnv;
      const handle: SandboxHandle = {
        id: name,
        rootDir: workspaceDir,
        homeDir: HOME_DIR,
        coldStart,
        ...(scratch ? { scratch: true } : {}),
        ...(Object.keys(env).length ? { env } : {}),
      };

      try {
        const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(HOME_DIR, opts.credentialPaths ?? [])}`;
        const prep = await execRaw(name, `mkdir -p ${shq(workspaceDir)}${credLinks}`, 60);
        if (prep.code !== 0)
          throw new Error(`e2b provision prep failed: ${(prep.stderr || prep.stdout).slice(0, 200)}`);

        await materializeRoLayers(
          workspace,
          layers,
          handle,
          {
            readFile: (h, rel) => sandbox.readFile(h, rel),
            writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data),
            exec: (script, t) => execRaw(name, script, t),
          },
          { manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "e2b" },
        );
        await installLayerTools?.({
          exec: (script, t) => execRaw(name, script, t),
          writeAbs: (abs, data) => writeAbsBytes(name, abs, data),
        });

        return handle;
      } catch (err) {
        await cleanupFailedProvision(sandbox, handle, err);
        throw err;
      }
    },

    async run(handle, command, execOpts?: ExecOptions): Promise<ExecResult> {
      const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
      const exports = Object.entries(handle.env ?? {})
        .map(([k, v]) => `export ${k}=${shq(v)}`)
        .join("; ");
      const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${handle.rootDir} 2>/dev/null; ${command}`;
      const signal = execOpts?.signal;
      if (!signal) return execRaw(handle.id, script, timeoutSec);
      const killUid = randomUUID();
      const fireKill = () => {
        execRaw(handle.id, killScript(killUid), 15).catch(swallowAs("e2b-sandbox: kill in-flight exec", undefined));
      };
      signal.throwIfAborted();
      const onAbort = () => fireKill();
      signal.addEventListener("abort", onAbort, { once: true });
      try {
        return await execRaw(handle.id, killableScript(script, killUid), timeoutSec);
      } finally {
        signal.removeEventListener("abort", onAbort);
      }
    },

    async writeFileBytes(handle, relPath, data): Promise<void> {
      await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data);
    },
    async writeFile(handle, relPath, data): Promise<void> {
      await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8"));
    },
    async readFileBytes(handle, relPath): Promise<Uint8Array | null> {
      return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath));
    },
    async readFile(handle, relPath): Promise<string | null> {
      const bytes = await sandbox.readFileBytes(handle, relPath);
      return bytes === null ? null : Buffer.from(bytes).toString("utf8");
    },

    exportFiles: execExport.exportFiles,

    ...(homeSnapshots
      ? {
          async persistHomeSnapshot(scopeId: string): Promise<void> {
            const name = sandboxScopeName(prefix, scopeId);
            scopeByName.set(name, scopeId);
            const { session } = await ensureSession(scopeId, name);
            await snapshotHome(scopeId, session);
          },
        }
      : {}),

    async computerStatus(scopeId: string): Promise<ComputerStatus> {
      const name = sandboxScopeName(prefix, scopeId);
      const stored = await store.get(scopeId);

      if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false };
      const machine = `e2b sandbox ${stored.sandboxId}`;
      let expiresAtMs: number | undefined;
      const recoveryError = stored.recoveryError ?? stored.preservationError;
      const recovery = {
        strategy: stored.nativePause ? ("provider_pause" as const) : ("workspace_snapshot" as const),
        state: stored.preservationState,
        ...(recoveryError ? { error: recoveryError } : {}),
        ...(stored.recoverySnapshotId ? { checkpointId: stored.recoverySnapshotId, checkpointExpiresAtMs: null } : {}),
        ...(stored.lastSnapshotMs ? { checkpointAtMs: stored.lastSnapshotMs } : {}),
      };
      try {
        if (client.info) {
          const info = await client.info(stored.sandboxId);
          expiresAtMs = info.state === "running" ? info.expiresAtMs : undefined;
          if (info.state === "paused")
            return {
              machine,
              listed: info.state,
              lifecycleState: "paused",
              provisioned: true,
              guestResponsive: false,
              recovery: { ...recovery, state: info.state, checkpointExpiresAtMs: null },
            };
        }
        scopeByName.set(name, scopeId);
        let session = sessionByName.get(name);
        if (!session) {
          session = await client.connect(stored.sandboxId);
          sessionByName.set(name, session);
        }
        const r = await session.runCommand("echo responsive", { timeoutMs: 30_000 });
        const metrics = await session.metrics().catch(swallowAs("e2b-sandbox: metrics", null));
        if (metrics?.diskTotalBytes) observed.diskGb = Math.round(metrics.diskTotalBytes / 2 ** 30);
        return {
          machine,
          expiresAtMs,
          recovery,
          provisioned: true,
          guestResponsive: r.exitCode === 0 && /responsive/.test(r.stdout),
          ...(metrics
            ? {
                resources: {
                  cpuUsedPct: metrics.cpuUsedPct,
                  memUsedMb: Math.round(metrics.memUsedBytes / 2 ** 20),
                  memTotalMb: Math.round(metrics.memTotalBytes / 2 ** 20),
                  diskUsedGb: Math.round((metrics.diskUsedBytes / 2 ** 30) * 10) / 10,
                  diskTotalGb: Math.round((metrics.diskTotalBytes / 2 ** 30) * 10) / 10,
                },
              }
            : {}),
        };
      } catch (e) {
        return {
          recovery,
          machine: `${machine} (${errMessage(e).slice(0, 120)})`,
          provisioned: !(e instanceof E2bSandboxGoneError),
          guestResponsive: false,
        };
      }
    },

    async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
      if (handle.scratch) {
        const key = scratchKeyByName.get(handle.id);
        return provisionQueue(key ? `scratch:${key}` : handle.id, async () => {
          const remaining = (activeScratch.get(handle.id) ?? 1) - 1;
          if (remaining > 0) {
            activeScratch.set(handle.id, remaining);
            return;
          }
          activeScratch.delete(handle.id);
          const session = sessionByName.get(handle.id);
          if (session) {
            if (tdOpts?.destroy) await session.kill();
            else await session.kill().catch(swallowAs("e2b-sandbox: scratch kill", undefined));
          }
          sessionByName.delete(handle.id);
          scratchKeyByName.delete(handle.id);
        });
      }
      if (tdOpts?.destroy && !scopeByName.has(handle.id)) return;
      const scope = scopeByName.get(handle.id) ?? "default";
      return provisionQueue(scope, () => teardownScope(handle, scope, tdOpts));
    },
  };

  async function teardownScope(handle: SandboxHandle, scope: string, tdOpts?: TeardownOptions): Promise<void> {
    const session = sessionByName.get(handle.id);
    if (tdOpts?.destroy) return destroyStoredScope(scope);
    if (!session) return;

    const stored = await store.get(scope);
    if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });
    if (!stored?.nativePause && snapshotDue(stored, tdOpts, snapshotIntervalMs)) {
      try {
        await snapshotHome(scope, session);
      } catch (e) {
        reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope);
      }
    }
    if (tdOpts?.keepWarm) {
      try {
        await session.keepAlive(keepWarmMs);
      } catch (e) {
        reportError("sandbox_preservation", "keep_warm_failed", errMessage(e), scope);
      }
      return;
    }
    if (stored?.nativePause && snapshotDue(stored, tdOpts, nativeSnapshotIntervalMs))
      await captureRecoverySnapshot(scope, session, stored.recoverySnapshotId);
    try {
      await session.pause();
      await store.merge(scope, { preservationState: "paused", preservationError: undefined });
      sessionByName.delete(handle.id);
    } catch (error) {
      await store.merge(scope, { preservationState: "pause_failed", preservationError: errMessage(error) });
      reportError("sandbox_preservation", "pause_failed", errMessage(error), scope);
      throw error;
    }
  }

  return sandbox;
}