Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

28.2 KB
Raw Blame
import { randomUUID } from "node:crypto";
import { setTimeout as wait } from "node:timers/promises";
import type { Readable } from "node:stream";
import { APIError, SpritesClient, type Checkpoint, type SpriteCheck, type StreamMessage } from "@fly/sprites";
import type { DurableMap } from "../persistence/durable-map.ts";
import type { AdvisoryLock } from "../persistence/advisory-lock.ts";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
import { jitteredBackoffMs, retryAfterMs, withAbort, withTimeout } from "../util/async.ts";
import { swallow, errMessage } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { createExecProcessSessions, processSessionDir, type ExecProcessIo } from "./exec-process-session.ts";
import {
  createBackendBlobStaging,
  createExecExport,
  createExecFileOps,
  type BlobStagingOptions,
} from "./exec-file-ops.ts";
import type { CredentialPathSpec } from "../credentials/resident-paths.ts";
import { ephemeralCredLinkPaths } from "../credentials/resident-paths.ts";
import { visibleNotInstalled, visibleTools } from "./sandbox.ts";
import { createExecSandboxBase, sandboxScopeName } from "./exec-sandbox-base.ts";
import { withConnectorSdk, type ConnectorSdkBundle } from "./connector-sdk.ts";
import { createLayerToolInstaller } from "./layer-tool-install.ts";
import {
  createHomeSnapshotOps,
  HOME_SNAPSHOT_PRUNE,
  snapshotDue,
  type HomeSnapshotStore,
  type SnapshotBookkeeping,
} from "./home-snapshot.ts";
import type { LayerInstallFile } from "../deployment/load-layer.ts";
import type {
  AgentComputerProfile,
  ComputerStatus,
  ExecPressure,
  ExecResult,
  Sandbox,
  SandboxHandle,
  StartProcessOptions,
  TeardownOptions,
} from "./sandbox.ts";

const HOME_DIR = "/home/sprite";
const HOME_TAR = `${HOME_DIR}/.qm-home.tar`;
export const SCRIPT_RUNNER = 's=$(mktemp) && cat > "$s" && sh "$s" </dev/null; rc=$?; rm -f "$s"; exit $rc';
const EXIT_GRACE_MS = 60_000;
const GUEST_PROBE_TIMEOUT_SEC = 15;
const KEEPALIVE_TIMEOUT_SEC = 20;
const CHECKPOINT_INTERVAL_MS = 5 * 60_000;
const CHECKPOINT_COMMENT = "qm turn end";
const KEEPALIVE_EXPIRE = "5m";
const KEEPALIVE_RENEW_SEC = 60;
const SPRITE_CPUS = 8;
const SPRITE_DISK_GB = 100;
const PRESSURE_RECORD_FULL60 = 75;
const PRESSURE_CLEAR_FULL60 = 40;

function parsePressure(ioFull10Raw: string, ioFull60Raw: string, load1Raw: string): ExecPressure | undefined {
  const ioFull10 = Number.parseFloat(ioFull10Raw);
  const ioFull60 = Number.parseFloat(ioFull60Raw);
  const load1 = Number.parseFloat(load1Raw);
  if (!Number.isFinite(ioFull60) || ioFull60 < 0) return undefined;
  return {
    ioFull10: Number.isFinite(ioFull10) && ioFull10 >= 0 ? ioFull10 : ioFull60,
    ioFull60,
    load1: Number.isFinite(load1) && load1 >= 0 ? load1 : 0,
  };
}

function withoutSdkUrl(message: string): string {
  const url = message.lastIndexOf(" (url: ");
  return url < 0 ? message : message.slice(0, url);
}

export function spritesErrorDetail(e: unknown): string {
  if (!(e instanceof APIError)) return errMessage(e);
  const parts = [e.message];
  if (e.statusCode !== undefined) parts.push(`http ${e.statusCode}`);
  if (e.errorCode && e.errorCode !== e.message) parts.push(e.errorCode);
  const retryAfter = e.getRetryAfterSeconds();
  if (retryAfter !== undefined && !e.message.includes("retry after")) parts.push(`retry after ${retryAfter}s`);
  return parts.join("; ");
}

export async function retrySpritesControl<T>(operation: () => Promise<T>, timeoutMs = 60_000): Promise<T> {
  const signal = AbortSignal.timeout(timeoutMs);
  for (let attempt = 1; ; attempt++) {
    try {
      return await withAbort(operation, signal);
    } catch (error) {
      signal.throwIfAborted();
      if (!(error instanceof APIError) || ![429, 500, 502, 503, 504].includes(error.statusCode ?? 0) || attempt >= 4)
        throw error;
      const seconds = error.getRetryAfterSeconds();
      const delay = seconds === undefined ? undefined : retryAfterMs(new Headers({ "retry-after": String(seconds) }));
      try {
        await wait(delay ?? jitteredBackoffMs(attempt), undefined, { signal });
      } catch (error) {
        signal.throwIfAborted();
        throw error;
      }
    }
  }
}

const isMissing = (e: unknown): boolean => e instanceof APIError && e.statusCode === 404;

const describeCheck = (check: SpriteCheck): string => `${check.status}${check.reason ? ` (${check.reason})` : ""}`;

export function processKeepaliveScript(processId: string): string {
  const task = `qm-proc-${processId}`;
  const api = "curl -sf --unix-socket /.sprite/api.sock -H 'Content-Type: application/json'";
  const loop = [
    `P="${processSessionDir(processId)}"`,
    `while [ ! -f "$P/code" ] && kill -0 "$(cat "$P/pid" 2>/dev/null)" 2>/dev/null; do`,
    `  ${api} -X PUT -d '{"expire":"${KEEPALIVE_EXPIRE}"}' http://sprite/v1/tasks/${task} >/dev/null 2>&1 || true`,
    `  i=0; while [ $i -lt ${KEEPALIVE_RENEW_SEC} ] && [ ! -f "$P/code" ]; do sleep 1; i=$((i+1)); done`,
    `done`,
    `${api} -X DELETE http://sprite/v1/tasks/${task} >/dev/null 2>&1 || true`,
  ].join("\n");
  return `LOOP=${shq(loop)}; if command -v setsid >/dev/null 2>&1; then setsid sh -c "$LOOP" >/dev/null 2>&1 & else sh -c "$LOOP" >/dev/null 2>&1 & fi; echo OK`;
}

export interface SpritesSandboxOptions extends BlobStagingOptions {
  token?: string;
  baseUrl?: string;
  namePrefix?: string;
  defaultTimeoutSec?: number;
  egressProxyUrl?: string;
  egressProxyAdditionalUrls?: string[];
  memoryMb?: number;
  checkpointIntervalMs?: number;
  snapshots?: HomeSnapshotStore;
  initializationStore: DurableMap<{ pending: boolean }>;
  advisoryLock: AdvisoryLock;
  extraTools?: string[];
  credentialPaths?: CredentialPathSpec[];
  connectorSdk?: () => Promise<ConnectorSdkBundle>;
  layerToolFiles?: () => readonly LayerInstallFile[];
  onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;
}

interface RawExec {
  rc: number;
  stdout: Buffer;
  stderr: Buffer;
}

export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSandboxOptions): Sandbox {
  if (!opts.token) throw new Error("SANDBOX_BACKEND=sprites requires SPRITES_TOKEN");
  const client = new SpritesClient(opts.token, opts.baseUrl ? { baseURL: opts.baseUrl } : {});
  const sprite = (name: string) => client.sprite(name);
  const prefix = opts.namePrefix ?? "qm";
  const initializationStore = opts.initializationStore;
  const advisoryLock = opts.advisoryLock;
  const withLifecycle = <T>(name: string, fn: () => Promise<T>): Promise<T> =>
    advisoryLock.withLock(`sprites-lifecycle:${name}`, fn);
  const requireInitialized = async (name: string): Promise<void> => {
    if ((await initializationStore.get(name))?.pending)
      throw new Error(`sprites ${name}: initialization is incomplete; refusing to capture or restore a checkpoint`);
  };
  const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;
  const checkpointIntervalMs = opts.checkpointIntervalMs ?? CHECKPOINT_INTERVAL_MS;

  const ensured = new Set<string>();
  const resourcesApplied = new Set<string>();
  const egressPolicyByName = new Map<string, string>();
  const pressureEpisodes = new Set<string>();
  const checkpointBooks = new Map<string, SnapshotBookkeeping>();

  const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {
    try {
      opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });
    } catch (e) {
      swallow(`sprites-sandbox: ${code} report`, e);
    }
  };

  async function attempt<T>(what: string, fn: () => Promise<T>): Promise<T> {
    try {
      return await fn();
    } catch (e) {
      throw new Error(`sprites ${what}: ${spritesErrorDetail(e)}`, { cause: e });
    }
  }

  async function spawnExec(name: string, argv: string[], stdin: Buffer, deadlineMs: number): Promise<RawExec> {
    const cmd = sprite(name).spawn(argv[0]!, argv.slice(1));
    let submissionStarted = false;
    const out: Buffer[] = [];
    const err: Buffer[] = [];
    const collect = (stream: Readable, into: Buffer[]): Promise<void> =>
      new Promise((resolve) => {
        stream.on("data", (chunk: Buffer) => into.push(chunk));
        stream.once("end", resolve);
      });
    const exited = new Promise<number>((resolve, reject) => {
      const fail = (detail: string): void => {
        const phase = submissionStarted ? "script submission started; execution unknown" : "script not submitted";
        reject(new Error(`sprites exec: WebSocket error (${phase}): ${detail}`));
      };
      cmd.on("error", (e) => fail(withoutSdkUrl(e.message)));
      cmd.once("exit", (code) => {
        if (code < 0) fail("connection closed without an exit status");
        else resolve(code);
      });
      cmd.once("spawn", () => {
        submissionStarted = true;
        cmd.stdin.end(stdin);
      });
    });
    try {
      const [rc] = await withTimeout(
        () => Promise.all([exited, collect(cmd.stdout, out), collect(cmd.stderr, err)]),
        deadlineMs,
        "sprites exec",
      );
      return { rc, stdout: Buffer.concat(out), stderr: Buffer.concat(err) };
    } catch (e) {
      cmd.kill();
      throw e;
    } finally {
      cmd.close();
    }
  }

  async function execRaw(name: string, script: string, timeoutSec: number): Promise<ExecResult> {
    const uid = randomUUID();
    const out = `/tmp/.exec-${uid}.out`;
    const err = `/tmp/.exec-${uid}.err`;
    const psi = `$(awk -F'[= ]+' '/^full/{print $3, $5; f=1} END{if(!f)print "-1 -1"}' /proc/pressure/io 2>/dev/null || echo '-1 -1')`;
    const load = `$(cut -d' ' -f1 /proc/loadavg 2>/dev/null || echo -1)`;
    const body = `/tmp/.exec-${uid}.sh`;
    const eof = `__QM_EOF_${uid}__`;
    const wrapped =
      `cat > ${body} <<'${eof}'\n${script}\n${eof}\n` +
      `timeout ${timeoutSec} sh ${body} > ${out} 2> ${err}; __rc=$?; printf '%s %s %s %s %s\\n' "$__rc" "$(wc -c < ${out})" "$(wc -c < ${err})" "${psi}" "${load}"; cat ${out} ${err}; rm -f ${out} ${err} ${body}`;
    const r = await spawnExec(
      name,
      ["sh", "-c", SCRIPT_RUNNER],
      Buffer.from(wrapped, "utf8"),
      timeoutSec * 1000 + EXIT_GRACE_MS,
    );
    const nl = r.stdout.indexOf(0x0a);
    const header = (nl < 0 ? r.stdout : r.stdout.subarray(0, nl)).toString("utf8").trim().split(/\s+/);
    if (r.rc !== 0 || nl < 0 || header.length < 3) {
      const detail = r.stderr.toString("utf8").trim() || r.stdout.toString("utf8").slice(0, 120);
      throw new Error(`sprites exec ${name}: bad envelope (rc=${r.rc}): ${detail}`);
    }
    const pressure = header.length === 6 ? parsePressure(header[3]!, header[4]!, header[5]!) : undefined;
    if (pressure) notePressure(name, pressure);
    const code = Number.parseInt(header[0]!, 10);
    const outLen = Number.parseInt(header[1]!, 10);
    const errLen = Number.parseInt(header[2]!, 10);
    const start = nl + 1;
    const outBuf = r.stdout.subarray(start, start + outLen);
    const errBuf = r.stdout.subarray(start + outLen, start + outLen + errLen);
    if (outBuf.length !== outLen || errBuf.length !== errLen) {
      throw new Error(
        `sprites exec ${name}: truncated stream (${outBuf.length}/${outLen} out, ${errBuf.length}/${errLen} err)`,
      );
    }
    return {
      stdout: outBuf.toString("utf8"),
      stderr: errBuf.toString("utf8"),
      code,
      timedOut: code === 124,
      ...(pressure ? { pressure } : {}),
    };
  }

  function notePressure(name: string, pressure: ExecPressure): void {
    if (pressure.ioFull60 >= PRESSURE_RECORD_FULL60 && !pressureEpisodes.has(name)) {
      pressureEpisodes.add(name);
      reportError(
        "agent_computer",
        "io_pressure_high",
        `sprite ${name}: io pressure full avg60=${pressure.ioFull60}% (load ${pressure.load1}) — disk-bound work is stalling`,
        base.scopeFor(name),
      );
    } else if (pressure.ioFull60 < PRESSURE_CLEAR_FULL60) {
      pressureEpisodes.delete(name);
    }
  }

  async function writeAbsBytes(name: string, absPath: string, data: Uint8Array): Promise<void> {
    const fs = sprite(name).filesystem();
    const tmp = `${absPath}.part.${randomUUID()}`;
    await attempt(`write ${absPath} failed`, async () => {
      await fs.writeFile(tmp, Buffer.from(data));
      await fs.rename(tmp, absPath);
    });
  }

  async function readAbsBytes(name: string, absPath: string): Promise<Uint8Array | null> {
    try {
      return await sprite(name).filesystem().readFile(absPath, null);
    } catch (e) {
      if ((e as { code?: string }).code === "ENOENT") return null;
      throw new Error(`sprites read ${absPath} failed: ${spritesErrorDetail(e)}`, { cause: e });
    }
  }

  const norm = (domain?: string): string => (domain ?? "").toLowerCase().replace(/\.$/, "");
  const egressProxyHosts = opts.egressProxyUrl
    ? [
        ...new Set(
          [opts.egressProxyUrl, ...(opts.egressProxyAdditionalUrls ?? [])].map((value) => {
            const url = new URL(value);
            if (!["https:", "http:"].includes(url.protocol) || !url.hostname || url.username || url.password)
              throw new Error("sprites egress proxy URLs must be HTTP(S) URLs without credentials");
            return norm(url.hostname);
          }),
        ),
      ].sort()
    : [];
  if (!opts.egressProxyUrl && opts.egressProxyAdditionalUrls?.length)
    throw new Error("sprites additional egress proxy URLs require a primary proxy URL");

  async function ensureEgressPolicy(name: string): Promise<void> {
    const rules = egressProxyHosts.map((domain) => ({ domain, action: "allow" as const }));
    const want = JSON.stringify(rules);
    if (egressPolicyByName.get(name) === want) return;
    const s = sprite(name);
    await attempt(`egress policy ${name}`, () => retrySpritesControl(() => s.updateNetworkPolicy({ rules })));
    const got = await attempt(`egress policy readback ${name}`, () => retrySpritesControl(() => s.getNetworkPolicy()));
    const actual = got.rules ?? [];
    const bound =
      actual.length === rules.length &&
      actual.every((rule) => rule.action?.toLowerCase() === "allow") &&
      JSON.stringify(actual.map((rule) => norm(rule.domain)).sort()) === JSON.stringify(egressProxyHosts);
    if (!bound)
      throw new Error(`sprites egress policy ${name}: readback mismatch (${JSON.stringify(got).slice(0, 200)})`);
    egressPolicyByName.set(name, want);
  }

  async function applyResources(name: string): Promise<void> {
    if (opts.memoryMb === undefined || resourcesApplied.has(name)) return;
    await attempt(`resources policy ${name}`, () =>
      sprite(name).updateResourcesPolicy({ memory: { limitMB: opts.memoryMb! } }),
    );
    resourcesApplied.add(name);
  }

  async function spriteExists(name: string): Promise<boolean> {
    try {
      await retrySpritesControl(() => client.getSprite(name));
      return true;
    } catch (e) {
      if (isMissing(e)) return false;
      throw new Error(`sprites get ${name}: ${spritesErrorDetail(e)}`, { cause: e });
    }
  }

  async function deleteSprite(name: string): Promise<void> {
    try {
      await retrySpritesControl(() => client.deleteSprite(name));
    } catch (e) {
      if (!isMissing(e)) throw new Error(`sprites delete ${name}: ${spritesErrorDetail(e)}`, { cause: e });
    }
  }

  const homeSnapshots = opts.snapshots
    ? createHomeSnapshotOps<string>({
        label: "sprites",
        homeDir: HOME_DIR,
        homeTarPath: HOME_TAR,
        prunePaths: [
          ...HOME_SNAPSHOT_PRUNE,
          ...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`),
        ],
        store: opts.snapshots,
        io: {
          runCommand: async (name, script, timeoutMs) => {
            const r = await execRaw(name, script, Math.ceil(timeoutMs / 1000));
            return { exitCode: r.code, stdout: r.stdout, stderr: r.stderr };
          },
          readFileBytes: readAbsBytes,
          writeFileBytes: writeAbsBytes,
        },
      })
    : undefined;

  async function exportHome(name: string, scope: string): Promise<void> {
    await requireInitialized(name);
    if (!homeSnapshots || !(await spriteExists(name))) return;
    await homeSnapshots.snapshotHome(scope, name);
  }

  async function settleStream(stream: AsyncIterable<StreamMessage>, what: string): Promise<void> {
    let failure: string | undefined;
    for await (const msg of stream) {
      if (msg.type === "error") failure = msg.error ?? msg.data ?? "error";
      else if (msg.type === "complete") failure = undefined;
    }
    if (failure !== undefined) throw new Error(`${what}: ${failure}`);
  }

  async function createCheckpoint(name: string): Promise<void> {
    await settleStream(await sprite(name).createCheckpoint(CHECKPOINT_COMMENT), `sprites checkpoint ${name}`);
  }

  async function latestCheckpoint(name: string): Promise<Checkpoint | undefined> {
    const list = await retrySpritesControl(() => sprite(name).listCheckpoints());
    return list
      .filter((c) => c.id !== "Current" && !c.health)
      .sort((a, b) => b.createTime.getTime() - a.createTime.getTime())[0];
  }

  async function restoreLatestCheckpoint(name: string): Promise<Checkpoint | undefined> {
    const latest = await latestCheckpoint(name);
    if (!latest) return undefined;
    await settleStream(await sprite(name).restoreCheckpoint(latest.id), `sprites restore ${name} ${latest.id}`);
    return latest;
  }

  async function checkpointIfDue(name: string, tdOpts?: TeardownOptions): Promise<void> {
    await requireInitialized(name);
    const book = checkpointBooks.get(name) ?? {};
    checkpointBooks.set(name, book);
    if (!tdOpts?.homeUnchanged) book.homeDirty = true;
    if (!snapshotDue(book, tdOpts, checkpointIntervalMs)) return;
    try {
      await createCheckpoint(name);
      book.lastSnapshotMs = Date.now();
      book.homeDirty = false;
    } catch (e) {
      reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), base.scopeFor(name));
    }
  }

  const forget = (name: string): void => {
    ensured.delete(name);
    resourcesApplied.delete(name);
    pressureEpisodes.delete(name);
    egressPolicyByName.delete(name);
    checkpointBooks.delete(name);
  };

  const base = createExecSandboxBase({
    workspace,
    label: "sprites",
    prefix,
    homeDir: HOME_DIR,
    defaultTimeoutSec,
    credentialPaths: opts.credentialPaths ?? [],
    installLayerTools: withConnectorSdk(
      HOME_DIR,
      createLayerToolInstaller(opts.layerToolFiles ?? (() => [])),
      opts.connectorSdk,
    ),
    combineLayerToolPrep: true,
    egressProxyUrl: opts.egressProxyUrl,
    deleteFailureCode: "sprite_delete_failed",
    onError: opts.onError,
    exec: execRaw,
    writeAbsBytes,
    readAbsBytes,
    ensureResident(name, onStatus) {
      return withLifecycle(name, async () => {
        const pending = (await initializationStore.get(name))?.pending;
        if (ensured.has(name) && !pending) return { coldStart: false };
        const exists = await spriteExists(name);
        if (exists && !pending) {
          await applyResources(name);
          ensured.add(name);
          return { coldStart: false };
        }
        await initializationStore.put(name, { pending: true });
        onStatus?.("Preparing the sandbox…");
        const scope = base.scopeFor(name);
        try {
          if (!exists) await attempt(`create ${name}`, () => client.createSprite(name, { waitForCapacity: true }));
          await applyResources(name);
          const hydrated = homeSnapshots && scope ? await homeSnapshots.hydrateHome(scope, name) : false;
          await initializationStore.delete(name);
          ensured.add(name);
          return { coldStart: !hydrated };
        } catch (e) {
          forget(name);
          reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);
          await deleteSprite(name).catch((deleteErr) =>
            swallow("sprites-sandbox: delete after failed hydrate", deleteErr),
          );
          throw e;
        }
      });
    },
    isProvisioned: (name) => ensured.has(name),
    async recreateScratch(name) {
      if (await spriteExists(name)) await deleteSprite(name);
      await attempt(`create ${name}`, () => client.createSprite(name, { waitForCapacity: true }));
      await applyResources(name);
      ensured.add(name);
    },
    deleteInstance: deleteSprite,
    forgetInstance: forget,
    ensureEgress: ensureEgressPolicy,
  });

  const profile: AgentComputerProfile = {
    backend: "sprites",
    writablePersistence: "resident_disk",
    processSessions: true,
    egressEnforcement: opts.egressProxyUrl ? "domain" : "none",
    spec: {
      os: "Ubuntu (25.10 for newly created sprites; existing sprites keep the release they were created with) — Fly Sprite microVM: the whole disk persists and idle sprites sleep; background processes are held awake while they run but do not survive a cold wake or restart",
      runtimes: ["Node 24", "Python 3"],
      get tools() {
        return visibleTools(["git", "curl", "jq", "tar", "python3", ...(opts.extraTools ?? [])]);
      },
      get notInstalled() {
        return visibleNotInstalled(["gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], opts.extraTools ?? []);
      },
      cpus: SPRITE_CPUS,
      ...(opts.memoryMb !== undefined ? { memoryMb: opts.memoryMb } : {}),
      diskGb: SPRITE_DISK_GB,
      homeDir: HOME_DIR,
      workdir: base.workspaceDir,
    },
  };

  const procIo: ExecProcessIo = {
    async run(handle, command, execOpts): Promise<ExecResult> {
      const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
      return execRaw(handle.id, command, timeoutSec);
    },
  };
  const procSessions = createExecProcessSessions(procIo);

  const execFileOps = createExecFileOps({
    combineRemoveAndList: true,
    label: "sprites",
    exec: (id, script, t) => execRaw(id, script, t),
    writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),
  });

  const blobStaging = createBackendBlobStaging("sprites", (id, script, t) => execRaw(id, script, t), opts);

  const execExport = createExecExport({
    label: "sprites",
    exec: (id, script, t) => execRaw(id, script, t),
    readAbsBytes,
    defaultHomeDir: HOME_DIR,
    ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),
  });

  async function startProcess(
    handle: SandboxHandle,
    command: string,
    spOpts?: StartProcessOptions,
  ): Promise<{ processId: string }> {
    const started = await procSessions.startProcess(handle, command, spOpts);
    try {
      const r = await execRaw(handle.id, processKeepaliveScript(started.processId), KEEPALIVE_TIMEOUT_SEC);
      if (r.code !== 0 || !r.stdout.includes("OK")) throw new Error(r.stderr.trim() || `exit ${r.code}`);
    } catch (e) {
      reportError(
        "agent_computer",
        "process_keepalive_failed",
        `sprite ${handle.id}: process ${started.processId} runs without a Tasks hold and may freeze when the sprite idles: ${errMessage(e)}`,
        base.scopeFor(handle.id),
      );
    }
    return started;
  }

  return {
    profile,
    startProcess,
    readProcess: procSessions.readProcess,
    writeStdin: procSessions.writeStdin,
    signalProcess: procSessions.signalProcess,
    listProcesses: procSessions.listProcesses,
    ...execFileOps,
    ...blobStaging,
    provision: base.provision,
    run: base.run,
    writeFileBytes: base.writeFileBytes,
    writeFile: base.writeFile,
    readFileBytes: base.readFileBytes,
    readFile: base.readFile,
    exportFiles: execExport.exportFiles,

    async destroyScope(scopeId: string): Promise<void> {
      const name = sandboxScopeName(prefix, scopeId);
      return base.provisionQueue(scopeId, () =>
        withLifecycle(name, async () => {
          if (!(await initializationStore.get(name))?.pending) await exportHome(name, scopeId);
          await deleteSprite(name);
          forget(name);
        }),
      );
    },

    ...(homeSnapshots
      ? {
          async persistHomeSnapshot(scopeId: string): Promise<void> {
            const name = sandboxScopeName(prefix, scopeId);
            return base.provisionQueue(scopeId, () =>
              withLifecycle(name, async () => {
                await requireInitialized(name);
                if (!(await spriteExists(name)))
                  throw new Error(`sprites persistHomeSnapshot: no sprite for ${scopeId}`);
                await homeSnapshots.snapshotHome(scopeId, name);
              }),
            );
          },
        }
      : {}),

    async computerStatus(scopeId: string): Promise<ComputerStatus> {
      const name = sandboxScopeName(prefix, scopeId);
      const [checked, latest] = await Promise.allSettled([
        retrySpritesControl(() => sprite(name).check()),
        latestCheckpoint(name),
      ]);
      if (checked.status === "rejected" && isMissing(checked.reason)) {
        return { machine: "no sprite provisioned yet", provisioned: false, guestResponsive: false };
      }
      const machine =
        checked.status === "fulfilled"
          ? describeCheck(checked.value)
          : `check failed: ${spritesErrorDetail(checked.reason)}`;
      const recovery: ComputerStatus["recovery"] = {
        strategy: "provider_snapshot",
        checkpointExpiresAtMs: null,
        ...(latest.status === "fulfilled" && latest.value
          ? { checkpointId: latest.value.id, checkpointAtMs: latest.value.createTime.getTime() }
          : {}),
        ...(latest.status === "rejected" ? { error: `checkpoint list failed: ${errMessage(latest.reason)}` } : {}),
      };
      let guestResponsive = false;
      let pressure: ExecPressure | undefined;
      let probeError: string | undefined;
      try {
        const probe = await execRaw(name, "true", GUEST_PROBE_TIMEOUT_SEC);
        guestResponsive = probe.code === 0;
        pressure = probe.pressure;
      } catch (e) {
        probeError = errMessage(e);
      }
      return {
        machine,
        provisioned: checked.status === "fulfilled",
        guestResponsive,
        recovery,
        ...(pressure ? { pressure } : {}),
        ...(probeError ? { probeError } : {}),
      };
    },

    restartComputer(scopeId: string): Promise<void> {
      const name = sandboxScopeName(prefix, scopeId);
      return base.provisionQueue(`restart:${name}`, () =>
        withLifecycle(name, async () => {
          await requireInitialized(name);
          forget(name);
          const s = sprite(name);
          const restartFailure = await s.restart().then(
            () => undefined,
            (e) => spritesErrorDetail(e),
          );
          if (restartFailure === undefined) return;
          let check: SpriteCheck;
          try {
            check = await retrySpritesControl(() => s.check());
          } catch (e) {
            throw new Error(
              `sprites restart ${name}: ${restartFailure}; check failed: ${spritesErrorDetail(e)}; no checkpoint was restored`,
              { cause: e },
            );
          }
          const fault = check.status === "unhealthy" ? `check reports ${describeCheck(check)}` : undefined;
          if (fault === undefined) {
            throw new Error(
              `sprites restart ${name}: ${restartFailure}; the health check reports no fault, so no checkpoint was restored`,
            );
          }
          let restored: Checkpoint | undefined;
          try {
            restored = await restoreLatestCheckpoint(name);
          } catch (e) {
            throw new Error(
              `sprites restart ${name}: ${restartFailure}; ${fault}; checkpoint restore failed: ${errMessage(e)}`,
              {
                cause: e,
              },
            );
          }
          if (!restored)
            throw new Error(`sprites restart ${name}: ${restartFailure}; ${fault}; no checkpoint to restore`);
          reportError(
            "agent_computer",
            "checkpoint_restored",
            `sprite ${name}: restart failed (${restartFailure}); ${fault}; restored checkpoint ${restored.id} from ${restored.createTime.toISOString()}`,
            base.scopeFor(name) ?? scopeId,
          );
        }),
      );
    },

    async teardown(handle: SandboxHandle, tdOpts?: TeardownOptions): Promise<void> {
      return withLifecycle(handle.id, async () => {
        if (!handle.scratch) {
          if (tdOpts?.destroy) {
            if (!(await initializationStore.get(handle.id))?.pending)
              await exportHome(handle.id, base.scopeFor(handle.id) ?? handle.id);
          } else await checkpointIfDue(handle.id, tdOpts);
        }
        return base.teardown(handle, tdOpts);
      });
    },
  };
}