Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

22.6 KB
Raw Blame
import { cleanupFailedProvision } from "./sandbox.ts";
import { DeleteObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { orgId as configOrgId } from "../config.ts";
import type { WorkspaceLayer } from "../types.ts";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
import type { AdvisoryLock } from "../persistence/advisory-lock.ts";
import type { DurableMap } from "../persistence/durable-map.ts";
import { createKeyedQueue } from "../util/async.ts";
import { scopeStorageKey } from "../../cli/src/scope-storage-key.ts";
import { swallowAs, errMessage } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { nonInteractiveShellPrefix } from "./sandbox-env.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
import { materializeRoLayers } from "./ro-layers.ts";
import {
  createBackendBlobStaging,
  createExecExport,
  createExecFileOps,
  posixJoin,
  type BlobStagingOptions,
} from "./exec-file-ops.ts";
import { AwsApiError, createMicrovmApi, createMicrovmClient, type AwsMicrovmApi } from "./aws-microvm-api.ts";
import type {
  AgentComputerProfile,
  ExecOptions,
  ExecResult,
  ProvisionOptions,
  Sandbox,
  SandboxHandle,
  TeardownOptions,
} from "./sandbox.ts";
import { execFailureDetail, visibleNotInstalled, visibleTools } from "./sandbox.ts";
import { createHomeSnapshotOps, createS3SnapshotStore, HOME_SNAPSHOT_PRUNE, snapshotDue } from "./home-snapshot.ts";
import {
  ephemeralCredLinkPaths,
  ephemeralCredLinkScript,
  type CredentialPathSpec,
} from "../credentials/resident-paths.ts";

const HOME_DIR = "/root";
const WORKSPACE_BASENAME = "workspace";
const WORKSPACE_DIR = `${HOME_DIR}/${WORKSPACE_BASENAME}`;
const PREP_TIMEOUT_SEC = 30;
const HOME_TAR = "/tmp/agent-home.tar";
const RO_LAYERS_TAR = ".ro-layers.tar";
const RO_LAYERS_MANIFEST = ".ro-layers.manifest";
const SNAPSHOT_PRUNE = [...HOME_SNAPSHOT_PRUNE, "./.npm", "./.aws"];

const DEFAULT_INGRESS = (region: string) =>
  `arn:aws:lambda:${region}:aws:network-connector:aws-network-connector:ALL_INGRESS`;
const DEFAULT_EGRESS = (region: string) =>
  `arn:aws:lambda:${region}:aws:network-connector:aws-network-connector:INTERNET_EGRESS`;

export interface StoredMicrovm {
  microvmId: string;
  endpoint: string;
  imageVersion?: string;
  createdAtMs: number;
  lastSnapshotMs?: number;
  lastActivityMs?: number;
  homeDirty?: boolean;
  provisioning?: boolean;
  orgId?: string;
}

export interface AwsSandboxOptions extends BlobStagingOptions {
  region: string;
  profile?: string;
  imageIdentifier: string;
  imageVersion?: string;
  executionRoleArn?: string;
  ingressConnectorArns?: string[];
  egressConnectorArns?: string[];
  s3Bucket: string;
  s3Prefix?: string;
  agentPort?: number;
  maxIdleDurationSeconds?: number;
  suspendedDurationSeconds?: number;
  maximumDurationInSeconds?: number;
  rotateAfterSeconds?: number;
  snapshotIntervalMs?: number;
  defaultTimeoutSec?: number;
  cpus?: number;
  memoryMb?: number;
  diskGb?: number;
  store: DurableMap<StoredMicrovm>;
  advisoryLock: AdvisoryLock;
  extraTools?: string[];
  credentialPaths?: CredentialPathSpec[];
  onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;
  api?: AwsMicrovmApi;
  s3?: Pick<S3Client, "send">;
  fetchImpl?: typeof fetch;
}

interface BodyRef {
  microvmId: string;
  endpoint: string;
}

export function createAwsSandbox(workspace: WorkspaceStore, opts: AwsSandboxOptions): Sandbox {
  const region = opts.region;
  const api =
    opts.api ??
    createMicrovmApi({
      region,
      ...(opts.profile ? { profile: opts.profile } : {}),
      ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
    });
  const s3: Pick<S3Client, "send"> =
    opts.s3 ?? new S3Client({ region, ...(opts.profile ? { profile: opts.profile } : {}) });
  const store = opts.store;
  const advisoryLock = opts.advisoryLock;
  const provisionQueue = createKeyedQueue<string>();
  const withScopeLock = <T>(scope: string, run: () => Promise<T>): Promise<T> =>
    provisionQueue(scope, () => advisoryLock.withLock(`aws-provision:${scope}`, run));
  const onError = opts.onError;

  const ingress = opts.ingressConnectorArns?.length ? opts.ingressConnectorArns : [DEFAULT_INGRESS(region)];
  const egress = opts.egressConnectorArns?.length ? opts.egressConnectorArns : [DEFAULT_EGRESS(region)];
  const s3Prefix = (opts.s3Prefix ?? "sandbox-home").replace(/\/+$/, "");
  const agentPort = opts.agentPort ?? 8080;
  const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;
  const maximumDurationInSeconds = opts.maximumDurationInSeconds ?? 28_800;
  const rotateAfterMs = (opts.rotateAfterSeconds ?? 27_000) * 1000;
  const snapshotIntervalMs = opts.snapshotIntervalMs ?? 0;
  const cpus = opts.cpus ?? 4;
  const memoryMb = opts.memoryMb ?? 8192;
  const diskGb = opts.diskGb ?? 8;
  const credentialPaths = opts.credentialPaths ?? [];

  const endpointById = new Map<string, string>();
  const scopeByMicrovm = new Map<string, string>();
  const scratchByKey = new Map<string, BodyRef>();
  const activeByMicrovm = new Map<string, number>();

  function reportError(category: string, code: string, message: string, scopeLabel?: string): void {
    onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });
  }

  const s3KeyFor = (scope: string): string => `${s3Prefix}/${scopeStorageKey(scope)}.tar`;

  const client = createMicrovmClient(api, {
    agentPort,
    tokenTtlMinutes: 30,
    ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
  });

  async function resolveEndpoint(id: string): Promise<string> {
    const cached = endpointById.get(id);
    if (cached) return cached;
    const desc = await api.getMicrovm(id);
    if (!desc.endpoint) throw new Error(`microVM ${id} has no endpoint`);
    endpointById.set(id, desc.endpoint);
    return desc.endpoint;
  }

  async function execRaw(id: string, command: string, timeoutSec: number): Promise<ExecResult> {
    return client.execRaw(id, await resolveEndpoint(id), command, timeoutSec);
  }

  async function writeAbsBytes(id: string, absPath: string, data: Uint8Array): Promise<void> {
    await client.writeAbs(id, await resolveEndpoint(id), absPath, data);
  }

  async function readAbsBytes(id: string, absPath: string): Promise<Uint8Array | null> {
    const res = await client.daemon(id, await resolveEndpoint(id), "/read", { path: absPath });
    if (res.status === 404) return null;
    if (res.status !== 200)
      throw new Error(`microVM read ${absPath} failed (${res.status}): ${res.text.slice(0, 200)}`);
    return Buffer.from((JSON.parse(res.text) as { b64: string }).b64, "base64");
  }

  async function ensureRunning(id: string, observed?: Awaited<ReturnType<AwsMicrovmApi["getMicrovm"]>>): Promise<void> {
    await client.ensureRunning(id, await resolveEndpoint(id), observed);
  }

  const homeSnapshots = createHomeSnapshotOps<string>({
    label: "aws",
    homeDir: HOME_DIR,
    homeTarPath: HOME_TAR,
    prunePaths: [...SNAPSHOT_PRUNE, ...ephemeralCredLinkPaths(credentialPaths).map(({ rel }) => `./${rel}`)],
    store: createS3SnapshotStore({ bucket: opts.s3Bucket, prefix: s3Prefix, s3, keyFor: s3KeyFor }),
    io: {
      runCommand: async (id, script, timeoutMs) => {
        const r = await execRaw(id, script, Math.ceil(timeoutMs / 1000));
        return { exitCode: r.code, stdout: r.stdout, stderr: r.stderr };
      },
      readFileBytes: readAbsBytes,
      writeFileBytes: writeAbsBytes,
    },
  });

  const snapshotHome = (scope: string, id: string): Promise<void> => homeSnapshots.snapshotHome(scope, id);
  const hydrateHome = (scope: string, id: string): Promise<boolean> => homeSnapshots.hydrateHome(scope, id);

  let resolvedImageArn: string | undefined;
  async function imageArn(): Promise<string> {
    if (resolvedImageArn) return resolvedImageArn;
    if (opts.imageIdentifier.startsWith("arn:")) {
      resolvedImageArn = opts.imageIdentifier;
      return resolvedImageArn;
    }
    const img = await api.findImage(opts.imageIdentifier);
    if (!img) throw new Error(`AWS sandbox image not found: ${opts.imageIdentifier}`);
    resolvedImageArn = img.imageArn;
    return resolvedImageArn;
  }

  async function launchBody(scope: string | undefined): Promise<{ id: string; endpoint: string }> {
    const run = await api.runMicrovm({
      imageIdentifier: await imageArn(),
      ...(opts.imageVersion ? { imageVersion: opts.imageVersion } : {}),
      ingressNetworkConnectors: ingress,
      egressNetworkConnectors: egress,
      ...(opts.executionRoleArn ? { executionRoleArn: opts.executionRoleArn } : {}),
      idlePolicy: {
        autoResumeEnabled: true,
        maxIdleDurationSeconds: opts.maxIdleDurationSeconds ?? 900,
        suspendedDurationSeconds: opts.suspendedDurationSeconds ?? 3600,
      },
      maximumDurationInSeconds,
      clientToken: `${scope ?? "scratch"}-${Date.now()}`,
    });
    try {
      if (scope)
        await store.put(scope, {
          microvmId: run.microvmId,
          endpoint: run.endpoint ?? "",
          createdAtMs: Date.now(),
          orgId: configOrgId(),
          provisioning: true,
        });
      const ready = await api.waitForState(run.microvmId, "RUNNING");
      const endpoint = run.endpoint ?? ready.endpoint;
      if (!endpoint) throw new Error(`microVM ${run.microvmId} has no endpoint`);
      endpointById.set(run.microvmId, endpoint);
      await client.waitDaemon(run.microvmId, endpoint);
      return { id: run.microvmId, endpoint };
    } catch (error) {
      try {
        await api.terminate(run.microvmId);
      } catch (cleanupError) {
        throw new AggregateError([error, cleanupError], `AWS launch ${run.microvmId} failed and termination failed`, {
          cause: cleanupError,
        });
      }
      throw error;
    }
  }

  async function ensureBody(scope: string): Promise<{ id: string; endpoint: string; coldStart: boolean }> {
    return withScopeLock(scope, async () => {
      const stored = await store.get(scope);
      if (stored) {
        const desc = await api.tryGetMicrovm(stored.microvmId);
        const alive = desc && desc.state !== "TERMINATED" && desc.state !== "TERMINATING";
        const stale = Date.now() - stored.createdAtMs > rotateAfterMs;
        if (alive && stored.provisioning)
          throw new Error(`AWS sandbox ${stored.microvmId} has incomplete provisioning; retire it before retrying`);
        if (alive && !stale) {
          endpointById.set(stored.microvmId, stored.endpoint);
          scopeByMicrovm.set(stored.microvmId, scope);
          await ensureRunning(stored.microvmId, desc);
          return { id: stored.microvmId, endpoint: stored.endpoint, coldStart: false };
        }
        if (alive && stale) {
          endpointById.set(stored.microvmId, stored.endpoint);
          await ensureRunning(stored.microvmId).catch(() => {});
          await snapshotHome(scope, stored.microvmId).catch((e) =>
            reportError("sandbox_snapshot", "rotate_snapshot_failed", errMessage(e), scope),
          );
          await api.terminate(stored.microvmId).catch(() => {});
        }
      }
      const body = await launchBody(scope);
      scopeByMicrovm.set(body.id, scope);
      let hydrated: boolean;
      try {
        hydrated = await hydrateHome(scope, body.id);
      } catch (e) {
        reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);
        scopeByMicrovm.delete(body.id);
        endpointById.delete(body.id);
        client.evict(body.id);
        await api.terminate(body.id).catch(swallowAs("aws-sandbox: terminate after failed hydrate", undefined));
        throw new Error(`aws provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`, {
          cause: e,
        });
      }
      await store.put(scope, {
        microvmId: body.id,
        endpoint: body.endpoint,
        ...(opts.imageVersion ? { imageVersion: opts.imageVersion } : {}),
        createdAtMs: Date.now(),
        ...(hydrated ? { lastSnapshotMs: Date.now(), homeDirty: false } : {}),
        orgId: configOrgId(),
      });
      return { id: body.id, endpoint: body.endpoint, coldStart: !hydrated };
    });
  }

  async function ensureScratch(key: string): Promise<{ id: string; endpoint: string; coldStart: boolean }> {
    return provisionQueue(`scratch:${key}`, async () => {
      const existing = scratchByKey.get(key);
      if (existing) {
        const desc = await api.tryGetMicrovm(existing.microvmId);
        if (desc && desc.state !== "TERMINATED" && desc.state !== "TERMINATING") {
          endpointById.set(existing.microvmId, existing.endpoint);
          await ensureRunning(existing.microvmId, desc);
          return { id: existing.microvmId, endpoint: existing.endpoint, coldStart: false };
        }
      }
      const body = await launchBody(undefined);
      scratchByKey.set(key, { microvmId: body.id, endpoint: body.endpoint });
      return { id: body.id, endpoint: body.endpoint, coldStart: true };
    });
  }

  const profile: AgentComputerProfile = {
    backend: "aws-microvm",
    writablePersistence: "snapshot_to_workspace",
    processSessions: true,
    parksOnTeardown: true,
    egressEnforcement: "none",
    spec: {
      os: "Amazon Linux 2023, glibc",
      runtimes: ["Node 24", "Python 3 (venv on PATH)"],
      get tools() {
        return visibleTools([
          "git",
          "curl",
          "wget",
          "jq",
          "unzip",
          "gnupg",
          "python3",
          "gh",
          "aws (CLI v2)",
          ...(opts.extraTools ?? []),
        ]);
      },
      get notInstalled() {
        return visibleNotInstalled(["gcloud", "kubectl", "glab"], opts.extraTools ?? []);
      },
      cpus,
      memoryMb,
      diskGb,
      homeDir: HOME_DIR,
      workdir: WORKSPACE_DIR,
    },
  };

  const procIo: ExecProcessIo = {
    async run(handle, command, execOpts): Promise<ExecResult> {
      const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
      await ensureRunning(handle.id);
      return execRaw(handle.id, command, timeoutSec);
    },
  };
  const procSessions = createExecProcessSessions(procIo);

  const execFileOps = createExecFileOps({
    label: "aws",
    exec: (id, script, t) => execRaw(id, script, t),
    writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),
  });

  const execExport = createExecExport({
    label: "aws",
    exec: (id, script, t) => execRaw(id, script, t),
    readAbsBytes,
    defaultHomeDir: HOME_DIR,
    ephemeralCredentialPrefixes: ephemeralCredLinkPaths(credentialPaths).map(({ rel }) => rel),
  });

  const blobStaging = createBackendBlobStaging("aws", (id, script, t) => execRaw(id, script, t), opts);

  async function terminateBody(id: string): Promise<void> {
    const body = await api.tryGetMicrovm(id);
    if (!body || body.state === "TERMINATED") return;
    try {
      if (body.state !== "TERMINATING") await api.terminate(id);
      await api.waitForState(id, "TERMINATED", { timeoutMs: 60_000 });
    } catch (error) {
      if (!(error instanceof AwsApiError && error.status === 404)) throw error;
    }
  }

  function destroyStoredScope(scopeId: string, expectedId?: string): Promise<void> {
    return withScopeLock(scopeId, async () => {
      const stored = await store.get(scopeId);
      const id = expectedId ?? stored?.microvmId;
      if (id) await terminateBody(id);
      if (!expectedId || !stored || stored.microvmId === expectedId) {
        await s3.send(new DeleteObjectCommand({ Bucket: opts.s3Bucket, Key: s3KeyFor(scopeId) }));
        await store.delete(scopeId);
      }
      if (id) {
        client.evict(id);
        endpointById.delete(id);
        scopeByMicrovm.delete(id);
        activeByMicrovm.delete(id);
      }
    });
  }

  const sandbox: Sandbox = {
    destroyScope: (scopeId) => destroyStoredScope(scopeId),
    profile,
    startProcess: procSessions.startProcess,
    readProcess: procSessions.readProcess,
    writeStdin: procSessions.writeStdin,
    signalProcess: procSessions.signalProcess,
    listProcesses: procSessions.listProcesses,
    ...execFileOps,
    ...blobStaging,

    async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise<SandboxHandle> {
      const scratch = provOpts?.scratch;
      const writable = layers.find((l) => l.mode === "rw") ?? layers[0];
      const scope = writable?.scopeId ?? "default";
      const body = scratch ? await ensureScratch(scratch.key) : await ensureBody(scope);
      const id = body.id;
      endpointById.set(id, body.endpoint);
      const coldStart = body.coldStart;

      const env = provOpts?.env && Object.keys(provOpts.env).length ? provOpts.env : undefined;
      const handle: SandboxHandle = {
        id,
        rootDir: WORKSPACE_DIR,
        homeDir: HOME_DIR,
        coldStart,
        ...(scratch ? { scratch: true } : {}),
        ...(env ? { env } : {}),
      };

      activeByMicrovm.set(id, (activeByMicrovm.get(id) ?? 0) + 1);
      try {
        const prepared = await execRaw(
          id,
          `mkdir -p ${shq(WORKSPACE_DIR)} && ${ephemeralCredLinkScript(HOME_DIR, credentialPaths)}`,
          PREP_TIMEOUT_SEC,
        );
        if (prepared.code !== 0)
          throw new Error(
            `AWS sandbox credential setup failed: ${execFailureDetail(prepared, PREP_TIMEOUT_SEC).slice(0, 200)}`,
          );

        await materializeRoLayers(
          workspace,
          layers,
          handle,
          {
            readFile: (h, rel) => sandbox.readFile(h, rel),
            writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data),
            exec: (script, t) => execRaw(id, script, t),
          },
          { manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "aws" },
        );

        return handle;
      } catch (error) {
        await cleanupFailedProvision(sandbox, handle, error);
        throw error;
      }
    },

    async run(handle, command, execOpts?: ExecOptions): Promise<ExecResult> {
      const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
      execOpts?.signal?.throwIfAborted();
      await ensureRunning(handle.id);
      const exports = Object.entries(handle.env ?? {})
        .map(([k, v]) => `export ${k}=${shq(v)}`)
        .join("; ");
      const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${handle.rootDir} 2>/dev/null; ${command}`;
      return execRaw(handle.id, script, timeoutSec);
    },

    async writeFileBytes(handle, relPath, data): Promise<void> {
      await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data);
    },
    async writeFile(handle, relPath, data): Promise<void> {
      await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8"));
    },
    async readFileBytes(handle, relPath): Promise<Uint8Array | null> {
      return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath));
    },
    async readFile(handle, relPath): Promise<string | null> {
      const bytes = await sandbox.readFileBytes(handle, relPath);
      return bytes === null ? null : Buffer.from(bytes).toString("utf8");
    },

    exportFiles: execExport.exportFiles,

    async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
      const remaining = (activeByMicrovm.get(handle.id) ?? 1) - 1;
      if (remaining > 0) {
        activeByMicrovm.set(handle.id, remaining);
        return;
      }
      activeByMicrovm.delete(handle.id);

      if (handle.scratch) {
        for (const [k, ref] of scratchByKey) if (ref.microvmId === handle.id) scratchByKey.delete(k);
        if (tdOpts?.destroy) await api.terminate(handle.id);
        else await api.terminate(handle.id).catch(swallowAs("aws-sandbox: scratch terminate", undefined));
        client.evict(handle.id);
        endpointById.delete(handle.id);
        return;
      }

      if (tdOpts?.keepWarm) return;

      const scope = scopeByMicrovm.get(handle.id) ?? (await scopeOf(handle.id));
      if (tdOpts?.destroy) {
        if (scope) await destroyStoredScope(scope, handle.id);
        else {
          await terminateBody(handle.id);
          client.evict(handle.id);
          endpointById.delete(handle.id);
        }
        return;
      }

      if (scope) {
        await withScopeLock(scope, async () => {
          const stored = await store.get(scope);
          if (stored?.microvmId !== handle.id) return;
          if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });
          if (snapshotDue(stored, tdOpts, snapshotIntervalMs)) {
            try {
              await snapshotHome(scope, handle.id);
              await store.merge(scope, { lastSnapshotMs: Date.now(), lastActivityMs: Date.now(), homeDirty: false });
            } catch (e) {
              reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope);
              await store.merge(scope, { lastActivityMs: Date.now() }).catch(() => {});
            }
          } else {
            await store.merge(scope, { lastActivityMs: Date.now() }).catch(() => {});
          }
          await api.suspend(handle.id).catch(swallowAs("aws-sandbox: suspend on teardown", undefined));
        });
      }
    },

    async reapDeepIdle(idleMs): Promise<{ reaped: number }> {
      if (!(idleMs > 0)) return { reaped: 0 };
      const cutoff = Date.now() - idleMs;
      let reaped = 0;
      for (const [scope, candidate] of await store.entries()) {
        await withScopeLock(scope, async () => {
          const rec = await store.get(scope);
          if (!rec || rec.microvmId !== candidate.microvmId) return;
          if (rec.orgId && rec.orgId !== configOrgId()) return;
          if (!rec.lastActivityMs || rec.lastActivityMs > cutoff) return;
          if (activeByMicrovm.has(rec.microvmId)) return;
          const desc = await api.tryGetMicrovm(rec.microvmId);
          if (!desc || desc.state === "TERMINATED" || desc.state === "TERMINATING") {
            await store.delete(scope).catch(() => {});
            return;
          }
          try {
            endpointById.set(rec.microvmId, rec.endpoint);
            const due = !rec.lastSnapshotMs || (rec.lastActivityMs ?? 0) > rec.lastSnapshotMs;
            if (due) {
              await ensureRunning(rec.microvmId);
              await snapshotHome(scope, rec.microvmId);
            }
            await api.terminate(rec.microvmId);
            await store.delete(scope);
            reaped++;
          } catch (e) {
            reportError("sandbox_reap", "deep_idle_reap_failed", errMessage(e), scope);
          }
        });
      }
      return { reaped };
    },
  };

  async function scopeOf(microvmId: string): Promise<string | undefined> {
    const cached = scopeByMicrovm.get(microvmId);
    if (cached) return cached;
    for (const [scope, rec] of await store.entries()) if (rec.microvmId === microvmId) return scope;
    return undefined;
  }

  return sandbox;
}