Multiplayer agent harness for work.
Add DigitalOcean Managed Agents as a sandbox backend (#2128)
Joshua France51385cfd3f59
diff --git a/cli/src/secrets.ts b/cli/src/secrets.tsindex c02dc5c7d..6560efe63 100644--- a/cli/src/secrets.ts+++ b/cli/src/secrets.ts@@ -170,6 +170,14 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [required: { when: { kind: "sandbox-backend", backend: "e2b" } },description: "E2B API key used by the e2b sandbox backend (from e2b.dev dashboard).",},+ {+ name: "DO_AGENTS_API_TOKEN",+ service: "core",+ required: { when: { kind: "sandbox-backend", backend: "do-managed-agents" } },+ description:+ "DigitalOcean IAM token for the DigitalOcean Managed Agents sandbox backend. Carries the team identity that owns the sandbox sessions, and authenticates both the session REST calls and the port-forward tunnel that carries exec and file transfer.",+ generate: "doctl auth init # then create a personal access token at cloud.digitalocean.com/account/api/tokens",+ },{name: "MODAL_TOKEN_ID",service: "core",diff --git a/deploy/do-managed-agents/Dockerfile b/deploy/do-managed-agents/Dockerfilenew file mode 100644index 000000000..fd913e088--- /dev/null+++ b/deploy/do-managed-agents/Dockerfile@@ -0,0 +1,13 @@+FROM node:24-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d++ENV DEBIAN_FRONTEND=noninteractive+RUN apt-get update && apt-get install -y --no-install-recommends \+ ca-certificates git curl wget jq tar xz-utils unzip zip ripgrep \+ python3 python3-pip python3-venv \+ openssh-client gnupg less vim-tiny procps \+ && rm -rf /var/lib/apt/lists/* \+ && node --version && npm --version++ENV HOME=/workspace/home+ENTRYPOINT ["sleep", "infinity"]+CMD []diff --git a/deploy/do-managed-agents/register-template.sh b/deploy/do-managed-agents/register-template.shnew file mode 100755index 000000000..3c6bf4c97--- /dev/null+++ b/deploy/do-managed-agents/register-template.sh@@ -0,0 +1,51 @@+#!/usr/bin/env bash+# Build the QM sandbox image and register it as a Managed Agents template+# on your own DigitalOcean team. Run once per team, and again after the+# Dockerfile changes.+#+# Needs: docker (buildx), doctl authenticated with a token that has+# registry and Managed Agents access, and an existing container registry.+#+# Usage: deploy/do-managed-agents/register-template.sh [tag]+# DO_REGISTRY registry name (default: first registry on the account)+# TEMPLATE_NAME template name (default: qm-sandbox)+set -euo pipefail++TAG="${1:-$(date +%Y%m%d%H%M%S)}"+TEMPLATE_NAME="${TEMPLATE_NAME:-qm-sandbox}"+DIR="$(cd "$(dirname "$0")" && pwd)"++for bin in docker doctl jq; do+ command -v "$bin" >/dev/null || { echo "missing: $bin" >&2; exit 1; }+done++DO_REGISTRY="${DO_REGISTRY:-$(doctl registry get -o json | jq -r 'if type=="array" then .[0].name else .name end // empty')}"+[ -n "$DO_REGISTRY" ] || { echo "no container registry found; create one or set DO_REGISTRY" >&2; exit 1; }+IMAGE="registry.digitalocean.com/$DO_REGISTRY/$TEMPLATE_NAME:$TAG"++echo "building $IMAGE"+docker build --platform linux/amd64 -t "$IMAGE" "$DIR"+doctl registry login >/dev/null+docker push "$IMAGE"++if doctl harness-runtime template get "$TEMPLATE_NAME" >/dev/null 2>&1; then+ echo "updating template $TEMPLATE_NAME"+ doctl harness-runtime template update "$TEMPLATE_NAME" --source-oci-ref "$IMAGE" >/dev/null+else+ echo "creating template $TEMPLATE_NAME"+ doctl harness-runtime template create --name "$TEMPLATE_NAME" \+ --base-template sandbox --source-oci-ref "$IMAGE" >/dev/null+fi++echo "waiting for the template build"+for _ in $(seq 1 60); do+ status="$(doctl harness-runtime template list-builds "$TEMPLATE_NAME" -o json \+ | jq -r 'sort_by(.created_at) | last | .status')"+ case "$status" in+ SUCCEEDED) echo "ready. Set DO_AGENTS_TEMPLATE=$TEMPLATE_NAME in your QM config."; exit 0 ;;+ FAILED) echo "template build failed" >&2; exit 1 ;;+ esac+ sleep 20+done+echo "timed out waiting for the template build (last status: $status)" >&2+exit 1diff --git a/docs/do-managed-agents.md b/docs/do-managed-agents.mdnew file mode 100644index 000000000..31e2b5eb3--- /dev/null+++ b/docs/do-managed-agents.md@@ -0,0 +1,39 @@+# DigitalOcean Managed Agents sandboxes++The `do-managed-agents` backend runs each QM scope on a DigitalOcean Managed Agents microVM. QM keeps its own agent loop; sessions start with `agent: none`, so DigitalOcean supplies the machine and nothing else.++## Set it up++1. Create a DigitalOcean API token with Managed Agents access.+2. Point core at it:++ ```sh+ SANDBOX_BACKEND=do-managed-agents+ DO_AGENTS_API_TOKEN=your-token+ ```++That's enough to run. Sessions land on DigitalOcean's bare base image.++## Optional++**A QM template.** For an image with the tools agents usually reach for (Node, git, jq, ripgrep, Python), register one on your team. You need Docker, an authenticated `doctl`, and a container registry:++```sh+deploy/do-managed-agents/register-template.sh+DO_AGENTS_TEMPLATE=qm-sandbox+```++Templates belong to a team, so each team registers its own. Rerun the script after changing the Dockerfile.++**A snapshot bucket.** When a scope parks, QM takes a DigitalOcean checkpoint, which lets it roll a wedged computer back in place. A checkpoint is tied to its session, though, so if the session is deleted the checkpoint goes with it. Set `DO_AGENTS_SNAPSHOT_S3_BUCKET` to also keep a copy of the home directory that survives that. Without it, the copy lives in core's memory and is lost when core restarts.++**An egress proxy.** Set `DO_AGENTS_EGRESS_PROXY_URL` and the proxy host becomes the session's only allowed destination.++Other knobs: `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_NAME_PREFIX` (use a distinct one per deployment sharing a team), `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and `DO_AGENTS_API_BASE_URL`.++## Good to know++- Commands and file transfers go over a port-forward to the microVM rather than DigitalOcean's REST exec, which caps output and runtime. Long commands are bounded only by `SANDBOX_TIMEOUT_SEC`.+- QM doesn't reap idle machines itself. DigitalOcean's idle timeout does, and paused sessions still count against team quota.+- Sessions allow `bash` outright, since QM already runs its own approval gates.+- `scripts/managed-agents-sandbox-smoke.ts` checks a token and template end to end before you point a deployment at them.diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.mdindex c8517560d..e7c36793b 100644--- a/docs/sandbox-preservation.md+++ b/docs/sandbox-preservation.md@@ -34,3 +34,5 @@ Modal machines are created with a one-core, 2 GiB reservation (`MODAL_CPUS`, `MOWhen `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound allowlist limited to the proxy host (a domain allowlist for an https proxy on port 443, a CIDR allowlist for an IP address; other proxy URLs are rejected at startup because Modal domain allowlists pass only TLS on 443), and the profile reports `egressEnforcement: "domain"`. Command execution, file transfer, and checkpoints travel over Modal's control plane and are unaffected. Without the setting, core warns that Modal sandboxes run fail-open.Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there.++Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home as a tar on that same interval measured from the last successful tar. The tar goes to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when it is set; without it the tar is held in core's memory and does not survive a core restart. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. Restart of a wedged computer calls `POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback` with that stored checkpoint: the session id stays, the response's sandbox id replaces the stored one, and any open port-forward is closed so the next command dials the restored microVM. A paused session is rolled back directly, without being resumed first. Restart with no checkpoint fails and leaves the session where it is. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar when one exists, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's.diff --git a/package-lock.json b/package-lock.jsonindex ecad8b459..66cdbbd4a 100644--- a/package-lock.json+++ b/package-lock.json@@ -21,6 +21,8 @@"@earendil-works/pi-ai": "https://github.com/yc-software/qm/releases/download/vendored-pi-ai-0.82.0-qm.1/earendil-works-pi-ai-0.82.0-qm.1.tgz","@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.6/earendil-works-pi-coding-agent-0.82.0-qm-security.6.tgz","@fly/sprites": "0.2.3",+ "@grpc/grpc-js": "^1.14.5",+ "@grpc/proto-loader": "^0.8.1","@openai/codex": "0.156.1","@opencode-ai/plugin": "1.18.31","@opencode-ai/sdk": "1.18.31",@@ -45,6 +47,7 @@"socks-proxy-agent": "10.1.0","tar-stream": "^3.2.0","typebox": "^1.1.38",+ "ws": "^8.21.1","zod": "4.4.3"},"devDependencies": {@@ -53,6 +56,7 @@"@types/node": "^24.0.0","@types/pg": "^8.11.10","@types/tar-stream": "^3.1.4",+ "@types/ws": "^8.18.1","@yc-software/qm": "file:./cli","devbar.sh": "1.1.0","esbuild": "0.28.1",diff --git a/package.json b/package.jsonindex 8de4ab481..75dad7683 100644--- a/package.json+++ b/package.json@@ -40,6 +40,8 @@"smoke:service-cred": "node scripts/service-credential-livetest.ts","smoke:aws-sandbox": "node scripts/aws-sandbox-smoke.ts","smoke:local-sandbox": "node scripts/local-sandbox-smoke.ts",+ "smoke:managed-agents-sandbox": "node --env-file-if-exists=.env scripts/managed-agents-sandbox-smoke.ts",+ "smoke:managed-agents-turn": "node --env-file-if-exists=.env scripts/managed-agents-turn-smoke.ts","build:modal-image": "node scripts/modal-build-image.ts","build:e2b-template": "node scripts/e2b-build-template.ts","build:aws-image": "node scripts/aws-build-sandbox-image.ts",@@ -78,6 +80,8 @@"@earendil-works/pi-ai": "https://github.com/yc-software/qm/releases/download/vendored-pi-ai-0.82.0-qm.1/earendil-works-pi-ai-0.82.0-qm.1.tgz","@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.6/earendil-works-pi-coding-agent-0.82.0-qm-security.6.tgz","@fly/sprites": "0.2.3",+ "@grpc/grpc-js": "^1.14.5",+ "@grpc/proto-loader": "^0.8.1","@openai/codex": "0.156.1","@opencode-ai/plugin": "1.18.31","@opencode-ai/sdk": "1.18.31",@@ -102,6 +106,7 @@"socks-proxy-agent": "10.1.0","tar-stream": "^3.2.0","typebox": "^1.1.38",+ "ws": "^8.21.1","zod": "4.4.3"},"overrides": {@@ -128,6 +133,7 @@"@types/node": "^24.0.0","@types/pg": "^8.11.10","@types/tar-stream": "^3.1.4",+ "@types/ws": "^8.18.1","@yc-software/qm": "file:./cli","devbar.sh": "1.1.0","esbuild": "0.28.1",diff --git a/scripts/dev/cli.ts b/scripts/dev/cli.tsindex a0436ab0f..f5614ade4 100644--- a/scripts/dev/cli.ts+++ b/scripts/dev/cli.ts@@ -671,7 +671,7 @@ async function main(): Promise<number> {});default:console.error(- "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]",+ "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|do-managed-agents|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]",);return EXIT.usage;}diff --git a/scripts/dev/lib/sandbox.ts b/scripts/dev/lib/sandbox.tsindex 9d4843274..3dc0b3512 100644--- a/scripts/dev/lib/sandbox.ts+++ b/scripts/dev/lib/sandbox.ts@@ -190,6 +190,44 @@ export async function resolveSandbox(opts: {return { backend: "superserve", env, detail: `Superserve (template ${template})`, publicApiUrl: apiUrl, warnings };}+ if (backend === "do-managed-agents") {+ const managedAgentsToken = opts.baseEnv.DO_AGENTS_API_TOKEN?.trim();+ if (!managedAgentsToken)+ throw new Error(+ "--sandbox do-managed-agents requires DO_AGENTS_API_TOKEN in the environment (a DigitalOcean API token for a team with Managed Agents access)",+ );+ let managedAgentsApiUrl = opts.baseEnv.PUBLIC_API_URL || null;+ if (!managedAgentsApiUrl) {+ managedAgentsApiUrl = await startQuickTunnel(opts.corePort, opts.lock, opts.log);+ if (!managedAgentsApiUrl)+ warnings.push(+ "cloudflared tunnel didn't come up -- agent self-API (crons/sends) won't be reachable from the sandbox",+ );+ }+ const env: Record<string, string> = {+ SANDBOX_BACKEND: "do-managed-agents",+ DO_AGENTS_API_TOKEN: managedAgentsToken,+ DO_AGENTS_NAME_PREFIX: opts.baseEnv.DO_AGENTS_NAME_PREFIX || "qmdev",+ };+ if (opts.baseEnv.DO_AGENTS_API_BASE_URL) env.DO_AGENTS_API_BASE_URL = opts.baseEnv.DO_AGENTS_API_BASE_URL;+ if (opts.baseEnv.DO_AGENTS_TEMPLATE) env.DO_AGENTS_TEMPLATE = opts.baseEnv.DO_AGENTS_TEMPLATE;+ if (opts.baseEnv.DO_AGENTS_SIZE_SLUG) env.DO_AGENTS_SIZE_SLUG = opts.baseEnv.DO_AGENTS_SIZE_SLUG;+ if (opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL)+ env.DO_AGENTS_EGRESS_PROXY_URL = opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL;+ else+ warnings.push(+ "DO_AGENTS_EGRESS_PROXY_URL unset -- do-managed-agents sandbox runs with NO egress enforcement; set it to QA the forced-proxy path",+ );+ if (managedAgentsApiUrl) env.PUBLIC_API_URL = managedAgentsApiUrl;+ return {+ backend: "do-managed-agents",+ env,+ detail: `Managed Agents (${opts.baseEnv.DO_AGENTS_API_BASE_URL || "api.digitalocean.com"})`,+ publicApiUrl: managedAgentsApiUrl,+ warnings,+ };+ }+if (backend === "porter") {const porterToken = opts.baseEnv.PORTER_DEPLOY_API_TOKEN;const projectId = opts.baseEnv.PORTER_DEPLOY_PROJECT_ID;diff --git a/scripts/dev/lib/types.ts b/scripts/dev/lib/types.tsindex f1491f783..f96294a41 100644--- a/scripts/dev/lib/types.ts+++ b/scripts/dev/lib/types.ts@@ -104,7 +104,8 @@ export interface StatusReport {children: Record<string, ChildStatus & { slack?: SlackHealth }>;}-export type DevSandboxBackend = "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve";+export type DevSandboxBackend =+ "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve" | "do-managed-agents";export type DevSandboxChoice = DevSandboxBackend | "auto";export interface BootSpec {diff --git a/scripts/managed-agents-sandbox-smoke.ts b/scripts/managed-agents-sandbox-smoke.tsnew file mode 100644index 000000000..96ba31e6f--- /dev/null+++ b/scripts/managed-agents-sandbox-smoke.ts@@ -0,0 +1,122 @@+#!/usr/bin/env node+import assert from "node:assert/strict";+import { randomBytes } from "node:crypto";+import { mkdtempSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { createManagedAgentsSandbox } from "../src/sandbox/managed-agents-sandbox.ts";+import { createSdkManagedAgentsClient } from "../src/sandbox/managed-agents-client.ts";+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";+import { scopeId } from "../src/types.ts";++const apiToken = process.env.DO_AGENTS_API_TOKEN;+const apiBaseUrl = process.env.DO_AGENTS_API_BASE_URL;+const template = process.env.DO_AGENTS_TEMPLATE ?? "";+const agent = process.env.DO_AGENTS_AGENT;+const keep = process.env.DO_AGENTS_SMOKE_KEEP === "1";++const log = (...a: unknown[]) => console.log("[managed-agents-smoke]", ...a);++async function main(): Promise<void> {+ if (!apiToken) throw new Error("DO_AGENTS_API_TOKEN is required");++ const client = createSdkManagedAgentsClient({+ apiToken,+ ...(apiBaseUrl ? { apiBaseUrl } : {}),+ ...(agent ? { agent } : {}),+ ...(template ? { template } : {}),+ });++ const prefix = `qms${randomBytes(3).toString("hex")}`;+ const sandbox = createManagedAgentsSandbox(+ createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-smoke-ws-"))),+ {+ client,+ namePrefix: prefix,+ },+ );+ const scope = scopeId("personal", "smoke");+ const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }];++ log(`provisioning a qm agent computer (prefix ${prefix})...`);+ const handle = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } });+ log("provisioned", handle.id, "coldStart", handle.coldStart);++ try {+ log("running a command through the qm Sandbox interface...");+ const r = await sandbox.run(handle, "pwd; echo QM_SMOKE=$QM_SMOKE; whoami");+ assert.equal(r.code, 0, `run exited ${r.code}: ${r.stderr}`);+ log("stdout:", JSON.stringify(r.stdout));+ assert.match(r.stdout, /QM_SMOKE=1/);++ log("writing and reading a workspace file through qm...");+ await sandbox.writeFile(handle, "notes/hello.txt", "written by qm");+ assert.equal(await sandbox.readFile(handle, "notes/hello.txt"), "written by qm");+ log("file round-trip OK");++ log("checking the toolchain qm expects...");+ const binaryName = (entry: string) => entry.trim().split(/\s+/)[0] ?? "";+ const declaredPresent = (sandbox.profile.spec?.tools ?? []).map(binaryName).filter(Boolean);+ const declaredAbsent = (sandbox.profile.spec?.notInstalled ?? []).map(binaryName).filter(Boolean);+ const plumbingNeeds = ["sh", "timeout", "tar", "base64", "sha256sum", "setsid", "mkfifo", "cut"];+ const probe = [...new Set([...plumbingNeeds, ...declaredPresent, ...declaredAbsent])];+ const tools = await sandbox.run(+ handle,+ `for t in ${probe.join(" ")}; do printf '%s=%s ' $t $(command -v $t >/dev/null 2>&1 && echo yes || echo NO); done; echo`,+ );+ log(tools.stdout.trim());+ const onPath = new Set(+ tools.stdout+ .trim()+ .split(/\s+/)+ .flatMap((pair) => {+ const [name, state] = pair.split("=");+ return name && state === "yes" ? [name] : [];+ }),+ );+ for (const t of plumbingNeeds)+ assert.ok(onPath.has(t), `qm's own file and process plumbing needs ${t}, which the guest does not have`);+ for (const t of declaredPresent)+ assert.ok(onPath.has(t), `the computer profile advertises ${t}, which the guest does not have`);+ for (const t of declaredAbsent)+ assert.ok(!onPath.has(t), `the computer profile says ${t} is absent, but the guest has it`);++ log("checking the guest carries no managed coding agent...");+ const bare = await sandbox.run(+ handle,+ "for t in codex claude claude-code opencode cursor hermes; do command -v $t >/dev/null && echo AGENT_ON_PATH=$t; done; echo ---; ps -eo comm= | sort -u | tr '\\n' ' '",+ );+ log("guest:", bare.stdout.trim());+ assert.doesNotMatch(bare.stdout, /AGENT_ON_PATH=/, "a bare sandbox must carry no agent CLI");++ log("checking the workspace survives a pause and resume...");+ await sandbox.run(handle, "echo persisted > survives-pause.txt");+ await sandbox.teardown(handle, {});+ const resumed = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } });+ const after = await sandbox.run(resumed, "cat survives-pause.txt");+ assert.match(after.stdout, /persisted/, "the workspace did not survive a pause and resume");+ log("workspace intact after pause and resume");++ if (process.env.DO_AGENTS_SMOKE_LONG === "1") {+ log("checking a command survives past the 4-minute REST exec clamp...");+ const long = await sandbox.run(resumed, "sleep 250; echo past-the-clamp", { timeoutMs: 400_000 });+ assert.equal(long.code, 0, `long run exited ${long.code}: ${long.stderr}`);+ assert.match(long.stdout, /past-the-clamp/);+ log("250s command completed over the tunnel");+ }++ log("ALL CHECKS PASSED");+ } finally {+ if (keep) {+ log("DO_AGENTS_SMOKE_KEEP=1 — leaving the sandbox alive");+ } else {+ log("tearing down and destroying the scope...");+ await sandbox.teardown(handle, { destroy: true }).catch((e: unknown) => log("teardown failed:", e));+ }+ }+}++main().catch((e: unknown) => {+ console.error("[managed-agents-smoke] FAILED", e);+ process.exit(1);+});diff --git a/scripts/managed-agents-turn-smoke.ts b/scripts/managed-agents-turn-smoke.tsnew file mode 100644index 000000000..396717de9--- /dev/null+++ b/scripts/managed-agents-turn-smoke.ts@@ -0,0 +1,52 @@+#!/usr/bin/env node+import { randomUUID } from "node:crypto";+import { stagingApiHeaders } from "../src/deployment/postdeploy-smoke.ts";++const core = process.env.CORE_URL ?? "http://localhost:8081";+const principal = process.env.PRINCIPAL ?? "managed-agents-smoke";+const sourceSecret = process.env.CORE_SIGNING_SECRET ?? "";+const portalSecret = process.env.PORTAL_IDENTITY_SECRET ?? "";++const log = (...a: unknown[]) => console.log("[managed-agents-turn]", ...a);++async function main(): Promise<void> {+ if (!sourceSecret || !portalSecret) throw new Error("CORE_SIGNING_SECRET and PORTAL_IDENTITY_SECRET are required");++ const nonce = randomUUID();+ const body = JSON.stringify({+ surface: "web",+ actor: { externalId: principal },+ conversation: { kind: "dm", threadRef: `web:${principal}:managed-agents-turn-${nonce}` },+ text: "Use your execute tool to run exactly this shell command in your agent computer: `echo DO_AGENTS_TURN_OK; uname -sr; pwd; whoami`. Then reply with the exact stdout you got back and nothing else.",+ origin: { kind: "human" },+ addressed: true,+ skipMemory: true,+ idempotencyKey: nonce,+ });++ const path = "/v1/turns";+ const headers = await stagingApiHeaders(principal, sourceSecret, portalSecret, "POST", path, body);++ log(`POST ${core}${path} (this provisions a Managed Agents microVM on the first execute)...`);+ const started = Date.now();+ const res = await fetch(`${core}${path}`, {+ method: "POST",+ headers: { ...headers, "content-type": "application/json" },+ body,+ signal: AbortSignal.timeout(600_000),+ });+ const text = await res.text();+ log(`HTTP ${res.status} in ${Math.round((Date.now() - started) / 1000)}s`);+ if (!res.ok) throw new Error(`turn failed: ${text.slice(0, 2000)}`);++ const turn = JSON.parse(text) as { status?: string; sessionId?: string; reply?: string };+ log("status:", turn.status, "session:", turn.sessionId);+ log("reply:", JSON.stringify(turn.reply));+ if (!turn.reply?.includes("DO_AGENTS_TURN_OK")) throw new Error("the model reply does not contain DO_AGENTS_TURN_OK");+ log("REAL QM TURN EXECUTED IN A Managed Agents SANDBOX");+}++main().catch((e: unknown) => {+ console.error("[managed-agents-turn] FAILED", e);+ process.exit(1);+});diff --git a/src/config.ts b/src/config.tsindex df35cd61d..b55f0203b 100644--- a/src/config.ts+++ b/src/config.ts@@ -3,6 +3,7 @@ import { isStrongSigningSecret } from "./auth/source-auth.ts";import { parseSandboxCapabilityTtlMs } from "./auth/capability-token.ts";import { parseScopeId } from "./types.ts";import type { LegacySandboxScopeDefaults } from "./sandbox/sandbox-resource-upgrade.ts";+import type { SandboxBackendName } from "./sandbox/sandbox-routing.ts";import { existsSync, readdirSync } from "node:fs";import {parseProviderBaseUrl,@@ -73,9 +74,8 @@ export interface Config {securityPosture: SecurityPosture;sharingPosture: SharingPosture;legacySandboxScopeDefaults?: LegacySandboxScopeDefaults;- sandboxBackend: "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";- sandboxSecondaryBackend?:- "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";+ sandboxBackend: SandboxBackendName;+ sandboxSecondaryBackend?: SandboxBackendName;deployProvider: "docker" | "aws" | "fly" | "porter";egressServiceHosts?: string[];brandingDefault?: OrgBranding;@@ -205,6 +205,7 @@ export interface Config {agent37Sandbox: Agent37SandboxEnv;superserveSandbox: SuperserveSandboxEnv;e2bSandbox: E2bSandboxEnv;+ managedAgentsSandbox: ManagedAgentsSandboxEnv;modalSandbox: ModalSandboxEnv;porterSandbox: PorterSandboxEnv;porterDeploy: PorterDeployEnv;@@ -431,6 +432,40 @@ function e2bSandboxEnv(env: NodeJS.ProcessEnv): E2bSandboxEnv {};}+interface ManagedAgentsSandboxEnv {+ apiToken?: string;+ apiBaseUrl?: string;+ template?: string;+ namePrefix?: string;+ sizeSlug?: string;+ idleTimeoutSec?: number;+ egressProxyUrl?: string;+ snapshotS3Bucket?: string;+ snapshotIntervalSec?: number;+ defaultTimeoutSec?: number;+}++function managedAgentsSandboxEnv(env: NodeJS.ProcessEnv): ManagedAgentsSandboxEnv {+ return {+ ...(env.DO_AGENTS_API_TOKEN ? { apiToken: env.DO_AGENTS_API_TOKEN } : {}),+ ...(env.DO_AGENTS_API_BASE_URL ? { apiBaseUrl: env.DO_AGENTS_API_BASE_URL } : {}),+ ...(env.DO_AGENTS_TEMPLATE ? { template: env.DO_AGENTS_TEMPLATE } : {}),+ ...(env.DO_AGENTS_NAME_PREFIX ? { namePrefix: env.DO_AGENTS_NAME_PREFIX } : {}),+ ...(env.DO_AGENTS_SIZE_SLUG ? { sizeSlug: env.DO_AGENTS_SIZE_SLUG } : {}),+ ...(numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) !== undefined+ ? { idleTimeoutSec: numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) }+ : {}),+ ...(env.DO_AGENTS_EGRESS_PROXY_URL ? { egressProxyUrl: env.DO_AGENTS_EGRESS_PROXY_URL } : {}),+ ...(env.DO_AGENTS_SNAPSHOT_S3_BUCKET ? { snapshotS3Bucket: env.DO_AGENTS_SNAPSHOT_S3_BUCKET } : {}),+ ...(numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) !== undefined+ ? { snapshotIntervalSec: numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) }+ : {}),+ ...(numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) !== undefined+ ? { defaultTimeoutSec: numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) }+ : {}),+ };+}+interface ModalSandboxEnv {nativeSnapshotsEnabled?: boolean;nativeSnapshotIntervalSec?: number;@@ -999,6 +1034,7 @@ export function enabledSandboxBackends(config: Config): Array<Config["sandboxBacagent37: Boolean(config.agent37Sandbox?.apiKey),superserve: Boolean(config.superserveSandbox?.apiKey && config.superserveSandbox?.template),e2b: Boolean(config.e2bSandbox?.apiKey),+ "do-managed-agents": Boolean(config.managedAgentsSandbox?.apiToken),modal: Boolean(config.modalSandbox?.tokenId && config.modalSandbox?.tokenSecret),aws: Boolean(config.awsSandbox?.s3Bucket),porter: Boolean(config.porterSandbox?.token),@@ -1022,11 +1058,12 @@ function sandboxBackendEnvStrict(value: string | undefined, name = "SANDBOX_BACKbackend === "modal" ||backend === "agent37" ||backend === "superserve" ||+ backend === "do-managed-agents" ||backend === "porter")return backend;throw new Error(- `${name}=${JSON.stringify(value)} is not recognized — use aws, local, sprites, smolmachines, e2b, modal, porter, agent37, or superserve, or unset it.`,+ `${name}=${JSON.stringify(value)} is not recognized — use aws, local, sprites, smolmachines, e2b, modal, porter, agent37, superserve, or do-managed-agents, or unset it.`,);}@@ -1223,6 +1260,16 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {"[config] SANDBOX_BACKEND=e2b without E2B_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set E2B_EGRESS_PROXY_URL to the public egress proxy so E2B's network rules admit only that host.",);}+ if (env.SANDBOX_BACKEND === "do-managed-agents" && !env.DO_AGENTS_EGRESS_PROXY_URL) {+ console.warn(+ "[config] SANDBOX_BACKEND=do-managed-agents without DO_AGENTS_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set DO_AGENTS_EGRESS_PROXY_URL to the public egress proxy so the session's egress allowlist admits only that host.",+ );+ }+ if (env.DO_AGENTS_API_TOKEN && !env.DO_AGENTS_SNAPSHOT_S3_BUCKET) {+ console.warn(+ "[config] do-managed-agents sandbox backend enabled without DO_AGENTS_SNAPSHOT_S3_BUCKET — pause preserves the microVM, but a destroyed session takes the home with it; set DO_AGENTS_SNAPSHOT_S3_BUCKET so a replacement session can hydrate from a portable snapshot.",+ );+ }if (env.MODAL_TOKEN_ID && env.MODAL_TOKEN_SECRET && !env.MODAL_EGRESS_PROXY_URL) {console.warn("[config] modal sandbox backend enabled without MODAL_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set MODAL_EGRESS_PROXY_URL to the public https egress proxy so Modal's outbound allowlist admits only that host.",@@ -1262,7 +1309,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {}if (env.NODE_ENV === "production" && !env.SANDBOX_BACKEND?.trim()) {throw new Error(- "SANDBOX_BACKEND must be set explicitly in production — use sprites, smolmachines, e2b, modal, porter, agent37, superserve, aws, or local.",+ "SANDBOX_BACKEND must be set explicitly in production — use sprites, smolmachines, e2b, modal, porter, agent37, superserve, do-managed-agents, aws, or local.",);}const sandboxBackend = sandboxBackendEnvStrict(env.SANDBOX_BACKEND);@@ -1662,6 +1709,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {porterSandbox: porterSandboxEnv(env),porterDeploy: porterDeployEnv(env),e2bSandbox: e2bSandboxEnv(env),+ managedAgentsSandbox: managedAgentsSandboxEnv(env),modalSandbox: modalSandboxEnv(env),awsDeploy: {...awsDeployEnv(env),diff --git a/src/deployment/secret-schema.ts b/src/deployment/secret-schema.tsindex 4c2f34383..3c9880f8f 100644--- a/src/deployment/secret-schema.ts+++ b/src/deployment/secret-schema.ts@@ -12,6 +12,7 @@ type SecretGate =| "porter"| "agent37"| "superserve"+ | "do-managed-agents"| "porter-deploy"| "fly-shared-deploy"| "fly-deploy"@@ -46,6 +47,7 @@ export const CORE_SECRET_SPECS: readonly RuntimeSecretSpec[] = [{ name: "AGENT37_API_KEY", requiredWhen: "agent37" },{ name: "SUPERSERVE_API_KEY", requiredWhen: "superserve" },{ name: "E2B_API_KEY", requiredWhen: "e2b" },+ { name: "DO_AGENTS_API_TOKEN", requiredWhen: "do-managed-agents" },{ name: "MODAL_TOKEN_ID", requiredWhen: "modal" },{ name: "MODAL_TOKEN_SECRET", requiredWhen: "modal" },{ name: "PORTER_DEPLOY_API_TOKEN", requiredWhen: ["porter", "porter-deploy"] },@@ -73,6 +75,7 @@ const GATE_PREDICATES: Readonly<Record<SecretGate, (env: NodeJS.ProcessEnv) => bporter: (env) => sandboxBackendSelected(env, "porter"),agent37: (env) => sandboxBackendSelected(env, "agent37"),superserve: (env) => sandboxBackendSelected(env, "superserve"),+ "do-managed-agents": (env) => sandboxBackendSelected(env, "do-managed-agents"),"porter-deploy": (env) => env.DEPLOY_PROVIDER === "porter","fly-shared-deploy": (env) => env.DEPLOY_PROVIDER === "fly" && Boolean(env.FLY_DEPLOY_SHARED_APP_NAME?.trim()),"fly-deploy": (env) => env.DEPLOY_PROVIDER === "fly",diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.tsnew file mode 100644index 000000000..7430b7511--- /dev/null+++ b/src/sandbox/managed-agents-client.ts@@ -0,0 +1,760 @@+import { fileURLToPath } from "node:url";+import { fetchWithRetry } from "../util/async.ts";+import { errMessage, swallowAs, withRequestId } from "../util/errors.ts";+import { openManagedAgentsTunnel, SANDBOX_AGENT_PORT, type ManagedAgentsTunnel } from "./managed-agents-tunnel.ts";++export interface ManagedAgentsCommandResult {+ stdout: string;+ stderr: string;+ exitCode: number;+ hitlRejected?: boolean;+}++interface ManagedAgentsRunOpts {+ timeoutMs?: number;+ workdir?: string;+ env?: Record<string, string>;+}++export type ManagedAgentsSessionState =+ "unspecified" | "provisioning" | "ready" | "detached" | "destroying" | "destroyed" | "failed" | "paused";++export interface ManagedAgentsSessionSummary {+ sessionId: string;+ sandboxId: string;+ name: string;+ state: ManagedAgentsSessionState;+}++export interface ManagedAgentsSessionInfo extends ManagedAgentsSessionSummary {+ createdAtMs?: number;+ errorMessage?: string;+ sizeSlug?: string;+ template?: string;+}++export interface ManagedAgentsCheckpoint {+ checkpointId: string;+ status: string;+ createdAtMs?: number;+}++export interface ManagedAgentsSession {+ readonly sessionId: string;+ readonly sandboxId: string;+ runCommand(command: string, opts?: ManagedAgentsRunOpts): Promise<ManagedAgentsCommandResult>;+ readFileBytes(absPath: string): Promise<Uint8Array | null>;+ writeFileBytes(absPath: string, data: Uint8Array): Promise<void>;+ createCheckpoint(label?: string): Promise<ManagedAgentsCheckpoint>;+ deleteCheckpoint(checkpointId: string): Promise<void>;+ close(): Promise<void>;+ pause(): Promise<void>;+ resume(): Promise<void>;+ kill(): Promise<void>;+}++export class ManagedAgentsSandboxGoneError extends Error {+ constructor(sessionId: string, detail: string) {+ super(`do-managed-agents session ${sessionId} is gone: ${detail}`);+ this.name = "ManagedAgentsSandboxGoneError";+ }+}++export class ManagedAgentsCommandLostError extends Error {+ constructor(sessionId: string, detail: string) {+ super(+ `do-managed-agents session ${sessionId} was lost while a command was running (${detail}); the command may have partially executed and was not retried`,+ );+ this.name = "ManagedAgentsCommandLostError";+ }+}++export class ManagedAgentsHitlRejectedError extends Error {+ constructor(sessionId: string) {+ super(+ `do-managed-agents rejected a command in session ${sessionId} at its own approval gate; set the session permissions to allow bash so qm's approval gates stay authoritative`,+ );+ this.name = "ManagedAgentsHitlRejectedError";+ }+}++interface ManagedAgentsCreateOpts {+ name: string;+ egressAllow?: string[];+}++export interface ManagedAgentsClient {+ readonly nativePause?: boolean;+ info(sessionId: string): Promise<ManagedAgentsSessionInfo>;+ create(opts: ManagedAgentsCreateOpts): Promise<ManagedAgentsSession>;+ connect(sessionId: string): Promise<ManagedAgentsSession>;+ list(name: string): Promise<ManagedAgentsSessionSummary[]>;+ rollback(sessionId: string, checkpointId: string): Promise<ManagedAgentsSessionInfo>;+ kill(sessionId: string): Promise<void>;+}++export interface SdkManagedAgentsClientOptions {+ apiToken: string;+ apiBaseUrl?: string;+ template?: string;+ agent?: string;+ sizeSlug?: string;+ idleTimeoutSec?: number;+ maxCommandMs?: number;+ stallProbeMs?: number;+ egressProxyUrl?: string;+}++const DEFAULT_API_BASE_URL = "https://api.digitalocean.com";+const DEFAULT_TEMPLATE = "";+const DEFAULT_AGENT = "none";+const DEFAULT_MAX_COMMAND_MS = 3600_000;+const CREATE_TIMEOUT_MS = 120_000;+const CHECKPOINT_TIMEOUT_MS = 5 * 60_000;+const REQUEST_TIMEOUT_MS = 30_000;+const READY_TIMEOUT_MS = 300_000;+const ROLLBACK_TIMEOUT_MS = READY_TIMEOUT_MS;+const READY_POLL_MS = 2_000;+const READY_FAST_POLL_MS = 250;+const READY_FAST_WINDOW_MS = 10_000;+const STALL_PROBE_MS = 45_000;+const READY_TRANSIENT_LIMIT = 5;+const UPLOAD_CHUNK_BYTES = 256 * 1024;+const DOWNLOAD_CHUNK_BYTES = 256 * 1024;+const MAX_EXEC_OUTPUT_BYTES = 16 * 1024 * 1024;+const CONNECT_TIMEOUT_MS = 30_000;+const GRPC_NOT_FOUND = 5;+const GRPC_INTERNAL = 13;+const GRPC_UNAVAILABLE = 14;++const PROTO_PATH = fileURLToPath(new URL("./managed-agents-sandbox-agent.proto", import.meta.url));++const STATE_BY_WIRE: Record<string, ManagedAgentsSessionState> = {+ SESSION_STATUS_UNSPECIFIED: "unspecified",+ SESSION_STATUS_PROVISIONING: "provisioning",+ SESSION_STATUS_READY: "ready",+ SESSION_STATUS_DETACHED: "detached",+ SESSION_STATUS_DESTROYING: "destroying",+ SESSION_STATUS_DESTROYED: "destroyed",+ SESSION_STATUS_FAILED: "failed",+ SESSION_STATUS_PAUSED: "paused",+};++const GONE_STATES: ReadonlySet<ManagedAgentsSessionState> = new Set<ManagedAgentsSessionState>([+ "destroying",+ "destroyed",+ "failed",+]);++const USABLE_STATES: ReadonlySet<ManagedAgentsSessionState> = new Set<ManagedAgentsSessionState>(["ready", "detached"]);++export const managedAgentsSessionState = (wire: string | undefined): ManagedAgentsSessionState =>+ STATE_BY_WIRE[wire ?? ""] ?? "unspecified";++export function managedAgentsEgressAllow(egressProxyUrl: string): string[] {+ const host = new URL(egressProxyUrl).hostname.replace(/^\[|\]$/g, "");+ if (!host) throw new Error(`DO_AGENTS_EGRESS_PROXY_URL ${egressProxyUrl} has no host to put on the egress allowlist`);+ return [host];+}++export function managedAgentsManifest(opts: {+ name: string;+ template: string;+ agent?: string;+ sizeSlug?: string;+ idleTimeoutSec?: number;+ egressAllow?: readonly string[];+}): string {+ const lines = [`name: ${JSON.stringify(opts.name)}`, `agent: ${JSON.stringify(opts.agent ?? DEFAULT_AGENT)}`];+ if (opts.template) lines.push(`template: ${JSON.stringify(opts.template)}`);+ if (opts.sizeSlug) lines.push(`size: ${JSON.stringify(opts.sizeSlug)}`);+ if (opts.idleTimeoutSec) lines.push(`idle_timeout: ${JSON.stringify(`${opts.idleTimeoutSec}s`)}`);+ if (opts.egressAllow?.length) {+ lines.push("egress:");+ for (const host of opts.egressAllow) lines.push(` - ${JSON.stringify(host)}`);+ }+ lines.push("permissions:", " default: allow", " rules:", " - tool: bash", " action: allow");+ return `${lines.join("\n")}\n`;+}++interface WireSession {+ id?: string;+ session_id?: string;+ sandbox_id?: string;+ name?: string;+ status?: string;+ template?: string;+ size_slug?: string;+ error_message?: string;+ created_at?: string;+}++interface WireCheckpoint {+ checkpoint_id?: string;+ status?: string;+ error_message?: string;+ created_at?: string;+}++const toSummary = (s: WireSession): ManagedAgentsSessionSummary => ({+ sessionId: s.session_id ?? s.id ?? "",+ sandboxId: s.sandbox_id ?? "",+ name: s.name ?? "",+ state: managedAgentsSessionState(s.status),+});++const toInfo = (s: WireSession): ManagedAgentsSessionInfo => {+ const createdAtMs = s.created_at ? Date.parse(s.created_at) : Number.NaN;+ return {+ ...toSummary(s),+ ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}),+ ...(s.error_message ? { errorMessage: s.error_message } : {}),+ ...(s.size_slug ? { sizeSlug: s.size_slug } : {}),+ ...(s.template ? { template: s.template } : {}),+ };+};++interface GrpcCallError {+ code?: number;+ details?: string;+ message?: string;+}++const grpcCode = (err: unknown): number | undefined =>+ typeof err === "object" && err !== null ? (err as GrpcCallError).code : undefined;++const isGuestGone = (err: unknown): boolean => {+ const code = grpcCode(err);+ return code === GRPC_NOT_FOUND || code === GRPC_UNAVAILABLE;+};++type ProtoLoader = typeof import("@grpc/proto-loader");+type Grpc = typeof import("@grpc/grpc-js");++interface GrpcDuplex {+ write(msg: unknown): void;+ end(): void;+ cancel(): void;+ on(event: string, cb: (arg: never) => void): void;+}++type GrpcWritable = Pick<GrpcDuplex, "write" | "end" | "on">;+type GrpcReadable = Pick<GrpcDuplex, "on" | "cancel">;++interface SandboxAgent {+ Exec(): GrpcDuplex;+ Upload(cb: (err: unknown, res?: { bytes_written?: string }) => void): GrpcWritable;+ Download(req: unknown): GrpcReadable;+ waitForReady(deadline: Date, cb: (err?: Error) => void): void;+ close(): void;+}++interface ExecOutputFrame {+ output?: "stdout" | "stderr" | "exit";+ stdout?: Buffer;+ stderr?: Buffer;+ exit?: { exit_code?: number; hitl_rejected?: boolean; timed_out?: boolean };+}++interface DownloadFrame {+ output?: "header" | "chunk" | "end";+ chunk?: Buffer;+}++let grpcModules: Promise<[ProtoLoader, Grpc]> | null = null;++function loadGrpc(): Promise<[ProtoLoader, Grpc]> {+ grpcModules ??= Promise.all([+ import("@grpc/proto-loader").catch((e: unknown) => {+ throw new Error(`@grpc/proto-loader is not installed: ${errMessage(e)}`);+ }),+ import("@grpc/grpc-js").catch((e: unknown) => {+ throw new Error(`@grpc/grpc-js is not installed: ${errMessage(e)}`);+ }),+ ]) as Promise<[ProtoLoader, Grpc]>;+ return grpcModules;+}++async function dialSandboxAgent(localPort: number, readyTimeoutMs: number): Promise<SandboxAgent> {+ const [loader, grpc] = await loadGrpc();+ const definition = loader.loadSync(PROTO_PATH, {+ keepCase: true,+ longs: String,+ enums: String,+ defaults: true,+ oneofs: true,+ });+ const pkg = grpc.loadPackageDefinition(definition) as unknown as {+ do: {+ teams: {+ hosted_agents: {+ runtime: {+ sandbox_agent: {+ v1: {+ SandboxAgentService: new (+ addr: string,+ creds: ReturnType<Grpc["credentials"]["createInsecure"]>,+ options: Record<string, number>,+ ) => SandboxAgent;+ };+ };+ };+ };+ };+ };+ };+ const Service = pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService;+ const agent = new Service(`127.0.0.1:${localPort}`, grpc.credentials.createInsecure(), {+ "grpc.max_receive_message_length": MAX_EXEC_OUTPUT_BYTES,+ "grpc.max_send_message_length": MAX_EXEC_OUTPUT_BYTES,+ });+ try {+ await new Promise<void>((resolve, reject) => {+ agent.waitForReady(new Date(Date.now() + readyTimeoutMs), (err?: Error) => (err ? reject(err) : resolve()));+ });+ } catch (e) {+ agent.close();+ throw e;+ }+ return agent;+}++export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions): ManagedAgentsClient {+ const apiBaseUrl = (opts.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/+$/, "");+ const template = opts.template ?? DEFAULT_TEMPLATE;+ const maxCommandMs = opts.maxCommandMs ?? DEFAULT_MAX_COMMAND_MS;+ const stallProbeMs = opts.stallProbeMs ?? STALL_PROBE_MS;+ const egressAllow = opts.egressProxyUrl ? managedAgentsEgressAllow(opts.egressProxyUrl) : undefined;++ function send(method: string, path: string, body?: string, contentType?: string, signal?: AbortSignal) {+ return fetch(`${apiBaseUrl}${path}`, {+ method,+ headers: {+ authorization: `Bearer ${opts.apiToken}`,+ accept: "application/json",+ ...(contentType ? { "content-type": contentType } : {}),+ },+ ...(body !== undefined ? { body } : {}),+ signal: signal ?? AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });+ }++ function call(+ method: string,+ path: string,+ body?: string,+ contentType?: string,+ timeoutMs = REQUEST_TIMEOUT_MS,+ ): Promise<Response> {+ const operation = (signal?: AbortSignal): Promise<Response> => send(method, path, body, contentType, signal);+ return method === "GET" || method === "DELETE"+ ? fetchWithRetry((signal) => operation(signal), "idempotent", { timeoutMs })+ : operation(AbortSignal.timeout(timeoutMs));+ }++ async function failure(action: string, sessionId: string, res: Response): Promise<Error> {+ const raw = await res.text().catch(swallowAs(`do-managed-agents ${action} ${sessionId}: read error body`, ""));+ let message = raw.slice(0, 200);+ try {+ const parsed = JSON.parse(raw) as { error?: { message?: string } };+ if (parsed.error?.message) message = parsed.error.message;+ } catch {+ message = raw.slice(0, 200);+ }+ const detail = withRequestId(`http ${res.status} ${message}`, res.headers);+ if (res.status === 404 || res.status === 410) return new ManagedAgentsSandboxGoneError(sessionId, detail);+ return new Error(`do-managed-agents ${action}: ${detail}`);+ }++ async function callJson<T>(+ method: string,+ path: string,+ sessionId: string,+ body?: string,+ contentType?: string,+ timeoutMs = REQUEST_TIMEOUT_MS,+ ): Promise<T> {+ const res = await call(method, path, body, contentType, timeoutMs);+ if (!res.ok) throw await failure(`${method} ${path}`, sessionId, res);+ if (res.status === 204) return undefined as T;+ return (await res.json()) as T;+ }++ const getInfo = async (sessionId: string): Promise<ManagedAgentsSessionInfo> => {+ const body = await callJson<{ session?: WireSession }>("GET", `/v2/agents/sessions/${sessionId}`, sessionId);+ if (!body.session) throw new ManagedAgentsSandboxGoneError(sessionId, "response carried no session");+ return toInfo(body.session);+ };++ async function awaitUsable(sessionId: string): Promise<ManagedAgentsSessionInfo> {+ const started = Date.now();+ const deadline = started + READY_TIMEOUT_MS;+ let resumed = false;+ let strikes = 0;+ const tolerate = (err: unknown): void => {+ if (err instanceof ManagedAgentsSandboxGoneError) throw err;+ if (++strikes > READY_TRANSIENT_LIMIT) throw err;+ };+ for (;;) {+ let info: ManagedAgentsSessionInfo | undefined;+ try {+ info = await getInfo(sessionId);+ strikes = 0;+ } catch (err) {+ tolerate(err);+ }+ if (info) {+ if (USABLE_STATES.has(info.state)) return info;+ if (GONE_STATES.has(info.state))+ throw new ManagedAgentsSandboxGoneError(sessionId, info.errorMessage ?? `status ${info.state}`);+ if (info.state === "paused" && !resumed) {+ try {+ await callJson<void>("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId);+ resumed = true;+ strikes = 0;+ } catch (err) {+ tolerate(err);+ }+ }+ }+ if (Date.now() >= deadline)+ throw new Error(`do-managed-agents session ${sessionId} did not become ready within ${READY_TIMEOUT_MS}ms`);+ await new Promise((resolve) =>+ setTimeout(resolve, Date.now() - started < READY_FAST_WINDOW_MS ? READY_FAST_POLL_MS : READY_POLL_MS),+ );+ }+ }++ async function rollbackToCheckpoint(sessionId: string, checkpointId: string): Promise<ManagedAgentsSessionInfo> {+ const body = await callJson<{ session?: WireSession }>(+ "POST",+ `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}/rollback`,+ sessionId,+ undefined,+ undefined,+ ROLLBACK_TIMEOUT_MS,+ );+ if (!body.session) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no session`);+ const restored = toInfo(body.session);+ if (!restored.sandboxId) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no sandbox id`);+ if (GONE_STATES.has(restored.state))+ throw new ManagedAgentsSandboxGoneError(sessionId, restored.errorMessage ?? `status ${restored.state}`);+ return restored;+ }++ function buildSession(info: ManagedAgentsSessionInfo): ManagedAgentsSession {+ const sessionId = info.sessionId;+ let guest: Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> | null = null;+ let lastTunnel: ManagedAgentsTunnel | null = null;++ const connect = (): Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> =>+ (guest ??= (async () => {+ const tunnel = await openManagedAgentsTunnel({+ apiBaseUrl,+ sessionId,+ remotePort: SANDBOX_AGENT_PORT,+ getToken: async () => opts.apiToken,+ });+ lastTunnel = tunnel;+ try {+ const dial = dialSandboxAgent(tunnel.localPort, CONNECT_TIMEOUT_MS);+ const abort = tunnel.whenFailed().then((detail) => {+ throw new Error(detail);+ });+ dial.catch(() => undefined);+ abort.catch(() => undefined);+ return { tunnel, agent: await Promise.race([dial, abort]) };+ } catch (e) {+ const detail = tunnel.lastFailure() ?? errMessage(e);+ await tunnel.close();+ throw new ManagedAgentsSandboxGoneError(sessionId, `guest unreachable over the port-forward: ${detail}`);+ }+ })().catch((e: unknown) => {+ guest = null;+ throw e;+ }));++ async function disconnect(): Promise<void> {+ const pending = guest;+ guest = null;+ if (!pending) return;+ await pending+ .then(async ({ tunnel, agent }) => {+ agent.close();+ await tunnel.close();+ })+ .catch(swallowAs(`do-managed-agents ${sessionId}: disconnect`, undefined));+ }++ async function download(absPath: string): Promise<Uint8Array | null> {+ const { agent } = await connect();+ return new Promise<Uint8Array | null>((resolve, reject) => {+ const chunks: Buffer[] = [];+ const stream = agent.Download({ path: absPath, as_archive: false, chunk_size_bytes: DOWNLOAD_CHUNK_BYTES });+ let settled = false;+ const settle = (fn: () => void): void => {+ if (settled) return;+ settled = true;+ fn();+ };+ stream.on("data", ((frame: DownloadFrame) => {+ if (frame.chunk?.length) chunks.push(frame.chunk);+ }) as (arg: never) => void);+ stream.on("error", ((err: unknown) => {+ settle(() => {+ if (grpcCode(err) === GRPC_NOT_FOUND) {+ resolve(null);+ return;+ }+ const detail = guestDetail(err);+ void disconnect();+ reject(+ grpcCode(err) === GRPC_UNAVAILABLE+ ? new ManagedAgentsSandboxGoneError(sessionId, detail)+ : new Error(`do-managed-agents download ${absPath} failed: ${detail}`, { cause: err }),+ );+ });+ }) as (arg: never) => void);+ stream.on("end", (() => settle(() => resolve(new Uint8Array(Buffer.concat(chunks))))) as (arg: never) => void);+ });+ }++ const guestDetail = (err: unknown): string => lastTunnel?.lastFailure() ?? errMessage(err);++ return {+ sessionId,+ sandboxId: info.sandboxId,++ async runCommand(command, runOpts): Promise<ManagedAgentsCommandResult> {+ const { agent } = await connect();+ const timeoutMs = Math.min(runOpts?.timeoutMs ?? maxCommandMs, maxCommandMs);+ return new Promise<ManagedAgentsCommandResult>((resolve, reject) => {+ const stream = agent.Exec();+ const out: Buffer[] = [];+ const errOut: Buffer[] = [];+ let outBytes = 0;+ let errBytes = 0;+ let exit: { exit_code?: number; hitl_rejected?: boolean } | undefined;+ let settled = false;+ let quietSince = Date.now();+ const settle = (fn: () => void): void => {+ if (settled) return;+ settled = true;+ clearInterval(watchdog);+ fn();+ };++ const watchdog = setInterval(() => {+ if (settled || Date.now() - quietSince < stallProbeMs) return;+ void getInfo(sessionId).then(+ (live) => {+ quietSince = Date.now();+ if (settled || USABLE_STATES.has(live.state)) return;+ settle(() => {+ void disconnect();+ reject(+ GONE_STATES.has(live.state)+ ? new ManagedAgentsSandboxGoneError(sessionId, `status ${live.state}`)+ : new ManagedAgentsCommandLostError(sessionId, `the session became ${live.state} mid-command`),+ );+ });+ },+ swallowAs(`do-managed-agents ${sessionId}: stall probe`, undefined),+ );+ }, stallProbeMs);+ watchdog.unref();++ stream.on("data", ((frame: ExecOutputFrame) => {+ quietSince = Date.now();+ if (frame.stdout && outBytes < MAX_EXEC_OUTPUT_BYTES) {+ out.push(frame.stdout);+ outBytes += frame.stdout.length;+ }+ if (frame.stderr && errBytes < MAX_EXEC_OUTPUT_BYTES) {+ errOut.push(frame.stderr);+ errBytes += frame.stderr.length;+ }+ if (frame.exit) exit = frame.exit;+ }) as (arg: never) => void);++ stream.on("error", ((err: unknown) => {+ settle(() => {+ const detail = guestDetail(err);+ void disconnect();+ reject(+ isGuestGone(err)+ ? new ManagedAgentsCommandLostError(sessionId, detail)+ : new Error(`do-managed-agents exec failed: ${detail}`),+ );+ });+ }) as (arg: never) => void);++ stream.on("end", (() => {+ settle(() => {+ if (exit?.hitl_rejected) {+ reject(new ManagedAgentsHitlRejectedError(sessionId));+ return;+ }+ if (exit?.exit_code === undefined) {+ void disconnect();+ reject(new ManagedAgentsCommandLostError(sessionId, "the command stream ended without an exit status"));+ return;+ }+ resolve({+ stdout: Buffer.concat(out).toString("utf8"),+ stderr: Buffer.concat(errOut).toString("utf8"),+ exitCode: exit.exit_code,+ });+ });+ }) as (arg: never) => void);++ stream.write({+ start: {+ argv: ["/bin/bash", "-c", command],+ ...(runOpts?.workdir ? { workdir: runOpts.workdir } : {}),+ ...(runOpts?.env ? { env: runOpts.env } : {}),+ timeout_seconds: Math.ceil(timeoutMs / 1000),+ },+ });+ stream.end();+ });+ },++ async readFileBytes(absPath): Promise<Uint8Array | null> {+ try {+ return await download(absPath);+ } catch (err) {+ if (grpcCode((err as { cause?: unknown }).cause) !== GRPC_INTERNAL) throw err;+ const probe = await this.runCommand(`test -e '${absPath.replaceAll("'", "'\\''")}'`, { timeoutMs: 30_000 });+ if (probe.exitCode === 1) return null;+ throw err;+ }+ },++ async writeFileBytes(absPath, data): Promise<void> {+ const { agent } = await connect();+ await new Promise<void>((resolve, reject) => {+ const stream = agent.Upload((err: unknown) => {+ if (!err) {+ resolve();+ return;+ }+ const detail = guestDetail(err);+ void disconnect();+ reject(+ isGuestGone(err)+ ? new ManagedAgentsSandboxGoneError(sessionId, detail)+ : new Error(`do-managed-agents upload ${absPath} failed: ${detail}`),+ );+ });+ stream.write({ header: { path: absPath, mode: 0o644, is_archive: false } });+ for (let offset = 0; offset < data.length; offset += UPLOAD_CHUNK_BYTES)+ stream.write({ chunk: Buffer.from(data.subarray(offset, offset + UPLOAD_CHUNK_BYTES)) });+ stream.write({ end: {} });+ stream.end();+ });+ },++ async createCheckpoint(label?: string): Promise<ManagedAgentsCheckpoint> {+ const body = await callJson<{ checkpoint?: WireCheckpoint }>(+ "POST",+ `/v2/agents/sessions/${sessionId}/checkpoints`,+ sessionId,+ JSON.stringify(label ? { label } : {}),+ "application/json",+ CHECKPOINT_TIMEOUT_MS,+ );+ const captured = body.checkpoint;+ if (!captured?.checkpoint_id)+ throw new Error(`do-managed-agents create checkpoint ${sessionId}: response carried no checkpoint id`);+ if (captured.status !== "READY") {+ const detail = captured.error_message ? `: ${captured.error_message}` : "";+ throw new Error(+ `do-managed-agents create checkpoint ${sessionId}: status ${captured.status ?? "missing"}${detail}`,+ );+ }+ const createdAtMs = captured.created_at ? Date.parse(captured.created_at) : Number.NaN;+ return {+ checkpointId: captured.checkpoint_id,+ status: captured.status,+ ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}),+ };+ },++ async deleteCheckpoint(checkpointId: string): Promise<void> {+ await callJson<void>(+ "DELETE",+ `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}`,+ sessionId,+ );+ },++ async close(): Promise<void> {+ await disconnect();+ },++ async pause(): Promise<void> {+ await disconnect();+ await callJson<void>("POST", `/v2/agents/sessions/${sessionId}/pause`, sessionId);+ },++ async resume(): Promise<void> {+ await callJson<void>("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId);+ },++ async kill(): Promise<void> {+ await disconnect();+ await callJson<void>("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId);+ },+ };+ }++ return {+ nativePause: true,++ info: getInfo,++ async create(createOpts): Promise<ManagedAgentsSession> {+ const manifest = managedAgentsManifest({+ name: createOpts.name,+ template,+ ...(opts.agent ? { agent: opts.agent } : {}),+ ...(opts.sizeSlug ? { sizeSlug: opts.sizeSlug } : {}),+ ...(opts.idleTimeoutSec ? { idleTimeoutSec: opts.idleTimeoutSec } : {}),+ ...((createOpts.egressAllow ?? egressAllow) ? { egressAllow: createOpts.egressAllow ?? egressAllow } : {}),+ });+ const body = await callJson<{ session?: WireSession }>(+ "POST",+ "/v2/agents/sessions",+ createOpts.name,+ manifest,+ "application/x-yaml",+ CREATE_TIMEOUT_MS,+ );+ const created = body.session ? toSummary(body.session) : null;+ if (!created?.sessionId) throw new Error("do-managed-agents create session: response carried no session id");+ return buildSession(await awaitUsable(created.sessionId));+ },++ async connect(sessionId): Promise<ManagedAgentsSession> {+ return buildSession(await awaitUsable(sessionId));+ },++ async rollback(sessionId, checkpointId): Promise<ManagedAgentsSessionInfo> {+ return rollbackToCheckpoint(sessionId, checkpointId);+ },++ async list(name): Promise<ManagedAgentsSessionSummary[]> {+ const body = await callJson<{ sessions?: WireSession[] }>(+ "GET",+ `/v2/agents/sessions?name=${encodeURIComponent(name)}`,+ name,+ );+ return (body.sessions ?? []).map(toSummary).filter((s) => s.name === name && !GONE_STATES.has(s.state));+ },++ async kill(sessionId): Promise<void> {+ await callJson<void>("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId);+ },+ };+}diff --git a/src/sandbox/managed-agents-sandbox-agent.proto b/src/sandbox/managed-agents-sandbox-agent.protonew file mode 100644index 000000000..59f5c8dda--- /dev/null+++ b/src/sandbox/managed-agents-sandbox-agent.proto@@ -0,0 +1,98 @@+syntax = "proto3";++package do.teams.hosted_agents.runtime.sandbox_agent.v1;++service SandboxAgentService {+ rpc Exec(stream ExecRequest) returns (stream ExecOutput);+ rpc Upload(stream UploadRequest) returns (UploadResponse);+ rpc Download(DownloadRequest) returns (stream DownloadChunk);+}++message ExecRequest {+ oneof request {+ ExecStart start = 1;+ bytes stdin = 2;+ ExecSignal signal = 3;+ }+}++message ExecStart {++ repeated string argv = 1;+ string workdir = 2;+ map<string, string> env = 3;++ int64 timeout_seconds = 4;+}++message ExecSignal {+ int32 signal = 1;+}++message ExecOutput {+ oneof output {+ bytes stdout = 1;+ bytes stderr = 2;+ ExecExit exit = 3;+ }+}++message ExecExit {+ int32 exit_code = 1;+ int64 duration_ms = 2;++ bool hitl_rejected = 3;+ bool timed_out = 4;+}++message UploadRequest {+ oneof request {+ UploadHeader header = 1;+ bytes chunk = 2;+ UploadEnd end = 3;+ }+}++message UploadHeader {++ string path = 1;++ uint32 mode = 2;++ bool is_archive = 3;+}++message UploadEnd {++ string sha256 = 1;+}++message UploadResponse {+ string path = 1;+ int64 bytes_written = 2;+}++message DownloadRequest {+ string path = 1;+ bool as_archive = 2;++ uint32 chunk_size_bytes = 3;+}++message DownloadChunk {+ oneof output {+ DownloadHeader header = 1;+ bytes chunk = 2;+ DownloadEnd end = 3;+ }+}++message DownloadHeader {++ int64 size_bytes = 1;+ bool is_archive = 2;+}++message DownloadEnd {+ string sha256 = 1;+}diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.tsnew file mode 100644index 000000000..0ebdf9afc--- /dev/null+++ b/src/sandbox/managed-agents-sandbox.ts@@ -0,0 +1,631 @@+import { cleanupFailedProvision } from "./sandbox.ts";+import { randomUUID } from "node:crypto";+import type { WorkspaceLayer } from "../types.ts";+import type { WorkspaceStore } from "../workspace/workspace-store.ts";+import type { DurableMap } from "../persistence/durable-map.ts";+import { createMemoryMap } from "../persistence/durable-map.ts";+import { createKeyedQueue } from "../util/async.ts";+import { swallowAs, errMessage } from "../util/errors.ts";+import { shq } from "../util/shell.ts";+import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, forceThroughProxyEnv } from "./sandbox-env.ts";+import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";+import { materializeRoLayers } from "./ro-layers.ts";+import { createLayerToolInstaller } from "./layer-tool-install.ts";+import type { LayerInstallFile } from "../deployment/load-layer.ts";+import {+ createBackendBlobStaging,+ createExecExport,+ createExecFileOps,+ posixJoin,+ type BlobStagingOptions,+} from "./exec-file-ops.ts";+import {+ ephemeralCredLinkScript,+ ephemeralCredLinkPaths,+ type CredentialPathSpec,+} from "../credentials/resident-paths.ts";+import { killableScript, killScript } from "./exec-kill.ts";+import { visibleNotInstalled, visibleTools } from "./sandbox.ts";+import { sandboxScopeName } from "./exec-sandbox-base.ts";+import {+ ManagedAgentsCommandLostError,+ ManagedAgentsSandboxGoneError,+ type ManagedAgentsClient,+ type ManagedAgentsSessionInfo,+ type ManagedAgentsSession,+} from "./managed-agents-client.ts";+import {+ createHomeSnapshotOps,+ createMemorySnapshotStore,+ HOME_SNAPSHOT_PRUNE,+ snapshotDue,+ type HomeSnapshotStore,+} from "./home-snapshot.ts";+import type {+ AgentComputerProfile,+ ComputerStatus,+ ExecOptions,+ ExecResult,+ ProvisionOptions,+ Sandbox,+ SandboxHandle,+ TeardownOptions,+} from "./sandbox.ts";++const HOME_DIR = "/workspace/home";+const WORKSPACE_BASENAME = "workspace";+const RO_LAYERS_TAR = ".ro-layers.tar";+const RO_LAYERS_MANIFEST = ".ro-layers.manifest";+const HOME_TAR = `${HOME_DIR}/.qm-home.tar`;++const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE;+const DEFAULT_SNAPSHOT_INTERVAL_MS = 5 * 60_000;+const CHECKPOINT_LABEL = "qm teardown";++export const QM_TEMPLATE_TOOLS = ["jq", "rg", "unzip", "wget"];++export interface StoredManagedAgentsSandbox {+ sessionId: string;+ sandboxId: string;+ preservationState?: "running" | "paused" | "pause_failed";+ preservationError?: string;+ createdAtMs: number;+ lastSnapshotMs?: number;+ homeDirty?: boolean;+ checkpointId?: string;+ checkpointAtMs?: number;+ checkpointError?: string;+ pendingCheckpointDeletes?: string[];+}++export interface ManagedAgentsSandboxOptions extends BlobStagingOptions {+ client: ManagedAgentsClient;+ namePrefix?: string;+ defaultTimeoutSec?: number;+ snapshotIntervalMs?: number;+ egressProxyUrl?: string;+ extraTools?: string[];+ credentialPaths?: CredentialPathSpec[];+ layerToolFiles?: () => readonly LayerInstallFile[];+ store?: DurableMap<StoredManagedAgentsSandbox>;+ snapshots?: HomeSnapshotStore;+ onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;+}++export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: ManagedAgentsSandboxOptions): Sandbox {+ const client = opts.client;+ const prefix = opts.namePrefix ?? "qm";+ const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;+ const snapshotIntervalMs = opts.snapshotIntervalMs ?? DEFAULT_SNAPSHOT_INTERVAL_MS;+ const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`;+ const store = opts.store ?? createMemoryMap<StoredManagedAgentsSandbox>();+ const snapshots = opts.snapshots ?? createMemorySnapshotStore();+ const provisionQueue = createKeyedQueue<string>();++ const sessionByName = new Map<string, ManagedAgentsSession>();+ const scopeByName = new Map<string, string>();+ const scratchKeyByName = new Map<string, string>();+ const activeScratch = new Map<string, number>();++ const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {+ opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });+ };++ const homeSnapshots = createHomeSnapshotOps<ManagedAgentsSession>({+ label: "do-managed-agents",+ homeDir: HOME_DIR,+ homeTarPath: HOME_TAR,+ prunePaths: [...SNAPSHOT_PRUNE, ...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`)],+ store: snapshots,+ io: {+ runCommand: (session, script, timeoutMs) => session.runCommand(script, { timeoutMs }),+ readFileBytes: (session, abs) => session.readFileBytes(abs),+ writeFileBytes: (session, abs, data) => session.writeFileBytes(abs, data),+ },+ });++ async function snapshotHome(scope: string, session: ManagedAgentsSession): Promise<void> {+ await homeSnapshots.snapshotHome(scope, session);+ await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false });+ }++ async function pendingDeletes(scope: string): Promise<string[]> {+ return (await store.get(scope))?.pendingCheckpointDeletes ?? [];+ }++ async function forgetCheckpoint(scope: string, session: ManagedAgentsSession, checkpointId: string): Promise<void> {+ try {+ await session.deleteCheckpoint(checkpointId);+ } catch (e) {+ reportError("sandbox_snapshot", "checkpoint_delete_failed", errMessage(e), scope);+ return;+ }+ const pending = await pendingDeletes(scope);+ await store.merge(scope, { pendingCheckpointDeletes: pending.filter((id) => id !== checkpointId) });+ }++ async function retryPendingCheckpointDeletes(scope: string, session: ManagedAgentsSession): Promise<void> {+ const current = (await store.get(scope))?.checkpointId;+ for (const checkpointId of await pendingDeletes(scope)) {+ if (checkpointId === current) continue;+ await forgetCheckpoint(scope, session, checkpointId);+ }+ }++ async function captureCheckpoint(scope: string, session: ManagedAgentsSession): Promise<void> {+ const stored = await store.get(scope);+ const previous = stored?.checkpointId;+ const captured = await session.createCheckpoint(CHECKPOINT_LABEL);+ const pending = stored?.pendingCheckpointDeletes ?? [];+ const queued =+ previous && previous !== captured.checkpointId && !pending.includes(previous) ? [...pending, previous] : pending;+ await store.merge(scope, {+ checkpointId: captured.checkpointId,+ checkpointAtMs: captured.createdAtMs ?? Date.now(),+ checkpointError: undefined,+ pendingCheckpointDeletes: queued,+ });+ if (previous && previous !== captured.checkpointId) await forgetCheckpoint(scope, session, previous);+ }++ const hydrateHome = (scope: string, session: ManagedAgentsSession): Promise<boolean> =>+ homeSnapshots.hydrateHome(scope, session);++ const recordSession = (scope: string, session: ManagedAgentsSession, createdAtMs: number): Promise<void> =>+ store.put(scope, {+ sessionId: session.sessionId,+ sandboxId: session.sandboxId,+ createdAtMs,+ preservationState: "running",+ });++ async function ensureSession(+ scope: string,+ name: string,+ onStatus?: (text: string) => void,+ ): Promise<{ session: ManagedAgentsSession; coldStart: boolean }> {+ return provisionQueue(scope, async () => {+ const cached = sessionByName.get(name);+ if (cached) return { session: cached, coldStart: false };++ const adopt = (session: ManagedAgentsSession): { session: ManagedAgentsSession; coldStart: boolean } => {+ sessionByName.set(name, session);+ return { session, coldStart: false };+ };++ const stored = await store.get(scope);+ if (stored) {+ try {+ onStatus?.("Resuming the sandbox…");+ const session = await client.connect(stored.sessionId);+ await store.merge(scope, { preservationState: "running", sandboxId: session.sandboxId });+ return adopt(session);+ } catch (err) {+ if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;+ }+ }++ for (const summary of await client.list(name)) {+ try {+ const session = await client.connect(summary.sessionId);+ await recordSession(scope, session, Date.now());+ return adopt(session);+ } catch (err) {+ if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;+ }+ }++ onStatus?.("Creating the sandbox…");+ const session = await client.create({ name });+ sessionByName.set(name, session);+ await recordSession(scope, session, Date.now());+ let hydrated: boolean;+ try {+ hydrated = await hydrateHome(scope, session);+ } catch (e) {+ reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);+ sessionByName.delete(name);+ await client.kill(session.sessionId).catch((err: unknown) => {+ reportError("sandbox_hydrate", "orphan_kill_failed", errMessage(err), scope);+ });+ throw new Error(+ `do-managed-agents provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`,+ {+ cause: e,+ },+ );+ }+ return { session, coldStart: !hydrated };+ });+ }++ async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> {+ const name = sandboxScopeName(`${prefix}-scratch`, key);+ return provisionQueue(`scratch:${key}`, async () => {+ scratchKeyByName.set(name, key);+ const active = activeScratch.get(name) ?? 0;+ if (active === 0 && !sessionByName.has(name)) {+ sessionByName.set(name, await client.create({ name }));+ }+ activeScratch.set(name, active + 1);+ return { name, coldStart: active === 0 };+ });+ }++ async function withSession<T>(name: string, action: (session: ManagedAgentsSession) => Promise<T>): Promise<T> {+ const scratchKey = scratchKeyByName.get(name);+ if (scratchKey === undefined && !scopeByName.has(name)) throw new Error("sandbox handle has been released");++ const reviveScratch = async (): Promise<ManagedAgentsSession> => {+ const session = await client.create({ name });+ sessionByName.set(name, session);+ return session;+ };+ const first =+ scratchKey !== undefined+ ? { session: sessionByName.get(name) ?? (await reviveScratch()) }+ : await ensureSession(scopeByName.get(name) ?? "default", name);+ try {+ return await action(first.session);+ } catch (err) {+ if (err instanceof ManagedAgentsCommandLostError) sessionByName.delete(name);+ if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;+ sessionByName.delete(name);+ const second =+ scratchKey !== undefined+ ? { session: await reviveScratch() }+ : await ensureSession(scopeByName.get(name) ?? "default", name);+ return action(second.session);+ }+ }++ async function execRaw(name: string, script: string, timeoutSec: number): Promise<ExecResult> {+ return withSession(name, async (session) => {+ const r = await session.runCommand(`sh -c ${shq(`timeout -k 5 ${timeoutSec} sh -c ${shq(script)}; exit $?`)}`, {+ timeoutMs: timeoutSec * 1000 + 30_000,+ });+ return {+ stdout: r.stdout,+ stderr: r.stderr,+ code: r.exitCode,+ timedOut: r.exitCode === 124 || r.exitCode === 137,+ };+ });+ }++ const profile: AgentComputerProfile = {+ backend: "do-managed-agents",+ writablePersistence: client.nativePause ? "provider_managed" : "snapshot_to_workspace",+ processSessions: true,+ parksOnTeardown: true,+ egressEnforcement: opts.egressProxyUrl ? "domain" : "none",+ spec: {+ os: "Linux — DigitalOcean Managed Agents Firecracker microVM (provider pause preserves state; publish durable work to git or Files)",+ runtimes: ["Node", "Python 3"],+ get tools() {+ return visibleTools(["git", "curl", "tar", "python3", ...(opts.extraTools ?? [])]);+ },+ get notInstalled() {+ return visibleNotInstalled(+ ["jq", "rg", "unzip", "wget", "gh", "aws", "gcloud", "kubectl", "flyctl", "glab"],+ opts.extraTools ?? [],+ );+ },+ homeDir: HOME_DIR,+ workdir: workspaceDir,+ },+ };++ const procIo: ExecProcessIo = {+ async run(handle, command, execOpts): Promise<ExecResult> {+ const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;+ return execRaw(handle.id, command, timeoutSec);+ },+ };+ const procSessions = createExecProcessSessions(procIo);++ const writeAbsBytes = (name: string, absPath: string, data: Uint8Array): Promise<void> =>+ withSession(name, (session) => session.writeFileBytes(absPath, data));+ const readAbsBytes = (name: string, absPath: string): Promise<Uint8Array | null> =>+ withSession(name, (session) => session.readFileBytes(absPath));+ const installLayerTools = opts.layerToolFiles ? createLayerToolInstaller(opts.layerToolFiles) : null;++ const execFileOps = createExecFileOps({+ label: "do-managed-agents",+ exec: (id, script, t) => execRaw(id, script, t),+ writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),+ });++ const execExport = createExecExport({+ label: "do-managed-agents",+ exec: (id, script, t) => execRaw(id, script, t),+ readAbsBytes,+ defaultHomeDir: HOME_DIR,+ ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),+ });++ const blobStaging = createBackendBlobStaging("do-managed-agents", (id, script, t) => execRaw(id, script, t), opts);++ async function destroyStoredScope(scope: string): Promise<void> {+ const name = sandboxScopeName(prefix, scope);+ const stored = await store.get(scope);+ const cached = sessionByName.get(name);+ const ids = new Set([stored?.sessionId, cached?.sessionId].filter((id): id is string => !!id));+ for (const id of ids) {+ try {+ if (cached && id === cached.sessionId) await cached.kill();+ else await client.kill(id);+ } catch (error) {+ if (!(error instanceof ManagedAgentsSandboxGoneError)) throw error;+ }+ }+ await store.delete(scope);+ sessionByName.delete(name);+ scopeByName.delete(name);+ }++ const sandbox: Sandbox = {+ destroyScope(scopeId: string): Promise<void> {+ return provisionQueue(scopeId, () => destroyStoredScope(scopeId));+ },++ profile,+ startProcess: procSessions.startProcess,+ readProcess: procSessions.readProcess,+ writeStdin: procSessions.writeStdin,+ signalProcess: procSessions.signalProcess,+ listProcesses: procSessions.listProcesses,+ ...execFileOps,+ ...blobStaging,++ async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise<SandboxHandle> {+ const scratch = provOpts?.scratch;+ const writable = layers.find((l) => l.mode === "rw") ?? layers[0];+ const scope = writable?.scopeId ?? "default";+ let name: string;+ let coldStart: boolean;+ if (scratch) {+ ({ name, coldStart } = await ensureScratch(scratch.key));+ } else {+ name = sandboxScopeName(prefix, scope);+ scopeByName.set(name, scope);+ ({ coldStart } = await ensureSession(scope, name, provOpts?.onStatus));+ }++ const forceEgress = !!opts.egressProxyUrl && !!provOpts?.egressToken;+ const turnEnv = Object.fromEntries(+ Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),+ );+ const env = {+ ...turnEnv,+ ...(forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!) : {}),+ };+ const handle: SandboxHandle = {+ id: name,+ rootDir: workspaceDir,+ homeDir: HOME_DIR,+ coldStart,+ ...(scratch ? { scratch: true } : {}),+ ...(Object.keys(env).length ? { env } : {}),+ };++ try {+ const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(HOME_DIR, opts.credentialPaths ?? [])}`;+ const prep = await execRaw(name, `mkdir -p ${shq(workspaceDir)}${credLinks}`, 60);+ if (prep.code !== 0)+ throw new Error(`do-managed-agents provision prep failed: ${(prep.stderr || prep.stdout).slice(0, 200)}`);++ await materializeRoLayers(+ workspace,+ layers,+ handle,+ {+ readFile: (h, rel) => sandbox.readFile(h, rel),+ writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data),+ exec: (script, t) => execRaw(name, script, t),+ },+ { manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "do-managed-agents" },+ );+ await installLayerTools?.({+ exec: (script, t) => execRaw(name, script, t),+ writeAbs: (abs, data) => writeAbsBytes(name, abs, data),+ });++ return handle;+ } catch (err) {+ await cleanupFailedProvision(sandbox, handle, err);+ throw err;+ }+ },++ async run(handle, command, execOpts?: ExecOptions): Promise<ExecResult> {+ const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;+ const exports = Object.entries(handle.env ?? {})+ .map(([k, v]) => `export ${k}=${shq(v)}`)+ .join("; ");+ const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${handle.rootDir} 2>/dev/null; ${command}`;+ const signal = execOpts?.signal;+ if (!signal) return execRaw(handle.id, script, timeoutSec);+ const killUid = randomUUID();+ const fireKill = (): void => {+ execRaw(handle.id, killScript(killUid), 15).catch(+ swallowAs("managed-agents-sandbox: kill in-flight exec", undefined),+ );+ };+ signal.throwIfAborted();+ const onAbort = (): void => fireKill();+ signal.addEventListener("abort", onAbort, { once: true });+ try {+ return await execRaw(handle.id, killableScript(script, killUid), timeoutSec);+ } finally {+ signal.removeEventListener("abort", onAbort);+ }+ },++ async writeFileBytes(handle, relPath, data): Promise<void> {+ await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data);+ },+ async writeFile(handle, relPath, data): Promise<void> {+ await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8"));+ },+ async readFileBytes(handle, relPath): Promise<Uint8Array | null> {+ return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath));+ },+ async readFile(handle, relPath): Promise<string | null> {+ const bytes = await sandbox.readFileBytes(handle, relPath);+ return bytes === null ? null : Buffer.from(bytes).toString("utf8");+ },++ exportFiles: execExport.exportFiles,++ async persistHomeSnapshot(scopeId: string): Promise<void> {+ const name = sandboxScopeName(prefix, scopeId);+ scopeByName.set(name, scopeId);+ const { session } = await ensureSession(scopeId, name);+ await snapshotHome(scopeId, session);+ },++ async restartComputer(scopeId: string): Promise<void> {+ const name = sandboxScopeName(prefix, scopeId);+ scopeByName.set(name, scopeId);+ return provisionQueue(scopeId, async () => {+ const stored = await store.get(scopeId);+ if (!stored?.sessionId) throw new Error("do-managed-agents restart: no sandbox provisioned yet");+ const checkpointId = stored.checkpointId;+ if (!checkpointId) throw new Error("do-managed-agents restart: no checkpoint to restore");+ const cached = sessionByName.get(name);+ sessionByName.delete(name);+ if (cached) await cached.close();+ let restored: ManagedAgentsSessionInfo;+ try {+ restored = await client.rollback(stored.sessionId, checkpointId);+ } catch (e) {+ reportError("agent_computer", "checkpoint_rollback_failed", errMessage(e), scopeId);+ throw e;+ }+ await store.merge(scopeId, {+ sandboxId: restored.sandboxId,+ preservationState: "running",+ preservationError: undefined,+ });+ reportError(+ "agent_computer",+ "checkpoint_restored",+ `session ${stored.sessionId}: restored checkpoint ${checkpointId} onto sandbox ${restored.sandboxId}`,+ scopeId,+ );+ });+ },++ async computerStatus(scopeId: string): Promise<ComputerStatus> {+ const name = sandboxScopeName(prefix, scopeId);+ const stored = await store.get(scopeId);++ if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false };+ const machine = `do-managed-agents sandbox ${stored.sandboxId || stored.sessionId}`;+ const checkpointAtMs = stored.checkpointAtMs ?? stored.lastSnapshotMs;+ const recoveryError = stored.preservationError ?? stored.checkpointError;+ const recovery = {+ strategy: "workspace_snapshot" as const,+ state: stored.preservationState,+ ...(recoveryError ? { error: recoveryError } : {}),+ ...(stored.checkpointId ? { checkpointId: stored.checkpointId } : {}),+ ...(checkpointAtMs ? { checkpointAtMs } : {}),+ };+ try {+ const info: ManagedAgentsSessionInfo = await client.info(stored.sessionId);+ if (info.state === "paused")+ return {+ machine,+ listed: info.state,+ lifecycleState: "paused",+ provisioned: true,+ guestResponsive: false,+ recovery: { ...recovery, state: "paused" },+ };+ scopeByName.set(name, scopeId);+ let session = sessionByName.get(name);+ if (!session) {+ session = await client.connect(stored.sessionId);+ sessionByName.set(name, session);+ }+ const r = await session.runCommand("echo responsive", { timeoutMs: 30_000 });+ return {+ machine,+ recovery,+ provisioned: true,+ guestResponsive: r.exitCode === 0 && r.stdout.trim() === "responsive",+ };+ } catch (e) {+ return {+ recovery,+ machine: `${machine} (${errMessage(e).slice(0, 120)})`,+ provisioned: !(e instanceof ManagedAgentsSandboxGoneError),+ guestResponsive: false,+ };+ }+ },++ async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {+ if (handle.scratch) {+ const key = scratchKeyByName.get(handle.id);+ return provisionQueue(key ? `scratch:${key}` : handle.id, async () => {+ const remaining = (activeScratch.get(handle.id) ?? 1) - 1;+ if (remaining > 0) {+ activeScratch.set(handle.id, remaining);+ return;+ }+ activeScratch.delete(handle.id);+ const session = sessionByName.get(handle.id);+ if (session) {+ if (tdOpts?.destroy) await session.kill();+ else await session.kill().catch(swallowAs("managed-agents-sandbox: scratch kill", undefined));+ }+ sessionByName.delete(handle.id);+ scratchKeyByName.delete(handle.id);+ });+ }+ if (tdOpts?.destroy && !scopeByName.has(handle.id)) return;+ const scope = scopeByName.get(handle.id) ?? "default";+ return provisionQueue(scope, () => teardownScope(handle, scope, tdOpts));+ },+ };++ async function teardownScope(handle: SandboxHandle, scope: string, tdOpts?: TeardownOptions): Promise<void> {+ const session = sessionByName.get(handle.id);+ if (tdOpts?.destroy) return destroyStoredScope(scope);+ if (!session) return;++ const stored = await store.get(scope);+ if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });+ await retryPendingCheckpointDeletes(scope, session);+ const homeDirty = stored?.homeDirty;+ if (snapshotDue({ lastSnapshotMs: stored?.checkpointAtMs, homeDirty }, tdOpts, snapshotIntervalMs)) {+ try {+ await captureCheckpoint(scope, session);+ } catch (e) {+ await store.merge(scope, { checkpointError: errMessage(e) });+ reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), scope);+ }+ }+ if (snapshotDue({ lastSnapshotMs: stored?.lastSnapshotMs, homeDirty }, tdOpts, snapshotIntervalMs)) {+ try {+ await snapshotHome(scope, session);+ } catch (e) {+ reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope);+ }+ }+ if (tdOpts?.keepWarm) return;+ try {+ await session.pause();+ await store.merge(scope, { preservationState: "paused", preservationError: undefined });+ sessionByName.delete(handle.id);+ } catch (error) {+ await store.merge(scope, { preservationState: "pause_failed", preservationError: errMessage(error) });+ reportError("sandbox_preservation", "pause_failed", errMessage(error), scope);+ throw error;+ }+ }++ return sandbox;+}diff --git a/src/sandbox/managed-agents-tunnel.ts b/src/sandbox/managed-agents-tunnel.tsnew file mode 100644index 000000000..a0557e730--- /dev/null+++ b/src/sandbox/managed-agents-tunnel.ts@@ -0,0 +1,151 @@+import { createServer, type Server, type Socket } from "node:net";+import { once } from "node:events";+import type WebSocket from "ws";++export const SANDBOX_AGENT_PORT = 8443;++const TUNNEL_CLOSE_SESSION_UNAVAILABLE = 4001;+const TUNNEL_CLOSE_GUEST_DIAL_FAILED = 4002;+const TUNNEL_CLOSE_UPSTREAM_ERROR = 4011;+const REJECTION_BODY_MAX = 300;+const PING_INTERVAL_MS = 30_000;++export interface ManagedAgentsTunnel {+ readonly localPort: number;+ lastFailure(): string | null;+ whenFailed(): Promise<string>;+ close(): Promise<void>;+}++export interface ManagedAgentsTunnelOptions {+ apiBaseUrl: string;+ sessionId: string;+ remotePort: number;+ getToken: () => Promise<string>;+ pingIntervalMs?: number;+}++export function tunnelUrl(apiBaseUrl: string, sessionId: string, remotePort: number): string {+ const url = new URL(apiBaseUrl);+ if (url.protocol !== "http:" && url.protocol !== "https:")+ throw new Error(`DO_AGENTS_API_BASE_URL must be http(s), got ${url.protocol}`);+ url.protocol = url.protocol === "https:" ? "wss:" : "ws:";+ url.pathname = `${url.pathname.replace(/\/$/, "")}/v2/agents/sessions/${sessionId}/port-forward/${remotePort}`;+ return url.toString();+}++type WebSocketCtor = typeof WebSocket;++let wsModule: WebSocketCtor | null = null;+async function loadWebSocket(): Promise<WebSocketCtor> {+ wsModule ??= (await import("ws")).default;+ return wsModule;+}++export async function openManagedAgentsTunnel(opts: ManagedAgentsTunnelOptions): Promise<ManagedAgentsTunnel> {+ const WebSocketImpl = await loadWebSocket();+ const url = tunnelUrl(opts.apiBaseUrl, opts.sessionId, opts.remotePort);+ const sockets = new Set<Socket>();+ let failure: string | null = null;+ let closed = false;+ let announceFailure: (detail: string) => void = () => undefined;+ const failed = new Promise<string>((resolve) => {+ announceFailure = resolve;+ });+ const fail = (detail: string): void => {+ failure = detail;+ announceFailure(detail);+ };++ const bridge = (local: Socket): void => {+ sockets.add(local);+ local.once("close", () => sockets.delete(local));+ void opts+ .getToken()+ .then((token) => {+ if (closed) {+ local.destroy();+ return;+ }+ const ws = new WebSocketImpl(url, { headers: { Authorization: `Bearer ${token}` } });+ let rejected = false;++ ws.on("unexpected-response", (_req, res) => {+ rejected = true;+ let body = "";+ res.on("data", (chunk: Buffer) => {+ body += chunk.toString("utf8");+ });+ res.on("end", () => {+ fail(`${res.statusCode ?? 0} ${body.trim().slice(0, REJECTION_BODY_MAX)}`.trim());+ local.destroy();+ });+ });++ ws.on("open", () => {+ const heartbeat = setInterval(() => {+ if (ws.readyState === WebSocketImpl.OPEN) ws.ping();+ }, opts.pingIntervalMs ?? PING_INTERVAL_MS);+ heartbeat.unref();+ ws.once("close", () => clearInterval(heartbeat));+ local.on("data", (chunk: Buffer) => {+ local.pause();+ ws.send(chunk, (err) => {+ if (err) return;+ local.resume();+ });+ });+ ws.on("message", (data: Buffer) => {+ if (!local.write(data)) {+ ws.pause();+ local.once("drain", () => ws.resume());+ }+ });+ });++ ws.on("close", (code: number, reason: Buffer) => {+ if (+ code === TUNNEL_CLOSE_SESSION_UNAVAILABLE ||+ code === TUNNEL_CLOSE_GUEST_DIAL_FAILED ||+ code === TUNNEL_CLOSE_UPSTREAM_ERROR+ )+ fail(`close ${code} ${reason.toString("utf8")}`.trim());+ local.destroy();+ });++ ws.on("error", (err: Error) => {+ if (!rejected) fail(err.message);+ local.destroy();+ });++ local.on("close", () => {+ if (ws.readyState === WebSocketImpl.OPEN || ws.readyState === WebSocketImpl.CONNECTING) ws.close();+ });+ local.on("error", () => ws.close());+ })+ .catch((err: unknown) => {+ fail(err instanceof Error ? err.message : String(err));+ local.destroy();+ });+ };++ const server: Server = createServer(bridge);+ server.listen(0, "127.0.0.1");+ await once(server, "listening");+ server.unref();+ const address = server.address();+ if (address === null || typeof address === "string")+ throw new Error("Managed Agents tunnel failed to bind a local port");++ return {+ localPort: address.port,+ lastFailure: () => failure,+ whenFailed: () => failed,+ close: async () => {+ closed = true;+ for (const socket of sockets) socket.destroy();+ sockets.clear();+ await new Promise<void>((resolve) => server.close(() => resolve()));+ },+ };+}diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.tsindex bca3a9757..8068eeb7a 100644--- a/src/sandbox/sandbox-routing.ts+++ b/src/sandbox/sandbox-routing.ts@@ -17,7 +17,16 @@ import {} from "./sandbox.ts";export type SandboxBackendName =- "sprites" | "aws" | "local" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";+ | "sprites"+ | "aws"+ | "local"+ | "smolmachines"+ | "e2b"+ | "modal"+ | "porter"+ | "agent37"+ | "superserve"+ | "do-managed-agents";export class NoDefaultSandboxError extends Error {constructor() {diff --git a/src/wiring.ts b/src/wiring.tsindex 063032ed1..62280b4ff 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -231,6 +231,12 @@ import {import { createSdkSuperserveClient } from "./sandbox/superserve-client.ts";import { createE2bSandbox, type StoredE2bSandbox } from "./sandbox/e2b-sandbox.ts";import { createSdkE2bClient } from "./sandbox/e2b-client.ts";+import {+ createManagedAgentsSandbox,+ QM_TEMPLATE_TOOLS,+ type StoredManagedAgentsSandbox,+} from "./sandbox/managed-agents-sandbox.ts";+import { createSdkManagedAgentsClient } from "./sandbox/managed-agents-client.ts";import { createS3SnapshotStore } from "./sandbox/home-snapshot.ts";import { createModalSandbox, type StoredModalSandbox } from "./sandbox/modal-sandbox.ts";import { createSdkModalClient } from "./sandbox/modal-client.ts";@@ -921,6 +927,41 @@ export function buildApp(onError: sandboxOnError,});};+ const managedAgentsBodies = artifactMap<StoredManagedAgentsSandbox>("do_managed_agents_sandbox_bodies");+ const buildManagedAgents = (): Sandbox => {+ const managedAgents = config.managedAgentsSandbox;+ if (!managedAgents.apiToken) throw new Error("SANDBOX_BACKEND=do-managed-agents requires DO_AGENTS_API_TOKEN");+ return createManagedAgentsSandbox(workspace, {+ client: createSdkManagedAgentsClient({+ apiToken: managedAgents.apiToken,+ ...(managedAgents.apiBaseUrl ? { apiBaseUrl: managedAgents.apiBaseUrl } : {}),+ ...(managedAgents.template ? { template: managedAgents.template } : {}),+ ...(managedAgents.sizeSlug ? { sizeSlug: managedAgents.sizeSlug } : {}),+ ...(managedAgents.idleTimeoutSec ? { idleTimeoutSec: managedAgents.idleTimeoutSec } : {}),+ ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}),+ }),+ ...(managedAgents.namePrefix ? { namePrefix: managedAgents.namePrefix } : {}),+ ...(managedAgents.defaultTimeoutSec ? { defaultTimeoutSec: managedAgents.defaultTimeoutSec } : {}),+ ...(managedAgents.snapshotIntervalSec !== undefined+ ? { snapshotIntervalMs: managedAgents.snapshotIntervalSec * 1000 }+ : {}),+ ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}),+ extraTools: [...(managedAgents.template ? QM_TEMPLATE_TOOLS : []), ...deploymentLayer.advertisedTools],+ credentialPaths: deploymentLayer.credentialPaths,+ layerToolFiles: () => deploymentLayer.installFiles,+ blobTransfer,+ ...(config.signingSecret ? { signingSecret: config.signingSecret } : {}),+ ...(config.capabilitySecret ? { capabilitySecret: config.capabilitySecret } : {}),+ ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}),+ store: managedAgentsBodies,+ ...(managedAgents.snapshotS3Bucket+ ? {+ snapshots: createS3SnapshotStore({ bucket: managedAgents.snapshotS3Bucket, prefix: "managed-agents-home" }),+ }+ : {}),+ onError: sandboxOnError,+ });+ };const buildModal = (): Sandbox => {const modal = config.modalSandbox;if (!modal.tokenId || !modal.tokenSecret)@@ -1070,6 +1111,7 @@ export function buildApp(porter: buildPorter,agent37: buildAgent37,superserve: buildSuperserve,+ "do-managed-agents": buildManagedAgents,};const enabledBackends = new Set(enabledSandboxBackends(config));const sandboxBackends: Partial<Record<SandboxBackendName, Sandbox>> = {diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.tsindex ea5634a72..9bf5081d0 100644--- a/test/live-slack-sandbox-providers.test.ts+++ b/test/live-slack-sandbox-providers.test.ts@@ -64,6 +64,7 @@ test("every provider has its own parallel execution scenario", () => {assert.deepEqual(sandboxProviders.toSorted(), ["agent37","aws",+ "do-managed-agents","e2b","local","modal",@@ -159,7 +160,7 @@ test("explicit provider selection retains strict coverage without treating defer"sandbox-execute-modal",],);- assert.equal(selectSandboxProviderScenarios("all").length, 9);+ assert.equal(selectSandboxProviderScenarios("all").length, 10);assert.deepEqual(selectSandboxProviderScenarios(undefined), []);for (const value of ["", "sprites,", "typo", "sprites,sprites", "all,sprites", "constructor", "__proto__"])assert.throws(() => selectSandboxProviderScenarios(value));diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.tsindex a8675f071..8e7183c90 100644--- a/test/live-slack/scenarios-sandbox-providers.ts+++ b/test/live-slack/scenarios-sandbox-providers.ts@@ -14,6 +14,7 @@ const providerCoverage: Record<SandboxBackendName, true> = {porter: true,agent37: true,superserve: true,+ "do-managed-agents": true,};export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[];diff --git a/test/managed-agents-client.test.ts b/test/managed-agents-client.test.tsnew file mode 100644index 000000000..d8cfa4748--- /dev/null+++ b/test/managed-agents-client.test.ts@@ -0,0 +1,301 @@+import { test, before, after } from "node:test";+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import { connect } from "node:net";+import { once } from "node:events";+import { join } from "node:path";+import {+ createSdkManagedAgentsClient,+ managedAgentsEgressAllow,+ managedAgentsManifest,+ managedAgentsSessionState,+ ManagedAgentsSandboxGoneError,+ ManagedAgentsCommandLostError,+ type ManagedAgentsClient,+ type ManagedAgentsSession,+} from "../src/sandbox/managed-agents-client.ts";+import { tunnelUrl, openManagedAgentsTunnel, SANDBOX_AGENT_PORT } from "../src/sandbox/managed-agents-tunnel.ts";+import { startFakeManagedAgentsService, type FakeManagedAgentsService } from "./support/fake-managed-agents-service.ts";++let service: FakeManagedAgentsService;+let client: ManagedAgentsClient;+const opened: ManagedAgentsSession[] = [];++const session = async (name: string): Promise<ManagedAgentsSession> => {+ const created = await client.create({ name });+ opened.push(created);+ return created;+};++before(async () => {+ service = await startFakeManagedAgentsService();+ client = createSdkManagedAgentsClient({+ apiToken: service.token,+ apiBaseUrl: service.apiBaseUrl,+ template: "base",+ });+});++after(async () => {+ await Promise.all(opened.map((s) => s.kill().catch(() => undefined)));+ await service?.close();+});++test("the port-forward url targets sandbox-agent over wss", () => {+ assert.equal(+ tunnelUrl("https://api.digitalocean.com", "sess-1", SANDBOX_AGENT_PORT),+ "wss://api.digitalocean.com/v2/agents/sessions/sess-1/port-forward/8443",+ );+ assert.equal(+ tunnelUrl("http://127.0.0.1:9000/edge", "sess-2", SANDBOX_AGENT_PORT),+ "ws://127.0.0.1:9000/edge/v2/agents/sessions/sess-2/port-forward/8443",+ );+ assert.throws(() => tunnelUrl("ftp://example.com", "s", 8443), /must be http/);+});++test("the manifest asks for a bare sandbox and pre-allows bash so Managed Agents never gates a command", () => {+ const yaml = managedAgentsManifest({ name: "qm-personal-tester", template: "base" });+ assert.match(yaml, /^name: "qm-personal-tester"$/m);+ assert.match(yaml, /^agent: "none"$/m);+ assert.match(yaml, /^template: "base"$/m);+ assert.match(yaml, /tool: bash/);+ assert.match(yaml, /action: allow/);+ assert.doesNotMatch(yaml, /persistent_workspace/, "Managed Agents deprecated the key and warns when it is sent");+});++test("the manifest omits the template so Managed Agents lands a bare session on its own base", () => {+ const yaml = managedAgentsManifest({ name: "n", template: "" });+ assert.match(yaml, /^agent: "none"$/m);+ assert.doesNotMatch(yaml, /^template:/m);+});++test("the manifest carries size, idle timeout and the egress allowlist only when configured", () => {+ const bare = managedAgentsManifest({ name: "n", template: "base" });+ assert.doesNotMatch(bare, /size:|idle_timeout:|egress:/);+ const full = managedAgentsManifest({+ name: "n",+ template: "base",+ sizeSlug: "mv-2vcpu-4gb",+ idleTimeoutSec: 600,+ egressAllow: ["proxy.example.com"],+ });+ assert.match(full, /^size: "mv-2vcpu-4gb"$/m);+ assert.match(full, /^idle_timeout: "600s"$/m);+ assert.match(full, /^ {2}- "proxy\.example\.com"$/m);+});++test("the vendored proto is the client subset and carries nothing provider-internal", () => {+ const proto = readFileSync(join(process.cwd(), "src/sandbox/managed-agents-sandbox-agent.proto"), "utf8");+ assert.match(proto, /service SandboxAgentService/);+ for (const rpc of ["Exec", "Upload", "Download"]) assert.match(proto, new RegExp(`rpc ${rpc}\\(`));+ for (const unused of ["Shell", "ImportFromURL", "ExportToURL", "Health", "StartChildProcess", "ProxyPort"])+ assert.doesNotMatch(proto, new RegExp(`rpc ${unused}\\(`), `${unused} is unused and should not be vendored`);+ for (const internal of [/do\/doge/, /dorpc/, /sandboxsvc/, /sandbox-service/, /microvm\.v1/, /MSANDBOX/, /169\.254/])+ assert.doesNotMatch(proto, internal, `${internal.source} is provider-internal`);+});++test("creating a session posts a yaml manifest and waits for ready", async () => {+ const s = await session("qm-create");+ assert.ok(s.sessionId);+ assert.equal(service.sessions().length, 1);+ assert.match(service.manifests()[0] ?? "", /template: "base"/);+ assert.equal(service.sessions()[0]?.template, "base");+});++test("a command runs in the guest over the port-forward tunnel", async () => {+ const s = await session("qm-exec");+ const result = await s.runCommand("echo hello-from-guest; echo oops >&2; exit 3");+ assert.equal(result.exitCode, 3);+ assert.match(result.stdout, /hello-from-guest/);+ assert.match(result.stderr, /oops/);+ assert.ok(service.tunnelCount() > 0, "exec must travel through the tunnel, not the REST exec endpoint");+});++test("a port-forward the edge refuses reads as gone, carrying the rejection, not as a half-run command", async () => {+ const s = await session("qm-no-tunnel");+ service.rejectPortForward("team is not entitled to port-forward");+ const err = await s.runCommand("echo never-runs").then(+ () => null,+ (e: unknown) => e,+ );+ assert.ok(err instanceof ManagedAgentsSandboxGoneError, `expected a retryable gone error, got ${String(err)}`);+ assert.match(err.message, /403/);+ assert.match(err.message, /not entitled to port-forward/);+ assert.doesNotMatch(err.message, /may have partially executed/);+ service.rejectPortForward(null);+});++test("a transient lifecycle failure while waiting for readiness is polled through, not surfaced", async () => {+ const created = await session("qm-transient-ready");+ service.failNextLifecycleCalls(2, 403);+ const reconnected = await client.connect(created.sessionId);+ opened.push(reconnected);+ assert.equal(reconnected.sessionId, created.sessionId);+});++test("a lifecycle failure that never clears is surfaced instead of polled to the deadline", async () => {+ const created = await session("qm-stuck-ready");+ service.failNextLifecycleCalls(50, 403);+ const err = await client.connect(created.sessionId).then(+ () => null,+ (e: unknown) => e,+ );+ service.failNextLifecycleCalls(0, 403);+ assert.ok(err instanceof Error, "expected the persistent failure to surface");+ assert.match(err.message, /403/);+ assert.doesNotMatch(err.message, /did not become ready/);+});++test("command env and working directory reach the guest", async () => {+ const s = await session("qm-env");+ const result = await s.runCommand("echo VAR=$MY_VAR", { env: { MY_VAR: "set-by-qm" } });+ assert.match(result.stdout, /VAR=set-by-qm/);+});++test("files round trip through the guest upload and download streams", async () => {+ const s = await session("qm-files");+ const body = Buffer.from("managed agents file payload");+ await s.writeFileBytes("/home/user/notes/a.txt", new Uint8Array(body));+ const read = await s.readFileBytes("/home/user/notes/a.txt");+ assert.ok(read);+ assert.equal(Buffer.from(read).toString("utf8"), "managed agents file payload");+});++test("a missing file reads as null rather than an error", async () => {+ const s = await session("qm-missing");+ assert.equal(await s.readFileBytes("/home/user/nope.txt"), null);+});++test("resuming a paused session does not wait out a fixed poll interval", async () => {+ const created = await session("qm-resume-latency");+ service.setStatus("qm-resume-latency", "SESSION_STATUS_PAUSED");+ const started = Date.now();+ const back = await client.connect(created.sessionId);+ opened.push(back);+ assert.equal(back.sessionId, created.sessionId);+ assert.ok(Date.now() - started < 1_000, `resume took ${Date.now() - started}ms`);+});++test("a session paused mid-command fails fast instead of stalling until the command timeout", async () => {+ const watchful = createSdkManagedAgentsClient({+ apiToken: service.token,+ apiBaseUrl: service.apiBaseUrl,+ stallProbeMs: 50,+ });+ const s = await watchful.create({ name: "qm-paused-midcommand" });+ opened.push(s);+ const started = Date.now();+ const running = s.runCommand("sleep 30", { timeoutMs: 120_000 });+ await new Promise((resolve) => setTimeout(resolve, 150));+ service.setStatus("qm-paused-midcommand", "SESSION_STATUS_PAUSED");+ const err = await running.then(+ () => null,+ (e: unknown) => e,+ );+ assert.ok(err instanceof ManagedAgentsCommandLostError, `expected a command-lost error, got ${String(err)}`);+ assert.ok(Date.now() - started < 10_000, `expected a fast failure, took ${Date.now() - started}ms`);+});++test("the tunnel heartbeats so an edge idle timeout cannot kill a long silent command", async () => {+ const created = await session("qm-heartbeat");+ const before = service.pingCount();+ const tunnel = await openManagedAgentsTunnel({+ apiBaseUrl: service.apiBaseUrl,+ sessionId: created.sessionId,+ remotePort: SANDBOX_AGENT_PORT,+ getToken: async () => service.token,+ pingIntervalMs: 20,+ });+ try {+ const probe = connect(tunnel.localPort, "127.0.0.1");+ await once(probe, "connect");+ await new Promise((resolve) => setTimeout(resolve, 400));+ assert.ok(service.pingCount() > before, `expected heartbeat pings, saw ${service.pingCount() - before}`);+ probe.destroy();+ } finally {+ await tunnel.close();+ }+});++test("a large file survives the chunked upload path", async () => {+ const s = await session("qm-large");+ const body = Buffer.alloc(900 * 1024, "x");+ await s.writeFileBytes("/home/user/big.bin", new Uint8Array(body));+ const read = await s.readFileBytes("/home/user/big.bin");+ assert.equal(read?.length, body.length);+});++test("list resolves a session by name and skips destroyed rows", async () => {+ const s = await session("qm-list");+ const found = await client.list("qm-list");+ assert.equal(found.length, 1);+ assert.equal(found[0]?.sessionId, s.sessionId);+ await s.kill();+ assert.deepEqual(await client.list("qm-list"), []);+});++test("create checkpoint posts to the session checkpoint route and returns the ready id", async () => {+ const s = await session("qm-checkpoint");+ const captured = await s.createCheckpoint("qm teardown");+ assert.match(captured.checkpointId, /^cp_/);+ assert.equal(captured.status, "READY");+ assert.ok(captured.createdAtMs);+ const stored = service.checkpoints(s.sessionId);+ assert.equal(stored.length, 1);+ assert.equal(stored[0]?.checkpoint_id, captured.checkpointId);+ assert.equal(stored[0]?.label, "qm teardown");+ assert.equal(stored[0]?.status, "READY");+ await s.deleteCheckpoint(captured.checkpointId);+ assert.equal(service.checkpoints(s.sessionId).length, 0);+});++test("rollback rewinds the same session onto a new sandbox without resuming it first", async () => {+ const s = await session("qm-rollback");+ const captured = await s.createCheckpoint("qm teardown");+ const previousSandbox = s.sandboxId;+ await s.pause();+ assert.equal((await client.info(s.sessionId)).state, "paused");+ const restored = await client.rollback(s.sessionId, captured.checkpointId);+ assert.equal(restored.sessionId, s.sessionId);+ assert.notEqual(restored.sandboxId, previousSandbox);+ assert.equal(restored.state, "ready");+ assert.equal((await client.info(s.sessionId)).sandboxId, restored.sandboxId);+ assert.equal(service.checkpoints(s.sessionId).length, 1);+ await assert.rejects(() => client.rollback(s.sessionId, "missing"), /404|no such checkpoint/);+});++test("connect resumes a paused session before handing it back", async () => {+ const s = await session("qm-paused");+ await s.pause();+ assert.equal((await client.info(s.sessionId)).state, "paused");+ const again = await client.connect(s.sessionId);+ assert.equal((await client.info(again.sessionId)).state, "ready");+ const result = await again.runCommand("echo awake");+ assert.match(result.stdout, /awake/);+});++test("a destroyed session reports a terminal state, and reconnecting to it fails fast", async () => {+ const s = await session("qm-gone");+ await client.kill(s.sessionId);+ assert.equal((await client.info(s.sessionId)).state, "destroyed");+ await assert.rejects(() => client.connect(s.sessionId), ManagedAgentsSandboxGoneError);+});++test("a session the control plane has never heard of reads as gone", async () => {+ await assert.rejects(() => client.info("11111111-2222-3333-4444-555555555555"), ManagedAgentsSandboxGoneError);+});++test("a bad token is refused by the control plane", async () => {+ const wrong = createSdkManagedAgentsClient({ apiToken: "nope", apiBaseUrl: service.apiBaseUrl, template: "base" });+ await assert.rejects(() => wrong.create({ name: "qm-unauth" }), /401|bad token/);+});++test("session status wire values map onto backend states", () => {+ assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready");+ assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused");+ assert.equal(managedAgentsSessionState("nonsense"), "unspecified");+});++test("the egress allowlist carries just the proxy host", () => {+ assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]);+});diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.tsnew file mode 100644index 000000000..d651e7474--- /dev/null+++ b/test/managed-agents-sandbox.test.ts@@ -0,0 +1,397 @@+import { test, after, beforeEach } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { pollProcess } from "../src/sandbox/process-poll.ts";+import {+ createManagedAgentsSandbox,+ QM_TEMPLATE_TOOLS,+ type StoredManagedAgentsSandbox,+} from "../src/sandbox/managed-agents-sandbox.ts";+import {+ managedAgentsEgressAllow,+ managedAgentsSessionState,+ ManagedAgentsHitlRejectedError,+} from "../src/sandbox/managed-agents-client.ts";+import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts";+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";+import type { HomeSnapshotStore } from "../src/sandbox/home-snapshot.ts";+import { supportsProcessSessions } from "../src/sandbox/sandbox.ts";+import { createMemoryMap, type DurableMap } from "../src/persistence/durable-map.ts";+import { scopeId } from "../src/types.ts";+import { installFakeManagedAgents, type FakeManagedAgents } from "./support/fake-managed-agents.ts";+import type { Sandbox } from "../src/sandbox/sandbox.ts";++let fake: FakeManagedAgents;+let sandbox: Sandbox;+const scope = scopeId("personal", "tester");+const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }];+const scopeName = (): string => sandboxScopeName("qmt", scope);++function make(extra: Record<string, unknown> = {}): Sandbox {+ return createManagedAgentsSandbox(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-ws-"))), {+ client: fake.client,+ namePrefix: "qmt",+ ...extra,+ });+}++beforeEach(() => {+ fake = installFakeManagedAgents();+ sandbox = make();+});++after(() => fake?.cleanup());++test("provision runs commands with env and cwd", async () => {+ const h = await sandbox.provision(layers, { env: { MY_VAR: "v1" } });+ assert.equal(h.coldStart, true);+ const r = await sandbox.run(h, "pwd; echo VAR=$MY_VAR");+ assert.equal(r.code, 0);+ assert.match(r.stdout, /workspace/);+ assert.match(r.stdout, /VAR=v1/);+});++test("one session per scope is reused across provisions", async () => {+ const first = await sandbox.provision(layers);+ await sandbox.teardown(first, { keepWarm: true });+ const second = await sandbox.provision(layers);+ assert.equal(second.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 1);+});++test("the session is named after the scope so an operator can find it", async () => {+ await sandbox.provision(layers);+ const current = fake.current(scopeName());+ assert.ok(current, "expected a session registered under the scope name");+ assert.ok(scopeName().length <= 64, "Managed Agents caps session names at 64 characters");+ assert.doesNotMatch(scopeName(), /^[0-9a-f]{8}-[0-9a-f]{4}-/, "Managed Agents rejects UUID-shaped session names");+});++test("teardown captures a provider checkpoint before pausing", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.run(h, "echo hi > kept.txt");+ await sandbox.teardown(h);+ const calls = fake.checkpointCalls();+ assert.equal(calls.length, 1);+ assert.equal(calls[0]?.label, "qm teardown");+ assert.equal(calls[0]?.sessionId, fake.current(scopeName())?.sessionId);+ const stored = await store.get(scope);+ assert.equal(stored?.checkpointId, calls[0]?.checkpointId);+ const status = await sandbox.computerStatus!(scope);+ assert.equal(status.recovery?.checkpointId, stored?.checkpointId);+ assert.equal(status.lifecycleState, "paused");+ assert.equal(fake.pauseCalls().length, 1);+});++test("restart rolls a paused session back to its checkpoint without resuming the old microVM", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ const restored: string[] = [];+ sandbox = make({+ store,+ onError: (e: { code: string }) => {+ if (e.code === "checkpoint_restored") restored.push(e.code);+ },+ });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const before = await store.get(scope);+ assert.ok(before?.checkpointId);+ assert.equal(before.preservationState, "paused");+ const resumes = fake.resumeCalls().length;+ const disconnects = fake.guestDisconnects().length;+ await sandbox.restartComputer!(scope);+ const after = await store.get(scope);+ assert.equal(after?.sessionId, before.sessionId);+ assert.notEqual(after?.sandboxId, before.sandboxId);+ assert.equal(after?.preservationState, "running");+ assert.equal(after?.checkpointId, before.checkpointId);+ assert.deepEqual(fake.rollbackCalls(), [{ sessionId: before.sessionId, checkpointId: before.checkpointId }]);+ assert.equal(fake.resumeCalls().length, resumes);+ assert.equal(fake.guestDisconnects().length, disconnects);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.deepEqual(restored, ["checkpoint_restored"]);+ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, false);+ assert.equal(fake.current(scopeName())?.sandboxId, after?.sandboxId);+});++test("restart drops a warm guest's port-forward before rolling back", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h, { keepWarm: true });+ const disconnects = fake.guestDisconnects().length;+ await sandbox.restartComputer!(scope);+ assert.equal(fake.guestDisconnects().length, disconnects + 1);+ assert.equal(fake.rollbackCalls().length, 1);+ assert.equal((await store.get(scope))?.preservationState, "running");+});++test("restart without a checkpoint does not call rollback", async () => {+ await sandbox.provision(layers);+ await assert.rejects(() => sandbox.restartComputer!(scope), /no checkpoint to restore/);+ assert.equal(fake.rollbackCalls().length, 0);+});++test("a failed rollback keeps the stored sandbox and the same session", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const before = await store.get(scope);+ fake.failNextRollback(scopeName());+ await assert.rejects(() => sandbox.restartComputer!(scope), /rollback/);+ assert.equal((await store.get(scope))?.sandboxId, before?.sandboxId);+ assert.equal(fake.rollbackCalls().length, 0);+ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 1);+});++test("a checkpoint failure is recorded and the session still pauses", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ fake.failNextCheckpoint(scopeName());+ await sandbox.teardown(h);+ assert.equal(fake.checkpointCalls().length, 0);+ assert.equal(fake.pauseCalls().length, 1);+ assert.equal(fake.current(scopeName())?.state, "paused");+ const status = await sandbox.computerStatus!(scope);+ assert.match(status.recovery?.error ?? "", /create checkpoint/);+});++test("checkpoint capture is throttled across a pause and resume", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const again = await sandbox.provision(layers);+ await sandbox.teardown(again);+ assert.equal(fake.checkpointCalls().length, 1);+});++test("a newer checkpoint replaces the previous one", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store, snapshotIntervalMs: -1 });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const first = (await store.get(scope))?.checkpointId;+ const again = await sandbox.provision(layers);+ await sandbox.teardown(again);+ const second = (await store.get(scope))?.checkpointId;+ assert.ok(first);+ assert.ok(second);+ assert.notEqual(first, second);+ assert.deepEqual(fake.deletedCheckpoints(), [first]);+ assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []);+});++test("a failed checkpoint delete is remembered and retried without another capture", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ const fast = make({ store, snapshotIntervalMs: -1 });+ const h = await fast.provision(layers);+ await fast.teardown(h);+ const first = (await store.get(scope))?.checkpointId;+ assert.ok(first);+ fake.failNextCheckpointDelete(scopeName());+ const again = await fast.provision(layers);+ await fast.teardown(again);+ assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes, [first]);+ assert.deepEqual(fake.deletedCheckpoints(), []);++ const slow = make({ store });+ const resumed = await slow.provision(layers);+ const captures = fake.checkpointCalls().length;+ await slow.teardown(resumed, { homeUnchanged: true });+ assert.equal(fake.checkpointCalls().length, captures);+ assert.deepEqual(fake.deletedCheckpoints(), [first]);+ assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []);+});++test("a failed home snapshot retries without recapturing the checkpoint", async () => {+ const uploads: number[] = [];+ const snapshots: HomeSnapshotStore = {+ open: async () => null,+ put: async () => undefined,+ createUpload: async () => {+ uploads.push(1);+ throw new Error("snapshot upload failed");+ },+ };+ sandbox = make({ snapshots });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const again = await sandbox.provision(layers);+ await sandbox.teardown(again);+ assert.equal(fake.checkpointCalls().length, 1);+ assert.equal(uploads.length, 2);+});++test("teardown pauses the session and the next provision resumes it", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.run(h, "echo hi > kept.txt");+ await sandbox.teardown(h);+ assert.deepEqual(fake.pauseCalls().length, 1);+ assert.equal(fake.current(scopeName())?.state, "paused");++ const again = await sandbox.provision(layers);+ assert.equal(fake.resumeCalls().length, 1);+ const r = await sandbox.run(again, "cat kept.txt");+ assert.match(r.stdout, /hi/);+ assert.equal(fake.createdCount(scopeName()), 1);+});++test("keepWarm teardown leaves the session running for background work", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h, { keepWarm: true });+ assert.equal(fake.pauseCalls().length, 0);+ assert.equal(fake.current(scopeName())?.state, "ready");+});++test("a lost session is re-adopted by name when the durable record is gone", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store });+ await sandbox.provision(layers);+ const before = fake.current(scopeName());+ await store.delete(scope);++ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.equal(fake.current(scopeName())?.sessionId, before?.sessionId);+});++test("a destroyed session is replaced on the next provision", async () => {+ await sandbox.provision(layers);+ fake.destroy(scopeName());+ const again = await sandbox.provision(layers);+ assert.ok(again.id);+ assert.equal(fake.createdCount(scopeName()), 2);+});++test("destroying a scope disconnects the live guest rather than only deleting the session", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.run(h, "true");+ const live = fake.current(scopeName());+ assert.ok(live, "expected a live session before destroying the scope");+ await sandbox.destroyScope!(scope);+ assert.deepEqual(fake.guestDisconnects(), [live.sessionId]);+ assert.equal(fake.current(scopeName()), null);+});++test("a command lost mid-flight is surfaced rather than silently retried", async () => {+ const h = await sandbox.provision(layers);+ fake.loseNextCommand(scopeName());+ await assert.rejects(() => sandbox.run(h, "echo nope"), /was lost while a command was running/);+});++test("a Managed Agents approval rejection names the permissions fix", async () => {+ const h = await sandbox.provision(layers);+ fake.rejectNextCommand(scopeName());+ await assert.rejects(() => sandbox.run(h, "echo nope"), ManagedAgentsHitlRejectedError);+ await assert.rejects(() => {+ fake.rejectNextCommand(scopeName());+ return sandbox.run(h, "echo nope");+ }, /permissions to allow bash/);+});++test("files round trip through the data plane", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "notes/a.txt", "hello managed agents");+ assert.equal(await sandbox.readFile(h, "notes/a.txt"), "hello managed agents");+ assert.equal(await sandbox.readFile(h, "notes/missing.txt"), null);+});++test("destroy removes the session and its durable record", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h, { destroy: true });+ assert.equal(await store.get(scope), null);+ assert.equal(fake.current(scopeName()), null);+});++test("computerStatus reports a paused machine without waking it", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const status = await sandbox.computerStatus!(scope);+ assert.equal(status.provisioned, true);+ assert.equal(status.guestResponsive, false);+ assert.equal(status.lifecycleState, "paused");+ assert.equal(fake.resumeCalls().length, 0);+});++test("computerStatus probes a running machine", async () => {+ await sandbox.provision(layers);+ const status = await sandbox.computerStatus!(scope);+ assert.equal(status.provisioned, true);+ assert.equal(status.guestResponsive, true);+});++test("computerStatus on an unprovisioned scope reports nothing provisioned", async () => {+ const status = await sandbox.computerStatus!(scope);+ assert.equal(status.provisioned, false);+ assert.equal(status.guestResponsive, false);+});++test("scratch sandboxes are reference counted and never persisted", async () => {+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();+ sandbox = make({ store });+ const a = await sandbox.provision(layers, { scratch: { key: "k1" } });+ const b = await sandbox.provision(layers, { scratch: { key: "k1" } });+ assert.equal(a.id, b.id);+ assert.equal(a.scratch, true);+ await sandbox.teardown(a);+ assert.ok(fake.current(a.id), "still held by the second handle");+ await sandbox.teardown(b);+ assert.equal(fake.current(a.id), null);+ assert.equal(await store.get(scope), null);+});++test("the profile advertises provider-managed persistence and parks on teardown", () => {+ assert.equal(sandbox.profile.backend, "do-managed-agents");+ assert.equal(sandbox.profile.writablePersistence, "provider_managed");+ assert.equal(sandbox.profile.parksOnTeardown, true);+ assert.equal(supportsProcessSessions(sandbox), true);+});++test("the QM template's tools are advertised as installed, not missing", () => {+ assert.ok(sandbox.profile.spec?.notInstalled?.includes("jq"));+ const spec = make({ extraTools: QM_TEMPLATE_TOOLS }).profile.spec;+ for (const tool of QM_TEMPLATE_TOOLS) {+ assert.ok(spec?.tools?.includes(tool), tool);+ assert.ok(!spec?.notInstalled?.includes(tool), tool);+ }+});++test("egress enforcement is only claimed when a proxy is configured", () => {+ assert.equal(sandbox.profile.egressEnforcement, "none");+ assert.equal(make({ egressProxyUrl: "https://egress.example.com" }).profile.egressEnforcement, "domain");+});++test("the egress allowlist carries just the proxy host", () => {+ assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]);+ assert.deepEqual(managedAgentsEgressAllow("http://10.0.0.7:8080"), ["10.0.0.7"]);+});++test("session status wire values map onto backend states", () => {+ assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready");+ assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused");+ assert.equal(managedAgentsSessionState("SESSION_STATUS_DESTROYED"), "destroyed");+ assert.equal(managedAgentsSessionState("nonsense"), "unspecified");+ assert.equal(managedAgentsSessionState(undefined), "unspecified");+});++test("process sessions stream output through the data plane exec path", async () => {+ assert.ok(supportsProcessSessions(sandbox));+ if (!supportsProcessSessions(sandbox)) return;+ const h = await sandbox.provision(layers);+ const { processId } = await sandbox.startProcess(h, "echo one; echo two");+ const { output, status } = await pollProcess(sandbox, h, processId, { deadlineMs: 5_000, waitMs: 100 });+ assert.equal(status.state, "exited");+ assert.match(output, /one/);+ assert.match(output, /two/);+});diff --git a/test/support/fake-managed-agents-service.ts b/test/support/fake-managed-agents-service.tsnew file mode 100644index 000000000..71cb864cd--- /dev/null+++ b/test/support/fake-managed-agents-service.ts@@ -0,0 +1,348 @@+import { createServer as createHttpServer, type Server as HttpServer, type ServerResponse } from "node:http";+import { connect, type Socket } from "node:net";+import { once } from "node:events";+import { fileURLToPath } from "node:url";+import { randomUUID } from "node:crypto";+import { spawn } from "node:child_process";+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import { WebSocketServer, type WebSocket } from "ws";+import * as loader from "@grpc/proto-loader";+import * as grpc from "@grpc/grpc-js";++const PROTO_PATH = fileURLToPath(new URL("../../src/sandbox/managed-agents-sandbox-agent.proto", import.meta.url));++interface SessionRow {+ session_id: string;+ sandbox_id: string;+ name: string;+ status: string;+ template: string;+ created_at: string;+}++export interface FakeManagedAgentsService {+ apiBaseUrl: string;+ token: string;+ sessions(): SessionRow[];+ manifests(): string[];+ execScripts(): string[];+ home(): string;+ checkpoints(sessionId: string): Array<{ checkpoint_id: string; label: string; status: string }>;+ setStatus(name: string, status: string): void;+ rejectPortForward(body: string | null): void;+ failNextLifecycleCalls(count: number, status: number): void;+ tunnelCount(): number;+ pingCount(): number;+ close(): Promise<void>;+}++function loadService(): grpc.ServiceDefinition {+ const definition = loader.loadSync(PROTO_PATH, {+ keepCase: true,+ longs: String,+ enums: String,+ defaults: true,+ oneofs: true,+ });+ const pkg = grpc.loadPackageDefinition(definition) as unknown as {+ do: {+ teams: {+ hosted_agents: { runtime: { sandbox_agent: { v1: { SandboxAgentService: grpc.ServiceClientConstructor } } } };+ };+ };+ };+ return pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService.service;+}++export async function startFakeManagedAgentsService(): Promise<FakeManagedAgentsService> {+ const token = "dop_v1_test-token";+ const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-guest-"));+ const rows = new Map<string, SessionRow>();+ const checkpoints = new Map<string, Array<{ checkpoint_id: string; label: string; status: string }>>();+ const manifests: string[] = [];+ const execScripts: string[] = [];+ let tunnels = 0;+ let pings = 0;+ let lifecycleFailures = 0;+ let lifecycleFailureStatus = 403;++ const guest = new grpc.Server();+ guest.addService(loadService(), {+ Exec: (call: grpc.ServerDuplexStream<Record<string, unknown>, unknown>) => {+ call.on("data", (frame: { start?: { argv?: string[]; workdir?: string; env?: Record<string, string> } }) => {+ if (!frame.start) return;+ const argv = frame.start.argv ?? [];+ const script = argv[argv.length - 1] ?? "";+ execScripts.push(script);+ try {+ const cwd = (frame.start.workdir || root).replace(/^\/home\/user/, root);+ mkdirSync(cwd, { recursive: true });+ const child = spawn("sh", ["-c", script.replace(/\/home\/user/g, root)], {+ cwd,+ env: { ...process.env, ...frame.start.env, HOME: root },+ });+ child.stdout.on("data", (chunk: Buffer) => call.write({ stdout: chunk }));+ child.stderr.on("data", (chunk: Buffer) => call.write({ stderr: chunk }));+ child.on("close", (code: number | null) => {+ call.write({ exit: { exit_code: code ?? -1, duration_ms: 1, hitl_rejected: false } });+ call.end();+ });+ child.on("error", (e: Error) => {+ call.write({ stderr: Buffer.from(e.message) });+ call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } });+ call.end();+ });+ } catch (e) {+ call.write({ stderr: Buffer.from(e instanceof Error ? e.message : String(e)) });+ call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } });+ call.end();+ }+ });+ },+ Upload: (call: grpc.ServerReadableStream<Record<string, unknown>, unknown>, cb: grpc.sendUnaryData<unknown>) => {+ let path = "";+ const chunks: Buffer[] = [];+ call.on("data", (frame: { header?: { path?: string }; chunk?: Buffer }) => {+ if (frame.header?.path) path = frame.header.path;+ if (frame.chunk?.length) chunks.push(frame.chunk);+ });+ call.on("end", () => {+ const hostPath = path.replace(/^\/home\/user/, root);+ mkdirSync(dirname(hostPath), { recursive: true });+ const body = Buffer.concat(chunks);+ writeFileSync(hostPath, body);+ cb(null, { path, bytes_written: String(body.length) });+ });+ },+ Download: (call: grpc.ServerWritableStream<{ path?: string }, unknown>) => {+ const hostPath = (call.request.path ?? "").replace(/^\/home\/user/, root);+ let body: Buffer;+ try {+ body = readFileSync(hostPath);+ } catch {+ call.emit("error", {+ code: grpc.status.INTERNAL,+ details: `envd download: download status 404: {"code":404,"message":"path '${call.request.path ?? ""}' does not exist"}`,+ });+ return;+ }+ call.write({ header: { size_bytes: String(body.length), is_archive: false } });+ call.write({ chunk: body });+ call.write({ end: { sha256: "" } });+ call.end();+ },+ });++ const guestPort = await new Promise<number>((resolve, reject) => {+ guest.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (err, port) =>+ err ? reject(err) : resolve(port),+ );+ });++ const byName = (name: string): SessionRow | undefined =>+ [...rows.values()].find((r) => r.name === name && r.status !== "SESSION_STATUS_DESTROYED");++ const json = (res: ServerResponse, code: number, body: unknown): void => {+ res.writeHead(code, { "content-type": "application/json" });+ res.end(JSON.stringify(body));+ };++ const http: HttpServer = createHttpServer((req, res) => {+ if (req.headers.authorization !== `Bearer ${token}`) {+ json(res, 401, { error: { code: "unauthorized", message: "bad token" } });+ return;+ }+ const url = new URL(req.url ?? "/", "http://localhost");+ const parts = url.pathname.split("/").filter(Boolean);+ const id = parts[3] ?? "";+ const action = parts[4] ?? "";++ if (req.method === "POST" && url.pathname === "/v2/agents/sessions") {+ let body = "";+ req.on("data", (c: Buffer) => (body += c.toString("utf8")));+ req.on("end", () => {+ manifests.push(body);+ const name = /^name:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "unnamed";+ const row: SessionRow = {+ session_id: randomUUID(),+ sandbox_id: `sbx-${rows.size + 1}`,+ name,+ status: "SESSION_STATUS_READY",+ template: /^template:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "",+ created_at: new Date().toISOString(),+ };+ rows.set(row.session_id, row);+ json(res, 201, { session: row });+ });+ return;+ }+ if (req.method === "GET" && url.pathname === "/v2/agents/sessions") {+ const name = url.searchParams.get("name");+ const all = [...rows.values()].filter((r) => !name || r.name === name);+ json(res, 200, { sessions: all, next_page_token: "" });+ return;+ }+ if (lifecycleFailures > 0) {+ lifecycleFailures -= 1;+ json(res, lifecycleFailureStatus, { error: { code: "forbidden", message: "transient" } });+ return;+ }+ if (req.method === "GET" && parts.length === 4) {+ const row = rows.get(id);+ if (!row) {+ json(res, 404, { error: { code: "not_found", message: "no such session" } });+ return;+ }+ json(res, 200, { session: row });+ return;+ }+ if (req.method === "DELETE" && parts.length === 4) {+ const row = rows.get(id);+ if (row) row.status = "SESSION_STATUS_DESTROYED";+ res.writeHead(204).end();+ return;+ }+ if (req.method === "POST" && action === "checkpoints" && parts.length === 5) {+ let body = "";+ req.on("data", (c: Buffer) => (body += c.toString("utf8")));+ req.on("end", () => {+ const row = rows.get(id);+ if (!row || row.status === "SESSION_STATUS_DESTROYED") {+ json(res, 404, { error: { code: "not_found", message: "no such session" } });+ return;+ }+ let label: string;+ try {+ label = (JSON.parse(body) as { label?: string }).label ?? "";+ } catch {+ label = "";+ }+ const checkpoint = {+ checkpoint_id: `cp_${randomUUID().slice(0, 8)}`,+ session_id: row.session_id,+ status: "READY",+ kind: "explicit",+ label,+ created_at: new Date().toISOString(),+ size_bytes: 1,+ };+ const list = checkpoints.get(row.session_id) ?? [];+ list.unshift(checkpoint);+ checkpoints.set(row.session_id, list);+ json(res, 200, { checkpoint });+ });+ return;+ }+ if (req.method === "POST" && action === "checkpoints" && parts[6] === "rollback" && parts.length === 7) {+ const row = rows.get(id);+ const checkpointId = decodeURIComponent(parts[5] ?? "");+ const list = checkpoints.get(id) ?? [];+ if (!row || row.status === "SESSION_STATUS_DESTROYED" || !list.some((c) => c.checkpoint_id === checkpointId)) {+ json(res, 404, { error: { code: "not_found", message: "no such checkpoint" } });+ return;+ }+ row.sandbox_id = `sbx-restored-${randomUUID().slice(0, 8)}`;+ row.status = "SESSION_STATUS_READY";+ json(res, 200, { session: row });+ return;+ }+ if (req.method === "DELETE" && action === "checkpoints" && parts.length === 6) {+ const checkpointId = decodeURIComponent(parts[5] ?? "");+ const list = checkpoints.get(id) ?? [];+ checkpoints.set(+ id,+ list.filter((c) => c.checkpoint_id !== checkpointId),+ );+ json(res, 200, {});+ return;+ }+ if (req.method === "POST" && (action === "pause" || action === "resume")) {+ const row = rows.get(id);+ if (!row) {+ json(res, 404, { error: { code: "not_found", message: "no such session" } });+ return;+ }+ row.status = action === "pause" ? "SESSION_STATUS_PAUSED" : "SESSION_STATUS_READY";+ res.writeHead(204).end();+ return;+ }+ json(res, 404, { error: { code: "not_found", message: url.pathname } });+ });++ const wss = new WebSocketServer({ noServer: true });+ let portForwardRejection: string | null = null;+ http.on("upgrade", (req, socket, head) => {+ const url = new URL(req.url ?? "/", "http://localhost");+ const match = /^\/v2\/agents\/sessions\/([^/]+)\/port-forward\/(\d+)$/.exec(url.pathname);+ if (req.headers.authorization !== `Bearer ${token}` || !match) {+ socket.destroy();+ return;+ }+ if (portForwardRejection !== null) {+ socket.end(+ `HTTP/1.1 403 Forbidden\r\nContent-Length: ${Buffer.byteLength(portForwardRejection)}\r\nConnection: close\r\n\r\n${portForwardRejection}`,+ );+ return;+ }+ const row = rows.get(match[1] ?? "");+ if (!row || row.status === "SESSION_STATUS_DESTROYED") {+ socket.destroy();+ return;+ }+ wss.handleUpgrade(req, socket, head, (ws: WebSocket) => {+ tunnels += 1;+ ws.on("ping", () => {+ pings += 1;+ });+ const upstream: Socket = connect(guestPort, "127.0.0.1");+ ws.on("message", (data: Buffer) => upstream.write(data));+ upstream.on("data", (chunk: Buffer) => ws.send(chunk));+ const shutdown = (): void => {+ upstream.destroy();+ if (ws.readyState === ws.OPEN) ws.close();+ };+ ws.on("close", shutdown);+ ws.on("error", shutdown);+ upstream.on("close", shutdown);+ upstream.on("error", shutdown);+ });+ });++ http.listen(0, "127.0.0.1");+ await once(http, "listening");+ const address = http.address();+ if (address === null || typeof address === "string") throw new Error("fake managed agents service failed to bind");++ return {+ apiBaseUrl: `http://127.0.0.1:${address.port}`,+ token,+ sessions: () => [...rows.values()],+ manifests: () => [...manifests],+ execScripts: () => [...execScripts],+ home: () => root,+ checkpoints: (sessionId) => [...(checkpoints.get(sessionId) ?? [])],+ setStatus: (name, status) => {+ const row = byName(name);+ if (row) row.status = status;+ },+ rejectPortForward: (body) => {+ portForwardRejection = body;+ },+ failNextLifecycleCalls: (count, status) => {+ lifecycleFailures = count;+ lifecycleFailureStatus = status;+ },+ tunnelCount: () => tunnels,+ pingCount: () => pings,+ close: async () => {+ for (const ws of wss.clients) ws.terminate();+ wss.close();+ http.closeAllConnections();+ await new Promise<void>((resolve) => http.close(() => resolve()));+ guest.forceShutdown();+ rmSync(root, { recursive: true, force: true });+ },+ };+}diff --git a/test/support/fake-managed-agents.ts b/test/support/fake-managed-agents.tsnew file mode 100644index 000000000..39754201d--- /dev/null+++ b/test/support/fake-managed-agents.ts@@ -0,0 +1,289 @@+import { spawnSync } from "node:child_process";+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import {+ ManagedAgentsCommandLostError,+ ManagedAgentsHitlRejectedError,+ ManagedAgentsSandboxGoneError,+ type ManagedAgentsClient,+ type ManagedAgentsCommandResult,+ type ManagedAgentsCheckpoint,+ type ManagedAgentsSession,+ type ManagedAgentsSessionInfo,+ type ManagedAgentsSessionState,+ type ManagedAgentsSessionSummary,+} from "../../src/sandbox/managed-agents-client.ts";+import { mkdtempSync } from "node:fs";++const GUEST_HOME_DIR = "/workspace/home";++interface FakeRecord {+ sessionId: string;+ sandboxId: string;+ name: string;+ state: ManagedAgentsSessionState;+ home: string;+ createdAt: number;+ loseNextCommand: boolean;+ rejectNextCommand: boolean;+ failNextCheckpoint: boolean;+ failNextCheckpointDelete: boolean;+ failNextRollback: boolean;+}++export interface FakeManagedAgents {+ client: ManagedAgentsClient;+ current(name: string): { sessionId: string; sandboxId: string; state: ManagedAgentsSessionState } | null;+ createdCount(name: string): number;+ homeDir(name: string): string;+ pause(name: string): void;+ destroy(name: string): void;+ loseNextCommand(name: string): void;+ rejectNextCommand(name: string): void;+ execScripts(): string[];+ pauseCalls(): string[];+ resumeCalls(): string[];+ guestDisconnects(): string[];+ checkpointCalls(): Array<{ sessionId: string; checkpointId: string; label?: string }>;+ deletedCheckpoints(): string[];+ rollbackCalls(): Array<{ sessionId: string; checkpointId: string }>;+ failNextCheckpoint(name: string): void;+ failNextCheckpointDelete(name: string): void;+ failNextRollback(name: string): void;+ cleanup(): void;+}++export function installFakeManagedAgents(): FakeManagedAgents {+ const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-"));+ const records = new Map<string, FakeRecord>();+ const execScripts: string[] = [];+ const pauseCalls: string[] = [];+ const resumeCalls: string[] = [];+ const guestDisconnects: string[] = [];+ const checkpointCalls: Array<{ sessionId: string; checkpointId: string; label?: string }> = [];+ const deletedCheckpoints: string[] = [];+ const rollbackCalls: Array<{ sessionId: string; checkpointId: string }> = [];+ let nextId = 1;+ let nextSandbox = 1;+ let nextCheckpoint = 1;+ let clock = 0;++ const gone = (state: ManagedAgentsSessionState): boolean =>+ state === "destroyed" || state === "destroying" || state === "failed";++ const byName = (name: string): FakeRecord | undefined => {+ const all = [...records.values()].filter((r) => r.name === name).sort((a, b) => b.createdAt - a.createdAt);+ return all.find((r) => !gone(r.state)) ?? all[0];+ };++ const need = (name: string): FakeRecord => {+ const r = byName(name);+ if (!r) throw new Error(`fake-managed-agents: no session named ${name}`);+ return r;+ };++ const remap = (r: FakeRecord, script: string): string => {+ const homeRe = r.home.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");+ const remapPath = new RegExp(`${homeRe}/tmp/|${homeRe}(?![A-Za-z0-9._-])|/tmp/`, "g");+ return (+ `export HOME=${JSON.stringify(r.home)}; ` ++ script+ .replace(/\btimeout \d+ /g, "")+ .replaceAll(GUEST_HOME_DIR, r.home)+ .replace(remapPath, (mm) => (mm.startsWith(r.home) ? mm : `${r.home}/tmp/`))+ );+ };++ const alive = (r: FakeRecord): void => {+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);+ if (r.state === "paused") r.state = "ready";+ };++ const info = (r: FakeRecord): ManagedAgentsSessionInfo => ({+ sessionId: r.sessionId,+ sandboxId: r.sandboxId,+ name: r.name,+ state: r.state,+ createdAtMs: r.createdAt,+ });++ const session = (r: FakeRecord): ManagedAgentsSession => ({+ sessionId: r.sessionId,+ sandboxId: r.sandboxId,+ async runCommand(command): Promise<ManagedAgentsCommandResult> {+ alive(r);+ if (r.loseNextCommand) {+ r.loseNextCommand = false;+ r.state = "destroyed";+ throw new ManagedAgentsCommandLostError(r.sessionId, "sandbox is not running anymore");+ }+ if (r.rejectNextCommand) {+ r.rejectNextCommand = false;+ throw new ManagedAgentsHitlRejectedError(r.sessionId);+ }+ execScripts.push(command);+ mkdirSync(join(r.home, "tmp"), { recursive: true });+ const spawned = spawnSync("sh", ["-c", remap(r, command)], {+ encoding: "buffer",+ maxBuffer: 128 * 1024 * 1024,+ env: { ...process.env, COPYFILE_DISABLE: "1" },+ });+ return {+ stdout: (spawned.stdout ?? Buffer.alloc(0)).toString("utf8"),+ stderr: (spawned.stderr ?? Buffer.alloc(0)).toString("utf8"),+ exitCode: spawned.status ?? (spawned.signal ? 137 : -1),+ };+ },+ async readFileBytes(absPath): Promise<Uint8Array | null> {+ alive(r);+ const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath;+ if (!existsSync(hostPath)) return null;+ return new Uint8Array(readFileSync(hostPath));+ },+ async writeFileBytes(absPath, data): Promise<void> {+ alive(r);+ const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath;+ mkdirSync(dirname(hostPath), { recursive: true });+ writeFileSync(hostPath, Buffer.from(data));+ },+ async createCheckpoint(label?: string): Promise<ManagedAgentsCheckpoint> {+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);+ if (r.failNextCheckpoint) {+ r.failNextCheckpoint = false;+ throw new Error("do-managed-agents create checkpoint: http 500 capture failed");+ }+ const checkpointId = `cp-${nextCheckpoint++}`;+ checkpointCalls.push({ sessionId: r.sessionId, checkpointId, ...(label ? { label } : {}) });+ return { checkpointId, status: "READY", createdAtMs: Date.now() };+ },+ async deleteCheckpoint(checkpointId: string): Promise<void> {+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);+ if (r.failNextCheckpointDelete) {+ r.failNextCheckpointDelete = false;+ throw new Error("do-managed-agents delete checkpoint: http 500 delete failed");+ }+ deletedCheckpoints.push(checkpointId);+ },+ async close(): Promise<void> {+ guestDisconnects.push(r.sessionId);+ },+ async pause(): Promise<void> {+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);+ guestDisconnects.push(r.sessionId);+ pauseCalls.push(r.sessionId);+ r.state = "paused";+ },+ async resume(): Promise<void> {+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);+ resumeCalls.push(r.sessionId);+ r.state = "ready";+ },+ async kill(): Promise<void> {+ guestDisconnects.push(r.sessionId);+ r.state = "destroyed";+ rmSync(r.home, { recursive: true, force: true });+ },+ });++ const client: ManagedAgentsClient = {+ nativePause: true,+ async info(sessionId): Promise<ManagedAgentsSessionInfo> {+ const r = records.get(sessionId);+ if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found");+ return info(r);+ },+ async create(opts): Promise<ManagedAgentsSession> {+ const id = `sess-${nextId++}`;+ const r: FakeRecord = {+ sessionId: id,+ sandboxId: `sbx-${id}`,+ name: opts.name,+ state: "ready",+ home: join(root, id),+ createdAt: ++clock,+ loseNextCommand: false,+ rejectNextCommand: false,+ failNextCheckpoint: false,+ failNextCheckpointDelete: false,+ failNextRollback: false,+ };+ mkdirSync(r.home, { recursive: true });+ records.set(id, r);+ return session(r);+ },+ async connect(sessionId): Promise<ManagedAgentsSession> {+ const r = records.get(sessionId);+ if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found");+ if (r.state === "paused") {+ resumeCalls.push(r.sessionId);+ r.state = "ready";+ }+ return session(r);+ },+ async list(name): Promise<ManagedAgentsSessionSummary[]> {+ return [...records.values()]+ .filter((r) => !gone(r.state) && r.name === name)+ .map((r) => ({ sessionId: r.sessionId, sandboxId: r.sandboxId, name: r.name, state: r.state }));+ },+ async rollback(sessionId, checkpointId): Promise<ManagedAgentsSessionInfo> {+ const r = records.get(sessionId);+ if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found");+ if (r.failNextRollback) {+ r.failNextRollback = false;+ throw new Error("do-managed-agents rollback: http 409 checkpoint is not ready");+ }+ rollbackCalls.push({ sessionId, checkpointId });+ r.sandboxId = `sbx-restored-${nextSandbox++}`;+ r.state = "ready";+ return info(r);+ },+ async kill(sessionId): Promise<void> {+ const r = records.get(sessionId);+ if (!r) return;+ r.state = "destroyed";+ rmSync(r.home, { recursive: true, force: true });+ },+ };++ return {+ client,+ current: (name) => {+ const r = byName(name);+ return r && !gone(r.state) ? { sessionId: r.sessionId, sandboxId: r.sandboxId, state: r.state } : null;+ },+ createdCount: (name) => [...records.values()].filter((r) => r.name === name).length,+ homeDir: (name) => need(name).home,+ pause: (name) => {+ need(name).state = "paused";+ },+ destroy: (name) => {+ const r = need(name);+ r.state = "destroyed";+ rmSync(r.home, { recursive: true, force: true });+ },+ loseNextCommand: (name) => {+ need(name).loseNextCommand = true;+ },+ rejectNextCommand: (name) => {+ need(name).rejectNextCommand = true;+ },+ checkpointCalls: () => [...checkpointCalls],+ deletedCheckpoints: () => [...deletedCheckpoints],+ failNextCheckpoint: (name) => {+ need(name).failNextCheckpoint = true;+ },+ failNextCheckpointDelete: (name) => {+ need(name).failNextCheckpointDelete = true;+ },+ rollbackCalls: () => [...rollbackCalls],+ failNextRollback: (name) => {+ need(name).failNextRollback = true;+ },+ execScripts: () => [...execScripts],+ pauseCalls: () => [...pauseCalls],+ resumeCalls: () => [...resumeCalls],+ guestDisconnects: () => [...guestDisconnects],+ cleanup: () => rmSync(root, { recursive: true, force: true }),+ };+}