Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Add DigitalOcean Managed Agents as a sandbox backend (#2128)

Joshua France51385cfd3f59
diff --git a/cli/src/secrets.ts b/cli/src/secrets.ts
index c02dc5c7d..6560efe63 100644
--- a/cli/src/secrets.ts
+++ b/cli/src/secrets.ts
@@ -170,6 +170,14 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [
required: { when: { kind: "sandbox-backend", backend: "e2b" } },
description: "E2B API key used by the e2b sandbox backend (from e2b.dev dashboard).",
},
+ {
+ name: "DO_AGENTS_API_TOKEN",
+ service: "core",
+ required: { when: { kind: "sandbox-backend", backend: "do-managed-agents" } },
+ description:
+ "DigitalOcean IAM token for the DigitalOcean Managed Agents sandbox backend. Carries the team identity that owns the sandbox sessions, and authenticates both the session REST calls and the port-forward tunnel that carries exec and file transfer.",
+ generate: "doctl auth init # then create a personal access token at cloud.digitalocean.com/account/api/tokens",
+ },
{
name: "MODAL_TOKEN_ID",
service: "core",
diff --git a/deploy/do-managed-agents/Dockerfile b/deploy/do-managed-agents/Dockerfile
new file mode 100644
index 000000000..fd913e088
--- /dev/null
+++ b/deploy/do-managed-agents/Dockerfile
@@ -0,0 +1,13 @@
+FROM node:24-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ ca-certificates git curl wget jq tar xz-utils unzip zip ripgrep \
+ python3 python3-pip python3-venv \
+ openssh-client gnupg less vim-tiny procps \
+ && rm -rf /var/lib/apt/lists/* \
+ && node --version && npm --version
+
+ENV HOME=/workspace/home
+ENTRYPOINT ["sleep", "infinity"]
+CMD []
diff --git a/deploy/do-managed-agents/register-template.sh b/deploy/do-managed-agents/register-template.sh
new file mode 100755
index 000000000..3c6bf4c97
--- /dev/null
+++ b/deploy/do-managed-agents/register-template.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Build the QM sandbox image and register it as a Managed Agents template
+# on your own DigitalOcean team. Run once per team, and again after the
+# Dockerfile changes.
+#
+# Needs: docker (buildx), doctl authenticated with a token that has
+# registry and Managed Agents access, and an existing container registry.
+#
+# Usage: deploy/do-managed-agents/register-template.sh [tag]
+# DO_REGISTRY registry name (default: first registry on the account)
+# TEMPLATE_NAME template name (default: qm-sandbox)
+set -euo pipefail
+
+TAG="${1:-$(date +%Y%m%d%H%M%S)}"
+TEMPLATE_NAME="${TEMPLATE_NAME:-qm-sandbox}"
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+for bin in docker doctl jq; do
+ command -v "$bin" >/dev/null || { echo "missing: $bin" >&2; exit 1; }
+done
+
+DO_REGISTRY="${DO_REGISTRY:-$(doctl registry get -o json | jq -r 'if type=="array" then .[0].name else .name end // empty')}"
+[ -n "$DO_REGISTRY" ] || { echo "no container registry found; create one or set DO_REGISTRY" >&2; exit 1; }
+IMAGE="registry.digitalocean.com/$DO_REGISTRY/$TEMPLATE_NAME:$TAG"
+
+echo "building $IMAGE"
+docker build --platform linux/amd64 -t "$IMAGE" "$DIR"
+doctl registry login >/dev/null
+docker push "$IMAGE"
+
+if doctl harness-runtime template get "$TEMPLATE_NAME" >/dev/null 2>&1; then
+ echo "updating template $TEMPLATE_NAME"
+ doctl harness-runtime template update "$TEMPLATE_NAME" --source-oci-ref "$IMAGE" >/dev/null
+else
+ echo "creating template $TEMPLATE_NAME"
+ doctl harness-runtime template create --name "$TEMPLATE_NAME" \
+ --base-template sandbox --source-oci-ref "$IMAGE" >/dev/null
+fi
+
+echo "waiting for the template build"
+for _ in $(seq 1 60); do
+ status="$(doctl harness-runtime template list-builds "$TEMPLATE_NAME" -o json \
+ | jq -r 'sort_by(.created_at) | last | .status')"
+ case "$status" in
+ SUCCEEDED) echo "ready. Set DO_AGENTS_TEMPLATE=$TEMPLATE_NAME in your QM config."; exit 0 ;;
+ FAILED) echo "template build failed" >&2; exit 1 ;;
+ esac
+ sleep 20
+done
+echo "timed out waiting for the template build (last status: $status)" >&2
+exit 1
diff --git a/docs/do-managed-agents.md b/docs/do-managed-agents.md
new file mode 100644
index 000000000..31e2b5eb3
--- /dev/null
+++ b/docs/do-managed-agents.md
@@ -0,0 +1,39 @@
+# DigitalOcean Managed Agents sandboxes
+
+The `do-managed-agents` backend runs each QM scope on a DigitalOcean Managed Agents microVM. QM keeps its own agent loop; sessions start with `agent: none`, so DigitalOcean supplies the machine and nothing else.
+
+## Set it up
+
+1. Create a DigitalOcean API token with Managed Agents access.
+2. Point core at it:
+
+ ```sh
+ SANDBOX_BACKEND=do-managed-agents
+ DO_AGENTS_API_TOKEN=your-token
+ ```
+
+That's enough to run. Sessions land on DigitalOcean's bare base image.
+
+## Optional
+
+**A QM template.** For an image with the tools agents usually reach for (Node, git, jq, ripgrep, Python), register one on your team. You need Docker, an authenticated `doctl`, and a container registry:
+
+```sh
+deploy/do-managed-agents/register-template.sh
+DO_AGENTS_TEMPLATE=qm-sandbox
+```
+
+Templates belong to a team, so each team registers its own. Rerun the script after changing the Dockerfile.
+
+**A snapshot bucket.** When a scope parks, QM takes a DigitalOcean checkpoint, which lets it roll a wedged computer back in place. A checkpoint is tied to its session, though, so if the session is deleted the checkpoint goes with it. Set `DO_AGENTS_SNAPSHOT_S3_BUCKET` to also keep a copy of the home directory that survives that. Without it, the copy lives in core's memory and is lost when core restarts.
+
+**An egress proxy.** Set `DO_AGENTS_EGRESS_PROXY_URL` and the proxy host becomes the session's only allowed destination.
+
+Other knobs: `DO_AGENTS_SIZE_SLUG`, `DO_AGENTS_IDLE_TIMEOUT_SEC`, `DO_AGENTS_NAME_PREFIX` (use a distinct one per deployment sharing a team), `DO_AGENTS_SNAPSHOT_INTERVAL_SEC`, and `DO_AGENTS_API_BASE_URL`.
+
+## Good to know
+
+- Commands and file transfers go over a port-forward to the microVM rather than DigitalOcean's REST exec, which caps output and runtime. Long commands are bounded only by `SANDBOX_TIMEOUT_SEC`.
+- QM doesn't reap idle machines itself. DigitalOcean's idle timeout does, and paused sessions still count against team quota.
+- Sessions allow `bash` outright, since QM already runs its own approval gates.
+- `scripts/managed-agents-sandbox-smoke.ts` checks a token and template end to end before you point a deployment at them.
diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.md
index c8517560d..e7c36793b 100644
--- a/docs/sandbox-preservation.md
+++ b/docs/sandbox-preservation.md
@@ -34,3 +34,5 @@ Modal machines are created with a one-core, 2 GiB reservation (`MODAL_CPUS`, `MO
When `MODAL_EGRESS_PROXY_URL` is set, machines are created with Modal's outbound allowlist limited to the proxy host (a domain allowlist for an https proxy on port 443, a CIDR allowlist for an IP address; other proxy URLs are rejected at startup because Modal domain allowlists pass only TLS on 443), and the profile reports `egressEnforcement: "domain"`. Command execution, file transfer, and checkpoints travel over Modal's control plane and are unaffected. Without the setting, core warns that Modal sandboxes run fail-open.
Turn environment reaches Modal commands through the exec `env` parameter rather than inlined `export` statements, exec deadlines are rounded up to whole seconds with a single thirty-second grace beyond the in-guest `timeout`, and a command longer than 64 KiB is written to `/tmp` through the filesystem API and executed from there.
+
+Managed Agents sessions name a sandbox template explicitly, which yields a microVM carrying no managed agent loop, and the disk persists across the provider's pause and resume. Used-turn teardown calls `POST /v2/agents/sessions/{id}/checkpoints` when the home changed, throttled by `DO_AGENTS_SNAPSHOT_INTERVAL_SEC` from the last successful capture, then snapshots the home as a tar on that same interval measured from the last successful tar. The tar goes to `DO_AGENTS_SNAPSHOT_S3_BUCKET` when it is set; without it the tar is held in core's memory and does not survive a core restart. A failed tar is retried without taking another checkpoint. The checkpoint is the provider's machine-state capture of the live session (`CreateCheckpoint` blocks until the capture is `READY`); only the newest is kept, and the previous one is deleted after the new capture succeeds. A delete that fails stays recorded and is retried on a later teardown. A capture failure is recorded on `recovery.error` and the pause still proceeds. A pause failure is recorded as `pause_failed` and the running session is retained for retry. Status reports `recovery.strategy: "workspace_snapshot"` plus `recovery.checkpointId`. Restart of a wedged computer calls `POST /v2/agents/sessions/{id}/checkpoints/{checkpoint_id}/rollback` with that stored checkpoint: the session id stays, the response's sandbox id replaces the stored one, and any open port-forward is closed so the next command dials the restored microVM. A paused session is rolled back directly, without being resumed first. Restart with no checkpoint fails and leaves the session where it is. A checkpoint is bound to its session, so a replacement session for a destroyed or reclaimed scope still hydrates from the portable tar when one exists, and a failed hydration kills the replacement rather than adopting a blank home over a saved one. Session lifecycle travels over the Managed Agents REST surface with a DigitalOcean IAM token; commands and file transfer travel over a bearer-authenticated WebSocket port-forward to the guest's `sandbox-agent` gRPC server, which has no request deadline, so command length is bounded by `SANDBOX_TIMEOUT_SEC` rather than by the control plane. Sessions are created with a permissions block that allows `bash` outright, because Managed Agents would otherwise hold every command at its own approval gate behind core's.
diff --git a/package-lock.json b/package-lock.json
index ecad8b459..66cdbbd4a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -21,6 +21,8 @@
"@earendil-works/pi-ai": "https://github.com/yc-software/qm/releases/download/vendored-pi-ai-0.82.0-qm.1/earendil-works-pi-ai-0.82.0-qm.1.tgz",
"@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.6/earendil-works-pi-coding-agent-0.82.0-qm-security.6.tgz",
"@fly/sprites": "0.2.3",
+ "@grpc/grpc-js": "^1.14.5",
+ "@grpc/proto-loader": "^0.8.1",
"@openai/codex": "0.156.1",
"@opencode-ai/plugin": "1.18.31",
"@opencode-ai/sdk": "1.18.31",
@@ -45,6 +47,7 @@
"socks-proxy-agent": "10.1.0",
"tar-stream": "^3.2.0",
"typebox": "^1.1.38",
+ "ws": "^8.21.1",
"zod": "4.4.3"
},
"devDependencies": {
@@ -53,6 +56,7 @@
"@types/node": "^24.0.0",
"@types/pg": "^8.11.10",
"@types/tar-stream": "^3.1.4",
+ "@types/ws": "^8.18.1",
"@yc-software/qm": "file:./cli",
"devbar.sh": "1.1.0",
"esbuild": "0.28.1",
diff --git a/package.json b/package.json
index 8de4ab481..75dad7683 100644
--- a/package.json
+++ b/package.json
@@ -40,6 +40,8 @@
"smoke:service-cred": "node scripts/service-credential-livetest.ts",
"smoke:aws-sandbox": "node scripts/aws-sandbox-smoke.ts",
"smoke:local-sandbox": "node scripts/local-sandbox-smoke.ts",
+ "smoke:managed-agents-sandbox": "node --env-file-if-exists=.env scripts/managed-agents-sandbox-smoke.ts",
+ "smoke:managed-agents-turn": "node --env-file-if-exists=.env scripts/managed-agents-turn-smoke.ts",
"build:modal-image": "node scripts/modal-build-image.ts",
"build:e2b-template": "node scripts/e2b-build-template.ts",
"build:aws-image": "node scripts/aws-build-sandbox-image.ts",
@@ -78,6 +80,8 @@
"@earendil-works/pi-ai": "https://github.com/yc-software/qm/releases/download/vendored-pi-ai-0.82.0-qm.1/earendil-works-pi-ai-0.82.0-qm.1.tgz",
"@earendil-works/pi-coding-agent": "https://github.com/yc-software/qm/releases/download/vendored-pi-coding-agent-0.82.0-security.6/earendil-works-pi-coding-agent-0.82.0-qm-security.6.tgz",
"@fly/sprites": "0.2.3",
+ "@grpc/grpc-js": "^1.14.5",
+ "@grpc/proto-loader": "^0.8.1",
"@openai/codex": "0.156.1",
"@opencode-ai/plugin": "1.18.31",
"@opencode-ai/sdk": "1.18.31",
@@ -102,6 +106,7 @@
"socks-proxy-agent": "10.1.0",
"tar-stream": "^3.2.0",
"typebox": "^1.1.38",
+ "ws": "^8.21.1",
"zod": "4.4.3"
},
"overrides": {
@@ -128,6 +133,7 @@
"@types/node": "^24.0.0",
"@types/pg": "^8.11.10",
"@types/tar-stream": "^3.1.4",
+ "@types/ws": "^8.18.1",
"@yc-software/qm": "file:./cli",
"devbar.sh": "1.1.0",
"esbuild": "0.28.1",
diff --git a/scripts/dev/cli.ts b/scripts/dev/cli.ts
index a0436ab0f..f5614ade4 100644
--- a/scripts/dev/cli.ts
+++ b/scripts/dev/cli.ts
@@ -671,7 +671,7 @@ async function main(): Promise<number> {
});
default:
console.error(
- "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]",
+ "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|do-managed-agents|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]",
);
return EXIT.usage;
}
diff --git a/scripts/dev/lib/sandbox.ts b/scripts/dev/lib/sandbox.ts
index 9d4843274..3dc0b3512 100644
--- a/scripts/dev/lib/sandbox.ts
+++ b/scripts/dev/lib/sandbox.ts
@@ -190,6 +190,44 @@ export async function resolveSandbox(opts: {
return { backend: "superserve", env, detail: `Superserve (template ${template})`, publicApiUrl: apiUrl, warnings };
}
+ if (backend === "do-managed-agents") {
+ const managedAgentsToken = opts.baseEnv.DO_AGENTS_API_TOKEN?.trim();
+ if (!managedAgentsToken)
+ throw new Error(
+ "--sandbox do-managed-agents requires DO_AGENTS_API_TOKEN in the environment (a DigitalOcean API token for a team with Managed Agents access)",
+ );
+ let managedAgentsApiUrl = opts.baseEnv.PUBLIC_API_URL || null;
+ if (!managedAgentsApiUrl) {
+ managedAgentsApiUrl = await startQuickTunnel(opts.corePort, opts.lock, opts.log);
+ if (!managedAgentsApiUrl)
+ warnings.push(
+ "cloudflared tunnel didn't come up -- agent self-API (crons/sends) won't be reachable from the sandbox",
+ );
+ }
+ const env: Record<string, string> = {
+ SANDBOX_BACKEND: "do-managed-agents",
+ DO_AGENTS_API_TOKEN: managedAgentsToken,
+ DO_AGENTS_NAME_PREFIX: opts.baseEnv.DO_AGENTS_NAME_PREFIX || "qmdev",
+ };
+ if (opts.baseEnv.DO_AGENTS_API_BASE_URL) env.DO_AGENTS_API_BASE_URL = opts.baseEnv.DO_AGENTS_API_BASE_URL;
+ if (opts.baseEnv.DO_AGENTS_TEMPLATE) env.DO_AGENTS_TEMPLATE = opts.baseEnv.DO_AGENTS_TEMPLATE;
+ if (opts.baseEnv.DO_AGENTS_SIZE_SLUG) env.DO_AGENTS_SIZE_SLUG = opts.baseEnv.DO_AGENTS_SIZE_SLUG;
+ if (opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL)
+ env.DO_AGENTS_EGRESS_PROXY_URL = opts.baseEnv.DO_AGENTS_EGRESS_PROXY_URL;
+ else
+ warnings.push(
+ "DO_AGENTS_EGRESS_PROXY_URL unset -- do-managed-agents sandbox runs with NO egress enforcement; set it to QA the forced-proxy path",
+ );
+ if (managedAgentsApiUrl) env.PUBLIC_API_URL = managedAgentsApiUrl;
+ return {
+ backend: "do-managed-agents",
+ env,
+ detail: `Managed Agents (${opts.baseEnv.DO_AGENTS_API_BASE_URL || "api.digitalocean.com"})`,
+ publicApiUrl: managedAgentsApiUrl,
+ warnings,
+ };
+ }
+
if (backend === "porter") {
const porterToken = opts.baseEnv.PORTER_DEPLOY_API_TOKEN;
const projectId = opts.baseEnv.PORTER_DEPLOY_PROJECT_ID;
diff --git a/scripts/dev/lib/types.ts b/scripts/dev/lib/types.ts
index f1491f783..f96294a41 100644
--- a/scripts/dev/lib/types.ts
+++ b/scripts/dev/lib/types.ts
@@ -104,7 +104,8 @@ export interface StatusReport {
children: Record<string, ChildStatus & { slack?: SlackHealth }>;
}
-export type DevSandboxBackend = "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve";
+export type DevSandboxBackend =
+ "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve" | "do-managed-agents";
export type DevSandboxChoice = DevSandboxBackend | "auto";
export interface BootSpec {
diff --git a/scripts/managed-agents-sandbox-smoke.ts b/scripts/managed-agents-sandbox-smoke.ts
new file mode 100644
index 000000000..96ba31e6f
--- /dev/null
+++ b/scripts/managed-agents-sandbox-smoke.ts
@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+import assert from "node:assert/strict";
+import { randomBytes } from "node:crypto";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createManagedAgentsSandbox } from "../src/sandbox/managed-agents-sandbox.ts";
+import { createSdkManagedAgentsClient } from "../src/sandbox/managed-agents-client.ts";
+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";
+import { scopeId } from "../src/types.ts";
+
+const apiToken = process.env.DO_AGENTS_API_TOKEN;
+const apiBaseUrl = process.env.DO_AGENTS_API_BASE_URL;
+const template = process.env.DO_AGENTS_TEMPLATE ?? "";
+const agent = process.env.DO_AGENTS_AGENT;
+const keep = process.env.DO_AGENTS_SMOKE_KEEP === "1";
+
+const log = (...a: unknown[]) => console.log("[managed-agents-smoke]", ...a);
+
+async function main(): Promise<void> {
+ if (!apiToken) throw new Error("DO_AGENTS_API_TOKEN is required");
+
+ const client = createSdkManagedAgentsClient({
+ apiToken,
+ ...(apiBaseUrl ? { apiBaseUrl } : {}),
+ ...(agent ? { agent } : {}),
+ ...(template ? { template } : {}),
+ });
+
+ const prefix = `qms${randomBytes(3).toString("hex")}`;
+ const sandbox = createManagedAgentsSandbox(
+ createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-smoke-ws-"))),
+ {
+ client,
+ namePrefix: prefix,
+ },
+ );
+ const scope = scopeId("personal", "smoke");
+ const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }];
+
+ log(`provisioning a qm agent computer (prefix ${prefix})...`);
+ const handle = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } });
+ log("provisioned", handle.id, "coldStart", handle.coldStart);
+
+ try {
+ log("running a command through the qm Sandbox interface...");
+ const r = await sandbox.run(handle, "pwd; echo QM_SMOKE=$QM_SMOKE; whoami");
+ assert.equal(r.code, 0, `run exited ${r.code}: ${r.stderr}`);
+ log("stdout:", JSON.stringify(r.stdout));
+ assert.match(r.stdout, /QM_SMOKE=1/);
+
+ log("writing and reading a workspace file through qm...");
+ await sandbox.writeFile(handle, "notes/hello.txt", "written by qm");
+ assert.equal(await sandbox.readFile(handle, "notes/hello.txt"), "written by qm");
+ log("file round-trip OK");
+
+ log("checking the toolchain qm expects...");
+ const binaryName = (entry: string) => entry.trim().split(/\s+/)[0] ?? "";
+ const declaredPresent = (sandbox.profile.spec?.tools ?? []).map(binaryName).filter(Boolean);
+ const declaredAbsent = (sandbox.profile.spec?.notInstalled ?? []).map(binaryName).filter(Boolean);
+ const plumbingNeeds = ["sh", "timeout", "tar", "base64", "sha256sum", "setsid", "mkfifo", "cut"];
+ const probe = [...new Set([...plumbingNeeds, ...declaredPresent, ...declaredAbsent])];
+ const tools = await sandbox.run(
+ handle,
+ `for t in ${probe.join(" ")}; do printf '%s=%s ' $t $(command -v $t >/dev/null 2>&1 && echo yes || echo NO); done; echo`,
+ );
+ log(tools.stdout.trim());
+ const onPath = new Set(
+ tools.stdout
+ .trim()
+ .split(/\s+/)
+ .flatMap((pair) => {
+ const [name, state] = pair.split("=");
+ return name && state === "yes" ? [name] : [];
+ }),
+ );
+ for (const t of plumbingNeeds)
+ assert.ok(onPath.has(t), `qm's own file and process plumbing needs ${t}, which the guest does not have`);
+ for (const t of declaredPresent)
+ assert.ok(onPath.has(t), `the computer profile advertises ${t}, which the guest does not have`);
+ for (const t of declaredAbsent)
+ assert.ok(!onPath.has(t), `the computer profile says ${t} is absent, but the guest has it`);
+
+ log("checking the guest carries no managed coding agent...");
+ const bare = await sandbox.run(
+ handle,
+ "for t in codex claude claude-code opencode cursor hermes; do command -v $t >/dev/null && echo AGENT_ON_PATH=$t; done; echo ---; ps -eo comm= | sort -u | tr '\\n' ' '",
+ );
+ log("guest:", bare.stdout.trim());
+ assert.doesNotMatch(bare.stdout, /AGENT_ON_PATH=/, "a bare sandbox must carry no agent CLI");
+
+ log("checking the workspace survives a pause and resume...");
+ await sandbox.run(handle, "echo persisted > survives-pause.txt");
+ await sandbox.teardown(handle, {});
+ const resumed = await sandbox.provision(layers, { env: { QM_SMOKE: "1" } });
+ const after = await sandbox.run(resumed, "cat survives-pause.txt");
+ assert.match(after.stdout, /persisted/, "the workspace did not survive a pause and resume");
+ log("workspace intact after pause and resume");
+
+ if (process.env.DO_AGENTS_SMOKE_LONG === "1") {
+ log("checking a command survives past the 4-minute REST exec clamp...");
+ const long = await sandbox.run(resumed, "sleep 250; echo past-the-clamp", { timeoutMs: 400_000 });
+ assert.equal(long.code, 0, `long run exited ${long.code}: ${long.stderr}`);
+ assert.match(long.stdout, /past-the-clamp/);
+ log("250s command completed over the tunnel");
+ }
+
+ log("ALL CHECKS PASSED");
+ } finally {
+ if (keep) {
+ log("DO_AGENTS_SMOKE_KEEP=1 — leaving the sandbox alive");
+ } else {
+ log("tearing down and destroying the scope...");
+ await sandbox.teardown(handle, { destroy: true }).catch((e: unknown) => log("teardown failed:", e));
+ }
+ }
+}
+
+main().catch((e: unknown) => {
+ console.error("[managed-agents-smoke] FAILED", e);
+ process.exit(1);
+});
diff --git a/scripts/managed-agents-turn-smoke.ts b/scripts/managed-agents-turn-smoke.ts
new file mode 100644
index 000000000..396717de9
--- /dev/null
+++ b/scripts/managed-agents-turn-smoke.ts
@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+import { randomUUID } from "node:crypto";
+import { stagingApiHeaders } from "../src/deployment/postdeploy-smoke.ts";
+
+const core = process.env.CORE_URL ?? "http://localhost:8081";
+const principal = process.env.PRINCIPAL ?? "managed-agents-smoke";
+const sourceSecret = process.env.CORE_SIGNING_SECRET ?? "";
+const portalSecret = process.env.PORTAL_IDENTITY_SECRET ?? "";
+
+const log = (...a: unknown[]) => console.log("[managed-agents-turn]", ...a);
+
+async function main(): Promise<void> {
+ if (!sourceSecret || !portalSecret) throw new Error("CORE_SIGNING_SECRET and PORTAL_IDENTITY_SECRET are required");
+
+ const nonce = randomUUID();
+ const body = JSON.stringify({
+ surface: "web",
+ actor: { externalId: principal },
+ conversation: { kind: "dm", threadRef: `web:${principal}:managed-agents-turn-${nonce}` },
+ text: "Use your execute tool to run exactly this shell command in your agent computer: `echo DO_AGENTS_TURN_OK; uname -sr; pwd; whoami`. Then reply with the exact stdout you got back and nothing else.",
+ origin: { kind: "human" },
+ addressed: true,
+ skipMemory: true,
+ idempotencyKey: nonce,
+ });
+
+ const path = "/v1/turns";
+ const headers = await stagingApiHeaders(principal, sourceSecret, portalSecret, "POST", path, body);
+
+ log(`POST ${core}${path} (this provisions a Managed Agents microVM on the first execute)...`);
+ const started = Date.now();
+ const res = await fetch(`${core}${path}`, {
+ method: "POST",
+ headers: { ...headers, "content-type": "application/json" },
+ body,
+ signal: AbortSignal.timeout(600_000),
+ });
+ const text = await res.text();
+ log(`HTTP ${res.status} in ${Math.round((Date.now() - started) / 1000)}s`);
+ if (!res.ok) throw new Error(`turn failed: ${text.slice(0, 2000)}`);
+
+ const turn = JSON.parse(text) as { status?: string; sessionId?: string; reply?: string };
+ log("status:", turn.status, "session:", turn.sessionId);
+ log("reply:", JSON.stringify(turn.reply));
+ if (!turn.reply?.includes("DO_AGENTS_TURN_OK")) throw new Error("the model reply does not contain DO_AGENTS_TURN_OK");
+ log("REAL QM TURN EXECUTED IN A Managed Agents SANDBOX");
+}
+
+main().catch((e: unknown) => {
+ console.error("[managed-agents-turn] FAILED", e);
+ process.exit(1);
+});
diff --git a/src/config.ts b/src/config.ts
index df35cd61d..b55f0203b 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -3,6 +3,7 @@ import { isStrongSigningSecret } from "./auth/source-auth.ts";
import { parseSandboxCapabilityTtlMs } from "./auth/capability-token.ts";
import { parseScopeId } from "./types.ts";
import type { LegacySandboxScopeDefaults } from "./sandbox/sandbox-resource-upgrade.ts";
+import type { SandboxBackendName } from "./sandbox/sandbox-routing.ts";
import { existsSync, readdirSync } from "node:fs";
import {
parseProviderBaseUrl,
@@ -73,9 +74,8 @@ export interface Config {
securityPosture: SecurityPosture;
sharingPosture: SharingPosture;
legacySandboxScopeDefaults?: LegacySandboxScopeDefaults;
- sandboxBackend: "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";
- sandboxSecondaryBackend?:
- "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";
+ sandboxBackend: SandboxBackendName;
+ sandboxSecondaryBackend?: SandboxBackendName;
deployProvider: "docker" | "aws" | "fly" | "porter";
egressServiceHosts?: string[];
brandingDefault?: OrgBranding;
@@ -205,6 +205,7 @@ export interface Config {
agent37Sandbox: Agent37SandboxEnv;
superserveSandbox: SuperserveSandboxEnv;
e2bSandbox: E2bSandboxEnv;
+ managedAgentsSandbox: ManagedAgentsSandboxEnv;
modalSandbox: ModalSandboxEnv;
porterSandbox: PorterSandboxEnv;
porterDeploy: PorterDeployEnv;
@@ -431,6 +432,40 @@ function e2bSandboxEnv(env: NodeJS.ProcessEnv): E2bSandboxEnv {
};
}
+interface ManagedAgentsSandboxEnv {
+ apiToken?: string;
+ apiBaseUrl?: string;
+ template?: string;
+ namePrefix?: string;
+ sizeSlug?: string;
+ idleTimeoutSec?: number;
+ egressProxyUrl?: string;
+ snapshotS3Bucket?: string;
+ snapshotIntervalSec?: number;
+ defaultTimeoutSec?: number;
+}
+
+function managedAgentsSandboxEnv(env: NodeJS.ProcessEnv): ManagedAgentsSandboxEnv {
+ return {
+ ...(env.DO_AGENTS_API_TOKEN ? { apiToken: env.DO_AGENTS_API_TOKEN } : {}),
+ ...(env.DO_AGENTS_API_BASE_URL ? { apiBaseUrl: env.DO_AGENTS_API_BASE_URL } : {}),
+ ...(env.DO_AGENTS_TEMPLATE ? { template: env.DO_AGENTS_TEMPLATE } : {}),
+ ...(env.DO_AGENTS_NAME_PREFIX ? { namePrefix: env.DO_AGENTS_NAME_PREFIX } : {}),
+ ...(env.DO_AGENTS_SIZE_SLUG ? { sizeSlug: env.DO_AGENTS_SIZE_SLUG } : {}),
+ ...(numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) !== undefined
+ ? { idleTimeoutSec: numEnvStrict("DO_AGENTS_IDLE_TIMEOUT_SEC", env.DO_AGENTS_IDLE_TIMEOUT_SEC) }
+ : {}),
+ ...(env.DO_AGENTS_EGRESS_PROXY_URL ? { egressProxyUrl: env.DO_AGENTS_EGRESS_PROXY_URL } : {}),
+ ...(env.DO_AGENTS_SNAPSHOT_S3_BUCKET ? { snapshotS3Bucket: env.DO_AGENTS_SNAPSHOT_S3_BUCKET } : {}),
+ ...(numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) !== undefined
+ ? { snapshotIntervalSec: numEnvStrict("DO_AGENTS_SNAPSHOT_INTERVAL_SEC", env.DO_AGENTS_SNAPSHOT_INTERVAL_SEC) }
+ : {}),
+ ...(numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) !== undefined
+ ? { defaultTimeoutSec: numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) }
+ : {}),
+ };
+}
+
interface ModalSandboxEnv {
nativeSnapshotsEnabled?: boolean;
nativeSnapshotIntervalSec?: number;
@@ -999,6 +1034,7 @@ export function enabledSandboxBackends(config: Config): Array<Config["sandboxBac
agent37: Boolean(config.agent37Sandbox?.apiKey),
superserve: Boolean(config.superserveSandbox?.apiKey && config.superserveSandbox?.template),
e2b: Boolean(config.e2bSandbox?.apiKey),
+ "do-managed-agents": Boolean(config.managedAgentsSandbox?.apiToken),
modal: Boolean(config.modalSandbox?.tokenId && config.modalSandbox?.tokenSecret),
aws: Boolean(config.awsSandbox?.s3Bucket),
porter: Boolean(config.porterSandbox?.token),
@@ -1022,11 +1058,12 @@ function sandboxBackendEnvStrict(value: string | undefined, name = "SANDBOX_BACK
backend === "modal" ||
backend === "agent37" ||
backend === "superserve" ||
+ backend === "do-managed-agents" ||
backend === "porter"
)
return backend;
throw new Error(
- `${name}=${JSON.stringify(value)} is not recognized — use aws, local, sprites, smolmachines, e2b, modal, porter, agent37, or superserve, or unset it.`,
+ `${name}=${JSON.stringify(value)} is not recognized — use aws, local, sprites, smolmachines, e2b, modal, porter, agent37, superserve, or do-managed-agents, or unset it.`,
);
}
@@ -1223,6 +1260,16 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
"[config] SANDBOX_BACKEND=e2b without E2B_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set E2B_EGRESS_PROXY_URL to the public egress proxy so E2B's network rules admit only that host.",
);
}
+ if (env.SANDBOX_BACKEND === "do-managed-agents" && !env.DO_AGENTS_EGRESS_PROXY_URL) {
+ console.warn(
+ "[config] SANDBOX_BACKEND=do-managed-agents without DO_AGENTS_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set DO_AGENTS_EGRESS_PROXY_URL to the public egress proxy so the session's egress allowlist admits only that host.",
+ );
+ }
+ if (env.DO_AGENTS_API_TOKEN && !env.DO_AGENTS_SNAPSHOT_S3_BUCKET) {
+ console.warn(
+ "[config] do-managed-agents sandbox backend enabled without DO_AGENTS_SNAPSHOT_S3_BUCKET — pause preserves the microVM, but a destroyed session takes the home with it; set DO_AGENTS_SNAPSHOT_S3_BUCKET so a replacement session can hydrate from a portable snapshot.",
+ );
+ }
if (env.MODAL_TOKEN_ID && env.MODAL_TOKEN_SECRET && !env.MODAL_EGRESS_PROXY_URL) {
console.warn(
"[config] modal sandbox backend enabled without MODAL_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set MODAL_EGRESS_PROXY_URL to the public https egress proxy so Modal's outbound allowlist admits only that host.",
@@ -1262,7 +1309,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
}
if (env.NODE_ENV === "production" && !env.SANDBOX_BACKEND?.trim()) {
throw new Error(
- "SANDBOX_BACKEND must be set explicitly in production — use sprites, smolmachines, e2b, modal, porter, agent37, superserve, aws, or local.",
+ "SANDBOX_BACKEND must be set explicitly in production — use sprites, smolmachines, e2b, modal, porter, agent37, superserve, do-managed-agents, aws, or local.",
);
}
const sandboxBackend = sandboxBackendEnvStrict(env.SANDBOX_BACKEND);
@@ -1662,6 +1709,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
porterSandbox: porterSandboxEnv(env),
porterDeploy: porterDeployEnv(env),
e2bSandbox: e2bSandboxEnv(env),
+ managedAgentsSandbox: managedAgentsSandboxEnv(env),
modalSandbox: modalSandboxEnv(env),
awsDeploy: {
...awsDeployEnv(env),
diff --git a/src/deployment/secret-schema.ts b/src/deployment/secret-schema.ts
index 4c2f34383..3c9880f8f 100644
--- a/src/deployment/secret-schema.ts
+++ b/src/deployment/secret-schema.ts
@@ -12,6 +12,7 @@ type SecretGate =
| "porter"
| "agent37"
| "superserve"
+ | "do-managed-agents"
| "porter-deploy"
| "fly-shared-deploy"
| "fly-deploy"
@@ -46,6 +47,7 @@ export const CORE_SECRET_SPECS: readonly RuntimeSecretSpec[] = [
{ name: "AGENT37_API_KEY", requiredWhen: "agent37" },
{ name: "SUPERSERVE_API_KEY", requiredWhen: "superserve" },
{ name: "E2B_API_KEY", requiredWhen: "e2b" },
+ { name: "DO_AGENTS_API_TOKEN", requiredWhen: "do-managed-agents" },
{ name: "MODAL_TOKEN_ID", requiredWhen: "modal" },
{ name: "MODAL_TOKEN_SECRET", requiredWhen: "modal" },
{ name: "PORTER_DEPLOY_API_TOKEN", requiredWhen: ["porter", "porter-deploy"] },
@@ -73,6 +75,7 @@ const GATE_PREDICATES: Readonly<Record<SecretGate, (env: NodeJS.ProcessEnv) => b
porter: (env) => sandboxBackendSelected(env, "porter"),
agent37: (env) => sandboxBackendSelected(env, "agent37"),
superserve: (env) => sandboxBackendSelected(env, "superserve"),
+ "do-managed-agents": (env) => sandboxBackendSelected(env, "do-managed-agents"),
"porter-deploy": (env) => env.DEPLOY_PROVIDER === "porter",
"fly-shared-deploy": (env) => env.DEPLOY_PROVIDER === "fly" && Boolean(env.FLY_DEPLOY_SHARED_APP_NAME?.trim()),
"fly-deploy": (env) => env.DEPLOY_PROVIDER === "fly",
diff --git a/src/sandbox/managed-agents-client.ts b/src/sandbox/managed-agents-client.ts
new file mode 100644
index 000000000..7430b7511
--- /dev/null
+++ b/src/sandbox/managed-agents-client.ts
@@ -0,0 +1,760 @@
+import { fileURLToPath } from "node:url";
+import { fetchWithRetry } from "../util/async.ts";
+import { errMessage, swallowAs, withRequestId } from "../util/errors.ts";
+import { openManagedAgentsTunnel, SANDBOX_AGENT_PORT, type ManagedAgentsTunnel } from "./managed-agents-tunnel.ts";
+
+export interface ManagedAgentsCommandResult {
+ stdout: string;
+ stderr: string;
+ exitCode: number;
+ hitlRejected?: boolean;
+}
+
+interface ManagedAgentsRunOpts {
+ timeoutMs?: number;
+ workdir?: string;
+ env?: Record<string, string>;
+}
+
+export type ManagedAgentsSessionState =
+ "unspecified" | "provisioning" | "ready" | "detached" | "destroying" | "destroyed" | "failed" | "paused";
+
+export interface ManagedAgentsSessionSummary {
+ sessionId: string;
+ sandboxId: string;
+ name: string;
+ state: ManagedAgentsSessionState;
+}
+
+export interface ManagedAgentsSessionInfo extends ManagedAgentsSessionSummary {
+ createdAtMs?: number;
+ errorMessage?: string;
+ sizeSlug?: string;
+ template?: string;
+}
+
+export interface ManagedAgentsCheckpoint {
+ checkpointId: string;
+ status: string;
+ createdAtMs?: number;
+}
+
+export interface ManagedAgentsSession {
+ readonly sessionId: string;
+ readonly sandboxId: string;
+ runCommand(command: string, opts?: ManagedAgentsRunOpts): Promise<ManagedAgentsCommandResult>;
+ readFileBytes(absPath: string): Promise<Uint8Array | null>;
+ writeFileBytes(absPath: string, data: Uint8Array): Promise<void>;
+ createCheckpoint(label?: string): Promise<ManagedAgentsCheckpoint>;
+ deleteCheckpoint(checkpointId: string): Promise<void>;
+ close(): Promise<void>;
+ pause(): Promise<void>;
+ resume(): Promise<void>;
+ kill(): Promise<void>;
+}
+
+export class ManagedAgentsSandboxGoneError extends Error {
+ constructor(sessionId: string, detail: string) {
+ super(`do-managed-agents session ${sessionId} is gone: ${detail}`);
+ this.name = "ManagedAgentsSandboxGoneError";
+ }
+}
+
+export class ManagedAgentsCommandLostError extends Error {
+ constructor(sessionId: string, detail: string) {
+ super(
+ `do-managed-agents session ${sessionId} was lost while a command was running (${detail}); the command may have partially executed and was not retried`,
+ );
+ this.name = "ManagedAgentsCommandLostError";
+ }
+}
+
+export class ManagedAgentsHitlRejectedError extends Error {
+ constructor(sessionId: string) {
+ super(
+ `do-managed-agents rejected a command in session ${sessionId} at its own approval gate; set the session permissions to allow bash so qm's approval gates stay authoritative`,
+ );
+ this.name = "ManagedAgentsHitlRejectedError";
+ }
+}
+
+interface ManagedAgentsCreateOpts {
+ name: string;
+ egressAllow?: string[];
+}
+
+export interface ManagedAgentsClient {
+ readonly nativePause?: boolean;
+ info(sessionId: string): Promise<ManagedAgentsSessionInfo>;
+ create(opts: ManagedAgentsCreateOpts): Promise<ManagedAgentsSession>;
+ connect(sessionId: string): Promise<ManagedAgentsSession>;
+ list(name: string): Promise<ManagedAgentsSessionSummary[]>;
+ rollback(sessionId: string, checkpointId: string): Promise<ManagedAgentsSessionInfo>;
+ kill(sessionId: string): Promise<void>;
+}
+
+export interface SdkManagedAgentsClientOptions {
+ apiToken: string;
+ apiBaseUrl?: string;
+ template?: string;
+ agent?: string;
+ sizeSlug?: string;
+ idleTimeoutSec?: number;
+ maxCommandMs?: number;
+ stallProbeMs?: number;
+ egressProxyUrl?: string;
+}
+
+const DEFAULT_API_BASE_URL = "https://api.digitalocean.com";
+const DEFAULT_TEMPLATE = "";
+const DEFAULT_AGENT = "none";
+const DEFAULT_MAX_COMMAND_MS = 3600_000;
+const CREATE_TIMEOUT_MS = 120_000;
+const CHECKPOINT_TIMEOUT_MS = 5 * 60_000;
+const REQUEST_TIMEOUT_MS = 30_000;
+const READY_TIMEOUT_MS = 300_000;
+const ROLLBACK_TIMEOUT_MS = READY_TIMEOUT_MS;
+const READY_POLL_MS = 2_000;
+const READY_FAST_POLL_MS = 250;
+const READY_FAST_WINDOW_MS = 10_000;
+const STALL_PROBE_MS = 45_000;
+const READY_TRANSIENT_LIMIT = 5;
+const UPLOAD_CHUNK_BYTES = 256 * 1024;
+const DOWNLOAD_CHUNK_BYTES = 256 * 1024;
+const MAX_EXEC_OUTPUT_BYTES = 16 * 1024 * 1024;
+const CONNECT_TIMEOUT_MS = 30_000;
+const GRPC_NOT_FOUND = 5;
+const GRPC_INTERNAL = 13;
+const GRPC_UNAVAILABLE = 14;
+
+const PROTO_PATH = fileURLToPath(new URL("./managed-agents-sandbox-agent.proto", import.meta.url));
+
+const STATE_BY_WIRE: Record<string, ManagedAgentsSessionState> = {
+ SESSION_STATUS_UNSPECIFIED: "unspecified",
+ SESSION_STATUS_PROVISIONING: "provisioning",
+ SESSION_STATUS_READY: "ready",
+ SESSION_STATUS_DETACHED: "detached",
+ SESSION_STATUS_DESTROYING: "destroying",
+ SESSION_STATUS_DESTROYED: "destroyed",
+ SESSION_STATUS_FAILED: "failed",
+ SESSION_STATUS_PAUSED: "paused",
+};
+
+const GONE_STATES: ReadonlySet<ManagedAgentsSessionState> = new Set<ManagedAgentsSessionState>([
+ "destroying",
+ "destroyed",
+ "failed",
+]);
+
+const USABLE_STATES: ReadonlySet<ManagedAgentsSessionState> = new Set<ManagedAgentsSessionState>(["ready", "detached"]);
+
+export const managedAgentsSessionState = (wire: string | undefined): ManagedAgentsSessionState =>
+ STATE_BY_WIRE[wire ?? ""] ?? "unspecified";
+
+export function managedAgentsEgressAllow(egressProxyUrl: string): string[] {
+ const host = new URL(egressProxyUrl).hostname.replace(/^\[|\]$/g, "");
+ if (!host) throw new Error(`DO_AGENTS_EGRESS_PROXY_URL ${egressProxyUrl} has no host to put on the egress allowlist`);
+ return [host];
+}
+
+export function managedAgentsManifest(opts: {
+ name: string;
+ template: string;
+ agent?: string;
+ sizeSlug?: string;
+ idleTimeoutSec?: number;
+ egressAllow?: readonly string[];
+}): string {
+ const lines = [`name: ${JSON.stringify(opts.name)}`, `agent: ${JSON.stringify(opts.agent ?? DEFAULT_AGENT)}`];
+ if (opts.template) lines.push(`template: ${JSON.stringify(opts.template)}`);
+ if (opts.sizeSlug) lines.push(`size: ${JSON.stringify(opts.sizeSlug)}`);
+ if (opts.idleTimeoutSec) lines.push(`idle_timeout: ${JSON.stringify(`${opts.idleTimeoutSec}s`)}`);
+ if (opts.egressAllow?.length) {
+ lines.push("egress:");
+ for (const host of opts.egressAllow) lines.push(` - ${JSON.stringify(host)}`);
+ }
+ lines.push("permissions:", " default: allow", " rules:", " - tool: bash", " action: allow");
+ return `${lines.join("\n")}\n`;
+}
+
+interface WireSession {
+ id?: string;
+ session_id?: string;
+ sandbox_id?: string;
+ name?: string;
+ status?: string;
+ template?: string;
+ size_slug?: string;
+ error_message?: string;
+ created_at?: string;
+}
+
+interface WireCheckpoint {
+ checkpoint_id?: string;
+ status?: string;
+ error_message?: string;
+ created_at?: string;
+}
+
+const toSummary = (s: WireSession): ManagedAgentsSessionSummary => ({
+ sessionId: s.session_id ?? s.id ?? "",
+ sandboxId: s.sandbox_id ?? "",
+ name: s.name ?? "",
+ state: managedAgentsSessionState(s.status),
+});
+
+const toInfo = (s: WireSession): ManagedAgentsSessionInfo => {
+ const createdAtMs = s.created_at ? Date.parse(s.created_at) : Number.NaN;
+ return {
+ ...toSummary(s),
+ ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}),
+ ...(s.error_message ? { errorMessage: s.error_message } : {}),
+ ...(s.size_slug ? { sizeSlug: s.size_slug } : {}),
+ ...(s.template ? { template: s.template } : {}),
+ };
+};
+
+interface GrpcCallError {
+ code?: number;
+ details?: string;
+ message?: string;
+}
+
+const grpcCode = (err: unknown): number | undefined =>
+ typeof err === "object" && err !== null ? (err as GrpcCallError).code : undefined;
+
+const isGuestGone = (err: unknown): boolean => {
+ const code = grpcCode(err);
+ return code === GRPC_NOT_FOUND || code === GRPC_UNAVAILABLE;
+};
+
+type ProtoLoader = typeof import("@grpc/proto-loader");
+type Grpc = typeof import("@grpc/grpc-js");
+
+interface GrpcDuplex {
+ write(msg: unknown): void;
+ end(): void;
+ cancel(): void;
+ on(event: string, cb: (arg: never) => void): void;
+}
+
+type GrpcWritable = Pick<GrpcDuplex, "write" | "end" | "on">;
+type GrpcReadable = Pick<GrpcDuplex, "on" | "cancel">;
+
+interface SandboxAgent {
+ Exec(): GrpcDuplex;
+ Upload(cb: (err: unknown, res?: { bytes_written?: string }) => void): GrpcWritable;
+ Download(req: unknown): GrpcReadable;
+ waitForReady(deadline: Date, cb: (err?: Error) => void): void;
+ close(): void;
+}
+
+interface ExecOutputFrame {
+ output?: "stdout" | "stderr" | "exit";
+ stdout?: Buffer;
+ stderr?: Buffer;
+ exit?: { exit_code?: number; hitl_rejected?: boolean; timed_out?: boolean };
+}
+
+interface DownloadFrame {
+ output?: "header" | "chunk" | "end";
+ chunk?: Buffer;
+}
+
+let grpcModules: Promise<[ProtoLoader, Grpc]> | null = null;
+
+function loadGrpc(): Promise<[ProtoLoader, Grpc]> {
+ grpcModules ??= Promise.all([
+ import("@grpc/proto-loader").catch((e: unknown) => {
+ throw new Error(`@grpc/proto-loader is not installed: ${errMessage(e)}`);
+ }),
+ import("@grpc/grpc-js").catch((e: unknown) => {
+ throw new Error(`@grpc/grpc-js is not installed: ${errMessage(e)}`);
+ }),
+ ]) as Promise<[ProtoLoader, Grpc]>;
+ return grpcModules;
+}
+
+async function dialSandboxAgent(localPort: number, readyTimeoutMs: number): Promise<SandboxAgent> {
+ const [loader, grpc] = await loadGrpc();
+ const definition = loader.loadSync(PROTO_PATH, {
+ keepCase: true,
+ longs: String,
+ enums: String,
+ defaults: true,
+ oneofs: true,
+ });
+ const pkg = grpc.loadPackageDefinition(definition) as unknown as {
+ do: {
+ teams: {
+ hosted_agents: {
+ runtime: {
+ sandbox_agent: {
+ v1: {
+ SandboxAgentService: new (
+ addr: string,
+ creds: ReturnType<Grpc["credentials"]["createInsecure"]>,
+ options: Record<string, number>,
+ ) => SandboxAgent;
+ };
+ };
+ };
+ };
+ };
+ };
+ };
+ const Service = pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService;
+ const agent = new Service(`127.0.0.1:${localPort}`, grpc.credentials.createInsecure(), {
+ "grpc.max_receive_message_length": MAX_EXEC_OUTPUT_BYTES,
+ "grpc.max_send_message_length": MAX_EXEC_OUTPUT_BYTES,
+ });
+ try {
+ await new Promise<void>((resolve, reject) => {
+ agent.waitForReady(new Date(Date.now() + readyTimeoutMs), (err?: Error) => (err ? reject(err) : resolve()));
+ });
+ } catch (e) {
+ agent.close();
+ throw e;
+ }
+ return agent;
+}
+
+export function createSdkManagedAgentsClient(opts: SdkManagedAgentsClientOptions): ManagedAgentsClient {
+ const apiBaseUrl = (opts.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/+$/, "");
+ const template = opts.template ?? DEFAULT_TEMPLATE;
+ const maxCommandMs = opts.maxCommandMs ?? DEFAULT_MAX_COMMAND_MS;
+ const stallProbeMs = opts.stallProbeMs ?? STALL_PROBE_MS;
+ const egressAllow = opts.egressProxyUrl ? managedAgentsEgressAllow(opts.egressProxyUrl) : undefined;
+
+ function send(method: string, path: string, body?: string, contentType?: string, signal?: AbortSignal) {
+ return fetch(`${apiBaseUrl}${path}`, {
+ method,
+ headers: {
+ authorization: `Bearer ${opts.apiToken}`,
+ accept: "application/json",
+ ...(contentType ? { "content-type": contentType } : {}),
+ },
+ ...(body !== undefined ? { body } : {}),
+ signal: signal ?? AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+ }
+
+ function call(
+ method: string,
+ path: string,
+ body?: string,
+ contentType?: string,
+ timeoutMs = REQUEST_TIMEOUT_MS,
+ ): Promise<Response> {
+ const operation = (signal?: AbortSignal): Promise<Response> => send(method, path, body, contentType, signal);
+ return method === "GET" || method === "DELETE"
+ ? fetchWithRetry((signal) => operation(signal), "idempotent", { timeoutMs })
+ : operation(AbortSignal.timeout(timeoutMs));
+ }
+
+ async function failure(action: string, sessionId: string, res: Response): Promise<Error> {
+ const raw = await res.text().catch(swallowAs(`do-managed-agents ${action} ${sessionId}: read error body`, ""));
+ let message = raw.slice(0, 200);
+ try {
+ const parsed = JSON.parse(raw) as { error?: { message?: string } };
+ if (parsed.error?.message) message = parsed.error.message;
+ } catch {
+ message = raw.slice(0, 200);
+ }
+ const detail = withRequestId(`http ${res.status} ${message}`, res.headers);
+ if (res.status === 404 || res.status === 410) return new ManagedAgentsSandboxGoneError(sessionId, detail);
+ return new Error(`do-managed-agents ${action}: ${detail}`);
+ }
+
+ async function callJson<T>(
+ method: string,
+ path: string,
+ sessionId: string,
+ body?: string,
+ contentType?: string,
+ timeoutMs = REQUEST_TIMEOUT_MS,
+ ): Promise<T> {
+ const res = await call(method, path, body, contentType, timeoutMs);
+ if (!res.ok) throw await failure(`${method} ${path}`, sessionId, res);
+ if (res.status === 204) return undefined as T;
+ return (await res.json()) as T;
+ }
+
+ const getInfo = async (sessionId: string): Promise<ManagedAgentsSessionInfo> => {
+ const body = await callJson<{ session?: WireSession }>("GET", `/v2/agents/sessions/${sessionId}`, sessionId);
+ if (!body.session) throw new ManagedAgentsSandboxGoneError(sessionId, "response carried no session");
+ return toInfo(body.session);
+ };
+
+ async function awaitUsable(sessionId: string): Promise<ManagedAgentsSessionInfo> {
+ const started = Date.now();
+ const deadline = started + READY_TIMEOUT_MS;
+ let resumed = false;
+ let strikes = 0;
+ const tolerate = (err: unknown): void => {
+ if (err instanceof ManagedAgentsSandboxGoneError) throw err;
+ if (++strikes > READY_TRANSIENT_LIMIT) throw err;
+ };
+ for (;;) {
+ let info: ManagedAgentsSessionInfo | undefined;
+ try {
+ info = await getInfo(sessionId);
+ strikes = 0;
+ } catch (err) {
+ tolerate(err);
+ }
+ if (info) {
+ if (USABLE_STATES.has(info.state)) return info;
+ if (GONE_STATES.has(info.state))
+ throw new ManagedAgentsSandboxGoneError(sessionId, info.errorMessage ?? `status ${info.state}`);
+ if (info.state === "paused" && !resumed) {
+ try {
+ await callJson<void>("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId);
+ resumed = true;
+ strikes = 0;
+ } catch (err) {
+ tolerate(err);
+ }
+ }
+ }
+ if (Date.now() >= deadline)
+ throw new Error(`do-managed-agents session ${sessionId} did not become ready within ${READY_TIMEOUT_MS}ms`);
+ await new Promise((resolve) =>
+ setTimeout(resolve, Date.now() - started < READY_FAST_WINDOW_MS ? READY_FAST_POLL_MS : READY_POLL_MS),
+ );
+ }
+ }
+
+ async function rollbackToCheckpoint(sessionId: string, checkpointId: string): Promise<ManagedAgentsSessionInfo> {
+ const body = await callJson<{ session?: WireSession }>(
+ "POST",
+ `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}/rollback`,
+ sessionId,
+ undefined,
+ undefined,
+ ROLLBACK_TIMEOUT_MS,
+ );
+ if (!body.session) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no session`);
+ const restored = toInfo(body.session);
+ if (!restored.sandboxId) throw new Error(`do-managed-agents rollback ${sessionId}: response carried no sandbox id`);
+ if (GONE_STATES.has(restored.state))
+ throw new ManagedAgentsSandboxGoneError(sessionId, restored.errorMessage ?? `status ${restored.state}`);
+ return restored;
+ }
+
+ function buildSession(info: ManagedAgentsSessionInfo): ManagedAgentsSession {
+ const sessionId = info.sessionId;
+ let guest: Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> | null = null;
+ let lastTunnel: ManagedAgentsTunnel | null = null;
+
+ const connect = (): Promise<{ tunnel: ManagedAgentsTunnel; agent: SandboxAgent }> =>
+ (guest ??= (async () => {
+ const tunnel = await openManagedAgentsTunnel({
+ apiBaseUrl,
+ sessionId,
+ remotePort: SANDBOX_AGENT_PORT,
+ getToken: async () => opts.apiToken,
+ });
+ lastTunnel = tunnel;
+ try {
+ const dial = dialSandboxAgent(tunnel.localPort, CONNECT_TIMEOUT_MS);
+ const abort = tunnel.whenFailed().then((detail) => {
+ throw new Error(detail);
+ });
+ dial.catch(() => undefined);
+ abort.catch(() => undefined);
+ return { tunnel, agent: await Promise.race([dial, abort]) };
+ } catch (e) {
+ const detail = tunnel.lastFailure() ?? errMessage(e);
+ await tunnel.close();
+ throw new ManagedAgentsSandboxGoneError(sessionId, `guest unreachable over the port-forward: ${detail}`);
+ }
+ })().catch((e: unknown) => {
+ guest = null;
+ throw e;
+ }));
+
+ async function disconnect(): Promise<void> {
+ const pending = guest;
+ guest = null;
+ if (!pending) return;
+ await pending
+ .then(async ({ tunnel, agent }) => {
+ agent.close();
+ await tunnel.close();
+ })
+ .catch(swallowAs(`do-managed-agents ${sessionId}: disconnect`, undefined));
+ }
+
+ async function download(absPath: string): Promise<Uint8Array | null> {
+ const { agent } = await connect();
+ return new Promise<Uint8Array | null>((resolve, reject) => {
+ const chunks: Buffer[] = [];
+ const stream = agent.Download({ path: absPath, as_archive: false, chunk_size_bytes: DOWNLOAD_CHUNK_BYTES });
+ let settled = false;
+ const settle = (fn: () => void): void => {
+ if (settled) return;
+ settled = true;
+ fn();
+ };
+ stream.on("data", ((frame: DownloadFrame) => {
+ if (frame.chunk?.length) chunks.push(frame.chunk);
+ }) as (arg: never) => void);
+ stream.on("error", ((err: unknown) => {
+ settle(() => {
+ if (grpcCode(err) === GRPC_NOT_FOUND) {
+ resolve(null);
+ return;
+ }
+ const detail = guestDetail(err);
+ void disconnect();
+ reject(
+ grpcCode(err) === GRPC_UNAVAILABLE
+ ? new ManagedAgentsSandboxGoneError(sessionId, detail)
+ : new Error(`do-managed-agents download ${absPath} failed: ${detail}`, { cause: err }),
+ );
+ });
+ }) as (arg: never) => void);
+ stream.on("end", (() => settle(() => resolve(new Uint8Array(Buffer.concat(chunks))))) as (arg: never) => void);
+ });
+ }
+
+ const guestDetail = (err: unknown): string => lastTunnel?.lastFailure() ?? errMessage(err);
+
+ return {
+ sessionId,
+ sandboxId: info.sandboxId,
+
+ async runCommand(command, runOpts): Promise<ManagedAgentsCommandResult> {
+ const { agent } = await connect();
+ const timeoutMs = Math.min(runOpts?.timeoutMs ?? maxCommandMs, maxCommandMs);
+ return new Promise<ManagedAgentsCommandResult>((resolve, reject) => {
+ const stream = agent.Exec();
+ const out: Buffer[] = [];
+ const errOut: Buffer[] = [];
+ let outBytes = 0;
+ let errBytes = 0;
+ let exit: { exit_code?: number; hitl_rejected?: boolean } | undefined;
+ let settled = false;
+ let quietSince = Date.now();
+ const settle = (fn: () => void): void => {
+ if (settled) return;
+ settled = true;
+ clearInterval(watchdog);
+ fn();
+ };
+
+ const watchdog = setInterval(() => {
+ if (settled || Date.now() - quietSince < stallProbeMs) return;
+ void getInfo(sessionId).then(
+ (live) => {
+ quietSince = Date.now();
+ if (settled || USABLE_STATES.has(live.state)) return;
+ settle(() => {
+ void disconnect();
+ reject(
+ GONE_STATES.has(live.state)
+ ? new ManagedAgentsSandboxGoneError(sessionId, `status ${live.state}`)
+ : new ManagedAgentsCommandLostError(sessionId, `the session became ${live.state} mid-command`),
+ );
+ });
+ },
+ swallowAs(`do-managed-agents ${sessionId}: stall probe`, undefined),
+ );
+ }, stallProbeMs);
+ watchdog.unref();
+
+ stream.on("data", ((frame: ExecOutputFrame) => {
+ quietSince = Date.now();
+ if (frame.stdout && outBytes < MAX_EXEC_OUTPUT_BYTES) {
+ out.push(frame.stdout);
+ outBytes += frame.stdout.length;
+ }
+ if (frame.stderr && errBytes < MAX_EXEC_OUTPUT_BYTES) {
+ errOut.push(frame.stderr);
+ errBytes += frame.stderr.length;
+ }
+ if (frame.exit) exit = frame.exit;
+ }) as (arg: never) => void);
+
+ stream.on("error", ((err: unknown) => {
+ settle(() => {
+ const detail = guestDetail(err);
+ void disconnect();
+ reject(
+ isGuestGone(err)
+ ? new ManagedAgentsCommandLostError(sessionId, detail)
+ : new Error(`do-managed-agents exec failed: ${detail}`),
+ );
+ });
+ }) as (arg: never) => void);
+
+ stream.on("end", (() => {
+ settle(() => {
+ if (exit?.hitl_rejected) {
+ reject(new ManagedAgentsHitlRejectedError(sessionId));
+ return;
+ }
+ if (exit?.exit_code === undefined) {
+ void disconnect();
+ reject(new ManagedAgentsCommandLostError(sessionId, "the command stream ended without an exit status"));
+ return;
+ }
+ resolve({
+ stdout: Buffer.concat(out).toString("utf8"),
+ stderr: Buffer.concat(errOut).toString("utf8"),
+ exitCode: exit.exit_code,
+ });
+ });
+ }) as (arg: never) => void);
+
+ stream.write({
+ start: {
+ argv: ["/bin/bash", "-c", command],
+ ...(runOpts?.workdir ? { workdir: runOpts.workdir } : {}),
+ ...(runOpts?.env ? { env: runOpts.env } : {}),
+ timeout_seconds: Math.ceil(timeoutMs / 1000),
+ },
+ });
+ stream.end();
+ });
+ },
+
+ async readFileBytes(absPath): Promise<Uint8Array | null> {
+ try {
+ return await download(absPath);
+ } catch (err) {
+ if (grpcCode((err as { cause?: unknown }).cause) !== GRPC_INTERNAL) throw err;
+ const probe = await this.runCommand(`test -e '${absPath.replaceAll("'", "'\\''")}'`, { timeoutMs: 30_000 });
+ if (probe.exitCode === 1) return null;
+ throw err;
+ }
+ },
+
+ async writeFileBytes(absPath, data): Promise<void> {
+ const { agent } = await connect();
+ await new Promise<void>((resolve, reject) => {
+ const stream = agent.Upload((err: unknown) => {
+ if (!err) {
+ resolve();
+ return;
+ }
+ const detail = guestDetail(err);
+ void disconnect();
+ reject(
+ isGuestGone(err)
+ ? new ManagedAgentsSandboxGoneError(sessionId, detail)
+ : new Error(`do-managed-agents upload ${absPath} failed: ${detail}`),
+ );
+ });
+ stream.write({ header: { path: absPath, mode: 0o644, is_archive: false } });
+ for (let offset = 0; offset < data.length; offset += UPLOAD_CHUNK_BYTES)
+ stream.write({ chunk: Buffer.from(data.subarray(offset, offset + UPLOAD_CHUNK_BYTES)) });
+ stream.write({ end: {} });
+ stream.end();
+ });
+ },
+
+ async createCheckpoint(label?: string): Promise<ManagedAgentsCheckpoint> {
+ const body = await callJson<{ checkpoint?: WireCheckpoint }>(
+ "POST",
+ `/v2/agents/sessions/${sessionId}/checkpoints`,
+ sessionId,
+ JSON.stringify(label ? { label } : {}),
+ "application/json",
+ CHECKPOINT_TIMEOUT_MS,
+ );
+ const captured = body.checkpoint;
+ if (!captured?.checkpoint_id)
+ throw new Error(`do-managed-agents create checkpoint ${sessionId}: response carried no checkpoint id`);
+ if (captured.status !== "READY") {
+ const detail = captured.error_message ? `: ${captured.error_message}` : "";
+ throw new Error(
+ `do-managed-agents create checkpoint ${sessionId}: status ${captured.status ?? "missing"}${detail}`,
+ );
+ }
+ const createdAtMs = captured.created_at ? Date.parse(captured.created_at) : Number.NaN;
+ return {
+ checkpointId: captured.checkpoint_id,
+ status: captured.status,
+ ...(Number.isFinite(createdAtMs) ? { createdAtMs } : {}),
+ };
+ },
+
+ async deleteCheckpoint(checkpointId: string): Promise<void> {
+ await callJson<void>(
+ "DELETE",
+ `/v2/agents/sessions/${sessionId}/checkpoints/${encodeURIComponent(checkpointId)}`,
+ sessionId,
+ );
+ },
+
+ async close(): Promise<void> {
+ await disconnect();
+ },
+
+ async pause(): Promise<void> {
+ await disconnect();
+ await callJson<void>("POST", `/v2/agents/sessions/${sessionId}/pause`, sessionId);
+ },
+
+ async resume(): Promise<void> {
+ await callJson<void>("POST", `/v2/agents/sessions/${sessionId}/resume`, sessionId);
+ },
+
+ async kill(): Promise<void> {
+ await disconnect();
+ await callJson<void>("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId);
+ },
+ };
+ }
+
+ return {
+ nativePause: true,
+
+ info: getInfo,
+
+ async create(createOpts): Promise<ManagedAgentsSession> {
+ const manifest = managedAgentsManifest({
+ name: createOpts.name,
+ template,
+ ...(opts.agent ? { agent: opts.agent } : {}),
+ ...(opts.sizeSlug ? { sizeSlug: opts.sizeSlug } : {}),
+ ...(opts.idleTimeoutSec ? { idleTimeoutSec: opts.idleTimeoutSec } : {}),
+ ...((createOpts.egressAllow ?? egressAllow) ? { egressAllow: createOpts.egressAllow ?? egressAllow } : {}),
+ });
+ const body = await callJson<{ session?: WireSession }>(
+ "POST",
+ "/v2/agents/sessions",
+ createOpts.name,
+ manifest,
+ "application/x-yaml",
+ CREATE_TIMEOUT_MS,
+ );
+ const created = body.session ? toSummary(body.session) : null;
+ if (!created?.sessionId) throw new Error("do-managed-agents create session: response carried no session id");
+ return buildSession(await awaitUsable(created.sessionId));
+ },
+
+ async connect(sessionId): Promise<ManagedAgentsSession> {
+ return buildSession(await awaitUsable(sessionId));
+ },
+
+ async rollback(sessionId, checkpointId): Promise<ManagedAgentsSessionInfo> {
+ return rollbackToCheckpoint(sessionId, checkpointId);
+ },
+
+ async list(name): Promise<ManagedAgentsSessionSummary[]> {
+ const body = await callJson<{ sessions?: WireSession[] }>(
+ "GET",
+ `/v2/agents/sessions?name=${encodeURIComponent(name)}`,
+ name,
+ );
+ return (body.sessions ?? []).map(toSummary).filter((s) => s.name === name && !GONE_STATES.has(s.state));
+ },
+
+ async kill(sessionId): Promise<void> {
+ await callJson<void>("DELETE", `/v2/agents/sessions/${sessionId}`, sessionId);
+ },
+ };
+}
diff --git a/src/sandbox/managed-agents-sandbox-agent.proto b/src/sandbox/managed-agents-sandbox-agent.proto
new file mode 100644
index 000000000..59f5c8dda
--- /dev/null
+++ b/src/sandbox/managed-agents-sandbox-agent.proto
@@ -0,0 +1,98 @@
+syntax = "proto3";
+
+package do.teams.hosted_agents.runtime.sandbox_agent.v1;
+
+service SandboxAgentService {
+ rpc Exec(stream ExecRequest) returns (stream ExecOutput);
+ rpc Upload(stream UploadRequest) returns (UploadResponse);
+ rpc Download(DownloadRequest) returns (stream DownloadChunk);
+}
+
+message ExecRequest {
+ oneof request {
+ ExecStart start = 1;
+ bytes stdin = 2;
+ ExecSignal signal = 3;
+ }
+}
+
+message ExecStart {
+
+ repeated string argv = 1;
+ string workdir = 2;
+ map<string, string> env = 3;
+
+ int64 timeout_seconds = 4;
+}
+
+message ExecSignal {
+ int32 signal = 1;
+}
+
+message ExecOutput {
+ oneof output {
+ bytes stdout = 1;
+ bytes stderr = 2;
+ ExecExit exit = 3;
+ }
+}
+
+message ExecExit {
+ int32 exit_code = 1;
+ int64 duration_ms = 2;
+
+ bool hitl_rejected = 3;
+ bool timed_out = 4;
+}
+
+message UploadRequest {
+ oneof request {
+ UploadHeader header = 1;
+ bytes chunk = 2;
+ UploadEnd end = 3;
+ }
+}
+
+message UploadHeader {
+
+ string path = 1;
+
+ uint32 mode = 2;
+
+ bool is_archive = 3;
+}
+
+message UploadEnd {
+
+ string sha256 = 1;
+}
+
+message UploadResponse {
+ string path = 1;
+ int64 bytes_written = 2;
+}
+
+message DownloadRequest {
+ string path = 1;
+ bool as_archive = 2;
+
+ uint32 chunk_size_bytes = 3;
+}
+
+message DownloadChunk {
+ oneof output {
+ DownloadHeader header = 1;
+ bytes chunk = 2;
+ DownloadEnd end = 3;
+ }
+}
+
+message DownloadHeader {
+
+ int64 size_bytes = 1;
+ bool is_archive = 2;
+}
+
+message DownloadEnd {
+ string sha256 = 1;
+}
diff --git a/src/sandbox/managed-agents-sandbox.ts b/src/sandbox/managed-agents-sandbox.ts
new file mode 100644
index 000000000..0ebdf9afc
--- /dev/null
+++ b/src/sandbox/managed-agents-sandbox.ts
@@ -0,0 +1,631 @@
+import { cleanupFailedProvision } from "./sandbox.ts";
+import { randomUUID } from "node:crypto";
+import type { WorkspaceLayer } from "../types.ts";
+import type { WorkspaceStore } from "../workspace/workspace-store.ts";
+import type { DurableMap } from "../persistence/durable-map.ts";
+import { createMemoryMap } from "../persistence/durable-map.ts";
+import { createKeyedQueue } from "../util/async.ts";
+import { swallowAs, errMessage } from "../util/errors.ts";
+import { shq } from "../util/shell.ts";
+import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, forceThroughProxyEnv } from "./sandbox-env.ts";
+import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
+import { materializeRoLayers } from "./ro-layers.ts";
+import { createLayerToolInstaller } from "./layer-tool-install.ts";
+import type { LayerInstallFile } from "../deployment/load-layer.ts";
+import {
+ createBackendBlobStaging,
+ createExecExport,
+ createExecFileOps,
+ posixJoin,
+ type BlobStagingOptions,
+} from "./exec-file-ops.ts";
+import {
+ ephemeralCredLinkScript,
+ ephemeralCredLinkPaths,
+ type CredentialPathSpec,
+} from "../credentials/resident-paths.ts";
+import { killableScript, killScript } from "./exec-kill.ts";
+import { visibleNotInstalled, visibleTools } from "./sandbox.ts";
+import { sandboxScopeName } from "./exec-sandbox-base.ts";
+import {
+ ManagedAgentsCommandLostError,
+ ManagedAgentsSandboxGoneError,
+ type ManagedAgentsClient,
+ type ManagedAgentsSessionInfo,
+ type ManagedAgentsSession,
+} from "./managed-agents-client.ts";
+import {
+ createHomeSnapshotOps,
+ createMemorySnapshotStore,
+ HOME_SNAPSHOT_PRUNE,
+ snapshotDue,
+ type HomeSnapshotStore,
+} from "./home-snapshot.ts";
+import type {
+ AgentComputerProfile,
+ ComputerStatus,
+ ExecOptions,
+ ExecResult,
+ ProvisionOptions,
+ Sandbox,
+ SandboxHandle,
+ TeardownOptions,
+} from "./sandbox.ts";
+
+const HOME_DIR = "/workspace/home";
+const WORKSPACE_BASENAME = "workspace";
+const RO_LAYERS_TAR = ".ro-layers.tar";
+const RO_LAYERS_MANIFEST = ".ro-layers.manifest";
+const HOME_TAR = `${HOME_DIR}/.qm-home.tar`;
+
+const SNAPSHOT_PRUNE = HOME_SNAPSHOT_PRUNE;
+const DEFAULT_SNAPSHOT_INTERVAL_MS = 5 * 60_000;
+const CHECKPOINT_LABEL = "qm teardown";
+
+export const QM_TEMPLATE_TOOLS = ["jq", "rg", "unzip", "wget"];
+
+export interface StoredManagedAgentsSandbox {
+ sessionId: string;
+ sandboxId: string;
+ preservationState?: "running" | "paused" | "pause_failed";
+ preservationError?: string;
+ createdAtMs: number;
+ lastSnapshotMs?: number;
+ homeDirty?: boolean;
+ checkpointId?: string;
+ checkpointAtMs?: number;
+ checkpointError?: string;
+ pendingCheckpointDeletes?: string[];
+}
+
+export interface ManagedAgentsSandboxOptions extends BlobStagingOptions {
+ client: ManagedAgentsClient;
+ namePrefix?: string;
+ defaultTimeoutSec?: number;
+ snapshotIntervalMs?: number;
+ egressProxyUrl?: string;
+ extraTools?: string[];
+ credentialPaths?: CredentialPathSpec[];
+ layerToolFiles?: () => readonly LayerInstallFile[];
+ store?: DurableMap<StoredManagedAgentsSandbox>;
+ snapshots?: HomeSnapshotStore;
+ onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;
+}
+
+export function createManagedAgentsSandbox(workspace: WorkspaceStore, opts: ManagedAgentsSandboxOptions): Sandbox {
+ const client = opts.client;
+ const prefix = opts.namePrefix ?? "qm";
+ const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;
+ const snapshotIntervalMs = opts.snapshotIntervalMs ?? DEFAULT_SNAPSHOT_INTERVAL_MS;
+ const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`;
+ const store = opts.store ?? createMemoryMap<StoredManagedAgentsSandbox>();
+ const snapshots = opts.snapshots ?? createMemorySnapshotStore();
+ const provisionQueue = createKeyedQueue<string>();
+
+ const sessionByName = new Map<string, ManagedAgentsSession>();
+ const scopeByName = new Map<string, string>();
+ const scratchKeyByName = new Map<string, string>();
+ const activeScratch = new Map<string, number>();
+
+ const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {
+ opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });
+ };
+
+ const homeSnapshots = createHomeSnapshotOps<ManagedAgentsSession>({
+ label: "do-managed-agents",
+ homeDir: HOME_DIR,
+ homeTarPath: HOME_TAR,
+ prunePaths: [...SNAPSHOT_PRUNE, ...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`)],
+ store: snapshots,
+ io: {
+ runCommand: (session, script, timeoutMs) => session.runCommand(script, { timeoutMs }),
+ readFileBytes: (session, abs) => session.readFileBytes(abs),
+ writeFileBytes: (session, abs, data) => session.writeFileBytes(abs, data),
+ },
+ });
+
+ async function snapshotHome(scope: string, session: ManagedAgentsSession): Promise<void> {
+ await homeSnapshots.snapshotHome(scope, session);
+ await store.merge(scope, { lastSnapshotMs: Date.now(), homeDirty: false });
+ }
+
+ async function pendingDeletes(scope: string): Promise<string[]> {
+ return (await store.get(scope))?.pendingCheckpointDeletes ?? [];
+ }
+
+ async function forgetCheckpoint(scope: string, session: ManagedAgentsSession, checkpointId: string): Promise<void> {
+ try {
+ await session.deleteCheckpoint(checkpointId);
+ } catch (e) {
+ reportError("sandbox_snapshot", "checkpoint_delete_failed", errMessage(e), scope);
+ return;
+ }
+ const pending = await pendingDeletes(scope);
+ await store.merge(scope, { pendingCheckpointDeletes: pending.filter((id) => id !== checkpointId) });
+ }
+
+ async function retryPendingCheckpointDeletes(scope: string, session: ManagedAgentsSession): Promise<void> {
+ const current = (await store.get(scope))?.checkpointId;
+ for (const checkpointId of await pendingDeletes(scope)) {
+ if (checkpointId === current) continue;
+ await forgetCheckpoint(scope, session, checkpointId);
+ }
+ }
+
+ async function captureCheckpoint(scope: string, session: ManagedAgentsSession): Promise<void> {
+ const stored = await store.get(scope);
+ const previous = stored?.checkpointId;
+ const captured = await session.createCheckpoint(CHECKPOINT_LABEL);
+ const pending = stored?.pendingCheckpointDeletes ?? [];
+ const queued =
+ previous && previous !== captured.checkpointId && !pending.includes(previous) ? [...pending, previous] : pending;
+ await store.merge(scope, {
+ checkpointId: captured.checkpointId,
+ checkpointAtMs: captured.createdAtMs ?? Date.now(),
+ checkpointError: undefined,
+ pendingCheckpointDeletes: queued,
+ });
+ if (previous && previous !== captured.checkpointId) await forgetCheckpoint(scope, session, previous);
+ }
+
+ const hydrateHome = (scope: string, session: ManagedAgentsSession): Promise<boolean> =>
+ homeSnapshots.hydrateHome(scope, session);
+
+ const recordSession = (scope: string, session: ManagedAgentsSession, createdAtMs: number): Promise<void> =>
+ store.put(scope, {
+ sessionId: session.sessionId,
+ sandboxId: session.sandboxId,
+ createdAtMs,
+ preservationState: "running",
+ });
+
+ async function ensureSession(
+ scope: string,
+ name: string,
+ onStatus?: (text: string) => void,
+ ): Promise<{ session: ManagedAgentsSession; coldStart: boolean }> {
+ return provisionQueue(scope, async () => {
+ const cached = sessionByName.get(name);
+ if (cached) return { session: cached, coldStart: false };
+
+ const adopt = (session: ManagedAgentsSession): { session: ManagedAgentsSession; coldStart: boolean } => {
+ sessionByName.set(name, session);
+ return { session, coldStart: false };
+ };
+
+ const stored = await store.get(scope);
+ if (stored) {
+ try {
+ onStatus?.("Resuming the sandbox…");
+ const session = await client.connect(stored.sessionId);
+ await store.merge(scope, { preservationState: "running", sandboxId: session.sandboxId });
+ return adopt(session);
+ } catch (err) {
+ if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;
+ }
+ }
+
+ for (const summary of await client.list(name)) {
+ try {
+ const session = await client.connect(summary.sessionId);
+ await recordSession(scope, session, Date.now());
+ return adopt(session);
+ } catch (err) {
+ if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;
+ }
+ }
+
+ onStatus?.("Creating the sandbox…");
+ const session = await client.create({ name });
+ sessionByName.set(name, session);
+ await recordSession(scope, session, Date.now());
+ let hydrated: boolean;
+ try {
+ hydrated = await hydrateHome(scope, session);
+ } catch (e) {
+ reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);
+ sessionByName.delete(name);
+ await client.kill(session.sessionId).catch((err: unknown) => {
+ reportError("sandbox_hydrate", "orphan_kill_failed", errMessage(err), scope);
+ });
+ throw new Error(
+ `do-managed-agents provision: home hydration failed (${errMessage(e)}); not risking the stored snapshot`,
+ {
+ cause: e,
+ },
+ );
+ }
+ return { session, coldStart: !hydrated };
+ });
+ }
+
+ async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> {
+ const name = sandboxScopeName(`${prefix}-scratch`, key);
+ return provisionQueue(`scratch:${key}`, async () => {
+ scratchKeyByName.set(name, key);
+ const active = activeScratch.get(name) ?? 0;
+ if (active === 0 && !sessionByName.has(name)) {
+ sessionByName.set(name, await client.create({ name }));
+ }
+ activeScratch.set(name, active + 1);
+ return { name, coldStart: active === 0 };
+ });
+ }
+
+ async function withSession<T>(name: string, action: (session: ManagedAgentsSession) => Promise<T>): Promise<T> {
+ const scratchKey = scratchKeyByName.get(name);
+ if (scratchKey === undefined && !scopeByName.has(name)) throw new Error("sandbox handle has been released");
+
+ const reviveScratch = async (): Promise<ManagedAgentsSession> => {
+ const session = await client.create({ name });
+ sessionByName.set(name, session);
+ return session;
+ };
+ const first =
+ scratchKey !== undefined
+ ? { session: sessionByName.get(name) ?? (await reviveScratch()) }
+ : await ensureSession(scopeByName.get(name) ?? "default", name);
+ try {
+ return await action(first.session);
+ } catch (err) {
+ if (err instanceof ManagedAgentsCommandLostError) sessionByName.delete(name);
+ if (!(err instanceof ManagedAgentsSandboxGoneError)) throw err;
+ sessionByName.delete(name);
+ const second =
+ scratchKey !== undefined
+ ? { session: await reviveScratch() }
+ : await ensureSession(scopeByName.get(name) ?? "default", name);
+ return action(second.session);
+ }
+ }
+
+ async function execRaw(name: string, script: string, timeoutSec: number): Promise<ExecResult> {
+ return withSession(name, async (session) => {
+ const r = await session.runCommand(`sh -c ${shq(`timeout -k 5 ${timeoutSec} sh -c ${shq(script)}; exit $?`)}`, {
+ timeoutMs: timeoutSec * 1000 + 30_000,
+ });
+ return {
+ stdout: r.stdout,
+ stderr: r.stderr,
+ code: r.exitCode,
+ timedOut: r.exitCode === 124 || r.exitCode === 137,
+ };
+ });
+ }
+
+ const profile: AgentComputerProfile = {
+ backend: "do-managed-agents",
+ writablePersistence: client.nativePause ? "provider_managed" : "snapshot_to_workspace",
+ processSessions: true,
+ parksOnTeardown: true,
+ egressEnforcement: opts.egressProxyUrl ? "domain" : "none",
+ spec: {
+ os: "Linux — DigitalOcean Managed Agents Firecracker microVM (provider pause preserves state; publish durable work to git or Files)",
+ runtimes: ["Node", "Python 3"],
+ get tools() {
+ return visibleTools(["git", "curl", "tar", "python3", ...(opts.extraTools ?? [])]);
+ },
+ get notInstalled() {
+ return visibleNotInstalled(
+ ["jq", "rg", "unzip", "wget", "gh", "aws", "gcloud", "kubectl", "flyctl", "glab"],
+ opts.extraTools ?? [],
+ );
+ },
+ homeDir: HOME_DIR,
+ workdir: workspaceDir,
+ },
+ };
+
+ const procIo: ExecProcessIo = {
+ async run(handle, command, execOpts): Promise<ExecResult> {
+ const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
+ return execRaw(handle.id, command, timeoutSec);
+ },
+ };
+ const procSessions = createExecProcessSessions(procIo);
+
+ const writeAbsBytes = (name: string, absPath: string, data: Uint8Array): Promise<void> =>
+ withSession(name, (session) => session.writeFileBytes(absPath, data));
+ const readAbsBytes = (name: string, absPath: string): Promise<Uint8Array | null> =>
+ withSession(name, (session) => session.readFileBytes(absPath));
+ const installLayerTools = opts.layerToolFiles ? createLayerToolInstaller(opts.layerToolFiles) : null;
+
+ const execFileOps = createExecFileOps({
+ label: "do-managed-agents",
+ exec: (id, script, t) => execRaw(id, script, t),
+ writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),
+ });
+
+ const execExport = createExecExport({
+ label: "do-managed-agents",
+ exec: (id, script, t) => execRaw(id, script, t),
+ readAbsBytes,
+ defaultHomeDir: HOME_DIR,
+ ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),
+ });
+
+ const blobStaging = createBackendBlobStaging("do-managed-agents", (id, script, t) => execRaw(id, script, t), opts);
+
+ async function destroyStoredScope(scope: string): Promise<void> {
+ const name = sandboxScopeName(prefix, scope);
+ const stored = await store.get(scope);
+ const cached = sessionByName.get(name);
+ const ids = new Set([stored?.sessionId, cached?.sessionId].filter((id): id is string => !!id));
+ for (const id of ids) {
+ try {
+ if (cached && id === cached.sessionId) await cached.kill();
+ else await client.kill(id);
+ } catch (error) {
+ if (!(error instanceof ManagedAgentsSandboxGoneError)) throw error;
+ }
+ }
+ await store.delete(scope);
+ sessionByName.delete(name);
+ scopeByName.delete(name);
+ }
+
+ const sandbox: Sandbox = {
+ destroyScope(scopeId: string): Promise<void> {
+ return provisionQueue(scopeId, () => destroyStoredScope(scopeId));
+ },
+
+ profile,
+ startProcess: procSessions.startProcess,
+ readProcess: procSessions.readProcess,
+ writeStdin: procSessions.writeStdin,
+ signalProcess: procSessions.signalProcess,
+ listProcesses: procSessions.listProcesses,
+ ...execFileOps,
+ ...blobStaging,
+
+ async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise<SandboxHandle> {
+ const scratch = provOpts?.scratch;
+ const writable = layers.find((l) => l.mode === "rw") ?? layers[0];
+ const scope = writable?.scopeId ?? "default";
+ let name: string;
+ let coldStart: boolean;
+ if (scratch) {
+ ({ name, coldStart } = await ensureScratch(scratch.key));
+ } else {
+ name = sandboxScopeName(prefix, scope);
+ scopeByName.set(name, scope);
+ ({ coldStart } = await ensureSession(scope, name, provOpts?.onStatus));
+ }
+
+ const forceEgress = !!opts.egressProxyUrl && !!provOpts?.egressToken;
+ const turnEnv = Object.fromEntries(
+ Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),
+ );
+ const env = {
+ ...turnEnv,
+ ...(forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!) : {}),
+ };
+ const handle: SandboxHandle = {
+ id: name,
+ rootDir: workspaceDir,
+ homeDir: HOME_DIR,
+ coldStart,
+ ...(scratch ? { scratch: true } : {}),
+ ...(Object.keys(env).length ? { env } : {}),
+ };
+
+ try {
+ const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(HOME_DIR, opts.credentialPaths ?? [])}`;
+ const prep = await execRaw(name, `mkdir -p ${shq(workspaceDir)}${credLinks}`, 60);
+ if (prep.code !== 0)
+ throw new Error(`do-managed-agents provision prep failed: ${(prep.stderr || prep.stdout).slice(0, 200)}`);
+
+ await materializeRoLayers(
+ workspace,
+ layers,
+ handle,
+ {
+ readFile: (h, rel) => sandbox.readFile(h, rel),
+ writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data),
+ exec: (script, t) => execRaw(name, script, t),
+ },
+ { manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "do-managed-agents" },
+ );
+ await installLayerTools?.({
+ exec: (script, t) => execRaw(name, script, t),
+ writeAbs: (abs, data) => writeAbsBytes(name, abs, data),
+ });
+
+ return handle;
+ } catch (err) {
+ await cleanupFailedProvision(sandbox, handle, err);
+ throw err;
+ }
+ },
+
+ async run(handle, command, execOpts?: ExecOptions): Promise<ExecResult> {
+ const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;
+ const exports = Object.entries(handle.env ?? {})
+ .map(([k, v]) => `export ${k}=${shq(v)}`)
+ .join("; ");
+ const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${handle.rootDir} 2>/dev/null; ${command}`;
+ const signal = execOpts?.signal;
+ if (!signal) return execRaw(handle.id, script, timeoutSec);
+ const killUid = randomUUID();
+ const fireKill = (): void => {
+ execRaw(handle.id, killScript(killUid), 15).catch(
+ swallowAs("managed-agents-sandbox: kill in-flight exec", undefined),
+ );
+ };
+ signal.throwIfAborted();
+ const onAbort = (): void => fireKill();
+ signal.addEventListener("abort", onAbort, { once: true });
+ try {
+ return await execRaw(handle.id, killableScript(script, killUid), timeoutSec);
+ } finally {
+ signal.removeEventListener("abort", onAbort);
+ }
+ },
+
+ async writeFileBytes(handle, relPath, data): Promise<void> {
+ await writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data);
+ },
+ async writeFile(handle, relPath, data): Promise<void> {
+ await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8"));
+ },
+ async readFileBytes(handle, relPath): Promise<Uint8Array | null> {
+ return readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath));
+ },
+ async readFile(handle, relPath): Promise<string | null> {
+ const bytes = await sandbox.readFileBytes(handle, relPath);
+ return bytes === null ? null : Buffer.from(bytes).toString("utf8");
+ },
+
+ exportFiles: execExport.exportFiles,
+
+ async persistHomeSnapshot(scopeId: string): Promise<void> {
+ const name = sandboxScopeName(prefix, scopeId);
+ scopeByName.set(name, scopeId);
+ const { session } = await ensureSession(scopeId, name);
+ await snapshotHome(scopeId, session);
+ },
+
+ async restartComputer(scopeId: string): Promise<void> {
+ const name = sandboxScopeName(prefix, scopeId);
+ scopeByName.set(name, scopeId);
+ return provisionQueue(scopeId, async () => {
+ const stored = await store.get(scopeId);
+ if (!stored?.sessionId) throw new Error("do-managed-agents restart: no sandbox provisioned yet");
+ const checkpointId = stored.checkpointId;
+ if (!checkpointId) throw new Error("do-managed-agents restart: no checkpoint to restore");
+ const cached = sessionByName.get(name);
+ sessionByName.delete(name);
+ if (cached) await cached.close();
+ let restored: ManagedAgentsSessionInfo;
+ try {
+ restored = await client.rollback(stored.sessionId, checkpointId);
+ } catch (e) {
+ reportError("agent_computer", "checkpoint_rollback_failed", errMessage(e), scopeId);
+ throw e;
+ }
+ await store.merge(scopeId, {
+ sandboxId: restored.sandboxId,
+ preservationState: "running",
+ preservationError: undefined,
+ });
+ reportError(
+ "agent_computer",
+ "checkpoint_restored",
+ `session ${stored.sessionId}: restored checkpoint ${checkpointId} onto sandbox ${restored.sandboxId}`,
+ scopeId,
+ );
+ });
+ },
+
+ async computerStatus(scopeId: string): Promise<ComputerStatus> {
+ const name = sandboxScopeName(prefix, scopeId);
+ const stored = await store.get(scopeId);
+
+ if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false };
+ const machine = `do-managed-agents sandbox ${stored.sandboxId || stored.sessionId}`;
+ const checkpointAtMs = stored.checkpointAtMs ?? stored.lastSnapshotMs;
+ const recoveryError = stored.preservationError ?? stored.checkpointError;
+ const recovery = {
+ strategy: "workspace_snapshot" as const,
+ state: stored.preservationState,
+ ...(recoveryError ? { error: recoveryError } : {}),
+ ...(stored.checkpointId ? { checkpointId: stored.checkpointId } : {}),
+ ...(checkpointAtMs ? { checkpointAtMs } : {}),
+ };
+ try {
+ const info: ManagedAgentsSessionInfo = await client.info(stored.sessionId);
+ if (info.state === "paused")
+ return {
+ machine,
+ listed: info.state,
+ lifecycleState: "paused",
+ provisioned: true,
+ guestResponsive: false,
+ recovery: { ...recovery, state: "paused" },
+ };
+ scopeByName.set(name, scopeId);
+ let session = sessionByName.get(name);
+ if (!session) {
+ session = await client.connect(stored.sessionId);
+ sessionByName.set(name, session);
+ }
+ const r = await session.runCommand("echo responsive", { timeoutMs: 30_000 });
+ return {
+ machine,
+ recovery,
+ provisioned: true,
+ guestResponsive: r.exitCode === 0 && r.stdout.trim() === "responsive",
+ };
+ } catch (e) {
+ return {
+ recovery,
+ machine: `${machine} (${errMessage(e).slice(0, 120)})`,
+ provisioned: !(e instanceof ManagedAgentsSandboxGoneError),
+ guestResponsive: false,
+ };
+ }
+ },
+
+ async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
+ if (handle.scratch) {
+ const key = scratchKeyByName.get(handle.id);
+ return provisionQueue(key ? `scratch:${key}` : handle.id, async () => {
+ const remaining = (activeScratch.get(handle.id) ?? 1) - 1;
+ if (remaining > 0) {
+ activeScratch.set(handle.id, remaining);
+ return;
+ }
+ activeScratch.delete(handle.id);
+ const session = sessionByName.get(handle.id);
+ if (session) {
+ if (tdOpts?.destroy) await session.kill();
+ else await session.kill().catch(swallowAs("managed-agents-sandbox: scratch kill", undefined));
+ }
+ sessionByName.delete(handle.id);
+ scratchKeyByName.delete(handle.id);
+ });
+ }
+ if (tdOpts?.destroy && !scopeByName.has(handle.id)) return;
+ const scope = scopeByName.get(handle.id) ?? "default";
+ return provisionQueue(scope, () => teardownScope(handle, scope, tdOpts));
+ },
+ };
+
+ async function teardownScope(handle: SandboxHandle, scope: string, tdOpts?: TeardownOptions): Promise<void> {
+ const session = sessionByName.get(handle.id);
+ if (tdOpts?.destroy) return destroyStoredScope(scope);
+ if (!session) return;
+
+ const stored = await store.get(scope);
+ if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });
+ await retryPendingCheckpointDeletes(scope, session);
+ const homeDirty = stored?.homeDirty;
+ if (snapshotDue({ lastSnapshotMs: stored?.checkpointAtMs, homeDirty }, tdOpts, snapshotIntervalMs)) {
+ try {
+ await captureCheckpoint(scope, session);
+ } catch (e) {
+ await store.merge(scope, { checkpointError: errMessage(e) });
+ reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), scope);
+ }
+ }
+ if (snapshotDue({ lastSnapshotMs: stored?.lastSnapshotMs, homeDirty }, tdOpts, snapshotIntervalMs)) {
+ try {
+ await snapshotHome(scope, session);
+ } catch (e) {
+ reportError("sandbox_snapshot", "teardown_snapshot_failed", errMessage(e), scope);
+ }
+ }
+ if (tdOpts?.keepWarm) return;
+ try {
+ await session.pause();
+ await store.merge(scope, { preservationState: "paused", preservationError: undefined });
+ sessionByName.delete(handle.id);
+ } catch (error) {
+ await store.merge(scope, { preservationState: "pause_failed", preservationError: errMessage(error) });
+ reportError("sandbox_preservation", "pause_failed", errMessage(error), scope);
+ throw error;
+ }
+ }
+
+ return sandbox;
+}
diff --git a/src/sandbox/managed-agents-tunnel.ts b/src/sandbox/managed-agents-tunnel.ts
new file mode 100644
index 000000000..a0557e730
--- /dev/null
+++ b/src/sandbox/managed-agents-tunnel.ts
@@ -0,0 +1,151 @@
+import { createServer, type Server, type Socket } from "node:net";
+import { once } from "node:events";
+import type WebSocket from "ws";
+
+export const SANDBOX_AGENT_PORT = 8443;
+
+const TUNNEL_CLOSE_SESSION_UNAVAILABLE = 4001;
+const TUNNEL_CLOSE_GUEST_DIAL_FAILED = 4002;
+const TUNNEL_CLOSE_UPSTREAM_ERROR = 4011;
+const REJECTION_BODY_MAX = 300;
+const PING_INTERVAL_MS = 30_000;
+
+export interface ManagedAgentsTunnel {
+ readonly localPort: number;
+ lastFailure(): string | null;
+ whenFailed(): Promise<string>;
+ close(): Promise<void>;
+}
+
+export interface ManagedAgentsTunnelOptions {
+ apiBaseUrl: string;
+ sessionId: string;
+ remotePort: number;
+ getToken: () => Promise<string>;
+ pingIntervalMs?: number;
+}
+
+export function tunnelUrl(apiBaseUrl: string, sessionId: string, remotePort: number): string {
+ const url = new URL(apiBaseUrl);
+ if (url.protocol !== "http:" && url.protocol !== "https:")
+ throw new Error(`DO_AGENTS_API_BASE_URL must be http(s), got ${url.protocol}`);
+ url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
+ url.pathname = `${url.pathname.replace(/\/$/, "")}/v2/agents/sessions/${sessionId}/port-forward/${remotePort}`;
+ return url.toString();
+}
+
+type WebSocketCtor = typeof WebSocket;
+
+let wsModule: WebSocketCtor | null = null;
+async function loadWebSocket(): Promise<WebSocketCtor> {
+ wsModule ??= (await import("ws")).default;
+ return wsModule;
+}
+
+export async function openManagedAgentsTunnel(opts: ManagedAgentsTunnelOptions): Promise<ManagedAgentsTunnel> {
+ const WebSocketImpl = await loadWebSocket();
+ const url = tunnelUrl(opts.apiBaseUrl, opts.sessionId, opts.remotePort);
+ const sockets = new Set<Socket>();
+ let failure: string | null = null;
+ let closed = false;
+ let announceFailure: (detail: string) => void = () => undefined;
+ const failed = new Promise<string>((resolve) => {
+ announceFailure = resolve;
+ });
+ const fail = (detail: string): void => {
+ failure = detail;
+ announceFailure(detail);
+ };
+
+ const bridge = (local: Socket): void => {
+ sockets.add(local);
+ local.once("close", () => sockets.delete(local));
+ void opts
+ .getToken()
+ .then((token) => {
+ if (closed) {
+ local.destroy();
+ return;
+ }
+ const ws = new WebSocketImpl(url, { headers: { Authorization: `Bearer ${token}` } });
+ let rejected = false;
+
+ ws.on("unexpected-response", (_req, res) => {
+ rejected = true;
+ let body = "";
+ res.on("data", (chunk: Buffer) => {
+ body += chunk.toString("utf8");
+ });
+ res.on("end", () => {
+ fail(`${res.statusCode ?? 0} ${body.trim().slice(0, REJECTION_BODY_MAX)}`.trim());
+ local.destroy();
+ });
+ });
+
+ ws.on("open", () => {
+ const heartbeat = setInterval(() => {
+ if (ws.readyState === WebSocketImpl.OPEN) ws.ping();
+ }, opts.pingIntervalMs ?? PING_INTERVAL_MS);
+ heartbeat.unref();
+ ws.once("close", () => clearInterval(heartbeat));
+ local.on("data", (chunk: Buffer) => {
+ local.pause();
+ ws.send(chunk, (err) => {
+ if (err) return;
+ local.resume();
+ });
+ });
+ ws.on("message", (data: Buffer) => {
+ if (!local.write(data)) {
+ ws.pause();
+ local.once("drain", () => ws.resume());
+ }
+ });
+ });
+
+ ws.on("close", (code: number, reason: Buffer) => {
+ if (
+ code === TUNNEL_CLOSE_SESSION_UNAVAILABLE ||
+ code === TUNNEL_CLOSE_GUEST_DIAL_FAILED ||
+ code === TUNNEL_CLOSE_UPSTREAM_ERROR
+ )
+ fail(`close ${code} ${reason.toString("utf8")}`.trim());
+ local.destroy();
+ });
+
+ ws.on("error", (err: Error) => {
+ if (!rejected) fail(err.message);
+ local.destroy();
+ });
+
+ local.on("close", () => {
+ if (ws.readyState === WebSocketImpl.OPEN || ws.readyState === WebSocketImpl.CONNECTING) ws.close();
+ });
+ local.on("error", () => ws.close());
+ })
+ .catch((err: unknown) => {
+ fail(err instanceof Error ? err.message : String(err));
+ local.destroy();
+ });
+ };
+
+ const server: Server = createServer(bridge);
+ server.listen(0, "127.0.0.1");
+ await once(server, "listening");
+ server.unref();
+ const address = server.address();
+ if (address === null || typeof address === "string")
+ throw new Error("Managed Agents tunnel failed to bind a local port");
+
+ return {
+ localPort: address.port,
+ lastFailure: () => failure,
+ whenFailed: () => failed,
+ close: async () => {
+ closed = true;
+ for (const socket of sockets) socket.destroy();
+ sockets.clear();
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ },
+ };
+}
diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.ts
index bca3a9757..8068eeb7a 100644
--- a/src/sandbox/sandbox-routing.ts
+++ b/src/sandbox/sandbox-routing.ts
@@ -17,7 +17,16 @@ import {
} from "./sandbox.ts";
export type SandboxBackendName =
- "sprites" | "aws" | "local" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";
+ | "sprites"
+ | "aws"
+ | "local"
+ | "smolmachines"
+ | "e2b"
+ | "modal"
+ | "porter"
+ | "agent37"
+ | "superserve"
+ | "do-managed-agents";
export class NoDefaultSandboxError extends Error {
constructor() {
diff --git a/src/wiring.ts b/src/wiring.ts
index 063032ed1..62280b4ff 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -231,6 +231,12 @@ import {
import { createSdkSuperserveClient } from "./sandbox/superserve-client.ts";
import { createE2bSandbox, type StoredE2bSandbox } from "./sandbox/e2b-sandbox.ts";
import { createSdkE2bClient } from "./sandbox/e2b-client.ts";
+import {
+ createManagedAgentsSandbox,
+ QM_TEMPLATE_TOOLS,
+ type StoredManagedAgentsSandbox,
+} from "./sandbox/managed-agents-sandbox.ts";
+import { createSdkManagedAgentsClient } from "./sandbox/managed-agents-client.ts";
import { createS3SnapshotStore } from "./sandbox/home-snapshot.ts";
import { createModalSandbox, type StoredModalSandbox } from "./sandbox/modal-sandbox.ts";
import { createSdkModalClient } from "./sandbox/modal-client.ts";
@@ -921,6 +927,41 @@ export function buildApp(
onError: sandboxOnError,
});
};
+ const managedAgentsBodies = artifactMap<StoredManagedAgentsSandbox>("do_managed_agents_sandbox_bodies");
+ const buildManagedAgents = (): Sandbox => {
+ const managedAgents = config.managedAgentsSandbox;
+ if (!managedAgents.apiToken) throw new Error("SANDBOX_BACKEND=do-managed-agents requires DO_AGENTS_API_TOKEN");
+ return createManagedAgentsSandbox(workspace, {
+ client: createSdkManagedAgentsClient({
+ apiToken: managedAgents.apiToken,
+ ...(managedAgents.apiBaseUrl ? { apiBaseUrl: managedAgents.apiBaseUrl } : {}),
+ ...(managedAgents.template ? { template: managedAgents.template } : {}),
+ ...(managedAgents.sizeSlug ? { sizeSlug: managedAgents.sizeSlug } : {}),
+ ...(managedAgents.idleTimeoutSec ? { idleTimeoutSec: managedAgents.idleTimeoutSec } : {}),
+ ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}),
+ }),
+ ...(managedAgents.namePrefix ? { namePrefix: managedAgents.namePrefix } : {}),
+ ...(managedAgents.defaultTimeoutSec ? { defaultTimeoutSec: managedAgents.defaultTimeoutSec } : {}),
+ ...(managedAgents.snapshotIntervalSec !== undefined
+ ? { snapshotIntervalMs: managedAgents.snapshotIntervalSec * 1000 }
+ : {}),
+ ...(managedAgents.egressProxyUrl ? { egressProxyUrl: managedAgents.egressProxyUrl } : {}),
+ extraTools: [...(managedAgents.template ? QM_TEMPLATE_TOOLS : []), ...deploymentLayer.advertisedTools],
+ credentialPaths: deploymentLayer.credentialPaths,
+ layerToolFiles: () => deploymentLayer.installFiles,
+ blobTransfer,
+ ...(config.signingSecret ? { signingSecret: config.signingSecret } : {}),
+ ...(config.capabilitySecret ? { capabilitySecret: config.capabilitySecret } : {}),
+ ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}),
+ store: managedAgentsBodies,
+ ...(managedAgents.snapshotS3Bucket
+ ? {
+ snapshots: createS3SnapshotStore({ bucket: managedAgents.snapshotS3Bucket, prefix: "managed-agents-home" }),
+ }
+ : {}),
+ onError: sandboxOnError,
+ });
+ };
const buildModal = (): Sandbox => {
const modal = config.modalSandbox;
if (!modal.tokenId || !modal.tokenSecret)
@@ -1070,6 +1111,7 @@ export function buildApp(
porter: buildPorter,
agent37: buildAgent37,
superserve: buildSuperserve,
+ "do-managed-agents": buildManagedAgents,
};
const enabledBackends = new Set(enabledSandboxBackends(config));
const sandboxBackends: Partial<Record<SandboxBackendName, Sandbox>> = {
diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.ts
index ea5634a72..9bf5081d0 100644
--- a/test/live-slack-sandbox-providers.test.ts
+++ b/test/live-slack-sandbox-providers.test.ts
@@ -64,6 +64,7 @@ test("every provider has its own parallel execution scenario", () => {
assert.deepEqual(sandboxProviders.toSorted(), [
"agent37",
"aws",
+ "do-managed-agents",
"e2b",
"local",
"modal",
@@ -159,7 +160,7 @@ test("explicit provider selection retains strict coverage without treating defer
"sandbox-execute-modal",
],
);
- assert.equal(selectSandboxProviderScenarios("all").length, 9);
+ assert.equal(selectSandboxProviderScenarios("all").length, 10);
assert.deepEqual(selectSandboxProviderScenarios(undefined), []);
for (const value of ["", "sprites,", "typo", "sprites,sprites", "all,sprites", "constructor", "__proto__"])
assert.throws(() => selectSandboxProviderScenarios(value));
diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.ts
index a8675f071..8e7183c90 100644
--- a/test/live-slack/scenarios-sandbox-providers.ts
+++ b/test/live-slack/scenarios-sandbox-providers.ts
@@ -14,6 +14,7 @@ const providerCoverage: Record<SandboxBackendName, true> = {
porter: true,
agent37: true,
superserve: true,
+ "do-managed-agents": true,
};
export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[];
diff --git a/test/managed-agents-client.test.ts b/test/managed-agents-client.test.ts
new file mode 100644
index 000000000..d8cfa4748
--- /dev/null
+++ b/test/managed-agents-client.test.ts
@@ -0,0 +1,301 @@
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { connect } from "node:net";
+import { once } from "node:events";
+import { join } from "node:path";
+import {
+ createSdkManagedAgentsClient,
+ managedAgentsEgressAllow,
+ managedAgentsManifest,
+ managedAgentsSessionState,
+ ManagedAgentsSandboxGoneError,
+ ManagedAgentsCommandLostError,
+ type ManagedAgentsClient,
+ type ManagedAgentsSession,
+} from "../src/sandbox/managed-agents-client.ts";
+import { tunnelUrl, openManagedAgentsTunnel, SANDBOX_AGENT_PORT } from "../src/sandbox/managed-agents-tunnel.ts";
+import { startFakeManagedAgentsService, type FakeManagedAgentsService } from "./support/fake-managed-agents-service.ts";
+
+let service: FakeManagedAgentsService;
+let client: ManagedAgentsClient;
+const opened: ManagedAgentsSession[] = [];
+
+const session = async (name: string): Promise<ManagedAgentsSession> => {
+ const created = await client.create({ name });
+ opened.push(created);
+ return created;
+};
+
+before(async () => {
+ service = await startFakeManagedAgentsService();
+ client = createSdkManagedAgentsClient({
+ apiToken: service.token,
+ apiBaseUrl: service.apiBaseUrl,
+ template: "base",
+ });
+});
+
+after(async () => {
+ await Promise.all(opened.map((s) => s.kill().catch(() => undefined)));
+ await service?.close();
+});
+
+test("the port-forward url targets sandbox-agent over wss", () => {
+ assert.equal(
+ tunnelUrl("https://api.digitalocean.com", "sess-1", SANDBOX_AGENT_PORT),
+ "wss://api.digitalocean.com/v2/agents/sessions/sess-1/port-forward/8443",
+ );
+ assert.equal(
+ tunnelUrl("http://127.0.0.1:9000/edge", "sess-2", SANDBOX_AGENT_PORT),
+ "ws://127.0.0.1:9000/edge/v2/agents/sessions/sess-2/port-forward/8443",
+ );
+ assert.throws(() => tunnelUrl("ftp://example.com", "s", 8443), /must be http/);
+});
+
+test("the manifest asks for a bare sandbox and pre-allows bash so Managed Agents never gates a command", () => {
+ const yaml = managedAgentsManifest({ name: "qm-personal-tester", template: "base" });
+ assert.match(yaml, /^name: "qm-personal-tester"$/m);
+ assert.match(yaml, /^agent: "none"$/m);
+ assert.match(yaml, /^template: "base"$/m);
+ assert.match(yaml, /tool: bash/);
+ assert.match(yaml, /action: allow/);
+ assert.doesNotMatch(yaml, /persistent_workspace/, "Managed Agents deprecated the key and warns when it is sent");
+});
+
+test("the manifest omits the template so Managed Agents lands a bare session on its own base", () => {
+ const yaml = managedAgentsManifest({ name: "n", template: "" });
+ assert.match(yaml, /^agent: "none"$/m);
+ assert.doesNotMatch(yaml, /^template:/m);
+});
+
+test("the manifest carries size, idle timeout and the egress allowlist only when configured", () => {
+ const bare = managedAgentsManifest({ name: "n", template: "base" });
+ assert.doesNotMatch(bare, /size:|idle_timeout:|egress:/);
+ const full = managedAgentsManifest({
+ name: "n",
+ template: "base",
+ sizeSlug: "mv-2vcpu-4gb",
+ idleTimeoutSec: 600,
+ egressAllow: ["proxy.example.com"],
+ });
+ assert.match(full, /^size: "mv-2vcpu-4gb"$/m);
+ assert.match(full, /^idle_timeout: "600s"$/m);
+ assert.match(full, /^ {2}- "proxy\.example\.com"$/m);
+});
+
+test("the vendored proto is the client subset and carries nothing provider-internal", () => {
+ const proto = readFileSync(join(process.cwd(), "src/sandbox/managed-agents-sandbox-agent.proto"), "utf8");
+ assert.match(proto, /service SandboxAgentService/);
+ for (const rpc of ["Exec", "Upload", "Download"]) assert.match(proto, new RegExp(`rpc ${rpc}\\(`));
+ for (const unused of ["Shell", "ImportFromURL", "ExportToURL", "Health", "StartChildProcess", "ProxyPort"])
+ assert.doesNotMatch(proto, new RegExp(`rpc ${unused}\\(`), `${unused} is unused and should not be vendored`);
+ for (const internal of [/do\/doge/, /dorpc/, /sandboxsvc/, /sandbox-service/, /microvm\.v1/, /MSANDBOX/, /169\.254/])
+ assert.doesNotMatch(proto, internal, `${internal.source} is provider-internal`);
+});
+
+test("creating a session posts a yaml manifest and waits for ready", async () => {
+ const s = await session("qm-create");
+ assert.ok(s.sessionId);
+ assert.equal(service.sessions().length, 1);
+ assert.match(service.manifests()[0] ?? "", /template: "base"/);
+ assert.equal(service.sessions()[0]?.template, "base");
+});
+
+test("a command runs in the guest over the port-forward tunnel", async () => {
+ const s = await session("qm-exec");
+ const result = await s.runCommand("echo hello-from-guest; echo oops >&2; exit 3");
+ assert.equal(result.exitCode, 3);
+ assert.match(result.stdout, /hello-from-guest/);
+ assert.match(result.stderr, /oops/);
+ assert.ok(service.tunnelCount() > 0, "exec must travel through the tunnel, not the REST exec endpoint");
+});
+
+test("a port-forward the edge refuses reads as gone, carrying the rejection, not as a half-run command", async () => {
+ const s = await session("qm-no-tunnel");
+ service.rejectPortForward("team is not entitled to port-forward");
+ const err = await s.runCommand("echo never-runs").then(
+ () => null,
+ (e: unknown) => e,
+ );
+ assert.ok(err instanceof ManagedAgentsSandboxGoneError, `expected a retryable gone error, got ${String(err)}`);
+ assert.match(err.message, /403/);
+ assert.match(err.message, /not entitled to port-forward/);
+ assert.doesNotMatch(err.message, /may have partially executed/);
+ service.rejectPortForward(null);
+});
+
+test("a transient lifecycle failure while waiting for readiness is polled through, not surfaced", async () => {
+ const created = await session("qm-transient-ready");
+ service.failNextLifecycleCalls(2, 403);
+ const reconnected = await client.connect(created.sessionId);
+ opened.push(reconnected);
+ assert.equal(reconnected.sessionId, created.sessionId);
+});
+
+test("a lifecycle failure that never clears is surfaced instead of polled to the deadline", async () => {
+ const created = await session("qm-stuck-ready");
+ service.failNextLifecycleCalls(50, 403);
+ const err = await client.connect(created.sessionId).then(
+ () => null,
+ (e: unknown) => e,
+ );
+ service.failNextLifecycleCalls(0, 403);
+ assert.ok(err instanceof Error, "expected the persistent failure to surface");
+ assert.match(err.message, /403/);
+ assert.doesNotMatch(err.message, /did not become ready/);
+});
+
+test("command env and working directory reach the guest", async () => {
+ const s = await session("qm-env");
+ const result = await s.runCommand("echo VAR=$MY_VAR", { env: { MY_VAR: "set-by-qm" } });
+ assert.match(result.stdout, /VAR=set-by-qm/);
+});
+
+test("files round trip through the guest upload and download streams", async () => {
+ const s = await session("qm-files");
+ const body = Buffer.from("managed agents file payload");
+ await s.writeFileBytes("/home/user/notes/a.txt", new Uint8Array(body));
+ const read = await s.readFileBytes("/home/user/notes/a.txt");
+ assert.ok(read);
+ assert.equal(Buffer.from(read).toString("utf8"), "managed agents file payload");
+});
+
+test("a missing file reads as null rather than an error", async () => {
+ const s = await session("qm-missing");
+ assert.equal(await s.readFileBytes("/home/user/nope.txt"), null);
+});
+
+test("resuming a paused session does not wait out a fixed poll interval", async () => {
+ const created = await session("qm-resume-latency");
+ service.setStatus("qm-resume-latency", "SESSION_STATUS_PAUSED");
+ const started = Date.now();
+ const back = await client.connect(created.sessionId);
+ opened.push(back);
+ assert.equal(back.sessionId, created.sessionId);
+ assert.ok(Date.now() - started < 1_000, `resume took ${Date.now() - started}ms`);
+});
+
+test("a session paused mid-command fails fast instead of stalling until the command timeout", async () => {
+ const watchful = createSdkManagedAgentsClient({
+ apiToken: service.token,
+ apiBaseUrl: service.apiBaseUrl,
+ stallProbeMs: 50,
+ });
+ const s = await watchful.create({ name: "qm-paused-midcommand" });
+ opened.push(s);
+ const started = Date.now();
+ const running = s.runCommand("sleep 30", { timeoutMs: 120_000 });
+ await new Promise((resolve) => setTimeout(resolve, 150));
+ service.setStatus("qm-paused-midcommand", "SESSION_STATUS_PAUSED");
+ const err = await running.then(
+ () => null,
+ (e: unknown) => e,
+ );
+ assert.ok(err instanceof ManagedAgentsCommandLostError, `expected a command-lost error, got ${String(err)}`);
+ assert.ok(Date.now() - started < 10_000, `expected a fast failure, took ${Date.now() - started}ms`);
+});
+
+test("the tunnel heartbeats so an edge idle timeout cannot kill a long silent command", async () => {
+ const created = await session("qm-heartbeat");
+ const before = service.pingCount();
+ const tunnel = await openManagedAgentsTunnel({
+ apiBaseUrl: service.apiBaseUrl,
+ sessionId: created.sessionId,
+ remotePort: SANDBOX_AGENT_PORT,
+ getToken: async () => service.token,
+ pingIntervalMs: 20,
+ });
+ try {
+ const probe = connect(tunnel.localPort, "127.0.0.1");
+ await once(probe, "connect");
+ await new Promise((resolve) => setTimeout(resolve, 400));
+ assert.ok(service.pingCount() > before, `expected heartbeat pings, saw ${service.pingCount() - before}`);
+ probe.destroy();
+ } finally {
+ await tunnel.close();
+ }
+});
+
+test("a large file survives the chunked upload path", async () => {
+ const s = await session("qm-large");
+ const body = Buffer.alloc(900 * 1024, "x");
+ await s.writeFileBytes("/home/user/big.bin", new Uint8Array(body));
+ const read = await s.readFileBytes("/home/user/big.bin");
+ assert.equal(read?.length, body.length);
+});
+
+test("list resolves a session by name and skips destroyed rows", async () => {
+ const s = await session("qm-list");
+ const found = await client.list("qm-list");
+ assert.equal(found.length, 1);
+ assert.equal(found[0]?.sessionId, s.sessionId);
+ await s.kill();
+ assert.deepEqual(await client.list("qm-list"), []);
+});
+
+test("create checkpoint posts to the session checkpoint route and returns the ready id", async () => {
+ const s = await session("qm-checkpoint");
+ const captured = await s.createCheckpoint("qm teardown");
+ assert.match(captured.checkpointId, /^cp_/);
+ assert.equal(captured.status, "READY");
+ assert.ok(captured.createdAtMs);
+ const stored = service.checkpoints(s.sessionId);
+ assert.equal(stored.length, 1);
+ assert.equal(stored[0]?.checkpoint_id, captured.checkpointId);
+ assert.equal(stored[0]?.label, "qm teardown");
+ assert.equal(stored[0]?.status, "READY");
+ await s.deleteCheckpoint(captured.checkpointId);
+ assert.equal(service.checkpoints(s.sessionId).length, 0);
+});
+
+test("rollback rewinds the same session onto a new sandbox without resuming it first", async () => {
+ const s = await session("qm-rollback");
+ const captured = await s.createCheckpoint("qm teardown");
+ const previousSandbox = s.sandboxId;
+ await s.pause();
+ assert.equal((await client.info(s.sessionId)).state, "paused");
+ const restored = await client.rollback(s.sessionId, captured.checkpointId);
+ assert.equal(restored.sessionId, s.sessionId);
+ assert.notEqual(restored.sandboxId, previousSandbox);
+ assert.equal(restored.state, "ready");
+ assert.equal((await client.info(s.sessionId)).sandboxId, restored.sandboxId);
+ assert.equal(service.checkpoints(s.sessionId).length, 1);
+ await assert.rejects(() => client.rollback(s.sessionId, "missing"), /404|no such checkpoint/);
+});
+
+test("connect resumes a paused session before handing it back", async () => {
+ const s = await session("qm-paused");
+ await s.pause();
+ assert.equal((await client.info(s.sessionId)).state, "paused");
+ const again = await client.connect(s.sessionId);
+ assert.equal((await client.info(again.sessionId)).state, "ready");
+ const result = await again.runCommand("echo awake");
+ assert.match(result.stdout, /awake/);
+});
+
+test("a destroyed session reports a terminal state, and reconnecting to it fails fast", async () => {
+ const s = await session("qm-gone");
+ await client.kill(s.sessionId);
+ assert.equal((await client.info(s.sessionId)).state, "destroyed");
+ await assert.rejects(() => client.connect(s.sessionId), ManagedAgentsSandboxGoneError);
+});
+
+test("a session the control plane has never heard of reads as gone", async () => {
+ await assert.rejects(() => client.info("11111111-2222-3333-4444-555555555555"), ManagedAgentsSandboxGoneError);
+});
+
+test("a bad token is refused by the control plane", async () => {
+ const wrong = createSdkManagedAgentsClient({ apiToken: "nope", apiBaseUrl: service.apiBaseUrl, template: "base" });
+ await assert.rejects(() => wrong.create({ name: "qm-unauth" }), /401|bad token/);
+});
+
+test("session status wire values map onto backend states", () => {
+ assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready");
+ assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused");
+ assert.equal(managedAgentsSessionState("nonsense"), "unspecified");
+});
+
+test("the egress allowlist carries just the proxy host", () => {
+ assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]);
+});
diff --git a/test/managed-agents-sandbox.test.ts b/test/managed-agents-sandbox.test.ts
new file mode 100644
index 000000000..d651e7474
--- /dev/null
+++ b/test/managed-agents-sandbox.test.ts
@@ -0,0 +1,397 @@
+import { test, after, beforeEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { pollProcess } from "../src/sandbox/process-poll.ts";
+import {
+ createManagedAgentsSandbox,
+ QM_TEMPLATE_TOOLS,
+ type StoredManagedAgentsSandbox,
+} from "../src/sandbox/managed-agents-sandbox.ts";
+import {
+ managedAgentsEgressAllow,
+ managedAgentsSessionState,
+ ManagedAgentsHitlRejectedError,
+} from "../src/sandbox/managed-agents-client.ts";
+import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts";
+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";
+import type { HomeSnapshotStore } from "../src/sandbox/home-snapshot.ts";
+import { supportsProcessSessions } from "../src/sandbox/sandbox.ts";
+import { createMemoryMap, type DurableMap } from "../src/persistence/durable-map.ts";
+import { scopeId } from "../src/types.ts";
+import { installFakeManagedAgents, type FakeManagedAgents } from "./support/fake-managed-agents.ts";
+import type { Sandbox } from "../src/sandbox/sandbox.ts";
+
+let fake: FakeManagedAgents;
+let sandbox: Sandbox;
+const scope = scopeId("personal", "tester");
+const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }];
+const scopeName = (): string => sandboxScopeName("qmt", scope);
+
+function make(extra: Record<string, unknown> = {}): Sandbox {
+ return createManagedAgentsSandbox(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "managed-agents-ws-"))), {
+ client: fake.client,
+ namePrefix: "qmt",
+ ...extra,
+ });
+}
+
+beforeEach(() => {
+ fake = installFakeManagedAgents();
+ sandbox = make();
+});
+
+after(() => fake?.cleanup());
+
+test("provision runs commands with env and cwd", async () => {
+ const h = await sandbox.provision(layers, { env: { MY_VAR: "v1" } });
+ assert.equal(h.coldStart, true);
+ const r = await sandbox.run(h, "pwd; echo VAR=$MY_VAR");
+ assert.equal(r.code, 0);
+ assert.match(r.stdout, /workspace/);
+ assert.match(r.stdout, /VAR=v1/);
+});
+
+test("one session per scope is reused across provisions", async () => {
+ const first = await sandbox.provision(layers);
+ await sandbox.teardown(first, { keepWarm: true });
+ const second = await sandbox.provision(layers);
+ assert.equal(second.coldStart, false);
+ assert.equal(fake.createdCount(scopeName()), 1);
+});
+
+test("the session is named after the scope so an operator can find it", async () => {
+ await sandbox.provision(layers);
+ const current = fake.current(scopeName());
+ assert.ok(current, "expected a session registered under the scope name");
+ assert.ok(scopeName().length <= 64, "Managed Agents caps session names at 64 characters");
+ assert.doesNotMatch(scopeName(), /^[0-9a-f]{8}-[0-9a-f]{4}-/, "Managed Agents rejects UUID-shaped session names");
+});
+
+test("teardown captures a provider checkpoint before pausing", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store });
+ const h = await sandbox.provision(layers);
+ await sandbox.run(h, "echo hi > kept.txt");
+ await sandbox.teardown(h);
+ const calls = fake.checkpointCalls();
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0]?.label, "qm teardown");
+ assert.equal(calls[0]?.sessionId, fake.current(scopeName())?.sessionId);
+ const stored = await store.get(scope);
+ assert.equal(stored?.checkpointId, calls[0]?.checkpointId);
+ const status = await sandbox.computerStatus!(scope);
+ assert.equal(status.recovery?.checkpointId, stored?.checkpointId);
+ assert.equal(status.lifecycleState, "paused");
+ assert.equal(fake.pauseCalls().length, 1);
+});
+
+test("restart rolls a paused session back to its checkpoint without resuming the old microVM", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ const restored: string[] = [];
+ sandbox = make({
+ store,
+ onError: (e: { code: string }) => {
+ if (e.code === "checkpoint_restored") restored.push(e.code);
+ },
+ });
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h);
+ const before = await store.get(scope);
+ assert.ok(before?.checkpointId);
+ assert.equal(before.preservationState, "paused");
+ const resumes = fake.resumeCalls().length;
+ const disconnects = fake.guestDisconnects().length;
+ await sandbox.restartComputer!(scope);
+ const after = await store.get(scope);
+ assert.equal(after?.sessionId, before.sessionId);
+ assert.notEqual(after?.sandboxId, before.sandboxId);
+ assert.equal(after?.preservationState, "running");
+ assert.equal(after?.checkpointId, before.checkpointId);
+ assert.deepEqual(fake.rollbackCalls(), [{ sessionId: before.sessionId, checkpointId: before.checkpointId }]);
+ assert.equal(fake.resumeCalls().length, resumes);
+ assert.equal(fake.guestDisconnects().length, disconnects);
+ assert.equal(fake.createdCount(scopeName()), 1);
+ assert.deepEqual(restored, ["checkpoint_restored"]);
+ const again = await sandbox.provision(layers);
+ assert.equal(again.coldStart, false);
+ assert.equal(fake.current(scopeName())?.sandboxId, after?.sandboxId);
+});
+
+test("restart drops a warm guest's port-forward before rolling back", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store });
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h, { keepWarm: true });
+ const disconnects = fake.guestDisconnects().length;
+ await sandbox.restartComputer!(scope);
+ assert.equal(fake.guestDisconnects().length, disconnects + 1);
+ assert.equal(fake.rollbackCalls().length, 1);
+ assert.equal((await store.get(scope))?.preservationState, "running");
+});
+
+test("restart without a checkpoint does not call rollback", async () => {
+ await sandbox.provision(layers);
+ await assert.rejects(() => sandbox.restartComputer!(scope), /no checkpoint to restore/);
+ assert.equal(fake.rollbackCalls().length, 0);
+});
+
+test("a failed rollback keeps the stored sandbox and the same session", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store });
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h);
+ const before = await store.get(scope);
+ fake.failNextRollback(scopeName());
+ await assert.rejects(() => sandbox.restartComputer!(scope), /rollback/);
+ assert.equal((await store.get(scope))?.sandboxId, before?.sandboxId);
+ assert.equal(fake.rollbackCalls().length, 0);
+ const again = await sandbox.provision(layers);
+ assert.equal(again.coldStart, false);
+ assert.equal(fake.createdCount(scopeName()), 1);
+});
+
+test("a checkpoint failure is recorded and the session still pauses", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store });
+ const h = await sandbox.provision(layers);
+ fake.failNextCheckpoint(scopeName());
+ await sandbox.teardown(h);
+ assert.equal(fake.checkpointCalls().length, 0);
+ assert.equal(fake.pauseCalls().length, 1);
+ assert.equal(fake.current(scopeName())?.state, "paused");
+ const status = await sandbox.computerStatus!(scope);
+ assert.match(status.recovery?.error ?? "", /create checkpoint/);
+});
+
+test("checkpoint capture is throttled across a pause and resume", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h);
+ const again = await sandbox.provision(layers);
+ await sandbox.teardown(again);
+ assert.equal(fake.checkpointCalls().length, 1);
+});
+
+test("a newer checkpoint replaces the previous one", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store, snapshotIntervalMs: -1 });
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h);
+ const first = (await store.get(scope))?.checkpointId;
+ const again = await sandbox.provision(layers);
+ await sandbox.teardown(again);
+ const second = (await store.get(scope))?.checkpointId;
+ assert.ok(first);
+ assert.ok(second);
+ assert.notEqual(first, second);
+ assert.deepEqual(fake.deletedCheckpoints(), [first]);
+ assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []);
+});
+
+test("a failed checkpoint delete is remembered and retried without another capture", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ const fast = make({ store, snapshotIntervalMs: -1 });
+ const h = await fast.provision(layers);
+ await fast.teardown(h);
+ const first = (await store.get(scope))?.checkpointId;
+ assert.ok(first);
+ fake.failNextCheckpointDelete(scopeName());
+ const again = await fast.provision(layers);
+ await fast.teardown(again);
+ assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes, [first]);
+ assert.deepEqual(fake.deletedCheckpoints(), []);
+
+ const slow = make({ store });
+ const resumed = await slow.provision(layers);
+ const captures = fake.checkpointCalls().length;
+ await slow.teardown(resumed, { homeUnchanged: true });
+ assert.equal(fake.checkpointCalls().length, captures);
+ assert.deepEqual(fake.deletedCheckpoints(), [first]);
+ assert.deepEqual((await store.get(scope))?.pendingCheckpointDeletes ?? [], []);
+});
+
+test("a failed home snapshot retries without recapturing the checkpoint", async () => {
+ const uploads: number[] = [];
+ const snapshots: HomeSnapshotStore = {
+ open: async () => null,
+ put: async () => undefined,
+ createUpload: async () => {
+ uploads.push(1);
+ throw new Error("snapshot upload failed");
+ },
+ };
+ sandbox = make({ snapshots });
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h);
+ const again = await sandbox.provision(layers);
+ await sandbox.teardown(again);
+ assert.equal(fake.checkpointCalls().length, 1);
+ assert.equal(uploads.length, 2);
+});
+
+test("teardown pauses the session and the next provision resumes it", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.run(h, "echo hi > kept.txt");
+ await sandbox.teardown(h);
+ assert.deepEqual(fake.pauseCalls().length, 1);
+ assert.equal(fake.current(scopeName())?.state, "paused");
+
+ const again = await sandbox.provision(layers);
+ assert.equal(fake.resumeCalls().length, 1);
+ const r = await sandbox.run(again, "cat kept.txt");
+ assert.match(r.stdout, /hi/);
+ assert.equal(fake.createdCount(scopeName()), 1);
+});
+
+test("keepWarm teardown leaves the session running for background work", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h, { keepWarm: true });
+ assert.equal(fake.pauseCalls().length, 0);
+ assert.equal(fake.current(scopeName())?.state, "ready");
+});
+
+test("a lost session is re-adopted by name when the durable record is gone", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store });
+ await sandbox.provision(layers);
+ const before = fake.current(scopeName());
+ await store.delete(scope);
+
+ const again = await sandbox.provision(layers);
+ assert.equal(again.coldStart, false);
+ assert.equal(fake.createdCount(scopeName()), 1);
+ assert.equal(fake.current(scopeName())?.sessionId, before?.sessionId);
+});
+
+test("a destroyed session is replaced on the next provision", async () => {
+ await sandbox.provision(layers);
+ fake.destroy(scopeName());
+ const again = await sandbox.provision(layers);
+ assert.ok(again.id);
+ assert.equal(fake.createdCount(scopeName()), 2);
+});
+
+test("destroying a scope disconnects the live guest rather than only deleting the session", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.run(h, "true");
+ const live = fake.current(scopeName());
+ assert.ok(live, "expected a live session before destroying the scope");
+ await sandbox.destroyScope!(scope);
+ assert.deepEqual(fake.guestDisconnects(), [live.sessionId]);
+ assert.equal(fake.current(scopeName()), null);
+});
+
+test("a command lost mid-flight is surfaced rather than silently retried", async () => {
+ const h = await sandbox.provision(layers);
+ fake.loseNextCommand(scopeName());
+ await assert.rejects(() => sandbox.run(h, "echo nope"), /was lost while a command was running/);
+});
+
+test("a Managed Agents approval rejection names the permissions fix", async () => {
+ const h = await sandbox.provision(layers);
+ fake.rejectNextCommand(scopeName());
+ await assert.rejects(() => sandbox.run(h, "echo nope"), ManagedAgentsHitlRejectedError);
+ await assert.rejects(() => {
+ fake.rejectNextCommand(scopeName());
+ return sandbox.run(h, "echo nope");
+ }, /permissions to allow bash/);
+});
+
+test("files round trip through the data plane", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.writeFile(h, "notes/a.txt", "hello managed agents");
+ assert.equal(await sandbox.readFile(h, "notes/a.txt"), "hello managed agents");
+ assert.equal(await sandbox.readFile(h, "notes/missing.txt"), null);
+});
+
+test("destroy removes the session and its durable record", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store });
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h, { destroy: true });
+ assert.equal(await store.get(scope), null);
+ assert.equal(fake.current(scopeName()), null);
+});
+
+test("computerStatus reports a paused machine without waking it", async () => {
+ const h = await sandbox.provision(layers);
+ await sandbox.teardown(h);
+ const status = await sandbox.computerStatus!(scope);
+ assert.equal(status.provisioned, true);
+ assert.equal(status.guestResponsive, false);
+ assert.equal(status.lifecycleState, "paused");
+ assert.equal(fake.resumeCalls().length, 0);
+});
+
+test("computerStatus probes a running machine", async () => {
+ await sandbox.provision(layers);
+ const status = await sandbox.computerStatus!(scope);
+ assert.equal(status.provisioned, true);
+ assert.equal(status.guestResponsive, true);
+});
+
+test("computerStatus on an unprovisioned scope reports nothing provisioned", async () => {
+ const status = await sandbox.computerStatus!(scope);
+ assert.equal(status.provisioned, false);
+ assert.equal(status.guestResponsive, false);
+});
+
+test("scratch sandboxes are reference counted and never persisted", async () => {
+ const store: DurableMap<StoredManagedAgentsSandbox> = createMemoryMap<StoredManagedAgentsSandbox>();
+ sandbox = make({ store });
+ const a = await sandbox.provision(layers, { scratch: { key: "k1" } });
+ const b = await sandbox.provision(layers, { scratch: { key: "k1" } });
+ assert.equal(a.id, b.id);
+ assert.equal(a.scratch, true);
+ await sandbox.teardown(a);
+ assert.ok(fake.current(a.id), "still held by the second handle");
+ await sandbox.teardown(b);
+ assert.equal(fake.current(a.id), null);
+ assert.equal(await store.get(scope), null);
+});
+
+test("the profile advertises provider-managed persistence and parks on teardown", () => {
+ assert.equal(sandbox.profile.backend, "do-managed-agents");
+ assert.equal(sandbox.profile.writablePersistence, "provider_managed");
+ assert.equal(sandbox.profile.parksOnTeardown, true);
+ assert.equal(supportsProcessSessions(sandbox), true);
+});
+
+test("the QM template's tools are advertised as installed, not missing", () => {
+ assert.ok(sandbox.profile.spec?.notInstalled?.includes("jq"));
+ const spec = make({ extraTools: QM_TEMPLATE_TOOLS }).profile.spec;
+ for (const tool of QM_TEMPLATE_TOOLS) {
+ assert.ok(spec?.tools?.includes(tool), tool);
+ assert.ok(!spec?.notInstalled?.includes(tool), tool);
+ }
+});
+
+test("egress enforcement is only claimed when a proxy is configured", () => {
+ assert.equal(sandbox.profile.egressEnforcement, "none");
+ assert.equal(make({ egressProxyUrl: "https://egress.example.com" }).profile.egressEnforcement, "domain");
+});
+
+test("the egress allowlist carries just the proxy host", () => {
+ assert.deepEqual(managedAgentsEgressAllow("https://egress.example.com:443/path"), ["egress.example.com"]);
+ assert.deepEqual(managedAgentsEgressAllow("http://10.0.0.7:8080"), ["10.0.0.7"]);
+});
+
+test("session status wire values map onto backend states", () => {
+ assert.equal(managedAgentsSessionState("SESSION_STATUS_READY"), "ready");
+ assert.equal(managedAgentsSessionState("SESSION_STATUS_PAUSED"), "paused");
+ assert.equal(managedAgentsSessionState("SESSION_STATUS_DESTROYED"), "destroyed");
+ assert.equal(managedAgentsSessionState("nonsense"), "unspecified");
+ assert.equal(managedAgentsSessionState(undefined), "unspecified");
+});
+
+test("process sessions stream output through the data plane exec path", async () => {
+ assert.ok(supportsProcessSessions(sandbox));
+ if (!supportsProcessSessions(sandbox)) return;
+ const h = await sandbox.provision(layers);
+ const { processId } = await sandbox.startProcess(h, "echo one; echo two");
+ const { output, status } = await pollProcess(sandbox, h, processId, { deadlineMs: 5_000, waitMs: 100 });
+ assert.equal(status.state, "exited");
+ assert.match(output, /one/);
+ assert.match(output, /two/);
+});
diff --git a/test/support/fake-managed-agents-service.ts b/test/support/fake-managed-agents-service.ts
new file mode 100644
index 000000000..71cb864cd
--- /dev/null
+++ b/test/support/fake-managed-agents-service.ts
@@ -0,0 +1,348 @@
+import { createServer as createHttpServer, type Server as HttpServer, type ServerResponse } from "node:http";
+import { connect, type Socket } from "node:net";
+import { once } from "node:events";
+import { fileURLToPath } from "node:url";
+import { randomUUID } from "node:crypto";
+import { spawn } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { WebSocketServer, type WebSocket } from "ws";
+import * as loader from "@grpc/proto-loader";
+import * as grpc from "@grpc/grpc-js";
+
+const PROTO_PATH = fileURLToPath(new URL("../../src/sandbox/managed-agents-sandbox-agent.proto", import.meta.url));
+
+interface SessionRow {
+ session_id: string;
+ sandbox_id: string;
+ name: string;
+ status: string;
+ template: string;
+ created_at: string;
+}
+
+export interface FakeManagedAgentsService {
+ apiBaseUrl: string;
+ token: string;
+ sessions(): SessionRow[];
+ manifests(): string[];
+ execScripts(): string[];
+ home(): string;
+ checkpoints(sessionId: string): Array<{ checkpoint_id: string; label: string; status: string }>;
+ setStatus(name: string, status: string): void;
+ rejectPortForward(body: string | null): void;
+ failNextLifecycleCalls(count: number, status: number): void;
+ tunnelCount(): number;
+ pingCount(): number;
+ close(): Promise<void>;
+}
+
+function loadService(): grpc.ServiceDefinition {
+ const definition = loader.loadSync(PROTO_PATH, {
+ keepCase: true,
+ longs: String,
+ enums: String,
+ defaults: true,
+ oneofs: true,
+ });
+ const pkg = grpc.loadPackageDefinition(definition) as unknown as {
+ do: {
+ teams: {
+ hosted_agents: { runtime: { sandbox_agent: { v1: { SandboxAgentService: grpc.ServiceClientConstructor } } } };
+ };
+ };
+ };
+ return pkg.do.teams.hosted_agents.runtime.sandbox_agent.v1.SandboxAgentService.service;
+}
+
+export async function startFakeManagedAgentsService(): Promise<FakeManagedAgentsService> {
+ const token = "dop_v1_test-token";
+ const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-guest-"));
+ const rows = new Map<string, SessionRow>();
+ const checkpoints = new Map<string, Array<{ checkpoint_id: string; label: string; status: string }>>();
+ const manifests: string[] = [];
+ const execScripts: string[] = [];
+ let tunnels = 0;
+ let pings = 0;
+ let lifecycleFailures = 0;
+ let lifecycleFailureStatus = 403;
+
+ const guest = new grpc.Server();
+ guest.addService(loadService(), {
+ Exec: (call: grpc.ServerDuplexStream<Record<string, unknown>, unknown>) => {
+ call.on("data", (frame: { start?: { argv?: string[]; workdir?: string; env?: Record<string, string> } }) => {
+ if (!frame.start) return;
+ const argv = frame.start.argv ?? [];
+ const script = argv[argv.length - 1] ?? "";
+ execScripts.push(script);
+ try {
+ const cwd = (frame.start.workdir || root).replace(/^\/home\/user/, root);
+ mkdirSync(cwd, { recursive: true });
+ const child = spawn("sh", ["-c", script.replace(/\/home\/user/g, root)], {
+ cwd,
+ env: { ...process.env, ...frame.start.env, HOME: root },
+ });
+ child.stdout.on("data", (chunk: Buffer) => call.write({ stdout: chunk }));
+ child.stderr.on("data", (chunk: Buffer) => call.write({ stderr: chunk }));
+ child.on("close", (code: number | null) => {
+ call.write({ exit: { exit_code: code ?? -1, duration_ms: 1, hitl_rejected: false } });
+ call.end();
+ });
+ child.on("error", (e: Error) => {
+ call.write({ stderr: Buffer.from(e.message) });
+ call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } });
+ call.end();
+ });
+ } catch (e) {
+ call.write({ stderr: Buffer.from(e instanceof Error ? e.message : String(e)) });
+ call.write({ exit: { exit_code: 127, duration_ms: 0, hitl_rejected: false } });
+ call.end();
+ }
+ });
+ },
+ Upload: (call: grpc.ServerReadableStream<Record<string, unknown>, unknown>, cb: grpc.sendUnaryData<unknown>) => {
+ let path = "";
+ const chunks: Buffer[] = [];
+ call.on("data", (frame: { header?: { path?: string }; chunk?: Buffer }) => {
+ if (frame.header?.path) path = frame.header.path;
+ if (frame.chunk?.length) chunks.push(frame.chunk);
+ });
+ call.on("end", () => {
+ const hostPath = path.replace(/^\/home\/user/, root);
+ mkdirSync(dirname(hostPath), { recursive: true });
+ const body = Buffer.concat(chunks);
+ writeFileSync(hostPath, body);
+ cb(null, { path, bytes_written: String(body.length) });
+ });
+ },
+ Download: (call: grpc.ServerWritableStream<{ path?: string }, unknown>) => {
+ const hostPath = (call.request.path ?? "").replace(/^\/home\/user/, root);
+ let body: Buffer;
+ try {
+ body = readFileSync(hostPath);
+ } catch {
+ call.emit("error", {
+ code: grpc.status.INTERNAL,
+ details: `envd download: download status 404: {"code":404,"message":"path '${call.request.path ?? ""}' does not exist"}`,
+ });
+ return;
+ }
+ call.write({ header: { size_bytes: String(body.length), is_archive: false } });
+ call.write({ chunk: body });
+ call.write({ end: { sha256: "" } });
+ call.end();
+ },
+ });
+
+ const guestPort = await new Promise<number>((resolve, reject) => {
+ guest.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (err, port) =>
+ err ? reject(err) : resolve(port),
+ );
+ });
+
+ const byName = (name: string): SessionRow | undefined =>
+ [...rows.values()].find((r) => r.name === name && r.status !== "SESSION_STATUS_DESTROYED");
+
+ const json = (res: ServerResponse, code: number, body: unknown): void => {
+ res.writeHead(code, { "content-type": "application/json" });
+ res.end(JSON.stringify(body));
+ };
+
+ const http: HttpServer = createHttpServer((req, res) => {
+ if (req.headers.authorization !== `Bearer ${token}`) {
+ json(res, 401, { error: { code: "unauthorized", message: "bad token" } });
+ return;
+ }
+ const url = new URL(req.url ?? "/", "http://localhost");
+ const parts = url.pathname.split("/").filter(Boolean);
+ const id = parts[3] ?? "";
+ const action = parts[4] ?? "";
+
+ if (req.method === "POST" && url.pathname === "/v2/agents/sessions") {
+ let body = "";
+ req.on("data", (c: Buffer) => (body += c.toString("utf8")));
+ req.on("end", () => {
+ manifests.push(body);
+ const name = /^name:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "unnamed";
+ const row: SessionRow = {
+ session_id: randomUUID(),
+ sandbox_id: `sbx-${rows.size + 1}`,
+ name,
+ status: "SESSION_STATUS_READY",
+ template: /^template:\s*"?([^"\n]+)"?/m.exec(body)?.[1] ?? "",
+ created_at: new Date().toISOString(),
+ };
+ rows.set(row.session_id, row);
+ json(res, 201, { session: row });
+ });
+ return;
+ }
+ if (req.method === "GET" && url.pathname === "/v2/agents/sessions") {
+ const name = url.searchParams.get("name");
+ const all = [...rows.values()].filter((r) => !name || r.name === name);
+ json(res, 200, { sessions: all, next_page_token: "" });
+ return;
+ }
+ if (lifecycleFailures > 0) {
+ lifecycleFailures -= 1;
+ json(res, lifecycleFailureStatus, { error: { code: "forbidden", message: "transient" } });
+ return;
+ }
+ if (req.method === "GET" && parts.length === 4) {
+ const row = rows.get(id);
+ if (!row) {
+ json(res, 404, { error: { code: "not_found", message: "no such session" } });
+ return;
+ }
+ json(res, 200, { session: row });
+ return;
+ }
+ if (req.method === "DELETE" && parts.length === 4) {
+ const row = rows.get(id);
+ if (row) row.status = "SESSION_STATUS_DESTROYED";
+ res.writeHead(204).end();
+ return;
+ }
+ if (req.method === "POST" && action === "checkpoints" && parts.length === 5) {
+ let body = "";
+ req.on("data", (c: Buffer) => (body += c.toString("utf8")));
+ req.on("end", () => {
+ const row = rows.get(id);
+ if (!row || row.status === "SESSION_STATUS_DESTROYED") {
+ json(res, 404, { error: { code: "not_found", message: "no such session" } });
+ return;
+ }
+ let label: string;
+ try {
+ label = (JSON.parse(body) as { label?: string }).label ?? "";
+ } catch {
+ label = "";
+ }
+ const checkpoint = {
+ checkpoint_id: `cp_${randomUUID().slice(0, 8)}`,
+ session_id: row.session_id,
+ status: "READY",
+ kind: "explicit",
+ label,
+ created_at: new Date().toISOString(),
+ size_bytes: 1,
+ };
+ const list = checkpoints.get(row.session_id) ?? [];
+ list.unshift(checkpoint);
+ checkpoints.set(row.session_id, list);
+ json(res, 200, { checkpoint });
+ });
+ return;
+ }
+ if (req.method === "POST" && action === "checkpoints" && parts[6] === "rollback" && parts.length === 7) {
+ const row = rows.get(id);
+ const checkpointId = decodeURIComponent(parts[5] ?? "");
+ const list = checkpoints.get(id) ?? [];
+ if (!row || row.status === "SESSION_STATUS_DESTROYED" || !list.some((c) => c.checkpoint_id === checkpointId)) {
+ json(res, 404, { error: { code: "not_found", message: "no such checkpoint" } });
+ return;
+ }
+ row.sandbox_id = `sbx-restored-${randomUUID().slice(0, 8)}`;
+ row.status = "SESSION_STATUS_READY";
+ json(res, 200, { session: row });
+ return;
+ }
+ if (req.method === "DELETE" && action === "checkpoints" && parts.length === 6) {
+ const checkpointId = decodeURIComponent(parts[5] ?? "");
+ const list = checkpoints.get(id) ?? [];
+ checkpoints.set(
+ id,
+ list.filter((c) => c.checkpoint_id !== checkpointId),
+ );
+ json(res, 200, {});
+ return;
+ }
+ if (req.method === "POST" && (action === "pause" || action === "resume")) {
+ const row = rows.get(id);
+ if (!row) {
+ json(res, 404, { error: { code: "not_found", message: "no such session" } });
+ return;
+ }
+ row.status = action === "pause" ? "SESSION_STATUS_PAUSED" : "SESSION_STATUS_READY";
+ res.writeHead(204).end();
+ return;
+ }
+ json(res, 404, { error: { code: "not_found", message: url.pathname } });
+ });
+
+ const wss = new WebSocketServer({ noServer: true });
+ let portForwardRejection: string | null = null;
+ http.on("upgrade", (req, socket, head) => {
+ const url = new URL(req.url ?? "/", "http://localhost");
+ const match = /^\/v2\/agents\/sessions\/([^/]+)\/port-forward\/(\d+)$/.exec(url.pathname);
+ if (req.headers.authorization !== `Bearer ${token}` || !match) {
+ socket.destroy();
+ return;
+ }
+ if (portForwardRejection !== null) {
+ socket.end(
+ `HTTP/1.1 403 Forbidden\r\nContent-Length: ${Buffer.byteLength(portForwardRejection)}\r\nConnection: close\r\n\r\n${portForwardRejection}`,
+ );
+ return;
+ }
+ const row = rows.get(match[1] ?? "");
+ if (!row || row.status === "SESSION_STATUS_DESTROYED") {
+ socket.destroy();
+ return;
+ }
+ wss.handleUpgrade(req, socket, head, (ws: WebSocket) => {
+ tunnels += 1;
+ ws.on("ping", () => {
+ pings += 1;
+ });
+ const upstream: Socket = connect(guestPort, "127.0.0.1");
+ ws.on("message", (data: Buffer) => upstream.write(data));
+ upstream.on("data", (chunk: Buffer) => ws.send(chunk));
+ const shutdown = (): void => {
+ upstream.destroy();
+ if (ws.readyState === ws.OPEN) ws.close();
+ };
+ ws.on("close", shutdown);
+ ws.on("error", shutdown);
+ upstream.on("close", shutdown);
+ upstream.on("error", shutdown);
+ });
+ });
+
+ http.listen(0, "127.0.0.1");
+ await once(http, "listening");
+ const address = http.address();
+ if (address === null || typeof address === "string") throw new Error("fake managed agents service failed to bind");
+
+ return {
+ apiBaseUrl: `http://127.0.0.1:${address.port}`,
+ token,
+ sessions: () => [...rows.values()],
+ manifests: () => [...manifests],
+ execScripts: () => [...execScripts],
+ home: () => root,
+ checkpoints: (sessionId) => [...(checkpoints.get(sessionId) ?? [])],
+ setStatus: (name, status) => {
+ const row = byName(name);
+ if (row) row.status = status;
+ },
+ rejectPortForward: (body) => {
+ portForwardRejection = body;
+ },
+ failNextLifecycleCalls: (count, status) => {
+ lifecycleFailures = count;
+ lifecycleFailureStatus = status;
+ },
+ tunnelCount: () => tunnels,
+ pingCount: () => pings,
+ close: async () => {
+ for (const ws of wss.clients) ws.terminate();
+ wss.close();
+ http.closeAllConnections();
+ await new Promise<void>((resolve) => http.close(() => resolve()));
+ guest.forceShutdown();
+ rmSync(root, { recursive: true, force: true });
+ },
+ };
+}
diff --git a/test/support/fake-managed-agents.ts b/test/support/fake-managed-agents.ts
new file mode 100644
index 000000000..39754201d
--- /dev/null
+++ b/test/support/fake-managed-agents.ts
@@ -0,0 +1,289 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import {
+ ManagedAgentsCommandLostError,
+ ManagedAgentsHitlRejectedError,
+ ManagedAgentsSandboxGoneError,
+ type ManagedAgentsClient,
+ type ManagedAgentsCommandResult,
+ type ManagedAgentsCheckpoint,
+ type ManagedAgentsSession,
+ type ManagedAgentsSessionInfo,
+ type ManagedAgentsSessionState,
+ type ManagedAgentsSessionSummary,
+} from "../../src/sandbox/managed-agents-client.ts";
+import { mkdtempSync } from "node:fs";
+
+const GUEST_HOME_DIR = "/workspace/home";
+
+interface FakeRecord {
+ sessionId: string;
+ sandboxId: string;
+ name: string;
+ state: ManagedAgentsSessionState;
+ home: string;
+ createdAt: number;
+ loseNextCommand: boolean;
+ rejectNextCommand: boolean;
+ failNextCheckpoint: boolean;
+ failNextCheckpointDelete: boolean;
+ failNextRollback: boolean;
+}
+
+export interface FakeManagedAgents {
+ client: ManagedAgentsClient;
+ current(name: string): { sessionId: string; sandboxId: string; state: ManagedAgentsSessionState } | null;
+ createdCount(name: string): number;
+ homeDir(name: string): string;
+ pause(name: string): void;
+ destroy(name: string): void;
+ loseNextCommand(name: string): void;
+ rejectNextCommand(name: string): void;
+ execScripts(): string[];
+ pauseCalls(): string[];
+ resumeCalls(): string[];
+ guestDisconnects(): string[];
+ checkpointCalls(): Array<{ sessionId: string; checkpointId: string; label?: string }>;
+ deletedCheckpoints(): string[];
+ rollbackCalls(): Array<{ sessionId: string; checkpointId: string }>;
+ failNextCheckpoint(name: string): void;
+ failNextCheckpointDelete(name: string): void;
+ failNextRollback(name: string): void;
+ cleanup(): void;
+}
+
+export function installFakeManagedAgents(): FakeManagedAgents {
+ const root = mkdtempSync(join(tmpdir(), "fake-managed-agents-"));
+ const records = new Map<string, FakeRecord>();
+ const execScripts: string[] = [];
+ const pauseCalls: string[] = [];
+ const resumeCalls: string[] = [];
+ const guestDisconnects: string[] = [];
+ const checkpointCalls: Array<{ sessionId: string; checkpointId: string; label?: string }> = [];
+ const deletedCheckpoints: string[] = [];
+ const rollbackCalls: Array<{ sessionId: string; checkpointId: string }> = [];
+ let nextId = 1;
+ let nextSandbox = 1;
+ let nextCheckpoint = 1;
+ let clock = 0;
+
+ const gone = (state: ManagedAgentsSessionState): boolean =>
+ state === "destroyed" || state === "destroying" || state === "failed";
+
+ const byName = (name: string): FakeRecord | undefined => {
+ const all = [...records.values()].filter((r) => r.name === name).sort((a, b) => b.createdAt - a.createdAt);
+ return all.find((r) => !gone(r.state)) ?? all[0];
+ };
+
+ const need = (name: string): FakeRecord => {
+ const r = byName(name);
+ if (!r) throw new Error(`fake-managed-agents: no session named ${name}`);
+ return r;
+ };
+
+ const remap = (r: FakeRecord, script: string): string => {
+ const homeRe = r.home.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+ const remapPath = new RegExp(`${homeRe}/tmp/|${homeRe}(?![A-Za-z0-9._-])|/tmp/`, "g");
+ return (
+ `export HOME=${JSON.stringify(r.home)}; ` +
+ script
+ .replace(/\btimeout \d+ /g, "")
+ .replaceAll(GUEST_HOME_DIR, r.home)
+ .replace(remapPath, (mm) => (mm.startsWith(r.home) ? mm : `${r.home}/tmp/`))
+ );
+ };
+
+ const alive = (r: FakeRecord): void => {
+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);
+ if (r.state === "paused") r.state = "ready";
+ };
+
+ const info = (r: FakeRecord): ManagedAgentsSessionInfo => ({
+ sessionId: r.sessionId,
+ sandboxId: r.sandboxId,
+ name: r.name,
+ state: r.state,
+ createdAtMs: r.createdAt,
+ });
+
+ const session = (r: FakeRecord): ManagedAgentsSession => ({
+ sessionId: r.sessionId,
+ sandboxId: r.sandboxId,
+ async runCommand(command): Promise<ManagedAgentsCommandResult> {
+ alive(r);
+ if (r.loseNextCommand) {
+ r.loseNextCommand = false;
+ r.state = "destroyed";
+ throw new ManagedAgentsCommandLostError(r.sessionId, "sandbox is not running anymore");
+ }
+ if (r.rejectNextCommand) {
+ r.rejectNextCommand = false;
+ throw new ManagedAgentsHitlRejectedError(r.sessionId);
+ }
+ execScripts.push(command);
+ mkdirSync(join(r.home, "tmp"), { recursive: true });
+ const spawned = spawnSync("sh", ["-c", remap(r, command)], {
+ encoding: "buffer",
+ maxBuffer: 128 * 1024 * 1024,
+ env: { ...process.env, COPYFILE_DISABLE: "1" },
+ });
+ return {
+ stdout: (spawned.stdout ?? Buffer.alloc(0)).toString("utf8"),
+ stderr: (spawned.stderr ?? Buffer.alloc(0)).toString("utf8"),
+ exitCode: spawned.status ?? (spawned.signal ? 137 : -1),
+ };
+ },
+ async readFileBytes(absPath): Promise<Uint8Array | null> {
+ alive(r);
+ const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath;
+ if (!existsSync(hostPath)) return null;
+ return new Uint8Array(readFileSync(hostPath));
+ },
+ async writeFileBytes(absPath, data): Promise<void> {
+ alive(r);
+ const hostPath = absPath.startsWith(GUEST_HOME_DIR) ? r.home + absPath.slice(GUEST_HOME_DIR.length) : absPath;
+ mkdirSync(dirname(hostPath), { recursive: true });
+ writeFileSync(hostPath, Buffer.from(data));
+ },
+ async createCheckpoint(label?: string): Promise<ManagedAgentsCheckpoint> {
+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);
+ if (r.failNextCheckpoint) {
+ r.failNextCheckpoint = false;
+ throw new Error("do-managed-agents create checkpoint: http 500 capture failed");
+ }
+ const checkpointId = `cp-${nextCheckpoint++}`;
+ checkpointCalls.push({ sessionId: r.sessionId, checkpointId, ...(label ? { label } : {}) });
+ return { checkpointId, status: "READY", createdAtMs: Date.now() };
+ },
+ async deleteCheckpoint(checkpointId: string): Promise<void> {
+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);
+ if (r.failNextCheckpointDelete) {
+ r.failNextCheckpointDelete = false;
+ throw new Error("do-managed-agents delete checkpoint: http 500 delete failed");
+ }
+ deletedCheckpoints.push(checkpointId);
+ },
+ async close(): Promise<void> {
+ guestDisconnects.push(r.sessionId);
+ },
+ async pause(): Promise<void> {
+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);
+ guestDisconnects.push(r.sessionId);
+ pauseCalls.push(r.sessionId);
+ r.state = "paused";
+ },
+ async resume(): Promise<void> {
+ if (gone(r.state)) throw new ManagedAgentsSandboxGoneError(r.sessionId, `status ${r.state}`);
+ resumeCalls.push(r.sessionId);
+ r.state = "ready";
+ },
+ async kill(): Promise<void> {
+ guestDisconnects.push(r.sessionId);
+ r.state = "destroyed";
+ rmSync(r.home, { recursive: true, force: true });
+ },
+ });
+
+ const client: ManagedAgentsClient = {
+ nativePause: true,
+ async info(sessionId): Promise<ManagedAgentsSessionInfo> {
+ const r = records.get(sessionId);
+ if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found");
+ return info(r);
+ },
+ async create(opts): Promise<ManagedAgentsSession> {
+ const id = `sess-${nextId++}`;
+ const r: FakeRecord = {
+ sessionId: id,
+ sandboxId: `sbx-${id}`,
+ name: opts.name,
+ state: "ready",
+ home: join(root, id),
+ createdAt: ++clock,
+ loseNextCommand: false,
+ rejectNextCommand: false,
+ failNextCheckpoint: false,
+ failNextCheckpointDelete: false,
+ failNextRollback: false,
+ };
+ mkdirSync(r.home, { recursive: true });
+ records.set(id, r);
+ return session(r);
+ },
+ async connect(sessionId): Promise<ManagedAgentsSession> {
+ const r = records.get(sessionId);
+ if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found");
+ if (r.state === "paused") {
+ resumeCalls.push(r.sessionId);
+ r.state = "ready";
+ }
+ return session(r);
+ },
+ async list(name): Promise<ManagedAgentsSessionSummary[]> {
+ return [...records.values()]
+ .filter((r) => !gone(r.state) && r.name === name)
+ .map((r) => ({ sessionId: r.sessionId, sandboxId: r.sandboxId, name: r.name, state: r.state }));
+ },
+ async rollback(sessionId, checkpointId): Promise<ManagedAgentsSessionInfo> {
+ const r = records.get(sessionId);
+ if (!r || gone(r.state)) throw new ManagedAgentsSandboxGoneError(sessionId, "session was not found");
+ if (r.failNextRollback) {
+ r.failNextRollback = false;
+ throw new Error("do-managed-agents rollback: http 409 checkpoint is not ready");
+ }
+ rollbackCalls.push({ sessionId, checkpointId });
+ r.sandboxId = `sbx-restored-${nextSandbox++}`;
+ r.state = "ready";
+ return info(r);
+ },
+ async kill(sessionId): Promise<void> {
+ const r = records.get(sessionId);
+ if (!r) return;
+ r.state = "destroyed";
+ rmSync(r.home, { recursive: true, force: true });
+ },
+ };
+
+ return {
+ client,
+ current: (name) => {
+ const r = byName(name);
+ return r && !gone(r.state) ? { sessionId: r.sessionId, sandboxId: r.sandboxId, state: r.state } : null;
+ },
+ createdCount: (name) => [...records.values()].filter((r) => r.name === name).length,
+ homeDir: (name) => need(name).home,
+ pause: (name) => {
+ need(name).state = "paused";
+ },
+ destroy: (name) => {
+ const r = need(name);
+ r.state = "destroyed";
+ rmSync(r.home, { recursive: true, force: true });
+ },
+ loseNextCommand: (name) => {
+ need(name).loseNextCommand = true;
+ },
+ rejectNextCommand: (name) => {
+ need(name).rejectNextCommand = true;
+ },
+ checkpointCalls: () => [...checkpointCalls],
+ deletedCheckpoints: () => [...deletedCheckpoints],
+ failNextCheckpoint: (name) => {
+ need(name).failNextCheckpoint = true;
+ },
+ failNextCheckpointDelete: (name) => {
+ need(name).failNextCheckpointDelete = true;
+ },
+ rollbackCalls: () => [...rollbackCalls],
+ failNextRollback: (name) => {
+ need(name).failNextRollback = true;
+ },
+ execScripts: () => [...execScripts],
+ pauseCalls: () => [...pauseCalls],
+ resumeCalls: () => [...resumeCalls],
+ guestDisconnects: () => [...guestDisconnects],
+ cleanup: () => rmSync(root, { recursive: true, force: true }),
+ };
+}