Multiplayer agent harness for work.
feat(sandbox): add Superserve backend and agent template (#1346)
Pavitra Bhalla7df57367c309
diff --git a/README.md b/README.mdindex 13fe21927..de7d118e2 100644--- a/README.md+++ b/README.md@@ -242,6 +242,7 @@ upstream source history to merge.- [`cli/README.md`](./cli/README.md) — the `qm` CLI and the deployment directory contract- [`docs/deploy-directory.md`](./docs/deploy-directory.md) — the deployment directory in full- [`docs/porter.md`](./docs/porter.md) — running qm on Porter+- [`docs/superserve.md`](./docs/superserve.md) — using Superserve for agent sandboxes- [`.env.example`](./.env.example) — every knob, documented in place- [`plugins/`](./plugins) — the surfaces (Slack, web UI, admin, portal)diff --git a/cli/src/config.ts b/cli/src/config.tsindex cbea1aaef..5ba969f86 100644--- a/cli/src/config.ts+++ b/cli/src/config.ts@@ -46,7 +46,7 @@ export interface PluginEntry {}export interface SandboxConfig {- backend?: "local" | "sprites" | "aws" | "agent37";+ backend?: "local" | "sprites" | "aws" | "agent37" | "superserve";app?: string;image?: string;baseImage?: string;@@ -211,8 +211,14 @@ export const dockerBasePort = (config: QmConfig): number => envNum("QM_BASE_PORTexport const isDigestPinned = (ref: string): boolean => /@sha256:[0-9a-f]{64}$/.test(ref);+const effectiveSandboxBackend = (config: Pick<QmConfig, "env" | "sandbox">): string | undefined =>+ config.env.core?.SANDBOX_BACKEND?.trim() || config.sandbox?.backend;+export const localSandboxActive = (config: QmConfig): boolean =>- config.target === "docker" && config.sandbox?.backend === "local";+ config.target === "docker" &&+ (config.sandbox?.backend === "superserve"+ ? effectiveSandboxBackend(config) === "local"+ : config.sandbox?.backend === "local" && effectiveSandboxBackend(config) !== "superserve");export function sandboxCoreEnv(config: QmConfig,@@ -227,8 +233,12 @@ export function sandboxCoreEnv(if (sb.image) env.LOCAL_SANDBOX_IMAGE = sb.image;return { env, missingSecrets };}- if (sb.backend === "agent37") {- env.SANDBOX_BACKEND = "agent37";+ const backend =+ sb.backend === "superserve" || effectiveSandboxBackend(config) === "superserve"+ ? effectiveSandboxBackend(config)+ : sb.backend;+ if (backend === "agent37" || backend === "superserve") {+ env.SANDBOX_BACKEND = backend;return { env, missingSecrets };}for (const [k, v] of Object.entries(sb.env ?? {})) env[`FLY_RESIDENT_ENV_${k}`] = v;@@ -682,6 +692,23 @@ function validate(raw: unknown, path: string): QmConfig {return v;});const sandbox = validateSandbox(o["sandbox"], path, target);+ if (+ sandbox?.backend === "superserve" &&+ env.core?.SANDBOX_BACKEND !== undefined &&+ !env.core.SANDBOX_BACKEND.trim()+ ) {+ throw new CliError(+ `${path}: "env.core.SANDBOX_BACKEND" is blank — name the backend core should run, or remove the key; every target forwards env.core verbatim, so a blank value leaves core with no backend`,+ );+ }+ if (+ (effectiveSandboxBackend({ env, sandbox }) === "superserve" || scopeUsesSuperserve(env.core)) &&+ !env.core?.SUPERSERVE_TEMPLATE?.trim()+ ) {+ throw new CliError(+ `${path}: the superserve sandbox backend requires env.core.SUPERSERVE_TEMPLATE (the ready qm-agent-<release> template); core refuses to start without it`,+ );+ }const out: QmConfig = {contract,@@ -945,6 +972,20 @@ export function validatePortalTrust(config: QmConfig, path = "config", secrets?:}}+function scopeUsesSuperserve(core: Record<string, string> | undefined): boolean {+ try {+ const scopes: unknown = JSON.parse(core?.SANDBOX_SCOPE_BACKENDS || "{}");+ return Boolean(+ scopes &&+ typeof scopes === "object" &&+ !Array.isArray(scopes) &&+ Object.values(scopes).some((value) => typeof value === "string" && value.trim() === "superserve"),+ );+ } catch {+ return false;+ }+}+export function requiresAwsMicrovmImage(config: QmConfig): boolean {if (config.target !== "aws") return false;const core = config.env.core;@@ -1470,10 +1511,11 @@ function validateSandbox(raw: unknown, path: string, target: Target): SandboxCono["backend"] !== "local" &&o["backend"] !== "sprites" &&o["backend"] !== "aws" &&- o["backend"] !== "agent37"+ o["backend"] !== "agent37" &&+ o["backend"] !== "superserve") {throw new CliError(- `${path}: "sandbox.backend" must be "local" (Docker containers on the deployment host), "sprites" (Fly Sprites), "aws" (Lambda MicroVM sandboxes), or "agent37"`,+ `${path}: "sandbox.backend" must be "local" (Docker containers on the deployment host), "sprites" (Fly Sprites), "aws" (Lambda MicroVM sandboxes), "agent37", or "superserve" (Superserve sandboxes)`,);}out.backend = o["backend"];@@ -1537,11 +1579,11 @@ function validateSandbox(raw: unknown, path: string, target: Target): SandboxCon);}}- if (out.backend === "agent37") {+ if (out.backend === "agent37" || out.backend === "superserve") {const stray = (["app", "image", "baseImage", "env", "secretEnv"] as const).filter((key) => out[key] !== undefined);if (stray.length) {throw new CliError(- `${path}: "sandbox.backend": "agent37" ignores ${stray.map((key) => `"sandbox.${key}"`).join(", ")} — remove them`,+ `${path}: "sandbox.backend": "${out.backend}" ignores ${stray.map((key) => `"sandbox.${key}"`).join(", ")} — remove them`,);}}diff --git a/cli/src/providers.ts b/cli/src/providers.tsindex af7cd671f..7676dd4db 100644--- a/cli/src/providers.ts+++ b/cli/src/providers.ts@@ -5,7 +5,7 @@ export type Target = (typeof HOSTING_PROVIDER_IDS)[number];export const isTarget = (value: unknown): value is Target =>typeof value === "string" && (HOSTING_PROVIDER_IDS as readonly string[]).includes(value);-export type SandboxBackendId = "local" | "sprites" | "aws" | "agent37";+export type SandboxBackendId = "local" | "sprites" | "aws" | "agent37" | "superserve";export interface SandboxBackendPolicy {/** Sandbox backends this hosting target can run. */@@ -16,9 +16,9 @@ export interface SandboxBackendPolicy {/** Keyed by hosting target so adding a target forces a sandbox-backend decision. */export const SANDBOX_BACKEND_POLICY: Record<Target, SandboxBackendPolicy> = {- docker: { allowed: ["local", "sprites", "agent37"], requireExplicit: false },- fly: { allowed: ["sprites", "agent37"], requireExplicit: false },- aws: { allowed: ["sprites", "aws", "agent37"], requireExplicit: true },+ docker: { allowed: ["local", "sprites", "agent37", "superserve"], requireExplicit: false },+ fly: { allowed: ["sprites", "agent37", "superserve"], requireExplicit: false },+ aws: { allowed: ["sprites", "aws", "agent37", "superserve"], requireExplicit: true },};export const targetsAllowingSandboxBackend = (backend: SandboxBackendId): Target[] =>diff --git a/cli/src/secrets.ts b/cli/src/secrets.tsindex 5e6ae16b0..aad5622f4 100644--- a/cli/src/secrets.ts+++ b/cli/src/secrets.ts@@ -205,6 +205,13 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [description: "Agent37 API key for the agent-computer substrate.",generate: "mint a key in the Agent37 dashboard (https://agent37.com/dashboard/cloud/api-keys)",},+ {+ name: "SUPERSERVE_API_KEY",+ service: "core",+ required: { when: { kind: "sandbox-backend", backend: "superserve" } },+ description: "API key for Superserve sandboxes.",+ generate: "create an API key in the Superserve console (https://console.superserve.ai/api-keys)",+ },{name: "DATABASE_URL",service: "core",diff --git a/cli/test/config.test.ts b/cli/test/config.test.tsindex e4b4cf940..13db9480b 100644--- a/cli/test/config.test.ts+++ b/cli/test/config.test.ts@@ -9,6 +9,7 @@ import {CONFIG_FILENAME,loadConfigAt,loadConfigInDir,+ localSandboxActive,mockHarnessWarning,sandboxCoreEnv,securityScreenEnv,@@ -886,11 +887,11 @@ test("sandbox shape errors: object, app non-empty string, env string-map, secret{ sandbox: { secretEnv: ["1BAD"] }, rx: /not a valid env var name/ },{sandbox: { backend: "k8s", app: "acme-sandboxes" },- rx: /"sandbox.backend" must be "local".*"sprites".*"aws".*or "agent37"/,+ rx: /"sandbox.backend" must be "local".*"sprites".*"aws".*"agent37".*or "superserve"/,},{sandbox: { backend: "fly", app: "acme-sandboxes" },- rx: /"sandbox.backend" must be "local".*"sprites".*"aws".*or "agent37"/,+ rx: /"sandbox.backend" must be "local".*"sprites".*"aws".*"agent37".*or "superserve"/,},{sandbox: { backend: "aws", app: "acme-sandboxes" },@@ -938,6 +939,135 @@ test("agent37 is a deployment backend on every target and rejects unused Fly set});});+test("superserve backend requires the agent template in env.core", () => {+ withConfig({ sandbox: { backend: "superserve" } }, ({ path }) => {+ assert.throws(() => loadConfigAt(path), /superserve sandbox backend requires env.core.SUPERSERVE_TEMPLATE/);+ });+ withConfig({ env: { core: { SANDBOX_BACKEND: "superserve" } } }, ({ path }) => {+ assert.throws(() => loadConfigAt(path), /superserve sandbox backend requires env.core.SUPERSERVE_TEMPLATE/);+ });+ withConfig({ sandbox: { backend: "superserve" }, env: { core: { SUPERSERVE_TEMPLATE: " " } } }, ({ path }) => {+ assert.throws(() => loadConfigAt(path), /superserve sandbox backend requires env.core.SUPERSERVE_TEMPLATE/);+ });+ withConfig(+ { sandbox: { backend: "superserve" }, env: { core: { SUPERSERVE_TEMPLATE: "qm-agent-1.0.0" } } },+ ({ path }) => {+ const { config } = loadConfigAt(path);+ assert.deepEqual(sandboxCoreEnv(config), { env: { SANDBOX_BACKEND: "superserve" }, missingSecrets: [] });+ },+ );+ withConfig(+ {+ sandbox: { backend: "local" },+ env: { core: { SANDBOX_SCOPE_BACKENDS: JSON.stringify({ personal: "superserve" }) } },+ },+ ({ path }) => {+ assert.throws(() => loadConfigAt(path), /superserve sandbox backend requires env.core.SUPERSERVE_TEMPLATE/);+ },+ );+ withConfig(+ {+ sandbox: { backend: "local" },+ env: {+ core: {+ SANDBOX_SCOPE_BACKENDS: JSON.stringify({ personal: "superserve" }),+ SUPERSERVE_TEMPLATE: "qm-agent-1.0.0",+ },+ },+ },+ ({ path }) => assert.doesNotThrow(() => loadConfigAt(path)),+ );+});++test("Superserve selection honors env.core.SANDBOX_BACKEND overrides", () => {+ withConfig({ sandbox: { backend: "local" }, env: { core: { SANDBOX_BACKEND: "superserve" } } }, ({ path }) => {+ assert.throws(() => loadConfigAt(path), /superserve sandbox backend requires env.core.SUPERSERVE_TEMPLATE/);+ });+ withConfig({ sandbox: { backend: "local" }, env: { core: { SANDBOX_BACKEND: " superserve " } } }, ({ path }) => {+ assert.throws(() => loadConfigAt(path), /superserve sandbox backend requires env.core.SUPERSERVE_TEMPLATE/);+ });+ withConfig({ sandbox: { backend: "superserve" }, env: { core: { SANDBOX_BACKEND: " " } } }, ({ path }) => {+ assert.throws(+ () => loadConfigAt(path),+ /"env.core.SANDBOX_BACKEND" is blank/,+ "a blank override is refused rather than rendered over the sandbox block",+ );+ });+ withConfig(+ {+ target: "fly",+ sandbox: { backend: "sprites", app: "acme-sandboxes", secretEnv: ["COMPANY_API_TOKEN"] },+ env: { core: { SANDBOX_BACKEND: "superserve", SUPERSERVE_TEMPLATE: "qm-agent-1.0.0" } },+ },+ ({ path }) => {+ const { config } = loadConfigAt(path);+ assert.deepEqual(+ sandboxCoreEnv(config),+ { env: { SANDBOX_BACKEND: "superserve" }, missingSecrets: [] },+ "the overridden backend's settings are not demanded or injected",+ );+ },+ );+ withConfig(+ {+ sandbox: { backend: "local" },+ env: { core: { SANDBOX_BACKEND: "superserve", SUPERSERVE_TEMPLATE: "qm-agent-1.0.0" } },+ },+ ({ path }) => {+ const { config } = loadConfigAt(path);+ assert.equal(config.sandbox?.backend, "local");+ },+ );+ withConfig({ sandbox: { backend: "superserve" }, env: { core: { SANDBOX_BACKEND: "local" } } }, ({ path }) => {+ const { config } = loadConfigAt(path);+ assert.equal(localSandboxActive(config), true, "docker prepares the local sandbox the override actually runs");+ assert.equal(sandboxCoreEnv(config).env.SANDBOX_BACKEND, "local");+ });+ withConfig(+ {+ sandbox: { backend: "local" },+ env: { core: { SANDBOX_BACKEND: "superserve", SUPERSERVE_TEMPLATE: "qm-agent-1.0.0" } },+ },+ ({ path }) => {+ assert.equal(+ localSandboxActive(loadConfigAt(path).config),+ false,+ "a remote override never mounts the host docker socket",+ );+ },+ );+});++test("Superserve configuration preserves existing backend selection and validation", () => {+ for (const override of ["agent37", "sprites", "", " "]) {+ withConfig({ sandbox: { backend: "local" }, env: { core: { SANDBOX_BACKEND: override } } }, ({ path }) => {+ const { config } = loadConfigAt(path);+ assert.equal(localSandboxActive(config), true);+ assert.equal(sandboxCoreEnv(config).env.SANDBOX_BACKEND, "local");+ });+ }+ withConfig({ sandbox: { backend: "agent37" }, env: { core: { SANDBOX_BACKEND: "sprites" } } }, ({ path }) =>+ assert.equal(sandboxCoreEnv(loadConfigAt(path).config).env.SANDBOX_BACKEND, "agent37"),+ );+ for (const scopes of ["invalid", "[]", "null", "42"]) {+ withConfig({ sandbox: { backend: "local" }, env: { core: { SANDBOX_SCOPE_BACKENDS: scopes } } }, ({ path }) =>+ assert.doesNotThrow(() => loadConfigAt(path)),+ );+ }+});++test("sandbox.backend alone makes the backend's credential a required secret on every target", () => {+ for (const target of ["docker", "fly"] as const) {+ withConfig(+ { target, sandbox: { backend: "superserve" }, env: { core: { SUPERSERVE_TEMPLATE: "qm-agent-1.0.0" } } },+ ({ path }) => {+ const secret = computedSecrets(loadConfigAt(path).config).find((s) => s.name === "SUPERSERVE_API_KEY");+ assert.ok(secret?.required, `${target} requires the key without duplicating SANDBOX_BACKEND under env.core`);+ },+ );+ }+});+test("aws target makes the sandbox substrate explicit: backend required with a sandbox block, aws forbids fly sandbox settings", () => {const aws = {accountId: "123456789012",diff --git a/cli/test/secrets.test.ts b/cli/test/secrets.test.tsindex 9c7375b7e..8b33aff1e 100644--- a/cli/test/secrets.test.ts+++ b/cli/test/secrets.test.ts@@ -401,6 +401,13 @@ test("scope-selected providers require both Modal and Sprites credentials", () =assert.ok(required.includes(name), name);});+test("a scope routed to superserve requires its API key even when it is not the primary backend", () => {+ const config = makeConfig({+ env: { core: { SANDBOX_BACKEND: "local", SANDBOX_SCOPE_BACKENDS: '{"channel":"superserve"}' } },+ });+ assert.equal(secretByName(config, "SUPERSERVE_API_KEY").required, true);+});+test("shared Fly publishing requires private peers only when selected", () => {for (const provider of ["fly", "aws"]) {for (const shared of [false, true]) {diff --git a/docs/superserve.md b/docs/superserve.mdnew file mode 100644index 000000000..b7b35b1fb--- /dev/null+++ b/docs/superserve.md@@ -0,0 +1,94 @@+# Superserve agent sandboxes++The `superserve` backend runs each QM scope in a sandbox with a persistent workspace,+file transfers, and background process sessions. Paused sandboxes resume on use.++## Configure++Build and verify an [agent template](../superserve/templates/README.md), then set:++```sh+SANDBOX_BACKEND=superserve+SUPERSERVE_API_KEY=your-superserve-api-key+SUPERSERVE_TEMPLATE=qm-agent-0.1.0+SUPERSERVE_NAME_PREFIX=my-qm+```++Store the API key as a core service secret. Production requires `DATABASE_URL` for+sandbox records, provisioning locks, and configuration ordering across instances.+Use a distinct name prefix for each independent deployment sharing a Superserve+team, including development instances.++For CLI deployments, add these fields to the deployment config and supply the API+key through the CLI's secret configuration:++```json+{+ "sandbox": { "backend": "superserve" },+ "env": {+ "core": {+ "SUPERSERVE_TEMPLATE": "qm-agent-0.1.0",+ "SUPERSERVE_NAME_PREFIX": "my-qm"+ }+ }+}+```++`env.core.SANDBOX_BACKEND` overrides `sandbox.backend`. To select Superserve for+individual scope kinds, use `SANDBOX_SCOPE_BACKENDS`; the same key, template, and+database requirements apply.++For local development, export the key and template and run+`npm run dev-instance -- --sandbox superserve`.++## Settings++| Variable | Default | Purpose |+| ------------------------------ | ----------- | ---------------------------------------------------------------------------------- |+| `SUPERSERVE_API_KEY` | Required | Team API key, held by core. |+| `SUPERSERVE_TEMPLATE` | Required | Ready template name. |+| `SUPERSERVE_BASE_URL` | SDK default | API endpoint override. |+| `SUPERSERVE_NAME_PREFIX` | `qm` | Namespace for scope discovery. |+| `SUPERSERVE_HOME_DIR` | `/root` | Guest home; workspace is `<home>/workspace`. |+| `SUPERSERVE_IDLE_PAUSE_SEC` | `900` | Active-time limit before pause, not an inactivity timer. |+| `SUPERSERVE_RETENTION_SEC` | `2592000` | Delete the sandbox after this many continuous seconds paused (30 days by default). |+| `SUPERSERVE_EGRESS_ALLOW` | Unset | Comma-separated outbound allow rules. |+| `SUPERSERVE_EGRESS_DENY` | Unset | Comma-separated outbound deny rules. |+| `SUPERSERVE_CONFIG_GENERATION` | Automatic | Explicit rollout sequence. |+| `SANDBOX_TIMEOUT_SEC` | `600` | Default command deadline in seconds. |++Ending a turn leaves its sandbox running until the active-time limit, so other+turns sharing it can continue. Background work can extend that limit to QM's+configured maximum background job lifetime. Scratch sandboxes are deleted when+their last local handle closes, with time limits as a cleanup fallback.++Command output is limited to 2 MiB per stream; write larger results to files.+Guest restart and the optional browser engine are not supported.++## Updates and retention++QM rediscovers sandboxes by scope metadata after a restart and checks their actual+egress policy before reuse. Changing the template replaces existing scope+sandboxes and deletes their resident files. An egress change can also require+replacement when a paused sandbox cannot accept the update. Export needed files+before either change.++A sandbox that remains paused for `SUPERSERVE_RETENTION_SEC` is automatically+deleted, including its disk. Explicit scope destruction also deletes the disk.++Automatic configuration ordering assigns a generation to each build and+configuration. Older instances cannot overwrite a newer sandbox configuration.+A rollback reuses the earlier generation: to apply it, set+`SUPERSERVE_CONFIG_GENERATION` to a strictly increasing rollout sequence and+continue using that sequence for subsequent deployments. Template and egress+changes during rollback have the same replacement behavior described above.++## Tests++Backend tests run without an account:++```sh+node --experimental-test-module-mocks --test test/superserve-*.test.ts+```++The template verifier requires an API key and creates a temporary sandbox.diff --git a/knip.json b/knip.jsonindex b0d00d855..19b7c755c 100644--- a/knip.json+++ b/knip.json@@ -32,6 +32,8 @@"scripts/mine-slack-scenarios.ts","scripts/skills-materialize-smoke.ts","scripts/slack-user-token.ts",+ "superserve/templates/qm-agent.ts",+ "superserve/templates/verify-qm-agent.ts","docs/qa/resource-search-demo.mjs"]},diff --git a/package-lock.json b/package-lock.jsonindex c83106e6f..26159bfa6 100644--- a/package-lock.json+++ b/package-lock.json@@ -28,6 +28,7 @@"@slack/socket-mode": "^2.0.7","@slack/web-api": "^7.19.0","@smithy/signature-v4": "^5.6.6",+ "@superserve/sdk": "^0.8.3","croner": "^10.0.1","e2b": "^2.39.0","emoji-datasource": "^16.0.0",@@ -6104,6 +6105,12 @@"integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==","license": "MIT"},+ "node_modules/@superserve/sdk": {+ "version": "0.8.3",+ "resolved": "https://registry.npmjs.org/@superserve/sdk/-/sdk-0.8.3.tgz",+ "integrity": "sha512-ukSYO1bz4g4W6/TUtPpuvstrFdY1hQgMk9zaf+hxvXmtM/3eCth8AgsBkxCitHe8gE96bIGwYmX0e9hV3zBmHQ==",+ "license": "Apache-2.0"+ },"node_modules/@tokenizer/inflate": {"version": "0.4.1","resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.4.1.tgz",diff --git a/package.json b/package.jsonindex 10459b009..4a3f56195 100644--- a/package.json+++ b/package.json@@ -83,6 +83,7 @@"@slack/socket-mode": "^2.0.7","@slack/web-api": "^7.19.0","@smithy/signature-v4": "^5.6.6",+ "@superserve/sdk": "^0.8.3","croner": "^10.0.1","e2b": "^2.39.0","emoji-datasource": "^16.0.0",diff --git a/scripts/dev/cli.ts b/scripts/dev/cli.tsindex 3a524af0e..4cf0a10e2 100644--- a/scripts/dev/cli.ts+++ b/scripts/dev/cli.ts@@ -41,7 +41,7 @@ import { sweepSlackTokenOrphans } from "./lib/orphans.ts";import { destroyLocalDevSandboxes } from "./lib/sandbox.ts";import { bestEffort, errMessage, formatAge, nowEpoch, sleep } from "./lib/util.ts";import { runDoctor } from "./commands/doctor.ts";-import type { BootPhaseEvent, BootResult, LeaseInfo } from "./lib/types.ts";+import type { BootPhaseEvent, BootResult, DevSandboxChoice, LeaseInfo } from "./lib/types.ts";import { CHILD_ORDER, EXIT } from "./lib/types.ts";import { validOrgId } from "../../cli/src/config.ts";@@ -258,7 +258,7 @@ async function bootOnSlot(slot: string, worktree: string, branch: string): Promibranch,callerEnv,watch: !opts["no-watch"] && callerEnv.DEV_INSTANCE_WATCH !== "0",- sandbox: opts.sandbox as "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "auto",+ sandbox: opts.sandbox as DevSandboxChoice,canaryChannel,strict: opts.strict,slack: withSlack,@@ -674,7 +674,7 @@ async function main(): Promise<number> {});default:console.error(- "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]",+ "usage: dev [up|down|status|restart|canary|logs|doctor] [--json] [--force] [--rotate] [--strict] [--sandbox local|sprites|smolmachines|e2b|porter|agent37|superserve|auto] [--surface web|slack|both] [--no-slack] [--no-watch] [--org id] [--fix]",);return EXIT.usage;}diff --git a/scripts/dev/lib/sandbox.ts b/scripts/dev/lib/sandbox.tsindex f56a24461..bf10c1c14 100644--- a/scripts/dev/lib/sandbox.ts+++ b/scripts/dev/lib/sandbox.ts@@ -1,13 +1,15 @@import { existsSync, openSync, readFileSync } from "node:fs";import { spawn } from "node:child_process";+import { createHash } from "node:crypto";import { join } from "node:path";import { writePidFile } from "./lease.ts";import { run } from "./proc.ts";import { ensureDockerDaemon } from "./postgres.ts";import { bestEffortValue, sleep } from "./util.ts";+import type { DevSandboxBackend, DevSandboxChoice } from "./types.ts";export interface SandboxResolution {- backend: "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37";+ backend: DevSandboxBackend;env: Record<string, string>;detail: string;publicApiUrl: string | null;@@ -24,7 +26,7 @@ async function localImagePresent(image: string): Promise<boolean> {export async function resolveSandbox(opts: {worktree: string;- requested: "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "auto";+ requested: DevSandboxChoice;corePort: number;lock: string;baseEnv: Record<string, string>;@@ -150,6 +152,44 @@ export async function resolveSandbox(opts: {return { backend: "agent37", env, detail: "Agent37 (api.agent37.com)", publicApiUrl: apiUrl, warnings };}+ if (backend === "superserve") {+ const apiKey = opts.baseEnv.SUPERSERVE_API_KEY?.trim();+ const template = opts.baseEnv.SUPERSERVE_TEMPLATE?.trim();+ if (!apiKey)+ throw new Error(+ "--sandbox superserve requires SUPERSERVE_API_KEY in the environment (create an API key in the Superserve console)",+ );+ if (!template)+ throw new Error(+ "--sandbox superserve requires SUPERSERVE_TEMPLATE in the environment (the ready qm-agent-<release> template that carries the agent toolchain)",+ );+ let apiUrl = opts.baseEnv.PUBLIC_API_URL || null;+ if (!apiUrl) {+ apiUrl = await startQuickTunnel(opts.corePort, opts.lock, opts.log);+ if (!apiUrl)+ warnings.push(+ "cloudflared tunnel didn't come up -- agent self-API (crons/sends) won't be reachable from the sandbox",+ );+ }+ const env: Record<string, string> = {+ SANDBOX_BACKEND: "superserve",+ SUPERSERVE_API_KEY: apiKey,+ SUPERSERVE_TEMPLATE: template,+ SUPERSERVE_NAME_PREFIX:+ opts.baseEnv.SUPERSERVE_NAME_PREFIX ||+ `qmdev-${createHash("sha1").update(opts.worktree).digest("hex").slice(0, 12)}`,+ };+ if (opts.baseEnv.SUPERSERVE_BASE_URL) env.SUPERSERVE_BASE_URL = opts.baseEnv.SUPERSERVE_BASE_URL;+ if (opts.baseEnv.SUPERSERVE_EGRESS_ALLOW) env.SUPERSERVE_EGRESS_ALLOW = opts.baseEnv.SUPERSERVE_EGRESS_ALLOW;+ if (opts.baseEnv.SUPERSERVE_EGRESS_DENY) env.SUPERSERVE_EGRESS_DENY = opts.baseEnv.SUPERSERVE_EGRESS_DENY;+ if (!opts.baseEnv.SUPERSERVE_EGRESS_ALLOW && !opts.baseEnv.SUPERSERVE_EGRESS_DENY)+ warnings.push(+ "SUPERSERVE_EGRESS_ALLOW/SUPERSERVE_EGRESS_DENY unset -- superserve sandbox runs with open egress; set one to QA the enforced path",+ );+ if (apiUrl) env.PUBLIC_API_URL = apiUrl;+ return { backend: "superserve", env, detail: `Superserve (template ${template})`, publicApiUrl: apiUrl, warnings };+ }+if (backend === "porter") {const porterToken = opts.baseEnv.PORTER_DEPLOY_API_TOKEN;const projectId = opts.baseEnv.PORTER_DEPLOY_PROJECT_ID;diff --git a/scripts/dev/lib/types.ts b/scripts/dev/lib/types.tsindex 354697832..f1491f783 100644--- a/scripts/dev/lib/types.ts+++ b/scripts/dev/lib/types.ts@@ -104,13 +104,16 @@ export interface StatusReport {children: Record<string, ChildStatus & { slack?: SlackHealth }>;}+export type DevSandboxBackend = "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "superserve";+export type DevSandboxChoice = DevSandboxBackend | "auto";+export interface BootSpec {slot: string;worktree: string;branch: string;callerEnv: Record<string, string>;watch: boolean;- sandbox: "local" | "sprites" | "smolmachines" | "e2b" | "porter" | "agent37" | "auto";+ sandbox: DevSandboxChoice;canaryChannel?: string;strict: boolean;slack?: boolean;diff --git a/src/config.ts b/src/config.tsindex 86aa63da9..f6fa32804 100644--- a/src/config.ts+++ b/src/config.ts@@ -66,8 +66,9 @@ export interface Config {sandboxResourcesEnabled: boolean;sharingPosture: SharingPosture;sandboxScopeDefaults?: SandboxScopeDefaults;- sandboxBackend: "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37";- sandboxSecondaryBackend?: "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37";+ sandboxBackend: "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";+ sandboxSecondaryBackend?:+ "aws" | "local" | "sprites" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";deployProvider: "docker" | "aws" | "fly" | "porter";egressServiceHosts?: string[];brandingDefault?: OrgBranding;@@ -194,6 +195,7 @@ export interface Config {spritesSandbox: SpritesSandboxEnv;smolmachinesSandbox: SmolmachinesSandboxEnv;agent37Sandbox: Agent37SandboxEnv;+ superserveSandbox: SuperserveSandboxEnv;e2bSandbox: E2bSandboxEnv;modalSandbox: ModalSandboxEnv;porterSandbox: PorterSandboxEnv;@@ -600,6 +602,58 @@ function agent37SandboxEnv(env: NodeJS.ProcessEnv): Agent37SandboxEnv {};}+interface SuperserveSandboxEnv {+ apiKey?: string;+ baseUrl?: string;+ namePrefix?: string;+ template?: string;+ homeDir?: string;+ idlePauseSec?: number;+ retentionSec?: number;+ egressAllow?: string[];+ egressDeny?: string[];+ defaultTimeoutSec?: number;+ configGeneration?: number;+}++const csvList = (value: string | undefined): string[] | undefined => {+ const items = (value ?? "")+ .split(",")+ .map((s) => s.trim())+ .filter(Boolean);+ return items.length ? items : undefined;+};++function superserveSandboxEnv(env: NodeJS.ProcessEnv): SuperserveSandboxEnv {+ const egressAllow = csvList(env.SUPERSERVE_EGRESS_ALLOW);+ const egressDeny = csvList(env.SUPERSERVE_EGRESS_DENY);+ const configGeneration = numEnvStrict("SUPERSERVE_CONFIG_GENERATION", env.SUPERSERVE_CONFIG_GENERATION);+ if (configGeneration !== undefined && (!Number.isSafeInteger(configGeneration) || configGeneration < 0)) {+ throw new Error(+ `SUPERSERVE_CONFIG_GENERATION=${JSON.stringify(env.SUPERSERVE_CONFIG_GENERATION)} must be a nonnegative safe integer, or unset it.`,+ );+ }+ return {+ ...(env.SUPERSERVE_API_KEY ? { apiKey: env.SUPERSERVE_API_KEY } : {}),+ ...(env.SUPERSERVE_BASE_URL?.trim() ? { baseUrl: env.SUPERSERVE_BASE_URL.trim() } : {}),+ ...(env.SUPERSERVE_NAME_PREFIX ? { namePrefix: env.SUPERSERVE_NAME_PREFIX } : {}),+ ...(env.SUPERSERVE_TEMPLATE?.trim() ? { template: env.SUPERSERVE_TEMPLATE.trim() } : {}),+ ...(env.SUPERSERVE_HOME_DIR ? { homeDir: env.SUPERSERVE_HOME_DIR } : {}),+ ...(numEnvStrict("SUPERSERVE_IDLE_PAUSE_SEC", env.SUPERSERVE_IDLE_PAUSE_SEC) !== undefined+ ? { idlePauseSec: numEnvStrict("SUPERSERVE_IDLE_PAUSE_SEC", env.SUPERSERVE_IDLE_PAUSE_SEC) }+ : {}),+ ...(numEnvStrict("SUPERSERVE_RETENTION_SEC", env.SUPERSERVE_RETENTION_SEC) !== undefined+ ? { retentionSec: numEnvStrict("SUPERSERVE_RETENTION_SEC", env.SUPERSERVE_RETENTION_SEC) }+ : {}),+ ...(egressAllow ? { egressAllow } : {}),+ ...(egressDeny ? { egressDeny } : {}),+ ...(numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) !== undefined+ ? { defaultTimeoutSec: numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) }+ : {}),+ ...(configGeneration !== undefined ? { configGeneration } : {}),+ };+}+interface AwsDeployEnv {region: string;profile?: string;@@ -888,6 +942,7 @@ export function enabledSandboxBackends(config: Config): Array<Config["sandboxBacsprites: Boolean(config.spritesSandbox?.token),smolmachines: Boolean(config.smolmachinesSandbox?.token),agent37: Boolean(config.agent37Sandbox?.apiKey),+ superserve: Boolean(config.superserveSandbox?.apiKey && config.superserveSandbox?.template),e2b: Boolean(config.e2bSandbox?.apiKey),modal: Boolean(config.modalSandbox?.tokenId && config.modalSandbox?.tokenSecret),aws: Boolean(config.awsSandbox?.s3Bucket),@@ -911,11 +966,12 @@ function sandboxBackendEnvStrict(value: string | undefined, name = "SANDBOX_BACKbackend === "e2b" ||backend === "modal" ||backend === "agent37" ||+ backend === "superserve" ||backend === "porter")return backend;throw new Error(- `${name}=${JSON.stringify(value)} is not recognized — use aws, local, sprites, smolmachines, e2b, modal, porter, or agent37, or unset it.`,+ `${name}=${JSON.stringify(value)} is not recognized — use aws, local, sprites, smolmachines, e2b, modal, porter, agent37, or superserve, or unset it.`,);}@@ -1122,7 +1178,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {}if (env.NODE_ENV === "production" && !env.SANDBOX_BACKEND?.trim()) {throw new Error(- "SANDBOX_BACKEND must be set explicitly in production — use sprites, smolmachines, e2b, modal, porter, agent37, aws, or local.",+ "SANDBOX_BACKEND must be set explicitly in production — use sprites, smolmachines, e2b, modal, porter, agent37, superserve, aws, or local.",);}const sandboxBackend = sandboxBackendEnvStrict(env.SANDBOX_BACKEND);@@ -1138,6 +1194,20 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {sandboxScopeDefaults[parsed] = sandboxBackendEnvStrict(value, "SANDBOX_SCOPE_BACKENDS." + kind);}}+ const superserveSelected =+ sandboxBackend === "superserve" || Object.values(sandboxScopeDefaults).includes("superserve");+ if (superserveSelected && !env.SUPERSERVE_TEMPLATE?.trim()) {+ throw new Error(+ "SANDBOX_BACKEND=superserve requires SUPERSERVE_TEMPLATE, the ready qm-agent-<release> template that carries the agent toolchain.",+ );+ }+ const superserveEnabled =+ superserveSelected || Boolean(env.SUPERSERVE_API_KEY?.trim() && env.SUPERSERVE_TEMPLATE?.trim());+ if (superserveEnabled && env.NODE_ENV === "production" && !env.DATABASE_URL?.trim()) {+ throw new Error(+ "the superserve sandbox backend requires DATABASE_URL in production: the config generation and the provisioning lock have to be durable across instances, or a blue-green rollout can destroy a scope's resident disk.",+ );+ }if (env.SANDBOX_SECONDARY_BACKEND?.trim()) {console.warn(@@ -1485,6 +1555,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {spritesSandbox: spritesSandboxEnv(env),smolmachinesSandbox: smolmachinesSandboxEnv(env),agent37Sandbox: agent37SandboxEnv(env),+ superserveSandbox: superserveSandboxEnv(env),porterSandbox: porterSandboxEnv(env),porterDeploy: porterDeployEnv(env),e2bSandbox: e2bSandboxEnv(env),diff --git a/src/deployment/secret-schema.ts b/src/deployment/secret-schema.tsindex d474798b9..d2006368d 100644--- a/src/deployment/secret-schema.ts+++ b/src/deployment/secret-schema.ts@@ -11,6 +11,7 @@ type SecretGate =| "modal"| "porter"| "agent37"+ | "superserve"| "porter-deploy"| "fly-shared-deploy"| "fly-deploy"@@ -43,6 +44,7 @@ export const CORE_SECRET_SPECS: readonly RuntimeSecretSpec[] = [{ name: "SPRITES_TOKEN", requiredWhen: "sprites" },{ name: "SMOLMACHINES_TOKEN", requiredWhen: "smolmachines" },{ name: "AGENT37_API_KEY", requiredWhen: "agent37" },+ { name: "SUPERSERVE_API_KEY", requiredWhen: "superserve" },{ name: "E2B_API_KEY", requiredWhen: "e2b" },{ name: "MODAL_TOKEN_ID", requiredWhen: "modal" },{ name: "MODAL_TOKEN_SECRET", requiredWhen: "modal" },@@ -74,6 +76,7 @@ const GATE_PREDICATES: Readonly<Record<SecretGate, (env: NodeJS.ProcessEnv) => bmodal: (env) => sandboxBackendSelected(env, "modal"),porter: (env) => sandboxBackendSelected(env, "porter"),agent37: (env) => sandboxBackendSelected(env, "agent37"),+ superserve: (env) => sandboxBackendSelected(env, "superserve"),"porter-deploy": (env) => env.DEPLOY_PROVIDER === "porter","fly-shared-deploy": (env) => env.DEPLOY_PROVIDER === "fly" && Boolean(env.FLY_DEPLOY_SHARED_APP_NAME?.trim()),"fly-deploy": (env) => env.DEPLOY_PROVIDER === "fly",diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.tsindex 60be520c1..06e62e786 100644--- a/src/sandbox/sandbox-routing.ts+++ b/src/sandbox/sandbox-routing.ts@@ -16,7 +16,8 @@ import {type TeardownOptions,} from "./sandbox.ts";-export type SandboxBackendName = "sprites" | "aws" | "local" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37";+export type SandboxBackendName =+ "sprites" | "aws" | "local" | "smolmachines" | "e2b" | "modal" | "porter" | "agent37" | "superserve";export type SandboxScopeDefaults = Partial<Record<ScopeKind, SandboxBackendName>>;diff --git a/src/sandbox/sandbox.ts b/src/sandbox/sandbox.tsindex 9d9f77e30..c85e7f728 100644--- a/src/sandbox/sandbox.ts+++ b/src/sandbox/sandbox.ts@@ -2,6 +2,7 @@ import type { EgressPolicy, WorkspaceLayer } from "../types.ts";export interface SandboxHandle {resourceId?: string;+ providerSandboxId?: string;id: string;rootDir: string;homeDir?: string;diff --git a/src/sandbox/superserve-client.ts b/src/sandbox/superserve-client.tsnew file mode 100644index 000000000..366eff40e--- /dev/null+++ b/src/sandbox/superserve-client.ts@@ -0,0 +1,280 @@+import { errMessage } from "../util/errors.ts";++export interface SuperserveCommandResult {+ stdout: string;+ stderr: string;+ exitCode: number;+ truncated?: boolean;+}++export interface SuperserveNetwork {+ allowOut?: string[];+ denyOut?: string[];+}++export type SuperserveSandboxState = "starting" | "active" | "pausing" | "paused" | "resuming" | "failed" | "deleted";++interface SuperserveSandboxSummary {+ id: string;+ name: string;+ status: SuperserveSandboxState;+ metadata: Record<string, string>;+}++export interface SuperserveSandboxInfo extends SuperserveSandboxSummary {+ vcpuCount?: number;+ memoryMib?: number;+ timeoutSeconds?: number;+ autoDeleteAtMs?: number;+ network?: SuperserveNetwork;+}++interface SuperserveRunOptions {+ timeoutMs?: number;+ maxOutputBytes?: number;+}++export interface SuperserveUpdate {+ network?: SuperserveNetwork;+ metadata?: Record<string, string>;+ timeoutSeconds?: number | null;+ autoDeleteSeconds?: number | null;+}++export interface SuperserveSession {+ readonly id: string;+ run(command: string, opts?: SuperserveRunOptions): Promise<SuperserveCommandResult>;+ readFileBytes(absPath: string): Promise<Uint8Array | null>;+ writeFileBytes(absPath: string, data: Uint8Array): Promise<void>;+ update(patch: SuperserveUpdate): Promise<void>;+ pause(): Promise<void>;+ kill(): Promise<void>;+}++export interface SuperserveCreateOptions {+ name: string;+ metadata: Record<string, string>;+ template?: string;+ timeoutSeconds?: number;+ autoDeleteSeconds?: number;+ network?: SuperserveNetwork;+}++export class SuperserveSandboxGoneError extends Error {+ constructor(sandboxId: string, detail: string) {+ super(`superserve sandbox ${sandboxId} is gone: ${detail}`);+ this.name = "SuperserveSandboxGoneError";+ }+}++export interface SuperserveClient {+ create(opts: SuperserveCreateOptions): Promise<SuperserveSession>;+ connect(sandboxId: string): Promise<SuperserveSession>;+ update(sandboxId: string, patch: SuperserveUpdate): Promise<void>;+ info(sandboxId: string, scopeMetadata?: Record<string, string>): Promise<SuperserveSandboxInfo>;+ list(metadata: Record<string, string>): Promise<SuperserveSandboxSummary[]>;+ kill(sandboxId: string): Promise<void>;+}++export interface SdkSuperserveClientOptions {+ apiKey: string;+ baseUrl?: string;+ template?: string;+ maxCommandMs?: number;+}++const DEFAULT_MAX_COMMAND_MS = 3600_000;+const DEFAULT_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;++export const clampUtf8 = (text: string, maxBytes: number): string => {+ if (maxBytes <= 0) return "";+ const encoded = Buffer.from(text, "utf8");+ if (encoded.length <= maxBytes) return text;+ let end = maxBytes;+ while (end > 0 && (encoded[end]! & 0xc0) === 0x80) end -= 1;+ return encoded.subarray(0, end).toString("utf8");+};+const GONE_STATES: ReadonlySet<string> = new Set(["deleted", "failed"]);++function hasStatus(err: unknown, statusCode: number): boolean {+ return typeof err === "object" && err !== null && (err as { statusCode?: unknown }).statusCode === statusCode;+}++function isGoneError(err: unknown): boolean {+ return hasStatus(err, 404) || hasStatus(err, 410);+}++export function createSdkSuperserveClient(opts: SdkSuperserveClientOptions): SuperserveClient {+ const maxCommandMs = opts.maxCommandMs ?? DEFAULT_MAX_COMMAND_MS;+ const connection = { apiKey: opts.apiKey, ...(opts.baseUrl ? { baseUrl: opts.baseUrl } : {}) };++ type Sdk = typeof import("@superserve/sdk");+ let sdk: Promise<Sdk> | null = null;+ const loadSdk = (): Promise<Sdk> =>+ (sdk ??= import("@superserve/sdk").catch((e: unknown) => {+ sdk = null;+ throw e;+ }));++ type SdkSandbox = Awaited<ReturnType<Sdk["Sandbox"]["create"]>>;++ const gone = (id: string, err: unknown): never => {+ throw new SuperserveSandboxGoneError(id, errMessage(err));+ };++ const wrap = (sbx: SdkSandbox): SuperserveSession => ({+ id: sbx.id,+ async run(command, runOpts): Promise<SuperserveCommandResult> {+ const timeoutMs = runOpts?.timeoutMs ?? maxCommandMs;+ const cap = runOpts?.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;+ try {+ const r = await sbx.commands.run(command, {+ timeoutMs,+ onStdout: () => undefined,+ });+ return {+ stdout: clampUtf8(r.stdout, cap),+ stderr: clampUtf8(r.stderr, cap),+ exitCode: r.exitCode,+ ...(r.truncated || Buffer.byteLength(r.stdout, "utf8") > cap || Buffer.byteLength(r.stderr, "utf8") > cap+ ? { truncated: true }+ : {}),+ };+ } catch (err) {+ if (isGoneError(err)) gone(sbx.id, err);+ throw err;+ }+ },+ async readFileBytes(absPath): Promise<Uint8Array | null> {+ try {+ return await sbx.files.read(absPath);+ } catch (err) {+ if (hasStatus(err, 404)) {+ let status: string;+ try {+ status = (await sbx.getInfo()).status;+ } catch (infoErr) {+ if (isGoneError(infoErr)) gone(sbx.id, infoErr);+ throw infoErr;+ }+ if (GONE_STATES.has(status)) gone(sbx.id, err);+ return null;+ }+ if (isGoneError(err)) gone(sbx.id, err);+ throw err;+ }+ },+ async writeFileBytes(absPath, data): Promise<void> {+ try {+ await sbx.files.write(absPath, data);+ } catch (err) {+ if (isGoneError(err)) gone(sbx.id, err);+ throw err;+ }+ },+ async update(patch): Promise<void> {+ try {+ await sbx.update(patch);+ } catch (err) {+ if (isGoneError(err)) gone(sbx.id, err);+ throw err;+ }+ },+ async pause(): Promise<void> {+ try {+ await sbx.pause();+ } catch (err) {+ if (isGoneError(err)) gone(sbx.id, err);+ throw err;+ }+ },+ async kill(): Promise<void> {+ await sbx.kill().catch((err) => {+ if (!isGoneError(err)) throw err;+ });+ },+ });++ const toInfo = (i: {+ id: string;+ name: string;+ status: SuperserveSandboxState;+ metadata: Record<string, string>;+ vcpuCount?: number;+ memoryMib?: number;+ timeoutSeconds?: number;+ autoDeleteAt?: Date;+ network?: SuperserveNetwork;+ }): SuperserveSandboxInfo => ({+ id: i.id,+ name: i.name,+ status: i.status,+ metadata: i.metadata ?? {},+ ...(i.network ? { network: i.network } : {}),+ ...(i.vcpuCount !== undefined ? { vcpuCount: i.vcpuCount } : {}),+ ...(i.memoryMib !== undefined ? { memoryMib: i.memoryMib } : {}),+ ...(i.timeoutSeconds !== undefined ? { timeoutSeconds: i.timeoutSeconds } : {}),+ ...(i.autoDeleteAt ? { autoDeleteAtMs: i.autoDeleteAt.getTime() } : {}),+ });++ return {+ async create(createOpts): Promise<SuperserveSession> {+ const { Sandbox } = await loadSdk();+ const sbx = await Sandbox.create({+ ...connection,+ name: createOpts.name,+ metadata: createOpts.metadata,+ ...((createOpts.template ?? opts.template) ? { fromTemplate: createOpts.template ?? opts.template } : {}),+ ...(createOpts.timeoutSeconds !== undefined ? { timeoutSeconds: createOpts.timeoutSeconds } : {}),+ ...(createOpts.autoDeleteSeconds !== undefined ? { autoDeleteSeconds: createOpts.autoDeleteSeconds } : {}),+ ...(createOpts.network ? { network: createOpts.network } : {}),+ previewAccess: "private",+ });+ return wrap(sbx);+ },+ async update(sandboxId, patch): Promise<void> {+ const { Sandbox } = await loadSdk();+ try {+ await Sandbox.updateById(sandboxId, patch, connection);+ } catch (err) {+ if (isGoneError(err)) gone(sandboxId, err);+ throw err;+ }+ },+ async connect(sandboxId): Promise<SuperserveSession> {+ const { Sandbox } = await loadSdk();+ try {+ return wrap(await Sandbox.connect(sandboxId, connection));+ } catch (err) {+ if (isGoneError(err)) gone(sandboxId, err);+ throw err;+ }+ },+ async info(sandboxId, scopeMetadata): Promise<SuperserveSandboxInfo> {+ const { Sandbox } = await loadSdk();+ const scoped = scopeMetadata && Object.keys(scopeMetadata).length ? scopeMetadata : undefined;+ try {+ const listed = await Sandbox.list({ ...connection, ...(scoped ? { metadata: scoped } : {}) });+ const hit = listed.find((s) => s.id === sandboxId);+ if (!hit || GONE_STATES.has(hit.status)) throw new SuperserveSandboxGoneError(sandboxId, "not listed");+ return toInfo(hit);+ } catch (err) {+ if (isGoneError(err)) gone(sandboxId, err);+ throw err;+ }+ },+ async list(metadata): Promise<SuperserveSandboxSummary[]> {+ const { Sandbox } = await loadSdk();+ const listed = await Sandbox.list({ ...connection, ...(Object.keys(metadata).length ? { metadata } : {}) });+ return listed+ .filter((s) => !GONE_STATES.has(s.status))+ .map((s) => ({ id: s.id, name: s.name, status: s.status, metadata: s.metadata ?? {} }));+ },+ async kill(sandboxId): Promise<void> {+ const { Sandbox } = await loadSdk();+ await Sandbox.killById(sandboxId, connection).catch((err) => {+ if (!isGoneError(err)) throw err;+ });+ },+ };+}diff --git a/src/sandbox/superserve-sandbox.ts b/src/sandbox/superserve-sandbox.tsnew file mode 100644index 000000000..9f92109e0--- /dev/null+++ b/src/sandbox/superserve-sandbox.ts@@ -0,0 +1,773 @@+import { AsyncLocalStorage } from "node:async_hooks";+import { createHash, randomUUID } from "node:crypto";+import type { WorkspaceLayer } from "../types.ts";+import type { WorkspaceStore } from "../workspace/workspace-store.ts";+import type { DurableMap } from "../persistence/durable-map.ts";+import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";+import { createMemoryMap } from "../persistence/durable-map.ts";+import { createKeyedQueue } from "../util/async.ts";+import { swallowAs, errMessage } from "../util/errors.ts";+import { shq } from "../util/shell.ts";+import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV } from "./sandbox-env.ts";+import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";+import { materializeRoLayers } from "./ro-layers.ts";+import { createLayerToolInstaller } from "./layer-tool-install.ts";+import type { LayerInstallFile } from "../deployment/load-layer.ts";+import {+ createBackendBlobStaging,+ createExecExport,+ createExecFileOps,+ posixJoin,+ type BlobStagingOptions,+} from "./exec-file-ops.ts";+import {+ ephemeralCredLinkScript,+ ephemeralCredLinkPaths,+ type CredentialPathSpec,+} from "../credentials/resident-paths.ts";+import { killableScript, killScript } from "./exec-kill.ts";+import { visibleNotInstalled, visibleTools } from "./sandbox.ts";+import { sandboxScopeName } from "./exec-sandbox-base.ts";+import {+ SuperserveSandboxGoneError,+ type SuperserveClient,+ type SuperserveNetwork,+ type SuperserveSession,+} from "./superserve-client.ts";+import type {+ AgentComputerProfile,+ ComputerStatus,+ ExecOptions,+ ExecResult,+ ProvisionOptions,+ Sandbox,+ SandboxHandle,+ TeardownOptions,+} from "./sandbox.ts";++const DEFAULT_HOME_DIR = "/root";+const WORKSPACE_BASENAME = "workspace";+const RO_LAYERS_TAR = ".ro-layers.tar";+const RO_LAYERS_MANIFEST = ".ro-layers.manifest";+const DEFAULT_IDLE_PAUSE_SEC = 15 * 60;+const DEFAULT_KEEP_WARM_SEC = 3600;+const DEFAULT_RETENTION_SEC = 30 * 24 * 3600;+const SCRATCH_IDLE_PAUSE_SEC = 10 * 60;+const SCRATCH_RETENTION_SEC = 24 * 3600;+const PREP_TIMEOUT_SEC = 60;+const TIMEOUT_EXIT_CODE = 124;+const TIMEOUT_KILLED_EXIT_CODE = 137;+const KILL_AFTER_SEC = 10;+const OUTPUT_CAP_BYTES = 2 * 1024 * 1024;+const TRUNCATED_NOTICE = "[superserve: output truncated at 2 MiB; redirect large output to a file]";++export const SUPERSERVE_METADATA = {+ scope: "qm_scope",+ prefix: "qm_prefix",+ kind: "qm_kind",+ egress: "qm_egress",+ template: "qm_template",+ epoch: "qm_config_epoch",+} as const;++export interface StoredSuperserveSandbox {+ sandboxId: string;+ createdAtMs: number;+}++export interface StoredConfigEpoch {+ generation: number;+}++const CONFIG_EPOCH_LOCK = "superserve-config-epoch";++export function createConfigEpochResolver(+ store: DurableMap<StoredConfigEpoch>,+ generationKey: string,+ lock: AdvisoryLock = createNoopAdvisoryLock(),+): () => Promise<number> {+ let pending: Promise<number> | undefined;+ const claim = (): Promise<number> =>+ lock.withLock(CONFIG_EPOCH_LOCK, async () => {+ const stored = await store.get(generationKey);+ if (stored) return stored.generation;+ const generation = (await store.entries()).reduce((high, [, seen]) => Math.max(high, seen.generation), 0) + 1;+ await store.put(generationKey, { generation });+ return (await store.get(generationKey))?.generation ?? generation;+ });+ return () => {+ pending ??= claim().catch((err: unknown) => {+ pending = undefined;+ throw err;+ });+ return pending;+ };+}++export interface SuperserveSandboxOptions extends BlobStagingOptions {+ client: SuperserveClient;+ namePrefix?: string;+ defaultTimeoutSec?: number;+ template?: string;+ homeDir?: string;+ idlePauseSec?: number;+ keepWarmSec?: number;+ configEpoch?: number | (() => Promise<number>);+ retentionSec?: number;+ egressAllow?: string[];+ egressDeny?: string[];+ extraTools?: string[];+ credentialPaths?: CredentialPathSpec[];+ layerToolFiles?: () => readonly LayerInstallFile[];+ store?: DurableMap<StoredSuperserveSandbox>;+ advisoryLock?: AdvisoryLock;+ onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;+}++interface Live {+ session: SuperserveSession;+ current: boolean;+}++const pinnedSandbox = new AsyncLocalStorage<string>();++const isConflict = (err: unknown): boolean =>+ typeof err === "object" && err !== null && (err as { statusCode?: unknown }).statusCode === 409;++export function createSuperserveSandbox(workspace: WorkspaceStore, opts: SuperserveSandboxOptions): Sandbox {+ const client = opts.client;+ const prefix = opts.namePrefix ?? "qm";+ const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;+ const configuredHome = opts.homeDir ?? DEFAULT_HOME_DIR;+ const idlePauseSec = opts.idlePauseSec ?? DEFAULT_IDLE_PAUSE_SEC;+ const keepWarmSec = Math.max(opts.keepWarmSec ?? DEFAULT_KEEP_WARM_SEC, idlePauseSec);+ const retentionSec = opts.retentionSec ?? DEFAULT_RETENTION_SEC;+ const network: SuperserveNetwork | undefined =+ opts.egressAllow?.length || opts.egressDeny?.length+ ? {+ ...(opts.egressAllow?.length ? { allowOut: opts.egressAllow } : {}),+ ...(opts.egressDeny?.length ? { denyOut: opts.egressDeny } : {}),+ }+ : undefined;+ const store = opts.store ?? createMemoryMap<StoredSuperserveSandbox>();+ const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock();+ const lockKey = (scope: string): string => `superserve-provision:${scope}`;+ const provisionQueue = createKeyedQueue<string>();++ const liveByName = new Map<string, Live>();+ const scopeByName = new Map<string, string>();+ const scratchKeyByName = new Map<string, string>();+ const activeScratch = new Map<string, number>();++ const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {+ opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });+ };++ const workspaceDirOf = (homeDir: string): string => `${homeDir}/${WORKSPACE_BASENAME}`;++ const adoptedNetwork: SuperserveNetwork = network ?? { allowOut: [], denyOut: [] };+ const normalizedNetwork = (value: SuperserveNetwork | undefined): string =>+ JSON.stringify([+ [...(value?.allowOut ?? [])].map((rule) => rule.trim()).sort(),+ [...(value?.denyOut ?? [])].map((rule) => rule.trim()).sort(),+ ]);+ const appliedNetwork = normalizedNetwork(adoptedNetwork);+ const egressTag = createHash("sha256")+ .update(JSON.stringify([[...(adoptedNetwork.allowOut ?? [])].sort(), [...(adoptedNetwork.denyOut ?? [])].sort()]))+ .digest("hex")+ .slice(0, 16);++ const bootEpoch = Date.now();+ const configEpoch = async (): Promise<number> =>+ typeof opts.configEpoch === "function" ? opts.configEpoch() : (opts.configEpoch ?? bootEpoch);+ void configEpoch().catch(swallowAs("superserve-sandbox: claim config generation", undefined));+ const scopeMetadata = async (name: string): Promise<Record<string, string>> => ({+ [SUPERSERVE_METADATA.epoch]: String(await configEpoch()),+ [SUPERSERVE_METADATA.scope]: name,+ [SUPERSERVE_METADATA.prefix]: prefix,+ [SUPERSERVE_METADATA.kind]: "scope",+ [SUPERSERVE_METADATA.egress]: egressTag,+ ...(opts.template ? { [SUPERSERVE_METADATA.template]: opts.template } : {}),+ });++ const scopeFilter = (name: string): Record<string, string> => ({ [SUPERSERVE_METADATA.scope]: name });++ const staleHandle = (name: string): Error =>+ new Error(+ `superserve sandbox for ${name} was replaced while this handle was held; provision it again before using it`,+ );++ const handleIsCurrent = (handle: SandboxHandle): boolean => {+ const live = liveByName.get(handle.id);+ return !handle.providerSandboxId || !live || live.session.id === handle.providerSandboxId;+ };++ const assertCurrent = (handle: SandboxHandle): void => {+ if (!handleIsCurrent(handle)) throw staleHandle(handle.id);+ };++ const pinnedToHandle = <T>(handle: SandboxHandle, action: () => T): T => {+ assertCurrent(handle);+ return handle.providerSandboxId ? pinnedSandbox.run(handle.providerSandboxId, action) : action();+ };++ const guardHandleOps = <T extends object>(ops: T): T =>+ Object.fromEntries(+ Object.entries(ops).map(([key, value]) => [+ key,+ typeof value === "function"+ ? (handle: SandboxHandle, ...args: unknown[]): unknown =>+ pinnedToHandle(handle, () => (value as (...a: unknown[]) => unknown)(handle, ...args))+ : value,+ ]),+ ) as T;++ const dropLive = (name: string, sandboxId: string): void => {+ if (liveByName.get(name)?.session.id === sandboxId) liveByName.delete(name);+ };++ async function ownsGuest(name: string): Promise<boolean> {+ const live = liveByName.get(name);+ if (!live) return false;+ if (!live.current) return false;+ try {+ const info = await client.info(live.session.id, scopeFilter(name));+ live.current = Number(info.metadata[SUPERSERVE_METADATA.epoch] ?? 0) <= (await configEpoch());+ return live.current;+ } catch (err) {+ if (!(err instanceof SuperserveSandboxGoneError)) throw err;+ dropLive(name, live.session.id);+ const scope = scopeByName.get(name);+ if (scope !== undefined) await forget(scope, live.session.id);+ throw new Error(`superserve sandbox for ${name} is gone; the next provision creates a replacement`, {+ cause: err,+ });+ }+ }++ async function forget(scope: string, sandboxId: string): Promise<void> {+ if (store.deleteIf) {+ await store.deleteIf(scope, (record) => record.sandboxId === sandboxId);+ return;+ }+ const stored = await store.get(scope);+ if (stored?.sandboxId === sandboxId) await store.delete(scope);+ }++ async function reconnect(+ scope: string,+ name: string,+ sandboxId: string,+ ): Promise<{ session: SuperserveSession; current: boolean }> {+ const info = await client.info(sandboxId, scopeFilter(name));+ const stampedEpoch = Number(info.metadata[SUPERSERVE_METADATA.epoch] ?? 0);+ if (stampedEpoch > (await configEpoch())) return { session: await client.connect(sandboxId), current: false };+ if (opts.template && info.metadata[SUPERSERVE_METADATA.template] !== opts.template) {+ await client.kill(sandboxId);+ const message = `sandbox ${sandboxId} was built from template ${info.metadata[SUPERSERVE_METADATA.template] ?? "unknown"}, not ${opts.template}; it was destroyed and the next provision creates a replacement`;+ reportError("sandbox_template", "template_changed", message, scope);+ throw new SuperserveSandboxGoneError(sandboxId, message);+ }+ const stale = normalizedNetwork(info.network) !== appliedNetwork;+ const retire = async (): Promise<never> => {+ await client.kill(sandboxId);+ const message = `sandbox ${sandboxId} was paused under a different egress policy; it was destroyed and the next provision creates a replacement`;+ reportError("sandbox_egress", "policy_changed", message, scope);+ throw new SuperserveSandboxGoneError(sandboxId, message);+ };+ try {+ await client.update(sandboxId, {+ timeoutSeconds: Math.max(info.timeoutSeconds ?? 0, idlePauseSec),+ autoDeleteSeconds: retentionSec,+ metadata: { ...info.metadata, ...(await scopeMetadata(info.name)) },+ ...(stale ? { network: adoptedNetwork } : {}),+ });+ } catch (err) {+ if (stale && isConflict(err)) await retire();+ throw err;+ }+ if (stale && normalizedNetwork((await client.info(sandboxId, scopeFilter(name))).network) !== appliedNetwork) {+ await retire();+ }+ return { session: await client.connect(sandboxId), current: true };+ }++ async function adopt(+ name: string,+ session: SuperserveSession,+ persist?: { scope: string; known?: StoredSuperserveSandbox },+ current = true,+ ): Promise<Live> {+ const live: Live = { session, current };+ if (persist)+ await store.put(persist.scope, { sandboxId: session.id, createdAtMs: persist.known?.createdAtMs ?? Date.now() });+ liveByName.set(name, live);+ return live;+ }++ async function ensureLive(+ scope: string,+ name: string,+ onStatus?: (text: string) => void,+ ): Promise<{ live: Live; coldStart: boolean }> {+ const cached = liveByName.get(name);+ if (cached) {+ try {+ const info = await client.info(cached.session.id, scopeFilter(name));+ if (normalizedNetwork(info.network) === appliedNetwork) {+ cached.current = Number(info.metadata[SUPERSERVE_METADATA.epoch] ?? 0) <= (await configEpoch());+ return { live: cached, coldStart: false };+ }+ dropLive(name, cached.session.id);+ } catch (err) {+ if (!(err instanceof SuperserveSandboxGoneError)) throw err;+ dropLive(name, cached.session.id);+ await forget(scope, cached.session.id);+ }+ }++ const stored = await store.get(scope);+ if (stored) {+ try {+ const { session, current } = await reconnect(scope, name, stored.sandboxId);+ const live = await adopt(name, session, { scope, known: stored }, current);+ return { live, coldStart: false };+ } catch (err) {+ if (!(err instanceof SuperserveSandboxGoneError)) throw err;+ await forget(scope, stored.sandboxId);+ }+ }++ const listed = await client.list({ [SUPERSERVE_METADATA.scope]: name });+ for (const summary of listed) {+ try {+ const { session, current } = await reconnect(scope, name, summary.id);+ const live = await adopt(name, session, { scope }, current);+ return { live, coldStart: false };+ } catch (err) {+ if (!(err instanceof SuperserveSandboxGoneError)) throw err;+ }+ }++ try {+ onStatus?.("Creating the sandbox…");+ } catch (error) {+ void error;+ }+ const session = await client.create({+ name,+ metadata: await scopeMetadata(name),+ ...(opts.template ? { template: opts.template } : {}),+ timeoutSeconds: idlePauseSec,+ autoDeleteSeconds: retentionSec,+ ...(network ? { network } : {}),+ });+ const live = await adopt(name, session, { scope });+ return { live, coldStart: true };+ }++ async function createScratch(name: string): Promise<Live> {+ const session = await client.create({+ name,+ metadata: { ...(await scopeMetadata(name)), [SUPERSERVE_METADATA.kind]: "scratch" },+ ...(opts.template ? { template: opts.template } : {}),+ timeoutSeconds: SCRATCH_IDLE_PAUSE_SEC,+ autoDeleteSeconds: SCRATCH_RETENTION_SEC,+ ...(network ? { network } : {}),+ });+ return adopt(name, session);+ }++ async function ensureScratch(key: string): Promise<{ name: string; coldStart: boolean }> {+ const name = sandboxScopeName(`${prefix}-scratch`, key);+ scratchKeyByName.set(name, key);+ const active = activeScratch.get(name) ?? 0;+ const cached = liveByName.get(name);+ if (cached) {+ try {+ const info = await client.info(cached.session.id, scopeFilter(name));+ if (normalizedNetwork(info.network) !== appliedNetwork) {+ await client.kill(cached.session.id);+ dropLive(name, cached.session.id);+ reportError(+ "sandbox_egress",+ "policy_changed",+ `scratch sandbox ${cached.session.id} no longer carried the configured egress policy; it was destroyed and replaced`,+ );+ }+ } catch (err) {+ if (!(err instanceof SuperserveSandboxGoneError)) throw err;+ dropLive(name, cached.session.id);+ }+ }+ const coldStart = !liveByName.has(name);+ if (coldStart) await createScratch(name);+ activeScratch.set(name, active + 1);+ return { name, coldStart };+ }++ async function withLive<T>(name: string, action: (live: Live) => Promise<T>): Promise<T> {+ const live = liveByName.get(name);+ if (!live) throw new Error(`superserve sandbox for ${name} is gone; provision it again before using this handle`);+ const pinned = pinnedSandbox.getStore();+ if (pinned && live.session.id !== pinned) throw staleHandle(name);+ try {+ return await action(live);+ } catch (err) {+ if (!(err instanceof SuperserveSandboxGoneError)) throw err;+ dropLive(name, live.session.id);+ const scope = scopeByName.get(name);+ if (scope !== undefined) await forget(scope, live.session.id);+ throw new Error(`superserve sandbox for ${name} is gone; the next provision creates a replacement`, {+ cause: err,+ });+ }+ }++ function spooledScript(script: string, timeoutSec: number): string {+ const cap = OUTPUT_CAP_BYTES;+ const capture = (file: string): string => `{ head -c ${cap} >"${file}"; wc -c >"${file}.rest"; }`;+ return [+ `o=$(mktemp) && e=$(mktemp) && r=$(mktemp) || exit 1`,+ `{ { timeout -k ${KILL_AFTER_SEC} ${timeoutSec} sh -c ${shq(script)}; echo $? >"$r"; } 2>&1 1>&3 3>&- | ${capture("$e")}; } 3>&1 | ${capture("$o")}`,+ `cat "$o"`,+ `cat "$e" >&2`,+ `if [ "$(cat "$o.rest")" -gt 0 ] || [ "$(cat "$e.rest")" -gt 0 ]; then echo ${shq(TRUNCATED_NOTICE)} >&2; fi`,+ `rc=$(cat "$r")`,+ `rm -f "$o" "$e" "$r" "$o.rest" "$e.rest"`,+ `exit "\${rc:-1}"`,+ ].join("; ");+ }++ async function execRaw(name: string, script: string, timeoutSec: number): Promise<ExecResult> {+ return withLive(name, async ({ session }) => {+ const t0 = Date.now();+ const r = await session.run(spooledScript(`export HOME=${shq(configuredHome)}; ${script}`, timeoutSec), {+ timeoutMs: timeoutSec * 1000 + 30_000,+ maxOutputBytes: OUTPUT_CAP_BYTES,+ });+ const elapsedMs = Date.now() - t0;+ const stderr = r.truncated ? `${r.stderr}\n${TRUNCATED_NOTICE}` : r.stderr;+ return {+ stdout: r.stdout,+ stderr,+ code: r.exitCode,+ timedOut:+ r.exitCode === TIMEOUT_EXIT_CODE ||+ (r.exitCode === TIMEOUT_KILLED_EXIT_CODE && elapsedMs >= timeoutSec * 1000),+ };+ });+ }++ const profile: AgentComputerProfile = {+ backend: "superserve",+ writablePersistence: "resident_disk",+ processSessions: true,+ egressEnforcement: "none",+ spec: {+ os: "Ubuntu — Superserve sandbox (disk persists across pause/resume; publish durable work to git or Files)",+ runtimes: ["Node", "Python 3"],+ get tools() {+ return visibleTools([+ "git",+ "curl",+ "jq",+ "tar",+ "python3",+ "node",+ "npm",+ "gh",+ "aws",+ "claude",+ "codex",+ "x-api",+ ...(opts.extraTools ?? []),+ ]);+ },+ get notInstalled() {+ return visibleNotInstalled(["gcloud", "kubectl", "flyctl", "glab"], opts.extraTools ?? []);+ },+ homeDir: configuredHome,+ workdir: workspaceDirOf(configuredHome),+ },+ };++ const procIo: ExecProcessIo = {+ async run(handle, command, execOpts): Promise<ExecResult> {+ const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;+ return execRaw(handle.id, command, timeoutSec);+ },+ };+ const procSessions = createExecProcessSessions(procIo);++ const writeAbsBytes = (name: string, absPath: string, data: Uint8Array): Promise<void> =>+ withLive(name, ({ session }) => session.writeFileBytes(absPath, data));+ const readAbsBytes = (name: string, absPath: string): Promise<Uint8Array | null> =>+ withLive(name, ({ session }) => session.readFileBytes(absPath));+ const installLayerTools = opts.layerToolFiles ? createLayerToolInstaller(opts.layerToolFiles) : null;++ const execFileOps = createExecFileOps({+ label: "superserve",+ exec: (id, script, t) => execRaw(id, script, t),+ writeInline: (id, abs, data) => writeAbsBytes(id, abs, data),+ });++ const execExport = createExecExport({+ label: "superserve",+ exec: (id, script, t) => execRaw(id, script, t),+ readAbsBytes,+ defaultHomeDir: configuredHome,+ ephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),+ });++ const blobStaging = createBackendBlobStaging("superserve", (id, script, t) => execRaw(id, script, t), opts);++ async function destroyStoredScope(scope: string): Promise<void> {+ const name = sandboxScopeName(prefix, scope);+ const stored = await store.get(scope);+ const cached = liveByName.get(name);+ const ids = new Set([stored?.sandboxId, cached?.session.id].filter((id): id is string => !!id));+ const listed = await client.list({ [SUPERSERVE_METADATA.scope]: name });+ for (const summary of listed) ids.add(summary.id);+ for (const id of ids) {+ try {+ await client.kill(id);+ } catch (error) {+ if (!(error instanceof SuperserveSandboxGoneError)) throw error;+ }+ }+ await store.delete(scope);+ liveByName.delete(name);+ scopeByName.delete(name);+ }++ const sandbox: Sandbox = {+ destroyScope(scopeId: string): Promise<void> {+ return provisionQueue(scopeId, () => advisoryLock.withLock(lockKey(scopeId), () => destroyStoredScope(scopeId)));+ },++ profile,+ ...guardHandleOps({+ startProcess: procSessions.startProcess,+ readProcess: procSessions.readProcess,+ writeStdin: procSessions.writeStdin,+ signalProcess: procSessions.signalProcess,+ listProcesses: procSessions.listProcesses,+ }),+ ...guardHandleOps(execFileOps),+ ...(blobStaging ? guardHandleOps(blobStaging) : {}),++ async provision(layers: WorkspaceLayer[], provOpts?: ProvisionOptions): Promise<SandboxHandle> {+ return pinnedSandbox.exit(async () => {+ const scratch = provOpts?.scratch;+ const writable = layers.find((l) => l.mode === "rw") ?? layers[0];+ const scope = writable?.scopeId ?? "default";+ let pendingHandle: SandboxHandle | undefined;+ const provisionAndPrepare = async (): Promise<SandboxHandle> => {+ let name: string;+ let coldStart: boolean;+ if (scratch) {+ ({ name, coldStart } = await ensureScratch(scratch.key));+ } else {+ name = sandboxScopeName(prefix, scope);+ scopeByName.set(name, scope);+ coldStart = (await ensureLive(scope, name, provOpts?.onStatus)).coldStart;+ }+ const homeDir = configuredHome;+ const workspaceDir = workspaceDirOf(homeDir);+ const providerSandboxId = liveByName.get(name)?.session.id;+ const env = Object.fromEntries(+ Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),+ );+ const handle: SandboxHandle = {+ id: name,+ rootDir: workspaceDir,+ homeDir,+ coldStart,+ ...(providerSandboxId ? { providerSandboxId } : {}),+ ...(scratch ? { scratch: true } : {}),+ ...(Object.keys(env).length ? { env } : {}),+ };+ pendingHandle = handle;+ const prepare = async (): Promise<SandboxHandle> => {+ assertCurrent(handle);+ if (!(await ownsGuest(name))) return handle;+ const credLinks = scratch ? "" : ` && ${ephemeralCredLinkScript(homeDir, opts.credentialPaths ?? [])}`;+ const prep = await execRaw(name, `mkdir -p ${shq(workspaceDir)}${credLinks}`, PREP_TIMEOUT_SEC);+ if (prep.code !== 0)+ throw new Error(`superserve provision prep failed: ${(prep.stderr || prep.stdout).slice(0, 200)}`);+ await materializeRoLayers(+ workspace,+ layers,+ handle,+ {+ readFile: (h, rel) => sandbox.readFile(h, rel),+ writeFileBytes: (h, rel, data) => sandbox.writeFileBytes(h, rel, data),+ exec: (script, t) => execRaw(name, script, t),+ },+ { manifest: RO_LAYERS_MANIFEST, tar: RO_LAYERS_TAR, label: "superserve" },+ );+ if (installLayerTools)+ await installLayerTools({+ exec: (script, t) => execRaw(name, script, t),+ writeAbs: (abs, data) => writeAbsBytes(name, abs, data),+ });+ assertCurrent(handle);+ return handle;+ };+ return providerSandboxId ? pinnedSandbox.run(providerSandboxId, prepare) : prepare();+ };+ try {+ return await (scratch+ ? provisionQueue(`scratch:${scratch.key}`, provisionAndPrepare)+ : provisionQueue(scope, () => advisoryLock.withLock(lockKey(scope), provisionAndPrepare)));+ } catch (err) {+ if (pendingHandle)+ await sandbox+ .teardown(pendingHandle)+ .catch(swallowAs("superserve-sandbox: teardown after failed provision", undefined));+ throw err;+ }+ });+ },++ async run(handle, command, execOpts?: ExecOptions): Promise<ExecResult> {+ return pinnedToHandle(handle, async () => {+ const timeoutSec = execOpts?.timeoutMs ? Math.ceil(execOpts.timeoutMs / 1000) : defaultTimeoutSec;+ const exports = Object.entries(handle.env ?? {})+ .map(([k, v]) => `export ${k}=${shq(v)}`)+ .join("; ");+ const script = `${nonInteractiveShellPrefix()}${exports ? exports + "; " : ""}cd ${shq(handle.rootDir)} || exit 1; ${command}`;+ const signal = execOpts?.signal;+ if (!signal) return execRaw(handle.id, script, timeoutSec);+ const killUid = randomUUID();+ const fireKill = () => {+ execRaw(handle.id, killScript(killUid), 15).catch(+ swallowAs("superserve-sandbox: kill in-flight exec", undefined),+ );+ };+ signal.throwIfAborted();+ const onAbort = () => fireKill();+ signal.addEventListener("abort", onAbort, { once: true });+ try {+ return await execRaw(handle.id, killableScript(script, killUid), timeoutSec);+ } finally {+ signal.removeEventListener("abort", onAbort);+ }+ });+ },++ async writeFileBytes(handle, relPath, data): Promise<void> {+ await pinnedToHandle(handle, () => writeAbsBytes(handle.id, posixJoin(handle.rootDir, relPath), data));+ },+ async writeFile(handle, relPath, data): Promise<void> {+ await sandbox.writeFileBytes(handle, relPath, Buffer.from(data, "utf8"));+ },+ async readFileBytes(handle, relPath): Promise<Uint8Array | null> {+ return pinnedToHandle(handle, () => readAbsBytes(handle.id, posixJoin(handle.rootDir, relPath)));+ },+ async readFile(handle, relPath): Promise<string | null> {+ const bytes = await sandbox.readFileBytes(handle, relPath);+ return bytes === null ? null : Buffer.from(bytes).toString("utf8");+ },++ exportFiles: (handle, exportOpts) => pinnedToHandle(handle, () => execExport.exportFiles(handle, exportOpts)),++ async computerStatus(scopeId: string): Promise<ComputerStatus> {+ const name = sandboxScopeName(prefix, scopeId);+ const stored = await store.get(scopeId);+ if (!stored) return { machine: "no sandbox provisioned yet", provisioned: false, guestResponsive: false };+ const machineOf = (sandboxId: string): string => `superserve sandbox ${sandboxId}`;+ const recovery = { strategy: "provider_pause" as const };+ let inspectedId = stored.sandboxId;+ try {+ const info = await client.info(stored.sandboxId, scopeFilter(name));+ if (info.status === "paused" || info.status === "pausing")+ return {+ machine: machineOf(stored.sandboxId),+ listed: info.status,+ lifecycleState: "paused",+ provisioned: true,+ guestResponsive: false,+ recovery: { ...recovery, state: info.status, checkpointExpiresAtMs: info.autoDeleteAtMs ?? null },+ };+ scopeByName.set(name, scopeId);+ const { live } = await provisionQueue(scopeId, () =>+ advisoryLock.withLock(lockKey(scopeId), () => ensureLive(scopeId, name)),+ );+ inspectedId = live.session.id;+ const probed = inspectedId === stored.sandboxId ? info : await client.info(inspectedId, scopeFilter(name));+ const r = await live.session.run("echo responsive", { timeoutMs: 30_000 });+ return {+ machine: machineOf(inspectedId),+ listed: probed.status,+ lifecycleState: "running",+ recovery: { ...recovery, state: probed.status },+ provisioned: true,+ guestResponsive: r.exitCode === 0 && /responsive/.test(r.stdout),+ };+ } catch (e) {+ const gone = e instanceof SuperserveSandboxGoneError;+ if (gone) {+ dropLive(name, inspectedId);+ await forget(scopeId, inspectedId);+ }+ return {+ recovery,+ machine: `${machineOf(inspectedId)} (${errMessage(e).slice(0, 120)})`,+ provisioned: !gone,+ guestResponsive: false,+ };+ }+ },++ async teardown(handle, tdOpts?: TeardownOptions): Promise<void> {+ if (handle.scratch) {+ const key = scratchKeyByName.get(handle.id);+ return provisionQueue(key ? `scratch:${key}` : handle.id, async () => {+ const remaining = (activeScratch.get(handle.id) ?? 1) - 1;+ if (remaining > 0) {+ activeScratch.set(handle.id, remaining);+ return;+ }+ activeScratch.delete(handle.id);+ const live = liveByName.get(handle.id);+ if (!live) return;+ try {+ await live.session.kill();+ } catch (err) {+ if (tdOpts?.destroy) throw err;+ swallowAs("superserve-sandbox: scratch kill", undefined)(err);+ }+ dropLive(handle.id, live.session.id);+ });+ }+ if (tdOpts?.destroy && !scopeByName.has(handle.id)) return;+ const scope = scopeByName.get(handle.id) ?? "default";+ return provisionQueue(scope, async () => {+ if (!tdOpts?.destroy && !handleIsCurrent(handle)) return;+ await teardownScope(handle.id, scope, tdOpts);+ });+ },+ };++ async function teardownScope(name: string, scope: string, tdOpts?: TeardownOptions): Promise<void> {+ if (tdOpts?.destroy) return advisoryLock.withLock(lockKey(scope), () => destroyStoredScope(scope));+ const live = liveByName.get(name);+ if (!live?.current) return;+ return advisoryLock.withLock(lockKey(scope), async () => {+ try {+ const info = await client.info(live.session.id, scopeFilter(name));+ live.current = Number(info.metadata[SUPERSERVE_METADATA.epoch] ?? 0) <= (await configEpoch());+ if (!live.current) return;+ await live.session.update({ timeoutSeconds: tdOpts?.keepWarm ? keepWarmSec : idlePauseSec });+ } catch (err) {+ if (!(err instanceof SuperserveSandboxGoneError)) throw err;+ dropLive(name, live.session.id);+ await forget(scope, live.session.id);+ }+ });+ }++ return sandbox;+}diff --git a/src/wiring.ts b/src/wiring.tsindex cda80aba4..c3dde7f93 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -43,7 +43,7 @@ import type { SessionShare, SessionShareStore } from "./sessions/session-share.timport { createModelOverlayStore, type ModelOverlayStore } from "./model/model-overlay-store.ts";import { mkdirSync } from "node:fs";import type { StagedEnvelope } from "./slack/envelope-staging.ts";-import { randomBytes, randomUUID } from "node:crypto";+import { createHash, randomBytes, randomUUID } from "node:crypto";import { join, resolve } from "node:path";import {baseModelProviders,@@ -201,6 +201,13 @@ import { createLocalSandbox } from "./sandbox/local-sandbox.ts";import { createSpritesSandbox } from "./sandbox/sprites-sandbox.ts";import { createSmolmachinesSandbox } from "./sandbox/smolmachines-sandbox.ts";import { createAgent37Sandbox } from "./sandbox/agent37-sandbox.ts";+import {+ createConfigEpochResolver,+ createSuperserveSandbox,+ type StoredConfigEpoch,+ type StoredSuperserveSandbox,+} from "./sandbox/superserve-sandbox.ts";+import { createSdkSuperserveClient } from "./sandbox/superserve-client.ts";import { createE2bSandbox, type StoredE2bSandbox } from "./sandbox/e2b-sandbox.ts";import { createSdkE2bClient } from "./sandbox/e2b-client.ts";import { createS3SnapshotStore } from "./sandbox/home-snapshot.ts";@@ -924,6 +931,60 @@ export function buildApp(...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}),onError: sandboxOnError,});+ const superserveBodies = artifactMap<StoredSuperserveSandbox>("superserve_sandbox_bodies");+ const superserveEpochs = artifactMap<StoredConfigEpoch>("superserve_config_epochs");+ const buildSuperserve = (): Sandbox => {+ const ss = config.superserveSandbox;+ if (!ss.apiKey) throw new Error("SANDBOX_BACKEND=superserve requires SUPERSERVE_API_KEY");+ if (!ss.template)+ throw new Error("SANDBOX_BACKEND=superserve requires SUPERSERVE_TEMPLATE (a ready qm-agent-<release> template)");+ const keepWarmSec = Math.ceil(config.backgroundJobTtlMaxMs / 1000);+ const generationKey = createHash("sha256")+ .update(+ JSON.stringify([+ config.buildSha ?? "",+ ss.template,+ ss.namePrefix ?? "",+ ss.homeDir ?? "",+ ss.idlePauseSec ?? null,+ ss.retentionSec ?? null,+ keepWarmSec,+ [...(ss.egressAllow ?? [])].sort(),+ [...(ss.egressDeny ?? [])].sort(),+ ]),+ )+ .digest("hex")+ .slice(0, 32);+ return createSuperserveSandbox(workspace, {+ configEpoch:+ ss.configGeneration ??+ (pgArtifactMap ? createConfigEpochResolver(superserveEpochs, generationKey, advisoryLock) : 0),+ client: createSdkSuperserveClient({+ apiKey: ss.apiKey,+ ...(ss.baseUrl ? { baseUrl: ss.baseUrl } : {}),+ ...(ss.template ? { template: ss.template } : {}),+ }),+ ...(ss.namePrefix ? { namePrefix: ss.namePrefix } : {}),+ template: ss.template,+ ...(ss.homeDir ? { homeDir: ss.homeDir } : {}),+ ...(ss.idlePauseSec !== undefined ? { idlePauseSec: ss.idlePauseSec } : {}),+ keepWarmSec,+ ...(ss.retentionSec !== undefined ? { retentionSec: ss.retentionSec } : {}),+ ...(ss.egressAllow ? { egressAllow: ss.egressAllow } : {}),+ ...(ss.egressDeny ? { egressDeny: ss.egressDeny } : {}),+ ...(ss.defaultTimeoutSec ? { defaultTimeoutSec: ss.defaultTimeoutSec } : {}),+ advisoryLock,+ extraTools: deploymentLayer.advertisedTools,+ credentialPaths: deploymentLayer.credentialPaths,+ layerToolFiles: () => deploymentLayer.installFiles,+ blobTransfer,+ ...(config.signingSecret ? { signingSecret: config.signingSecret } : {}),+ ...(config.capabilitySecret ? { capabilitySecret: config.capabilitySecret } : {}),+ ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}),+ store: superserveBodies,+ onError: sandboxOnError,+ });+ };const buildAws = (): Sandbox => {if (!config.awsSandbox.s3Bucket) throw new Error("SANDBOX_BACKEND=aws requires AWS_SANDBOX_S3_BUCKET");return createAwsSandbox(workspace, {@@ -961,6 +1022,7 @@ export function buildApp(aws: buildAws,porter: buildPorter,agent37: buildAgent37,+ superserve: buildSuperserve,};const enabledBackends = new Set(enabledSandboxBackends(config));const sandboxBackends: Partial<Record<SandboxBackendName, Sandbox>> = {@@ -978,11 +1040,21 @@ export function buildApp(rollout: artifactMap<SandboxResourceRollout>("sandbox_resource_rollout"),legacyScopes: async () => (await sessions.distinctScopes()).map((scope) => scope.scopeId),legacySandboxes: async () => {- const [e2b, modal, aws] = await Promise.all([e2bBodies.entries(), modalBodies.entries(), awsBodies.entries()]);+ const [e2b, modal, aws, superserve] = await Promise.all([+ e2bBodies.entries(),+ modalBodies.entries(),+ awsBodies.entries(),+ superserveBodies.entries(),+ ]);return [...e2b.map(([scopeId, body]) => ({ scopeId, backend: "e2b" as const, machineId: body.sandboxId })),...modal.map(([scopeId, body]) => ({ scopeId, backend: "modal" as const, machineId: body.sandboxId })),...aws.map(([scopeId, body]) => ({ scopeId, backend: "aws" as const, machineId: body.microvmId })),+ ...superserve.map(([scopeId, body]) => ({+ scopeId,+ backend: "superserve" as const,+ machineId: body.sandboxId,+ })),];},records: artifactMap<SandboxResource>("sandbox_resources"),diff --git a/superserve/templates/README.md b/superserve/templates/README.mdnew file mode 100644index 000000000..2b4c6cf20--- /dev/null+++ b/superserve/templates/README.md@@ -0,0 +1,59 @@+# Agent template++`qm-agent.ts` builds the `qm-agent-<release>` template used by the Superserve+sandbox backend.++## Contents++The template uses `ubuntu:24.04` on linux/amd64 and includes:++- Shell and file utilities, Git, curl, jq, and SSH.+- Node and npm, Python with a virtual environment at `/opt/agent-venv`.+- Coding-agent CLIs, `gh`, `aws`, and `x-api`.++Tool versions are pinned in the build script. Downloaded Node and CLI archives+are checksum-verified. Python and pip wrappers select the virtual environment+regardless of the exec environment's `PATH`. Commands run as root; QM sets `HOME`+to `SUPERSERVE_HOME_DIR` (default `/root`) and uses `<home>/workspace`.++Deployment tools and skills are installed during provisioning. The optional+browser engine is not included.++The default Superserve sandbox has 2 vCPUs, 2048 MiB memory, and 8192 MiB disk. Override these with+`--vcpu`, `--memory-mib`, and `--disk-mib` when building.++## Build++```sh+export SUPERSERVE_API_KEY=your-superserve-api-key+node superserve/templates/qm-agent.ts --release 0.1.0 --wait+```++`--release` names the template; use the release tag your deployment runs.+`--wait` streams logs until the build completes. Without it, the script returns+after queuing the build. Use `--base-url` or `SUPERSERVE_BASE_URL` to override the+API endpoint.++A ready template with the same name is reused. Failed builds are replaced;+`--force` also replaces a ready template. Names are unique within a team.++Set `SUPERSERVE_TEMPLATE=qm-agent-<release>` on core after the build succeeds.+The backend requires a ready template with this toolchain.++Rebuilding under the same name does not update existing sandboxes. Changing+`SUPERSERVE_TEMPLATE` to a different name replaces them and deletes their resident+files. Export needed files first; see [updates and retention](../../docs/superserve.md#updates-and-retention).++## Verify++```sh+node superserve/templates/verify-qm-agent.ts --release 0.1.0+```++The verifier creates a sandbox, checks its tools, workspace, Python environment,+and command timeout behavior, then deletes it. It exits nonzero if verification+or cleanup fails. `--template` selects a template directly instead of by release.++`--keep` retains the sandbox for inspection and schedules deletion one hour after+it pauses. Otherwise, automatic deletion on pause backs up explicit cleanup,+including when the verifier is interrupted.diff --git a/superserve/templates/common.ts b/superserve/templates/common.tsnew file mode 100644index 000000000..8cef14e1c--- /dev/null+++ b/superserve/templates/common.ts@@ -0,0 +1,47 @@+import { dirname, join } from "node:path";+import { fileURLToPath } from "node:url";+import { Template, type TemplateInfo } from "@superserve/sdk";++const HERE = dirname(fileURLToPath(import.meta.url));+export const REPO_ROOT = join(HERE, "..", "..");++const TEMPLATE_NAME_PREFIX = "qm-agent";++export interface Connection {+ apiKey: string;+ baseUrl?: string;+}++export function resolveConnection(baseUrlFlag?: string): Connection {+ const apiKey = process.env.SUPERSERVE_API_KEY?.trim();+ if (!apiKey) throw new Error("SUPERSERVE_API_KEY is required");+ const baseUrl = (baseUrlFlag ?? process.env.SUPERSERVE_BASE_URL)?.trim();+ return baseUrl ? { apiKey, baseUrl } : { apiKey };+}++export function requireRelease(value?: string): string {+ const release = value?.trim();+ if (!release) {+ throw new Error(+ "--release <qm-release> is required: the QM release tag the deployment runs, naming the template qm-agent-<release>",+ );+ }+ return release;+}++export function templateNameForRelease(release: string): string {+ const cleaned = release.trim().replace(/^v/, "");+ if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(cleaned)) {+ throw new Error(`release ${JSON.stringify(release)} is not a valid template name suffix`);+ }+ return `${TEMPLATE_NAME_PREFIX}-${cleaned}`;+}++export function fmtMs(ms: number): string {+ return ms < 10_000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(1)}s`;+}++export async function findTemplateByName(name: string, conn: Connection): Promise<TemplateInfo | undefined> {+ const matches = await Template.list({ ...conn, namePrefix: name });+ return matches.find((t) => t.name === name);+}diff --git a/superserve/templates/qm-agent.ts b/superserve/templates/qm-agent.tsnew file mode 100644index 000000000..cb1d79a6d--- /dev/null+++ b/superserve/templates/qm-agent.ts@@ -0,0 +1,262 @@+#!/usr/bin/env node+import { readFileSync } from "node:fs";+import { join } from "node:path";+import { parseArgs } from "node:util";+import { BuildError, Template, type BuildStep, type TemplateInfo } from "@superserve/sdk";+import {+ REPO_ROOT,+ requireRelease,+ findTemplateByName,+ fmtMs,+ resolveConnection,+ templateNameForRelease,+} from "./common.ts";++const CLAUDE_CODE_VERSION = "2.1.210";+const CODEX_VERSION = "0.144.4";+const GH_VERSION = "2.93.0";+const GH_SHA256_AMD64 = "02d1290eba130e0b896f3709ffff22e1c75a51475ddb70476a85abc6b5807af0";+const AWSCLI_VERSION = "2.34.54";+const AWSCLI_SHA256_X86_64 = "de278754dec97e0f6e9b4e8167d4bd1a27004c3e56e9c2da10002597f76ca35a";+const NODE_VERSION = "24.18.0";+const NODE_SHA256_X64 = "55aa7153f9d88f28d765fcdad5ae6945b5c0f98a36881703817e4c450fa76742";+const AGENT_VENV = "/opt/agent-venv";+const BASE_IMAGE = "ubuntu:24.04";++const DEFAULT_VCPU = 2;+const DEFAULT_MEMORY_MIB = 2048;+const DEFAULT_DISK_MIB = 8192;++const APT_PACKAGES = [+ "bash",+ "coreutils",+ "findutils",+ "grep",+ "sed",+ "gawk",+ "git",+ "curl",+ "wget",+ "jq",+ "unzip",+ "tar",+ "xz-utils",+ "openssh-client",+ "python3",+ "python3-venv",+ "python3-pip",+ "ca-certificates",+ "gnupg",+];++function sh(lines: string[]): string {+ return ["set -eu", ...lines].join("\n");+}++function buildSteps(): BuildStep[] {+ const xApi = readFileSync(join(REPO_ROOT, "fly", "tools", "x-api"), "utf8");+ if (xApi.includes("\nQM_X_API_EOF\n")) throw new Error("x-api contains the heredoc delimiter");+ return [+ { env: { key: "DEBIAN_FRONTEND", value: "noninteractive" } },+ {+ run: sh([+ "apt-get update",+ `apt-get install -y --no-install-recommends ${APT_PACKAGES.join(" ")}`,+ "rm -rf /var/lib/apt/lists/*",+ ]),+ },+ {+ run: sh([+ `curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" -o /tmp/node.tar.xz`,+ `echo "${NODE_SHA256_X64} /tmp/node.tar.xz" | sha256sum -c -`,+ "tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner" ++ " --exclude=CHANGELOG.md --exclude=LICENSE --exclude=README.md",+ "rm -f /tmp/node.tar.xz",+ "node --version",+ "npm --version",+ ]),+ },+ {+ run: sh([+ "npm install -g --allow-scripts=@anthropic-ai/claude-code" ++ ` @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} @openai/codex@${CODEX_VERSION}`,+ "npm cache clean --force",+ "claude --version",+ "codex --version",+ ]),+ },+ {+ run: sh([+ `curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" -o /tmp/gh.tgz`,+ `echo "${GH_SHA256_AMD64} /tmp/gh.tgz" | sha256sum -c -`,+ "tar xzf /tmp/gh.tgz -C /tmp",+ `install "/tmp/gh_${GH_VERSION}_linux_amd64/bin/gh" /usr/local/bin/gh`,+ "rm -rf /tmp/gh*",+ "gh --version",+ "rm -rf /root/.local/state/gh",+ ]),+ },+ {+ run: sh([+ `curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWSCLI_VERSION}.zip" -o /tmp/awscliv2.zip`,+ `echo "${AWSCLI_SHA256_X86_64} /tmp/awscliv2.zip" | sha256sum -c -`,+ "unzip -q /tmp/awscliv2.zip -d /tmp",+ "/tmp/aws/install",+ "rm -rf /tmp/aws /tmp/awscliv2.zip",+ "aws --version",+ ]),+ },+ {+ run: sh([+ `python3 -m venv ${AGENT_VENV}`,+ `${AGENT_VENV}/bin/pip install --no-cache-dir --upgrade pip`,+ `${AGENT_VENV}/bin/python --version`,+ ]),+ },+ {+ run: sh([+ "for t in python python3 pip pip3; do",+ ` printf '#!/bin/sh\\nexec ${AGENT_VENV}/bin/%s "$@"\\n' "$t" > "/usr/local/bin/$t"`,+ ' chmod +x "/usr/local/bin/$t"',+ "done",+ `/usr/local/bin/python -c 'import sys; print(sys.prefix)' | grep -qx ${AGENT_VENV}`,+ "/usr/local/bin/pip --version",+ ]),+ },+ { env: { key: "VIRTUAL_ENV", value: AGENT_VENV } },+ { env: { key: "PATH", value: `${AGENT_VENV}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` } },+ {+ run: sh([+ "cat > /usr/local/bin/x-api <<'QM_X_API_EOF'",+ xApi.replace(/\n$/, ""),+ "QM_X_API_EOF",+ "chmod +x /usr/local/bin/x-api",+ ]),+ },+ { workdir: "/root/workspace" },+ ];+}++interface Args {+ release: string;+ wait: boolean;+ force: boolean;+ baseUrl?: string;+ vcpu: number;+ memoryMib: number;+ diskMib: number;+}++function parseCli(argv: string[]): Args {+ const { values } = parseArgs({+ args: argv,+ options: {+ release: { type: "string" },+ wait: { type: "boolean", default: false },+ force: { type: "boolean", default: false },+ "base-url": { type: "string" },+ vcpu: { type: "string" },+ "memory-mib": { type: "string" },+ "disk-mib": { type: "string" },+ help: { type: "boolean", default: false },+ },+ });+ if (values.help) {+ console.log(+ [+ "usage: node superserve/templates/qm-agent.ts --release <qm-release> [--wait] [--force]",+ " [--base-url <url>] [--vcpu N] [--memory-mib N] [--disk-mib N]",+ "env: SUPERSERVE_API_KEY (required), SUPERSERVE_BASE_URL (optional)",+ ].join("\n"),+ );+ process.exit(0);+ }+ const num = (flag: string, raw: string | undefined, fallback: number): number => {+ if (raw === undefined) return fallback;+ const n = Number(raw);+ if (!Number.isInteger(n) || n <= 0) throw new Error(`--${flag} must be a positive integer`);+ return n;+ };+ return {+ release: requireRelease(values.release),+ wait: values.wait,+ force: values.force,+ ...(values["base-url"] ? { baseUrl: values["base-url"] } : {}),+ vcpu: num("vcpu", values.vcpu, DEFAULT_VCPU),+ memoryMib: num("memory-mib", values["memory-mib"], DEFAULT_MEMORY_MIB),+ diskMib: num("disk-mib", values["disk-mib"], DEFAULT_DISK_MIB),+ };+}++function describe(info: TemplateInfo | Template): string {+ const size = info.sizeBytes ? ` size=${(info.sizeBytes / 1024 / 1024).toFixed(0)}MiB` : "";+ return `${info.name} id=${info.id} status=${info.status} shape=${info.vcpu}vcpu/${info.memoryMib}MiB/${info.diskMib}MiB${size}`;+}++async function findExisting(name: string, conn: ReturnType<typeof resolveConnection>): Promise<Template | undefined> {+ const info = await findTemplateByName(name, conn);+ return info ? Template.connect(info.id, conn) : undefined;+}++async function waitForBuild(template: Template): Promise<TemplateInfo> {+ const started = Date.now();+ const info = await template.waitUntilReady({+ onLog: (ev) => {+ const text = ev.text.replace(/\n$/, "");+ if (text) console.log(`[build:${ev.stream}] ${text}`);+ },+ });+ console.log(`[qm-agent] build finished in ${fmtMs(Date.now() - started)}`);+ return info;+}++async function main(): Promise<void> {+ const args = parseCli(process.argv.slice(2));+ const conn = resolveConnection(args.baseUrl);+ const name = templateNameForRelease(args.release);+ console.log(`[qm-agent] template=${name} base=${BASE_IMAGE}${conn.baseUrl ? ` api=${conn.baseUrl}` : ""}`);++ const existing = await findExisting(name, conn);+ if (existing) {+ if (existing.status === "ready" && !args.force) {+ console.log(`[qm-agent] already ready: ${describe(existing)}`);+ return;+ }+ if (existing.status === "pending" || existing.status === "building") {+ if (!args.force) {+ console.log(`[qm-agent] build in progress: ${describe(existing)}`);+ if (args.wait) console.log(`[qm-agent] ready: ${describe(await waitForBuild(existing))}`);+ return;+ }+ if (existing.latestBuildId) await existing.cancelBuild(existing.latestBuildId).catch(() => undefined);+ }+ console.log(`[qm-agent] deleting existing template (${existing.status}${args.force ? ", --force" : ""})`);+ await existing.delete();+ }++ const template = await Template.create({+ ...conn,+ name,+ from: BASE_IMAGE,+ vcpu: args.vcpu,+ memoryMib: args.memoryMib,+ diskMib: args.diskMib,+ steps: buildSteps(),+ });+ console.log(`[qm-agent] build queued: ${describe(template)} build=${template.latestBuildId ?? "?"}`);+ if (!args.wait) {+ console.log(`[qm-agent] re-run with --wait to block on the build, or verify later with verify-qm-agent.ts`);+ return;+ }+ const info = await waitForBuild(template);+ console.log(`[qm-agent] ready: ${describe(info)}`);+}++main().catch((e: unknown) => {+ if (e instanceof BuildError) {+ console.error(`[qm-agent] build failed (${e.code}) build=${e.buildId}: ${e.message}`);+ } else {+ console.error(`[qm-agent] ${e instanceof Error ? e.message : String(e)}`);+ }+ process.exitCode = 1;+});diff --git a/superserve/templates/verify-qm-agent.ts b/superserve/templates/verify-qm-agent.tsnew file mode 100644index 000000000..34f49d68c--- /dev/null+++ b/superserve/templates/verify-qm-agent.ts@@ -0,0 +1,220 @@+#!/usr/bin/env node+import { setTimeout as sleep } from "node:timers/promises";+import { parseArgs } from "node:util";+import { Sandbox } from "@superserve/sdk";+import { shq } from "../../src/util/shell.ts";+import { requireRelease, findTemplateByName, fmtMs, resolveConnection, templateNameForRelease } from "./common.ts";++const EXPECTED_TOOLS = [+ "sh",+ "bash",+ "git",+ "curl",+ "wget",+ "jq",+ "tar",+ "xz",+ "unzip",+ "timeout",+ "ssh",+ "python3",+ "pip",+ "node",+ "npm",+ "claude",+ "codex",+ "gh",+ "aws",+ "x-api",+];++const VERSION_COMMANDS: Record<string, string> = {+ node: "node --version",+ npm: "npm --version",+ python3: "python3 --version",+ "agent-venv python": "/opt/agent-venv/bin/python --version",+ "agent-venv pip": "/opt/agent-venv/bin/pip --version",+ "python (PATH)": "python --version",+ "pip (PATH)": "pip --version",+ claude: "claude --version",+ codex: "codex --version",+ gh: "gh --version",+ aws: "aws --version",+ git: "git --version",+};++const SANDBOX_TIMEOUT_SECONDS = 300;+const RELEASE_DEADLINE_MS = 15_000;+const KEEP_AUTO_DELETE_SECONDS = 3_600;+const SIGNALS = ["SIGINT", "SIGTERM"] as const;+const SIGNAL_EXIT_CODES: Record<string, number> = { SIGINT: 130, SIGTERM: 143 };++interface Args {+ template: string;+ keep: boolean;+ baseUrl?: string;+}++function parseCli(argv: string[]): Args {+ const { values } = parseArgs({+ args: argv,+ options: {+ release: { type: "string" },+ template: { type: "string" },+ keep: { type: "boolean", default: false },+ "base-url": { type: "string" },+ help: { type: "boolean", default: false },+ },+ });+ if (values.help) {+ console.log(+ [+ "usage: node superserve/templates/verify-qm-agent.ts [--release <qm-release> | --template <name>] [--keep] [--base-url <url>]",+ "env: SUPERSERVE_API_KEY (required), SUPERSERVE_BASE_URL (optional)",+ ].join("\n"),+ );+ process.exit(0);+ }+ return {+ template: values.template ?? templateNameForRelease(requireRelease(values.release)),+ keep: values.keep,+ ...(values["base-url"] ? { baseUrl: values["base-url"] } : {}),+ };+}++async function run(sandbox: Sandbox, script: string): Promise<{ exitCode: number; stdout: string; stderr: string }> {+ const r = await sandbox.commands.run(`timeout 120 sh -c ${shq(script)}`, { timeoutMs: 150_000 });+ return { exitCode: r.exitCode, stdout: r.stdout, stderr: r.stderr };+}++async function main(): Promise<void> {+ const args = parseCli(process.argv.slice(2));+ const conn = resolveConnection(args.baseUrl);+ const failures: string[] = [];+ const fail = (msg: string): void => {+ failures.push(msg);+ console.log(`FAIL ${msg}`);+ };++ const template = await findTemplateByName(args.template, conn);+ if (!template) throw new Error(`template ${args.template} not found (build it with qm-agent.ts first)`);+ if (template.status !== "ready") throw new Error(`template ${args.template} is ${template.status}, not ready`);+ const sandboxName = `qm-agent-verify-${Date.now().toString(36)}`;+ console.log(+ `[verify] template=${template.name} id=${template.id} sandbox=${sandboxName}${conn.baseUrl ? ` api=${conn.baseUrl}` : ""}`,+ );++ const t0 = Date.now();+ const creating = Sandbox.create({+ ...conn,+ name: sandboxName,+ fromTemplate: { id: template.id, name: template.name },+ metadata: { qm_kind: "verify", qm_template: args.template },+ timeoutSeconds: SANDBOX_TIMEOUT_SECONDS,+ autoDeleteSeconds: args.keep ? KEEP_AUTO_DELETE_SECONDS : 0,+ });++ const releaseOnce = async (): Promise<void> => {+ const created = await creating.catch(() => undefined);+ if (!created) return;+ if (args.keep) {+ console.log(+ `[verify] keeping sandbox ${created.id} (--keep; auto-deletes ${KEEP_AUTO_DELETE_SECONDS}s after it pauses)`,+ );+ return;+ }+ try {+ await created.kill();+ console.log(`[verify] killed sandbox ${created.id}`);+ } catch (e: unknown) {+ fail(+ `could not kill sandbox ${created.id}: ${e instanceof Error ? e.message : String(e)}; ` ++ "it is deleted as soon as it pauses",+ );+ }+ };+ let releasing: Promise<void> | undefined;+ const release = (): Promise<void> => (releasing ??= releaseOnce());+ const onSignal = (signal: NodeJS.Signals): void => {+ console.error(`[verify] ${signal} received; releasing sandbox ${sandboxName}`);+ void Promise.race([release(), sleep(RELEASE_DEADLINE_MS)]).then(() => process.exit(SIGNAL_EXIT_CODES[signal] ?? 1));+ };+ for (const signal of SIGNALS) process.on(signal, onSignal);++ const sandbox = await creating;+ const tCreated = Date.now();++ try {+ const first = await sandbox.commands.run("true", { timeoutMs: 60_000 });+ const tFirst = Date.now();+ console.log(+ `[verify] sandbox=${sandbox.id} create=${fmtMs(tCreated - t0)} first-exec=${fmtMs(tFirst - tCreated)} ` ++ `cold-boot-to-first-exec=${fmtMs(tFirst - t0)} (exit=${first.exitCode})`,+ );+ if (first.exitCode !== 0) fail(`first exec exited ${first.exitCode}: ${first.stderr.trim()}`);++ console.log("--- environment ---");+ const env = await run(+ sandbox,+ 'printf "HOME=%s\\nUSER=%s\\nwhoami=%s\\nuname=%s\\npwd=%s\\nPATH=%s\\nVIRTUAL_ENV=%s\\n" "${HOME:-}" "${USER:-}" "$(whoami)" "$(uname -a)" "$(pwd)" "$PATH" "${VIRTUAL_ENV:-}"',+ );+ process.stdout.write(env.stdout);+ if (env.exitCode !== 0) fail(`environment probe exited ${env.exitCode}: ${env.stderr.trim()}`);+ const home = /^HOME=(.*)$/m.exec(env.stdout)?.[1] ?? "";+ if (!home.startsWith("/")) fail(`$HOME is not set for the login user (got ${JSON.stringify(home)})`);+ const whoami = /^whoami=(.*)$/m.exec(env.stdout)?.[1] ?? "";+ if (whoami.trim() !== "root") {+ fail(`expected the template to run as root; got whoami=${JSON.stringify(whoami)}`);+ }+ const workspaceOk = await run(sandbox, 'test -d "/root/workspace" && test -w "/root/workspace" && echo writable');+ console.log(`/root/workspace exists and is writable: ${workspaceOk.stdout.trim() === "writable" ? "yes" : "no"}`);+ if (workspaceOk.stdout.trim() !== "writable") {+ fail("/root/workspace is missing or not writable");+ }++ console.log("--- tool inventory ---");+ const inv = await run(+ sandbox,+ `for t in ${EXPECTED_TOOLS.join(" ")}; do if p=$(command -v "$t" 2>/dev/null); then echo "ok $t $p"; else echo "missing $t"; fi; done`,+ );+ process.stdout.write(inv.stdout);+ for (const tool of EXPECTED_TOOLS) {+ if (!new RegExp(`^ok ${tool} `, "m").test(inv.stdout)) fail(`missing tool: ${tool}`);+ }++ console.log("--- versions ---");+ for (const [label, cmd] of Object.entries(VERSION_COMMANDS)) {+ const r = await run(sandbox, cmd);+ const out = (r.stdout.trim() || r.stderr.trim()).split("\n")[0] ?? "";+ console.log(`${r.exitCode === 0 ? "ok" : "err"} ${label}: ${out}`);+ if (r.exitCode !== 0) fail(`${label}: ${cmd} exited ${r.exitCode}`);+ }++ console.log("--- backend exec shape ---");+ const to = await run(sandbox, "timeout 2 sh -c 'sleep 5'; echo exit=$?");+ console.log(`timeout kills a long command: ${to.stdout.trim()} (expect exit=124)`);+ if (!to.stdout.includes("exit=124")) fail("timeout(1) did not terminate the command with exit 124");+ const venv = await run(sandbox, "command -v python && python -c 'import sys; print(sys.prefix)'");+ console.log(`venv on PATH: ${venv.stdout.trim().replace(/\n/g, " ")}`);+ if (!venv.stdout.includes("/opt/agent-venv")) fail("/opt/agent-venv is not the default python on PATH");+ const pip = await run(sandbox, "pip --version");+ console.log(`pip on PATH: ${pip.stdout.trim()}`);+ if (!pip.stdout.includes("/opt/agent-venv")) fail("pip on PATH is not the /opt/agent-venv pip");+ } finally {+ await release();+ for (const signal of SIGNALS) process.off(signal, onSignal);+ }++ if (failures.length) {+ console.log(`[verify] FAILED: ${failures.length} problem(s)`);+ for (const f of failures) console.log(` - ${f}`);+ process.exitCode = 1;+ } else {+ console.log(`[verify] OK: ${EXPECTED_TOOLS.length} tools present`);+ }+}++main().catch((e: unknown) => {+ console.error(`[verify] ${e instanceof Error ? e.message : String(e)}`);+ process.exitCode = 1;+});diff --git a/test/config.test.ts b/test/config.test.tsindex 94e6ba4c4..d1ce62239 100644--- a/test/config.test.ts+++ b/test/config.test.ts@@ -5,6 +5,7 @@ import {harnessCarriedModelAuth,baseModelProviders,boolEnv,+ enabledSandboxBackends,loadConfig,numEnv,CONFIG_DEFAULTS,@@ -388,6 +389,18 @@ test("HARNESS=claude uses native Claude authentication and does not require an Aassert.equal(loadConfig({ HARNESS: "claude", CLAUDE_MODEL: "claude-opus-4-8" }).claudeModel, "claude-opus-4-8");});+test("SUPERSERVE_CONFIG_GENERATION accepts only nonnegative safe integers", () => {+ for (const value of ["9007199254740993", "-1", "0.5", "NaN", "Infinity"]) {+ assert.throws(() => loadConfig({ SUPERSERVE_CONFIG_GENERATION: value }), /SUPERSERVE_CONFIG_GENERATION/, value);+ }+ for (const value of [undefined, "", " "]) {+ assert.equal(loadConfig({ SUPERSERVE_CONFIG_GENERATION: value }).superserveSandbox.configGeneration, undefined);+ }+ for (const value of ["0", "7", " 42 ", String(Number.MAX_SAFE_INTEGER)]) {+ assert.equal(loadConfig({ SUPERSERVE_CONFIG_GENERATION: value }).superserveSandbox.configGeneration, Number(value));+ }+});+test("SANDBOX_BACKEND: unset defaults to local (dev only); the retired secondary variable is tolerated", () => {assert.equal(loadConfig({}).sandboxBackend, "local");assert.throws(@@ -397,6 +410,116 @@ test("SANDBOX_BACKEND: unset defaults to local (dev only); the retired secondaryassert.throws(() => loadConfig({ SANDBOX_BACKEND: "sprites" }), /SPRITES_TOKEN/);assert.throws(() => loadConfig({ SANDBOX_BACKEND: "agent37" }), /AGENT37_API_KEY/);assert.equal(loadConfig({ SANDBOX_BACKEND: "agent37", AGENT37_API_KEY: "sk_live_k" }).sandboxBackend, "agent37");+ assert.throws(() => loadConfig({ SANDBOX_BACKEND: "superserve" }), /SUPERSERVE_API_KEY/);+ assert.throws(+ () => loadConfig({ SANDBOX_BACKEND: "superserve", SUPERSERVE_API_KEY: "ss_live_k" }),+ /SUPERSERVE_TEMPLATE/,+ );+ assert.throws(+ () => loadConfig({ SANDBOX_BACKEND: " superserve ", SUPERSERVE_API_KEY: "ss_live_k" }),+ /SUPERSERVE_TEMPLATE/,+ );+ assert.throws(+ () => loadConfig({ SANDBOX_BACKEND: " superserve ", SUPERSERVE_TEMPLATE: "qm-agent-1.0.0" }),+ /SUPERSERVE_API_KEY/,+ );+ assert.equal(+ loadConfig({+ SANDBOX_BACKEND: "superserve",+ SUPERSERVE_TEMPLATE: "qm-agent-1.0.0",+ SUPERSERVE_API_KEY: "ss_live_k",+ SUPERSERVE_CONFIG_GENERATION: "7",+ }).superserveSandbox.configGeneration,+ 7,+ "a deployment that tracks its own rollouts stamps the generation its sandboxes carry",+ );+ assert.throws(+ () =>+ loadConfig({+ SANDBOX_BACKEND: "superserve",+ SUPERSERVE_TEMPLATE: "qm-agent-1.0.0",+ SUPERSERVE_API_KEY: "ss_live_k",+ SUPERSERVE_CONFIG_GENERATION: "later",+ }),+ /SUPERSERVE_CONFIG_GENERATION/,+ );+ assert.equal(+ loadConfig({+ SANDBOX_BACKEND: "superserve",+ SUPERSERVE_TEMPLATE: "qm-agent-1.0.0",+ SUPERSERVE_API_KEY: "ss_live_k",+ }).sandboxBackend,+ "superserve",+ );+ const superserveProd = {+ ...productionEnv,+ SANDBOX_BACKEND: "superserve",+ SUPERSERVE_TEMPLATE: "qm-agent-1.0.0",+ SUPERSERVE_API_KEY: "ss_live_k",+ };+ assert.throws(+ () => loadConfig(superserveProd),+ /superserve sandbox backend requires DATABASE_URL in production/,+ "without a durable store the generation and provisioning lock are per-process",+ );+ assert.throws(+ () => loadConfig({ ...superserveProd, SANDBOX_BACKEND: "local" }),+ /superserve sandbox backend requires DATABASE_URL in production/,+ "the requirement follows the credentials that enable it, not just the primary backend",+ );+ assert.doesNotThrow(() => loadConfig({ ...superserveProd, DATABASE_URL: "postgres://qm@localhost/qm" }));+ assert.doesNotThrow(+ () =>+ loadConfig({+ SANDBOX_BACKEND: "superserve",+ SUPERSERVE_TEMPLATE: "qm-agent-1.0.0",+ SUPERSERVE_API_KEY: "ss_live_k",+ }),+ "a single-process dev instance needs no durable store",+ );+ assert.ok(+ !enabledSandboxBackends(loadConfig({ SANDBOX_BACKEND: "local", SUPERSERVE_API_KEY: "ss_live_k" })).includes(+ "superserve",+ ),+ "a stray key without a template must not enable the secondary backend",+ );+ assert.throws(+ () =>+ loadConfig({+ SANDBOX_BACKEND: "local",+ SANDBOX_SCOPE_BACKENDS: '{"channel":"superserve"}',+ SUPERSERVE_API_KEY: "ss_live_k",+ }),+ /SUPERSERVE_TEMPLATE/,+ "a scope routed to superserve needs the template even when it is not the primary backend",+ );+ assert.doesNotThrow(() =>+ loadConfig({+ SANDBOX_BACKEND: "local",+ SANDBOX_SCOPE_BACKENDS: '{"channel":"superserve"}',+ SUPERSERVE_TEMPLATE: "qm-agent-1.0.0",+ SUPERSERVE_API_KEY: "ss_live_k",+ }),+ );+ assert.ok(+ enabledSandboxBackends(+ loadConfig({ SANDBOX_BACKEND: "local", SUPERSERVE_API_KEY: "ss_live_k", SUPERSERVE_TEMPLATE: "qm-agent-1.0.0" }),+ ).includes("superserve"),+ );+ assert.ok(+ !enabledSandboxBackends(+ loadConfig({ SANDBOX_BACKEND: "local", SUPERSERVE_API_KEY: "ss_live_k", SUPERSERVE_TEMPLATE: " " }),+ ).includes("superserve"),+ "a blank template must not enable the secondary backend",+ );+ assert.equal(+ loadConfig({+ SANDBOX_BACKEND: "superserve",+ SUPERSERVE_TEMPLATE: " qm-agent-1.0.0 ",+ SUPERSERVE_API_KEY: "ss_live_k",+ }).superserveSandbox?.template,+ "qm-agent-1.0.0",+ );const config = loadConfig({ SANDBOX_SECONDARY_BACKEND: "smolmachines" });assert.equal(config.sandboxBackend, "local");assert.ok(!("sandboxSecondaryBackend" in config));diff --git a/test/dev-superserve.test.ts b/test/dev-superserve.test.tsnew file mode 100644index 000000000..b57c5f6b4--- /dev/null+++ b/test/dev-superserve.test.ts@@ -0,0 +1,46 @@+import assert from "node:assert/strict";+import { test } from "node:test";+import { resolveSandbox } from "../scripts/dev/lib/sandbox.ts";++const resolve = (worktree: string, env: Record<string, string> = {}) =>+ resolveSandbox({+ worktree,+ requested: "superserve",+ corePort: 8080,+ lock: "/tmp/qm-dev-superserve-test",+ baseEnv: {+ SUPERSERVE_API_KEY: "test-key",+ SUPERSERVE_TEMPLATE: "qm-agent-test",+ PUBLIC_API_URL: "https://core.example.com",+ ...env,+ },+ log() {},+ });++test("Superserve dev namespaces isolate worktrees and survive instance restarts", async () => {+ const first = await resolve("/worktrees/template-a");+ const restarted = await resolve("/worktrees/template-a");+ const second = await resolve("/worktrees/template-b");+ const prefix = first.env.SUPERSERVE_NAME_PREFIX;+ assert.ok(prefix);+ assert.match(prefix, /^qmdev-[a-f0-9]{12}$/);+ assert.equal(restarted.env.SUPERSERVE_NAME_PREFIX, prefix);+ assert.notEqual(second.env.SUPERSERVE_NAME_PREFIX, prefix);+});++test("Superserve dev namespaces preserve an explicit override", async () => {+ const result = await resolve("/worktrees/template-a", { SUPERSERVE_NAME_PREFIX: "my-dev-namespace" });+ assert.equal(result.env.SUPERSERVE_NAME_PREFIX, "my-dev-namespace");+});++test("Superserve dev settings trim credentials and template and reject blank values before boot", async () => {+ const result = await resolve("/worktrees/template-a", {+ SUPERSERVE_API_KEY: " test-key ",+ SUPERSERVE_TEMPLATE: " qm-agent-test ",+ });+ assert.equal(result.env.SUPERSERVE_API_KEY, "test-key");+ assert.equal(result.env.SUPERSERVE_TEMPLATE, "qm-agent-test");+ for (const key of ["SUPERSERVE_API_KEY", "SUPERSERVE_TEMPLATE"]) {+ await assert.rejects(resolve("/worktrees/template-a", { [key]: " " }), new RegExp(`requires ${key}`));+ }+});diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.tsindex b0838648c..ea5634a72 100644--- a/test/live-slack-sandbox-providers.test.ts+++ b/test/live-slack-sandbox-providers.test.ts@@ -70,6 +70,7 @@ test("every provider has its own parallel execution scenario", () => {"porter","smolmachines","sprites",+ "superserve",]);assert.equal(sandboxProviderScenarios.length, sandboxProviders.length);assert.ok(sandboxProviderScenarios.every((s) => s.lane === "parallel" && s.tags?.includes("provider-execution")));@@ -158,7 +159,7 @@ test("explicit provider selection retains strict coverage without treating defer"sandbox-execute-modal",],);- assert.equal(selectSandboxProviderScenarios("all").length, 8);+ assert.equal(selectSandboxProviderScenarios("all").length, 9);assert.deepEqual(selectSandboxProviderScenarios(undefined), []);for (const value of ["", "sprites,", "typo", "sprites,sprites", "all,sprites", "constructor", "__proto__"])assert.throws(() => selectSandboxProviderScenarios(value));diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.tsindex c67e8093a..a8675f071 100644--- a/test/live-slack/scenarios-sandbox-providers.ts+++ b/test/live-slack/scenarios-sandbox-providers.ts@@ -13,6 +13,7 @@ const providerCoverage: Record<SandboxBackendName, true> = {modal: true,porter: true,agent37: true,+ superserve: true,};export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[];diff --git a/test/superserve-client.test.ts b/test/superserve-client.test.tsnew file mode 100644index 000000000..cf1fa5f37--- /dev/null+++ b/test/superserve-client.test.ts@@ -0,0 +1,104 @@+import assert from "node:assert/strict";+import { test, type TestContext } from "node:test";+import { createSdkSuperserveClient, SuperserveSandboxGoneError } from "../src/sandbox/superserve-client.ts";++const SANDBOX = {+ id: "test-sandbox",+ name: "test-sandbox",+ status: "active",+ metadata: {},+ created_at: "2026-01-01T00:00:00Z",+ access_token: "test-access-token",+};++function mockApi(t: TestContext, respond: (url: URL, init?: RequestInit) => Response | Promise<Response>): void {+ t.mock.method(globalThis, "fetch", async (input: string | URL | Request, init?: RequestInit) => {+ const url = new URL(input instanceof Request ? input.url : input);+ if (url.pathname.endsWith("/activate")) return Response.json(SANDBOX);+ return respond(url, init);+ });+}++const connect = () =>+ createSdkSuperserveClient({ apiKey: "test-api-key", baseUrl: "https://sandbox.example.com" }).connect(SANDBOX.id);++function stream(events: object[]): Response {+ return new Response(events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""));+}++function apiError(status: number, message: string): Response {+ return Response.json({ error: { message } }, { status });+}++test("SDK command results retain terminal errors after streamed stderr", async (t) => {+ mockApi(t, () =>+ stream([{ stderr: "prior warning\n" }, { finished: true, exit_code: 124, error: "command timed out" }]),+ );+ const result = await (await connect()).run("sleep 100");+ assert.equal(result.stderr, "prior warning\ncommand timed out");+ assert.equal(result.exitCode, 124);+});++test("SDK command results cap each stream on UTF-8 boundaries and report truncation", async (t) => {+ mockApi(t, () =>+ stream([+ { stdout: "漢漢", stderr: "warning" },+ { finished: true, exit_code: 1, error: "terminal failure" },+ ]),+ );+ const result = await (await connect()).run("command", { maxOutputBytes: 4 });+ assert.deepEqual(result, { stdout: "漢", stderr: "warn", exitCode: 1, truncated: true });+});++test("SDK terminal errors alone are included in truncation accounting", async (t) => {+ mockApi(t, () => stream([{ finished: true, exit_code: 1, error: "terminal failure" }]));+ const result = await (await connect()).run("command", { maxOutputBytes: 4 });+ assert.deepEqual(result, { stdout: "", stderr: "term", exitCode: 1, truncated: true });+});++test("SDK command transport errors containing HTTP status digits do not mark sandboxes gone", async (t) => {+ mockApi(t, () => {+ throw new Error("connect ECONNREFUSED 127.0.0.1:4100");+ });+ await assert.rejects((await connect()).run("true"), (error: unknown) => {+ assert.ok(error instanceof Error);+ assert.equal(error.name, "SandboxError");+ assert.match(error.message, /ECONNREFUSED/);+ return true;+ });+});++for (const status of [404, 410]) {+ test(`SDK HTTP ${status} command errors mark sandboxes gone independently of message wording`, async (t) => {+ mockApi(t, () => apiError(status, "Sandbox has expired"));+ await assert.rejects((await connect()).run("true"), SuperserveSandboxGoneError);+ });+}++test("SDK missing file returns null while its sandbox remains active", async (t) => {+ mockApi(t, (url) => (url.pathname === "/files" ? apiError(404, "No such file") : Response.json(SANDBOX)));+ assert.equal(await (await connect()).readFileBytes("/missing"), null);+});++for (const status of ["deleted", "failed", "absent"]) {+ test(`SDK file 404 distinguishes sandbox state ${status} from a missing file`, async (t) => {+ mockApi(t, (url) => {+ if (url.pathname === "/files") return apiError(404, "No such file");+ return status === "absent" ? apiError(404, "Missing resource") : Response.json({ ...SANDBOX, status });+ });+ await assert.rejects((await connect()).readFileBytes("/missing"), SuperserveSandboxGoneError);+ });+}++test("SDK HTTP 410 file errors mark sandboxes gone", async (t) => {+ mockApi(t, () => apiError(410, "Sandbox has expired"));+ await assert.rejects((await connect()).readFileBytes("/file"), SuperserveSandboxGoneError);+});++test("SDK file validation errors containing 404 are preserved", async (t) => {+ mockApi(t, (url) => {+ assert.equal(url.pathname, "/files");+ return apiError(400, "Cannot read directory /root/404");+ });+ await assert.rejects((await connect()).readFileBytes("/root/404"), { name: "ValidationError" });+});diff --git a/test/superserve-output-cap.test.ts b/test/superserve-output-cap.test.tsnew file mode 100644index 000000000..ebd8b07a6--- /dev/null+++ b/test/superserve-output-cap.test.ts@@ -0,0 +1,21 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { clampUtf8 } from "../src/sandbox/superserve-client.ts";++test("clampUtf8 bounds output by bytes, not UTF-16 code units", () => {+ assert.equal(clampUtf8("a".repeat(100), 10), "a".repeat(10));+ assert.equal(Buffer.byteLength(clampUtf8("漢".repeat(10), 10), "utf8"), 9, "never exceeds the byte budget");+ assert.equal(clampUtf8("漢".repeat(10), 10), "漢漢漢");+ assert.equal(clampUtf8("", 10), "");+ assert.equal(clampUtf8("abc", 0), "");+ assert.equal(clampUtf8("abc", 10), "abc", "short input is returned untouched");+});++test("clampUtf8 cuts on a codepoint boundary and keeps a replacement character the input really had", () => {+ const withReplacement = `${"a".repeat(7)}�b`;+ assert.equal(Buffer.byteLength(withReplacement, "utf8"), 11);+ assert.equal(clampUtf8(withReplacement, 10), `${"a".repeat(7)}�`, "a real U+FFFD survives truncation");+ assert.ok(!clampUtf8("漢".repeat(10), 10).includes("�"), "truncation never invents one");+ assert.ok(!clampUtf8("😀".repeat(4), 6).includes("�"), "surrogate pairs are cut whole");+ assert.equal(clampUtf8("😀😀", 4), "😀");+});diff --git a/test/superserve-sandbox.test.ts b/test/superserve-sandbox.test.tsnew file mode 100644index 000000000..a2f41ef17--- /dev/null+++ b/test/superserve-sandbox.test.ts@@ -0,0 +1,971 @@+import { test, afterEach, beforeEach } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import {+ createConfigEpochResolver,+ createSuperserveSandbox,+ SUPERSERVE_METADATA,+ type StoredConfigEpoch,+ type StoredSuperserveSandbox,+} from "../src/sandbox/superserve-sandbox.ts";+import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts";+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";+import { supportsProcessSessions, computerVerdict } from "../src/sandbox/sandbox.ts";+import { createMemoryMap, type DurableMap } from "../src/persistence/durable-map.ts";+import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";+import { setTimeout as delay } from "node:timers/promises";+import { scopeId } from "../src/types.ts";+import { shq } from "../src/util/shell.ts";+import { installFakeSuperserve, type FakeSuperserve } from "./support/fake-superserve.ts";+import type { Sandbox } from "../src/sandbox/sandbox.ts";++let fake: FakeSuperserve;+let sandbox: Sandbox;+const workspaceRoots: string[] = [];++function newWorkspace() {+ const root = mkdtempSync(join(tmpdir(), "superserve-ws-"));+ workspaceRoots.push(root);+ return createLocalWorkspaceStore(root);+}+const scope = scopeId("personal", "tester");+const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }];+const scopeName = (): string => sandboxScopeName("qmt", scope);++function make(extra: Record<string, unknown> = {}): Sandbox {+ return createSuperserveSandbox(newWorkspace(), {+ client: fake.client,+ namePrefix: "qmt",+ ...extra,+ });+}++beforeEach(() => {+ fake = installFakeSuperserve();+ sandbox = make();+});+afterEach(() => {+ fake?.cleanup();+ for (const root of workspaceRoots.splice(0)) rmSync(root, { recursive: true, force: true });+});++test("profile advertises resident disk, process sessions, and the template's toolchain", () => {+ assert.equal(sandbox.profile.backend, "superserve");+ assert.equal(sandbox.profile.writablePersistence, "resident_disk");+ assert.equal(supportsProcessSessions(sandbox), true);+ for (const tool of ["node", "gh", "aws", "claude", "codex"])+ assert.ok(sandbox.profile.spec?.tools?.includes(tool), tool);+ assert.ok(!sandbox.profile.spec?.notInstalled?.includes("gh"));+});++test("output is capped while the command runs, and the exit code survives", async () => {+ const h = await sandbox.provision(layers);+ const r = await sandbox.run(h, "head -c 3000000 /dev/zero | tr '\\0' a; echo err-side >&2; exit 7");+ assert.equal(r.code, 7);+ assert.equal(r.stdout.length, 2 * 1024 * 1024);+ assert.match(r.stderr, /err-side/);+ assert.match(r.stderr, /truncated/);+ const small = await sandbox.run(h, "printf ok; printf bad >&2; exit 0");+ assert.equal(small.stdout, "ok");+ assert.equal(small.stderr, "bad");+ assert.doesNotMatch(small.stderr, /truncated/);+});++test("commands are run under a timeout that force-kills a process ignoring SIGTERM", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.run(h, "true");+ assert.ok(fake.execScripts().some((s) => /\btimeout -k \d+ \d+ sh -c /.test(s)));+});++test("an immediate SIGKILL unrelated to the timeout deadline is not reported as timed out", async () => {+ const h = await sandbox.provision(layers);+ const r = await sandbox.run(h, "kill -9 $$");+ assert.equal(r.code, 137);+ assert.equal(r.timedOut, false, "a self-kill that completes instantly cannot be timeout -k's own escalation");+});++test("a command force-killed by timeout -k's own SIGKILL escalation is reported as timed out", async () => {+ const h = await sandbox.provision(layers);+ fake.beforeNextRun(() => new Promise((resolve) => setTimeout(resolve, 1100)));+ const r = await sandbox.run(h, "kill -9 $$", { timeoutMs: 1 });+ assert.equal(r.code, 137);+ assert.equal(r.timedOut, true, "an exit that takes at least the full deadline is timeout -k's own escalation");+});++test("provision creates one sandbox per scope with scope metadata and lifecycle knobs", async () => {+ const h = await sandbox.provision(layers, { env: { MY_VAR: "v1" } });+ assert.equal(h.coldStart, true);+ const r = await sandbox.run(h, "pwd; echo VAR=$MY_VAR");+ assert.ok(+ fake.execScripts().some((script) => script.includes("cd " + shq("/root/workspace").replace(/'/g, "'\\''"))),+ "workspace path quoted",+ );+ assert.equal(r.code, 0);+ assert.match(r.stdout, /workspace/);+ assert.match(r.stdout, /VAR=v1/);++ const record = fake.current(scopeName());+ assert.ok(record);+ assert.equal(record.metadata[SUPERSERVE_METADATA.scope], scopeName());+ assert.equal(record.metadata[SUPERSERVE_METADATA.prefix], "qmt");+ assert.equal(record.metadata[SUPERSERVE_METADATA.kind], "scope");+ assert.equal(record.timeoutSeconds, 15 * 60);+ assert.equal(record.autoDeleteSeconds, 30 * 24 * 3600);+});++test("the default HOME and workspace paths match the template", async () => {+ const h = await sandbox.provision(layers);+ assert.equal(h.homeDir, "/root");+ assert.equal(h.rootDir, "/root/workspace");+});++test("egress allow/deny lists are applied at create time", async () => {+ sandbox = make({ egressAllow: ["api.example.com", "*.github.com"], egressDeny: ["0.0.0.0/0"] });+ await sandbox.provision(layers);+ assert.deepEqual(fake.current(scopeName())?.network, {+ allowOut: ["api.example.com", "*.github.com"],+ denyOut: ["0.0.0.0/0"],+ });+});++test("a paused sandbox is resumed as-is when the egress policy is unchanged", async () => {+ sandbox = make({ egressDeny: ["0.0.0.0/0"], egressAllow: ["api.example.com"] });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const again = make({ egressDeny: ["0.0.0.0/0"], egressAllow: ["api.example.com"] });+ await again.provision(layers);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.equal(fake.current(scopeName())?.status, "active");+});++test("a paused sandbox under a different egress policy is destroyed and replaced, never resumed", async () => {+ const errors: string[] = [];+ sandbox = make();+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "old.txt", "stale\n");+ await sandbox.teardown(h);+ fake.pause(scopeName());+ const oldId = fake.current(scopeName())!.id;++ const tightened = make({+ egressDeny: ["0.0.0.0/0"],+ egressAllow: ["api.example.com"],+ onError: (e: { category: string; code: string }) => errors.push(`${e.category}:${e.code}`),+ });+ const replaced = await tightened.provision(layers);+ assert.equal(replaced.coldStart, true);+ assert.equal(fake.createdCount(scopeName()), 2);+ assert.notEqual(fake.current(scopeName())!.id, oldId);+ assert.deepEqual(fake.current(scopeName())?.network, { allowOut: ["api.example.com"], denyOut: ["0.0.0.0/0"] });+ assert.equal(fake.calls().indexOf(`connect:${oldId}`, fake.calls().indexOf(`pause:${oldId}`)), -1, "never resumed");+ assert.ok(fake.calls().includes(`kill:${oldId}`));+ assert.deepEqual(errors, ["sandbox_egress:policy_changed"]);+ assert.equal(await tightened.readFile(replaced, "old.txt"), null);+});++test("a paused sandbox that predates egress stamping keeps its disk when the policy still matches", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "resident.txt", "months of work\n");+ const record = fake.current(scopeName())!;+ const originalId = record.id;+ delete record.metadata[SUPERSERVE_METADATA.egress];+ await sandbox.teardown(h);+ fake.pause(scopeName());++ const upgraded = make({ store });+ const adopted = await upgraded.provision(layers);+ assert.equal(adopted.coldStart, false, "an unstamped sandbox is adopted, never destroyed");+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.equal(fake.current(scopeName())!.id, originalId);+ assert.equal(await upgraded.readFile(adopted, "resident.txt"), "months of work\n");+ assert.ok(fake.current(scopeName())!.metadata[SUPERSERVE_METADATA.egress], "and it is stamped on adoption");+});++test("network reconciliation reads the actual policy independently of metadata", async () => {+ sandbox = make({ egressDeny: ["0.0.0.0/0"], egressAllow: ["api.example.com"] });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ const record = fake.current(scopeName())!;+ const stampedId = record.id;+ const stamp = record.metadata[SUPERSERVE_METADATA.egress];+ record.network = { allowOut: ["evil.example.com"], denyOut: [] };+ fake.pause(scopeName());++ const again = make({ egressDeny: ["0.0.0.0/0"], egressAllow: ["api.example.com"] });+ await again.provision(layers);+ assert.equal(+ fake.current(scopeName())!.metadata[SUPERSERVE_METADATA.egress],+ stamp,+ "the stamp still claimed a match",+ );+ assert.notEqual(fake.current(scopeName())!.id, stampedId, "but drifted network state is caught anyway");+ assert.ok(fake.calls().includes(`kill:${stampedId}`));+ assert.deepEqual(fake.current(scopeName())?.network, { allowOut: ["api.example.com"], denyOut: ["0.0.0.0/0"] });+});++test("a paused sandbox keeps its disk when its policy can be updated without resuming", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ fake.acceptNetworkUpdateWhilePaused();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "resident.txt", "months of work\n");+ await sandbox.teardown(h);+ fake.pause(scopeName());+ const keptId = fake.current(scopeName())!.id;++ const tightened = make({ store, egressDeny: ["0.0.0.0/0"], egressAllow: ["api.example.com"] });+ const adopted = await tightened.provision(layers);+ assert.equal(fake.createdCount(scopeName()), 1, "a routine policy change never destroys the disk");+ assert.equal(fake.current(scopeName())!.id, keptId);+ assert.deepEqual(fake.current(scopeName())?.network, { allowOut: ["api.example.com"], denyOut: ["0.0.0.0/0"] });+ assert.equal(await tightened.readFile(adopted, "resident.txt"), "months of work\n");+ const calls = fake.calls();+ assert.ok(+ calls.lastIndexOf(`update:${keptId}`) < calls.lastIndexOf(`connect:${keptId}`),+ "the policy lands before the sandbox is resumed",+ );+});++test("an unapplied policy update prevents resuming the sandbox under the old policy", async () => {+ fake.ignoreNetworkUpdateWhilePaused();+ const errors: string[] = [];+ sandbox = make();+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ fake.pause(scopeName());+ const oldId = fake.current(scopeName())!.id;++ const tightened = make({+ egressDeny: ["0.0.0.0/0"],+ egressAllow: ["api.example.com"],+ onError: (e: { category: string; code: string }) => errors.push(`${e.category}:${e.code}`),+ });+ await tightened.provision(layers);+ assert.notEqual(fake.current(scopeName())!.id, oldId, "an unverified policy is never trusted");+ assert.ok(fake.calls().includes(`kill:${oldId}`));+ assert.equal(fake.calls().indexOf(`connect:${oldId}`, fake.calls().indexOf(`pause:${oldId}`)), -1, "never resumed");+ assert.deepEqual(errors, ["sandbox_egress:policy_changed"]);+});++test("a cached sandbox whose network drifted is reconciled before the next command", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store, egressDeny: ["0.0.0.0/0"], egressAllow: ["api.example.com"] });+ const h = await sandbox.provision(layers);+ const id = fake.current(scopeName())!.id;+ fake.current(scopeName())!.network = { allowOut: ["evil.example.com"], denyOut: [] };++ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, false, "the cached sandbox is reused");+ assert.equal(fake.current(scopeName())!.id, id);+ assert.deepEqual(+ fake.current(scopeName())?.network,+ { allowOut: ["api.example.com"], denyOut: ["0.0.0.0/0"] },+ "drift on a cached session is repaired rather than trusted",+ );+ assert.equal((await sandbox.run(h, "echo ok")).stdout.trim(), "ok");+});++test("a cached scratch sandbox whose network drifted is replaced", async () => {+ sandbox = make({ egressDeny: ["0.0.0.0/0"] });+ const first = await sandbox.provision(layers, { scratch: { key: "job" } });+ const firstId = fake.current(first.id)!.id;+ fake.current(first.id)!.network = { allowOut: ["evil.example.com"], denyOut: [] };++ const second = await sandbox.provision(layers, { scratch: { key: "job" } });+ assert.equal(second.coldStart, true);+ assert.notEqual(fake.current(first.id)!.id, firstId);+ assert.deepEqual(fake.current(first.id)?.network, { denyOut: ["0.0.0.0/0"] });+ assert.ok(fake.calls().includes(`kill:${firstId}`));+});++test("an active sandbox gets a changed egress policy applied before its first command", async () => {+ sandbox = make();+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h, { keepWarm: true });+ const id = fake.current(scopeName())!.id;++ const tightened = make({ egressDeny: ["0.0.0.0/0"], egressAllow: ["api.example.com"] });+ await tightened.provision(layers);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.deepEqual(fake.current(scopeName())?.network, { allowOut: ["api.example.com"], denyOut: ["0.0.0.0/0"] });+ const calls = fake.calls();+ const connectAt = calls.lastIndexOf(`connect:${id}`);+ const policyAt = calls.lastIndexOf(`update:${id}`, connectAt);+ const firstRunAt = calls.indexOf(`run:${id}`, connectAt);+ assert.ok(+ policyAt >= 0 && policyAt < connectAt && connectAt < firstRunAt,+ "policy applied before activation and the first command",+ );+ const meta = fake.current(scopeName())!.metadata;+ assert.equal(meta[SUPERSERVE_METADATA.scope], scopeName());+ assert.equal(meta[SUPERSERVE_METADATA.kind], "scope");+ assert.ok(meta[SUPERSERVE_METADATA.egress]);++ const relaxed = make({ idlePauseSec: 120, retentionSec: 3600 });+ const relaxedHandle = await relaxed.provision(layers);+ assert.deepEqual(fake.current(scopeName())?.network, { allowOut: [], denyOut: [] });+ assert.equal(fake.current(scopeName())?.autoDeleteSeconds, 3600);+ await relaxed.teardown(relaxedHandle);+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 120, "a shorter idle pause lands at the next plain teardown");+});++test("computerStatus observing a deleted sandbox clears cached state so the next provision replaces it", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ await sandbox.provision(layers);+ fake.expire(scopeName());+ const status = await sandbox.computerStatus!(scope);+ assert.equal(status.provisioned, false);+ assert.equal(await store.get(scope), null);+ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, true);+ assert.equal(fake.createdCount(scopeName()), 2);+});++test("template and prefix flow through to creation", async () => {+ sandbox = make({ template: "qm-agent-1.2.3" });+ await sandbox.provision(layers);+ assert.equal(fake.current(scopeName())?.template, "qm-agent-1.2.3");+});++test("an already-aborted signal never executes a command", async () => {+ const handle = await sandbox.provision(layers);+ const before = fake.execScripts().length;+ await assert.rejects(sandbox.run(handle, "echo must-not-run", { signal: AbortSignal.abort() }), /aborted/i);+ assert.equal(fake.execScripts().length, before);+});++test("streams and exit codes are exact", async () => {+ const h = await sandbox.provision(layers);+ const r = await sandbox.run(h, "echo out; echo err >&2; exit 3");+ assert.equal(r.code, 3);+ assert.equal(r.stdout.trim(), "out");+ assert.equal(r.stderr.trim(), "err");+});++test("file roundtrip incl. large binary and missing file", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "a/b.txt", "hello\n");+ assert.equal(await sandbox.readFile(h, "a/b.txt"), "hello\n");+ assert.equal(await sandbox.readFile(h, "nope.txt"), null);+ const big = Buffer.alloc(1300 * 1024);+ for (let i = 0; i < big.length; i++) big[i] = (i * 13) % 256;+ await sandbox.writeFileBytes(h, "big.bin", big);+ const back = await sandbox.readFileBytes(h, "big.bin");+ assert.ok(back && Buffer.from(back).equals(big));+ const seen = await sandbox.run(h, "wc -c < big.bin");+ assert.equal(seen.stdout.trim(), String(big.length));+});++test("importFiles, listDir and removeDir work through exec", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.importFiles!(h, [+ { path: "dir/one.txt", data: Buffer.from("1") },+ { path: "dir/two.txt", data: Buffer.from("2") },+ ]);+ const listed = await sandbox.listDir(h, "dir");+ assert.deepEqual(listed.sort(), ["dir/one.txt", "dir/two.txt"]);+ await sandbox.removeDir(h, "dir");+ assert.equal(await sandbox.readFile(h, "dir/one.txt"), null);+});++test("provisioning the same scope twice reuses the sandbox", async () => {+ const a = await sandbox.provision(layers);+ const b = await sandbox.provision(layers);+ assert.equal(a.id, b.id);+ assert.equal(b.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 1);+});++test("concurrent cores serialize read-only layer preparation", async () => {+ const workspace = newWorkspace();+ const shared = scopeId("org", "shared");+ await workspace.write(shared, "policy.txt", "current");+ const layered = [...layers, { scopeId: shared, mountPath: "global", mode: "ro" as const }];+ const advisoryLock = createMemoryAdvisoryLock();+ let writes = 0;+ let overlappingWrites = false;+ const wrap = (session: Awaited<ReturnType<typeof fake.client.create>>) => ({+ ...session,+ async writeFileBytes(path: string, data: Uint8Array): Promise<void> {+ if (!path.endsWith(".ro-layers.tar")) return session.writeFileBytes(path, data);+ overlappingWrites ||= writes > 0;+ writes += 1;+ try {+ await session.writeFileBytes(path, data);+ await delay(100);+ } finally {+ writes -= 1;+ }+ },+ });+ const client = {+ ...fake.client,+ create: async (...args: Parameters<typeof fake.client.create>) => wrap(await fake.client.create(...args)),+ connect: async (...args: Parameters<typeof fake.client.connect>) => wrap(await fake.client.connect(...args)),+ };+ const first = createSuperserveSandbox(workspace, { client, advisoryLock, configEpoch: 1 });+ const second = createSuperserveSandbox(workspace, { client, advisoryLock, configEpoch: 1 });+ const results = await Promise.allSettled([first.provision(layered), second.provision(layered)]);+ assert.equal(overlappingWrites, false, "each core owns the shared archive until extraction finishes");+ assert.deepEqual(+ results.map((result) => result.status),+ ["fulfilled", "fulfilled"],+ );+});++test("an older core leaves newer read-only layers intact", async () => {+ const olderWorkspace = newWorkspace();+ const newerWorkspace = newWorkspace();+ const shared = scopeId("org", "shared");+ await olderWorkspace.write(shared, "policy.txt", "old");+ await newerWorkspace.write(shared, "policy.txt", "new");+ const layered = [...layers, { scopeId: shared, mountPath: "global", mode: "ro" as const }];+ const advisoryLock = createMemoryAdvisoryLock();+ const older = createSuperserveSandbox(olderWorkspace, { client: fake.client, advisoryLock, configEpoch: 1 });+ const newer = createSuperserveSandbox(newerWorkspace, { client: fake.client, advisoryLock, configEpoch: 2 });+ await older.provision(layered);+ const upgraded = await newer.provision(layered);+ const held = await older.provision(layered);+ assert.equal(await newer.readFile(upgraded, "global/policy.txt"), "new");+ assert.equal((await older.run(held, "cat global/policy.txt")).stdout, "new");+});++test("teardown preserves the sandbox until automatic pause; resume preserves its disk", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "keep.txt", "still here\n");+ await sandbox.teardown(h);+ assert.equal(fake.current(scopeName())?.status, "active");+ assert.ok(!fake.calls().some((c) => c.startsWith("pause:")));++ fake.pause(scopeName());+ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.equal(await sandbox.readFile(again, "keep.txt"), "still here\n");+ assert.equal(fake.current(scopeName())?.status, "active");+});++test("a sandbox built from an older template is replaced on adoption", async () => {+ const errors: string[] = [];+ sandbox = make({ template: "qm-agent-1.0.0" });+ const h = await sandbox.provision(layers);+ const oldId = fake.current(scopeName())!.id;+ assert.equal(fake.current(scopeName())?.metadata[SUPERSERVE_METADATA.template], "qm-agent-1.0.0");+ await sandbox.teardown(h);++ const upgraded = make({+ template: "qm-agent-1.1.0",+ onError: (e: { category: string; code: string }) => errors.push(`${e.category}:${e.code}`),+ });+ const replaced = await upgraded.provision(layers);+ assert.equal(replaced.coldStart, true);+ assert.notEqual(fake.current(scopeName())!.id, oldId);+ assert.equal(fake.current(scopeName())?.metadata[SUPERSERVE_METADATA.template], "qm-agent-1.1.0");+ assert.ok(fake.calls().includes(`kill:${oldId}`));+ assert.deepEqual(errors, ["sandbox_template:template_changed"]);++ const same = make({ template: "qm-agent-1.1.0" });+ const again = await same.provision(layers);+ assert.equal(again.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 2);+});++test("an older core never reverts a sandbox a newer core already reconfigured", async () => {+ const older = make({ template: "qm-agent-1.0.0", configEpoch: 1_000, idlePauseSec: 600, retentionSec: 3_600 });+ const first = await older.provision(layers);+ await older.teardown(first);++ const newer = make({+ template: "qm-agent-1.1.0",+ configEpoch: 2_000,+ egressDeny: ["0.0.0.0/0"],+ idlePauseSec: 1_200,+ retentionSec: 7_200,+ });+ await newer.provision(layers);+ assert.equal(fake.createdCount(scopeName()), 2);+ const upgradedId = fake.current(scopeName())!.id;++ const olderAgain = make({ template: "qm-agent-1.0.0", configEpoch: 1_000, idlePauseSec: 600, retentionSec: 3_600 });+ const h = await olderAgain.provision(layers);+ assert.equal(h.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 2, "no third sandbox");+ const record = fake.current(scopeName())!;+ assert.equal(record.id, upgradedId);+ assert.equal(record.metadata[SUPERSERVE_METADATA.template], "qm-agent-1.1.0");+ assert.equal(record.metadata[SUPERSERVE_METADATA.epoch], "2000");+ assert.deepEqual(record.network, { denyOut: ["0.0.0.0/0"] });+ assert.equal(record.timeoutSeconds, 1_200, "older core leaves the newer idle pause alone");+ assert.equal(record.autoDeleteSeconds, 7_200, "older core leaves the newer retention alone");+ assert.equal((await olderAgain.run(h, "echo ok")).stdout.trim(), "ok");+ await olderAgain.teardown(h, { keepWarm: true });+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 1_200, "older core's teardown does not touch the timeout");+});++test("an older core with a cached session stops configuring once a newer core takes over", async () => {+ const older = make({ configEpoch: 1_000, idlePauseSec: 600 });+ const h = await older.provision(layers);+ await older.teardown(h);++ const newer = make({ configEpoch: 2_000, idlePauseSec: 1_800 });+ await newer.teardown(await newer.provision(layers));+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 1_800);++ const again = await older.provision(layers);+ assert.equal(again.coldStart, false);+ await older.teardown(again);+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 1_800, "cached older core no longer rewrites the timeout");+});++test("a core without a durable generation never outranks one that has one", async () => {+ const durable = make({ configEpoch: 3, idlePauseSec: 1_800, template: "qm-agent-1.1.0" });+ await durable.teardown(await durable.provision(layers));+ const stamped = fake.current(scopeName())!.id;++ const ephemeral = make({ configEpoch: 0, idlePauseSec: 600, template: "qm-agent-1.0.0" });+ const h = await ephemeral.provision(layers);+ await ephemeral.teardown(h, { keepWarm: true });++ assert.equal(fake.current(scopeName())?.id, stamped, "it never destroys the durable core's sandbox");+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 1_800, "and never rewrites its lifecycle");+});++test("an older core never reinstalls deployment tools over a newer generation's", async () => {+ let reconciles = 0;+ const toolFiles = (): { to: string; mode: string; content: string }[] => {+ reconciles += 1;+ return [];+ };+ const older = make({ configEpoch: 1_000, layerToolFiles: toolFiles });+ await older.teardown(await older.provision(layers));+ const reconciledByOwner = reconciles;+ assert.ok(reconciledByOwner > 0, "the owning generation reconciles its guest tools");++ const newer = make({ configEpoch: 2_000 });+ await newer.teardown(await newer.provision(layers));++ await older.provision(layers);+ assert.equal(reconciles, reconciledByOwner, "the older generation leaves the newer one's guest tools alone");+});++test("a teardown rechecks the sandbox's stamp before it rewrites the lifecycle timeout", async () => {+ const older = make({ configEpoch: 1_000, idlePauseSec: 600 });+ const held = await older.provision(layers);++ const newer = make({ configEpoch: 2_000, idlePauseSec: 1_800 });+ await newer.teardown(await newer.provision(layers));+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 1_800);++ await older.teardown(held, { keepWarm: true });+ assert.equal(+ fake.current(scopeName())?.timeoutSeconds,+ 1_800,+ "a handle provisioned before the newer core took over no longer rewrites the timeout",+ );+});++test("a newer core stamps its epoch even when only lifecycle settings changed", async () => {+ const older = make({ configEpoch: 1_000, idlePauseSec: 600 });+ await older.teardown(await older.provision(layers));+ assert.equal(fake.current(scopeName())?.metadata[SUPERSERVE_METADATA.epoch], "1000");++ const newer = make({ configEpoch: 2_000, idlePauseSec: 900, retentionSec: 7_200 });+ await newer.provision(layers);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.equal(fake.current(scopeName())?.metadata[SUPERSERVE_METADATA.epoch], "2000");+ assert.equal(fake.current(scopeName())?.autoDeleteSeconds, 7_200);++ const olderAgain = make({ configEpoch: 1_000, idlePauseSec: 600, retentionSec: 3_600 });+ await olderAgain.teardown(await olderAgain.provision(layers));+ assert.equal(fake.current(scopeName())?.autoDeleteSeconds, 7_200);+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 900);+});++test("the config epoch is claimed once per deployment generation, so a restarted core cannot outrank a newer one", async () => {+ const epochs: DurableMap<StoredConfigEpoch> = createMemoryMap();+ const claimed = await createConfigEpochResolver(epochs, "release-1")();+ assert.equal(claimed, 1);+ assert.equal(await createConfigEpochResolver(epochs, "release-1")(), claimed, "a restart reuses its own generation");+ const newRelease = await createConfigEpochResolver(epochs, "release-2")();+ assert.equal(newRelease, claimed + 1, "each new generation is strictly higher than every generation before it");+ assert.equal(await createConfigEpochResolver(epochs, "release-1")(), claimed, "a restart still ranks below it");++ const older = make({ configEpoch: createConfigEpochResolver(epochs, "release-1"), idlePauseSec: 600 });+ await older.teardown(await older.provision(layers));+ assert.equal(fake.current(scopeName())?.metadata[SUPERSERVE_METADATA.epoch], String(claimed));++ const newer = make({ configEpoch: createConfigEpochResolver(epochs, "release-2"), idlePauseSec: 1_800 });+ await newer.teardown(await newer.provision(layers));+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 1_800);++ const restarted = make({ configEpoch: createConfigEpochResolver(epochs, "release-1"), idlePauseSec: 600 });+ await restarted.teardown(await restarted.provision(layers));+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 1_800, "the restarted older release stays passive");+ assert.equal(fake.current(scopeName())?.metadata[SUPERSERVE_METADATA.epoch], String(newRelease));+});++test("a command that finds its sandbox gone leaves a replacement provisioned meanwhile in place", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ const lostId = fake.current(scopeName())!.id;+ fake.beforeNextRun(async () => {+ fake.expire(scopeName());+ await sandbox.provision(layers);+ });+ await assert.rejects(sandbox.run(h, "echo back"), /is gone/);+ const replacement = fake.current(scopeName())!;+ assert.notEqual(replacement.id, lostId);+ assert.equal((await store.get(scope))?.sandboxId, replacement.id);+ await assert.rejects(+ sandbox.run(h, "echo again"),+ /provision it again/,+ "the old handle never adopts the replacement",+ );+ const fresh = await sandbox.provision(layers);+ assert.equal(fresh.coldStart, false, "the replacement is still cached for the next provision");+ assert.equal(fake.current(scopeName())!.id, replacement.id);+ assert.equal((await sandbox.run(fresh, "echo again")).stdout.trim(), "again");+});++test("a handle never runs against a replacement another turn is still provisioning", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const held = await sandbox.provision(layers);+ const lostId = fake.current(scopeName())!.id;+ fake.expire(scopeName());++ let duringPrep: unknown;+ fake.beforeNextRun(async () => {+ duringPrep = await sandbox.run(held, "pwd").catch((e: unknown) => e);+ });+ const replaced = await sandbox.provision(layers);++ assert.notEqual(fake.current(scopeName())!.id, lostId);+ assert.ok(duringPrep instanceof Error, "the stale handle is rejected instead of running in an unprepared sandbox");+ assert.match((duringPrep as Error).message, /provision it again/);+ assert.match((await sandbox.run(replaced, "pwd")).stdout.trim(), /\/workspace$/, "the fresh handle is prepared");+ const idleBefore = fake.current(scopeName())?.timeoutSeconds;+ await sandbox.teardown(held, { keepWarm: true });+ assert.equal(fake.current(scopeName())?.timeoutSeconds, idleBefore, "a stale teardown leaves the replacement alone");+});++test("a sandbox that disappears while checking ownership fails provisioning instead of returning a handle to it", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store, layerToolFiles: () => [] });+ const first = await sandbox.provision(layers);+ const lostId = fake.current(scopeName())!.id;+ assert.equal((await sandbox.run(first, "echo ok")).stdout.trim(), "ok");++ fake.beforeNextInfo(async () => {+ fake.beforeNextInfo(async () => {+ fake.expire(scopeName());+ });+ });+ await assert.rejects(sandbox.provision(layers), /is gone/);++ assert.equal(await store.get(scope), null, "the durable record for the gone sandbox is cleared");+ const fresh = await sandbox.provision(layers);+ assert.equal(+ fresh.coldStart,+ true,+ "the next provision creates a genuine replacement rather than reusing the gone id",+ );+ assert.notEqual(fake.current(scopeName())!.id, lostId);+});++test("a gone sandbox never forgets a replacement another instance already recorded", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ const oldId = fake.current(scopeName())!.id;+ fake.expire(scopeName());+ const other = make({ store });+ const replacement = await other.provision(layers);+ assert.equal(replacement.coldStart, true);+ const newId = (await store.get(scope))!.sandboxId;+ assert.notEqual(newId, oldId);++ await assert.rejects(sandbox.run(h, "echo x"), /is gone/);+ assert.equal((await store.get(scope))?.sandboxId, newId, "the replacement's record survives");+ const status = await sandbox.computerStatus!(scope);+ assert.equal(status.provisioned, true);+});++test("reconnecting keeps a longer keep-warm timeout until a plain teardown restores it", async () => {+ sandbox = make({ idlePauseSec: 600, keepWarmSec: 5400 });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h, { keepWarm: true });+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 5400);++ const other = make({ idlePauseSec: 600, keepWarmSec: 5400 });+ await other.computerStatus!(scope);+ const probed = await other.provision(layers);+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 5400, "another instance's probe keeps the warm window");+ await other.teardown(probed);+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 600);+});++test("keepWarm teardown extends the active-time limit; a plain teardown restores it", async () => {+ sandbox = make({ idlePauseSec: 600, keepWarmSec: 5400 });+ const h = await sandbox.provision(layers);+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 600);+ await sandbox.teardown(h, { keepWarm: true });+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 5400);+ assert.equal(fake.current(scopeName())?.status, "active");+ await sandbox.teardown(h);+ assert.equal(fake.current(scopeName())?.timeoutSeconds, 600);+});++test("a concurrent handle keeps working after another handle's teardown", async () => {+ const a = await sandbox.provision(layers);+ const b = await sandbox.provision(layers);+ await sandbox.teardown(a);+ const r = await sandbox.run(b, "echo still-running");+ assert.equal(r.code, 0);+ assert.match(r.stdout, /still-running/);+});++test("a fresh core with an empty store rediscovers the sandbox by scope metadata", async () => {+ const first = await sandbox.provision(layers);+ await sandbox.writeFile(first, "state.txt", "from before\n");+ await sandbox.teardown(first);++ const restarted = make();+ const h = await restarted.provision(layers);+ assert.equal(h.coldStart, false);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.equal(await restarted.readFile(h, "state.txt"), "from before\n");+});++test("a durable store lets a restarted core reconnect without listing", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const first = await sandbox.provision(layers);+ await sandbox.teardown(first);+ const stored = await store.get(scope);+ assert.ok(stored);++ const restarted = make({ store });+ const before = fake.calls().filter((c) => c.startsWith("create:")).length;+ const h = await restarted.provision(layers);+ assert.equal(h.coldStart, false);+ assert.equal(fake.calls().filter((c) => c.startsWith("create:")).length, before);+ assert.ok(fake.calls().includes(`connect:${stored.sandboxId}`));+});++test("a sandbox that disappeared is replaced on the next provision", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h);+ fake.expire(scopeName());++ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, true);+ assert.equal(fake.createdCount(scopeName()), 2);+ assert.notEqual((await store.get(scope))?.sandboxId, undefined);+});++test("a sandbox lost mid-session fails the command and is replaced by the next provision", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ fake.expire(scopeName());+ await assert.rejects(sandbox.run(h, "echo back"), /is gone/);+ await assert.rejects(sandbox.run(h, "echo again"), /provision it again/);+ await assert.rejects(sandbox.readFile(h, "x.txt"), /provision it again/);+ assert.equal(fake.createdCount(scopeName()), 1);+ assert.equal(await store.get(scope), null);++ const again = await sandbox.provision(layers);+ assert.equal(again.coldStart, true);+ assert.equal(fake.createdCount(scopeName()), 2);+ assert.equal((await sandbox.run(again, "echo back")).stdout.trim(), "back");+});++test("destroyScope surfaces a failed listing instead of forgetting the scope", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ await sandbox.provision(layers);+ fake.failNextList(new Error("superserve unavailable"));+ await assert.rejects(sandbox.destroyScope!(scope), /unavailable/);+ assert.ok(await store.get(scope), "record survives so retirement can be retried");+ assert.ok(fake.current(scopeName()), "sandbox untouched");+ await sandbox.destroyScope!(scope);+ assert.equal(fake.current(scopeName()), null);+});++test("destroy teardown and destroyScope kill the sandbox and forget the scope", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ const h = await sandbox.provision(layers);+ await sandbox.teardown(h, { destroy: true });+ assert.equal(fake.current(scopeName()), null);+ assert.equal(await store.get(scope), null);++ const h2 = await sandbox.provision(layers);+ assert.equal(h2.coldStart, true);+ await sandbox.destroyScope!(scope);+ assert.equal(fake.current(scopeName()), null);+ assert.equal(await store.get(scope), null);+});++test("destroyScope also removes sandboxes only findable by metadata", async () => {+ await sandbox.provision(layers);+ const orphanOwner = make();+ await orphanOwner.destroyScope!(scope);+ assert.equal(fake.current(scopeName()), null);+});++test("computerStatus reports paused, running, and gone", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store });+ assert.equal(computerVerdict(await sandbox.computerStatus!(scope)), "down");++ const h = await sandbox.provision(layers);+ const running = await sandbox.computerStatus!(scope);+ assert.equal(running.guestResponsive, true);+ assert.equal(running.lifecycleState, "running");+ assert.equal(computerVerdict(running), "ok");++ await sandbox.teardown(h);+ fake.pause(scopeName());+ const paused = await sandbox.computerStatus!(scope);+ assert.equal(paused.lifecycleState, "paused");+ assert.equal(paused.recovery?.strategy, "provider_pause");+ assert.equal(computerVerdict(paused), "ok");++ fake.expire(scopeName());+ const gone = await sandbox.computerStatus!(scope);+ assert.equal(gone.provisioned, false);+ assert.equal(computerVerdict(gone), "down");+});++test("computerStatus reports the sandbox it actually probed after a replacement", async () => {+ const store: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ sandbox = make({ store, template: "qm-agent-1.0.0" });+ await sandbox.provision(layers);+ const firstId = fake.current(scopeName())!.id;++ const upgraded = make({ store, template: "qm-agent-1.1.0" });+ const status = await upgraded.computerStatus!(scope);+ const replacementId = fake.current(scopeName())!.id;++ assert.notEqual(replacementId, firstId);+ assert.match(status.machine, new RegExp(replacementId));+ assert.doesNotMatch(status.machine, new RegExp(firstId), "never reports the sandbox it replaced as responsive");+});++test("scratch sandboxes are separate, shared while active, and killed on last teardown", async () => {+ const a = await sandbox.provision(layers, { scratch: { key: "k1" } });+ const b = await sandbox.provision(layers, { scratch: { key: "k1" } });+ assert.equal(a.id, b.id);+ assert.equal(a.scratch, true);+ assert.equal(b.coldStart, false);+ const scratchName = a.id;+ assert.equal(fake.current(scratchName)?.metadata[SUPERSERVE_METADATA.kind], "scratch");+ assert.equal(fake.current(scratchName)?.autoDeleteSeconds, 24 * 3600);++ await sandbox.teardown(a);+ assert.ok(fake.current(scratchName), "still alive while another user holds it");+ await sandbox.teardown(b);+ assert.equal(fake.current(scratchName), null);+ assert.equal(fake.current(scopeName()), null, "scratch never touches the scope sandbox");+});++test("destroying a scratch sandbox surfaces a failed kill instead of reporting success", async () => {+ const h = await sandbox.provision(layers, { scratch: { key: "creds" } });+ fake.failNextKill(new Error("superserve unavailable"));+ await assert.rejects(sandbox.teardown(h, { destroy: true }), /unavailable/);+ assert.notEqual(fake.current(h.id), null, "the credential-bearing sandbox is still there to retry");+ await sandbox.teardown(h, { destroy: true });+ assert.equal(fake.current(h.id), null);+});++test("a best-effort scratch teardown still tolerates a failed kill", async () => {+ const h = await sandbox.provision(layers, { scratch: { key: "job" } });+ fake.failNextKill(new Error("superserve unavailable"));+ await sandbox.teardown(h);+});++test("the last scratch handle to close kills the replacement even when it was provisioned earlier", async () => {+ const first = await sandbox.provision(layers, { scratch: { key: "k1" } });+ fake.expire(first.id);+ const replacement = await sandbox.provision(layers, { scratch: { key: "k1" } });+ assert.equal(replacement.coldStart, true, "a deleted scratch sandbox is recreated on the next provision");+ assert.notEqual(fake.current(first.id), null);++ await sandbox.teardown(replacement);+ assert.notEqual(fake.current(first.id), null, "still referenced by the older handle");+ await sandbox.teardown(first);+ assert.equal(fake.current(first.id), null, "the replacement is killed once nothing references it");+});++test("a scratch sandbox lost while handles are active is recreated for the next scratch provision", async () => {+ const a = await sandbox.provision(layers, { scratch: { key: "job" } });+ const scratchName = a.id;+ fake.expire(scratchName);+ await assert.rejects(sandbox.run(a, "echo x"), /is gone/);+ const b = await sandbox.provision(layers, { scratch: { key: "job" } });+ assert.equal(b.coldStart, true);+ assert.equal(fake.createdCount(scratchName), 2);+ assert.equal((await sandbox.run(b, "echo back")).stdout.trim(), "back");+});++test("process sessions run in the background and can be read back", async () => {+ const h = await sandbox.provision(layers);+ assert.ok(supportsProcessSessions(sandbox));+ const { processId } = await sandbox.startProcess(h, "echo started; sleep 0.2; echo finished");+ let out = "";+ for (let i = 0; i < 40; i++) {+ const r = await sandbox.readProcess(h, processId, { sinceCursor: 0 });+ out = r.chunks;+ if (r.status.state === "exited") break;+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ assert.match(out, /started/);+ assert.match(out, /finished/);+ const listed = await sandbox.listProcesses(h);+ assert.ok(listed.some((p) => p.processId === processId));+});++test("no access token or key ever appears in exec scripts", async () => {+ sandbox = make();+ const h = await sandbox.provision(layers);+ await sandbox.run(h, "true");+ for (const script of fake.execScripts()) {+ assert.doesNotMatch(script, /ss_live_|X-Access-Token|access_token/);+ }+});++test("a sandbox is not cached when its durable record cannot be written", async () => {+ const inner: DurableMap<StoredSuperserveSandbox> = createMemoryMap();+ let failPuts = 1;+ const store: DurableMap<StoredSuperserveSandbox> = {+ ...inner,+ put: async (key, value) => {+ if (failPuts-- > 0) throw new Error("persistence unavailable");+ return inner.put(key, value);+ },+ };+ sandbox = make({ store });+ await assert.rejects(sandbox.provision(layers), /persistence unavailable/);+ const h = await sandbox.provision(layers);+ assert.ok(await inner.get(scope), "record written on the retry");+ assert.equal((await inner.get(scope))?.sandboxId, fake.current(scopeName())?.id);+ assert.equal((await sandbox.run(h, "echo ok")).stdout.trim(), "ok");+});++test("run refuses to execute when its workspace directory is missing", async () => {+ const handle = await sandbox.provision(layers);+ rmSync(join(fake.homeDir(scopeName()), "workspace"), { recursive: true });+ const result = await sandbox.run(handle, "echo should-not-run");+ assert.notEqual(result.code, 0);+ assert.equal(result.stdout, "");+});diff --git a/test/support/fake-superserve.ts b/test/support/fake-superserve.tsnew file mode 100644index 000000000..01ac682a2--- /dev/null+++ b/test/support/fake-superserve.ts@@ -0,0 +1,275 @@+import { spawnSync } from "node:child_process";+import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import {+ SuperserveSandboxGoneError,+ type SuperserveClient,+ type SuperserveCommandResult,+ type SuperserveCreateOptions,+ type SuperserveNetwork,+ type SuperserveSandboxInfo,+ type SuperserveSandboxState,+ type SuperserveSession,+ type SuperserveUpdate,+} from "../../src/sandbox/superserve-client.ts";++interface FakeRecord {+ id: string;+ name: string;+ status: SuperserveSandboxState;+ expired: boolean;+ metadata: Record<string, string>;+ network?: SuperserveNetwork;+ timeoutSeconds?: number;+ autoDeleteSeconds?: number;+ template?: string;+ home: string;+ createdAt: number;+ resumes: number;+}++export interface FakeSuperserve {+ client: SuperserveClient;+ current(name: string): FakeRecord | null;+ createdCount(name: string): number;+ homeDir(name: string): string;+ pause(name: string): void;+ expire(name: string): void;+ execScripts(): string[];+ calls(): string[];+ failNextList(error: Error): void;+ failNextKill(error: Error): void;+ acceptNetworkUpdateWhilePaused(): void;+ ignoreNetworkUpdateWhilePaused(): void;+ beforeNextRun(hook: () => Promise<void>): void;+ beforeNextInfo(hook: () => Promise<void>): void;+ cleanup(): void;+}++export function installFakeSuperserve(): FakeSuperserve {+ const root = mkdtempSync(join(tmpdir(), "fake-superserve-"));+ const records = new Map<string, FakeRecord>();+ const execScripts: string[] = [];+ const calls: string[] = [];+ let nextId = 1;+ let clock = 0;+ let listFailure: Error | null = null;+ let runHook: (() => Promise<void>) | null = null;+ let infoHook: (() => Promise<void>) | null = null;+ let killFailure: Error | null = null;+ let pausedNetworkUpdates = false;+ let silentlyIgnorePausedNetwork = false;++ const byName = (name: string): FakeRecord | undefined => {+ const all = [...records.values()].filter((r) => r.name === name).sort((a, b) => b.createdAt - a.createdAt);+ return all.find((r) => !r.expired) ?? all[0];+ };++ const gone = (r: FakeRecord): never => {+ throw new SuperserveSandboxGoneError(r.id, "sandbox was not found");+ };++ const alive = (r: FakeRecord): void => {+ if (r.expired) gone(r);+ if (r.status === "paused") {+ r.status = "active";+ r.resumes += 1;+ }+ };++ const remap = (r: FakeRecord, script: string): string =>+ script+ .replace(/\btimeout (?:-k \d+ )?\d+ /g, "")+ .replace(/(^|[^A-Za-z0-9._/-])\/tmp\//g, `$1${r.home}/tmp/`)+ .replace(/(^|[^A-Za-z0-9._/-])\/root(?=\/|$|[^A-Za-z0-9._/-])/g, `$1${r.home}`);++ const hostPath = (r: FakeRecord, absPath: string): string => {+ if (absPath.startsWith("/tmp/")) return join(r.home, "tmp", absPath.slice(5));+ if (absPath === "/root" || absPath.startsWith("/root/")) return join(r.home, absPath.slice(5));+ return absPath;+ };++ const session = (r: FakeRecord): SuperserveSession => ({+ id: r.id,+ async run(command): Promise<SuperserveCommandResult> {+ const hook = runHook;+ runHook = null;+ if (hook) await hook();+ alive(r);+ calls.push(`run:${r.id}`);+ execScripts.push(command);+ mkdirSync(join(r.home, "tmp"), { recursive: true });+ const spawned = spawnSync("sh", ["-c", remap(r, command)], {+ encoding: "buffer",+ maxBuffer: 128 * 1024 * 1024,+ env: { ...process.env, COPYFILE_DISABLE: "1" },+ });+ return {+ stdout: (spawned.stdout ?? Buffer.alloc(0)).toString("utf8"),+ stderr: (spawned.stderr ?? Buffer.alloc(0)).toString("utf8"),+ exitCode: spawned.status ?? (spawned.signal ? 137 : -1),+ };+ },+ async readFileBytes(absPath): Promise<Uint8Array | null> {+ alive(r);+ const p = hostPath(r, absPath);+ if (!existsSync(p)) return null;+ return new Uint8Array(readFileSync(p));+ },+ async writeFileBytes(absPath, data): Promise<void> {+ alive(r);+ const p = hostPath(r, absPath);+ mkdirSync(dirname(p), { recursive: true });+ writeFileSync(p, Buffer.from(data));+ },+ async update(patch: SuperserveUpdate): Promise<void> {+ if (r.expired) gone(r);+ calls.push(`update:${r.id}`);+ const dropNetwork = patch.network !== undefined && r.status !== "active" && silentlyIgnorePausedNetwork;+ if (patch.network !== undefined && r.status !== "active" && !pausedNetworkUpdates && !dropNetwork)+ throw Object.assign(new Error("Sandbox must be active to update network config"), { statusCode: 409 });+ if (patch.network !== undefined && !dropNetwork) r.network = patch.network;+ if (patch.metadata !== undefined) r.metadata = { ...patch.metadata };+ if (patch.timeoutSeconds !== undefined) r.timeoutSeconds = patch.timeoutSeconds ?? undefined;+ if (patch.autoDeleteSeconds !== undefined) r.autoDeleteSeconds = patch.autoDeleteSeconds ?? undefined;+ },+ async pause(): Promise<void> {+ if (r.expired) gone(r);+ calls.push(`pause:${r.id}`);+ r.status = "paused";+ },+ async kill(): Promise<void> {+ if (killFailure) {+ const failure = killFailure;+ killFailure = null;+ throw failure;+ }+ calls.push(`kill:${r.id}`);+ r.expired = true;+ r.status = "deleted";+ rmSync(r.home, { recursive: true, force: true });+ },+ });++ const info = (r: FakeRecord): SuperserveSandboxInfo => ({+ id: r.id,+ name: r.name,+ status: r.status,+ metadata: r.metadata,+ ...(r.network ? { network: r.network } : {}),+ vcpuCount: 2,+ memoryMib: 2048,+ ...(r.timeoutSeconds !== undefined ? { timeoutSeconds: r.timeoutSeconds } : {}),+ });++ const client: SuperserveClient = {+ async create(opts: SuperserveCreateOptions): Promise<SuperserveSession> {+ calls.push(`create:${opts.name}`);+ const id = `sbx-${nextId++}`;+ const r: FakeRecord = {+ id,+ name: opts.name,+ status: "active",+ expired: false,+ metadata: { ...opts.metadata },+ ...(opts.network ? { network: opts.network } : {}),+ ...(opts.timeoutSeconds !== undefined ? { timeoutSeconds: opts.timeoutSeconds } : {}),+ ...(opts.autoDeleteSeconds !== undefined ? { autoDeleteSeconds: opts.autoDeleteSeconds } : {}),+ ...(opts.template ? { template: opts.template } : {}),+ home: join(root, id),+ createdAt: ++clock,+ resumes: 0,+ };+ mkdirSync(r.home, { recursive: true });+ records.set(id, r);+ return session(r);+ },+ async update(sandboxId, patch): Promise<void> {+ const r = records.get(sandboxId);+ if (!r || r.expired) throw new SuperserveSandboxGoneError(sandboxId, "sandbox was not found");+ await session(r).update(patch);+ },+ async connect(sandboxId): Promise<SuperserveSession> {+ calls.push(`connect:${sandboxId}`);+ const r = records.get(sandboxId);+ if (!r || r.expired) throw new SuperserveSandboxGoneError(sandboxId, "sandbox was not found");+ alive(r);+ return session(r);+ },+ async info(sandboxId, scopeMetadata): Promise<SuperserveSandboxInfo> {+ const hook = infoHook;+ infoHook = null;+ if (hook) await hook();+ const r = records.get(sandboxId);+ const matches = Object.entries(scopeMetadata ?? {}).every(([k, v]) => r?.metadata[k] === v);+ if (!r || r.expired || !matches) throw new SuperserveSandboxGoneError(sandboxId, "sandbox was not found");+ return info(r);+ },+ async list(metadata): Promise<SuperserveSandboxInfo[]> {+ if (listFailure) {+ const error = listFailure;+ listFailure = null;+ throw error;+ }+ return [...records.values()]+ .filter((r) => !r.expired && Object.entries(metadata).every(([k, v]) => r.metadata[k] === v))+ .map(info);+ },+ async kill(sandboxId): Promise<void> {+ calls.push(`kill:${sandboxId}`);+ const r = records.get(sandboxId);+ if (!r) return;+ r.expired = true;+ r.status = "deleted";+ rmSync(r.home, { recursive: true, force: true });+ },+ };++ return {+ client,+ current: (name) => {+ const r = byName(name);+ return r && !r.expired ? r : null;+ },+ createdCount: (name) => [...records.values()].filter((r) => r.name === name).length,+ homeDir: (name) => {+ const r = byName(name);+ if (!r) throw new Error(`fake-superserve: no sandbox named ${name}`);+ return r.home;+ },+ pause: (name) => {+ const r = byName(name);+ if (r) r.status = "paused";+ },+ expire: (name) => {+ const r = byName(name);+ if (r) {+ r.expired = true;+ r.status = "deleted";+ rmSync(r.home, { recursive: true, force: true });+ }+ },+ execScripts: () => [...execScripts],+ calls: () => [...calls],+ failNextList: (error) => {+ listFailure = error;+ },+ beforeNextRun: (hook) => {+ runHook = hook;+ },+ beforeNextInfo: (hook) => {+ infoHook = hook;+ },+ failNextKill: (error) => {+ killFailure = error;+ },+ acceptNetworkUpdateWhilePaused: () => {+ pausedNetworkUpdates = true;+ },+ ignoreNetworkUpdateWhilePaused: () => {+ silentlyIgnorePausedNetwork = true;+ },+ cleanup: () => rmSync(root, { recursive: true, force: true }),+ };+}diff --git a/tsconfig.json b/tsconfig.jsonindex 3454e08a4..961a30b07 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -17,5 +17,5 @@"skipLibCheck": true,"forceConsistentCasingInFileNames": true},- "include": ["src/**/*.ts", "test/**/*.ts", "scripts/**/*.ts"]+ "include": ["src/**/*.ts", "test/**/*.ts", "scripts/**/*.ts", "superserve/**/*.ts"]}