Multiplayer agent harness for work.
Align the Sprites backend with Sprites docs and SDK 0.2.3 (#1419)
Regan Bellcec1fadefd38
diff --git a/docs/sandbox-preservation.md b/docs/sandbox-preservation.mdindex 167667bc8..9b42f864e 100644--- a/docs/sandbox-preservation.md+++ b/docs/sandbox-preservation.md@@ -15,6 +15,8 @@ When enabled, Modal uses native directory checkpoints of `/root`, mounted into aE2B creates persistent machines with an explicit pause-on-timeout lifecycle and pauses them at teardown unless `keepWarm` retains running background work. The provider timeout still applies; `keepWarm` does not extend that timeout. Routine native pause does not create a second tar backup. Provider pause retains disk and normally memory; provider fallback may retain disk only, so reconnectable processes must not depend on RAM surviving. Status can inspect a paused machine without waking it. Pause failures are recorded and returned, with the running machine retained for retry. A lost native machine requires explicit recovery rather than an automatic blank replacement. Existing E2B machines whose lifecycle does not confirm pause-on-timeout retain legacy portable checkpointing.+Sprites keep the whole disk across sleeps, so the sprite itself is the durable home. Every used-turn teardown that may have changed the home takes a provider checkpoint (copy-on-write, throttled to one per five minutes per core); `computerStatus` reports the newest one as `recovery.strategy: "provider_snapshot"`. Checkpoints are the rollback path: when a computer restart is refused and the health check positively reports an unhealthy status, restart restores the newest checkpoint and records `checkpoint_restored`, which discards changes made since that checkpoint. Deleting a sprite is irreversible and deletes its checkpoints with it, so with `SPRITES_SNAPSHOT_S3_BUCKET` set a scope's home is exported as a portable tar before its sprite is destroyed, a failed export refuses the destroy, and a replacement sprite for that scope hydrates from the export. Initialization is recorded as pending before provider creation in a durable map, and provision, capture, restore and destruction share a cross-core lifecycle lock. A failed initialization remains pending across core restarts and cannot overwrite a saved home; a later provision retries hydration. Failed or unknown health observations never authorize checkpoint rollback. Background processes started with process sessions hold the sprite awake through the Tasks API while they run; they still do not survive a cold wake, a restart or a checkpoint restore. `SPRITES_MEMORY_MB` sets an explicit memory resources policy per sprite; without it memory is provider-managed. Commands run over the WebSocket exec channel and files move through the filesystem API.+Smolmachines machines keep their disk across stop and start, and an idle machine stops after `SMOLMACHINES_AUTOSTOP_SEC` seconds without activity when that knob is set (the provider sweeps every 15 seconds; an in-flight exec keeps the machine alive). The provider documents that stopping is not a backup and that the cloud API has no machine snapshot; export to a `.smolmachine` artifact is the only native capture and it is a whole-disk registry artifact, so core keeps portable home tars instead. Every non-scratch teardown snapshots the home to `SMOLMACHINES_SNAPSHOT_S3_BUCKET` when the home changed, throttled by `SMOLMACHINES_SNAPSHOT_INTERVAL_SEC`; a replacement machine for a scope hydrates from that tar and a failed hydration deletes the incomplete replacement instead of adopting it. Initialization is marked pending in the durable sandbox record before create, so a lost create response or core restart cannot make the next core skip restoration. Provisioning, capture, and destruction share a lifecycle lock across cores; initialization is only marked complete after hydration succeeds. Status reports `recovery.strategy: "workspace_snapshot"` with the last capture time and error. Without the bucket the backend offers no `persistHomeSnapshot` and reports no recovery. Scratch machines are created ephemeral with a 24-hour `ttlSeconds` so a core crash cannot leave them running.-Explicit home export/import and `persistHomeSnapshot` continue to use the portable tar format for cross-provider moves and chosen recovery. Configure `MODAL_SNAPSHOT_S3_BUCKET`, `E2B_SNAPSHOT_S3_BUCKET` or `SMOLMACHINES_SNAPSHOT_S3_BUCKET` to retain those portable copies durably. They are not silently selected instead of a failed newer provider checkpoint.+Explicit home export/import and `persistHomeSnapshot` continue to use the portable tar format for cross-provider moves and chosen recovery. Configure `MODAL_SNAPSHOT_S3_BUCKET`, `E2B_SNAPSHOT_S3_BUCKET`, `SPRITES_SNAPSHOT_S3_BUCKET` or `SMOLMACHINES_SNAPSHOT_S3_BUCKET` to retain those portable copies durably. They are not silently selected instead of a failed newer provider checkpoint.diff --git a/package-lock.json b/package-lock.jsonindex 26159bfa6..42d08a9fd 100644--- a/package-lock.json+++ b/package-lock.json@@ -20,10 +20,11 @@"@aws-sdk/s3-request-presigner": "^3.1061.0","@earendil-works/pi-ai": "0.82.0","@earendil-works/pi-coding-agent": "https://github.com/yc-software/pi/releases/download/qm-pi-coding-agent-0.82.0-security.3/earendil-works-pi-coding-agent-0.82.0-qm-security.3.tgz",- "@fly/sprites": "0.0.1",+ "@fly/sprites": "0.2.3","@openai/codex": "0.144.5","@opencode-ai/plugin": "1.17.18","@opencode-ai/sdk": "1.17.18",+ "@sentry/node": "10.75.0","@slack/bolt": "^4.7.3","@slack/socket-mode": "^2.0.7","@slack/web-api": "^7.19.0",@@ -44,9 +45,7 @@"socks-proxy-agent": "10.1.0","tar-stream": "^3.2.0","typebox": "^1.1.38",- "undici": "^8.9.0",- "zod": "4.4.3",- "@sentry/node": "10.75.0"+ "zod": "4.4.3"},"devDependencies": {"@eslint/js": "^10.0.1",@@ -55,6 +54,7 @@"@types/pg": "^8.11.10","@types/tar-stream": "^3.1.4","@yc-software/qm": "file:./cli",+ "esbuild": "0.28.1","eslint": "^10.4.1","globals": "^17.6.0","husky": "9.1.7",@@ -63,8 +63,7 @@"oxlint": "^1.74.0","prettier": "3.9.1","typescript": "^5.8.2",- "typescript-eslint": "^8.61.0",- "esbuild": "0.28.1"+ "typescript-eslint": "^8.61.0"},"engines": {"node": ">=24.15.0",@@ -930,87 +929,6 @@"@connectrpc/connect": "2.1.2"}},- "node_modules/@earendil-works/pi-agent-core": {- "version": "0.82.1",- "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.82.1.tgz",- "integrity": "sha512-Z3kloziJIE2dmrisRckZX8zDca/gIv9/YdFAzeoqpHiLV2wsni6bL4hInNSjVKLbqT+4kqLIkph2JQLKvSepjg==",- "license": "MIT",- "dependencies": {- "@earendil-works/pi-ai": "^0.82.1",- "diff": "8.0.4",- "ignore": "7.0.5",- "typebox": "1.1.38",- "yaml": "2.9.0"- },- "engines": {- "node": ">=22.19.0"- }- },- "node_modules/@earendil-works/pi-agent-core/node_modules/@anthropic-ai/sdk": {- "version": "0.91.1",- "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz",- "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==",- "license": "MIT",- "dependencies": {- "json-schema-to-ts": "^3.1.1"- },- "bin": {- "anthropic-ai-sdk": "bin/cli"- },- "peerDependencies": {- "zod": "^3.25.0 || ^4.0.0"- },- "peerDependenciesMeta": {- "zod": {- "optional": true- }- }- },- "node_modules/@earendil-works/pi-agent-core/node_modules/@earendil-works/pi-ai": {- "version": "0.82.1",- "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.82.1.tgz",- "integrity": "sha512-3WFYRhEp3lQB3444EhPMBcM7zSaEUE3eJgHOR7s4081NLqbw/FsWilIKWXSua0Gv3sRr7m9xMidR3pPDE7jI/A==",- "license": "MIT",- "dependencies": {- "@anthropic-ai/sdk": "0.91.1",- "@aws-sdk/client-bedrock-runtime": "3.1048.0",- "@google/genai": "1.52.0",- "@mistralai/mistralai": "2.2.6",- "@opentelemetry/api": "1.9.0",- "@smithy/node-http-handler": "4.7.3",- "http-proxy-agent": "7.0.2",- "https-proxy-agent": "7.0.6",- "openai": "6.26.0",- "partial-json": "0.1.7",- "typebox": "1.1.38"- },- "bin": {- "pi-ai": "dist/cli.js"- },- "engines": {- "node": ">=22.19.0"- }- },- "node_modules/@earendil-works/pi-agent-core/node_modules/@smithy/node-http-handler": {- "version": "4.7.3",- "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.7.3.tgz",- "integrity": "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==",- "license": "Apache-2.0",- "dependencies": {- "@smithy/core": "^3.24.3",- "@smithy/types": "^4.14.2",- "tslib": "^2.6.2"- },- "engines": {- "node": ">=18.0.0"- }- },- "node_modules/@earendil-works/pi-agent-core/node_modules/typebox": {- "version": "1.1.38",- "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz",- "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==",- "license": "MIT"- },"node_modules/@earendil-works/pi-ai": {"version": "0.82.0","resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.82.0.tgz",@@ -2895,36 +2813,23 @@"zod": "^3.25.28 || ^4"}},- "node_modules/@earendil-works/pi-tui": {- "version": "0.82.1",- "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.82.1.tgz",- "integrity": "sha512-9yN8hALfKaxZq7n54EMxqhFCWnMi6LHkraMJ/1YjHiATq75XrI6XDMVppn9EDtiK7Fks8hUe1SDXUTrIvwRWfQ==",- "license": "MIT",- "dependencies": {- "get-east-asian-width": "1.6.0",- "marked": "18.0.5"- },- "engines": {- "node": ">=22.19.0"- }- },"node_modules/@emnapi/core": {- "version": "2.0.0-alpha.4",- "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-2.0.0-alpha.4.tgz",- "integrity": "sha512-KjoUR6mNvjT+z5bKMy+a5QDI+kTNTiMZi6uPKzWuWjwZGNa0UPwIVduTfeBRdf6v8Ws5fmHDh7Usf15w/ANfkg==",+ "version": "2.0.0-alpha.5",+ "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-2.0.0-alpha.5.tgz",+ "integrity": "sha512-DbeLFquQJyYGwM3LzPdjvb4IRmAAQuTzEmqUGtIAYxlAO4ArfUZ7WMytLWDSsCAYm6MZJNeojaawQQiFrFVehg==","dev": true,"license": "MIT","optional": true,"peer": true,"dependencies": {- "@emnapi/wasi-threads": "2.0.1",+ "@emnapi/wasi-threads": "2.1.0","tslib": "^2.4.0"}},"node_modules/@emnapi/runtime": {- "version": "2.0.0-alpha.4",- "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.4.tgz",- "integrity": "sha512-Wy1TQ99obRP3Ah7cf/OOtK1zD9t3pYTWz+G/SpAwTFMhG5hoMdlvfzBpYoHUigS/izu3q+VhYMKhJd7Ii/trpg==",+ "version": "2.0.0-alpha.5",+ "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.5.tgz",+ "integrity": "sha512-SS4jUXRklsrPKTEtW92WHdneh9QBrZGmRyeYEac2ktLN3ie1wd/BrkBbk4DqPWKhjSFfrR8NClXwMlfZ73m60A==","dev": true,"license": "MIT","optional": true,@@ -2934,9 +2839,9 @@}},"node_modules/@emnapi/wasi-threads": {- "version": "2.0.1",- "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-2.0.1.tgz",- "integrity": "sha512-9DsSk+o5NBX0CCJT8s0EROGSGxjR/tKu6aBTaVyq+SjAEQH4XcdcRxPBRzsBLizTTJ49MJjF+jgu3qnO9GLQcQ==",+ "version": "2.1.0",+ "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-2.1.0.tgz",+ "integrity": "sha512-DM18NXgQ5coifswrwhFgrqeW/UiFRaLj0aLKU0cp0FoYjEfP/hj/yUeF+Hz754ViG+jQmG2HIeDCUeQsPnRwOA==","dev": true,"license": "MIT","optional": true,@@ -3627,14 +3532,28 @@}},"node_modules/@fly/sprites": {- "version": "0.0.1",- "resolved": "https://registry.npmjs.org/@fly/sprites/-/sprites-0.0.1.tgz",- "integrity": "sha512-1s+dIVi/pTMP4Aj4Mkg+4LoZ/+a0Kp6l9piPRxvpgEKm11b/eRiZgJwVytwAHeI/vtg2fuwcFExjtXOEfny/TA==",+ "version": "0.2.3",+ "resolved": "https://registry.npmjs.org/@fly/sprites/-/sprites-0.2.3.tgz",+ "integrity": "sha512-LB3xmHpHLdVECTQej+Vyi/aFFTc/Otf8bZ8GwVe/Hhlx4kIFS0K+9O5B8ru8Llg+cEwvYmzyeV69vNZPpCt0fw==",+ "bundleDependencies": [+ "@fly/client-signals"+ ],"license": "MIT",+ "dependencies": {+ "@fly/client-signals": "file:vendor/client-signals"+ },"engines": {"node": ">=24.0.0"}},+ "node_modules/@fly/sprites/node_modules/@fly/client-signals": {+ "version": "0.4.4",+ "inBundle": true,+ "license": "Apache-2.0",+ "engines": {+ "node": ">=20"+ }+ },"node_modules/@google/genai": {"version": "1.52.0","resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz",@@ -3790,185 +3709,6 @@"url": "https://opencollective.com/js-sdsl"}},- "node_modules/@mariozechner/clipboard": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.9.tgz",- "integrity": "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA==",- "license": "MIT",- "optional": true,- "engines": {- "node": ">= 10"- },- "optionalDependencies": {- "@mariozechner/clipboard-darwin-arm64": "0.3.9",- "@mariozechner/clipboard-darwin-universal": "0.3.9",- "@mariozechner/clipboard-darwin-x64": "0.3.9",- "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9",- "@mariozechner/clipboard-linux-arm64-musl": "0.3.9",- "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9",- "@mariozechner/clipboard-linux-x64-gnu": "0.3.9",- "@mariozechner/clipboard-linux-x64-musl": "0.3.9",- "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9",- "@mariozechner/clipboard-win32-x64-msvc": "0.3.9"- }- },- "node_modules/@mariozechner/clipboard-darwin-arm64": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-arm64/-/clipboard-darwin-arm64-0.3.9.tgz",- "integrity": "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ==",- "cpu": [- "arm64"- ],- "license": "MIT",- "optional": true,- "os": [- "darwin"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-darwin-universal": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-universal/-/clipboard-darwin-universal-0.3.9.tgz",- "integrity": "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ==",- "license": "MIT",- "optional": true,- "os": [- "darwin"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-darwin-x64": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-x64/-/clipboard-darwin-x64-0.3.9.tgz",- "integrity": "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg==",- "cpu": [- "x64"- ],- "license": "MIT",- "optional": true,- "os": [- "darwin"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-linux-arm64-gnu": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-gnu/-/clipboard-linux-arm64-gnu-0.3.9.tgz",- "integrity": "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw==",- "cpu": [- "arm64"- ],- "license": "MIT",- "optional": true,- "os": [- "linux"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-linux-arm64-musl": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-musl/-/clipboard-linux-arm64-musl-0.3.9.tgz",- "integrity": "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ==",- "cpu": [- "arm64"- ],- "license": "MIT",- "optional": true,- "os": [- "linux"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-linux-riscv64-gnu": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-riscv64-gnu/-/clipboard-linux-riscv64-gnu-0.3.9.tgz",- "integrity": "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw==",- "cpu": [- "riscv64"- ],- "license": "MIT",- "optional": true,- "os": [- "linux"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-linux-x64-gnu": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-gnu/-/clipboard-linux-x64-gnu-0.3.9.tgz",- "integrity": "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw==",- "cpu": [- "x64"- ],- "license": "MIT",- "optional": true,- "os": [- "linux"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-linux-x64-musl": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-musl/-/clipboard-linux-x64-musl-0.3.9.tgz",- "integrity": "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ==",- "cpu": [- "x64"- ],- "license": "MIT",- "optional": true,- "os": [- "linux"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-win32-arm64-msvc": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-arm64-msvc/-/clipboard-win32-arm64-msvc-0.3.9.tgz",- "integrity": "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ==",- "cpu": [- "arm64"- ],- "license": "MIT",- "optional": true,- "os": [- "win32"- ],- "engines": {- "node": ">= 10"- }- },- "node_modules/@mariozechner/clipboard-win32-x64-msvc": {- "version": "0.3.9",- "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-x64-msvc/-/clipboard-win32-x64-msvc-0.3.9.tgz",- "integrity": "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA==",- "cpu": [- "x64"- ],- "license": "MIT",- "optional": true,- "os": [- "win32"- ],- "engines": {- "node": ">= 10"- }- },"node_modules/@mistralai/mistralai": {"version": "2.2.6","resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-2.2.6.tgz",@@ -4573,7 +4313,6 @@"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.11.0.tgz","integrity": "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==","license": "Apache-2.0",- "peer": true,"dependencies": {"@opentelemetry/semantic-conventions": "^1.29.0"},@@ -4589,7 +4328,6 @@"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.220.0.tgz","integrity": "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==","license": "Apache-2.0",- "peer": true,"dependencies": {"@opentelemetry/api-logs": "0.220.0","import-in-the-middle": "^3.0.0",@@ -4640,7 +4378,6 @@"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz","integrity": "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==","license": "Apache-2.0",- "peer": true,"dependencies": {"@opentelemetry/core": "2.11.0","@opentelemetry/resources": "2.11.0",@@ -5860,12 +5597,6 @@"node": ">=18"}},- "node_modules/@silvia-odwyer/photon-node": {- "version": "0.3.4",- "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz",- "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==",- "license": "Apache-2.0"- },"node_modules/@slack/bolt": {"version": "4.7.3","resolved": "https://registry.npmjs.org/@slack/bolt/-/bolt-4.7.3.tgz",@@ -7394,15 +7125,6 @@"node": ">=8"}},- "node_modules/diff": {- "version": "8.0.4",- "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz",- "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==",- "license": "BSD-3-Clause",- "engines": {- "node": ">=0.3.1"- }- },"node_modules/dockerfile-ast": {"version": "0.7.1","resolved": "https://registry.npmjs.org/dockerfile-ast/-/dockerfile-ast-0.7.1.tgz",@@ -8433,6 +8155,7 @@"version": "1.6.0","resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz","integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==",+ "dev": true,"license": "MIT","engines": {"node": ">=18"@@ -8572,12 +8295,6 @@"url": "https://github.com/sponsors/ljharb"}},- "node_modules/graceful-fs": {- "version": "4.2.11",- "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",- "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",- "license": "ISC"- },"node_modules/has-symbols": {"version": "1.1.0","resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",@@ -8617,15 +8334,6 @@"node": ">= 0.4"}},- "node_modules/highlight.js": {- "version": "10.7.3",- "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz",- "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==",- "license": "BSD-3-Clause",- "engines": {- "node": "*"- }- },"node_modules/hono": {"version": "4.13.5","resolved": "https://registry.npmjs.org/hono/-/hono-4.13.5.tgz",@@ -8635,18 +8343,6 @@"node": ">=16.9.0"}},- "node_modules/hosted-git-info": {- "version": "9.0.3",- "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz",- "integrity": "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==",- "license": "ISC",- "dependencies": {- "lru-cache": "^11.1.0"- },- "engines": {- "node": "^20.17.0 || >=22.9.0"- }- },"node_modules/http-errors": {"version": "2.0.1","resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",@@ -8755,6 +8451,7 @@"version": "7.0.5","resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz","integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==",+ "dev": true,"license": "MIT","engines": {"node": ">= 4"@@ -8896,6 +8593,7 @@"version": "2.7.0","resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz","integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==",+ "dev": true,"license": "MIT","bin": {"jiti": "lib/jiti-cli.mjs"@@ -9355,18 +9053,6 @@"node": ">=12"}},- "node_modules/marked": {- "version": "18.0.5",- "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.5.tgz",- "integrity": "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w==",- "license": "MIT",- "bin": {- "marked": "bin/marked.js"- },- "engines": {- "node": ">= 20"- }- },"node_modules/math-intrinsics": {"version": "1.1.0","resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",@@ -10532,26 +10218,6 @@],"license": "MIT"},- "node_modules/proper-lockfile": {- "version": "4.1.2",- "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",- "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==",- "license": "MIT",- "dependencies": {- "graceful-fs": "^4.2.4",- "retry": "^0.12.0",- "signal-exit": "^3.0.2"- }- },- "node_modules/proper-lockfile/node_modules/retry": {- "version": "0.12.0",- "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",- "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",- "license": "MIT",- "engines": {- "node": ">= 4"- }- },"node_modules/protobufjs": {"version": "7.6.5","resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz",@@ -11069,12 +10735,6 @@"url": "https://github.com/sponsors/ljharb"}},- "node_modules/signal-exit": {- "version": "3.0.7",- "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",- "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",- "license": "ISC"- },"node_modules/slice-ansi": {"version": "8.0.0","resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-8.0.0.tgz",diff --git a/package.json b/package.jsonindex d343fbcb4..7c5c4b7ce 100644--- a/package.json+++ b/package.json@@ -74,7 +74,7 @@"@aws-sdk/s3-request-presigner": "^3.1061.0","@earendil-works/pi-ai": "0.82.0","@earendil-works/pi-coding-agent": "https://github.com/yc-software/pi/releases/download/qm-pi-coding-agent-0.82.0-security.3/earendil-works-pi-coding-agent-0.82.0-qm-security.3.tgz",- "@fly/sprites": "0.0.1",+ "@fly/sprites": "0.2.3","@openai/codex": "0.144.5","@opencode-ai/plugin": "1.17.18","@opencode-ai/sdk": "1.17.18",@@ -99,7 +99,6 @@"socks-proxy-agent": "10.1.0","tar-stream": "^3.2.0","typebox": "^1.1.38",- "undici": "^8.9.0","zod": "4.4.3"},"overrides": {@@ -115,7 +114,7 @@"fast-json-stringify": {"fast-uri": "4.1.4"},- "undici": "$undici",+ "undici": "8.9.0","protobufjs": "7.6.5"},"devDependencies": {diff --git a/src/config.ts b/src/config.tsindex cd864abfb..4b11df2e8 100644--- a/src/config.ts+++ b/src/config.ts@@ -348,6 +348,8 @@ interface SpritesSandboxEnv {baseUrl?: string;namePrefix?: string;egressProxyUrl?: string;+ snapshotS3Bucket?: string;+ memoryMb?: number;defaultTimeoutSec?: number;}@@ -357,6 +359,10 @@ function spritesSandboxEnv(env: NodeJS.ProcessEnv): SpritesSandboxEnv {...(env.SPRITES_BASE_URL ? { baseUrl: env.SPRITES_BASE_URL } : {}),...(env.SPRITES_NAME_PREFIX ? { namePrefix: env.SPRITES_NAME_PREFIX } : {}),...(env.SPRITES_EGRESS_PROXY_URL ? { egressProxyUrl: env.SPRITES_EGRESS_PROXY_URL } : {}),+ ...(env.SPRITES_SNAPSHOT_S3_BUCKET ? { snapshotS3Bucket: env.SPRITES_SNAPSHOT_S3_BUCKET } : {}),+ ...(numEnvStrict("SPRITES_MEMORY_MB", env.SPRITES_MEMORY_MB) !== undefined+ ? { memoryMb: numEnvStrict("SPRITES_MEMORY_MB", env.SPRITES_MEMORY_MB) }+ : {}),...(numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) !== undefined? { defaultTimeoutSec: numEnvStrict("SANDBOX_TIMEOUT_SEC", env.SANDBOX_TIMEOUT_SEC) }: {}),@@ -1175,6 +1181,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {"[config] SANDBOX_BACKEND=sprites without SPRITES_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set SPRITES_EGRESS_PROXY_URL to the public egress proxy to force sandbox traffic through it.",);}+ if (env.SANDBOX_BACKEND === "sprites" && !env.SPRITES_SNAPSHOT_S3_BUCKET) {+ console.warn(+ "[config] SANDBOX_BACKEND=sprites without SPRITES_SNAPSHOT_S3_BUCKET — retiring a computer deletes its sprite and every checkpoint irreversibly with no exported home; set SPRITES_SNAPSHOT_S3_BUCKET to export the home to S3 before a sprite is destroyed and to rehydrate a replacement.",+ );+ }if (env.SANDBOX_BACKEND === "smolmachines" && !env.SMOLMACHINES_EGRESS_PROXY_URL) {console.warn("[config] SANDBOX_BACKEND=smolmachines without SMOLMACHINES_EGRESS_PROXY_URL — machines are created with open outbound networking and NO egress enforcement (fail-open); set SMOLMACHINES_EGRESS_PROXY_URL to allow-list only the egress proxy.",diff --git a/src/sandbox/exec-process-session.ts b/src/sandbox/exec-process-session.tsindex 5b9ba2e8f..1b8ea6347 100644--- a/src/sandbox/exec-process-session.ts+++ b/src/sandbox/exec-process-session.ts@@ -25,6 +25,7 @@ export interface ExecProcessSessions {}const PROC_BASE = "${HOME:-/root}/.agent-proc";+export const processSessionDir = (processId: string): string => `${PROC_BASE}/${processId}`;const REBOOT_RC = 137;const BOOT_ID_SH = `cat /proc/sys/kernel/random/boot_id 2>/dev/null || sysctl -n kern.boottime 2>/dev/null || true`;const REAP_SH = [@@ -83,7 +84,7 @@ export function createExecProcessSessions(io: ExecProcessIo): ExecProcessSession.map(([k, v]) => `export ${k}=${shq(v)}`).join("; ");const script = [- `P="${PROC_BASE}/${processId}"`,+ `P="${processSessionDir(processId)}"`,`mkdir -p "$P"`,`printf '%s' '${b64(command)}' | base64 -d > "$P/cmd"`,...(envExports ? [`printf '%s' '${b64(envExports)}' | base64 -d > "$P/env"`] : []),@@ -110,7 +111,7 @@ export function createExecProcessSessions(io: ExecProcessIo): ExecProcessSessionconst waitMs = Math.max(0, opts?.waitMs ?? 0);const iters = Math.ceil(waitMs / 100);const script = [- `P="${PROC_BASE}/${processId}"`,+ `P="${processSessionDir(processId)}"`,`[ -d "$P" ] || { echo "MISSING=1"; exit 0; }`,REAP_SH,`_reap "$P"`,@@ -147,7 +148,7 @@ export function createExecProcessSessions(io: ExecProcessIo): ExecProcessSessionasync writeStdin(handle, processId, data): Promise<void> {assertId(processId);const script = [- `P="${PROC_BASE}/${processId}"`,+ `P="${processSessionDir(processId)}"`,`[ -p "$P/in" ] || { echo "MISSING=1"; exit 1; }`,`printf '%s' '${b64(data)}' | base64 -d > "$P/in"`,].join("\n");@@ -164,7 +165,7 @@ export function createExecProcessSessions(io: ExecProcessIo): ExecProcessSession? `[ -f "$P/code" ] || echo 137 > "$P/code"`: `if ! kill -0 -"$pid" 2>/dev/null && ! kill -0 "$pid" 2>/dev/null; then [ -f "$P/code" ] || echo 143 > "$P/code"; fi`;const script = [- `P="${PROC_BASE}/${processId}"`,+ `P="${processSessionDir(processId)}"`,`pid=$(cat "$P/pid" 2>/dev/null) || exit 0`,`kill -${sig} -"$pid" 2>/dev/null || kill -${sig} "$pid" 2>/dev/null || true`,sentinelLine,diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.tsindex 15257e47f..f73d787b4 100644--- a/src/sandbox/sprites-sandbox.ts+++ b/src/sandbox/sprites-sandbox.ts@@ -1,11 +1,14 @@import { randomUUID } from "node:crypto";-import { Agent, fetch as undiciFetch } from "undici";-import { SpritesClient } from "@fly/sprites";+import { setTimeout as wait } from "node:timers/promises";+import type { Readable } from "node:stream";+import { APIError, SpritesClient, type Checkpoint, type SpriteCheck, type StreamMessage } from "@fly/sprites";+import { createMemoryMap, type DurableMap } from "../persistence/durable-map.ts";+import { createMemoryAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";import type { WorkspaceStore } from "../workspace/workspace-store.ts";-import { fetchWithRetry, sleep } from "../util/async.ts";-import { httpFailure, swallow, swallowAs, errMessage } from "../util/errors.ts";+import { jitteredBackoffMs, retryAfterMs, withAbort, withTimeout } from "../util/async.ts";+import { swallow, errMessage } from "../util/errors.ts";import { shq } from "../util/shell.ts";-import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";+import { createExecProcessSessions, processSessionDir, type ExecProcessIo } from "./exec-process-session.ts";import {createBackendBlobStaging,createExecExport,@@ -18,28 +21,39 @@ import { visibleNotInstalled, visibleTools } from "./sandbox.ts";import { createExecSandboxBase, sandboxScopeName } from "./exec-sandbox-base.ts";import { withConnectorSdk, type ConnectorSdkBundle } from "./connector-sdk.ts";import { createLayerToolInstaller } from "./layer-tool-install.ts";+import {+ createHomeSnapshotOps,+ HOME_SNAPSHOT_PRUNE,+ snapshotDue,+ type HomeSnapshotStore,+ type SnapshotBookkeeping,+} from "./home-snapshot.ts";import type { LayerInstallFile } from "../deployment/load-layer.ts";-import type { AgentComputerProfile, ExecPressure, ExecResult, Sandbox } from "./sandbox.ts";+import type {+ AgentComputerProfile,+ ComputerStatus,+ ExecPressure,+ ExecResult,+ Sandbox,+ SandboxHandle,+ StartProcessOptions,+ TeardownOptions,+} from "./sandbox.ts";const HOME_DIR = "/home/sprite";-const spritesDispatcher = new Agent({ headersTimeout: 0, bodyTimeout: 0, allowH2: false });+const HOME_TAR = `${HOME_DIR}/.qm-home.tar`;export const SCRIPT_RUNNER = 's=$(mktemp) && cat > "$s" && sh "$s" </dev/null; rc=$?; rm -f "$s"; exit $rc';-export const FETCH_SUBSTRATE = Symbol.for("qm.fetchSubstrate");-const defaultSpritesFetch: typeof fetch = (input, init) => {- const g = globalThis.fetch as typeof fetch & { [FETCH_SUBSTRATE]?: boolean };- return g[FETCH_SUBSTRATE] ? g(input, init) : (undiciFetch as unknown as typeof fetch)(input, init);-};-const MISSING_RC = 44;-const READ_CHUNK = 512 * 1024;const EXIT_GRACE_MS = 60_000;-const RESTART_TIMEOUT_MS = 60_000;-const CHECK_TIMEOUT_MS = 30_000;const GUEST_PROBE_TIMEOUT_SEC = 15;-const DEFAULT_SPRITES_BASE_URL = "https://api.sprites.dev";+const KEEPALIVE_TIMEOUT_SEC = 20;+const CHECKPOINT_INTERVAL_MS = 5 * 60_000;+const CHECKPOINT_COMMENT = "qm turn end";+const KEEPALIVE_EXPIRE = "5m";+const KEEPALIVE_RENEW_SEC = 60;+const SPRITE_CPUS = 8;+const SPRITE_DISK_GB = 100;const PRESSURE_RECORD_FULL60 = 75;const PRESSURE_CLEAR_FULL60 = 40;-const FORCED_RESTART_ATTEMPTS = 3;-const FORCED_RESTART_RETRY_MS = 5_000;function parsePressure(ioFull10Raw: string, ioFull60Raw: string, load1Raw: string): ExecPressure | undefined {const ioFull10 = Number.parseFloat(ioFull10Raw);@@ -53,10 +67,53 @@ function parsePressure(ioFull10Raw: string, ioFull60Raw: string, load1Raw: strin};}-export interface SpritesClientLike {- getSprite(name: string): Promise<unknown>;- createSprite(name: string): Promise<unknown>;- deleteSprite(name: string): Promise<void>;+export function spritesErrorDetail(e: unknown): string {+ if (!(e instanceof APIError)) return errMessage(e);+ const parts = [e.message];+ if (e.statusCode !== undefined) parts.push(`http ${e.statusCode}`);+ if (e.errorCode && e.errorCode !== e.message) parts.push(e.errorCode);+ const retryAfter = e.getRetryAfterSeconds();+ if (retryAfter !== undefined && !e.message.includes("retry after")) parts.push(`retry after ${retryAfter}s`);+ return parts.join("; ");+}++export async function retrySpritesControl<T>(operation: () => Promise<T>, timeoutMs = 60_000): Promise<T> {+ const signal = AbortSignal.timeout(timeoutMs);+ for (let attempt = 1; ; attempt++) {+ try {+ return await withAbort(operation, signal);+ } catch (error) {+ signal.throwIfAborted();+ if (!(error instanceof APIError) || ![429, 500, 502, 503, 504].includes(error.statusCode ?? 0) || attempt >= 4)+ throw error;+ const seconds = error.getRetryAfterSeconds();+ const delay = seconds === undefined ? undefined : retryAfterMs(new Headers({ "retry-after": String(seconds) }));+ try {+ await wait(delay ?? jitteredBackoffMs(attempt), undefined, { signal });+ } catch (error) {+ signal.throwIfAborted();+ throw error;+ }+ }+ }+}++const isMissing = (e: unknown): boolean => e instanceof APIError && e.statusCode === 404;++const describeCheck = (check: SpriteCheck): string => `${check.status}${check.reason ? ` (${check.reason})` : ""}`;++export function processKeepaliveScript(processId: string): string {+ const task = `qm-proc-${processId}`;+ const api = "curl -sf --unix-socket /.sprite/api.sock -H 'Content-Type: application/json'";+ const loop = [+ `P="${processSessionDir(processId)}"`,+ `while [ ! -f "$P/code" ] && kill -0 "$(cat "$P/pid" 2>/dev/null)" 2>/dev/null; do`,+ ` ${api} -X PUT -d '{"expire":"${KEEPALIVE_EXPIRE}"}' http://sprite/v1/tasks/${task} >/dev/null 2>&1 || true`,+ ` i=0; while [ $i -lt ${KEEPALIVE_RENEW_SEC} ] && [ ! -f "$P/code" ]; do sleep 1; i=$((i+1)); done`,+ `done`,+ `${api} -X DELETE http://sprite/v1/tasks/${task} >/dev/null 2>&1 || true`,+ ].join("\n");+ return `LOOP=${shq(loop)}; if command -v setsid >/dev/null 2>&1; then setsid sh -c "$LOOP" >/dev/null 2>&1 & else sh -c "$LOOP" >/dev/null 2>&1 & fi; echo OK`;}export interface SpritesSandboxOptions extends BlobStagingOptions {@@ -65,66 +122,89 @@ export interface SpritesSandboxOptions extends BlobStagingOptions {namePrefix?: string;defaultTimeoutSec?: number;egressProxyUrl?: string;+ memoryMb?: number;+ checkpointIntervalMs?: number;+ snapshots?: HomeSnapshotStore;+ initializationStore?: DurableMap<{ pending: boolean }>;+ advisoryLock?: AdvisoryLock;extraTools?: string[];credentialPaths?: CredentialPathSpec[];connectorSdk?: () => Promise<ConnectorSdkBundle>;layerToolFiles?: () => readonly LayerInstallFile[];- client?: SpritesClientLike;- fetchImpl?: typeof fetch;onError?: (e: { category: string; code: string; message: string; scopeLabel?: string }) => void;}+interface RawExec {+ rc: number;+ stdout: Buffer;+ stderr: Buffer;+}+export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSandboxOptions = {}): Sandbox {- if ((!opts.client || !opts.fetchImpl) && !opts.token)- throw new Error("SANDBOX_BACKEND=sprites requires SPRITES_TOKEN");- const client: SpritesClientLike =- opts.client ?? (new SpritesClient(opts.token!, opts.baseUrl ? { baseURL: opts.baseUrl } : {}) as SpritesClientLike);- const rawFetch = opts.fetchImpl ?? defaultSpritesFetch;- const fetchImpl: typeof fetch = (input, init) =>- rawFetch(input, { dispatcher: spritesDispatcher, ...init } as RequestInit);- const baseUrl = (opts.baseUrl ?? DEFAULT_SPRITES_BASE_URL).replace(/\/+$/, "");+ if (!opts.token) throw new Error("SANDBOX_BACKEND=sprites requires SPRITES_TOKEN");+ const client = new SpritesClient(opts.token, opts.baseUrl ? { baseURL: opts.baseUrl } : {});+ const sprite = (name: string) => client.sprite(name);const prefix = opts.namePrefix ?? "qm";+ const initializationStore = opts.initializationStore ?? createMemoryMap<{ pending: boolean }>();+ const advisoryLock = opts.advisoryLock ?? createMemoryAdvisoryLock();+ const withLifecycle = <T>(name: string, fn: () => Promise<T>): Promise<T> =>+ advisoryLock.withLock(`sprites-lifecycle:${name}`, fn);+ const requireInitialized = async (name: string): Promise<void> => {+ if ((await initializationStore.get(name))?.pending)+ throw new Error(`sprites ${name}: initialization is incomplete; refusing to capture or restore a checkpoint`);+ };const defaultTimeoutSec = opts.defaultTimeoutSec ?? 600;+ const checkpointIntervalMs = opts.checkpointIntervalMs ?? CHECKPOINT_INTERVAL_MS;const ensured = new Set<string>();+ const resourcesApplied = new Set<string>();+ const egressPolicyByName = new Map<string, string>();+ const pressureEpisodes = new Set<string>();+ const checkpointBooks = new Map<string, SnapshotBookkeeping>();++ const reportError = (category: string, code: string, message: string, scopeLabel?: string): void => {+ try {+ opts.onError?.({ category, code, message, ...(scopeLabel ? { scopeLabel } : {}) });+ } catch (e) {+ swallow(`sprites-sandbox: ${code} report`, e);+ }+ };- interface RawExec {- rc: number;- stdout: Buffer;- stderr: Buffer;+ async function attempt<T>(what: string, fn: () => Promise<T>): Promise<T> {+ try {+ return await fn();+ } catch (e) {+ throw new Error(`sprites ${what}: ${spritesErrorDetail(e)}`, { cause: e });+ }}- async function postExec(name: string, argv: string[], timeoutSec: number, body?: Uint8Array): Promise<RawExec> {- const qs = new URLSearchParams();- if (body) qs.append("stdin", "true");- for (const a of argv) qs.append("cmd", a);- const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}/exec?${qs}`, {- method: "POST",- headers: { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/octet-stream" },- ...(body ? { body: Buffer.from(body) } : {}),- signal: AbortSignal.timeout(timeoutSec * 1000 + EXIT_GRACE_MS),- } as RequestInit);- if (!res.ok) throw new Error(`sprites exec ${name}: ${await httpFailure(res)}`);- const raw = Buffer.from(await res.arrayBuffer());- let rc = 0;+ async function spawnExec(name: string, argv: string[], stdin: Buffer, deadlineMs: number): Promise<RawExec> {+ const cmd = sprite(name).spawn(argv[0]!, argv.slice(1));const out: Buffer[] = [];const err: Buffer[] = [];- let i = 0;- while (i < raw.length) {- const id = raw[i]!;- if (id === 3) {- rc = raw[i + 1] ?? 0;- i += 2;- continue;- }- i++;- const start = i;- while (i < raw.length && raw[i]! >= 4) i++;- const payload = raw.subarray(start, i);- if (id === 1) out.push(payload);- else if (id === 2) err.push(payload);+ const collect = (stream: Readable, into: Buffer[]): Promise<void> =>+ new Promise((resolve) => {+ stream.on("data", (chunk: Buffer) => into.push(chunk));+ stream.once("end", resolve);+ });+ const exited = new Promise<number>((resolve, reject) => {+ cmd.on("error", (e) => reject(new Error(`sprites exec ${name}: ${errMessage(e)}`, { cause: e })));+ cmd.once("exit", resolve);+ cmd.once("spawn", () => cmd.stdin.end(stdin));+ });+ try {+ const [rc] = await withTimeout(+ () => Promise.all([exited, collect(cmd.stdout, out), collect(cmd.stderr, err)]),+ deadlineMs,+ `sprites exec ${name}`,+ );+ return { rc, stdout: Buffer.concat(out), stderr: Buffer.concat(err) };+ } catch (e) {+ cmd.kill();+ throw e;+ } finally {+ cmd.close();}- return { rc, stdout: Buffer.concat(out), stderr: Buffer.concat(err) };}async function execRaw(name: string, script: string, timeoutSec: number): Promise<ExecResult> {@@ -137,26 +217,27 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanconst eof = `__QM_EOF_${uid}__`;const wrapped =`cat > ${body} <<'${eof}'\n${script}\n${eof}\n` +- `timeout ${timeoutSec} sh ${body} > ${out} 2> ${err}; __rc=$?; printf '%s %s %s %s %s\\n' "$__rc" "$(wc -c < ${out})" "$(wc -c < ${err})" "${psi}" "${load}"; base64 < ${out}; base64 < ${err}; rm -f ${out} ${err} ${body}`;- const r = await postExec(name, ["sh", "-c", SCRIPT_RUNNER], timeoutSec, Buffer.from(wrapped, "utf8"));- const text = r.stdout.toString("utf8");- const nl = text.indexOf("\n");- const header = text- .slice(0, nl < 0 ? undefined : nl)- .trim()- .split(/\s+/);+ `timeout ${timeoutSec} sh ${body} > ${out} 2> ${err}; __rc=$?; printf '%s %s %s %s %s\\n' "$__rc" "$(wc -c < ${out})" "$(wc -c < ${err})" "${psi}" "${load}"; cat ${out} ${err}; rm -f ${out} ${err} ${body}`;+ const r = await spawnExec(+ name,+ ["sh", "-c", SCRIPT_RUNNER],+ Buffer.from(wrapped, "utf8"),+ timeoutSec * 1000 + EXIT_GRACE_MS,+ );+ const nl = r.stdout.indexOf(0x0a);+ const header = (nl < 0 ? r.stdout : r.stdout.subarray(0, nl)).toString("utf8").trim().split(/\s+/);if (r.rc !== 0 || nl < 0 || header.length < 3) {- throw new Error(`sprites exec ${name}: bad envelope (rc=${r.rc}): ${text.slice(0, 120)}`);+ const detail = r.stderr.toString("utf8").trim() || r.stdout.toString("utf8").slice(0, 120);+ throw new Error(`sprites exec ${name}: bad envelope (rc=${r.rc}): ${detail}`);}const pressure = header.length === 6 ? parsePressure(header[3]!, header[4]!, header[5]!) : undefined;if (pressure) notePressure(name, pressure);const code = Number.parseInt(header[0]!, 10);const outLen = Number.parseInt(header[1]!, 10);const errLen = Number.parseInt(header[2]!, 10);- const b64 = text.slice(nl + 1).replace(/\s+/g, "");- const outB64 = Math.ceil(outLen / 3) * 4;- const outBuf = Buffer.from(b64.slice(0, outB64), "base64");- const errBuf = Buffer.from(b64.slice(outB64), "base64");+ const start = nl + 1;+ const outBuf = r.stdout.subarray(start, start + outLen);+ const errBuf = r.stdout.subarray(start + outLen, start + outLen + errLen);if (outBuf.length !== outLen || errBuf.length !== errLen) {throw new Error(`sprites exec ${name}: truncated stream (${outBuf.length}/${outLen} out, ${errBuf.length}/${errLen} err)`,@@ -171,104 +252,158 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan};}- const pressureEpisodes = new Set<string>();function notePressure(name: string, pressure: ExecPressure): void {if (pressure.ioFull60 >= PRESSURE_RECORD_FULL60 && !pressureEpisodes.has(name)) {pressureEpisodes.add(name);- const scope = base.scopeFor(name);- try {- opts.onError?.({- category: "agent_computer",- code: "io_pressure_high",- message: `sprite ${name}: io pressure full avg60=${pressure.ioFull60}% (load ${pressure.load1}) — disk-bound work is stalling`,- ...(scope ? { scopeLabel: scope } : {}),- });- } catch (e) {- swallow("sprites-sandbox: io pressure report", e);- }+ reportError(+ "agent_computer",+ "io_pressure_high",+ `sprite ${name}: io pressure full avg60=${pressure.ioFull60}% (load ${pressure.load1}) — disk-bound work is stalling`,+ base.scopeFor(name),+ );} else if (pressure.ioFull60 < PRESSURE_CLEAR_FULL60) {pressureEpisodes.delete(name);}}async function writeAbsBytes(name: string, absPath: string, data: Uint8Array): Promise<void> {+ const fs = sprite(name).filesystem();const tmp = `${absPath}.part.${randomUUID()}`;- const script =- `mkdir -p "$(dirname ${shq(absPath)})" && cat > ${shq(tmp)} && ` +- `mv -f ${shq(tmp)} ${shq(absPath)} && wc -c < ${shq(absPath)}`;- const r = await postExec(name, ["sh", "-c", script], 120, data);- const written = Number.parseInt(r.stdout.toString("utf8").trim(), 10);- if (r.rc !== 0 || written !== data.length) {- throw new Error(- `sprites write ${absPath} failed (rc=${r.rc}, ${Number.isFinite(written) ? written : 0}/${data.length} bytes)`,- );+ await attempt(`write ${absPath} failed`, async () => {+ await fs.writeFile(tmp, Buffer.from(data));+ await fs.rename(tmp, absPath);+ });+ }++ async function readAbsBytes(name: string, absPath: string): Promise<Uint8Array | null> {+ try {+ return await sprite(name).filesystem().readFile(absPath, null);+ } catch (e) {+ if ((e as { code?: string }).code === "ENOENT") return null;+ throw new Error(`sprites read ${absPath} failed: ${spritesErrorDetail(e)}`, { cause: e });}}const egressProxyHost = opts.egressProxyUrl ? new URL(opts.egressProxyUrl).hostname : undefined;- const egressPolicyByName = new Map<string, string>();async function ensureEgressPolicy(name: string): Promise<void> {- const rules = [{ domain: egressProxyHost!, action: "allow" }];+ const rules = [{ domain: egressProxyHost!, action: "allow" as const }];const want = JSON.stringify(rules);if (egressPolicyByName.get(name) === want) return;- const url = `${baseUrl}/v1/sprites/${encodeURIComponent(name)}/policy/network`;- const headers = { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/json" };- const res = await fetchWithRetry(- (signal) =>- fetchImpl(url, {- method: "POST",- headers,- body: JSON.stringify({ rules }),- signal,- }),- "idempotent",- { timeoutMs: 30_000 },- );- if (!res.ok) throw new Error(`sprites egress policy ${name}: ${await httpFailure(res)}`);- const check = await fetchWithRetry((signal) => fetchImpl(url, { headers, signal }), "idempotent", {- timeoutMs: 30_000,- });- const got = (await check.json().catch(() => null)) as {- rules?: Array<{ domain?: string; action?: string }>;- } | null;+ const s = sprite(name);+ await attempt(`egress policy ${name}`, () => retrySpritesControl(() => s.updateNetworkPolicy({ rules })));+ const got = await attempt(`egress policy readback ${name}`, () => retrySpritesControl(() => s.getNetworkPolicy()));const norm = (d?: string) => (d ?? "").toLowerCase().replace(/\.$/, "");- const only = got?.rules?.length === 1 ? got.rules[0] : undefined;+ const only = got.rules?.length === 1 ? got.rules[0] : undefined;const bound = !!only && norm(only.domain) === norm(egressProxyHost) && only.action?.toLowerCase() === "allow";- if (!check.ok || !bound)+ if (!bound)throw new Error(`sprites egress policy ${name}: readback mismatch (${JSON.stringify(got).slice(0, 200)})`);egressPolicyByName.set(name, want);}- async function readAbsBytes(name: string, absPath: string): Promise<Uint8Array | null> {- const script =- `[ -e ${shq(absPath)} ] || exit ${MISSING_RC}; s=$(wc -c < ${shq(absPath)}); echo "$s"; ` +- `if [ "$s" -le ${READ_CHUNK} ]; then base64 < ${shq(absPath)}; fi`;- const r = await postExec(name, ["sh", "-c", script], 120);- if (r.rc === MISSING_RC) return null;- const text = r.stdout.toString("utf8");- if (r.rc !== 0) throw new Error(`sprites read ${absPath} failed (${r.rc}): ${text.slice(0, 200)}`);- const nl = text.indexOf("\n");- const declared = Number.parseInt(text.slice(0, nl < 0 ? undefined : nl).trim(), 10);- if (!Number.isFinite(declared)) throw new Error(`sprites read ${absPath}: bad size (${text.slice(0, 40)})`);- const parts: Buffer[] = [];- if (declared <= READ_CHUNK) {- parts.push(Buffer.from(text.slice(nl + 1).replace(/\s+/g, ""), "base64"));- } else {- for (let i = 0; i < Math.ceil(declared / READ_CHUNK); i++) {- const chunk = `dd if=${shq(absPath)} bs=${READ_CHUNK} skip=${i} count=1 2>/dev/null | base64`;- const c = await postExec(name, ["sh", "-c", chunk], 120);- if (c.rc !== 0) throw new Error(`sprites read ${absPath} chunk ${i} failed (${c.rc})`);- parts.push(Buffer.from(c.stdout.toString("utf8").replace(/\s+/g, ""), "base64"));- }+ async function applyResources(name: string): Promise<void> {+ if (opts.memoryMb === undefined || resourcesApplied.has(name)) return;+ await attempt(`resources policy ${name}`, () =>+ sprite(name).updateResourcesPolicy({ memory: { limitMB: opts.memoryMb! } }),+ );+ resourcesApplied.add(name);+ }++ async function spriteExists(name: string): Promise<boolean> {+ try {+ await retrySpritesControl(() => client.getSprite(name));+ return true;+ } catch (e) {+ if (isMissing(e)) return false;+ throw new Error(`sprites get ${name}: ${spritesErrorDetail(e)}`, { cause: e });+ }+ }++ async function deleteSprite(name: string): Promise<void> {+ try {+ await retrySpritesControl(() => client.deleteSprite(name));+ } catch (e) {+ if (!isMissing(e)) throw new Error(`sprites delete ${name}: ${spritesErrorDetail(e)}`, { cause: e });+ }+ }++ const homeSnapshots = opts.snapshots+ ? createHomeSnapshotOps<string>({+ label: "sprites",+ homeDir: HOME_DIR,+ homeTarPath: HOME_TAR,+ prunePaths: [+ ...HOME_SNAPSHOT_PRUNE,+ ...ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => `./${rel}`),+ ],+ store: opts.snapshots,+ io: {+ runCommand: async (name, script, timeoutMs) => {+ const r = await execRaw(name, script, Math.ceil(timeoutMs / 1000));+ return { exitCode: r.code, stdout: r.stdout, stderr: r.stderr };+ },+ readFileBytes: readAbsBytes,+ writeFileBytes: writeAbsBytes,+ },+ })+ : undefined;++ async function exportHome(name: string, scope: string): Promise<void> {+ await requireInitialized(name);+ if (!homeSnapshots || !(await spriteExists(name))) return;+ await homeSnapshots.snapshotHome(scope, name);+ }++ async function settleStream(stream: AsyncIterable<StreamMessage>, what: string): Promise<void> {+ let failure: string | undefined;+ for await (const msg of stream) {+ if (msg.type === "error") failure = msg.error ?? msg.data ?? "error";+ else if (msg.type === "complete") failure = undefined;}- const data = Buffer.concat(parts);- if (data.length !== declared) {- throw new Error(`sprites read ${absPath}: truncated (${data.length}/${declared})`);+ if (failure !== undefined) throw new Error(`${what}: ${failure}`);+ }++ async function createCheckpoint(name: string): Promise<void> {+ await settleStream(await sprite(name).createCheckpoint(CHECKPOINT_COMMENT), `sprites checkpoint ${name}`);+ }++ async function latestCheckpoint(name: string): Promise<Checkpoint | undefined> {+ const list = await retrySpritesControl(() => sprite(name).listCheckpoints());+ return list+ .filter((c) => c.id !== "Current" && !c.health)+ .sort((a, b) => b.createTime.getTime() - a.createTime.getTime())[0];+ }++ async function restoreLatestCheckpoint(name: string): Promise<Checkpoint | undefined> {+ const latest = await latestCheckpoint(name);+ if (!latest) return undefined;+ await settleStream(await sprite(name).restoreCheckpoint(latest.id), `sprites restore ${name} ${latest.id}`);+ return latest;+ }++ async function checkpointIfDue(name: string, tdOpts?: TeardownOptions): Promise<void> {+ await requireInitialized(name);+ const book = checkpointBooks.get(name) ?? {};+ checkpointBooks.set(name, book);+ if (!tdOpts?.homeUnchanged) book.homeDirty = true;+ if (!snapshotDue(book, tdOpts, checkpointIntervalMs)) return;+ try {+ await createCheckpoint(name);+ book.lastSnapshotMs = Date.now();+ book.homeDirty = false;+ } catch (e) {+ reportError("sandbox_snapshot", "checkpoint_failed", errMessage(e), base.scopeFor(name));}- return data;}+ const forget = (name: string): void => {+ ensured.delete(name);+ resourcesApplied.delete(name);+ pressureEpisodes.delete(name);+ egressPolicyByName.delete(name);+ checkpointBooks.delete(name);+ };+const base = createExecSandboxBase({workspace,label: "sprites",@@ -288,54 +423,49 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanexec: execRaw,writeAbsBytes,readAbsBytes,- async ensureResident(name, onStatus) {- if (ensured.has(name)) return { coldStart: false };- let exists = false;- try {- await client.getSprite(name);- exists = true;- } catch (error) {- void error;- }- if (!exists) {+ ensureResident(name, onStatus) {+ return withLifecycle(name, async () => {+ const pending = (await initializationStore.get(name))?.pending;+ if (ensured.has(name) && !pending) return { coldStart: false };+ const exists = await spriteExists(name);+ if (exists && !pending) {+ await applyResources(name);+ ensured.add(name);+ return { coldStart: false };+ }+ await initializationStore.put(name, { pending: true });try {- onStatus?.("Creating the sandbox…");+ onStatus?.("Preparing the sandbox…");} catch (error) {void error;}+ const scope = base.scopeFor(name);try {- await client.createSprite(name);+ if (!exists) await attempt(`create ${name}`, () => client.createSprite(name, { waitForCapacity: true }));+ await applyResources(name);+ const hydrated = homeSnapshots && scope ? await homeSnapshots.hydrateHome(scope, name) : false;+ await initializationStore.delete(name);ensured.add(name);- return { coldStart: true };- } catch (createErr) {- try {- await client.getSprite(name);- } catch {- throw createErr;- }+ return { coldStart: !hydrated };+ } catch (e) {+ forget(name);+ reportError("sandbox_hydrate", "hydrate_failed", errMessage(e), scope);+ await deleteSprite(name).catch((deleteErr) =>+ swallow("sprites-sandbox: delete after failed hydrate", deleteErr),+ );+ throw e;}- }- ensured.add(name);- return { coldStart: false };+ });},isProvisioned: (name) => ensured.has(name),async recreateScratch(name) {- let stale = true;- try {- await client.getSprite(name);- } catch {- stale = false;- }- if (stale) await client.deleteSprite(name).catch(swallowAs("sprites-sandbox: stale scratch delete", undefined));- await client.createSprite(name);+ if (await spriteExists(name)) await deleteSprite(name);+ await attempt(`create ${name}`, () => client.createSprite(name, { waitForCapacity: true }));+ await applyResources(name);ensured.add(name);},- deleteInstance: (name) => client.deleteSprite(name),- forgetInstance: (name) => {- ensured.delete(name);- pressureEpisodes.delete(name);- egressPolicyByName.delete(name);- },+ deleteInstance: deleteSprite,+ forgetInstance: forget,ensureEgress: ensureEgressPolicy,});@@ -345,7 +475,7 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanprocessSessions: true,egressEnforcement: opts.egressProxyUrl ? "domain" : "none",spec: {- os: "Ubuntu 26.04 LTS — Fly Sprite microVM (auto-sleeps when idle; the whole disk persists)",+ os: "Ubuntu (25.10 for newly created sprites; existing sprites keep the release they were created with) — Fly Sprite microVM: the whole disk persists and idle sprites sleep; background processes are held awake while they run but do not survive a cold wake or restart",runtimes: ["Node 24", "Python 3"],get tools() {return visibleTools(["git", "curl", "jq", "tar", "python3", ...(opts.extraTools ?? [])]);@@ -353,7 +483,9 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanget notInstalled() {return visibleNotInstalled(["gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], opts.extraTools ?? []);},- diskGb: 100,+ cpus: SPRITE_CPUS,+ ...(opts.memoryMb !== undefined ? { memoryMb: opts.memoryMb } : {}),+ diskGb: SPRITE_DISK_GB,homeDir: HOME_DIR,workdir: base.workspaceDir,},@@ -384,9 +516,29 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanephemeralCredentialPrefixes: ephemeralCredLinkPaths(opts.credentialPaths ?? []).map(({ rel }) => rel),});+ async function startProcess(+ handle: SandboxHandle,+ command: string,+ spOpts?: StartProcessOptions,+ ): Promise<{ processId: string }> {+ const started = await procSessions.startProcess(handle, command, spOpts);+ try {+ const r = await execRaw(handle.id, processKeepaliveScript(started.processId), KEEPALIVE_TIMEOUT_SEC);+ if (r.code !== 0 || !r.stdout.includes("OK")) throw new Error(r.stderr.trim() || `exit ${r.code}`);+ } catch (e) {+ reportError(+ "agent_computer",+ "process_keepalive_failed",+ `sprite ${handle.id}: process ${started.processId} runs without a Tasks hold and may freeze when the sprite idles: ${errMessage(e)}`,+ base.scopeFor(handle.id),+ );+ }+ return started;+ }+return {profile,- startProcess: procSessions.startProcess,+ startProcess,readProcess: procSessions.readProcess,writeStdin: procSessions.writeStdin,signalProcess: procSessions.signalProcess,@@ -402,46 +554,53 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanexportFiles: execExport.exportFiles,async destroyScope(scopeId: string): Promise<void> {- return base.provisionQueue(scopeId, async () => {- const name = sandboxScopeName(prefix, scopeId);- const res = await fetchWithRetry(- (signal) =>- fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}`, {- method: "DELETE",- headers: { authorization: `Bearer ${opts.token ?? ""}` },- signal,- }),- "idempotent",- { timeoutMs: RESTART_TIMEOUT_MS },- );- if (!res.ok && res.status !== 404) throw new Error(`sprites delete ${name}: ${await httpFailure(res)}`);- ensured.delete(name);- pressureEpisodes.delete(name);- egressPolicyByName.delete(name);- });+ const name = sandboxScopeName(prefix, scopeId);+ return base.provisionQueue(scopeId, () =>+ withLifecycle(name, async () => {+ if (!(await initializationStore.get(name))?.pending) await exportHome(name, scopeId);+ await deleteSprite(name);+ forget(name);+ }),+ );},- async computerStatus(scopeId: string) {+ ...(homeSnapshots+ ? {+ async persistHomeSnapshot(scopeId: string): Promise<void> {+ const name = sandboxScopeName(prefix, scopeId);+ return base.provisionQueue(scopeId, () =>+ withLifecycle(name, async () => {+ await requireInitialized(name);+ if (!(await spriteExists(name)))+ throw new Error(`sprites persistHomeSnapshot: no sprite for ${scopeId}`);+ await homeSnapshots.snapshotHome(scopeId, name);+ }),+ );+ },+ }+ : {}),++ async computerStatus(scopeId: string): Promise<ComputerStatus> {const name = sandboxScopeName(prefix, scopeId);- const spriteJson = async (path: string): Promise<{ status?: string } | null> => {- const res = await fetchWithRetry(- (signal) =>- fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}${path}`, {- headers: { authorization: `Bearer ${opts.token ?? ""}` },- signal,- }),- "idempotent",- { timeoutMs: CHECK_TIMEOUT_MS },- );- if (!res.ok) throw new Error(await httpFailure(res));- return (await res.json().catch(() => null)) as { status?: string } | null;- };- const [machineOut, listedOut] = await Promise.allSettled([spriteJson("/check"), spriteJson("")]);+ const [checked, latest] = await Promise.allSettled([+ retrySpritesControl(() => sprite(name).check()),+ latestCheckpoint(name),+ ]);+ if (checked.status === "rejected" && isMissing(checked.reason)) {+ return { machine: "no sprite provisioned yet", provisioned: false, guestResponsive: false };+ }const machine =- machineOut.status === "fulfilled"- ? (machineOut.value?.status ?? "check failed: no status")- : `check failed: ${errMessage(machineOut.reason)}`;- const listed = listedOut.status === "fulfilled" ? listedOut.value?.status : undefined;+ checked.status === "fulfilled"+ ? describeCheck(checked.value)+ : `check failed: ${spritesErrorDetail(checked.reason)}`;+ const recovery: ComputerStatus["recovery"] = {+ strategy: "provider_snapshot",+ checkpointExpiresAtMs: null,+ ...(latest.status === "fulfilled" && latest.value+ ? { checkpointId: latest.value.id, checkpointAtMs: latest.value.createTime.getTime() }+ : {}),+ ...(latest.status === "rejected" ? { error: `checkpoint list failed: ${errMessage(latest.reason)}` } : {}),+ };let guestResponsive = false;let pressure: ExecPressure | undefined;try {@@ -453,40 +612,73 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan}return {machine,- ...(listed ? { listed } : {}),- provisioned: machineOut.status === "fulfilled",+ provisioned: checked.status === "fulfilled",guestResponsive,+ recovery,...(pressure ? { pressure } : {}),};},restartComputer(scopeId: string): Promise<void> {const name = sandboxScopeName(prefix, scopeId);- return base.provisionQueue(`restart:${name}`, async () => {- ensured.delete(name);- egressPolicyByName.delete(name);- const restart = (force: boolean) =>- fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}/restart${force ? "?force=true" : ""}`, {- method: "POST",- headers: { authorization: `Bearer ${opts.token ?? ""}` },- signal: AbortSignal.timeout(RESTART_TIMEOUT_MS),- });- const plain = await restart(false);- if (plain.ok) return;- const plainDetail = await httpFailure(plain);- if (plain.status !== 409 && plain.status !== 502) throw new Error(`sprites restart ${name}: ${plainDetail}`);- let forcedDetail = "";- for (let attempt = 0; attempt < FORCED_RESTART_ATTEMPTS; attempt++) {- if (attempt > 0) await sleep(FORCED_RESTART_RETRY_MS);- const forced = await restart(true);- if (forced.ok) return;- forcedDetail = await httpFailure(forced);- if (forced.status !== 409 && forced.status !== 502) break;+ return base.provisionQueue(`restart:${name}`, () =>+ withLifecycle(name, async () => {+ await requireInitialized(name);+ forget(name);+ const s = sprite(name);+ const restartFailure = await s.restart().then(+ () => undefined,+ (e) => spritesErrorDetail(e),+ );+ if (restartFailure === undefined) return;+ let check: SpriteCheck;+ try {+ check = await retrySpritesControl(() => s.check());+ } catch (e) {+ throw new Error(+ `sprites restart ${name}: ${restartFailure}; check failed: ${spritesErrorDetail(e)}; no checkpoint was restored`,+ { cause: e },+ );+ }+ const fault = check.status === "unhealthy" ? `check reports ${describeCheck(check)}` : undefined;+ if (fault === undefined) {+ throw new Error(+ `sprites restart ${name}: ${restartFailure}; the health check reports no fault, so no checkpoint was restored`,+ );+ }+ let restored: Checkpoint | undefined;+ try {+ restored = await restoreLatestCheckpoint(name);+ } catch (e) {+ throw new Error(+ `sprites restart ${name}: ${restartFailure}; ${fault}; checkpoint restore failed: ${errMessage(e)}`,+ {+ cause: e,+ },+ );+ }+ if (!restored)+ throw new Error(`sprites restart ${name}: ${restartFailure}; ${fault}; no checkpoint to restore`);+ reportError(+ "agent_computer",+ "checkpoint_restored",+ `sprite ${name}: restart failed (${restartFailure}); ${fault}; restored checkpoint ${restored.id} from ${restored.createTime.toISOString()}`,+ base.scopeFor(name) ?? scopeId,+ );+ }),+ );+ },++ async teardown(handle: SandboxHandle, tdOpts?: TeardownOptions): Promise<void> {+ return withLifecycle(handle.id, async () => {+ if (!handle.scratch) {+ if (tdOpts?.destroy) {+ if (!(await initializationStore.get(handle.id))?.pending)+ await exportHome(handle.id, base.scopeFor(handle.id) ?? handle.id);+ } else await checkpointIfDue(handle.id, tdOpts);}- throw new Error(`sprites restart ${name}: ${plainDetail}; forced retry: ${forcedDetail}`);+ return base.teardown(handle, tdOpts);});},-- teardown: base.teardown,};}diff --git a/src/wiring.ts b/src/wiring.tsindex 87adc7cb8..240922d16 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -839,9 +839,15 @@ export function buildApp(...config.localSandbox,onError: sandboxOnError,});- const buildSprites = (): Sandbox =>- createSpritesSandbox(workspace, {- ...config.spritesSandbox,+ const buildSprites = (): Sandbox => {+ const { snapshotS3Bucket, ...sprites } = config.spritesSandbox;+ return createSpritesSandbox(workspace, {+ ...sprites,+ initializationStore: artifactMap<{ pending: boolean }>("sprites_initialization"),+ advisoryLock,+ ...(snapshotS3Bucket+ ? { snapshots: createS3SnapshotStore({ bucket: snapshotS3Bucket, prefix: "sprites-home" }) }+ : {}),blobTransfer,extraTools: deploymentLayer.advertisedTools,credentialPaths: deploymentLayer.credentialPaths,@@ -852,6 +858,7 @@ export function buildApp(...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}),onError: sandboxOnError,});+ };const smolmachinesBodies = artifactMap<StoredSmolmachinesSandbox>("smolmachines_sandbox_bodies");const buildSmolmachines = (): Sandbox => {const { snapshotS3Bucket, snapshotIntervalSec, ...smol } = config.smolmachinesSandbox;diff --git a/test/agent-computer-profile.test.ts b/test/agent-computer-profile.test.tsindex 58eba5bab..ccadbe6d5 100644--- a/test/agent-computer-profile.test.ts+++ b/test/agent-computer-profile.test.ts@@ -45,6 +45,8 @@ test("the sprites sandbox declares the Agent Computer contract (persistent per-segressEnforcement: "none",});assert.match(spec?.os ?? "", /Ubuntu/);+ assert.equal(spec?.cpus, 8);+ assert.equal(spec?.diskGb, 100);assert.equal(spec?.homeDir, "/home/sprite");assert.equal(spec?.workdir, "/home/sprite/workspace");assert.ok(spec?.notInstalled?.includes("gh"));diff --git a/test/connector-invariants.test.ts b/test/connector-invariants.test.tsindex f0e3f4c24..ec3b39a7f 100644--- a/test/connector-invariants.test.ts+++ b/test/connector-invariants.test.ts@@ -221,7 +221,7 @@ test("a read-only wake never reaches the sandbox (execute stripped), so no execconst res = await built.app.turn(wake("[wake] glance only", true));assert.equal(res.status, "ok");assert.ok(- !fakeSprites.calls.some((c) => c.method === "POST" && c.path.endsWith("/exec")),+ !fakeSprites.calls.some((c) => c.method === "WS" && c.path.endsWith("/exec")),"a read-only wake spins no sandbox exec",);});diff --git a/test/device-flow-persist.test.ts b/test/device-flow-persist.test.tsindex 338096bc1..cc24893f0 100644--- a/test/device-flow-persist.test.ts+++ b/test/device-flow-persist.test.ts@@ -38,8 +38,7 @@ function sprites() {const dir = mkdtempSync(join(tmpdir(), "dfp-ws-"));return createSpritesSandbox(createLocalWorkspaceStore(dir), {token: "test-token",- client: ff.client,- fetchImpl: ff.fetchImpl,+ baseUrl: ff.baseUrl,});}const rw = (scope: string) => [{ scopeId: scope, mountPath: "", mode: "rw" as const }];@@ -374,8 +373,7 @@ test("ACMECLI quarantine removes the canonical root even with no record or a staconst dir = mkdtempSync(join(tmpdir(), "dfp-ws-"));const sb = createSpritesSandbox(createLocalWorkspaceStore(dir), {token: "test-token",- client: ff.client,- fetchImpl: ff.fetchImpl,+ baseUrl: ff.baseUrl,credentialPaths: [{ path: ".acmecli", kind: "directory" }],});const k = kc();diff --git a/test/exec-kill.test.ts b/test/exec-kill.test.tsindex e2f866c05..5079dfb8c 100644--- a/test/exec-kill.test.ts+++ b/test/exec-kill.test.ts@@ -37,8 +37,7 @@ function sprites(): Sandbox {const dir = mkdtempSync(join(tmpdir(), "sprites-kill-ws-"));return createSpritesSandbox(createLocalWorkspaceStore(dir), {token: "test-token",- client: ff.client,- fetchImpl: ff.fetchImpl,+ baseUrl: ff.baseUrl,});}const rw = [{ scopeId: scopeId("personal", "U1"), mountPath: "", mode: "rw" as const }];diff --git a/test/file-sharing.test.ts b/test/file-sharing.test.tsindex 5d0ff0f24..4e14df1f3 100644--- a/test/file-sharing.test.ts+++ b/test/file-sharing.test.ts@@ -600,7 +600,7 @@ test("removeDir wipes a per-turn spool dir (and only it), tolerating an absent dconst ff = installFakeSprites();after(() => ff.cleanup());const ws = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "fs-rm-")));- const sandbox = createSpritesSandbox(ws, { token: "test-token", client: ff.client, fetchImpl: ff.fetchImpl });+ const sandbox = createSpritesSandbox(ws, { token: "test-token", baseUrl: ff.baseUrl });const handle = await sandbox.provision([{ scopeId: "personal:U1", mountPath: "", mode: "rw" }]);await sandbox.writeFileBytes(handle, "spool/one.txt", new Uint8Array(Buffer.from("1")));await sandbox.writeFileBytes(handle, "spool/two.txt", new Uint8Array(Buffer.from("2")));@@ -620,7 +620,7 @@ test("a binary file round-trips through the sandbox (base64-over-exec) without uconst ff = installFakeSprites();after(() => ff.cleanup());const ws = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "fs-bin-")));- const sandbox = createSpritesSandbox(ws, { token: "test-token", client: ff.client, fetchImpl: ff.fetchImpl });+ const sandbox = createSpritesSandbox(ws, { token: "test-token", baseUrl: ff.baseUrl });const handle = await sandbox.provision([{ scopeId: "personal:U1", mountPath: "", mode: "rw" }]);const raw = new Uint8Array([0x00, 0x9f, 0x92, 0x96, 0xff, 0xfe]);await sandbox.writeFileBytes(handle, "keep.bin", raw);diff --git a/test/sandbox-destroy-scope.test.ts b/test/sandbox-destroy-scope.test.tsindex f4d5d696c..0b6bc70e4 100644--- a/test/sandbox-destroy-scope.test.ts+++ b/test/sandbox-destroy-scope.test.ts@@ -16,6 +16,7 @@ import { createAgent37Sandbox } from "../src/sandbox/agent37-sandbox.ts";import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts";import { installFakeSmolmachines } from "./support/fake-smolmachines.ts";+import { installFakeSprites } from "./support/fake-sprites.ts";import { installFakeAgent37 } from "./support/fake-agent37.ts";const workspace = () => createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "destroy-scope-")));@@ -41,21 +42,25 @@ test("Local scope deletion removes deterministic resources without provisioning,});test("Sprites scope deletion sends only DELETE, retries provider errors, and accepts missing bodies", async () => {- const calls: string[] = [];- let status = 503;- const sandbox = createSpritesSandbox(workspace(), {- token: "test",- fetchImpl: async (input, init) => {- calls.push(`${init?.method} ${new URL(String(input)).pathname}`);- return new Response(null, { status, headers: { "retry-after": "0" } });- },- });- await assert.rejects(sandbox.destroyScope!("scope"), /503/);- status = 204;- await sandbox.destroyScope!("scope");- status = 404;- await sandbox.destroyScope!("scope");- assert.deepEqual(calls, Array(6).fill(`DELETE /v1/sprites/${sandboxScopeName("qm", "scope")}`));+ const fake = installFakeSprites();+ try {+ const sandbox = createSpritesSandbox(workspace(), { token: "test", baseUrl: fake.baseUrl });+ await sandbox.provision([{ scopeId: "scope", mountPath: "", mode: "rw" }]);+ fake.calls.length = 0;+ fake.refuseDelete(503);+ await assert.rejects(sandbox.destroyScope!("scope"), /503/);+ assert.equal(fake.names().length, 1);+ fake.refuseDelete();+ await sandbox.destroyScope!("scope");+ assert.deepEqual(fake.names(), []);+ await sandbox.destroyScope!("scope");+ assert.deepEqual(+ fake.calls.map((c) => `${c.method} ${c.path}`),+ Array(6).fill(`DELETE /v1/sprites/${sandboxScopeName("qm", "scope")}`),+ );+ } finally {+ fake.cleanup();+ }});for (const [name, create, resource] of [diff --git a/test/sandbox-noninteractive.test.ts b/test/sandbox-noninteractive.test.tsindex 8e0530ce6..04df8c946 100644--- a/test/sandbox-noninteractive.test.ts+++ b/test/sandbox-noninteractive.test.ts@@ -19,8 +19,7 @@ function spritesHandle(env?: Record<string, string>) {const dir = mkdtempSync(join(tmpdir(), "noninteractive-"));const sandbox = createSpritesSandbox(createLocalWorkspaceStore(dir), {token: "test-token",- client: ff.client,- fetchImpl: ff.fetchImpl,+ baseUrl: ff.baseUrl,});const layers = [{ scopeId: scopeId("personal", "U1"), mountPath: "", mode: "rw" as const }];return { sandbox, layers, ...(env ? { env } : {}) };diff --git a/test/scope-reach.test.ts b/test/scope-reach.test.tsindex ea5bf3319..200f35e46 100644--- a/test/scope-reach.test.ts+++ b/test/scope-reach.test.ts@@ -348,7 +348,7 @@ test("DM + directory + flag: execute(scope:#room) runs on that channel's own comassert.equal(res.status, "ok");assert.equal(res.reply, "still here");assert.ok(- fakeSprites.calls.some((c) => c.method === "POST" && /\/sprites\/qm-channel-c-ph-[^/]+\/exec$/.test(c.path)),+ fakeSprites.calls.some((c) => c.method === "WS" && /\/sprites\/qm-channel-c-ph-[^/]+\/exec$/.test(c.path)),"the command landed on the channel's computer",);});diff --git a/test/sprites-sandbox.test.ts b/test/sprites-sandbox.test.tsindex 4a77d32ff..5f834f94d 100644--- a/test/sprites-sandbox.test.ts+++ b/test/sprites-sandbox.test.ts@@ -1,18 +1,28 @@+import { createMemoryMap } from "../src/persistence/durable-map.ts";+import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";import { pollProcess } from "../src/sandbox/process-poll.ts";import { test, after, beforeEach } from "node:test";import assert from "node:assert/strict";import { mkdtempSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import { createSpritesSandbox } from "../src/sandbox/sprites-sandbox.ts";+import {+ createSpritesSandbox,+ processKeepaliveScript,+ retrySpritesControl,+ spritesErrorDetail,+} from "../src/sandbox/sprites-sandbox.ts";import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts";import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";import { execFailureDetail, supportsProcessSessions, supportsBlobStaging } from "../src/sandbox/sandbox.ts";+import { createMemorySnapshotStore } from "../src/sandbox/home-snapshot.ts";+import { sleep } from "../src/util/async.ts";import { createMemoryBlobTransferStore } from "../src/persistence/blob-transfer.ts";import { scopeId } from "../src/types.ts";import { mintCapabilityToken, EGRESS_PROXY_AUD } from "../src/auth/capability-token.ts";import { installFakeSprites, FAKE_SPRITES_TOKEN, type FakeSprites } from "./support/fake-sprites.ts";import type { Sandbox } from "../src/sandbox/sandbox.ts";+import { APIError } from "@fly/sprites";let fake: FakeSprites;let sandbox: Sandbox;@@ -23,13 +33,19 @@ function make(extra: Record<string, unknown> = {}): Sandbox {return createSpritesSandbox(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "sprites-ws-"))), {token: FAKE_SPRITES_TOKEN,namePrefix: "qmt",- client: fake.client,- fetchImpl: fake.fetchImpl,+ baseUrl: fake.baseUrl,...extra,});}+const proxyToken = () =>+ mintCapabilityToken(+ { actorId: "tester", scopeId: scope, aud: EGRESS_PROXY_AUD, exp: Date.now() + 600_000 },+ "secret",+ );+beforeEach(() => {+ fake?.cleanup();fake = installFakeSprites();sandbox = make();});@@ -52,24 +68,55 @@ test("streams and exit codes are exact", async () => {assert.equal(r.stderr.trim(), "err");});-test("file roundtrip incl. large binary and missing file", async () => {+test("commands run over the WebSocket exec endpoint with the script in the stream, never the URL", async () => {+ const h = await sandbox.provision(layers);+ const huge = `echo start; : ${"x".repeat(1024 * 1024)}; echo end`;+ const r = await sandbox.run(h, huge);+ assert.equal(r.code, 0);+ assert.match(r.stdout, /start\s+end/);+ const execs = fake.calls.filter((c) => c.method === "WS");+ assert.ok(execs.length > 0, "commands ride the WebSocket exec channel");+ assert.ok(+ !fake.calls.some((c) => c.method === "POST" && c.path.endsWith("/exec")),+ "the frame-ambiguous HTTP exec fallback is never used",+ );+ assert.ok(Math.max(...execs.map((c) => c.path.length)) < 2048, "exec URLs stay small");+ assert.ok(+ execs.some((c) => (c.script?.length ?? 0) > 1024 * 1024),+ "the megabyte script travels as stdin frames",+ );+});++test("file roundtrip incl. large binary and missing file goes through the filesystem API", async () => {const h = await sandbox.provision(layers);await sandbox.writeFile(h, "a/b.txt", "hello\n");assert.equal(await sandbox.readFile(h, "a/b.txt"), "hello\n");assert.equal(await sandbox.readFile(h, "nope.txt"), null);- const big = Buffer.alloc(200 * 1024);- for (let i = 0; i < big.length; i++) big[i] = (i * 7) % 256;- await sandbox.writeFileBytes(h, "big.bin", big);- const back = await sandbox.readFileBytes(h, "big.bin");- assert.ok(back && Buffer.from(back).equals(big));const huge = Buffer.alloc(1300 * 1024);for (let i = 0; i < huge.length; i++) huge[i] = (i * 13) % 256;await sandbox.writeFileBytes(h, "huge.bin", huge);const hugeBack = await sandbox.readFileBytes(h, "huge.bin");assert.ok(hugeBack && Buffer.from(hugeBack).equals(huge));+ assert.ok(fake.calls.some((c) => c.method === "PUT" && c.path.includes("/fs/write?")));+ assert.ok(fake.calls.some((c) => c.method === "GET" && c.path.includes("/fs/read?")));+ assert.ok(!fake.execScripts().some((s) => /base64|dd if=/.test(s)), "bytes never round-trip through shell encoding");+});++test("a file write lands on a temp path and is renamed into place, never streamed into the target", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "cfg.txt", "value\n");+ assert.equal(await sandbox.readFile(h, "cfg.txt"), "value\n");+ const write = fake.calls.find((c) => c.method === "PUT" && c.path.includes("cfg.txt"));+ assert.ok(write, "expected a filesystem write");+ assert.match(write!.path, /cfg\.txt\.part\./, "the payload lands on a temp path");+ const writeIdx = fake.calls.indexOf(write!);+ assert.ok(+ fake.calls.slice(writeIdx).some((c) => c.method === "POST" && c.path.endsWith("/fs/rename")),+ "and is renamed over the target afterwards",+ );});-test("process sessions capability works end to end", async () => {+test("process sessions capability works end to end and holds the sprite awake with a Tasks heartbeat", async () => {assert.ok(supportsProcessSessions(sandbox));if (!supportsProcessSessions(sandbox)) return;const h = await sandbox.provision(layers);@@ -78,15 +125,29 @@ test("process sessions capability works end to end", async () => {assert.equal(status.state, "exited");assert.match(output, /one/);assert.match(output, /two/);+ const keepalive = fake.execScripts().find((s) => s.includes("/.sprite/api.sock"));+ assert.ok(keepalive, "a keepalive sidecar was launched next to the process");+ assert.match(keepalive!, new RegExp(`/v1/tasks/qm-proc-${processId}`));+ assert.match(keepalive!, /-X PUT/, "renews by upsert so the first call creates the task");+ assert.match(keepalive!, /-X DELETE/, "and releases the hold when the process is gone");+ const listed = await sandbox.listProcesses(h);+ assert.deepEqual(+ listed.map((p) => p.command),+ ["echo one; echo two"],+ "the sidecar is not a visible process session and the command stays readable",+ );+});++test("the keepalive script stops renewing once the process has recorded an exit code", () => {+ const script = processKeepaliveScript("00000000-0000-0000-0000-000000000000");+ assert.match(script, /while \[ ! -f "\$P\/code" \]/);+ assert.match(script, /"expire":"5m"/);+ assert.match(script, /setsid/);});test("background processes inherit the force-through proxy env", async () => {const s = make({ egressProxyUrl: "https://proxy.example.com" });- const token = await mintCapabilityToken(- { actorId: "tester", scopeId: scope, aud: EGRESS_PROXY_AUD, exp: Date.now() + 600_000 },- "secret",- );- const h = await s.provision(layers, { egressToken: token });+ const h = await s.provision(layers, { egressToken: await proxyToken() });assert.ok(supportsProcessSessions(s));if (!supportsProcessSessions(s)) return;const { processId } = await s.startProcess(h, "echo PROXY=$HTTPS_PROXY");@@ -138,37 +199,23 @@ test("force-through pins the platform policy and injects proxy env", async () =>test("force-through strips agent-supplied proxy vars", async () => {const s = make({ egressProxyUrl: "https://proxy.example.com" });- const token = await mintCapabilityToken(- { actorId: "tester", scopeId: scope, aud: EGRESS_PROXY_AUD, exp: Date.now() + 600_000 },- "secret",- );- const h = await s.provision(layers, { egressToken: token, env: { HTTPS_PROXY: "http://evil:1", FOO: "keep" } });+ const h = await s.provision(layers, {+ egressToken: await proxyToken(),+ env: { HTTPS_PROXY: "http://evil:1", FOO: "keep" },+ });assert.ok(!h.env?.HTTPS_PROXY?.includes("evil"));assert.equal(h.env?.FOO, "keep");});test("force-through fails closed if the policy readback doesn't bind", async () => {- const brokenFetch: typeof fetch = async (input, init) => {- const url = new URL(typeof input === "string" ? input : input.toString());- if (url.pathname.endsWith("/policy/network") && (init?.method ?? "GET") === "GET") {- return Response.json({ rules: [] });- }- return fake.fetchImpl(input, init);- };- const s = make({ egressProxyUrl: "https://proxy.example.com", fetchImpl: brokenFetch });- const token = await mintCapabilityToken(- { actorId: "tester", scopeId: scope, aud: EGRESS_PROXY_AUD, exp: Date.now() + 600_000 },- "secret",- );- await assert.rejects(s.provision(layers, { egressToken: token }), /readback mismatch/);+ const s = make({ egressProxyUrl: "https://proxy.example.com" });+ fake.breakPolicyReadback(sandboxScopeName("qmt", scope));+ await assert.rejects(s.provision(layers, { egressToken: await proxyToken() }), /readback mismatch/);});test("a recreated sprite gets the egress policy re-pinned, never served from a stale cache", async () => {const s = make({ egressProxyUrl: "https://proxy.example.com" });- const token = await mintCapabilityToken(- { actorId: "tester", scopeId: scope, aud: EGRESS_PROXY_AUD, exp: Date.now() + 600_000 },- "secret",- );+ const token = await proxyToken();const a = await s.provision(layers, { scratch: { key: "job-pol" }, egressToken: token });assert.deepEqual(fake.policy(a.id), [{ domain: "proxy.example.com", action: "allow" }]);await s.teardown(a);@@ -179,20 +226,10 @@ test("a recreated sprite gets the egress policy re-pinned, never served from a s});test("a failed egress setup releases the scratch lease instead of wedging the key", async () => {- const brokenFetch: typeof fetch = async (input, init) => {- const url = new URL(typeof input === "string" ? input : input.toString());- if (url.pathname.endsWith("/policy/network") && (init?.method ?? "GET") === "GET") {- return Response.json({ rules: [] });- }- return fake.fetchImpl(input, init);- };- const s = make({ egressProxyUrl: "https://proxy.example.com", fetchImpl: brokenFetch });- const token = await mintCapabilityToken(- { actorId: "tester", scopeId: scope, aud: EGRESS_PROXY_AUD, exp: Date.now() + 600_000 },- "secret",- );+ const s = make({ egressProxyUrl: "https://proxy.example.com" });+ fake.breakPolicyReadback(sandboxScopeName("qmt-scratch", "job-egress"));await assert.rejects(- s.provision(layers, { scratch: { key: "job-egress" }, egressToken: token }),+ s.provision(layers, { scratch: { key: "job-egress" }, egressToken: await proxyToken() }),/readback mismatch/,);const retry = await s.provision(layers, { scratch: { key: "job-egress" } });@@ -211,112 +248,77 @@ test("scratch sprites are ephemeral and shared leases survive until the last relassert.ok(fake.names().includes(a.id));await sandbox.teardown(b);assert.ok(!fake.names().includes(b.id));+ assert.equal(fake.checkpoints(a.id).length, 0, "scratch boxes are never checkpointed");});-test("teardown is a no-op park; destroy deletes the sprite", async () => {+test("teardown checkpoints a changed home, throttles repeats, and destroy deletes the sprite", async () => {const h = await sandbox.provision(layers);await sandbox.teardown(h);- assert.ok(fake.names().includes(h.id));+ assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a turn that may have changed the home ends in a checkpoint");+ await sandbox.teardown(h);+ assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a second teardown inside the interval is throttled");+ assert.ok(fake.names().includes(h.id), "parking keeps the sprite");await sandbox.teardown(h, { destroy: true });assert.ok(!fake.names().includes(h.id));});-test("exportFiles tars workspace + home over the exec channel (the publish fast path)", async () => {- const h = await sandbox.provision(layers);- await sandbox.run(- h,- [- "mkdir -p app/.cache",- "printf hi > app/index.html",- "printf junk > app/.cache/x",- 'printf note > "$HOME/.profile-note"',- ].join(" && "),- );-- const got = await sandbox.exportFiles!(h);- const paths = got.map((e) => `${e.area}:${e.path}`).sort();- assert.ok(paths.includes("workspace:app/index.html"), `workspace file packed (got ${paths.join(", ")})`);- assert.ok(paths.includes("home:.profile-note"), "home file packed");- assert.ok(!paths.some((p) => p.includes(".cache")), "content caches pruned by default");- assert.ok(!paths.some((p) => p.startsWith("home:workspace/")), "workspace pruned from the home area");- assert.equal(Buffer.from(got.find((e) => e.path === "app/index.html")!.data).toString("utf8"), "hi");+test("an unchanged home is not checkpointed once its state is known", async () => {+ const s = make({ checkpointIntervalMs: 1 });+ const h = await s.provision(layers);+ await s.teardown(h, { homeUnchanged: true });+ assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a sprite this core has not checkpointed yet is saved once");+ await sleep(5);+ await s.teardown(h, { homeUnchanged: true });+ assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a quiet turn after that adds nothing");+ await s.teardown(h);+ assert.deepEqual(fake.checkpoints(h.id), ["v1", "v2"], "a turn that may have changed the home is saved");+ await sleep(5);+ await s.teardown(h, { homeUnchanged: true });+ assert.deepEqual(fake.checkpoints(h.id), ["v1", "v2"]);+});++test("a checkpoint failure is reported and never fails the teardown", async () => {+ const events: Array<{ code: string }> = [];+ const s = make({ onError: (e: { code: string }) => events.push(e) });+ const h = await s.provision(layers);+ fake.fail502(h.id);+ fake.unhealthy(h.id, "checkpoint store offline");+ await s.teardown(h);+ assert.ok(fake.checkpoints(h.id).length <= 1);});-test("exportFiles keepContentCaches ships cache-named build output (publish parity)", async () => {+test("computerStatus reports checkpoint recovery and a healthy machine whose shell has stopped answering", async () => {const h = await sandbox.provision(layers);- await sandbox.run(h, "mkdir -p site/.cache && printf real > site/.cache/bundle.js");- const got = await sandbox.exportFiles!(h, {- include: ["workspace"],- keepContentCaches: true,- exclude: () => false,- });- assert.ok(- got.some((e) => e.path === "site/.cache/bundle.js"),- "cache-named build output crosses when publish asks for it",- );-});--test("blob staging is advertised only when the channel is actually wired", async () => {- assert.equal(- supportsBlobStaging(make()),- false,- "without blobTransfer/secret/apiBaseUrl the capability must not be claimed — copyHome probes for it",- );- const wired = make({- blobTransfer: createMemoryBlobTransferStore(),- capabilitySecret: "blob-secret",- apiBaseUrl: "http://core.internal:8080",- });- assert.equal(supportsBlobStaging(wired), true, "wired up, sprites can move bytes by reference");-});+ const fresh = await sandbox.computerStatus!(scope);+ assert.equal(fresh.machine, "healthy");+ assert.equal(fresh.provisioned, true);+ assert.equal(fresh.guestResponsive, true);+ assert.equal(fresh.listed, undefined, "the list-view status is stale by design and is not surfaced");+ assert.deepEqual(fresh.recovery, { strategy: "provider_snapshot", checkpointExpiresAtMs: null });-test("stageOut posts to core's blob endpoint by streaming, never by buffering in the guest", async () => {- const sb = make({- blobTransfer: createMemoryBlobTransferStore(),- capabilitySecret: "blob-secret",- apiBaseUrl: "http://core.internal:8080",- });- const h = await sb.provision(layers);- await assert.rejects(() => sb.stageOut!(h, "artifacts/big.bin"), /sprites stageOut/);-- const script = fake.execScripts().find((s: string) => s.includes("/v1/blobs"))!;- assert.ok(script, "the stageOut curl reached the guest");- assert.match(script, /--upload-file/, "streams from disk rather than buffering in the guest");- assert.doesNotMatch(script, /--data-binary/, "the OOM shape must never come back");- assert.match(script, /-X POST/, "--upload-file alone would send PUT");- assert.match(script, /x-content-sha256/, "core verifies the upload end-to-end");-});--test("stageIn pulls a blob into the guest atomically (temp then mv)", async () => {- const sb = make({- blobTransfer: createMemoryBlobTransferStore(),- capabilitySecret: "blob-secret",- apiBaseUrl: "http://core.internal:8080",- });- const h = await sb.provision(layers);- await assert.rejects(() => sb.stageIn!(h, "inbox/big.bin", "f".repeat(32)), /sprites stageIn/);-- const script = fake.execScripts().find((s: string) => s.includes("/v1/blobs/"))!;- assert.match(script, /-o .*\.part/, "downloads to a temp file");- assert.match(script, /mv -f /, "and only then moves it into place");- assert.match(script, /curl -fsS/, "-f so an HTTP error fails loudly instead of writing the error body");+ await sandbox.teardown(h);+ fake.fail502(h.id);+ const wedged = await sandbox.computerStatus!(scope);+ assert.equal(wedged.machine, "healthy");+ assert.equal(wedged.guestResponsive, false);+ assert.equal(wedged.recovery?.checkpointId, "v1");+ assert.equal(typeof wedged.recovery?.checkpointAtMs, "number");});-test("a prep failure that writes nothing still names its cause", () => {- assert.equal(- execFailureDetail({ stdout: "", stderr: "", code: 124, timedOut: true }, 60),- "timed out after 60s with no output",- "a wedged guest filesystem kills the script before it can explain itself",- );- assert.equal(execFailureDetail({ stdout: "", stderr: "", code: 1, timedOut: false }, 60), "exit 1 with no output");- assert.equal(execFailureDetail({ stdout: "out", stderr: " boom\n", code: 1, timedOut: false }, 60), "boom");- assert.equal(execFailureDetail({ stdout: " out\n", stderr: "", code: 1, timedOut: false }, 60), "out");+test("computerStatus names a faulted check and an unprovisioned scope", async () => {+ const none = await sandbox.computerStatus!(scope);+ assert.deepEqual(none, { machine: "no sprite provisioned yet", provisioned: false, guestResponsive: false });+ const h = await sandbox.provision(layers);+ fake.unhealthy(h.id, "disk unreachable");+ const s = await sandbox.computerStatus!(scope);+ assert.equal(s.machine, "unhealthy (disk unreachable)");+ assert.equal(s.provisioned, true);});test("restartComputer reboots the scope's sprite and heals a wedged exec channel", async () => {const h = await sandbox.provision(layers);fake.fail502(h.id);- await assert.rejects(sandbox.run(h, "echo back"), /http 502/);+ await assert.rejects(sandbox.run(h, "echo back"), /WebSocket error/);await sandbox.restartComputer!(scope);assert.deepEqual(fake.restarts(), [h.id]);@@ -326,22 +328,40 @@ test("restartComputer reboots the scope's sprite and heals a wedged exec channelassert.equal(after.stdout.trim(), "back");});-test("computerStatus reports a healthy machine whose shell has stopped answering", async () => {+test("a refused restart on a faulted sprite restores the latest checkpoint", async () => {+ const events: Array<{ code: string; message: string }> = [];+ const s = make({ onError: (e: { code: string; message: string }) => events.push(e) });+ const h = await s.provision(layers);+ await s.writeFile(h, "keep.txt", "before\n");+ await s.teardown(h);+ await s.writeFile(h, "keep.txt", "after\n");+ fake.refuseRestart(h.id);+ fake.unhealthy(h.id, "machine unreachable");+ fake.fail502(h.id);++ await s.restartComputer!(scope);+ assert.deepEqual(fake.restarts(), []);+ assert.equal(await s.readFile(h, "keep.txt"), "before\n", "the home is back at the checkpoint");+ const restored = events.find((e) => e.code === "checkpoint_restored");+ assert.ok(restored, "the destructive recovery is reported");+ assert.match(restored!.message, /http 502/);+ assert.match(restored!.message, /machine unreachable/);+ assert.match(restored!.message, /restored checkpoint v1/);+});++test("a refused restart with a healthy check restores nothing and surfaces the refusal", async () => {const h = await sandbox.provision(layers);- assert.deepEqual(await sandbox.computerStatus!(scope), {- machine: "healthy",- listed: "warm",- provisioned: true,- guestResponsive: true,- });+ await sandbox.teardown(h);+ fake.refuseRestart(h.id);+ await assert.rejects(sandbox.restartComputer!(scope), /http 502.*reports no fault/s);+ assert.equal(await sandbox.readFile(h, ".ro-layers.manifest"), null);+});- fake.fail502(h.id);- assert.deepEqual(await sandbox.computerStatus!(scope), {- machine: "healthy",- listed: "warm",- provisioned: true,- guestResponsive: false,- });+test("a refused restart with no checkpoint to fall back on names all three failures", async () => {+ const h = await sandbox.provision(layers);+ fake.refuseRestart(h.id);+ fake.unhealthy(h.id, "boot loop");+ await assert.rejects(sandbox.restartComputer!(scope), /http 502.*boot loop.*no checkpoint to restore/s);});test("a command that ran before the response was lost is never re-executed", async () => {@@ -355,32 +375,6 @@ test("a command that ran before the response was lost is never re-executed", asyassert.equal(ledger, "entry\n", "the side effect must have happened exactly once");});-test("an inline write lands atomically, so a reboot mid-write cannot truncate the target", async () => {- const h = await sandbox.provision(layers);- await sandbox.writeFile(h, "cfg.txt", "value\n");- assert.equal(await sandbox.readFile(h, "cfg.txt"), "value\n");-- const write = fake.execScripts().find((s) => s.includes("cfg.txt") && s.includes("cat >"));- assert.ok(write, "expected an inline write script");- assert.match(write!, /\.part\./, "the payload must land on a temp path");- assert.match(write!, /mv -f/, "and be renamed over the target, never streamed into it");-});--test("a refused plain restart falls back to a forced one", async () => {- const h = await sandbox.provision(layers);- fake.refuseRestart(h.id);- await sandbox.restartComputer!(scope);- assert.deepEqual(fake.restarts(), [`${h.id}?force=true`]);-});--test("a restart refused even with force surfaces both failures", async () => {- const h = await sandbox.provision(layers);- fake.refuseRestart(h.id);- fake.refuseForcedRestart(h.id);- await assert.rejects(sandbox.restartComputer!(scope), /http 502 .*forced retry: http 502/s);- assert.deepEqual(fake.restarts(), []);-});-test("exec results carry io pressure when the guest exposes it, and omit it when it can't be read", async () => {const h = await sandbox.provision(layers);const bare = await sandbox.run(h, "echo ok");@@ -413,12 +407,11 @@ test("sustained io pressure is reported once per episode, then re-arms after itassert.equal(events.filter((e) => e.code === "io_pressure_high").length, 2, "a new episode records again");});-test("computerStatus carries the list-view status and guest pressure alongside the health check", async () => {+test("computerStatus carries guest pressure alongside the health check", async () => {const h = await sandbox.provision(layers);fake.setPressure(h.id, { full10: 60, full60: 55, load1: 8 });const s = await sandbox.computerStatus!(scope);assert.equal(s.machine, "healthy");- assert.equal(s.listed, "warm");assert.equal(s.guestResponsive, true);assert.deepEqual(s.pressure, { ioFull10: 60, ioFull60: 55, load1: 8 });});@@ -439,106 +432,281 @@ test("concurrent restart calls for one sprite collapse into sequential requests"assert.deepEqual(fake.restarts(), [h.id, h.id], "serialized, one request per call, never interleaved forcing");});-test("exec fetches carry a dispatcher with undici's 5-minute header/body caps disabled", async () => {- let seen: unknown;- const spyFetch: typeof fetch = async (input, init) => {- const url = new URL(typeof input === "string" ? input : input.toString());- if (url.pathname.endsWith("/exec")) seen = (init as { dispatcher?: unknown } | undefined)?.dispatcher;- return fake.fetchImpl(input, init);- };- const s = make({ fetchImpl: spyFetch });- const h = await s.provision(layers);- await s.run(h, "echo ok");- assert.ok(seen, "exec requests must not ride the default dispatcher, whose ~300s caps kill long execs");-});--test("the script travels in the request body, so command size never reaches the URL", async () => {- const seen: Array<{ pathLength: number; bodyBytes: number }> = [];- const spyFetch: typeof fetch = async (input, init) => {- const url = new URL(typeof input === "string" ? input : input.toString());- if (url.pathname.endsWith("/exec")) {- const body = init?.body as Buffer | undefined;- seen.push({ pathLength: url.pathname.length + url.search.length, bodyBytes: body ? body.byteLength : 0 });- }- return fake.fetchImpl(input, init);- };- const s = make({ fetchImpl: spyFetch });- const h = await s.provision(layers);- const huge = `echo start; : ${"x".repeat(1024 * 1024)}; echo end`;- const r = await s.run(h, huge);+test("commands cannot swallow the script from stdin", async () => {+ const h = await sandbox.provision(layers);+ const r = await sandbox.run(h, "cat; echo after-cat");assert.equal(r.code, 0);- assert.match(r.stdout, /start\s+end/);- const largest = Math.max(...seen.map((x) => x.pathLength));- assert.ok(largest < 2048, `every exec URL must stay small, saw ${largest} chars`);+ assert.equal(r.stdout.trim(), "after-cat");+});++test("a creation rate limit surfaces the provider's code and retry hint", async () => {+ fake.rateLimitCreate(42);+ await assert.rejects(sandbox.provision(layers), (e: Error) => {+ assert.match(e.message, /sprites create /);+ assert.match(e.message, /sprite_creation_rate_limited/);+ assert.match(e.message, /retry after: 42s/);+ assert.match(e.message, /http 429/);+ assert.ok(e.cause instanceof APIError);+ return true;+ });+ const created = await sandbox.provision(layers);+ assert.equal(created.coldStart, true, "the next attempt creates normally");assert.ok(- seen.some((x) => x.bodyBytes > 1024 * 1024),- "the megabyte script must ride in the body",+ fake.calls.some((c) => c.method === "POST" && c.path === "/v1/sprites"),+ "creation goes through the SDK client",);});-test("commands cannot swallow the script from stdin", async () => {+test("spritesErrorDetail keeps plain errors as they are and enriches API errors", () => {+ assert.equal(spritesErrorDetail(new Error("boom")), "boom");+ const limited = new APIError("Too many", {+ statusCode: 429,+ errorCode: "concurrent_sprite_limit_exceeded",+ retryAfterHeader: 7,+ });+ assert.equal(spritesErrorDetail(limited), "Too many; http 429; concurrent_sprite_limit_exceeded; retry after 7s");+});++test("a configured memory limit is applied as a resources policy and advertised in the profile", async () => {+ const s = make({ memoryMb: 4096 });+ assert.equal(s.profile.spec?.memoryMb, 4096);+ assert.equal(s.profile.spec?.cpus, 8);+ const h = await s.provision(layers);+ assert.deepEqual(fake.resources(h.id), { limitMB: 4096 });+ assert.equal(sandbox.profile.spec?.memoryMb, undefined, "no knob, no claim");+ assert.equal(fake.resources(sandboxScopeName("qmt", scope))?.limitMB, 4096);+});++test("the profile is honest about the base release and what survives sleep", () => {+ const os = sandbox.profile.spec?.os ?? "";+ assert.match(os, /Ubuntu \(25\.10 for newly created sprites/);+ assert.doesNotMatch(os, /LTS/);+ assert.match(os, /do not survive a cold wake/);+});++test("exportFiles tars workspace + home over the exec channel (the publish fast path)", async () => {const h = await sandbox.provision(layers);- const r = await sandbox.run(h, "cat; echo after-cat");- assert.equal(r.code, 0);- assert.equal(r.stdout.trim(), "after-cat");+ await sandbox.run(+ h,+ [+ "mkdir -p app/.cache",+ "printf hi > app/index.html",+ "printf junk > app/.cache/x",+ 'printf note > "$HOME/.profile-note"',+ ].join(" && "),+ );++ const got = await sandbox.exportFiles!(h);+ const paths = got.map((e) => `${e.area}:${e.path}`).sort();+ assert.ok(paths.includes("workspace:app/index.html"), `workspace file packed (got ${paths.join(", ")})`);+ assert.ok(paths.includes("home:.profile-note"), "home file packed");+ assert.ok(!paths.some((p) => p.includes(".cache")), "content caches pruned by default");+ assert.ok(!paths.some((p) => p.startsWith("home:workspace/")), "workspace pruned from the home area");+ assert.equal(Buffer.from(got.find((e) => e.path === "app/index.html")!.data).toString("utf8"), "hi");});-test("every sprites fetch rides one HTTP/1.1 dispatcher, so a bad request fails alone", async () => {- const dispatchers = new Map<string, unknown>();- const spyFetch: typeof fetch = async (input, init) => {- const url = new URL(typeof input === "string" ? input : input.toString());- dispatchers.set(- url.pathname.split("/").slice(4).join("/") || "root",- (init as { dispatcher?: unknown })?.dispatcher,- );- return fake.fetchImpl(input, init);- };- const s = make({ fetchImpl: spyFetch, egressProxyUrl: "https://egress.example" });- const token = await mintCapabilityToken(- {- actorId: "tester",- scopeId: scope,- aud: EGRESS_PROXY_AUD,- egress: { allowedHosts: ["api.anthropic.com"], deniedHosts: [] },- exp: Date.now() + 600_000,- },- "secret",+test("exportFiles keepContentCaches ships cache-named build output (publish parity)", async () => {+ const h = await sandbox.provision(layers);+ await sandbox.run(h, "mkdir -p site/.cache && printf real > site/.cache/bundle.js");+ const got = await sandbox.exportFiles!(h, {+ include: ["workspace"],+ keepContentCaches: true,+ exclude: () => false,+ });+ assert.ok(+ got.some((e) => e.path === "site/.cache/bundle.js"),+ "cache-named build output crosses when publish asks for it",);- const h = await s.provision(layers, { egressToken: token });- await s.run(h, "echo ok");- await s.computerStatus!(scope);- const distinct = new Set(dispatchers.values());+});++test("blob staging is advertised only when the channel is actually wired", async () => {assert.equal(- distinct.size,- 1,- `expected one shared dispatcher, saw ${distinct.size} across ${[...dispatchers.keys()]}`,+ supportsBlobStaging(make()),+ false,+ "without blobTransfer/secret/apiBaseUrl the capability must not be claimed — copyHome probes for it",+ );+ const wired = make({+ blobTransfer: createMemoryBlobTransferStore(),+ capabilitySecret: "blob-secret",+ apiBaseUrl: "http://core.internal:8080",+ });+ assert.equal(supportsBlobStaging(wired), true, "wired up, sprites can move bytes by reference");+});++test("stageOut posts to core's blob endpoint by streaming, never by buffering in the guest", async () => {+ const sb = make({+ blobTransfer: createMemoryBlobTransferStore(),+ capabilitySecret: "blob-secret",+ apiBaseUrl: "http://core.internal:8080",+ });+ const h = await sb.provision(layers);+ await assert.rejects(() => sb.stageOut!(h, "artifacts/big.bin"), /sprites stageOut/);++ const script = fake.execScripts().find((s: string) => s.includes("/v1/blobs"))!;+ assert.ok(script, "the stageOut curl reached the guest");+ assert.match(script, /--upload-file/, "streams from disk rather than buffering in the guest");+ assert.doesNotMatch(script, /--data-binary/, "the OOM shape must never come back");+ assert.match(script, /-X POST/, "--upload-file alone would send PUT");+ assert.match(script, /x-content-sha256/, "core verifies the upload end-to-end");+});++test("stageIn pulls a blob into the guest atomically (temp then mv)", async () => {+ const sb = make({+ blobTransfer: createMemoryBlobTransferStore(),+ capabilitySecret: "blob-secret",+ apiBaseUrl: "http://core.internal:8080",+ });+ const h = await sb.provision(layers);+ await assert.rejects(() => sb.stageIn!(h, "inbox/big.bin", "f".repeat(32)), /sprites stageIn/);++ const script = fake.execScripts().find((s: string) => s.includes("/v1/blobs/"))!;+ assert.match(script, /-o .*\.part/, "downloads to a temp file");+ assert.match(script, /mv -f /, "and only then moves it into place");+ assert.match(script, /curl -fsS/, "-f so an HTTP error fails loudly instead of writing the error body");+});++test("a prep failure that writes nothing still names its cause", () => {+ assert.equal(+ execFailureDetail({ stdout: "", stderr: "", code: 124, timedOut: true }, 60),+ "timed out after 60s with no output",+ "a wedged guest filesystem kills the script before it can explain itself",+ );+ assert.equal(execFailureDetail({ stdout: "", stderr: "", code: 1, timedOut: false }, 60), "exit 1 with no output");+ assert.equal(execFailureDetail({ stdout: "out", stderr: " boom\n", code: 1, timedOut: false }, 60), "boom");+ assert.equal(execFailureDetail({ stdout: " out\n", stderr: "", code: 1, timedOut: false }, 60), "out");+});++test("destroying a scope exports the home to the snapshot store first and a replacement hydrates from it", async () => {+ const snapshots = createMemorySnapshotStore();+ const s = make({ snapshots });+ const h = await s.provision(layers);+ await s.run(h, 'printf survivor > "$HOME/notes.txt"');+ assert.equal(await snapshots.open(scope), null);++ await s.destroyScope!(scope);+ assert.ok(!fake.names().includes(h.id), "the sprite is gone");+ const stored = await snapshots.open(scope);+ assert.ok(stored && stored.size > 0, "but its home was exported before deletion");++ const again = await s.provision(layers);+ assert.equal(again.coldStart, false, "the replacement is hydrated, not cold");+ const back = await s.run(again, 'cat "$HOME/notes.txt"');+ assert.equal(back.stdout, "survivor");+ assert.equal(typeof s.persistHomeSnapshot, "function", "explicit export is offered when a store is wired");+ assert.equal(typeof make().persistHomeSnapshot, "undefined");+});++test("a failed export refuses to destroy the sprite", async () => {+ const snapshots = createMemorySnapshotStore();+ const s = make({ snapshots });+ const h = await s.provision(layers);+ fake.fail502(h.id);+ await assert.rejects(s.destroyScope!(scope), /WebSocket error/);+ assert.ok(fake.names().includes(h.id), "an irreversible delete never follows a lost export");+});++test("without a snapshot store a destroy is a single delete that tolerates a missing sprite", async () => {+ await sandbox.destroyScope!(scope);+ assert.deepEqual(+ fake.calls.map((c) => `${c.method} ${c.path}`),+ [`DELETE /v1/sprites/${sandboxScopeName("qmt", scope)}`],);- const dispatcher = [...distinct][0] as Record<symbol, { allowH2?: boolean }>;- const optionsKey = Object.getOwnPropertySymbols(dispatcher).find((k) => String(k).includes("options"));- assert.ok(optionsKey, "undici Agent must expose its options");- assert.equal(dispatcher[optionsKey]?.allowH2, false);+ fake.refuseDelete(503);+ await assert.rejects(sandbox.destroyScope!(scope), /sprites delete .*http 503/);});test("control-plane reads retry 429 with Retry-After while exec is never retried", async () => {const h = await sandbox.provision(layers);fake.failNext(429, { headers: { "retry-after": "0" }, match: (c) => c.path.endsWith("/check") });- assert.deepEqual(await sandbox.computerStatus!(scope), {- machine: "healthy",- listed: "warm",- provisioned: true,- guestResponsive: true,- });+ const status = await sandbox.computerStatus!(scope);+ assert.equal(status.provisioned, true);+ assert.equal(status.guestResponsive, true);assert.equal(fake.calls.filter((c) => c.path.endsWith("/check")).length, 2);- const before = fake.calls.filter((c) => c.path.endsWith("/exec")).length;- fake.failNext(502, { headers: { "fly-request-id": "fly-9" }, match: (c) => c.path.endsWith("/exec") });- await assert.rejects(sandbox.run(h, "echo hi"), /sprites exec .*: http 502 .*\[request id fly-9\]/);- assert.equal(fake.calls.filter((c) => c.path.endsWith("/exec")).length, before + 1);+ const before = fake.execScripts().length;+ fake.stallAfterRun(h.id);+ await assert.rejects(sandbox.run(h, "echo hi"), /connection reset/);+ assert.equal(fake.execScripts().length, before + 1);for (let i = 1; i <= 4; i++)fake.failNext(503, {headers: { "retry-after": "0", "x-request-id": `req-${i}` },match: (c) => c.method === "DELETE",});- await assert.rejects(sandbox.destroyScope!(scope), /sprites delete .*: http 503 .*\[request id req-4\]/);+ await assert.rejects(sandbox.destroyScope!(scope), /sprites delete .*injected 503/);+});++test("control retries share an elapsed-time budget across backoff and SDK requests", async () => {+ let calls = 0;+ await assert.rejects(+ retrySpritesControl(async () => {+ calls++;+ throw new APIError("rate limited", { statusCode: 429, retryAfterHeader: 30 });+ }, 30),+ { name: "TimeoutError" },+ );+ assert.equal(calls, 1);+ calls = 0;+ await assert.rejects(+ retrySpritesControl(async () => {+ calls++;+ if (calls === 1) throw new APIError("unavailable", { statusCode: 503, retryAfterHeader: 0 });+ return new Promise((resolve) => setTimeout(() => resolve("late response"), 100));+ }, 30),+ { name: "TimeoutError" },+ );+ assert.equal(calls, 2);+});++for (const outcome of ["healthy", "unknown", "403", "503"]) {+ test(`refused restart with ${outcome} health observation never rolls back newer files`, async () => {+ const h = await sandbox.provision(layers);+ await sandbox.writeFile(h, "ledger", "old");+ await sandbox.teardown(h);+ await sandbox.writeFile(h, "ledger", "new");+ fake.refuseRestart(h.id);+ if (outcome === "healthy" || outcome === "unknown") fake.health(h.id, outcome, "machine is running");+ else+ for (let i = 0; i < (outcome === "503" ? 4 : 1); i++)+ fake.failNext(Number(outcome), { headers: { "retry-after": "0" }, match: (c) => c.path.endsWith("/check") });+ await assert.rejects(sandbox.restartComputer!(scope), /no checkpoint was restored/);+ assert.equal(await sandbox.readFile(h, "ledger"), "new");+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/restore")).length, 0);+ });+}++test("failed hydration and failed deletion remain pending across adapters without overwriting the saved home", async () => {+ const snapshots = createMemorySnapshotStore();+ const initializationStore = createMemoryMap<{ pending: boolean }>();+ const advisoryLock = createMemoryAdvisoryLock();+ let failOpen = false;+ let opens = 0;+ const wrapped = {+ ...snapshots,+ open: async (id: string) => {+ opens++;+ if (failOpen) throw new Error("snapshot unavailable");+ return snapshots.open(id);+ },+ };+ const options = { snapshots: wrapped, initializationStore, advisoryLock };+ const a = make(options);+ const h = await a.provision(layers);+ await a.writeFile(h, "ledger", "saved");+ await a.destroyScope!(scope);+ failOpen = true;+ fake.refuseDelete(403);+ await assert.rejects(a.provision(layers), /snapshot unavailable/);+ assert.deepEqual(await initializationStore.get(h.id), { pending: true });+ const b = make(options);+ await assert.rejects(b.persistHomeSnapshot!(scope), /initialization is incomplete/);+ await assert.rejects(b.restartComputer!(scope), /initialization is incomplete/);+ const before = opens;+ await assert.rejects(b.provision(layers), /snapshot unavailable/);+ assert.equal(opens, before + 1);+ failOpen = false;+ fake.refuseDelete();+ const restored = await make(options).provision(layers);+ assert.equal(await b.readFile(restored, "ledger"), "saved");+ assert.equal(await initializationStore.get(h.id), null);});diff --git a/test/support/fake-sprites.ts b/test/support/fake-sprites.tsindex 0f01a25c1..acf108399 100644--- a/test/support/fake-sprites.ts+++ b/test/support/fake-sprites.ts@@ -1,9 +1,18 @@import { spawnSync } from "node:child_process";-import { FETCH_SUBSTRATE, SCRIPT_RUNNER } from "../../src/sandbox/sprites-sandbox.ts";-import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";+import {+ chmodSync,+ cpSync,+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ renameSync,+ rmSync,+ writeFileSync,+} from "node:fs";import { tmpdir } from "node:os";-import { join } from "node:path";-import type { SpritesClientLike } from "../../src/sandbox/sprites-sandbox.ts";+import { dirname, join } from "node:path";+import { SCRIPT_RUNNER } from "../../src/sandbox/sprites-sandbox.ts";export interface NetworkRule {domain: string;@@ -13,7 +22,6 @@ export interface NetworkRule {export interface SpritesCall {method: string;path: string;- /** For exec calls: the script (last `cmd` argv element). */script?: string;}@@ -23,19 +31,23 @@ export interface InjectedFailure {}export interface FakeSprites {- client: SpritesClientLike;- fetchImpl: typeof fetch;+ baseUrl: string;calls: SpritesCall[];- /** Host-side dir standing in for the sprite's disk — wipe it to simulate a replaced computer. */homeDir(name: string): string;names(): string[];policy(name: string): NetworkRule[] | null;+ resources(name: string): { limitMB: number } | null;+ checkpoints(name: string): string[];execScripts(): string[];stallAfterRun(name: string): void;fail502(name: string): void;failNext(status: number, opts?: InjectedFailure): void;refuseRestart(name: string): void;- refuseForcedRestart(name: string): void;+ unhealthy(name: string, reason: string): void;+ health(name: string, status: string, reason: string): void;+ refuseDelete(status?: number): void;+ rateLimitCreate(retryAfterSeconds: number): void;+ breakPolicyReadback(name: string): void;setPressure(name: string, p: { full10: number; full60: number; load1: number }): void;restarts(): string[];reset(): void;@@ -44,18 +56,130 @@ export interface FakeSprites {const TOKEN = "test-token";const API_ORIGIN = "https://api.sprites.dev";+const GUEST_HOME = "/home/sprite";-export function installFakeSprites(): FakeSprites {+interface Engine {+ fetch(url: URL, init?: RequestInit): Promise<Response>;+ exec(url: URL): FakeExec | null;+}++interface FakeExec {+ run(stdin: Buffer): { frames: Buffer[]; dropAfterRun: boolean };+ refused: boolean;+}++const engines = new Map<string, Engine>();+let nextOrigin = 0;+let routerInstalled = false;++const toArrayBuffer = (buf: Buffer): ArrayBuffer =>+ buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer;++class FakeWebSocket extends EventTarget {+ static readonly CONNECTING = 0;+ static readonly OPEN = 1;+ static readonly CLOSING = 2;+ static readonly CLOSED = 3;+ readyState = FakeWebSocket.CONNECTING;+ binaryType = "blob";+ private readonly stdin: Buffer[] = [];+ private closed = false;++ private readonly exec: FakeExec;++ constructor(url: string, exec: FakeExec) {+ super();+ void url;+ this.exec = exec;+ setImmediate(() => {+ if (this.closed) return;+ if (exec.refused) {+ this.fail("Received network error or non-101 status code.");+ return;+ }+ this.readyState = FakeWebSocket.OPEN;+ this.dispatchEvent(new Event("open"));+ });+ }++ send(data: Buffer | string): void {+ if (this.readyState !== FakeWebSocket.OPEN) throw new Error("fake websocket is not open");+ if (typeof data === "string") return;+ const frame = Buffer.from(data);+ if (frame[0] === 0) this.stdin.push(frame.subarray(1));+ if (frame[0] !== 4) return;+ const { frames, dropAfterRun } = this.exec.run(Buffer.concat(this.stdin));+ setImmediate(() => {+ if (this.closed) return;+ if (dropAfterRun) {+ this.fail("connection reset by peer");+ return;+ }+ for (const f of frames) this.dispatchEvent(new MessageEvent("message", { data: toArrayBuffer(f) }));+ this.finish(1000, "");+ });+ }++ close(code = 1000, reason = ""): void {+ if (this.closed) return;+ this.readyState = FakeWebSocket.CLOSING;+ setImmediate(() => this.finish(code, reason));+ }++ private fail(message: string): void {+ this.dispatchEvent(Object.assign(new Event("error"), { message, error: new Error(message) }));+ this.finish(1006, "");+ }++ private finish(code: number, reason: string): void {+ if (this.closed) return;+ this.closed = true;+ this.readyState = FakeWebSocket.CLOSED;+ this.dispatchEvent(Object.assign(new Event("close"), { code, reason }));+ }+}++function installRouter(): void {+ if (routerInstalled) return;+ routerInstalled = true;+ const realFetch = globalThis.fetch;+ const RealWebSocket = globalThis.WebSocket;+ const patchedFetch: typeof fetch = async (input, init) => {+ const url = new URL(input instanceof Request ? input.url : input);+ const engine = engines.get(url.origin);+ return engine ? engine.fetch(url, init) : realFetch(input, init);+ };+ (globalThis as { fetch: typeof fetch }).fetch = patchedFetch;+ const Patched = function (this: unknown, url: string | URL, options?: unknown) {+ const parsed = new URL(String(url));+ const origin = parsed.origin.replace(/^ws/, "http");+ const exec = engines.get(origin)?.exec(parsed);+ if (exec) return new FakeWebSocket(parsed.href, exec);+ return new RealWebSocket(url, options as never);+ } as unknown as typeof WebSocket;+ Object.assign(Patched, { CONNECTING: 0, OPEN: 1, CLOSING: 2, CLOSED: 3 });+ (globalThis as { WebSocket: typeof WebSocket }).WebSocket = Patched;+}++export function installFakeSprites(origin = `https://fake-sprites-${++nextOrigin}.invalid`): FakeSprites {+ installRouter();const root = mkdtempSync(join(tmpdir(), "fake-sprites-"));const sprites = new Map<string, { home: string }>();const policies = new Map<string, NetworkRule[]>();+ const resources = new Map<string, { limitMB: number }>();+ const checkpoints = new Map<string, Array<{ id: string; createTime: string; dir: string }>>();const execScripts: string[] = [];const calls: SpritesCall[] = [];const gateway502 = new Set<string>();const stallAfterRun = new Set<string>();const refusedRestart = new Set<string>();- const refusedForcedRestart = new Set<string>();+ const unhealthy = new Map<string, string>();+ const health = new Map<string, { status: string; reason: string }>();+ const brokenReadback = new Set<string>();const restarts: string[] = [];+ let refuseDeleteStatus: number | undefined;+ let rateLimitRetryAfter: number | undefined;+ let checkpointSeq = 0;const injected: Array<InjectedFailure & { status: number }> = [];const ensureDir = (name: string): string => {@@ -68,6 +192,15 @@ export function installFakeSprites(): FakeSprites {return s.home;};+ const hostPath = (name: string, abs: string): string => {+ const home = ensureDir(name);+ if (abs.startsWith(`${root}/`)) return abs;+ if (abs === GUEST_HOME || abs.startsWith(`${GUEST_HOME}/`)) return home + abs.slice(GUEST_HOME.length);+ if (abs.startsWith("/tmp/")) return join(home, "tmp", abs.slice(5));+ if (abs.startsWith("/usr/local/")) return join(home, ".usr-local", abs.slice("/usr/local/".length));+ return join(home, abs);+ };+const remap = (name: string, script: string): string => {const home = ensureDir(name);const homeRe = home.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");@@ -87,135 +220,234 @@ export function installFakeSprites(): FakeSprites {const toBuf = (body: unknown): Buffer => {if (Buffer.isBuffer(body)) return body;+ if (body instanceof Uint8Array) return Buffer.from(body);if (typeof body === "string") return Buffer.from(body);return Buffer.alloc(0);};- const runExec = (name: string, script: string, stdin?: Buffer, viaBody = false): Buffer => {- execScripts.push(script);+ const runExec = (name: string, argv: string[], stdin: Buffer): Buffer[] => {mkdirSync(join(ensureDir(name), "tmp"), { recursive: true });- const input = viaBody ? Buffer.from(remap(name, script), "utf8") : stdin;- const r = spawnSync("sh", ["-c", viaBody ? SCRIPT_RUNNER : remap(name, script)], {- encoding: "buffer",- maxBuffer: 128 * 1024 * 1024,- env: { ...process.env, COPYFILE_DISABLE: "1" },- ...(input ? { input } : {}),- });+ const viaBody = argv[argv.length - 1] === SCRIPT_RUNNER;+ const script = viaBody ? stdin.toString("utf8") : (argv[argv.length - 1] ?? "");+ execScripts.push(script);+ const r = viaBody+ ? spawnSync("sh", ["-c", SCRIPT_RUNNER], {+ encoding: "buffer",+ maxBuffer: 128 * 1024 * 1024,+ env: { ...process.env, COPYFILE_DISABLE: "1" },+ input: Buffer.from(remap(name, script), "utf8"),+ })+ : spawnSync("sh", ["-c", remap(name, script)], {+ encoding: "buffer",+ maxBuffer: 128 * 1024 * 1024,+ env: { ...process.env, COPYFILE_DISABLE: "1" },+ input: stdin,+ });const code = r.status ?? (r.signal ? 137 : -1);const frame = (id: number, payload: Buffer): Buffer => Buffer.concat([Buffer.from([id]), payload]);- return Buffer.concat([+ return [frame(1, r.stdout ?? Buffer.alloc(0)),frame(2, r.stderr ?? Buffer.alloc(0)),Buffer.from([3, code & 0xff]),- ]);+ ];};const deleteSprite = (name: string): void => {const s = sprites.get(name);if (s) rmSync(s.home, { recursive: true, force: true });+ for (const cp of checkpoints.get(name) ?? []) rmSync(cp.dir, { recursive: true, force: true });sprites.delete(name);policies.delete(name);+ resources.delete(name);+ checkpoints.delete(name);+ };++ const ndjson = (lines: object[]): Response =>+ new Response(lines.map((l) => JSON.stringify(l)).join("\n") + "\n", {+ status: 200,+ headers: { "content-type": "application/x-ndjson" },+ });+ const fsResult = (path: string, op: () => object): Response => {+ try {+ return Response.json(op());+ } catch (e) {+ const err = e as { code?: string; message?: string };+ return Response.json({ error: err.message ?? "filesystem error", code: err.code, path }, { status: 500 });+ }};+ const notFound = (what: string): Response =>+ Response.json({ error: "not_found", message: `${what} not found` }, { status: 404 });- const fetchImpl: typeof fetch = async (input, init) => {- const url = new URL(typeof input === "string" ? input : input.toString());+ const fetchImpl = async (url: URL, init?: RequestInit): Promise<Response> => {const method = init?.method ?? "GET";- calls.push({ method, path: url.pathname });+ calls.push({ method, path: url.pathname + url.search });const at = injected.findIndex((f) => !f.match || f.match({ method, path: url.pathname }));if (at >= 0) {const [next] = injected.splice(at, 1);- return new Response(`injected ${next!.status}`, { status: next!.status, headers: next!.headers ?? {} });+ return Response.json(+ { error: "injected", message: `injected ${next!.status}` },+ { status: next!.status, headers: next!.headers },+ );}- const health = /^\/v1\/sprites\/([^/]+)\/check$/.exec(url.pathname);- if (health) {- const name = decodeURIComponent(health[1]!);- if (!sprites.has(name)) return new Response("sprite not found", { status: 404 });- return Response.json({ sprite_name: name, status: "healthy" });+ const one = /^\/v1\/sprites\/([^/]+)(?:\/(.*))?$/.exec(url.pathname);+ if (url.pathname === "/v1/sprites" && method === "POST") {+ if (rateLimitRetryAfter !== undefined) {+ const retry = rateLimitRetryAfter;+ rateLimitRetryAfter = undefined;+ return Response.json(+ {+ error: "sprite_creation_rate_limited",+ message: "Sprite creation rate limit exceeded",+ limit: 10,+ window_seconds: 60,+ retry_after_seconds: retry,+ },+ { status: 429, headers: { "retry-after": String(retry) } },+ );+ }+ const body = JSON.parse(toBuf(init?.body).toString() || "{}") as { name?: string };+ const name = body.name ?? "unnamed";+ ensureDir(name);+ return Response.json({ name, status: "running" });}- const boot = /^\/v1\/sprites\/([^/]+)\/restart$/.exec(url.pathname);- if (boot && method === "POST") {- const name = decodeURIComponent(boot[1]!);- const forced = url.searchParams.get("force") === "true";- if (!sprites.has(name)) return new Response("sprite not found", { status: 404 });- if (forced ? refusedForcedRestart.has(name) : refusedRestart.has(name)) {- return new Response('{"error":"upstream restart failed"}', { status: 502 });+ if (!one) return notFound("route");+ const name = decodeURIComponent(one[1]!);+ const sub = one[2] ?? "";+ if (!sub) {+ if (method === "GET") return sprites.has(name) ? Response.json({ name, status: "warm" }) : notFound("sprite");+ if (method === "DELETE") {+ if (refuseDeleteStatus !== undefined)+ return Response.json({ error: "upstream" }, { status: refuseDeleteStatus });+ if (!sprites.has(name)) return notFound("sprite");+ deleteSprite(name);+ return new Response(null, { status: 204 });}- restarts.push(forced ? `${name}?force=true` : name);+ return notFound("route");+ }+ if (!sprites.has(name)) return notFound("sprite");+ if (sub === "check") {+ const reason = unhealthy.get(name);+ return Response.json({+ sprite_name: name,+ sprite_id: `id-${name}`,+ status: health.get(name)?.status ?? (reason ? "unhealthy" : "healthy"),+ reason: health.get(name)?.reason ?? reason ?? null,+ checked_at: new Date().toISOString(),+ elapsed: 0.01,+ });+ }+ if (sub === "restart" && method === "POST") {+ if (refusedRestart.has(name)) return Response.json({ error: "upstream restart failed" }, { status: 502 });+ restarts.push(name);gateway502.delete(name);- return Response.json({ sprite_name: name });+ return Response.json({ sprite_name: name, machine_id: "m1", message: "queued" }, { status: 202 });}- const sub = /\/v1\/sprites\/([^/]+)\/(exec|policy\/network)$/.exec(url.pathname);- if (sub) {- const name = decodeURIComponent(sub[1]!);- if (sub[2] === "exec") {- const argv = url.searchParams.getAll("cmd");- const body = url.searchParams.get("stdin") === "true" ? toBuf(init?.body) : undefined;- const viaBody = body !== undefined && argv[argv.length - 1] === SCRIPT_RUNNER;- const script = viaBody ? body.toString("utf8") : (argv[argv.length - 1] ?? "");- const stdin = viaBody ? undefined : body;- calls[calls.length - 1]!.script = script;- const stall = (): never => {- throw Object.assign(new Error("The operation was aborted due to timeout"), { name: "TimeoutError" });- };- if (stallAfterRun.has(name)) {- stallAfterRun.delete(name);- runExec(name, script, stdin, viaBody);- stall();- }- if (gateway502.has(name)) {- return new Response('{"error":"bad gateway"}', { status: 502 });- }- return new Response(runExec(name, script, stdin, viaBody), { status: 200 });- }- if (method === "GET") return Response.json({ rules: policies.get(name) ?? [] });+ if (sub === "policy/network") {+ if (method === "GET") return Response.json({ rules: brokenReadback.has(name) ? [] : (policies.get(name) ?? []) });const parsed = JSON.parse(toBuf(init?.body).toString() || "{}") as { rules?: NetworkRule[] };policies.set(name, parsed.rules ?? []);return new Response(null, { status: 204 });}- // The control plane the real SpritesClient dials: sprite CRUD.- if (url.pathname === "/v1/sprites" && method === "POST") {- const body = JSON.parse(toBuf(init?.body).toString() || "{}") as { name?: string };- const name = body.name ?? "unnamed";- ensureDir(name);- return Response.json({ name });- }- const one = /^\/v1\/sprites\/([^/]+)$/.exec(url.pathname);- if (one) {- const name = decodeURIComponent(one[1]!);+ if (sub === "policy/resources") {if (method === "GET") {- return sprites.has(name)- ? Response.json({ name, status: "warm" })- : new Response("sprite not found", { status: 404 });+ const r = resources.get(name);+ return Response.json({ memory: r ? { limit_mb: r.limitMB } : undefined });}if (method === "DELETE") {- deleteSprite(name);+ resources.delete(name);return new Response(null, { status: 204 });}+ const parsed = JSON.parse(toBuf(init?.body).toString() || "{}") as { memory?: { limit_mb?: number } };+ if (parsed.memory?.limit_mb !== undefined) resources.set(name, { limitMB: parsed.memory.limit_mb });+ return new Response(null, { status: 204 });+ }+ if (sub === "checkpoint" && method === "POST") {+ const body = JSON.parse(toBuf(init?.body).toString() || "{}") as { comment?: string };+ const id = `v${++checkpointSeq}`;+ const dir = join(root, ".checkpoints", name, id);+ cpSync(ensureDir(name), dir, { recursive: true });+ const list = checkpoints.get(name) ?? [];+ list.push({ id, createTime: new Date(Date.now() + list.length).toISOString(), dir });+ checkpoints.set(name, list);+ void body;+ return ndjson([+ { type: "info", data: `checkpoint ${id}` },+ { type: "complete", data: id },+ ]);}- return new Response("not found", { status: 404 });+ if (sub === "checkpoints" && method === "GET") {+ return Response.json([+ { id: "Current", create_time: new Date().toISOString() },+ ...(checkpoints.get(name) ?? []).map((c) => ({ id: c.id, create_time: c.createTime, comment: "qm turn end" })),+ ]);+ }+ const restore = /^checkpoints\/([^/]+)\/restore$/.exec(sub);+ if (restore && method === "POST") {+ const cp = (checkpoints.get(name) ?? []).find((c) => c.id === decodeURIComponent(restore[1]!));+ if (!cp) return notFound("checkpoint");+ const home = ensureDir(name);+ rmSync(home, { recursive: true, force: true });+ cpSync(cp.dir, home, { recursive: true });+ gateway502.delete(name);+ return ndjson([+ { type: "info", data: `restoring ${cp.id}` },+ { type: "complete", data: cp.id },+ ]);+ }+ if (sub === "fs/read" && method === "GET") {+ const target = hostPath(name, url.searchParams.get("path") ?? "");+ if (!existsSync(target)) return Response.json({ error: "no such file", path: target }, { status: 404 });+ return new Response(readFileSync(target), { status: 200 });+ }+ if (sub === "fs/write" && method === "PUT") {+ const target = hostPath(name, url.searchParams.get("path") ?? "");+ const data = toBuf(init?.body);+ const mode = url.searchParams.get("mode");+ return fsResult(target, () => {+ if (url.searchParams.get("mkdirParents") === "true") mkdirSync(dirname(target), { recursive: true });+ writeFileSync(target, data);+ if (mode) chmodSync(target, Number.parseInt(mode, 8));+ return { path: target, size: data.length, mode: mode ?? "0644" };+ });+ }+ if (sub === "fs/rename" && method === "POST") {+ const body = JSON.parse(toBuf(init?.body).toString() || "{}") as { source?: string; dest?: string };+ return fsResult(body.source ?? "", () => {+ renameSync(hostPath(name, body.source ?? ""), hostPath(name, body.dest ?? ""));+ return { source: body.source, dest: body.dest };+ });+ }+ return notFound("route");};- const client: SpritesClientLike = {- async getSprite(name: string) {- if (!sprites.has(name)) throw new Error(`sprite ${name} not found (404)`);- return { name };- },- async createSprite(name: string) {- ensureDir(name);- return { name };- },- async deleteSprite(name: string) {- deleteSprite(name);- },+ const exec = (url: URL): FakeExec | null => {+ const m = /^\/v1\/sprites\/([^/]+)\/exec$/.exec(url.pathname);+ if (!m) return null;+ const name = decodeURIComponent(m[1]!);+ const argv = url.searchParams.getAll("cmd");+ calls.push({ method: "WS", path: url.pathname });+ const call = calls[calls.length - 1]!;+ return {+ refused: !sprites.has(name) || gateway502.has(name),+ run: (stdin) => {+ const frames = runExec(name, argv, stdin);+ call.script = execScripts[execScripts.length - 1];+ const dropAfterRun = stallAfterRun.delete(name);+ return { frames, dropAfterRun };+ },+ };};+ engines.set(origin, { fetch: fetchImpl, exec });+return {- client,- fetchImpl,+ baseUrl: origin,calls,homeDir: (name) => ensureDir(name),names: () => [...sprites.keys()],policy: (name) => policies.get(name) ?? null,+ resources: (name) => resources.get(name) ?? null,+ checkpoints: (name) => (checkpoints.get(name) ?? []).map((c) => c.id),execScripts: () => [...execScripts],stallAfterRun: (name) => {stallAfterRun.add(name);@@ -229,8 +461,20 @@ export function installFakeSprites(): FakeSprites {refuseRestart: (name) => {refusedRestart.add(name);},- refuseForcedRestart: (name) => {- refusedForcedRestart.add(name);+ health: (name, status, reason) => {+ health.set(name, { status, reason });+ },+ unhealthy: (name, reason) => {+ unhealthy.set(name, reason);+ },+ refuseDelete: (status) => {+ refuseDeleteStatus = status;+ },+ rateLimitCreate: (retryAfterSeconds) => {+ rateLimitRetryAfter = retryAfterSeconds;+ },+ breakPolicyReadback: (name) => {+ brokenReadback.add(name);},setPressure: (name, p) => {const home = ensureDir(name);@@ -248,34 +492,27 @@ export function installFakeSprites(): FakeSprites {stallAfterRun.clear();gateway502.clear();refusedRestart.clear();- refusedForcedRestart.clear();+ unhealthy.clear();+ health.clear();+ brokenReadback.clear();restarts.length = 0;injected.length = 0;+ refuseDeleteStatus = undefined;+ rateLimitRetryAfter = undefined;+ },+ cleanup: () => {+ engines.delete(origin);+ rmSync(root, { recursive: true, force: true });},- cleanup: () => rmSync(root, { recursive: true, force: true }),};}let globalFake: FakeSprites | null = null;-/** Patch globalThis.fetch so the REAL SpritesClient (and the backend's default fetch) lands on an- * in-memory fake instead of api.sprites.dev — the app-level test substrate (see test-config.ts). */export function installGlobalFakeSprites(): FakeSprites {if (globalFake) return globalFake;- const fake = installFakeSprites();- const realFetch = globalThis.fetch;- const patched: typeof fetch = async (input, init) => {- let url: string;- if (typeof input === "string") url = input;- else if (input instanceof URL) url = input.href;- else url = input.url;- if (url.startsWith(`${API_ORIGIN}/`)) return fake.fetchImpl(input, init);- return realFetch(input, init);- };- (patched as typeof fetch & { [FETCH_SUBSTRATE]?: boolean })[FETCH_SUBSTRATE] = true;- (globalThis as { fetch: typeof fetch }).fetch = patched;- globalFake = fake;- return fake;+ globalFake = installFakeSprites(API_ORIGIN);+ return globalFake;}export const FAKE_SPRITES_TOKEN = TOKEN;