Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

37.7 KB
Raw Blame
import { SandboxProvisionCleanupError } from "../src/sandbox/sandbox.ts";
import { createMemoryMap } from "../src/persistence/durable-map.ts";
import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";
import { pollProcess } from "../src/sandbox/process-poll.ts";
import { test, after, beforeEach } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { inspect } from "node:util";
import {
  createSpritesSandbox,
  processKeepaliveScript,
  retrySpritesControl,
  spritesErrorDetail,
} from "../src/sandbox/sprites-sandbox.ts";
import { sandboxScopeName } from "../src/sandbox/exec-sandbox-base.ts";
import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";
import { execFailureDetail, supportsProcessSessions, supportsBlobStaging } from "../src/sandbox/sandbox.ts";
import { createMemorySnapshotStore } from "../src/sandbox/home-snapshot.ts";
import { sleep } from "../src/util/async.ts";
import { createMemoryBlobTransferStore } from "../src/persistence/blob-transfer.ts";
import { scopeId } from "../src/types.ts";
import { mintCapabilityToken, EGRESS_PROXY_AUD } from "../src/auth/capability-token.ts";
import { installFakeSprites, FAKE_SPRITES_TOKEN, type FakeSprites } from "./support/fake-sprites.ts";
import type { Sandbox } from "../src/sandbox/sandbox.ts";
import { APIError, SpriteCommand } from "@fly/sprites";

let fake: FakeSprites;
let sandbox: Sandbox;
const scope = scopeId("personal", "tester");
const layers = [{ scopeId: scope, mountPath: "/", mode: "rw" as const }];

function make(extra: Record<string, unknown> = {}): Sandbox {
  return createSpritesSandbox(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "sprites-ws-"))), {
    initializationStore: createMemoryMap(),
    advisoryLock: createMemoryAdvisoryLock(),
    token: FAKE_SPRITES_TOKEN,
    namePrefix: "qmt",
    baseUrl: fake.baseUrl,
    ...extra,
  });
}

const proxyToken = () =>
  mintCapabilityToken(
    { actorId: "tester", scopeId: scope, aud: EGRESS_PROXY_AUD, exp: Date.now() + 600_000 },
    "secret",
  );

beforeEach(() => {
  fake?.cleanup();
  fake = installFakeSprites();
  sandbox = make();
});
after(() => fake?.cleanup());

test("provision runs commands with env and cwd", async () => {
  const h = await sandbox.provision(layers, { env: { MY_VAR: "v1" } });
  assert.equal(h.coldStart, true);
  const r = await sandbox.run(h, "pwd; echo VAR=$MY_VAR");
  assert.equal(r.code, 0);
  assert.match(r.stdout, /\/home\/sprite\/workspace|workspace/);
  assert.match(r.stdout, /VAR=v1/);
});

test("streams and exit codes are exact", async () => {
  const h = await sandbox.provision(layers);
  const r = await sandbox.run(h, "echo out; echo err >&2; exit 3");
  assert.equal(r.code, 3);
  assert.equal(r.stdout.trim(), "out");
  assert.equal(r.stderr.trim(), "err");
});

test("commands run over the WebSocket exec endpoint with the script in the stream, never the URL", async () => {
  const h = await sandbox.provision(layers);
  const huge = `echo start; : ${"x".repeat(1024 * 1024)}; echo end`;
  const r = await sandbox.run(h, huge);
  assert.equal(r.code, 0);
  assert.match(r.stdout, /start\s+end/);
  const execs = fake.calls.filter((c) => c.method === "WS");
  assert.ok(execs.length > 0, "commands ride the WebSocket exec channel");
  assert.ok(
    !fake.calls.some((c) => c.method === "POST" && c.path.endsWith("/exec")),
    "the frame-ambiguous HTTP exec fallback is never used",
  );
  assert.ok(Math.max(...execs.map((c) => c.path.length)) < 2048, "exec URLs stay small");
  assert.ok(
    execs.some((c) => (c.script?.length ?? 0) > 1024 * 1024),
    "the megabyte script travels as stdin frames",
  );
});

test("file roundtrip incl. large binary and missing file goes through the filesystem API", async () => {
  const h = await sandbox.provision(layers);
  await sandbox.writeFile(h, "a/b.txt", "hello\n");
  assert.equal(await sandbox.readFile(h, "a/b.txt"), "hello\n");
  assert.equal(await sandbox.readFile(h, "nope.txt"), null);
  const huge = Buffer.alloc(1300 * 1024);
  for (let i = 0; i < huge.length; i++) huge[i] = (i * 13) % 256;
  await sandbox.writeFileBytes(h, "huge.bin", huge);
  const hugeBack = await sandbox.readFileBytes(h, "huge.bin");
  assert.ok(hugeBack && Buffer.from(hugeBack).equals(huge));
  assert.ok(fake.calls.some((c) => c.method === "PUT" && c.path.includes("/fs/write?")));
  assert.ok(fake.calls.some((c) => c.method === "GET" && c.path.includes("/fs/read?")));
  assert.ok(!fake.execScripts().some((s) => /base64|dd if=/.test(s)), "bytes never round-trip through shell encoding");
});

test("a file write lands on a temp path and is renamed into place, never streamed into the target", async () => {
  const h = await sandbox.provision(layers);
  await sandbox.writeFile(h, "cfg.txt", "value\n");
  assert.equal(await sandbox.readFile(h, "cfg.txt"), "value\n");
  const write = fake.calls.find((c) => c.method === "PUT" && c.path.includes("cfg.txt"));
  assert.ok(write, "expected a filesystem write");
  assert.match(write!.path, /cfg\.txt\.part\./, "the payload lands on a temp path");
  const writeIdx = fake.calls.indexOf(write!);
  assert.ok(
    fake.calls.slice(writeIdx).some((c) => c.method === "POST" && c.path.endsWith("/fs/rename")),
    "and is renamed over the target afterwards",
  );
});

test("process sessions capability works end to end and holds the sprite awake with a Tasks heartbeat", async () => {
  assert.ok(supportsProcessSessions(sandbox));
  if (!supportsProcessSessions(sandbox)) return;
  const h = await sandbox.provision(layers);
  const { processId } = await sandbox.startProcess(h, "echo one; echo two");
  const { output, status } = await pollProcess(sandbox, h, processId, { deadlineMs: 5_000, waitMs: 100 });
  assert.equal(status.state, "exited");
  assert.match(output, /one/);
  assert.match(output, /two/);
  const keepalive = fake.execScripts().find((s) => s.includes("/.sprite/api.sock"));
  assert.ok(keepalive, "a keepalive sidecar was launched next to the process");
  assert.match(keepalive!, new RegExp(`/v1/tasks/qm-proc-${processId}`));
  assert.match(keepalive!, /-X PUT/, "renews by upsert so the first call creates the task");
  assert.match(keepalive!, /-X DELETE/, "and releases the hold when the process is gone");
  const listed = await sandbox.listProcesses(h);
  assert.deepEqual(
    listed.map((p) => p.command),
    ["echo one; echo two"],
    "the sidecar is not a visible process session and the command stays readable",
  );
});

test("the keepalive script stops renewing once the process has recorded an exit code", () => {
  const script = processKeepaliveScript("00000000-0000-0000-0000-000000000000");
  assert.match(script, /while \[ ! -f "\$P\/code" \]/);
  assert.match(script, /"expire":"5m"/);
  assert.match(script, /setsid/);
});

test("background processes inherit the force-through proxy env", async () => {
  const s = make({ egressProxyUrl: "https://proxy.example.com" });
  const h = await s.provision(layers, { egressToken: await proxyToken() });
  assert.ok(supportsProcessSessions(s));
  if (!supportsProcessSessions(s)) return;
  const { processId } = await s.startProcess(h, "echo PROXY=$HTTPS_PROXY");
  let cursor = 0,
    chunks = "",
    state = "running";
  for (let i = 0; i < 10 && state === "running"; i++) {
    const r = await s.readProcess(h, processId, { sinceCursor: cursor });
    chunks += r.chunks;
    cursor = r.cursor;
    state = r.status.state;
  }
  assert.match(chunks, /PROXY=https?:\/\/[^ ]*proxy\.example\.com/);
});

test("scope name is stable and slugged", () => {
  const a = sandboxScopeName("qmt", "person:tester");
  assert.equal(a, sandboxScopeName("qmt", "person:tester"));
  assert.match(a, /^qmt-person-tester-[0-9a-f]{6}$/);
});

test("scope names fit the 63-char provider limit for long prefixes, scratch included", () => {
  const scope = "person:[email protected]";
  const ids = [scope, "conversation:9f8e7d6c-5b4a-3210-fedc-ba9876543210", "owner:" + scope, ""];
  for (const prefix of ["qm", "qm-long-generic-fleet-tenant-prefix", "x".repeat(70) + "-"]) {
    for (const p of [prefix, `${prefix}-scratch`]) {
      const names = ids.map((id) => sandboxScopeName(p, id));
      for (const n of names) {
        assert.ok(n.length <= 63, `${n} is ${n.length} chars`);
        assert.match(n, /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/);
      }
      assert.equal(new Set(names).size, names.length);
    }
    assert.notEqual(sandboxScopeName(prefix, scope), sandboxScopeName(`${prefix}-scratch`, scope));
  }
  // Names that already fit are unchanged, so existing resident computers keep their identity.
  assert.equal(sandboxScopeName("qmt", "person:tester"), "qmt-person-tester-7b537e");
  assert.match(
    sandboxScopeName("qm-long-generic-fleet-tenant-prefix", "person:tester"),
    /^qm-long-generic-fleet-tenant-prefix-person-tester-[0-9a-f]{6}$/,
  );
});

test("no egress force-through without a proxy url: no policy, no proxy env", async () => {
  assert.equal(sandbox.profile.egressEnforcement, "none");
  const h = await sandbox.provision(layers, { egressToken: "ignored" });
  assert.equal(fake.policy(h.id), null);
  assert.equal(h.env?.HTTPS_PROXY, undefined);
});

test("force-through pins the platform policy and injects proxy env", async () => {
  const s = make({ egressProxyUrl: "https://proxy.example.com" });
  assert.equal(s.profile.egressEnforcement, "domain");
  const token = await mintCapabilityToken(
    {
      actorId: "tester",
      scopeId: scope,
      aud: EGRESS_PROXY_AUD,
      egress: { allowedHosts: ["api.anthropic.com"], deniedHosts: [] },
      exp: Date.now() + 600_000,
    },
    "secret",
  );
  const h = await s.provision(layers, { egressToken: token });
  const pol = fake.policy(h.id);
  assert.deepEqual(pol, [{ domain: "proxy.example.com", action: "allow" }]);
  assert.equal(new URL(h.env!.HTTPS_PROXY!).hostname, "proxy.example.com");
  assert.ok(h.env?.HTTPS_PROXY?.includes(token));
  assert.equal(h.env?.NO_PROXY, "localhost,127.0.0.1,::1");
});

test("proxy migration keeps both hosts reachable across cutover and rollback", async () => {
  const token = await proxyToken();
  const oldUrl = "https://old.example.com";
  const newUrl = "https://new.example.com";
  for (const [primary, additional] of [
    [oldUrl, newUrl],
    [newUrl, oldUrl],
    [oldUrl, newUrl],
  ]) {
    const s = make({ egressProxyUrl: primary, egressProxyAdditionalUrls: [additional, primary] });
    const h = await s.provision(layers, { egressToken: token });
    assert.deepEqual(fake.policy(h.id), [
      { domain: "new.example.com", action: "allow" },
      { domain: "old.example.com", action: "allow" },
    ]);
    assert.equal(new URL(h.env!.HTTPS_PROXY!).hostname, new URL(primary!).hostname);
  }
});

test("proxy migration rejects mismatched provider policy and invalid configuration", async () => {
  const s = make({
    egressProxyUrl: "https://new.example.com",
    egressProxyAdditionalUrls: ["https://old.example.com"],
  });
  fake.breakPolicyReadback(sandboxScopeName("qmt", scope));
  await assert.rejects(s.provision(layers, { egressToken: await proxyToken() }), /readback mismatch/);
  assert.throws(() => make({ egressProxyAdditionalUrls: ["https://old.example.com"] }), /require a primary/);
  assert.throws(
    () => make({ egressProxyUrl: "https://new.example.com", egressProxyAdditionalUrls: ["file:///tmp/x"] }),
    /HTTP\(S\)/,
  );
});

test("force-through strips agent-supplied proxy vars", async () => {
  const s = make({ egressProxyUrl: "https://proxy.example.com" });
  const h = await s.provision(layers, {
    egressToken: await proxyToken(),
    env: { HTTPS_PROXY: "http://evil:1", FOO: "keep" },
  });
  assert.ok(!h.env?.HTTPS_PROXY?.includes("evil"));
  assert.equal(h.env?.FOO, "keep");
});

test("force-through makes git send proxy credentials up front, keeping turn git config", async () => {
  const s = make({ egressProxyUrl: "https://proxy.example.com" });
  const h = await s.provision(layers, {
    egressToken: await proxyToken(),
    env: { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "user.name", GIT_CONFIG_VALUE_0: "Turn User" },
  });
  const r = await s.run(h, "echo $GIT_HTTP_PROXY_AUTHMETHOD; git config --get user.name");
  assert.equal(r.code, 0, r.stderr);
  assert.deepEqual(r.stdout.trim().split("\n"), ["basic", "Turn User"]);
  assert.deepEqual(
    [h.env?.GIT_CONFIG_COUNT, h.env?.GIT_CONFIG_KEY_0, h.env?.GIT_CONFIG_VALUE_0],
    ["1", "user.name", "Turn User"],
  );
});

test("git proxy auth from the proxy env stays off when egress is not forced", async () => {
  const h = await sandbox.provision(layers, { egressToken: "ignored" });
  const r = await sandbox.run(h, 'echo "${GIT_HTTP_PROXY_AUTHMETHOD:-unset}"');
  assert.equal(r.stdout.trim(), "unset");
});

test("force-through fails closed if the policy readback doesn't bind", async () => {
  const s = make({ egressProxyUrl: "https://proxy.example.com" });
  fake.breakPolicyReadback(sandboxScopeName("qmt", scope));
  await assert.rejects(s.provision(layers, { egressToken: await proxyToken() }), /readback mismatch/);
});

test("a recreated sprite gets the egress policy re-pinned, never served from a stale cache", async () => {
  const s = make({ egressProxyUrl: "https://proxy.example.com" });
  const token = await proxyToken();
  const a = await s.provision(layers, { scratch: { key: "job-pol" }, egressToken: token });
  assert.deepEqual(fake.policy(a.id), [{ domain: "proxy.example.com", action: "allow" }]);
  await s.teardown(a);
  assert.equal(fake.policy(a.id), null, "the sprite and its platform policy are gone after release");
  const b = await s.provision(layers, { scratch: { key: "job-pol" }, egressToken: token });
  assert.equal(b.id, a.id);
  assert.deepEqual(fake.policy(b.id), [{ domain: "proxy.example.com", action: "allow" }]);
});

test("a failed egress setup releases the scratch lease instead of wedging the key", async () => {
  const s = make({ egressProxyUrl: "https://proxy.example.com" });
  fake.breakPolicyReadback(sandboxScopeName("qmt-scratch", "job-egress"));
  await assert.rejects(
    s.provision(layers, { scratch: { key: "job-egress" }, egressToken: await proxyToken() }),
    /readback mismatch/,
  );
  const retry = await s.provision(layers, { scratch: { key: "job-egress" } });
  assert.equal(retry.scratch, true);
  assert.equal(retry.coldStart, true, "the failed provision released its lease and the box was recreated");
});

test("scratch sprites are ephemeral and shared leases survive until the last release", async () => {
  const a = await sandbox.provision(layers, { scratch: { key: "job-1" } });
  const b = await sandbox.provision(layers, { scratch: { key: "job-1" } });
  assert.equal(a.scratch, true);
  assert.equal(a.id, b.id);
  assert.equal(a.coldStart, true);
  assert.equal(b.coldStart, false);
  await sandbox.teardown(a);
  assert.ok(fake.names().includes(a.id));
  await sandbox.teardown(b);
  assert.ok(!fake.names().includes(b.id));
  assert.equal(fake.checkpoints(a.id).length, 0, "scratch boxes are never checkpointed");
});

test("teardown checkpoints a changed home, throttles repeats, and destroy deletes the sprite", async () => {
  const h = await sandbox.provision(layers);
  await sandbox.teardown(h);
  assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a turn that may have changed the home ends in a checkpoint");
  await sandbox.teardown(h);
  assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a second teardown inside the interval is throttled");
  assert.ok(fake.names().includes(h.id), "parking keeps the sprite");
  await sandbox.teardown(h, { destroy: true });
  assert.ok(!fake.names().includes(h.id));
});

test("an unchanged home is not checkpointed once its state is known", async () => {
  const s = make({ checkpointIntervalMs: 1 });
  const h = await s.provision(layers);
  await s.teardown(h, { homeUnchanged: true });
  assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a sprite this core has not checkpointed yet is saved once");
  await sleep(5);
  await s.teardown(h, { homeUnchanged: true });
  assert.deepEqual(fake.checkpoints(h.id), ["v1"], "a quiet turn after that adds nothing");
  await s.teardown(h);
  assert.deepEqual(fake.checkpoints(h.id), ["v1", "v2"], "a turn that may have changed the home is saved");
  await sleep(5);
  await s.teardown(h, { homeUnchanged: true });
  assert.deepEqual(fake.checkpoints(h.id), ["v1", "v2"]);
});

test("a checkpoint failure is reported and never fails the teardown", async () => {
  const events: Array<{ code: string }> = [];
  const s = make({ onError: (e: { code: string }) => events.push(e) });
  const h = await s.provision(layers);
  fake.fail502(h.id);
  fake.unhealthy(h.id, "checkpoint store offline");
  await s.teardown(h);
  assert.ok(fake.checkpoints(h.id).length <= 1);
});

test("computerStatus reports checkpoint recovery and a healthy machine whose shell has stopped answering", async () => {
  const h = await sandbox.provision(layers);
  const fresh = await sandbox.computerStatus!(scope);
  assert.equal(fresh.machine, "healthy");
  assert.equal(fresh.provisioned, true);
  assert.equal(fresh.guestResponsive, true);
  assert.equal(fresh.listed, undefined, "the list-view status is stale by design and is not surfaced");
  assert.deepEqual(fresh.recovery, { strategy: "provider_snapshot", checkpointExpiresAtMs: null });

  await sandbox.teardown(h);
  fake.fail502(h.id);
  const wedged = await sandbox.computerStatus!(scope);
  assert.equal(wedged.machine, "healthy");
  assert.equal(wedged.guestResponsive, false);
  assert.equal(wedged.recovery?.checkpointId, "v1");
  assert.equal(typeof wedged.recovery?.checkpointAtMs, "number");
});

test("computerStatus names a faulted check and an unprovisioned scope", async () => {
  const none = await sandbox.computerStatus!(scope);
  assert.deepEqual(none, { machine: "no sprite provisioned yet", provisioned: false, guestResponsive: false });
  const h = await sandbox.provision(layers);
  fake.unhealthy(h.id, "disk unreachable");
  const s = await sandbox.computerStatus!(scope);
  assert.equal(s.machine, "unhealthy (disk unreachable)");
  assert.equal(s.provisioned, true);
});

test("restartComputer reboots the scope's sprite and heals a wedged exec channel", async () => {
  const h = await sandbox.provision(layers);
  fake.fail502(h.id);
  await assert.rejects(sandbox.run(h, "echo back"), /WebSocket error/);

  await sandbox.restartComputer!(scope);
  assert.deepEqual(fake.restarts(), [h.id]);

  const after = await sandbox.run(h, "echo back");
  assert.equal(after.code, 0);
  assert.equal(after.stdout.trim(), "back");
});

test("a refused restart on a faulted sprite restores the latest checkpoint", async () => {
  const events: Array<{ code: string; message: string }> = [];
  const s = make({ onError: (e: { code: string; message: string }) => events.push(e) });
  const h = await s.provision(layers);
  await s.writeFile(h, "keep.txt", "before\n");
  await s.teardown(h);
  await s.writeFile(h, "keep.txt", "after\n");
  fake.refuseRestart(h.id);
  fake.unhealthy(h.id, "machine unreachable");
  fake.fail502(h.id);

  await s.restartComputer!(scope);
  assert.deepEqual(fake.restarts(), []);
  assert.equal(await s.readFile(h, "keep.txt"), "before\n", "the home is back at the checkpoint");
  const restored = events.find((e) => e.code === "checkpoint_restored");
  assert.ok(restored, "the destructive recovery is reported");
  assert.match(restored!.message, /http 502/);
  assert.match(restored!.message, /machine unreachable/);
  assert.match(restored!.message, /restored checkpoint v1/);
});

test("a refused restart with a healthy check restores nothing and surfaces the refusal", async () => {
  const h = await sandbox.provision(layers);
  await sandbox.teardown(h);
  fake.refuseRestart(h.id);
  await assert.rejects(sandbox.restartComputer!(scope), /http 502.*reports no fault/s);
  assert.equal(await sandbox.readFile(h, ".ro-layers.manifest"), null);
});

test("a refused restart with no checkpoint to fall back on names all three failures", async () => {
  const h = await sandbox.provision(layers);
  fake.refuseRestart(h.id);
  fake.unhealthy(h.id, "boot loop");
  await assert.rejects(sandbox.restartComputer!(scope), /http 502.*boot loop.*no checkpoint to restore/s);
});

for (const submitted of [false, true]) {
  for (const closeOnly of [false, true]) {
    test(`exec failures report submission state without replay or private details (submitted=${submitted}, closeOnly=${closeOnly})`, async (t) => {
      const h = await sandbox.provision(layers);
      if (closeOnly) {
        const dispatch = EventTarget.prototype.dispatchEvent;
        t.mock.method(EventTarget.prototype, "dispatchEvent", function (this: EventTarget, event: Event) {
          return event.type === "error" || dispatch.call(this, event);
        });
      }
      const before = fake.calls.filter((call) => call.method === "WS").length;
      if (submitted) fake.stallAfterRun(h.id);
      else fake.fail502(h.id);

      await assert.rejects(sandbox.run(h, "echo entry >> /home/sprite/workspace/ledger"), (error: Error) => {
        assert.match(
          error.message,
          submitted ? /script submission started; execution unknown/ : /script not submitted/,
        );
        for (const value of [h.id, fake.baseUrl.replace("https:", "wss:"), "cmd=", "echo entry"])
          assert.ok(!inspect(error).includes(value));
        return true;
      });

      assert.equal(fake.calls.filter((call) => call.method === "WS").length, before + 1);
      assert.equal(await sandbox.readFile(h, "ledger"), submitted ? "entry\n" : null);
    });
  }
}

test("exec diagnostics keep the provider's transport reason but not its URL or raw cause", async (t) => {
  const h = await sandbox.provision(layers);
  let reason = "";
  t.mock.method(SpriteCommand.prototype, "start", async () => {
    throw new Error(`WebSocket error: ${reason} (url: wss://example.invalid/TypeError?token=example-secret)`, {
      cause: new Error("example-secret"),
    });
  });
  for (reason of ["TypeError", "unclassified provider failure"]) {
    await assert.rejects(sandbox.run(h, "echo ignored"), (error: Error) => {
      assert.match(error.message, /script not submitted/);
      assert.ok(error.message.endsWith(`WebSocket error: ${reason}`));
      assert.doesNotMatch(inspect(error), /example-secret|example\.invalid|token=/);
      return true;
    });
  }
});

test("exec results carry io pressure when the guest exposes it, and omit it when it can't be read", async () => {
  const h = await sandbox.provision(layers);
  const bare = await sandbox.run(h, "echo ok");
  assert.equal(bare.pressure, undefined);

  fake.setPressure(h.id, { full10: 85.17, full60: 86.14, load1: 30.78 });
  const r = await sandbox.run(h, "echo ok");
  assert.equal(r.code, 0);
  assert.deepEqual(r.pressure, { ioFull10: 85.17, ioFull60: 86.14, load1: 30.78 });
});

test("sustained io pressure is reported once per episode, then re-arms after it clears", async () => {
  const events: Array<{ code: string }> = [];
  const s = make({ onError: (e: { code: string }) => events.push(e) });
  const h = await s.provision(layers);

  fake.setPressure(h.id, { full10: 90, full60: 88, load1: 25 });
  await s.run(h, "echo a");
  await s.run(h, "echo b");
  assert.deepEqual(
    events.filter((e) => e.code === "io_pressure_high").length,
    1,
    "a continuing episode records exactly one event",
  );

  fake.setPressure(h.id, { full10: 5, full60: 5, load1: 1 });
  await s.run(h, "echo c");
  fake.setPressure(h.id, { full10: 90, full60: 88, load1: 25 });
  await s.run(h, "echo d");
  assert.equal(events.filter((e) => e.code === "io_pressure_high").length, 2, "a new episode records again");
});

test("computerStatus carries guest pressure alongside the health check", async () => {
  const h = await sandbox.provision(layers);
  fake.setPressure(h.id, { full10: 60, full60: 55, load1: 8 });
  const s = await sandbox.computerStatus!(scope);
  assert.equal(s.machine, "healthy");
  assert.equal(s.guestResponsive, true);
  assert.deepEqual(s.pressure, { ioFull10: 60, ioFull60: 55, load1: 8 });
});

test("a garbled pressure read never costs the caller a completed command's result", async () => {
  const h = await sandbox.provision(layers);
  fake.setPressure(h.id, { full10: 60, full60: 55, load1: 8 });
  writeFileSync(join(fake.homeDir(h.id), ".proc-loadavg"), "");
  const r = await sandbox.run(h, "echo survived");
  assert.equal(r.code, 0);
  assert.equal(r.stdout.trim(), "survived");
  assert.equal(r.pressure, undefined, "partial telemetry is dropped, not surfaced or fatal");
});

test("concurrent restart calls for one sprite collapse into sequential requests", async () => {
  const h = await sandbox.provision(layers);
  await Promise.all([sandbox.restartComputer!(scope), sandbox.restartComputer!(scope)]);
  assert.deepEqual(fake.restarts(), [h.id, h.id], "serialized, one request per call, never interleaved forcing");
});

test("commands cannot swallow the script from stdin", async () => {
  const h = await sandbox.provision(layers);
  const r = await sandbox.run(h, "cat; echo after-cat");
  assert.equal(r.code, 0);
  assert.equal(r.stdout.trim(), "after-cat");
});

test("a creation rate limit surfaces the provider's code and retry hint", async () => {
  fake.rateLimitCreate(42);
  await assert.rejects(sandbox.provision(layers), (e: Error) => {
    assert.match(e.message, /sprites create /);
    assert.match(e.message, /sprite_creation_rate_limited/);
    assert.match(e.message, /retry after: 42s/);
    assert.match(e.message, /http 429/);
    assert.ok(e.cause instanceof APIError);
    return true;
  });
  const created = await sandbox.provision(layers);
  assert.equal(created.coldStart, true, "the next attempt creates normally");
  assert.ok(
    fake.calls.some((c) => c.method === "POST" && c.path === "/v1/sprites"),
    "creation goes through the SDK client",
  );
});

test("spritesErrorDetail keeps plain errors as they are and enriches API errors", () => {
  assert.equal(spritesErrorDetail(new Error("boom")), "boom");
  const limited = new APIError("Too many", {
    statusCode: 429,
    errorCode: "concurrent_sprite_limit_exceeded",
    retryAfterHeader: 7,
  });
  assert.equal(spritesErrorDetail(limited), "Too many; http 429; concurrent_sprite_limit_exceeded; retry after 7s");
});

test("a configured memory limit is applied as a resources policy and advertised in the profile", async () => {
  const s = make({ memoryMb: 4096 });
  assert.equal(s.profile.spec?.memoryMb, 4096);
  assert.equal(s.profile.spec?.cpus, 8);
  const h = await s.provision(layers);
  assert.deepEqual(fake.resources(h.id), { limitMB: 4096 });
  assert.equal(sandbox.profile.spec?.memoryMb, undefined, "no knob, no claim");
  assert.equal(fake.resources(sandboxScopeName("qmt", scope))?.limitMB, 4096);
});

test("the profile is honest about the base release and what survives sleep", () => {
  const os = sandbox.profile.spec?.os ?? "";
  assert.match(os, /Ubuntu \(25\.10 for newly created sprites/);
  assert.doesNotMatch(os, /LTS/);
  assert.match(os, /do not survive a cold wake/);
});

test("exportFiles tars workspace + home over the exec channel (the publish fast path)", async () => {
  const h = await sandbox.provision(layers);
  await sandbox.run(
    h,
    [
      "mkdir -p app/.cache",
      "printf hi > app/index.html",
      "printf junk > app/.cache/x",
      'printf note > "$HOME/.profile-note"',
    ].join(" && "),
  );

  const got = await sandbox.exportFiles!(h);
  const paths = got.map((e) => `${e.area}:${e.path}`).sort();
  assert.ok(paths.includes("workspace:app/index.html"), `workspace file packed (got ${paths.join(", ")})`);
  assert.ok(paths.includes("home:.profile-note"), "home file packed");
  assert.ok(!paths.some((p) => p.includes(".cache")), "content caches pruned by default");
  assert.ok(!paths.some((p) => p.startsWith("home:workspace/")), "workspace pruned from the home area");
  assert.equal(Buffer.from(got.find((e) => e.path === "app/index.html")!.data).toString("utf8"), "hi");
});

test("exportFiles keepContentCaches ships cache-named build output (publish parity)", async () => {
  const h = await sandbox.provision(layers);
  await sandbox.run(h, "mkdir -p site/.cache && printf real > site/.cache/bundle.js");
  const got = await sandbox.exportFiles!(h, {
    include: ["workspace"],
    keepContentCaches: true,
    exclude: () => false,
  });
  assert.ok(
    got.some((e) => e.path === "site/.cache/bundle.js"),
    "cache-named build output crosses when publish asks for it",
  );
});

test("blob staging is advertised only when the channel is actually wired", async () => {
  assert.equal(
    supportsBlobStaging(make()),
    false,
    "without blobTransfer/secret/apiBaseUrl the capability must not be claimed — copyHome probes for it",
  );
  const wired = make({
    blobTransfer: createMemoryBlobTransferStore(),
    capabilitySecret: "blob-secret",
    apiBaseUrl: "http://core.internal:8080",
  });
  assert.equal(supportsBlobStaging(wired), true, "wired up, sprites can move bytes by reference");
});

test("stageOut posts to core's blob endpoint by streaming, never by buffering in the guest", async () => {
  const sb = make({
    blobTransfer: createMemoryBlobTransferStore(),
    capabilitySecret: "blob-secret",
    apiBaseUrl: "http://core.internal:8080",
  });
  const h = await sb.provision(layers);
  await assert.rejects(() => sb.stageOut!(h, "artifacts/big.bin"), /sprites stageOut/);

  const script = fake.execScripts().find((s: string) => s.includes("/v1/blobs"))!;
  assert.ok(script, "the stageOut curl reached the guest");
  assert.match(script, /--upload-file/, "streams from disk rather than buffering in the guest");
  assert.doesNotMatch(script, /--data-binary/, "the OOM shape must never come back");
  assert.match(script, /-X POST/, "--upload-file alone would send PUT");
  assert.match(script, /x-content-sha256/, "core verifies the upload end-to-end");
});

test("stageIn pulls a blob into the guest atomically (temp then mv)", async () => {
  const sb = make({
    blobTransfer: createMemoryBlobTransferStore(),
    capabilitySecret: "blob-secret",
    apiBaseUrl: "http://core.internal:8080",
  });
  const h = await sb.provision(layers);
  await assert.rejects(() => sb.stageIn!(h, "inbox/big.bin", "f".repeat(32)), /sprites stageIn/);

  const script = fake.execScripts().find((s: string) => s.includes("/v1/blobs/"))!;
  assert.match(script, /-o .*\.part/, "downloads to a temp file");
  assert.match(script, /mv -f /, "and only then moves it into place");
  assert.match(script, /curl -fsS/, "-f so an HTTP error fails loudly instead of writing the error body");
});

test("a prep failure that writes nothing still names its cause", () => {
  assert.equal(
    execFailureDetail({ stdout: "", stderr: "", code: 124, timedOut: true }, 60),
    "timed out after 60s with no output",
    "a wedged guest filesystem kills the script before it can explain itself",
  );
  assert.equal(execFailureDetail({ stdout: "", stderr: "", code: 1, timedOut: false }, 60), "exit 1 with no output");
  assert.equal(execFailureDetail({ stdout: "out", stderr: " boom\n", code: 1, timedOut: false }, 60), "boom");
  assert.equal(execFailureDetail({ stdout: " out\n", stderr: "", code: 1, timedOut: false }, 60), "out");
});

test("destroying a scope exports the home to the snapshot store first and a replacement hydrates from it", async () => {
  const snapshots = createMemorySnapshotStore();
  const s = make({ snapshots });
  const h = await s.provision(layers);
  await s.run(h, 'printf survivor > "$HOME/notes.txt"');
  assert.equal(await snapshots.open(scope), null);

  await s.destroyScope!(scope);
  assert.ok(!fake.names().includes(h.id), "the sprite is gone");
  const stored = await snapshots.open(scope);
  assert.ok(stored && stored.size > 0, "but its home was exported before deletion");

  const again = await s.provision(layers);
  assert.equal(again.coldStart, false, "the replacement is hydrated, not cold");
  const back = await s.run(again, 'cat "$HOME/notes.txt"');
  assert.equal(back.stdout, "survivor");
  assert.equal(typeof s.persistHomeSnapshot, "function", "explicit export is offered when a store is wired");
  assert.equal(typeof make().persistHomeSnapshot, "undefined");
});

test("a failed export refuses to destroy the sprite", async () => {
  const snapshots = createMemorySnapshotStore();
  const s = make({ snapshots });
  const h = await s.provision(layers);
  fake.fail502(h.id);
  await assert.rejects(s.destroyScope!(scope), /WebSocket error/);
  assert.ok(fake.names().includes(h.id), "an irreversible delete never follows a lost export");
});

test("without a snapshot store a destroy is a single delete that tolerates a missing sprite", async () => {
  await sandbox.destroyScope!(scope);
  assert.deepEqual(
    fake.calls.map((c) => `${c.method} ${c.path}`),
    [`DELETE /v1/sprites/${sandboxScopeName("qmt", scope)}`],
  );
  fake.refuseDelete(503);
  await assert.rejects(sandbox.destroyScope!(scope), /sprites delete .*http 503/);
});

test("control-plane reads retry 429 with Retry-After while exec is never retried", async () => {
  const h = await sandbox.provision(layers);
  fake.failNext(429, { headers: { "retry-after": "0" }, match: (c) => c.path.endsWith("/check") });
  const status = await sandbox.computerStatus!(scope);
  assert.equal(status.provisioned, true);
  assert.equal(status.guestResponsive, true);
  assert.equal(fake.calls.filter((c) => c.path.endsWith("/check")).length, 2);

  const before = fake.execScripts().length;
  fake.stallAfterRun(h.id);
  await assert.rejects(sandbox.run(h, "echo hi"), /connection reset/);
  assert.equal(fake.execScripts().length, before + 1);

  for (let i = 1; i <= 4; i++)
    fake.failNext(503, {
      headers: { "retry-after": "0", "x-request-id": `req-${i}` },
      match: (c) => c.method === "DELETE",
    });
  await assert.rejects(sandbox.destroyScope!(scope), /sprites delete .*injected 503/);
});

test("control retries share an elapsed-time budget across backoff and SDK requests", async () => {
  let calls = 0;
  await assert.rejects(
    retrySpritesControl(async () => {
      calls++;
      throw new APIError("rate limited", { statusCode: 429, retryAfterHeader: 30 });
    }, 30),
    { name: "TimeoutError" },
  );
  assert.equal(calls, 1);
  calls = 0;
  await assert.rejects(
    retrySpritesControl(async () => {
      calls++;
      if (calls === 1) throw new APIError("unavailable", { statusCode: 503, retryAfterHeader: 0 });
      return new Promise((resolve) => setTimeout(() => resolve("late response"), 100));
    }, 30),
    { name: "TimeoutError" },
  );
  assert.equal(calls, 2);
});

for (const outcome of ["healthy", "unknown", "403", "503"]) {
  test(`refused restart with ${outcome} health observation never rolls back newer files`, async () => {
    const h = await sandbox.provision(layers);
    await sandbox.writeFile(h, "ledger", "old");
    await sandbox.teardown(h);
    await sandbox.writeFile(h, "ledger", "new");
    fake.refuseRestart(h.id);
    if (outcome === "healthy" || outcome === "unknown") fake.health(h.id, outcome, "machine is running");
    else
      for (let i = 0; i < (outcome === "503" ? 4 : 1); i++)
        fake.failNext(Number(outcome), { headers: { "retry-after": "0" }, match: (c) => c.path.endsWith("/check") });
    await assert.rejects(sandbox.restartComputer!(scope), /no checkpoint was restored/);
    assert.equal(await sandbox.readFile(h, "ledger"), "new");
    assert.equal(fake.calls.filter((c) => c.path.endsWith("/restore")).length, 0);
  });
}

test("failed hydration and failed deletion remain pending across adapters without overwriting the saved home", async () => {
  const snapshots = createMemorySnapshotStore();
  const initializationStore = createMemoryMap<{ pending: boolean }>();
  const advisoryLock = createMemoryAdvisoryLock();
  let failOpen = false;
  let opens = 0;
  const wrapped = {
    ...snapshots,
    open: async (id: string) => {
      opens++;
      if (failOpen) throw new Error("snapshot unavailable");
      return snapshots.open(id);
    },
  };
  const options = { snapshots: wrapped, initializationStore, advisoryLock };
  const a = make(options);
  const h = await a.provision(layers);
  await a.writeFile(h, "ledger", "saved");
  await a.destroyScope!(scope);
  failOpen = true;
  fake.refuseDelete(403);
  await assert.rejects(a.provision(layers), /snapshot unavailable/);
  assert.deepEqual(await initializationStore.get(h.id), { pending: true });
  const b = make(options);
  await assert.rejects(b.persistHomeSnapshot!(scope), /initialization is incomplete/);
  await assert.rejects(b.restartComputer!(scope), /initialization is incomplete/);
  const before = opens;
  await assert.rejects(b.provision(layers), /snapshot unavailable/);
  assert.equal(opens, before + 1);
  failOpen = false;
  fake.refuseDelete();
  const restored = await make(options).provision(layers);
  assert.equal(await b.readFile(restored, "ledger"), "saved");
  assert.equal(await initializationStore.get(h.id), null);
});

test("scratch allocation is deleted when resource initialization fails", async () => {
  const box = make({ memoryMb: 4096 });
  fake.failNext(400, { match: (call) => call.path.endsWith("/policy/resources") });
  await assert.rejects(box.provision(layers, { scratch: { key: "partial-allocation" } }));
  assert.ok(!fake.names().some((name) => name.includes("scratch")));
});

test("failed scratch initialization and deletion retain a safe retryable identity", async () => {
  const box = make({ memoryMb: 4096 });
  fake.failNext(400, { match: (call) => call.path.endsWith("/policy/resources") });
  fake.failNext(400, { match: (call) => call.method === "DELETE" });
  let pending: import("../src/sandbox/sandbox.ts").SandboxHandle | undefined;
  await assert.rejects(box.provision(layers, { scratch: { key: "failed-rollback" } }), (error: Error) => {
    assert.ok(error instanceof SandboxProvisionCleanupError);
    pending = error.handle;
    assert.equal(pending.backend, "sprites");
    assert.ok(fake.names().includes(pending.id));
    assert.ok(error.cause);
    return true;
  });
  assert.ok(pending);
  await box.teardown(pending, { destroy: true });
  assert.ok(!fake.names().includes(pending.id));
});