Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

42.3 KB
Raw Blame
import { createBackgroundBroker } from "../src/connectors/background-exec-broker.ts";
import { createMemoryProcessRegistry } from "../src/processes/process-registry.ts";
import { supportsProcessSessions } from "../src/sandbox/sandbox.ts";
import { createTurnSandboxes, type TurnSandboxContext } from "../src/core/orchestrator/sandboxes.ts";
import { test } from "node:test";
import assert from "node:assert/strict";
import { createSandboxResources, type SandboxResource, type SandboxDefault } from "../src/sandbox/sandbox-resources.ts";
import { createSandboxRouter } from "../src/sandbox/sandbox-routing.ts";
import {
  upgradeLegacySandboxes,
  legacySandboxBackendForScope,
  legacySandboxId,
  type LegacyRoute,
  type SandboxResourceUpgradeMarker,
} from "../src/sandbox/sandbox-resource-upgrade.ts";
import { createMemoryMap } from "../src/persistence/durable-map.ts";
import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";
import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
import type { Sandbox } from "../src/sandbox/sandbox.ts";

function fixture(configure?: (backend: Sandbox) => void, legacyScopes = ["personal:alice"]) {
  const records = createMemoryMap<SandboxResource>();
  const defaults = createMemoryMap<SandboxDefault>();
  const routes = createMemoryMap<LegacyRoute>();
  const marker = createMemoryMap<SandboxResourceUpgradeMarker>();
  const disks = new Map<string, Map<string, string>>();
  const provisioned: string[] = [];
  const backend: Sandbox = {
    profile: { backend: "local", writablePersistence: "resident_disk", processSessions: false },
    async provision(layers) {
      const id = layers.find((layer) => layer.mode === "rw")!.scopeId;
      provisioned.push(id);
      if (!disks.has(id)) disks.set(id, new Map());
      return { id, rootDir: "/workspace" };
    },
    async run(handle) {
      return { stdout: handle.id, stderr: "", code: 0, timedOut: false };
    },
    async readFile(handle, path) {
      return disks.get(handle.id)?.get(path) ?? null;
    },
    async writeFile(handle, path, data) {
      disks.get(handle.id)!.set(path, data);
    },
    async readFileBytes() {
      return null;
    },
    async writeFileBytes() {},
    async listDir() {
      return [];
    },
    async removeDir() {},
    async teardown() {},
    async destroyScope(id) {
      disks.delete(id);
    },
    async computerStatus(scopeId) {
      return { machine: scopeId, guestResponsive: true };
    },
    async restartComputer(scopeId) {
      provisioned.push(`restart:${scopeId}`);
    },
  };
  configure?.(backend);
  const lock = createMemoryAdvisoryLock();
  const upgrade =
    (overrides: Partial<Parameters<typeof upgradeLegacySandboxes>[0]> = {}) =>
    () =>
      upgradeLegacySandboxes({
        availableBackends: ["local"],
        records,
        defaults,
        marker,
        lock,
        routes: () => routes.entries(),
        legacyScopes: async () => legacyScopes,
        legacyBackend: () => "local",
        ...overrides,
      });
  const options = {
    upgrade: upgrade(),
    records,
    defaults,
    backends: { local: backend },
    defaultBackend: "local",
    lock,
    canUseScope: async (actor: string, scope: string) =>
      actor === "admin" || scope === `personal:${actor}` || (actor === "alice" && scope === "channel:team"),
  } satisfies Parameters<typeof createSandboxResources>[0];
  const resources = createSandboxResources(options);
  const router = createSandboxRouter({ backends: { local: backend }, defaultBackend: "local", resources });
  const layers = [{ scopeId: "personal:alice", mode: "rw" as const, mountPath: "/" }];
  return { records, defaults, routes, resources, router, provisioned, layers, backend, options, marker, upgrade };
}

test("blank sandbox identities coexist and default changes never copy files or redirect existing handles", async () => {
  const { resources, router, layers } = fixture();
  const old = await router.provision(layers);
  await router.writeFile(old, "secret", "old disk");
  const first = await resources.create("alice", "personal:alice", "local", "build");
  const second = await resources.create("alice", "personal:alice", "local", "analysis");
  const a = await router.provision(layers, { sandboxId: first.id });
  const b = await router.provision(layers, { sandboxId: second.id });
  assert.notEqual(a.id, b.id);
  assert.equal(await router.readFile(a, "secret"), null);
  await router.writeFile(a, "output", "A");
  await resources.setDefault("alice", "personal:alice", first.id);
  assert.equal((await router.provision(layers)).id, a.id);
  await resources.setDefault("alice", "personal:alice", second.id);
  assert.equal((await router.provision(layers)).id, b.id);
  assert.equal((await router.run(a, "pwd")).stdout, a.id);
  assert.equal(await router.readFile(a, "output"), "A");
  assert.equal(await router.readFile(b, "output"), null);
  assert.equal(await router.readFile(old, "secret"), "old disk");
  assert.equal(a.scopeId, "personal:alice");
  assert.equal(a.resourceId, first.id);
});

test("provisioning a running computer does not wait behind a long-running command on it", async () => {
  const { resources, router, layers } = fixture();
  const handle = await router.provision(layers);
  const command = Promise.withResolvers<void>();
  const running = resources.use(handle.resourceId!, () => command.promise);

  assert.equal((await router.provision(layers)).id, handle.id);
  command.resolve();
  await running;
});

test("explicit sandbox profiles follow selected storage rather than the parent's default provider", async () => {
  const { backend, options } = fixture();
  const modal: Sandbox = { ...backend, profile: { ...backend.profile, backend: "modal" } };
  const backends = { local: backend, modal };
  const resources = createSandboxResources({ ...options, backends });
  const router = createSandboxRouter({ backends, defaultBackend: "local", resources });
  const worker = await resources.create("alice", "personal:alice", "modal", "Worker");
  assert.equal((await router.profileFor!("personal:alice")).backend, "local");
  assert.equal((await router.profileFor!("personal:alice", worker.id)).backend, "modal");
  await assert.rejects(router.profileFor!("personal:alice", ""), /sandbox not found/);
  await assert.rejects(
    router.provision([{ scopeId: "personal:alice", mode: "rw", mountPath: "" }], { sandboxId: "" }),
    /sandbox not found/,
  );
  assert.equal((await router.profileFor!("personal:alice")).backend, "local");
});

test("unset defaults remain unset durably while explicit execution remains usable", async () => {
  const { resources, router, layers, defaults } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  await resources.setDefault("alice", "personal:alice", null);
  assert.equal((await defaults.get("personal:alice"))?.sandboxId, null);
  await assert.rejects(
    router.provision(layers),
    /sandbox list, create, set_default, then retry before reporting blocked/,
  );
  const explicit = await router.provision(layers, { sandboxId: record.id });
  assert.equal(explicit.resourceId, record.id);
  const listed = await resources.list("alice", "personal:alice");
  assert.equal(listed.defaultSandboxId, null);
  assert.equal(listed.defaultMode, "none");
  assert.equal(await resources.resolve("personal:new"), null);
});

test("legacy adoption is deterministic and reconnects the existing backing identity", async () => {
  const { resources, router, layers, records, backend } = fixture();
  const old = await backend.provision(layers);
  await backend.writeFile(old, "file", "keep");
  const lists = await Promise.all(Array.from({ length: 8 }, () => resources.list("alice", "personal:alice")));
  const id = lists[0]!.defaultSandboxId!;
  assert.equal(id, legacySandboxId("personal:alice", "local"));
  assert.ok(lists.every((list) => list.defaultSandboxId === id));
  assert.equal((await records.all()).length, 1);
  assert.equal((await records.get(id))?.backingScopeId, "personal:alice");
  const adopted = await router.provision(layers);
  assert.equal(adopted.resourceId, id);
  assert.equal(await router.readFile(adopted, "file"), "keep");
});

test("scope ACL protects inventory and target access and prevents cross-scope default credential relocation", async () => {
  const { resources } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  await assert.rejects(resources.access("bob", record.id), /permission/);
  await assert.rejects(resources.list("bob", "personal:alice"), /permission/);
  await assert.rejects(resources.create("bob", "personal:alice", "local"), /permission/);
  await assert.rejects(resources.setDefault("bob", "personal:alice", record.id), /permission/);
  await assert.rejects(resources.setDefault("admin", "personal:bob", record.id), /belong to this scope/);
  assert.equal((await resources.access("admin", record.id)).id, record.id);
});

test("status and restart resolve the selected backing machine rather than legacy scope", async () => {
  const { resources, router, provisioned } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  await resources.setDefault("alice", "personal:alice", record.id);
  assert.equal((await router.computerStatus!("personal:alice")).machine, record.backingScopeId);
  await router.restartComputer!("personal:alice");
  assert.deepEqual(provisioned, [record.backingScopeId, `restart:${record.backingScopeId}`]);
});

test("listing an untouched scope never invents a legacy machine", async () => {
  const { resources, provisioned } = fixture(undefined, []);
  assert.deepEqual(await resources.list("alice", "personal:alice"), {
    sandboxes: [],
    defaultSandboxId: null,
    defaultMode: "none",
    providers: [{ name: "local", actions: ["create", "status", "restart", "retire"] }],
  });
  assert.deepEqual(provisioned, []);
  await assert.rejects(resources.create("alice", "personal:alice", "__proto__"), /unavailable/);
  await assert.rejects(resources.create("alice", "personal:alice", "toString"), /unavailable/);
});

test("turn default changes invalidate cached provisioning while explicit calls dedupe and cleanup each computer once", async () => {
  const { resources, router, layers, backend } = fixture();
  const released: string[] = [];
  backend.teardown = async (handle) => {
    released.push(handle.id);
  };
  const a = await resources.create("alice", "personal:alice", "local");
  const b = await resources.create("alice", "personal:alice", "local");
  await resources.setDefault("alice", "personal:alice", a.id);
  const turn = createTurnSandboxes({
    deps: { sandbox: router, sandboxResources: resources },
    input: { origin: { kind: "user" } },
    actor: { id: "alice", type: "internal" },
    session: { id: "s" },
    resolution: { layers },
    scopeId: "personal:alice",
    memoryScopeId: "personal:alice",
    transferId: "t",
    turnSessionDir: "turn/s",
    turnFilesDir: "turn/s/t",
    connectorEnv: { AGENT_API_TOKEN: "scope-token" },
    ownerAuthAvailable: false,
    visibleSkills: [],
    visibleSkillsForTurn: async () => [],
    emitGapWork: () => {},
    perf: { credsMs: 0 },
  } as unknown as TurnSandboxContext);
  const old = await turn.provision();
  await resources.setDefault("alice", "personal:alice", b.id);
  turn.invalidateProvision();
  const next = await turn.provision();
  assert.notEqual(old.id, next.id);
  assert.equal((await router.run(old, "still old")).stdout, old.id);
  const [x, y] = await Promise.all([turn.provisionResource(a.id), turn.provisionResource(a.id)]);
  assert.equal(x, y);
  await turn.reclaimBox();
  assert.deepEqual(released.sort(), [a.backingScopeId, b.backingScopeId].sort());
});

test("retirement refuses the default, waits for an active command, and prevents future execution", async () => {
  const { resources, router, backend, layers } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  await resources.setDefault("alice", "personal:alice", record.id);
  await assert.rejects(resources.retire("alice", record.id), /default/);
  const handle = await router.provision(layers);
  await resources.setDefault("alice", "personal:alice", null);
  let finish!: () => void;
  let started!: () => void;
  const running = new Promise<void>((resolve) => {
    started = resolve;
  });
  backend.run = async () => {
    started();
    await new Promise<void>((resolve) => {
      finish = resolve;
    });
    return { stdout: "done", stderr: "", code: 0, timedOut: false };
  };
  let destroyed = false;
  backend.destroyScope = async () => {
    destroyed = true;
  };
  const command = router.run(handle, "work");
  await running;
  const retiring = resources.retire("alice", record.id);
  assert.equal(destroyed, false);
  finish();
  await command;
  await retiring;
  assert.equal(destroyed, true);
  await assert.rejects(router.run(handle, "no resurrection"), /retired/);
  await assert.rejects(resources.setDefault("alice", "personal:alice", record.id), /retired/);
});

test("every handle operation rejects retirement before reaching a backend that could revive the machine", async () => {
  let backendCalls = 0;
  const hit = async (): Promise<never> => {
    backendCalls++;
    throw new Error("backend reached");
  };
  const { resources, router, layers } = fixture((backend) => {
    backend.profile.processSessions = true;
    backend.startProcess = hit;
    backend.readProcess = hit;
    backend.writeStdin = hit;
    backend.signalProcess = hit;
    backend.listProcesses = async () => [];
    backend.exportFiles = hit;
    backend.stageIn = hit;
    backend.stageOut = hit;
    backend.importFiles = hit;
  });
  const record = await resources.create("alice", "personal:alice", "local");
  const handle = await router.provision(layers, { sandboxId: record.id });
  await resources.retire("alice", record.id);
  const operations: Array<() => Promise<unknown>> = [
    () => router.run(handle, "work"),
    () => router.readFile(handle, "file"),
    () => router.readFileBytes(handle, "file"),
    () => router.writeFile(handle, "file", "data"),
    () => router.writeFileBytes(handle, "file", new Uint8Array()),
    () => router.listDir(handle, "."),
    () => router.removeDir(handle, "dir"),
    () => router.teardown(handle),
    () => router.startProcess!(handle, "work"),
    () => router.readProcess!(handle, "process"),
    () => router.writeStdin!(handle, "process", "input"),
    () => router.signalProcess!(handle, "process", "TERM"),
    () => router.listProcesses!(handle),
    () => router.exportFiles!(handle),
    () => router.stageIn!(handle, "file", "blob"),
    () => router.stageOut!(handle, "file"),
    () => router.importFiles!(handle, []),
    () => resources.status("alice", record.id),
    () => resources.restart("alice", record.id),
  ];
  for (const operation of operations) await assert.rejects(operation(), /retired/);
  assert.equal(backendCalls, 0);
});

for (const fail of [false, true])
  test(`retirement waits for outstanding creation ${fail ? "failure" : "success"} and remains terminal`, async () => {
    const { resources, records, backend } = fixture();
    const provision = backend.provision;
    let release!: () => void;
    let entered!: () => void;
    const gate = new Promise<void>((resolve) => {
      release = resolve;
    });
    const started = new Promise<void>((resolve) => {
      entered = resolve;
    });
    let first = true;
    backend.provision = async (layers, options) => {
      if (first) {
        first = false;
        entered();
        await gate;
        if (fail) throw new Error("create failed");
      }
      return provision(layers, options);
    };
    const creating = resources.create("alice", "personal:alice", "local");
    const completed = creating.then(
      () => undefined,
      (error) => {
        assert.match(String(error), /create failed/);
      },
    );
    await started;
    const record = (await records.all())[0]!;
    let retired = false;
    const retiring = resources.retire("alice", record.id).then(() => {
      retired = true;
    });
    await new Promise((resolve) => setImmediate(resolve));
    assert.equal(retired, false);
    assert.equal((await records.get(record.id))?.state, "provisioning");
    release();
    await Promise.all([completed, retiring]);
    assert.equal((await records.get(record.id))?.state, "retired");
    await assert.rejects(
      resources.use(record.id, async () => {}),
      /retired/,
    );
  });

test("activation preserves routes, cold identities and explicit nulls without calling a provider", async () => {
  const { options, records, defaults, routes, marker, provisioned, upgrade } = fixture(undefined, []);
  await routes.put("personal:routed", { backend: "modal" });
  await defaults.put("personal:unset", { sandboxId: null });
  const managed: SandboxResource = {
    id: "managed",
    backend: "local",
    ownerScopeId: "personal:selected",
    backingScopeId: "sandbox-managed",
    name: "managed",
    createdBy: "selected",
    createdAt: "2026-01-01",
    legacy: false,
    state: "ready",
  };
  await records.put(managed.id, managed);
  await defaults.put(managed.ownerScopeId, { sandboxId: managed.id });
  const resources = createSandboxResources({
    ...options,
    upgrade: upgrade({
      availableBackends: ["local", "modal"],
      legacyScopes: async () => ["personal:old-session", "personal:unset"],
      legacySandboxes: async () => [
        { scopeId: "personal:routed", backend: "modal", machineId: "sb-old" },
        { scopeId: "personal:routed", backend: "e2b", machineId: "e2b-cold" },
        { scopeId: "sandbox-managed", backend: "local", machineId: "managed-machine" },
      ],
    }),
  });
  const routed = await resources.resolve("personal:routed");
  assert.equal(routed?.backend, "modal");
  assert.equal(routed?.backingScopeId, "personal:routed");
  assert.equal(routed?.machineId, "sb-old");
  assert.equal(routed?.state, "unverified");
  assert.equal((await resources.resolve("personal:old-session"))?.machineId, undefined);
  assert.equal(await resources.resolve("personal:unset"), null);
  assert.equal((await resources.resolve(managed.ownerScopeId))?.id, "managed");
  assert.equal(await resources.resolve("personal:new-after-activation"), null);
  assert.ok(await marker.get("explicit-defaults"));
  const inventory = await resources.list("admin", "personal:routed");
  assert.ok(inventory.sandboxes.some((r) => r.backend === "e2b" && r.machineId === "e2b-cold"));
  assert.ok(!inventory.sandboxes.some((r) => r.legacy && r.backingScopeId === "sandbox-managed"));
  assert.deepEqual(inventory.sandboxes.find((r) => r.id === routed!.id)?.availableActions, []);
  assert.deepEqual(provisioned, []);
  assert.equal((await routes.get("personal:routed"))?.backend, "modal");
  await routes.put("personal:later", { backend: "local" });
  const restarted = createSandboxResources(options);
  assert.equal(await restarted.resolve("personal:later"), null);
  assert.equal((await restarted.resolve("personal:routed"))?.id, routed?.id);
});

test("activation retries partial durable writes without losing defaults or creating machines", async () => {
  const { options, defaults, marker, records, provisioned } = fixture(undefined, ["personal:first", "personal:second"]);
  const put = defaults.putIfAbsent;
  let fail = true;
  defaults.putIfAbsent = async (id, value) => {
    if (id === "personal:second" && fail) throw new Error("database interrupted");
    return put(id, value);
  };
  const resources = createSandboxResources(options);
  await assert.rejects(resources.resolve("personal:first"), /database interrupted/);
  assert.equal(await marker.get("explicit-defaults"), null);
  const first = await defaults.get("personal:first");
  await defaults.put("personal:first", { sandboxId: null });
  fail = false;
  await Promise.all([resources.resolve("personal:first"), resources.resolve("personal:second")]);
  assert.ok(first?.sandboxId);
  assert.equal(await resources.resolve("personal:first"), null);
  assert.equal((await records.all()).length, 2);
  assert.ok(await marker.get("explicit-defaults"));
  assert.deepEqual(provisioned, []);
});

test("boot activation freezes legacy scope adoption before a new session arrives", async () => {
  const known = ["personal:old"];
  const { options, defaults, provisioned } = fixture(undefined, known);
  const resources = createSandboxResources(options);
  await resources.initialize();
  known.push("personal:new-session");
  assert.equal(await resources.resolve("personal:new-session"), null);
  assert.equal(await defaults.get("personal:new-session"), null);
  assert.ok((await resources.resolve("personal:old"))?.id);
  assert.deepEqual(provisioned, []);
});

test("retirement deletes an inferred missing computer without provisioning, status or restore", async () => {
  const { resources, backend, provisioned } = fixture();
  await resources.initialize();
  const record = await resources.resolve("personal:alice");
  assert.equal(record?.state, "unverified");
  await resources.setDefault("alice", "personal:alice", null);
  const destroyed: string[] = [];
  backend.provision = async () => {
    throw new Error("must not restore");
  };
  backend.computerStatus = async () => {
    throw new Error("must not probe");
  };
  backend.destroyScope = async (scope) => {
    destroyed.push(scope);
  };
  await resources.retire("alice", record!.id);
  await resources.retire("alice", record!.id);
  assert.deepEqual(destroyed, ["personal:alice"]);
  assert.deepEqual(provisioned, []);
});

test("unsupported retirement is hidden and refuses before inventory mutation", async () => {
  const { resources, records } = fixture((backend) => {
    delete backend.destroyScope;
  });
  const record = await resources.create("alice", "personal:alice", "local");
  const before = await records.get(record.id);
  const list = await resources.list("alice", "personal:alice");
  assert.ok(!list.providers[0]!.actions.includes("retire"));
  assert.ok(!list.sandboxes.find((r) => r.id === record.id)!.availableActions!.includes("retire"));
  await assert.rejects(resources.retire("alice", record.id), /retirement unavailable/);
  assert.deepEqual(await records.get(record.id), before);
});

test("failed retirement stays unroutable and retries cleanup by backing scope after a core restart", async () => {
  const { resources, backend, records, options } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  backend.destroyScope = async () => {
    throw new Error("provider unavailable");
  };
  await assert.rejects(resources.retire("alice", record.id), /provider unavailable/);
  assert.equal((await records.get(record.id))?.cleanupPending, true);
  assert.equal((await records.get(record.id))?.state, "retired");
  await assert.rejects(resources.setDefault("alice", "personal:alice", record.id), /retired/);
  assert.deepEqual(
    (await resources.list("alice", "personal:alice")).sandboxes.find((r) => r.id === record.id)?.availableActions,
    ["retire"],
  );
  const destroyed: string[] = [];
  backend.destroyScope = async (scope) => {
    destroyed.push(scope);
  };
  backend.provision = async () => {
    throw new Error("must not restore");
  };
  const restarted = createSandboxResources(options);
  await restarted.retire("alice", record.id);
  assert.deepEqual(destroyed, [record.backingScopeId]);
  assert.equal((await records.get(record.id))?.cleanupPending, false);
  assert.equal((await records.get(record.id))?.error, undefined);
});

test("a pending retirement without an error remains retryable after a crash", async () => {
  const { resources, records, options, backend } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  await records.put(record.id, { ...record, state: "retired", cleanupPending: true });
  let destroyed = false;
  backend.destroyScope = async () => {
    destroyed = true;
  };
  await createSandboxResources(options).retire("alice", record.id);
  assert.equal(destroyed, true);
  assert.equal((await records.get(record.id))?.cleanupPending, false);
});

test("retirement preserves core live-work and owning-scope guards before direct deletion", async () => {
  const { resources, records, options, backend } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  let destroyed = false;
  backend.destroyScope = async () => {
    destroyed = true;
  };
  const guarded = createSandboxResources({
    ...options,
    beforeRetire: async () => {
      throw new Error("live background work");
    },
  });
  await assert.rejects(guarded.retire("mallory", record.id), /permission/);
  await assert.rejects(guarded.retire("alice", record.id), /live background work/);
  assert.equal(destroyed, false);
  assert.equal((await records.get(record.id))?.state, "ready");
});

test("retirement waits for background startup to commit its live registry row", async () => {
  const { options, backend, layers } = fixture((sandbox) => {
    sandbox.profile.processSessions = true;
    sandbox.startProcess = async () => ({ processId: "job" });
    sandbox.readProcess = async () => ({ chunks: "", cursor: 0, status: { state: "running" } });
    sandbox.writeStdin = async () => {};
    sandbox.signalProcess = async () => {};
    sandbox.listProcesses = async () => [];
  });
  const registry = createMemoryProcessRegistry();
  const resources = createSandboxResources({
    ...options,
    beforeRetire: async (record) => {
      if ((await registry.liveByScope(record.ownerScopeId)).some((r) => r.sandboxId === record.id))
        throw new Error("live background work");
    },
  });
  const router = createSandboxRouter({ resources, backends: { local: backend }, defaultBackend: "local" });
  assert.ok(supportsProcessSessions(router));
  const record = await resources.create("alice", "personal:alice", "local");
  const handle = await router.provision(layers, { sandboxId: record.id });
  const entering = Promise.withResolvers<void>();
  const release = Promise.withResolvers<void>();
  const register = registry.register.bind(registry);
  registry.register = async (row) => {
    entering.resolve();
    await release.promise;
    return register(row);
  };
  const broker = createBackgroundBroker({ sandbox: router, registry, scopeId: "personal:alice", pollMs: 0 });
  const starting = broker.start(handle, "sleep 60", "Run background tests");
  await entering.promise;
  let destroyed = false;
  backend.destroyScope = async () => {
    destroyed = true;
  };
  const retiring = assert.rejects(resources.retire("alice", record.id), /live background work/);
  release.resolve();
  await Promise.all([starting, retiring]);
  assert.equal(destroyed, false);
  assert.equal((await registry.get("job"))?.sandboxId, record.id);
});

test("failed background registration kills its process and releases the resource lock", async () => {
  const { resources, router, backend, layers } = fixture((sandbox) => {
    sandbox.profile.processSessions = true;
    sandbox.startProcess = async () => ({ processId: "unregistered" });
    sandbox.readProcess = async () => ({ chunks: "", cursor: 0, status: { state: "running" } });
    sandbox.writeStdin = async () => {};
    sandbox.signalProcess = async () => {};
    sandbox.listProcesses = async () => [];
  });
  assert.ok(supportsProcessSessions(router));
  const record = await resources.create("alice", "personal:alice", "local");
  const handle = await router.provision(layers, { sandboxId: record.id });
  const registry = createMemoryProcessRegistry();
  registry.register = async () => {
    throw new Error("registry unavailable");
  };
  const signals: string[] = [];
  backend.signalProcess = async (_handle, id, signal) => {
    signals.push(`${id}:${signal}`);
  };
  const broker = createBackgroundBroker({ sandbox: router, registry, scopeId: "personal:alice", pollMs: 0 });
  await assert.rejects(broker.start(handle, "sleep 60", "Run background tests"), /registry unavailable/);
  assert.deepEqual(signals, ["unregistered:KILL"]);
  await resources.retire("alice", record.id);
  assert.equal((await resources.get(record.id)).cleanupPending, false);
});

test("resource activation honors scope defaults and preserves explicit legacy provider routes", async () => {
  const { options, routes, backend, upgrade } = fixture(undefined, [
    "personal:alice",
    "channel:room",
    "personal:existing",
  ]);
  await routes.put("personal:existing", { backend: "aws" });
  const scopeDefaults = { personal: "modal", channel: "sprites" } as const;
  const resources = createSandboxResources({
    ...options,
    upgrade: upgrade({
      availableBackends: ["aws", "modal", "sprites"],
      legacyBackend: (scope) => legacySandboxBackendForScope(scope, "sprites", scopeDefaults),
    }),
    backends: { modal: backend, sprites: backend, aws: backend },
    defaultBackend: "sprites",
  });
  assert.equal((await resources.resolve("personal:alice"))?.backend, "modal");
  assert.equal((await resources.resolve("channel:room"))?.backend, "sprites");
  assert.equal((await resources.resolve("personal:existing"))?.backend, "aws");
  assert.equal(await resources.resolve("personal:new"), null);
  assert.equal(resources.defaultBackend(), "sprites");
  assert.equal(legacySandboxBackendForScope("org", "sprites", scopeDefaults), "sprites");
});

for (const [kind, waiting] of [
  ["modal", "command"],
  ["modal", "checkpoint"],
  ["sprites", "command"],
  ["sprites", "checkpoint"],
] as const) {
  test(`${kind} ${waiting} allows other tools while restart waits`, { timeout: 10000 }, async () => {
    const { options, backend, layers } = fixture();
    const resources = createSandboxResources({ ...options, backends: { [kind]: backend }, defaultBackend: kind });
    const router = createSandboxRouter({ backends: { [kind]: backend }, defaultBackend: kind, resources });
    const record = await resources.create("alice", "personal:alice", kind);
    await resources.setDefault("alice", "personal:alice", record.id);
    const handle = await router.provision(layers);
    const entered = Promise.withResolvers<void>();
    const release = Promise.withResolvers<void>();
    const hold = async () => {
      entered.resolve();
      await release.promise;
    };
    backend.teardown = hold;
    backend.run = async (_handle, command) => {
      if (command === "long") await hold();
      return { stdout: command, stderr: "", code: 0, timedOut: false };
    };
    const long = waiting === "command" ? router.run(handle, "long") : router.teardown(handle);
    let restarted = false;
    backend.restartComputer = async () => {
      restarted = true;
    };
    let restarting: Promise<void> | undefined;
    let timer: ReturnType<typeof setTimeout> | undefined;
    try {
      await entered.promise;
      const result = await Promise.race([
        router.run(handle, "responsive"),
        new Promise<never>((_, reject) => {
          timer = setTimeout(() => reject(new Error("ordinary tool was serialized")), 1000);
        }),
      ]);
      assert.equal(result.stdout, "responsive");
      assert.equal((await resources.status("alice", record.id)).guestResponsive, true);
      restarting =
        waiting === "command" ? resources.restart("alice", record.id) : router.restartComputer!("personal:alice");
      await new Promise<void>((resolve) => setImmediate(resolve));
      assert.equal(restarted, false);
    } finally {
      clearTimeout(timer);
      release.resolve();
      await long;
      await restarting;
    }
    assert.equal(restarted, true);
    await resources.setDefault("alice", "personal:alice", null);
    await resources.retire("alice", record.id);
    await assert.rejects(router.run(handle, "retired"), /retired/);
  });
}

test("Modal provisioning and destructive cleanup wait for active operations", { timeout: 10000 }, async () => {
  const { options, backend, layers } = fixture();
  const resources = createSandboxResources({ ...options, backends: { modal: backend }, defaultBackend: "modal" });
  const router = createSandboxRouter({ backends: { modal: backend }, defaultBackend: "modal", resources });
  const record = await resources.create("alice", "personal:alice", "modal");
  const handle = await router.provision(layers, { sandboxId: record.id });
  const entered = Promise.withResolvers<void>();
  const release = Promise.withResolvers<void>();
  backend.run = async () => {
    entered.resolve();
    await release.promise;
    return { stdout: "done", stderr: "", code: 0, timedOut: false };
  };
  const running = router.run(handle, "long");
  await entered.promise;
  let provisioned = false;
  let destroyed = false;
  backend.provision = async () => {
    provisioned = true;
    return handle;
  };
  backend.teardown = async () => {
    destroyed = true;
  };
  const provision = router.provision(layers, { sandboxId: record.id });
  const teardown = router.teardown(handle, { destroy: true });
  try {
    await new Promise<void>((resolve) => setImmediate(resolve));
    assert.equal(provisioned, false);
    assert.equal(destroyed, false);
  } finally {
    release.resolve();
    await Promise.all([running, provision, teardown]);
  }
  assert.equal(provisioned, true);
  assert.equal(destroyed, true);
});

test("concurrent commands on one Sprites computer run together", { timeout: 10000 }, async () => {
  const { options, backend, layers } = fixture();
  const resources = createSandboxResources({ ...options, backends: { sprites: backend }, defaultBackend: "sprites" });
  const router = createSandboxRouter({ backends: { sprites: backend }, defaultBackend: "sprites", resources });
  const record = await resources.create("alice", "personal:alice", "sprites");
  const handle = await router.provision(layers, { sandboxId: record.id });
  const release = Promise.withResolvers<void>();
  let running = 0;
  let peak = 0;
  backend.run = async (_handle, command) => {
    peak = Math.max(peak, ++running);
    if (peak === 3) release.resolve();
    await release.promise;
    running--;
    return { stdout: command, stderr: "", code: 0, timedOut: false };
  };
  const results = await Promise.all(["a", "b", "c"].map((command) => router.run(handle, command)));
  assert.deepEqual(
    results.map((result) => result.stdout),
    ["a", "b", "c"],
  );
  assert.equal(peak, 3);
});

test("parking teardown waits for active commands on the computer", { timeout: 10000 }, async () => {
  const { options, backend, layers } = fixture();
  const parking: Sandbox = { ...backend, profile: { ...backend.profile, parksOnTeardown: true } };
  const resources = createSandboxResources({ ...options, backends: { e2b: parking }, defaultBackend: "e2b" });
  const router = createSandboxRouter({ backends: { e2b: parking }, defaultBackend: "e2b", resources });
  const record = await resources.create("alice", "personal:alice", "e2b");
  const handle = await router.provision(layers, { sandboxId: record.id });
  const entered = Promise.withResolvers<void>();
  const release = Promise.withResolvers<void>();
  parking.run = async () => {
    entered.resolve();
    await release.promise;
    return { stdout: "done", stderr: "", code: 0, timedOut: false };
  };
  let parked = false;
  parking.teardown = async () => {
    parked = true;
  };
  const running = router.run(handle, "long");
  await entered.promise;
  const teardown = router.teardown(handle);
  try {
    await new Promise<void>((resolve) => setImmediate(resolve));
    assert.equal(parked, false);
  } finally {
    release.resolve();
    await Promise.all([running, teardown]);
  }
  assert.equal(parked, true);
});

test("a verified live turn can create only its own new scope computer without directory mutations", async () => {
  const { resources, router } = fixture(undefined, []);
  const scope = "channel:external-slack:T1:policy:C1";
  await resources.initialize();
  assert.equal(await resources.resolve(scope), null);
  await assert.rejects(resources.create("alice", scope, "local"), /permission/);
  let current = true;
  const turn = resources.forTurn({ actorId: "alice", scopeId: scope, isCurrent: async () => current });
  const computer = await turn.create("alice", scope, "local", "external work");
  await turn.setDefault("alice", scope, computer.id);
  const handle = await router.provision([{ scopeId: scope, mode: "rw", mountPath: "" }]);
  assert.equal(handle.scopeId, scope);
  await router.writeFile(handle, "result", "safe-output");
  assert.equal(await router.readFile(handle, "result"), "safe-output");
  await assert.rejects(turn.create("bob", scope, "local"), /permission/);
  await assert.rejects(turn.create("alice", "personal:bob", "local"), /permission/);
  current = false;
  await assert.rejects(turn.access("alice", computer.id), /permission/);
  await assert.rejects(turn.setDefault("alice", scope, computer.id), /permission/);
});

test("legacy import refuses an unavailable selected provider before writing any adoption state", async () => {
  const { options, upgrade, records, defaults, marker } = fixture();
  const resources = createSandboxResources({ ...options, upgrade: upgrade({ availableBackends: [] }) });
  await assert.rejects(resources.initialize(), /personal:alice: backend local is not configured/);
  assert.deepEqual(await records.all(), []);
  assert.deepEqual(await defaults.all(), []);
  assert.equal(await marker.get("explicit-defaults"), null);
  await createSandboxResources(options).initialize();
  assert.ok((await defaults.get("personal:alice"))?.sandboxId);
});

test("legacy import does not validate obsolete provider choices after explicit selection or clearing", async () => {
  const { options, upgrade, defaults, marker } = fixture(undefined, ["personal:cleared", "personal:selected"]);
  await defaults.put("personal:cleared", { sandboxId: null });
  await defaults.put("personal:selected", { sandboxId: "newer-computer" });
  await createSandboxResources({ ...options, upgrade: upgrade({ availableBackends: [] }) }).initialize();
  assert.deepEqual(await defaults.get("personal:cleared"), { sandboxId: null });
  assert.deepEqual(await defaults.get("personal:selected"), { sandboxId: "newer-computer" });
  assert.ok(await marker.get("explicit-defaults"));
});

test("a computer stays held until its last live run releases it, and ended runs stop counting", async () => {
  const { resources } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  await resources.hold(record.id, "a");
  await resources.hold(record.id, "b");
  await resources.hold(record.id, "b");
  const live = new Set(["a", "b"]);
  const isLive = async (runId: string) => live.has(runId);
  assert.equal(await resources.release(record.id, "a", isLive), true);
  live.delete("b");
  assert.equal(await resources.release(record.id, undefined, isLive), false);
  assert.deepEqual((await resources.get(record.id)).holders, []);
  const listed = (await resources.list("alice", "personal:alice")).sandboxes.find((s) => s.id === record.id);
  assert.equal(listed?.holders, undefined);
});

test("two concurrent turns on one parking computer: only the last turn to finish parks it", async () => {
  const { options, backend, layers } = fixture();
  const parking: Sandbox = { ...backend, profile: { ...backend.profile, parksOnTeardown: true } };
  const resources = createSandboxResources({ ...options, backends: { e2b: parking }, defaultBackend: "e2b" });
  const router = createSandboxRouter({ backends: { e2b: parking }, defaultBackend: "e2b", resources });
  const record = await resources.create("alice", "personal:alice", "e2b");
  await resources.setDefault("alice", "personal:alice", record.id);
  const teardowns: unknown[] = [];
  parking.teardown = async (_handle, opts) => {
    teardowns.push(opts ?? {});
  };
  const { runs } = createMemoryRunStore();
  const start = async (sessionId: string) => {
    await runs.enqueue({ sessionId, request: { origin: { kind: "direct" }, text: "x" } as never });
    return (await runs.claim("worker", 60_000))!;
  };
  const turn = (runId: string, sessionId: string) =>
    createTurnSandboxes({
      deps: { sandbox: router, sandboxResources: resources, runs },
      input: { origin: { kind: "user" }, runId },
      actor: { id: "alice", type: "internal" },
      session: { id: sessionId },
      resolution: { layers },
      scopeId: "personal:alice",
      memoryScopeId: "personal:alice",
      transferId: runId,
      turnSessionDir: `turn/${sessionId}`,
      turnFilesDir: `turn/${sessionId}/${runId}`,
      connectorEnv: {},
      ownerAuthAvailable: false,
      visibleSkills: [],
      visibleSkillsForTurn: async () => [],
      emitGapWork: () => {},
      perf: { credsMs: 0 },
    } as unknown as TurnSandboxContext);
  const a = await start("sa");
  const b = await start("sb");
  const first = turn(a.id, "sa");
  const second = turn(b.id, "sb");
  await Promise.all([first.provision(), second.provision()]);
  await first.reclaimBox();
  await runs.complete(a.id, a.leaseToken!, { status: "ok", reply: "done" });
  assert.deepEqual(teardowns, [{ keepWarm: true }]);
  await second.reclaimBox();
  assert.deepEqual(teardowns, [{ keepWarm: true }, {}]);
  assert.deepEqual((await resources.get(record.id)).holders, []);
});

test("same-name skills from distinct sources keep their materialized assets separate", async () => {
  const { resources, router, layers } = fixture();
  const record = await resources.create("alice", "personal:alice", "local");
  await resources.setDefault("alice", "personal:alice", record.id);
  const turn = createTurnSandboxes({
    deps: { sandbox: router, sandboxResources: resources },
    input: { origin: { kind: "user" } },
    actor: { id: "alice", type: "internal" },
    session: { id: "s" },
    resolution: { layers },
    scopeId: "personal:alice",
    memoryScopeId: "personal:alice",
    transferId: "t",
    turnSessionDir: "turn/s",
    turnFilesDir: "turn/s/t",
    connectorEnv: {},
    ownerAuthAvailable: false,
    emitGapWork: () => {},
    visibleSkillsForTurn: async (sources: readonly string[] = ["org:example", "personal:alice"]) =>
      sources.map((source) => ({
        skill: {
          id: source,
          scopeId: source,
          version: 1,
          manifest: {
            name: "helper",
            body: "Run skills/helper/value.py",
            files: [{ path: "value.py", content: source }],
          },
        },
        shadowed: [],
      })),
  } as unknown as TurnSandboxContext);
  const a = await turn.useSkill("helper", "SKILL.md", undefined, "org:example");
  const b = await turn.useSkill("helper", "SKILL.md", undefined, "personal:alice");
  const again = await turn.useSkill("helper", "SKILL.md", undefined, "org:example");
  assert.notEqual(a.dir, b.dir);
  assert.equal(a.dir, again.dir);
  const handle = await turn.provision();
  assert.equal(await router.readFile(handle, `${a.dir!.replace("/workspace/", "")}/value.py`), "org:example");
  assert.equal(await router.readFile(handle, `${b.dir!.replace("/workspace/", "")}/value.py`), "personal:alice");
});