Multiplayer agent harness for work.
Stop serializing every sandbox operation behind one machine lock (#1748)
Regan Bellc616b40fdd1d
diff --git a/src/sandbox/aws-sandbox.ts b/src/sandbox/aws-sandbox.tsindex fb1a2df1b..c67882a39 100644--- a/src/sandbox/aws-sandbox.ts+++ b/src/sandbox/aws-sandbox.ts@@ -321,6 +321,7 @@ export function createAwsSandbox(workspace: WorkspaceStore, opts: AwsSandboxOptibackend: "aws-microvm",writablePersistence: "snapshot_to_workspace",processSessions: true,+ parksOnTeardown: true,egressEnforcement: "none",spec: {os: "Amazon Linux 2023, glibc",diff --git a/src/sandbox/e2b-sandbox.ts b/src/sandbox/e2b-sandbox.tsindex ef1c31db4..08b45c02d 100644--- a/src/sandbox/e2b-sandbox.ts+++ b/src/sandbox/e2b-sandbox.ts@@ -303,6 +303,7 @@ export function createE2bSandbox(workspace: WorkspaceStore, opts: E2bSandboxOptibackend: "e2b",writablePersistence: client.nativePause ? "provider_managed" : "snapshot_to_workspace",processSessions: true,+ parksOnTeardown: true,egressEnforcement: opts.egressProxyUrl ? "domain" : "none",spec: {os: "Linux — E2B Firecracker sandbox (provider pause preserves state; publish durable work to git or Files)",diff --git a/src/sandbox/local-sandbox.ts b/src/sandbox/local-sandbox.tsindex 895b2c570..219b1d036 100644--- a/src/sandbox/local-sandbox.ts+++ b/src/sandbox/local-sandbox.ts@@ -345,6 +345,7 @@ export function createLocalSandbox(workspace: WorkspaceStore, opts: LocalSandboxbackend: "local-docker",writablePersistence: "resident_disk",processSessions: true,+ parksOnTeardown: true,egressEnforcement: "none",spec: {os: `Debian 12 (bookworm), glibc — local Docker container on a ${arch()} host`,diff --git a/src/sandbox/sandbox-resources.ts b/src/sandbox/sandbox-resources.tsindex 3af0c5c10..1555e7c30 100644--- a/src/sandbox/sandbox-resources.ts+++ b/src/sandbox/sandbox-resources.ts@@ -192,11 +192,8 @@ export function createSandboxResources(opts: {return record;};const use = async <T>(id: string, action: () => Promise<T>, exclusive = false): Promise<T> => {- const record = await get(id);- const lock =- exclusive || record.backend !== "modal" || !opts.lock.withSharedLock- ? opts.lock.withLock- : opts.lock.withSharedLock;+ await get(id);+ const lock = exclusive || !opts.lock.withSharedLock ? opts.lock.withLock : opts.lock.withSharedLock;return lock(`sandbox-resource:${id}`, async () => {const current = await get(id);if (current.state === "retired") throw new Error("sandbox has been retired");diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.tsindex 76f1a428d..9ef52c6da 100644--- a/src/sandbox/sandbox-routing.ts+++ b/src/sandbox/sandbox-routing.ts@@ -232,7 +232,11 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {teardown(handle, tdOpts?: TeardownOptions): Promise<void> {const action = () => forHandle(handle).teardown(handle, tdOpts);return handle.resourceId && opts.resources- ? opts.resources.use(handle.resourceId, action, handle.backend !== "modal" || !!tdOpts?.destroy)+ ? opts.resources.use(+ handle.resourceId,+ action,+ !!tdOpts?.destroy || !!forHandle(handle).profile.parksOnTeardown,+ ): action();},diff --git a/src/sandbox/sandbox.ts b/src/sandbox/sandbox.tsindex b2a6c8fe0..a3f2a7798 100644--- a/src/sandbox/sandbox.ts+++ b/src/sandbox/sandbox.ts@@ -62,6 +62,7 @@ export interface AgentComputerProfile {backend: string;writablePersistence: WritablePersistence;processSessions: boolean;+ parksOnTeardown?: boolean;egressEnforcement?: EgressEnforcement;spec?: AgentComputerSpec;}diff --git a/test/sandbox-resources.test.ts b/test/sandbox-resources.test.tsindex 316a9e3af..3263195de 100644--- a/test/sandbox-resources.test.ts+++ b/test/sandbox-resources.test.ts@@ -837,12 +837,17 @@ test("resource activation honors scope defaults and preserves explicit legacy prassert.equal(resources.defaultBackend("channel:new"), "sprites");});-for (const waiting of ["command", "checkpoint"] as const) {- test(`Modal ${waiting} allows other tools while restart waits`, { timeout: 10000 }, async () => {+for (const [kind, waiting] of [+ ["modal", "command"],+ ["modal", "checkpoint"],+ ["sprites", "command"],+ ["sprites", "checkpoint"],+] as const) {+ test(`${kind} ${waiting} allows other tools while restart waits`, { timeout: 10000 }, async () => {const { options, backend, layers, routes } = fixture();- const resources = createSandboxResources({ ...options, backends: { modal: backend }, defaultBackend: "modal" });- const router = createSandboxRouter({ routes, backends: { modal: backend }, defaultBackend: "modal", resources });- const record = await resources.create("alice", "personal:alice", "modal");+ const resources = createSandboxResources({ ...options, backends: { [kind]: backend }, defaultBackend: kind });+ const router = createSandboxRouter({ routes, backends: { [kind]: backend }, defaultBackend: kind, resources });+ const record = await resources.create("alice", "personal:alice", kind);await resources.setDefault("alice", "personal:alice", record.id);const handle = await router.provision(layers);const entered = Promise.withResolvers<void>();@@ -928,6 +933,61 @@ test("Modal provisioning and destructive cleanup wait for active operations", {assert.equal(destroyed, true);});+test("concurrent commands on one Sprites computer run together", { timeout: 10000 }, async () => {+ const { options, backend, layers, routes } = fixture();+ const resources = createSandboxResources({ ...options, backends: { sprites: backend }, defaultBackend: "sprites" });+ const router = createSandboxRouter({ routes, backends: { sprites: backend }, defaultBackend: "sprites", resources });+ const record = await resources.create("alice", "personal:alice", "sprites");+ const handle = await router.provision(layers, { sandboxId: record.id });+ const release = Promise.withResolvers<void>();+ let running = 0;+ let peak = 0;+ backend.run = async (_handle, command) => {+ peak = Math.max(peak, ++running);+ if (peak === 3) release.resolve();+ await release.promise;+ running--;+ return { stdout: command, stderr: "", code: 0, timedOut: false };+ };+ const results = await Promise.all(["a", "b", "c"].map((command) => router.run(handle, command)));+ assert.deepEqual(+ results.map((result) => result.stdout),+ ["a", "b", "c"],+ );+ assert.equal(peak, 3);+});++test("parking teardown waits for active commands on the computer", { timeout: 10000 }, async () => {+ const { options, backend, layers, routes } = fixture();+ const parking: Sandbox = { ...backend, profile: { ...backend.profile, parksOnTeardown: true } };+ const resources = createSandboxResources({ ...options, backends: { e2b: parking }, defaultBackend: "e2b" });+ const router = createSandboxRouter({ routes, backends: { e2b: parking }, defaultBackend: "e2b", resources });+ const record = await resources.create("alice", "personal:alice", "e2b");+ const handle = await router.provision(layers, { sandboxId: record.id });+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ parking.run = async () => {+ entered.resolve();+ await release.promise;+ return { stdout: "done", stderr: "", code: 0, timedOut: false };+ };+ let parked = false;+ parking.teardown = async () => {+ parked = true;+ };+ const running = router.run(handle, "long");+ await entered.promise;+ const teardown = router.teardown(handle);+ try {+ await new Promise<void>((resolve) => setImmediate(resolve));+ assert.equal(parked, false);+ } finally {+ release.resolve();+ await Promise.all([running, teardown]);+ }+ assert.equal(parked, true);+});+test("a verified live turn can create only its own new scope computer without directory mutations", async () => {const { resources, router } = fixture(undefined, []);const scope = "channel:external-slack:T1:policy:C1";