Multiplayer agent harness for work.
Pause a shared computer only after the last turn using it releases it (#2179)
Regan Belldaeb2deb9038
diff --git a/src/core/orchestrator/sandboxes.ts b/src/core/orchestrator/sandboxes.tsindex aea3ee182..bfc1185d3 100644--- a/src/core/orchestrator/sandboxes.ts+++ b/src/core/orchestrator/sandboxes.ts@@ -6,6 +6,7 @@ import type { GapPhase } from "../../sessions/session-store.ts";import { type SandboxHandle, supportsProcessSessions, SandboxProvisionCleanupError } from "../../sandbox/sandbox.ts";import type { SandboxAccessPlan } from "../../sandbox/sandbox-resources.ts";import { reconcileProcesses } from "../../processes/reconcile.ts";+import { leaseLapsed } from "../../runs/run-store.ts";import { finishProcessCredentials } from "../../credentials/execute-files.ts";import {materializeSkillTree as laySkillTree,@@ -170,6 +171,18 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {const resourcePendingHandles = new Map<string, SandboxHandle>();const resourcePending = new Map<string, Promise<SandboxHandle>>();let provisionInFlight: Promise<SandboxHandle> | null = null;+ const holdResource = async (handle: SandboxHandle): Promise<void> => {+ if (handle.resourceId && deps.runs && input.runId)+ await deps.sandboxResources?.hold(handle.resourceId, input.runId);+ };+ const releaseHold = async (handle: SandboxHandle): Promise<boolean> => {+ const runs = deps.runs;+ if (!handle.resourceId || !runs || !deps.sandboxResources) return false;+ return deps.sandboxResources.release(handle.resourceId, input.runId, async (runId) => {+ const run = await runs.get(runId);+ return run?.status === "running" && !leaseLapsed(run, Date.now());+ });+ };const provision = (eager = false): Promise<SandboxHandle> => {if (!eager) box.used = true;provisionInFlight ??= doProvision(eager ? () => {} : emitGapWork, eager).catch((err) => {@@ -193,6 +206,7 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {...(onSandboxStatus ? { onStatus: onSandboxStatus } : {}),});box.pending = handle;+ await holdResource(handle);emit("provision", provisionStart, Date.now());box.provisionMs = Date.now() - provisionStart;deps.auditLog?.record({@@ -383,6 +397,7 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {...(egressToken ? { egressToken } : {}),});resourcePendingHandles.set(id, handle);+ await holdResource(handle);if (!crossScope) {await prepareTurnFiles(handle);}@@ -500,6 +515,7 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {},);reachBoxes.set(target, handle);+ await holdResource(handle);return handle;};const clearTurnFiles = async (handle: SandboxHandle): Promise<void> => {@@ -576,8 +592,8 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {reachBoxes.clear();await Promise.all(reachEntries.map(async ([target, h]) => {- let keepReachWarm = false;- if (deps.processes && supportsProcessSessions(deps.sandbox)) {+ let keepReachWarm = await releaseHold(h).catch(swallowAs("orchestrator: reach hold release", true));+ if (!keepReachWarm && deps.processes && supportsProcessSessions(deps.sandbox)) {try {keepReachWarm = await hasLiveProcesses(h, target);} catch (e) {@@ -662,9 +678,11 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {await clearTurnFiles(handle);} finally {await deps.sandbox.teardown(handle, {- keepWarm: await hasLiveProcesses(handle, memoryScopeId).catch(- swallowAs("orchestrator: live process check", true),- ),+ keepWarm:+ (await releaseHold(handle).catch(swallowAs("orchestrator: resource hold release", true))) ||+ (await hasLiveProcesses(handle, memoryScopeId).catch(+ swallowAs("orchestrator: live process check", true),+ )),});}})(),@@ -686,8 +704,8 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {return;}await clearTurnFiles(handle);- let keepWarm = false;- if (deps.processes && supportsProcessSessions(deps.sandbox)) {+ let keepWarm = await releaseHold(handle).catch(swallowAs("orchestrator: hold release", true));+ if (!keepWarm && deps.processes && supportsProcessSessions(deps.sandbox)) {try {keepWarm = await hasLiveProcesses(handle, memoryScopeId);} catch (e) {diff --git a/src/sandbox/sandbox-resources.ts b/src/sandbox/sandbox-resources.tsindex 1b5bebbc9..e267514f5 100644--- a/src/sandbox/sandbox-resources.ts+++ b/src/sandbox/sandbox-resources.ts@@ -19,6 +19,7 @@ export interface SandboxResource {availableActions?: string[];machineId?: string;cleanupPending?: boolean;+ holders?: string[];spec?: AgentComputerSpec;error?: string;}@@ -60,6 +61,8 @@ export interface SandboxResources {restart(actorId: string, id: string): Promise<void>;retire(actorId: string, id: string): Promise<void>;use<T>(id: string, action: () => Promise<T>, exclusive?: boolean): Promise<T>;+ hold(id: string, runId: string): Promise<void>;+ release(id: string, runId: string | undefined, isLive: (runId: string) => Promise<boolean>): Promise<boolean>;setDefault(actorId: string, scopeId: ScopeId, id: string | null): Promise<void>;resolve(scopeId: ScopeId): Promise<SandboxResource | null>;get(id: string): Promise<SandboxResource>;@@ -126,6 +129,28 @@ export function createSandboxResources(opts: {initialize,get,use,+ async hold(id, runId) {+ await opts.records.update!(id, (current) =>+ current.holders?.includes(runId) ? current : { ...current, holders: [...(current.holders ?? []), runId] },+ );+ },+ async release(id, runId, isLive) {+ const record = runId+ ? await opts.records.update!(id, (current) => ({+ ...current,+ holders: (current.holders ?? []).filter((holder) => holder !== runId),+ }))+ : await opts.records.get(id);+ const others = record?.holders ?? [];+ const live = await Promise.all(others.map(isLive));+ const ended = others.filter((_, index) => !live[index]);+ if (ended.length)+ await opts.records.update!(id, (current) => ({+ ...current,+ holders: (current.holders ?? []).filter((holder) => !ended.includes(holder)),+ }));+ return live.includes(true);+ },async retire(actorId, id) {await initialize();const record = await get(id);@@ -189,7 +214,7 @@ export function createSandboxResources(opts: {let availableActions = actionsFor(backend).filter((action) => action !== "create");if (record.state === "retired")availableActions = (record.cleanupPending || record.error) && backend?.destroyScope ? ["retire"] : [];- sandboxes.push({ ...record, availableActions });+ sandboxes.push({ ...record, holders: undefined, availableActions });}const providers = (Object.entries(opts.backends) as Array<[SandboxBackendName, Sandbox]>).filter(([, backend]) => !!backend)diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.tsindex 49b517730..bca3a9757 100644--- a/src/sandbox/sandbox-routing.ts+++ b/src/sandbox/sandbox-routing.ts@@ -161,7 +161,11 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {teardown(handle, tdOpts?: TeardownOptions): Promise<void> {const action = () => forHandle(handle).teardown(handle, tdOpts);return handle.resourceId- ? resources.use(handle.resourceId, action, !!tdOpts?.destroy || !!forHandle(handle).profile.parksOnTeardown)+ ? resources.use(+ handle.resourceId,+ action,+ !!tdOpts?.destroy || (!tdOpts?.keepWarm && !!forHandle(handle).profile.parksOnTeardown),+ ): action();},diff --git a/test/sandbox-resources.test.ts b/test/sandbox-resources.test.tsindex d896fdd8e..0ba157cf4 100644--- a/test/sandbox-resources.test.ts+++ b/test/sandbox-resources.test.ts@@ -15,6 +15,7 @@ import {} from "../src/sandbox/sandbox-resource-upgrade.ts";import { createMemoryMap } from "../src/persistence/durable-map.ts";import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";import type { Sandbox } from "../src/sandbox/sandbox.ts";function fixture(configure?: (backend: Sandbox) => void, legacyScopes = ["personal:alice"]) {@@ -837,3 +838,67 @@ test("legacy import does not validate obsolete provider choices after explicit sassert.deepEqual(await defaults.get("personal:selected"), { sandboxId: "newer-computer" });assert.ok(await marker.get("explicit-defaults"));});++test("a computer stays held until its last live run releases it, and ended runs stop counting", async () => {+ const { resources } = fixture();+ const record = await resources.create("alice", "personal:alice", "local");+ await resources.hold(record.id, "a");+ await resources.hold(record.id, "b");+ await resources.hold(record.id, "b");+ const live = new Set(["a", "b"]);+ const isLive = async (runId: string) => live.has(runId);+ assert.equal(await resources.release(record.id, "a", isLive), true);+ live.delete("b");+ assert.equal(await resources.release(record.id, undefined, isLive), false);+ assert.deepEqual((await resources.get(record.id)).holders, []);+ const listed = (await resources.list("alice", "personal:alice")).sandboxes.find((s) => s.id === record.id);+ assert.equal(listed?.holders, undefined);+});++test("two concurrent turns on one parking computer: only the last turn to finish parks it", async () => {+ const { options, backend, layers } = fixture();+ const parking: Sandbox = { ...backend, profile: { ...backend.profile, parksOnTeardown: true } };+ const resources = createSandboxResources({ ...options, backends: { e2b: parking }, defaultBackend: "e2b" });+ const router = createSandboxRouter({ backends: { e2b: parking }, defaultBackend: "e2b", resources });+ const record = await resources.create("alice", "personal:alice", "e2b");+ await resources.setDefault("alice", "personal:alice", record.id);+ const teardowns: unknown[] = [];+ parking.teardown = async (_handle, opts) => {+ teardowns.push(opts ?? {});+ };+ const { runs } = createMemoryRunStore();+ const start = async (sessionId: string) => {+ await runs.enqueue({ sessionId, request: { origin: { kind: "direct" }, text: "x" } as never });+ return (await runs.claim("worker", 60_000))!;+ };+ const turn = (runId: string, sessionId: string) =>+ createTurnSandboxes({+ deps: { sandbox: router, sandboxResources: resources, runs },+ input: { origin: { kind: "user" }, runId },+ actor: { id: "alice", type: "internal" },+ session: { id: sessionId },+ resolution: { layers },+ scopeId: "personal:alice",+ memoryScopeId: "personal:alice",+ transferId: runId,+ turnSessionDir: `turn/${sessionId}`,+ turnFilesDir: `turn/${sessionId}/${runId}`,+ connectorEnv: {},+ ownerAuthAvailable: false,+ visibleSkills: [],+ visibleSkillsForTurn: async () => [],+ emitGapWork: () => {},+ perf: { credsMs: 0 },+ } as unknown as TurnSandboxContext);+ const a = await start("sa");+ const b = await start("sb");+ const first = turn(a.id, "sa");+ const second = turn(b.id, "sb");+ await Promise.all([first.provision(), second.provision()]);+ await first.reclaimBox();+ await runs.complete(a.id, a.leaseToken!, { status: "ok", reply: "done" });+ assert.deepEqual(teardowns, [{ keepWarm: true }]);+ await second.reclaimBox();+ assert.deepEqual(teardowns, [{ keepWarm: true }, {}]);+ assert.deepEqual((await resources.get(record.id)).holders, []);+});diff --git a/test/sandbox-routing.test.ts b/test/sandbox-routing.test.tsindex 580620bfc..f9d719fe9 100644--- a/test/sandbox-routing.test.ts+++ b/test/sandbox-routing.test.ts@@ -339,6 +339,18 @@ function computerBackends() {return { aws, sprites };}+test("parking backends lock the computer exclusively only to park it, not to keep it warm", async () => {+ const { router, sprites, resources } = build({ "personal:a": resource("r1", "sprites") });+ sprites.profile.parksOnTeardown = true;+ const h = await router.provision(layersFor("personal:a"));+ await router.teardown(h, { keepWarm: true });+ await router.teardown(h);+ assert.deepEqual(resources.locks.slice(1), [+ { id: "r1", exclusive: false },+ { id: "r1", exclusive: true },+ ]);+});+test("computer status/restart target the resource's backing scope under its lock", async () => {const { aws, sprites } = computerBackends();const resources = stubResources({ "personal:s": resource("s", "sprites") });