Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Pause a shared computer only after the last turn using it releases it (#2179)

Regan Belldaeb2deb9038
diff --git a/src/core/orchestrator/sandboxes.ts b/src/core/orchestrator/sandboxes.ts
index aea3ee182..bfc1185d3 100644
--- a/src/core/orchestrator/sandboxes.ts
+++ b/src/core/orchestrator/sandboxes.ts
@@ -6,6 +6,7 @@ import type { GapPhase } from "../../sessions/session-store.ts";
import { type SandboxHandle, supportsProcessSessions, SandboxProvisionCleanupError } from "../../sandbox/sandbox.ts";
import type { SandboxAccessPlan } from "../../sandbox/sandbox-resources.ts";
import { reconcileProcesses } from "../../processes/reconcile.ts";
+import { leaseLapsed } from "../../runs/run-store.ts";
import { finishProcessCredentials } from "../../credentials/execute-files.ts";
import {
materializeSkillTree as laySkillTree,
@@ -170,6 +171,18 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {
const resourcePendingHandles = new Map<string, SandboxHandle>();
const resourcePending = new Map<string, Promise<SandboxHandle>>();
let provisionInFlight: Promise<SandboxHandle> | null = null;
+ const holdResource = async (handle: SandboxHandle): Promise<void> => {
+ if (handle.resourceId && deps.runs && input.runId)
+ await deps.sandboxResources?.hold(handle.resourceId, input.runId);
+ };
+ const releaseHold = async (handle: SandboxHandle): Promise<boolean> => {
+ const runs = deps.runs;
+ if (!handle.resourceId || !runs || !deps.sandboxResources) return false;
+ return deps.sandboxResources.release(handle.resourceId, input.runId, async (runId) => {
+ const run = await runs.get(runId);
+ return run?.status === "running" && !leaseLapsed(run, Date.now());
+ });
+ };
const provision = (eager = false): Promise<SandboxHandle> => {
if (!eager) box.used = true;
provisionInFlight ??= doProvision(eager ? () => {} : emitGapWork, eager).catch((err) => {
@@ -193,6 +206,7 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {
...(onSandboxStatus ? { onStatus: onSandboxStatus } : {}),
});
box.pending = handle;
+ await holdResource(handle);
emit("provision", provisionStart, Date.now());
box.provisionMs = Date.now() - provisionStart;
deps.auditLog?.record({
@@ -383,6 +397,7 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {
...(egressToken ? { egressToken } : {}),
});
resourcePendingHandles.set(id, handle);
+ await holdResource(handle);
if (!crossScope) {
await prepareTurnFiles(handle);
}
@@ -500,6 +515,7 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {
},
);
reachBoxes.set(target, handle);
+ await holdResource(handle);
return handle;
};
const clearTurnFiles = async (handle: SandboxHandle): Promise<void> => {
@@ -576,8 +592,8 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {
reachBoxes.clear();
await Promise.all(
reachEntries.map(async ([target, h]) => {
- let keepReachWarm = false;
- if (deps.processes && supportsProcessSessions(deps.sandbox)) {
+ let keepReachWarm = await releaseHold(h).catch(swallowAs("orchestrator: reach hold release", true));
+ if (!keepReachWarm && deps.processes && supportsProcessSessions(deps.sandbox)) {
try {
keepReachWarm = await hasLiveProcesses(h, target);
} catch (e) {
@@ -662,9 +678,11 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {
await clearTurnFiles(handle);
} finally {
await deps.sandbox.teardown(handle, {
- keepWarm: await hasLiveProcesses(handle, memoryScopeId).catch(
- swallowAs("orchestrator: live process check", true),
- ),
+ keepWarm:
+ (await releaseHold(handle).catch(swallowAs("orchestrator: resource hold release", true))) ||
+ (await hasLiveProcesses(handle, memoryScopeId).catch(
+ swallowAs("orchestrator: live process check", true),
+ )),
});
}
})(),
@@ -686,8 +704,8 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) {
return;
}
await clearTurnFiles(handle);
- let keepWarm = false;
- if (deps.processes && supportsProcessSessions(deps.sandbox)) {
+ let keepWarm = await releaseHold(handle).catch(swallowAs("orchestrator: hold release", true));
+ if (!keepWarm && deps.processes && supportsProcessSessions(deps.sandbox)) {
try {
keepWarm = await hasLiveProcesses(handle, memoryScopeId);
} catch (e) {
diff --git a/src/sandbox/sandbox-resources.ts b/src/sandbox/sandbox-resources.ts
index 1b5bebbc9..e267514f5 100644
--- a/src/sandbox/sandbox-resources.ts
+++ b/src/sandbox/sandbox-resources.ts
@@ -19,6 +19,7 @@ export interface SandboxResource {
availableActions?: string[];
machineId?: string;
cleanupPending?: boolean;
+ holders?: string[];
spec?: AgentComputerSpec;
error?: string;
}
@@ -60,6 +61,8 @@ export interface SandboxResources {
restart(actorId: string, id: string): Promise<void>;
retire(actorId: string, id: string): Promise<void>;
use<T>(id: string, action: () => Promise<T>, exclusive?: boolean): Promise<T>;
+ hold(id: string, runId: string): Promise<void>;
+ release(id: string, runId: string | undefined, isLive: (runId: string) => Promise<boolean>): Promise<boolean>;
setDefault(actorId: string, scopeId: ScopeId, id: string | null): Promise<void>;
resolve(scopeId: ScopeId): Promise<SandboxResource | null>;
get(id: string): Promise<SandboxResource>;
@@ -126,6 +129,28 @@ export function createSandboxResources(opts: {
initialize,
get,
use,
+ async hold(id, runId) {
+ await opts.records.update!(id, (current) =>
+ current.holders?.includes(runId) ? current : { ...current, holders: [...(current.holders ?? []), runId] },
+ );
+ },
+ async release(id, runId, isLive) {
+ const record = runId
+ ? await opts.records.update!(id, (current) => ({
+ ...current,
+ holders: (current.holders ?? []).filter((holder) => holder !== runId),
+ }))
+ : await opts.records.get(id);
+ const others = record?.holders ?? [];
+ const live = await Promise.all(others.map(isLive));
+ const ended = others.filter((_, index) => !live[index]);
+ if (ended.length)
+ await opts.records.update!(id, (current) => ({
+ ...current,
+ holders: (current.holders ?? []).filter((holder) => !ended.includes(holder)),
+ }));
+ return live.includes(true);
+ },
async retire(actorId, id) {
await initialize();
const record = await get(id);
@@ -189,7 +214,7 @@ export function createSandboxResources(opts: {
let availableActions = actionsFor(backend).filter((action) => action !== "create");
if (record.state === "retired")
availableActions = (record.cleanupPending || record.error) && backend?.destroyScope ? ["retire"] : [];
- sandboxes.push({ ...record, availableActions });
+ sandboxes.push({ ...record, holders: undefined, availableActions });
}
const providers = (Object.entries(opts.backends) as Array<[SandboxBackendName, Sandbox]>)
.filter(([, backend]) => !!backend)
diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.ts
index 49b517730..bca3a9757 100644
--- a/src/sandbox/sandbox-routing.ts
+++ b/src/sandbox/sandbox-routing.ts
@@ -161,7 +161,11 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {
teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
const action = () => forHandle(handle).teardown(handle, tdOpts);
return handle.resourceId
- ? resources.use(handle.resourceId, action, !!tdOpts?.destroy || !!forHandle(handle).profile.parksOnTeardown)
+ ? resources.use(
+ handle.resourceId,
+ action,
+ !!tdOpts?.destroy || (!tdOpts?.keepWarm && !!forHandle(handle).profile.parksOnTeardown),
+ )
: action();
},
diff --git a/test/sandbox-resources.test.ts b/test/sandbox-resources.test.ts
index d896fdd8e..0ba157cf4 100644
--- a/test/sandbox-resources.test.ts
+++ b/test/sandbox-resources.test.ts
@@ -15,6 +15,7 @@ import {
} from "../src/sandbox/sandbox-resource-upgrade.ts";
import { createMemoryMap } from "../src/persistence/durable-map.ts";
import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";
+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
import type { Sandbox } from "../src/sandbox/sandbox.ts";
function fixture(configure?: (backend: Sandbox) => void, legacyScopes = ["personal:alice"]) {
@@ -837,3 +838,67 @@ test("legacy import does not validate obsolete provider choices after explicit s
assert.deepEqual(await defaults.get("personal:selected"), { sandboxId: "newer-computer" });
assert.ok(await marker.get("explicit-defaults"));
});
+
+test("a computer stays held until its last live run releases it, and ended runs stop counting", async () => {
+ const { resources } = fixture();
+ const record = await resources.create("alice", "personal:alice", "local");
+ await resources.hold(record.id, "a");
+ await resources.hold(record.id, "b");
+ await resources.hold(record.id, "b");
+ const live = new Set(["a", "b"]);
+ const isLive = async (runId: string) => live.has(runId);
+ assert.equal(await resources.release(record.id, "a", isLive), true);
+ live.delete("b");
+ assert.equal(await resources.release(record.id, undefined, isLive), false);
+ assert.deepEqual((await resources.get(record.id)).holders, []);
+ const listed = (await resources.list("alice", "personal:alice")).sandboxes.find((s) => s.id === record.id);
+ assert.equal(listed?.holders, undefined);
+});
+
+test("two concurrent turns on one parking computer: only the last turn to finish parks it", async () => {
+ const { options, backend, layers } = fixture();
+ const parking: Sandbox = { ...backend, profile: { ...backend.profile, parksOnTeardown: true } };
+ const resources = createSandboxResources({ ...options, backends: { e2b: parking }, defaultBackend: "e2b" });
+ const router = createSandboxRouter({ backends: { e2b: parking }, defaultBackend: "e2b", resources });
+ const record = await resources.create("alice", "personal:alice", "e2b");
+ await resources.setDefault("alice", "personal:alice", record.id);
+ const teardowns: unknown[] = [];
+ parking.teardown = async (_handle, opts) => {
+ teardowns.push(opts ?? {});
+ };
+ const { runs } = createMemoryRunStore();
+ const start = async (sessionId: string) => {
+ await runs.enqueue({ sessionId, request: { origin: { kind: "direct" }, text: "x" } as never });
+ return (await runs.claim("worker", 60_000))!;
+ };
+ const turn = (runId: string, sessionId: string) =>
+ createTurnSandboxes({
+ deps: { sandbox: router, sandboxResources: resources, runs },
+ input: { origin: { kind: "user" }, runId },
+ actor: { id: "alice", type: "internal" },
+ session: { id: sessionId },
+ resolution: { layers },
+ scopeId: "personal:alice",
+ memoryScopeId: "personal:alice",
+ transferId: runId,
+ turnSessionDir: `turn/${sessionId}`,
+ turnFilesDir: `turn/${sessionId}/${runId}`,
+ connectorEnv: {},
+ ownerAuthAvailable: false,
+ visibleSkills: [],
+ visibleSkillsForTurn: async () => [],
+ emitGapWork: () => {},
+ perf: { credsMs: 0 },
+ } as unknown as TurnSandboxContext);
+ const a = await start("sa");
+ const b = await start("sb");
+ const first = turn(a.id, "sa");
+ const second = turn(b.id, "sb");
+ await Promise.all([first.provision(), second.provision()]);
+ await first.reclaimBox();
+ await runs.complete(a.id, a.leaseToken!, { status: "ok", reply: "done" });
+ assert.deepEqual(teardowns, [{ keepWarm: true }]);
+ await second.reclaimBox();
+ assert.deepEqual(teardowns, [{ keepWarm: true }, {}]);
+ assert.deepEqual((await resources.get(record.id)).holders, []);
+});
diff --git a/test/sandbox-routing.test.ts b/test/sandbox-routing.test.ts
index 580620bfc..f9d719fe9 100644
--- a/test/sandbox-routing.test.ts
+++ b/test/sandbox-routing.test.ts
@@ -339,6 +339,18 @@ function computerBackends() {
return { aws, sprites };
}
+test("parking backends lock the computer exclusively only to park it, not to keep it warm", async () => {
+ const { router, sprites, resources } = build({ "personal:a": resource("r1", "sprites") });
+ sprites.profile.parksOnTeardown = true;
+ const h = await router.provision(layersFor("personal:a"));
+ await router.teardown(h, { keepWarm: true });
+ await router.teardown(h);
+ assert.deepEqual(resources.locks.slice(1), [
+ { id: "r1", exclusive: false },
+ { id: "r1", exclusive: true },
+ ]);
+});
+
test("computer status/restart target the resource's backing scope under its lock", async () => {
const { aws, sprites } = computerBackends();
const resources = stubResources({ "personal:s": resource("s", "sprites") });