Multiplayer agent harness for work.
Keep Modal commands responsive during checkpoint saves (#1401)
Regan Bellf97638f55dfa
diff --git a/src/persistence/advisory-lock.ts b/src/persistence/advisory-lock.tsindex 22e81ec30..0f600cc87 100644--- a/src/persistence/advisory-lock.ts+++ b/src/persistence/advisory-lock.ts@@ -1,8 +1,10 @@-import type { PgPool } from "./pg-pool.ts";-import { createKeyedQueue, sleep } from "../util/async.ts";+import { AsyncLocalStorage } from "node:async_hooks";+import type { PgPool, PoolClient } from "./pg-pool.ts";+import { sleep } from "../util/async.ts";export interface AdvisoryLock {withLock<T>(key: string, fn: () => Promise<T>): Promise<T>;+ withSharedLock?<T>(key: string, fn: () => Promise<T>): Promise<T>;tryWithLock?<T>(key: string, fn: () => Promise<T>): Promise<T | null>;}@@ -14,6 +16,9 @@ export function createNoopAdvisoryLock(): AdvisoryLock {async withLock<T>(_key: string, fn: () => Promise<T>): Promise<T> {return fn();},+ async withSharedLock<T>(_key: string, fn: () => Promise<T>): Promise<T> {+ return fn();+ },async tryWithLock<T>(_key: string, fn: () => Promise<T>): Promise<T | null> {return fn();},@@ -21,22 +26,34 @@ export function createNoopAdvisoryLock(): AdvisoryLock {}export function createMemoryAdvisoryLock(): AdvisoryLock {- const queue = createKeyedQueue<string>();- const held = new Set<string>();- const withLock = <T>(key: string, fn: () => Promise<T>): Promise<T> =>- queue(key, async () => {- held.add(key);- try {- return await fn();- } finally {- held.delete(key);- }- });+ const states = new Map<string, { tail: Promise<void>; readers: Set<Promise<void>>; pending: number }>();+ const run = async <T>(key: string, fn: () => Promise<T>, shared: boolean): Promise<T> => {+ let state = states.get(key);+ if (!state) {+ state = { tail: Promise.resolve(), readers: new Set(), pending: 0 };+ states.set(key, state);+ }+ const done = Promise.withResolvers<void>();+ const before = shared ? state.tail : Promise.all([state.tail, ...state.readers]);+ state.pending++;+ if (shared) state.readers.add(done.promise);+ else state.tail = done.promise;+ try {+ await before;+ return await fn();+ } finally {+ state.readers.delete(done.promise);+ state.pending--;+ done.resolve();+ if (!state.pending) states.delete(key);+ }+ };return {- withLock,+ withLock: (key, fn) => run(key, fn, false),+ withSharedLock: (key, fn) => run(key, fn, true),async tryWithLock(key, fn) {- if (held.has(key)) return null;- return withLock(key, fn);+ if (states.has(key)) return null;+ return run(key, fn, false);},};}@@ -48,50 +65,60 @@ export function createPostgresAdvisoryLock(const timeoutMs = opts.timeoutMs ?? DEFAULT_ADVISORY_LOCK_TIMEOUT_MS;const pollMs = opts.pollMs ?? DEFAULT_ADVISORY_LOCK_POLL_MS;- return {- async withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {- const deadline = Date.now() + timeoutMs;- const pool = await pg.sessionPool();- for (;;) {- const client = await pool.connect();+ type Context = {+ client: PoolClient;+ references: number;+ keys: Map<string, { shared: boolean; count: number }>;+ };+ const sessions = new AsyncLocalStorage<{ context: Context; active: boolean }>();+ const withClient = async <T>(action: (context: Context) => Promise<T>): Promise<T> => {+ const parent = sessions.getStore();+ const context = parent?.active+ ? parent.context+ : { client: await (await pg.sessionPool()).connect(), references: 0, keys: new Map() };+ context.references++;+ const lease = { context, active: true };+ try {+ return await sessions.run(lease, () => action(context));+ } finally {+ lease.active = false;+ if (--context.references === 0) context.client.release();+ }+ };+ const run = async <T>(key: string, fn: () => Promise<T>, shared: boolean, wait: boolean): Promise<T | null> => {+ const deadline = Date.now() + timeoutMs;+ for (;;) {+ const attempt = await withClient(async (context): Promise<{ acquired: false } | { acquired: true; value: T }> => {+ const { client, keys } = context;+ const held = keys.get(key);+ if (held && !(shared && held.shared)) return { acquired: false };+ const reservation = held ?? { shared, count: 0 };+ reservation.count++;+ keys.set(key, reservation);try {const res = await client.query<{ locked: boolean }>(- "SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked",+ `SELECT pg_try_advisory_lock${shared ? "_shared" : ""}(hashtextextended($1, 0)) AS locked`,[key],);- const held = res.rows[0]?.locked === true;- if (held) {- try {- return await fn();- } finally {- await client.query("SELECT pg_advisory_unlock(hashtextextended($1, 0))", [key]);- }+ if (res.rows[0]?.locked !== true) return { acquired: false };+ try {+ return { acquired: true, value: await fn() };+ } finally {+ await client.query(`SELECT pg_advisory_unlock${shared ? "_shared" : ""}(hashtextextended($1, 0))`, [key]);}} finally {- client.release();- }- if (Date.now() >= deadline) throw new Error(`timeout acquiring advisory lock for ${key}`);- await sleep(pollMs);- }- },-- async tryWithLock<T>(key: string, fn: () => Promise<T>): Promise<T | null> {- const pool = await pg.sessionPool();- const client = await pool.connect();- try {- const res = await client.query<{ locked: boolean }>(- "SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked",- [key],- );- if (res.rows[0]?.locked !== true) return null;- try {- return await fn();- } finally {- await client.query("SELECT pg_advisory_unlock(hashtextextended($1, 0))", [key]);+ if (--reservation.count === 0) keys.delete(key);}- } finally {- client.release();- }- },+ });+ if (attempt.acquired) return attempt.value;+ if (!wait) return null;+ if (Date.now() >= deadline) throw new Error(`timeout acquiring advisory lock for ${key}`);+ await sleep(pollMs);+ }+ };+ return {+ withLock: <T>(key: string, fn: () => Promise<T>) => run(key, fn, false, true) as Promise<T>,+ withSharedLock: <T>(key: string, fn: () => Promise<T>) => run(key, fn, true, true) as Promise<T>,+ tryWithLock: <T>(key: string, fn: () => Promise<T>) => run(key, fn, false, false),};}diff --git a/src/sandbox/modal-sandbox.ts b/src/sandbox/modal-sandbox.tsindex 698566603..b32ada11c 100644--- a/src/sandbox/modal-sandbox.ts+++ b/src/sandbox/modal-sandbox.ts@@ -1,6 +1,7 @@import { randomUUID } from "node:crypto";import type { WorkspaceLayer } from "../types.ts";import type { WorkspaceStore } from "../workspace/workspace-store.ts";+import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";import type { DurableMap } from "../persistence/durable-map.ts";import { createMemoryMap } from "../persistence/durable-map.ts";import { orgId as configOrgId } from "../config.ts";@@ -68,6 +69,7 @@ export interface StoredModalSandbox {snapshotGeneration?: number;createdAtMs: number;lastSnapshotMs?: number;+ lastSnapshotAttemptMs?: number;lastActivityMs?: number;homeDirty?: boolean;orgId?: string;@@ -75,6 +77,7 @@ export interface StoredModalSandbox {export interface ModalSandboxOptions extends BlobStagingOptions {client: ModalClient;+ advisoryLock?: AdvisoryLock;namePrefix?: string;defaultTimeoutSec?: number;snapshotIntervalMs?: number;@@ -102,6 +105,8 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox!!stored?.nativeSnapshotId || !!(client.nativeSnapshots && opts.nativeSnapshotsEnabled);const snapshotIntervalMs = (stored?: StoredModalSandbox | null): number =>usesNativeSnapshots(stored) ? (opts.nativeSnapshotIntervalMs ?? 5 * 60_000) : (opts.snapshotIntervalMs ?? 0);+ const lastSnapshotAttemptMs = (stored?: StoredModalSandbox | null): number =>+ Math.max(stored?.lastSnapshotMs ?? 0, usesNativeSnapshots(stored) ? (stored?.lastSnapshotAttemptMs ?? 0) : 0);const rotateAfterMs = opts.rotateAfterMs ?? 20 * 3600_000;const reapIdleMs = opts.reapIdleMs ?? 6 * 3600_000;const fileChunkBytes = opts.fileChunkBytes ?? 64 * 1024 * 1024;@@ -110,7 +115,10 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxconst workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`;const store = opts.store ?? createMemoryMap<StoredModalSandbox>();const snapshots = opts.snapshots ?? createMemorySnapshotStore();- const provisionQueue = createKeyedQueue<string>();+ const localQueue = createKeyedQueue<string>();+ const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock();+ const provisionQueue = <T>(scope: string, action: () => Promise<T>): Promise<T> =>+ localQueue(scope, () => advisoryLock.withLock(`modal-provision:${scope}`, action));const sessionByName = new Map<string, ModalSession>();const scopeByName = new Map<string, string>();@@ -150,7 +158,11 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxif (!store.update) throw new Error("native Modal checkpoints require an atomic durable store");const requested = await store.update(scope, (current) => {if (current.sandboxId !== session.sandboxId) throw new Error("Modal checkpoint source has been replaced");- return { ...current, snapshotGeneration: (current.snapshotGeneration ?? 0) + 1 };+ return {+ ...current,+ snapshotGeneration: (current.snapshotGeneration ?? 0) + 1,+ lastSnapshotAttemptMs: Date.now(),+ };});if (!requested) throw new Error("Modal checkpoint source is no longer tracked");const capturedAtMs = Date.now();@@ -163,6 +175,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxnativeSnapshotId: snapshot.imageId,nativeSnapshotExpiresAtMs: snapshot.expiresAtMs,recoveryError: undefined,+ lastSnapshotAttemptMs: undefined,lastSnapshotMs: capturedAtMs,homeDirty: false,}@@ -217,6 +230,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxawait store.put(scope, {...previous,hydrationPending: true,+ lastSnapshotAttemptMs: undefined,...(client.lifetimeMs ? { expiresAtMs: Date.now() + client.lifetimeMs } : {}),sandboxId: session.sandboxId,createdAtMs: Date.now(),@@ -263,6 +277,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxscope: string,name: string,onStatus?: (text: string) => void,+ allowRotation = true,): Promise<{ session: ModalSession; coldStart: boolean }> {return provisionQueue(scope, async () => {const adopt = async (session: ModalSession): Promise<{ session: ModalSession; coldStart: boolean }> => {@@ -278,6 +293,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox"Modal home hydration was interrupted; use computer restart to discard the incomplete replacement and retry the retained checkpoint",);const stale =+ allowRotation &&!!stored &&Date.now() - stored.createdAtMs > rotateAfterMs &&Date.now() >= (rotationHoldUntil.get(scope) ?? 0);@@ -372,7 +388,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox});}- async function withSession<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {+ async function withSessionUnlocked<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {const scratchKey = scratchKeyByName.get(name);const reviveScratch = async (): Promise<ModalSession> => {const session = await client.create({});@@ -382,7 +398,11 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxconst first =scratchKey !== undefined? { session: sessionByName.get(name) ?? (await reviveScratch()) }- : await ensureSession(scopeByName.get(name) ?? "default", name);+ : {+ session:+ sessionByName.get(name) ??+ (await ensureSession(scopeByName.get(name) ?? "default", name, undefined, false)).session,+ };try {return await action(first.session);} catch (err) {@@ -391,11 +411,17 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxconst second =scratchKey !== undefined? { session: await reviveScratch() }- : await ensureSession(scopeByName.get(name) ?? "default", name);+ : await ensureSession(scopeByName.get(name) ?? "default", name, undefined, false);return action(second.session);}}+ function withSession<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {+ if (scratchKeyByName.has(name)) return withSessionUnlocked(name, action);+ const shared = advisoryLock.withSharedLock ?? advisoryLock.withLock;+ return shared(`modal-use:${scopeByName.get(name) ?? "default"}`, () => withSessionUnlocked(name, action));+ }+const lastTouchMs = new Map<string, number>();async function touchActivity(name: string): Promise<void> {const scope = scopeByName.get(name);@@ -745,7 +771,9 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxif (!session) return;const stored = await store.get(scope);if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });- if (snapshotDue(stored, tdOpts, snapshotIntervalMs(stored))) {+ if (+ snapshotDue({ ...stored, lastSnapshotMs: lastSnapshotAttemptMs(stored) }, tdOpts, snapshotIntervalMs(stored))+ ) {try {await snapshotHome(scope, session);await store.merge(scope, { lastActivityMs: Date.now() });@@ -769,10 +797,17 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandboxif (usesNativeSnapshots(rec) &&(!rec.expiresAtMs || rec.expiresAtMs > Date.now()) &&- (!rec.lastSnapshotMs || Date.now() - rec.lastSnapshotMs > snapshotIntervalMs(rec))+ Date.now() - lastSnapshotAttemptMs(rec) > snapshotIntervalMs(rec)) {await provisionQueue(scope, async () => {try {+ const current = await store.get(scope);+ if (+ !current ||+ current.sandboxId !== rec.sandboxId ||+ Date.now() - lastSnapshotAttemptMs(current) <= snapshotIntervalMs(current)+ )+ return;const session = await client.fromId(rec.sandboxId);await snapshotHome(scope, session);} catch (error) {@@ -799,26 +834,28 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox}continue;}- reaped += await provisionQueue(scope, async (): Promise<number> => {- const current = await store.get(scope);- if (!current || current.sandboxId !== rec.sandboxId) return 0;- if (!current.lastActivityMs || current.lastActivityMs > cutoff) return 0;- try {- await snapshotHome(scope, session);- await session.terminate();- sessionByName.delete(name);- if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope);- return 1;- } catch (e) {- if (e instanceof ModalSandboxGoneError) {+ reaped += await advisoryLock.withLock(`modal-use:${scope}`, () =>+ provisionQueue(scope, async (): Promise<number> => {+ const current = await store.get(scope);+ if (!current || current.sandboxId !== rec.sandboxId) return 0;+ if (!current.lastActivityMs || current.lastActivityMs > cutoff) return 0;+ try {+ await snapshotHome(scope, session);+ await session.terminate();sessionByName.delete(name);- if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope).catch(() => undefined);- } else {- reportError("sandbox_reap", "deep_idle_reap_failed", errMessage(e), scope);+ if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope);+ return 1;+ } catch (e) {+ if (e instanceof ModalSandboxGoneError) {+ sessionByName.delete(name);+ if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope).catch(() => undefined);+ } else {+ reportError("sandbox_reap", "deep_idle_reap_failed", errMessage(e), scope);+ }+ return 0;}- return 0;- }- });+ }),+ );}return { reaped };},diff --git a/src/sandbox/sandbox-resources.ts b/src/sandbox/sandbox-resources.tsindex c5a39e3ee..b6462de88 100644--- a/src/sandbox/sandbox-resources.ts+++ b/src/sandbox/sandbox-resources.ts@@ -63,7 +63,7 @@ export interface SandboxResources {status(actorId: string, id: string): Promise<ComputerStatus>;restart(actorId: string, id: string): Promise<void>;retire(actorId: string, id: string): Promise<void>;- use<T>(id: string, action: () => Promise<T>): Promise<T>;+ use<T>(id: string, action: () => Promise<T>, exclusive?: boolean): Promise<T>;setDefault(actorId: string, scopeId: ScopeId, id: string | null): Promise<void>;resolve(scopeId: ScopeId): Promise<SandboxResource | null | undefined>;get(id: string): Promise<SandboxResource>;@@ -180,12 +180,18 @@ export function createSandboxResources(opts: {if (!record) throw new Error(`sandbox not found: ${id}`);return record;};- const use = <T>(id: string, action: () => Promise<T>): Promise<T> =>- opts.lock.withLock(`sandbox-resource:${id}`, async () => {- const record = await get(id);- if (record.state === "retired") throw new Error("sandbox has been retired");+ const use = async <T>(id: string, action: () => Promise<T>, exclusive = false): Promise<T> => {+ const record = await get(id);+ const lock =+ exclusive || record.backend !== "modal" || !opts.lock.withSharedLock+ ? opts.lock.withLock+ : opts.lock.withSharedLock;+ return lock(`sandbox-resource:${id}`, async () => {+ const current = await get(id);+ if (current.state === "retired") throw new Error("sandbox has been retired");return action();});+ };const recordLegacy = async (scopeId: string, backend: SandboxBackendName, handle: SandboxHandle): Promise<string> => {const id = legacyId(scopeId, backend);await opts.records.putIfAbsent(id, {@@ -269,7 +275,7 @@ export function createSandboxResources(opts: {await authorize(actorId, record.ownerScopeId);const backend = opts.backends[record.backend];if (!backend?.restartComputer) throw new Error(`sandbox restart unavailable: ${record.backend}`);- await use(id, () => backend.restartComputer!(record.backingScopeId));+ await use(id, () => backend.restartComputer!(record.backingScopeId), true);},async resolve(scopeId) {await initialize();diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.tsindex 06e62e786..0f23279b9 100644--- a/src/sandbox/sandbox-routing.ts+++ b/src/sandbox/sandbox-routing.ts@@ -170,7 +170,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {const routedLayers = layers.map((layer) =>layer.mode === "rw" ? { ...layer, scopeId: resource.backingScopeId } : layer,);- const handle = await opts.resources!.use(resource.id, () => sandbox.provision(routedLayers, provOpts));+ const handle = await opts.resources!.use(resource.id, () => sandbox.provision(routedLayers, provOpts), true);return { ...handle, backend: resource.backend, scopeId: resource.ownerScopeId, resourceId: resource.id };}const { name, sandbox } = await pick(scope);@@ -214,7 +214,10 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {});},teardown(handle, tdOpts?: TeardownOptions): Promise<void> {- return useHandle(handle, () => forHandle(handle).teardown(handle, tdOpts));+ const action = () => forHandle(handle).teardown(handle, tdOpts);+ return handle.resourceId && opts.resources+ ? opts.resources.use(handle.resourceId, action, handle.backend !== "modal" || !!tdOpts?.destroy)+ : action();},...(some(supportsProcessSessions)@@ -286,7 +289,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {restartComputer: async (scopeId: string) => {const target = await computerTarget(scopeId);const action = () => requireCap(target.sandbox, "restartComputer", scopeId).restartComputer(target.scopeId);- return target.resourceId && opts.resources ? opts.resources.use(target.resourceId, action) : action();+ return target.resourceId && opts.resources ? opts.resources.use(target.resourceId, action, true) : action();},}: {}),diff --git a/src/wiring.ts b/src/wiring.tsindex c3dde7f93..1a6599316 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -882,6 +882,7 @@ export function buildApp(if (!modal.tokenId || !modal.tokenSecret)throw new Error("SANDBOX_BACKEND=modal requires MODAL_TOKEN_ID and MODAL_TOKEN_SECRET");return createModalSandbox(workspace, {+ advisoryLock,client: createSdkModalClient({tokenId: modal.tokenId,tokenSecret: modal.tokenSecret,diff --git a/test/advisory-lock.test.ts b/test/advisory-lock.test.tsindex 359bfc15c..98b968c5c 100644--- a/test/advisory-lock.test.ts+++ b/test/advisory-lock.test.ts@@ -178,3 +178,174 @@ test("pg sandbox activation fences publication from a separate compatible readerawait pgB.close();}});++test(+ "pg shared sandbox use overlaps across cores and excludes lifecycle mutations",+ { skip, timeout: 10000 },+ async () => {+ const pgA = createPgPool(URL!);+ const pgB = createPgPool(URL!);+ const pgWriter = createPgPool(URL!);+ const release = Promise.withResolvers<void>();+ const entered = Promise.withResolvers<void>();+ const key = `sandbox-shared:${crypto.randomUUID()}`;+ const a = createPostgresAdvisoryLock(pgA, { pollMs: 10 });+ const b = createPostgresAdvisoryLock(pgB, { pollMs: 10, timeoutMs: 200 });+ const writer = createPostgresAdvisoryLock(pgWriter, { pollMs: 10, timeoutMs: 100 });+ const running = a.withSharedLock!(key, async () => {+ entered.resolve();+ await release.promise;+ });+ try {+ await entered.promise;+ assert.equal(await b.withSharedLock!(key, async () => "concurrent"), "concurrent");+ await assert.rejects(+ writer.withLock(key, async () => assert.fail("mutated active sandbox")),+ /timeout acquiring/,+ );+ assert.equal(await writer.tryWithLock!(key, async () => "exclusive"), null);+ release.resolve();+ await running;+ assert.equal(await writer.withLock(key, async () => "exclusive"), "exclusive");+ await assert.rejects(+ b.withSharedLock!(key, async () => {+ throw new Error("failed operation");+ }),+ /failed operation/,+ );+ assert.equal(await writer.tryWithLock!(key, async () => "released"), "released");+ await writer.withLock(key, async () => {+ await assert.rejects(+ b.withSharedLock!(key, async () => assert.fail("entered during mutation")),+ /timeout acquiring/,+ );+ });+ } finally {+ release.resolve();+ await running;+ await Promise.all([pgA.close(), pgB.close(), pgWriter.close()]);+ }+ },+);++test("pg nested locks reuse one connection without bypassing sibling exclusion", { skip, timeout: 10000 }, async () => {+ const { Pool } = await import("pg");+ const pool = new Pool({ connectionString: URL, max: 1, connectionTimeoutMillis: 200 });+ const pg = createPgPool(URL!);+ const lock = createPostgresAdvisoryLock({ ...pg, sessionPool: async () => pool }, { pollMs: 5, timeoutMs: 100 });+ try {+ assert.equal(+ await lock.withSharedLock!("resource", () =>+ lock.withSharedLock!("use", () => lock.withLock("recover", async () => 42)),+ ),+ 42,+ );+ await lock.withLock("parent", async () => {+ let active = 0;+ let maximum = 0;+ const action = () =>+ lock.withLock("writer", async () => {+ maximum = Math.max(maximum, ++active);+ await sleep(15);+ active--;+ });+ await Promise.all([action(), action(), action()]);+ assert.equal(maximum, 1);+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ let writing = false;+ const reader = lock.withSharedLock!("mixed", async () => {+ entered.resolve();+ await release.promise;+ });+ await entered.promise;+ assert.equal(await lock.tryWithLock!("mixed", async () => true), null);+ const writer = lock.withLock("mixed", async () => {+ writing = true;+ });+ await sleep(15);+ assert.equal(writing, false);+ release.resolve();+ await Promise.all([reader, writer]);+ await assert.rejects(+ lock.withLock("parent", async () => true),+ /timeout acquiring/,+ );+ await assert.rejects(+ lock.withLock("throws", async () => {+ throw new Error("callback failure");+ }),+ /callback failure/,+ );+ assert.equal(await lock.withLock("throws", async () => true), true);+ });+ assert.equal(pool.idleCount, 1);+ } finally {+ await pool.end();+ await pg.close();+ }+});++test(+ "pg nested lock leases outlive their parent and late inherited work leases anew",+ { skip, timeout: 10000 },+ async () => {+ const { Pool } = await import("pg");+ const pool = new Pool({ connectionString: URL, max: 1, connectionTimeoutMillis: 200 });+ const pg = createPgPool(URL!);+ const lock = createPostgresAdvisoryLock({ ...pg, sessionPool: async () => pool }, { pollMs: 5 });+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const late = Promise.withResolvers<void>();+ let child!: Promise<void>;+ let delayed!: Promise<boolean>;+ try {+ await lock.withLock("root", async () => {+ child = lock.withLock("child", async () => {+ entered.resolve();+ await release.promise;+ assert.equal(await lock.withLock("grandchild", async () => true), true);+ });+ delayed = late.promise.then(() => lock.withLock("late", async () => true));+ await entered.promise;+ });+ assert.equal(pool.idleCount, 0);+ release.resolve();+ await child;+ assert.equal(pool.idleCount, 1);+ late.resolve();+ assert.equal(await delayed, true);+ assert.equal(pool.idleCount, 1);+ } finally {+ release.resolve();+ late.resolve();+ await pool.end();+ await pg.close();+ }+ },+);++test("pg contended polling releases connections for unrelated keys", { skip, timeout: 10000 }, async () => {+ const { Pool } = await import("pg");+ const pool = new Pool({ connectionString: URL, max: 2, connectionTimeoutMillis: 100 });+ const pg = createPgPool(URL!);+ const lock = createPostgresAdvisoryLock({ ...pg, sessionPool: async () => pool }, { pollMs: 5 });+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const holding = lock.withLock("occupied", async () => {+ entered.resolve();+ await release.promise;+ });+ await entered.promise;+ const waiting = lock.withLock("occupied", async () => true);+ try {+ await sleep(20);+ assert.equal(await lock.withLock("unrelated", async () => true), true);+ assert.equal(await lock.tryWithLock!("unrelated", async () => true), true);+ } finally {+ release.resolve();+ await Promise.all([holding, waiting]);+ await pool.end();+ await pg.close();+ }+});diff --git a/test/modal-sandbox.test.ts b/test/modal-sandbox.test.tsindex ae39463fd..f2d8488eb 100644--- a/test/modal-sandbox.test.ts+++ b/test/modal-sandbox.test.ts@@ -1,3 +1,4 @@+import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";import { pollProcess } from "../src/sandbox/process-poll.ts";import { test, after, beforeEach } from "node:test";import assert from "node:assert/strict";@@ -675,6 +676,7 @@ test("a late checkpoint from another core cannot replace a newer committed checkconst two = await second.provision(layers);const older = first.teardown(one);await entered;+ await store.merge(scope, { lastSnapshotAttemptMs: 1 });await second.teardown(two);const newest = (await store.get(scope))?.nativeSnapshotId;release();@@ -857,3 +859,179 @@ test("repeated destroy teardown never targets an unrelated default scope", asyncassert.deepEqual(await store.get("default"), defaultRecord);assert.equal(await store.get(scope), null);});++test("warm commands and process controls do not wait for a checkpoint", { timeout: 10000 }, async () => {+ const counting = instrumentedSnapshotStore();+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const s = make({+ snapshots: {+ ...counting.store,+ createUpload: async (id: string) => {+ const upload = await counting.store.createUpload!(id);+ entered.resolve();+ await release.promise;+ return upload;+ },+ },+ });+ const handle = await s.provision(layers);+ await s.writeFile(handle, "keep.txt", "before snapshot");+ const saving = s.teardown(handle);+ try {+ await entered.promise;+ assert.equal((await s.run(handle, "echo responsive")).stdout.trim(), "responsive");+ assert.equal(await s.readFile(handle, "keep.txt"), "before snapshot");+ assert.ok(supportsProcessSessions(s));+ assert.deepEqual(await s.listProcesses(handle), []);+ } finally {+ release.resolve();+ await saving;+ }+});++test("ordinary calls never rotate an aged warm sandbox", async () => {+ const store = createMemoryMap<StoredModalSandbox>();+ const s = make({ store, rotateAfterMs: 1 });+ const handle = await s.provision(layers);+ await store.merge(scope, { createdAtMs: 0 });+ const id = (await store.get(scope))!.sandboxId;+ await s.run(handle, "echo same machine");+ await s.writeFile(handle, "keep.txt", "same machine");+ assert.equal((await store.get(scope))!.sandboxId, id);+ assert.equal(fake.createdCount(scopeName()), 1);+ await s.provision(layers);+ assert.notEqual((await store.get(scope))!.sandboxId, id);+});++test("checkpoint writers from separate cores share the durable lifecycle lock", { timeout: 10000 }, async () => {+ const store = createMemoryMap<StoredModalSandbox>();+ const advisoryLock = createMemoryAdvisoryLock();+ const counting = instrumentedSnapshotStore();+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ let writers = 0;+ let maximum = 0;+ const snapshots = {+ ...counting.store,+ createUpload: async (id: string) => {+ writers++;+ maximum = Math.max(maximum, writers);+ const upload = await counting.store.createUpload!(id);+ if (counting.puts() === 0) {+ entered.resolve();+ await release.promise;+ }+ return {+ ...upload,+ complete: async () => {+ await upload.complete();+ writers--;+ },+ };+ },+ };+ const first = make({ store, snapshots, advisoryLock });+ const second = make({ store, snapshots, advisoryLock });+ const a = await first.provision(layers);+ const b = await second.provision(layers);+ const saving = first.teardown(a);+ await entered.promise;+ const exporting = second.persistHomeSnapshot!(scope);+ try {+ assert.equal((await second.run(b, "echo second core")).stdout.trim(), "second core");+ } finally {+ release.resolve();+ await Promise.all([saving, exporting]);+ }+ assert.equal(maximum, 1);+ assert.equal(counting.puts(), 2);+});++test("failed native checkpoints wait for the interval across cleanup and maintenance", async () => {+ fake.cleanup();+ fake = installFakeModal({ native: true });+ const store = createMemoryMap<StoredModalSandbox>();+ let attempts = 0;+ let fail = false;+ const wrap = (session: Awaited<ReturnType<ModalClient["create"]>>) => ({+ ...session,+ async snapshotHome() {+ attempts++;+ if (fail) throw new Error("Timeout expired");+ return session.snapshotHome!();+ },+ });+ const client: ModalClient = {+ ...fake.client,+ create: async (options) => wrap(await fake.client.create(options)),+ fromId: async (id) => wrap(await fake.client.fromId(id)),+ };+ const s = make({ store, client });+ const h = await s.provision(layers);+ await s.teardown(h);+ const saved = (await store.get(scope))!.nativeSnapshotId;+ fail = true;+ await store.merge(scope, { lastSnapshotMs: 1, lastSnapshotAttemptMs: 1 });+ await s.teardown(h);+ assert.equal(attempts, 2);+ await s.teardown(h);+ await s.reapDeepIdle!(6 * 3600_000);+ assert.equal(attempts, 2);+ const restarted = make({ store, client });+ await restarted.reapDeepIdle!(6 * 3600_000);+ assert.equal(attempts, 2);+ assert.equal((await store.get(scope))!.nativeSnapshotId, saved);+ assert.equal((await store.get(scope))!.recoveryError, "Timeout expired");+ await store.merge(scope, { lastSnapshotAttemptMs: 1 });+ await s.teardown(h);+ assert.equal(attempts, 3);+});++test("deep idle termination excludes writes from another core until recovery", { timeout: 10000 }, async () => {+ fake.cleanup();+ fake = installFakeModal({ native: true });+ const store = createMemoryMap<StoredModalSandbox>();+ const advisoryLock = createMemoryAdvisoryLock();+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ let block = false;+ const wrap = (session: Awaited<ReturnType<ModalClient["create"]>>) => ({+ ...session,+ async snapshotHome() {+ const snapshot = await session.snapshotHome!();+ if (block) {+ entered.resolve();+ await release.promise;+ }+ return snapshot;+ },+ });+ const client: ModalClient = {+ ...fake.client,+ create: async (options) => wrap(await fake.client.create(options)),+ fromId: async (id) => wrap(await fake.client.fromId(id)),+ };+ const first = make({ store, client, advisoryLock });+ const second = make({ store, client, advisoryLock });+ const one = await first.provision(layers);+ const two = await second.provision(layers);+ await first.teardown(one);+ await store.merge(scope, { lastActivityMs: Date.now() - 7 * 3600_000 });+ block = true;+ const reaping = first.reapDeepIdle!(6 * 3600_000);+ await entered.promise;+ let completed = false;+ const writing = second.writeFile(two, "after-checkpoint.txt", "must survive").then(() => {+ completed = true;+ });+ try {+ await new Promise((resolve) => setTimeout(resolve, 30));+ assert.equal(completed, false);+ } finally {+ release.resolve();+ await Promise.all([reaping, writing]);+ }+ assert.equal(await second.readFile(two, "after-checkpoint.txt"), "must survive");+ assert.equal(fake.createdCount(scopeName()), 2);+});diff --git a/test/sandbox-resources.test.ts b/test/sandbox-resources.test.tsindex 62ad8ca6a..07ff142ac 100644--- a/test/sandbox-resources.test.ts+++ b/test/sandbox-resources.test.ts@@ -834,3 +834,94 @@ test("resource activation honors scope defaults and preserves explicit legacy prassert.equal(resources.defaultBackend("personal:new"), "modal");assert.equal(resources.defaultBackend("channel:new"), "sprites");});++for (const waiting of ["command", "checkpoint"] as const) {+ test(`Modal ${waiting} allows other tools while restart waits`, { timeout: 10000 }, async () => {+ const { options, backend, layers, routes } = fixture();+ const resources = createSandboxResources({ ...options, backends: { modal: backend }, defaultBackend: "modal" });+ const router = createSandboxRouter({ routes, backends: { modal: backend }, defaultBackend: "modal", resources });+ const record = await resources.create("alice", "personal:alice", "modal");+ await resources.setDefault("alice", "personal:alice", record.id);+ const handle = await router.provision(layers);+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const hold = async () => {+ entered.resolve();+ await release.promise;+ };+ backend.teardown = hold;+ backend.run = async (_handle, command) => {+ if (command === "long") await hold();+ return { stdout: command, stderr: "", code: 0, timedOut: false };+ };+ const long = waiting === "command" ? router.run(handle, "long") : router.teardown(handle);+ let restarted = false;+ backend.restartComputer = async () => {+ restarted = true;+ };+ let restarting: Promise<void> | undefined;+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ await entered.promise;+ const result = await Promise.race([+ router.run(handle, "responsive"),+ new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error("ordinary tool was serialized")), 1000);+ }),+ ]);+ assert.equal(result.stdout, "responsive");+ assert.equal((await resources.status("alice", record.id)).guestResponsive, true);+ restarting =+ waiting === "command" ? resources.restart("alice", record.id) : router.restartComputer!("personal:alice");+ await new Promise<void>((resolve) => setImmediate(resolve));+ assert.equal(restarted, false);+ } finally {+ clearTimeout(timer);+ release.resolve();+ await long;+ await restarting;+ }+ assert.equal(restarted, true);+ await resources.setDefault("alice", "personal:alice", null);+ await resources.retire("alice", record.id);+ await assert.rejects(router.run(handle, "retired"), /retired/);+ });+}++test("Modal provisioning and destructive cleanup wait for active operations", { timeout: 10000 }, async () => {+ const { options, backend, layers, routes } = fixture();+ const resources = createSandboxResources({ ...options, backends: { modal: backend }, defaultBackend: "modal" });+ const router = createSandboxRouter({ routes, backends: { modal: backend }, defaultBackend: "modal", resources });+ const record = await resources.create("alice", "personal:alice", "modal");+ const handle = await router.provision(layers, { sandboxId: record.id });+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ backend.run = async () => {+ entered.resolve();+ await release.promise;+ return { stdout: "done", stderr: "", code: 0, timedOut: false };+ };+ const running = router.run(handle, "long");+ await entered.promise;+ let provisioned = false;+ let destroyed = false;+ backend.provision = async () => {+ provisioned = true;+ return handle;+ };+ backend.teardown = async () => {+ destroyed = true;+ };+ const provision = router.provision(layers, { sandboxId: record.id });+ const teardown = router.teardown(handle, { destroy: true });+ try {+ await new Promise<void>((resolve) => setImmediate(resolve));+ assert.equal(provisioned, false);+ assert.equal(destroyed, false);+ } finally {+ release.resolve();+ await Promise.all([running, provision, teardown]);+ }+ assert.equal(provisioned, true);+ assert.equal(destroyed, true);+});