Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Keep Modal commands responsive during checkpoint saves (#1401)

Regan Bellf97638f55dfa
diff --git a/src/persistence/advisory-lock.ts b/src/persistence/advisory-lock.ts
index 22e81ec30..0f600cc87 100644
--- a/src/persistence/advisory-lock.ts
+++ b/src/persistence/advisory-lock.ts
@@ -1,8 +1,10 @@
-import type { PgPool } from "./pg-pool.ts";
-import { createKeyedQueue, sleep } from "../util/async.ts";
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { PgPool, PoolClient } from "./pg-pool.ts";
+import { sleep } from "../util/async.ts";
export interface AdvisoryLock {
withLock<T>(key: string, fn: () => Promise<T>): Promise<T>;
+ withSharedLock?<T>(key: string, fn: () => Promise<T>): Promise<T>;
tryWithLock?<T>(key: string, fn: () => Promise<T>): Promise<T | null>;
}
@@ -14,6 +16,9 @@ export function createNoopAdvisoryLock(): AdvisoryLock {
async withLock<T>(_key: string, fn: () => Promise<T>): Promise<T> {
return fn();
},
+ async withSharedLock<T>(_key: string, fn: () => Promise<T>): Promise<T> {
+ return fn();
+ },
async tryWithLock<T>(_key: string, fn: () => Promise<T>): Promise<T | null> {
return fn();
},
@@ -21,22 +26,34 @@ export function createNoopAdvisoryLock(): AdvisoryLock {
}
export function createMemoryAdvisoryLock(): AdvisoryLock {
- const queue = createKeyedQueue<string>();
- const held = new Set<string>();
- const withLock = <T>(key: string, fn: () => Promise<T>): Promise<T> =>
- queue(key, async () => {
- held.add(key);
- try {
- return await fn();
- } finally {
- held.delete(key);
- }
- });
+ const states = new Map<string, { tail: Promise<void>; readers: Set<Promise<void>>; pending: number }>();
+ const run = async <T>(key: string, fn: () => Promise<T>, shared: boolean): Promise<T> => {
+ let state = states.get(key);
+ if (!state) {
+ state = { tail: Promise.resolve(), readers: new Set(), pending: 0 };
+ states.set(key, state);
+ }
+ const done = Promise.withResolvers<void>();
+ const before = shared ? state.tail : Promise.all([state.tail, ...state.readers]);
+ state.pending++;
+ if (shared) state.readers.add(done.promise);
+ else state.tail = done.promise;
+ try {
+ await before;
+ return await fn();
+ } finally {
+ state.readers.delete(done.promise);
+ state.pending--;
+ done.resolve();
+ if (!state.pending) states.delete(key);
+ }
+ };
return {
- withLock,
+ withLock: (key, fn) => run(key, fn, false),
+ withSharedLock: (key, fn) => run(key, fn, true),
async tryWithLock(key, fn) {
- if (held.has(key)) return null;
- return withLock(key, fn);
+ if (states.has(key)) return null;
+ return run(key, fn, false);
},
};
}
@@ -48,50 +65,60 @@ export function createPostgresAdvisoryLock(
const timeoutMs = opts.timeoutMs ?? DEFAULT_ADVISORY_LOCK_TIMEOUT_MS;
const pollMs = opts.pollMs ?? DEFAULT_ADVISORY_LOCK_POLL_MS;
- return {
- async withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
- const deadline = Date.now() + timeoutMs;
- const pool = await pg.sessionPool();
- for (;;) {
- const client = await pool.connect();
+ type Context = {
+ client: PoolClient;
+ references: number;
+ keys: Map<string, { shared: boolean; count: number }>;
+ };
+ const sessions = new AsyncLocalStorage<{ context: Context; active: boolean }>();
+ const withClient = async <T>(action: (context: Context) => Promise<T>): Promise<T> => {
+ const parent = sessions.getStore();
+ const context = parent?.active
+ ? parent.context
+ : { client: await (await pg.sessionPool()).connect(), references: 0, keys: new Map() };
+ context.references++;
+ const lease = { context, active: true };
+ try {
+ return await sessions.run(lease, () => action(context));
+ } finally {
+ lease.active = false;
+ if (--context.references === 0) context.client.release();
+ }
+ };
+ const run = async <T>(key: string, fn: () => Promise<T>, shared: boolean, wait: boolean): Promise<T | null> => {
+ const deadline = Date.now() + timeoutMs;
+ for (;;) {
+ const attempt = await withClient(async (context): Promise<{ acquired: false } | { acquired: true; value: T }> => {
+ const { client, keys } = context;
+ const held = keys.get(key);
+ if (held && !(shared && held.shared)) return { acquired: false };
+ const reservation = held ?? { shared, count: 0 };
+ reservation.count++;
+ keys.set(key, reservation);
try {
const res = await client.query<{ locked: boolean }>(
- "SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked",
+ `SELECT pg_try_advisory_lock${shared ? "_shared" : ""}(hashtextextended($1, 0)) AS locked`,
[key],
);
- const held = res.rows[0]?.locked === true;
- if (held) {
- try {
- return await fn();
- } finally {
- await client.query("SELECT pg_advisory_unlock(hashtextextended($1, 0))", [key]);
- }
+ if (res.rows[0]?.locked !== true) return { acquired: false };
+ try {
+ return { acquired: true, value: await fn() };
+ } finally {
+ await client.query(`SELECT pg_advisory_unlock${shared ? "_shared" : ""}(hashtextextended($1, 0))`, [key]);
}
} finally {
- client.release();
- }
- if (Date.now() >= deadline) throw new Error(`timeout acquiring advisory lock for ${key}`);
- await sleep(pollMs);
- }
- },
-
- async tryWithLock<T>(key: string, fn: () => Promise<T>): Promise<T | null> {
- const pool = await pg.sessionPool();
- const client = await pool.connect();
- try {
- const res = await client.query<{ locked: boolean }>(
- "SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked",
- [key],
- );
- if (res.rows[0]?.locked !== true) return null;
- try {
- return await fn();
- } finally {
- await client.query("SELECT pg_advisory_unlock(hashtextextended($1, 0))", [key]);
+ if (--reservation.count === 0) keys.delete(key);
}
- } finally {
- client.release();
- }
- },
+ });
+ if (attempt.acquired) return attempt.value;
+ if (!wait) return null;
+ if (Date.now() >= deadline) throw new Error(`timeout acquiring advisory lock for ${key}`);
+ await sleep(pollMs);
+ }
+ };
+ return {
+ withLock: <T>(key: string, fn: () => Promise<T>) => run(key, fn, false, true) as Promise<T>,
+ withSharedLock: <T>(key: string, fn: () => Promise<T>) => run(key, fn, true, true) as Promise<T>,
+ tryWithLock: <T>(key: string, fn: () => Promise<T>) => run(key, fn, false, false),
};
}
diff --git a/src/sandbox/modal-sandbox.ts b/src/sandbox/modal-sandbox.ts
index 698566603..b32ada11c 100644
--- a/src/sandbox/modal-sandbox.ts
+++ b/src/sandbox/modal-sandbox.ts
@@ -1,6 +1,7 @@
import { randomUUID } from "node:crypto";
import type { WorkspaceLayer } from "../types.ts";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
+import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";
import type { DurableMap } from "../persistence/durable-map.ts";
import { createMemoryMap } from "../persistence/durable-map.ts";
import { orgId as configOrgId } from "../config.ts";
@@ -68,6 +69,7 @@ export interface StoredModalSandbox {
snapshotGeneration?: number;
createdAtMs: number;
lastSnapshotMs?: number;
+ lastSnapshotAttemptMs?: number;
lastActivityMs?: number;
homeDirty?: boolean;
orgId?: string;
@@ -75,6 +77,7 @@ export interface StoredModalSandbox {
export interface ModalSandboxOptions extends BlobStagingOptions {
client: ModalClient;
+ advisoryLock?: AdvisoryLock;
namePrefix?: string;
defaultTimeoutSec?: number;
snapshotIntervalMs?: number;
@@ -102,6 +105,8 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
!!stored?.nativeSnapshotId || !!(client.nativeSnapshots && opts.nativeSnapshotsEnabled);
const snapshotIntervalMs = (stored?: StoredModalSandbox | null): number =>
usesNativeSnapshots(stored) ? (opts.nativeSnapshotIntervalMs ?? 5 * 60_000) : (opts.snapshotIntervalMs ?? 0);
+ const lastSnapshotAttemptMs = (stored?: StoredModalSandbox | null): number =>
+ Math.max(stored?.lastSnapshotMs ?? 0, usesNativeSnapshots(stored) ? (stored?.lastSnapshotAttemptMs ?? 0) : 0);
const rotateAfterMs = opts.rotateAfterMs ?? 20 * 3600_000;
const reapIdleMs = opts.reapIdleMs ?? 6 * 3600_000;
const fileChunkBytes = opts.fileChunkBytes ?? 64 * 1024 * 1024;
@@ -110,7 +115,10 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
const workspaceDir = `${HOME_DIR}/${WORKSPACE_BASENAME}`;
const store = opts.store ?? createMemoryMap<StoredModalSandbox>();
const snapshots = opts.snapshots ?? createMemorySnapshotStore();
- const provisionQueue = createKeyedQueue<string>();
+ const localQueue = createKeyedQueue<string>();
+ const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock();
+ const provisionQueue = <T>(scope: string, action: () => Promise<T>): Promise<T> =>
+ localQueue(scope, () => advisoryLock.withLock(`modal-provision:${scope}`, action));
const sessionByName = new Map<string, ModalSession>();
const scopeByName = new Map<string, string>();
@@ -150,7 +158,11 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
if (!store.update) throw new Error("native Modal checkpoints require an atomic durable store");
const requested = await store.update(scope, (current) => {
if (current.sandboxId !== session.sandboxId) throw new Error("Modal checkpoint source has been replaced");
- return { ...current, snapshotGeneration: (current.snapshotGeneration ?? 0) + 1 };
+ return {
+ ...current,
+ snapshotGeneration: (current.snapshotGeneration ?? 0) + 1,
+ lastSnapshotAttemptMs: Date.now(),
+ };
});
if (!requested) throw new Error("Modal checkpoint source is no longer tracked");
const capturedAtMs = Date.now();
@@ -163,6 +175,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
nativeSnapshotId: snapshot.imageId,
nativeSnapshotExpiresAtMs: snapshot.expiresAtMs,
recoveryError: undefined,
+ lastSnapshotAttemptMs: undefined,
lastSnapshotMs: capturedAtMs,
homeDirty: false,
}
@@ -217,6 +230,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
await store.put(scope, {
...previous,
hydrationPending: true,
+ lastSnapshotAttemptMs: undefined,
...(client.lifetimeMs ? { expiresAtMs: Date.now() + client.lifetimeMs } : {}),
sandboxId: session.sandboxId,
createdAtMs: Date.now(),
@@ -263,6 +277,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
scope: string,
name: string,
onStatus?: (text: string) => void,
+ allowRotation = true,
): Promise<{ session: ModalSession; coldStart: boolean }> {
return provisionQueue(scope, async () => {
const adopt = async (session: ModalSession): Promise<{ session: ModalSession; coldStart: boolean }> => {
@@ -278,6 +293,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
"Modal home hydration was interrupted; use computer restart to discard the incomplete replacement and retry the retained checkpoint",
);
const stale =
+ allowRotation &&
!!stored &&
Date.now() - stored.createdAtMs > rotateAfterMs &&
Date.now() >= (rotationHoldUntil.get(scope) ?? 0);
@@ -372,7 +388,7 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
});
}
- async function withSession<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {
+ async function withSessionUnlocked<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {
const scratchKey = scratchKeyByName.get(name);
const reviveScratch = async (): Promise<ModalSession> => {
const session = await client.create({});
@@ -382,7 +398,11 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
const first =
scratchKey !== undefined
? { session: sessionByName.get(name) ?? (await reviveScratch()) }
- : await ensureSession(scopeByName.get(name) ?? "default", name);
+ : {
+ session:
+ sessionByName.get(name) ??
+ (await ensureSession(scopeByName.get(name) ?? "default", name, undefined, false)).session,
+ };
try {
return await action(first.session);
} catch (err) {
@@ -391,11 +411,17 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
const second =
scratchKey !== undefined
? { session: await reviveScratch() }
- : await ensureSession(scopeByName.get(name) ?? "default", name);
+ : await ensureSession(scopeByName.get(name) ?? "default", name, undefined, false);
return action(second.session);
}
}
+ function withSession<T>(name: string, action: (session: ModalSession) => Promise<T>): Promise<T> {
+ if (scratchKeyByName.has(name)) return withSessionUnlocked(name, action);
+ const shared = advisoryLock.withSharedLock ?? advisoryLock.withLock;
+ return shared(`modal-use:${scopeByName.get(name) ?? "default"}`, () => withSessionUnlocked(name, action));
+ }
+
const lastTouchMs = new Map<string, number>();
async function touchActivity(name: string): Promise<void> {
const scope = scopeByName.get(name);
@@ -745,7 +771,9 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
if (!session) return;
const stored = await store.get(scope);
if (!tdOpts?.homeUnchanged) await store.merge(scope, { homeDirty: true });
- if (snapshotDue(stored, tdOpts, snapshotIntervalMs(stored))) {
+ if (
+ snapshotDue({ ...stored, lastSnapshotMs: lastSnapshotAttemptMs(stored) }, tdOpts, snapshotIntervalMs(stored))
+ ) {
try {
await snapshotHome(scope, session);
await store.merge(scope, { lastActivityMs: Date.now() });
@@ -769,10 +797,17 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
if (
usesNativeSnapshots(rec) &&
(!rec.expiresAtMs || rec.expiresAtMs > Date.now()) &&
- (!rec.lastSnapshotMs || Date.now() - rec.lastSnapshotMs > snapshotIntervalMs(rec))
+ Date.now() - lastSnapshotAttemptMs(rec) > snapshotIntervalMs(rec)
) {
await provisionQueue(scope, async () => {
try {
+ const current = await store.get(scope);
+ if (
+ !current ||
+ current.sandboxId !== rec.sandboxId ||
+ Date.now() - lastSnapshotAttemptMs(current) <= snapshotIntervalMs(current)
+ )
+ return;
const session = await client.fromId(rec.sandboxId);
await snapshotHome(scope, session);
} catch (error) {
@@ -799,26 +834,28 @@ export function createModalSandbox(workspace: WorkspaceStore, opts: ModalSandbox
}
continue;
}
- reaped += await provisionQueue(scope, async (): Promise<number> => {
- const current = await store.get(scope);
- if (!current || current.sandboxId !== rec.sandboxId) return 0;
- if (!current.lastActivityMs || current.lastActivityMs > cutoff) return 0;
- try {
- await snapshotHome(scope, session);
- await session.terminate();
- sessionByName.delete(name);
- if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope);
- return 1;
- } catch (e) {
- if (e instanceof ModalSandboxGoneError) {
+ reaped += await advisoryLock.withLock(`modal-use:${scope}`, () =>
+ provisionQueue(scope, async (): Promise<number> => {
+ const current = await store.get(scope);
+ if (!current || current.sandboxId !== rec.sandboxId) return 0;
+ if (!current.lastActivityMs || current.lastActivityMs > cutoff) return 0;
+ try {
+ await snapshotHome(scope, session);
+ await session.terminate();
sessionByName.delete(name);
- if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope).catch(() => undefined);
- } else {
- reportError("sandbox_reap", "deep_idle_reap_failed", errMessage(e), scope);
+ if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope);
+ return 1;
+ } catch (e) {
+ if (e instanceof ModalSandboxGoneError) {
+ sessionByName.delete(name);
+ if (!(await store.get(scope))?.nativeSnapshotId) await store.delete(scope).catch(() => undefined);
+ } else {
+ reportError("sandbox_reap", "deep_idle_reap_failed", errMessage(e), scope);
+ }
+ return 0;
}
- return 0;
- }
- });
+ }),
+ );
}
return { reaped };
},
diff --git a/src/sandbox/sandbox-resources.ts b/src/sandbox/sandbox-resources.ts
index c5a39e3ee..b6462de88 100644
--- a/src/sandbox/sandbox-resources.ts
+++ b/src/sandbox/sandbox-resources.ts
@@ -63,7 +63,7 @@ export interface SandboxResources {
status(actorId: string, id: string): Promise<ComputerStatus>;
restart(actorId: string, id: string): Promise<void>;
retire(actorId: string, id: string): Promise<void>;
- use<T>(id: string, action: () => Promise<T>): Promise<T>;
+ use<T>(id: string, action: () => Promise<T>, exclusive?: boolean): Promise<T>;
setDefault(actorId: string, scopeId: ScopeId, id: string | null): Promise<void>;
resolve(scopeId: ScopeId): Promise<SandboxResource | null | undefined>;
get(id: string): Promise<SandboxResource>;
@@ -180,12 +180,18 @@ export function createSandboxResources(opts: {
if (!record) throw new Error(`sandbox not found: ${id}`);
return record;
};
- const use = <T>(id: string, action: () => Promise<T>): Promise<T> =>
- opts.lock.withLock(`sandbox-resource:${id}`, async () => {
- const record = await get(id);
- if (record.state === "retired") throw new Error("sandbox has been retired");
+ const use = async <T>(id: string, action: () => Promise<T>, exclusive = false): Promise<T> => {
+ const record = await get(id);
+ const lock =
+ exclusive || record.backend !== "modal" || !opts.lock.withSharedLock
+ ? opts.lock.withLock
+ : opts.lock.withSharedLock;
+ return lock(`sandbox-resource:${id}`, async () => {
+ const current = await get(id);
+ if (current.state === "retired") throw new Error("sandbox has been retired");
return action();
});
+ };
const recordLegacy = async (scopeId: string, backend: SandboxBackendName, handle: SandboxHandle): Promise<string> => {
const id = legacyId(scopeId, backend);
await opts.records.putIfAbsent(id, {
@@ -269,7 +275,7 @@ export function createSandboxResources(opts: {
await authorize(actorId, record.ownerScopeId);
const backend = opts.backends[record.backend];
if (!backend?.restartComputer) throw new Error(`sandbox restart unavailable: ${record.backend}`);
- await use(id, () => backend.restartComputer!(record.backingScopeId));
+ await use(id, () => backend.restartComputer!(record.backingScopeId), true);
},
async resolve(scopeId) {
await initialize();
diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.ts
index 06e62e786..0f23279b9 100644
--- a/src/sandbox/sandbox-routing.ts
+++ b/src/sandbox/sandbox-routing.ts
@@ -170,7 +170,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {
const routedLayers = layers.map((layer) =>
layer.mode === "rw" ? { ...layer, scopeId: resource.backingScopeId } : layer,
);
- const handle = await opts.resources!.use(resource.id, () => sandbox.provision(routedLayers, provOpts));
+ const handle = await opts.resources!.use(resource.id, () => sandbox.provision(routedLayers, provOpts), true);
return { ...handle, backend: resource.backend, scopeId: resource.ownerScopeId, resourceId: resource.id };
}
const { name, sandbox } = await pick(scope);
@@ -214,7 +214,10 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {
});
},
teardown(handle, tdOpts?: TeardownOptions): Promise<void> {
- return useHandle(handle, () => forHandle(handle).teardown(handle, tdOpts));
+ const action = () => forHandle(handle).teardown(handle, tdOpts);
+ return handle.resourceId && opts.resources
+ ? opts.resources.use(handle.resourceId, action, handle.backend !== "modal" || !!tdOpts?.destroy)
+ : action();
},
...(some(supportsProcessSessions)
@@ -286,7 +289,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {
restartComputer: async (scopeId: string) => {
const target = await computerTarget(scopeId);
const action = () => requireCap(target.sandbox, "restartComputer", scopeId).restartComputer(target.scopeId);
- return target.resourceId && opts.resources ? opts.resources.use(target.resourceId, action) : action();
+ return target.resourceId && opts.resources ? opts.resources.use(target.resourceId, action, true) : action();
},
}
: {}),
diff --git a/src/wiring.ts b/src/wiring.ts
index c3dde7f93..1a6599316 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -882,6 +882,7 @@ export function buildApp(
if (!modal.tokenId || !modal.tokenSecret)
throw new Error("SANDBOX_BACKEND=modal requires MODAL_TOKEN_ID and MODAL_TOKEN_SECRET");
return createModalSandbox(workspace, {
+ advisoryLock,
client: createSdkModalClient({
tokenId: modal.tokenId,
tokenSecret: modal.tokenSecret,
diff --git a/test/advisory-lock.test.ts b/test/advisory-lock.test.ts
index 359bfc15c..98b968c5c 100644
--- a/test/advisory-lock.test.ts
+++ b/test/advisory-lock.test.ts
@@ -178,3 +178,174 @@ test("pg sandbox activation fences publication from a separate compatible reader
await pgB.close();
}
});
+
+test(
+ "pg shared sandbox use overlaps across cores and excludes lifecycle mutations",
+ { skip, timeout: 10000 },
+ async () => {
+ const pgA = createPgPool(URL!);
+ const pgB = createPgPool(URL!);
+ const pgWriter = createPgPool(URL!);
+ const release = Promise.withResolvers<void>();
+ const entered = Promise.withResolvers<void>();
+ const key = `sandbox-shared:${crypto.randomUUID()}`;
+ const a = createPostgresAdvisoryLock(pgA, { pollMs: 10 });
+ const b = createPostgresAdvisoryLock(pgB, { pollMs: 10, timeoutMs: 200 });
+ const writer = createPostgresAdvisoryLock(pgWriter, { pollMs: 10, timeoutMs: 100 });
+ const running = a.withSharedLock!(key, async () => {
+ entered.resolve();
+ await release.promise;
+ });
+ try {
+ await entered.promise;
+ assert.equal(await b.withSharedLock!(key, async () => "concurrent"), "concurrent");
+ await assert.rejects(
+ writer.withLock(key, async () => assert.fail("mutated active sandbox")),
+ /timeout acquiring/,
+ );
+ assert.equal(await writer.tryWithLock!(key, async () => "exclusive"), null);
+ release.resolve();
+ await running;
+ assert.equal(await writer.withLock(key, async () => "exclusive"), "exclusive");
+ await assert.rejects(
+ b.withSharedLock!(key, async () => {
+ throw new Error("failed operation");
+ }),
+ /failed operation/,
+ );
+ assert.equal(await writer.tryWithLock!(key, async () => "released"), "released");
+ await writer.withLock(key, async () => {
+ await assert.rejects(
+ b.withSharedLock!(key, async () => assert.fail("entered during mutation")),
+ /timeout acquiring/,
+ );
+ });
+ } finally {
+ release.resolve();
+ await running;
+ await Promise.all([pgA.close(), pgB.close(), pgWriter.close()]);
+ }
+ },
+);
+
+test("pg nested locks reuse one connection without bypassing sibling exclusion", { skip, timeout: 10000 }, async () => {
+ const { Pool } = await import("pg");
+ const pool = new Pool({ connectionString: URL, max: 1, connectionTimeoutMillis: 200 });
+ const pg = createPgPool(URL!);
+ const lock = createPostgresAdvisoryLock({ ...pg, sessionPool: async () => pool }, { pollMs: 5, timeoutMs: 100 });
+ try {
+ assert.equal(
+ await lock.withSharedLock!("resource", () =>
+ lock.withSharedLock!("use", () => lock.withLock("recover", async () => 42)),
+ ),
+ 42,
+ );
+ await lock.withLock("parent", async () => {
+ let active = 0;
+ let maximum = 0;
+ const action = () =>
+ lock.withLock("writer", async () => {
+ maximum = Math.max(maximum, ++active);
+ await sleep(15);
+ active--;
+ });
+ await Promise.all([action(), action(), action()]);
+ assert.equal(maximum, 1);
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ let writing = false;
+ const reader = lock.withSharedLock!("mixed", async () => {
+ entered.resolve();
+ await release.promise;
+ });
+ await entered.promise;
+ assert.equal(await lock.tryWithLock!("mixed", async () => true), null);
+ const writer = lock.withLock("mixed", async () => {
+ writing = true;
+ });
+ await sleep(15);
+ assert.equal(writing, false);
+ release.resolve();
+ await Promise.all([reader, writer]);
+ await assert.rejects(
+ lock.withLock("parent", async () => true),
+ /timeout acquiring/,
+ );
+ await assert.rejects(
+ lock.withLock("throws", async () => {
+ throw new Error("callback failure");
+ }),
+ /callback failure/,
+ );
+ assert.equal(await lock.withLock("throws", async () => true), true);
+ });
+ assert.equal(pool.idleCount, 1);
+ } finally {
+ await pool.end();
+ await pg.close();
+ }
+});
+
+test(
+ "pg nested lock leases outlive their parent and late inherited work leases anew",
+ { skip, timeout: 10000 },
+ async () => {
+ const { Pool } = await import("pg");
+ const pool = new Pool({ connectionString: URL, max: 1, connectionTimeoutMillis: 200 });
+ const pg = createPgPool(URL!);
+ const lock = createPostgresAdvisoryLock({ ...pg, sessionPool: async () => pool }, { pollMs: 5 });
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const late = Promise.withResolvers<void>();
+ let child!: Promise<void>;
+ let delayed!: Promise<boolean>;
+ try {
+ await lock.withLock("root", async () => {
+ child = lock.withLock("child", async () => {
+ entered.resolve();
+ await release.promise;
+ assert.equal(await lock.withLock("grandchild", async () => true), true);
+ });
+ delayed = late.promise.then(() => lock.withLock("late", async () => true));
+ await entered.promise;
+ });
+ assert.equal(pool.idleCount, 0);
+ release.resolve();
+ await child;
+ assert.equal(pool.idleCount, 1);
+ late.resolve();
+ assert.equal(await delayed, true);
+ assert.equal(pool.idleCount, 1);
+ } finally {
+ release.resolve();
+ late.resolve();
+ await pool.end();
+ await pg.close();
+ }
+ },
+);
+
+test("pg contended polling releases connections for unrelated keys", { skip, timeout: 10000 }, async () => {
+ const { Pool } = await import("pg");
+ const pool = new Pool({ connectionString: URL, max: 2, connectionTimeoutMillis: 100 });
+ const pg = createPgPool(URL!);
+ const lock = createPostgresAdvisoryLock({ ...pg, sessionPool: async () => pool }, { pollMs: 5 });
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const holding = lock.withLock("occupied", async () => {
+ entered.resolve();
+ await release.promise;
+ });
+ await entered.promise;
+ const waiting = lock.withLock("occupied", async () => true);
+ try {
+ await sleep(20);
+ assert.equal(await lock.withLock("unrelated", async () => true), true);
+ assert.equal(await lock.tryWithLock!("unrelated", async () => true), true);
+ } finally {
+ release.resolve();
+ await Promise.all([holding, waiting]);
+ await pool.end();
+ await pg.close();
+ }
+});
diff --git a/test/modal-sandbox.test.ts b/test/modal-sandbox.test.ts
index ae39463fd..f2d8488eb 100644
--- a/test/modal-sandbox.test.ts
+++ b/test/modal-sandbox.test.ts
@@ -1,3 +1,4 @@
+import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";
import { pollProcess } from "../src/sandbox/process-poll.ts";
import { test, after, beforeEach } from "node:test";
import assert from "node:assert/strict";
@@ -675,6 +676,7 @@ test("a late checkpoint from another core cannot replace a newer committed check
const two = await second.provision(layers);
const older = first.teardown(one);
await entered;
+ await store.merge(scope, { lastSnapshotAttemptMs: 1 });
await second.teardown(two);
const newest = (await store.get(scope))?.nativeSnapshotId;
release();
@@ -857,3 +859,179 @@ test("repeated destroy teardown never targets an unrelated default scope", async
assert.deepEqual(await store.get("default"), defaultRecord);
assert.equal(await store.get(scope), null);
});
+
+test("warm commands and process controls do not wait for a checkpoint", { timeout: 10000 }, async () => {
+ const counting = instrumentedSnapshotStore();
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const s = make({
+ snapshots: {
+ ...counting.store,
+ createUpload: async (id: string) => {
+ const upload = await counting.store.createUpload!(id);
+ entered.resolve();
+ await release.promise;
+ return upload;
+ },
+ },
+ });
+ const handle = await s.provision(layers);
+ await s.writeFile(handle, "keep.txt", "before snapshot");
+ const saving = s.teardown(handle);
+ try {
+ await entered.promise;
+ assert.equal((await s.run(handle, "echo responsive")).stdout.trim(), "responsive");
+ assert.equal(await s.readFile(handle, "keep.txt"), "before snapshot");
+ assert.ok(supportsProcessSessions(s));
+ assert.deepEqual(await s.listProcesses(handle), []);
+ } finally {
+ release.resolve();
+ await saving;
+ }
+});
+
+test("ordinary calls never rotate an aged warm sandbox", async () => {
+ const store = createMemoryMap<StoredModalSandbox>();
+ const s = make({ store, rotateAfterMs: 1 });
+ const handle = await s.provision(layers);
+ await store.merge(scope, { createdAtMs: 0 });
+ const id = (await store.get(scope))!.sandboxId;
+ await s.run(handle, "echo same machine");
+ await s.writeFile(handle, "keep.txt", "same machine");
+ assert.equal((await store.get(scope))!.sandboxId, id);
+ assert.equal(fake.createdCount(scopeName()), 1);
+ await s.provision(layers);
+ assert.notEqual((await store.get(scope))!.sandboxId, id);
+});
+
+test("checkpoint writers from separate cores share the durable lifecycle lock", { timeout: 10000 }, async () => {
+ const store = createMemoryMap<StoredModalSandbox>();
+ const advisoryLock = createMemoryAdvisoryLock();
+ const counting = instrumentedSnapshotStore();
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ let writers = 0;
+ let maximum = 0;
+ const snapshots = {
+ ...counting.store,
+ createUpload: async (id: string) => {
+ writers++;
+ maximum = Math.max(maximum, writers);
+ const upload = await counting.store.createUpload!(id);
+ if (counting.puts() === 0) {
+ entered.resolve();
+ await release.promise;
+ }
+ return {
+ ...upload,
+ complete: async () => {
+ await upload.complete();
+ writers--;
+ },
+ };
+ },
+ };
+ const first = make({ store, snapshots, advisoryLock });
+ const second = make({ store, snapshots, advisoryLock });
+ const a = await first.provision(layers);
+ const b = await second.provision(layers);
+ const saving = first.teardown(a);
+ await entered.promise;
+ const exporting = second.persistHomeSnapshot!(scope);
+ try {
+ assert.equal((await second.run(b, "echo second core")).stdout.trim(), "second core");
+ } finally {
+ release.resolve();
+ await Promise.all([saving, exporting]);
+ }
+ assert.equal(maximum, 1);
+ assert.equal(counting.puts(), 2);
+});
+
+test("failed native checkpoints wait for the interval across cleanup and maintenance", async () => {
+ fake.cleanup();
+ fake = installFakeModal({ native: true });
+ const store = createMemoryMap<StoredModalSandbox>();
+ let attempts = 0;
+ let fail = false;
+ const wrap = (session: Awaited<ReturnType<ModalClient["create"]>>) => ({
+ ...session,
+ async snapshotHome() {
+ attempts++;
+ if (fail) throw new Error("Timeout expired");
+ return session.snapshotHome!();
+ },
+ });
+ const client: ModalClient = {
+ ...fake.client,
+ create: async (options) => wrap(await fake.client.create(options)),
+ fromId: async (id) => wrap(await fake.client.fromId(id)),
+ };
+ const s = make({ store, client });
+ const h = await s.provision(layers);
+ await s.teardown(h);
+ const saved = (await store.get(scope))!.nativeSnapshotId;
+ fail = true;
+ await store.merge(scope, { lastSnapshotMs: 1, lastSnapshotAttemptMs: 1 });
+ await s.teardown(h);
+ assert.equal(attempts, 2);
+ await s.teardown(h);
+ await s.reapDeepIdle!(6 * 3600_000);
+ assert.equal(attempts, 2);
+ const restarted = make({ store, client });
+ await restarted.reapDeepIdle!(6 * 3600_000);
+ assert.equal(attempts, 2);
+ assert.equal((await store.get(scope))!.nativeSnapshotId, saved);
+ assert.equal((await store.get(scope))!.recoveryError, "Timeout expired");
+ await store.merge(scope, { lastSnapshotAttemptMs: 1 });
+ await s.teardown(h);
+ assert.equal(attempts, 3);
+});
+
+test("deep idle termination excludes writes from another core until recovery", { timeout: 10000 }, async () => {
+ fake.cleanup();
+ fake = installFakeModal({ native: true });
+ const store = createMemoryMap<StoredModalSandbox>();
+ const advisoryLock = createMemoryAdvisoryLock();
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ let block = false;
+ const wrap = (session: Awaited<ReturnType<ModalClient["create"]>>) => ({
+ ...session,
+ async snapshotHome() {
+ const snapshot = await session.snapshotHome!();
+ if (block) {
+ entered.resolve();
+ await release.promise;
+ }
+ return snapshot;
+ },
+ });
+ const client: ModalClient = {
+ ...fake.client,
+ create: async (options) => wrap(await fake.client.create(options)),
+ fromId: async (id) => wrap(await fake.client.fromId(id)),
+ };
+ const first = make({ store, client, advisoryLock });
+ const second = make({ store, client, advisoryLock });
+ const one = await first.provision(layers);
+ const two = await second.provision(layers);
+ await first.teardown(one);
+ await store.merge(scope, { lastActivityMs: Date.now() - 7 * 3600_000 });
+ block = true;
+ const reaping = first.reapDeepIdle!(6 * 3600_000);
+ await entered.promise;
+ let completed = false;
+ const writing = second.writeFile(two, "after-checkpoint.txt", "must survive").then(() => {
+ completed = true;
+ });
+ try {
+ await new Promise((resolve) => setTimeout(resolve, 30));
+ assert.equal(completed, false);
+ } finally {
+ release.resolve();
+ await Promise.all([reaping, writing]);
+ }
+ assert.equal(await second.readFile(two, "after-checkpoint.txt"), "must survive");
+ assert.equal(fake.createdCount(scopeName()), 2);
+});
diff --git a/test/sandbox-resources.test.ts b/test/sandbox-resources.test.ts
index 62ad8ca6a..07ff142ac 100644
--- a/test/sandbox-resources.test.ts
+++ b/test/sandbox-resources.test.ts
@@ -834,3 +834,94 @@ test("resource activation honors scope defaults and preserves explicit legacy pr
assert.equal(resources.defaultBackend("personal:new"), "modal");
assert.equal(resources.defaultBackend("channel:new"), "sprites");
});
+
+for (const waiting of ["command", "checkpoint"] as const) {
+ test(`Modal ${waiting} allows other tools while restart waits`, { timeout: 10000 }, async () => {
+ const { options, backend, layers, routes } = fixture();
+ const resources = createSandboxResources({ ...options, backends: { modal: backend }, defaultBackend: "modal" });
+ const router = createSandboxRouter({ routes, backends: { modal: backend }, defaultBackend: "modal", resources });
+ const record = await resources.create("alice", "personal:alice", "modal");
+ await resources.setDefault("alice", "personal:alice", record.id);
+ const handle = await router.provision(layers);
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const hold = async () => {
+ entered.resolve();
+ await release.promise;
+ };
+ backend.teardown = hold;
+ backend.run = async (_handle, command) => {
+ if (command === "long") await hold();
+ return { stdout: command, stderr: "", code: 0, timedOut: false };
+ };
+ const long = waiting === "command" ? router.run(handle, "long") : router.teardown(handle);
+ let restarted = false;
+ backend.restartComputer = async () => {
+ restarted = true;
+ };
+ let restarting: Promise<void> | undefined;
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await entered.promise;
+ const result = await Promise.race([
+ router.run(handle, "responsive"),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(() => reject(new Error("ordinary tool was serialized")), 1000);
+ }),
+ ]);
+ assert.equal(result.stdout, "responsive");
+ assert.equal((await resources.status("alice", record.id)).guestResponsive, true);
+ restarting =
+ waiting === "command" ? resources.restart("alice", record.id) : router.restartComputer!("personal:alice");
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ assert.equal(restarted, false);
+ } finally {
+ clearTimeout(timer);
+ release.resolve();
+ await long;
+ await restarting;
+ }
+ assert.equal(restarted, true);
+ await resources.setDefault("alice", "personal:alice", null);
+ await resources.retire("alice", record.id);
+ await assert.rejects(router.run(handle, "retired"), /retired/);
+ });
+}
+
+test("Modal provisioning and destructive cleanup wait for active operations", { timeout: 10000 }, async () => {
+ const { options, backend, layers, routes } = fixture();
+ const resources = createSandboxResources({ ...options, backends: { modal: backend }, defaultBackend: "modal" });
+ const router = createSandboxRouter({ routes, backends: { modal: backend }, defaultBackend: "modal", resources });
+ const record = await resources.create("alice", "personal:alice", "modal");
+ const handle = await router.provision(layers, { sandboxId: record.id });
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ backend.run = async () => {
+ entered.resolve();
+ await release.promise;
+ return { stdout: "done", stderr: "", code: 0, timedOut: false };
+ };
+ const running = router.run(handle, "long");
+ await entered.promise;
+ let provisioned = false;
+ let destroyed = false;
+ backend.provision = async () => {
+ provisioned = true;
+ return handle;
+ };
+ backend.teardown = async () => {
+ destroyed = true;
+ };
+ const provision = router.provision(layers, { sandboxId: record.id });
+ const teardown = router.teardown(handle, { destroy: true });
+ try {
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ assert.equal(provisioned, false);
+ assert.equal(destroyed, false);
+ } finally {
+ release.resolve();
+ await Promise.all([running, provision, teardown]);
+ }
+ assert.equal(provisioned, true);
+ assert.equal(destroyed, true);
+});