Multiplayer agent harness for work.
Reduce repeated recovery work and run lookup cost (#1644)
Joshua Francef6f072bc942a
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.tsindex 6e73a864e..4fb614d8d 100644--- a/src/runs/memory-run-store.ts+++ b/src/runs/memory-run-store.ts@@ -169,10 +169,12 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti);},async pendingReturns(limit = 100, afterId = "") {+ const now = Date.now();return [...runs.values()].filter((run) =>isTerminal(run.status) &&+ (retryAfter.get(run.id) ?? 0) <= now &&(!returned.has(run.id) ||(() => {const wake = runs.get(byKey.get(`subagent-return:${run.id}`) ?? "");@@ -187,6 +189,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntiasync markReturned(runId) {returned.add(runId);},+ async deferReturn(runId, delayMs) {+ const run = runs.get(runId);+ if (run && isTerminal(run.status)) retryAfter.set(runId, Date.now() + Math.max(0, delayMs));+ },onTerminal(listener) {terminalListeners.push(listener);},diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex 08e07b90f..1d2ca1296 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -119,6 +119,13 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla`CREATE INDEX IF NOT EXISTS idx_runs_pending_child_returns ON runs(id) WHERE status IN ('done','failed') AND returned_at IS NULL AND session_id LIKE 'agent:main:subagent:%'`,],},+ {+ id: "runs/store/0005-session-history",+ statements: [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_session_created_seq ON runs(session_id, created_at DESC, seq DESC)`,+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_created ON runs(created_at DESC)`,+ ],+ },],[{@@ -384,14 +391,20 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaAND wake.idempotency_key LIKE 'subagent-return:%'AND child.status IN ('done','failed') AND child.session_id LIKE 'agent:main:subagent:%'AND child.id > $2- ) pending ORDER BY id LIMIT $1`,- [limit, afterId],+ ) pending WHERE retry_after <= $3 ORDER BY id LIMIT $1`,+ [limit, afterId, Date.now()],);return rows.map(rowToRun);},async markReturned(runId) {await q("UPDATE runs SET returned_at = $2 WHERE id = $1 AND status IN ('done','failed')", [runId, Date.now()]);},+ async deferReturn(runId, delayMs) {+ await q("UPDATE runs SET retry_after = $2 WHERE id = $1 AND status IN ('done','failed')", [+ runId,+ Date.now() + Math.max(0, delayMs),+ ]);+ },onTerminal(listener): void {terminalListeners.push(listener);},diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 4cc0fabc1..185eba9db 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -84,6 +84,7 @@ export interface RunStore {latestForThread(threadRef: string, opts?: { excludePrivateMessages?: boolean }): Promise<Run | null>;pendingReturns(limit?: number, afterId?: string): Promise<Run[]>;markReturned(runId: string): Promise<void>;+ deferReturn(runId: string, delayMs: number): Promise<void>;onTerminal(listener: (run: Run) => void): void;diff --git a/src/wiring.ts b/src/wiring.tsindex 708812da5..815350bc8 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -2214,22 +2214,27 @@ export function buildApp(});let lastSignalPrune = 0;const returnSessionRun = (run: Run) =>- advisoryLock.withLock("session-tree-admission", async () => {- const settled = await deliverSubagentMail(- {- sessions,- runs,- maxAttempts,- mailbox: sessionMailbox,- prepareRequest: prepareSessionRequest,- delegationEnabled: (actorId) => featureFlags.enabled("responsive_spine", scopeId("personal", actorId)),- deliveries,- signals: runSignals,- },- run,- );- if (settled) await runs.markReturned(run.id);- });+ advisoryLock+ .withLock("session-tree-admission", async () => {+ const settled = await deliverSubagentMail(+ {+ sessions,+ runs,+ maxAttempts,+ mailbox: sessionMailbox,+ prepareRequest: prepareSessionRequest,+ delegationEnabled: (actorId) => featureFlags.enabled("responsive_spine", scopeId("personal", actorId)),+ deliveries,+ signals: runSignals,+ },+ run,+ );+ if (settled) await runs.markReturned(run.id);+ })+ .catch(async (error: unknown) => {+ await runs.deferReturn(run.id, 60_000);+ throw error;+ });runs.onTerminal((run) => {if (run.sessionId.startsWith("agent:main:subagent:"))void returnSessionRun(run).catch(swallowAs("sessions: return", undefined));diff --git a/test/run-retry-delay.test.ts b/test/run-retry-delay.test.tsindex 87ddc41f9..eb6951b2e 100644--- a/test/run-retry-delay.test.ts+++ b/test/run-retry-delay.test.ts@@ -78,8 +78,130 @@ for (const backend of ["memory", "postgres"] as const) {}},);++ test(+ `${backend}: failed returns wait durably without delaying new work or acknowledging results`,+ { skip: backend === "postgres" && !process.env.DATABASE_URL },+ async (t) => {+ let connectionString = process.env.DATABASE_URL!;+ let admin: Pool | undefined;+ const schema = `return_retry_${randomUUID().replaceAll("-", "")}`;+ if (backend === "postgres") {+ const pg = (await import("pg")).default;+ admin = new pg.Pool({ connectionString });+ await admin.query(`CREATE SCHEMA ${schema}`);+ const url = new URL(connectionString);+ url.searchParams.set("options", `-c search_path=${schema}`);+ connectionString = url.toString();+ }+ let { runs } = backend === "memory" ? createMemoryRunStore() : createPostgresRunStore(connectionString);+ try {+ const child = (await runs.enqueue({ sessionId: `agent:main:subagent:${randomUUID()}`, request })).run;+ await runs.deferReturn(child.id, 60_000);+ const lease = await runs.claimById(child.id, "worker", 60_000);+ assert.ok(lease?.leaseToken);+ await runs.complete(child.id, lease.leaseToken, { status: "ok", reply: "saved result" });+ const completed = await runs.get(child.id);+ const wake = (await runs.enqueue({ sessionId: randomUUID(), request, dedupKey: `subagent-return:${child.id}` }))+ .run;+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });+ await runs.deferReturn(child.id, 60_000);+ if (backend === "postgres") {+ await runs.close?.();+ runs = createPostgresRunStore(connectionString).runs;+ }+ assert.deepEqual(await runs.pendingReturns(), []);+ assert.deepEqual(await runs.get(child.id), completed);+ const fresh = (await runs.enqueue({ sessionId: `agent:main:subagent:${randomUUID()}`, request })).run;+ const freshLease = await runs.claimById(fresh.id, "worker", 60_000);+ await runs.fail(fresh.id, freshLease!.leaseToken!, "failed task", { retry: false });+ assert.deepEqual(+ (await runs.pendingReturns()).map((run) => run.id),+ [fresh.id],+ );+ await runs.deferReturn(fresh.id, 60_000);+ t.mock.timers.tick(59_999);+ assert.deepEqual(await runs.pendingReturns(), []);+ t.mock.timers.tick(1);+ assert.equal((await runs.pendingReturns()).length, 2);+ await runs.markReturned(child.id);+ await runs.deferReturn(child.id, 60_000);+ assert.ok(!(await runs.pendingReturns()).some((run) => run.id === child.id));+ t.mock.timers.tick(60_000);+ assert.ok((await runs.pendingReturns()).some((run) => run.id === child.id));+ await runs.withdraw(wake.id, { unstartedOnly: true });+ assert.deepEqual(+ (await runs.pendingReturns()).map((run) => run.id),+ [fresh.id],+ );+ } finally {+ t.mock.timers.reset();+ await runs.close?.();+ if (admin) {+ try {+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);+ } finally {+ await admin.end();+ }+ }+ }+ },+ );}+test(+ "postgres: latest session history uses its ordered index and excludes private messages",+ {+ skip: !process.env.DATABASE_URL,+ },+ async () => {+ const pg = (await import("pg")).default;+ const admin = new pg.Pool({ connectionString: process.env.DATABASE_URL });+ const schema = `run_history_${randomUUID().replaceAll("-", "")}`;+ await admin.query(`CREATE SCHEMA ${schema}`);+ const url = new URL(process.env.DATABASE_URL!);+ url.searchParams.set("options", `-c search_path=${schema}`);+ const runtime = createPostgresRunStore(url.toString());+ const pool = new pg.Pool({ connectionString: url.toString() });+ try {+ const sessionId = randomUUID();+ const publicRun = (await runtime.runs.enqueue({ sessionId, request })).run;+ const privateRun = (+ await runtime.runs.enqueue({ sessionId, request: { ...request, privateSessionMessage: true } })+ ).run;+ await pool.query("UPDATE runs SET created_at = 1 WHERE session_id = $1", [sessionId]);+ assert.equal((await runtime.runs.latestForThread(sessionId))?.id, privateRun.id);+ assert.equal((await runtime.runs.latestForThread(sessionId, { excludePrivateMessages: true }))?.id, publicRun.id);+ const client = await pool.connect();+ try {+ await client.query("SET enable_seqscan = off");+ const { rows } = await client.query(+ "EXPLAIN (FORMAT JSON) SELECT * FROM runs WHERE session_id = $1 AND COALESCE(request::jsonb->>'privateSessionMessage', 'false') <> 'true' ORDER BY created_at DESC, seq DESC LIMIT 1",+ [sessionId],+ );+ const plan = JSON.stringify(rows);+ assert.match(plan, /idx_runs_session_created_seq/);+ assert.doesNotMatch(plan, /"Node Type":"Sort"/);+ const recent = await client.query(+ "EXPLAIN (FORMAT JSON) SELECT * FROM runs ORDER BY created_at DESC LIMIT 500",+ );+ assert.match(JSON.stringify(recent.rows), /idx_runs_created/);+ assert.doesNotMatch(JSON.stringify(recent.rows), /"Node Type":"Sort"/);+ } finally {+ client.release();+ }+ } finally {+ await runtime.close();+ await pool.end();+ try {+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);+ } finally {+ await admin.end();+ }+ }+ },+);+test("memory: one claim uses one clock snapshot across the retry boundary", async (t) => {const { runs } = createMemoryRunStore();const first = (await runs.enqueue({ sessionId: "same", request })).run;diff --git a/test/session-syscalls.test.ts b/test/session-syscalls.test.tsindex e6d6be04f..95222dfe9 100644--- a/test/session-syscalls.test.ts+++ b/test/session-syscalls.test.ts@@ -639,7 +639,7 @@ test("writes revalidate access even while a target is running", async () => {assert.deepEqual(await r.signals.takePending(run!.id), []);});-test("completion cannot expand the finished run's audience during roster refresh", async () => {+test("completion cannot expand the finished run's audience and retries after its return delay", async (t) => {const r = await rig();const opened = await r.syscallsFor(r.room).open({ task: "private research" });assert.ok(opened.ok);@@ -647,24 +647,52 @@ test("completion cannot expand the finished run's audience during roster refreshconst [run] = await r.runs.inFlightForThread(child.threadRef);const claimed = await r.runs.claimById(run!.id, "worker", 60_000);await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: "private result" });- await assert.rejects(- deliverSubagentMail(- {- mailbox: r.mailbox,- sessions: r.sessions,- runs: r.runs,- maxAttempts: 3,- prepareRequest: async (request) => ({- ...request,- conversation: { ...request.conversation, audience: [actor, { id: "new-member", type: "internal" }] },- }),- },- (await r.runs.get(run!.id))!,- ),- /audience/,- );+ let expanded = true;+ let attempts = 0;+ const recover = async () => {+ for (const pending of await r.runs.pendingReturns()) {+ attempts++;+ try {+ if (+ await deliverSubagentMail(+ {+ mailbox: r.mailbox,+ sessions: r.sessions,+ runs: r.runs,+ maxAttempts: 3,+ prepareRequest: async (request) => ({+ ...request,+ conversation: {+ ...request.conversation,+ audience: expanded ? [actor, { id: "new-member", type: "internal" }] : [actor],+ },+ }),+ },+ pending,+ )+ )+ await r.runs.markReturned(pending.id);+ } catch (error) {+ await r.runs.deferReturn(pending.id, 60_000);+ throw error;+ }+ }+ };+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });+ await assert.rejects(recover(), /audience/);assert.deepEqual(await r.runs.inFlightForThread(r.room.threadRef), []);- assert.equal((await r.runs.pendingReturns()).length, 1);+ assert.deepEqual(await r.mailbox.pending(r.room.id), []);+ expanded = false;+ for (let i = 0; i < 59; i++) {+ t.mock.timers.tick(1_000);+ await recover();+ }+ assert.equal(attempts, 1);+ t.mock.timers.tick(1_000);+ await recover();+ assert.equal(attempts, 2);+ assert.equal((await r.mailbox.pending(r.room.id)).length, 1);+ assert.deepEqual(await r.runs.pendingReturns(), []);});test("private session replies stay read-only, queue behind running work, and do not return to a parent", async () => {