Multiplayer agent harness for work.
Coalesce subagent completion wakeups until parents are idle (#1480)
Regan Bell68f7efa1706b
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.tsindex 01897d938..6e73a864e 100644--- a/src/runs/memory-run-store.ts+++ b/src/runs/memory-run-store.ts@@ -173,7 +173,11 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti.filter((run) =>isTerminal(run.status) &&- !returned.has(run.id) &&+ (!returned.has(run.id) ||+ (() => {+ const wake = runs.get(byKey.get(`subagent-return:${run.id}`) ?? "");+ return wake?.status === "pending" && wake.attempts === 0 && wake.turnUserSeq === null;+ })()) &&run.id > afterId &&run.sessionId.startsWith("agent:main:subagent:"),)@@ -219,9 +223,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntireturn true;},- async withdraw(runId) {+ async withdraw(runId, opts) {const run = runs.get(runId);if (!run || run.status !== "pending") return false;+ if (opts?.unstartedOnly && (run.attempts !== 0 || run.turnUserSeq !== null)) return false;runs.delete(runId);retryAfter.delete(runId);if (run.dedupKey) byKey.delete(run.dedupKey);diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex 780b3703f..afab775d2 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -372,9 +372,17 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla},async pendingReturns(limit = 100, afterId = "") {const { rows } = await q(- `SELECT * FROM runs WHERE status IN ('done','failed') AND returned_at IS NULL- AND session_id LIKE 'agent:main:subagent:%'- AND id > $2 ORDER BY id LIMIT $1`,+ `SELECT * FROM (+ SELECT * FROM runs WHERE status IN ('done','failed') AND returned_at IS NULL+ AND session_id LIKE 'agent:main:subagent:%' AND id > $2+ UNION+ SELECT child.* FROM runs wake JOIN runs child+ ON child.id = substring(wake.idempotency_key FROM length('subagent-return:') + 1)+ WHERE wake.status = 'pending' AND wake.attempts = 0 AND wake.turn_user_seq IS NULL+ AND wake.idempotency_key LIKE 'subagent-return:%'+ AND child.status IN ('done','failed') AND child.session_id LIKE 'agent:main:subagent:%'+ AND child.id > $2+ ) pending ORDER BY id LIMIT $1`,[limit, afterId],);return rows.map(rowToRun);@@ -414,8 +422,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClareturn (rowCount ?? 0) > 0;},- async withdraw(runId: string): Promise<boolean> {- const { rowCount } = await q("DELETE FROM runs WHERE id = $1 AND status = 'pending'", [runId]);+ async withdraw(runId: string, opts): Promise<boolean> {+ const { rowCount } = await q(+ "DELETE FROM runs WHERE id = $1 AND status = 'pending' AND (NOT $2::boolean OR (attempts = 0 AND turn_user_seq IS NULL))",+ [runId, Boolean(opts?.unstartedOnly)],+ );return (rowCount ?? 0) > 0;},diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 701067398..4cc0fabc1 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -93,7 +93,7 @@ export interface RunStore {inFlightForThread(sessionId: string): Promise<Run[]>;- withdraw(runId: string): Promise<boolean>;+ withdraw(runId: string, opts?: { unstartedOnly?: boolean }): Promise<boolean>;steerQueued(queuedRunId: string, targetRunId: string, signal: RunSignal, signals: RunSignalStore): Promise<boolean>;editPendingText(runId: string, text: string, expectedText: string): Promise<boolean>;diff --git a/src/sessions/session-syscalls.ts b/src/sessions/session-syscalls.tsindex f142dc821..1b0d7f3f1 100644--- a/src/sessions/session-syscalls.ts+++ b/src/sessions/session-syscalls.ts@@ -769,17 +769,18 @@ export interface SubagentMailDeps {delegationEnabled?: (actorId: string) => Promise<boolean>;mailbox: SessionMailbox;sessions: Pick<SessionStore, "get" | "getByThread" | "getEntries" | "latestEntrySeq" | "visibleEntries">;- runs: Pick<RunStore, "enqueue"> & Partial<Pick<RunStore, "latestForThread" | "get">>;+ runs: Pick<RunStore, "enqueue" | "inFlightForThread" | "getByDedupKey" | "withdraw"> &+ Partial<Pick<RunStore, "latestForThread" | "get">>;maxAttempts: number;prepareRequest?: (request: OrchestratorInput) => Promise<OrchestratorInput>;}-export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Promise<void> {- if (!isSubagentThreadRef(run.sessionId) || run.request.privateSessionMessage) return;+export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Promise<boolean> {+ if (!isSubagentThreadRef(run.sessionId) || run.request.privateSessionMessage) return true;const child = await deps.sessions.getByThread(run.sessionId);- if (!child?.parentSessionId || !child.spawnMeta) return;+ if (!child?.parentSessionId || !child.spawnMeta) return true;const parent = await deps.sessions.get(child.parentSessionId);- if (!parent) return;+ if (!parent) return true;if (parent.threadRef.startsWith("swarm:")) throw new Error("subagent returns cannot enter swarm workers");const latestParent = await deps.runs.latestForThread?.(parent.threadRef, { excludePrivateMessages: true });if (parent.scopeId !== child.scopeId) throw new Error("parent and child contexts no longer match");@@ -882,11 +883,20 @@ export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Pro(await deps.delegationEnabled?.(prepared.actor.id)) === true,)) {+ const dedupKey = `subagent-return:${run.id}`;+ const existing = await deps.runs.getByDedupKey(dedupKey);+ const unread = (await deps.mailbox.pending(parent.id)).some((message) => message.id === `subagent-mail-${run.id}`);+ if (!unread) {+ if (existing?.status === "pending") await deps.runs.withdraw(existing.id, { unstartedOnly: true });+ return true;+ }+ if (existing && (existing.status === "done" || existing.status === "failed")) return true;+ if ((await deps.runs.inFlightForThread(parent.threadRef)).length) return false;const wake ="A delegated task finished. Check internal messages with session wait (timeoutMs: 0), then report any new result or blocker relevant to the user's request. If it was already handled or no message is available, end without posting.";await deps.runs.enqueue({sessionId: parent.threadRef,- dedupKey: `subagent-return:${run.id}`,+ dedupKey,request: {...prepared,surfaceTools: true,@@ -902,5 +912,7 @@ export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Pro},maxAttempts: deps.maxAttempts,});+ return false;}+ return true;}diff --git a/src/wiring.ts b/src/wiring.tsindex de987eb31..782b649e7 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -2113,7 +2113,7 @@ export function buildApp(let lastSignalPrune = 0;const returnSessionRun = (run: Run) =>advisoryLock.withLock("session-tree-admission", async () => {- await deliverSubagentMail(+ const settled = await deliverSubagentMail({sessions,runs,@@ -2124,7 +2124,7 @@ export function buildApp(},run,);- await runs.markReturned(run.id);+ if (settled) await runs.markReturned(run.id);});runs.onTerminal((run) => {if (run.sessionId.startsWith("agent:main:subagent:"))diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.tsindex 532d2686a..888dc3af7 100644--- a/test/postgres-store.test.ts+++ b/test/postgres-store.test.ts@@ -2567,3 +2567,53 @@ test("pg session status survives restart, is shared, and clears", { skip }, asynawait restarted.updateStatus(session.id, null);assert.equal((await first.get(session.id))?.status ?? null, null);});++test(+ "pg legacy completion wakeups remain discoverable after return and restart until withdrawn",+ { skip },+ async () => {+ const first = createPostgresRunStore(URL!);+ const { run } = await first.runs.enqueue({+ sessionId: `agent:main:subagent:${randomUUID()}`,+ request: turn("child"),+ });+ const claimed = await first.runs.claimById(run.id, "child", 30_000);+ await first.runs.complete(run.id, claimed!.leaseToken!, { status: "ok", reply: "done" });+ const { run: wake } = await first.runs.enqueue({+ sessionId: `parent-${randomUUID()}`,+ dedupKey: `subagent-return:${run.id}`,+ request: turn("completion"),+ });+ await first.runs.markReturned(run.id);+ await first.close();+ const second = createPostgresRunStore(URL!);+ try {+ assert.ok((await second.runs.pendingReturns(1000)).some((pending) => pending.id === run.id));+ assert.ok(!(await second.runs.pendingReturns(1000, run.id)).some((pending) => pending.id === run.id));+ assert.equal(await second.runs.withdraw(wake.id), true);+ assert.ok(!(await second.runs.pendingReturns(1000)).some((pending) => pending.id === run.id));+ } finally {+ await second.close();+ }+ },+);++test("pg unstarted withdrawal preserves claimed and released turns atomically", { skip }, async () => {+ const store = createPostgresRunStore(URL!);+ try {+ const { run } = await store.runs.enqueue({ sessionId: `wake-retry-${randomUUID()}`, request: turn("completion") });+ const claimed = await store.runs.claimById(run.id, "worker", 30_000);+ assert.equal(await store.runs.withdraw(run.id, { unstartedOnly: true }), false);+ await store.runs.releaseLease(run.id, claimed!.leaseToken!);+ assert.equal(await store.runs.withdraw(run.id, { unstartedOnly: true }), false);+ assert.equal((await store.runs.get(run.id))!.status, "pending");+ const { run: fresh } = await store.runs.enqueue({+ sessionId: `wake-fresh-${randomUUID()}`,+ request: turn("completion"),+ });+ assert.equal(await store.runs.withdraw(fresh.id, { unstartedOnly: true }), true);+ assert.equal(await store.runs.get(fresh.id), null);+ } finally {+ await store.close();+ }+});diff --git a/test/session-syscalls.test.ts b/test/session-syscalls.test.tsindex 3ab9e6bdd..b20fbaaba 100644--- a/test/session-syscalls.test.ts+++ b/test/session-syscalls.test.ts@@ -12,6 +12,7 @@ import {renderSubagentMail,SUBAGENT_TREE_RUN_CAP,requiresDelegation,+ sessionTreeRunCount,delegatedAuthorizationOrigin,} from "../src/sessions/session-syscalls.ts";import { scopeId, type Conversation, type Principal, type ScopeId, type Session } from "../src/types.ts";@@ -428,8 +429,17 @@ test("terminal children remain recoverable until their return is acknowledged",assert.equal(wakes[0]!.dedupKey, `subagent-return:${pending!.id}`);assert.match(wakes[0]!.request.text, /Check internal messages/);assert.equal(wakes[0]!.request.surfaceTools, true);+ await r.mailbox.acknowledge(r.room.id, [`subagent-mail-${pending!.id}`]);+ assert.equal(+ await deliverSubagentMail(+ { mailbox: r.mailbox, sessions: r.sessions, runs: r.runs, maxAttempts: 3, delegationEnabled: async () => true },+ pending!,+ ),+ true,+ );await r.runs.markReturned(pending!.id);assert.deepEqual(await r.runs.pendingReturns(), []);+ assert.deepEqual(await r.runs.inFlightForThread(r.room.threadRef), []);});test("adopting into a fresh web parent never delivers to the old parent's surface", async () => {@@ -837,7 +847,11 @@ test("queued results recheck source-entry visibility after participant tenure chtest("enabled coordinators and their children can delegate", async () => {const r = await rig();const factory = createSessionSyscalls({ ...r, maxAttempts: 3, enabled: async () => true });- const api = factory.forTurn({ session: r.room, scopeId: scope, request: { actor, conversation, surface: "slack" } });+ const api = factory.forTurn({+ session: r.room,+ scopeId: scope,+ request: { actor, conversation, surface: "slack" },+ });const opened = await api.open({ task: "Inspect the logs" });assert.ok(opened.ok);const child = await freshSession(r.sessions, opened.sessionId);@@ -876,6 +890,11 @@ test("completion wakes preserve a spine trigger's delivery destination", async (const queued = (await r.runs.inFlightForThread(child.threadRef))[0]!;const claimed = await r.runs.claimById(queued.id, "w1", 60000);await r.runs.complete(queued.id, claimed!.leaseToken!, { status: "ok", reply: "done" });+ for (const pending of await r.runs.inFlightForThread(r.room.threadRef)) {+ const parentTurn = await r.runs.claimById(pending.id, "parent", 60_000);+ assert.ok(parentTurn);+ await r.runs.complete(pending.id, parentTurn.leaseToken!, { status: "silent" });+ }await deliverSubagentMail({ ...r, maxAttempts: 3, delegationEnabled: async () => true },(await r.runs.get(queued.id))!,@@ -1081,6 +1100,11 @@ test("a later participant cannot change a completion's actor, rollout, destinati});const claimed = await r.runs.claimById(queued.id, "w1", 60000);await r.runs.complete(queued.id, claimed!.leaseToken!, { status: "ok", reply: "done" });+ for (const pending of await r.runs.inFlightForThread(room.threadRef)) {+ const parentTurn = await r.runs.claimById(pending.id, "parent", 60_000);+ assert.ok(parentTurn);+ await r.runs.complete(pending.id, parentTurn.leaseToken!, { status: "silent" });+ }await deliverSubagentMail({ ...r, maxAttempts: 3, delegationEnabled: async (id) => id === actor.id },(await r.runs.get(queued.id))!,@@ -1097,6 +1121,11 @@ test("completion wakes preserve live authorization for subsequent delegated stepconst r = await delegatedHumanRig();const claimed = await r.runs.claimById(r.run.id, "w1", 60000);await r.runs.complete(r.run.id, claimed!.leaseToken!, { status: "ok", reply: "step one" });+ for (const pending of await r.runs.inFlightForThread(r.room.threadRef)) {+ const parentTurn = await r.runs.claimById(pending.id, "parent", 60_000);+ assert.ok(parentTurn);+ await r.runs.complete(pending.id, parentTurn.leaseToken!, { status: "silent" });+ }await deliverSubagentMail({ ...r, maxAttempts: 3, delegationEnabled: async () => true },(await r.runs.get(r.run.id))!,@@ -1117,3 +1146,128 @@ test("completion wakes preserve live authorization for subsequent delegated stepassert.equal(await delegatedAuthorizationOrigin(wake.request, r), undefined);assert.equal(await delegatedAuthorizationOrigin(next.request, r), undefined);});++test("completed children do not fill a working parent's run slots and consumed returns settle", async () => {+ const r = await rig();+ const parent = await r.runs.enqueue({+ sessionId: r.room.threadRef,+ request: { actor, conversation, surface: "slack", origin: { kind: "human" }, text: "work" },+ });+ await r.runs.claimById(parent.run.id, "parent", 60_000);+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };+ for (let i = 0; i < 12; i++) {+ const opened = await r.syscallsFor(r.room).open({ task: `work ${i}` });+ assert.ok(opened.ok);+ const child = await freshSession(r.sessions, opened.sessionId);+ const [run] = await r.runs.inFlightForThread(child.threadRef);+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: `result ${i}` });+ const finished = (await r.runs.get(run!.id))!;+ assert.equal(await deliverSubagentMail(deps, finished), false);+ assert.equal(await sessionTreeRunCount(r.sessions, r.runs, r.room), 1);+ }+ const pending = await r.runs.pendingReturns();+ assert.equal(pending.length, 12);+ let received = 0;+ while (received < 12) {+ const messages = await r.syscallsFor(r.room).receive!();+ assert.ok(messages.length > 0);+ received += messages.length;+ await r.syscallsFor(r.room).acknowledge!(messages.map((message) => message.id));+ }+ assert.equal(received, 12);+ for (const run of pending) {+ assert.equal(await deliverSubagentMail(deps, run), true);+ await r.runs.markReturned(run.id);+ }+ assert.deepEqual(await r.runs.pendingReturns(), []);+ assert.equal(await sessionTreeRunCount(r.sessions, r.runs, r.room), 1);+});++test("unread completions survive parent shutdown and share one pending wakeup", async () => {+ const r = await rig();+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };+ const parent = await r.runs.enqueue({+ sessionId: r.room.threadRef,+ request: { actor, conversation, surface: "slack", origin: { kind: "human" }, text: "work" },+ });+ const active = await r.runs.claimById(parent.run.id, "parent", 60_000);+ assert.deepEqual(await r.syscallsFor(r.room).receive!(), []);+ for (let i = 0; i < 3; i++) {+ const opened = await r.syscallsFor(r.room).open({ task: `late result ${i}` });+ assert.ok(opened.ok);+ const child = await freshSession(r.sessions, opened.sessionId);+ const [run] = await r.runs.inFlightForThread(child.threadRef);+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: `late ${i}` });+ assert.equal(await deliverSubagentMail(deps, (await r.runs.get(run!.id))!), false);+ }+ await r.runs.complete(parent.run.id, active!.leaseToken!, { status: "silent" });+ for (const run of await r.runs.pendingReturns()) assert.equal(await deliverSubagentMail(deps, run), false);+ const wakes = await r.runs.inFlightForThread(r.room.threadRef);+ assert.equal(wakes.length, 1);+ const wake = await r.runs.claimById(wakes[0]!.id, "parent-restarted", 60_000);+ const mail = await r.syscallsFor(r.room).receive!();+ assert.equal(mail.length, 3);+ await r.syscallsFor(r.room).acknowledge!(mail.map((message) => message.id));+ for (const run of await r.runs.pendingReturns()) {+ assert.equal(await deliverSubagentMail(deps, run), true);+ await r.runs.markReturned(run.id);+ }+ assert.equal((await r.runs.get(wake!.id))!.status, "running");+ await r.runs.complete(wake!.id, wake!.leaseToken!, { status: "silent" });+ assert.deepEqual(await r.runs.pendingReturns(), []);+ assert.deepEqual(await r.runs.inFlightForThread(r.room.threadRef), []);+});++test("legacy returned children with consumed mail release queued wakeups without cancelling user work", async () => {+ const r = await rig();+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };+ const opened = await r.syscallsFor(r.room).open({ task: "old completion" });+ assert.ok(opened.ok);+ const child = await freshSession(r.sessions, opened.sessionId);+ const [run] = await r.runs.inFlightForThread(child.threadRef);+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: "done" });+ const finished = (await r.runs.get(run!.id))!;+ assert.equal(await deliverSubagentMail(deps, finished), false);+ await r.runs.markReturned(finished.id);+ const user = await r.runs.enqueue({+ sessionId: r.room.threadRef,+ request: { actor, conversation, origin: { kind: "human" }, text: "new user work" },+ });+ await r.mailbox.acknowledge(r.room.id, [`subagent-mail-${finished.id}`]);+ assert.deepEqual(+ (await r.runs.pendingReturns()).map((pending) => pending.id),+ [finished.id],+ );+ assert.equal(await deliverSubagentMail(deps, finished), true);+ await r.runs.markReturned(finished.id);+ assert.deepEqual(await r.runs.pendingReturns(), []);+ assert.deepEqual(+ (await r.runs.inFlightForThread(r.room.threadRef)).map((pending) => pending.id),+ [user.run.id],+ );+});++test("a completion wake that consumed mail and then retries retains its turn and delegation authority", async () => {+ const r = await rig();+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };+ const opened = await r.syscallsFor(r.room).open({ task: "finish" });+ assert.ok(opened.ok);+ const child = await freshSession(r.sessions, opened.sessionId);+ const [run] = await r.runs.inFlightForThread(child.threadRef);+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: "done" });+ const finished = (await r.runs.get(run!.id))!;+ await deliverSubagentMail(deps, finished);+ const wake = (await r.runs.inFlightForThread(r.room.threadRef))[0]!;+ const running = await r.runs.claimById(wake.id, "parent", 60_000);+ await r.mailbox.acknowledge(r.room.id, [`subagent-mail-${finished.id}`]);+ await r.runs.fail(wake.id, running!.leaseToken!, "transient failure", { retryAfterMs: 1000 });+ assert.equal(await deliverSubagentMail(deps, finished), true);+ await r.runs.markReturned(finished.id);+ assert.equal((await r.runs.get(wake.id))!.status, "pending");+ assert.equal((await r.runs.get(wake.id))!.attempts, 1);+ assert.deepEqual(await r.runs.pendingReturns(), []);+});