Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Coalesce subagent completion wakeups until parents are idle (#1480)

Regan Bell68f7efa1706b
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.ts
index 01897d938..6e73a864e 100644
--- a/src/runs/memory-run-store.ts
+++ b/src/runs/memory-run-store.ts
@@ -173,7 +173,11 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
.filter(
(run) =>
isTerminal(run.status) &&
- !returned.has(run.id) &&
+ (!returned.has(run.id) ||
+ (() => {
+ const wake = runs.get(byKey.get(`subagent-return:${run.id}`) ?? "");
+ return wake?.status === "pending" && wake.attempts === 0 && wake.turnUserSeq === null;
+ })()) &&
run.id > afterId &&
run.sessionId.startsWith("agent:main:subagent:"),
)
@@ -219,9 +223,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
return true;
},
- async withdraw(runId) {
+ async withdraw(runId, opts) {
const run = runs.get(runId);
if (!run || run.status !== "pending") return false;
+ if (opts?.unstartedOnly && (run.attempts !== 0 || run.turnUserSeq !== null)) return false;
runs.delete(runId);
retryAfter.delete(runId);
if (run.dedupKey) byKey.delete(run.dedupKey);
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index 780b3703f..afab775d2 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -372,9 +372,17 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
},
async pendingReturns(limit = 100, afterId = "") {
const { rows } = await q(
- `SELECT * FROM runs WHERE status IN ('done','failed') AND returned_at IS NULL
- AND session_id LIKE 'agent:main:subagent:%'
- AND id > $2 ORDER BY id LIMIT $1`,
+ `SELECT * FROM (
+ SELECT * FROM runs WHERE status IN ('done','failed') AND returned_at IS NULL
+ AND session_id LIKE 'agent:main:subagent:%' AND id > $2
+ UNION
+ SELECT child.* FROM runs wake JOIN runs child
+ ON child.id = substring(wake.idempotency_key FROM length('subagent-return:') + 1)
+ WHERE wake.status = 'pending' AND wake.attempts = 0 AND wake.turn_user_seq IS NULL
+ AND wake.idempotency_key LIKE 'subagent-return:%'
+ AND child.status IN ('done','failed') AND child.session_id LIKE 'agent:main:subagent:%'
+ AND child.id > $2
+ ) pending ORDER BY id LIMIT $1`,
[limit, afterId],
);
return rows.map(rowToRun);
@@ -414,8 +422,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
return (rowCount ?? 0) > 0;
},
- async withdraw(runId: string): Promise<boolean> {
- const { rowCount } = await q("DELETE FROM runs WHERE id = $1 AND status = 'pending'", [runId]);
+ async withdraw(runId: string, opts): Promise<boolean> {
+ const { rowCount } = await q(
+ "DELETE FROM runs WHERE id = $1 AND status = 'pending' AND (NOT $2::boolean OR (attempts = 0 AND turn_user_seq IS NULL))",
+ [runId, Boolean(opts?.unstartedOnly)],
+ );
return (rowCount ?? 0) > 0;
},
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 701067398..4cc0fabc1 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -93,7 +93,7 @@ export interface RunStore {
inFlightForThread(sessionId: string): Promise<Run[]>;
- withdraw(runId: string): Promise<boolean>;
+ withdraw(runId: string, opts?: { unstartedOnly?: boolean }): Promise<boolean>;
steerQueued(queuedRunId: string, targetRunId: string, signal: RunSignal, signals: RunSignalStore): Promise<boolean>;
editPendingText(runId: string, text: string, expectedText: string): Promise<boolean>;
diff --git a/src/sessions/session-syscalls.ts b/src/sessions/session-syscalls.ts
index f142dc821..1b0d7f3f1 100644
--- a/src/sessions/session-syscalls.ts
+++ b/src/sessions/session-syscalls.ts
@@ -769,17 +769,18 @@ export interface SubagentMailDeps {
delegationEnabled?: (actorId: string) => Promise<boolean>;
mailbox: SessionMailbox;
sessions: Pick<SessionStore, "get" | "getByThread" | "getEntries" | "latestEntrySeq" | "visibleEntries">;
- runs: Pick<RunStore, "enqueue"> & Partial<Pick<RunStore, "latestForThread" | "get">>;
+ runs: Pick<RunStore, "enqueue" | "inFlightForThread" | "getByDedupKey" | "withdraw"> &
+ Partial<Pick<RunStore, "latestForThread" | "get">>;
maxAttempts: number;
prepareRequest?: (request: OrchestratorInput) => Promise<OrchestratorInput>;
}
-export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Promise<void> {
- if (!isSubagentThreadRef(run.sessionId) || run.request.privateSessionMessage) return;
+export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Promise<boolean> {
+ if (!isSubagentThreadRef(run.sessionId) || run.request.privateSessionMessage) return true;
const child = await deps.sessions.getByThread(run.sessionId);
- if (!child?.parentSessionId || !child.spawnMeta) return;
+ if (!child?.parentSessionId || !child.spawnMeta) return true;
const parent = await deps.sessions.get(child.parentSessionId);
- if (!parent) return;
+ if (!parent) return true;
if (parent.threadRef.startsWith("swarm:")) throw new Error("subagent returns cannot enter swarm workers");
const latestParent = await deps.runs.latestForThread?.(parent.threadRef, { excludePrivateMessages: true });
if (parent.scopeId !== child.scopeId) throw new Error("parent and child contexts no longer match");
@@ -882,11 +883,20 @@ export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Pro
(await deps.delegationEnabled?.(prepared.actor.id)) === true,
)
) {
+ const dedupKey = `subagent-return:${run.id}`;
+ const existing = await deps.runs.getByDedupKey(dedupKey);
+ const unread = (await deps.mailbox.pending(parent.id)).some((message) => message.id === `subagent-mail-${run.id}`);
+ if (!unread) {
+ if (existing?.status === "pending") await deps.runs.withdraw(existing.id, { unstartedOnly: true });
+ return true;
+ }
+ if (existing && (existing.status === "done" || existing.status === "failed")) return true;
+ if ((await deps.runs.inFlightForThread(parent.threadRef)).length) return false;
const wake =
"A delegated task finished. Check internal messages with session wait (timeoutMs: 0), then report any new result or blocker relevant to the user's request. If it was already handled or no message is available, end without posting.";
await deps.runs.enqueue({
sessionId: parent.threadRef,
- dedupKey: `subagent-return:${run.id}`,
+ dedupKey,
request: {
...prepared,
surfaceTools: true,
@@ -902,5 +912,7 @@ export async function deliverSubagentMail(deps: SubagentMailDeps, run: Run): Pro
},
maxAttempts: deps.maxAttempts,
});
+ return false;
}
+ return true;
}
diff --git a/src/wiring.ts b/src/wiring.ts
index de987eb31..782b649e7 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -2113,7 +2113,7 @@ export function buildApp(
let lastSignalPrune = 0;
const returnSessionRun = (run: Run) =>
advisoryLock.withLock("session-tree-admission", async () => {
- await deliverSubagentMail(
+ const settled = await deliverSubagentMail(
{
sessions,
runs,
@@ -2124,7 +2124,7 @@ export function buildApp(
},
run,
);
- await runs.markReturned(run.id);
+ if (settled) await runs.markReturned(run.id);
});
runs.onTerminal((run) => {
if (run.sessionId.startsWith("agent:main:subagent:"))
diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.ts
index 532d2686a..888dc3af7 100644
--- a/test/postgres-store.test.ts
+++ b/test/postgres-store.test.ts
@@ -2567,3 +2567,53 @@ test("pg session status survives restart, is shared, and clears", { skip }, asyn
await restarted.updateStatus(session.id, null);
assert.equal((await first.get(session.id))?.status ?? null, null);
});
+
+test(
+ "pg legacy completion wakeups remain discoverable after return and restart until withdrawn",
+ { skip },
+ async () => {
+ const first = createPostgresRunStore(URL!);
+ const { run } = await first.runs.enqueue({
+ sessionId: `agent:main:subagent:${randomUUID()}`,
+ request: turn("child"),
+ });
+ const claimed = await first.runs.claimById(run.id, "child", 30_000);
+ await first.runs.complete(run.id, claimed!.leaseToken!, { status: "ok", reply: "done" });
+ const { run: wake } = await first.runs.enqueue({
+ sessionId: `parent-${randomUUID()}`,
+ dedupKey: `subagent-return:${run.id}`,
+ request: turn("completion"),
+ });
+ await first.runs.markReturned(run.id);
+ await first.close();
+ const second = createPostgresRunStore(URL!);
+ try {
+ assert.ok((await second.runs.pendingReturns(1000)).some((pending) => pending.id === run.id));
+ assert.ok(!(await second.runs.pendingReturns(1000, run.id)).some((pending) => pending.id === run.id));
+ assert.equal(await second.runs.withdraw(wake.id), true);
+ assert.ok(!(await second.runs.pendingReturns(1000)).some((pending) => pending.id === run.id));
+ } finally {
+ await second.close();
+ }
+ },
+);
+
+test("pg unstarted withdrawal preserves claimed and released turns atomically", { skip }, async () => {
+ const store = createPostgresRunStore(URL!);
+ try {
+ const { run } = await store.runs.enqueue({ sessionId: `wake-retry-${randomUUID()}`, request: turn("completion") });
+ const claimed = await store.runs.claimById(run.id, "worker", 30_000);
+ assert.equal(await store.runs.withdraw(run.id, { unstartedOnly: true }), false);
+ await store.runs.releaseLease(run.id, claimed!.leaseToken!);
+ assert.equal(await store.runs.withdraw(run.id, { unstartedOnly: true }), false);
+ assert.equal((await store.runs.get(run.id))!.status, "pending");
+ const { run: fresh } = await store.runs.enqueue({
+ sessionId: `wake-fresh-${randomUUID()}`,
+ request: turn("completion"),
+ });
+ assert.equal(await store.runs.withdraw(fresh.id, { unstartedOnly: true }), true);
+ assert.equal(await store.runs.get(fresh.id), null);
+ } finally {
+ await store.close();
+ }
+});
diff --git a/test/session-syscalls.test.ts b/test/session-syscalls.test.ts
index 3ab9e6bdd..b20fbaaba 100644
--- a/test/session-syscalls.test.ts
+++ b/test/session-syscalls.test.ts
@@ -12,6 +12,7 @@ import {
renderSubagentMail,
SUBAGENT_TREE_RUN_CAP,
requiresDelegation,
+ sessionTreeRunCount,
delegatedAuthorizationOrigin,
} from "../src/sessions/session-syscalls.ts";
import { scopeId, type Conversation, type Principal, type ScopeId, type Session } from "../src/types.ts";
@@ -428,8 +429,17 @@ test("terminal children remain recoverable until their return is acknowledged",
assert.equal(wakes[0]!.dedupKey, `subagent-return:${pending!.id}`);
assert.match(wakes[0]!.request.text, /Check internal messages/);
assert.equal(wakes[0]!.request.surfaceTools, true);
+ await r.mailbox.acknowledge(r.room.id, [`subagent-mail-${pending!.id}`]);
+ assert.equal(
+ await deliverSubagentMail(
+ { mailbox: r.mailbox, sessions: r.sessions, runs: r.runs, maxAttempts: 3, delegationEnabled: async () => true },
+ pending!,
+ ),
+ true,
+ );
await r.runs.markReturned(pending!.id);
assert.deepEqual(await r.runs.pendingReturns(), []);
+ assert.deepEqual(await r.runs.inFlightForThread(r.room.threadRef), []);
});
test("adopting into a fresh web parent never delivers to the old parent's surface", async () => {
@@ -837,7 +847,11 @@ test("queued results recheck source-entry visibility after participant tenure ch
test("enabled coordinators and their children can delegate", async () => {
const r = await rig();
const factory = createSessionSyscalls({ ...r, maxAttempts: 3, enabled: async () => true });
- const api = factory.forTurn({ session: r.room, scopeId: scope, request: { actor, conversation, surface: "slack" } });
+ const api = factory.forTurn({
+ session: r.room,
+ scopeId: scope,
+ request: { actor, conversation, surface: "slack" },
+ });
const opened = await api.open({ task: "Inspect the logs" });
assert.ok(opened.ok);
const child = await freshSession(r.sessions, opened.sessionId);
@@ -876,6 +890,11 @@ test("completion wakes preserve a spine trigger's delivery destination", async (
const queued = (await r.runs.inFlightForThread(child.threadRef))[0]!;
const claimed = await r.runs.claimById(queued.id, "w1", 60000);
await r.runs.complete(queued.id, claimed!.leaseToken!, { status: "ok", reply: "done" });
+ for (const pending of await r.runs.inFlightForThread(r.room.threadRef)) {
+ const parentTurn = await r.runs.claimById(pending.id, "parent", 60_000);
+ assert.ok(parentTurn);
+ await r.runs.complete(pending.id, parentTurn.leaseToken!, { status: "silent" });
+ }
await deliverSubagentMail(
{ ...r, maxAttempts: 3, delegationEnabled: async () => true },
(await r.runs.get(queued.id))!,
@@ -1081,6 +1100,11 @@ test("a later participant cannot change a completion's actor, rollout, destinati
});
const claimed = await r.runs.claimById(queued.id, "w1", 60000);
await r.runs.complete(queued.id, claimed!.leaseToken!, { status: "ok", reply: "done" });
+ for (const pending of await r.runs.inFlightForThread(room.threadRef)) {
+ const parentTurn = await r.runs.claimById(pending.id, "parent", 60_000);
+ assert.ok(parentTurn);
+ await r.runs.complete(pending.id, parentTurn.leaseToken!, { status: "silent" });
+ }
await deliverSubagentMail(
{ ...r, maxAttempts: 3, delegationEnabled: async (id) => id === actor.id },
(await r.runs.get(queued.id))!,
@@ -1097,6 +1121,11 @@ test("completion wakes preserve live authorization for subsequent delegated step
const r = await delegatedHumanRig();
const claimed = await r.runs.claimById(r.run.id, "w1", 60000);
await r.runs.complete(r.run.id, claimed!.leaseToken!, { status: "ok", reply: "step one" });
+ for (const pending of await r.runs.inFlightForThread(r.room.threadRef)) {
+ const parentTurn = await r.runs.claimById(pending.id, "parent", 60_000);
+ assert.ok(parentTurn);
+ await r.runs.complete(pending.id, parentTurn.leaseToken!, { status: "silent" });
+ }
await deliverSubagentMail(
{ ...r, maxAttempts: 3, delegationEnabled: async () => true },
(await r.runs.get(r.run.id))!,
@@ -1117,3 +1146,128 @@ test("completion wakes preserve live authorization for subsequent delegated step
assert.equal(await delegatedAuthorizationOrigin(wake.request, r), undefined);
assert.equal(await delegatedAuthorizationOrigin(next.request, r), undefined);
});
+
+test("completed children do not fill a working parent's run slots and consumed returns settle", async () => {
+ const r = await rig();
+ const parent = await r.runs.enqueue({
+ sessionId: r.room.threadRef,
+ request: { actor, conversation, surface: "slack", origin: { kind: "human" }, text: "work" },
+ });
+ await r.runs.claimById(parent.run.id, "parent", 60_000);
+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };
+ for (let i = 0; i < 12; i++) {
+ const opened = await r.syscallsFor(r.room).open({ task: `work ${i}` });
+ assert.ok(opened.ok);
+ const child = await freshSession(r.sessions, opened.sessionId);
+ const [run] = await r.runs.inFlightForThread(child.threadRef);
+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);
+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: `result ${i}` });
+ const finished = (await r.runs.get(run!.id))!;
+ assert.equal(await deliverSubagentMail(deps, finished), false);
+ assert.equal(await sessionTreeRunCount(r.sessions, r.runs, r.room), 1);
+ }
+ const pending = await r.runs.pendingReturns();
+ assert.equal(pending.length, 12);
+ let received = 0;
+ while (received < 12) {
+ const messages = await r.syscallsFor(r.room).receive!();
+ assert.ok(messages.length > 0);
+ received += messages.length;
+ await r.syscallsFor(r.room).acknowledge!(messages.map((message) => message.id));
+ }
+ assert.equal(received, 12);
+ for (const run of pending) {
+ assert.equal(await deliverSubagentMail(deps, run), true);
+ await r.runs.markReturned(run.id);
+ }
+ assert.deepEqual(await r.runs.pendingReturns(), []);
+ assert.equal(await sessionTreeRunCount(r.sessions, r.runs, r.room), 1);
+});
+
+test("unread completions survive parent shutdown and share one pending wakeup", async () => {
+ const r = await rig();
+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };
+ const parent = await r.runs.enqueue({
+ sessionId: r.room.threadRef,
+ request: { actor, conversation, surface: "slack", origin: { kind: "human" }, text: "work" },
+ });
+ const active = await r.runs.claimById(parent.run.id, "parent", 60_000);
+ assert.deepEqual(await r.syscallsFor(r.room).receive!(), []);
+ for (let i = 0; i < 3; i++) {
+ const opened = await r.syscallsFor(r.room).open({ task: `late result ${i}` });
+ assert.ok(opened.ok);
+ const child = await freshSession(r.sessions, opened.sessionId);
+ const [run] = await r.runs.inFlightForThread(child.threadRef);
+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);
+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: `late ${i}` });
+ assert.equal(await deliverSubagentMail(deps, (await r.runs.get(run!.id))!), false);
+ }
+ await r.runs.complete(parent.run.id, active!.leaseToken!, { status: "silent" });
+ for (const run of await r.runs.pendingReturns()) assert.equal(await deliverSubagentMail(deps, run), false);
+ const wakes = await r.runs.inFlightForThread(r.room.threadRef);
+ assert.equal(wakes.length, 1);
+ const wake = await r.runs.claimById(wakes[0]!.id, "parent-restarted", 60_000);
+ const mail = await r.syscallsFor(r.room).receive!();
+ assert.equal(mail.length, 3);
+ await r.syscallsFor(r.room).acknowledge!(mail.map((message) => message.id));
+ for (const run of await r.runs.pendingReturns()) {
+ assert.equal(await deliverSubagentMail(deps, run), true);
+ await r.runs.markReturned(run.id);
+ }
+ assert.equal((await r.runs.get(wake!.id))!.status, "running");
+ await r.runs.complete(wake!.id, wake!.leaseToken!, { status: "silent" });
+ assert.deepEqual(await r.runs.pendingReturns(), []);
+ assert.deepEqual(await r.runs.inFlightForThread(r.room.threadRef), []);
+});
+
+test("legacy returned children with consumed mail release queued wakeups without cancelling user work", async () => {
+ const r = await rig();
+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };
+ const opened = await r.syscallsFor(r.room).open({ task: "old completion" });
+ assert.ok(opened.ok);
+ const child = await freshSession(r.sessions, opened.sessionId);
+ const [run] = await r.runs.inFlightForThread(child.threadRef);
+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);
+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: "done" });
+ const finished = (await r.runs.get(run!.id))!;
+ assert.equal(await deliverSubagentMail(deps, finished), false);
+ await r.runs.markReturned(finished.id);
+ const user = await r.runs.enqueue({
+ sessionId: r.room.threadRef,
+ request: { actor, conversation, origin: { kind: "human" }, text: "new user work" },
+ });
+ await r.mailbox.acknowledge(r.room.id, [`subagent-mail-${finished.id}`]);
+ assert.deepEqual(
+ (await r.runs.pendingReturns()).map((pending) => pending.id),
+ [finished.id],
+ );
+ assert.equal(await deliverSubagentMail(deps, finished), true);
+ await r.runs.markReturned(finished.id);
+ assert.deepEqual(await r.runs.pendingReturns(), []);
+ assert.deepEqual(
+ (await r.runs.inFlightForThread(r.room.threadRef)).map((pending) => pending.id),
+ [user.run.id],
+ );
+});
+
+test("a completion wake that consumed mail and then retries retains its turn and delegation authority", async () => {
+ const r = await rig();
+ const deps = { ...r, maxAttempts: 3, delegationEnabled: async () => true };
+ const opened = await r.syscallsFor(r.room).open({ task: "finish" });
+ assert.ok(opened.ok);
+ const child = await freshSession(r.sessions, opened.sessionId);
+ const [run] = await r.runs.inFlightForThread(child.threadRef);
+ const claimed = await r.runs.claimById(run!.id, "child", 60_000);
+ await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: "done" });
+ const finished = (await r.runs.get(run!.id))!;
+ await deliverSubagentMail(deps, finished);
+ const wake = (await r.runs.inFlightForThread(r.room.threadRef))[0]!;
+ const running = await r.runs.claimById(wake.id, "parent", 60_000);
+ await r.mailbox.acknowledge(r.room.id, [`subagent-mail-${finished.id}`]);
+ await r.runs.fail(wake.id, running!.leaseToken!, "transient failure", { retryAfterMs: 1000 });
+ assert.equal(await deliverSubagentMail(deps, finished), true);
+ await r.runs.markReturned(finished.id);
+ assert.equal((await r.runs.get(wake.id))!.status, "pending");
+ assert.equal((await r.runs.get(wake.id))!.attempts, 1);
+ assert.deepEqual(await r.runs.pendingReturns(), []);
+});