Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Reduce repeated recovery work and run lookup cost (#1644)

Joshua Francef6f072bc942a
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.ts
index 6e73a864e..4fb614d8d 100644
--- a/src/runs/memory-run-store.ts
+++ b/src/runs/memory-run-store.ts
@@ -169,10 +169,12 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
);
},
async pendingReturns(limit = 100, afterId = "") {
+ const now = Date.now();
return [...runs.values()]
.filter(
(run) =>
isTerminal(run.status) &&
+ (retryAfter.get(run.id) ?? 0) <= now &&
(!returned.has(run.id) ||
(() => {
const wake = runs.get(byKey.get(`subagent-return:${run.id}`) ?? "");
@@ -187,6 +189,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
async markReturned(runId) {
returned.add(runId);
},
+ async deferReturn(runId, delayMs) {
+ const run = runs.get(runId);
+ if (run && isTerminal(run.status)) retryAfter.set(runId, Date.now() + Math.max(0, delayMs));
+ },
onTerminal(listener) {
terminalListeners.push(listener);
},
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index 08e07b90f..1d2ca1296 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -119,6 +119,13 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
`CREATE INDEX IF NOT EXISTS idx_runs_pending_child_returns ON runs(id) WHERE status IN ('done','failed') AND returned_at IS NULL AND session_id LIKE 'agent:main:subagent:%'`,
],
},
+ {
+ id: "runs/store/0005-session-history",
+ statements: [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_session_created_seq ON runs(session_id, created_at DESC, seq DESC)`,
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_created ON runs(created_at DESC)`,
+ ],
+ },
],
[
{
@@ -384,14 +391,20 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
AND wake.idempotency_key LIKE 'subagent-return:%'
AND child.status IN ('done','failed') AND child.session_id LIKE 'agent:main:subagent:%'
AND child.id > $2
- ) pending ORDER BY id LIMIT $1`,
- [limit, afterId],
+ ) pending WHERE retry_after <= $3 ORDER BY id LIMIT $1`,
+ [limit, afterId, Date.now()],
);
return rows.map(rowToRun);
},
async markReturned(runId) {
await q("UPDATE runs SET returned_at = $2 WHERE id = $1 AND status IN ('done','failed')", [runId, Date.now()]);
},
+ async deferReturn(runId, delayMs) {
+ await q("UPDATE runs SET retry_after = $2 WHERE id = $1 AND status IN ('done','failed')", [
+ runId,
+ Date.now() + Math.max(0, delayMs),
+ ]);
+ },
onTerminal(listener): void {
terminalListeners.push(listener);
},
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 4cc0fabc1..185eba9db 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -84,6 +84,7 @@ export interface RunStore {
latestForThread(threadRef: string, opts?: { excludePrivateMessages?: boolean }): Promise<Run | null>;
pendingReturns(limit?: number, afterId?: string): Promise<Run[]>;
markReturned(runId: string): Promise<void>;
+ deferReturn(runId: string, delayMs: number): Promise<void>;
onTerminal(listener: (run: Run) => void): void;
diff --git a/src/wiring.ts b/src/wiring.ts
index 708812da5..815350bc8 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -2214,22 +2214,27 @@ export function buildApp(
});
let lastSignalPrune = 0;
const returnSessionRun = (run: Run) =>
- advisoryLock.withLock("session-tree-admission", async () => {
- const settled = await deliverSubagentMail(
- {
- sessions,
- runs,
- maxAttempts,
- mailbox: sessionMailbox,
- prepareRequest: prepareSessionRequest,
- delegationEnabled: (actorId) => featureFlags.enabled("responsive_spine", scopeId("personal", actorId)),
- deliveries,
- signals: runSignals,
- },
- run,
- );
- if (settled) await runs.markReturned(run.id);
- });
+ advisoryLock
+ .withLock("session-tree-admission", async () => {
+ const settled = await deliverSubagentMail(
+ {
+ sessions,
+ runs,
+ maxAttempts,
+ mailbox: sessionMailbox,
+ prepareRequest: prepareSessionRequest,
+ delegationEnabled: (actorId) => featureFlags.enabled("responsive_spine", scopeId("personal", actorId)),
+ deliveries,
+ signals: runSignals,
+ },
+ run,
+ );
+ if (settled) await runs.markReturned(run.id);
+ })
+ .catch(async (error: unknown) => {
+ await runs.deferReturn(run.id, 60_000);
+ throw error;
+ });
runs.onTerminal((run) => {
if (run.sessionId.startsWith("agent:main:subagent:"))
void returnSessionRun(run).catch(swallowAs("sessions: return", undefined));
diff --git a/test/run-retry-delay.test.ts b/test/run-retry-delay.test.ts
index 87ddc41f9..eb6951b2e 100644
--- a/test/run-retry-delay.test.ts
+++ b/test/run-retry-delay.test.ts
@@ -78,8 +78,130 @@ for (const backend of ["memory", "postgres"] as const) {
}
},
);
+
+ test(
+ `${backend}: failed returns wait durably without delaying new work or acknowledging results`,
+ { skip: backend === "postgres" && !process.env.DATABASE_URL },
+ async (t) => {
+ let connectionString = process.env.DATABASE_URL!;
+ let admin: Pool | undefined;
+ const schema = `return_retry_${randomUUID().replaceAll("-", "")}`;
+ if (backend === "postgres") {
+ const pg = (await import("pg")).default;
+ admin = new pg.Pool({ connectionString });
+ await admin.query(`CREATE SCHEMA ${schema}`);
+ const url = new URL(connectionString);
+ url.searchParams.set("options", `-c search_path=${schema}`);
+ connectionString = url.toString();
+ }
+ let { runs } = backend === "memory" ? createMemoryRunStore() : createPostgresRunStore(connectionString);
+ try {
+ const child = (await runs.enqueue({ sessionId: `agent:main:subagent:${randomUUID()}`, request })).run;
+ await runs.deferReturn(child.id, 60_000);
+ const lease = await runs.claimById(child.id, "worker", 60_000);
+ assert.ok(lease?.leaseToken);
+ await runs.complete(child.id, lease.leaseToken, { status: "ok", reply: "saved result" });
+ const completed = await runs.get(child.id);
+ const wake = (await runs.enqueue({ sessionId: randomUUID(), request, dedupKey: `subagent-return:${child.id}` }))
+ .run;
+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });
+ await runs.deferReturn(child.id, 60_000);
+ if (backend === "postgres") {
+ await runs.close?.();
+ runs = createPostgresRunStore(connectionString).runs;
+ }
+ assert.deepEqual(await runs.pendingReturns(), []);
+ assert.deepEqual(await runs.get(child.id), completed);
+ const fresh = (await runs.enqueue({ sessionId: `agent:main:subagent:${randomUUID()}`, request })).run;
+ const freshLease = await runs.claimById(fresh.id, "worker", 60_000);
+ await runs.fail(fresh.id, freshLease!.leaseToken!, "failed task", { retry: false });
+ assert.deepEqual(
+ (await runs.pendingReturns()).map((run) => run.id),
+ [fresh.id],
+ );
+ await runs.deferReturn(fresh.id, 60_000);
+ t.mock.timers.tick(59_999);
+ assert.deepEqual(await runs.pendingReturns(), []);
+ t.mock.timers.tick(1);
+ assert.equal((await runs.pendingReturns()).length, 2);
+ await runs.markReturned(child.id);
+ await runs.deferReturn(child.id, 60_000);
+ assert.ok(!(await runs.pendingReturns()).some((run) => run.id === child.id));
+ t.mock.timers.tick(60_000);
+ assert.ok((await runs.pendingReturns()).some((run) => run.id === child.id));
+ await runs.withdraw(wake.id, { unstartedOnly: true });
+ assert.deepEqual(
+ (await runs.pendingReturns()).map((run) => run.id),
+ [fresh.id],
+ );
+ } finally {
+ t.mock.timers.reset();
+ await runs.close?.();
+ if (admin) {
+ try {
+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);
+ } finally {
+ await admin.end();
+ }
+ }
+ }
+ },
+ );
}
+test(
+ "postgres: latest session history uses its ordered index and excludes private messages",
+ {
+ skip: !process.env.DATABASE_URL,
+ },
+ async () => {
+ const pg = (await import("pg")).default;
+ const admin = new pg.Pool({ connectionString: process.env.DATABASE_URL });
+ const schema = `run_history_${randomUUID().replaceAll("-", "")}`;
+ await admin.query(`CREATE SCHEMA ${schema}`);
+ const url = new URL(process.env.DATABASE_URL!);
+ url.searchParams.set("options", `-c search_path=${schema}`);
+ const runtime = createPostgresRunStore(url.toString());
+ const pool = new pg.Pool({ connectionString: url.toString() });
+ try {
+ const sessionId = randomUUID();
+ const publicRun = (await runtime.runs.enqueue({ sessionId, request })).run;
+ const privateRun = (
+ await runtime.runs.enqueue({ sessionId, request: { ...request, privateSessionMessage: true } })
+ ).run;
+ await pool.query("UPDATE runs SET created_at = 1 WHERE session_id = $1", [sessionId]);
+ assert.equal((await runtime.runs.latestForThread(sessionId))?.id, privateRun.id);
+ assert.equal((await runtime.runs.latestForThread(sessionId, { excludePrivateMessages: true }))?.id, publicRun.id);
+ const client = await pool.connect();
+ try {
+ await client.query("SET enable_seqscan = off");
+ const { rows } = await client.query(
+ "EXPLAIN (FORMAT JSON) SELECT * FROM runs WHERE session_id = $1 AND COALESCE(request::jsonb->>'privateSessionMessage', 'false') <> 'true' ORDER BY created_at DESC, seq DESC LIMIT 1",
+ [sessionId],
+ );
+ const plan = JSON.stringify(rows);
+ assert.match(plan, /idx_runs_session_created_seq/);
+ assert.doesNotMatch(plan, /"Node Type":"Sort"/);
+ const recent = await client.query(
+ "EXPLAIN (FORMAT JSON) SELECT * FROM runs ORDER BY created_at DESC LIMIT 500",
+ );
+ assert.match(JSON.stringify(recent.rows), /idx_runs_created/);
+ assert.doesNotMatch(JSON.stringify(recent.rows), /"Node Type":"Sort"/);
+ } finally {
+ client.release();
+ }
+ } finally {
+ await runtime.close();
+ await pool.end();
+ try {
+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);
+ } finally {
+ await admin.end();
+ }
+ }
+ },
+);
+
test("memory: one claim uses one clock snapshot across the retry boundary", async (t) => {
const { runs } = createMemoryRunStore();
const first = (await runs.enqueue({ sessionId: "same", request })).run;
diff --git a/test/session-syscalls.test.ts b/test/session-syscalls.test.ts
index e6d6be04f..95222dfe9 100644
--- a/test/session-syscalls.test.ts
+++ b/test/session-syscalls.test.ts
@@ -639,7 +639,7 @@ test("writes revalidate access even while a target is running", async () => {
assert.deepEqual(await r.signals.takePending(run!.id), []);
});
-test("completion cannot expand the finished run's audience during roster refresh", async () => {
+test("completion cannot expand the finished run's audience and retries after its return delay", async (t) => {
const r = await rig();
const opened = await r.syscallsFor(r.room).open({ task: "private research" });
assert.ok(opened.ok);
@@ -647,24 +647,52 @@ test("completion cannot expand the finished run's audience during roster refresh
const [run] = await r.runs.inFlightForThread(child.threadRef);
const claimed = await r.runs.claimById(run!.id, "worker", 60_000);
await r.runs.complete(run!.id, claimed!.leaseToken!, { status: "ok", reply: "private result" });
- await assert.rejects(
- deliverSubagentMail(
- {
- mailbox: r.mailbox,
- sessions: r.sessions,
- runs: r.runs,
- maxAttempts: 3,
- prepareRequest: async (request) => ({
- ...request,
- conversation: { ...request.conversation, audience: [actor, { id: "new-member", type: "internal" }] },
- }),
- },
- (await r.runs.get(run!.id))!,
- ),
- /audience/,
- );
+ let expanded = true;
+ let attempts = 0;
+ const recover = async () => {
+ for (const pending of await r.runs.pendingReturns()) {
+ attempts++;
+ try {
+ if (
+ await deliverSubagentMail(
+ {
+ mailbox: r.mailbox,
+ sessions: r.sessions,
+ runs: r.runs,
+ maxAttempts: 3,
+ prepareRequest: async (request) => ({
+ ...request,
+ conversation: {
+ ...request.conversation,
+ audience: expanded ? [actor, { id: "new-member", type: "internal" }] : [actor],
+ },
+ }),
+ },
+ pending,
+ )
+ )
+ await r.runs.markReturned(pending.id);
+ } catch (error) {
+ await r.runs.deferReturn(pending.id, 60_000);
+ throw error;
+ }
+ }
+ };
+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });
+ await assert.rejects(recover(), /audience/);
assert.deepEqual(await r.runs.inFlightForThread(r.room.threadRef), []);
- assert.equal((await r.runs.pendingReturns()).length, 1);
+ assert.deepEqual(await r.mailbox.pending(r.room.id), []);
+ expanded = false;
+ for (let i = 0; i < 59; i++) {
+ t.mock.timers.tick(1_000);
+ await recover();
+ }
+ assert.equal(attempts, 1);
+ t.mock.timers.tick(1_000);
+ await recover();
+ assert.equal(attempts, 2);
+ assert.equal((await r.mailbox.pending(r.room.id)).length, 1);
+ assert.deepEqual(await r.runs.pendingReturns(), []);
});
test("private session replies stay read-only, queue behind running work, and do not return to a parent", async () => {