Multiplayer agent harness for work.
import { randomUUID } from "node:crypto";
import { EventEmitter } from "node:events";
import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";
import {
countedClaims,
isTerminal,
leaseLapsed,
releasesDedupKey,
runPriority,
stopErrorCode,
withTraceCarrier,
} from "./run-store.ts";
import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";
export interface MemoryRuntime {
runs: RunStore;
ledger: ToolLedger;
}
const FENCE_HOLD_MS = 600_000;
export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntime {
const maxClaims = opts?.maxClaims ?? Number.POSITIVE_INFINITY;
const runs = new Map<string, Run>();
const retryAfter = new Map<string, number>();
const byKey = new Map<string, string>();
const ledger = new Map<string, string>();
const events = new EventEmitter();
events.setMaxListeners(0);
const returned = new Set<string>();
const terminalListeners: Array<(run: Run) => void> = [];
function sessionUnavailable(sessionId: string, now: number): boolean {
for (const r of runs.values()) {
if (
r.sessionId === sessionId &&
(r.status === "running" || (r.status === "pending" && (retryAfter.get(r.id) ?? 0) > now))
)
return true;
}
return false;
}
function pendingRuns(now: number): Run[] {
const heads = new Map<string, Run>();
for (const r of [...runs.values()].sort((a, b) => a.createdAt - b.createdAt)) {
if (r.status === "pending" && !heads.has(r.sessionId) && !sessionUnavailable(r.sessionId, now))
heads.set(r.sessionId, r);
}
return [...heads.values()].sort((a, b) => a.priority - b.priority || a.createdAt - b.createdAt);
}
function settle(run: Run): void {
if (!isTerminal(run.status)) return;
events.emit(run.id, run);
for (const listener of terminalListeners) listener(run);
}
const store: RunStore = {
...(Number.isFinite(maxClaims) ? { maxClaims } : {}),
async enqueue({ sessionId, request, dedupKey, maxAttempts = 3 }: EnqueueInput): Promise<EnqueueResult> {
if (dedupKey) {
const existingId = byKey.get(dedupKey);
if (existingId) {
const existing = runs.get(existingId);
if (existing) return { run: existing, deduped: true };
}
}
const run: Run = {
id: randomUUID(),
sessionId,
status: "pending",
request: withTraceCarrier(request),
result: null,
deliveryState: null,
turnUserSeq: null,
dedupKey: dedupKey ?? null,
attempts: 0,
errorAttempts: 0,
maxAttempts,
priority: runPriority(
request,
request.delegatingRunId ? (runs.get(request.delegatingRunId)?.priority ?? null) : null,
),
preemptions: 0,
leaseToken: null,
leaseExpiresAt: null,
workerId: null,
createdAt: Date.now(),
startedAt: null,
finishedAt: null,
};
runs.set(run.id, run);
if (dedupKey) byKey.set(dedupKey, run.id);
return { run, deduped: false };
},
async claim(workerId, ttlMs) {
const now = Date.now();
const run = pendingRuns(now)[0];
if (!run) return null;
return lease(run, workerId, ttlMs);
},
async personWaiting(createdBefore) {
return pendingRuns(Date.now()).some((run) => run.priority === 0 && run.createdAt <= createdBefore);
},
async claimById(runId, workerId, ttlMs) {
const run = runs.get(runId);
if (!run || pendingRuns(Date.now()).find((pending) => pending.sessionId === run.sessionId)?.id !== runId)
return null;
return lease(run, workerId, ttlMs);
},
async claimForSession(sessionId, workerId, ttlMs) {
const run = pendingRuns(Date.now()).find((pending) => pending.sessionId === sessionId);
return run ? lease(run, workerId, ttlMs) : null;
},
async heartbeat(runId, leaseToken, ttlMs) {
const run = runs.get(runId);
if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;
run.leaseExpiresAt = Date.now() + ttlMs;
return true;
},
async releaseLease(runId, leaseToken, stop) {
const run = runs.get(runId);
if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;
if (stop?.kind === "parked") run.preemptions += 1;
run.status = "pending";
run.leaseToken = null;
run.leaseExpiresAt = null;
run.workerId = null;
if (stop) run.lastStop = stop;
return true;
},
async complete(runId, leaseToken, result) {
const run = runs.get(runId);
if (!run || run.leaseToken !== leaseToken) return false;
run.status = "done";
run.result = result;
run.leaseToken = null;
run.leaseExpiresAt = null;
run.finishedAt = Date.now();
if (releasesDedupKey(result) && run.dedupKey) {
byKey.delete(run.dedupKey);
run.dedupKey = null;
}
settle(run);
return true;
},
async fail(runId, leaseToken, error, opts) {
const run = runs.get(runId);
if (!run || run.leaseToken !== leaseToken) return { requeued: false };
return {
requeued: retire(run, error, opts?.retry !== false, {
countsAsError: true,
retryAfterMs: opts?.retryAfterMs,
...(opts?.stop ? { stop: opts.stop } : {}),
}).requeued,
};
},
async noteTurnUserSeq(runId: string, seq: number) {
const run = runs.get(runId);
if (!run || run.turnUserSeq !== null) return false;
run.turnUserSeq = seq;
return true;
},
async setDeliveryState(runId: string, leaseToken: string | null, state: RunDeliveryState) {
const run = runs.get(runId);
if (!run) return false;
if (leaseToken !== null && run.leaseToken !== leaseToken) return false;
run.deliveryState = { ...run.deliveryState, ...state };
return true;
},
async latestForThreads(threadRefs, opts) {
const wanted = new Set(threadRefs);
const latest = new Map<string, Run>();
for (const run of runs.values()) {
if (!wanted.has(run.sessionId) || (opts?.excludePrivateMessages && run.request.privateSessionMessage)) continue;
if (!latest.has(run.sessionId) || latest.get(run.sessionId)!.createdAt <= run.createdAt)
latest.set(run.sessionId, run);
}
return latest;
},
async latestForThread(threadRef, opts) {
return (
[...runs.values()]
.reverse()
.filter(
(run) =>
run.sessionId === threadRef && !(opts?.excludePrivateMessages && run.request.privateSessionMessage),
)
.sort((a, b) => b.createdAt - a.createdAt)
.at(0) ?? null
);
},
async pendingReturns(limit = 100, afterId = "") {
const now = Date.now();
return [...runs.values()]
.filter(
(run) =>
isTerminal(run.status) &&
(retryAfter.get(run.id) ?? 0) <= now &&
(!returned.has(run.id) ||
(() => {
const wake = runs.get(byKey.get(`subagent-return:${run.id}`) ?? "");
return wake?.status === "pending" && wake.attempts === 0 && wake.turnUserSeq === null;
})()) &&
run.id > afterId &&
run.sessionId.startsWith("agent:main:subagent:"),
)
.sort((a, b) => a.id.localeCompare(b.id))
.slice(0, limit);
},
async terminalFinished(after, beforeMs, limit) {
return [...runs.values()]
.filter(
(run) =>
isTerminal(run.status) &&
run.finishedAt !== null &&
run.finishedAt <= beforeMs &&
(run.finishedAt > after.finishedAt || (run.finishedAt === after.finishedAt && run.id > after.id)),
)
.sort((a, b) => a.finishedAt! - b.finishedAt! || (a.id < b.id ? -1 : 1))
.slice(0, limit);
},
async markReturned(runId) {
returned.add(runId);
},
async deferReturn(runId, delayMs) {
const run = runs.get(runId);
if (run && isTerminal(run.status)) retryAfter.set(runId, Date.now() + Math.max(0, delayMs));
},
onTerminal(listener) {
terminalListeners.push(listener);
},
async get(runId) {
return runs.get(runId) ?? null;
},
async getByDedupKey(dedupKey) {
const id = byKey.get(dedupKey);
return id ? (runs.get(id) ?? null) : null;
},
async activeForThread(sessionId) {
const inFlight = [...runs.values()].filter((r) => r.sessionId === sessionId && !isTerminal(r.status));
return (
inFlight.sort((a, b) => {
if ((a.status === "running") !== (b.status === "running")) return a.status === "running" ? -1 : 1;
return a.createdAt - b.createdAt;
})[0] ?? null
);
},
async inFlightForThread(sessionId) {
return [...runs.values()]
.filter((r) => r.sessionId === sessionId && !isTerminal(r.status))
.sort((a, b) => a.createdAt - b.createdAt);
},
async editPendingText(runId, text, expectedText) {
const run = runs.get(runId);
if (!run || run.status !== "pending" || run.attempts !== 0 || run.turnUserSeq !== null) return false;
if ((run.request.displayText ?? run.request.text) !== expectedText) return false;
run.request = { ...run.request, text, displayText: text };
return true;
},
async withdraw(runId, opts) {
const run = runs.get(runId);
if (!run || run.status !== "pending") return false;
if (opts?.unstartedOnly && (run.attempts !== 0 || run.turnUserSeq !== null)) return false;
runs.delete(runId);
retryAfter.delete(runId);
if (run.dedupKey) byKey.delete(run.dedupKey);
return true;
},
async steerQueued(queuedRunId, targetRunId, signal, signals) {
const queued = runs.get(queuedRunId);
const target = runs.get(targetRunId);
if (!queued || queued.status !== "pending" || !target || isTerminal(target.status) || queuedRunId === targetRunId)
return false;
if ((queued.request.displayText ?? queued.request.text) !== signal.request?.text) return false;
runs.delete(queuedRunId);
try {
await signals.send(targetRunId, signal);
} catch (error) {
runs.set(queuedRunId, queued);
throw error;
}
retryAfter.delete(queuedRunId);
if (queued.dedupKey) byKey.delete(queued.dedupKey);
return true;
},
async activeSessionIds() {
const ids = new Set<string>();
for (const r of runs.values()) if (!isTerminal(r.status)) ids.add(r.sessionId);
return [...ids];
},
async list({ limit = 200, threadRef }: { limit?: number; threadRef?: string } = {}) {
return [...runs.values()]
.filter(
(run) =>
!threadRef ||
run.sessionId === threadRef ||
run.sessionId.startsWith(`${threadRef}:task:`) ||
run.sessionId.startsWith(`${threadRef}:status:`),
)
.sort((a, b) => b.createdAt - a.createdAt)
.slice(0, limit);
},
async reapExpired(
onRetired?: (sessionIds: string[]) => Promise<void>,
opts?: { maxAgeMs?: number; onReap?: (event: ReapEvent) => void },
) {
const now = Date.now();
const expired = [...runs.values()].filter((run) => leaseLapsed(run, now));
let requeued = 0;
let parked = 0;
for (const run of expired) {
const tooOld = opts?.maxAgeMs !== undefined && run.startedAt !== null && now - run.startedAt > opts.maxAgeMs;
const reason = tooOld ? "run exceeded max age (reaped)" : "lease expired (reaped)";
const workerId = run.workerId;
if (run.status !== "running" || run.leaseExpiresAt === null || run.leaseExpiresAt > now) continue;
run.leaseToken = randomUUID();
run.leaseExpiresAt = now + FENCE_HOLD_MS;
if (onRetired) await onRetired([run.sessionId]);
const r = retire(run, reason, !tooOld);
if (!r.applied) continue;
if (r.requeued) requeued++;
else parked++;
opts?.onReap?.({
runId: run.id,
sessionId: run.sessionId,
workerId,
attempts: run.attempts,
errorAttempts: run.errorAttempts,
outcome: r.requeued ? "requeued" : "parked",
});
}
return { requeued, parked };
},
waitFor(runId, timeoutMs = 60_000) {
const run = runs.get(runId);
if (run && isTerminal(run.status)) return Promise.resolve(run);
return new Promise<Run>((resolvePromise, reject) => {
const timer = setTimeout(() => {
events.off(runId, onSettle);
reject(new Error(`run ${runId} did not finish within ${timeoutMs}ms`));
}, timeoutMs);
timer.unref?.();
function onSettle(r: Run): void {
clearTimeout(timer);
resolvePromise(r);
}
events.once(runId, onSettle);
});
},
};
function lease(run: Run, workerId: string, ttlMs: number): Run {
retryAfter.delete(run.id);
run.status = "running";
run.leaseToken = randomUUID();
run.leaseExpiresAt = Date.now() + ttlMs;
run.workerId = workerId;
run.attempts += 1;
run.startedAt = run.startedAt ?? Date.now();
return { ...run };
}
function retire(
run: Run,
error: string,
retry: boolean,
opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number; stop?: RunStop },
): { requeued: boolean; applied: boolean } {
if (run.status !== "running") return { requeued: false, applied: false };
if (opts?.ifExpiredAt !== undefined && (run.leaseExpiresAt === null || run.leaseExpiresAt > opts.ifExpiredAt)) {
return { requeued: false, applied: false };
}
run.leaseToken = null;
run.leaseExpiresAt = null;
run.workerId = null;
if (opts?.countsAsError) run.errorAttempts += 1;
const overClaimed = countedClaims(run) >= maxClaims;
if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {
run.status = "pending";
run.lastStop = opts?.stop ?? (opts?.countsAsError ? { kind: "error" } : { kind: "lease_expired" });
retryAfter.set(run.id, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0));
return { requeued: true, applied: true };
}
run.status = "failed";
const reason =
!opts?.countsAsError && overClaimed && retry && run.errorAttempts < run.maxAttempts
? `run parked after ${countedClaims(run)} claims without completing (suspected crash loop)`
: error;
run.result = { status: "failed", sessionId: run.sessionId, reason, ...stopErrorCode(opts?.stop) };
run.finishedAt = Date.now();
settle(run);
return { requeued: false, applied: true };
}
const toolLedger: ToolLedger = {
async begin(runId, attempt, callIndex): Promise<LedgerBegin> {
const output = ledger.get(`${runId}:${attempt}:${callIndex}`);
return output !== undefined ? { cached: true, output } : { cached: false };
},
async record(runId, attempt, callIndex, output) {
ledger.set(`${runId}:${attempt}:${callIndex}`, output);
},
};
return { runs: store, ledger: toolLedger };
}