Multiplayer agent harness for work.
import type { ProcessRecord, ProcessRegistry } from "./process-registry.ts";
import { createSweeper } from "../util/sweeper.ts";
import type { LeaderLease } from "../persistence/leader-lease.ts";
import { awaitProcessExit } from "../sandbox/await-process-exit.ts";
import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";
import type { ProcessSandbox, SandboxHandle } from "../sandbox/sandbox.ts";
import { errMessage, swallow } from "../util/errors.ts";
const PROCESS_REAPER_LEASE_KEY = "processes:reaper";
export interface ReaperKillHookOptions {
termGraceMs?: number;
killGraceMs?: number;
onExit?: (handle: SandboxHandle, processId: string) => Promise<void>;
}
export function createReaperKillHook(
sandbox: ProcessSandbox,
opts?: ReaperKillHookOptions,
): (rec: ProcessRecord) => Promise<void> {
const termGraceMs = opts?.termGraceMs ?? 5_000;
const killGraceMs = opts?.killGraceMs ?? 2_000;
return async (rec) => {
if (!rec.sandboxId) return;
const handle = await sandbox.provision([{ scopeId: rec.scopeId, mode: "rw", mountPath: "" }], {
sandboxId: rec.sandboxId,
});
try {
try {
await sandbox.signalProcess(handle, rec.processId, "TERM");
let status = await awaitProcessExit(sandbox, handle, rec.processId, termGraceMs);
if (status.state !== "exited") {
await sandbox.signalProcess(handle, rec.processId, "KILL");
status = await awaitProcessExit(sandbox, handle, rec.processId, killGraceMs);
}
if (status.state !== "exited") throw new Error(`process survived TERM+KILL: ${rec.processId}`);
} catch (e) {
if (!(e instanceof ProcessSessionGoneError)) throw e;
}
await opts
?.onExit?.(handle, rec.processId)
.catch((e) =>
console.error(
"[process-reaper] credential writeback failed:",
errMessage(e),
...(e instanceof AggregateError ? e.errors.map(errMessage) : []),
),
);
} finally {
await sandbox.teardown(handle, { keepWarm: true });
}
};
}
export interface ProcessReaper {
start(): void;
stop(): Promise<void>;
sweep(): Promise<{ reaped: number }>;
}
export interface ProcessReaperOptions {
intervalMs: number;
kill?: (rec: ProcessRecord) => Promise<void>;
onReaped?: (rec: ProcessRecord) => Promise<void>;
leaderLease: LeaderLease;
}
export function createProcessReaper(registry: ProcessRegistry, opts: ProcessReaperOptions): ProcessReaper {
const leaderLease = opts.leaderLease;
async function sweep(): Promise<{ reaped: number }> {
const expired = await registry.listExpired();
let reaped = 0;
for (const rec of expired) {
if (opts.kill) {
try {
await opts.kill(rec);
} catch (error) {
swallow(`process reaper: kill ${rec.processId}`, error);
continue;
}
}
const flipped = await registry.markStatus(rec.processId, "reaped");
if (!flipped) continue;
reaped++;
if (opts.onReaped)
await opts
.onReaped(rec)
.catch((err) => console.error("[process-reaper] onReaped hook failed:", errMessage(err)));
}
return { reaped };
}
const sweeper = createSweeper(() => leaderLease.hold(PROCESS_REAPER_LEASE_KEY, sweep), opts.intervalMs);
return {
start: () => sweeper.start(),
stop: () => sweeper.stop(),
sweep,
};
}