Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

3.4 KB
Raw Blame
import type { ProcessRecord, ProcessRegistry } from "./process-registry.ts";
import { createSweeper } from "../util/sweeper.ts";
import type { LeaderLease } from "../persistence/leader-lease.ts";
import { awaitProcessExit } from "../sandbox/await-process-exit.ts";
import { ProcessSessionGoneError } from "../sandbox/exec-process-session.ts";
import type { ProcessSandbox, SandboxHandle } from "../sandbox/sandbox.ts";
import { errMessage, swallow } from "../util/errors.ts";

const PROCESS_REAPER_LEASE_KEY = "processes:reaper";

export interface ReaperKillHookOptions {
  termGraceMs?: number;
  killGraceMs?: number;
  onExit?: (handle: SandboxHandle, processId: string) => Promise<void>;
}

export function createReaperKillHook(
  sandbox: ProcessSandbox,
  opts?: ReaperKillHookOptions,
): (rec: ProcessRecord) => Promise<void> {
  const termGraceMs = opts?.termGraceMs ?? 5_000;
  const killGraceMs = opts?.killGraceMs ?? 2_000;
  return async (rec) => {
    if (!rec.sandboxId) return;
    const handle = await sandbox.provision([{ scopeId: rec.scopeId, mode: "rw", mountPath: "" }], {
      sandboxId: rec.sandboxId,
    });
    try {
      try {
        await sandbox.signalProcess(handle, rec.processId, "TERM");
        let status = await awaitProcessExit(sandbox, handle, rec.processId, termGraceMs);
        if (status.state !== "exited") {
          await sandbox.signalProcess(handle, rec.processId, "KILL");
          status = await awaitProcessExit(sandbox, handle, rec.processId, killGraceMs);
        }
        if (status.state !== "exited") throw new Error(`process survived TERM+KILL: ${rec.processId}`);
      } catch (e) {
        if (!(e instanceof ProcessSessionGoneError)) throw e;
      }
      await opts
        ?.onExit?.(handle, rec.processId)
        .catch((e) =>
          console.error(
            "[process-reaper] credential writeback failed:",
            errMessage(e),
            ...(e instanceof AggregateError ? e.errors.map(errMessage) : []),
          ),
        );
    } finally {
      await sandbox.teardown(handle, { keepWarm: true });
    }
  };
}

export interface ProcessReaper {
  start(): void;
  stop(): Promise<void>;
  sweep(): Promise<{ reaped: number }>;
}

export interface ProcessReaperOptions {
  intervalMs: number;
  kill?: (rec: ProcessRecord) => Promise<void>;
  onReaped?: (rec: ProcessRecord) => Promise<void>;
  leaderLease: LeaderLease;
}

export function createProcessReaper(registry: ProcessRegistry, opts: ProcessReaperOptions): ProcessReaper {
  const leaderLease = opts.leaderLease;
  async function sweep(): Promise<{ reaped: number }> {
    const expired = await registry.listExpired();
    let reaped = 0;
    for (const rec of expired) {
      if (opts.kill) {
        try {
          await opts.kill(rec);
        } catch (error) {
          swallow(`process reaper: kill ${rec.processId}`, error);
          continue;
        }
      }
      const flipped = await registry.markStatus(rec.processId, "reaped");
      if (!flipped) continue;
      reaped++;
      if (opts.onReaped)
        await opts
          .onReaped(rec)
          .catch((err) => console.error("[process-reaper] onReaped hook failed:", errMessage(err)));
    }
    return { reaped };
  }

  const sweeper = createSweeper(() => leaderLease.hold(PROCESS_REAPER_LEASE_KEY, sweep), opts.intervalMs);
  return {
    start: () => sweeper.start(),
    stop: () => sweeper.stop(),
    sweep,
  };
}