Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

15.0 KB
Raw Blame
import { randomUUID } from "node:crypto";
import { EventEmitter } from "node:events";
import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";
import {
  countedClaims,
  isTerminal,
  leaseLapsed,
  releasesDedupKey,
  runPriority,
  stopErrorCode,
  withTraceCarrier,
} from "./run-store.ts";
import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";

export interface MemoryRuntime {
  runs: RunStore;
  ledger: ToolLedger;
}

const FENCE_HOLD_MS = 600_000;

export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntime {
  const maxClaims = opts?.maxClaims ?? Number.POSITIVE_INFINITY;
  const runs = new Map<string, Run>();
  const retryAfter = new Map<string, number>();
  const byKey = new Map<string, string>();
  const ledger = new Map<string, string>();
  const events = new EventEmitter();
  events.setMaxListeners(0);
  const returned = new Set<string>();
  const terminalListeners: Array<(run: Run) => void> = [];

  function sessionUnavailable(sessionId: string, now: number): boolean {
    for (const r of runs.values()) {
      if (
        r.sessionId === sessionId &&
        (r.status === "running" || (r.status === "pending" && (retryAfter.get(r.id) ?? 0) > now))
      )
        return true;
    }
    return false;
  }

  function pendingRuns(now: number): Run[] {
    const heads = new Map<string, Run>();
    for (const r of [...runs.values()].sort((a, b) => a.createdAt - b.createdAt)) {
      if (r.status === "pending" && !heads.has(r.sessionId) && !sessionUnavailable(r.sessionId, now))
        heads.set(r.sessionId, r);
    }
    return [...heads.values()].sort((a, b) => a.priority - b.priority || a.createdAt - b.createdAt);
  }

  function settle(run: Run): void {
    if (!isTerminal(run.status)) return;
    events.emit(run.id, run);
    for (const listener of terminalListeners) listener(run);
  }

  const store: RunStore = {
    ...(Number.isFinite(maxClaims) ? { maxClaims } : {}),

    async enqueue({ sessionId, request, dedupKey, maxAttempts = 3 }: EnqueueInput): Promise<EnqueueResult> {
      if (dedupKey) {
        const existingId = byKey.get(dedupKey);
        if (existingId) {
          const existing = runs.get(existingId);
          if (existing) return { run: existing, deduped: true };
        }
      }
      const run: Run = {
        id: randomUUID(),
        sessionId,
        status: "pending",
        request: withTraceCarrier(request),
        result: null,
        deliveryState: null,
        turnUserSeq: null,
        dedupKey: dedupKey ?? null,
        attempts: 0,
        errorAttempts: 0,
        maxAttempts,
        priority: runPriority(
          request,
          request.delegatingRunId ? (runs.get(request.delegatingRunId)?.priority ?? null) : null,
        ),
        preemptions: 0,
        leaseToken: null,
        leaseExpiresAt: null,
        workerId: null,
        createdAt: Date.now(),
        startedAt: null,
        finishedAt: null,
      };
      runs.set(run.id, run);
      if (dedupKey) byKey.set(dedupKey, run.id);
      return { run, deduped: false };
    },

    async claim(workerId, ttlMs) {
      const now = Date.now();
      const run = pendingRuns(now)[0];
      if (!run) return null;
      return lease(run, workerId, ttlMs);
    },

    async personWaiting(createdBefore) {
      return pendingRuns(Date.now()).some((run) => run.priority === 0 && run.createdAt <= createdBefore);
    },

    async claimById(runId, workerId, ttlMs) {
      const run = runs.get(runId);
      if (!run || pendingRuns(Date.now()).find((pending) => pending.sessionId === run.sessionId)?.id !== runId)
        return null;
      return lease(run, workerId, ttlMs);
    },

    async claimForSession(sessionId, workerId, ttlMs) {
      const run = pendingRuns(Date.now()).find((pending) => pending.sessionId === sessionId);
      return run ? lease(run, workerId, ttlMs) : null;
    },

    async heartbeat(runId, leaseToken, ttlMs) {
      const run = runs.get(runId);
      if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;
      run.leaseExpiresAt = Date.now() + ttlMs;
      return true;
    },

    async releaseLease(runId, leaseToken, stop) {
      const run = runs.get(runId);
      if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;
      if (stop?.kind === "parked") run.preemptions += 1;
      run.status = "pending";
      run.leaseToken = null;
      run.leaseExpiresAt = null;
      run.workerId = null;
      if (stop) run.lastStop = stop;
      return true;
    },

    async complete(runId, leaseToken, result) {
      const run = runs.get(runId);
      if (!run || run.leaseToken !== leaseToken) return false;
      run.status = "done";
      run.result = result;
      run.leaseToken = null;
      run.leaseExpiresAt = null;
      run.finishedAt = Date.now();
      if (releasesDedupKey(result) && run.dedupKey) {
        byKey.delete(run.dedupKey);
        run.dedupKey = null;
      }
      settle(run);
      return true;
    },

    async fail(runId, leaseToken, error, opts) {
      const run = runs.get(runId);
      if (!run || run.leaseToken !== leaseToken) return { requeued: false };
      return {
        requeued: retire(run, error, opts?.retry !== false, {
          countsAsError: true,
          retryAfterMs: opts?.retryAfterMs,
          ...(opts?.stop ? { stop: opts.stop } : {}),
        }).requeued,
      };
    },

    async noteTurnUserSeq(runId: string, seq: number) {
      const run = runs.get(runId);
      if (!run || run.turnUserSeq !== null) return false;
      run.turnUserSeq = seq;
      return true;
    },

    async setDeliveryState(runId: string, leaseToken: string | null, state: RunDeliveryState) {
      const run = runs.get(runId);
      if (!run) return false;
      if (leaseToken !== null && run.leaseToken !== leaseToken) return false;
      run.deliveryState = { ...run.deliveryState, ...state };
      return true;
    },

    async latestForThreads(threadRefs, opts) {
      const wanted = new Set(threadRefs);
      const latest = new Map<string, Run>();
      for (const run of runs.values()) {
        if (!wanted.has(run.sessionId) || (opts?.excludePrivateMessages && run.request.privateSessionMessage)) continue;
        if (!latest.has(run.sessionId) || latest.get(run.sessionId)!.createdAt <= run.createdAt)
          latest.set(run.sessionId, run);
      }
      return latest;
    },
    async latestForThread(threadRef, opts) {
      return (
        [...runs.values()]
          .reverse()
          .filter(
            (run) =>
              run.sessionId === threadRef && !(opts?.excludePrivateMessages && run.request.privateSessionMessage),
          )
          .sort((a, b) => b.createdAt - a.createdAt)
          .at(0) ?? null
      );
    },
    async pendingReturns(limit = 100, afterId = "") {
      const now = Date.now();
      return [...runs.values()]
        .filter(
          (run) =>
            isTerminal(run.status) &&
            (retryAfter.get(run.id) ?? 0) <= now &&
            (!returned.has(run.id) ||
              (() => {
                const wake = runs.get(byKey.get(`subagent-return:${run.id}`) ?? "");
                return wake?.status === "pending" && wake.attempts === 0 && wake.turnUserSeq === null;
              })()) &&
            run.id > afterId &&
            run.sessionId.startsWith("agent:main:subagent:"),
        )
        .sort((a, b) => a.id.localeCompare(b.id))
        .slice(0, limit);
    },
    async terminalFinished(after, beforeMs, limit) {
      return [...runs.values()]
        .filter(
          (run) =>
            isTerminal(run.status) &&
            run.finishedAt !== null &&
            run.finishedAt <= beforeMs &&
            (run.finishedAt > after.finishedAt || (run.finishedAt === after.finishedAt && run.id > after.id)),
        )
        .sort((a, b) => a.finishedAt! - b.finishedAt! || (a.id < b.id ? -1 : 1))
        .slice(0, limit);
    },
    async markReturned(runId) {
      returned.add(runId);
    },
    async deferReturn(runId, delayMs) {
      const run = runs.get(runId);
      if (run && isTerminal(run.status)) retryAfter.set(runId, Date.now() + Math.max(0, delayMs));
    },
    onTerminal(listener) {
      terminalListeners.push(listener);
    },

    async get(runId) {
      return runs.get(runId) ?? null;
    },
    async getByDedupKey(dedupKey) {
      const id = byKey.get(dedupKey);
      return id ? (runs.get(id) ?? null) : null;
    },

    async activeForThread(sessionId) {
      const inFlight = [...runs.values()].filter((r) => r.sessionId === sessionId && !isTerminal(r.status));
      return (
        inFlight.sort((a, b) => {
          if ((a.status === "running") !== (b.status === "running")) return a.status === "running" ? -1 : 1;
          return a.createdAt - b.createdAt;
        })[0] ?? null
      );
    },

    async inFlightForThread(sessionId) {
      return [...runs.values()]
        .filter((r) => r.sessionId === sessionId && !isTerminal(r.status))
        .sort((a, b) => a.createdAt - b.createdAt);
    },

    async editPendingText(runId, text, expectedText) {
      const run = runs.get(runId);
      if (!run || run.status !== "pending" || run.attempts !== 0 || run.turnUserSeq !== null) return false;
      if ((run.request.displayText ?? run.request.text) !== expectedText) return false;
      run.request = { ...run.request, text, displayText: text };
      return true;
    },

    async withdraw(runId, opts) {
      const run = runs.get(runId);
      if (!run || run.status !== "pending") return false;
      if (opts?.unstartedOnly && (run.attempts !== 0 || run.turnUserSeq !== null)) return false;
      runs.delete(runId);
      retryAfter.delete(runId);
      if (run.dedupKey) byKey.delete(run.dedupKey);
      return true;
    },

    async steerQueued(queuedRunId, targetRunId, signal, signals) {
      const queued = runs.get(queuedRunId);
      const target = runs.get(targetRunId);
      if (!queued || queued.status !== "pending" || !target || isTerminal(target.status) || queuedRunId === targetRunId)
        return false;
      if ((queued.request.displayText ?? queued.request.text) !== signal.request?.text) return false;
      runs.delete(queuedRunId);
      try {
        await signals.send(targetRunId, signal);
      } catch (error) {
        runs.set(queuedRunId, queued);
        throw error;
      }
      retryAfter.delete(queuedRunId);
      if (queued.dedupKey) byKey.delete(queued.dedupKey);
      return true;
    },

    async activeSessionIds() {
      const ids = new Set<string>();
      for (const r of runs.values()) if (!isTerminal(r.status)) ids.add(r.sessionId);
      return [...ids];
    },

    async list({ limit = 200, threadRef }: { limit?: number; threadRef?: string } = {}) {
      return [...runs.values()]
        .filter(
          (run) =>
            !threadRef ||
            run.sessionId === threadRef ||
            run.sessionId.startsWith(`${threadRef}:task:`) ||
            run.sessionId.startsWith(`${threadRef}:status:`),
        )
        .sort((a, b) => b.createdAt - a.createdAt)
        .slice(0, limit);
    },

    async reapExpired(
      onRetired?: (sessionIds: string[]) => Promise<void>,
      opts?: { maxAgeMs?: number; onReap?: (event: ReapEvent) => void },
    ) {
      const now = Date.now();
      const expired = [...runs.values()].filter((run) => leaseLapsed(run, now));
      let requeued = 0;
      let parked = 0;
      for (const run of expired) {
        const tooOld = opts?.maxAgeMs !== undefined && run.startedAt !== null && now - run.startedAt > opts.maxAgeMs;
        const reason = tooOld ? "run exceeded max age (reaped)" : "lease expired (reaped)";
        const workerId = run.workerId;
        if (run.status !== "running" || run.leaseExpiresAt === null || run.leaseExpiresAt > now) continue;
        run.leaseToken = randomUUID();
        run.leaseExpiresAt = now + FENCE_HOLD_MS;
        if (onRetired) await onRetired([run.sessionId]);
        const r = retire(run, reason, !tooOld);
        if (!r.applied) continue;
        if (r.requeued) requeued++;
        else parked++;
        opts?.onReap?.({
          runId: run.id,
          sessionId: run.sessionId,
          workerId,
          attempts: run.attempts,
          errorAttempts: run.errorAttempts,
          outcome: r.requeued ? "requeued" : "parked",
        });
      }
      return { requeued, parked };
    },

    waitFor(runId, timeoutMs = 60_000) {
      const run = runs.get(runId);
      if (run && isTerminal(run.status)) return Promise.resolve(run);
      return new Promise<Run>((resolvePromise, reject) => {
        const timer = setTimeout(() => {
          events.off(runId, onSettle);
          reject(new Error(`run ${runId} did not finish within ${timeoutMs}ms`));
        }, timeoutMs);
        timer.unref?.();
        function onSettle(r: Run): void {
          clearTimeout(timer);
          resolvePromise(r);
        }
        events.once(runId, onSettle);
      });
    },
  };

  function lease(run: Run, workerId: string, ttlMs: number): Run {
    retryAfter.delete(run.id);
    run.status = "running";
    run.leaseToken = randomUUID();
    run.leaseExpiresAt = Date.now() + ttlMs;
    run.workerId = workerId;
    run.attempts += 1;
    run.startedAt = run.startedAt ?? Date.now();
    return { ...run };
  }

  function retire(
    run: Run,
    error: string,
    retry: boolean,
    opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number; stop?: RunStop },
  ): { requeued: boolean; applied: boolean } {
    if (run.status !== "running") return { requeued: false, applied: false };
    if (opts?.ifExpiredAt !== undefined && (run.leaseExpiresAt === null || run.leaseExpiresAt > opts.ifExpiredAt)) {
      return { requeued: false, applied: false };
    }
    run.leaseToken = null;
    run.leaseExpiresAt = null;
    run.workerId = null;
    if (opts?.countsAsError) run.errorAttempts += 1;
    const overClaimed = countedClaims(run) >= maxClaims;
    if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {
      run.status = "pending";
      run.lastStop = opts?.stop ?? (opts?.countsAsError ? { kind: "error" } : { kind: "lease_expired" });
      retryAfter.set(run.id, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0));
      return { requeued: true, applied: true };
    }
    run.status = "failed";
    const reason =
      !opts?.countsAsError && overClaimed && retry && run.errorAttempts < run.maxAttempts
        ? `run parked after ${countedClaims(run)} claims without completing (suspected crash loop)`
        : error;
    run.result = { status: "failed", sessionId: run.sessionId, reason, ...stopErrorCode(opts?.stop) };
    run.finishedAt = Date.now();
    settle(run);
    return { requeued: false, applied: true };
  }

  const toolLedger: ToolLedger = {
    async begin(runId, attempt, callIndex): Promise<LedgerBegin> {
      const output = ledger.get(`${runId}:${attempt}:${callIndex}`);
      return output !== undefined ? { cached: true, output } : { cached: false };
    },
    async record(runId, attempt, callIndex, output) {
      ledger.set(`${runId}:${attempt}:${callIndex}`, output);
    },
  };

  return { runs: store, ledger: toolLedger };
}