Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

11.6 KB
Raw Blame
import type { AdmittedWork } from "../util/admitted-work.ts";
import { randomUUID } from "node:crypto";
import type { ErrorLog } from "../admin/error-log.ts";
import { conversationScope } from "../resolution/resolution-service.ts";
import type { TurnResult } from "../types.ts";
import type { Orchestrator } from "../core/orchestrator.ts";
import { isNonRetryable, NonRetryableTurnError, ProviderTurnError, turnFailureMessage } from "../core/turn-error.ts";
import { resolveTurnOrigin } from "../core/turn-origin.ts";
import {
  countedClaims,
  errorParks,
  handbackStop,
  MAX_PREEMPTIONS,
  PARKED,
  type Run,
  type RunStop,
  type RunStore,
} from "./run-store.ts";
import { errMessage, errorAlreadyReported, reportFailure, swallow } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
import { retryDelay } from "./retry-delay.ts";
import { resolveSwarmSettings } from "../swarms/swarm-settings.ts";
import { traceQueued } from "../../plugins/chassis/src/error-reporting.ts";

export interface ProcessDeps {
  runs: RunStore;
  orchestrator: Orchestrator;
  leaseTtlMs: number;
  heartbeatIntervalMs?: number;
  errors?: ErrorLog;
}

export const LEASE_LOST_CONSECUTIVE = 3;

const CLAIM_FAIL_REPORT_CONSECUTIVE = 20;

export async function processRun(
  deps: ProcessDeps,
  run: Run,
  opts?: { background?: boolean; shutdown?: AbortSignal },
): Promise<TurnResult> {
  const { trace, ...request } = run.request;
  return traceQueued(
    trace,
    "run",
    {
      "messaging.system": "qm",
      "messaging.destination.name": "runs",
      "messaging.message.id": run.id,
      "messaging.message.retry.count": Math.max(0, run.attempts - 1),
      "messaging.message.receive.latency":
        run.startedAt !== null ? Math.max(0, run.startedAt - run.createdAt) : undefined,
      "qm.surface": request.surface,
      "qm.origin": resolveTurnOrigin(request).kind,
    },
    () => attemptRun(deps, { ...run, request }, opts),
    (result) => opts?.shutdown?.aborted === true || result.stopped === true || result.status === "queued",
  );
}

async function attemptRun(
  deps: ProcessDeps,
  run: Run,
  opts?: { background?: boolean; shutdown?: AbortSignal },
): Promise<TurnResult> {
  const token = run.leaseToken;
  if (token === null) throw new Error(`processRun called with an unleased run ${run.id}`);
  const intervalMs = deps.heartbeatIntervalMs ?? Math.max(1_000, Math.floor(deps.leaseTtlMs / 3));
  const cancel = new AbortController();
  const onShutdown = (): void => cancel.abort();
  if (opts?.shutdown?.aborted) onShutdown();
  else opts?.shutdown?.addEventListener("abort", onShutdown, { once: true });
  let workDeadline: ReturnType<typeof setTimeout> | undefined;
  let started = false;
  let consecutiveLost = 0;
  let leaseLost = false;
  const beat = setInterval(() => {
    void deps.runs
      .heartbeat(run.id, token, deps.leaseTtlMs)
      .then((alive) => {
        if (alive) {
          consecutiveLost = 0;
          return;
        }
        consecutiveLost += 1;
        if (consecutiveLost >= LEASE_LOST_CONSECUTIVE && !leaseLost) {
          leaseLost = true;
          clearInterval(beat);
          console.warn(
            `[worker] run ${run.id} lost its lease after ${consecutiveLost} consecutive beats; cancelling the in-process turn`,
          );
          cancel.abort();
        }
      })
      .catch((err: unknown) => {
        consecutiveLost = 0;
        console.warn(`[worker] heartbeat failed for run ${run.id} (transient, ignored): ${errMessage(err)}`);
      });
  }, intervalMs);
  let beatStopped = false;
  const stopBeat = (): void => {
    if (beatStopped) return;
    beatStopped = true;
    clearInterval(beat);
  };
  try {
    if (opts?.shutdown?.aborted) return { status: "queued", sessionId: run.sessionId, runId: run.id };
    if (run.request.swarm && !run.request.swarmGoal) {
      const { turnMs } = resolveSwarmSettings({ turnMs: run.request.turnWallClockMs });
      workDeadline = setTimeout(() => cancel.abort(), turnMs);
    }
    if (run.request.swarm && countedClaims(run) > 3) throw new NonRetryableTurnError("swarm claim budget exhausted");
    started = true;
    const result = await deps.orchestrator.handleTurn({
      ...run.request,
      origin: resolveTurnOrigin(run.request),
      runId: run.id,
      attempt: run.attempts,
      ...(run.lastStop ? { resumeCause: run.lastStop } : {}),
      runLeaseToken: token,
      finalAttempt: errorParks(run, deps.runs.maxClaims),
      background: opts?.background ?? false,
      cancel: cancel.signal,
      ...(opts?.shutdown ? { shutdown: opts.shutdown } : {}),
      ...(run.startedAt !== null ? { runStartedAt: run.startedAt } : {}),
    });
    stopBeat();
    if (opts?.shutdown?.aborted) return result;
    if (!(await deps.runs.complete(run.id, token, result))) {
      throw new Error(`run ${run.id} lost its lease before completion`);
    }
    return result;
  } catch (err) {
    stopBeat();
    if (opts?.shutdown?.aborted) throw err;
    console.error(`[worker] run ${run.id} turn failed: ${errMessage(err)}`);
    if (!errorAlreadyReported(err))
      deps.errors?.record(
        {
          category: "turn",
          code: "error",
          message: `run ${run.id}: ${errMessage(err)}`,
          scopeLabel: conversationScope(run.request.conversation, run.request.actor.id),
        },
        err,
      );
    await deps.runs.fail(run.id, token, turnFailureMessage(err), {
      retry: !isNonRetryable(err),
      retryAfterMs: retryDelay(run.errorAttempts),
      stop: runStopFor(err),
    });
    throw err;
  } finally {
    clearTimeout(workDeadline);
    stopBeat();
    opts?.shutdown?.removeEventListener("abort", onShutdown);
    if (opts?.shutdown?.aborted)
      await deps.runs
        .releaseLease(run.id, token, started ? handbackStop(opts.shutdown) : undefined)
        .catch((e) => swallow(`worker: shutdown handback failed run=${run.id}; lease will expire after exit`, e));
  }
}

export interface WorkerDeps extends ProcessDeps {
  pollMs?: number;
  recoveryPollMs?: number;
  workerId?: string;
  canClaim?: () => boolean;
  admittedWork?: AdmittedWork;
}

export interface Worker {
  start(): void;
  held(): { run: Run; claimedAt: number; shutdown: AbortController } | null;
  preempt(runId: string): void;
  stopClaims(): Promise<void>;
  drained(): Promise<void>;
  stop(drainMs?: number): Promise<void>;
  releaseInFlight(): Promise<void>;
}

const STOP_DRAIN_MS = 2_000;

export function createWorker(deps: WorkerDeps): Worker {
  const workerId = deps.workerId ?? `w-${randomUUID().slice(0, 8)}`;
  const pollMs = deps.pollMs ?? 50;
  const recoveryPollMs = deps.recoveryPollMs ?? 5_000;
  const notifications = Boolean(deps.runs.subscribeAvailable);
  let generation = 0;
  let wake: (() => void) | null = null;
  let unsubscribe: (() => void) | undefined;
  const notify = (): void => {
    generation++;
    wake?.();
  };
  async function waitForWork(observed: number): Promise<void> {
    if (stopped || observed !== generation) return;
    await new Promise<void>((resolve) => {
      const done = (): void => {
        clearTimeout(timer);
        wake = null;
        resolve();
      };
      const timer = setTimeout(done, notifications ? recoveryPollMs : pollMs);
      wake = done;
    });
  }
  let stopped = false;
  let loopDone: Promise<void> | null = null;
  let claimDone: Promise<void> | null = null;
  let inFlight: {
    run: Run;
    claimedAt: number;
    shutdown: AbortController;
    done: Promise<void>;
  } | null = null;

  async function loop(): Promise<void> {
    let claimFailures = 0;
    while (!stopped) {
      if (deps.canClaim && !deps.canClaim()) {
        await sleep(pollMs);
        continue;
      }
      const observed = generation;
      let run: Run | null;
      let claimed!: () => void;
      claimDone = new Promise<void>((resolve) => {
        claimed = resolve;
      });
      try {
        run = await deps.runs.claim(workerId, deps.leaseTtlMs);
        claimFailures = 0;
      } catch (e) {
        claimed();
        claimDone = null;
        claimFailures += 1;
        if (claimFailures === CLAIM_FAIL_REPORT_CONSECUTIVE) reportFailure("worker: persistent claim failure", e);
        swallow("worker: claim failed (transient, retrying)", e);
        await sleep(Math.min(pollMs * 2 ** Math.min(claimFailures, 5), 5_000));
        continue;
      }
      if (!run) {
        claimed();
        claimDone = null;
        await waitForWork(observed);
        continue;
      }
      if (stopped || (deps.canClaim && !deps.canClaim())) {
        if (run.leaseToken !== null)
          await deps.runs
            .releaseLease(run.id, run.leaseToken)
            .catch((e) => swallow("worker: post-stop claim handback failed", e));
        claimed();
        claimDone = null;
        break;
      }
      const shutdown = new AbortController();
      let settled!: () => void;
      const current: NonNullable<typeof inFlight> = {
        run,
        claimedAt: Date.now(),
        shutdown,
        done: new Promise<void>((resolve) => {
          settled = resolve;
        }),
      };
      inFlight = current;
      console.log(`[worker] claimed worker=${workerId} run=${run.id} thread=${run.sessionId}`);
      claimed();
      claimDone = null;
      try {
        const work = () => processRun(deps, run, { background: true, shutdown: shutdown.signal });
        if (deps.admittedWork) await deps.admittedWork.run(work);
        else await work();
      } catch (e) {
        if (!shutdown.signal.aborted) swallow("worker: background run crashed", e);
      } finally {
        if (shutdown.signal.aborted)
          console.log(`[worker] shutdown settled worker=${workerId} run=${run.id} thread=${run.sessionId}`);
        inFlight = null;
        settled();
      }
    }
  }

  function stopClaims(): Promise<void> {
    stopped = true;
    unsubscribe?.();
    unsubscribe = undefined;
    notify();
    return claimDone ?? Promise.resolve();
  }

  return {
    held: () => inFlight,
    preempt(runId) {
      if (inFlight?.run.id !== runId || inFlight.shutdown.signal.aborted) return;
      inFlight.shutdown.abort(PARKED);
    },
    start() {
      if (loopDone) return;
      stopped = false;
      unsubscribe = deps.runs.subscribeAvailable?.(notify, {
        pollMs,
        onResync: notify,
      });
      loopDone = loop().finally(() => {
        loopDone = null;
      });
    },
    async releaseInFlight() {
      await stopClaims();
      const held = inFlight;
      if (!held) return;
      held.shutdown.abort();
      await held.done;
    },
    stopClaims,
    drained: () => loopDone ?? Promise.resolve(),
    async stop(drainMs = STOP_DRAIN_MS) {
      void stopClaims();
      await Promise.race([loopDone, sleep(drainMs, { unref: true })]);
    },
  };
}

function runStopFor(err: unknown): RunStop {
  if (err instanceof ProviderTurnError)
    return { kind: "error", code: err.code, ...(err.status !== undefined ? { status: err.status } : {}) };
  return { kind: "error" };
}

export function createRunPreemption(deps: {
  runs: RunStore;
  workers: readonly Worker[];
  canClaim?: () => boolean;
  graceMs?: number;
}): { sweep(): Promise<boolean> } {
  return {
    async sweep() {
      if (deps.canClaim && !deps.canClaim()) return false;
      const held = deps.workers.map((worker) => ({ worker, held: worker.held() }));
      if (held.some(({ held }) => !held || held.shutdown.signal.aborted)) return false;
      const victim = held
        .filter(({ held }) => held!.run.priority === 2 && held!.run.preemptions < MAX_PREEMPTIONS)
        .sort((a, b) => b.held!.claimedAt - a.held!.claimedAt)[0];
      if (!victim || !(await deps.runs.personWaiting(Date.now() - (deps.graceMs ?? 1_000)))) return false;
      victim.worker.preempt(victim.held!.run.id);
      return true;
    },
  };
}