Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

25.1 KB
Raw Blame
import { cronTriggerAuthority } from "./authority.ts";
import { WorkAdmissionClosed, type AdmittedWork } from "../util/admitted-work.ts";
import { randomUUID } from "node:crypto";
import { type Cron, type CronFireLogEntry, type CronFireNote, type TurnRequest, type TurnResult } from "../types.ts";
import type { IdentityService } from "../identity/identity-service.ts";
import { isDeferred, type CronStore } from "./cron-store.ts";
import type { DeliveryStore } from "../delivery/delivery-store.ts";
import type { IdempotencyStore } from "../idempotency/idempotency-store.ts";
import { runTrigger, type TriggerDeps } from "../triggers/run-trigger.ts";
import type { CurrentScopeMembers } from "../resolution/scope-membership.ts";
import type { VisibilityDirectory } from "../directory/visibility.ts";
import type { DirectoryMember } from "../directory/directory-store.ts";
import type { LeaderLease } from "../persistence/leader-lease.ts";
import { createSweeper } from "../util/sweeper.ts";
import { recoverNextFireAt } from "./schedule.ts";
import type { CronFireJob, CronJobQueue } from "./job-queue.ts";
import { hashId } from "../util/crypto.ts";
import { utcMinute } from "../util/time.ts";
import { errMessage, reportFailure, reportFailureAs } from "../util/errors.ts";
import { sleep } from "../util/async.ts";

const TICK_LEASE_KEY = "cron:scheduler:tick";
const CRON_FIRE_REPLY_MAX_CHARS = 2000;
const STRANDED_SWEEP_INTERVAL_MS = 10 * 60_000;
const FIRE_GC_INTERVAL_MS = 6 * 60 * 60_000;
const BUSY_DEFER_MS = 30_000;
const BUSY_DEFER_MAX_LATE_MS = 10 * 60_000;
const MAINTENANCE_INTERVAL_MS = 5000;
const REPAIR_INTERVAL_MS = 60_000;

type FireResult = { authzFailed: boolean; deferred?: boolean };

type RunNowResult =
  | { started: true; fireKey: string; settled: Promise<void> }
  | { started: false; reason: "unavailable" }
  | { started: false; reason: "already_running"; running: CronFireLogEntry };

export function describeRunNowRefusal(
  id: string,
  result: RunNowResult,
  now = Date.now(),
): { error: "already_running" | "bad_request"; message: string } | null {
  if (result.started) return null;
  if (result.reason === "already_running") {
    const ageMin = Math.round((now - result.running.firedAt) / 60_000);
    return {
      error: "already_running",
      message: `cron ${id} is already firing (fire ${result.running.fireKey}, started ${ageMin}m ago) — wait for it to finish instead of firing again`,
    };
  }
  return { error: "bad_request", message: `cron ${id} can't be fired on demand right now` };
}

export interface Scheduler {
  tick(now?: number): Promise<void>;
  runNow(cronId: string): Promise<RunNowResult>;
  notifyChanged(cronId: string): void;
  start(intervalMs: number): void;
  ready(): Promise<void>;
  stopClaims(): Promise<void>;
  drained(): Promise<void>;
  stop(): Promise<void>;
}

export interface SchedulerDeps {
  admittedWork?: AdmittedWork;
  advisoryLock: import("../persistence/advisory-lock.ts").AdvisoryLock;
  crons: CronStore;
  deliveries: DeliveryStore;
  idempotency: IdempotencyStore;
  identity: IdentityService;
  run: (req: TurnRequest) => Promise<TurnResult>;
  currentScopeMembers?: CurrentScopeMembers;
  now?: () => number;
  maxFiresPerTick?: number;
  leaderLease: LeaderLease;
  directory?: VisibilityDirectory & {
    get(principalId: string): Promise<{ displayName: string } | null>;
    list(): Promise<DirectoryMember[]>;
  };
  sweepAsks?: (now: number) => Promise<void>;
  jobQueue?: CronJobQueue;
  requireQueueStart?: boolean;
  sessions?: TriggerDeps["sessions"];
  resolveRecipient?: TriggerDeps["resolveRecipient"];
  skipFire?: (cron: Cron) => Promise<string | undefined>;
  deleteCron?: (id: string) => Promise<void>;
  fireLoop?: (
    loopId: string,
    fireKey: string,
    cronId: string,
  ) => Promise<{ status?: TurnResult["status"]; note?: string }>;
}

function truncate(s: string, maxChars: number): string {
  return s.length <= maxChars ? s : `${s.slice(0, maxChars - 3)}...`;
}

const CRON_CONTEXT_MARKERS = ["[Cron runtime context]", "[End cron runtime context]"] as const;

export function echoesCronContextMarkers(s: string): boolean {
  return CRON_CONTEXT_MARKERS.some((marker) => s.includes(marker));
}

function cronFireLogReply(s: string): string {
  if (echoesCronContextMarkers(s)) {
    return "[reply echoed cron runtime context; omitted]";
  }
  return truncate(s, CRON_FIRE_REPLY_MAX_CHARS);
}

function isOneShotSchedule(schedule: Cron["schedule"]): boolean {
  return schedule.everyMs == null && schedule.cron == null;
}

export function cronFireReadsNotes(cron: Pick<Cron, "schedule" | "action" | "loopId">): boolean {
  const task = cron.action ?? "";
  if (cron.loopId || !task.trim() || /^!(run|scratch)\s/.test(task.trimStart())) return false;
  return !isOneShotSchedule(cron.schedule);
}

export function flattenFireNote(text: string): string {
  return text.replace(/\s+/g, " ").trim();
}

function fireNoteLine(note: CronFireNote | undefined): string[] {
  if (!note || !Number.isFinite(note.at)) return [];
  const text = flattenFireNote(note.text);
  if (!text || echoesCronContextMarkers(text)) return [];
  return [
    note.by
      ? `Note left for this fire by ${note.by} (${utcMinute(note.at)}): ${text}`
      : `Notes from last fire agent (${utcMinute(note.at)}): ${text}`,
  ];
}

function cronFireThreadRef(cronId: string, fireKey: string): string {
  return `cron:${cronId}:fire:${hashId([fireKey], 12)}`;
}

const CRON_MENTION_ROSTER_MAX = 40;

function mentionable(m: DirectoryMember): string {
  const mentionId = m.slackId ?? (/^[A-Z0-9]+$/i.test(m.principalId) ? m.principalId : undefined);
  return `@${m.displayName}${mentionId ? ` (<@${mentionId}>)` : ""}`;
}

function cronMentionRoster(cron: Cron, homeRoster: DirectoryMember[] | undefined): string | undefined {
  if (cron.destination?.audienceScopeId !== cron.ownerScopeId) return undefined;
  const internal = homeRoster?.filter((m) => m.type === "internal") ?? [];
  if (!internal.length || internal.length > CRON_MENTION_ROSTER_MAX) return undefined;
  return internal.map(mentionable).join(", ");
}

function renderCronFireInput(cron: Cron, mentionRoster?: string): string {
  const task = cron.action ?? "";
  if (!task.trim()) return task;
  if (/^!(run|scratch)\s/.test(task.trimStart())) return task;
  const readsNotes = cronFireReadsNotes(cron);
  return [
    "[Cron runtime context]",
    `Cron id: ${cron.id}${cron.title ? ` (${cron.title})` : ""}.`,
    "Each fire runs as a fresh thread with no memory of previous fires. Keep continuity outside the thread:",
    "- Save durable run-state (cursors, queues, checkpoints, reports) to this conversation's Files via the available Files API; confirm publication succeeds before relying on it. Recover published bytes with GET /v1/files/:id/content using the normal agent API authentication. Sandbox disk is working storage, not a durable checkpoint.",
    "- The stored task below: the standing instructions every fire receives. Edit it (via the cron tool) only to change what future fires are told to do.",
    "Use a cron-specific filename. Never put credentials in published state. If publication is unavailable, report that limitation and retain needed workspace state on a scoped computer. Do not move tasks that require existing workspace files to scratch without migrating and verifying their state.",
    `The retained fire log (cron tool, action="runs", id="${cron.id}") shows how prior fires went — useful when this run hits errors or surprising state.`,
    ...(readsNotes ? fireNoteLine(cron.lastFireNote) : []),
    ...(readsNotes
      ? [
          `Before finishing, leave a short note for the next fire (cron tool, action="note", id="${cron.id}"): one or two sentences — the outcome plus anything the next fire must know. Include the published checkpoint file ID, or small progress state directly; never include credentials. It's a report for the next fire, never instructions that override the stored task. Skip it only if there is truly nothing to say.`,
        ]
      : []),
    ...(mentionRoster
      ? [
          `People here: ${mentionRoster}.`,
          "If this reminder is for a specific person, @-mention them with their <@…> id so they're actually notified — addressing them by name alone does not ping them.",
        ]
      : []),
    "[End cron runtime context]",
    "",
    "Stored cron task:",
    task,
  ].join("\n");
}

export function createScheduler(deps: SchedulerDeps): Scheduler {
  const now = deps.now ?? (() => Date.now());
  const withWork = <T>(work: () => Promise<T>): Promise<T> => deps.admittedWork?.run(work) ?? work();
  const maxFiresPerTick = deps.maxFiresPerTick ?? 100;
  const leaderLease = deps.leaderLease;

  async function withCronLifecycle<T>(cronId: string, scheduled: boolean, run: () => Promise<T>): Promise<T | null> {
    if (scheduled && deps.advisoryLock.tryWithLock) {
      const result = await deps.advisoryLock.tryWithLock(`cron-lifecycle:${cronId}`, run);
      if (result === null) await deps.crons.defer(cronId, now() + BUSY_DEFER_MS);
      return result;
    }
    return deps.advisoryLock.withLock(`cron-lifecycle:${cronId}`, run);
  }

  async function fire(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {
    const current = await deps.crons.get(cron.id);
    if (!current || current.archived || !current.enabled) {
      await deps.crons.recordFire(cron.id, {
        fireKey,
        threadRef: cronFireThreadRef(cron.id, fireKey),
        firedAt: t,
        endedAt: now(),
        status: "silent",
        note: "Cron is no longer enabled",
      });
      return { authzFailed: true };
    }
    const skipped =
      scheduledAt !== undefined
        ? await deps.skipFire?.(current).catch(reportFailureAs(`scheduler: skip check for cron ${cron.id}`, undefined))
        : undefined;
    if (skipped) {
      await deps.crons.recordFire(cron.id, {
        fireKey,
        threadRef: cronFireThreadRef(cron.id, fireKey),
        firedAt: t,
        endedAt: now(),
        scheduledAt,
        status: "silent",
        note: skipped,
      });
      return { authzFailed: false };
    }
    return fireEnabled(current, t, fireKey, scheduledAt);
  }

  async function fireEnabled(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {
    const threadRef = cronFireThreadRef(cron.id, fireKey);
    const runningEntry: CronFireLogEntry = {
      fireKey,
      threadRef,
      firedAt: t,
      ...(scheduledAt !== undefined ? { scheduledAt } : {}),
      status: "running",
    };
    if (cron.loopId) {
      await deps.crons.beginFire(cron.id, runningEntry);
      let result: { status?: TurnResult["status"]; note?: string };
      if (!deps.fireLoop) {
        result = { status: "failed", note: "loop service unavailable" };
      } else
        try {
          result = await deps.fireLoop(cron.loopId, fireKey, cron.id);
        } catch (e) {
          result = { status: "failed", note: errMessage(e) };
        }
      await deps.crons.recordFire(cron.id, {
        fireKey,
        threadRef,
        firedAt: t,
        endedAt: now(),
        ...(scheduledAt !== undefined ? { scheduledAt } : {}),
        status: result.status ?? "ok",
        ...(result.note ? { note: truncate(result.note, CRON_FIRE_REPLY_MAX_CHARS) } : {}),
      });
      if (isOneShotSchedule(cron.schedule)) await deps.crons.setEnabled(cron.id, false);
      return { authzFailed: false };
    }
    let outcome: Awaited<ReturnType<typeof runTrigger>>;
    try {
      outcome = await runTrigger(
        {
          deliveries: deps.deliveries,
          idempotency: deps.idempotency,
          identity: deps.identity,
          run: deps.run,
          ...(deps.directory ? { directory: deps.directory } : {}),
          ...(deps.currentScopeMembers ? { currentScopeMembers: deps.currentScopeMembers } : {}),
          ...(deps.sessions ? { sessions: deps.sessions } : {}),
          ...(deps.resolveRecipient ? { resolveRecipient: deps.resolveRecipient } : {}),
        },
        {
          ...cronTriggerAuthority(cron),
          input: (homeRoster) => renderCronFireInput(cron, cronMentionRoster(cron, homeRoster)),
          fireKey,
          threadRef,
          surface: "cron",
          ...(cron.runtime ? { runtime: cron.runtime } : {}),
          ...(cron.title ? { title: cron.title } : {}),
          onClaimed: async () => {
            await deps.crons.beginFire(cron.id, runningEntry);
          },
          ...(cron.message !== undefined ? { message: cron.message } : {}),
          ...(cron.destination ? { destination: cron.destination } : {}),
          deferWhenBusy: scheduledAt !== undefined && t - scheduledAt <= BUSY_DEFER_MAX_LATE_MS,
        },
      );
    } catch (e) {
      await deps.crons.recordFire(cron.id, {
        fireKey,
        threadRef,
        firedAt: t,
        endedAt: now(),
        ...(scheduledAt !== undefined ? { scheduledAt } : {}),
        status: "failed",
        note: truncate(errMessage(e), CRON_FIRE_REPLY_MAX_CHARS),
      });
      throw e;
    }
    if (outcome.deferred) {
      const deferUntil = now() + BUSY_DEFER_MS;
      await deps.crons.recordFire(cron.id, {
        fireKey,
        threadRef,
        firedAt: t,
        endedAt: now(),
        ...(scheduledAt !== undefined ? { scheduledAt } : {}),
        status: "deferred",
        note: `session busy — retrying at ${utcMinute(deferUntil)}`,
      });
      await deps.crons.defer(cron.id, deferUntil);
      return { authzFailed: false, deferred: true };
    }
    if (outcome.ran || outcome.authzFailed) {
      await deps.crons.recordFire(cron.id, {
        fireKey,
        threadRef,
        firedAt: t,
        endedAt: now(),
        ...(scheduledAt !== undefined ? { scheduledAt } : {}),
        status: outcome.status ?? (outcome.authzFailed ? "refused" : "ok"),
        ...(outcome.note ? { note: truncate(outcome.note, CRON_FIRE_REPLY_MAX_CHARS) } : {}),
        ...(outcome.reply !== undefined ? { reply: cronFireLogReply(outcome.reply) } : {}),
        ...(outcome.sessionId ? { sessionId: outcome.sessionId } : {}),
      });
    }
    if (outcome.ownerLeft) await (deps.deleteCron ? deps.deleteCron(cron.id) : deps.crons.delete(cron.id));
    else if (outcome.authzFailed) await deps.crons.setEnabled(cron.id, false);
    if (outcome.authzFailed) return { authzFailed: true };
    if (isOneShotSchedule(cron.schedule)) await deps.crons.setEnabled(cron.id, false);
    return { authzFailed: false };
  }

  let lastStrandedSweep = 0;
  const sweepStranded = async (t: number): Promise<void> => {
    if (t - lastStrandedSweep < STRANDED_SWEEP_INTERVAL_MS) return;
    lastStrandedSweep = t;
    try {
      const swept = await deps.crons.sweepStrandedFires(t);
      if (swept > 0) console.warn(`[scheduler] closed ${swept} stranded running fire(s) as failed`);
    } catch (e) {
      reportFailure("scheduler: stranded-fire sweep", e);
    }
  };

  let lastFireGc = 0;
  const gcFires = async (t: number): Promise<void> => {
    if (t - lastFireGc < FIRE_GC_INTERVAL_MS) return;
    lastFireGc = t;
    try {
      const pruned = await deps.crons.pruneFires(t);
      if (pruned > 0) console.log(`[scheduler] pruned ${pruned} old cron fire row(s)`);
    } catch (e) {
      reportFailure("scheduler: cron fire gc", e);
    }
  };

  const fireDue = async (t: number, observed: number): Promise<void> => {
    const due = await deps.crons.due(t);
    let batch = due;
    if (due.length > maxFiresPerTick) {
      const ordered = [...due].sort((a, b) => (a.lastAttemptAt ?? 0) - (b.lastAttemptAt ?? 0));
      batch = [];
      for (const cron of ordered) {
        if (batch.length >= maxFiresPerTick) break;
        try {
          await deps.crons.markAttempted(cron.id, t);
          batch.push(cron);
        } catch (e) {
          reportFailure("scheduler: attempt mark (holding this cron back)", e);
        }
      }
      console.warn(`[scheduler] fan-out capped: firing ${batch.length}/${due.length} due crons this tick`);
    }
    for (const cron of batch) {
      if (stopped || observed !== epoch) break;
      try {
        await withCronLifecycle(cron.id, true, async () => {
          if (stopped || observed !== epoch) return;
          const { authzFailed, deferred } = await fire(
            cron,
            t,
            `cron:${cron.id}:${cron.scheduledAt}`,
            cron.scheduledAt,
          );
          if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);
        });
      } catch (e) {
        reportFailure("scheduler: fire", e);
      }
    }
  };

  const tick = async (nowArg?: number): Promise<void> => {
    const t = nowArg ?? now();
    const observed = epoch;
    await withWork(() =>
      leaderLease.hold(TICK_LEASE_KEY, async () => {
        await fireDue(t, observed);
        await sweepStranded(t);
        await gcFires(t);
        await deps.sweepAsks?.(t).catch(reportFailureAs("scheduler: ask sweep", undefined));
      }),
    );
  };

  const makeSweeper = () =>
    createSweeper(() => tick().catch(reportFailureAs("scheduler: tick", undefined)), 1000, {
      label: "scheduler",
    });

  let sweeper = makeSweeper();

  function nextSlot(cron: Cron): number | undefined {
    return recoverNextFireAt(cron.schedule, cron.createdAt, cron.lastFiredAt, cron.nextFireAt);
  }

  function nextJob(cron: Cron): CronFireJob | undefined {
    const slot = nextSlot(cron);
    if (slot === undefined) return undefined;
    return {
      cronId: cron.id,
      scheduledAt: slot,
      ...(cron.deferUntil !== undefined ? { notBefore: cron.deferUntil } : {}),
    };
  }

  async function enqueueNext(cronId: string): Promise<void> {
    const cron = await deps.crons.get(cronId);
    if (!cron || cron.archived || !cron.enabled) return;
    const job = nextJob(cron);
    if (job) await deps.jobQueue!.enqueueFire(job);
  }

  async function fireJob(job: CronFireJob): Promise<void> {
    const observed = epoch;
    const cron = await deps.crons.get(job.cronId);
    if (!cron || cron.archived || !cron.enabled) return;
    const slot = nextSlot(cron);
    if (slot !== job.scheduledAt) return;
    const t = now();
    if (t < slot) {
      await deps.jobQueue!.enqueueFire(job);
      return;
    }
    if (isDeferred(cron, t)) {
      await deps.jobQueue!.enqueueFire({ ...job, notBefore: cron.deferUntil! });
      return;
    }
    if (stopped || observed !== epoch) return;
    const result = await withCronLifecycle(job.cronId, true, async () => {
      if (stopped || observed !== epoch) return;
      if (!(await deps.crons.claimSlot(job.cronId, slot, t))) return;
      if (stopped || observed !== epoch) {
        await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
        return;
      }
      try {
        const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${slot}`, slot);
        if (authzFailed || deferred) {
          await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
          if (deferred) await enqueueNext(job.cronId);
          return;
        }
      } catch (e) {
        reportFailure("scheduler: fire", e);
        await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
        return;
      }
      await enqueueNext(job.cronId);
    });
    if (result === null) await enqueueNext(job.cronId);
  }

  async function reconcile(): Promise<void> {
    try {
      for (const cron of await deps.crons.list()) {
        if (stopped) break;
        if (cron.archived || !cron.enabled) continue;
        const job = nextJob(cron);
        if (job) await deps.jobQueue!.enqueueFire(job);
      }
    } catch (e) {
      reportFailure("scheduler: reconcile", e);
    }
  }

  async function maintain(): Promise<void> {
    await sweepStranded(now());
    await gcFires(now());
    await deps.sweepAsks?.(now()).catch(reportFailureAs("scheduler: ask sweep", undefined));
  }

  let stopSignal = Promise.withResolvers<void>();
  const leaseGuard = createSweeper(
    () =>
      deps.jobQueue!.healthy()
        ? leaderLease.hold(TICK_LEASE_KEY, async (lost) => {
            let lockLost = false;
            void lost.then(() => (lockLost = true));
            let repairAt = 0;
            while (!stopped && !lockLost && deps.jobQueue!.healthy()) {
              if (Date.now() >= repairAt) {
                repairAt = Date.now() + REPAIR_INTERVAL_MS;
                await runQueueTask(reconcile);
              }
              await runQueueTask(maintain);
              await Promise.race([sleep(MAINTENANCE_INTERVAL_MS, { unref: true }), lost, stopSignal.promise]);
            }
          })
        : undefined,
    1000,
    { label: "scheduler lease guard", immediate: true },
  );

  let stopped = false;
  let epoch = 0;
  let started = false;
  let stopFailed = false;
  let starting: Promise<void> | null = null;
  let stopping: Promise<void> | null = null;
  let pausing: Promise<void> | null = null;
  const oldSweeps = new Set<Promise<void>>();
  const retireSweep = (work: Promise<void>) => {
    oldSweeps.add(work);
    void work.finally(() => oldSweeps.delete(work));
  };
  const pending = new Set<Promise<unknown>>();
  async function runQueueTask(work: () => Promise<void>): Promise<void> {
    if (stopped) return;
    const task = withWork(work);
    pending.add(task);
    try {
      await task;
    } finally {
      pending.delete(task);
    }
  }
  return {
    tick,
    async runNow(cronId) {
      try {
        const preparing = withWork(async (): Promise<RunNowResult> => {
          const cron = await deps.crons.get(cronId);
          if (!cron || cron.archived || !cron.enabled) return { started: false, reason: "unavailable" };
          const t = now();
          const fireKey = `cron:${cron.id}:manual:${randomUUID()}`;
          const begin = await deps.crons.beginFire(
            cronId,
            { fireKey, threadRef: cronFireThreadRef(cron.id, fireKey), firedAt: t, status: "running" },
            { exclusive: true },
          );
          if (!begin.begun) {
            return begin.running
              ? { started: false, reason: "already_running", running: begin.running }
              : { started: false, reason: "unavailable" };
          }
          const settled = withWork(() => withCronLifecycle(cron.id, false, () => fire(cron, t, fireKey))).then(
            () => undefined,
            reportFailureAs("scheduler: manual fire", undefined, `cron=${cronId}`),
          );
          pending.add(settled);
          void settled.finally(() => pending.delete(settled));
          return { started: true, fireKey, settled };
        });
        pending.add(preparing);
        try {
          return await preparing;
        } finally {
          pending.delete(preparing);
        }
      } catch (error) {
        if (error instanceof WorkAdmissionClosed) return { started: false, reason: "unavailable" };
        throw error;
      }
    },
    notifyChanged(cronId) {
      if (!deps.jobQueue) return;
      void enqueueNext(cronId).catch(reportFailureAs("scheduler: cron enqueue", undefined, `cron=${cronId}`));
    },
    start(intervalMs) {
      if (started || stopping || pausing || stopFailed) return;
      started = true;
      stopped = false;
      stopSignal = Promise.withResolvers<void>();
      if (!deps.jobQueue) {
        sweeper.start(intervalMs);
        return;
      }
      const observed = epoch;
      starting = deps.jobQueue
        .start({
          onFire: (job) => (observed === epoch ? runQueueTask(() => fireJob(job)) : Promise.resolve()),
        })
        .then(
          () => {
            if (!stopped) leaseGuard.start();
          },
          (e: unknown) => {
            if (deps.requireQueueStart) throw e;
            reportFailure("scheduler: cron job queue start (falling back to interval ticks)", e);
            if (!stopped) sweeper.start(intervalMs);
          },
        );
    },
    async ready() {
      await starting;
    },
    stopClaims() {
      if (pausing) return pausing;
      stopped = true;
      started = false;
      epoch++;
      stopSignal.resolve();
      retireSweep(sweeper.stop());
      sweeper = makeSweeper();
      const guard = leaseGuard.stop();
      pausing = (async () => {
        await starting?.catch(() => {});
        if (deps.jobQueue?.stopClaims) await deps.jobQueue.stopClaims();
        else await deps.jobQueue?.stop();
        await guard;
        stopFailed = false;
      })()
        .catch((error) => {
          stopFailed = true;
          throw error;
        })
        .finally(() => {
          pausing = null;
        });
      return pausing;
    },
    async drained() {
      while (oldSweeps.size || pending.size) await Promise.all([...oldSweeps, ...pending]);
    },
    stop() {
      if (stopping) return stopping;
      stopped = true;
      epoch++;
      stopSignal.resolve();
      started = false;
      const sweeps = [sweeper.stop(), leaseGuard.stop()];
      stopping = (async () => {
        await starting?.catch(() => {});
        if (deps.jobQueue?.stopClaims) await deps.jobQueue.stopClaims();
        await pausing;
        await Promise.all(sweeps);
        await Promise.all(oldSweeps);
        while (pending.size) await Promise.allSettled(pending);
        await deps.jobQueue?.stop();
        stopFailed = false;
      })()
        .catch((error: unknown) => {
          stopFailed = true;
          throw error;
        })
        .finally(() => {
          stopping = null;
        });
      return stopping;
    },
  };
}