Multiplayer agent harness for work.
Drain and resume background loops without closing runtime stores (#1290)
Regan Bella84080966cac
diff --git a/src/cron/job-queue.ts b/src/cron/job-queue.tsindex 49f4fc6aa..b279d10bc 100644--- a/src/cron/job-queue.ts+++ b/src/cron/job-queue.ts@@ -99,7 +99,7 @@ export function createPgBossCronQueue(},async stop() {started = false;- ticker?.stop();+ await ticker?.stop();try {await boss.stop({ close: true, graceful: false });} finally {diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.tsindex b65f018e0..270b9d800 100644--- a/src/cron/scheduler.ts+++ b/src/cron/scheduler.ts@@ -59,7 +59,7 @@ export interface Scheduler {runNow(cronId: string): Promise<RunNowResult>;notifyChanged(cronId: string): void;start(intervalMs: number): void;- stop(): void;+ stop(): Promise<void>;}export interface SchedulerDeps {@@ -433,6 +433,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {await deps.sweepAsks?.(now()).catch((e: unknown) => console.error("[scheduler] ask sweep failed:", errMessage(e)));}+ let stopSignal = Promise.withResolvers<void>();const leaseGuard = createSweeper(() =>deps.jobQueue!.healthy()@@ -440,7 +441,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {let lockLost = false;void lost.then(() => (lockLost = true));while (!stopped && !lockLost && deps.jobQueue!.healthy())- await Promise.race([sleep(1000, { unref: true }), lost]);+ await Promise.race([sleep(1000, { unref: true }), lost, stopSignal.promise]);}): undefined,1000,@@ -448,6 +449,21 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {);let stopped = false;+ let started = false;+ let stopFailed = false;+ let starting: Promise<void> | null = null;+ let stopping: Promise<void> | null = null;+ const pending = new Set<Promise<void>>();+ async function runQueueTask(work: () => Promise<void>): Promise<void> {+ if (stopped) return;+ const task = work();+ pending.add(task);+ try {+ await task;+ } finally {+ pending.delete(task);+ }+ }return {tick,async runNow(cronId) {@@ -476,28 +492,53 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {void enqueueNext(cronId).catch((e: unknown) => console.error("[scheduler] cron enqueue failed:", errMessage(e)));},start(intervalMs) {+ if (started || stopping || stopFailed) return;+ started = true;+ stopped = false;+ stopSignal = Promise.withResolvers<void>();if (!deps.jobQueue) {sweeper.start(intervalMs);return;}- stopped = false;- void deps.jobQueue.start({ onFire: fireJob, onTick: reconcile }, intervalMs).then(- () => {- if (!stopped) leaseGuard.start();- },- (e: unknown) => {- console.error("[scheduler] cron job queue failed to start; falling back to interval ticks:", errMessage(e));- if (!stopped) sweeper.start(intervalMs);- },- );+ starting = deps.jobQueue+ .start(+ {+ onFire: (job) => runQueueTask(() => fireJob(job)),+ onTick: () => runQueueTask(reconcile),+ },+ intervalMs,+ )+ .then(+ () => {+ if (!stopped) leaseGuard.start();+ },+ (e: unknown) => {+ console.error("[scheduler] cron job queue failed to start; falling back to interval ticks:", errMessage(e));+ if (!stopped) sweeper.start(intervalMs);+ },+ );},stop() {+ if (stopping) return stopping;stopped = true;- sweeper.stop();- leaseGuard.stop();- void deps.jobQueue- ?.stop()- .catch((e: unknown) => console.error("[scheduler] cron job queue stop failed:", errMessage(e)));+ stopSignal.resolve();+ started = false;+ const sweeps = [sweeper.stop(), leaseGuard.stop()];+ stopping = (async () => {+ await starting;+ await deps.jobQueue?.stop();+ await Promise.all(sweeps);+ await Promise.allSettled(pending);+ stopFailed = false;+ })()+ .catch((error: unknown) => {+ stopFailed = true;+ throw error;+ })+ .finally(() => {+ stopping = null;+ });+ return stopping;},};}diff --git a/src/deploy/fly-tunnel-manager.ts b/src/deploy/fly-tunnel-manager.tsindex 643da771a..62ff9e989 100644--- a/src/deploy/fly-tunnel-manager.ts+++ b/src/deploy/fly-tunnel-manager.ts@@ -124,7 +124,7 @@ export function createFlyTunnelManager(opts: {},async stop(): Promise<void> {stopped = true;- sweeper.stop();+ await sweeper.stop();await starting?.catch(() => {});await tunnel?.stop();},diff --git a/src/files/direct-file-upload.ts b/src/files/direct-file-upload.tsindex 293c1dc71..aac8ca91a 100644--- a/src/files/direct-file-upload.ts+++ b/src/files/direct-file-upload.ts@@ -54,7 +54,7 @@ export interface DirectFileUploads {abort(id: string): Promise<void>;sweep(): Promise<void>;start(): void;- stop(): void;+ stop(): Promise<void>;}export function createDirectFileUploads(options: {diff --git a/src/index.ts b/src/index.tsindex 26e891ab5..335ab20e9 100644--- a/src/index.ts+++ b/src/index.ts@@ -130,7 +130,7 @@ function shutdown(signal: string): void {void slackRuntime.stop().catch((e: unknown) => console.error("[qm] slack plugin stop failed:", errMessage(e)));for (const runtime of slackAccountRuntimes)void runtime.stop().catch((e: unknown) => console.error("[qm] slack account stop failed:", errMessage(e)));- built.scheduler.stop();+ void built.scheduler.stop().catch((e: unknown) => console.error("[qm] scheduler stop failed:", errMessage(e)));built.suggestedActivityMaintenance.stop();built.deploymentLayerRefresh.stop();server.close();diff --git a/src/monitors/monitor-poller.ts b/src/monitors/monitor-poller.tsindex 41baafa86..865f9c55b 100644--- a/src/monitors/monitor-poller.ts+++ b/src/monitors/monitor-poller.ts@@ -24,7 +24,7 @@ const DEFAULT_MIN_FIRE_INTERVAL_MS = 60_000;export interface MonitorPoller {tick(now?: number): Promise<void>;start(intervalMs: number): void;- stop(): void;+ stop(): Promise<void>;}export interface MonitorPollerDeps {diff --git a/src/processes/process-reaper.ts b/src/processes/process-reaper.tsindex 6ac8c42a9..881fd97f1 100644--- a/src/processes/process-reaper.ts+++ b/src/processes/process-reaper.ts@@ -42,7 +42,7 @@ export function createReaperKillHook(export interface ProcessReaper {start(): void;- stop(): void;+ stop(): Promise<void>;sweep(): Promise<{ reaped: number }>;}diff --git a/src/runs/drain.ts b/src/runs/drain.tsindex 7ac4a3b80..c3138e73c 100644--- a/src/runs/drain.ts+++ b/src/runs/drain.ts@@ -4,7 +4,7 @@ import { createSweeper, type Sweeper } from "../util/sweeper.ts";export interface DrainController {start(): void;- stop(): void;+ stop(): Promise<void>;canClaim(): boolean;noteBusy(): void;}@@ -43,11 +43,11 @@ export function createDrainController(opts: {);return {start: () => sweeper.start(),- stop: () => {- sweeper.stop();+ stop: async () => {+ await sweeper.stop();if (protectionOn && opts.protection) {protectionOn = false;- void opts.protection.set(false);+ await opts.protection.set(false);}},canClaim: () => !superseded,diff --git a/src/runs/reaper.ts b/src/runs/reaper.tsindex 1f2ee888a..363f4bf7d 100644--- a/src/runs/reaper.ts+++ b/src/runs/reaper.ts@@ -19,7 +19,7 @@ function recordReapEvent(errors: ErrorLog, e: ReapEvent): void {export interface Reaper {start(): void;- stop(): void;+ stop(): Promise<void>;sweep(): Promise<{ requeued: number; parked: number }>;}diff --git a/src/skills/skill-sync-engine.ts b/src/skills/skill-sync-engine.tsindex 23a55d04c..97bda5d7b 100644--- a/src/skills/skill-sync-engine.ts+++ b/src/skills/skill-sync-engine.ts@@ -10,7 +10,7 @@ const DEFAULT_INTERVAL_MS = 300_000;export interface SkillSyncEngine {tick(now?: number): Promise<void>;start(intervalMs: number): void;- stop(): void;+ stop(): Promise<void>;}export interface SkillSyncDeps {diff --git a/src/slack/index.ts b/src/slack/index.tsindex 7035cae2a..180d86dba 100644--- a/src/slack/index.ts+++ b/src/slack/index.ts@@ -489,11 +489,12 @@ export async function startSlackPlugin(return {async stop(): Promise<void> {if (stopped) {+ await replaySweeper?.stop();await app.stop();return;}stopped = true;- replaySweeper?.stop();+ await replaySweeper?.stop();if (deliveriesTimer) clearInterval(deliveriesTimer);if (emojiCatalogTimer) clearInterval(emojiCatalogTimer);if (followerRetry) clearTimeout(followerRetry);diff --git a/src/swarms/swarm-service.ts b/src/swarms/swarm-service.tsindex c878ffbd1..2d8ca05d6 100644--- a/src/swarms/swarm-service.ts+++ b/src/swarms/swarm-service.ts@@ -60,7 +60,7 @@ interface Authority {export interface SwarmService {start(): void;- stop(): void;+ stop(): Promise<void>;sweep(): Promise<void>;inspect(caller: SwarmCaller): Promise<{id: string;diff --git a/src/util/sweeper.ts b/src/util/sweeper.tsindex a707822e2..f626975df 100644--- a/src/util/sweeper.ts+++ b/src/util/sweeper.ts@@ -2,7 +2,7 @@ import { swallow, swallowAs } from "./errors.ts";export interface Sweeper {start(intervalMs?: number): void;- stop(): void;+ stop(): Promise<void>;}export function createSweeper(@@ -12,16 +12,20 @@ export function createSweeper(): Sweeper {const label = opts.label ?? "sweeper";let timer: ReturnType<typeof setInterval> | null = null;+ const pending = new Set<Promise<void>>();+ let stopping: Promise<void> | null = null;const sweep = (): void => {try {- void Promise.resolve(fn()).catch(swallowAs(`${label}: sweep failed`, undefined));+ const work = Promise.resolve(fn()).then(() => {}, swallowAs(`${label}: sweep failed`, undefined));+ pending.add(work);+ void work.finally(() => pending.delete(work));} catch (e) {swallow(`${label}: sweep failed`, e);}};return {start(intervalMs?: number) {- if (timer) return;+ if (timer || stopping) return;timer = setInterval(sweep, intervalMs ?? defaultIntervalMs);timer.unref?.();if (opts.immediate) sweep();@@ -29,6 +33,12 @@ export function createSweeper(stop() {if (timer) clearInterval(timer);timer = null;+ stopping ??= Promise.all(pending)+ .then(() => {})+ .finally(() => {+ stopping = null;+ });+ return stopping;},};}diff --git a/src/wake/sweep.ts b/src/wake/sweep.tsindex 1694741ea..d4a3851d6 100644--- a/src/wake/sweep.ts+++ b/src/wake/sweep.ts@@ -16,7 +16,7 @@ export interface WakeSweepOptions {export interface WakeSweep {start(): void;- stop(): void;+ stop(): Promise<void>;sweep(): Promise<{ swept: number; fresh: number }>;}diff --git a/src/wiring.ts b/src/wiring.tsindex 3e1baaff4..8c576602c 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -384,12 +384,15 @@ import { createSlackInstallationStore, type SlackInstallationStore } from "./surexport interface Runtime {start(): void;+ startBackground(): void;+ stopBackground(): Promise<void>;+ backgroundDrained(): Promise<void>;stop(): Promise<void>;releaseInFlightRuns(): Promise<void>;}export function stopWithBackstop(- runtime: Runtime,+ runtime: Pick<Runtime, "stop" | "releaseInFlightRuns">,shutdownDrainMs: number,label: string,beforeExit?: () => void,@@ -2256,50 +2259,87 @@ export function buildApp({ immediate: true },): null;+ let backgroundRunning = false;+ let backgroundStopping: Promise<void> | null = null;+ let backgroundGeneration = 0;+ function startBackground(): void {+ if (backgroundRunning || backgroundStopping) return;+ backgroundRunning = true;+ drain.start();+ const generation = ++backgroundGeneration;+ for (const worker of workers) {+ const drained = worker.drained();+ worker.start();+ void drained+ .then(() => {+ if (backgroundRunning && generation === backgroundGeneration) worker.start();+ })+ .catch(swallowAs("wiring: worker resume failed", undefined));+ }+ reaper.start();+ processReaper?.start();+ monitorPoller?.start(config.monitorPollMs);+ monitorRetentionSweeper.start();+ if (config.skillSyncPollMs > 0) skillSyncEngine.start(config.skillSyncPollMs);+ blobSweeper.start();+ fileUploads?.start();+ idleSweeper?.start();+ keepWarmSweeper.start();+ deepIdleSweeper?.start();+ wakeSweep.start();+ swarms?.start();+ orphanedSignalSweeper.start();+ sessionReturnSweeper.start();+ }+ function stopBackground(): Promise<void> {+ if (backgroundStopping) return backgroundStopping;+ backgroundRunning = false;+ backgroundGeneration++;+ const stopping = [+ reaper.stop(),+ processReaper?.stop(),+ monitorPoller?.stop(),+ monitorRetentionSweeper.stop(),+ skillSyncEngine.stop(),+ idleSweeper?.stop(),+ keepWarmSweeper.stop(),+ deepIdleSweeper?.stop(),+ blobSweeper.stop(),+ fileUploads?.stop(),+ wakeSweep.stop(),+ swarms?.stop(),+ orphanedSignalSweeper.stop(),+ sessionReturnSweeper.stop(),+ ...workers.map((worker) => worker.stopClaims()),+ ];+ backgroundStopping = Promise.all(stopping)+ .then(() => {})+ .finally(() => {+ backgroundStopping = null;+ });+ return backgroundStopping;+ }const runtime: Runtime = {start() {flyTunnel?.monitor();- if (!config.backgroundWorkEnabled) return;- for (const w of workers) w.start();- reaper.start();- processReaper?.start();- monitorPoller?.start(config.monitorPollMs);- monitorRetentionSweeper.start();- if (config.skillSyncPollMs > 0) skillSyncEngine.start(config.skillSyncPollMs);- blobSweeper.start();- fileUploads?.start();- idleSweeper?.start();- keepWarmSweeper.start();- deepIdleSweeper?.start();- wakeSweep.start();- swarms?.start();- orphanedSignalSweeper.start();- sessionReturnSweeper.start();- drain.start();+ if (config.backgroundWorkEnabled) startBackground();+ },+ startBackground,+ stopBackground,+ async backgroundDrained() {+ await backgroundStopping;+ await Promise.all(workers.map((worker) => worker.drained()));},async releaseInFlightRuns() {await Promise.all(workers.map((w) => w.releaseInFlight()));},async stop() {- reaper.stop();- processReaper?.stop();- monitorPoller?.stop();- monitorRetentionSweeper.stop();- skillSyncEngine.stop();- idleSweeper?.stop();- keepWarmSweeper.stop();- deepIdleSweeper?.stop();- blobSweeper.stop();- fileUploads?.stop();- wakeSweep.stop();- swarms?.stop();- orphanedSignalSweeper.stop();- sessionReturnSweeper.stop();+ await stopBackground();await Promise.all(workers.map((w) => w.stop(config.shutdownDrainMs))).catch(swallowAs("wiring: worker drain failed", undefined),);await Promise.all(workers.map((w) => w.releaseInFlight()));- drain.stop();+ await drain.stop();runs.close?.();void runSignals.close?.();void sessionStateBus.close?.();diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.tsindex 2b7937bc2..575a6ac98 100644--- a/test/cron-scheduler.test.ts+++ b/test/cron-scheduler.test.ts@@ -1430,3 +1430,71 @@ test("queue mode: a busy fire releases its slot and is re-queued to run after thassert.deepEqual(enqueued.pop(), { cronId: cron.id, scheduledAt: 36_000 }, "the next slot is chained without a hold");scheduler.stop();});++test("scheduler stop waits for delayed queue startup and supports an awaited restart", async () => {+ const gate = Promise.withResolvers<void>();+ let starts = 0;+ let stops = 0;+ const scheduler = createScheduler({+ crons: createCronStore(),+ deliveries: createDeliveryStore(),+ idempotency: createIdempotencyStore(),+ identity: createIdentityService(),+ run: async () => ({ status: "ok", reply: "unused" }),+ jobQueue: {+ start: async () => {+ starts++;+ await gate.promise;+ },+ enqueueFire: async () => {},+ healthy: () => false,+ stop: async () => {+ stops++;+ },+ },+ });+ scheduler.start(1000);+ scheduler.start(1000);+ const stopping = scheduler.stop();+ assert.equal(scheduler.stop(), stopping);+ scheduler.start(1000);+ assert.equal(starts, 1);+ assert.equal(stops, 0);+ gate.resolve();+ await stopping;+ assert.equal(stops, 1);+ scheduler.start(1000);+ await scheduler.stop();+ assert.equal(starts, 2);+ assert.equal(stops, 2);+});++test("scheduler cannot resume after uncertain queue shutdown until a stop retry succeeds", async () => {+ let starts = 0;+ let stops = 0;+ const scheduler = createScheduler({+ crons: createCronStore(),+ deliveries: createDeliveryStore(),+ idempotency: createIdempotencyStore(),+ identity: createIdentityService(),+ run: async () => ({ status: "ok", reply: "unused" }),+ jobQueue: {+ start: async () => {+ starts++;+ },+ enqueueFire: async () => {},+ healthy: () => false,+ stop: async () => {+ if (++stops === 1) throw new Error("uncertain queue stop");+ },+ },+ });+ scheduler.start(1000);+ await assert.rejects(scheduler.stop(), /uncertain queue stop/);+ scheduler.start(1000);+ assert.equal(starts, 1);+ await scheduler.stop();+ scheduler.start(1000);+ await scheduler.stop();+ assert.equal(starts, 2);+});diff --git a/test/file-upload-routes.test.ts b/test/file-upload-routes.test.tsindex 3f16cf757..bb1c45661 100644--- a/test/file-upload-routes.test.ts+++ b/test/file-upload-routes.test.ts@@ -52,7 +52,7 @@ const fake: DirectFileUploads = {},async sweep() {},start() {},- stop() {},+ async stop() {},};const token = (actorId: string, scope = personal) =>mintCapabilityToken({ actorId, scopeId: scope, aud: CONTROL_PLANE_AUD, exp: Date.now() + 60_000 }, secret);diff --git a/test/sweeper.test.ts b/test/sweeper.test.tsindex 8f74be02f..f0e493e71 100644--- a/test/sweeper.test.ts+++ b/test/sweeper.test.ts@@ -118,3 +118,51 @@ test("createSweeper unrefs its timer so it never keeps the process alive", () =>}assert.deepEqual(calls, ["unref"], "the interval was unref()'d on start");});++test("stop waits for all admitted sweeps and fences a restart until they settle", async () => {+ const first = Promise.withResolvers<void>();+ let ticks = 0;+ const s = createSweeper(+ async () => {+ ticks++;+ await first.promise;+ },+ 60_000,+ { immediate: true },+ );+ s.start();+ assert.equal(ticks, 1);+ let settled = false;+ const stopping = s.stop();+ assert.equal(s.stop(), stopping);+ void stopping.then(() => {+ settled = true;+ });+ s.start();+ await sleep(10);+ assert.equal(ticks, 1);+ assert.equal(settled, false);+ first.resolve();+ await stopping;+ s.start();+ assert.equal(ticks, 2);+ await s.stop();+});++test("stop drains overlapping callbacks including failed work before acknowledging", async () => {+ const gate = Promise.withResolvers<void>();+ let active = 0;+ const s = createSweeper(async () => {+ active++;+ await gate.promise;+ active--;+ throw new Error("expected sweep failure");+ }, 5);+ s.start();+ await sleep(20);+ assert.ok(active > 1);+ const stopping = s.stop();+ gate.resolve();+ await stopping;+ assert.equal(active, 0);+});diff --git a/test/worker-reaper.test.ts b/test/worker-reaper.test.tsindex 7ecb4dae3..8ddda17ec 100644--- a/test/worker-reaper.test.ts+++ b/test/worker-reaper.test.ts@@ -618,3 +618,57 @@ test("stopClaims waits for an outstanding claim to be handed back before acknowlassert.equal(turns, 0);assert.equal((await runs.get(pending.id))?.status, "pending");});++test("runtime pauses without closing stores and restores worker capacity after a busy rollback", async () => {+ const built = buildApp(+ testConfig({ dataDir: mkdtempSync(join(tmpdir(), "wr-pause-")), workers: 1, leaseTtlMs: 5_000 }),+ );+ const completing = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const complete = built.runs.complete.bind(built.runs);+ let first = true;+ built.runs.complete = async (...args) => {+ if (first) {+ first = false;+ completing.resolve();+ await release.promise;+ }+ return complete(...args);+ };+ const enqueue = (threadRef: string) =>+ built.app.turn({+ surface: "test",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef },+ text: "hello",+ async: true,+ });+ try {+ const a = await enqueue("pause-first");+ built.runtime.start();+ await completing.promise;+ await built.runtime.stopBackground();+ assert.equal((await built.runs.get(a.runId!))?.status, "running");+ const b = await enqueue("pause-second");+ assert.equal((await built.runs.get(b.runId!))?.status, "pending");+ let drained = false;+ void built.runtime.backgroundDrained().then(() => {+ drained = true;+ });+ await sleep(10);+ assert.equal(drained, false);+ built.runtime.startBackground();+ release.resolve();+ const result = await built.runs.waitFor(b.runId!, 5_000);+ assert.equal(result.status, "done");+ assert.equal(result.attempts, 1);+ await built.runtime.stopBackground();+ await built.runtime.backgroundDrained();+ built.runtime.startBackground();+ const c = await enqueue("pause-third");+ assert.equal((await built.runs.waitFor(c.runId!, 5_000)).status, "done");+ } finally {+ release.resolve();+ await built.runtime.stop();+ }+});