Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Drain and resume background loops without closing runtime stores (#1290)

Regan Bella84080966cac
diff --git a/src/cron/job-queue.ts b/src/cron/job-queue.ts
index 49f4fc6aa..b279d10bc 100644
--- a/src/cron/job-queue.ts
+++ b/src/cron/job-queue.ts
@@ -99,7 +99,7 @@ export function createPgBossCronQueue(
},
async stop() {
started = false;
- ticker?.stop();
+ await ticker?.stop();
try {
await boss.stop({ close: true, graceful: false });
} finally {
diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.ts
index b65f018e0..270b9d800 100644
--- a/src/cron/scheduler.ts
+++ b/src/cron/scheduler.ts
@@ -59,7 +59,7 @@ export interface Scheduler {
runNow(cronId: string): Promise<RunNowResult>;
notifyChanged(cronId: string): void;
start(intervalMs: number): void;
- stop(): void;
+ stop(): Promise<void>;
}
export interface SchedulerDeps {
@@ -433,6 +433,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
await deps.sweepAsks?.(now()).catch((e: unknown) => console.error("[scheduler] ask sweep failed:", errMessage(e)));
}
+ let stopSignal = Promise.withResolvers<void>();
const leaseGuard = createSweeper(
() =>
deps.jobQueue!.healthy()
@@ -440,7 +441,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
let lockLost = false;
void lost.then(() => (lockLost = true));
while (!stopped && !lockLost && deps.jobQueue!.healthy())
- await Promise.race([sleep(1000, { unref: true }), lost]);
+ await Promise.race([sleep(1000, { unref: true }), lost, stopSignal.promise]);
})
: undefined,
1000,
@@ -448,6 +449,21 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
);
let stopped = false;
+ let started = false;
+ let stopFailed = false;
+ let starting: Promise<void> | null = null;
+ let stopping: Promise<void> | null = null;
+ const pending = new Set<Promise<void>>();
+ async function runQueueTask(work: () => Promise<void>): Promise<void> {
+ if (stopped) return;
+ const task = work();
+ pending.add(task);
+ try {
+ await task;
+ } finally {
+ pending.delete(task);
+ }
+ }
return {
tick,
async runNow(cronId) {
@@ -476,28 +492,53 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
void enqueueNext(cronId).catch((e: unknown) => console.error("[scheduler] cron enqueue failed:", errMessage(e)));
},
start(intervalMs) {
+ if (started || stopping || stopFailed) return;
+ started = true;
+ stopped = false;
+ stopSignal = Promise.withResolvers<void>();
if (!deps.jobQueue) {
sweeper.start(intervalMs);
return;
}
- stopped = false;
- void deps.jobQueue.start({ onFire: fireJob, onTick: reconcile }, intervalMs).then(
- () => {
- if (!stopped) leaseGuard.start();
- },
- (e: unknown) => {
- console.error("[scheduler] cron job queue failed to start; falling back to interval ticks:", errMessage(e));
- if (!stopped) sweeper.start(intervalMs);
- },
- );
+ starting = deps.jobQueue
+ .start(
+ {
+ onFire: (job) => runQueueTask(() => fireJob(job)),
+ onTick: () => runQueueTask(reconcile),
+ },
+ intervalMs,
+ )
+ .then(
+ () => {
+ if (!stopped) leaseGuard.start();
+ },
+ (e: unknown) => {
+ console.error("[scheduler] cron job queue failed to start; falling back to interval ticks:", errMessage(e));
+ if (!stopped) sweeper.start(intervalMs);
+ },
+ );
},
stop() {
+ if (stopping) return stopping;
stopped = true;
- sweeper.stop();
- leaseGuard.stop();
- void deps.jobQueue
- ?.stop()
- .catch((e: unknown) => console.error("[scheduler] cron job queue stop failed:", errMessage(e)));
+ stopSignal.resolve();
+ started = false;
+ const sweeps = [sweeper.stop(), leaseGuard.stop()];
+ stopping = (async () => {
+ await starting;
+ await deps.jobQueue?.stop();
+ await Promise.all(sweeps);
+ await Promise.allSettled(pending);
+ stopFailed = false;
+ })()
+ .catch((error: unknown) => {
+ stopFailed = true;
+ throw error;
+ })
+ .finally(() => {
+ stopping = null;
+ });
+ return stopping;
},
};
}
diff --git a/src/deploy/fly-tunnel-manager.ts b/src/deploy/fly-tunnel-manager.ts
index 643da771a..62ff9e989 100644
--- a/src/deploy/fly-tunnel-manager.ts
+++ b/src/deploy/fly-tunnel-manager.ts
@@ -124,7 +124,7 @@ export function createFlyTunnelManager(opts: {
},
async stop(): Promise<void> {
stopped = true;
- sweeper.stop();
+ await sweeper.stop();
await starting?.catch(() => {});
await tunnel?.stop();
},
diff --git a/src/files/direct-file-upload.ts b/src/files/direct-file-upload.ts
index 293c1dc71..aac8ca91a 100644
--- a/src/files/direct-file-upload.ts
+++ b/src/files/direct-file-upload.ts
@@ -54,7 +54,7 @@ export interface DirectFileUploads {
abort(id: string): Promise<void>;
sweep(): Promise<void>;
start(): void;
- stop(): void;
+ stop(): Promise<void>;
}
export function createDirectFileUploads(options: {
diff --git a/src/index.ts b/src/index.ts
index 26e891ab5..335ab20e9 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -130,7 +130,7 @@ function shutdown(signal: string): void {
void slackRuntime.stop().catch((e: unknown) => console.error("[qm] slack plugin stop failed:", errMessage(e)));
for (const runtime of slackAccountRuntimes)
void runtime.stop().catch((e: unknown) => console.error("[qm] slack account stop failed:", errMessage(e)));
- built.scheduler.stop();
+ void built.scheduler.stop().catch((e: unknown) => console.error("[qm] scheduler stop failed:", errMessage(e)));
built.suggestedActivityMaintenance.stop();
built.deploymentLayerRefresh.stop();
server.close();
diff --git a/src/monitors/monitor-poller.ts b/src/monitors/monitor-poller.ts
index 41baafa86..865f9c55b 100644
--- a/src/monitors/monitor-poller.ts
+++ b/src/monitors/monitor-poller.ts
@@ -24,7 +24,7 @@ const DEFAULT_MIN_FIRE_INTERVAL_MS = 60_000;
export interface MonitorPoller {
tick(now?: number): Promise<void>;
start(intervalMs: number): void;
- stop(): void;
+ stop(): Promise<void>;
}
export interface MonitorPollerDeps {
diff --git a/src/processes/process-reaper.ts b/src/processes/process-reaper.ts
index 6ac8c42a9..881fd97f1 100644
--- a/src/processes/process-reaper.ts
+++ b/src/processes/process-reaper.ts
@@ -42,7 +42,7 @@ export function createReaperKillHook(
export interface ProcessReaper {
start(): void;
- stop(): void;
+ stop(): Promise<void>;
sweep(): Promise<{ reaped: number }>;
}
diff --git a/src/runs/drain.ts b/src/runs/drain.ts
index 7ac4a3b80..c3138e73c 100644
--- a/src/runs/drain.ts
+++ b/src/runs/drain.ts
@@ -4,7 +4,7 @@ import { createSweeper, type Sweeper } from "../util/sweeper.ts";
export interface DrainController {
start(): void;
- stop(): void;
+ stop(): Promise<void>;
canClaim(): boolean;
noteBusy(): void;
}
@@ -43,11 +43,11 @@ export function createDrainController(opts: {
);
return {
start: () => sweeper.start(),
- stop: () => {
- sweeper.stop();
+ stop: async () => {
+ await sweeper.stop();
if (protectionOn && opts.protection) {
protectionOn = false;
- void opts.protection.set(false);
+ await opts.protection.set(false);
}
},
canClaim: () => !superseded,
diff --git a/src/runs/reaper.ts b/src/runs/reaper.ts
index 1f2ee888a..363f4bf7d 100644
--- a/src/runs/reaper.ts
+++ b/src/runs/reaper.ts
@@ -19,7 +19,7 @@ function recordReapEvent(errors: ErrorLog, e: ReapEvent): void {
export interface Reaper {
start(): void;
- stop(): void;
+ stop(): Promise<void>;
sweep(): Promise<{ requeued: number; parked: number }>;
}
diff --git a/src/skills/skill-sync-engine.ts b/src/skills/skill-sync-engine.ts
index 23a55d04c..97bda5d7b 100644
--- a/src/skills/skill-sync-engine.ts
+++ b/src/skills/skill-sync-engine.ts
@@ -10,7 +10,7 @@ const DEFAULT_INTERVAL_MS = 300_000;
export interface SkillSyncEngine {
tick(now?: number): Promise<void>;
start(intervalMs: number): void;
- stop(): void;
+ stop(): Promise<void>;
}
export interface SkillSyncDeps {
diff --git a/src/slack/index.ts b/src/slack/index.ts
index 7035cae2a..180d86dba 100644
--- a/src/slack/index.ts
+++ b/src/slack/index.ts
@@ -489,11 +489,12 @@ export async function startSlackPlugin(
return {
async stop(): Promise<void> {
if (stopped) {
+ await replaySweeper?.stop();
await app.stop();
return;
}
stopped = true;
- replaySweeper?.stop();
+ await replaySweeper?.stop();
if (deliveriesTimer) clearInterval(deliveriesTimer);
if (emojiCatalogTimer) clearInterval(emojiCatalogTimer);
if (followerRetry) clearTimeout(followerRetry);
diff --git a/src/swarms/swarm-service.ts b/src/swarms/swarm-service.ts
index c878ffbd1..2d8ca05d6 100644
--- a/src/swarms/swarm-service.ts
+++ b/src/swarms/swarm-service.ts
@@ -60,7 +60,7 @@ interface Authority {
export interface SwarmService {
start(): void;
- stop(): void;
+ stop(): Promise<void>;
sweep(): Promise<void>;
inspect(caller: SwarmCaller): Promise<{
id: string;
diff --git a/src/util/sweeper.ts b/src/util/sweeper.ts
index a707822e2..f626975df 100644
--- a/src/util/sweeper.ts
+++ b/src/util/sweeper.ts
@@ -2,7 +2,7 @@ import { swallow, swallowAs } from "./errors.ts";
export interface Sweeper {
start(intervalMs?: number): void;
- stop(): void;
+ stop(): Promise<void>;
}
export function createSweeper(
@@ -12,16 +12,20 @@ export function createSweeper(
): Sweeper {
const label = opts.label ?? "sweeper";
let timer: ReturnType<typeof setInterval> | null = null;
+ const pending = new Set<Promise<void>>();
+ let stopping: Promise<void> | null = null;
const sweep = (): void => {
try {
- void Promise.resolve(fn()).catch(swallowAs(`${label}: sweep failed`, undefined));
+ const work = Promise.resolve(fn()).then(() => {}, swallowAs(`${label}: sweep failed`, undefined));
+ pending.add(work);
+ void work.finally(() => pending.delete(work));
} catch (e) {
swallow(`${label}: sweep failed`, e);
}
};
return {
start(intervalMs?: number) {
- if (timer) return;
+ if (timer || stopping) return;
timer = setInterval(sweep, intervalMs ?? defaultIntervalMs);
timer.unref?.();
if (opts.immediate) sweep();
@@ -29,6 +33,12 @@ export function createSweeper(
stop() {
if (timer) clearInterval(timer);
timer = null;
+ stopping ??= Promise.all(pending)
+ .then(() => {})
+ .finally(() => {
+ stopping = null;
+ });
+ return stopping;
},
};
}
diff --git a/src/wake/sweep.ts b/src/wake/sweep.ts
index 1694741ea..d4a3851d6 100644
--- a/src/wake/sweep.ts
+++ b/src/wake/sweep.ts
@@ -16,7 +16,7 @@ export interface WakeSweepOptions {
export interface WakeSweep {
start(): void;
- stop(): void;
+ stop(): Promise<void>;
sweep(): Promise<{ swept: number; fresh: number }>;
}
diff --git a/src/wiring.ts b/src/wiring.ts
index 3e1baaff4..8c576602c 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -384,12 +384,15 @@ import { createSlackInstallationStore, type SlackInstallationStore } from "./sur
export interface Runtime {
start(): void;
+ startBackground(): void;
+ stopBackground(): Promise<void>;
+ backgroundDrained(): Promise<void>;
stop(): Promise<void>;
releaseInFlightRuns(): Promise<void>;
}
export function stopWithBackstop(
- runtime: Runtime,
+ runtime: Pick<Runtime, "stop" | "releaseInFlightRuns">,
shutdownDrainMs: number,
label: string,
beforeExit?: () => void,
@@ -2256,50 +2259,87 @@ export function buildApp(
{ immediate: true },
)
: null;
+ let backgroundRunning = false;
+ let backgroundStopping: Promise<void> | null = null;
+ let backgroundGeneration = 0;
+ function startBackground(): void {
+ if (backgroundRunning || backgroundStopping) return;
+ backgroundRunning = true;
+ drain.start();
+ const generation = ++backgroundGeneration;
+ for (const worker of workers) {
+ const drained = worker.drained();
+ worker.start();
+ void drained
+ .then(() => {
+ if (backgroundRunning && generation === backgroundGeneration) worker.start();
+ })
+ .catch(swallowAs("wiring: worker resume failed", undefined));
+ }
+ reaper.start();
+ processReaper?.start();
+ monitorPoller?.start(config.monitorPollMs);
+ monitorRetentionSweeper.start();
+ if (config.skillSyncPollMs > 0) skillSyncEngine.start(config.skillSyncPollMs);
+ blobSweeper.start();
+ fileUploads?.start();
+ idleSweeper?.start();
+ keepWarmSweeper.start();
+ deepIdleSweeper?.start();
+ wakeSweep.start();
+ swarms?.start();
+ orphanedSignalSweeper.start();
+ sessionReturnSweeper.start();
+ }
+ function stopBackground(): Promise<void> {
+ if (backgroundStopping) return backgroundStopping;
+ backgroundRunning = false;
+ backgroundGeneration++;
+ const stopping = [
+ reaper.stop(),
+ processReaper?.stop(),
+ monitorPoller?.stop(),
+ monitorRetentionSweeper.stop(),
+ skillSyncEngine.stop(),
+ idleSweeper?.stop(),
+ keepWarmSweeper.stop(),
+ deepIdleSweeper?.stop(),
+ blobSweeper.stop(),
+ fileUploads?.stop(),
+ wakeSweep.stop(),
+ swarms?.stop(),
+ orphanedSignalSweeper.stop(),
+ sessionReturnSweeper.stop(),
+ ...workers.map((worker) => worker.stopClaims()),
+ ];
+ backgroundStopping = Promise.all(stopping)
+ .then(() => {})
+ .finally(() => {
+ backgroundStopping = null;
+ });
+ return backgroundStopping;
+ }
const runtime: Runtime = {
start() {
flyTunnel?.monitor();
- if (!config.backgroundWorkEnabled) return;
- for (const w of workers) w.start();
- reaper.start();
- processReaper?.start();
- monitorPoller?.start(config.monitorPollMs);
- monitorRetentionSweeper.start();
- if (config.skillSyncPollMs > 0) skillSyncEngine.start(config.skillSyncPollMs);
- blobSweeper.start();
- fileUploads?.start();
- idleSweeper?.start();
- keepWarmSweeper.start();
- deepIdleSweeper?.start();
- wakeSweep.start();
- swarms?.start();
- orphanedSignalSweeper.start();
- sessionReturnSweeper.start();
- drain.start();
+ if (config.backgroundWorkEnabled) startBackground();
+ },
+ startBackground,
+ stopBackground,
+ async backgroundDrained() {
+ await backgroundStopping;
+ await Promise.all(workers.map((worker) => worker.drained()));
},
async releaseInFlightRuns() {
await Promise.all(workers.map((w) => w.releaseInFlight()));
},
async stop() {
- reaper.stop();
- processReaper?.stop();
- monitorPoller?.stop();
- monitorRetentionSweeper.stop();
- skillSyncEngine.stop();
- idleSweeper?.stop();
- keepWarmSweeper.stop();
- deepIdleSweeper?.stop();
- blobSweeper.stop();
- fileUploads?.stop();
- wakeSweep.stop();
- swarms?.stop();
- orphanedSignalSweeper.stop();
- sessionReturnSweeper.stop();
+ await stopBackground();
await Promise.all(workers.map((w) => w.stop(config.shutdownDrainMs))).catch(
swallowAs("wiring: worker drain failed", undefined),
);
await Promise.all(workers.map((w) => w.releaseInFlight()));
- drain.stop();
+ await drain.stop();
runs.close?.();
void runSignals.close?.();
void sessionStateBus.close?.();
diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.ts
index 2b7937bc2..575a6ac98 100644
--- a/test/cron-scheduler.test.ts
+++ b/test/cron-scheduler.test.ts
@@ -1430,3 +1430,71 @@ test("queue mode: a busy fire releases its slot and is re-queued to run after th
assert.deepEqual(enqueued.pop(), { cronId: cron.id, scheduledAt: 36_000 }, "the next slot is chained without a hold");
scheduler.stop();
});
+
+test("scheduler stop waits for delayed queue startup and supports an awaited restart", async () => {
+ const gate = Promise.withResolvers<void>();
+ let starts = 0;
+ let stops = 0;
+ const scheduler = createScheduler({
+ crons: createCronStore(),
+ deliveries: createDeliveryStore(),
+ idempotency: createIdempotencyStore(),
+ identity: createIdentityService(),
+ run: async () => ({ status: "ok", reply: "unused" }),
+ jobQueue: {
+ start: async () => {
+ starts++;
+ await gate.promise;
+ },
+ enqueueFire: async () => {},
+ healthy: () => false,
+ stop: async () => {
+ stops++;
+ },
+ },
+ });
+ scheduler.start(1000);
+ scheduler.start(1000);
+ const stopping = scheduler.stop();
+ assert.equal(scheduler.stop(), stopping);
+ scheduler.start(1000);
+ assert.equal(starts, 1);
+ assert.equal(stops, 0);
+ gate.resolve();
+ await stopping;
+ assert.equal(stops, 1);
+ scheduler.start(1000);
+ await scheduler.stop();
+ assert.equal(starts, 2);
+ assert.equal(stops, 2);
+});
+
+test("scheduler cannot resume after uncertain queue shutdown until a stop retry succeeds", async () => {
+ let starts = 0;
+ let stops = 0;
+ const scheduler = createScheduler({
+ crons: createCronStore(),
+ deliveries: createDeliveryStore(),
+ idempotency: createIdempotencyStore(),
+ identity: createIdentityService(),
+ run: async () => ({ status: "ok", reply: "unused" }),
+ jobQueue: {
+ start: async () => {
+ starts++;
+ },
+ enqueueFire: async () => {},
+ healthy: () => false,
+ stop: async () => {
+ if (++stops === 1) throw new Error("uncertain queue stop");
+ },
+ },
+ });
+ scheduler.start(1000);
+ await assert.rejects(scheduler.stop(), /uncertain queue stop/);
+ scheduler.start(1000);
+ assert.equal(starts, 1);
+ await scheduler.stop();
+ scheduler.start(1000);
+ await scheduler.stop();
+ assert.equal(starts, 2);
+});
diff --git a/test/file-upload-routes.test.ts b/test/file-upload-routes.test.ts
index 3f16cf757..bb1c45661 100644
--- a/test/file-upload-routes.test.ts
+++ b/test/file-upload-routes.test.ts
@@ -52,7 +52,7 @@ const fake: DirectFileUploads = {
},
async sweep() {},
start() {},
- stop() {},
+ async stop() {},
};
const token = (actorId: string, scope = personal) =>
mintCapabilityToken({ actorId, scopeId: scope, aud: CONTROL_PLANE_AUD, exp: Date.now() + 60_000 }, secret);
diff --git a/test/sweeper.test.ts b/test/sweeper.test.ts
index 8f74be02f..f0e493e71 100644
--- a/test/sweeper.test.ts
+++ b/test/sweeper.test.ts
@@ -118,3 +118,51 @@ test("createSweeper unrefs its timer so it never keeps the process alive", () =>
}
assert.deepEqual(calls, ["unref"], "the interval was unref()'d on start");
});
+
+test("stop waits for all admitted sweeps and fences a restart until they settle", async () => {
+ const first = Promise.withResolvers<void>();
+ let ticks = 0;
+ const s = createSweeper(
+ async () => {
+ ticks++;
+ await first.promise;
+ },
+ 60_000,
+ { immediate: true },
+ );
+ s.start();
+ assert.equal(ticks, 1);
+ let settled = false;
+ const stopping = s.stop();
+ assert.equal(s.stop(), stopping);
+ void stopping.then(() => {
+ settled = true;
+ });
+ s.start();
+ await sleep(10);
+ assert.equal(ticks, 1);
+ assert.equal(settled, false);
+ first.resolve();
+ await stopping;
+ s.start();
+ assert.equal(ticks, 2);
+ await s.stop();
+});
+
+test("stop drains overlapping callbacks including failed work before acknowledging", async () => {
+ const gate = Promise.withResolvers<void>();
+ let active = 0;
+ const s = createSweeper(async () => {
+ active++;
+ await gate.promise;
+ active--;
+ throw new Error("expected sweep failure");
+ }, 5);
+ s.start();
+ await sleep(20);
+ assert.ok(active > 1);
+ const stopping = s.stop();
+ gate.resolve();
+ await stopping;
+ assert.equal(active, 0);
+});
diff --git a/test/worker-reaper.test.ts b/test/worker-reaper.test.ts
index 7ecb4dae3..8ddda17ec 100644
--- a/test/worker-reaper.test.ts
+++ b/test/worker-reaper.test.ts
@@ -618,3 +618,57 @@ test("stopClaims waits for an outstanding claim to be handed back before acknowl
assert.equal(turns, 0);
assert.equal((await runs.get(pending.id))?.status, "pending");
});
+
+test("runtime pauses without closing stores and restores worker capacity after a busy rollback", async () => {
+ const built = buildApp(
+ testConfig({ dataDir: mkdtempSync(join(tmpdir(), "wr-pause-")), workers: 1, leaseTtlMs: 5_000 }),
+ );
+ const completing = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const complete = built.runs.complete.bind(built.runs);
+ let first = true;
+ built.runs.complete = async (...args) => {
+ if (first) {
+ first = false;
+ completing.resolve();
+ await release.promise;
+ }
+ return complete(...args);
+ };
+ const enqueue = (threadRef: string) =>
+ built.app.turn({
+ surface: "test",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef },
+ text: "hello",
+ async: true,
+ });
+ try {
+ const a = await enqueue("pause-first");
+ built.runtime.start();
+ await completing.promise;
+ await built.runtime.stopBackground();
+ assert.equal((await built.runs.get(a.runId!))?.status, "running");
+ const b = await enqueue("pause-second");
+ assert.equal((await built.runs.get(b.runId!))?.status, "pending");
+ let drained = false;
+ void built.runtime.backgroundDrained().then(() => {
+ drained = true;
+ });
+ await sleep(10);
+ assert.equal(drained, false);
+ built.runtime.startBackground();
+ release.resolve();
+ const result = await built.runs.waitFor(b.runId!, 5_000);
+ assert.equal(result.status, "done");
+ assert.equal(result.attempts, 1);
+ await built.runtime.stopBackground();
+ await built.runtime.backgroundDrained();
+ built.runtime.startBackground();
+ const c = await enqueue("pause-third");
+ assert.equal((await built.runs.waitFor(c.runId!, 5_000)).status, "done");
+ } finally {
+ release.resolve();
+ await built.runtime.stop();
+ }
+});