Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: drain scheduled handlers before closing the database

[._.]/ Adam Eivybe58ed280760
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index 91667a45b7..b0734699fd 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -45,6 +45,7 @@ import { markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdo
import { signalDetachedGroups } from '../lib/credentialBootstrap.js';
import { createHttpDrain } from '../lib/httpDrain.js';
import { createEventLoopLagProbe } from '../lib/eventLoopLagProbe.js';
+import { quiesce as quiesceScheduledEvents, drain as drainScheduledEvents } from './eventScheduler.js';
import { setUserCatalogTypes } from '../lib/catalogTypes.js';
import { runMigrations } from '../../scripts/run-migrations.js';
@@ -1158,6 +1159,14 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// during teardown must not promote a waiting job the process is about to
// abandon — the next boot fails a running record but restores a queued one.
quiesceMediaJobQueue();
+ quiesceScheduledEvents();
+ // Start alongside the request drain so a stuck background handler spends
+ // no additional shutdown budget. The DB remains open until both settle.
+ const scheduledEventsDrained = withGrace('Scheduled events', HTTP_DRAIN_WINDOW_MS, ({ finish, finishWithError }) =>
+ drainScheduledEvents().then(
+ () => finish('✅ Scheduled events drained'),
+ (err) => finishWithError('⚠️ Scheduled events drain failed', err),
+ ));
const timing = createEventLoopLagProbe().begin('tailcat');
// Concurrent steps settle out of phase order; recorded as ms since the signal.
const settledMs = {};
@@ -1326,6 +1335,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
(err) => finishWithError('⚠️ Peer sync shutdown failed', err),
));
+ await scheduledEventsDrained;
timing.mark('dbPool');
const { close } = await import('../lib/db.js');
if (typeof close === 'function') {
diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.js
index a1bf7afc99..19e3aa1ed4 100644
--- a/server/services/bootstrap.shutdown.test.js
+++ b/server/services/bootstrap.shutdown.test.js
@@ -117,6 +117,22 @@ describe('shutdown handler — request drain ordering (#8323)', () => {
// Issue #11756: the IdeaLoom export debounce is in memory, so shutdown must
// flush it — after the servers close (nothing can schedule another export),
// bounded by a grace, overlapping the DB close, and awaited before exit.
+describe('shutdown handler — scheduled handlers finish before the DB pool closes', () => {
+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');
+
+ it('quiesces and starts a bounded drain before any await, then joins it before DB close', () => {
+ const firstAwait = code.indexOf('await ');
+ expect(code.indexOf('quiesceScheduledEvents()')).toBeGreaterThan(-1);
+ expect(code.indexOf('quiesceScheduledEvents()')).toBeLessThan(firstAwait);
+ expect(code.indexOf('drainScheduledEvents()')).toBeGreaterThan(code.indexOf('quiesceScheduledEvents()'));
+ expect(code.indexOf('drainScheduledEvents()')).toBeLessThan(firstAwait);
+ expect(code).toContain("withGrace('Scheduled events', HTTP_DRAIN_WINDOW_MS,");
+ const joinAt = code.indexOf('await scheduledEventsDrained');
+ expect(joinAt).toBeGreaterThan(code.indexOf('closeServer('));
+ expect(joinAt).toBeLessThan(code.indexOf("withGrace('DB pool'"));
+ });
+});
+
describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {
const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');
diff --git a/server/services/eventScheduler.js b/server/services/eventScheduler.js
index e4c4c28b49..9bc2d40c14 100644
--- a/server/services/eventScheduler.js
+++ b/server/services/eventScheduler.js
@@ -50,6 +50,11 @@ const scheduledEvents = new Map()
// Active timers
const activeTimers = new Map()
+// Cancellation removes an event from the registry while its handler can still
+// be writing. Keep those runs until shutdown has drained them before DB close.
+const activeRuns = new Set()
+let quiesced = false
+
// Event history
const eventHistory = []
const MAX_HISTORY = 500
@@ -418,7 +423,7 @@ function schedule(config) {
nextRunAt: null,
lastRunAt: parsedLastRunAt,
runCount: 0,
- active: true,
+ active: !quiesced,
lastRunSucceeded: null,
lastRunSkipped: null,
lastSkipReason: null,
@@ -474,7 +479,7 @@ function schedule(config) {
* @param {Object} event - Event object
*/
function scheduleNextRun(event) {
- if (!event.active || !event.nextRunAt) return
+ if (quiesced || !event.active || !event.nextRunAt) return
const delay = event.nextRunAt - Date.now()
if (delay < 0) {
@@ -600,9 +605,12 @@ function recordEventHistory(event, { startTime, success, error, skipped, skipRea
*/
function runEvent(event) {
if (event.inFlight) return event.inFlight
+ if (quiesced) return Promise.resolve()
event.inFlight = executeEvent(event).finally(() => {
+ activeRuns.delete(event.inFlight)
event.inFlight = null
})
+ activeRuns.add(event.inFlight)
return event.inFlight
}
@@ -727,6 +735,7 @@ function pause(id) {
* @returns {boolean} - True if event was found and resumed
*/
function resume(id) {
+ if (quiesced) return false
const event = scheduledEvents.get(id)
if (!event) return false
@@ -843,6 +852,17 @@ function cancelAll() {
return count
}
+/** Stop admitting runs for process exit; active handlers retain their ownership. */
+function quiesce() {
+ quiesced = true
+ for (const event of scheduledEvents.values()) deactivate(event)
+}
+
+/** Quiesce first; the shutdown caller owns the bounded wait before closing DB. */
+function drain() {
+ return Promise.allSettled([...activeRuns])
+}
+
/**
* Trigger an event immediately (for testing or manual runs). A trigger that
* lands while the event's timer run is still in flight joins that run rather
@@ -853,6 +873,7 @@ function cancelAll() {
* on a run already in progress.
*/
async function triggerNow(id) {
+ if (quiesced) return false
const event = scheduledEvents.get(id)
if (!event) return false
@@ -907,6 +928,8 @@ export {
getHistory,
getStats,
cancelAll,
+ quiesce,
+ drain,
triggerNow,
parseCronToNextRun,
parseCronToPrevRun,
diff --git a/server/services/eventScheduler.shutdown.test.js b/server/services/eventScheduler.shutdown.test.js
new file mode 100644
index 0000000000..df34954c8c
--- /dev/null
+++ b/server/services/eventScheduler.shutdown.test.js
@@ -0,0 +1,50 @@
+import { afterEach, expect, it, vi } from 'vitest';
+import { cancel, drain, getHistory, getStats, quiesce, resume, schedule, triggerNow } from './eventScheduler.js';
+
+afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+});
+
+// A cancelled handler must finish its DB write before shutdown ends the pool;
+// its late follow-up must also stay disarmed, even after it leaves the registry.
+it('drains cancelled handlers before DB close and refuses new work during shutdown', async () => {
+ vi.useFakeTimers();
+ const errors = vi.spyOn(console, 'error').mockImplementation(() => {});
+ let release;
+ const gate = new Promise(resolve => { release = resolve; });
+ let poolEnded = false;
+ const query = vi.fn(() => {
+ if (poolEnded) throw new Error('Cannot use a pool after calling end on the pool');
+ });
+ const followUp = vi.fn();
+ schedule({ id: 'writing', type: 'interval', intervalMs: 100, handler: async () => {
+ await gate;
+ query();
+ schedule({ id: 'follow-up', type: 'once', delayMs: 1, handler: followUp });
+ } });
+ const running = triggerNow('writing');
+ cancel('writing');
+ schedule({ id: 'waiting', type: 'once', delayMs: 1, handler: followUp });
+
+ quiesce();
+ quiesce();
+ const closing = drain().then(() => { poolEnded = true; });
+ await Promise.resolve();
+ expect(poolEnded).toBe(false);
+ expect(await triggerNow('waiting')).toBe(false);
+ expect(resume('waiting')).toBe(false);
+ await vi.advanceTimersByTimeAsync(10);
+ expect(followUp).not.toHaveBeenCalled();
+
+ release();
+ await running;
+ await closing;
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(poolEnded).toBe(true);
+ expect(query).toHaveBeenCalledOnce();
+ expect(followUp).not.toHaveBeenCalled();
+ expect(getStats()).toMatchObject({ activeEvents: 0, activeTimers: 0 });
+ expect(getHistory({ eventId: 'writing' })[0]).toMatchObject({ success: true, error: null });
+ expect(errors).not.toHaveBeenCalled();
+});