Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: drain scheduled handlers before closing the database
[._.]/ Adam Eivybe58ed280760
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex 91667a45b7..b0734699fd 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -45,6 +45,7 @@ import { markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdoimport { signalDetachedGroups } from '../lib/credentialBootstrap.js';import { createHttpDrain } from '../lib/httpDrain.js';import { createEventLoopLagProbe } from '../lib/eventLoopLagProbe.js';+import { quiesce as quiesceScheduledEvents, drain as drainScheduledEvents } from './eventScheduler.js';import { setUserCatalogTypes } from '../lib/catalogTypes.js';import { runMigrations } from '../../scripts/run-migrations.js';@@ -1158,6 +1159,14 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// during teardown must not promote a waiting job the process is about to// abandon — the next boot fails a running record but restores a queued one.quiesceMediaJobQueue();+ quiesceScheduledEvents();+ // Start alongside the request drain so a stuck background handler spends+ // no additional shutdown budget. The DB remains open until both settle.+ const scheduledEventsDrained = withGrace('Scheduled events', HTTP_DRAIN_WINDOW_MS, ({ finish, finishWithError }) =>+ drainScheduledEvents().then(+ () => finish('✅ Scheduled events drained'),+ (err) => finishWithError('⚠️ Scheduled events drain failed', err),+ ));const timing = createEventLoopLagProbe().begin('tailcat');// Concurrent steps settle out of phase order; recorded as ms since the signal.const settledMs = {};@@ -1326,6 +1335,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>(err) => finishWithError('⚠️ Peer sync shutdown failed', err),));+ await scheduledEventsDrained;timing.mark('dbPool');const { close } = await import('../lib/db.js');if (typeof close === 'function') {diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.jsindex a1bf7afc99..19e3aa1ed4 100644--- a/server/services/bootstrap.shutdown.test.js+++ b/server/services/bootstrap.shutdown.test.js@@ -117,6 +117,22 @@ describe('shutdown handler — request drain ordering (#8323)', () => {// Issue #11756: the IdeaLoom export debounce is in memory, so shutdown must// flush it — after the servers close (nothing can schedule another export),// bounded by a grace, overlapping the DB close, and awaited before exit.+describe('shutdown handler — scheduled handlers finish before the DB pool closes', () => {+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');++ it('quiesces and starts a bounded drain before any await, then joins it before DB close', () => {+ const firstAwait = code.indexOf('await ');+ expect(code.indexOf('quiesceScheduledEvents()')).toBeGreaterThan(-1);+ expect(code.indexOf('quiesceScheduledEvents()')).toBeLessThan(firstAwait);+ expect(code.indexOf('drainScheduledEvents()')).toBeGreaterThan(code.indexOf('quiesceScheduledEvents()'));+ expect(code.indexOf('drainScheduledEvents()')).toBeLessThan(firstAwait);+ expect(code).toContain("withGrace('Scheduled events', HTTP_DRAIN_WINDOW_MS,");+ const joinAt = code.indexOf('await scheduledEventsDrained');+ expect(joinAt).toBeGreaterThan(code.indexOf('closeServer('));+ expect(joinAt).toBeLessThan(code.indexOf("withGrace('DB pool'"));+ });+});+describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');diff --git a/server/services/eventScheduler.js b/server/services/eventScheduler.jsindex e4c4c28b49..9bc2d40c14 100644--- a/server/services/eventScheduler.js+++ b/server/services/eventScheduler.js@@ -50,6 +50,11 @@ const scheduledEvents = new Map()// Active timersconst activeTimers = new Map()+// Cancellation removes an event from the registry while its handler can still+// be writing. Keep those runs until shutdown has drained them before DB close.+const activeRuns = new Set()+let quiesced = false+// Event historyconst eventHistory = []const MAX_HISTORY = 500@@ -418,7 +423,7 @@ function schedule(config) {nextRunAt: null,lastRunAt: parsedLastRunAt,runCount: 0,- active: true,+ active: !quiesced,lastRunSucceeded: null,lastRunSkipped: null,lastSkipReason: null,@@ -474,7 +479,7 @@ function schedule(config) {* @param {Object} event - Event object*/function scheduleNextRun(event) {- if (!event.active || !event.nextRunAt) return+ if (quiesced || !event.active || !event.nextRunAt) returnconst delay = event.nextRunAt - Date.now()if (delay < 0) {@@ -600,9 +605,12 @@ function recordEventHistory(event, { startTime, success, error, skipped, skipRea*/function runEvent(event) {if (event.inFlight) return event.inFlight+ if (quiesced) return Promise.resolve()event.inFlight = executeEvent(event).finally(() => {+ activeRuns.delete(event.inFlight)event.inFlight = null})+ activeRuns.add(event.inFlight)return event.inFlight}@@ -727,6 +735,7 @@ function pause(id) {* @returns {boolean} - True if event was found and resumed*/function resume(id) {+ if (quiesced) return falseconst event = scheduledEvents.get(id)if (!event) return false@@ -843,6 +852,17 @@ function cancelAll() {return count}+/** Stop admitting runs for process exit; active handlers retain their ownership. */+function quiesce() {+ quiesced = true+ for (const event of scheduledEvents.values()) deactivate(event)+}++/** Quiesce first; the shutdown caller owns the bounded wait before closing DB. */+function drain() {+ return Promise.allSettled([...activeRuns])+}+/*** Trigger an event immediately (for testing or manual runs). A trigger that* lands while the event's timer run is still in flight joins that run rather@@ -853,6 +873,7 @@ function cancelAll() {* on a run already in progress.*/async function triggerNow(id) {+ if (quiesced) return falseconst event = scheduledEvents.get(id)if (!event) return false@@ -907,6 +928,8 @@ export {getHistory,getStats,cancelAll,+ quiesce,+ drain,triggerNow,parseCronToNextRun,parseCronToPrevRun,diff --git a/server/services/eventScheduler.shutdown.test.js b/server/services/eventScheduler.shutdown.test.jsnew file mode 100644index 0000000000..df34954c8c--- /dev/null+++ b/server/services/eventScheduler.shutdown.test.js@@ -0,0 +1,50 @@+import { afterEach, expect, it, vi } from 'vitest';+import { cancel, drain, getHistory, getStats, quiesce, resume, schedule, triggerNow } from './eventScheduler.js';++afterEach(() => {+ vi.useRealTimers();+ vi.restoreAllMocks();+});++// A cancelled handler must finish its DB write before shutdown ends the pool;+// its late follow-up must also stay disarmed, even after it leaves the registry.+it('drains cancelled handlers before DB close and refuses new work during shutdown', async () => {+ vi.useFakeTimers();+ const errors = vi.spyOn(console, 'error').mockImplementation(() => {});+ let release;+ const gate = new Promise(resolve => { release = resolve; });+ let poolEnded = false;+ const query = vi.fn(() => {+ if (poolEnded) throw new Error('Cannot use a pool after calling end on the pool');+ });+ const followUp = vi.fn();+ schedule({ id: 'writing', type: 'interval', intervalMs: 100, handler: async () => {+ await gate;+ query();+ schedule({ id: 'follow-up', type: 'once', delayMs: 1, handler: followUp });+ } });+ const running = triggerNow('writing');+ cancel('writing');+ schedule({ id: 'waiting', type: 'once', delayMs: 1, handler: followUp });++ quiesce();+ quiesce();+ const closing = drain().then(() => { poolEnded = true; });+ await Promise.resolve();+ expect(poolEnded).toBe(false);+ expect(await triggerNow('waiting')).toBe(false);+ expect(resume('waiting')).toBe(false);+ await vi.advanceTimersByTimeAsync(10);+ expect(followUp).not.toHaveBeenCalled();++ release();+ await running;+ await closing;+ await vi.advanceTimersByTimeAsync(1000);+ expect(poolEnded).toBe(true);+ expect(query).toHaveBeenCalledOnce();+ expect(followUp).not.toHaveBeenCalled();+ expect(getStats()).toMatchObject({ activeEvents: 0, activeTimers: 0 });+ expect(getHistory({ eventId: 'writing' })[0]).toMatchObject({ success: true, error: null });+ expect(errors).not.toHaveBeenCalled();+});