Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop graceful shutdown from blocking on share-bucket watcher close (#12853)

[._.]/ Adam Eivyd0c36988de6c
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index 5a67f36b7d..6e054fbc58 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -44,6 +44,7 @@ import { conflictJournalStore } from '../lib/conflictJournal.js';
import { markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdown.js';
import { signalDetachedGroups } from '../lib/credentialBootstrap.js';
import { createHttpDrain } from '../lib/httpDrain.js';
+import { createEventLoopLagProbe } from '../lib/eventLoopLagProbe.js';
import { setUserCatalogTypes } from '../lib/catalogTypes.js';
import { runMigrations } from '../../scripts/run-migrations.js';
@@ -146,7 +147,7 @@ import { initSyncOrchestrator } from './syncOrchestrator.js';
import { initMediaJobQueue, resumeMediaJobCompletions, flushMediaJobQueue, quiesceMediaJobQueue } from './mediaJobQueue/index.js';
import { initSpriteLocalAnimationHook } from './sprites/localAnimationJobHook.js';
import { initLoraTraining } from './loraTraining/index.js';
-import { initSharing, shutdownSharing, drainPeerSyncOperations } from './sharing/index.js';
+import { initSharing, quiesceSharingForExit, drainPeerSyncOperations } from './sharing/index.js';
import { initMortalLoomStore } from './mortalLoomStore.js';
import { initUniverseBuilderCollectionHook } from './universeBuilderCollectionHook.js';
import { initCatalogImageAttachHook } from './catalogImageAttachHook.js';
@@ -1108,6 +1109,19 @@ const HTTP_DRAIN_WINDOW_MS = 5000;
// pool closes; drain + HTTP close + this + DB pool grace stay under the ceiling.
const PEER_SYNC_STOP_GRACE_MS = 1000;
+// A shutdown at or above this logs one numbers-only line naming where the time
+// went (#12853). A step's loop lag is time the event loop could not run timers —
+// a synchronous block — rather than asynchronous waiting.
+const SLOW_SHUTDOWN_MS = 1000;
+const describeSlowShutdown = (summary, settledMs) => {
+ if (summary.totalMs < SLOW_SHUTDOWN_MS) return null;
+ const phases = Object.entries(summary.phases);
+ const [slowest] = phases.reduce((max, entry) => (entry[1].ms > max[1].ms ? entry : max), phases[0]);
+ const steps = phases.map(([name, { ms, lagMs }]) => `${name}Ms=${ms} ${name}LagMs=${lagMs}`);
+ const settled = Object.entries(settledMs).map(([name, ms]) => `${name}SettledMs=${ms}`);
+ return `🐢 Graceful shutdown slow totalMs=${summary.totalMs} slowestStep=${slowest} loopLagMs=${summary.lagMs} loopLagMaxMs=${summary.lagMaxMs} ${[...steps, ...settled].join(' ')}`;
+};
+
/**
* Wire SIGTERM/SIGINT to the graceful-shutdown state machine. Idempotent per
* signal: once shutdown starts, later signals are ignored.
@@ -1132,6 +1146,14 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// during teardown must not promote a waiting job the process is about to
// abandon — the next boot fails a running record but restores a queued one.
quiesceMediaJobQueue();
+ const timing = createEventLoopLagProbe().begin('tailcat');
+ // Concurrent steps settle out of phase order; recorded as ms since the signal.
+ const settledMs = {};
+ const recordSettled = (name) => () => { settledMs[name] = timing.snapshot().totalMs; };
+ const reportIfSlow = () => {
+ const line = describeSlowShutdown(timing.end(), settledMs);
+ if (line) console.log(line);
+ };
await import('./tailcatPeer.js').then(({ stopAllForwards }) => stopAllForwards())
.catch((err) => logBootstrapFailure('❌ Tailcat forward shutdown failed', err));
await import('./tailcatServe.js').then(({ stopServeProcess }) => stopServeProcess())
@@ -1140,6 +1162,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// -shutdown would `pm2 stop` a model server the user never asked to lose,
// and PortOS is about to stop being the thing that could restart it.
stopIdleReaper();
+ timing.mark('modelHost');
await import('./fleetLlmHost.js').then(({ stopFleetLlmHost }) => stopFleetLlmHost())
.catch((err) => logBootstrapFailure('❌ Dedicated model host shutdown failed', err));
// Same reasoning for the hosted-session sweeper: a tick mid-shutdown would
@@ -1163,6 +1186,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
const forceExitTimer = setTimeout(() => {
console.error('⚠️ Graceful shutdown timed out, forcing exit');
+ reportIfSlow();
process.exit(1);
}, GRACEFUL_SHUTDOWN_TIMEOUT_MS);
// Don't let the safety timer itself keep the event loop alive — if every
@@ -1191,6 +1215,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
const markerWritten = withGrace('Host-shutdown marker', 1500, ({ finish, finishWithError }) =>
writeHostShutdownMarker({ agentIds: [...activeAgents.keys(), ...dispatchingAgentIds], signal })
.then(() => finish(), (err) => finishWithError('⚠️ Host-shutdown marker failed', err)));
+ markerWritten.then(recordSettled('marker'));
// Reach every child spawned into its own process group — agent runs, CLI
// runs and vision calls alike. A credential-bootstrap-wrapped child is
@@ -1210,6 +1235,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// pm2's TreeKill would reap it anyway, but a direct SIGTERM keeps a manual
// `kill` of the server from orphaning a Codex process holding the user's
// sign-in, and it is a no-op when nothing was ever spawned.
+ timing.mark('codex');
await withGrace('Codex app-server', 2000, ({ finish, finishWithError }) =>
stopCodexAppServer().then(
() => finish(),
@@ -1228,15 +1254,18 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// still waits for that server's connections to end, so while a slow request
// is draining this grace can lapse first — Socket.IO itself is torn down by
// then, and the drain wait below bounds the rest.
+ timing.mark('socketIo');
await withGrace('Socket.IO', 3000, ({ finish, finishWithError }) =>
io.close((err) => {
if (err && err.code !== 'ERR_SERVER_NOT_RUNNING') finishWithError('⚠️ Error closing Socket.IO', err);
else finish('✅ Socket.IO closed');
}));
+ timing.mark('ingress');
await import('./tailcatIngress.js').then(({ stopTailcatIngress }) => stopTailcatIngress())
.catch((err) => logBootstrapFailure('❌ Tailcat ingress shutdown failed', err));
// Let accepted requests finish (bounded by the drain window), then force the
// rest closed below — a handler mid-write loses its response only at the deadline.
+ timing.mark('requestDrain');
const { remaining } = await requestsDrained;
if (remaining > 0) console.warn(`⚠️ ${remaining} request(s) still running after the ${HTTP_DRAIN_WINDOW_MS}ms drain — closing them`);
// Close BOTH servers explicitly. The drain already stopped both listeners
@@ -1246,6 +1275,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// Upgraded (WebSocket) sockets are invisible to closeAllConnections() but
// keep server.close() pending; Socket.IO has already sent its close frames.
httpDrain.destroyUpgraded();
+ timing.mark('httpClose');
await Promise.all([
closeServer(httpServer, 'HTTP server'),
closeServer(localHttpServer, 'Local HTTP mirror')
@@ -1257,13 +1287,16 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// schedule another export now that the servers are closed.
const ideaLoomFlushed = withGrace('IdeaLoom auto-sync flush', 2500, ({ finish, finishWithError }) =>
flushIdeaLoomAutoSync().then(() => finish('✅ IdeaLoom auto-sync flushed'), (err) => finishWithError('⚠️ IdeaLoom auto-sync flush failed', err)));
+ ideaLoomFlushed.then(recordSettled('ideaLoom'));
// Flush the media-job queue now that no request can admit another job
// (#8325): terminal transitions and the latest progress reach the snapshot a
- // restart restores from. The file write is independent of Postgres, so it
- // runs alongside the DB close below and is awaited before exit — bounded by
- // its own timeout so it spends no budget beyond the DB close's.
+ // restart restores from. Not earlier: quiescing stops dispatch, not enqueue.
+ // Started ahead of the sharing and DB steps, so its write is already in
+ // flight when they run (#12853); awaited before exit and bounded by its own
+ // timeout so it spends no budget beyond the DB close's.
const mediaQueueFlushed = flushMediaJobQueue().then((outcome) => {
+ recordSettled('mediaFlush')();
if (outcome.ok) console.log('✅ Media job queue flushed');
else console.error(`❌ Media job queue flush failed: ${outcome.error}`);
});
@@ -1271,12 +1304,16 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// Stop peer sync before the pool ends: a peer:online convergence walk still
// running would otherwise log one pool-ended stack per remaining subscription.
// Abandoning it is safe — the next boot's peer:online pass repeats it (#11750).
+ // The exit form leaves share-bucket watchers open: closing them blocks the
+ // main thread for seconds on macOS, and process.exit releases them (#12853).
+ timing.mark('peerSync');
await withGrace('Peer sync', PEER_SYNC_STOP_GRACE_MS, ({ finish, finishWithError }) =>
- shutdownSharing().then(() => drainPeerSyncOperations()).then(
+ Promise.resolve().then(quiesceSharingForExit).then(() => drainPeerSyncOperations()).then(
() => finish(),
(err) => finishWithError('⚠️ Peer sync shutdown failed', err),
));
+ timing.mark('dbPool');
const { close } = await import('../lib/db.js');
if (typeof close === 'function') {
// Bound the DB pool close: pool.end() waits for every checked-out client to
@@ -1288,10 +1325,12 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
console.warn('ℹ️ DB pool close not available; skipping DB shutdown');
}
+ timing.mark('flushWait');
await markerWritten;
await mediaQueueFlushed;
await ideaLoomFlushed;
clearTimeout(forceExitTimer);
+ reportIfSlow();
process.exit(0);
};
process.on('SIGTERM', () => shutdown('SIGTERM'));
diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.js
index 3b9b16c3f1..50e6d380d1 100644
--- a/server/services/bootstrap.shutdown.test.js
+++ b/server/services/bootstrap.shutdown.test.js
@@ -26,6 +26,7 @@ import { dirname, join } from 'path';
// a destructured or defaulted parameter.
import { extractDeclaration, stripCommentsAndNormalize } from '../lib/mirrorParity.js';
import { logFailureWithStack } from '../lib/failureLogging.js';
+import { createEventLoopLagProbe, __createManualClock } from '../lib/eventLoopLagProbe.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
const SRC = readFileSync(join(__dirname, 'bootstrap.js'), 'utf-8').replace(/\r\n/g, '\n');
@@ -138,9 +139,9 @@ describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {
describe('shutdown handler — peer sync stops before the DB pool closes (#12585)', () => {
const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');
- it('calls shutdownSharing() then drains, under a grace, between the HTTP close and the DB pool close', () => {
- const stopAt = code.indexOf('shutdownSharing()');
- expect(stopAt, 'shutdownSharing() is not called in shutdown()').toBeGreaterThan(-1);
+ it('stops sharing then drains, under a grace, between the HTTP close and the DB pool close', () => {
+ const stopAt = code.indexOf('quiesceSharingForExit');
+ expect(stopAt, 'quiesceSharingForExit is not called in shutdown()').toBeGreaterThan(-1);
expect(code).toMatch(/withGrace\('Peer sync', PEER_SYNC_STOP_GRACE_MS,/);
expect(code.indexOf('drainPeerSyncOperations()')).toBeGreaterThan(stopAt);
expect(stopAt).toBeGreaterThan(code.indexOf('closeServer('));
@@ -253,3 +254,52 @@ describe('shutdown handler — media-job queue quiesce + flush (#8325, #8691)',
expect(awaitAt).toBeLessThan(code.indexOf('process.exit(0)'));
});
});
+
+// #12853: closing the share-bucket watchers blocked the main thread for seconds
+// on macOS, starving the marker and media-flush timers and, past PM2's kill
+// window, skipping the rest of the teardown.
+describe('shutdown handler — no blocking teardown at exit (#12853)', () => {
+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');
+
+ it('leaves share-bucket watchers open and starts the durable writes before stopping sharing', () => {
+ expect(code).not.toMatch(/shutdownSharing|shutdownAllWatchers|detachWatcher/);
+ const stopAt = code.indexOf('quiesceSharingForExit');
+ expect(code.indexOf('writeHostShutdownMarker(')).toBeLessThan(stopAt);
+ expect(code.indexOf('flushMediaJobQueue(')).toBeLessThan(stopAt);
+ });
+
+ it('reports a slow shutdown on both exits', () => {
+ const exits = [...code.matchAll(/process\.exit\([01]\)/g)];
+ expect(exits).toHaveLength(2);
+ for (const exit of exits) {
+ expect(code.slice(0, exit.index).trimEnd().endsWith('reportIfSlow();')).toBe(true);
+ }
+ });
+
+ const describeSlowShutdown = runInNewContext(`
+ ${SRC.match(/const SLOW_SHUTDOWN_MS = \d+;/)[0]}
+ ${extractDeclaration(SRC, 'describeSlowShutdown')}
+ describeSlowShutdown;
+ `, {});
+
+ it('names the step that blocked the loop in one numbers-only line, and stays silent when fast', () => {
+ const clock = __createManualClock();
+ const span = createEventLoopLagProbe({ clock }).begin('httpClose');
+ clock.advance(20);
+ span.mark('peerSync');
+ clock.advance(100);
+ clock.stall(6000);
+ clock.advance(100);
+ span.mark('dbPool');
+ clock.advance(30);
+ const line = describeSlowShutdown(span.end(), { marker: 6250, mediaFlush: 6240 });
+ expect(line).toMatch(/^🐢 Graceful shutdown slow totalMs=6250 slowestStep=peerSync /);
+ // Most of the step was loop lag: a synchronous block, not slow I/O.
+ expect(Number(line.match(/ peerSyncMs=6200 peerSyncLagMs=(\d+) /)?.[1])).toBeGreaterThan(5000);
+ expect(line).toContain('markerSettledMs=6250 mediaFlushSettledMs=6240');
+
+ const fast = createEventLoopLagProbe({ clock }).begin('httpClose');
+ clock.advance(300);
+ expect(describeSlowShutdown(fast.end(), {})).toBeNull();
+ });
+});
diff --git a/server/services/sharing/index.js b/server/services/sharing/index.js
index 4a7e6a2585..0d2c788472 100644
--- a/server/services/sharing/index.js
+++ b/server/services/sharing/index.js
@@ -9,7 +9,7 @@
import { ensureDir, PATHS } from '../../lib/fileUtils.js';
import { join } from 'path';
-import { attachAllWatchers, attachWatcher, detachWatcher, shutdownAllWatchers, listAttachedWatchers } from './watcher.js';
+import { attachAllWatchers, attachWatcher, detachWatcher, shutdownAllWatchers, quiesceAllWatchersForExit, listAttachedWatchers } from './watcher.js';
import { sharingEvents } from './importer.js';
import { installSubscriptionListener } from './subscriptions.js';
import { installPeerSyncListener, uninstallPeerSyncListener, peerSyncEvents, syncMediaLibraryWithAllPeers, syncCosHistoryWithAllPeers, syncCosTasksWithAllPeers, syncEidoverseFoundationsWithAllPeers } from './peerSync.js';
@@ -173,11 +173,28 @@ export async function initSharing({ io: socketIo } = {}) {
}
// Resolves once tracked peer-sync background work (peer:online walks, debounced
-// pushes) has settled. Call after shutdownSharing() and before closing the DB pool.
+// pushes) has settled. Call after shutdownSharing() or quiesceSharingForExit() and
+// before closing the DB pool.
export { drainBackgroundOperations as drainPeerSyncOperations } from './peerSubscriptions.js';
export async function shutdownSharing() {
await shutdownAllWatchers();
+ releaseSharingRuntime();
+}
+
+/**
+ * Process-exit form of shutdownSharing(): the same admission stop and listener
+ * teardown, without closing the share-bucket watchers, whose close blocks the
+ * main thread for seconds on macOS (#12853). process.exit releases them. No
+ * convergence walk can start afterwards, so the DB pool still closes after
+ * peer sync stops (#12585). Keep shutdownSharing() for re-init and tests.
+ */
+export function quiesceSharingForExit() {
+ quiesceAllWatchersForExit();
+ releaseSharingRuntime();
+}
+
+function releaseSharingRuntime() {
stopMediaLibrarySweep();
sharingEvents.removeAllListeners();
// Detach the recordEvents + instanceEvents listeners that
diff --git a/server/services/sharing/watcher.js b/server/services/sharing/watcher.js
index 0082d6805f..f13d600b61 100644
--- a/server/services/sharing/watcher.js
+++ b/server/services/sharing/watcher.js
@@ -27,6 +27,8 @@ const backlogQueues = new Map(); // bucketId → { running: Promise, queued: Pro
const queueLifecycle = createFileWriteQueue();
let shutdownAdmitted = false;
let shutdownGeneration = 0;
+// Terminal: set only on the way to process.exit (quiesceAllWatchersForExit).
+let exitQuiesced = false;
/**
* Coalesce backlog-scan requests per bucket. A flood of records/ or assets/
@@ -93,11 +95,13 @@ async function attachWatcherNow(bucketId) {
// A path under assets/ or records/ is bundle-side sync we should retry on,
// not a manifest to process. Manifests live in manifests/ (always *.json).
const isBundleSync = (p) => p.startsWith(`${assetsDir}${sep}`) || p.startsWith(`${recordsDir}${sep}`);
+ // Handles left open for process exit still deliver events; drop them.
+ const live = (handler) => async (path) => { if (!exitQuiesced) await handler(path); };
// try/catch around async event handlers — see AGENTS.md "PTY/child-process
// /setTimeout/setInterval callbacks" rule (chokidar events fire outside the
// request lifecycle, no Express middleware to bubble the throw to).
- w.on('add', async (path) => {
+ w.on('add', live(async (path) => {
try {
if (isBundleSync(path)) { await queueBacklog(bucketId); return; }
const file = basename(path);
@@ -106,8 +110,8 @@ async function attachWatcherNow(bucketId) {
} catch (err) {
console.error(`❌ sharing.watcher: processManifest threw for ${basename(path)}: ${err?.message || err}`);
}
- });
- w.on('change', async (path) => {
+ }));
+ w.on('change', live(async (path) => {
try {
if (isBundleSync(path)) { await queueBacklog(bucketId); return; }
// A manifest *changing* after first write is unusual (atomicWrite
@@ -119,8 +123,8 @@ async function attachWatcherNow(bucketId) {
} catch (err) {
console.error(`❌ sharing.watcher: processManifest threw on change ${basename(path)}: ${err?.message || err}`);
}
- });
- w.on('unlink', async (path) => {
+ }));
+ w.on('unlink', live(async (path) => {
// Only manifest deletions are unshare signals. Record/asset unlinks are
// expected during cloud-sync churn and not actionable here.
if (!path.startsWith(`${manifestsDir}${sep}`)) return;
@@ -135,7 +139,7 @@ async function attachWatcherNow(bucketId) {
} catch (err) {
console.error(`❌ sharing.watcher: handleUnshare threw for ${file}: ${err?.message || err}`);
}
- });
+ }));
w.on('error', (err) => {
console.error(`❌ sharing.watcher: bucket=${bucket.name} ${err?.message || err}`);
});
@@ -207,6 +211,19 @@ export async function shutdownAllWatchers() {
});
}
+/**
+ * Process-exit teardown: stop admission and drop further events, but leave the
+ * handles open for process.exit to release. Closing a chokidar watcher stops
+ * its macOS FSEvents streams synchronously on the main thread — several seconds
+ * per bucket on a live install (#12853), long enough to starve every shutdown
+ * timer behind it. Use shutdownAllWatchers() when the process keeps running.
+ */
+export function quiesceAllWatchersForExit() {
+ shutdownAdmitted = true;
+ shutdownGeneration++;
+ exitQuiesced = true;
+}
+
export function listAttachedWatchers() {
return [...watchers.keys()];
}
diff --git a/server/services/sharing/watcher.test.js b/server/services/sharing/watcher.test.js
index 9d4139f07b..ea137c0f08 100644
--- a/server/services/sharing/watcher.test.js
+++ b/server/services/sharing/watcher.test.js
@@ -323,6 +323,23 @@ describe('share-bucket watcher attachment lifecycle', () => {
expect(liveHandles()).toEqual([]);
});
+ it('exit quiesce keeps handles open, drops their events and refuses later admission (#12853)', async () => {
+ const { watcher, quiesceAllWatchersForExit, attachWatcher, attachAllWatchers, listAttachedWatchers, paths, root } = await attach();
+ const queuedInit = attachAllWatchers();
+ quiesceAllWatchersForExit();
+ await queuedInit;
+ expect(watcher.close).not.toHaveBeenCalled();
+ expect(listBuckets).not.toHaveBeenCalled();
+ await expect(attachWatcher('bucket-late')).rejects.toThrow('watchers are shut down');
+ await deliver(watcher, 'add', paths.join(root, 'manifests', 'example-manifest.json'));
+ await deliver(watcher, 'change', paths.join(root, 'assets', 'blobs', 'example-blob'));
+ await deliver(watcher, 'unlink', paths.join(root, 'manifests', 'example-manifest.json'));
+ expect(processManifest).not.toHaveBeenCalled();
+ expect(processBacklog).not.toHaveBeenCalled();
+ expect(handleUnshare).not.toHaveBeenCalled();
+ expect(listAttachedWatchers()).toEqual(['bucket-example']);
+ });
+
it('failed closure retains ownership, drains other closes, and allows teardown to retry', async () => {
const { watcher, attachWatcher, listAttachedWatchers } = await attach();
const other = await attachWatcher('bucket-other');