Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop graceful shutdown from blocking on share-bucket watcher close (#12853)
[._.]/ Adam Eivyd0c36988de6c
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex 5a67f36b7d..6e054fbc58 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -44,6 +44,7 @@ import { conflictJournalStore } from '../lib/conflictJournal.js';import { markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdown.js';import { signalDetachedGroups } from '../lib/credentialBootstrap.js';import { createHttpDrain } from '../lib/httpDrain.js';+import { createEventLoopLagProbe } from '../lib/eventLoopLagProbe.js';import { setUserCatalogTypes } from '../lib/catalogTypes.js';import { runMigrations } from '../../scripts/run-migrations.js';@@ -146,7 +147,7 @@ import { initSyncOrchestrator } from './syncOrchestrator.js';import { initMediaJobQueue, resumeMediaJobCompletions, flushMediaJobQueue, quiesceMediaJobQueue } from './mediaJobQueue/index.js';import { initSpriteLocalAnimationHook } from './sprites/localAnimationJobHook.js';import { initLoraTraining } from './loraTraining/index.js';-import { initSharing, shutdownSharing, drainPeerSyncOperations } from './sharing/index.js';+import { initSharing, quiesceSharingForExit, drainPeerSyncOperations } from './sharing/index.js';import { initMortalLoomStore } from './mortalLoomStore.js';import { initUniverseBuilderCollectionHook } from './universeBuilderCollectionHook.js';import { initCatalogImageAttachHook } from './catalogImageAttachHook.js';@@ -1108,6 +1109,19 @@ const HTTP_DRAIN_WINDOW_MS = 5000;// pool closes; drain + HTTP close + this + DB pool grace stay under the ceiling.const PEER_SYNC_STOP_GRACE_MS = 1000;+// A shutdown at or above this logs one numbers-only line naming where the time+// went (#12853). A step's loop lag is time the event loop could not run timers —+// a synchronous block — rather than asynchronous waiting.+const SLOW_SHUTDOWN_MS = 1000;+const describeSlowShutdown = (summary, settledMs) => {+ if (summary.totalMs < SLOW_SHUTDOWN_MS) return null;+ const phases = Object.entries(summary.phases);+ const [slowest] = phases.reduce((max, entry) => (entry[1].ms > max[1].ms ? entry : max), phases[0]);+ const steps = phases.map(([name, { ms, lagMs }]) => `${name}Ms=${ms} ${name}LagMs=${lagMs}`);+ const settled = Object.entries(settledMs).map(([name, ms]) => `${name}SettledMs=${ms}`);+ return `🐢 Graceful shutdown slow totalMs=${summary.totalMs} slowestStep=${slowest} loopLagMs=${summary.lagMs} loopLagMaxMs=${summary.lagMaxMs} ${[...steps, ...settled].join(' ')}`;+};+/*** Wire SIGTERM/SIGINT to the graceful-shutdown state machine. Idempotent per* signal: once shutdown starts, later signals are ignored.@@ -1132,6 +1146,14 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// during teardown must not promote a waiting job the process is about to// abandon — the next boot fails a running record but restores a queued one.quiesceMediaJobQueue();+ const timing = createEventLoopLagProbe().begin('tailcat');+ // Concurrent steps settle out of phase order; recorded as ms since the signal.+ const settledMs = {};+ const recordSettled = (name) => () => { settledMs[name] = timing.snapshot().totalMs; };+ const reportIfSlow = () => {+ const line = describeSlowShutdown(timing.end(), settledMs);+ if (line) console.log(line);+ };await import('./tailcatPeer.js').then(({ stopAllForwards }) => stopAllForwards()).catch((err) => logBootstrapFailure('❌ Tailcat forward shutdown failed', err));await import('./tailcatServe.js').then(({ stopServeProcess }) => stopServeProcess())@@ -1140,6 +1162,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// -shutdown would `pm2 stop` a model server the user never asked to lose,// and PortOS is about to stop being the thing that could restart it.stopIdleReaper();+ timing.mark('modelHost');await import('./fleetLlmHost.js').then(({ stopFleetLlmHost }) => stopFleetLlmHost()).catch((err) => logBootstrapFailure('❌ Dedicated model host shutdown failed', err));// Same reasoning for the hosted-session sweeper: a tick mid-shutdown would@@ -1163,6 +1186,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>const forceExitTimer = setTimeout(() => {console.error('⚠️ Graceful shutdown timed out, forcing exit');+ reportIfSlow();process.exit(1);}, GRACEFUL_SHUTDOWN_TIMEOUT_MS);// Don't let the safety timer itself keep the event loop alive — if every@@ -1191,6 +1215,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>const markerWritten = withGrace('Host-shutdown marker', 1500, ({ finish, finishWithError }) =>writeHostShutdownMarker({ agentIds: [...activeAgents.keys(), ...dispatchingAgentIds], signal }).then(() => finish(), (err) => finishWithError('⚠️ Host-shutdown marker failed', err)));+ markerWritten.then(recordSettled('marker'));// Reach every child spawned into its own process group — agent runs, CLI// runs and vision calls alike. A credential-bootstrap-wrapped child is@@ -1210,6 +1235,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// pm2's TreeKill would reap it anyway, but a direct SIGTERM keeps a manual// `kill` of the server from orphaning a Codex process holding the user's// sign-in, and it is a no-op when nothing was ever spawned.+ timing.mark('codex');await withGrace('Codex app-server', 2000, ({ finish, finishWithError }) =>stopCodexAppServer().then(() => finish(),@@ -1228,15 +1254,18 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// still waits for that server's connections to end, so while a slow request// is draining this grace can lapse first — Socket.IO itself is torn down by// then, and the drain wait below bounds the rest.+ timing.mark('socketIo');await withGrace('Socket.IO', 3000, ({ finish, finishWithError }) =>io.close((err) => {if (err && err.code !== 'ERR_SERVER_NOT_RUNNING') finishWithError('⚠️ Error closing Socket.IO', err);else finish('✅ Socket.IO closed');}));+ timing.mark('ingress');await import('./tailcatIngress.js').then(({ stopTailcatIngress }) => stopTailcatIngress()).catch((err) => logBootstrapFailure('❌ Tailcat ingress shutdown failed', err));// Let accepted requests finish (bounded by the drain window), then force the// rest closed below — a handler mid-write loses its response only at the deadline.+ timing.mark('requestDrain');const { remaining } = await requestsDrained;if (remaining > 0) console.warn(`⚠️ ${remaining} request(s) still running after the ${HTTP_DRAIN_WINDOW_MS}ms drain — closing them`);// Close BOTH servers explicitly. The drain already stopped both listeners@@ -1246,6 +1275,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// Upgraded (WebSocket) sockets are invisible to closeAllConnections() but// keep server.close() pending; Socket.IO has already sent its close frames.httpDrain.destroyUpgraded();+ timing.mark('httpClose');await Promise.all([closeServer(httpServer, 'HTTP server'),closeServer(localHttpServer, 'Local HTTP mirror')@@ -1257,13 +1287,16 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// schedule another export now that the servers are closed.const ideaLoomFlushed = withGrace('IdeaLoom auto-sync flush', 2500, ({ finish, finishWithError }) =>flushIdeaLoomAutoSync().then(() => finish('✅ IdeaLoom auto-sync flushed'), (err) => finishWithError('⚠️ IdeaLoom auto-sync flush failed', err)));+ ideaLoomFlushed.then(recordSettled('ideaLoom'));// Flush the media-job queue now that no request can admit another job// (#8325): terminal transitions and the latest progress reach the snapshot a- // restart restores from. The file write is independent of Postgres, so it- // runs alongside the DB close below and is awaited before exit — bounded by- // its own timeout so it spends no budget beyond the DB close's.+ // restart restores from. Not earlier: quiescing stops dispatch, not enqueue.+ // Started ahead of the sharing and DB steps, so its write is already in+ // flight when they run (#12853); awaited before exit and bounded by its own+ // timeout so it spends no budget beyond the DB close's.const mediaQueueFlushed = flushMediaJobQueue().then((outcome) => {+ recordSettled('mediaFlush')();if (outcome.ok) console.log('✅ Media job queue flushed');else console.error(`❌ Media job queue flush failed: ${outcome.error}`);});@@ -1271,12 +1304,16 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// Stop peer sync before the pool ends: a peer:online convergence walk still// running would otherwise log one pool-ended stack per remaining subscription.// Abandoning it is safe — the next boot's peer:online pass repeats it (#11750).+ // The exit form leaves share-bucket watchers open: closing them blocks the+ // main thread for seconds on macOS, and process.exit releases them (#12853).+ timing.mark('peerSync');await withGrace('Peer sync', PEER_SYNC_STOP_GRACE_MS, ({ finish, finishWithError }) =>- shutdownSharing().then(() => drainPeerSyncOperations()).then(+ Promise.resolve().then(quiesceSharingForExit).then(() => drainPeerSyncOperations()).then(() => finish(),(err) => finishWithError('⚠️ Peer sync shutdown failed', err),));+ timing.mark('dbPool');const { close } = await import('../lib/db.js');if (typeof close === 'function') {// Bound the DB pool close: pool.end() waits for every checked-out client to@@ -1288,10 +1325,12 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>console.warn('ℹ️ DB pool close not available; skipping DB shutdown');}+ timing.mark('flushWait');await markerWritten;await mediaQueueFlushed;await ideaLoomFlushed;clearTimeout(forceExitTimer);+ reportIfSlow();process.exit(0);};process.on('SIGTERM', () => shutdown('SIGTERM'));diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.jsindex 3b9b16c3f1..50e6d380d1 100644--- a/server/services/bootstrap.shutdown.test.js+++ b/server/services/bootstrap.shutdown.test.js@@ -26,6 +26,7 @@ import { dirname, join } from 'path';// a destructured or defaulted parameter.import { extractDeclaration, stripCommentsAndNormalize } from '../lib/mirrorParity.js';import { logFailureWithStack } from '../lib/failureLogging.js';+import { createEventLoopLagProbe, __createManualClock } from '../lib/eventLoopLagProbe.js';const __dirname = dirname(fileURLToPath(import.meta.url));const SRC = readFileSync(join(__dirname, 'bootstrap.js'), 'utf-8').replace(/\r\n/g, '\n');@@ -138,9 +139,9 @@ describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {describe('shutdown handler — peer sync stops before the DB pool closes (#12585)', () => {const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');- it('calls shutdownSharing() then drains, under a grace, between the HTTP close and the DB pool close', () => {- const stopAt = code.indexOf('shutdownSharing()');- expect(stopAt, 'shutdownSharing() is not called in shutdown()').toBeGreaterThan(-1);+ it('stops sharing then drains, under a grace, between the HTTP close and the DB pool close', () => {+ const stopAt = code.indexOf('quiesceSharingForExit');+ expect(stopAt, 'quiesceSharingForExit is not called in shutdown()').toBeGreaterThan(-1);expect(code).toMatch(/withGrace\('Peer sync', PEER_SYNC_STOP_GRACE_MS,/);expect(code.indexOf('drainPeerSyncOperations()')).toBeGreaterThan(stopAt);expect(stopAt).toBeGreaterThan(code.indexOf('closeServer('));@@ -253,3 +254,52 @@ describe('shutdown handler — media-job queue quiesce + flush (#8325, #8691)',expect(awaitAt).toBeLessThan(code.indexOf('process.exit(0)'));});});++// #12853: closing the share-bucket watchers blocked the main thread for seconds+// on macOS, starving the marker and media-flush timers and, past PM2's kill+// window, skipping the rest of the teardown.+describe('shutdown handler — no blocking teardown at exit (#12853)', () => {+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');++ it('leaves share-bucket watchers open and starts the durable writes before stopping sharing', () => {+ expect(code).not.toMatch(/shutdownSharing|shutdownAllWatchers|detachWatcher/);+ const stopAt = code.indexOf('quiesceSharingForExit');+ expect(code.indexOf('writeHostShutdownMarker(')).toBeLessThan(stopAt);+ expect(code.indexOf('flushMediaJobQueue(')).toBeLessThan(stopAt);+ });++ it('reports a slow shutdown on both exits', () => {+ const exits = [...code.matchAll(/process\.exit\([01]\)/g)];+ expect(exits).toHaveLength(2);+ for (const exit of exits) {+ expect(code.slice(0, exit.index).trimEnd().endsWith('reportIfSlow();')).toBe(true);+ }+ });++ const describeSlowShutdown = runInNewContext(`+ ${SRC.match(/const SLOW_SHUTDOWN_MS = \d+;/)[0]}+ ${extractDeclaration(SRC, 'describeSlowShutdown')}+ describeSlowShutdown;+ `, {});++ it('names the step that blocked the loop in one numbers-only line, and stays silent when fast', () => {+ const clock = __createManualClock();+ const span = createEventLoopLagProbe({ clock }).begin('httpClose');+ clock.advance(20);+ span.mark('peerSync');+ clock.advance(100);+ clock.stall(6000);+ clock.advance(100);+ span.mark('dbPool');+ clock.advance(30);+ const line = describeSlowShutdown(span.end(), { marker: 6250, mediaFlush: 6240 });+ expect(line).toMatch(/^🐢 Graceful shutdown slow totalMs=6250 slowestStep=peerSync /);+ // Most of the step was loop lag: a synchronous block, not slow I/O.+ expect(Number(line.match(/ peerSyncMs=6200 peerSyncLagMs=(\d+) /)?.[1])).toBeGreaterThan(5000);+ expect(line).toContain('markerSettledMs=6250 mediaFlushSettledMs=6240');++ const fast = createEventLoopLagProbe({ clock }).begin('httpClose');+ clock.advance(300);+ expect(describeSlowShutdown(fast.end(), {})).toBeNull();+ });+});diff --git a/server/services/sharing/index.js b/server/services/sharing/index.jsindex 4a7e6a2585..0d2c788472 100644--- a/server/services/sharing/index.js+++ b/server/services/sharing/index.js@@ -9,7 +9,7 @@import { ensureDir, PATHS } from '../../lib/fileUtils.js';import { join } from 'path';-import { attachAllWatchers, attachWatcher, detachWatcher, shutdownAllWatchers, listAttachedWatchers } from './watcher.js';+import { attachAllWatchers, attachWatcher, detachWatcher, shutdownAllWatchers, quiesceAllWatchersForExit, listAttachedWatchers } from './watcher.js';import { sharingEvents } from './importer.js';import { installSubscriptionListener } from './subscriptions.js';import { installPeerSyncListener, uninstallPeerSyncListener, peerSyncEvents, syncMediaLibraryWithAllPeers, syncCosHistoryWithAllPeers, syncCosTasksWithAllPeers, syncEidoverseFoundationsWithAllPeers } from './peerSync.js';@@ -173,11 +173,28 @@ export async function initSharing({ io: socketIo } = {}) {}// Resolves once tracked peer-sync background work (peer:online walks, debounced-// pushes) has settled. Call after shutdownSharing() and before closing the DB pool.+// pushes) has settled. Call after shutdownSharing() or quiesceSharingForExit() and+// before closing the DB pool.export { drainBackgroundOperations as drainPeerSyncOperations } from './peerSubscriptions.js';export async function shutdownSharing() {await shutdownAllWatchers();+ releaseSharingRuntime();+}++/**+ * Process-exit form of shutdownSharing(): the same admission stop and listener+ * teardown, without closing the share-bucket watchers, whose close blocks the+ * main thread for seconds on macOS (#12853). process.exit releases them. No+ * convergence walk can start afterwards, so the DB pool still closes after+ * peer sync stops (#12585). Keep shutdownSharing() for re-init and tests.+ */+export function quiesceSharingForExit() {+ quiesceAllWatchersForExit();+ releaseSharingRuntime();+}++function releaseSharingRuntime() {stopMediaLibrarySweep();sharingEvents.removeAllListeners();// Detach the recordEvents + instanceEvents listeners thatdiff --git a/server/services/sharing/watcher.js b/server/services/sharing/watcher.jsindex 0082d6805f..f13d600b61 100644--- a/server/services/sharing/watcher.js+++ b/server/services/sharing/watcher.js@@ -27,6 +27,8 @@ const backlogQueues = new Map(); // bucketId → { running: Promise, queued: Proconst queueLifecycle = createFileWriteQueue();let shutdownAdmitted = false;let shutdownGeneration = 0;+// Terminal: set only on the way to process.exit (quiesceAllWatchersForExit).+let exitQuiesced = false;/*** Coalesce backlog-scan requests per bucket. A flood of records/ or assets/@@ -93,11 +95,13 @@ async function attachWatcherNow(bucketId) {// A path under assets/ or records/ is bundle-side sync we should retry on,// not a manifest to process. Manifests live in manifests/ (always *.json).const isBundleSync = (p) => p.startsWith(`${assetsDir}${sep}`) || p.startsWith(`${recordsDir}${sep}`);+ // Handles left open for process exit still deliver events; drop them.+ const live = (handler) => async (path) => { if (!exitQuiesced) await handler(path); };// try/catch around async event handlers — see AGENTS.md "PTY/child-process// /setTimeout/setInterval callbacks" rule (chokidar events fire outside the// request lifecycle, no Express middleware to bubble the throw to).- w.on('add', async (path) => {+ w.on('add', live(async (path) => {try {if (isBundleSync(path)) { await queueBacklog(bucketId); return; }const file = basename(path);@@ -106,8 +110,8 @@ async function attachWatcherNow(bucketId) {} catch (err) {console.error(`❌ sharing.watcher: processManifest threw for ${basename(path)}: ${err?.message || err}`);}- });- w.on('change', async (path) => {+ }));+ w.on('change', live(async (path) => {try {if (isBundleSync(path)) { await queueBacklog(bucketId); return; }// A manifest *changing* after first write is unusual (atomicWrite@@ -119,8 +123,8 @@ async function attachWatcherNow(bucketId) {} catch (err) {console.error(`❌ sharing.watcher: processManifest threw on change ${basename(path)}: ${err?.message || err}`);}- });- w.on('unlink', async (path) => {+ }));+ w.on('unlink', live(async (path) => {// Only manifest deletions are unshare signals. Record/asset unlinks are// expected during cloud-sync churn and not actionable here.if (!path.startsWith(`${manifestsDir}${sep}`)) return;@@ -135,7 +139,7 @@ async function attachWatcherNow(bucketId) {} catch (err) {console.error(`❌ sharing.watcher: handleUnshare threw for ${file}: ${err?.message || err}`);}- });+ }));w.on('error', (err) => {console.error(`❌ sharing.watcher: bucket=${bucket.name} ${err?.message || err}`);});@@ -207,6 +211,19 @@ export async function shutdownAllWatchers() {});}+/**+ * Process-exit teardown: stop admission and drop further events, but leave the+ * handles open for process.exit to release. Closing a chokidar watcher stops+ * its macOS FSEvents streams synchronously on the main thread — several seconds+ * per bucket on a live install (#12853), long enough to starve every shutdown+ * timer behind it. Use shutdownAllWatchers() when the process keeps running.+ */+export function quiesceAllWatchersForExit() {+ shutdownAdmitted = true;+ shutdownGeneration++;+ exitQuiesced = true;+}+export function listAttachedWatchers() {return [...watchers.keys()];}diff --git a/server/services/sharing/watcher.test.js b/server/services/sharing/watcher.test.jsindex 9d4139f07b..ea137c0f08 100644--- a/server/services/sharing/watcher.test.js+++ b/server/services/sharing/watcher.test.js@@ -323,6 +323,23 @@ describe('share-bucket watcher attachment lifecycle', () => {expect(liveHandles()).toEqual([]);});+ it('exit quiesce keeps handles open, drops their events and refuses later admission (#12853)', async () => {+ const { watcher, quiesceAllWatchersForExit, attachWatcher, attachAllWatchers, listAttachedWatchers, paths, root } = await attach();+ const queuedInit = attachAllWatchers();+ quiesceAllWatchersForExit();+ await queuedInit;+ expect(watcher.close).not.toHaveBeenCalled();+ expect(listBuckets).not.toHaveBeenCalled();+ await expect(attachWatcher('bucket-late')).rejects.toThrow('watchers are shut down');+ await deliver(watcher, 'add', paths.join(root, 'manifests', 'example-manifest.json'));+ await deliver(watcher, 'change', paths.join(root, 'assets', 'blobs', 'example-blob'));+ await deliver(watcher, 'unlink', paths.join(root, 'manifests', 'example-manifest.json'));+ expect(processManifest).not.toHaveBeenCalled();+ expect(processBacklog).not.toHaveBeenCalled();+ expect(handleUnshare).not.toHaveBeenCalled();+ expect(listAttachedWatchers()).toEqual(['bucket-example']);+ });+it('failed closure retains ownership, drains other closes, and allows teardown to retry', async () => {const { watcher, attachWatcher, listAttachedWatchers } = await attach();const other = await attachWatcher('bucket-other');