Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: honor shutdown before teardown waits and startup continuation
[._.]/ Adam Eivy913989f80e0c
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex b0734699fd..51d5cdd4d3 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -41,7 +41,7 @@ import { verifyCollectionVersions } from '../lib/collectionStore.js';import { startIdleReaper, stopIdleReaper } from '../lib/managedDaemon.js';import { adoptNpmGlobalBinDir } from '../lib/npmGlobalBin.js';import { conflictJournalStore } from '../lib/conflictJournal.js';-import { markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdown.js';+import { isHostShuttingDown, markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdown.js';import { signalDetachedGroups } from '../lib/credentialBootstrap.js';import { createHttpDrain } from '../lib/httpDrain.js';import { createEventLoopLagProbe } from '../lib/eventLoopLagProbe.js';@@ -447,7 +447,7 @@ const startBackgroundServices = ({ spawnerReady, io, httpsEnabled, port }) => {// Explicit call (not a module-level side effect) so test imports of cos.js// don't spin up its event listeners and timers. The spawner gate itself lives// in bootstrapSequence.js.- initCosAfterSpawner({ spawnerReady, initCos: () => cos.init() });+ initCosAfterSpawner({ spawnerReady, initCos: () => cos.init(), isStopping: isHostShuttingDown });// Restore only an installed, opted-in world, then its bridge, resident// presence, and pending design. No install, download, or AI provider call.@@ -968,6 +968,7 @@ const announceListening = ({ io, httpServer, localHttpServer, httpsEnabled, port*/export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled, port, host, spawnerReady }) =>runPostRouteSequence(gateStepsForSmokeBoot({+ isStopping: isHostShuttingDown,startBackgroundServices: () => startBackgroundServices({ spawnerReady, io, httpsEnabled, port }),// Instance identity + sync log come up before requests are accepted, so a@@ -1036,6 +1037,7 @@ export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled,loadSeriesCoverBackfill: async () => (await import('../scripts/backfillSeriesCoverImages.js')).backfillSeriesCoverImages,startListening: () => httpServer.listen(port, host, () => runPostListenSequence(gateStepsForSmokeBoot({+ isStopping: isHostShuttingDown,announceListening: () => announceListening({ io, httpServer, localHttpServer, httpsEnabled, port }),// Process-level safety net, wired with the io instance so unhandled// failures also surface in the UI.@@ -1175,24 +1177,6 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>const line = describeSlowShutdown(timing.end(), settledMs);if (line) console.log(line);};- await import('./tailcatPeer.js').then(({ stopAllForwards }) => stopAllForwards())- .catch((err) => logBootstrapFailure('❌ Tailcat forward shutdown failed', err));- await import('./tailcatServe.js').then(({ stopServeProcess }) => stopServeProcess())- .catch((err) => logBootstrapFailure('❌ Tailcat serve shutdown failed', err));- // Disarm the idle reaper before anything awaits: a sweep that fires mid- // -shutdown would `pm2 stop` a model server the user never asked to lose,- // and PortOS is about to stop being the thing that could restart it.- stopIdleReaper();- timing.mark('modelHost');- await import('./fleetLlmHost.js').then(({ stopFleetLlmHost }) => stopFleetLlmHost())- .catch((err) => logBootstrapFailure('❌ Dedicated model host shutdown failed', err));- // Same reasoning for the hosted-session sweeper: a tick mid-shutdown would- // emit into a namespace we are about to close.- stopHostedSessionSweep();- // And the Beeper transport: an open WebSocket is a live handle that would- // hold the event loop past the last server close, and a reconnect firing- // mid-shutdown would emit into a socket namespace we are about to close.- stopBeeperSocket();// Diagnostic context for the shutdown trigger. ppid tells us whether the// signal came from PM2 (parent is the PM2 god process), a TTY (parent is// the user's shell), or some external orchestrator. pm_* env vars are set@@ -1238,6 +1222,24 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>.then(() => finish(), (err) => finishWithError('⚠️ Host-shutdown marker failed', err)));markerWritten.then(recordSettled('marker'));+ // No teardown await may delay the hard deadline, recovery snapshot, or+ // disarming timers that could launch work while shutdown is suspended.+ stopIdleReaper();+ stopHostedSessionSweep();+ stopBeeperSocket();+ await import('./tailcatPeer.js').then(({ stopAllForwards }) => stopAllForwards())+ .catch((err) => logBootstrapFailure('❌ Tailcat forward shutdown failed', err));+ await import('./tailcatServe.js').then(({ stopServeProcess }) => stopServeProcess())+ .catch((err) => logBootstrapFailure('❌ Tailcat serve shutdown failed', err));+ timing.mark('modelHost');+ // Usage persistence is best effort; a stalled ledger write must leave+ // time for the media queue and DB to settle inside the shutdown ceiling.+ await withGrace('Dedicated model host', 1000, ({ finish, finishWithError }) =>+ import('./fleetLlmHost.js').then(({ stopFleetLlmHost }) => stopFleetLlmHost()).then(+ () => finish(),+ (err) => finishWithError('❌ Dedicated model host shutdown failed', err),+ ));+// Reach every child spawned into its own process group — agent runs, CLI// runs and vision calls alike. A credential-bootstrap-wrapped child is// spawned `detached: true` so stop/timeout/cancel can signal its wholediff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.jsindex a04d15e8e1..78960e4d5a 100644--- a/server/services/bootstrap.shutdown.test.js+++ b/server/services/bootstrap.shutdown.test.js@@ -31,6 +31,85 @@ import { createEventLoopLagProbe, __createManualClock } from '../lib/eventLoopLaconst __dirname = dirname(fileURLToPath(import.meta.url));const SRC = readFileSync(join(__dirname, 'bootstrap.js'), 'utf-8').replace(/\r\n/g, '\n');+// Execute the signal boundary without importing the live service graph. Module+// loading is injected so a suspended teardown uses fake time, never a real disk.+const loadShutdown = (loadModule) => {+ const signals = new Map();+ const exit = vi.fn();+ const marker = vi.fn(async () => {});+ const flush = vi.fn(async () => ({ ok: true }));+ const close = vi.fn(async () => {});+ const noop = () => {};+ runInNewContext(`+ ${extractDeclaration(SRC, 'withGrace')}+ ${extractDeclaration(SRC, 'closeServer')}+ const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 10000;+ const HTTP_DRAIN_WINDOW_MS = 5000;+ const PEER_SYNC_STOP_GRACE_MS = 1000;+ ${extractDeclaration(SRC, 'registerShutdownHandlers').replace(/^export /, '').replace(/\bimport\(/g, 'loadModule(')}+ registerShutdownHandlers({ io: { close: done => done() }, httpServer: null, localHttpServer: null });+ `, {+ process: { on: (signal, handler) => signals.set(signal, handler), exit, env: {}, stdin: {}, pid: 1, ppid: 2 },+ console: { log: noop, warn: noop, error: noop }, setTimeout, clearTimeout,+ createHttpDrain: () => ({ begin: async () => ({ remaining: 0 }), destroyUpgraded: noop }),+ markHostShuttingDown: noop, quiesceMediaJobQueue: noop, quiesceScheduledEvents: noop,+ drainScheduledEvents: async () => {},+ createEventLoopLagProbe: () => ({ begin: () => ({ mark: noop, end: noop, snapshot: () => ({ totalMs: 0 }) }) }),+ describeSlowShutdown: () => null,+ stopIdleReaper: noop, stopHostedSessionSweep: noop, stopBeeperSocket: noop,+ activeAgents: new Map([['example-agent', {}]]), dispatchingAgentIds: new Set(),+ writeHostShutdownMarker: marker, signalDetachedGroups: noop, stopCodexAppServer: async () => {},+ flushIdeaLoomAutoSync: async () => {}, flushMediaJobQueue: flush,+ quiesceSharingForExit: noop, drainPeerSyncOperations: async () => {},+ logBootstrapFailure: noop,+ loadModule: (path) => path === '../lib/db.js' ? Promise.resolve({ close }) : loadModule(path),+ });+ return { stop: () => signals.get('SIGTERM')(), exit, marker, flush, close };+};++describe('shutdown signal deadline and recovery', () => {+ afterEach(() => vi.useRealTimers());++ it('starts the hard deadline at the signal even when the first teardown import stalls', async () => {+ vi.useFakeTimers();+ let release;+ const gate = new Promise(resolve => { release = resolve; });+ const lifecycle = loadShutdown(() => gate);+ const stopping = lifecycle.stop();+ try {+ await vi.advanceTimersByTimeAsync(9999);+ expect(lifecycle.exit).not.toHaveBeenCalled();+ await vi.advanceTimersByTimeAsync(1);+ expect(lifecycle.exit).toHaveBeenCalledWith(1);+ expect(lifecycle.marker).toHaveBeenCalledOnce();+ } finally {+ release({ stopAllForwards: () => {}, stopServeProcess: () => {}, stopFleetLlmHost: async () => {}, stopTailcatIngress: () => {} });+ await stopping;+ }+ });++ it('preserves agent recovery and flushes the queue and DB when model-host usage cannot flush', async () => {+ vi.useFakeTimers();+ let release;+ const gate = new Promise(resolve => { release = resolve; });+ const lifecycle = loadShutdown(async () => ({+ stopAllForwards: () => {}, stopServeProcess: () => {},+ stopFleetLlmHost: () => gate, stopTailcatIngress: () => {},+ }));+ const stopping = lifecycle.stop();+ try {+ await vi.advanceTimersByTimeAsync(1500);+ expect(lifecycle.marker).toHaveBeenCalledOnce();+ expect(lifecycle.flush).toHaveBeenCalledOnce();+ expect(lifecycle.close).toHaveBeenCalledOnce();+ expect(lifecycle.exit).toHaveBeenCalledWith(0);+ } finally {+ release();+ await stopping;+ }+ });+});+describe('shutdown handler — host-restart bookkeeping (#3202)', () => {const shutdownBody = extractDeclaration(SRC, 'shutdown');diff --git a/server/services/bootstrapSequence.js b/server/services/bootstrapSequence.jsindex 9449bd57d6..61eb81f73c 100644--- a/server/services/bootstrapSequence.js+++ b/server/services/bootstrapSequence.js@@ -129,9 +129,9 @@ export const runPreRouteSequence = async ({* for pending tasks while it initializes; emitted before the spawner registered* its listener, those events are dropped and the tasks sit forever.*/-export const initCosAfterSpawner = ({ spawnerReady, initCos }) =>+export const initCosAfterSpawner = ({ spawnerReady, initCos, isStopping = () => false }) =>Promise.resolve(spawnerReady)- .then(() => initCos())+ .then(() => { if (!isStopping()) return initCos(); }).catch(logFailure('CoS init failed'));/**@@ -386,43 +386,49 @@ export const runPostRouteSequence = ({runDatabasePhase: databasePhase,loadSeriesCoverBackfill,startListening,+ isStopping = () => false,onFatal = () => process.exit(1)}) => {+ if (isStopping()) return Promise.resolve();+ // An awaited boot step can settle after SIGTERM has closed the listeners+ // and quiesced workers. Never re-arm recovery or open a listener afterwards.+ const whileRunning = (step) => () => { if (!isStopping()) return step(); };startBackgroundServices();return Promise.resolve()- .then(() => ensureSelf())- .then(() => initSyncLog())- .then(() => {+ .then(whileRunning(ensureSelf))+ .then(whileRunning(initSyncLog))+ .then(whileRunning(() => {bestEffort(recoverStuckClassifications(), logFailure('Brain recovery failed'));return bestEffort(recoverInterruptedRepoClones(), logFailure('Brain clone recovery failed'));- })- .then(() => initMediaJobQueue({ deferCompletionRecovery: true }))- .then(() => bestEffort(initLoraTraining(), logFailure('loraTraining init failed')))- .then(() => initMediaJobDependentHooks())- .then(() => {+ }))+ .then(whileRunning(() => initMediaJobQueue({ deferCompletionRecovery: true })))+ .then(whileRunning(() => bestEffort(initLoraTraining(), logFailure('loraTraining init failed'))))+ .then(whileRunning(initMediaJobDependentHooks))+ .then(whileRunning(() => {// Recovery may wait behind a backup cut; arm it without blocking boot.resumeMediaJobCompletions();- })- .then(() => {+ }))+ .then(whileRunning(() => {// Sharing attaches chokidar watchers + drains the offline manifest// backlog. Fire-and-forget — a failed bucket shouldn't block boot.bestEffort(initSharing(), logFailure('Sharing init failed'));- })- .then(() => {+ }))+ .then(whileRunning(() => {// Nudges Creative Director projects the media-queue reload just marked// 'failed (interrupted by restart)' so they don't sit frozen. Owns its own// failure path (it must resolve the CD recovery gate either way).recoverCreativeDirectorProjects();- })- .then(() => databasePhase())- .then(async () => {+ }))+ .then(whileRunning(databasePhase))+ .then(whileRunning(async () => {// Load awaited, run not — see point 6 above.const backfillSeriesCoverImages = await loadSeriesCoverBackfill();- bestEffort(backfillSeriesCoverImages(), logFailure('series cover backfill failed at boot'));- })- .then(() => startListening())+ if (!isStopping()) bestEffort(backfillSeriesCoverImages(), logFailure('series cover backfill failed at boot'));+ }))+ .then(whileRunning(startListening)).catch((err) => {+ if (isStopping()) return;console.error(`❌ Instance init failed: ${err.message}`);onFatal();});@@ -443,12 +449,15 @@ export const runPostListenSequence = ({relayUnloggedBrainRecords,startPolling,initSyncOrchestrator,- startBrainScheduler+ startBrainScheduler,+ isStopping = () => false}) => {+ if (isStopping()) return Promise.resolve();announceListening();setupProcessErrorHandlers();return bestEffort(Promise.resolve(backfillOriginInstanceId()).then(() => {+ if (isStopping()) return;bestEffort(relayUnloggedBrainRecords(), logFailure('Brain relay sweep failed'));startPolling();initSyncOrchestrator();diff --git a/server/services/bootstrapSequence.test.js b/server/services/bootstrapSequence.test.jsindex 53a73aa7f0..9aabfce5cb 100644--- a/server/services/bootstrapSequence.test.js+++ b/server/services/bootstrapSequence.test.js@@ -223,6 +223,17 @@ describe('runPreRouteSequence — pre-route boot order', () => {});describe('initCosAfterSpawner', () => {+ it('does not start CoS when shutdown interrupts spawner initialization', async () => {+ const spawner = deferred();+ const initCos = vi.fn();+ let stopping = false;+ const done = initCosAfterSpawner({ spawnerReady: spawner.promise, initCos, isStopping: () => stopping });+ stopping = true;+ spawner.resolve();+ await done;+ expect(initCos).not.toHaveBeenCalled();+ });+it('waits for the spawner before CoS init, so task:ready listeners exist', async () => {const spawner = deferred();const initCos = vi.fn(() => Promise.resolve());@@ -839,6 +850,26 @@ describe('runPostRouteSequence — post-route boot order', () => {expect(recorder.calls).toContain('startListening');});+ it.each(['ensureSelf', 'runDatabasePhase', 'loadSeriesCoverBackfill'])('stops the boot chain when shutdown interrupts %s', async (stepName) => {+ const recorder = createRecorder();+ const gate = deferred();+ let stopping = false;+ const deps = buildDeps(recorder, {+ [stepName]: recorder.step(stepName, () => gate.promise),+ isStopping: () => stopping,+ });+ const done = runPostRouteSequence(deps);+ await flush();+ expect(recorder.calls).toContain(stepName);+ stopping = true;+ const callsAtShutdown = [...recorder.calls];+ gate.resolve(recorder.step('backfillSeriesCoverImages'));+ await done;+ expect(recorder.calls).toEqual(callsAtShutdown);+ expect(deps.startListening).not.toHaveBeenCalled();+ expect(deps.onFatal).not.toHaveBeenCalled();+ });+it('is fatal when an awaited step rejects, and never opens the listener', async () => {const recorder = createRecorder();const deps = buildDeps(recorder, {@@ -919,6 +950,20 @@ describe('runPostListenSequence — inside the listen callback', () => {expect(recorder.calls).toContain('startPolling');});+ it('keeps peer sync and scheduling disarmed when shutdown interrupts the origin backfill', async () => {+ const recorder = createRecorder();+ const gate = deferred();+ let stopping = false;+ const done = runPostListenSequence(buildDeps(recorder, {+ backfillOriginInstanceId: recorder.step('backfillOriginInstanceId', () => gate.promise),+ isStopping: () => stopping,+ }));+ stopping = true;+ gate.resolve();+ await done;+ expect(recorder.calls).toEqual(['announceListening', 'setupProcessErrorHandlers', 'backfillOriginInstanceId']);+ });+it('starts peer sync even when the brain relay sweep fails', async () => {const recorder = createRecorder();await runPostListenSequence(buildDeps(recorder, {