Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: honor shutdown before teardown waits and startup continuation

[._.]/ Adam Eivy913989f80e0c
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index b0734699fd..51d5cdd4d3 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -41,7 +41,7 @@ import { verifyCollectionVersions } from '../lib/collectionStore.js';
import { startIdleReaper, stopIdleReaper } from '../lib/managedDaemon.js';
import { adoptNpmGlobalBinDir } from '../lib/npmGlobalBin.js';
import { conflictJournalStore } from '../lib/conflictJournal.js';
-import { markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdown.js';
+import { isHostShuttingDown, markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdown.js';
import { signalDetachedGroups } from '../lib/credentialBootstrap.js';
import { createHttpDrain } from '../lib/httpDrain.js';
import { createEventLoopLagProbe } from '../lib/eventLoopLagProbe.js';
@@ -447,7 +447,7 @@ const startBackgroundServices = ({ spawnerReady, io, httpsEnabled, port }) => {
// Explicit call (not a module-level side effect) so test imports of cos.js
// don't spin up its event listeners and timers. The spawner gate itself lives
// in bootstrapSequence.js.
- initCosAfterSpawner({ spawnerReady, initCos: () => cos.init() });
+ initCosAfterSpawner({ spawnerReady, initCos: () => cos.init(), isStopping: isHostShuttingDown });
// Restore only an installed, opted-in world, then its bridge, resident
// presence, and pending design. No install, download, or AI provider call.
@@ -968,6 +968,7 @@ const announceListening = ({ io, httpServer, localHttpServer, httpsEnabled, port
*/
export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled, port, host, spawnerReady }) =>
runPostRouteSequence(gateStepsForSmokeBoot({
+ isStopping: isHostShuttingDown,
startBackgroundServices: () => startBackgroundServices({ spawnerReady, io, httpsEnabled, port }),
// Instance identity + sync log come up before requests are accepted, so a
@@ -1036,6 +1037,7 @@ export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled,
loadSeriesCoverBackfill: async () => (await import('../scripts/backfillSeriesCoverImages.js')).backfillSeriesCoverImages,
startListening: () => httpServer.listen(port, host, () => runPostListenSequence(gateStepsForSmokeBoot({
+ isStopping: isHostShuttingDown,
announceListening: () => announceListening({ io, httpServer, localHttpServer, httpsEnabled, port }),
// Process-level safety net, wired with the io instance so unhandled
// failures also surface in the UI.
@@ -1175,24 +1177,6 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
const line = describeSlowShutdown(timing.end(), settledMs);
if (line) console.log(line);
};
- await import('./tailcatPeer.js').then(({ stopAllForwards }) => stopAllForwards())
- .catch((err) => logBootstrapFailure('❌ Tailcat forward shutdown failed', err));
- await import('./tailcatServe.js').then(({ stopServeProcess }) => stopServeProcess())
- .catch((err) => logBootstrapFailure('❌ Tailcat serve shutdown failed', err));
- // Disarm the idle reaper before anything awaits: a sweep that fires mid
- // -shutdown would `pm2 stop` a model server the user never asked to lose,
- // and PortOS is about to stop being the thing that could restart it.
- stopIdleReaper();
- timing.mark('modelHost');
- await import('./fleetLlmHost.js').then(({ stopFleetLlmHost }) => stopFleetLlmHost())
- .catch((err) => logBootstrapFailure('❌ Dedicated model host shutdown failed', err));
- // Same reasoning for the hosted-session sweeper: a tick mid-shutdown would
- // emit into a namespace we are about to close.
- stopHostedSessionSweep();
- // And the Beeper transport: an open WebSocket is a live handle that would
- // hold the event loop past the last server close, and a reconnect firing
- // mid-shutdown would emit into a socket namespace we are about to close.
- stopBeeperSocket();
// Diagnostic context for the shutdown trigger. ppid tells us whether the
// signal came from PM2 (parent is the PM2 god process), a TTY (parent is
// the user's shell), or some external orchestrator. pm_* env vars are set
@@ -1238,6 +1222,24 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
.then(() => finish(), (err) => finishWithError('⚠️ Host-shutdown marker failed', err)));
markerWritten.then(recordSettled('marker'));
+ // No teardown await may delay the hard deadline, recovery snapshot, or
+ // disarming timers that could launch work while shutdown is suspended.
+ stopIdleReaper();
+ stopHostedSessionSweep();
+ stopBeeperSocket();
+ await import('./tailcatPeer.js').then(({ stopAllForwards }) => stopAllForwards())
+ .catch((err) => logBootstrapFailure('❌ Tailcat forward shutdown failed', err));
+ await import('./tailcatServe.js').then(({ stopServeProcess }) => stopServeProcess())
+ .catch((err) => logBootstrapFailure('❌ Tailcat serve shutdown failed', err));
+ timing.mark('modelHost');
+ // Usage persistence is best effort; a stalled ledger write must leave
+ // time for the media queue and DB to settle inside the shutdown ceiling.
+ await withGrace('Dedicated model host', 1000, ({ finish, finishWithError }) =>
+ import('./fleetLlmHost.js').then(({ stopFleetLlmHost }) => stopFleetLlmHost()).then(
+ () => finish(),
+ (err) => finishWithError('❌ Dedicated model host shutdown failed', err),
+ ));
+
// Reach every child spawned into its own process group — agent runs, CLI
// runs and vision calls alike. A credential-bootstrap-wrapped child is
// spawned `detached: true` so stop/timeout/cancel can signal its whole
diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.js
index a04d15e8e1..78960e4d5a 100644
--- a/server/services/bootstrap.shutdown.test.js
+++ b/server/services/bootstrap.shutdown.test.js
@@ -31,6 +31,85 @@ import { createEventLoopLagProbe, __createManualClock } from '../lib/eventLoopLa
const __dirname = dirname(fileURLToPath(import.meta.url));
const SRC = readFileSync(join(__dirname, 'bootstrap.js'), 'utf-8').replace(/\r\n/g, '\n');
+// Execute the signal boundary without importing the live service graph. Module
+// loading is injected so a suspended teardown uses fake time, never a real disk.
+const loadShutdown = (loadModule) => {
+ const signals = new Map();
+ const exit = vi.fn();
+ const marker = vi.fn(async () => {});
+ const flush = vi.fn(async () => ({ ok: true }));
+ const close = vi.fn(async () => {});
+ const noop = () => {};
+ runInNewContext(`
+ ${extractDeclaration(SRC, 'withGrace')}
+ ${extractDeclaration(SRC, 'closeServer')}
+ const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 10000;
+ const HTTP_DRAIN_WINDOW_MS = 5000;
+ const PEER_SYNC_STOP_GRACE_MS = 1000;
+ ${extractDeclaration(SRC, 'registerShutdownHandlers').replace(/^export /, '').replace(/\bimport\(/g, 'loadModule(')}
+ registerShutdownHandlers({ io: { close: done => done() }, httpServer: null, localHttpServer: null });
+ `, {
+ process: { on: (signal, handler) => signals.set(signal, handler), exit, env: {}, stdin: {}, pid: 1, ppid: 2 },
+ console: { log: noop, warn: noop, error: noop }, setTimeout, clearTimeout,
+ createHttpDrain: () => ({ begin: async () => ({ remaining: 0 }), destroyUpgraded: noop }),
+ markHostShuttingDown: noop, quiesceMediaJobQueue: noop, quiesceScheduledEvents: noop,
+ drainScheduledEvents: async () => {},
+ createEventLoopLagProbe: () => ({ begin: () => ({ mark: noop, end: noop, snapshot: () => ({ totalMs: 0 }) }) }),
+ describeSlowShutdown: () => null,
+ stopIdleReaper: noop, stopHostedSessionSweep: noop, stopBeeperSocket: noop,
+ activeAgents: new Map([['example-agent', {}]]), dispatchingAgentIds: new Set(),
+ writeHostShutdownMarker: marker, signalDetachedGroups: noop, stopCodexAppServer: async () => {},
+ flushIdeaLoomAutoSync: async () => {}, flushMediaJobQueue: flush,
+ quiesceSharingForExit: noop, drainPeerSyncOperations: async () => {},
+ logBootstrapFailure: noop,
+ loadModule: (path) => path === '../lib/db.js' ? Promise.resolve({ close }) : loadModule(path),
+ });
+ return { stop: () => signals.get('SIGTERM')(), exit, marker, flush, close };
+};
+
+describe('shutdown signal deadline and recovery', () => {
+ afterEach(() => vi.useRealTimers());
+
+ it('starts the hard deadline at the signal even when the first teardown import stalls', async () => {
+ vi.useFakeTimers();
+ let release;
+ const gate = new Promise(resolve => { release = resolve; });
+ const lifecycle = loadShutdown(() => gate);
+ const stopping = lifecycle.stop();
+ try {
+ await vi.advanceTimersByTimeAsync(9999);
+ expect(lifecycle.exit).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(1);
+ expect(lifecycle.exit).toHaveBeenCalledWith(1);
+ expect(lifecycle.marker).toHaveBeenCalledOnce();
+ } finally {
+ release({ stopAllForwards: () => {}, stopServeProcess: () => {}, stopFleetLlmHost: async () => {}, stopTailcatIngress: () => {} });
+ await stopping;
+ }
+ });
+
+ it('preserves agent recovery and flushes the queue and DB when model-host usage cannot flush', async () => {
+ vi.useFakeTimers();
+ let release;
+ const gate = new Promise(resolve => { release = resolve; });
+ const lifecycle = loadShutdown(async () => ({
+ stopAllForwards: () => {}, stopServeProcess: () => {},
+ stopFleetLlmHost: () => gate, stopTailcatIngress: () => {},
+ }));
+ const stopping = lifecycle.stop();
+ try {
+ await vi.advanceTimersByTimeAsync(1500);
+ expect(lifecycle.marker).toHaveBeenCalledOnce();
+ expect(lifecycle.flush).toHaveBeenCalledOnce();
+ expect(lifecycle.close).toHaveBeenCalledOnce();
+ expect(lifecycle.exit).toHaveBeenCalledWith(0);
+ } finally {
+ release();
+ await stopping;
+ }
+ });
+});
+
describe('shutdown handler — host-restart bookkeeping (#3202)', () => {
const shutdownBody = extractDeclaration(SRC, 'shutdown');
diff --git a/server/services/bootstrapSequence.js b/server/services/bootstrapSequence.js
index 9449bd57d6..61eb81f73c 100644
--- a/server/services/bootstrapSequence.js
+++ b/server/services/bootstrapSequence.js
@@ -129,9 +129,9 @@ export const runPreRouteSequence = async ({
* for pending tasks while it initializes; emitted before the spawner registered
* its listener, those events are dropped and the tasks sit forever.
*/
-export const initCosAfterSpawner = ({ spawnerReady, initCos }) =>
+export const initCosAfterSpawner = ({ spawnerReady, initCos, isStopping = () => false }) =>
Promise.resolve(spawnerReady)
- .then(() => initCos())
+ .then(() => { if (!isStopping()) return initCos(); })
.catch(logFailure('CoS init failed'));
/**
@@ -386,43 +386,49 @@ export const runPostRouteSequence = ({
runDatabasePhase: databasePhase,
loadSeriesCoverBackfill,
startListening,
+ isStopping = () => false,
onFatal = () => process.exit(1)
}) => {
+ if (isStopping()) return Promise.resolve();
+ // An awaited boot step can settle after SIGTERM has closed the listeners
+ // and quiesced workers. Never re-arm recovery or open a listener afterwards.
+ const whileRunning = (step) => () => { if (!isStopping()) return step(); };
startBackgroundServices();
return Promise.resolve()
- .then(() => ensureSelf())
- .then(() => initSyncLog())
- .then(() => {
+ .then(whileRunning(ensureSelf))
+ .then(whileRunning(initSyncLog))
+ .then(whileRunning(() => {
bestEffort(recoverStuckClassifications(), logFailure('Brain recovery failed'));
return bestEffort(recoverInterruptedRepoClones(), logFailure('Brain clone recovery failed'));
- })
- .then(() => initMediaJobQueue({ deferCompletionRecovery: true }))
- .then(() => bestEffort(initLoraTraining(), logFailure('loraTraining init failed')))
- .then(() => initMediaJobDependentHooks())
- .then(() => {
+ }))
+ .then(whileRunning(() => initMediaJobQueue({ deferCompletionRecovery: true })))
+ .then(whileRunning(() => bestEffort(initLoraTraining(), logFailure('loraTraining init failed'))))
+ .then(whileRunning(initMediaJobDependentHooks))
+ .then(whileRunning(() => {
// Recovery may wait behind a backup cut; arm it without blocking boot.
resumeMediaJobCompletions();
- })
- .then(() => {
+ }))
+ .then(whileRunning(() => {
// Sharing attaches chokidar watchers + drains the offline manifest
// backlog. Fire-and-forget — a failed bucket shouldn't block boot.
bestEffort(initSharing(), logFailure('Sharing init failed'));
- })
- .then(() => {
+ }))
+ .then(whileRunning(() => {
// Nudges Creative Director projects the media-queue reload just marked
// 'failed (interrupted by restart)' so they don't sit frozen. Owns its own
// failure path (it must resolve the CD recovery gate either way).
recoverCreativeDirectorProjects();
- })
- .then(() => databasePhase())
- .then(async () => {
+ }))
+ .then(whileRunning(databasePhase))
+ .then(whileRunning(async () => {
// Load awaited, run not — see point 6 above.
const backfillSeriesCoverImages = await loadSeriesCoverBackfill();
- bestEffort(backfillSeriesCoverImages(), logFailure('series cover backfill failed at boot'));
- })
- .then(() => startListening())
+ if (!isStopping()) bestEffort(backfillSeriesCoverImages(), logFailure('series cover backfill failed at boot'));
+ }))
+ .then(whileRunning(startListening))
.catch((err) => {
+ if (isStopping()) return;
console.error(`❌ Instance init failed: ${err.message}`);
onFatal();
});
@@ -443,12 +449,15 @@ export const runPostListenSequence = ({
relayUnloggedBrainRecords,
startPolling,
initSyncOrchestrator,
- startBrainScheduler
+ startBrainScheduler,
+ isStopping = () => false
}) => {
+ if (isStopping()) return Promise.resolve();
announceListening();
setupProcessErrorHandlers();
return bestEffort(
Promise.resolve(backfillOriginInstanceId()).then(() => {
+ if (isStopping()) return;
bestEffort(relayUnloggedBrainRecords(), logFailure('Brain relay sweep failed'));
startPolling();
initSyncOrchestrator();
diff --git a/server/services/bootstrapSequence.test.js b/server/services/bootstrapSequence.test.js
index 53a73aa7f0..9aabfce5cb 100644
--- a/server/services/bootstrapSequence.test.js
+++ b/server/services/bootstrapSequence.test.js
@@ -223,6 +223,17 @@ describe('runPreRouteSequence — pre-route boot order', () => {
});
describe('initCosAfterSpawner', () => {
+ it('does not start CoS when shutdown interrupts spawner initialization', async () => {
+ const spawner = deferred();
+ const initCos = vi.fn();
+ let stopping = false;
+ const done = initCosAfterSpawner({ spawnerReady: spawner.promise, initCos, isStopping: () => stopping });
+ stopping = true;
+ spawner.resolve();
+ await done;
+ expect(initCos).not.toHaveBeenCalled();
+ });
+
it('waits for the spawner before CoS init, so task:ready listeners exist', async () => {
const spawner = deferred();
const initCos = vi.fn(() => Promise.resolve());
@@ -839,6 +850,26 @@ describe('runPostRouteSequence — post-route boot order', () => {
expect(recorder.calls).toContain('startListening');
});
+ it.each(['ensureSelf', 'runDatabasePhase', 'loadSeriesCoverBackfill'])('stops the boot chain when shutdown interrupts %s', async (stepName) => {
+ const recorder = createRecorder();
+ const gate = deferred();
+ let stopping = false;
+ const deps = buildDeps(recorder, {
+ [stepName]: recorder.step(stepName, () => gate.promise),
+ isStopping: () => stopping,
+ });
+ const done = runPostRouteSequence(deps);
+ await flush();
+ expect(recorder.calls).toContain(stepName);
+ stopping = true;
+ const callsAtShutdown = [...recorder.calls];
+ gate.resolve(recorder.step('backfillSeriesCoverImages'));
+ await done;
+ expect(recorder.calls).toEqual(callsAtShutdown);
+ expect(deps.startListening).not.toHaveBeenCalled();
+ expect(deps.onFatal).not.toHaveBeenCalled();
+ });
+
it('is fatal when an awaited step rejects, and never opens the listener', async () => {
const recorder = createRecorder();
const deps = buildDeps(recorder, {
@@ -919,6 +950,20 @@ describe('runPostListenSequence — inside the listen callback', () => {
expect(recorder.calls).toContain('startPolling');
});
+ it('keeps peer sync and scheduling disarmed when shutdown interrupts the origin backfill', async () => {
+ const recorder = createRecorder();
+ const gate = deferred();
+ let stopping = false;
+ const done = runPostListenSequence(buildDeps(recorder, {
+ backfillOriginInstanceId: recorder.step('backfillOriginInstanceId', () => gate.promise),
+ isStopping: () => stopping,
+ }));
+ stopping = true;
+ gate.resolve();
+ await done;
+ expect(recorder.calls).toEqual(['announceListening', 'setupProcessErrorHandlers', 'backfillOriginInstanceId']);
+ });
+
it('starts peer sync even when the brain relay sweep fails', async () => {
const recorder = createRecorder();
await runPostListenSequence(buildDeps(recorder, {