Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12585]): stop peer sync before the DB pool closes at shutdown
[._.]/ Adam Eivy1914771bdca7
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex e5cf6e56ad..6e649980c1 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -144,7 +144,7 @@ import { initSyncOrchestrator } from './syncOrchestrator.js';import { initMediaJobQueue, resumeMediaJobCompletions, flushMediaJobQueue, quiesceMediaJobQueue } from './mediaJobQueue/index.js';import { initSpriteLocalAnimationHook } from './sprites/localAnimationJobHook.js';import { initLoraTraining } from './loraTraining/index.js';-import { initSharing } from './sharing/index.js';+import { initSharing, shutdownSharing, drainPeerSyncOperations } from './sharing/index.js';import { initMortalLoomStore } from './mortalLoomStore.js';import { initUniverseBuilderCollectionHook } from './universeBuilderCollectionHook.js';import { initCatalogImageAttachHook } from './catalogImageAttachHook.js';@@ -1091,9 +1091,13 @@ const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 10000;// How long an already-accepted ordinary API request may keep running after the// signal (#8323). Measured from the signal, so it overlaps the teardown steps// before the HTTP close rather than adding to them: the drain (5s) + HTTP close-// grace (1s) + DB pool grace (3s) stays under the ceiling above.+// grace (1s) + peer sync stop (1s) + DB pool grace (2.5s) stays under the ceiling above.const HTTP_DRAIN_WINDOW_MS = 5000;+// Bound on stopping peer sync and settling its in-flight pushes before the DB+// pool closes; drain + HTTP close + this + DB pool grace stay under the ceiling.+const PEER_SYNC_STOP_GRACE_MS = 1000;+/*** Wire SIGTERM/SIGINT to the graceful-shutdown state machine. Idempotent per* signal: once shutdown starts, later signals are ignored.@@ -1254,12 +1258,21 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>else console.error(`❌ Media job queue flush failed: ${outcome.error}`);});+ // Stop peer sync before the pool ends: a peer:online convergence walk still+ // running would otherwise log one pool-ended stack per remaining subscription.+ // Abandoning it is safe — the next boot's peer:online pass repeats it (#11750).+ await withGrace('Peer sync', PEER_SYNC_STOP_GRACE_MS, ({ finish, finishWithError }) =>+ shutdownSharing().then(() => drainPeerSyncOperations()).then(+ () => finish(),+ (err) => finishWithError('⚠️ Peer sync shutdown failed', err),+ ));+const { close } = await import('../lib/db.js');if (typeof close === 'function') {// Bound the DB pool close: pool.end() waits for every checked-out client to// be released, so one hung/leaked connection (e.g. a LISTEN channel) would// otherwise stall shutdown until the force-exit timer.- await withGrace('DB pool', 3000, ({ finish, finishWithError }) =>+ await withGrace('DB pool', 2500, ({ finish, finishWithError }) =>close().then(() => finish('✅ DB pool closed'), (err) => finishWithError('⚠️ DB pool close failed', err)));} else {console.warn('ℹ️ DB pool close not available; skipping DB shutdown');diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.jsindex 4dcf9c1845..c25e1f28a3 100644--- a/server/services/bootstrap.shutdown.test.js+++ b/server/services/bootstrap.shutdown.test.js@@ -103,7 +103,7 @@ describe('shutdown handler — request drain ordering (#8323)', () => {const constant = (name) => Number(SRC.match(new RegExp(`const ${name} = (\\d+);`))?.[1]);const closeGrace = Number(SRC.match(/const closeServer = \(server, label, graceMs = (\d+)\)/)?.[1]);const dbGrace = Number(code.match(/withGrace\('DB pool', (\d+)/)?.[1]);- expect(constant('HTTP_DRAIN_WINDOW_MS') + closeGrace + dbGrace)+ expect(constant('HTTP_DRAIN_WINDOW_MS') + closeGrace + constant('PEER_SYNC_STOP_GRACE_MS') + dbGrace).toBeLessThan(constant('GRACEFUL_SHUTDOWN_TIMEOUT_MS'));});});@@ -132,6 +132,22 @@ describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {});});+// Issue #12585: a peer:online convergence walk still running when the pool ends+// logs a pool-ended stack per remaining subscription, so sharing stops (and its+// background work settles, bounded) after the servers close and before the pool.+describe('shutdown handler — peer sync stops before the DB pool closes (#12585)', () => {+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');++ it('calls shutdownSharing() then drains, under a grace, between the HTTP close and the DB pool close', () => {+ const stopAt = code.indexOf('shutdownSharing()');+ expect(stopAt, 'shutdownSharing() is not called in shutdown()').toBeGreaterThan(-1);+ expect(code).toMatch(/withGrace\('Peer sync', PEER_SYNC_STOP_GRACE_MS,/);+ expect(code.indexOf('drainPeerSyncOperations()')).toBeGreaterThan(stopAt);+ expect(stopAt).toBeGreaterThan(code.indexOf('closeServer('));+ expect(stopAt).toBeLessThan(code.indexOf("withGrace('DB pool'"));+ });+});+// Execute only the close boundary; importing bootstrap loads the live service// graph. Fake time pins the shutdown deadline without production sleeps.describe('bounded server shutdown', () => {diff --git a/server/services/sharing/index.js b/server/services/sharing/index.jsindex 22ca479522..4a7e6a2585 100644--- a/server/services/sharing/index.js+++ b/server/services/sharing/index.js@@ -172,6 +172,10 @@ export async function initSharing({ io: socketIo } = {}) {return result;}+// Resolves once tracked peer-sync background work (peer:online walks, debounced+// pushes) has settled. Call after shutdownSharing() and before closing the DB pool.+export { drainBackgroundOperations as drainPeerSyncOperations } from './peerSubscriptions.js';+export async function shutdownSharing() {await shutdownAllWatchers();stopMediaLibrarySweep();diff --git a/server/services/sharing/peerSync.js b/server/services/sharing/peerSync.jsindex 32c1f14e14..7b190012e4 100644--- a/server/services/sharing/peerSync.js+++ b/server/services/sharing/peerSync.js@@ -579,9 +579,9 @@ async function getIssueSeriesId(issueId) {* edit) gets another attempt.*/export async function retryPendingPushesForPeer(peerId) {- if (!isNonBlankStr(peerId)) return { walked: 0, pushed: 0 };+ if (!isNonBlankStr(peerId) || stopping) return { walked: 0, pushed: 0 };const subs = await listPeerSubscriptions({ peerId });- if (subs.length === 0) return { walked: 0, pushed: 0 };+ if (subs.length === 0 || stopping) return { walked: 0, pushed: 0 };// Separate counter for the log line — only count subs that were never// pushed (genuine retries) so steady-state convergence runs stay quiet.const neverPushedCount = subs.filter(s => !s.lastPushedAt).length;@@ -600,6 +600,7 @@ export async function retryPendingPushesForPeer(peerId) {// terminal write covering all N records' stamps.await withBaseHashFlushBatch(async () => {for (const sub of subs) {+ if (stopping) break;// peer:online fired — re-probe schema-blocked subs immediately (peer// may have upgraded since the last 409). Edit-triggered pushes still// respect the cooldown.@@ -750,10 +751,15 @@ export async function syncNowForPeer(peerId) {let onUpdated = null;let onDeleted = null;let onPeerOnline = null;+// Set by uninstallPeerSyncListener (shutdown). Convergence walks poll it so they+// stop quietly instead of pushing into a closing DB pool; the next boot's+// peer:online pass repeats whatever was abandoned (#11750).+let stopping = false;/** Attach the `recordEvents` + `peer:online` listeners — call once during sharing init. */export function installPeerSyncListener() {if (onUpdated) return;+ stopping = false;onUpdated = ({ recordKind, recordId }) => {trackBackgroundOperation(triggerPushForRecord(recordKind, recordId).catch((err) => {logFailureWithStack(`⚠️ peerSync: listener error for ${recordKind}/${recordId}`, err);@@ -804,6 +810,7 @@ export function installPeerSyncListener() {// peerHasCategory owns the (kind → category) mapping and the fullSync// short-circuit, so iterate the kinds and let it decide.for (const kind of PEER_SUBSCRIBABLE_KINDS) {+ if (stopping) return;// peerHasCategory short-circuits true for a full-sync peer, so a peer// that came online with its category bits off (or a freshly-added// full-sync peer whose instanceId we only just learned) still back-@@ -814,7 +821,9 @@ export function installPeerSyncListener() {});}}+ if (stopping) return;await retryPendingPushesForPeer(peer.instanceId).catch(() => {});+ if (stopping) return;// A full-sync peer added (via defaultPeerFullSync) or toggled before its// instanceId was known couldn't be reciprocated by updatePeer — no identity// yet. peer:online is the first point we know it, so request the mutual@@ -837,6 +846,7 @@ export function installPeerSyncListener() {* test teardown when a follow-up test re-creates events).*/export function uninstallPeerSyncListener() {+ stopping = true;for (const t of pendingTimers.values()) clearTimeout(t);pendingTimers.clear();if (onUpdated) recordEvents.off('updated', onUpdated);@@ -862,6 +872,8 @@ export async function __resetForTests() {await drainBackgroundOperations();await drainWriteTail();await flushBaseHashes();+ // Direct callers of retryPendingPushesForPeer in later tests never re-install.+ stopping = false;}/**diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.jsindex e9cc32c979..f926312338 100644--- a/server/services/sharing/peerSync.test.js+++ b/server/services/sharing/peerSync.test.js@@ -265,6 +265,7 @@ import {peerSyncEvents,__resetForTests,__drainForTests,+ uninstallPeerSyncListener,} from './peerSync.js';import { buildPushPayload, __resetPushFailureLogForTests } from './peerSyncPush.js';@@ -1475,6 +1476,38 @@ describe('peerSync', () => {expect(pushCalls()).toBe(afterFirst);});+ it('shutdown mid-walk stops pushing the remaining subscriptions and logs nothing (#12585)', async () => {+ vi.mocked(listUniverses).mockResolvedValue([{ id: 'u1' }, { id: 'u2' }, { id: 'u3' }]);+ vi.mocked(getUniverse).mockImplementation(async (id) => ({ id }));+ // Initial pushes fail so every sub is still unpushed when the walk starts.+ vi.mocked(peerFetch).mockResolvedValue(null);+ for (const id of ['u1', 'u2', 'u3']) await subscribePeer({ peerId: 'peer-a', recordKind: 'universe', recordId: id });+ await __drainForTests();++ // Shutdown lands while the first retry push is in flight; the later pushes+ // would hit a closed pool in production.+ vi.mocked(peerFetch).mockReset();+ vi.mocked(peerFetch).mockImplementation(async () => {+ uninstallPeerSyncListener();+ return { ok: true, json: async () => ({ missingAssets: [] }) };+ });+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ const result = await retryPendingPushesForPeer('peer-a');+ expect(vi.mocked(peerFetch)).toHaveBeenCalledTimes(1);+ expect(result.pushed).toBe(1);+ // A walk that starts after shutdown does nothing at all.+ expect(await retryPendingPushesForPeer('peer-a')).toEqual({ walked: 0, pushed: 0 });+ expect(vi.mocked(peerFetch)).toHaveBeenCalledTimes(1);+ expect(errorSpy).not.toHaveBeenCalled();+ expect(warnSpy).not.toHaveBeenCalled();+ } finally {+ errorSpy.mockRestore();+ warnSpy.mockRestore();+ }+ });+it('returns {walked: 0, pushed: 0} when the peer has no subscriptions', async () => {const result = await retryPendingPushesForPeer('peer-without-subs');expect(result).toEqual({ walked: 0, pushed: 0 });