Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12585]): stop peer sync before the DB pool closes at shutdown

[._.]/ Adam Eivy1914771bdca7
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index e5cf6e56ad..6e649980c1 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -144,7 +144,7 @@ import { initSyncOrchestrator } from './syncOrchestrator.js';
import { initMediaJobQueue, resumeMediaJobCompletions, flushMediaJobQueue, quiesceMediaJobQueue } from './mediaJobQueue/index.js';
import { initSpriteLocalAnimationHook } from './sprites/localAnimationJobHook.js';
import { initLoraTraining } from './loraTraining/index.js';
-import { initSharing } from './sharing/index.js';
+import { initSharing, shutdownSharing, drainPeerSyncOperations } from './sharing/index.js';
import { initMortalLoomStore } from './mortalLoomStore.js';
import { initUniverseBuilderCollectionHook } from './universeBuilderCollectionHook.js';
import { initCatalogImageAttachHook } from './catalogImageAttachHook.js';
@@ -1091,9 +1091,13 @@ const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 10000;
// How long an already-accepted ordinary API request may keep running after the
// signal (#8323). Measured from the signal, so it overlaps the teardown steps
// before the HTTP close rather than adding to them: the drain (5s) + HTTP close
-// grace (1s) + DB pool grace (3s) stays under the ceiling above.
+// grace (1s) + peer sync stop (1s) + DB pool grace (2.5s) stays under the ceiling above.
const HTTP_DRAIN_WINDOW_MS = 5000;
+// Bound on stopping peer sync and settling its in-flight pushes before the DB
+// pool closes; drain + HTTP close + this + DB pool grace stay under the ceiling.
+const PEER_SYNC_STOP_GRACE_MS = 1000;
+
/**
* Wire SIGTERM/SIGINT to the graceful-shutdown state machine. Idempotent per
* signal: once shutdown starts, later signals are ignored.
@@ -1254,12 +1258,21 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
else console.error(`❌ Media job queue flush failed: ${outcome.error}`);
});
+ // Stop peer sync before the pool ends: a peer:online convergence walk still
+ // running would otherwise log one pool-ended stack per remaining subscription.
+ // Abandoning it is safe — the next boot's peer:online pass repeats it (#11750).
+ await withGrace('Peer sync', PEER_SYNC_STOP_GRACE_MS, ({ finish, finishWithError }) =>
+ shutdownSharing().then(() => drainPeerSyncOperations()).then(
+ () => finish(),
+ (err) => finishWithError('⚠️ Peer sync shutdown failed', err),
+ ));
+
const { close } = await import('../lib/db.js');
if (typeof close === 'function') {
// Bound the DB pool close: pool.end() waits for every checked-out client to
// be released, so one hung/leaked connection (e.g. a LISTEN channel) would
// otherwise stall shutdown until the force-exit timer.
- await withGrace('DB pool', 3000, ({ finish, finishWithError }) =>
+ await withGrace('DB pool', 2500, ({ finish, finishWithError }) =>
close().then(() => finish('✅ DB pool closed'), (err) => finishWithError('⚠️ DB pool close failed', err)));
} else {
console.warn('ℹ️ DB pool close not available; skipping DB shutdown');
diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.js
index 4dcf9c1845..c25e1f28a3 100644
--- a/server/services/bootstrap.shutdown.test.js
+++ b/server/services/bootstrap.shutdown.test.js
@@ -103,7 +103,7 @@ describe('shutdown handler — request drain ordering (#8323)', () => {
const constant = (name) => Number(SRC.match(new RegExp(`const ${name} = (\\d+);`))?.[1]);
const closeGrace = Number(SRC.match(/const closeServer = \(server, label, graceMs = (\d+)\)/)?.[1]);
const dbGrace = Number(code.match(/withGrace\('DB pool', (\d+)/)?.[1]);
- expect(constant('HTTP_DRAIN_WINDOW_MS') + closeGrace + dbGrace)
+ expect(constant('HTTP_DRAIN_WINDOW_MS') + closeGrace + constant('PEER_SYNC_STOP_GRACE_MS') + dbGrace)
.toBeLessThan(constant('GRACEFUL_SHUTDOWN_TIMEOUT_MS'));
});
});
@@ -132,6 +132,22 @@ describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {
});
});
+// Issue #12585: a peer:online convergence walk still running when the pool ends
+// logs a pool-ended stack per remaining subscription, so sharing stops (and its
+// background work settles, bounded) after the servers close and before the pool.
+describe('shutdown handler — peer sync stops before the DB pool closes (#12585)', () => {
+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');
+
+ it('calls shutdownSharing() then drains, under a grace, between the HTTP close and the DB pool close', () => {
+ const stopAt = code.indexOf('shutdownSharing()');
+ expect(stopAt, 'shutdownSharing() is not called in shutdown()').toBeGreaterThan(-1);
+ expect(code).toMatch(/withGrace\('Peer sync', PEER_SYNC_STOP_GRACE_MS,/);
+ expect(code.indexOf('drainPeerSyncOperations()')).toBeGreaterThan(stopAt);
+ expect(stopAt).toBeGreaterThan(code.indexOf('closeServer('));
+ expect(stopAt).toBeLessThan(code.indexOf("withGrace('DB pool'"));
+ });
+});
+
// Execute only the close boundary; importing bootstrap loads the live service
// graph. Fake time pins the shutdown deadline without production sleeps.
describe('bounded server shutdown', () => {
diff --git a/server/services/sharing/index.js b/server/services/sharing/index.js
index 22ca479522..4a7e6a2585 100644
--- a/server/services/sharing/index.js
+++ b/server/services/sharing/index.js
@@ -172,6 +172,10 @@ export async function initSharing({ io: socketIo } = {}) {
return result;
}
+// Resolves once tracked peer-sync background work (peer:online walks, debounced
+// pushes) has settled. Call after shutdownSharing() and before closing the DB pool.
+export { drainBackgroundOperations as drainPeerSyncOperations } from './peerSubscriptions.js';
+
export async function shutdownSharing() {
await shutdownAllWatchers();
stopMediaLibrarySweep();
diff --git a/server/services/sharing/peerSync.js b/server/services/sharing/peerSync.js
index 32c1f14e14..7b190012e4 100644
--- a/server/services/sharing/peerSync.js
+++ b/server/services/sharing/peerSync.js
@@ -579,9 +579,9 @@ async function getIssueSeriesId(issueId) {
* edit) gets another attempt.
*/
export async function retryPendingPushesForPeer(peerId) {
- if (!isNonBlankStr(peerId)) return { walked: 0, pushed: 0 };
+ if (!isNonBlankStr(peerId) || stopping) return { walked: 0, pushed: 0 };
const subs = await listPeerSubscriptions({ peerId });
- if (subs.length === 0) return { walked: 0, pushed: 0 };
+ if (subs.length === 0 || stopping) return { walked: 0, pushed: 0 };
// Separate counter for the log line — only count subs that were never
// pushed (genuine retries) so steady-state convergence runs stay quiet.
const neverPushedCount = subs.filter(s => !s.lastPushedAt).length;
@@ -600,6 +600,7 @@ export async function retryPendingPushesForPeer(peerId) {
// terminal write covering all N records' stamps.
await withBaseHashFlushBatch(async () => {
for (const sub of subs) {
+ if (stopping) break;
// peer:online fired — re-probe schema-blocked subs immediately (peer
// may have upgraded since the last 409). Edit-triggered pushes still
// respect the cooldown.
@@ -750,10 +751,15 @@ export async function syncNowForPeer(peerId) {
let onUpdated = null;
let onDeleted = null;
let onPeerOnline = null;
+// Set by uninstallPeerSyncListener (shutdown). Convergence walks poll it so they
+// stop quietly instead of pushing into a closing DB pool; the next boot's
+// peer:online pass repeats whatever was abandoned (#11750).
+let stopping = false;
/** Attach the `recordEvents` + `peer:online` listeners — call once during sharing init. */
export function installPeerSyncListener() {
if (onUpdated) return;
+ stopping = false;
onUpdated = ({ recordKind, recordId }) => {
trackBackgroundOperation(triggerPushForRecord(recordKind, recordId).catch((err) => {
logFailureWithStack(`⚠️ peerSync: listener error for ${recordKind}/${recordId}`, err);
@@ -804,6 +810,7 @@ export function installPeerSyncListener() {
// peerHasCategory owns the (kind → category) mapping and the fullSync
// short-circuit, so iterate the kinds and let it decide.
for (const kind of PEER_SUBSCRIBABLE_KINDS) {
+ if (stopping) return;
// peerHasCategory short-circuits true for a full-sync peer, so a peer
// that came online with its category bits off (or a freshly-added
// full-sync peer whose instanceId we only just learned) still back-
@@ -814,7 +821,9 @@ export function installPeerSyncListener() {
});
}
}
+ if (stopping) return;
await retryPendingPushesForPeer(peer.instanceId).catch(() => {});
+ if (stopping) return;
// A full-sync peer added (via defaultPeerFullSync) or toggled before its
// instanceId was known couldn't be reciprocated by updatePeer — no identity
// yet. peer:online is the first point we know it, so request the mutual
@@ -837,6 +846,7 @@ export function installPeerSyncListener() {
* test teardown when a follow-up test re-creates events).
*/
export function uninstallPeerSyncListener() {
+ stopping = true;
for (const t of pendingTimers.values()) clearTimeout(t);
pendingTimers.clear();
if (onUpdated) recordEvents.off('updated', onUpdated);
@@ -862,6 +872,8 @@ export async function __resetForTests() {
await drainBackgroundOperations();
await drainWriteTail();
await flushBaseHashes();
+ // Direct callers of retryPendingPushesForPeer in later tests never re-install.
+ stopping = false;
}
/**
diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.js
index e9cc32c979..f926312338 100644
--- a/server/services/sharing/peerSync.test.js
+++ b/server/services/sharing/peerSync.test.js
@@ -265,6 +265,7 @@ import {
peerSyncEvents,
__resetForTests,
__drainForTests,
+ uninstallPeerSyncListener,
} from './peerSync.js';
import { buildPushPayload, __resetPushFailureLogForTests } from './peerSyncPush.js';
@@ -1475,6 +1476,38 @@ describe('peerSync', () => {
expect(pushCalls()).toBe(afterFirst);
});
+ it('shutdown mid-walk stops pushing the remaining subscriptions and logs nothing (#12585)', async () => {
+ vi.mocked(listUniverses).mockResolvedValue([{ id: 'u1' }, { id: 'u2' }, { id: 'u3' }]);
+ vi.mocked(getUniverse).mockImplementation(async (id) => ({ id }));
+ // Initial pushes fail so every sub is still unpushed when the walk starts.
+ vi.mocked(peerFetch).mockResolvedValue(null);
+ for (const id of ['u1', 'u2', 'u3']) await subscribePeer({ peerId: 'peer-a', recordKind: 'universe', recordId: id });
+ await __drainForTests();
+
+ // Shutdown lands while the first retry push is in flight; the later pushes
+ // would hit a closed pool in production.
+ vi.mocked(peerFetch).mockReset();
+ vi.mocked(peerFetch).mockImplementation(async () => {
+ uninstallPeerSyncListener();
+ return { ok: true, json: async () => ({ missingAssets: [] }) };
+ });
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ try {
+ const result = await retryPendingPushesForPeer('peer-a');
+ expect(vi.mocked(peerFetch)).toHaveBeenCalledTimes(1);
+ expect(result.pushed).toBe(1);
+ // A walk that starts after shutdown does nothing at all.
+ expect(await retryPendingPushesForPeer('peer-a')).toEqual({ walked: 0, pushed: 0 });
+ expect(vi.mocked(peerFetch)).toHaveBeenCalledTimes(1);
+ expect(errorSpy).not.toHaveBeenCalled();
+ expect(warnSpy).not.toHaveBeenCalled();
+ } finally {
+ errorSpy.mockRestore();
+ warnSpy.mockRestore();
+ }
+ });
+
it('returns {walked: 0, pushed: 0} when the peer has no subscriptions', async () => {
const result = await retryPendingPushesForPeer('peer-without-subs');
expect(result).toEqual({ walked: 0, pushed: 0 });