Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: persist Brain memory resyncs and flush IdeaLoom export on shutdown (#11756)
[._.]/ Adam Eivy56bfe5139ffd
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex adf06fb4a8..e19fc83b8c 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -120,7 +120,8 @@ import { startOrphanedPartialGc } from './orphanedPartialGc.js';import { startBeeperAttachmentGc } from './beeperAttachmentGc.js';import { startTribePurge } from './tribePurge.js';import { startScreenshotsGc } from './screenshotsGc.js';-import { initBridge as initBrainMemoryBridge } from './brainMemoryBridge.js';+import { initBridge as initBrainMemoryBridge, replayPersistedResync as replayBrainMemoryResync } from './brainMemoryBridge.js';+import { flushAutoSync as flushIdeaLoomAutoSync } from './idealoomAutoSync.js';import { initDrillCache } from './meatspacePostDrillCache.js';import { registerPostReminderSchedule } from './meatspacePostReminder.js';import { registerAutobiographyReminderSchedule } from './autobiographyReminder.js';@@ -985,6 +986,10 @@ export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled,runDatabasePhase: async () => {await runDatabaseBootPhase();+ // Brain→memory resyncs a previous process queued but never finished. Runs+ // after initSyncLog and the DB phase (the memory index needs both), in the+ // background so a large backlog can't delay accepting requests.+ replayBrainMemoryResync().catch((err) => logBootstrapFailure('❌ Brain memory resync replay failed', err));const { bindPeerExecutionIo, initializePeerExecution } = await import('./peerExecutionRuntime.js');bindPeerExecutionIo(io);await initializePeerExecution().catch(err => console.error(`❌ Peer execution recovery remains held: ${err.message}`));@@ -1222,6 +1227,13 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>closeServer(localHttpServer, 'Local HTTP mirror')]);+ // An IdeaLoom vault export still inside its 5s debounce would otherwise be+ // lost. A local file write, so it overlaps the DB close below inside that+ // close's budget rather than adding to the shutdown ceiling; no request can+ // schedule another export now that the servers are closed.+ const ideaLoomFlushed = withGrace('IdeaLoom auto-sync flush', 2500, ({ finish, finishWithError }) =>+ flushIdeaLoomAutoSync().then(() => finish('✅ IdeaLoom auto-sync flushed'), (err) => finishWithError('⚠️ IdeaLoom auto-sync flush failed', err)));+// Flush the media-job queue now that no request can admit another job// (#8325): terminal transitions and the latest progress reach the snapshot a// restart restores from. The file write is independent of Postgres, so it@@ -1245,6 +1257,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>await markerWritten;await mediaQueueFlushed;+ await ideaLoomFlushed;clearTimeout(forceExitTimer);process.exit(0);};diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.jsindex 0cabb826ba..93f6f3b1ea 100644--- a/server/services/bootstrap.shutdown.test.js+++ b/server/services/bootstrap.shutdown.test.js@@ -100,6 +100,30 @@ describe('shutdown handler — request drain ordering (#8323)', () => {});});+// Issue #11756: the IdeaLoom export debounce is in memory, so shutdown must+// flush it — after the servers close (nothing can schedule another export),+// bounded by a grace, overlapping the DB close, and awaited before exit.+describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');++ it('flushes auto-sync under a bounded grace after the servers close and awaits it before exit', () => {+ const flushAt = code.indexOf('flushIdeaLoomAutoSync()');+ expect(flushAt, 'flushAutoSync() is not called in shutdown()').toBeGreaterThan(-1);+ expect(code).toMatch(/const ideaLoomFlushed = withGrace\('IdeaLoom auto-sync flush', \d+,/);+ expect(flushAt).toBeGreaterThan(code.indexOf('closeServer('));+ expect(flushAt).toBeLessThan(code.indexOf("withGrace('DB pool'"));+ const awaitAt = code.indexOf('await ideaLoomFlushed');+ expect(awaitAt).toBeGreaterThan(code.indexOf("withGrace('DB pool'"));+ expect(awaitAt).toBeLessThan(code.indexOf('process.exit(0)'));+ });++ it('keeps the flush grace within the DB close grace so the ceiling is unchanged', () => {+ const flushGrace = Number(code.match(/withGrace\('IdeaLoom auto-sync flush', (\d+)/)?.[1]);+ const dbGrace = Number(code.match(/withGrace\('DB pool', (\d+)/)?.[1]);+ expect(flushGrace).toBeLessThanOrEqual(dbGrace);+ });+});+// Execute only the close boundary; importing bootstrap loads the live service// graph. Fake time pins the shutdown deadline without production sleeps.describe('bounded server shutdown', () => {diff --git a/server/services/brainMemoryBridge.js b/server/services/brainMemoryBridge.jsindex 59eddfb007..12d02b60d0 100644--- a/server/services/brainMemoryBridge.js+++ b/server/services/brainMemoryBridge.js@@ -20,6 +20,7 @@ import { atomicWrite, ensureDir, PATHS } from '../lib/fileUtils.js';import { iterateJournals, getJournal } from './brainJournal.js';const BRIDGE_MAP_PATH = join(PATHS.brain, 'memory-bridge-map.json');+const PENDING_RESYNC_PATH = join(PATHS.brain, 'memory-resync-pending.json');// brainType → { memoryType, category }const TYPE_MAP = {@@ -539,6 +540,63 @@ let resyncFlushing = false; // single-flight guard so two flushes can't overlapconst JOURNAL_RESYNC_DEBOUNCE_MS = 30_000;const journalResyncTimers = new Map(); // journal id (the ISO date) → timer+// Durable mirror of every resync that has been triggered but not yet settled:+// queued entries, the batch a flush is working through, and journal days still+// inside their debounce window. Persisted so a restart replays the work instead+// of leaving stale embeddings or still-searchable deleted records behind.+const durablePending = new Map(); // bridgeKey → { type, id, hardDelete }+let pendingWriteTail = Promise.resolve();+let pendingWriteQueued = false;++function persistPending() {+ // Calls made while a write is queued or running coalesce into one snapshot+ // taken when the write actually starts.+ if (pendingWriteQueued) return;+ pendingWriteQueued = true;+ pendingWriteTail = pendingWriteTail.then(async () => {+ pendingWriteQueued = false;+ const dir = dirname(PENDING_RESYNC_PATH);+ if (!existsSync(dir)) await ensureDir(dir);+ await atomicWrite(PENDING_RESYNC_PATH, { entries: [...durablePending.values()] });+ }).catch((err) => {+ console.error(`❌ Brain bridge pending-resync persist failed: ${err.message}`);+ });+}++// An entry is only finished once nothing newer is waiting for the same key.+function settleDurable(key, journalId) {+ if (pendingResync.has(key) || (journalId && journalResyncTimers.has(journalId))) return;+ if (durablePending.delete(key)) persistPending();+}++/** Resolves once every queued pending-set write has reached disk. Exported for tests. */+export const flushPendingResyncWrites = () => pendingWriteTail;++/**+ * Replay resyncs a previous process queued but never finished. Re-queues them+ * through the normal path (sticky hard-delete included); an absent or empty+ * file makes no call at all.+ */+export async function replayPersistedResync() {+ const raw = await readFile(PENDING_RESYNC_PATH, 'utf-8').catch((err) => {+ if (err.code === 'ENOENT') return null;+ throw err;+ });+ if (raw === null) return 0;+ let entries;+ try {+ entries = JSON.parse(raw)?.entries;+ } catch {+ console.warn('⚠️ Corrupt brain pending-resync file; ignoring');+ return 0;+ }+ if (!Array.isArray(entries) || entries.length === 0) return 0;+ const valid = entries.filter((entry) => entry && typeof entry.id === 'string' && TYPE_MAP[entry.type]);+ queueResync(valid.map(({ type, id, hardDelete }) => ({ type, id, hardDelete: hardDelete === true })));+ console.log(`🧠🔗 Replaying ${valid.length} brain resync(s) interrupted by a restart`);+ return valid.length;+}+/*** Archive the memory entry mapped to a brain record (if any). Used when the* canonical record is gone (tombstoned), archived, or otherwise should no@@ -642,6 +700,7 @@ export async function flushPendingResync() {await resyncBrainRecord(type, id, { hardDelete }).catch((err) => {console.error(`❌ Brain bridge resync failed for ${type}/${id}: ${err.message}`);});+ settleDurable(bridgeKey(type, id), type === 'journals' ? id : null);}}} finally {@@ -657,6 +716,7 @@ export async function flushPendingResync() {*/export function queueResync(records) {if (!Array.isArray(records)) return;+ let queued = false;for (const { type, id, hardDelete } of records) {if (!id || !TYPE_MAP[type]) continue;const key = bridgeKey(type, id);@@ -667,8 +727,12 @@ export function queueResync(records) {// resyncBrainRecord's record-present branch upserts a resurrected record// regardless of the flag, so a true intent can never hard-delete a live row.const prevHardDelete = pendingResync.get(key)?.hardDelete;- pendingResync.set(key, { type, id, hardDelete: !!hardDelete || !!prevHardDelete });+ const entry = { type, id, hardDelete: !!hardDelete || !!prevHardDelete };+ pendingResync.set(key, entry);+ durablePending.set(key, entry);+ queued = true;}+ if (queued) persistPending();if (pendingResync.size === 0 || resyncTimer || resyncFlushing) return;resyncTimer = setTimeout(() => {flushPendingResync().catch((err) => {@@ -765,6 +829,11 @@ export function initBridge() {*/function queueJournalResync(id) {clearTimeout(journalResyncTimers.get(id));+ const key = bridgeKey('journals', id);+ if (!durablePending.has(key)) {+ durablePending.set(key, { type: 'journals', id, hardDelete: false });+ persistPending();+ }const timer = setTimeout(() => {journalResyncTimers.delete(id);queueResync([{ type: 'journals', id }]);@@ -782,6 +851,7 @@ function queueJournalResync(id) {export function cancelPendingJournalResync(id) {clearTimeout(journalResyncTimers.get(id));journalResyncTimers.delete(id);+ settleDurable(bridgeKey('journals', id), id);}async function handleJournalDeleted(entry) {diff --git a/server/services/brainMemoryBridge.test.js b/server/services/brainMemoryBridge.test.jsindex c3b2dab0e5..14f8c986d8 100644--- a/server/services/brainMemoryBridge.test.js+++ b/server/services/brainMemoryBridge.test.js@@ -7,6 +7,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';// live in module scope so assertions can reach them after re-import.let bridgeFileContents = null; // null = file absent+let pendingFileContents = null; // memory-resync-pending.json; null = absent+const isPendingPath = (path) => String(path).endsWith('memory-resync-pending.json');const createMemory = vi.fn(async (data) => ({ id: `mem-${data.content?.slice(0, 6) || 'x'}` }));const updateMemory = vi.fn(async () => ({ id: 'updated' }));@@ -21,16 +23,24 @@ const getDigests = vi.fn(async () => []);const getReviews = vi.fn(async () => []);const listJournals = vi.fn(async () => ({ records: [] }));const getJournal = vi.fn();-const atomicWrite = vi.fn(async (filePath, data) => {+const writeThroughMock = async (filePath, data) => {const payload = (typeof data === 'string' || Buffer.isBuffer(data)) ? data : JSON.stringify(data, null, 2);const { writeFile } = await import('fs/promises');return writeFile(filePath, payload);-});+};+const atomicWrite = vi.fn(writeThroughMock);vi.mock('fs', () => ({ existsSync: vi.fn(() => bridgeFileContents !== null) }));vi.mock('fs/promises', () => ({- readFile: vi.fn(async () => bridgeFileContents ?? '{}'),- writeFile: vi.fn(async (_path, data) => { bridgeFileContents = data; }),+ readFile: vi.fn(async (path) => {+ if (!isPendingPath(path)) return bridgeFileContents ?? '{}';+ if (pendingFileContents === null) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });+ return pendingFileContents;+ }),+ writeFile: vi.fn(async (path, data) => {+ if (isPendingPath(path)) pendingFileContents = data;+ else bridgeFileContents = data;+ }),}));vi.mock('../lib/fileUtils.js', () => ({PATHS: { brain: '/tmp/test-brain' },@@ -65,6 +75,7 @@ async function loadBridge() {beforeEach(async () => {vi.clearAllMocks();bridgeFileContents = null;+ pendingFileContents = null;// The mocked brainEvents emitter is created by the vi.mock factory, which// outlives vi.resetModules() — so each test's initBridge() stacks another// set of listeners on the SAME emitter, and one emit fans out to every@@ -842,14 +853,18 @@ describe('brain bridge durable identity (#11007)', () => {let enteredWrite;const entered = new Promise(resolve => { enteredWrite = resolve; });const held = new Promise(resolve => { releaseWrite = resolve; });- atomicWrite.mockImplementationOnce(async (_path, data) => {- const snapshot = JSON.stringify(data);- enteredWrite();- await held;- bridgeFileContents = snapshot;- });await createMemory.withImplementation(async data => ({ id: `memory-${data.content}` }), async () => {bridge.queueResync([{ type: 'people', id: 'example' }]);+ // Hold only the bridge map's first write; pending-set persists pass through.+ let holdNext = true;+ atomicWrite.mockImplementation(async (path, data) => {+ if (!holdNext || isPendingPath(path)) return writeThroughMock(path, data);+ holdNext = false;+ const snapshot = JSON.stringify(data);+ enteredWrite();+ await held;+ bridgeFileContents = snapshot;+ });const flush = bridge.flushPendingResync();await entered;try {@@ -903,3 +918,111 @@ describe('brain catch-up beyond old caps', () => {expect(await bridge.getEmbeddingCoverage()).toEqual({ total: 2002, missing: 0 });});});++describe('brainMemoryBridge — pending resyncs survive a restart (#11756)', () => {+ const persisted = () => JSON.parse(pendingFileContents).entries.map(({ type, id, hardDelete }) => `${type}:${id}:${hardDelete}`);++ // The first process "dies" mid-batch; fake timers keep its debounce timer from+ // firing into a later test.+ beforeEach(() => vi.useFakeTimers());+ afterEach(() => vi.useRealTimers());++ it('finishes records left behind after the first one settled', async () => {+ const first = await loadBridge();+ let hang;+ getById.mockImplementation(async (_type, id) => {+ if (id === 'b') await new Promise((resolve) => { hang = resolve; }); // process dies here+ return { id, name: id };+ });++ first.queueResync([{ type: 'people', id: 'a' }, { type: 'people', id: 'b' }, { type: 'people', id: 'c' }]);+ first.flushPendingResync(); // never settles: b is in flight, c still queued+ await vi.waitFor(() => expect(hang).toBeTypeOf('function'));+ await first.flushPendingResyncWrites();+ expect(persisted()).toEqual(['people:b:false', 'people:c:false']); // a was removed once it settled++ getById.mockClear();+ getById.mockImplementation(async (_type, id) => ({ id, name: id }));+ const second = await loadBridge();+ expect(await second.replayPersistedResync()).toBe(2);+ await second.flushPendingResync();++ expect(getById.mock.calls.map(([, id]) => id)).toEqual(['b', 'c']);+ await second.flushPendingResyncWrites();+ expect(persisted()).toEqual([]);+ });++ it('archives a synced-in delete and hard-deletes a local one after the restart', async () => {+ const first = await loadBridge();+ first.queueResync([{ type: 'people', id: 'synced' }]);+ first.queueResync([{ type: 'people', id: 'local', hardDelete: true }]);+ first.queueResync([{ type: 'people', id: 'local' }]); // coalesced soft touch must not downgrade+ await first.flushPendingResyncWrites();++ const second = await loadBridge();+ bridgeFileContents = JSON.stringify({+ [second.bridgeKey('people', 'synced')]: 'mem-synced',+ [second.bridgeKey('people', 'local')]: 'mem-local',+ });+ getById.mockResolvedValue(null); // both tombstoned+ await second.replayPersistedResync();+ await second.flushPendingResync();++ expect(updateMemory).toHaveBeenCalledWith('mem-synced', { status: 'archived' });+ expect(purgeMemory).toHaveBeenCalledWith('mem-local');+ expect(purgeMemory).not.toHaveBeenCalledWith('mem-synced');+ });++ it('keeps an entry re-queued while its resync was in flight', async () => {+ const bridge = await loadBridge();+ let release;+ getById.mockImplementationOnce(async () => { await new Promise((resolve) => { release = resolve; }); return { id: 'a', name: 'a' }; });+ getById.mockResolvedValue({ id: 'a', name: 'a' });++ bridge.queueResync([{ type: 'people', id: 'a' }]);+ const flush = bridge.flushPendingResync();+ await vi.waitFor(() => expect(release).toBeTypeOf('function'));+ bridge.queueResync([{ type: 'people', id: 'a' }]); // edited again mid-resync+ release();+ await flush;+ await bridge.flushPendingResyncWrites();++ expect(getById).toHaveBeenCalledTimes(2);+ expect(persisted()).toEqual([]); // removed only after the re-queued pass settled+ });++ it('persists a journal day inside its debounce window and replays it', async () => {+ const first = await loadBridge();+ const { brainEvents } = await import('./brainStorage.js');+ first.initBridge();+ brainEvents.emit('journals:upserted', { entry: { id: '2026-01-02' } });+ await first.flushPendingResyncWrites();+ expect(persisted()).toEqual(['journals:2026-01-02:false']);++ getJournal.mockResolvedValue({ id: '2026-01-02', date: '2026-01-02', content: 'hello' });+ const second = await loadBridge();+ await second.replayPersistedResync();+ await second.flushPendingResync();++ expect(getJournal).toHaveBeenCalledWith('2026-01-02');+ expect(createMemory).toHaveBeenCalledTimes(1);+ });++ it('makes no embedding or store call at boot when nothing is pending', async () => {+ const absent = await loadBridge();+ expect(await absent.replayPersistedResync()).toBe(0);++ pendingFileContents = JSON.stringify({ entries: [] });+ const empty = await loadBridge();+ expect(await empty.replayPersistedResync()).toBe(0);++ pendingFileContents = '{not json';+ const corrupt = await loadBridge();+ expect(await corrupt.replayPersistedResync()).toBe(0);+ await corrupt.flushPendingResync();++ expect(generateMemoryEmbedding).not.toHaveBeenCalled();+ expect(getById).not.toHaveBeenCalled();+ expect(atomicWrite).not.toHaveBeenCalled();+ });+});