Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: persist Brain memory resyncs and flush IdeaLoom export on shutdown (#11756)

[._.]/ Adam Eivy56bfe5139ffd
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index adf06fb4a8..e19fc83b8c 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -120,7 +120,8 @@ import { startOrphanedPartialGc } from './orphanedPartialGc.js';
import { startBeeperAttachmentGc } from './beeperAttachmentGc.js';
import { startTribePurge } from './tribePurge.js';
import { startScreenshotsGc } from './screenshotsGc.js';
-import { initBridge as initBrainMemoryBridge } from './brainMemoryBridge.js';
+import { initBridge as initBrainMemoryBridge, replayPersistedResync as replayBrainMemoryResync } from './brainMemoryBridge.js';
+import { flushAutoSync as flushIdeaLoomAutoSync } from './idealoomAutoSync.js';
import { initDrillCache } from './meatspacePostDrillCache.js';
import { registerPostReminderSchedule } from './meatspacePostReminder.js';
import { registerAutobiographyReminderSchedule } from './autobiographyReminder.js';
@@ -985,6 +986,10 @@ export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled,
runDatabasePhase: async () => {
await runDatabaseBootPhase();
+ // Brain→memory resyncs a previous process queued but never finished. Runs
+ // after initSyncLog and the DB phase (the memory index needs both), in the
+ // background so a large backlog can't delay accepting requests.
+ replayBrainMemoryResync().catch((err) => logBootstrapFailure('❌ Brain memory resync replay failed', err));
const { bindPeerExecutionIo, initializePeerExecution } = await import('./peerExecutionRuntime.js');
bindPeerExecutionIo(io);
await initializePeerExecution().catch(err => console.error(`❌ Peer execution recovery remains held: ${err.message}`));
@@ -1222,6 +1227,13 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
closeServer(localHttpServer, 'Local HTTP mirror')
]);
+ // An IdeaLoom vault export still inside its 5s debounce would otherwise be
+ // lost. A local file write, so it overlaps the DB close below inside that
+ // close's budget rather than adding to the shutdown ceiling; no request can
+ // schedule another export now that the servers are closed.
+ const ideaLoomFlushed = withGrace('IdeaLoom auto-sync flush', 2500, ({ finish, finishWithError }) =>
+ flushIdeaLoomAutoSync().then(() => finish('✅ IdeaLoom auto-sync flushed'), (err) => finishWithError('⚠️ IdeaLoom auto-sync flush failed', err)));
+
// Flush the media-job queue now that no request can admit another job
// (#8325): terminal transitions and the latest progress reach the snapshot a
// restart restores from. The file write is independent of Postgres, so it
@@ -1245,6 +1257,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
await markerWritten;
await mediaQueueFlushed;
+ await ideaLoomFlushed;
clearTimeout(forceExitTimer);
process.exit(0);
};
diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.js
index 0cabb826ba..93f6f3b1ea 100644
--- a/server/services/bootstrap.shutdown.test.js
+++ b/server/services/bootstrap.shutdown.test.js
@@ -100,6 +100,30 @@ describe('shutdown handler — request drain ordering (#8323)', () => {
});
});
+// Issue #11756: the IdeaLoom export debounce is in memory, so shutdown must
+// flush it — after the servers close (nothing can schedule another export),
+// bounded by a grace, overlapping the DB close, and awaited before exit.
+describe('shutdown handler — IdeaLoom auto-sync flush (#11756)', () => {
+ const code = stripCommentsAndNormalize(extractDeclaration(SRC, 'shutdown') || '');
+
+ it('flushes auto-sync under a bounded grace after the servers close and awaits it before exit', () => {
+ const flushAt = code.indexOf('flushIdeaLoomAutoSync()');
+ expect(flushAt, 'flushAutoSync() is not called in shutdown()').toBeGreaterThan(-1);
+ expect(code).toMatch(/const ideaLoomFlushed = withGrace\('IdeaLoom auto-sync flush', \d+,/);
+ expect(flushAt).toBeGreaterThan(code.indexOf('closeServer('));
+ expect(flushAt).toBeLessThan(code.indexOf("withGrace('DB pool'"));
+ const awaitAt = code.indexOf('await ideaLoomFlushed');
+ expect(awaitAt).toBeGreaterThan(code.indexOf("withGrace('DB pool'"));
+ expect(awaitAt).toBeLessThan(code.indexOf('process.exit(0)'));
+ });
+
+ it('keeps the flush grace within the DB close grace so the ceiling is unchanged', () => {
+ const flushGrace = Number(code.match(/withGrace\('IdeaLoom auto-sync flush', (\d+)/)?.[1]);
+ const dbGrace = Number(code.match(/withGrace\('DB pool', (\d+)/)?.[1]);
+ expect(flushGrace).toBeLessThanOrEqual(dbGrace);
+ });
+});
+
// Execute only the close boundary; importing bootstrap loads the live service
// graph. Fake time pins the shutdown deadline without production sleeps.
describe('bounded server shutdown', () => {
diff --git a/server/services/brainMemoryBridge.js b/server/services/brainMemoryBridge.js
index 59eddfb007..12d02b60d0 100644
--- a/server/services/brainMemoryBridge.js
+++ b/server/services/brainMemoryBridge.js
@@ -20,6 +20,7 @@ import { atomicWrite, ensureDir, PATHS } from '../lib/fileUtils.js';
import { iterateJournals, getJournal } from './brainJournal.js';
const BRIDGE_MAP_PATH = join(PATHS.brain, 'memory-bridge-map.json');
+const PENDING_RESYNC_PATH = join(PATHS.brain, 'memory-resync-pending.json');
// brainType → { memoryType, category }
const TYPE_MAP = {
@@ -539,6 +540,63 @@ let resyncFlushing = false; // single-flight guard so two flushes can't overlap
const JOURNAL_RESYNC_DEBOUNCE_MS = 30_000;
const journalResyncTimers = new Map(); // journal id (the ISO date) → timer
+// Durable mirror of every resync that has been triggered but not yet settled:
+// queued entries, the batch a flush is working through, and journal days still
+// inside their debounce window. Persisted so a restart replays the work instead
+// of leaving stale embeddings or still-searchable deleted records behind.
+const durablePending = new Map(); // bridgeKey → { type, id, hardDelete }
+let pendingWriteTail = Promise.resolve();
+let pendingWriteQueued = false;
+
+function persistPending() {
+ // Calls made while a write is queued or running coalesce into one snapshot
+ // taken when the write actually starts.
+ if (pendingWriteQueued) return;
+ pendingWriteQueued = true;
+ pendingWriteTail = pendingWriteTail.then(async () => {
+ pendingWriteQueued = false;
+ const dir = dirname(PENDING_RESYNC_PATH);
+ if (!existsSync(dir)) await ensureDir(dir);
+ await atomicWrite(PENDING_RESYNC_PATH, { entries: [...durablePending.values()] });
+ }).catch((err) => {
+ console.error(`❌ Brain bridge pending-resync persist failed: ${err.message}`);
+ });
+}
+
+// An entry is only finished once nothing newer is waiting for the same key.
+function settleDurable(key, journalId) {
+ if (pendingResync.has(key) || (journalId && journalResyncTimers.has(journalId))) return;
+ if (durablePending.delete(key)) persistPending();
+}
+
+/** Resolves once every queued pending-set write has reached disk. Exported for tests. */
+export const flushPendingResyncWrites = () => pendingWriteTail;
+
+/**
+ * Replay resyncs a previous process queued but never finished. Re-queues them
+ * through the normal path (sticky hard-delete included); an absent or empty
+ * file makes no call at all.
+ */
+export async function replayPersistedResync() {
+ const raw = await readFile(PENDING_RESYNC_PATH, 'utf-8').catch((err) => {
+ if (err.code === 'ENOENT') return null;
+ throw err;
+ });
+ if (raw === null) return 0;
+ let entries;
+ try {
+ entries = JSON.parse(raw)?.entries;
+ } catch {
+ console.warn('⚠️ Corrupt brain pending-resync file; ignoring');
+ return 0;
+ }
+ if (!Array.isArray(entries) || entries.length === 0) return 0;
+ const valid = entries.filter((entry) => entry && typeof entry.id === 'string' && TYPE_MAP[entry.type]);
+ queueResync(valid.map(({ type, id, hardDelete }) => ({ type, id, hardDelete: hardDelete === true })));
+ console.log(`🧠🔗 Replaying ${valid.length} brain resync(s) interrupted by a restart`);
+ return valid.length;
+}
+
/**
* Archive the memory entry mapped to a brain record (if any). Used when the
* canonical record is gone (tombstoned), archived, or otherwise should no
@@ -642,6 +700,7 @@ export async function flushPendingResync() {
await resyncBrainRecord(type, id, { hardDelete }).catch((err) => {
console.error(`❌ Brain bridge resync failed for ${type}/${id}: ${err.message}`);
});
+ settleDurable(bridgeKey(type, id), type === 'journals' ? id : null);
}
}
} finally {
@@ -657,6 +716,7 @@ export async function flushPendingResync() {
*/
export function queueResync(records) {
if (!Array.isArray(records)) return;
+ let queued = false;
for (const { type, id, hardDelete } of records) {
if (!id || !TYPE_MAP[type]) continue;
const key = bridgeKey(type, id);
@@ -667,8 +727,12 @@ export function queueResync(records) {
// resyncBrainRecord's record-present branch upserts a resurrected record
// regardless of the flag, so a true intent can never hard-delete a live row.
const prevHardDelete = pendingResync.get(key)?.hardDelete;
- pendingResync.set(key, { type, id, hardDelete: !!hardDelete || !!prevHardDelete });
+ const entry = { type, id, hardDelete: !!hardDelete || !!prevHardDelete };
+ pendingResync.set(key, entry);
+ durablePending.set(key, entry);
+ queued = true;
}
+ if (queued) persistPending();
if (pendingResync.size === 0 || resyncTimer || resyncFlushing) return;
resyncTimer = setTimeout(() => {
flushPendingResync().catch((err) => {
@@ -765,6 +829,11 @@ export function initBridge() {
*/
function queueJournalResync(id) {
clearTimeout(journalResyncTimers.get(id));
+ const key = bridgeKey('journals', id);
+ if (!durablePending.has(key)) {
+ durablePending.set(key, { type: 'journals', id, hardDelete: false });
+ persistPending();
+ }
const timer = setTimeout(() => {
journalResyncTimers.delete(id);
queueResync([{ type: 'journals', id }]);
@@ -782,6 +851,7 @@ function queueJournalResync(id) {
export function cancelPendingJournalResync(id) {
clearTimeout(journalResyncTimers.get(id));
journalResyncTimers.delete(id);
+ settleDurable(bridgeKey('journals', id), id);
}
async function handleJournalDeleted(entry) {
diff --git a/server/services/brainMemoryBridge.test.js b/server/services/brainMemoryBridge.test.js
index c3b2dab0e5..14f8c986d8 100644
--- a/server/services/brainMemoryBridge.test.js
+++ b/server/services/brainMemoryBridge.test.js
@@ -7,6 +7,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
// live in module scope so assertions can reach them after re-import.
let bridgeFileContents = null; // null = file absent
+let pendingFileContents = null; // memory-resync-pending.json; null = absent
+const isPendingPath = (path) => String(path).endsWith('memory-resync-pending.json');
const createMemory = vi.fn(async (data) => ({ id: `mem-${data.content?.slice(0, 6) || 'x'}` }));
const updateMemory = vi.fn(async () => ({ id: 'updated' }));
@@ -21,16 +23,24 @@ const getDigests = vi.fn(async () => []);
const getReviews = vi.fn(async () => []);
const listJournals = vi.fn(async () => ({ records: [] }));
const getJournal = vi.fn();
-const atomicWrite = vi.fn(async (filePath, data) => {
+const writeThroughMock = async (filePath, data) => {
const payload = (typeof data === 'string' || Buffer.isBuffer(data)) ? data : JSON.stringify(data, null, 2);
const { writeFile } = await import('fs/promises');
return writeFile(filePath, payload);
-});
+};
+const atomicWrite = vi.fn(writeThroughMock);
vi.mock('fs', () => ({ existsSync: vi.fn(() => bridgeFileContents !== null) }));
vi.mock('fs/promises', () => ({
- readFile: vi.fn(async () => bridgeFileContents ?? '{}'),
- writeFile: vi.fn(async (_path, data) => { bridgeFileContents = data; }),
+ readFile: vi.fn(async (path) => {
+ if (!isPendingPath(path)) return bridgeFileContents ?? '{}';
+ if (pendingFileContents === null) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });
+ return pendingFileContents;
+ }),
+ writeFile: vi.fn(async (path, data) => {
+ if (isPendingPath(path)) pendingFileContents = data;
+ else bridgeFileContents = data;
+ }),
}));
vi.mock('../lib/fileUtils.js', () => ({
PATHS: { brain: '/tmp/test-brain' },
@@ -65,6 +75,7 @@ async function loadBridge() {
beforeEach(async () => {
vi.clearAllMocks();
bridgeFileContents = null;
+ pendingFileContents = null;
// The mocked brainEvents emitter is created by the vi.mock factory, which
// outlives vi.resetModules() — so each test's initBridge() stacks another
// set of listeners on the SAME emitter, and one emit fans out to every
@@ -842,14 +853,18 @@ describe('brain bridge durable identity (#11007)', () => {
let enteredWrite;
const entered = new Promise(resolve => { enteredWrite = resolve; });
const held = new Promise(resolve => { releaseWrite = resolve; });
- atomicWrite.mockImplementationOnce(async (_path, data) => {
- const snapshot = JSON.stringify(data);
- enteredWrite();
- await held;
- bridgeFileContents = snapshot;
- });
await createMemory.withImplementation(async data => ({ id: `memory-${data.content}` }), async () => {
bridge.queueResync([{ type: 'people', id: 'example' }]);
+ // Hold only the bridge map's first write; pending-set persists pass through.
+ let holdNext = true;
+ atomicWrite.mockImplementation(async (path, data) => {
+ if (!holdNext || isPendingPath(path)) return writeThroughMock(path, data);
+ holdNext = false;
+ const snapshot = JSON.stringify(data);
+ enteredWrite();
+ await held;
+ bridgeFileContents = snapshot;
+ });
const flush = bridge.flushPendingResync();
await entered;
try {
@@ -903,3 +918,111 @@ describe('brain catch-up beyond old caps', () => {
expect(await bridge.getEmbeddingCoverage()).toEqual({ total: 2002, missing: 0 });
});
});
+
+describe('brainMemoryBridge — pending resyncs survive a restart (#11756)', () => {
+ const persisted = () => JSON.parse(pendingFileContents).entries.map(({ type, id, hardDelete }) => `${type}:${id}:${hardDelete}`);
+
+ // The first process "dies" mid-batch; fake timers keep its debounce timer from
+ // firing into a later test.
+ beforeEach(() => vi.useFakeTimers());
+ afterEach(() => vi.useRealTimers());
+
+ it('finishes records left behind after the first one settled', async () => {
+ const first = await loadBridge();
+ let hang;
+ getById.mockImplementation(async (_type, id) => {
+ if (id === 'b') await new Promise((resolve) => { hang = resolve; }); // process dies here
+ return { id, name: id };
+ });
+
+ first.queueResync([{ type: 'people', id: 'a' }, { type: 'people', id: 'b' }, { type: 'people', id: 'c' }]);
+ first.flushPendingResync(); // never settles: b is in flight, c still queued
+ await vi.waitFor(() => expect(hang).toBeTypeOf('function'));
+ await first.flushPendingResyncWrites();
+ expect(persisted()).toEqual(['people:b:false', 'people:c:false']); // a was removed once it settled
+
+ getById.mockClear();
+ getById.mockImplementation(async (_type, id) => ({ id, name: id }));
+ const second = await loadBridge();
+ expect(await second.replayPersistedResync()).toBe(2);
+ await second.flushPendingResync();
+
+ expect(getById.mock.calls.map(([, id]) => id)).toEqual(['b', 'c']);
+ await second.flushPendingResyncWrites();
+ expect(persisted()).toEqual([]);
+ });
+
+ it('archives a synced-in delete and hard-deletes a local one after the restart', async () => {
+ const first = await loadBridge();
+ first.queueResync([{ type: 'people', id: 'synced' }]);
+ first.queueResync([{ type: 'people', id: 'local', hardDelete: true }]);
+ first.queueResync([{ type: 'people', id: 'local' }]); // coalesced soft touch must not downgrade
+ await first.flushPendingResyncWrites();
+
+ const second = await loadBridge();
+ bridgeFileContents = JSON.stringify({
+ [second.bridgeKey('people', 'synced')]: 'mem-synced',
+ [second.bridgeKey('people', 'local')]: 'mem-local',
+ });
+ getById.mockResolvedValue(null); // both tombstoned
+ await second.replayPersistedResync();
+ await second.flushPendingResync();
+
+ expect(updateMemory).toHaveBeenCalledWith('mem-synced', { status: 'archived' });
+ expect(purgeMemory).toHaveBeenCalledWith('mem-local');
+ expect(purgeMemory).not.toHaveBeenCalledWith('mem-synced');
+ });
+
+ it('keeps an entry re-queued while its resync was in flight', async () => {
+ const bridge = await loadBridge();
+ let release;
+ getById.mockImplementationOnce(async () => { await new Promise((resolve) => { release = resolve; }); return { id: 'a', name: 'a' }; });
+ getById.mockResolvedValue({ id: 'a', name: 'a' });
+
+ bridge.queueResync([{ type: 'people', id: 'a' }]);
+ const flush = bridge.flushPendingResync();
+ await vi.waitFor(() => expect(release).toBeTypeOf('function'));
+ bridge.queueResync([{ type: 'people', id: 'a' }]); // edited again mid-resync
+ release();
+ await flush;
+ await bridge.flushPendingResyncWrites();
+
+ expect(getById).toHaveBeenCalledTimes(2);
+ expect(persisted()).toEqual([]); // removed only after the re-queued pass settled
+ });
+
+ it('persists a journal day inside its debounce window and replays it', async () => {
+ const first = await loadBridge();
+ const { brainEvents } = await import('./brainStorage.js');
+ first.initBridge();
+ brainEvents.emit('journals:upserted', { entry: { id: '2026-01-02' } });
+ await first.flushPendingResyncWrites();
+ expect(persisted()).toEqual(['journals:2026-01-02:false']);
+
+ getJournal.mockResolvedValue({ id: '2026-01-02', date: '2026-01-02', content: 'hello' });
+ const second = await loadBridge();
+ await second.replayPersistedResync();
+ await second.flushPendingResync();
+
+ expect(getJournal).toHaveBeenCalledWith('2026-01-02');
+ expect(createMemory).toHaveBeenCalledTimes(1);
+ });
+
+ it('makes no embedding or store call at boot when nothing is pending', async () => {
+ const absent = await loadBridge();
+ expect(await absent.replayPersistedResync()).toBe(0);
+
+ pendingFileContents = JSON.stringify({ entries: [] });
+ const empty = await loadBridge();
+ expect(await empty.replayPersistedResync()).toBe(0);
+
+ pendingFileContents = '{not json';
+ const corrupt = await loadBridge();
+ expect(await corrupt.replayPersistedResync()).toBe(0);
+ await corrupt.flushPendingResync();
+
+ expect(generateMemoryEmbedding).not.toHaveBeenCalled();
+ expect(getById).not.toHaveBeenCalled();
+ expect(atomicWrite).not.toHaveBeenCalled();
+ });
+});