Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: catch up all Brain digests, reviews, and journal days (#11009)
Adam Eivy11346fc1f741
diff --git a/server/services/brainJournal.js b/server/services/brainJournal.jsindex 72f0e0759d..d44a04057c 100644--- a/server/services/brainJournal.js+++ b/server/services/brainJournal.js@@ -527,17 +527,28 @@ async function removeFromObsidian(entry) {// Sized so an ordinary transient blip cannot trip it but a systemic outage does.const MAX_CONSECUTIVE_RESYNC_SKIPS = 25;+// Hydrate only one page at a time during whole-store catch-up.+export async function* iterateJournals() {+ const limit = 100;+ let offset = 0;+ while (true) {+ const { records, total } = await listJournals({ offset, limit, includeContent: true });+ for (const record of records) yield record;+ offset += records.length;+ if (offset >= total || records.length === 0) return;+ }+}+export async function resyncAllToObsidian() {const settings = await getSettings();if (!settings.obsidianVaultId) return { synced: 0, skipped: 0 };- const { records } = await listJournals({ limit: 10000, includeContent: true });let synced = 0;let skipped = 0;let consecutiveSkips = 0;let stoppedEarly = false;const locationUpdates = new Map();- for (const entry of records) {+ for await (const entry of iterateJournals()) {// force:true so this bulk resync still writes even when the user has// turned off the per-write autoSync — they explicitly clicked "Re-sync// all entries now", which is the manual-sync escape hatch.@@ -549,7 +560,7 @@ export async function resyncAllToObsidian() {}skipped += 1;consecutiveSkips += 1;- // Circuit breaker. This is a FOREGROUND request over up to 10,000 entries,+ // Circuit breaker. This is a FOREGROUND request over all entries,// and an evicted note now costs up to MATERIALIZE_TIMEOUT_MS (20s) before it// is refused (#3706) — so a wedged iCloud or an unplugged vault would turn// "Re-sync all entries now" into a multi-hour request the user cannot cancel.diff --git a/server/services/brainJournal.test.js b/server/services/brainJournal.test.jsindex 3edf30ca01..28c8c36113 100644--- a/server/services/brainJournal.test.js+++ b/server/services/brainJournal.test.js@@ -465,6 +465,16 @@ describe('brainJournal', () => {await journal.updateSettings({ obsidianVaultId: 'v1', autoSync: false, obsidianFolder: 'Daily Log' });});+ it('re-mirrors the oldest day beyond 10000 entries', async () => {+ for (let i = 0; i < 10001; i += 1) {+ const date = isoDay(i);+ journalRecords.set(date, { date, content: 'Day ' + i, segments: [] });+ }+ obsidian.upsertNote.mockResolvedValue('Daily Log/note.md');+ expect(await journal.resyncAllToObsidian()).toEqual({ synced: 10001, skipped: 0, stoppedEarly: false });+ expect(obsidian.upsertNote.mock.calls.some(call => JSON.stringify(call).includes(isoDay(0)))).toBe(true);+ });+it('writes all changed Obsidian locations in one sidecar save', async () => {for (let i = 0; i < 3; i += 1) await journal.appendJournal(isoDay(i), `day ${i}`);obsidian.upsertNote.mockResolvedValue('Daily Log/note.md');diff --git a/server/services/brainMemoryBridge.js b/server/services/brainMemoryBridge.jsindex 8144ca5dd7..59eddfb007 100644--- a/server/services/brainMemoryBridge.js+++ b/server/services/brainMemoryBridge.js@@ -17,7 +17,7 @@ import * as brainStorage from './brainStorage.js';import * as memory from './memoryBackend.js';import * as embeddings from './memoryEmbeddings.js';import { atomicWrite, ensureDir, PATHS } from '../lib/fileUtils.js';-import { listJournals, getJournal } from './brainJournal.js';+import { iterateJournals, getJournal } from './brainJournal.js';const BRIDGE_MAP_PATH = join(PATHS.brain, 'memory-bridge-map.json');@@ -38,10 +38,9 @@ const TYPE_MAP = {// their own getters rather than brainStorage.getAll.const JSONL_TYPES = ['digests', 'reviews'];-// Whole-store read for an append-only JSONL type (the `1000` cap is "all of-// them" — these files are small and cached by brainStorage).+// Explicit whole-store read; UI getters retain their latest-page default.const readJsonlStore = (type) =>- (type === 'digests' ? brainStorage.getDigests : brainStorage.getReviews)(1000);+ (type === 'digests' ? brainStorage.getDigests : brainStorage.getReviews)(Infinity);// The id-keyed entity stores the bridge mirrors. DERIVED from TYPE_MAP (minus// the JSONL stores and the Daily Log, both of which have their own walk) so@@ -373,8 +372,7 @@ export async function syncAllBrainData({ dryRun = false, refresh = false, onlyMi// 'sync:applied' event handlers instead (see initBridge). With `refresh`,// already-mapped days are re-embedded to heal pre-#1080 staleness.{- const { records: journals } = await listJournals({ limit: 10000, includeContent: true });- for (const record of journals) {+ for await (const record of iterateJournals()) {const key = bridgeKey('journals', record.id);// Already-mapped days are skipped in both real and dry-run modes so// dry-run stats match actual-run stats (rather than claiming todiff --git a/server/services/brainMemoryBridge.test.js b/server/services/brainMemoryBridge.test.jsindex 4d5451726e..c3b2dab0e5 100644--- a/server/services/brainMemoryBridge.test.js+++ b/server/services/brainMemoryBridge.test.js@@ -54,7 +54,7 @@ vi.mock('./brainStorage.js', () => {const { EventEmitter } = require('events');return { brainEvents: new EventEmitter(), getById, getAll, listLiveIds, getDigests, getReviews };});-vi.mock('./brainJournal.js', () => ({ listJournals, getJournal }));+vi.mock('./brainJournal.js', () => ({ listJournals, getJournal, iterateJournals: async function* () { const { records } = await listJournals(); yield* records; } }));// Re-import a fresh bridge module (clears the cached bridge map).async function loadBridge() {@@ -869,3 +869,37 @@ describe('brain bridge durable identity (#11007)', () => {});});});++describe('brain catch-up beyond old caps', () => {+ it('embeds the oldest journal beyond 10000 days', async () => {+ const bridge = await loadBridge();+ getAll.mockResolvedValue([]);+ getDigests.mockResolvedValue([]);+ getReviews.mockResolvedValue([]);+ const records = Array.from({ length: 10001 }, (_, i) => ({+ id: new Date(Date.UTC(2000, 0, i + 1)).toISOString().slice(0, 10),+ date: new Date(Date.UTC(2000, 0, i + 1)).toISOString().slice(0, 10),+ content: 'Journal day ' + i+ })).reverse();+ listJournals.mockResolvedValue({ records });+ expect((await bridge.syncAllBrainData({ onlyMissing: true })).synced).toBe(10001);+ // A second pass skips every mapped day, including the oldest.+ expect((await bridge.syncAllBrainData({ onlyMissing: true })).skipped).toBe(10001);+ });++ it('counts and embeds the oldest of 1001 digests and reviews', async () => {+ const bridge = await loadBridge();+ getAll.mockResolvedValue([]);+ listJournals.mockResolvedValue({ records: [] });+ listLiveIds.mockResolvedValue([]);+ const records = Array.from({ length: 1001 }, (_, i) => ({ id: 'record-' + i, digestText: 'Digest', reviewText: 'Review' }));+ getDigests.mockImplementation(async (limit = 10) => records.slice(0, limit));+ getReviews.mockImplementation(async (limit = 10) => records.slice(0, limit));+ expect(await bridge.getEmbeddingCoverage()).toEqual({ total: 2002, missing: 2002 });+ const stats = await bridge.syncAllBrainData({ onlyMissing: true });+ expect(stats.synced).toBe(2002);+ expect(getDigests).toHaveBeenCalledWith(Infinity);+ expect(getReviews).toHaveBeenCalledWith(Infinity);+ expect(await bridge.getEmbeddingCoverage()).toEqual({ total: 2002, missing: 0 });+ });+});diff --git a/server/services/brainStorage.js b/server/services/brainStorage.jsindex 3158ecf47a..3655f5cca1 100644--- a/server/services/brainStorage.js+++ b/server/services/brainStorage.js@@ -949,7 +949,7 @@ export async function getInboxLogCounts() {export async function getDigests(limit = 10) {let records = await loadJsonlStore('digests');records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));- return records.slice(0, limit);+ return limit === Infinity ? records : records.slice(0, limit);}/**@@ -989,7 +989,7 @@ export async function createDigest(digest) {export async function getReviews(limit = 10) {let records = await loadJsonlStore('reviews');records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));- return records.slice(0, limit);+ return limit === Infinity ? records : records.slice(0, limit);}/**diff --git a/server/services/brainStorage.test.js b/server/services/brainStorage.test.jsindex dafb7a354c..9b22045441 100644--- a/server/services/brainStorage.test.js+++ b/server/services/brainStorage.test.js@@ -1074,3 +1074,16 @@ describe('brainStorage meta strict reads (#4115)', () => {expect(meta.confidenceThreshold, 'ENOENT is the one errno that proves absence').toBe(0.6);});});++it('keeps latest-ten defaults while allowing complete JSONL catch-up', async () => {+ const records = Array.from({ length: 1001 }, (_, i) => ({ id: 'digest-' + i, generatedAt: new Date(Date.UTC(2020, 0, i + 1)).toISOString() }));+ const dir = join(getTempRoot(), 'brain');+ mkdirSync(dir, { recursive: true });+ for (const type of ['digests', 'reviews']) {+ writeFileSync(join(dir, type + '.jsonl'), records.map(record => JSON.stringify(record)).join('\n') + '\n');+ }+ brainStorage.invalidateAllCaches();+ expect(await brainStorage.getDigests()).toHaveLength(10);+ expect((await brainStorage.getDigests(Infinity)).at(-1).id).toBe('digest-0');+ expect((await brainStorage.getReviews(Infinity)).at(-1).id).toBe('digest-0');+});