Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: catch up all Brain digests, reviews, and journal days (#11009)

Adam Eivy11346fc1f741
diff --git a/server/services/brainJournal.js b/server/services/brainJournal.js
index 72f0e0759d..d44a04057c 100644
--- a/server/services/brainJournal.js
+++ b/server/services/brainJournal.js
@@ -527,17 +527,28 @@ async function removeFromObsidian(entry) {
// Sized so an ordinary transient blip cannot trip it but a systemic outage does.
const MAX_CONSECUTIVE_RESYNC_SKIPS = 25;
+// Hydrate only one page at a time during whole-store catch-up.
+export async function* iterateJournals() {
+ const limit = 100;
+ let offset = 0;
+ while (true) {
+ const { records, total } = await listJournals({ offset, limit, includeContent: true });
+ for (const record of records) yield record;
+ offset += records.length;
+ if (offset >= total || records.length === 0) return;
+ }
+}
+
export async function resyncAllToObsidian() {
const settings = await getSettings();
if (!settings.obsidianVaultId) return { synced: 0, skipped: 0 };
- const { records } = await listJournals({ limit: 10000, includeContent: true });
let synced = 0;
let skipped = 0;
let consecutiveSkips = 0;
let stoppedEarly = false;
const locationUpdates = new Map();
- for (const entry of records) {
+ for await (const entry of iterateJournals()) {
// force:true so this bulk resync still writes even when the user has
// turned off the per-write autoSync — they explicitly clicked "Re-sync
// all entries now", which is the manual-sync escape hatch.
@@ -549,7 +560,7 @@ export async function resyncAllToObsidian() {
}
skipped += 1;
consecutiveSkips += 1;
- // Circuit breaker. This is a FOREGROUND request over up to 10,000 entries,
+ // Circuit breaker. This is a FOREGROUND request over all entries,
// and an evicted note now costs up to MATERIALIZE_TIMEOUT_MS (20s) before it
// is refused (#3706) — so a wedged iCloud or an unplugged vault would turn
// "Re-sync all entries now" into a multi-hour request the user cannot cancel.
diff --git a/server/services/brainJournal.test.js b/server/services/brainJournal.test.js
index 3edf30ca01..28c8c36113 100644
--- a/server/services/brainJournal.test.js
+++ b/server/services/brainJournal.test.js
@@ -465,6 +465,16 @@ describe('brainJournal', () => {
await journal.updateSettings({ obsidianVaultId: 'v1', autoSync: false, obsidianFolder: 'Daily Log' });
});
+ it('re-mirrors the oldest day beyond 10000 entries', async () => {
+ for (let i = 0; i < 10001; i += 1) {
+ const date = isoDay(i);
+ journalRecords.set(date, { date, content: 'Day ' + i, segments: [] });
+ }
+ obsidian.upsertNote.mockResolvedValue('Daily Log/note.md');
+ expect(await journal.resyncAllToObsidian()).toEqual({ synced: 10001, skipped: 0, stoppedEarly: false });
+ expect(obsidian.upsertNote.mock.calls.some(call => JSON.stringify(call).includes(isoDay(0)))).toBe(true);
+ });
+
it('writes all changed Obsidian locations in one sidecar save', async () => {
for (let i = 0; i < 3; i += 1) await journal.appendJournal(isoDay(i), `day ${i}`);
obsidian.upsertNote.mockResolvedValue('Daily Log/note.md');
diff --git a/server/services/brainMemoryBridge.js b/server/services/brainMemoryBridge.js
index 8144ca5dd7..59eddfb007 100644
--- a/server/services/brainMemoryBridge.js
+++ b/server/services/brainMemoryBridge.js
@@ -17,7 +17,7 @@ import * as brainStorage from './brainStorage.js';
import * as memory from './memoryBackend.js';
import * as embeddings from './memoryEmbeddings.js';
import { atomicWrite, ensureDir, PATHS } from '../lib/fileUtils.js';
-import { listJournals, getJournal } from './brainJournal.js';
+import { iterateJournals, getJournal } from './brainJournal.js';
const BRIDGE_MAP_PATH = join(PATHS.brain, 'memory-bridge-map.json');
@@ -38,10 +38,9 @@ const TYPE_MAP = {
// their own getters rather than brainStorage.getAll.
const JSONL_TYPES = ['digests', 'reviews'];
-// Whole-store read for an append-only JSONL type (the `1000` cap is "all of
-// them" — these files are small and cached by brainStorage).
+// Explicit whole-store read; UI getters retain their latest-page default.
const readJsonlStore = (type) =>
- (type === 'digests' ? brainStorage.getDigests : brainStorage.getReviews)(1000);
+ (type === 'digests' ? brainStorage.getDigests : brainStorage.getReviews)(Infinity);
// The id-keyed entity stores the bridge mirrors. DERIVED from TYPE_MAP (minus
// the JSONL stores and the Daily Log, both of which have their own walk) so
@@ -373,8 +372,7 @@ export async function syncAllBrainData({ dryRun = false, refresh = false, onlyMi
// 'sync:applied' event handlers instead (see initBridge). With `refresh`,
// already-mapped days are re-embedded to heal pre-#1080 staleness.
{
- const { records: journals } = await listJournals({ limit: 10000, includeContent: true });
- for (const record of journals) {
+ for await (const record of iterateJournals()) {
const key = bridgeKey('journals', record.id);
// Already-mapped days are skipped in both real and dry-run modes so
// dry-run stats match actual-run stats (rather than claiming to
diff --git a/server/services/brainMemoryBridge.test.js b/server/services/brainMemoryBridge.test.js
index 4d5451726e..c3b2dab0e5 100644
--- a/server/services/brainMemoryBridge.test.js
+++ b/server/services/brainMemoryBridge.test.js
@@ -54,7 +54,7 @@ vi.mock('./brainStorage.js', () => {
const { EventEmitter } = require('events');
return { brainEvents: new EventEmitter(), getById, getAll, listLiveIds, getDigests, getReviews };
});
-vi.mock('./brainJournal.js', () => ({ listJournals, getJournal }));
+vi.mock('./brainJournal.js', () => ({ listJournals, getJournal, iterateJournals: async function* () { const { records } = await listJournals(); yield* records; } }));
// Re-import a fresh bridge module (clears the cached bridge map).
async function loadBridge() {
@@ -869,3 +869,37 @@ describe('brain bridge durable identity (#11007)', () => {
});
});
});
+
+describe('brain catch-up beyond old caps', () => {
+ it('embeds the oldest journal beyond 10000 days', async () => {
+ const bridge = await loadBridge();
+ getAll.mockResolvedValue([]);
+ getDigests.mockResolvedValue([]);
+ getReviews.mockResolvedValue([]);
+ const records = Array.from({ length: 10001 }, (_, i) => ({
+ id: new Date(Date.UTC(2000, 0, i + 1)).toISOString().slice(0, 10),
+ date: new Date(Date.UTC(2000, 0, i + 1)).toISOString().slice(0, 10),
+ content: 'Journal day ' + i
+ })).reverse();
+ listJournals.mockResolvedValue({ records });
+ expect((await bridge.syncAllBrainData({ onlyMissing: true })).synced).toBe(10001);
+ // A second pass skips every mapped day, including the oldest.
+ expect((await bridge.syncAllBrainData({ onlyMissing: true })).skipped).toBe(10001);
+ });
+
+ it('counts and embeds the oldest of 1001 digests and reviews', async () => {
+ const bridge = await loadBridge();
+ getAll.mockResolvedValue([]);
+ listJournals.mockResolvedValue({ records: [] });
+ listLiveIds.mockResolvedValue([]);
+ const records = Array.from({ length: 1001 }, (_, i) => ({ id: 'record-' + i, digestText: 'Digest', reviewText: 'Review' }));
+ getDigests.mockImplementation(async (limit = 10) => records.slice(0, limit));
+ getReviews.mockImplementation(async (limit = 10) => records.slice(0, limit));
+ expect(await bridge.getEmbeddingCoverage()).toEqual({ total: 2002, missing: 2002 });
+ const stats = await bridge.syncAllBrainData({ onlyMissing: true });
+ expect(stats.synced).toBe(2002);
+ expect(getDigests).toHaveBeenCalledWith(Infinity);
+ expect(getReviews).toHaveBeenCalledWith(Infinity);
+ expect(await bridge.getEmbeddingCoverage()).toEqual({ total: 2002, missing: 0 });
+ });
+});
diff --git a/server/services/brainStorage.js b/server/services/brainStorage.js
index 3158ecf47a..3655f5cca1 100644
--- a/server/services/brainStorage.js
+++ b/server/services/brainStorage.js
@@ -949,7 +949,7 @@ export async function getInboxLogCounts() {
export async function getDigests(limit = 10) {
let records = await loadJsonlStore('digests');
records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));
- return records.slice(0, limit);
+ return limit === Infinity ? records : records.slice(0, limit);
}
/**
@@ -989,7 +989,7 @@ export async function createDigest(digest) {
export async function getReviews(limit = 10) {
let records = await loadJsonlStore('reviews');
records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));
- return records.slice(0, limit);
+ return limit === Infinity ? records : records.slice(0, limit);
}
/**
diff --git a/server/services/brainStorage.test.js b/server/services/brainStorage.test.js
index dafb7a354c..9b22045441 100644
--- a/server/services/brainStorage.test.js
+++ b/server/services/brainStorage.test.js
@@ -1074,3 +1074,16 @@ describe('brainStorage meta strict reads (#4115)', () => {
expect(meta.confidenceThreshold, 'ENOENT is the one errno that proves absence').toBe(0.6);
});
});
+
+it('keeps latest-ten defaults while allowing complete JSONL catch-up', async () => {
+ const records = Array.from({ length: 1001 }, (_, i) => ({ id: 'digest-' + i, generatedAt: new Date(Date.UTC(2020, 0, i + 1)).toISOString() }));
+ const dir = join(getTempRoot(), 'brain');
+ mkdirSync(dir, { recursive: true });
+ for (const type of ['digests', 'reviews']) {
+ writeFileSync(join(dir, type + '.jsonl'), records.map(record => JSON.stringify(record)).join('\n') + '\n');
+ }
+ brainStorage.invalidateAllCaches();
+ expect(await brainStorage.getDigests()).toHaveLength(10);
+ expect((await brainStorage.getDigests(Infinity)).at(-1).id).toBe('digest-0');
+ expect((await brainStorage.getReviews(Infinity)).at(-1).id).toBe('digest-0');
+});