Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Brain → CoS Memory Bridge
*
* Mirrors brain records (projects, ideas, admin, memories/journal, digests, reviews, people,
* songs) into the CoS memory system so agents can semantically search user-captured thoughts.
*
* Brain JSON files remain the operational data store for the brain UI.
* This bridge creates/updates corresponding entries in the memories table
* tagged with sourceAppId='brain'.
*/
import { readFile } from 'fs/promises';
import { existsSync } from 'fs';
import { join, dirname } from 'path';
import { brainEvents } from './brainStorage.js';
import * as brainStorage from './brainStorage.js';
import * as memory from './memoryBackend.js';
import * as embeddings from './memoryEmbeddings.js';
import { atomicWrite, ensureDir, PATHS } from '../lib/fileUtils.js';
import { iterateJournals, getJournal } from './brainJournal.js';
const BRIDGE_MAP_PATH = join(PATHS.brain, 'memory-bridge-map.json');
const PENDING_RESYNC_PATH = join(PATHS.brain, 'memory-resync-pending.json');
// brainType → { memoryType, category }
const TYPE_MAP = {
people: { type: 'context', category: 'people' },
projects: { type: 'fact', category: 'project' },
ideas: { type: 'observation', category: 'ideas' },
admin: { type: 'fact', category: 'admin' },
memories: { type: 'observation', category: 'personal' },
digests: { type: 'context', category: 'digest' },
reviews: { type: 'context', category: 'review' },
journals: { type: 'observation', category: 'daily-log' },
songs: { type: 'context', category: 'songbook' }
};
// Append-only JSONL stores — no per-record events, no deletes, walked with
// their own getters rather than brainStorage.getAll.
const JSONL_TYPES = ['digests', 'reviews'];
// Explicit whole-store read; UI getters retain their latest-page default.
const readJsonlStore = (type) =>
(type === 'digests' ? brainStorage.getDigests : brainStorage.getReviews)(Infinity);
// The id-keyed entity stores the bridge mirrors. DERIVED from TYPE_MAP (minus
// the JSONL stores and the Daily Log, both of which have their own walk) so
// adding a type to the map above automatically enrolls it in the bulk sync, the
// embedding-coverage tally, the refresh reconcile, and the live event
// listeners. Those four sites used to restate the list, and a new type wired
// into TYPE_MAP but missed in one of them would sync but never re-embed (or
// count, or archive) — silently.
const BRIDGED_ENTITY_TYPES = Object.keys(TYPE_MAP)
.filter((type) => type !== 'journals' && !JSONL_TYPES.includes(type));
// Entity stores plus the Daily Log: everything keyed by a canonical record the
// bridge can re-read, which is what the coverage tally and the refresh
// reconcile walk. (JSONL stores are append-only, so they can never orphan a
// mapped memory entry.)
const RECONCILABLE_TYPES = [...BRIDGED_ENTITY_TYPES, 'journals'];
// ─── Bridge Map ─────────────────────────────────────────────────────────────
// Maps "brainType:brainId" → memoryId so updates hit the same memory entry
let bridgeMap = null;
let bridgeMapLoading = null;
let mapWriteTail = Promise.resolve();
export async function loadBridgeMap({ readOnly = false } = {}) {
if (bridgeMap) return bridgeMap;
if (bridgeMapLoading) return bridgeMapLoading;
const load = async () => {
let legacyMap = {};
let rebuild = false;
const raw = await readFile(BRIDGE_MAP_PATH, 'utf-8').catch(err => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (raw !== null) {
try {
legacyMap = JSON.parse(raw);
if (!legacyMap || Array.isArray(legacyMap) || typeof legacyMap !== 'object') {
throw new Error('Invalid bridge map shape');
}
legacyMap = Object.fromEntries(Object.entries(legacyMap)
.filter(([, id]) => typeof id === 'string' && id));
} catch {
console.warn('⚠️ Corrupt brain bridge cache; rebuilding from durable memory links');
legacyMap = {};
rebuild = true;
}
}
// Backfill old installs before trusting the cache. DB links win over stale
// file snapshots; missing rows are excluded so bulk catch-up can heal them.
const map = await memory.getBrainMemoryLinks(legacyMap, { readOnly, ...(rebuild ? { recover: true } : {}) });
if (!readOnly) {
bridgeMap = map;
if (rebuild) await saveBridgeMap();
}
return map;
};
// Read-only previews neither migrate links nor cache an unmigrated map.
if (readOnly) return load();
bridgeMapLoading = load();
try {
return await bridgeMapLoading;
} finally {
bridgeMapLoading = null;
}
}
function saveBridgeMap() {
// Snapshot only after the preceding rename completes. All writers, including
// bulk catch-up and delete handlers, share this tail.
const write = mapWriteTail.then(async () => {
const dir = dirname(BRIDGE_MAP_PATH);
if (!existsSync(dir)) await ensureDir(dir);
await atomicWrite(BRIDGE_MAP_PATH, bridgeMap);
});
mapWriteTail = write.catch(() => {});
return write;
}
export function bridgeKey(brainType, brainId) {
return `${brainType}:${brainId}`;
}
// ─── Content Composers ─────────────────────────────────────────────────────
// Each brain type has different fields; compose a single content string for memory storage.
function composePeopleContent(r) {
const parts = [`Person: ${r.name}`];
if (r.context) parts.push(r.context);
if (r.followUps?.length) parts.push(`Follow-ups: ${r.followUps.join('; ')}`);
return parts.join('\n');
}
function composeProjectContent(r) {
const parts = [`Project: ${r.name}`, `Status: ${r.status}`];
if (r.nextAction) parts.push(`Next action: ${r.nextAction}`);
if (r.notes) parts.push(r.notes);
return parts.join('\n');
}
function composeIdeaContent(r) {
const parts = [`Idea: ${r.title}`];
if (r.oneLiner) parts.push(r.oneLiner);
if (r.notes) parts.push(r.notes);
return parts.join('\n');
}
function composeAdminContent(r) {
const parts = [`Admin: ${r.title}`, `Status: ${r.status}`];
if (r.dueDate) parts.push(`Due: ${r.dueDate}`);
if (r.nextAction) parts.push(`Next action: ${r.nextAction}`);
if (r.notes) parts.push(r.notes);
return parts.join('\n');
}
function composeJournalContent(r) {
const parts = [];
if (r.title) parts.push(r.title);
if (r.content) parts.push(r.content);
if (r.mood) parts.push(`Mood: ${r.mood}`);
return parts.join('\n');
}
function composeDigestContent(r) {
const parts = [];
if (r.digestText) parts.push(r.digestText);
if (r.topActions?.length) parts.push(`Top actions: ${r.topActions.join('; ')}`);
if (r.stuckThing) parts.push(`Stuck on: ${r.stuckThing}`);
if (r.smallWin) parts.push(`Small win: ${r.smallWin}`);
return parts.join('\n');
}
function composeReviewContent(r) {
const parts = [];
if (r.reviewText) parts.push(r.reviewText);
if (r.whatHappened?.length) parts.push(`What happened: ${r.whatHappened.join('; ')}`);
if (r.biggestOpenLoops?.length) parts.push(`Open loops: ${r.biggestOpenLoops.join('; ')}`);
if (r.suggestedActionsNextWeek?.length) parts.push(`Suggested actions: ${r.suggestedActionsNextWeek.join('; ')}`);
if (r.recurringTheme) parts.push(`Recurring theme: ${r.recurringTheme}`);
return parts.join('\n');
}
function composeDailyLogContent(r) {
const parts = [`Daily Log — ${r.date}`];
if (r.content) parts.push(r.content);
return parts.join('\n');
}
// SongBook repertoire entry (brain type `songs`). The sheet body
// (`content.text`, up to 200k chars of tab/ChordPro notation) is deliberately
// LEFT OUT: it is positional notation, not prose, so it adds no semantic signal
// to an embedding while blowing the embedder's char budget — one long song
// would push the record into generateMemoryEmbedding's over-budget LLM
// summarization branch for nothing. What the user actually searches for is the
// song's identity and their own notes about it.
function composeSongContent(r) {
const parts = [`Song: ${r.title}`];
if (r.artist) parts.push(`Artist: ${r.artist}`);
if (r.instrument) parts.push(`Instrument: ${r.instrument}`);
if (r.stage) parts.push(`Stage: ${r.stage}`);
const setup = [
r.key && `Key: ${r.key}`,
r.tuning && `Tuning: ${r.tuning}`,
// `capo: 0` is the schema default and means "no capo" — not "capo at fret
// 0" — so it is legitimately omitted rather than rendered as `Capo: 0`.
// Explicit `> 0` so the intent doesn't read as an accidental truthiness bug.
r.capo > 0 ? `Capo: ${r.capo}` : null
].filter(Boolean);
if (setup.length) parts.push(setup.join(' · '));
if (r.notes) parts.push(r.notes);
return parts.join('\n');
}
export const CONTENT_COMPOSERS = {
people: composePeopleContent,
projects: composeProjectContent,
ideas: composeIdeaContent,
admin: composeAdminContent,
memories: composeJournalContent,
digests: composeDigestContent,
reviews: composeReviewContent,
journals: composeDailyLogContent,
songs: composeSongContent
};
// ─── Core Mapping ───────────────────────────────────────────────────────────
/**
* Convert a brain record into a memory-create payload.
*/
export function brainRecordToMemory(brainType, record) {
const mapping = TYPE_MAP[brainType];
if (!mapping) return null;
const composer = CONTENT_COMPOSERS[brainType];
const content = composer(record);
if (!content?.trim()) return null;
const summary = content.length > 200 ? content.substring(0, 197) + '...' : content;
const recordTags = record.tags || [];
const tags = [...new Set([...recordTags, 'brain', brainType])];
return {
type: mapping.type,
content,
summary,
category: mapping.category,
tags,
confidence: 1.0,
importance: 0.6,
sourceAppId: 'brain',
sourceAgentId: 'brain-bridge'
};
}
// ─── Sync ───────────────────────────────────────────────────────────────────
/**
* Upsert a single brain record into the memory system.
* Creates a new memory or updates the existing one based on the bridge map.
*/
export async function syncBrainRecord(brainType, record, { deferMapSave = false, onMapChanged } = {}) {
const memoryData = brainRecordToMemory(brainType, record);
if (!memoryData) return null;
const map = await loadBridgeMap();
const key = bridgeKey(brainType, record.id);
const existingMemoryId = map[key];
// Generate embedding. Pass the source hints (NOT persisted on memoryData —
// the memory schema has no such columns) so the embedder can summarize the
// FULL archived transcript for an over-budget chatgpt-import record rather
// than the already-capped content preview.
const embedding = await embeddings
.generateMemoryEmbedding(memoryData, { source: record.source, sourceRef: record.sourceRef })
.catch(() => null);
const saved = await memory.upsertBrainMemory(key, memoryData, embedding, existingMemoryId);
if (map[key] !== saved.id) {
map[key] = saved.id;
onMapChanged?.();
if (!deferMapSave) await saveBridgeMap();
}
console.log(`🧠🔗 Synced brain→memory: ${brainType}/${record.id} → ${saved.id}`);
return saved.id;
}
/**
* Bulk sync all existing brain data into the memory system.
* Used for initial migration and catch-up.
*
* @param {object} [opts]
* @param {boolean} [opts.dryRun=false] Report what would sync without writing.
* @param {boolean} [opts.refresh=false] Re-embed records that ALREADY have a
* bridge-map entry instead of skipping them. This is the recovery path for
* records that diverged before the per-record sync:applied signal existed
* (issue #1080): a peer's edit/delete of an already-mapped record used to
* leave the local memory copy stale and searchable forever, and a normal
* bulk sync skipped it precisely because it was mapped. `refresh` forces a
* re-embed of every live record so a one-time catch-up heals the staleness.
* In refresh mode a reconcile pass also archives memory entries whose brain
* record was deleted/archived on a peer before this fix (orphaned map keys
* the live-record walks can't reach) — reported as `archived`.
* @returns {Promise<{synced:number, skipped:number, errors:number, archived:number}>}
*/
// True when a record is already embedded: mapped AND its memory has a vector.
// Shared by the onlyMissing sync path and the read-only coverage tally.
const makeEmbeddedChecker = (map, missingMemIds) => (key) => {
const memId = map[key];
return !!memId && !missingMemIds.has(memId);
};
export async function syncAllBrainData({ dryRun = false, refresh = false, onlyMissing = false } = {}) {
// Revalidate cached ids once per bulk walk: a memory can be purged after
// this process loaded the cache. A stale key must not count as embedded.
const map = await memory.getBrainMemoryLinks(
await loadBridgeMap({ readOnly: dryRun }), { readOnly: dryRun });
const stats = { synced: 0, skipped: 0, errors: 0, archived: 0 };
let bridgeMapChanged = false;
const deferredMapSave = {
deferMapSave: true,
onMapChanged: () => { bridgeMapChanged = true; },
};
try {
// `onlyMissing` is the cheap, targeted backfill: embed only records that lack
// an embedding (unmapped, or mapped to a memory whose vector is NULL because
// generation failed) and skip everything already embedded. Unlike `refresh`
// it never re-embeds healthy records, so it's safe to run without a warning.
const missingMemIds = onlyMissing
? await memory.getMemoryIdsMissingEmbedding().catch(() => new Set())
: null;
const isEmbedded = makeEmbeddedChecker(map, missingMemIds);
// Entity stores (JSON-based)
for (const type of BRIDGED_ENTITY_TYPES) {
const records = await brainStorage.getAll(type);
for (const record of records) {
// Skip archived records
if (record.archived) {
stats.skipped++;
continue;
}
const key = bridgeKey(type, record.id);
if (onlyMissing) {
if (isEmbedded(key)) { stats.skipped++; continue; }
} else if (map[key] && !dryRun && !refresh) {
stats.skipped++;
continue;
}
if (dryRun) {
console.log(`🧠🔗 [dry-run] Would sync ${type}/${record.id}: ${record.name || record.title || '(untitled)'}`);
stats.synced++;
continue;
}
const memoryId = await syncBrainRecord(type, record, deferredMapSave).catch(err => {
console.error(`❌ Failed to sync ${type}/${record.id}: ${err.message}`);
stats.errors++;
return null;
});
if (memoryId) stats.synced++;
}
}
// Daily log entries — one memory per day. Without `refresh`, already-mapped
// days are skipped (initial/backfill import only) and content updates /
// deletions flow through the 'journals:upserted' / 'journals:deleted' and
// 'sync:applied' event handlers instead (see initBridge). With `refresh`,
// already-mapped days are re-embedded to heal pre-#1080 staleness.
{
for await (const record of iterateJournals()) {
const key = bridgeKey('journals', record.id);
// Already-mapped days are skipped in both real and dry-run modes so
// dry-run stats match actual-run stats (rather than claiming to
// re-sync every day every time) — unless refresh is forcing a re-embed.
if (onlyMissing) {
if (isEmbedded(key)) { stats.skipped += 1; continue; }
} else if (map[key] && !refresh) {
stats.skipped += 1;
continue;
}
if (dryRun) {
console.log(`🧠🔗 [dry-run] Would sync journals/${record.date}`);
stats.synced += 1;
continue;
}
const memoryId = await syncBrainRecord('journals', record, deferredMapSave).catch((err) => {
console.error(`❌ Failed to sync journals/${record.date}: ${err.message}`);
stats.errors += 1;
return null;
});
if (memoryId) stats.synced += 1;
}
}
// JSONL stores (digests, reviews)
for (const type of JSONL_TYPES) {
const records = await readJsonlStore(type);
for (const record of records) {
const key = bridgeKey(type, record.id);
if (onlyMissing) {
if (isEmbedded(key)) { stats.skipped++; continue; }
} else if (map[key] && !dryRun && !refresh) {
stats.skipped++;
continue;
}
if (dryRun) {
console.log(`🧠🔗 [dry-run] Would sync ${type}/${record.id}`);
stats.synced++;
continue;
}
const memoryId = await syncBrainRecord(type, record, deferredMapSave).catch(err => {
console.error(`❌ Failed to sync ${type}/${record.id}: ${err.message}`);
stats.errors++;
return null;
});
if (memoryId) stats.synced++;
}
}
// Refresh-mode reconcile (issue #1080 recovery): the live-record walks above
// re-embed edited records, but they iterate ONLY live records — getAll /
// listJournals strip tombstones and archived entities are skipped — so a
// record DELETED or ARCHIVED on a peer BEFORE this fix shipped leaves an
// orphaned bridge-map entry whose memory copy stays active and searchable.
// The going-forward sync:applied path archives those, but it can't reach the
// pre-fix backlog. Walk the bridge map and archive any mapped entry whose
// canonical record no longer resolves as live. Only for deletable, bridge-
// mirrored stores: digests/reviews are append-only (never deleted) and
// links/buckets/inbox aren't mirrored, so neither can orphan a memory entry.
if (refresh && !dryRun) {
const reconcilableTypes = new Set(RECONCILABLE_TYPES);
for (const mapKey of Object.keys(map)) {
const sep = mapKey.indexOf(':');
if (sep === -1) continue;
const type = mapKey.slice(0, sep);
const id = mapKey.slice(sep + 1);
if (!reconcilableTypes.has(type)) continue;
const record = type === 'journals'
? await getJournal(id)
: await brainStorage.getById(type, id);
if (record && !record.archived) continue; // still live — already re-embedded above
const archived = await archiveMappedMemory(type, id).catch(err => {
console.error(`❌ Failed to archive stale ${type}/${id}: ${err.message}`);
stats.errors++;
return false;
});
if (archived) stats.archived++;
}
}
return stats;
} finally {
if (bridgeMapChanged) {
await saveBridgeMap().catch((err) => {
console.error(`❌ Failed to save brain bridge map after bulk sync: ${err.message}`);
stats.errors++;
});
}
}
}
/**
* Count how many active brain records lack an embedding — the headline number
* for the "N memories missing embeddings · Embed missing" affordance on the
* graph. A record is "missing" when it's unmapped OR its mapped memory has a
* NULL embedding — scoped to ACTIVE memories, since getMemoryIdsMissingEmbedding
* only reports active rows (an archived/non-active mapped memory is treated as
* embedded and is not a backfill target). Read-only; walks the same stores as
* the onlyMissing sync so the count matches what "Embed missing" would process.
*
* Only IDENTITY is needed to tally — each store's live ids, checked against the
* bridge map — so the entity/journal walks go through `brainStorage.listLiveIds`
* rather than `getAll` / `listJournals` (issue #3508). Those loaded and parsed
* every record body across every collection, then threw all of it away but the
* `id` and `archived` fields, turning a status poll into hundreds of disk reads.
* `listLiveIds` reads only the ids it has never resolved.
*/
export async function getEmbeddingCoverage() {
const map = await memory.getBrainMemoryLinks(
await loadBridgeMap({ readOnly: true }), { readOnly: true });
const missingMemIds = await memory.getMemoryIdsMissingEmbedding().catch(() => new Set());
const isEmbedded = makeEmbeddedChecker(map, missingMemIds);
let total = 0;
let missing = 0;
const tally = (key) => { total += 1; if (!isEmbedded(key)) missing += 1; };
// Entity stores plus the daily log — the journal record id IS its date, the
// same key syncAllBrainData bridges under. `listLiveIds` already drops
// tombstones and archived records, and it has no page cap (the old
// `listJournals({ limit: 10000 })` silently stopped counting past 10k days).
for (const type of RECONCILABLE_TYPES) {
for (const id of await brainStorage.listLiveIds(type)) tally(bridgeKey(type, id));
}
// digests/reviews are append-only JSONL: one whole-file read each, behind
// brainStorage's own cache — not a per-record disk walk, so they stay as-is.
for (const type of JSONL_TYPES) {
const records = await readJsonlStore(type);
for (const record of records) tally(bridgeKey(type, record.id));
}
return { total, missing };
}
// ─── Synced-in record resync (issue #1080) ──────────────────────────────────
// Peer-synced brain writes go through brainStorage.applyRemoteRecord, which is
// deliberately event-silent (no brainEvents) to prevent cross-peer echo loops
// (#1077). That silence means the per-record bridge listeners below never fire
// for synced-in records, so a record created/edited/deleted on peer A was never
// re-vectorized into peer B's memory index — it stayed stale and searchable
// until a full bulk resync (which itself SKIPPED already-mapped records). The
// sync apply paths now emit a LOCAL-ONLY 'sync:applied' signal (it never feeds
// the sync log, so no echo) carrying the {type, id} of every applied change;
// we re-read each record's canonical state and re-embed or archive accordingly.
const RESYNC_DEBOUNCE_MS = 250;
const pendingResync = new Map(); // bridgeKey → { type, id, hardDelete } (dedups repeated touches)
let resyncTimer = null;
let resyncFlushing = false; // single-flight guard so two flushes can't overlap
// Journals re-embed on a much longer window than the other brain types, per
// day. The daily-log editor autosaves while the user types, so 'journals:upserted'
// now fires every couple of seconds for the SAME day — where it used to fire
// once per explicit Save. Re-embedding per event would mean an embedding call
// per typing pause, and — once the day's entry outgrows the embedding char
// budget — a full LLM summarization per event too (see generateMemoryEmbedding's
// over-budget branch), which is exactly the kind of unannounced provider traffic
// the AI policy rules out. Coalescing per day collapses a whole writing session
// into one re-embed after the user settles; resyncBrainRecord re-reads the
// canonical entry at flush time, so the trailing flush always embeds the newest
// content — and embedding once the sentence is finished beats embedding it
// half-typed 100 times.
const JOURNAL_RESYNC_DEBOUNCE_MS = 30_000;
const journalResyncTimers = new Map(); // journal id (the ISO date) → timer
// Durable mirror of every resync that has been triggered but not yet settled:
// queued entries, the batch a flush is working through, and journal days still
// inside their debounce window. Persisted so a restart replays the work instead
// of leaving stale embeddings or still-searchable deleted records behind.
const durablePending = new Map(); // bridgeKey → { type, id, hardDelete }
let pendingWriteTail = Promise.resolve();
let pendingWriteQueued = false;
function persistPending() {
// Calls made while a write is queued or running coalesce into one snapshot
// taken when the write actually starts.
if (pendingWriteQueued) return;
pendingWriteQueued = true;
pendingWriteTail = pendingWriteTail.then(async () => {
pendingWriteQueued = false;
const dir = dirname(PENDING_RESYNC_PATH);
if (!existsSync(dir)) await ensureDir(dir);
await atomicWrite(PENDING_RESYNC_PATH, { entries: [...durablePending.values()] });
}).catch((err) => {
console.error(`❌ Brain bridge pending-resync persist failed: ${err.message}`);
});
}
// An entry is only finished once nothing newer is waiting for the same key.
function settleDurable(key, journalId) {
if (pendingResync.has(key) || (journalId && journalResyncTimers.has(journalId))) return;
if (durablePending.delete(key)) persistPending();
}
/** Resolves once every queued pending-set write has reached disk. Exported for tests. */
export const flushPendingResyncWrites = () => pendingWriteTail;
/**
* Replay resyncs a previous process queued but never finished. Re-queues them
* through the normal path (sticky hard-delete included); an absent or empty
* file makes no call at all.
*/
export async function replayPersistedResync() {
const raw = await readFile(PENDING_RESYNC_PATH, 'utf-8').catch((err) => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (raw === null) return 0;
let entries;
try {
entries = JSON.parse(raw)?.entries;
} catch {
console.warn('⚠️ Corrupt brain pending-resync file; ignoring');
return 0;
}
if (!Array.isArray(entries) || entries.length === 0) return 0;
const valid = entries.filter((entry) => entry && typeof entry.id === 'string' && TYPE_MAP[entry.type]);
queueResync(valid.map(({ type, id, hardDelete }) => ({ type, id, hardDelete: hardDelete === true })));
console.log(`🧠🔗 Replaying ${valid.length} brain resync(s) interrupted by a restart`);
return valid.length;
}
/**
* Archive the memory entry mapped to a brain record (if any). Used when the
* canonical record is gone (tombstoned), archived, or otherwise should no
* longer be searchable.
*/
async function archiveMappedMemory(brainType, id) {
const map = await loadBridgeMap();
const memoryId = map[bridgeKey(brainType, id)];
if (!memoryId) return false;
await memory.updateMemory(memoryId, { status: 'archived' });
console.log(`🧠🔗 Archived brain→memory: ${brainType}/${id} → ${memoryId}`);
return true;
}
/**
* Hard-delete the memory entry mapped to a brain record (issue #1318). Used for
* a GENUINE local user delete: archiving the row would hide it from search but
* leave the row + its 768-dim embedding in Postgres forever (e.g. ~1,482 dead
* vectors after a cleared ChatGPT import). A hard delete drops the row and its
* embedding and removes the bridge-map entry so a later create can't collide
* with a stale mapping. NOT used for synced-in deletes — those stay soft-
* archived (see resyncBrainRecord) so a peer un-delete can resurrect them.
*/
async function hardDeleteMappedMemory(brainType, id) {
const map = await loadBridgeMap();
const key = bridgeKey(brainType, id);
const memoryId = map[key];
if (!memoryId) return false;
await memory.purgeMemory(memoryId);
delete map[key];
await saveBridgeMap();
console.log(`🧠🔗 Hard-deleted brain→memory: ${brainType}/${id} → ${memoryId}`);
return true;
}
/**
* Re-vectorize (or archive) a single brain record from its CANONICAL stored
* state — used after a peer sync applies a change. Reading the store rather
* than trusting an event payload makes this self-healing and order-independent:
* whatever the final converged state is (live record, archived, or tombstoned),
* the memory copy is brought into line.
* - record present & not archived → upsert + re-embed (syncBrainRecord)
* - record absent (tombstoned) → hard-delete on a genuine local delete
* (hardDelete), else archive the mapped memory entry (synced/recoverable)
* - record present but archived → archive (always soft — recoverable)
* - type not mirrored by the bridge (links/buckets/inbox) → no-op
*
* @param {object} [opts]
* @param {boolean} [opts.hardDelete=false] The trigger was a real local user
* delete (a `{type}:deleted` event), so a tombstoned record should hard-prune
* the mapped memory row + embedding rather than soft-archive it (issue #1318).
* Synced-in deletes (sync:applied) pass false so they stay resurrectable.
*/
export async function resyncBrainRecord(brainType, id, { hardDelete = false } = {}) {
if (!id || !TYPE_MAP[brainType]) return;
// getById returns null for tombstones (deleted records), so a synced-in
// delete naturally lands in the archive/hard-delete branch.
const record = brainType === 'journals'
? await getJournal(id)
: JSONL_TYPES.includes(brainType)
? (await readJsonlStore(brainType)).find(record => record.id === id)
: await brainStorage.getById(brainType, id);
if (!record) {
// Truly gone (tombstoned). A genuine local user delete hard-prunes the row;
// a sync-driven delete stays soft so a peer un-delete can resurrect it.
if (hardDelete) await hardDeleteMappedMemory(brainType, id);
else await archiveMappedMemory(brainType, id);
return;
}
if (record.archived) {
// Present but archived (not deleted) — always soft, never hard-delete, even
// if the queued trigger carried a hardDelete intent.
await archiveMappedMemory(brainType, id);
return;
}
await syncBrainRecord(brainType, record);
}
/**
* Drain the pending resync queue sequentially. Single-flight + re-draining:
* only one flush runs at a time (the `resyncFlushing` guard), and it loops
* until the queue is empty so records enqueued mid-flush are still processed —
* without a second flush running concurrently. Sequential (not parallel) so a
* large catch-up sync doesn't fire hundreds of concurrent embedding calls and
* saturate the embedding backend; single-flight also prevents two flushes from
* both creating a memory for the same not-yet-mapped record (duplicate entries).
* Exported for deterministic flushing in tests.
*/
export async function flushPendingResync() {
if (resyncTimer) { clearTimeout(resyncTimer); resyncTimer = null; }
if (resyncFlushing) return; // an in-flight flush will pick up newly-queued items
resyncFlushing = true;
try {
// The queue only grows during an `await` below; the loop re-checks size at
// the top of each pass, and the gap between the final size===0 check and
// clearing `resyncFlushing` contains no await — so nothing can be stranded.
while (pendingResync.size > 0) {
const batch = [...pendingResync.values()];
pendingResync.clear();
for (const { type, id, hardDelete } of batch) {
await resyncBrainRecord(type, id, { hardDelete }).catch((err) => {
console.error(`❌ Brain bridge resync failed for ${type}/${id}: ${err.message}`);
});
settleDurable(bridgeKey(type, id), type === 'journals' ? id : null);
}
}
} finally {
resyncFlushing = false;
}
}
/**
* Queue applied-change records for a debounced resync. Records touching a type
* the bridge doesn't mirror are dropped here so they never schedule a flush.
* No timer is armed while a flush is in flight — that flush re-drains the queue
* itself, so a second concurrent flush can never start.
*/
export function queueResync(records) {
if (!Array.isArray(records)) return;
let queued = false;
for (const { type, id, hardDelete } of records) {
if (!id || !TYPE_MAP[type]) continue;
const key = bridgeKey(type, id);
// Sticky hard-delete on coalesce: once a real local `{type}:deleted` marks a
// key for a hard prune, a later touch for the same key before the flush — a
// concurrent `sync:applied` carrying no flag, say — must NOT downgrade it to
// a soft archive, or the dead embedding lingers. Safe to keep sticky because
// resyncBrainRecord's record-present branch upserts a resurrected record
// regardless of the flag, so a true intent can never hard-delete a live row.
const prevHardDelete = pendingResync.get(key)?.hardDelete;
const entry = { type, id, hardDelete: !!hardDelete || !!prevHardDelete };
pendingResync.set(key, entry);
durablePending.set(key, entry);
queued = true;
}
if (queued) persistPending();
if (pendingResync.size === 0 || resyncTimer || resyncFlushing) return;
resyncTimer = setTimeout(() => {
flushPendingResync().catch((err) => {
console.error(`❌ Brain bridge resync flush failed: ${err.message}`);
});
}, RESYNC_DEBOUNCE_MS);
// Don't keep the event loop alive just for a pending resync flush.
if (typeof resyncTimer.unref === 'function') resyncTimer.unref();
}
// ─── Event Handlers ─────────────────────────────────────────────────────────
// Entity stores emit "{type}:upserted" / "{type}:deleted" with the single
// affected record. Both are routed through the debounced+sequential
// queueResync path (not handled inline) — see initBridge for why.
// JSONL stores still emit "{type}:added" with a single record.
function handleJsonlAdded(brainType, record) {
if (!record?.id) return;
queueResync([{ type: brainType, id: record.id }]);
}
// ─── Init ───────────────────────────────────────────────────────────────────
/**
* Initialize the brain→memory bridge.
* Attaches event listeners to brainEvents so new/updated brain records
* are automatically mirrored to the CoS memory system.
*/
export function initBridge() {
// Entity store changes — route through the debounced + sequential queueResync
// path rather than embedding inline per event. A single create/update already
// emits one per-record event (not a store-wide fan-out), but a BULK producer —
// e.g. the ChatGPT import creating ~1,400 conversations back-to-back — fires
// that many `:upserted` events in a tight loop. Handling each inline would
// launch ~1,400 concurrent embedding calls + Postgres writes, saturating the
// embedding backend (most time out → records persist with a NULL embedding,
// silently unsearchable) and exhausting the PG pool. queueResync coalesces the
// burst (250ms debounce + dedup) and drains it sequentially — the same
// protection the peer-sync `sync:applied` path already relies on. It also
// re-reads each record's CANONICAL state (resyncBrainRecord), so the archived/
// deleted/tombstoned branches are handled there uniformly and an event whose
// payload is already stale by flush time still converges correctly.
for (const type of BRIDGED_ENTITY_TYPES) {
brainEvents.on(`${type}:upserted`, ({ id }) => queueResync([{ type, id }]));
// A `:deleted` event is a genuine LOCAL user delete (remove() emits it;
// applyRemoteRecord is event-silent), so hard-prune the mapped vector row
// rather than soft-archive it forever (issue #1318).
brainEvents.on(`${type}:deleted`, ({ id }) => queueResync([{ type, id, hardDelete: true }]));
}
// JSONL appends (digests, reviews)
for (const type of JSONL_TYPES) {
brainEvents.on(`${type}:added`, (record) => handleJsonlAdded(type, record));
}
// Daily log — per-entry events so a single append doesn't re-embed every
// day of the user's history. (An earlier version listened for the
// store-wide 'journals:changed' event, which would trigger O(totalDays)
// embedding calls per dictation segment and saturate the embedding
// backend.) appendJournal/setJournalContent fire 'journals:upserted' with
// the single affected entry; deleteJournal fires 'journals:deleted'.
// Debounced per day because the editor autosaves — see queueJournalResync.
brainEvents.on('journals:upserted', ({ entry }) => {
if (entry?.id) queueJournalResync(entry.id);
});
// handleJournalDeleted is async and awaits loadBridgeMap(); wrap the call
// in a .catch so a rejection becomes a logged error instead of an
// unhandled-rejection warning (or a process crash under strict modes).
brainEvents.on('journals:deleted', ({ entry }) => {
// Drop any debounced re-embed for the day first — the hard-delete below
// prunes the mapped memory, and a later flush would only re-read a tombstone.
if (entry?.id) cancelPendingJournalResync(entry.id);
handleJournalDeleted(entry).catch((err) => {
console.error(`❌ Brain bridge delete sync failed for journals/${entry?.id}: ${err.message}`);
});
});
// Peer-synced records (issue #1080) — applyRemoteRecord is event-silent to
// prevent echo loops, so the per-record listeners above never fire for
// synced-in changes. The sync apply paths emit this local-only signal with
// the touched {type, id}s; we re-read each from the store and re-embed or
// archive. Debounced + sequential so a large catch-up doesn't saturate the
// embedding backend.
brainEvents.on('sync:applied', ({ records } = {}) => queueResync(records));
console.log('🧠🔗 Brain→Memory bridge initialized');
}
/**
* Coalesce repeated saves of one day into a single re-embed, then hand off to
* the shared queueResync path (dedup + sequential drain + canonical re-read).
* Trailing-edge per day: each save pushes the flush out, so a writing session
* costs one re-embed once the user stops. See JOURNAL_RESYNC_DEBOUNCE_MS.
*/
function queueJournalResync(id) {
clearTimeout(journalResyncTimers.get(id));
const key = bridgeKey('journals', id);
if (!durablePending.has(key)) {
durablePending.set(key, { type: 'journals', id, hardDelete: false });
persistPending();
}
const timer = setTimeout(() => {
journalResyncTimers.delete(id);
queueResync([{ type: 'journals', id }]);
}, JOURNAL_RESYNC_DEBOUNCE_MS);
// Don't hold the event loop open for a pending journal re-embed.
if (typeof timer.unref === 'function') timer.unref();
journalResyncTimers.set(id, timer);
}
/**
* Drop a day's pending re-embed. Called when the day is deleted: the delete
* path prunes the mapped memory immediately, so letting the debounce fire
* afterwards would just re-read a tombstone. Also exported for tests.
*/
export function cancelPendingJournalResync(id) {
clearTimeout(journalResyncTimers.get(id));
journalResyncTimers.delete(id);
settleDurable(bridgeKey('journals', id), id);
}
async function handleJournalDeleted(entry) {
if (!entry?.id) return;
// deleteJournal (the local user action) is the only emitter of
// 'journals:deleted'; synced-in journal deletes flow through sync:applied →
// resyncBrainRecord and stay soft-archived. So a local journal delete hard-
// prunes the mapped memory row + embedding (issue #1318).
await hardDeleteMappedMemory('journals', entry.id);
}