Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

38.2 KB
Raw Blame
/**
 * Brain → CoS Memory Bridge
 *
 * Mirrors brain records (projects, ideas, admin, memories/journal, digests, reviews, people,
 * songs) into the CoS memory system so agents can semantically search user-captured thoughts.
 *
 * Brain JSON files remain the operational data store for the brain UI.
 * This bridge creates/updates corresponding entries in the memories table
 * tagged with sourceAppId='brain'.
 */

import { readFile } from 'fs/promises';
import { existsSync } from 'fs';
import { join, dirname } from 'path';
import { brainEvents } from './brainStorage.js';
import * as brainStorage from './brainStorage.js';
import * as memory from './memoryBackend.js';
import * as embeddings from './memoryEmbeddings.js';
import { atomicWrite, ensureDir, PATHS } from '../lib/fileUtils.js';
import { iterateJournals, getJournal } from './brainJournal.js';

const BRIDGE_MAP_PATH = join(PATHS.brain, 'memory-bridge-map.json');
const PENDING_RESYNC_PATH = join(PATHS.brain, 'memory-resync-pending.json');

// brainType → { memoryType, category }
const TYPE_MAP = {
  people:   { type: 'context',     category: 'people' },
  projects: { type: 'fact',        category: 'project' },
  ideas:    { type: 'observation', category: 'ideas' },
  admin:    { type: 'fact',        category: 'admin' },
  memories: { type: 'observation', category: 'personal' },
  digests:  { type: 'context',     category: 'digest' },
  reviews:  { type: 'context',     category: 'review' },
  journals: { type: 'observation', category: 'daily-log' },
  songs:    { type: 'context',     category: 'songbook' }
};

// Append-only JSONL stores — no per-record events, no deletes, walked with
// their own getters rather than brainStorage.getAll.
const JSONL_TYPES = ['digests', 'reviews'];

// Explicit whole-store read; UI getters retain their latest-page default.
const readJsonlStore = (type) =>
  (type === 'digests' ? brainStorage.getDigests : brainStorage.getReviews)(Infinity);

// The id-keyed entity stores the bridge mirrors. DERIVED from TYPE_MAP (minus
// the JSONL stores and the Daily Log, both of which have their own walk) so
// adding a type to the map above automatically enrolls it in the bulk sync, the
// embedding-coverage tally, the refresh reconcile, and the live event
// listeners. Those four sites used to restate the list, and a new type wired
// into TYPE_MAP but missed in one of them would sync but never re-embed (or
// count, or archive) — silently.
const BRIDGED_ENTITY_TYPES = Object.keys(TYPE_MAP)
  .filter((type) => type !== 'journals' && !JSONL_TYPES.includes(type));

// Entity stores plus the Daily Log: everything keyed by a canonical record the
// bridge can re-read, which is what the coverage tally and the refresh
// reconcile walk. (JSONL stores are append-only, so they can never orphan a
// mapped memory entry.)
const RECONCILABLE_TYPES = [...BRIDGED_ENTITY_TYPES, 'journals'];

// ─── Bridge Map ─────────────────────────────────────────────────────────────
// Maps "brainType:brainId" → memoryId so updates hit the same memory entry

let bridgeMap = null;
let bridgeMapLoading = null;
let mapWriteTail = Promise.resolve();

export async function loadBridgeMap({ readOnly = false } = {}) {
  if (bridgeMap) return bridgeMap;
  if (bridgeMapLoading) return bridgeMapLoading;
  const load = async () => {
    let legacyMap = {};
    let rebuild = false;
    const raw = await readFile(BRIDGE_MAP_PATH, 'utf-8').catch(err => {
      if (err.code === 'ENOENT') return null;
      throw err;
    });
    if (raw !== null) {
      try {
        legacyMap = JSON.parse(raw);
        if (!legacyMap || Array.isArray(legacyMap) || typeof legacyMap !== 'object') {
          throw new Error('Invalid bridge map shape');
        }
        legacyMap = Object.fromEntries(Object.entries(legacyMap)
          .filter(([, id]) => typeof id === 'string' && id));
      } catch {
        console.warn('⚠️ Corrupt brain bridge cache; rebuilding from durable memory links');
        legacyMap = {};
        rebuild = true;
      }
    }
    // Backfill old installs before trusting the cache. DB links win over stale
    // file snapshots; missing rows are excluded so bulk catch-up can heal them.
    const map = await memory.getBrainMemoryLinks(legacyMap, { readOnly, ...(rebuild ? { recover: true } : {}) });
    if (!readOnly) {
      bridgeMap = map;
      if (rebuild) await saveBridgeMap();
    }
    return map;
  };
  // Read-only previews neither migrate links nor cache an unmigrated map.
  if (readOnly) return load();
  bridgeMapLoading = load();
  try {
    return await bridgeMapLoading;
  } finally {
    bridgeMapLoading = null;
  }
}

function saveBridgeMap() {
  // Snapshot only after the preceding rename completes. All writers, including
  // bulk catch-up and delete handlers, share this tail.
  const write = mapWriteTail.then(async () => {
    const dir = dirname(BRIDGE_MAP_PATH);
    if (!existsSync(dir)) await ensureDir(dir);
    await atomicWrite(BRIDGE_MAP_PATH, bridgeMap);
  });
  mapWriteTail = write.catch(() => {});
  return write;
}

export function bridgeKey(brainType, brainId) {
  return `${brainType}:${brainId}`;
}

// ─── Content Composers ─────────────────────────────────────────────────────
// Each brain type has different fields; compose a single content string for memory storage.

function composePeopleContent(r) {
  const parts = [`Person: ${r.name}`];
  if (r.context) parts.push(r.context);
  if (r.followUps?.length) parts.push(`Follow-ups: ${r.followUps.join('; ')}`);
  return parts.join('\n');
}

function composeProjectContent(r) {
  const parts = [`Project: ${r.name}`, `Status: ${r.status}`];
  if (r.nextAction) parts.push(`Next action: ${r.nextAction}`);
  if (r.notes) parts.push(r.notes);
  return parts.join('\n');
}

function composeIdeaContent(r) {
  const parts = [`Idea: ${r.title}`];
  if (r.oneLiner) parts.push(r.oneLiner);
  if (r.notes) parts.push(r.notes);
  return parts.join('\n');
}

function composeAdminContent(r) {
  const parts = [`Admin: ${r.title}`, `Status: ${r.status}`];
  if (r.dueDate) parts.push(`Due: ${r.dueDate}`);
  if (r.nextAction) parts.push(`Next action: ${r.nextAction}`);
  if (r.notes) parts.push(r.notes);
  return parts.join('\n');
}

function composeJournalContent(r) {
  const parts = [];
  if (r.title) parts.push(r.title);
  if (r.content) parts.push(r.content);
  if (r.mood) parts.push(`Mood: ${r.mood}`);
  return parts.join('\n');
}

function composeDigestContent(r) {
  const parts = [];
  if (r.digestText) parts.push(r.digestText);
  if (r.topActions?.length) parts.push(`Top actions: ${r.topActions.join('; ')}`);
  if (r.stuckThing) parts.push(`Stuck on: ${r.stuckThing}`);
  if (r.smallWin) parts.push(`Small win: ${r.smallWin}`);
  return parts.join('\n');
}

function composeReviewContent(r) {
  const parts = [];
  if (r.reviewText) parts.push(r.reviewText);
  if (r.whatHappened?.length) parts.push(`What happened: ${r.whatHappened.join('; ')}`);
  if (r.biggestOpenLoops?.length) parts.push(`Open loops: ${r.biggestOpenLoops.join('; ')}`);
  if (r.suggestedActionsNextWeek?.length) parts.push(`Suggested actions: ${r.suggestedActionsNextWeek.join('; ')}`);
  if (r.recurringTheme) parts.push(`Recurring theme: ${r.recurringTheme}`);
  return parts.join('\n');
}

function composeDailyLogContent(r) {
  const parts = [`Daily Log — ${r.date}`];
  if (r.content) parts.push(r.content);
  return parts.join('\n');
}

// SongBook repertoire entry (brain type `songs`). The sheet body
// (`content.text`, up to 200k chars of tab/ChordPro notation) is deliberately
// LEFT OUT: it is positional notation, not prose, so it adds no semantic signal
// to an embedding while blowing the embedder's char budget — one long song
// would push the record into generateMemoryEmbedding's over-budget LLM
// summarization branch for nothing. What the user actually searches for is the
// song's identity and their own notes about it.
function composeSongContent(r) {
  const parts = [`Song: ${r.title}`];
  if (r.artist) parts.push(`Artist: ${r.artist}`);
  if (r.instrument) parts.push(`Instrument: ${r.instrument}`);
  if (r.stage) parts.push(`Stage: ${r.stage}`);
  const setup = [
    r.key && `Key: ${r.key}`,
    r.tuning && `Tuning: ${r.tuning}`,
    // `capo: 0` is the schema default and means "no capo" — not "capo at fret
    // 0" — so it is legitimately omitted rather than rendered as `Capo: 0`.
    // Explicit `> 0` so the intent doesn't read as an accidental truthiness bug.
    r.capo > 0 ? `Capo: ${r.capo}` : null
  ].filter(Boolean);
  if (setup.length) parts.push(setup.join(' · '));
  if (r.notes) parts.push(r.notes);
  return parts.join('\n');
}

export const CONTENT_COMPOSERS = {
  people: composePeopleContent,
  projects: composeProjectContent,
  ideas: composeIdeaContent,
  admin: composeAdminContent,
  memories: composeJournalContent,
  digests: composeDigestContent,
  reviews: composeReviewContent,
  journals: composeDailyLogContent,
  songs: composeSongContent
};

// ─── Core Mapping ───────────────────────────────────────────────────────────

/**
 * Convert a brain record into a memory-create payload.
 */
export function brainRecordToMemory(brainType, record) {
  const mapping = TYPE_MAP[brainType];
  if (!mapping) return null;

  const composer = CONTENT_COMPOSERS[brainType];
  const content = composer(record);
  if (!content?.trim()) return null;

  const summary = content.length > 200 ? content.substring(0, 197) + '...' : content;
  const recordTags = record.tags || [];
  const tags = [...new Set([...recordTags, 'brain', brainType])];

  return {
    type: mapping.type,
    content,
    summary,
    category: mapping.category,
    tags,
    confidence: 1.0,
    importance: 0.6,
    sourceAppId: 'brain',
    sourceAgentId: 'brain-bridge'
  };
}

// ─── Sync ───────────────────────────────────────────────────────────────────

/**
 * Upsert a single brain record into the memory system.
 * Creates a new memory or updates the existing one based on the bridge map.
 */
export async function syncBrainRecord(brainType, record, { deferMapSave = false, onMapChanged } = {}) {
  const memoryData = brainRecordToMemory(brainType, record);
  if (!memoryData) return null;

  const map = await loadBridgeMap();
  const key = bridgeKey(brainType, record.id);
  const existingMemoryId = map[key];

  // Generate embedding. Pass the source hints (NOT persisted on memoryData —
  // the memory schema has no such columns) so the embedder can summarize the
  // FULL archived transcript for an over-budget chatgpt-import record rather
  // than the already-capped content preview.
  const embedding = await embeddings
    .generateMemoryEmbedding(memoryData, { source: record.source, sourceRef: record.sourceRef })
    .catch(() => null);

  const saved = await memory.upsertBrainMemory(key, memoryData, embedding, existingMemoryId);
  if (map[key] !== saved.id) {
    map[key] = saved.id;
    onMapChanged?.();
    if (!deferMapSave) await saveBridgeMap();
  }
  console.log(`🧠🔗 Synced brain→memory: ${brainType}/${record.id} → ${saved.id}`);
  return saved.id;
}

/**
 * Bulk sync all existing brain data into the memory system.
 * Used for initial migration and catch-up.
 *
 * @param {object}  [opts]
 * @param {boolean} [opts.dryRun=false]  Report what would sync without writing.
 * @param {boolean} [opts.refresh=false] Re-embed records that ALREADY have a
 *   bridge-map entry instead of skipping them. This is the recovery path for
 *   records that diverged before the per-record sync:applied signal existed
 *   (issue #1080): a peer's edit/delete of an already-mapped record used to
 *   leave the local memory copy stale and searchable forever, and a normal
 *   bulk sync skipped it precisely because it was mapped. `refresh` forces a
 *   re-embed of every live record so a one-time catch-up heals the staleness.
 *   In refresh mode a reconcile pass also archives memory entries whose brain
 *   record was deleted/archived on a peer before this fix (orphaned map keys
 *   the live-record walks can't reach) — reported as `archived`.
 * @returns {Promise<{synced:number, skipped:number, errors:number, archived:number}>}
 */
// True when a record is already embedded: mapped AND its memory has a vector.
// Shared by the onlyMissing sync path and the read-only coverage tally.
const makeEmbeddedChecker = (map, missingMemIds) => (key) => {
  const memId = map[key];
  return !!memId && !missingMemIds.has(memId);
};

export async function syncAllBrainData({ dryRun = false, refresh = false, onlyMissing = false } = {}) {
  // Revalidate cached ids once per bulk walk: a memory can be purged after
  // this process loaded the cache. A stale key must not count as embedded.
  const map = await memory.getBrainMemoryLinks(
    await loadBridgeMap({ readOnly: dryRun }), { readOnly: dryRun });
  const stats = { synced: 0, skipped: 0, errors: 0, archived: 0 };
  let bridgeMapChanged = false;
  const deferredMapSave = {
    deferMapSave: true,
    onMapChanged: () => { bridgeMapChanged = true; },
  };

  try {

  // `onlyMissing` is the cheap, targeted backfill: embed only records that lack
  // an embedding (unmapped, or mapped to a memory whose vector is NULL because
  // generation failed) and skip everything already embedded. Unlike `refresh`
  // it never re-embeds healthy records, so it's safe to run without a warning.
  const missingMemIds = onlyMissing
    ? await memory.getMemoryIdsMissingEmbedding().catch(() => new Set())
    : null;
  const isEmbedded = makeEmbeddedChecker(map, missingMemIds);

  // Entity stores (JSON-based)
  for (const type of BRIDGED_ENTITY_TYPES) {
    const records = await brainStorage.getAll(type);
    for (const record of records) {
      // Skip archived records
      if (record.archived) {
        stats.skipped++;
        continue;
      }
      const key = bridgeKey(type, record.id);
      if (onlyMissing) {
        if (isEmbedded(key)) { stats.skipped++; continue; }
      } else if (map[key] && !dryRun && !refresh) {
        stats.skipped++;
        continue;
      }
      if (dryRun) {
        console.log(`🧠🔗 [dry-run] Would sync ${type}/${record.id}: ${record.name || record.title || '(untitled)'}`);
        stats.synced++;
        continue;
      }
      const memoryId = await syncBrainRecord(type, record, deferredMapSave).catch(err => {
        console.error(`❌ Failed to sync ${type}/${record.id}: ${err.message}`);
        stats.errors++;
        return null;
      });
      if (memoryId) stats.synced++;
    }
  }

  // Daily log entries — one memory per day. Without `refresh`, already-mapped
  // days are skipped (initial/backfill import only) and content updates /
  // deletions flow through the 'journals:upserted' / 'journals:deleted' and
  // 'sync:applied' event handlers instead (see initBridge). With `refresh`,
  // already-mapped days are re-embedded to heal pre-#1080 staleness.
  {
    for await (const record of iterateJournals()) {
      const key = bridgeKey('journals', record.id);
      // Already-mapped days are skipped in both real and dry-run modes so
      // dry-run stats match actual-run stats (rather than claiming to
      // re-sync every day every time) — unless refresh is forcing a re-embed.
      if (onlyMissing) {
        if (isEmbedded(key)) { stats.skipped += 1; continue; }
      } else if (map[key] && !refresh) {
        stats.skipped += 1;
        continue;
      }
      if (dryRun) {
        console.log(`🧠🔗 [dry-run] Would sync journals/${record.date}`);
        stats.synced += 1;
        continue;
      }
      const memoryId = await syncBrainRecord('journals', record, deferredMapSave).catch((err) => {
        console.error(`❌ Failed to sync journals/${record.date}: ${err.message}`);
        stats.errors += 1;
        return null;
      });
      if (memoryId) stats.synced += 1;
    }
  }

  // JSONL stores (digests, reviews)
  for (const type of JSONL_TYPES) {
    const records = await readJsonlStore(type);
    for (const record of records) {
      const key = bridgeKey(type, record.id);
      if (onlyMissing) {
        if (isEmbedded(key)) { stats.skipped++; continue; }
      } else if (map[key] && !dryRun && !refresh) {
        stats.skipped++;
        continue;
      }
      if (dryRun) {
        console.log(`🧠🔗 [dry-run] Would sync ${type}/${record.id}`);
        stats.synced++;
        continue;
      }
      const memoryId = await syncBrainRecord(type, record, deferredMapSave).catch(err => {
        console.error(`❌ Failed to sync ${type}/${record.id}: ${err.message}`);
        stats.errors++;
        return null;
      });
      if (memoryId) stats.synced++;
    }
  }

  // Refresh-mode reconcile (issue #1080 recovery): the live-record walks above
  // re-embed edited records, but they iterate ONLY live records — getAll /
  // listJournals strip tombstones and archived entities are skipped — so a
  // record DELETED or ARCHIVED on a peer BEFORE this fix shipped leaves an
  // orphaned bridge-map entry whose memory copy stays active and searchable.
  // The going-forward sync:applied path archives those, but it can't reach the
  // pre-fix backlog. Walk the bridge map and archive any mapped entry whose
  // canonical record no longer resolves as live. Only for deletable, bridge-
  // mirrored stores: digests/reviews are append-only (never deleted) and
  // links/buckets/inbox aren't mirrored, so neither can orphan a memory entry.
  if (refresh && !dryRun) {
    const reconcilableTypes = new Set(RECONCILABLE_TYPES);
    for (const mapKey of Object.keys(map)) {
      const sep = mapKey.indexOf(':');
      if (sep === -1) continue;
      const type = mapKey.slice(0, sep);
      const id = mapKey.slice(sep + 1);
      if (!reconcilableTypes.has(type)) continue;
      const record = type === 'journals'
        ? await getJournal(id)
        : await brainStorage.getById(type, id);
      if (record && !record.archived) continue; // still live — already re-embedded above
      const archived = await archiveMappedMemory(type, id).catch(err => {
        console.error(`❌ Failed to archive stale ${type}/${id}: ${err.message}`);
        stats.errors++;
        return false;
      });
      if (archived) stats.archived++;
    }
  }

    return stats;
  } finally {
    if (bridgeMapChanged) {
      await saveBridgeMap().catch((err) => {
        console.error(`❌ Failed to save brain bridge map after bulk sync: ${err.message}`);
        stats.errors++;
      });
    }
  }
}

/**
 * Count how many active brain records lack an embedding — the headline number
 * for the "N memories missing embeddings · Embed missing" affordance on the
 * graph. A record is "missing" when it's unmapped OR its mapped memory has a
 * NULL embedding — scoped to ACTIVE memories, since getMemoryIdsMissingEmbedding
 * only reports active rows (an archived/non-active mapped memory is treated as
 * embedded and is not a backfill target). Read-only; walks the same stores as
 * the onlyMissing sync so the count matches what "Embed missing" would process.
 *
 * Only IDENTITY is needed to tally — each store's live ids, checked against the
 * bridge map — so the entity/journal walks go through `brainStorage.listLiveIds`
 * rather than `getAll` / `listJournals` (issue #3508). Those loaded and parsed
 * every record body across every collection, then threw all of it away but the
 * `id` and `archived` fields, turning a status poll into hundreds of disk reads.
 * `listLiveIds` reads only the ids it has never resolved.
 */
export async function getEmbeddingCoverage() {
  const map = await memory.getBrainMemoryLinks(
    await loadBridgeMap({ readOnly: true }), { readOnly: true });
  const missingMemIds = await memory.getMemoryIdsMissingEmbedding().catch(() => new Set());
  const isEmbedded = makeEmbeddedChecker(map, missingMemIds);

  let total = 0;
  let missing = 0;
  const tally = (key) => { total += 1; if (!isEmbedded(key)) missing += 1; };

  // Entity stores plus the daily log — the journal record id IS its date, the
  // same key syncAllBrainData bridges under. `listLiveIds` already drops
  // tombstones and archived records, and it has no page cap (the old
  // `listJournals({ limit: 10000 })` silently stopped counting past 10k days).
  for (const type of RECONCILABLE_TYPES) {
    for (const id of await brainStorage.listLiveIds(type)) tally(bridgeKey(type, id));
  }

  // digests/reviews are append-only JSONL: one whole-file read each, behind
  // brainStorage's own cache — not a per-record disk walk, so they stay as-is.
  for (const type of JSONL_TYPES) {
    const records = await readJsonlStore(type);
    for (const record of records) tally(bridgeKey(type, record.id));
  }

  return { total, missing };
}

// ─── Synced-in record resync (issue #1080) ──────────────────────────────────
// Peer-synced brain writes go through brainStorage.applyRemoteRecord, which is
// deliberately event-silent (no brainEvents) to prevent cross-peer echo loops
// (#1077). That silence means the per-record bridge listeners below never fire
// for synced-in records, so a record created/edited/deleted on peer A was never
// re-vectorized into peer B's memory index — it stayed stale and searchable
// until a full bulk resync (which itself SKIPPED already-mapped records). The
// sync apply paths now emit a LOCAL-ONLY 'sync:applied' signal (it never feeds
// the sync log, so no echo) carrying the {type, id} of every applied change;
// we re-read each record's canonical state and re-embed or archive accordingly.

const RESYNC_DEBOUNCE_MS = 250;
const pendingResync = new Map(); // bridgeKey → { type, id, hardDelete } (dedups repeated touches)
let resyncTimer = null;
let resyncFlushing = false; // single-flight guard so two flushes can't overlap

// Journals re-embed on a much longer window than the other brain types, per
// day. The daily-log editor autosaves while the user types, so 'journals:upserted'
// now fires every couple of seconds for the SAME day — where it used to fire
// once per explicit Save. Re-embedding per event would mean an embedding call
// per typing pause, and — once the day's entry outgrows the embedding char
// budget — a full LLM summarization per event too (see generateMemoryEmbedding's
// over-budget branch), which is exactly the kind of unannounced provider traffic
// the AI policy rules out. Coalescing per day collapses a whole writing session
// into one re-embed after the user settles; resyncBrainRecord re-reads the
// canonical entry at flush time, so the trailing flush always embeds the newest
// content — and embedding once the sentence is finished beats embedding it
// half-typed 100 times.
const JOURNAL_RESYNC_DEBOUNCE_MS = 30_000;
const journalResyncTimers = new Map(); // journal id (the ISO date) → timer

// Durable mirror of every resync that has been triggered but not yet settled:
// queued entries, the batch a flush is working through, and journal days still
// inside their debounce window. Persisted so a restart replays the work instead
// of leaving stale embeddings or still-searchable deleted records behind.
const durablePending = new Map(); // bridgeKey → { type, id, hardDelete }
let pendingWriteTail = Promise.resolve();
let pendingWriteQueued = false;

function persistPending() {
  // Calls made while a write is queued or running coalesce into one snapshot
  // taken when the write actually starts.
  if (pendingWriteQueued) return;
  pendingWriteQueued = true;
  pendingWriteTail = pendingWriteTail.then(async () => {
    pendingWriteQueued = false;
    const dir = dirname(PENDING_RESYNC_PATH);
    if (!existsSync(dir)) await ensureDir(dir);
    await atomicWrite(PENDING_RESYNC_PATH, { entries: [...durablePending.values()] });
  }).catch((err) => {
    console.error(`❌ Brain bridge pending-resync persist failed: ${err.message}`);
  });
}

// An entry is only finished once nothing newer is waiting for the same key.
function settleDurable(key, journalId) {
  if (pendingResync.has(key) || (journalId && journalResyncTimers.has(journalId))) return;
  if (durablePending.delete(key)) persistPending();
}

/** Resolves once every queued pending-set write has reached disk. Exported for tests. */
export const flushPendingResyncWrites = () => pendingWriteTail;

/**
 * Replay resyncs a previous process queued but never finished. Re-queues them
 * through the normal path (sticky hard-delete included); an absent or empty
 * file makes no call at all.
 */
export async function replayPersistedResync() {
  const raw = await readFile(PENDING_RESYNC_PATH, 'utf-8').catch((err) => {
    if (err.code === 'ENOENT') return null;
    throw err;
  });
  if (raw === null) return 0;
  let entries;
  try {
    entries = JSON.parse(raw)?.entries;
  } catch {
    console.warn('⚠️ Corrupt brain pending-resync file; ignoring');
    return 0;
  }
  if (!Array.isArray(entries) || entries.length === 0) return 0;
  const valid = entries.filter((entry) => entry && typeof entry.id === 'string' && TYPE_MAP[entry.type]);
  queueResync(valid.map(({ type, id, hardDelete }) => ({ type, id, hardDelete: hardDelete === true })));
  console.log(`🧠🔗 Replaying ${valid.length} brain resync(s) interrupted by a restart`);
  return valid.length;
}

/**
 * Archive the memory entry mapped to a brain record (if any). Used when the
 * canonical record is gone (tombstoned), archived, or otherwise should no
 * longer be searchable.
 */
async function archiveMappedMemory(brainType, id) {
  const map = await loadBridgeMap();
  const memoryId = map[bridgeKey(brainType, id)];
  if (!memoryId) return false;
  await memory.updateMemory(memoryId, { status: 'archived' });
  console.log(`🧠🔗 Archived brain→memory: ${brainType}/${id} → ${memoryId}`);
  return true;
}

/**
 * Hard-delete the memory entry mapped to a brain record (issue #1318). Used for
 * a GENUINE local user delete: archiving the row would hide it from search but
 * leave the row + its 768-dim embedding in Postgres forever (e.g. ~1,482 dead
 * vectors after a cleared ChatGPT import). A hard delete drops the row and its
 * embedding and removes the bridge-map entry so a later create can't collide
 * with a stale mapping. NOT used for synced-in deletes — those stay soft-
 * archived (see resyncBrainRecord) so a peer un-delete can resurrect them.
 */
async function hardDeleteMappedMemory(brainType, id) {
  const map = await loadBridgeMap();
  const key = bridgeKey(brainType, id);
  const memoryId = map[key];
  if (!memoryId) return false;
  await memory.purgeMemory(memoryId);
  delete map[key];
  await saveBridgeMap();
  console.log(`🧠🔗 Hard-deleted brain→memory: ${brainType}/${id} → ${memoryId}`);
  return true;
}

/**
 * Re-vectorize (or archive) a single brain record from its CANONICAL stored
 * state — used after a peer sync applies a change. Reading the store rather
 * than trusting an event payload makes this self-healing and order-independent:
 * whatever the final converged state is (live record, archived, or tombstoned),
 * the memory copy is brought into line.
 *   - record present & not archived → upsert + re-embed (syncBrainRecord)
 *   - record absent (tombstoned) → hard-delete on a genuine local delete
 *     (hardDelete), else archive the mapped memory entry (synced/recoverable)
 *   - record present but archived → archive (always soft — recoverable)
 *   - type not mirrored by the bridge (links/buckets/inbox) → no-op
 *
 * @param {object}  [opts]
 * @param {boolean} [opts.hardDelete=false] The trigger was a real local user
 *   delete (a `{type}:deleted` event), so a tombstoned record should hard-prune
 *   the mapped memory row + embedding rather than soft-archive it (issue #1318).
 *   Synced-in deletes (sync:applied) pass false so they stay resurrectable.
 */
export async function resyncBrainRecord(brainType, id, { hardDelete = false } = {}) {
  if (!id || !TYPE_MAP[brainType]) return;
  // getById returns null for tombstones (deleted records), so a synced-in
  // delete naturally lands in the archive/hard-delete branch.
  const record = brainType === 'journals'
    ? await getJournal(id)
    : JSONL_TYPES.includes(brainType)
      ? (await readJsonlStore(brainType)).find(record => record.id === id)
      : await brainStorage.getById(brainType, id);
  if (!record) {
    // Truly gone (tombstoned). A genuine local user delete hard-prunes the row;
    // a sync-driven delete stays soft so a peer un-delete can resurrect it.
    if (hardDelete) await hardDeleteMappedMemory(brainType, id);
    else await archiveMappedMemory(brainType, id);
    return;
  }
  if (record.archived) {
    // Present but archived (not deleted) — always soft, never hard-delete, even
    // if the queued trigger carried a hardDelete intent.
    await archiveMappedMemory(brainType, id);
    return;
  }
  await syncBrainRecord(brainType, record);
}

/**
 * Drain the pending resync queue sequentially. Single-flight + re-draining:
 * only one flush runs at a time (the `resyncFlushing` guard), and it loops
 * until the queue is empty so records enqueued mid-flush are still processed —
 * without a second flush running concurrently. Sequential (not parallel) so a
 * large catch-up sync doesn't fire hundreds of concurrent embedding calls and
 * saturate the embedding backend; single-flight also prevents two flushes from
 * both creating a memory for the same not-yet-mapped record (duplicate entries).
 * Exported for deterministic flushing in tests.
 */
export async function flushPendingResync() {
  if (resyncTimer) { clearTimeout(resyncTimer); resyncTimer = null; }
  if (resyncFlushing) return; // an in-flight flush will pick up newly-queued items
  resyncFlushing = true;
  try {
    // The queue only grows during an `await` below; the loop re-checks size at
    // the top of each pass, and the gap between the final size===0 check and
    // clearing `resyncFlushing` contains no await — so nothing can be stranded.
    while (pendingResync.size > 0) {
      const batch = [...pendingResync.values()];
      pendingResync.clear();
      for (const { type, id, hardDelete } of batch) {
        await resyncBrainRecord(type, id, { hardDelete }).catch((err) => {
          console.error(`❌ Brain bridge resync failed for ${type}/${id}: ${err.message}`);
        });
        settleDurable(bridgeKey(type, id), type === 'journals' ? id : null);
      }
    }
  } finally {
    resyncFlushing = false;
  }
}

/**
 * Queue applied-change records for a debounced resync. Records touching a type
 * the bridge doesn't mirror are dropped here so they never schedule a flush.
 * No timer is armed while a flush is in flight — that flush re-drains the queue
 * itself, so a second concurrent flush can never start.
 */
export function queueResync(records) {
  if (!Array.isArray(records)) return;
  let queued = false;
  for (const { type, id, hardDelete } of records) {
    if (!id || !TYPE_MAP[type]) continue;
    const key = bridgeKey(type, id);
    // Sticky hard-delete on coalesce: once a real local `{type}:deleted` marks a
    // key for a hard prune, a later touch for the same key before the flush — a
    // concurrent `sync:applied` carrying no flag, say — must NOT downgrade it to
    // a soft archive, or the dead embedding lingers. Safe to keep sticky because
    // resyncBrainRecord's record-present branch upserts a resurrected record
    // regardless of the flag, so a true intent can never hard-delete a live row.
    const prevHardDelete = pendingResync.get(key)?.hardDelete;
    const entry = { type, id, hardDelete: !!hardDelete || !!prevHardDelete };
    pendingResync.set(key, entry);
    durablePending.set(key, entry);
    queued = true;
  }
  if (queued) persistPending();
  if (pendingResync.size === 0 || resyncTimer || resyncFlushing) return;
  resyncTimer = setTimeout(() => {
    flushPendingResync().catch((err) => {
      console.error(`❌ Brain bridge resync flush failed: ${err.message}`);
    });
  }, RESYNC_DEBOUNCE_MS);
  // Don't keep the event loop alive just for a pending resync flush.
  if (typeof resyncTimer.unref === 'function') resyncTimer.unref();
}

// ─── Event Handlers ─────────────────────────────────────────────────────────
// Entity stores emit "{type}:upserted" / "{type}:deleted" with the single
// affected record. Both are routed through the debounced+sequential
// queueResync path (not handled inline) — see initBridge for why.
// JSONL stores still emit "{type}:added" with a single record.

function handleJsonlAdded(brainType, record) {
  if (!record?.id) return;
  queueResync([{ type: brainType, id: record.id }]);
}

// ─── Init ───────────────────────────────────────────────────────────────────

/**
 * Initialize the brain→memory bridge.
 * Attaches event listeners to brainEvents so new/updated brain records
 * are automatically mirrored to the CoS memory system.
 */
export function initBridge() {
  // Entity store changes — route through the debounced + sequential queueResync
  // path rather than embedding inline per event. A single create/update already
  // emits one per-record event (not a store-wide fan-out), but a BULK producer —
  // e.g. the ChatGPT import creating ~1,400 conversations back-to-back — fires
  // that many `:upserted` events in a tight loop. Handling each inline would
  // launch ~1,400 concurrent embedding calls + Postgres writes, saturating the
  // embedding backend (most time out → records persist with a NULL embedding,
  // silently unsearchable) and exhausting the PG pool. queueResync coalesces the
  // burst (250ms debounce + dedup) and drains it sequentially — the same
  // protection the peer-sync `sync:applied` path already relies on. It also
  // re-reads each record's CANONICAL state (resyncBrainRecord), so the archived/
  // deleted/tombstoned branches are handled there uniformly and an event whose
  // payload is already stale by flush time still converges correctly.
  for (const type of BRIDGED_ENTITY_TYPES) {
    brainEvents.on(`${type}:upserted`, ({ id }) => queueResync([{ type, id }]));
    // A `:deleted` event is a genuine LOCAL user delete (remove() emits it;
    // applyRemoteRecord is event-silent), so hard-prune the mapped vector row
    // rather than soft-archive it forever (issue #1318).
    brainEvents.on(`${type}:deleted`, ({ id }) => queueResync([{ type, id, hardDelete: true }]));
  }

  // JSONL appends (digests, reviews)
  for (const type of JSONL_TYPES) {
    brainEvents.on(`${type}:added`, (record) => handleJsonlAdded(type, record));
  }

  // Daily log — per-entry events so a single append doesn't re-embed every
  // day of the user's history. (An earlier version listened for the
  // store-wide 'journals:changed' event, which would trigger O(totalDays)
  // embedding calls per dictation segment and saturate the embedding
  // backend.) appendJournal/setJournalContent fire 'journals:upserted' with
  // the single affected entry; deleteJournal fires 'journals:deleted'.
  // Debounced per day because the editor autosaves — see queueJournalResync.
  brainEvents.on('journals:upserted', ({ entry }) => {
    if (entry?.id) queueJournalResync(entry.id);
  });
  // handleJournalDeleted is async and awaits loadBridgeMap(); wrap the call
  // in a .catch so a rejection becomes a logged error instead of an
  // unhandled-rejection warning (or a process crash under strict modes).
  brainEvents.on('journals:deleted', ({ entry }) => {
    // Drop any debounced re-embed for the day first — the hard-delete below
    // prunes the mapped memory, and a later flush would only re-read a tombstone.
    if (entry?.id) cancelPendingJournalResync(entry.id);
    handleJournalDeleted(entry).catch((err) => {
      console.error(`❌ Brain bridge delete sync failed for journals/${entry?.id}: ${err.message}`);
    });
  });

  // Peer-synced records (issue #1080) — applyRemoteRecord is event-silent to
  // prevent echo loops, so the per-record listeners above never fire for
  // synced-in changes. The sync apply paths emit this local-only signal with
  // the touched {type, id}s; we re-read each from the store and re-embed or
  // archive. Debounced + sequential so a large catch-up doesn't saturate the
  // embedding backend.
  brainEvents.on('sync:applied', ({ records } = {}) => queueResync(records));

  console.log('🧠🔗 Brain→Memory bridge initialized');
}

/**
 * Coalesce repeated saves of one day into a single re-embed, then hand off to
 * the shared queueResync path (dedup + sequential drain + canonical re-read).
 * Trailing-edge per day: each save pushes the flush out, so a writing session
 * costs one re-embed once the user stops. See JOURNAL_RESYNC_DEBOUNCE_MS.
 */
function queueJournalResync(id) {
  clearTimeout(journalResyncTimers.get(id));
  const key = bridgeKey('journals', id);
  if (!durablePending.has(key)) {
    durablePending.set(key, { type: 'journals', id, hardDelete: false });
    persistPending();
  }
  const timer = setTimeout(() => {
    journalResyncTimers.delete(id);
    queueResync([{ type: 'journals', id }]);
  }, JOURNAL_RESYNC_DEBOUNCE_MS);
  // Don't hold the event loop open for a pending journal re-embed.
  if (typeof timer.unref === 'function') timer.unref();
  journalResyncTimers.set(id, timer);
}

/**
 * Drop a day's pending re-embed. Called when the day is deleted: the delete
 * path prunes the mapped memory immediately, so letting the debounce fire
 * afterwards would just re-read a tombstone. Also exported for tests.
 */
export function cancelPendingJournalResync(id) {
  clearTimeout(journalResyncTimers.get(id));
  journalResyncTimers.delete(id);
  settleDurable(bridgeKey('journals', id), id);
}

async function handleJournalDeleted(entry) {
  if (!entry?.id) return;
  // deleteJournal (the local user action) is the only emitter of
  // 'journals:deleted'; synced-in journal deletes flow through sync:applied →
  // resyncBrainRecord and stay soft-archived. So a local journal delete hard-
  // prunes the mapped memory row + embedding (issue #1318).
  await hardDeleteMappedMemory('journals', entry.id);
}