Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

62.3 KB
Raw Blame
/**
 * Brain Storage Service
 *
 * Handles file-based persistence for the Brain feature.
 * - Per-record `collectionStore` dirs for entity stores (people, projects,
 *   ideas, admin, …, journals, inbox, songs, threads) — `data/brain/<type>/<id>/index.json`
 * - JSON for the single `meta.json` settings doc
 * - JSONL for append-only generated logs (digests, reviews)
 *
 * Storage layout (issue #725): each entity store is a `collectionStore` — one
 * file per record, read through to disk with no whole-store in-memory cache. A
 * write touches one record's file and per-id write queues let writes to
 * DIFFERENT records proceed in parallel (the scalability win over the old
 * monolithic `data/brain/<type>.json` + 2s-TTL cache + single global mutex).
 * Migration 200 splits the legacy monolithic files into this shape.
 *
 * The old single global write mutex is replaced by the store's per-id
 * `queueRecordWrite(id, fn)`: two writes to the SAME (type, id) serialize —
 * local create/update AND remote peer applies AND tombstone GC all queue on the
 * same per-id tail, so a peer sync landing mid-way through a local write can't
 * read a stale snapshot and drop the other write (the AGENTS.md-sanctioned
 * "serialize two write paths that mutate the same record" case). Writes to
 * different records no longer serialize against each other. brainJournal layers
 * its own storeMutex ON TOP for local sidecar coordination; journal transforms
 * run inside this per-id queue too, so peer applies cannot invalidate their
 * snapshot. Locks only nest storeMutex → this queue (one direction).
 *
 * FEDERATION is unaffected by the layout. Brain federates strictly per-record
 * through this module's API seams (`getRawRecords` / `applyRemoteRecord`) plus
 * the `brainSyncLog` delta log — never by shipping a whole-store file — so the
 * on-disk container shape is orthogonal to the wire format (no `schemaVersions.js`
 * gate; brain is intentionally ungated there).
 */

import { readFile, appendFile } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { v4 as uuidv4 } from '../lib/uuid.js';
import EventEmitter from 'events';
import { atomicWrite, ensureDir, readJSONFile, safeJSONParse, safeDate, PATHS } from '../lib/fileUtils.js';
import { getInstanceId } from './instanceIdentity.js';
import * as brainSyncLog from './brainSyncLog.js';
import { createCollectionStore } from '../lib/collectionStore.js';
import { linkIsRepo, normalizeRepoLinkFields } from '../lib/repoLinkFields.js';
import { supersedingTimestamp } from '../lib/tombstones.js';
import { parseTsMs, compareNewerWins } from '../lib/lwwTimestamp.js';
import { mergeIncomingLinkRecord } from '../lib/brainLinkLocalFields.js';

const DATA_DIR = PATHS.brain;

// The entity stores that participate in peer sync (records with IDs).
// Canonical list — sync, tombstone GC, origin backfill, and the per-type
// collectionStore construction below all derive from it so adding a type can't
// silently drop out of one of those paths. (Boot-time migrations keep their own
// frozen copy by necessity — migrations run before the service layer is wired
// up — see scripts/migrations/080-*.js and 200-*.js.)
//
// `journals` (the Daily Log) and `inbox` were added in migration 081; both are
// `{ id: record }` stores exactly like the others (journals keyed by date,
// inbox by uuid), so the delta log, anti-entropy reconcile, tombstone GC, and
// originInstanceId backfill all cover them with no per-type branching.
export const BRAIN_ENTITY_TYPES = Object.freeze([
  'people', 'projects', 'ideas', 'admin', 'memories', 'links', 'buckets',
  'journals', 'inbox', 'songs', 'threads',
]);

// The type-level storage-layout version. Bumped by a migration that changes the
// on-disk layout; distinct from any per-record field-shape version (brain
// records carry none). Migration 200 stamps this on every brain type index.
export const BRAIN_STORE_SCHEMA_VERSION = 1;

// One collectionStore per entity type, rooted at data/brain/<type>/. No
// `sanitizeRecord` — brain records (and their in-place tombstones) are stored
// and loaded verbatim, exactly as the old monolithic map did.
const brainStores = Object.freeze(Object.fromEntries(
  BRAIN_ENTITY_TYPES.map((type) => [type, createCollectionStore({
    dir: join(DATA_DIR, type),
    type,
    schemaVersion: BRAIN_STORE_SCHEMA_VERSION,
  })]),
));

function storeFor(type) {
  const store = brainStores[type];
  if (!store) throw new Error(`brainStorage: unknown entity type "${type}"`);
  return store;
}

/**
 * The per-type collectionStores, for the boot-time schema-version verifier in
 * server/index.js (`verifyCollectionVersions`).
 */
export function brainCollectionStores() {
  return BRAIN_ENTITY_TYPES.map((type) => brainStores[type]);
}

// A tombstone is a deleted-record marker kept IN PLACE as the record's stored
// value (its `{id}/index.json`) — rather than removing the record dir — so the
// last-writer-wins guard in applyRemoteRecord can reject a stale `create`
// echoed back from a peer. Without it, a hard delete leaves `existing ===
// undefined`, the LWW guard is skipped, and the record resurrects — then the
// newer delete re-kills it, and both ops relay to every peer forever (the
// federated brain-sync loop). Shape:
// { _deleted: true, updatedAt, originInstanceId, deletedAt }. The `updatedAt`
// is the LWW clock; `deletedAt` is the GC clock.
const isTombstone = (rec) => !!(rec && rec._deleted);

// Build the in-place tombstone marker. `updatedAt` is the LWW clock (must be the
// delete's timestamp); `deletedAt` is the GC clock — equal at birth, kept
// separate so the GC sweep reads its own field.
const makeTombstone = (updatedAt, originInstanceId) => ({
  _deleted: true,
  updatedAt,
  originInstanceId: originInstanceId ?? 'unknown',
  deletedAt: updatedAt,
});

// GC grace period: how long a tombstone is retained before it can be hard-
// pruned. Must comfortably exceed the longest realistic peer-offline window so
// a peer that reconnects still sees the delete (not a since-vanished id that it
// would re-create). 30 days matches the sharing-side tombstone grace buffer.
export const BRAIN_TOMBSTONE_GRACE_MS = 30 * 24 * 60 * 60 * 1000;

// Non-collection files: the single settings doc and the append-only JSONL logs.
const FILES = {
  meta: join(DATA_DIR, 'meta.json'),
  digests: join(DATA_DIR, 'digests.jsonl'),
  reviews: join(DATA_DIR, 'reviews.jsonl'),
};

// Event emitter for brain data changes
export const brainEvents = new EventEmitter();

/**
 * Announce a persisted entity-store change that did NOT emit the usual
 * `${type}:upserted` / `${type}:deleted` event.
 *
 * Some write paths are deliberately event-silent — `applyRemoteRecord` (peer
 * applies, silent to prevent the #1077 cross-peer echo loop),
 * `upsertWithId({ emitEvent: false })` (the caller emits its own richer event),
 * tombstone pruning, and origin-instance backfill. Read-through consumers did
 * not care, but derived in-memory state does: without this signal,
 * brainSearchIndex or the reconcile checksum cache could stay stale.
 *
 * LOCAL-ONLY, like `sync:applied`: it carries no record payload, is never fed
 * to the sync log, and is never relayed to a peer, so it cannot amplify an
 * echo. Listeners must treat it as "re-read this type from canonical state".
 */
function emitRecordChanged(type, id) {
  brainEvents.emit('record:changed', { type, id });
}

// In-memory caches — only for the non-collection files. Entity stores read
// through to disk (per issue #725: no whole-store cache to go stale when a
// per-record write bypasses it).
const caches = {
  meta: { data: null, timestamp: 0 },
  digests: { data: null, timestamp: 0 },
  reviews: { data: null, timestamp: 0 },
};

const CACHE_TTL_MS = 2000;

// Default settings
const DEFAULT_META = {
  version: 1,
  confidenceThreshold: 0.6,
  dailyDigestTime: '00:00',
  weeklyReviewTime: '00:00',
  weeklyReviewDay: 'sunday',
  defaultProvider: 'lmstudio',
  defaultModel: 'gptoss-20b',
  // Opt-in: digests/reviews call an AI provider, so nothing runs until enabled.
  scheduledReviewsEnabled: false,
  lastDailyDigest: null,
  lastWeeklyReview: null
};

/**
 * Ensure brain data directory exists
 */
export async function ensureBrainDir() {
  await ensureDir(DATA_DIR);
}

/**
 * Generate a new UUID
 */
export function generateId() {
  return uuidv4();
}

/**
 * Get current ISO timestamp
 */
export function now() {
  return new Date().toISOString();
}

// =============================================================================
// META / SETTINGS
// =============================================================================

/**
 * Load brain settings
 */
export async function loadMeta() {
  const cache = caches.meta;
  if (cache.data && (Date.now() - cache.timestamp) < CACHE_TTL_MS) {
    return cache.data;
  }

  await ensureBrainDir();

  // Strict (#4115): `updateMeta` is `loadMeta → spread updates → saveMeta`, so a
  // swallowed unreadable meta.json writes DEFAULT_META over every brain setting
  // on the next update — and the result is cached for CACHE_TTL_MS, so the fake
  // default outlives the read that produced it.
  const loaded = await readJSONFile(FILES.meta, null, { strict: true });
  cache.data = loaded ? { ...DEFAULT_META, ...loaded } : { ...DEFAULT_META };
  cache.timestamp = Date.now();
  return cache.data;
}

/**
 * Save brain settings
 */
export async function saveMeta(meta) {
  await ensureBrainDir();
  await atomicWrite(FILES.meta, meta);
  caches.meta.data = meta;
  caches.meta.timestamp = Date.now();
  brainEvents.emit('meta:changed', meta);
}

/**
 * Update brain settings (partial update)
 */
export async function updateMeta(updates) {
  const meta = await loadMeta();
  const updated = { ...meta, ...updates };
  await saveMeta(updated);
  return updated;
}

// =============================================================================
// ENTITY STORES (per-record collectionStore, read-through)
// =============================================================================

/**
 * Assemble the raw `{ id: record }` map for a type from its per-record files,
 * INCLUDING tombstones. Read-through: one `loadOne` per id, in parallel. This
 * is the seam the reconcile/anti-entropy path and getAll/getRawRecords build on
 * — identical return shape to the old monolithic `loadJsonStore(type).records`.
 */
async function loadRawMap(type) {
  const store = storeFor(type);
  const ids = await store.listIds();
  const entries = await Promise.all(ids.map(async (id) => {
    const record = await store.loadOne(id);
    return record ? [id, record] : null;
  }));
  return Object.fromEntries(entries.filter(Boolean));
}

/**
 * Get all records from a store (tombstones excluded — they exist only to anchor
 * the LWW sync guard, never as user-visible records).
 */
export async function getAll(type) {
  const map = await loadRawMap(type);
  return Object.entries(map)
    .filter(([, record]) => !isTombstone(record))
    .map(([id, record]) => ({ id, ...record }));
}

// ─── Per-record projection indexes (issues #3508, #3509) ────────────────────
// A projection index answers one question about EVERY record in a store — "is
// this id live?" (#3508), "what are this link's sort/filter keys?" (#3509) —
// WITHOUT re-reading and re-parsing every record body on every call, the way
// `getAll(type)` must.
//
// Cost model per call: one `listIds()` (readdir + lstat, no body reads) plus a
// `loadOne` for ids the index has never resolved. Steady state is zero body
// reads; a store the caller has already walked answers from memory.
//
// FRESHNESS — the same three signal classes brainSearchIndex documents:
//   1. Adds and hard-deletes (create, pruneTombstones) — covered by the
//      `listIds()` diff itself; membership is re-read from disk every call.
//   2. IN-PLACE mutations that keep the id: an `archived` flip, or `remove()`
//      writing a tombstone over the record. The id survives the diff, so these
//      ride the per-record `${type}:upserted` / `${type}:deleted` events.
//   3. Event-SILENT writes — `applyRemoteRecord` (silent to prevent the #1077
//      echo loop) and `upsertWithId({ emitEvent: false })` — ride the local-only
//      `record:changed` invalidation signal.
//
// No `null = not built` sentinel is needed here (unlike brainSearchIndex):
// `listIds()` is the membership authority on every call, so an empty index is a
// legitimate "nothing resolved yet" state and never causes a redundant re-scan.
//
// Two write paths are deliberately event-silent and NOT covered by the three
// classes above, because neither can change what any projection reads:
// `pruneTombstones` hard-deletes the record dir, so the id leaves `listIds()`
// and the diff drops it; `backfillOriginInstanceId` only stamps
// `originInstanceId`, a field no projection looks at, and runs once at boot.
// A NEW projection that reads either signal must invalidate from those paths.
const RECORD_INDEX_MAX_ATTEMPTS = 3;

// Every index registers itself here so ONE invalidation drops the touched id
// from all of them — an index that missed a signal would serve a stale
// projection until the next unrelated change to that record.
const recordIndexes = [];

/**
 * Build a per-type projection index. `project(record)` maps the parsed on-disk
 * record (or `null` for a missing/corrupt one) to the value to cache for that
 * id. Registers itself for invalidation; call `resolveRecordIndex` to read it.
 */
function createRecordIndex(project) {
  const index = {
    project,
    byType: Object.fromEntries(BRAIN_ENTITY_TYPES.map((type) => [type, new Map()])),
    // Bumped by every invalidation, so a resolve pass that raced an invalidation
    // is retried instead of writing the value it read just before the change landed.
    generation: Object.fromEntries(BRAIN_ENTITY_TYPES.map((type) => [type, 0])),
  };
  recordIndexes.push(index);
  return index;
}

function invalidateRecordIndexes(type, id) {
  if (!id) return;
  for (const index of recordIndexes) {
    const byType = index.byType[type];
    if (!byType) continue;
    byType.delete(id);
    index.generation[type] += 1;
  }
}

/**
 * Resolve a projection index for one store. Returns `[id, value]` pairs in
 * `listIds()` order — every present id, including ones whose projection is
 * `null` (callers decide what a null projection means).
 */
async function resolveRecordIndex(indexState, type, attempt) {
  // `storeFor` throws for an unknown type, and the index maps are keyed off the
  // same BRAIN_ENTITY_TYPES list — so past this line `byType[type]` exists.
  const store = storeFor(type);
  const index = indexState.byType[type];
  const generation = indexState.generation[type];

  const ids = await store.listIds();
  const present = new Set(ids);
  for (const id of [...index.keys()]) {
    if (!present.has(id)) index.delete(id);
  }

  const unresolved = ids.filter((id) => !index.has(id));
  const loaded = await Promise.all(unresolved.map((id) => store.loadOne(id)));
  const resolved = new Map(unresolved.map((id, i) => [id, indexState.project(loaded[i])]));

  // An invalidation landed while we were reading, so what we loaded may already
  // describe the pre-change record. Re-run — the retry re-reads only the ids the
  // invalidation dropped, so it is cheap. Past the attempt ceiling we answer
  // from what we have but do NOT write it into the index: caching a value we
  // know raced would strand it there until the next change to that record.
  const raced = indexState.generation[type] !== generation;
  if (raced && attempt + 1 < RECORD_INDEX_MAX_ATTEMPTS) {
    return resolveRecordIndex(indexState, type, attempt + 1);
  }
  if (raced) {
    // Ceiling reached. The invalidation may also have dropped ids this pass
    // took FROM the index (so they were never in `unresolved`) — read those
    // before answering, or a record we merely stopped knowing about would be
    // reported as absent and undercount the result.
    const unknown = ids.filter((id) => !resolved.has(id) && !index.has(id));
    const reloaded = await Promise.all(unknown.map((id) => store.loadOne(id)));
    unknown.forEach((id, i) => resolved.set(id, indexState.project(reloaded[i])));
  } else {
    for (const [id, value] of resolved) index.set(id, value);
  }

  return ids.map((id) => [id, resolved.has(id) ? resolved.get(id) : index.get(id)]);
}

// Live = present, not a tombstone, and not archived. `loadOne` returns null for
// a missing/corrupt/unparseable record, which `getAll` also drops — so it is
// not live either.
const isLiveRecord = (record) => !!record && !isTombstone(record) && !record.archived;
const liveIdIndex = createRecordIndex(isLiveRecord);

/**
 * Ids of the LIVE records in a store — present, not tombstoned, not archived —
 * without loading record bodies for ids the index has already resolved.
 *
 * This is the cheap counterpart to `getAll(type).filter(r => !r.archived)` for
 * callers that only need identity (embedding-coverage tallies, membership
 * checks). Returns a fresh array in `listIds()` order each call.
 */
export async function listLiveIds(type) {
  const rows = await resolveRecordIndex(liveIdIndex, type, 0);
  return rows.filter(([, live]) => live === true).map(([id]) => id);
}

// Wire the freshness signals once, at module load. Invalidation only drops the
// touched id, so the next resolve re-reads exactly one record.
//
// The `journals` type is the one that needs checking rather than assuming:
// brainJournal re-emits `journals:upserted` / `journals:deleted` under the SAME
// names with its own richer `{ entry }` / `{ date, entry }` payloads, which
// carry no top-level `id`. Those extra emits are harmless no-ops here — every
// journal write still routes through brainStorage first (`putEntry` →
// `upsertWithId({ emitEvent: false })` → `record:changed`; `deleteJournal` →
// `remove()` → `journals:deleted` with `{ id }`), which is what actually
// invalidates.
for (const type of BRAIN_ENTITY_TYPES) {
  brainEvents.on(`${type}:upserted`, ({ id } = {}) => invalidateRecordIndexes(type, id));
  brainEvents.on(`${type}:deleted`, ({ id } = {}) => invalidateRecordIndexes(type, id));
}
brainEvents.on('record:changed', ({ type, id } = {}) => invalidateRecordIndexes(type, id));

/**
 * Get the RAW records map for a store, INCLUDING tombstones, keyed by id.
 *
 * Unlike `getAll` (which strips tombstones for user-facing reads), the sync
 * reconcile path needs tombstones too: they carry the LWW `updatedAt` clock a
 * peer must see to keep a delete from resurrecting. Returns a fresh `{ id:
 * record }` map — safe to mutate the map, but the record objects are the parsed
 * on-disk values, so callers treat them as read-only (the reconcile path only
 * serializes them, never mutates).
 */
export async function getRawRecords(type) {
  return loadRawMap(type);
}

/**
 * Get a record by ID
 */
export async function getById(type, id) {
  const record = await storeFor(type).loadOne(id);
  return record && !isTombstone(record) ? { id, ...record } : null;
}

/**
 * Create a new record
 */
export async function create(type, recordData) {
  const store = storeFor(type);
  const originInstanceId = await getInstanceId();
  const id = generateId();
  const timestamp = now();

  const record = {
    ...recordData,
    originInstanceId,
    createdAt: timestamp,
    updatedAt: timestamp
  };

  // Fresh uuid → no read-modify-write; saveOne queues per-id internally.
  await store.saveOne(id, record);
  brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
  await brainSyncLog.appendLocalChanges([{ op: 'create', type, id, record, originInstanceId }]);

  console.log(`🧠 Created ${type} record: ${id}`);
  return { id, ...record };
}

/**
 * Update a record
 */
export async function update(type, id, updates) {
  const store = storeFor(type);
  // A malformed id can't name a valid per-record dir — treat as not-found
  // rather than letting the store's write-queue assertion throw on it.
  if (!store.isValidId(id)) return null;
  return store.queueRecordWrite(id, async () => {
    const existing = await store.loadOne(id);
    // A tombstoned record is gone — treat it as not-found rather than reviving it.
    if (!existing || isTombstone(existing)) {
      return null;
    }

    const record = {
      ...existing,
      ...updates,
      // Preserve immutable fields — originInstanceId tracks the creating instance
      originInstanceId: existing.originInstanceId,
      createdAt: existing.createdAt,
      // Local edits must outrank the version read inside this queue, even
      // within one millisecond or after receiving a peer's faster clock.
      updatedAt: supersedingTimestamp(existing.updatedAt, now())
    };

    await store.saveOneNow(id, record);
    brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
    await brainSyncLog.appendLocalChanges([{ op: 'update', type, id, record, originInstanceId: record.originInstanceId }]);

    console.log(`🧠 Updated ${type} record: ${id}`);
    return { id, ...record };
  });
}

/**
 * Locked read-modify-write update.
 *
 * `update(type, id, partial)` merges a partial the CALLER computed — usually
 * from a record snapshot read OUTSIDE the store write queue. When the partial
 * is derived from the current record (append to / filter an array field like
 * `attachments`), a concurrent writer landing between the snapshot read and
 * the update silently gets clobbered (and the clobber wins LWW federation).
 *
 * `updateWith` closes that window: INSIDE the per-id write queue it re-reads the
 * fresh record, calls `fn({ id, ...record })` → a partial-updates object (or
 * null/undefined to abort without writing), then merges/persists with exactly
 * `update()`'s semantics — immutable originInstanceId/createdAt, fresh
 * updatedAt stamp, `${type}:upserted` event, and a sync log append. Returns the
 * updated `{ id, ...record }`, or null when the record is missing/tombstoned or
 * fn aborted.
 */
export async function updateWith(type, id, fn) {
  const store = storeFor(type);
  if (!store.isValidId(id)) return null;
  return store.queueRecordWrite(id, async () => {
    const existing = await store.loadOne(id);
    // A tombstoned record is gone — treat it as not-found rather than reviving it.
    if (!existing || isTombstone(existing)) {
      return null;
    }

    const updates = await fn({ id, ...existing });
    if (!updates) return null;

    const record = {
      ...existing,
      ...updates,
      // Preserve immutable fields — originInstanceId tracks the creating instance
      originInstanceId: existing.originInstanceId,
      createdAt: existing.createdAt,
      updatedAt: supersedingTimestamp(existing.updatedAt, now())
    };

    await store.saveOneNow(id, record);
    brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
    await brainSyncLog.appendLocalChanges([{ op: 'update', type, id, record, originInstanceId: record.originInstanceId }]);

    console.log(`🧠 Updated ${type} record: ${id}`);
    return { id, ...record };
  });
}

/**
 * Upsert a record under a CALLER-PROVIDED id (full replace, create-if-missing).
 *
 * `create()` mints a uuid, which is wrong for stores whose identity is a natural
 * key — the Daily Log keys entries by calendar date so the same day converges
 * across peers instead of forking into per-machine uuids. This primitive lets
 * such a store own its id while still riding the exact entity-store contract:
 * it preserves `originInstanceId`/`createdAt` from an existing live record,
 * stamps a fresh `updatedAt` (the LWW clock), appends a create/update entry to
 * the sync log, and emits `${type}:upserted` unless the caller suppresses it.
 *
 * `recordData` is stored verbatim except for the three managed fields — callers
 * pass the FULL desired record (this does not merge unknown fields the way
 * `update()` does). `emitEvent:false` lets a caller (e.g. brainJournal) emit its
 * own richer, bridge-shaped event instead of the generic one.
 * A function receives the fresh live `{ id, ...record }` (or null) INSIDE the
 * per-id queue and returns the full replacement. Returning null/undefined
 * leaves storage, events and the sync log untouched and returns the live record.
 * Use this form for derived content and preconditions that must serialize with
 * peer applies as well as other local writes.
 * createOnly:true returns null for any existing record, including tombstones.
 */
export async function upsertWithId(type, id, recordData, { emitEvent = true, createOnly = false } = {}) {
  const store = storeFor(type);
  // A malformed id can't name a valid per-record dir — mirror the other write
  // paths and no-op rather than letting the store's write-queue assertion throw.
  if (!store.isValidId(id)) return null;
  // Resolve our instance id outside the queue (independent read); the existing
  // record's origin is preferred inside the queue when one is present.
  const fallbackOrigin = await getInstanceId();
  return store.queueRecordWrite(id, async () => {
    const existing = await store.loadOne(id);
    // Ingestion must not overwrite an edit or resurrect a tombstone.
    // Test presence inside the queue so a peer or local write cannot race.
    if (createOnly && existing) return null;
    const live = existing && !isTombstone(existing) ? existing : null;
    const input = typeof recordData === 'function'
      ? await recordData(live ? { id, ...live } : null)
      : recordData;
    if (typeof recordData === 'function' && input == null) return live ? { id, ...live } : null;
    const timestamp = now();
    const originInstanceId = live?.originInstanceId ?? fallbackOrigin;

    const record = {
      ...input,
      originInstanceId,
      createdAt: live?.createdAt ?? timestamp,
      updatedAt: supersedingTimestamp(existing?.updatedAt, timestamp)
    };

    await store.saveOneNow(id, record);
    if (emitEvent) {
      brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
    } else {
      // A suppressed per-record event still has to reach derived projections.
      emitRecordChanged(type, id);
    }
    await brainSyncLog.appendLocalChanges([{ op: live ? 'update' : 'create', type, id, record, originInstanceId }]);

    console.log(`🧠 Upserted ${type} record: ${id}`);
    return { id, ...record };
  });
}

/**
 * Apply many record updates to one store.
 *
 * With the old monolithic file, a batch (e.g. a chip reorder) had to be one
 * atomic load-modify-save or N concurrent single-record `update()` calls would
 * read overlapping baselines of the shared file and last-save-wins. Per-record
 * files remove that race entirely — writes to different ids no longer share a
 * file — so this now serializes each record on its own per-id queue and merges
 * against the freshest persisted record. `updates` is an array of { id,
 * ...fields }; unknown/tombstoned ids are skipped. Returns the updated records.
 */
export async function updateMany(type, updates) {
  const store = storeFor(type);
  const applied = [];
  for (const { id, ...fields } of updates) {
    if (!store.isValidId(id)) continue;
    const record = await store.queueRecordWrite(id, async () => {
      const existing = await store.loadOne(id);
      if (!existing || isTombstone(existing)) return null;
      const next = {
        ...existing,
        ...fields,
        // Preserve immutable fields, exactly as update() does.
        originInstanceId: existing.originInstanceId,
        createdAt: existing.createdAt,
        updatedAt: supersedingTimestamp(existing.updatedAt, now())
      };
      await store.saveOneNow(id, next);
      return next;
    });
    if (!record) continue;
    // Emit + append the sync-log entry as EACH record lands, not once after the
    // whole batch. Per-record files mean there is no all-or-nothing batch to
    // preserve, so if a later id's write throws, the records already persisted
    // must still have federated — a batch-tail loop would skip them entirely and
    // leave earlier successes local-only.
    applied.push({ id, record });
    brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
    await brainSyncLog.appendLocalChanges([{ op: 'update', type, id, record, originInstanceId: record.originInstanceId }]);
  }
  if (applied.length === 0) return [];
  console.log(`🧠 Updated ${applied.length} ${type} records in one batch`);
  return applied.map(({ id, record }) => ({ id, ...record }));
}

/**
 * Apply a batch whose records can be persisted independently and whose sync
 * entries should be appended together. Per-record queues still protect each
 * record's read-modify-write, while Promise.allSettled lets successful writes
 * reach the batch log before a failed sibling is surfaced to the caller.
 */
async function updateManyWithBatchLog(type, updates) {
  const store = storeFor(type);
  const results = await Promise.allSettled(updates.map(({ id, ...fields }) => {
    if (!store.isValidId(id)) return null;
    return store.queueRecordWrite(id, async () => {
      const existing = await store.loadOne(id);
      if (!existing || isTombstone(existing)) return null;

      const record = {
        ...existing,
        ...fields,
        // Preserve immutable fields, exactly as update() does.
        originInstanceId: existing.originInstanceId,
        createdAt: existing.createdAt,
        updatedAt: supersedingTimestamp(existing.updatedAt, now())
      };

      await store.saveOneNow(id, record);
      return { id, record };
    });
  }));
  const failures = results.filter(({ status }) => status === 'rejected');
  const applied = results
    .filter(({ status }) => status === 'fulfilled')
    .map(({ value }) => value)
    .filter(Boolean);

  for (const { id, record } of applied) {
    brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
  }

  if (applied.length > 0) {
    await brainSyncLog.appendLocalChanges(applied.map(({ id, record }) => ({
      op: 'update',
      type,
      id,
      record,
      originInstanceId: record.originInstanceId,
    })));
  }

  if (failures.length > 0) throw failures[0].reason;
  if (applied.length === 0) return [];

  console.log(`🧠 Updated ${applied.length} ${type} records in one batch`);
  return applied.map(({ id, record }) => ({ id, ...record }));
}

/**
 * Delete a record
 */
export async function remove(type, id) {
  const store = storeFor(type);
  if (!store.isValidId(id)) return false;
  return store.queueRecordWrite(id, async () => {
    const existing = await store.loadOne(id);
    // Already gone (absent) or already tombstoned — nothing to delete, and
    // re-tombstoning would mint a redundant sync-log entry that relays needlessly.
    if (!existing || isTombstone(existing)) {
      return false;
    }

    const originInstanceId = existing.originInstanceId ?? 'unknown';
    const deletedRecord = { id, ...existing };
    const ts = supersedingTimestamp(existing.updatedAt, now());
    // Retain a tombstone in place (save it as the record's value, not a hard
    // delete of the record dir) so a stale `create` echoed from a peer is
    // rejected by the LWW guard in applyRemoteRecord.
    await store.saveOneNow(id, makeTombstone(ts, originInstanceId));
    brainEvents.emit(`${type}:deleted`, { id, record: deletedRecord });
    // Wire format unchanged: the sync-log delete entry still carries only
    // { updatedAt } so an older peer (no tombstone support) applies it as a
    // plain hard delete exactly as before.
    await brainSyncLog.appendLocalChanges([{ op: 'delete', type, id, record: { updatedAt: ts }, originInstanceId }]);

    console.log(`🧠 Deleted ${type} record: ${id}`);
    return true;
  });
}

/**
 * Query records with filters
 */
export async function query(type, filters = {}) {
  const records = await getAll(type);

  return records.filter(record => {
    for (const [key, value] of Object.entries(filters)) {
      if (record[key] !== value) return false;
    }
    return true;
  });
}

// =============================================================================
// JSONL APPEND LOGS (digests, reviews)
// =============================================================================

/**
 * Load all records from a JSONL file
 */
async function loadJsonlStore(type) {
  const cache = caches[type];
  if (cache.data && (Date.now() - cache.timestamp) < CACHE_TTL_MS) {
    return cache.data;
  }

  await ensureBrainDir();
  const filePath = FILES[type];

  if (!existsSync(filePath)) {
    cache.data = [];
    cache.timestamp = Date.now();
    return cache.data;
  }

  const content = await readFile(filePath, 'utf-8');
  const lines = content.trim().split('\n').filter(line => line.trim());
  cache.data = lines.map(line => safeJSONParse(line, null)).filter(item => item !== null);
  cache.timestamp = Date.now();
  return cache.data;
}

/**
 * Append a record to a JSONL file
 */
async function appendJsonl(type, record) {
  await ensureBrainDir();
  const line = JSON.stringify(record) + '\n';
  await appendFile(FILES[type], line);

  // Invalidate cache so next read gets fresh data
  caches[type].data = null;
  caches[type].timestamp = 0;

  brainEvents.emit(`${type}:added`, record);
}

// =============================================================================
// INBOX LOG OPERATIONS
// =============================================================================

// The inbox is an id-keyed entity store (see BRAIN_ENTITY_TYPES) so it
// federates through the same delta-log + LWW + tombstone pipeline as every other
// brain type. These wrappers keep the historical getInboxLog/createInboxLog/…
// API (capturedAt sort, status counts) on top of the generic entity primitives.

// Cursor pages share one contract: rows sort newest-first (timestamp descending)
// with the id ascending as the tiebreak, and the cursor is the last row's
// `<zero-padded timestamp>_<id>` key. Continuation therefore keeps older rows
// and, at the cursor's own timestamp, only ids that sort AFTER the cursor id --
// comparing the encoded key as a string would pick the smaller ids instead and
// repeat/skip rows whenever a timestamp tie crosses a page boundary. The cursor
// row itself need not still exist. A malformed cursor yields the first page.
const cursorKeyOf = (id, timestampMs) => `${String(timestampMs).padStart(15, '0')}_${id}`;

function rowsAfterCursor(rows, cursor, timestampOf) {
  const match = /^(\d+)_(.*)$/s.exec(cursor);
  if (!match) return rows;
  const cursorMs = Number(match[1]);
  const cursorId = match[2];
  return rows.filter(([id, summary]) => {
    const ms = timestampOf(summary);
    return ms < cursorMs || (ms === cursorMs && id.localeCompare(cursorId) > 0);
  });
}

/**
 * One page of inbox entries (newest-first by capturedAt), optional status/search filters and cursor pagination.
 */
export async function getInboxPage(options = {}) {
  const { status, search, cursor, limit = 50, offset = 0 } = options;
  const boundedLimit = Math.min(Math.max(1, limit), 100);
  const rows = await resolveRecordIndex(summaryIndex, 'inbox', 0);
  const searchLower = search ? search.trim().toLowerCase() : null;

  const matching = rows.filter(([id, summary]) => {
    if (!summary) return false;
    if (status && summary.status !== status) return false;
    if (searchLower) {
      const textMatch = summary.capturedText?.toLowerCase().includes(searchLower);
      const titleMatch = summary.title?.toLowerCase().includes(searchLower);
      if (!textMatch && !titleMatch) return false;
    }
    return true;
  });

  // Bulk captures can share one timestamp. The id tiebreak keeps adjacent
  // cursor/offset pages stable so an entry is never duplicated or skipped.
  matching.sort(([idA, a], [idB, b]) => (b.capturedAtMs - a.capturedAtMs) || idA.localeCompare(idB));
  const total = matching.length;

  let remaining = matching;
  if (cursor) {
    remaining = rowsAfterCursor(matching, cursor, (s) => s.capturedAtMs);
  } else if (offset > 0) {
    remaining = matching.slice(offset);
  }

  const pageRows = remaining.slice(0, boundedLimit);
  const pageIds = pageRows.map(([id]) => id);
  const page = await Promise.all(pageIds.map((id) => getById('inbox', id)));
  const entries = page.filter(Boolean);

  const hasMore = remaining.length > boundedLimit;
  const nextCursor = hasMore && pageRows.length > 0
    ? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].capturedAtMs)
    : null;

  return {
    entries,
    items: entries,
    total,
    nextCursor
  };
}

/**
 * Get all inbox log entries (newest-first by capturedAt), optional status filter.
 */
export async function getInboxLog(options = {}) {
  const page = await getInboxPage(options);
  const arr = page.entries;
  arr.total = page.total;
  arr.nextCursor = page.nextCursor;
  arr.items = page.items;
  return arr;
}

/**
 * Get inbox log entry by ID
 */
export async function getInboxLogById(id) {
  return getById('inbox', id);
}

// The inbox's concurrent same-record write paths (a capture's create immediately
// followed by a background-classification update on the same entry, plus the
// boot recovery sweep) are serialized by create/update/remove's own per-id write
// queue — no separate inbox lock is needed.

/**
 * Create inbox log entry. `capturedAt` is the user-facing capture time (kept
 * distinct from the sync `createdAt`/`updatedAt` clocks stamped by create()).
 */
export async function createInboxLog(entry) {
  return create('inbox', { ...entry, capturedAt: entry.capturedAt || now() });
}

/**
 * Update inbox log entry (partial merge — returns null if absent/tombstoned).
 */
export async function updateInboxLog(id, updates) {
  return update('inbox', id, updates);
}

/**
 * Delete inbox log entry (tombstones in place for sync convergence).
 */
export async function deleteInboxLog(id) {
  return remove('inbox', id);
}

/**
 * Get inbox log count by status
 */
export async function getInboxLogCounts() {
  const rows = await resolveRecordIndex(summaryIndex, 'inbox', 0);

  const counts = {
    total: 0,
    classifying: 0,
    filed: 0,
    needs_review: 0,
    corrected: 0,
    done: 0,
    error: 0
  };

  for (const [, summary] of rows) {
    if (!summary) continue;
    counts.total++;
    if (counts[summary.status] !== undefined) {
      counts[summary.status]++;
    }
  }

  return counts;
}

// =============================================================================
// DIGEST OPERATIONS
// =============================================================================

/**
 * Get all digests
 */
export async function getDigests(limit = 10) {
  let records = await loadJsonlStore('digests');
  records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));
  return limit === Infinity ? records : records.slice(0, limit);
}

/**
 * Get latest digest
 */
export async function getLatestDigest() {
  const digests = await getDigests(1);
  return digests[0] || null;
}

/**
 * Create digest entry
 */
export async function createDigest(digest) {
  const record = {
    id: generateId(),
    ...digest,
    generatedAt: now()
  };

  await appendJsonl('digests', record);

  // Update meta with last digest time
  await updateMeta({ lastDailyDigest: record.generatedAt });

  console.log(`🧠 Created daily digest: ${record.id}`);
  return record;
}

// =============================================================================
// REVIEW OPERATIONS
// =============================================================================

/**
 * Get all reviews
 */
export async function getReviews(limit = 10) {
  let records = await loadJsonlStore('reviews');
  records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));
  return limit === Infinity ? records : records.slice(0, limit);
}

/**
 * Get latest review
 */
export async function getLatestReview() {
  const reviews = await getReviews(1);
  return reviews[0] || null;
}

/**
 * Create review entry
 */
export async function createReview(review) {
  const record = {
    id: generateId(),
    ...review,
    generatedAt: now()
  };

  await appendJsonl('reviews', record);

  // Update meta with last review time
  await updateMeta({ lastWeeklyReview: record.generatedAt });

  console.log(`🧠 Created weekly review: ${record.id}`);
  return record;
}

// =============================================================================
// CONVENIENCE EXPORTS FOR ENTITY TYPES
// =============================================================================

// People
export const getPeople = (filters) => filters ? query('people', filters) : getAll('people');
export const getPersonById = (id) => getById('people', id);
export const createPerson = (data) => create('people', data);
export const updatePerson = (id, data) => update('people', id, data);
export const deletePerson = (id) => remove('people', id);

// Projects
export const getProjects = (filters) => filters ? query('projects', filters) : getAll('projects');
export const getProjectById = (id) => getById('projects', id);
export const createProject = (data) => create('projects', data);
export const updateProject = (id, data) => update('projects', id, data);
export const deleteProject = (id) => remove('projects', id);

// Ideas
export const getIdeas = (filters) => filters ? query('ideas', filters) : getAll('ideas');
export const getIdeaById = (id) => getById('ideas', id);
export const createIdea = (data) => create('ideas', data);
export const updateIdea = (id, data) => update('ideas', id, data);
export const deleteIdea = (id) => remove('ideas', id);

// Admin
export const getAdminItems = (filters) => filters ? query('admin', filters) : getAll('admin');
export const getAdminById = (id) => getById('admin', id);
export const createAdminItem = (data) => create('admin', data);
export const updateAdminItem = (id, data) => update('admin', id, data);
export const deleteAdminItem = (id) => remove('admin', id);

// Memories
/**
 * Effective recency timestamp (ms epoch) for ordering a memory entry newest-first.
 *
 * Imported conversations (ChatGPT) carry the original conversation clock in
 * `sourceUpdatedAt` / `sourceCreatedAt`. A ChatGPT export is NOT ordered
 * chronologically, and every entry from one bulk import shares the same
 * `createdAt`/`updatedAt` (the import time) — so sorting on the storage clock
 * leaves imports in arbitrary export order (the user-reported bug). Prefer the
 * source clock when present, falling back to the storage clock for hand-written
 * entries. Returns 0 for a missing/unparseable timestamp so it sorts last.
 */
export const memoryRecencyMs = (record) => {
  for (const candidate of [
    record?.sourceUpdatedAt,
    record?.sourceCreatedAt,
    record?.updatedAt,
    record?.createdAt,
  ]) {
    const t = safeDate(candidate); // epoch ms, or 0 for missing/unparseable
    if (t) return t;
  }
  return 0;
};

export const getMemoryEntries = async () => {
  const entries = await getAll('memories');
  // Decorate-sort-undecorate: compute each record's recency once (a bulk ChatGPT
  // import can be hundreds of entries — recomputing it inside the comparator
  // would parse every timestamp O(n log n) times).
  return entries
    .map((entry) => ({ entry, recency: memoryRecencyMs(entry) }))
    .sort((a, b) => b.recency - a.recency)
    .map(({ entry }) => entry);
};
export const getMemoryEntryById = (id) => getById('memories', id);
export const createMemoryEntry = (data) => create('memories', data);
export const updateMemoryEntry = (id, data) => update('memories', id, data);
export const deleteMemoryEntry = (id) => remove('memories', id);

// Links
//
// EVERY link leaves this module through `readLink`/`readLinks`, which apply
// `normalizeRepoLinkFields` so a record written by an older PortOS (or arriving
// from a peer still on the GitHub-only field names) reads as a repo — see
// lib/repoLinkFields.js. Route through them when adding an accessor; the
// persisted payload itself is never rewritten (the derivation belongs to
// `createLinkFromUrl`, which owns the URL).
const readLink = async (record) => normalizeRepoLinkFields(await record);
const readLinks = async (records) => (await records).map(normalizeRepoLinkFields);
export const getLinks = (filters) => readLinks(filters ? query('links', filters) : getAll('links'));
export const getLinkById = (id) => readLink(getById('links', id));
export const createLink = (data) => readLink(create('links', data));
export const updateLink = (id, data) => readLink(update('links', id, data));
// Batch reorder: per-id write queues so a multi-chip drag merges each link
// against its freshest persisted record.
export const reorderLinks = (updates) => readLinks(updateMany('links', updates));
export const deleteLink = (id) => remove('links', id);

// ─── Link summary index (issue #3509) ───────────────────────────────────────
// Every ordering, filtering, counting, and membership question the links API
// asks is answerable from four fields. Caching just those in a projection index
// (see createRecordIndex above) means `GET /api/brain/links` reads ONLY the
// requested page's record bodies instead of the whole collection — page cost
// goes from O(N) parsed records to O(limit).
//
// `null` marks an id that is not a user-visible link (absent, unparseable, or
// tombstoned) — exactly what `getAll` drops. Tombstones stay in the id listing,
// so this must be a null projection rather than an omitted entry.
const projectLinkSummary = (record) => (record && !isTombstone(record)
  ? {
    // Pre-parsed sort key: `getAll` + `new Date()` in a comparator re-parses the
    // same timestamp O(n log n) times. `safeDate` yields 0 for a missing or
    // unparseable createdAt, so such a link sorts last deterministically
    // instead of poisoning the comparator with NaN.
    createdAtMs: safeDate(record.createdAt),
    // `linkType` is stored RAW so the route's strict `===` comparison keeps its
    // exact semantics. `isRepo` is resolved through `linkIsRepo` instead, so a
    // legacy record still carrying only `isGitHubRepo` filters correctly without
    // the index having to be rebuilt behind the migration.
    linkType: record.linkType,
    isRepo: linkIsRepo(record),
    url: record.url,
  }
  : null);
const linkSummaryIndex = createRecordIndex(projectLinkSummary);

// The Brain header and inbox gates need only these two fields from each
// user-visible record. Keeping that projection beside the link summary index
// lets their frequent count reads reuse the same per-record invalidation
// machinery without parsing full record bodies in steady state.
const projectSummary = (record) => (record && !isTombstone(record)
  ? {
    status: record.status,
    capturedAtMs: safeDate(record.capturedAt),
    capturedText: record.capturedText || record.text || '',
    title: record.title || record.extracted?.title || record.extracted?.name || '',
  }
  : null);
const summaryIndex = createRecordIndex(projectSummary);

export const MEMORY_LIST_CONTENT_CHARS = 300;

/**
 * Compact row projection for memory cards. Transcripts and long notes can reach
 * thousands of characters; the card only renders a short teaser, so clamp the
 * content at the transport boundary to keep list pages bounded. Full records
 * remain available from GET /api/brain/memories/:id.
 */
export function toMemoryListItem(memory) {
  if (!memory) return memory;
  if (typeof memory.content !== 'string') return memory;
  if (memory.content.length <= MEMORY_LIST_CONTENT_CHARS) return memory;
  return {
    ...memory,
    content: memory.content.slice(0, MEMORY_LIST_CONTENT_CHARS),
    contentTruncated: true,
  };
}

const projectEntitySummary = (record) => (record && !isTombstone(record)
  ? {
    status: record.status || null,
    archived: !!record.archived,
    sortKey: memoryRecencyMs(record) || safeDate(record.updatedAt || record.createdAt),
    searchText: [
      record.name,
      record.title,
      record.context,
      typeof record.content === 'string' ? record.content.slice(0, 500) : '',
      record.notes,
      record.oneLiner,
      record.nextAction,
      record.mood,
      ...(Array.isArray(record.tags) ? record.tags : []),
      ...(Array.isArray(record.followUps) ? record.followUps : []),
    ].filter(Boolean).join(' ').toLowerCase(),
  }
  : null);
const entitySummaryIndex = createRecordIndex(projectEntitySummary);

const resolveLinkSummaries = () => resolveRecordIndex(linkSummaryIndex, 'links', 0);

/**
 * One page of links, newest-first by `createdAt`.
 *
 * Filtering, ordering, and the `total` count all come off the cached summary
 * index; only the page's own records are loaded from disk. `linkType` /
 * `isRepo` are optional and compared strictly, matching the filters the route
 * used to apply in memory.
 */
export async function getLinksPage({ linkType, isRepo, limit = 50, offset = 0 } = {}) {
  const rows = await resolveLinkSummaries();
  // The two filters guard differently ON PURPOSE, mirroring the route code this
  // replaced: `linkType` was a truthiness check (no link type is the empty
  // string, so `''` means "don't filter"), while `isRepo` must honour an
  // explicit `false` and so can only be skipped when it is genuinely absent.
  const matching = rows.filter(([, summary]) => summary
    && (!linkType || summary.linkType === linkType)
    && (isRepo === undefined || summary.isRepo === isRepo));

  // Newest-first, with the id as a deterministic tiebreak: a bulk import stamps
  // many links with the SAME createdAt, and an unstable order across two page
  // requests would drop or duplicate records at the slice boundary.
  matching.sort(([idA, a], [idB, b]) => (b.createdAtMs - a.createdAtMs) || idA.localeCompare(idB));

  const total = matching.length;
  const pageIds = matching.slice(offset, offset + limit).map(([id]) => id);
  const page = await readLinks(Promise.all(pageIds.map((id) => getById('links', id))));
  // A record deleted between the index read and the body read comes back null —
  // drop it rather than emitting a hole in the page.
  return { links: page.filter(Boolean), total };
}

/**
 * Ids of the user-visible (non-tombstoned) links, from the same summary index —
 * a membership check that parses no record bodies in steady state.
 */
export async function listLinkIds() {
  const rows = await resolveLinkSummaries();
  return rows.filter(([, summary]) => summary).map(([id]) => id);
}

/**
 * Find link by URL. Resolves the id from the summary index (no bodies) and
 * loads only the one match.
 */
export async function getLinkByUrl(url) {
  const rows = await resolveLinkSummaries();
  const hit = rows.find(([, summary]) => summary && summary.url === url);
  return hit ? readLink(getById('links', hit[0])) : null;
}

/**
 * One page of entity records (people, projects, ideas, admin, memories) with
 * stable cursor pagination, query-wide search, status filter, and compact row
 * projections for memory list items.
 */
export async function getEntityPage(type, options = {}) {
  const {
    status,
    search,
    cursor,
    limit = 25,
    offset = 0,
    includeArchived = false,
  } = options;
  const boundedLimit = Math.min(Math.max(1, limit), 100);
  const rows = await resolveRecordIndex(entitySummaryIndex, type, 0);
  const searchLower = search ? search.trim().toLowerCase() : null;

  const matching = rows.filter(([, summary]) => {
    if (!summary) return false;
    if (!includeArchived && summary.archived) return false;
    if (status && summary.status !== status) return false;
    if (searchLower && !summary.searchText.includes(searchLower)) return false;
    return true;
  });

  // Sort newest-first with id as stable tiebreak
  matching.sort(([idA, a], [idB, b]) => (b.sortKey - a.sortKey) || idA.localeCompare(idB));
  const total = matching.length;

  let remaining = matching;
  if (cursor) {
    remaining = rowsAfterCursor(matching, cursor, (s) => s.sortKey);
  } else if (offset > 0) {
    remaining = matching.slice(offset);
  }

  const pageRows = remaining.slice(0, boundedLimit);
  const pageIds = pageRows.map(([id]) => id);
  const rawPage = await Promise.all(pageIds.map((id) => getById(type, id)));
  let items = rawPage.filter(Boolean);

  if (type === 'memories') {
    items = items.map(toMemoryListItem);
  }

  const hasMore = remaining.length > boundedLimit;
  const nextCursor = hasMore && pageRows.length > 0
    ? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].sortKey)
    : null;

  return {
    items,
    total,
    nextCursor,
  };
}

// Buckets (bookmark groups for links)
export const getBuckets = (filters) => filters ? query('buckets', filters) : getAll('buckets');
export const getBucketById = (id) => getById('buckets', id);
export const createBucket = (data) => create('buckets', data);
export const updateBucket = (id, data) => update('buckets', id, data);
// Bucket order changes are one user gesture; persist the independent records
// together and append their federation entries with one sync-log write.
export const reorderBuckets = (updates) => updateManyWithBatchLog('buckets', updates);
export const deleteBucket = (id) => remove('buckets', id);

// Threads (#7664)
//
// A thread is one tracked topic or commitment — an open loop in the bullet
// journal sense, NOT a message thread (messageSync.js owns that word too). The
// accessors are the generic store API under thread-shaped names, exactly like
// the siblings above; the ref registry that gives a thread its links lives in
// lib/threadRefKinds.js + services/threadRefs.js.
export const getThreads = (filters) => filters ? query('threads', filters) : getAll('threads');
export const getThreadById = (id) => getById('threads', id);
export const createThread = (data) => create('threads', data);
export const updateThread = (id, data) => update('threads', id, data);
export const deleteThread = (id) => remove('threads', id);

// =============================================================================
// REMOTE SYNC OPERATIONS (no per-record events, no sync log — echo prevention)
// =============================================================================
// The only event these emit is the local-only `record:changed` invalidation
// signal (see emitRecordChanged) — it never leaves this instance.

/**
 * The LWW rejection for an incoming op whose clock is not newer than ours.
 *
 * `local_current` means we already hold exactly the state the op carries — same
 * LWW clock, same delete-ness — so the op is ALREADY APPLIED here (an echo, or a
 * delta re-pulled after a crash between our save and our relay append). The
 * wire-shape of our copy rides along so the caller can relay it if our sync log
 * lacks it (#8316). Anything else is `local_newer`: a genuinely stale op.
 */
function rejectedAgainst(existing, record, incomingIsDelete) {
  if (existing.updatedAt !== record.updatedAt || isTombstone(existing) !== incomingIsDelete) {
    return { applied: false, reason: 'local_newer' };
  }
  const current = isTombstone(existing)
    ? { updatedAt: existing.updatedAt, originInstanceId: existing.originInstanceId }
    : { ...existing };
  return { applied: false, reason: 'local_current', current };
}

/**
 * Apply a remote record to a store (last-writer-wins by updatedAt).
 * Rejections carry `reason` `local_newer` (stale op) or `local_current` (op
 * already applied — `current` is our copy in wire shape); see rejectedAgainst.
 */
export async function applyRemoteRecord(type, id, record, op) {
  const store = storeFor(type);
  // Reject a malformed id (can't name a valid per-record dir) or a
  // prototype-polluting name — the store's `isValidId` owns both the format
  // allowlist AND the reserved-key denylist (`__proto__`/`constructor`/
  // `prototype`), so a bad id returns `invalid_id` here instead of throwing on
  // the write-queue assertion or appending a phantom relay entry that can never
  // converge (this covers BOTH the delta-sync and reconcile-snapshot callers).
  if (!store.isValidId(id)) {
    return { applied: false, reason: 'invalid_id' };
  }
  // Both delta pulls and snapshots bypass the HTTP push schema. Reject invalid
  // clocks here too, before any write or relay, while preserving missing-clock
  // skips for legacy peers. Parseable offsets/legacy formats remain supported.
  if (!record?.updatedAt) {
    return { applied: false, reason: 'missing_timestamp' };
  }
  if (parseTsMs(record.updatedAt) === null) {
    return { applied: false, reason: 'invalid_timestamp' };
  }
  return store.queueRecordWrite(id, async () => {
    const existing = await store.loadOne(id);

    if (op === 'delete') {
      // LWW: skip if our copy (live record OR existing tombstone) is at least as
      // new as the incoming delete. The tombstone-vs-tombstone case makes a
      // repeated delete idempotent → not relayed → the echo loop converges.
      if (existing && !compareNewerWins(record.updatedAt, existing.updatedAt)) {
        return rejectedAgainst(existing, record, true);
      }
      // Tombstone in place even when no local record exists. A delete that
      // arrives before we ever saw a create still leaves a marker, so a later
      // stale create for that id is rejected instead of resurrecting.
      await store.saveOneNow(id, makeTombstone(
        record.updatedAt,
        record.originInstanceId ?? existing?.originInstanceId
      ));
    } else {
      // Guard also fires when `existing` is a tombstone — a stale create (older
      // updatedAt than the recorded delete) is rejected, breaking the
      // resurrection loop. A genuinely newer create (later updatedAt than the
      // tombstone) still wins and legitimately revives the record.
      if (existing && !compareNewerWins(record.updatedAt, existing.updatedAt)) {
        return rejectedAgainst(existing, record, record._deleted === true);
      }
      // Defense-in-depth: a create carrying `_deleted` (a future peer, or a
      // direct caller bypassing brainSync's reroute) must persist as a proper
      // tombstone, never as a malformed live record missing `deletedAt`.
      await store.saveOneNow(id, record._deleted
        ? makeTombstone(record.updatedAt, record.originInstanceId ?? existing?.originInstanceId)
        : (type === 'links' ? mergeIncomingLinkRecord(record, existing) : { ...record }));
    }

    // Still no per-record event (that would echo back to the peer, #1077) —
    // just the local invalidation signal so derived projections re-read.
    emitRecordChanged(type, id);
    return { applied: true };
  });
}

/**
 * Hard-prune tombstones older than `cutoffMs` (a Date.now()-style epoch ms).
 * Called by the brain tombstone GC sweep on the orchestrator's interval.
 * Per-id re-check inside the write queue so a tombstone a concurrent apply just
 * refreshed isn't pruned on a stale read. Returns the number of tombstones
 * removed.
 */
export async function pruneTombstones(type, cutoffMs) {
  const store = storeFor(type);
  const ids = await store.listIds();
  // Different ids are independent files — prune them in parallel (each still
  // serialized on its own per-id queue so the stale-read recheck holds). The
  // count is order-independent.
  const results = await Promise.all(ids.map((id) => store.queueRecordWrite(id, async () => {
    const record = await store.loadOne(id);
    if (!isTombstone(record)) return false;
    const deletedAt = Date.parse(record.deletedAt ?? record.updatedAt ?? '');
    if (Number.isFinite(deletedAt) && deletedAt < cutoffMs) {
      await store.deleteOneNow(id);
      emitRecordChanged(type, id);
      return true;
    }
    return false;
  })));
  return results.filter(Boolean).length;
}

/**
 * Backfill originInstanceId on records missing it (run once at startup)
 */
export async function backfillOriginInstanceId() {
  const instanceId = await getInstanceId();

  // Every record across all types is an independent file — backfill them in
  // parallel (per-id writes still serialize on their own queue). This runs at
  // startup and gates sync start, so the parallelism directly shortens boot.
  const perType = await Promise.all(BRAIN_ENTITY_TYPES.map(async (type) => {
    const store = storeFor(type);
    const ids = await store.listIds();
    const changes = await Promise.all(ids.map((id) => store.queueRecordWrite(id, async () => {
      const record = await store.loadOne(id);
      // Tombstones always carry originInstanceId; skip them and absent records.
      if (!record || isTombstone(record) || record.originInstanceId) return false;
      await store.saveOneNow(id, { ...record, originInstanceId: instanceId });
      emitRecordChanged(type, id);
      return true;
    })));
    return changes.filter(Boolean).length;
  }));

  const totalBackfilled = perType.reduce((sum, n) => sum + n, 0);
  if (totalBackfilled > 0) {
    console.log(`🧠 Backfilled originInstanceId on ${totalBackfilled} records`);
  }
}

// =============================================================================
// UTILITY FUNCTIONS
// =============================================================================

/**
 * Invalidate the non-collection caches (meta + JSONL logs) and every projection
 * index. Entity store BODIES still read through to disk, so there is no
 * whole-record entity cache to clear.
 */
export function invalidateAllCaches() {
  for (const key of Object.keys(caches)) {
    caches[key].data = null;
    caches[key].timestamp = 0;
  }
  for (const index of recordIndexes) {
    for (const type of BRAIN_ENTITY_TYPES) {
      index.byType[type].clear();
      index.generation[type] += 1;
    }
  }
  // Live restores replace files without record events. Drop derived caches too,
  // without turning cache invalidation into a write or federation announcement.
  brainEvents.emit('caches:invalidated');
}

/**
 * Get brain data summary (for dashboard)
 */
export async function getSummary() {
  const types = ['people', 'projects', 'ideas', 'admin', 'memories', 'buckets'];
  const [typeRows, linkRows, inboxCounts, meta] = await Promise.all([
    Promise.all(types.map((type) => resolveRecordIndex(summaryIndex, type, 0))),
    resolveLinkSummaries(),
    getInboxLogCounts(),
    loadMeta()
  ]);
  const summaries = Object.fromEntries(types.map((type, i) => [
    type,
    typeRows[i].map(([, summary]) => summary).filter(Boolean),
  ]));
  const links = linkRows.map(([, summary]) => summary).filter(Boolean);

  return {
    counts: {
      people: summaries.people.length,
      projects: summaries.projects.length,
      ideas: summaries.ideas.length,
      admin: summaries.admin.length,
      memories: summaries.memories.length,
      links: links.length,
      buckets: summaries.buckets.length,
      inbox: inboxCounts
    },
    activeProjects: summaries.projects.filter(p => p.status === 'active').length,
    activeIdeas: summaries.ideas.filter(i => !i.status || i.status === 'active').length,
    openAdmin: summaries.admin.filter(a => a.status === 'open').length,
    repos: links.filter(l => l.isRepo).length,
    needsReview: inboxCounts.needs_review,
    lastDailyDigest: meta.lastDailyDigest,
    lastWeeklyReview: meta.lastWeeklyReview
  };
}