Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Brain Storage Service
*
* Handles file-based persistence for the Brain feature.
* - Per-record `collectionStore` dirs for entity stores (people, projects,
* ideas, admin, …, journals, inbox, songs, threads) — `data/brain/<type>/<id>/index.json`
* - JSON for the single `meta.json` settings doc
* - JSONL for append-only generated logs (digests, reviews)
*
* Storage layout (issue #725): each entity store is a `collectionStore` — one
* file per record, read through to disk with no whole-store in-memory cache. A
* write touches one record's file and per-id write queues let writes to
* DIFFERENT records proceed in parallel (the scalability win over the old
* monolithic `data/brain/<type>.json` + 2s-TTL cache + single global mutex).
* Migration 200 splits the legacy monolithic files into this shape.
*
* The old single global write mutex is replaced by the store's per-id
* `queueRecordWrite(id, fn)`: two writes to the SAME (type, id) serialize —
* local create/update AND remote peer applies AND tombstone GC all queue on the
* same per-id tail, so a peer sync landing mid-way through a local write can't
* read a stale snapshot and drop the other write (the AGENTS.md-sanctioned
* "serialize two write paths that mutate the same record" case). Writes to
* different records no longer serialize against each other. brainJournal layers
* its own storeMutex ON TOP for local sidecar coordination; journal transforms
* run inside this per-id queue too, so peer applies cannot invalidate their
* snapshot. Locks only nest storeMutex → this queue (one direction).
*
* FEDERATION is unaffected by the layout. Brain federates strictly per-record
* through this module's API seams (`getRawRecords` / `applyRemoteRecord`) plus
* the `brainSyncLog` delta log — never by shipping a whole-store file — so the
* on-disk container shape is orthogonal to the wire format (no `schemaVersions.js`
* gate; brain is intentionally ungated there).
*/
import { readFile, appendFile } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { v4 as uuidv4 } from '../lib/uuid.js';
import EventEmitter from 'events';
import { atomicWrite, ensureDir, readJSONFile, safeJSONParse, safeDate, PATHS } from '../lib/fileUtils.js';
import { getInstanceId } from './instanceIdentity.js';
import * as brainSyncLog from './brainSyncLog.js';
import { createCollectionStore } from '../lib/collectionStore.js';
import { linkIsRepo, normalizeRepoLinkFields } from '../lib/repoLinkFields.js';
import { supersedingTimestamp } from '../lib/tombstones.js';
import { parseTsMs, compareNewerWins } from '../lib/lwwTimestamp.js';
import { mergeIncomingLinkRecord } from '../lib/brainLinkLocalFields.js';
const DATA_DIR = PATHS.brain;
// The entity stores that participate in peer sync (records with IDs).
// Canonical list — sync, tombstone GC, origin backfill, and the per-type
// collectionStore construction below all derive from it so adding a type can't
// silently drop out of one of those paths. (Boot-time migrations keep their own
// frozen copy by necessity — migrations run before the service layer is wired
// up — see scripts/migrations/080-*.js and 200-*.js.)
//
// `journals` (the Daily Log) and `inbox` were added in migration 081; both are
// `{ id: record }` stores exactly like the others (journals keyed by date,
// inbox by uuid), so the delta log, anti-entropy reconcile, tombstone GC, and
// originInstanceId backfill all cover them with no per-type branching.
export const BRAIN_ENTITY_TYPES = Object.freeze([
'people', 'projects', 'ideas', 'admin', 'memories', 'links', 'buckets',
'journals', 'inbox', 'songs', 'threads',
]);
// The type-level storage-layout version. Bumped by a migration that changes the
// on-disk layout; distinct from any per-record field-shape version (brain
// records carry none). Migration 200 stamps this on every brain type index.
export const BRAIN_STORE_SCHEMA_VERSION = 1;
// One collectionStore per entity type, rooted at data/brain/<type>/. No
// `sanitizeRecord` — brain records (and their in-place tombstones) are stored
// and loaded verbatim, exactly as the old monolithic map did.
const brainStores = Object.freeze(Object.fromEntries(
BRAIN_ENTITY_TYPES.map((type) => [type, createCollectionStore({
dir: join(DATA_DIR, type),
type,
schemaVersion: BRAIN_STORE_SCHEMA_VERSION,
})]),
));
function storeFor(type) {
const store = brainStores[type];
if (!store) throw new Error(`brainStorage: unknown entity type "${type}"`);
return store;
}
/**
* The per-type collectionStores, for the boot-time schema-version verifier in
* server/index.js (`verifyCollectionVersions`).
*/
export function brainCollectionStores() {
return BRAIN_ENTITY_TYPES.map((type) => brainStores[type]);
}
// A tombstone is a deleted-record marker kept IN PLACE as the record's stored
// value (its `{id}/index.json`) — rather than removing the record dir — so the
// last-writer-wins guard in applyRemoteRecord can reject a stale `create`
// echoed back from a peer. Without it, a hard delete leaves `existing ===
// undefined`, the LWW guard is skipped, and the record resurrects — then the
// newer delete re-kills it, and both ops relay to every peer forever (the
// federated brain-sync loop). Shape:
// { _deleted: true, updatedAt, originInstanceId, deletedAt }. The `updatedAt`
// is the LWW clock; `deletedAt` is the GC clock.
const isTombstone = (rec) => !!(rec && rec._deleted);
// Build the in-place tombstone marker. `updatedAt` is the LWW clock (must be the
// delete's timestamp); `deletedAt` is the GC clock — equal at birth, kept
// separate so the GC sweep reads its own field.
const makeTombstone = (updatedAt, originInstanceId) => ({
_deleted: true,
updatedAt,
originInstanceId: originInstanceId ?? 'unknown',
deletedAt: updatedAt,
});
// GC grace period: how long a tombstone is retained before it can be hard-
// pruned. Must comfortably exceed the longest realistic peer-offline window so
// a peer that reconnects still sees the delete (not a since-vanished id that it
// would re-create). 30 days matches the sharing-side tombstone grace buffer.
export const BRAIN_TOMBSTONE_GRACE_MS = 30 * 24 * 60 * 60 * 1000;
// Non-collection files: the single settings doc and the append-only JSONL logs.
const FILES = {
meta: join(DATA_DIR, 'meta.json'),
digests: join(DATA_DIR, 'digests.jsonl'),
reviews: join(DATA_DIR, 'reviews.jsonl'),
};
// Event emitter for brain data changes
export const brainEvents = new EventEmitter();
/**
* Announce a persisted entity-store change that did NOT emit the usual
* `${type}:upserted` / `${type}:deleted` event.
*
* Some write paths are deliberately event-silent — `applyRemoteRecord` (peer
* applies, silent to prevent the #1077 cross-peer echo loop),
* `upsertWithId({ emitEvent: false })` (the caller emits its own richer event),
* tombstone pruning, and origin-instance backfill. Read-through consumers did
* not care, but derived in-memory state does: without this signal,
* brainSearchIndex or the reconcile checksum cache could stay stale.
*
* LOCAL-ONLY, like `sync:applied`: it carries no record payload, is never fed
* to the sync log, and is never relayed to a peer, so it cannot amplify an
* echo. Listeners must treat it as "re-read this type from canonical state".
*/
function emitRecordChanged(type, id) {
brainEvents.emit('record:changed', { type, id });
}
// In-memory caches — only for the non-collection files. Entity stores read
// through to disk (per issue #725: no whole-store cache to go stale when a
// per-record write bypasses it).
const caches = {
meta: { data: null, timestamp: 0 },
digests: { data: null, timestamp: 0 },
reviews: { data: null, timestamp: 0 },
};
const CACHE_TTL_MS = 2000;
// Default settings
const DEFAULT_META = {
version: 1,
confidenceThreshold: 0.6,
dailyDigestTime: '00:00',
weeklyReviewTime: '00:00',
weeklyReviewDay: 'sunday',
defaultProvider: 'lmstudio',
defaultModel: 'gptoss-20b',
// Opt-in: digests/reviews call an AI provider, so nothing runs until enabled.
scheduledReviewsEnabled: false,
lastDailyDigest: null,
lastWeeklyReview: null
};
/**
* Ensure brain data directory exists
*/
export async function ensureBrainDir() {
await ensureDir(DATA_DIR);
}
/**
* Generate a new UUID
*/
export function generateId() {
return uuidv4();
}
/**
* Get current ISO timestamp
*/
export function now() {
return new Date().toISOString();
}
// =============================================================================
// META / SETTINGS
// =============================================================================
/**
* Load brain settings
*/
export async function loadMeta() {
const cache = caches.meta;
if (cache.data && (Date.now() - cache.timestamp) < CACHE_TTL_MS) {
return cache.data;
}
await ensureBrainDir();
// Strict (#4115): `updateMeta` is `loadMeta → spread updates → saveMeta`, so a
// swallowed unreadable meta.json writes DEFAULT_META over every brain setting
// on the next update — and the result is cached for CACHE_TTL_MS, so the fake
// default outlives the read that produced it.
const loaded = await readJSONFile(FILES.meta, null, { strict: true });
cache.data = loaded ? { ...DEFAULT_META, ...loaded } : { ...DEFAULT_META };
cache.timestamp = Date.now();
return cache.data;
}
/**
* Save brain settings
*/
export async function saveMeta(meta) {
await ensureBrainDir();
await atomicWrite(FILES.meta, meta);
caches.meta.data = meta;
caches.meta.timestamp = Date.now();
brainEvents.emit('meta:changed', meta);
}
/**
* Update brain settings (partial update)
*/
export async function updateMeta(updates) {
const meta = await loadMeta();
const updated = { ...meta, ...updates };
await saveMeta(updated);
return updated;
}
// =============================================================================
// ENTITY STORES (per-record collectionStore, read-through)
// =============================================================================
/**
* Assemble the raw `{ id: record }` map for a type from its per-record files,
* INCLUDING tombstones. Read-through: one `loadOne` per id, in parallel. This
* is the seam the reconcile/anti-entropy path and getAll/getRawRecords build on
* — identical return shape to the old monolithic `loadJsonStore(type).records`.
*/
async function loadRawMap(type) {
const store = storeFor(type);
const ids = await store.listIds();
const entries = await Promise.all(ids.map(async (id) => {
const record = await store.loadOne(id);
return record ? [id, record] : null;
}));
return Object.fromEntries(entries.filter(Boolean));
}
/**
* Get all records from a store (tombstones excluded — they exist only to anchor
* the LWW sync guard, never as user-visible records).
*/
export async function getAll(type) {
const map = await loadRawMap(type);
return Object.entries(map)
.filter(([, record]) => !isTombstone(record))
.map(([id, record]) => ({ id, ...record }));
}
// ─── Per-record projection indexes (issues #3508, #3509) ────────────────────
// A projection index answers one question about EVERY record in a store — "is
// this id live?" (#3508), "what are this link's sort/filter keys?" (#3509) —
// WITHOUT re-reading and re-parsing every record body on every call, the way
// `getAll(type)` must.
//
// Cost model per call: one `listIds()` (readdir + lstat, no body reads) plus a
// `loadOne` for ids the index has never resolved. Steady state is zero body
// reads; a store the caller has already walked answers from memory.
//
// FRESHNESS — the same three signal classes brainSearchIndex documents:
// 1. Adds and hard-deletes (create, pruneTombstones) — covered by the
// `listIds()` diff itself; membership is re-read from disk every call.
// 2. IN-PLACE mutations that keep the id: an `archived` flip, or `remove()`
// writing a tombstone over the record. The id survives the diff, so these
// ride the per-record `${type}:upserted` / `${type}:deleted` events.
// 3. Event-SILENT writes — `applyRemoteRecord` (silent to prevent the #1077
// echo loop) and `upsertWithId({ emitEvent: false })` — ride the local-only
// `record:changed` invalidation signal.
//
// No `null = not built` sentinel is needed here (unlike brainSearchIndex):
// `listIds()` is the membership authority on every call, so an empty index is a
// legitimate "nothing resolved yet" state and never causes a redundant re-scan.
//
// Two write paths are deliberately event-silent and NOT covered by the three
// classes above, because neither can change what any projection reads:
// `pruneTombstones` hard-deletes the record dir, so the id leaves `listIds()`
// and the diff drops it; `backfillOriginInstanceId` only stamps
// `originInstanceId`, a field no projection looks at, and runs once at boot.
// A NEW projection that reads either signal must invalidate from those paths.
const RECORD_INDEX_MAX_ATTEMPTS = 3;
// Every index registers itself here so ONE invalidation drops the touched id
// from all of them — an index that missed a signal would serve a stale
// projection until the next unrelated change to that record.
const recordIndexes = [];
/**
* Build a per-type projection index. `project(record)` maps the parsed on-disk
* record (or `null` for a missing/corrupt one) to the value to cache for that
* id. Registers itself for invalidation; call `resolveRecordIndex` to read it.
*/
function createRecordIndex(project) {
const index = {
project,
byType: Object.fromEntries(BRAIN_ENTITY_TYPES.map((type) => [type, new Map()])),
// Bumped by every invalidation, so a resolve pass that raced an invalidation
// is retried instead of writing the value it read just before the change landed.
generation: Object.fromEntries(BRAIN_ENTITY_TYPES.map((type) => [type, 0])),
};
recordIndexes.push(index);
return index;
}
function invalidateRecordIndexes(type, id) {
if (!id) return;
for (const index of recordIndexes) {
const byType = index.byType[type];
if (!byType) continue;
byType.delete(id);
index.generation[type] += 1;
}
}
/**
* Resolve a projection index for one store. Returns `[id, value]` pairs in
* `listIds()` order — every present id, including ones whose projection is
* `null` (callers decide what a null projection means).
*/
async function resolveRecordIndex(indexState, type, attempt) {
// `storeFor` throws for an unknown type, and the index maps are keyed off the
// same BRAIN_ENTITY_TYPES list — so past this line `byType[type]` exists.
const store = storeFor(type);
const index = indexState.byType[type];
const generation = indexState.generation[type];
const ids = await store.listIds();
const present = new Set(ids);
for (const id of [...index.keys()]) {
if (!present.has(id)) index.delete(id);
}
const unresolved = ids.filter((id) => !index.has(id));
const loaded = await Promise.all(unresolved.map((id) => store.loadOne(id)));
const resolved = new Map(unresolved.map((id, i) => [id, indexState.project(loaded[i])]));
// An invalidation landed while we were reading, so what we loaded may already
// describe the pre-change record. Re-run — the retry re-reads only the ids the
// invalidation dropped, so it is cheap. Past the attempt ceiling we answer
// from what we have but do NOT write it into the index: caching a value we
// know raced would strand it there until the next change to that record.
const raced = indexState.generation[type] !== generation;
if (raced && attempt + 1 < RECORD_INDEX_MAX_ATTEMPTS) {
return resolveRecordIndex(indexState, type, attempt + 1);
}
if (raced) {
// Ceiling reached. The invalidation may also have dropped ids this pass
// took FROM the index (so they were never in `unresolved`) — read those
// before answering, or a record we merely stopped knowing about would be
// reported as absent and undercount the result.
const unknown = ids.filter((id) => !resolved.has(id) && !index.has(id));
const reloaded = await Promise.all(unknown.map((id) => store.loadOne(id)));
unknown.forEach((id, i) => resolved.set(id, indexState.project(reloaded[i])));
} else {
for (const [id, value] of resolved) index.set(id, value);
}
return ids.map((id) => [id, resolved.has(id) ? resolved.get(id) : index.get(id)]);
}
// Live = present, not a tombstone, and not archived. `loadOne` returns null for
// a missing/corrupt/unparseable record, which `getAll` also drops — so it is
// not live either.
const isLiveRecord = (record) => !!record && !isTombstone(record) && !record.archived;
const liveIdIndex = createRecordIndex(isLiveRecord);
/**
* Ids of the LIVE records in a store — present, not tombstoned, not archived —
* without loading record bodies for ids the index has already resolved.
*
* This is the cheap counterpart to `getAll(type).filter(r => !r.archived)` for
* callers that only need identity (embedding-coverage tallies, membership
* checks). Returns a fresh array in `listIds()` order each call.
*/
export async function listLiveIds(type) {
const rows = await resolveRecordIndex(liveIdIndex, type, 0);
return rows.filter(([, live]) => live === true).map(([id]) => id);
}
// Wire the freshness signals once, at module load. Invalidation only drops the
// touched id, so the next resolve re-reads exactly one record.
//
// The `journals` type is the one that needs checking rather than assuming:
// brainJournal re-emits `journals:upserted` / `journals:deleted` under the SAME
// names with its own richer `{ entry }` / `{ date, entry }` payloads, which
// carry no top-level `id`. Those extra emits are harmless no-ops here — every
// journal write still routes through brainStorage first (`putEntry` →
// `upsertWithId({ emitEvent: false })` → `record:changed`; `deleteJournal` →
// `remove()` → `journals:deleted` with `{ id }`), which is what actually
// invalidates.
for (const type of BRAIN_ENTITY_TYPES) {
brainEvents.on(`${type}:upserted`, ({ id } = {}) => invalidateRecordIndexes(type, id));
brainEvents.on(`${type}:deleted`, ({ id } = {}) => invalidateRecordIndexes(type, id));
}
brainEvents.on('record:changed', ({ type, id } = {}) => invalidateRecordIndexes(type, id));
/**
* Get the RAW records map for a store, INCLUDING tombstones, keyed by id.
*
* Unlike `getAll` (which strips tombstones for user-facing reads), the sync
* reconcile path needs tombstones too: they carry the LWW `updatedAt` clock a
* peer must see to keep a delete from resurrecting. Returns a fresh `{ id:
* record }` map — safe to mutate the map, but the record objects are the parsed
* on-disk values, so callers treat them as read-only (the reconcile path only
* serializes them, never mutates).
*/
export async function getRawRecords(type) {
return loadRawMap(type);
}
/**
* Get a record by ID
*/
export async function getById(type, id) {
const record = await storeFor(type).loadOne(id);
return record && !isTombstone(record) ? { id, ...record } : null;
}
/**
* Create a new record
*/
export async function create(type, recordData) {
const store = storeFor(type);
const originInstanceId = await getInstanceId();
const id = generateId();
const timestamp = now();
const record = {
...recordData,
originInstanceId,
createdAt: timestamp,
updatedAt: timestamp
};
// Fresh uuid → no read-modify-write; saveOne queues per-id internally.
await store.saveOne(id, record);
brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
await brainSyncLog.appendLocalChanges([{ op: 'create', type, id, record, originInstanceId }]);
console.log(`🧠 Created ${type} record: ${id}`);
return { id, ...record };
}
/**
* Update a record
*/
export async function update(type, id, updates) {
const store = storeFor(type);
// A malformed id can't name a valid per-record dir — treat as not-found
// rather than letting the store's write-queue assertion throw on it.
if (!store.isValidId(id)) return null;
return store.queueRecordWrite(id, async () => {
const existing = await store.loadOne(id);
// A tombstoned record is gone — treat it as not-found rather than reviving it.
if (!existing || isTombstone(existing)) {
return null;
}
const record = {
...existing,
...updates,
// Preserve immutable fields — originInstanceId tracks the creating instance
originInstanceId: existing.originInstanceId,
createdAt: existing.createdAt,
// Local edits must outrank the version read inside this queue, even
// within one millisecond or after receiving a peer's faster clock.
updatedAt: supersedingTimestamp(existing.updatedAt, now())
};
await store.saveOneNow(id, record);
brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
await brainSyncLog.appendLocalChanges([{ op: 'update', type, id, record, originInstanceId: record.originInstanceId }]);
console.log(`🧠 Updated ${type} record: ${id}`);
return { id, ...record };
});
}
/**
* Locked read-modify-write update.
*
* `update(type, id, partial)` merges a partial the CALLER computed — usually
* from a record snapshot read OUTSIDE the store write queue. When the partial
* is derived from the current record (append to / filter an array field like
* `attachments`), a concurrent writer landing between the snapshot read and
* the update silently gets clobbered (and the clobber wins LWW federation).
*
* `updateWith` closes that window: INSIDE the per-id write queue it re-reads the
* fresh record, calls `fn({ id, ...record })` → a partial-updates object (or
* null/undefined to abort without writing), then merges/persists with exactly
* `update()`'s semantics — immutable originInstanceId/createdAt, fresh
* updatedAt stamp, `${type}:upserted` event, and a sync log append. Returns the
* updated `{ id, ...record }`, or null when the record is missing/tombstoned or
* fn aborted.
*/
export async function updateWith(type, id, fn) {
const store = storeFor(type);
if (!store.isValidId(id)) return null;
return store.queueRecordWrite(id, async () => {
const existing = await store.loadOne(id);
// A tombstoned record is gone — treat it as not-found rather than reviving it.
if (!existing || isTombstone(existing)) {
return null;
}
const updates = await fn({ id, ...existing });
if (!updates) return null;
const record = {
...existing,
...updates,
// Preserve immutable fields — originInstanceId tracks the creating instance
originInstanceId: existing.originInstanceId,
createdAt: existing.createdAt,
updatedAt: supersedingTimestamp(existing.updatedAt, now())
};
await store.saveOneNow(id, record);
brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
await brainSyncLog.appendLocalChanges([{ op: 'update', type, id, record, originInstanceId: record.originInstanceId }]);
console.log(`🧠 Updated ${type} record: ${id}`);
return { id, ...record };
});
}
/**
* Upsert a record under a CALLER-PROVIDED id (full replace, create-if-missing).
*
* `create()` mints a uuid, which is wrong for stores whose identity is a natural
* key — the Daily Log keys entries by calendar date so the same day converges
* across peers instead of forking into per-machine uuids. This primitive lets
* such a store own its id while still riding the exact entity-store contract:
* it preserves `originInstanceId`/`createdAt` from an existing live record,
* stamps a fresh `updatedAt` (the LWW clock), appends a create/update entry to
* the sync log, and emits `${type}:upserted` unless the caller suppresses it.
*
* `recordData` is stored verbatim except for the three managed fields — callers
* pass the FULL desired record (this does not merge unknown fields the way
* `update()` does). `emitEvent:false` lets a caller (e.g. brainJournal) emit its
* own richer, bridge-shaped event instead of the generic one.
* A function receives the fresh live `{ id, ...record }` (or null) INSIDE the
* per-id queue and returns the full replacement. Returning null/undefined
* leaves storage, events and the sync log untouched and returns the live record.
* Use this form for derived content and preconditions that must serialize with
* peer applies as well as other local writes.
* createOnly:true returns null for any existing record, including tombstones.
*/
export async function upsertWithId(type, id, recordData, { emitEvent = true, createOnly = false } = {}) {
const store = storeFor(type);
// A malformed id can't name a valid per-record dir — mirror the other write
// paths and no-op rather than letting the store's write-queue assertion throw.
if (!store.isValidId(id)) return null;
// Resolve our instance id outside the queue (independent read); the existing
// record's origin is preferred inside the queue when one is present.
const fallbackOrigin = await getInstanceId();
return store.queueRecordWrite(id, async () => {
const existing = await store.loadOne(id);
// Ingestion must not overwrite an edit or resurrect a tombstone.
// Test presence inside the queue so a peer or local write cannot race.
if (createOnly && existing) return null;
const live = existing && !isTombstone(existing) ? existing : null;
const input = typeof recordData === 'function'
? await recordData(live ? { id, ...live } : null)
: recordData;
if (typeof recordData === 'function' && input == null) return live ? { id, ...live } : null;
const timestamp = now();
const originInstanceId = live?.originInstanceId ?? fallbackOrigin;
const record = {
...input,
originInstanceId,
createdAt: live?.createdAt ?? timestamp,
updatedAt: supersedingTimestamp(existing?.updatedAt, timestamp)
};
await store.saveOneNow(id, record);
if (emitEvent) {
brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
} else {
// A suppressed per-record event still has to reach derived projections.
emitRecordChanged(type, id);
}
await brainSyncLog.appendLocalChanges([{ op: live ? 'update' : 'create', type, id, record, originInstanceId }]);
console.log(`🧠 Upserted ${type} record: ${id}`);
return { id, ...record };
});
}
/**
* Apply many record updates to one store.
*
* With the old monolithic file, a batch (e.g. a chip reorder) had to be one
* atomic load-modify-save or N concurrent single-record `update()` calls would
* read overlapping baselines of the shared file and last-save-wins. Per-record
* files remove that race entirely — writes to different ids no longer share a
* file — so this now serializes each record on its own per-id queue and merges
* against the freshest persisted record. `updates` is an array of { id,
* ...fields }; unknown/tombstoned ids are skipped. Returns the updated records.
*/
export async function updateMany(type, updates) {
const store = storeFor(type);
const applied = [];
for (const { id, ...fields } of updates) {
if (!store.isValidId(id)) continue;
const record = await store.queueRecordWrite(id, async () => {
const existing = await store.loadOne(id);
if (!existing || isTombstone(existing)) return null;
const next = {
...existing,
...fields,
// Preserve immutable fields, exactly as update() does.
originInstanceId: existing.originInstanceId,
createdAt: existing.createdAt,
updatedAt: supersedingTimestamp(existing.updatedAt, now())
};
await store.saveOneNow(id, next);
return next;
});
if (!record) continue;
// Emit + append the sync-log entry as EACH record lands, not once after the
// whole batch. Per-record files mean there is no all-or-nothing batch to
// preserve, so if a later id's write throws, the records already persisted
// must still have federated — a batch-tail loop would skip them entirely and
// leave earlier successes local-only.
applied.push({ id, record });
brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
await brainSyncLog.appendLocalChanges([{ op: 'update', type, id, record, originInstanceId: record.originInstanceId }]);
}
if (applied.length === 0) return [];
console.log(`🧠 Updated ${applied.length} ${type} records in one batch`);
return applied.map(({ id, record }) => ({ id, ...record }));
}
/**
* Apply a batch whose records can be persisted independently and whose sync
* entries should be appended together. Per-record queues still protect each
* record's read-modify-write, while Promise.allSettled lets successful writes
* reach the batch log before a failed sibling is surfaced to the caller.
*/
async function updateManyWithBatchLog(type, updates) {
const store = storeFor(type);
const results = await Promise.allSettled(updates.map(({ id, ...fields }) => {
if (!store.isValidId(id)) return null;
return store.queueRecordWrite(id, async () => {
const existing = await store.loadOne(id);
if (!existing || isTombstone(existing)) return null;
const record = {
...existing,
...fields,
// Preserve immutable fields, exactly as update() does.
originInstanceId: existing.originInstanceId,
createdAt: existing.createdAt,
updatedAt: supersedingTimestamp(existing.updatedAt, now())
};
await store.saveOneNow(id, record);
return { id, record };
});
}));
const failures = results.filter(({ status }) => status === 'rejected');
const applied = results
.filter(({ status }) => status === 'fulfilled')
.map(({ value }) => value)
.filter(Boolean);
for (const { id, record } of applied) {
brainEvents.emit(`${type}:upserted`, { id, record: { id, ...record } });
}
if (applied.length > 0) {
await brainSyncLog.appendLocalChanges(applied.map(({ id, record }) => ({
op: 'update',
type,
id,
record,
originInstanceId: record.originInstanceId,
})));
}
if (failures.length > 0) throw failures[0].reason;
if (applied.length === 0) return [];
console.log(`🧠 Updated ${applied.length} ${type} records in one batch`);
return applied.map(({ id, record }) => ({ id, ...record }));
}
/**
* Delete a record
*/
export async function remove(type, id) {
const store = storeFor(type);
if (!store.isValidId(id)) return false;
return store.queueRecordWrite(id, async () => {
const existing = await store.loadOne(id);
// Already gone (absent) or already tombstoned — nothing to delete, and
// re-tombstoning would mint a redundant sync-log entry that relays needlessly.
if (!existing || isTombstone(existing)) {
return false;
}
const originInstanceId = existing.originInstanceId ?? 'unknown';
const deletedRecord = { id, ...existing };
const ts = supersedingTimestamp(existing.updatedAt, now());
// Retain a tombstone in place (save it as the record's value, not a hard
// delete of the record dir) so a stale `create` echoed from a peer is
// rejected by the LWW guard in applyRemoteRecord.
await store.saveOneNow(id, makeTombstone(ts, originInstanceId));
brainEvents.emit(`${type}:deleted`, { id, record: deletedRecord });
// Wire format unchanged: the sync-log delete entry still carries only
// { updatedAt } so an older peer (no tombstone support) applies it as a
// plain hard delete exactly as before.
await brainSyncLog.appendLocalChanges([{ op: 'delete', type, id, record: { updatedAt: ts }, originInstanceId }]);
console.log(`🧠 Deleted ${type} record: ${id}`);
return true;
});
}
/**
* Query records with filters
*/
export async function query(type, filters = {}) {
const records = await getAll(type);
return records.filter(record => {
for (const [key, value] of Object.entries(filters)) {
if (record[key] !== value) return false;
}
return true;
});
}
// =============================================================================
// JSONL APPEND LOGS (digests, reviews)
// =============================================================================
/**
* Load all records from a JSONL file
*/
async function loadJsonlStore(type) {
const cache = caches[type];
if (cache.data && (Date.now() - cache.timestamp) < CACHE_TTL_MS) {
return cache.data;
}
await ensureBrainDir();
const filePath = FILES[type];
if (!existsSync(filePath)) {
cache.data = [];
cache.timestamp = Date.now();
return cache.data;
}
const content = await readFile(filePath, 'utf-8');
const lines = content.trim().split('\n').filter(line => line.trim());
cache.data = lines.map(line => safeJSONParse(line, null)).filter(item => item !== null);
cache.timestamp = Date.now();
return cache.data;
}
/**
* Append a record to a JSONL file
*/
async function appendJsonl(type, record) {
await ensureBrainDir();
const line = JSON.stringify(record) + '\n';
await appendFile(FILES[type], line);
// Invalidate cache so next read gets fresh data
caches[type].data = null;
caches[type].timestamp = 0;
brainEvents.emit(`${type}:added`, record);
}
// =============================================================================
// INBOX LOG OPERATIONS
// =============================================================================
// The inbox is an id-keyed entity store (see BRAIN_ENTITY_TYPES) so it
// federates through the same delta-log + LWW + tombstone pipeline as every other
// brain type. These wrappers keep the historical getInboxLog/createInboxLog/…
// API (capturedAt sort, status counts) on top of the generic entity primitives.
// Cursor pages share one contract: rows sort newest-first (timestamp descending)
// with the id ascending as the tiebreak, and the cursor is the last row's
// `<zero-padded timestamp>_<id>` key. Continuation therefore keeps older rows
// and, at the cursor's own timestamp, only ids that sort AFTER the cursor id --
// comparing the encoded key as a string would pick the smaller ids instead and
// repeat/skip rows whenever a timestamp tie crosses a page boundary. The cursor
// row itself need not still exist. A malformed cursor yields the first page.
const cursorKeyOf = (id, timestampMs) => `${String(timestampMs).padStart(15, '0')}_${id}`;
function rowsAfterCursor(rows, cursor, timestampOf) {
const match = /^(\d+)_(.*)$/s.exec(cursor);
if (!match) return rows;
const cursorMs = Number(match[1]);
const cursorId = match[2];
return rows.filter(([id, summary]) => {
const ms = timestampOf(summary);
return ms < cursorMs || (ms === cursorMs && id.localeCompare(cursorId) > 0);
});
}
/**
* One page of inbox entries (newest-first by capturedAt), optional status/search filters and cursor pagination.
*/
export async function getInboxPage(options = {}) {
const { status, search, cursor, limit = 50, offset = 0 } = options;
const boundedLimit = Math.min(Math.max(1, limit), 100);
const rows = await resolveRecordIndex(summaryIndex, 'inbox', 0);
const searchLower = search ? search.trim().toLowerCase() : null;
const matching = rows.filter(([id, summary]) => {
if (!summary) return false;
if (status && summary.status !== status) return false;
if (searchLower) {
const textMatch = summary.capturedText?.toLowerCase().includes(searchLower);
const titleMatch = summary.title?.toLowerCase().includes(searchLower);
if (!textMatch && !titleMatch) return false;
}
return true;
});
// Bulk captures can share one timestamp. The id tiebreak keeps adjacent
// cursor/offset pages stable so an entry is never duplicated or skipped.
matching.sort(([idA, a], [idB, b]) => (b.capturedAtMs - a.capturedAtMs) || idA.localeCompare(idB));
const total = matching.length;
let remaining = matching;
if (cursor) {
remaining = rowsAfterCursor(matching, cursor, (s) => s.capturedAtMs);
} else if (offset > 0) {
remaining = matching.slice(offset);
}
const pageRows = remaining.slice(0, boundedLimit);
const pageIds = pageRows.map(([id]) => id);
const page = await Promise.all(pageIds.map((id) => getById('inbox', id)));
const entries = page.filter(Boolean);
const hasMore = remaining.length > boundedLimit;
const nextCursor = hasMore && pageRows.length > 0
? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].capturedAtMs)
: null;
return {
entries,
items: entries,
total,
nextCursor
};
}
/**
* Get all inbox log entries (newest-first by capturedAt), optional status filter.
*/
export async function getInboxLog(options = {}) {
const page = await getInboxPage(options);
const arr = page.entries;
arr.total = page.total;
arr.nextCursor = page.nextCursor;
arr.items = page.items;
return arr;
}
/**
* Get inbox log entry by ID
*/
export async function getInboxLogById(id) {
return getById('inbox', id);
}
// The inbox's concurrent same-record write paths (a capture's create immediately
// followed by a background-classification update on the same entry, plus the
// boot recovery sweep) are serialized by create/update/remove's own per-id write
// queue — no separate inbox lock is needed.
/**
* Create inbox log entry. `capturedAt` is the user-facing capture time (kept
* distinct from the sync `createdAt`/`updatedAt` clocks stamped by create()).
*/
export async function createInboxLog(entry) {
return create('inbox', { ...entry, capturedAt: entry.capturedAt || now() });
}
/**
* Update inbox log entry (partial merge — returns null if absent/tombstoned).
*/
export async function updateInboxLog(id, updates) {
return update('inbox', id, updates);
}
/**
* Delete inbox log entry (tombstones in place for sync convergence).
*/
export async function deleteInboxLog(id) {
return remove('inbox', id);
}
/**
* Get inbox log count by status
*/
export async function getInboxLogCounts() {
const rows = await resolveRecordIndex(summaryIndex, 'inbox', 0);
const counts = {
total: 0,
classifying: 0,
filed: 0,
needs_review: 0,
corrected: 0,
done: 0,
error: 0
};
for (const [, summary] of rows) {
if (!summary) continue;
counts.total++;
if (counts[summary.status] !== undefined) {
counts[summary.status]++;
}
}
return counts;
}
// =============================================================================
// DIGEST OPERATIONS
// =============================================================================
/**
* Get all digests
*/
export async function getDigests(limit = 10) {
let records = await loadJsonlStore('digests');
records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));
return limit === Infinity ? records : records.slice(0, limit);
}
/**
* Get latest digest
*/
export async function getLatestDigest() {
const digests = await getDigests(1);
return digests[0] || null;
}
/**
* Create digest entry
*/
export async function createDigest(digest) {
const record = {
id: generateId(),
...digest,
generatedAt: now()
};
await appendJsonl('digests', record);
// Update meta with last digest time
await updateMeta({ lastDailyDigest: record.generatedAt });
console.log(`🧠 Created daily digest: ${record.id}`);
return record;
}
// =============================================================================
// REVIEW OPERATIONS
// =============================================================================
/**
* Get all reviews
*/
export async function getReviews(limit = 10) {
let records = await loadJsonlStore('reviews');
records = records.sort((a, b) => new Date(b.generatedAt) - new Date(a.generatedAt));
return limit === Infinity ? records : records.slice(0, limit);
}
/**
* Get latest review
*/
export async function getLatestReview() {
const reviews = await getReviews(1);
return reviews[0] || null;
}
/**
* Create review entry
*/
export async function createReview(review) {
const record = {
id: generateId(),
...review,
generatedAt: now()
};
await appendJsonl('reviews', record);
// Update meta with last review time
await updateMeta({ lastWeeklyReview: record.generatedAt });
console.log(`🧠 Created weekly review: ${record.id}`);
return record;
}
// =============================================================================
// CONVENIENCE EXPORTS FOR ENTITY TYPES
// =============================================================================
// People
export const getPeople = (filters) => filters ? query('people', filters) : getAll('people');
export const getPersonById = (id) => getById('people', id);
export const createPerson = (data) => create('people', data);
export const updatePerson = (id, data) => update('people', id, data);
export const deletePerson = (id) => remove('people', id);
// Projects
export const getProjects = (filters) => filters ? query('projects', filters) : getAll('projects');
export const getProjectById = (id) => getById('projects', id);
export const createProject = (data) => create('projects', data);
export const updateProject = (id, data) => update('projects', id, data);
export const deleteProject = (id) => remove('projects', id);
// Ideas
export const getIdeas = (filters) => filters ? query('ideas', filters) : getAll('ideas');
export const getIdeaById = (id) => getById('ideas', id);
export const createIdea = (data) => create('ideas', data);
export const updateIdea = (id, data) => update('ideas', id, data);
export const deleteIdea = (id) => remove('ideas', id);
// Admin
export const getAdminItems = (filters) => filters ? query('admin', filters) : getAll('admin');
export const getAdminById = (id) => getById('admin', id);
export const createAdminItem = (data) => create('admin', data);
export const updateAdminItem = (id, data) => update('admin', id, data);
export const deleteAdminItem = (id) => remove('admin', id);
// Memories
/**
* Effective recency timestamp (ms epoch) for ordering a memory entry newest-first.
*
* Imported conversations (ChatGPT) carry the original conversation clock in
* `sourceUpdatedAt` / `sourceCreatedAt`. A ChatGPT export is NOT ordered
* chronologically, and every entry from one bulk import shares the same
* `createdAt`/`updatedAt` (the import time) — so sorting on the storage clock
* leaves imports in arbitrary export order (the user-reported bug). Prefer the
* source clock when present, falling back to the storage clock for hand-written
* entries. Returns 0 for a missing/unparseable timestamp so it sorts last.
*/
export const memoryRecencyMs = (record) => {
for (const candidate of [
record?.sourceUpdatedAt,
record?.sourceCreatedAt,
record?.updatedAt,
record?.createdAt,
]) {
const t = safeDate(candidate); // epoch ms, or 0 for missing/unparseable
if (t) return t;
}
return 0;
};
export const getMemoryEntries = async () => {
const entries = await getAll('memories');
// Decorate-sort-undecorate: compute each record's recency once (a bulk ChatGPT
// import can be hundreds of entries — recomputing it inside the comparator
// would parse every timestamp O(n log n) times).
return entries
.map((entry) => ({ entry, recency: memoryRecencyMs(entry) }))
.sort((a, b) => b.recency - a.recency)
.map(({ entry }) => entry);
};
export const getMemoryEntryById = (id) => getById('memories', id);
export const createMemoryEntry = (data) => create('memories', data);
export const updateMemoryEntry = (id, data) => update('memories', id, data);
export const deleteMemoryEntry = (id) => remove('memories', id);
// Links
//
// EVERY link leaves this module through `readLink`/`readLinks`, which apply
// `normalizeRepoLinkFields` so a record written by an older PortOS (or arriving
// from a peer still on the GitHub-only field names) reads as a repo — see
// lib/repoLinkFields.js. Route through them when adding an accessor; the
// persisted payload itself is never rewritten (the derivation belongs to
// `createLinkFromUrl`, which owns the URL).
const readLink = async (record) => normalizeRepoLinkFields(await record);
const readLinks = async (records) => (await records).map(normalizeRepoLinkFields);
export const getLinks = (filters) => readLinks(filters ? query('links', filters) : getAll('links'));
export const getLinkById = (id) => readLink(getById('links', id));
export const createLink = (data) => readLink(create('links', data));
export const updateLink = (id, data) => readLink(update('links', id, data));
// Batch reorder: per-id write queues so a multi-chip drag merges each link
// against its freshest persisted record.
export const reorderLinks = (updates) => readLinks(updateMany('links', updates));
export const deleteLink = (id) => remove('links', id);
// ─── Link summary index (issue #3509) ───────────────────────────────────────
// Every ordering, filtering, counting, and membership question the links API
// asks is answerable from four fields. Caching just those in a projection index
// (see createRecordIndex above) means `GET /api/brain/links` reads ONLY the
// requested page's record bodies instead of the whole collection — page cost
// goes from O(N) parsed records to O(limit).
//
// `null` marks an id that is not a user-visible link (absent, unparseable, or
// tombstoned) — exactly what `getAll` drops. Tombstones stay in the id listing,
// so this must be a null projection rather than an omitted entry.
const projectLinkSummary = (record) => (record && !isTombstone(record)
? {
// Pre-parsed sort key: `getAll` + `new Date()` in a comparator re-parses the
// same timestamp O(n log n) times. `safeDate` yields 0 for a missing or
// unparseable createdAt, so such a link sorts last deterministically
// instead of poisoning the comparator with NaN.
createdAtMs: safeDate(record.createdAt),
// `linkType` is stored RAW so the route's strict `===` comparison keeps its
// exact semantics. `isRepo` is resolved through `linkIsRepo` instead, so a
// legacy record still carrying only `isGitHubRepo` filters correctly without
// the index having to be rebuilt behind the migration.
linkType: record.linkType,
isRepo: linkIsRepo(record),
url: record.url,
}
: null);
const linkSummaryIndex = createRecordIndex(projectLinkSummary);
// The Brain header and inbox gates need only these two fields from each
// user-visible record. Keeping that projection beside the link summary index
// lets their frequent count reads reuse the same per-record invalidation
// machinery without parsing full record bodies in steady state.
const projectSummary = (record) => (record && !isTombstone(record)
? {
status: record.status,
capturedAtMs: safeDate(record.capturedAt),
capturedText: record.capturedText || record.text || '',
title: record.title || record.extracted?.title || record.extracted?.name || '',
}
: null);
const summaryIndex = createRecordIndex(projectSummary);
export const MEMORY_LIST_CONTENT_CHARS = 300;
/**
* Compact row projection for memory cards. Transcripts and long notes can reach
* thousands of characters; the card only renders a short teaser, so clamp the
* content at the transport boundary to keep list pages bounded. Full records
* remain available from GET /api/brain/memories/:id.
*/
export function toMemoryListItem(memory) {
if (!memory) return memory;
if (typeof memory.content !== 'string') return memory;
if (memory.content.length <= MEMORY_LIST_CONTENT_CHARS) return memory;
return {
...memory,
content: memory.content.slice(0, MEMORY_LIST_CONTENT_CHARS),
contentTruncated: true,
};
}
const projectEntitySummary = (record) => (record && !isTombstone(record)
? {
status: record.status || null,
archived: !!record.archived,
sortKey: memoryRecencyMs(record) || safeDate(record.updatedAt || record.createdAt),
searchText: [
record.name,
record.title,
record.context,
typeof record.content === 'string' ? record.content.slice(0, 500) : '',
record.notes,
record.oneLiner,
record.nextAction,
record.mood,
...(Array.isArray(record.tags) ? record.tags : []),
...(Array.isArray(record.followUps) ? record.followUps : []),
].filter(Boolean).join(' ').toLowerCase(),
}
: null);
const entitySummaryIndex = createRecordIndex(projectEntitySummary);
const resolveLinkSummaries = () => resolveRecordIndex(linkSummaryIndex, 'links', 0);
/**
* One page of links, newest-first by `createdAt`.
*
* Filtering, ordering, and the `total` count all come off the cached summary
* index; only the page's own records are loaded from disk. `linkType` /
* `isRepo` are optional and compared strictly, matching the filters the route
* used to apply in memory.
*/
export async function getLinksPage({ linkType, isRepo, limit = 50, offset = 0 } = {}) {
const rows = await resolveLinkSummaries();
// The two filters guard differently ON PURPOSE, mirroring the route code this
// replaced: `linkType` was a truthiness check (no link type is the empty
// string, so `''` means "don't filter"), while `isRepo` must honour an
// explicit `false` and so can only be skipped when it is genuinely absent.
const matching = rows.filter(([, summary]) => summary
&& (!linkType || summary.linkType === linkType)
&& (isRepo === undefined || summary.isRepo === isRepo));
// Newest-first, with the id as a deterministic tiebreak: a bulk import stamps
// many links with the SAME createdAt, and an unstable order across two page
// requests would drop or duplicate records at the slice boundary.
matching.sort(([idA, a], [idB, b]) => (b.createdAtMs - a.createdAtMs) || idA.localeCompare(idB));
const total = matching.length;
const pageIds = matching.slice(offset, offset + limit).map(([id]) => id);
const page = await readLinks(Promise.all(pageIds.map((id) => getById('links', id))));
// A record deleted between the index read and the body read comes back null —
// drop it rather than emitting a hole in the page.
return { links: page.filter(Boolean), total };
}
/**
* Ids of the user-visible (non-tombstoned) links, from the same summary index —
* a membership check that parses no record bodies in steady state.
*/
export async function listLinkIds() {
const rows = await resolveLinkSummaries();
return rows.filter(([, summary]) => summary).map(([id]) => id);
}
/**
* Find link by URL. Resolves the id from the summary index (no bodies) and
* loads only the one match.
*/
export async function getLinkByUrl(url) {
const rows = await resolveLinkSummaries();
const hit = rows.find(([, summary]) => summary && summary.url === url);
return hit ? readLink(getById('links', hit[0])) : null;
}
/**
* One page of entity records (people, projects, ideas, admin, memories) with
* stable cursor pagination, query-wide search, status filter, and compact row
* projections for memory list items.
*/
export async function getEntityPage(type, options = {}) {
const {
status,
search,
cursor,
limit = 25,
offset = 0,
includeArchived = false,
} = options;
const boundedLimit = Math.min(Math.max(1, limit), 100);
const rows = await resolveRecordIndex(entitySummaryIndex, type, 0);
const searchLower = search ? search.trim().toLowerCase() : null;
const matching = rows.filter(([, summary]) => {
if (!summary) return false;
if (!includeArchived && summary.archived) return false;
if (status && summary.status !== status) return false;
if (searchLower && !summary.searchText.includes(searchLower)) return false;
return true;
});
// Sort newest-first with id as stable tiebreak
matching.sort(([idA, a], [idB, b]) => (b.sortKey - a.sortKey) || idA.localeCompare(idB));
const total = matching.length;
let remaining = matching;
if (cursor) {
remaining = rowsAfterCursor(matching, cursor, (s) => s.sortKey);
} else if (offset > 0) {
remaining = matching.slice(offset);
}
const pageRows = remaining.slice(0, boundedLimit);
const pageIds = pageRows.map(([id]) => id);
const rawPage = await Promise.all(pageIds.map((id) => getById(type, id)));
let items = rawPage.filter(Boolean);
if (type === 'memories') {
items = items.map(toMemoryListItem);
}
const hasMore = remaining.length > boundedLimit;
const nextCursor = hasMore && pageRows.length > 0
? cursorKeyOf(pageRows[pageRows.length - 1][0], pageRows[pageRows.length - 1][1].sortKey)
: null;
return {
items,
total,
nextCursor,
};
}
// Buckets (bookmark groups for links)
export const getBuckets = (filters) => filters ? query('buckets', filters) : getAll('buckets');
export const getBucketById = (id) => getById('buckets', id);
export const createBucket = (data) => create('buckets', data);
export const updateBucket = (id, data) => update('buckets', id, data);
// Bucket order changes are one user gesture; persist the independent records
// together and append their federation entries with one sync-log write.
export const reorderBuckets = (updates) => updateManyWithBatchLog('buckets', updates);
export const deleteBucket = (id) => remove('buckets', id);
// Threads (#7664)
//
// A thread is one tracked topic or commitment — an open loop in the bullet
// journal sense, NOT a message thread (messageSync.js owns that word too). The
// accessors are the generic store API under thread-shaped names, exactly like
// the siblings above; the ref registry that gives a thread its links lives in
// lib/threadRefKinds.js + services/threadRefs.js.
export const getThreads = (filters) => filters ? query('threads', filters) : getAll('threads');
export const getThreadById = (id) => getById('threads', id);
export const createThread = (data) => create('threads', data);
export const updateThread = (id, data) => update('threads', id, data);
export const deleteThread = (id) => remove('threads', id);
// =============================================================================
// REMOTE SYNC OPERATIONS (no per-record events, no sync log — echo prevention)
// =============================================================================
// The only event these emit is the local-only `record:changed` invalidation
// signal (see emitRecordChanged) — it never leaves this instance.
/**
* The LWW rejection for an incoming op whose clock is not newer than ours.
*
* `local_current` means we already hold exactly the state the op carries — same
* LWW clock, same delete-ness — so the op is ALREADY APPLIED here (an echo, or a
* delta re-pulled after a crash between our save and our relay append). The
* wire-shape of our copy rides along so the caller can relay it if our sync log
* lacks it (#8316). Anything else is `local_newer`: a genuinely stale op.
*/
function rejectedAgainst(existing, record, incomingIsDelete) {
if (existing.updatedAt !== record.updatedAt || isTombstone(existing) !== incomingIsDelete) {
return { applied: false, reason: 'local_newer' };
}
const current = isTombstone(existing)
? { updatedAt: existing.updatedAt, originInstanceId: existing.originInstanceId }
: { ...existing };
return { applied: false, reason: 'local_current', current };
}
/**
* Apply a remote record to a store (last-writer-wins by updatedAt).
* Rejections carry `reason` `local_newer` (stale op) or `local_current` (op
* already applied — `current` is our copy in wire shape); see rejectedAgainst.
*/
export async function applyRemoteRecord(type, id, record, op) {
const store = storeFor(type);
// Reject a malformed id (can't name a valid per-record dir) or a
// prototype-polluting name — the store's `isValidId` owns both the format
// allowlist AND the reserved-key denylist (`__proto__`/`constructor`/
// `prototype`), so a bad id returns `invalid_id` here instead of throwing on
// the write-queue assertion or appending a phantom relay entry that can never
// converge (this covers BOTH the delta-sync and reconcile-snapshot callers).
if (!store.isValidId(id)) {
return { applied: false, reason: 'invalid_id' };
}
// Both delta pulls and snapshots bypass the HTTP push schema. Reject invalid
// clocks here too, before any write or relay, while preserving missing-clock
// skips for legacy peers. Parseable offsets/legacy formats remain supported.
if (!record?.updatedAt) {
return { applied: false, reason: 'missing_timestamp' };
}
if (parseTsMs(record.updatedAt) === null) {
return { applied: false, reason: 'invalid_timestamp' };
}
return store.queueRecordWrite(id, async () => {
const existing = await store.loadOne(id);
if (op === 'delete') {
// LWW: skip if our copy (live record OR existing tombstone) is at least as
// new as the incoming delete. The tombstone-vs-tombstone case makes a
// repeated delete idempotent → not relayed → the echo loop converges.
if (existing && !compareNewerWins(record.updatedAt, existing.updatedAt)) {
return rejectedAgainst(existing, record, true);
}
// Tombstone in place even when no local record exists. A delete that
// arrives before we ever saw a create still leaves a marker, so a later
// stale create for that id is rejected instead of resurrecting.
await store.saveOneNow(id, makeTombstone(
record.updatedAt,
record.originInstanceId ?? existing?.originInstanceId
));
} else {
// Guard also fires when `existing` is a tombstone — a stale create (older
// updatedAt than the recorded delete) is rejected, breaking the
// resurrection loop. A genuinely newer create (later updatedAt than the
// tombstone) still wins and legitimately revives the record.
if (existing && !compareNewerWins(record.updatedAt, existing.updatedAt)) {
return rejectedAgainst(existing, record, record._deleted === true);
}
// Defense-in-depth: a create carrying `_deleted` (a future peer, or a
// direct caller bypassing brainSync's reroute) must persist as a proper
// tombstone, never as a malformed live record missing `deletedAt`.
await store.saveOneNow(id, record._deleted
? makeTombstone(record.updatedAt, record.originInstanceId ?? existing?.originInstanceId)
: (type === 'links' ? mergeIncomingLinkRecord(record, existing) : { ...record }));
}
// Still no per-record event (that would echo back to the peer, #1077) —
// just the local invalidation signal so derived projections re-read.
emitRecordChanged(type, id);
return { applied: true };
});
}
/**
* Hard-prune tombstones older than `cutoffMs` (a Date.now()-style epoch ms).
* Called by the brain tombstone GC sweep on the orchestrator's interval.
* Per-id re-check inside the write queue so a tombstone a concurrent apply just
* refreshed isn't pruned on a stale read. Returns the number of tombstones
* removed.
*/
export async function pruneTombstones(type, cutoffMs) {
const store = storeFor(type);
const ids = await store.listIds();
// Different ids are independent files — prune them in parallel (each still
// serialized on its own per-id queue so the stale-read recheck holds). The
// count is order-independent.
const results = await Promise.all(ids.map((id) => store.queueRecordWrite(id, async () => {
const record = await store.loadOne(id);
if (!isTombstone(record)) return false;
const deletedAt = Date.parse(record.deletedAt ?? record.updatedAt ?? '');
if (Number.isFinite(deletedAt) && deletedAt < cutoffMs) {
await store.deleteOneNow(id);
emitRecordChanged(type, id);
return true;
}
return false;
})));
return results.filter(Boolean).length;
}
/**
* Backfill originInstanceId on records missing it (run once at startup)
*/
export async function backfillOriginInstanceId() {
const instanceId = await getInstanceId();
// Every record across all types is an independent file — backfill them in
// parallel (per-id writes still serialize on their own queue). This runs at
// startup and gates sync start, so the parallelism directly shortens boot.
const perType = await Promise.all(BRAIN_ENTITY_TYPES.map(async (type) => {
const store = storeFor(type);
const ids = await store.listIds();
const changes = await Promise.all(ids.map((id) => store.queueRecordWrite(id, async () => {
const record = await store.loadOne(id);
// Tombstones always carry originInstanceId; skip them and absent records.
if (!record || isTombstone(record) || record.originInstanceId) return false;
await store.saveOneNow(id, { ...record, originInstanceId: instanceId });
emitRecordChanged(type, id);
return true;
})));
return changes.filter(Boolean).length;
}));
const totalBackfilled = perType.reduce((sum, n) => sum + n, 0);
if (totalBackfilled > 0) {
console.log(`🧠 Backfilled originInstanceId on ${totalBackfilled} records`);
}
}
// =============================================================================
// UTILITY FUNCTIONS
// =============================================================================
/**
* Invalidate the non-collection caches (meta + JSONL logs) and every projection
* index. Entity store BODIES still read through to disk, so there is no
* whole-record entity cache to clear.
*/
export function invalidateAllCaches() {
for (const key of Object.keys(caches)) {
caches[key].data = null;
caches[key].timestamp = 0;
}
for (const index of recordIndexes) {
for (const type of BRAIN_ENTITY_TYPES) {
index.byType[type].clear();
index.generation[type] += 1;
}
}
// Live restores replace files without record events. Drop derived caches too,
// without turning cache invalidation into a write or federation announcement.
brainEvents.emit('caches:invalidated');
}
/**
* Get brain data summary (for dashboard)
*/
export async function getSummary() {
const types = ['people', 'projects', 'ideas', 'admin', 'memories', 'buckets'];
const [typeRows, linkRows, inboxCounts, meta] = await Promise.all([
Promise.all(types.map((type) => resolveRecordIndex(summaryIndex, type, 0))),
resolveLinkSummaries(),
getInboxLogCounts(),
loadMeta()
]);
const summaries = Object.fromEntries(types.map((type, i) => [
type,
typeRows[i].map(([, summary]) => summary).filter(Boolean),
]));
const links = linkRows.map(([, summary]) => summary).filter(Boolean);
return {
counts: {
people: summaries.people.length,
projects: summaries.projects.length,
ideas: summaries.ideas.length,
admin: summaries.admin.length,
memories: summaries.memories.length,
links: links.length,
buckets: summaries.buckets.length,
inbox: inboxCounts
},
activeProjects: summaries.projects.filter(p => p.status === 'active').length,
activeIdeas: summaries.ideas.filter(i => !i.status || i.status === 'active').length,
openAdmin: summaries.admin.filter(a => a.status === 'open').length,
repos: links.filter(l => l.isRepo).length,
needsReview: inboxCounts.needs_review,
lastDailyDigest: meta.lastDailyDigest,
lastWeeklyReview: meta.lastWeeklyReview
};
}