Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Catch-up before reads (design D5, task 1.8): before a search or a change
* probe answers, the index ingests every record appended to the brain
* store since its watermark for each shard.
*
* The cheap path is the common one. At most once per
* {@link catchUpIntervalMs} per process and index, the shards are listed
* (directory listings cached by directory mtime) and stat'ed, and the
* stats are compared with the watermarks this process last saw. When
* nothing moved, the check ends there: no store file is opened and the
* index is not opened either.
*
* When a shard moved, the index is opened for writing (only when no other
* writer holds it, so a running rebuild never blocks a read) and each
* changed shard is read from its stored offset through the store reader.
* A shard whose identity changed, that shrank below its offset, or whose
* verified prefix no longer hashes the same is re-read from offset 0; the
* re-read is idempotent by record id, and records the shard no longer
* holds are removed. A shard that disappeared takes its records with it,
* except those a sibling shard still holds (`orphans.ts`).
*
* An index that is missing or not at the current schema is left alone:
* the read's self-heal builds or upgrades it, and the rebuild ingests the
* store into the staging file before the swap ({@link ingestStoreInto}).
*/
import { Database } from "bun:sqlite";
import { readDeviceId } from "../../config.ts";
import { closeDatabase } from "../../sqlite-close.ts";
import { observeClock } from "../../store/clock-store.ts";
import { compareHlc, parseHlc } from "../../store/hlc.ts";
import { readBlobBody, readShard } from "../../store/reader.ts";
import { blobPath, partitionDir } from "../../store/store-paths.ts";
import { SIGNAL_KIND, type StorePartition } from "../../store/constants.ts";
import { SIGNAL_STATE_KIND } from "../../store/fold-signals.ts";
import { LANE_ENTRY_KIND } from "../../store/lane-records.ts";
import { TOMBSTONE_KINDS, voidedRecordIds } from "../../store/removal-backlog.ts";
import { LATEST_SCHEMA_VERSION, readSchemaVersion } from "../schema.ts";
import { Store } from "../store.ts";
import {
deleteWatermark,
getRecord,
hasBrainRecordTables,
holdsRecordKind,
pendingRecords,
readWatermarks,
recordIdsOfShard,
writeWatermark,
type StoreWatermark,
} from "../store/brain-records.ts";
import {
isWriterLockHeld,
keepWriterLeasesFresh,
WRITER_LEASE_BLOCKING_BUDGET_MS,
writerLeaseCheckIn,
} from "../store/writer-lock.ts";
import { SearchError, type ResolvedSearchConfig } from "../types.ts";
import { STORE_CATCH_UP_INTERVAL_ENV, STORE_CATCH_UP_INTERVAL_MS } from "./constants.ts";
import { completePendingRecord, ingestRecord, removeRecord } from "./ingest.ts";
import type { IndexedStore } from "./location.ts";
import { settleOrphans, type Orphans } from "./orphans.ts";
import { prefixHash } from "./prefix-hash.ts";
import { ShardScanner, statShardFile, type ShardStat } from "./shard-scan.ts";
/** Why a catch-up did not look at, or did not write, the store. */
export type CatchUpSkip =
| "no-store"
| "throttled"
| "no-index"
| "not-current"
| "locked"
| "failed";
export interface CatchUpResult {
/** Null when the check ran to the end (whether or not anything changed). */
readonly skipped: CatchUpSkip | null;
/** Shards read in this pass. */
readonly shardsRead: number;
/** Records added to the index. */
readonly added: number;
/** Records removed because their shard no longer holds them. */
readonly removed: number;
/** Pending blob bodies that arrived and were indexed. */
readonly completed: number;
/** The failure, when {@link skipped} is `failed`. */
readonly error?: string;
}
export interface CatchUpOptions {
/** Ignore the throttle (a write path that must see the store now). */
readonly force?: boolean;
}
type FileStat = Pick<ShardStat, "identity" | "size" | "mtimeMs">;
/** Per-process memory of one index's catch-up. */
interface ProcessState {
readonly scanner: ShardScanner;
readonly storeRoot: string;
/** The partitions the scanner lists: a machine that leaves one gets a new state. */
readonly partitions: string;
lastCheckMs: number;
indexIdentity: string | null;
/** Shard stats as of the watermarks last seen, or null when unknown. */
known: Map<string, FileStat> | null;
/**
* Blob files whose arrival would complete a pending record, each with
* its stat when last looked at (null: absent or not looked at yet). A
* blob that is present but fails verification keeps its stat here, so
* it triggers a write pass again only once it changes.
*/
pendingBlobs: ReadonlyMap<string, FileStat | null>;
}
const states = new Map<string, ProcessState>();
const NOTHING = { shardsRead: 0, added: 0, removed: 0, completed: 0 } as const;
/**
* Forget what this process last saw of the index at `dbPath`, so the next
* catch-up reads its watermarks again: for a writer that emptied the
* store tables behind catch-up's back.
*/
export function forgetCatchUpState(dbPath: string): void {
states.delete(dbPath);
}
/** The throttle between two metadata passes, from the environment or the default. */
export function catchUpIntervalMs(env: NodeJS.ProcessEnv = process.env): number {
const raw = env[STORE_CATCH_UP_INTERVAL_ENV]?.trim();
if (raw === undefined || raw === "") return STORE_CATCH_UP_INTERVAL_MS;
const n = Number(raw);
return Number.isInteger(n) && n >= 0 ? n : STORE_CATCH_UP_INTERVAL_MS;
}
function stateFor(config: ResolvedSearchConfig, store: IndexedStore): ProcessState {
const existing = states.get(config.dbPath);
const partitions = store.partitions.join(",");
if (
existing !== undefined &&
existing.storeRoot === store.root &&
existing.partitions === partitions
) {
return existing;
}
const fresh: ProcessState = {
scanner: new ShardScanner(store),
storeRoot: store.root,
partitions,
lastCheckMs: Number.NEGATIVE_INFINITY,
indexIdentity: null,
known: null,
pendingBlobs: new Map(),
};
states.set(config.dbPath, fresh);
return fresh;
}
function sameStat(a: FileStat, b: FileStat): boolean {
return a.identity === b.identity && a.size === b.size && a.mtimeMs === b.mtimeMs;
}
/** Whether any pending blob looks different from when it was last looked at. */
function blobMoved(pendingBlobs: ReadonlyMap<string, FileStat | null>): boolean {
for (const [path, before] of pendingBlobs) {
const now = statShardFile(path);
if (now === null ? before !== null : before === null || !sameStat(before, now)) return true;
}
return false;
}
function unchanged(known: Map<string, FileStat>, shards: ReadonlyArray<ShardStat>): boolean {
if (known.size !== shards.length) return false;
return shards.every((s) => {
const k = known.get(s.key);
return k !== undefined && sameStat(k, s);
});
}
function knownFrom(marks: Map<string, StoreWatermark>): Map<string, FileStat> {
const out = new Map<string, FileStat>();
for (const [key, m] of marks)
out.set(key, { identity: m.identity, size: m.size, mtimeMs: m.mtimeMs });
return out;
}
function pendingBlobPaths(store: IndexedStore, db: Database): string[] {
return pendingRecords(db).map((r) =>
blobPath(partitionDir(store.root, r.partition as StorePartition), r.sum),
);
}
/** Watermarks and pending blobs of a current index, read-only; null when it is not current. */
function peekIndex(
dbPath: string,
store: IndexedStore,
): { known: Map<string, FileStat>; pendingBlobs: Map<string, FileStat | null> } | null {
let db: Database;
try {
db = new Database(dbPath, { readonly: true });
} catch {
return null;
}
try {
if (readSchemaVersion(db) !== LATEST_SCHEMA_VERSION || !hasBrainRecordTables(db)) return null;
// Not looked at yet: a blob already present triggers one pass.
const pendingBlobs = new Map(pendingBlobPaths(store, db).map((p) => [p, null] as const));
return { known: knownFrom(readWatermarks(db)), pendingBlobs };
} catch {
return null;
} finally {
closeDatabase(db);
}
}
interface SyncTotals {
shardsRead: number;
/** Tombstones (`signal-removed`, `preference-remove`, `lane-entry-remove`) ingested this pass. */
tombstones: number;
added: number;
/** Records added this pass of a kind a tombstone can void (a signal, a preference, a lane entry). */
voidableAdded: number;
removed: number;
completed: number;
/** Largest clock ingested from another device. */
remoteHlc: string | null;
/**
* The blobs still pending after the pass, each stat'ed just BEFORE the
* read that found it missing or broken: a blob that arrives between that
* read and the end of the pass then looks moved on the next check, so it
* is read again instead of being taken as already looked at.
*/
pendingBlobs: Map<string, FileStat | null>;
}
/**
* Read one changed shard and ingest its records. Records it was credited
* with and no longer holds go to `orphans` rather than out of the index:
* a compaction may hold them in a sibling shard. Returns the watermark to
* write once the orphans are settled.
*/
function readOneShard(
index: Store,
config: ResolvedSearchConfig,
store: IndexedStore,
shard: ShardStat,
mark: StoreWatermark | undefined,
totals: SyncTotals,
device: string | null,
orphans: Orphans,
): { watermark: StoreWatermark; fromStart: boolean } {
const db = index.brainRecordsConnection();
const resume =
mark !== undefined &&
mark.identity === shard.identity &&
shard.size >= mark.offset &&
prefixHash(shard.path, mark.offset) === mark.prefixHash;
const from = resume ? mark.offset : 0;
const read = readShard(shard.path, {
offset: from,
partitionRoot: partitionDir(store.root, shard.partition),
});
totals.shardsRead++;
const seen = new Set<string>();
for (const r of read.records) {
// A shard read from its start can hold every record the store has; the
// writer lock this pass runs under beats as it goes.
writerLeaseCheckIn();
const outcome = ingestRecord(index, config, r.record, shard, r.bodyState === "pending");
// A copy refused for sitting in the other partition does not hold the
// record: it must not keep an orphaned row alive.
if (outcome === "refused") continue;
if (TOMBSTONE_KINDS.has(r.record.kind)) totals.tombstones++;
seen.add(r.record.id);
orphans.delete(r.record.id);
if (outcome === "added") {
totals.added++;
if (isVoidableKind(r.record.kind)) totals.voidableAdded++;
}
if (r.record.device !== device) {
const hlc = parseHlc(r.record.hlc);
const best = totals.remoteHlc === null ? null : parseHlc(totals.remoteHlc);
if (hlc !== null && (best === null || compareHlc(hlc, best) > 0))
totals.remoteHlc = r.record.hlc;
}
}
if (!resume && mark !== undefined) {
for (const id of recordIdsOfShard(db, shard.key)) if (!seen.has(id)) orphans.set(id, shard.key);
}
const count = (kind: string): number => read.diagnostics.filter((d) => d.kind === kind).length;
const post = statShardFile(shard.path) ?? shard;
// The stat (and the prefix hash) are taken after the read, so they may
// describe a newer file than the one read. Only an untouched file may
// vouch for the read; an append keeps the prefix, so the next pass
// resumes; anything else (a same-size swap or rewrite by a sync tool)
// leaves no prefix hash, so the next pass reads the shard from 0.
const sameFile = post.identity === shard.identity;
const untouched =
sameFile &&
post.size === read.size &&
(shard.size !== read.size || post.mtimeMs === shard.mtimeMs);
const appended = sameFile && post.size > read.size;
const watermark: StoreWatermark = {
shard: shard.key,
partition: shard.partition,
identity: post.identity,
size: read.size,
mtimeMs: untouched ? post.mtimeMs : -1,
offset: read.nextOffset,
prefixHash: untouched || appended ? prefixHash(shard.path, read.nextOffset) : "",
unknownVersions: (resume ? mark.unknownVersions : 0) + count("unknown-version"),
corruptLines: (resume ? mark.corruptLines : 0) + count("corrupt-line"),
};
return { watermark, fromStart: !resume };
}
/** Every preference record kind starts with it (`preference-set`, `preference-evidence`, ...). */
const PREFERENCE_KIND_PREFIX = "preference-";
/** A record kind a tombstone can void: a signal's, a preference's, a lane entry's. */
function isVoidableKind(kind: string): boolean {
return (
kind === SIGNAL_KIND ||
kind === SIGNAL_STATE_KIND ||
kind === LANE_ENTRY_KIND ||
kind.startsWith(PREFERENCE_KIND_PREFIX)
);
}
/**
* Take out of `index` every record a tombstone voided (a forgotten
* signal, a removed preference, a removed lane entry), over all `events`
* shards. Returns how many rows
* left the index.
*/
function dropVoidedRecords(
index: Store,
store: IndexedStore,
shards: ReadonlyArray<ShardStat>,
): number {
const records = shards
.filter((shard) => shard.stream === "events")
.flatMap((shard) =>
readShard(shard.path, {
partitionRoot: partitionDir(store.root, shard.partition),
}).records.map((r) => r.record),
);
const db = index.brainRecordsConnection();
let removed = 0;
for (const id of voidedRecordIds(records)) {
const row = getRecord(db, id);
if (row === null) continue;
removeRecord(index, row);
removed++;
}
return removed;
}
/**
* Bring `index` level with the shards in `shards`; returns what changed.
*
* A shard read from its start is one synchronous read of the whole file,
* which a check-in cannot interrupt, so the pass runs with the writer lock
* it holds kept fresh off this thread; the check-ins below cover the rest.
*/
function syncShards(
index: Store,
config: ResolvedSearchConfig,
store: IndexedStore,
shards: ReadonlyArray<ShardStat>,
): SyncTotals {
return keepWriterLeasesFresh(WRITER_LEASE_BLOCKING_BUDGET_MS, () =>
syncShardsInPass(index, config, store, shards),
);
}
function syncShardsInPass(
index: Store,
config: ResolvedSearchConfig,
store: IndexedStore,
shards: ReadonlyArray<ShardStat>,
): SyncTotals {
const db = index.brainRecordsConnection();
const marks = readWatermarks(db);
const device = readDeviceId(config.configPath);
const totals: SyncTotals = {
shardsRead: 0,
tombstones: 0,
added: 0,
voidableAdded: 0,
removed: 0,
completed: 0,
remoteHlc: null,
pendingBlobs: new Map(),
};
try {
const orphans: Orphans = new Map();
const present = new Set<string>();
// Every shard read from its start this pass (the orphan settling skips
// re-reading them), and of those the `events` shards, the only ones
// holding records a tombstone voids.
const readInFull = new Set<string>();
const eventsReadInFull = new Set<string>();
const next: StoreWatermark[] = [];
for (const shard of shards) {
writerLeaseCheckIn();
present.add(shard.key);
const mark = marks.get(shard.key);
if (mark !== undefined && sameStat(mark, shard)) continue;
const read = readOneShard(index, config, store, shard, mark, totals, device, orphans);
if (read.fromStart) {
readInFull.add(shard.key);
if (shard.stream === "events") eventsReadInFull.add(shard.key);
}
next.push(read.watermark);
}
const vanished = [...marks.keys()].filter((key) => !present.has(key));
for (const key of vanished) {
for (const id of recordIdsOfShard(db, key)) if (!orphans.has(id)) orphans.set(id, key);
}
writerLeaseCheckIn();
const settled = settleOrphans(index, config, store, shards, orphans, readInFull);
writerLeaseCheckIn();
totals.removed += settled.removed;
// A forgotten signal leaves the index with its tombstone (FORMAT.md,
// "Removal"): after a pass that read one, that re-read an events shard
// from its start, or that added a record a known tombstone may void (a
// line from before the tombstone that synced in late, appended to a
// shard already read), every record the tombstones voided is taken out.
if (
totals.tombstones > 0 ||
((eventsReadInFull.size > 0 || totals.voidableAdded > 0) &&
[...TOMBSTONE_KINDS].some((kind) => holdsRecordKind(db, kind)))
) {
totals.removed += dropVoidedRecords(index, store, shards);
}
// Watermarks move only once the orphans are settled: a pass that
// stops before that reads the same shards again next time, and a
// shard whose orphans could not be settled keeps its old watermark
// (a vanished one keeps its row), so the next pass meets them again.
for (const w of next) if (!settled.retry.has(w.shard)) writeWatermark(db, w);
for (const key of vanished) if (!settled.retry.has(key)) deleteWatermark(db, key);
for (const pending of pendingRecords(db)) {
const partitionRoot = partitionDir(store.root, pending.partition as StorePartition);
const path = blobPath(partitionRoot, pending.sum);
const before = statShardFile(path);
// A blob that is absent, still arriving, or that can never hold the
// body (hash-valid but not the record's JSON object) keeps the
// record pending; it never throws out of the pass.
const blob = readBlobBody(partitionRoot, pending.sum);
if (blob.status !== "ok") {
totals.pendingBlobs.set(path, before);
continue;
}
completePendingRecord(index, config, pending, blob.body);
totals.completed++;
}
} finally {
// Whatever was already written moves the revision, even when a later
// shard throws: the next pass finds those shards level and changes
// nothing, so it would never bump for them.
if (totals.added + totals.removed + totals.completed > 0) index.bumpIndexRevision();
}
if (device !== null && totals.remoteHlc !== null) {
try {
observeClock(store.deviceStateDir, device, parseHlc(totals.remoteHlc)!);
} catch {
// The receive rule is best effort here: the next catch-up raises it.
}
}
return totals;
}
/**
* Ingest the whole store into an index that is open for writing: the
* rebuild path, which builds a staging index before the swap. Every
* shard is read from offset 0 there, so records an older binary skipped
* for an unknown format version are picked up.
*/
export function ingestStoreInto(index: Store, config: ResolvedSearchConfig): void {
const store = config.brainStore;
if (store === null || store === undefined) return;
syncShards(index, config, store, new ShardScanner(store).scan());
}
/** Catch the index of `config` up with its brain store; never throws. */
export async function catchUpBrainStore(
config: ResolvedSearchConfig,
opts: CatchUpOptions = {},
): Promise<CatchUpResult> {
const store = config.brainStore;
if (store === null || store === undefined) return { skipped: "no-store", ...NOTHING };
const state = stateFor(config, store);
const now = Date.now();
if (opts.force !== true && now - state.lastCheckMs < catchUpIntervalMs()) {
return { skipped: "throttled", ...NOTHING };
}
state.lastCheckMs = now;
try {
const indexIdentity = statShardFile(config.dbPath)?.identity ?? null;
if (indexIdentity === null) {
state.known = null;
return { skipped: "no-index", ...NOTHING };
}
if (indexIdentity !== state.indexIdentity) {
state.known = null;
state.indexIdentity = indexIdentity;
}
const shards = state.scanner.scan();
if (state.known === null) {
const peek = peekIndex(config.dbPath, store);
if (peek === null) return { skipped: "not-current", ...NOTHING };
state.known = peek.known;
state.pendingBlobs = peek.pendingBlobs;
}
if (!blobMoved(state.pendingBlobs) && unchanged(state.known, shards)) {
return { skipped: null, ...NOTHING };
}
if (isWriterLockHeld(config.dbPath)) return { skipped: "locked", ...NOTHING };
const index = await Store.open(config, { mode: "write" });
try {
const totals = syncShards(index, config, store, shards);
state.known = knownFrom(readWatermarks(index.brainRecordsConnection()));
state.pendingBlobs = totals.pendingBlobs;
return {
skipped: null,
shardsRead: totals.shardsRead,
added: totals.added,
removed: totals.removed,
completed: totals.completed,
};
} finally {
await index.close();
}
} catch (e) {
state.known = null;
if (e instanceof SearchError && e.code === "INDEX_LOCKED")
return { skipped: "locked", ...NOTHING };
return { skipped: "failed", ...NOTHING, error: e instanceof Error ? e.message : String(e) };
}
}