Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Stage 0 of the dream pass: read the whole `Brain/` tree into memory.
*
* Pure with respect to the vault: it opens no writer, takes no clock and
* touches nothing on disk, which is why it is the one part of the pass
* that is provably runnable on its own (see `dream-step.ts`).
*
* A file whose frontmatter will not parse is collected into `corrupted`
* rather than aborting the scan; the planning phase turns each entry into
* a `skip-corrupted-frontmatter` log event.
*/
import { legacyVaultOf } from "./no-vault.ts";
import { existsSync, readdirSync } from "node:fs";
import { basename, join } from "node:path";
import { remoteDenyVisibilityTokens } from "../graph/privacy-markers.ts";
import type { FrontmatterMap } from "../types.ts";
import type {
CorruptedEntry,
PreferenceRecord,
RetiredRecord,
ScanResult,
SignalRecord,
} from "./dream-plan.ts";
import {
isTombstoned,
SUPERSEDED_BY_KEY,
buildChainLookup,
resolveChainTip,
} from "./lifecycle/tombstone-read.ts";
import { chainVerdict } from "../search/correction-coupling.ts";
import { brainDirs, brainDirsOf } from "./paths.ts";
import {
legacyMemoryPage,
storeMemoryPages,
type MemoryFolder,
type MemoryPage,
} from "./memory-pages.ts";
import { parsePreferenceMeta } from "./preference.ts";
import { parseSignalPage } from "./signal.ts";
import {
OPERATION,
progressCounter,
withProgress,
type ProgressCounter,
type ProgressSink,
} from "./progress.ts";
import type { Safeguard } from "./safeguard.ts";
const MARKDOWN_EXT = ".md";
/** The one stage this read has: it walks the four Brain directories. */
const SCAN_STAGE = "scan";
/**
* What a caller may hand the scan.
*
* Both members are optional and absence means nobody asked, which is the
* house observer idiom. They arrived together because the scan is one of
* the two units `runDreamStep` runs on its own: reached that way it is a
* whole call rather than a phase of one, and a call that can walk a large
* tree needs a deadline and something to say while it does.
*/
export interface ScanBrainOptions {
/**
* Cooperative deadline; checked once per file read and once per
* directory.
*
* `dream()` DOES pass this one, and the asymmetry with `onProgress`
* below is deliberate rather than an oversight. A deadline is about the
* wall-clock the work costs, and the work costs the same inside a pass
* as it does on its own: the scan is where a large `Brain/` tree is
* read, so a pass whose scan was unguarded would sit past its budget
* with the next checkpoint hundreds of files away. A stream, by
* contrast, is about who is speaking, and inside a pass that is the
* pass.
*/
readonly safeguard?: Safeguard;
/**
* Where this scan reports, when it is the whole run.
*
* `dream()` deliberately does NOT pass one: its own counter already
* owns a `scan` stage and hands the same sink a second stream would
* duplicate. A counter with no sink emits nothing, so the pass costs
* the full run exactly what it did before.
*/
readonly onProgress?: ProgressSink;
}
/** The per-file boundary every collector crosses: deadline, then tick. */
interface DirectoryWalk {
step(): void;
}
/**
* Absolute paths of the `.md` files directly inside `dir`, or nothing when it
* does not exist. Sorted by name in code-unit order: `readdirSync` returns the
* filesystem's storage order (hash order on ext4, name order on NTFS and
* APFS), and every list the dream pass writes - `moved_to_processed`, the
* evidence a promoted preference cites - inherits this order, so an unsorted
* walk made the same corpus write different bytes on different machines.
*/
function markdownFilesIn(dir: string): string[] {
if (!existsSync(dir)) return [];
const out: string[] = [];
for (const name of readdirSync(dir).toSorted()) {
if (name.endsWith(MARKDOWN_EXT)) out.push(join(dir, name));
}
return out;
}
/**
* The pages of one memory folder in the order {@link markdownFilesIn}
* lists them: the files on an unswitched machine, the pages rebuilt from
* the store's folds on a store-only one (task 2.10), already sorted by
* name the same way.
*/
function memoryPagesIn(vault: string | null, folder: MemoryFolder): MemoryPage[] {
return (
storeMemoryPages(vault, folder) ??
markdownFilesIn(brainDirs(legacyVaultOf(vault, "memoryPagesIn"))[folder]).map((full) =>
legacyMemoryPage(legacyVaultOf(vault, "memoryPagesIn"), folder, basename(full)),
)
);
}
/**
* A record's `visibility:` tokens plus any informal privacy marker's
* reservation (`remoteDenyVisibilityTokens`), as a spreadable slice: `{}`
* for default visibility, so a record from an untagged page keeps its
* exact shape.
*
* The scan is the one place the pass holds a page's raw frontmatter, so
* the marker union rides in here: the planner modules are pure folds over
* the records' token lists (`strictestVisibilityOf` in the topic plan and
* the cross-topic promotion), and a page saying `access: private` under
* the informal key must reserve every page drafted from it exactly as a
* formal `visibility: private` does.
*/
function visibilitySlice(meta: FrontmatterMap): { visibility?: ReadonlyArray<string> } {
const tokens = remoteDenyVisibilityTokens(meta);
return tokens.length > 0 ? { visibility: Object.freeze(tokens) } : {};
}
/**
* Signals from one directory. `inbox/`, `processed/` and `archived/` differ
* only in the flags they stamp on each record, so they share this walk.
*/
function collectSignals(
files: ReadonlyArray<MemoryPage>,
active: boolean,
signals: SignalRecord[],
corrupted: CorruptedEntry[],
walk: DirectoryWalk,
archived = false,
): void {
for (const page of files) {
walk.step();
const full = page.path;
try {
// Belief lifecycle suite (t_7d5a3589): a tombstoned signal is
// excluded from the dream pass so it is never re-clustered.
const [meta, body] = page.read();
if (isTombstoned(meta)) continue;
signals.push({
path: full,
signal: parseSignalPage(meta, body, full),
active,
...(archived ? { archived: true as const } : {}),
...visibilitySlice(meta),
});
} catch {
corrupted.push({ path: full });
}
}
}
function collectPreferences(
files: ReadonlyArray<MemoryPage>,
preferences: PreferenceRecord[],
corrupted: CorruptedEntry[],
walk: DirectoryWalk,
tombstoned: CorruptedEntry[],
): void {
for (const page of files) {
walk.step();
const full = page.path;
const rawMeta = page.read()[0];
// The tombstoned file stays on disk for audit; the pass never plans
// over it, but its path is still recorded (finding 64): the cross-
// topic slug check must see a `pref-cross-*` filename it would
// otherwise re-plan and collide with under `overwrite: false`.
if (isTombstoned(rawMeta)) {
tombstoned.push({ path: full });
continue;
}
// Belief lifecycle suite (A4): capture the raw superseded_by pointer
// (the typed parser drops it) so accelerated chain decay can see a
// temporally-superseded ancestor.
const supersededBy =
typeof rawMeta["superseded_by"] === "string" && rawMeta["superseded_by"] !== ""
? (rawMeta["superseded_by"] as string)
: null;
try {
preferences.push({
path: full,
pref: parsePreferenceMeta(rawMeta, full),
supersededBy,
...visibilitySlice(rawMeta),
});
} catch {
corrupted.push({ path: full });
}
}
}
function collectRetired(
vault: string | null,
files: ReadonlyArray<MemoryPage>,
retired: RetiredRecord[],
corrupted: CorruptedEntry[],
walk: DirectoryWalk,
): void {
// The serve-with-correction chain index, built on the first
// pointer-bearing record and reused for the rest of the walk: a vault
// whose retired records carry no pointer never pays for it. The scan
// runs at the operator's own reach, so every on-disk page is indexed -
// resolution here asks only whether the chain terminates, not who may
// read it.
let chainLookup: ReturnType<typeof buildChainLookup> | null = null;
for (const page of files) {
walk.step();
const full = page.path;
// Retired files we only need for topic + id (for supersede
// bookkeeping) plus the optional `user_rejected_reason` that
// drives signal-suppression (v0.10.1, _summary §6). We do a
// lightweight frontmatter parse to avoid the strict folder
// invariant check failing on permissive setups.
try {
const [meta] = page.read();
const topic = typeof meta["topic"] === "string" ? meta["topic"] : "";
const id = typeof meta["id"] === "string" ? meta["id"] : "";
const principle = typeof meta["principle"] === "string" ? meta["principle"] : "";
const scope = typeof meta["scope"] === "string" ? meta["scope"] : undefined;
const userReason =
typeof meta["user_rejected_reason"] === "string"
? (meta["user_rejected_reason"] as string).trim()
: "";
if (topic && id) {
// Serve-with-correction coupling (truth-correctable-time-aware,
// contract item 3): a retired record whose frontmatter declares
// a successor - the serveable-retired regime - is scanned only
// when that chain resolves to a tip, and is dropped fail-closed
// when a hop dangles, so the pass never plans over a correction
// that is not there. A record with no pointer is not in the
// predicate's regime and is scanned as before.
const pointer = meta[SUPERSEDED_BY_KEY];
if (typeof pointer === "string" && pointer.trim() !== "") {
chainLookup ??= buildChainLookup(vault);
const verdict = chainVerdict(id, resolveChainTip(pointer, chainLookup));
if (verdict.action === "drop") continue;
}
retired.push({
path: full,
topic,
id,
principle,
...(scope ? { scope } : {}),
...(userReason ? { user_rejected_reason: userReason } : {}),
...visibilitySlice(meta),
});
}
} catch {
corrupted.push({ path: full });
}
}
}
/**
* Read the whole `Brain/` tree into memory. Pure: it opens no writer, takes
* no clock and touches nothing on disk, which is why it is the one part of
* the pass that is provably runnable on its own (see `dream-step.ts`).
*/
export function scanBrain(vault: string | null, opts: ScanBrainOptions = {}): ScanResult {
const progress = progressCounter(OPERATION.dream, opts.onProgress);
return withProgress(progress, () => scanBrainRun(vault, opts, progress));
}
function scanBrainRun(
vault: string | null,
opts: ScanBrainOptions,
progress: ProgressCounter,
): ScanResult {
const dirs = brainDirsOf(vault);
// The four listings are `readdirSync` with no parse, so front-loading
// them costs a directory read each and buys the stage its denominator:
// the file count IS the unit this stage ticks in, and a reader watching
// a bare counter cannot tell a scan a tenth of the way through from one
// about to finish. The stage still opens before the first checkpoint,
// so an already-elapsed guard leaves a stream that spoke once.
const inbox = memoryPagesIn(vault, "inbox");
const processed = memoryPagesIn(vault, "processed");
// Archived signals are read as inactive history: the plan never counts
// them, but a preference whose evidence link resolves to one still
// derives its sign from it (see `deriveActiveSign`).
const archived = memoryPagesIn(vault, "archived");
const preferenceFiles = memoryPagesIn(vault, "preferences");
const retiredFiles = memoryPagesIn(vault, "retired");
progress.start(
SCAN_STAGE,
inbox.length +
processed.length +
archived.length +
preferenceFiles.length +
retiredFiles.length,
);
const signals: SignalRecord[] = [];
const preferences: PreferenceRecord[] = [];
const retired: RetiredRecord[] = [];
const corrupted: CorruptedEntry[] = [];
const tombstonedPreferences: CorruptedEntry[] = [];
// One boundary for all four collectors: the per-file read is where the
// work is, so it is where the deadline is honoured and where a reader
// learns the scan is still moving. Collapsed into one object because
// four collectors taking two more parameters each is four chances to
// thread one of them and forget the other.
const walk: DirectoryWalk = {
step: () => {
opts.safeguard?.checkpoint();
progress.advance(SCAN_STAGE);
},
};
// The directory boundary is a checkpoint too, and not only for
// symmetry: a tree with no markdown in it crosses no per-file boundary
// at all, and a deadline that only a populated vault can trip is a
// deadline that is absent exactly where a caller cannot predict it.
// The tick stays per file - a directory is not a unit of work a reader
// can count.
const collectors: ReadonlyArray<() => void> = [
() => collectSignals(inbox, true, signals, corrupted, walk),
() => collectSignals(processed, false, signals, corrupted, walk),
() => collectSignals(archived, false, signals, corrupted, walk, true),
() => collectPreferences(preferenceFiles, preferences, corrupted, walk, tombstonedPreferences),
() => collectRetired(vault, retiredFiles, retired, corrupted, walk),
];
for (const collect of collectors) {
opts.safeguard?.checkpoint();
collect();
}
return {
signals,
preferences,
retired,
corrupted,
tombstonedPreferences,
archivedNames: archivedNamesOf(vault, dirs.archived),
};
}
/** The archived folder's entry names; the rebuilt pages' names on a store-only machine. */
function archivedNamesOf(vault: string | null, dir: string): ReadonlySet<string> {
const stored = storeMemoryPages(vault, "archived");
return stored === null ? entryNamesIn(dir) : new Set(stored.map((page) => page.name));
}
/** Every entry name directly inside `dir`, or none when it does not exist. */
function entryNamesIn(dir: string): ReadonlySet<string> {
if (!existsSync(dir)) return new Set();
return new Set(readdirSync(dir));
}
/** What the archive rule needs from the inbox, read the way the pass reads it. */
export interface InboxArchiveScan {
/** Markdown files directly in `inbox/`, the files the pass walks. */
readonly inboxFiles: number;
/** Active inbox signal records: tombstoned and unparseable files left out. */
readonly signals: ReadonlyArray<SignalRecord>;
/** Every entry name in `inbox/archived/`. */
readonly archivedNames: ReadonlySet<string>;
}
/**
* The inbox half of {@link scanBrain}, for the `inbox-archivable` doctor
* check: the same walk and the same per-file skips, so the check selects
* from exactly the records the pass would.
*/
export function scanInboxForArchive(vault: string | null): InboxArchiveScan {
const dirs = brainDirsOf(vault);
const inbox = memoryPagesIn(vault, "inbox");
const signals: SignalRecord[] = [];
collectSignals(inbox, true, signals, [], { step: () => {} });
return {
inboxFiles: inbox.length,
signals,
archivedNames: archivedNamesOf(vault, dirs.archived),
};
}