Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Brain log (`Brain/log/<YYYY-MM-DD>.md`) parser and appender.
*
* The log is the audit trail of every state-changing Brain operation.
* One Markdown file per UTC day; events appear as level-2 headings
* formatted as `## <HH:MM:SS>Z — <event-kind>` followed by a bullet
* list of `- <key>: <value>` lines. Lists are encoded as repeated
* `- key:` entries with sub-bullets (the simple parser accepts both
* `- key: value` and the indented sub-bullet form for arrays).
*
* Two operations:
*
* - {@link parseLogDay} reads one day's file and returns an array of
* {@link BrainLogEntry}. Malformed entries (broken header, invalid
* bullet shape) are skipped and surfaced via the `warnings` array
* so the dream loop and digest never blow up on a partially-edited
* log. Returns an empty array when the file does not exist.
*
* - {@link appendLogEvent} writes a new event to today's log. If the
* file is absent, it is created with a canonical frontmatter
* header (`kind: brain-log`, `date`, `tags`) and the title line.
* Subsequent appends preserve previous bytes verbatim — the
* log is append-only. Each event is ONE append per file under the
* log directory lock (`log-append-io.ts`): the appender reads only
* the shard's tail (the previous chain link, the trailing
* whitespace), never the whole day, which it used to read and
* rewrite through a temp file for every event.
*
* Every appended event carries an `origin_channel` bullet beside the
* `agent` one: the server-derived channel of the process that wrote it
* (`src/core/origin-channel.ts`). Events written before that shipped
* carry no such bullet and are never rewritten to add one.
*
* Every appended JSONL row also carries a per-shard hash chain
* ({@link LOG_CHAIN_FIELDS}, who-wrote-what Task E): once the log is the
* attribution record for every write, a line someone deleted or edited
* has to be detectable. The markdown twin is a derived rendering of the
* same events and is deliberately NOT chained - a human-editable view is
* the wrong thing to hold to a hash, and the JSONL sidecar is the
* machine-primary surface every reader already prefers.
*/
import { existsSync, mkdirSync, readFileSync } from "node:fs";
import { basename } from "node:path";
import { appendSeparated, scanLinesFromEnd } from "./log-append-io.ts";
import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import {
brainDirs,
brainDirsForWrite,
logPath,
logShardJsonlPath,
logShardPath,
validateIsoDate,
} from "./paths.ts";
import { WRITE_LANE } from "./freeze-marker.ts";
import { isValidDeviceId } from "../config.ts";
import { listShardedFiles, resolveAppendShardId } from "./ledger-shards.ts";
import type { DualWriteRecord } from "../store/dual-write.ts";
import { routeWrite, type StoreWriteError } from "../store/store-write.ts";
import { acquireDirLock } from "../store/local-lock.ts";
import { logEventRecord } from "../store/dual-write-records.ts";
import { operationPrivacy, type PrivacyVerdict } from "../store/operation-privacy.ts";
import { logRefsNamePrivatePage } from "../store/reference-privacy.ts";
import { ORIGIN_CHANNEL_FIELD, originChannelStamp } from "../origin-channel.ts";
import { BRAIN_LOG_EVENT_KIND, BRAIN_LOG_EVENT_KIND_SET, type BrainLogEventKind } from "./types.ts";
import { PREF_ID_PREFIX } from "./dream-plan.ts";
/** A topic that can stand as a preference slug: no path separator, no dot run. */
const RECONCILE_TOPIC_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
// ----- Public types ---------------------------------------------------------
/** Parsed bullet payload of a single log entry. */
export type BrainLogEntryPayload = Readonly<Record<string, string | ReadonlyArray<string>>>;
/**
* A parsed log entry. `timestamp` is the full ISO-8601 UTC moment
* reconstructed from the file's `date` and the heading's `HH:MM:SS`
* stamp. `body` keeps the bullet payload verbatim (string for single
* values, string[] for repeated keys).
*/
export interface BrainLogEntry {
readonly timestamp: string;
readonly eventType: BrainLogEventKind;
readonly agent?: string;
readonly body: BrainLogEntryPayload;
/**
* Set on an event read from the store's `private` partition: a reader
* below local reach withholds it (`logEntriesAtReach`), as it withholds
* a private page. Never set on a day file's entry (it has no partition).
*/
readonly fromPrivatePartition?: true;
}
/** Warning surfaced by {@link parseLogDay} for a malformed sub-block. */
export interface BrainLogParseWarning {
readonly path: string;
readonly lineNumber: number;
readonly message: string;
}
export interface ParseLogDayResult {
readonly entries: ReadonlyArray<BrainLogEntry>;
readonly warnings: ReadonlyArray<BrainLogParseWarning>;
}
/** Where {@link appendLogEvent} put the event. */
export interface AppendLogEventResult {
/**
* The file that now holds the event: the Markdown day page on an
* unswitched machine, the store shard of its `log-event` record on a
* store-only one, and the empty string when a fail-open store write did
* not land.
*/
readonly logPath: string;
/** `legacy`: the vault's log day files (plus the dual write); `store`: the store only. */
readonly stored: "legacy" | "store";
/** The store record id on a store-only machine, else `null`. */
readonly recordId: string | null;
/**
* The store shard of that record as `<partition>/<partition-relative
* path>` (`shared/log/<device>/<YYYY-MM>.jsonl`) on a store-only
* machine, else `null`.
*/
readonly storeShard: string | null;
/**
* Why a fail-open store write did not land (the error doctor also
* counts); absent when the event landed. Branch on this, never on an
* empty `logPath`.
*/
readonly failed?: StoreWriteError;
}
/**
* The wikilink a retirement record stamps as `retired_by`, naming the
* log shard THIS device appends to for `date`.
*
* The link and the event it cites must land on the same file. The
* appender resolves its shard with {@link resolveAppendShardId}; a link
* built from the bare date (the pre-shard shape) names a file this
* device never writes once a device id is configured, and the doctor
* reads the retirement's provenance as a broken link. This helper is
* the one place that spelling exists, so a writer cannot drift from the
* appender: with the empty shard id it emits the exact historical
* `[[Brain/log/<date>]]`, with a device id `[[Brain/log/<date>.<id>]]`
* - the Obsidian basename of the file {@link logShardPath} writes.
*
* Pass `deviceId` explicitly when the caller already resolved the shard
* for its own `appendLogEvent` call (merge does), so one resolution
* feeds both writes. Omitted, the helper resolves the same way the
* appender defaults to. Historical records are never rewritten: only
* links written after this ship carry the shard segment.
*/
export function logShardWikilink(date: string, deviceId?: string): string {
const shardId = deviceId ?? resolveAppendShardId();
if (shardId !== "" && !isValidDeviceId(shardId)) {
throw new Error(
`logShardWikilink: invalid deviceId ${JSON.stringify(shardId)} - ` +
"expected a lowercase slug matching the device_id config shape",
);
}
return shardId === "" ? `[[Brain/log/${date}]]` : `[[Brain/log/${date}.${shardId}]]`;
}
// ----- Constants ------------------------------------------------------------
// `## HH:MM:SSZ — <kind>` — both the em dash (—) and the hyphen are
// accepted to tolerate hand-edits. The mandatory `Z` after the time
// reflects the design doc's "UTC time" formatting (§5.5 example).
const HEADER_RE = /^##\s+(\d{2}):(\d{2}):(\d{2})Z\s+[—-]\s+([a-z][a-z0-9-]*)\s*$/;
// Bullet keys are limited to ASCII identifiers (with `_`) plus
// hyphens. We accept any text after the colon, including wikilinks,
// quoted strings, and multi-word values — the parser does not try to
// interpret them, only to record the key/value split.
const BULLET_RE = /^-\s+([A-Za-z_][A-Za-z0-9_-]*)\s*:\s*(.*)$/;
// Sub-bullets nest underneath a parent bullet that ends with a bare
// colon (no value). The leading whitespace varies in real-world
// edits; we accept 2-or-more spaces or one or more tabs.
const SUB_BULLET_RE = /^(?:\s{2,}|\t+)-\s+(.+)$/;
// ----- Public API -----------------------------------------------------------
/**
* Parse `Brain/log/<date>.md` if present. Returns an empty array when
* the file does not exist — there is no Brain log for that day yet.
*
* `parseLogDay` is tolerant of garbage: malformed headers, unknown
* event kinds, or stray bullets get reported as warnings and the rest
* of the file is still returned. This keeps the dream loop alive even
* when a manual edit corrupts a single block.
*/
export function parseLogDay(vault: string, date: string): ParseLogDayResult {
const validDate = validateIsoDate(date);
return parseLogDayFile(vault, validDate, logPath(vault, validDate));
}
/**
* Parse one specific log markdown file for `date`. The per-device
* shard layout (Memory Integrity Suite) means a day can have several
* markdown files (`<date>.md`, `<date>.<deviceId>.md`); the shard
* readers in log-jsonl.ts call this once per file and merge.
*/
export function parseLogDayFile(vault: string, date: string, path: string): ParseLogDayResult {
const validDate = validateIsoDate(date);
if (!existsSync(path)) {
return { entries: [], warnings: [] };
}
let text: string;
try {
text = readFileSync(path, "utf8");
} catch {
return { entries: [], warnings: [] };
}
const entries: BrainLogEntry[] = [];
const warnings: BrainLogParseWarning[] = [];
// Strip the YAML frontmatter and the top-level title to expose the
// event blocks. We do this with a manual scan rather than the
// frontmatter parser because we need the raw line numbers for the
// warning trail.
const lines = text.split(/\r?\n/);
let cursor = 0;
if (lines[0]?.trimEnd() === "---") {
cursor = 1;
while (cursor < lines.length && lines[cursor]?.trimEnd() !== "---") {
cursor++;
}
if (cursor < lines.length) cursor++; // skip closing '---'
}
// Each event block starts at a `## ...` line and ends at the next
// `## ...` or EOF. We collect them in encounter order so the
// caller's "stable order" guarantee is honoured.
while (cursor < lines.length) {
const line = lines[cursor]!;
if (!line.startsWith("## ")) {
cursor++;
continue;
}
const headerLineNumber = cursor + 1;
const header = HEADER_RE.exec(line);
if (!header) {
warnings.push({
path,
lineNumber: headerLineNumber,
message: `malformed event header: ${line}`,
});
cursor++;
continue;
}
const hh = header[1]!;
const mm = header[2]!;
const ss = header[3]!;
const kindStr = header[4]!;
if (!BRAIN_LOG_EVENT_KIND_SET.has(kindStr)) {
warnings.push({
path,
lineNumber: headerLineNumber,
message: `unknown event kind: ${kindStr}`,
});
cursor++;
// Skip the malformed block's body so we don't try to parse its
// bullets as a top-level event.
while (cursor < lines.length && !lines[cursor]!.startsWith("## ")) {
cursor++;
}
continue;
}
// Read bullets until the next event header or EOF.
cursor++;
const blockStart = cursor;
while (cursor < lines.length && !lines[cursor]!.startsWith("## ")) {
cursor++;
}
const blockEnd = cursor;
const { payload, blockWarnings } = parseBulletBlock(
lines.slice(blockStart, blockEnd),
blockStart + 1,
path,
);
for (const w of blockWarnings) warnings.push(w);
entries.push({
timestamp: `${validDate}T${hh}:${mm}:${ss}Z`,
eventType: kindStr as BrainLogEventKind,
agent: typeof payload.agent === "string" ? payload.agent : undefined,
body: Object.freeze(payload),
});
}
return { entries, warnings };
}
/**
* Append a new event to the log for the day of `event.timestamp`. If
* the log file does not yet exist, it is created with the canonical
* frontmatter + title header; otherwise the existing file is read,
* concatenated with the new block, and written back atomically. We
* never edit prior blocks — append-only is the contract.
*
* Two writes happening at the same UTC second receive the same
* `<HH:MM:SS>` header; the on-disk order is the order of calls to
* this function. That's the stability guarantee the test suite asks
* for.
*/
/** Longest wait for the log directory lock before an append gives up. */
export const LOG_LOCK_WAIT_MS = 2_000;
/**
* Acquire the per-log-day directory lock with a bounded wait.
*
* `proper-lockfile`'s sync API refuses the `retries` option (it cannot
* block the event loop on a callback), so the wait is spelled out in
* `acquireDirLock` (`store/local-lock.ts`): jittered, growing sleeps
* while the lock is held, up to {@link LOG_LOCK_WAIT_MS}, then an
* `ELOCKED` refusal. A Windows lock directory still being deleted by the
* previous holder counts as held, not as a permission error. Any other
* error is rethrown immediately (permission, fs corruption, …).
*
* Exported so the fixtures that reproduce log-lock contention hold the
* SAME lock the appender takes (the attestation digest and verify
* suites): if this site or its options move, the fixtures fail loudly
* instead of silently reproducing nothing.
*
* Two seconds is short enough to feel synchronous to a coding agent and
* long enough for a writer to get its turn among several agents
* appending at once on a host whose fsync is slow (Windows), where a
* fixed handful of 50 ms retries lost every round to the writers that
* re-took the lock the moment they released it.
*/
export function acquireLogLock(logDir: string): () => void {
return acquireDirLock(logDir, {
staleMs: 10_000,
budgetMs: LOG_LOCK_WAIT_MS,
label: "log lock",
remedy: "another process is appending to the Brain log; retry the write",
});
}
export interface AppendLogEventOptions {
/**
* Per-device shard id (Memory Integrity Suite). Defaults to
* {@link resolveAppendShardId}, which reads the device-local config and
* falls back to the empty string - the legacy un-sharded
* `<date>.jsonl` / `<date>.md` pair - when the config cannot be
* resolved. That fallback and the reasoning behind it now live in
* `ledger-shards.ts`, shared with every other append-only ledger.
*
* Passing the empty string here forces the legacy pair explicitly.
*/
readonly deviceId?: string;
/**
* Brain store dual write: the privacy verdict of the operation this
* event belongs to (`store/operation-privacy.ts`). A private verdict
* sends the event's store record to `private` even when the event names
* no private page; an event that names one goes there regardless.
* Judged only when dual writes are on.
*/
readonly storePrivate?: PrivacyVerdict;
/**
* Hooks and other best-effort paths (design D14 item 1): on a
* store-only machine a store failure is counted in the device-local
* store-write status and the call returns instead of throwing. Without
* it a store-only failure throws `StoreWriteError` with its remedy. No
* effect on an unswitched machine, whose legacy append still throws as
* before and whose dual write never does.
*/
readonly failOpen?: boolean;
/** The machine config the store-only switch and the store root come from. */
readonly configPath?: string;
}
/**
* Fields of a log event body that are stamps rather than page references:
* the server-derived origin channel. The writing `agent` is NOT opaque: a
* caller supplies it as a free string, so it is judged like any other
* field. An operation that judges the event it is about to append judges
* its body with these same opaque fields ({@link logEventPrivacyContent}),
* so its verdict covers the appender's own.
*/
export const LOG_EVENT_OPAQUE_FIELDS: ReadonlySet<string> = new Set([ORIGIN_CHANNEL_FIELD]);
/** A log event body as content an operation's privacy verdict judges. */
export function logEventPrivacyContent(body: Readonly<Record<string, unknown>>): {
readonly value: unknown;
readonly opaque: ReadonlySet<string>;
} {
return { value: body, opaque: LOG_EVENT_OPAQUE_FIELDS };
}
export function appendLogEvent(
vault: string | null,
event: BrainLogEntry,
opts: AppendLogEventOptions = {},
): AppendLogEventResult {
if (!BRAIN_LOG_EVENT_KIND_SET.has(event.eventType)) {
throw new Error(
`appendLogEvent: unknown event kind '${event.eventType}' — must be one of ${Object.values(
BRAIN_LOG_EVENT_KIND,
).join(", ")}`,
);
}
let deviceId: string;
if (opts.deviceId !== undefined) {
if (opts.deviceId !== "" && !isValidDeviceId(opts.deviceId)) {
throw new Error(
`appendLogEvent: invalid deviceId ${JSON.stringify(opts.deviceId)} - ` +
"expected a lowercase slug matching the device_id config shape",
);
}
deviceId = opts.deviceId;
} else {
deviceId = resolveAppendShardId();
}
// Parse the timestamp once to extract date + HHMMSS deterministically.
// We deliberately do not accept Date objects: the caller controls the
// canonical representation so two calls with identical input produce
// byte-identical output (idempotency requirement).
const ts = parseIsoUtc(event.timestamp);
const topLevelAgent = (event as { agent?: unknown }).agent;
const withAgent =
typeof topLevelAgent === "string" && typeof event.body["agent"] !== "string"
? { ...event.body, agent: topLevelAgent }
: { ...event.body };
// The server-derived origin channel (Unit C), folded into the body the
// same way `agent` is so both surfaces this appender writes carry it
// and `parseLogDay` reads it back with no parser change.
//
// Assigned LAST and unconditionally: `agent` yields to a body key
// because the caller may legitimately name a different agent for the
// event than for the call, but the channel is derived from the process
// and a body that spells it is a caller naming its own provenance -
// the one thing this field exists not to be.
const eventBody: BrainLogEntryPayload = Object.freeze({
...withAgent,
[ORIGIN_CHANNEL_FIELD]: originChannelStamp(),
});
const diskEvent: BrainLogEntry = { ...event, body: eventBody };
// The store record (the dual write on an unswitched machine, the only
// write on a store-only one, design D14 item 1). An event that names a
// private page (a private preference's id and principle, a private
// note's link, in any field and inside free text), whose body carries a
// `<private>` region or the placeholder a stripped one leaves, or that
// belongs to a private operation goes to `private` only. A reconcile
// event's guessed preference is judged only when its page exists
// (`logRefsNamePrivatePage`). Store-only records carry no hash chain:
// store integrity is the record sums plus partition digests (D14 item
// 10); the chain stays a property of the legacy day files.
const storeRecord = (): DualWriteRecord => {
const guessed = reconcileTopicPreferenceId(diskEvent);
const isPrivate = operationPrivacy(vault, {
inherit: [
opts.storePrivate,
() => guessed !== undefined && logRefsNamePrivatePage(vault, [guessed], guessed),
],
content: [logEventPrivacyContent(diskEvent.body)],
});
return logEventRecord(diskEvent, isPrivate());
};
const routed = routeWrite({
legacy: () => appendLegacyLogEvent(requireLegacyVault(vault), diskEvent, ts, deviceId),
store: storeRecord,
context: {
vault,
...(opts.configPath !== undefined ? { configPath: opts.configPath } : {}),
},
...(opts.failOpen === true ? { failOpen: true } : {}),
});
if (routed.mode === "legacy") {
return { logPath: routed.value, stored: "legacy", recordId: null, storeShard: null };
}
const landed = routed.records[0];
if (landed === undefined) {
return {
logPath: "",
stored: "store",
recordId: null,
storeShard: null,
...(routed.failed !== undefined ? { failed: routed.failed } : {}),
};
}
// The shard path ends in its partition-relative path, below the
// partition folder whose name is the partition.
const partitionDir = landed.path.slice(0, landed.path.length - landed.rel.length - 1);
return {
logPath: landed.path,
stored: "store",
recordId: landed.record.id,
storeShard: `${basename(partitionDir)}/${landed.rel}`,
};
}
/** An unswitched machine has a legacy tree, so it has a vault. */
function requireLegacyVault(vault: string | null): string {
if (vault === null || vault.trim() === "") {
throw new Error("appendLogEvent: the legacy log needs a vault");
}
return vault;
}
/**
* The legacy append (unswitched machines): one JSONL row with its hash
* chain link and one Markdown block, under the log directory lock.
* Returns the Markdown day page's path.
*/
function appendLegacyLogEvent(
vault: string,
diskEvent: BrainLogEntry,
ts: { readonly date: string; readonly hms: string },
deviceId: string,
): string {
const path = logShardPath(vault, ts.date, deviceId);
const jsonlPath = logShardJsonlPath(vault, ts.date, deviceId);
// The AUDIT lane (who-wrote-what, Task C). This appender is one of the
// two callers that skip the freeze check, and it is the reason the lane
// exists at all: `freeze`, `unfreeze` and `write-refused` are events
// about a frozen vault, so a freeze that silenced the log would erase
// the record of itself and of every write it refused. The identity
// assertion still runs - a log line landing in the wrong store is as
// wrong as a note landing there.
const logDir = brainDirsForWrite(vault, undefined, WRITE_LANE.audit).log;
// §23 (v0.10.8): each event lands in both `<date>.jsonl` (machine
// surface, primary for `readLogDay`) and `<date>.md` (human-facing
// Obsidian view). JSONL is written first so a partial failure
// (e.g. ENOSPC between the two writes) leaves the machine-readable
// source authoritative — discipline-report and future tooling keep
// returning the correct counts; only the human Obsidian view lags
// until the next successful append rewrites the markdown.
//
// The pair shares one directory-level lock so two concurrent
// appenders cannot interleave each other's halves.
mkdirSync(logDir, { recursive: true });
const release = acquireLogLock(logDir);
try {
// ---- JSONL sidecar (machine surface, primary) ---------------------
// One row per event. The row is a deterministic projection of the
// markdown body so the markdown and JSONL representations describe
// the same event byte-for-byte after JSON.parse.
// The chain (Task E) needs only the previous link: the shard is read
// backwards from its end and the scan stops at the newest line that
// carries one, so the cost is the tail, not the day. Holding the lock
// is what makes the link CORRECT - two appenders reading the same
// head would otherwise write two lines claiming the same predecessor.
//
// A torn last line (a crash mid-append, no trailing newline) is
// skipped by the head scan exactly as before, and `appendSeparated`
// starts the new row on its own line, so the fragment stays one
// malformed line the reader warns about instead of corrupting the
// row written after it.
const line = renderJsonlLine(diskEvent, scanLinesFromEnd(jsonlPath, chainHashOfLine));
appendSeparated(jsonlPath, `${line}\n`, "\n", `${line}\n`);
// ---- Markdown (human surface) -------------------------------------
// One blank line between blocks and a single trailing newline, the
// same bytes the former whole-file rewrite produced; the block always
// ends in a newline, so the file always does too.
const block = renderEventBlock(diskEvent, ts.hms);
appendSeparated(path, block, "\n\n", renderFileHeader(ts.date) + "\n" + block);
} finally {
release();
}
return path;
}
// ----- Renderers ------------------------------------------------------------
/** The header of a log day page: its frontmatter and title line. */
export function renderLogDayHeader(date: string): string {
return renderFileHeader(date);
}
function renderFileHeader(date: string): string {
// Frontmatter is hand-rendered (not via `formatFrontmatter`) because
// the tags array must be a stable, comma-separated inline list, and
// we need byte-deterministic output for the test suite.
const lines: string[] = [
"---",
"kind: brain-log",
`date: ${date}`,
"tags: [brain, brain/log]",
"---",
"",
`# Brain log — ${date}`,
];
return lines.join("\n") + "\n";
}
/**
* One parsed entry rendered as the markdown block {@link appendLogEvent}
* writes for it, so a reader that drops or trims entries can serve the
* rest in the shape the shard would have had without them.
*/
export function renderLogEntryBlock(entry: BrainLogEntry): string {
return renderEventBlock(entry, parseIsoUtc(entry.timestamp).hms);
}
function renderEventBlock(event: BrainLogEntry, hms: string): string {
const lines: string[] = [`## ${hms}Z — ${event.eventType}`];
// Stable iteration order: keys in the order they appear in the
// payload object. The caller is responsible for the order they care
// about; we don't sort because some events benefit from a logical
// grouping (e.g. `run_id` first).
for (const [key, value] of Object.entries(event.body)) {
if (Array.isArray(value)) {
// Repeated keys are encoded as a parent bullet with the bare key
// followed by indented sub-bullets — matches the design-doc
// example for `dream.new_unconfirmed`.
lines.push(`- ${key}:`);
for (const item of value) {
lines.push(` - ${item}`);
}
} else {
lines.push(`- ${key}: ${value}`);
}
}
return lines.join("\n") + "\n";
}
/**
* Render the JSONL projection of an event (§23, v0.10.8; chained by
* who-wrote-what Task E). The row shape is
* `{ ts, kind, payload, prev, h }` where `payload` is a one-to-one map
* of the markdown body bullets. Array bullets become JSON arrays;
* scalar bullets become JSON strings. The function never sorts keys,
* so byte-identical inputs plus a byte-identical `prev` produce
* byte-identical rows.
*
* The chain fields come LAST so the projection the readers have always
* consumed still reads as the head of the row, and `readJsonl` ignores
* them entirely - the chain is an audit surface, never a read gate.
*/
function renderJsonlLine(event: BrainLogEntry, prev: string | null): string {
const payload: Record<string, string | ReadonlyArray<string>> = {};
for (const [key, value] of Object.entries(event.body)) {
payload[key] = value;
}
const ts = event.timestamp;
const kind: string = event.eventType;
return JSON.stringify({
ts,
kind,
payload,
[LOG_CHAIN_FIELDS.prev]: prev,
[LOG_CHAIN_FIELDS.h]: logChainHash(prev, ts, kind, payload),
});
}
// ----- The per-shard hash chain ---------------------------------------------
/**
* The two chain-carrying keys of a JSONL log row, named once because
* three surfaces spell them: the renderer above, the head scanner
* below, and the verifier in `log-chain.ts`.
*
* They are the ONLY keys the chain adds. Everything else a row carries
* is hashed, so a field a future writer adds to the projection is
* covered by the chain automatically.
*/
export const LOG_CHAIN_FIELDS = Object.freeze({
/** The preceding chained line's {@link LOG_CHAIN_FIELDS.h}, `null` at genesis. */
prev: "prev",
/** This line's chain hash. */
h: "h",
} as const);
/**
* The chain hash a row carries: sha256 over the canonical JSON of its
* link plus its whole projection.
*
* `prev` is inside the hash rather than beside it, which is what makes
* the structure a chain: editing any earlier line changes every later
* hash, so a tampered history cannot be re-linked without rewriting the
* whole shard from the edit forward.
*
* Defined here rather than in `log-chain.ts` even though the verifier is
* that module's subject: the appender needs it, `log-chain.ts` needs the
* shard listing from `log-jsonl.ts`, and `log-jsonl.ts` needs this
* module's parser - so a hash living with the verifier would close an
* import cycle. `log-chain.ts` re-exports it, so callers reading about
* the chain find it where the design puts it.
*/
export function logChainHash(
prev: string | null,
ts: string,
kind: string,
payload: Readonly<Record<string, string | ReadonlyArray<string>>>,
): string {
return sha256Hex(canonicalJson({ prev, ts, kind, payload }));
}
/**
* The chain head of a shard already read into memory: the `h` of the
* LAST line that carries one, or `null` when no line does.
*
* Scanned from the end and stopped at the first hit, so an append costs
* one JSON parse rather than one per line of the day. Lines that carry
* no `h` are the pre-chain history of a vault that upgraded mid-day;
* they are skipped rather than treated as a break, and the first chained
* line written after them anchors the shard's chain with `prev: null`.
* A line that will not parse is skipped for the same reason - the
* appender's job is to link to the newest link that exists, and naming
* the damage is the verifier's.
*/
function lastChainHash(jsonl: string): string | null {
const lines = jsonl.split(/\r?\n/);
for (let i = lines.length - 1; i >= 0; i--) {
const h = chainHashOfLine(lines[i]!);
if (h !== null) return h;
}
return null;
}
/**
* The chain hash one JSONL line carries, or `null` for a blank, an
* unparseable or a pre-chain line. The per-line rule of
* {@link lastChainHash}, shared with the appender's backward file scan
* so the two can never disagree about which line is the head.
*/
function chainHashOfLine(line: string): string | null {
if (line.trim() === "") return null;
let parsed: unknown;
try {
parsed = JSON.parse(line);
} catch {
return null;
}
if (parsed === null || typeof parsed !== "object") return null;
const h = (parsed as Record<string, unknown>)[LOG_CHAIN_FIELDS.h];
return typeof h === "string" && h !== "" ? h : null;
}
/**
* The JSONL-only half of the log's shard grammar, for the head collector
* below. The recogniser itself lives in `ledger-shards.ts` like every
* other ledger's; the markdown twin is deliberately absent here because
* it is not chained and carries no head to collect. Exported so the
* attestation record reader validates manifest shard names against the
* ONE grammar the collector names them with - a second spelling would
* let the two drift apart.
*/
export const LOG_HEAD_GRAMMAR = Object.freeze({
base: "\\d{4}-\\d{2}-\\d{2}",
extensions: Object.freeze(["jsonl"]),
});
/**
* Every JSONL shard's chain head, keyed by the shard's file name.
*
* Exported for the attestation manifest (delivery trust): a signed
* record binds the log state, and the log shards per device per day, so
* what gets attested is this MAP and never a single head - there is no
* total order across shards, and one file's head says nothing about
* another's.
*
* The key is the shard's FILE NAME (`2026-06-03.jsonl`,
* `2026-06-03.device-b.jsonl`) rather than the bare device id the appender
* resolves: the same device writes one shard per day, so the device id
* alone would collide in a map that spans days. The file name is the
* shard's unique, stable identity in `Brain/log/`, and it is what a
* verifier needs to re-resolve the head to bytes.
*
* Each head is read under the SAME per-directory lock the appender takes
* ({@link acquireLogLock}), for the reason documented at the append site:
* a head read without the lock can name a line the appender has already
* superseded, and a signature computed over it would vouch for state that
* never existed as a whole. A store that cannot be locked or read refuses
* by throwing, exactly as {@link appendLogEvent} does - refusing to attest
* is the honest answer, and a silent empty map would be a signature over
* nothing. Shards whose lines carry no chain hash at all (pre-chain
* history) contribute no entry; their names are still covered by the
* attested ledger inventory, so nothing escapes the manifest.
*
* Returns `{}` for a vault with no log directory - there is nothing to
* attest, which is a fact about the store, not a failure.
*/
export function collectLogShardHeads(vault: string): Readonly<Record<string, string>> {
const logDir = brainDirs(vault).log;
if (!existsSync(logDir)) return {};
const heads: Record<string, string> = {};
const release = acquireLogLock(logDir);
try {
// The listing runs INSIDE the lock, beside the head reads it feeds:
// a shard created between an outside listing and the lock (a day
// rollover, a new device's first append) would escape the map and a
// signed manifest would vouch for a log that omits it.
const files = listShardedFiles(logDir, LOG_HEAD_GRAMMAR);
for (const file of files) {
const head = lastChainHash(readFileSync(file.path, "utf8"));
if (head !== null) heads[file.name] = head;
}
} finally {
release();
}
return heads;
}
// ----- Bullet block parser --------------------------------------------------
interface ParseBulletResult {
readonly payload: Record<string, string | ReadonlyArray<string>>;
readonly blockWarnings: ReadonlyArray<BrainLogParseWarning>;
}
function parseBulletBlock(
lines: ReadonlyArray<string>,
startLineNumber: number,
path: string,
): ParseBulletResult {
const payload: Record<string, string | ReadonlyArray<string>> = {};
const blockWarnings: BrainLogParseWarning[] = [];
// We walk the block linearly. Top-level bullets either carry their
// value inline (`- key: value`) or open a sub-bullet list (`- key:`
// followed by ` - <item>` lines). We close the current sub-list as
// soon as we hit either another top-level bullet or a blank line.
let cursor = 0;
let currentListKey: string | null = null;
let currentList: string[] | null = null;
const flushList = (): void => {
if (currentListKey !== null && currentList !== null) {
payload[currentListKey] = currentList;
}
currentListKey = null;
currentList = null;
};
while (cursor < lines.length) {
const raw = lines[cursor]!;
const lineNumber = startLineNumber + cursor;
const trimmed = raw.trim();
if (trimmed === "") {
flushList();
cursor++;
continue;
}
const sub = SUB_BULLET_RE.exec(raw);
if (sub && currentList !== null) {
currentList.push(sub[1]!.trim());
cursor++;
continue;
}
const bullet = BULLET_RE.exec(raw);
if (!bullet) {
blockWarnings.push({
path,
lineNumber,
message: `malformed bullet: ${raw}`,
});
flushList();
cursor++;
continue;
}
flushList();
const key = bullet[1]!;
const value = bullet[2]!;
if (value === "") {
currentListKey = key;
currentList = [];
} else {
payload[key] = value;
}
cursor++;
}
flushList();
return { payload, blockWarnings };
}
// ----- Time utilities ------------------------------------------------------
interface IsoUtcParts {
readonly date: string;
readonly hms: string;
}
/**
* Parse an ISO-8601 UTC timestamp into its date and `HH:MM:SS` parts.
* Accepts both `2026-05-14T10:42:00Z` and `2026-05-14T10:42:00.123Z`
* forms; sub-second precision is truncated to seconds because the log
* heading shape is `HH:MM:SS` only.
*/
function parseIsoUtc(timestamp: string): IsoUtcParts {
const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?Z$/.exec(timestamp);
if (!m) {
throw new Error(
`appendLogEvent: timestamp must be ISO-8601 UTC (YYYY-MM-DDTHH:MM:SSZ); got ${JSON.stringify(timestamp)}`,
);
}
return { date: m[1]!, hms: `${m[2]!}:${m[3]!}:${m[4]!}` };
}
/**
* Every artifact one log entry names, as references an
* {@link ArtifactRefView} can resolve.
*
* A log entry is the one record shape whose SUBJECT is another artifact:
* it says what was applied to which preference, over which page. Two
* surfaces filter log rows by what they name - the `osb://log/{date}`
* resource and `brain_query` - and a second copy of the answer would be
* how one of the two comes to disclose more than the other.
*
* EVERY string in the payload is offered, rather than a named list of
* keys. The list was `path` / `preference` / `signal` / `artifact`, and a
* census of the writers in this tree found artifact references in
* `target`, `subject_a`, `subject_b`, `successor`, `predecessor`,
* `source`, `source_path`, `note`, `file`, `files`, `retired`,
* `superseded_by` and `conflicts` as well - so the list was not a
* measurement of what a row discloses, it was a sample of it, and every
* key outside it made its row unconditionally visible. Enumerating a
* payload whose writers are open-ended is the defect, not the particular
* keys that were missing from the enumeration.
*
* Offering too much is SAFE in a way offering too little is not: a string
* that resolves to no artifact on disk is read as naming nothing and lets
* its row through unchanged, which is the same answer the view gives an
* absent key. So the fail direction of the widening is the harmless one.
*
* Array-valued keys are flattened rather than dropped.
* {@link BrainLogEntryPayload} is `string | ReadonlyArray<string>` and
* `parseLogDay` produces an array whenever a key repeats or uses the
* indented sub-bullet form - so a `typeof === "string"` test answered
* `undefined` for a real multi-artifact reference, which the view reads
* as "names nothing".
*/
export function logEntryArtifactRefs(entry: BrainLogEntry): ReadonlyArray<string | undefined> {
const body = (entry.body ?? {}) as Record<string, unknown>;
const refs: Array<string | undefined> = [reconcileTopicPreferenceId(entry)];
for (const value of Object.values(body)) {
if (typeof value === "string") refs.push(value);
else if (Array.isArray(value)) {
for (const item of value) if (typeof item === "string") refs.push(item);
}
}
return refs;
}
/**
* The preference a reconcile event is about, by id.
*
* A reconcile event names its subject by `topic`, not by id, and a
* preference's slug is its topic: the open question on topic `x` is about
* `pref-x`. A topic that cannot be a slug names no preference.
*/
export function reconcileTopicPreferenceId(entry: BrainLogEntry): string | undefined {
if (entry.eventType !== BRAIN_LOG_EVENT_KIND.reconcile) return undefined;
const topic = (entry.body ?? {})["topic"];
return typeof topic === "string" && RECONCILE_TOPIC_SLUG_RE.test(topic)
? `${PREF_ID_PREFIX}${topic}`
: undefined;
}