Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
17.3 KB
Raw Blame
/**
 * Brain log reader (§23, v0.10.8; per-device shards in the Memory
 * Integrity Suite). A day's events live in one or more files:
 *
 *   - `<date>.jsonl` / `<date>.md`                - legacy single pair,
 *     treated as the shard with the empty shard id;
 *   - `<date>.<deviceId>.jsonl` / `.md`           - one pair per device.
 *
 * `readLogDay` merges every shard of a day sorted by
 * (timestamp, shardId, line) - deterministic across devices regardless
 * of Syncthing arrival order. Per shard the JSONL sidecar is preferred
 * and the markdown is the fallback, so pre-v0.10.8 markdown history
 * keeps reading even when new shards exist for the same date.
 *
 * Every machine consumer of `Brain/log/` discovers dates through
 * `listLogDates` and reads through `readLogDay`, so the shard layout
 * lives in exactly one module. Malformed JSONL lines surface as
 * `warnings` instead of aborting the read, matching `parseLogDay`'s
 * tolerance contract.
 *
 * The per-shard hash chain (who-wrote-what, Task E) is invisible here by
 * design: `coerceEntry` reads `ts`, `kind` and `payload` and nothing
 * else, so a row's `prev` and `h` never reach a reader and a broken
 * chain never withholds an event. Verification is a separate pass over
 * the same files ({@link scanJsonlRows}, consumed by `log-chain.ts`).
 */

import { readFileSync } from "node:fs";

import {
  listShardedFiles,
  listSyncConflictFiles,
  mergeShardedRows,
  parseShardedName,
  type LedgerShardGrammar,
  type ShardedRow,
} from "./ledger-shards.ts";
import { brainDirs, validateIsoDate } from "./paths.ts";
import { parseLogDayFile, type BrainLogEntry, type BrainLogParseWarning } from "./log.ts";
import { neutralizeSingleLine } from "./untrusted-source.ts";
import { BRAIN_LOG_EVENT_KIND_SET, type BrainLogEventKind } from "./types.ts";

export interface ReadLogDayResult {
  readonly entries: ReadonlyArray<BrainLogEntry>;
  /**
   * The shard id each entry in {@link entries} came from, at the same
   * index; the empty string for the legacy un-sharded pair.
   *
   * A parallel array rather than a field on the entry, because
   * `BrainLogEntry` is the shape the markdown parser, the JSONL parser
   * and every writer already agree on, and a key that only one of the
   * three could fill would be absent exactly where a reader needs it.
   * The shard id IS the device id (see `logShardPath`), which is how the
   * note-write reader answers "which machine wrote this" without a field
   * in the payload.
   */
  readonly entryShardIds: ReadonlyArray<string>;
  /**
   * The 0-based index each entry in {@link entries} occupies among its
   * own shard's PARSED entries, in file order - not the merge position,
   * and not the physical file line: rows that fail to parse (reported
   * as warnings) are uncounted, so a shard with skipped rows numbers
   * its entries against a shorter prefix than the file really has. The
   * merge orders by (timestamp, shardId, line), so when a shard's
   * timestamps go backwards (backfill, clock skew) its merge positions
   * diverge from these; a consumer that must name a durable position
   * within one shard's read (the log catch-up watermark) reads this,
   * because appends only ever extend a shard file and the parsed-entry
   * index never needs retraction.
   *
   * A parallel array for the same reason {@link entryShardIds} is: the
   * shard-internal line belongs beside the shard id, and neither is a
   * field every `BrainLogEntry` producer can fill.
   */
  readonly entryLines: ReadonlyArray<number>;
  readonly source: "jsonl" | "markdown-fallback";
  readonly warnings: ReadonlyArray<BrainLogParseWarning>;
}

// Canonical ISO-8601 UTC timestamp shape emitted by `renderJsonlLine`
// in `log.ts`. Accept the same shape `parseIsoUtc` recognises so the
// JSONL reader cannot leak a value that the markdown side would
// reject. Sub-second precision is allowed because `JSON.stringify`
// of a `Date` produces it; `parseIsoUtc` strips it back to seconds.
//
// Sibling regex: `ISO_8601_RE` in `src/cli/coerce.ts`. That one is
// looser (accepts `±HH:MM` offset, caps millisecond precision at 3
// digits) because it has to admit whatever a human typed on the
// CLI; this one is strict because it only ever sees values the
// writer side just produced in canonical UTC. The two intentionally
// do not share a constant — drift in either direction would silently
// break the contract of the other surface.
const ISO_UTC_TS_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/;

/**
 * The longest stretch of a log-controlled value a parse warning carries.
 * Warnings are model-facing - they ride MCP tool output and the CLI write
 * reports - so the value is fenced here at the source: neutralized to one
 * line by {@link neutralizeSingleLine}, then capped. The row's `path` and
 * `lineNumber` fields identify it; the message never needs the full bytes.
 */
const MAX_WARNING_VALUE_CHARS = 120;

/** Fence one log-controlled value for interpolation into a warning. */
function fenceWarningValue(value: string): string {
  const neutralized = neutralizeSingleLine(value);
  return neutralized.length > MAX_WARNING_VALUE_CHARS
    ? `${neutralized.slice(0, MAX_WARNING_VALUE_CHARS)}…`
    : neutralized;
}

/**
 * One log file name: `<date>.jsonl`, `<date>.md`, or the sharded
 * `<date>.<deviceId>.jsonl` / `.md`. The grammar itself, the device-id
 * slug and the sync-conflict exclusion live in `ledger-shards.ts`, which
 * every append-only ledger in the vault now shares; this declaration is
 * the log's own base and extensions and nothing more.
 */
export const LOG_GRAMMAR: LedgerShardGrammar = Object.freeze({
  base: "\\d{4}-\\d{2}-\\d{2}",
  extensions: Object.freeze(["jsonl", "md"]),
});

/** The shape of one recognised `Brain/log/` daily file name. */
export interface ParsedLogFileName {
  readonly date: string;
  /** Empty string for the legacy un-sharded pair. */
  readonly shardId: string;
  readonly ext: "jsonl" | "md";
}

/**
 * Decide whether `name` is a day of the Brain log, and what day.
 *
 * The four shapes are exactly what `logPath`, `logJsonlPath`,
 * `logShardPath` and `logShardJsonlPath` write, which the shard
 * round-trip test asserts by feeding those builders' own output back
 * through here - a recogniser checked only against hand-written names
 * proves nothing about what the writers produce.
 *
 * Exported because a second consumer needs the same answer: the snapshot
 * diff classifier buckets files under two extracted `Brain/` trees, and
 * the copy of this shape it used to carry predated per-device sharding, so
 * both the markdown shard and the machine-primary JSONL fell through to
 * its catch-all class and `rollback --dry-run` mislabelled the log surface
 * it is most likely to be showing. One recogniser in the module that owns
 * the layout, rather than a third copy of the pattern.
 */
export function parseLogFileName(name: string): ParsedLogFileName | null {
  const parsed = parseShardedName(name, LOG_GRAMMAR);
  if (parsed === null) return null;
  return { date: parsed.base, shardId: parsed.shardId, ext: parsed.ext as "jsonl" | "md" };
}

export interface LogShardFile extends ParsedLogFileName {
  readonly path: string;
  readonly name: string;
}

/**
 * Every recognised log file under `Brain/log/`, sorted by name. Exported
 * so a caller that reads MANY dates in one operation (e.g. the backlink
 * index) can list once and pass the result to {@link readLogDay} instead
 * of paying one `readdirSync` + sort per date.
 */
export function listLogShardFiles(vault: string): LogShardFile[] {
  return listShardedFiles(brainDirs(vault).log, LOG_GRAMMAR).map((file) => ({
    date: file.base,
    shardId: file.shardId,
    ext: file.ext as "jsonl" | "md",
    path: file.path,
    name: file.name,
  }));
}

/**
 * Sorted unique dates that have at least one log file (any shard, any
 * extension). The single date-discovery helper for every reader that
 * used to scan the directory itself.
 */
export function listLogDates(vault: string): string[] {
  const dates = new Set<string>();
  for (const f of listLogShardFiles(vault)) dates.add(f.date);
  return [...dates].toSorted();
}

/**
 * `Brain/log/*.sync-conflict-*` copies left behind by Syncthing. The
 * shard layout prevents NEW conflicts; doctor surfaces any leftovers
 * for a manual union+dedup merge.
 */
export function listLogSyncConflicts(vault: string): string[] {
  return listSyncConflictFiles(brainDirs(vault).log);
}

/** Markdown log files (legacy + shards) for doctor's per-file lint. */
export function listLogMarkdownFiles(vault: string): Array<{ date: string; path: string }> {
  return listLogShardFiles(vault)
    .filter((f) => f.ext === "md")
    .map((f) => ({ date: f.date, path: f.path }));
}

/**
 * Read one day of Brain log events, merged across every shard sorted
 * by (timestamp, shardId, line). Per shard the JSONL sidecar wins;
 * markdown is the fallback. When no file exists, returns an empty
 * result with `source: "jsonl"` (a convenient default — the caller
 * treats it as "no events for that date" regardless of source).
 *
 * `preloadedShards` lets a caller iterating many dates in one operation
 * (e.g. `collectLog`) list the directory once and pass the result in,
 * instead of every call re-running `readdirSync` + sort - output is
 * identical either way, this only skips redundant directory scans.
 */
export function readLogDay(
  vault: string,
  date: string,
  preloadedShards?: ReadonlyArray<LogShardFile>,
): ReadLogDayResult {
  const validDate = validateIsoDate(date);
  const shards = (preloadedShards ?? listLogShardFiles(vault)).filter((f) => f.date === validDate);
  if (shards.length === 0) {
    return { entries: [], entryShardIds: [], entryLines: [], source: "jsonl", warnings: [] };
  }

  // Group by shard id; per shard prefer .jsonl over .md.
  const byShard = new Map<string, { jsonl?: LogShardFile; md?: LogShardFile }>();
  for (const f of shards) {
    const slot = byShard.get(f.shardId) ?? {};
    slot[f.ext] = f;
    byShard.set(f.shardId, slot);
  }

  const tagged: Array<ShardedRow<BrainLogEntry>> = [];
  const warnings: BrainLogParseWarning[] = [];
  let usedMarkdown = false;

  for (const shardId of [...byShard.keys()].toSorted()) {
    const slot = byShard.get(shardId)!;
    if (slot.jsonl) {
      const r = readJsonl(slot.jsonl.path);
      r.entries.forEach((entry, i) => tagged.push({ value: entry, shardId, line: i }));
      warnings.push(...r.warnings);
      continue;
    }
    if (slot.md) {
      usedMarkdown = true;
      const r = parseLogDayFile(vault, validDate, slot.md.path);
      r.entries.forEach((entry, i) => tagged.push({ value: entry, shardId, line: i }));
      warnings.push(...r.warnings);
    }
  }

  // The merge orders by (timestamp, shard id, line) and returns values
  // only, so the shard id and the shard-internal file line ride inside
  // the value to come out beside their entry at the same index.
  const merged = mergeShardedRows(
    tagged.map((row) => ({
      ...row,
      value: { entry: row.value, shardId: row.shardId, line: row.line },
    })),
    (row) => row.entry.timestamp,
  );
  return {
    entries: merged.map((row) => row.entry),
    entryShardIds: merged.map((row) => row.shardId),
    entryLines: merged.map((row) => row.line),
    source: usedMarkdown ? "markdown-fallback" : "jsonl",
    warnings,
  };
}

/**
 * One non-blank line of a JSONL shard as the chain verifier sees it:
 * where it sits in the file and what it parsed to, with `row: null` for
 * a line that is not a JSON object.
 *
 * `lineNumber` is 1-based, because the number an operator needs is the
 * one their editor shows them.
 */
export interface RawJsonlLine {
  readonly lineNumber: number;
  readonly row: Readonly<Record<string, unknown>> | null;
}

/** Every non-blank line of one shard, plus whether the bytes were obtained. */
export interface RawJsonlScan {
  /** False when the file could not be read; `lines` is then empty. */
  readonly readable: boolean;
  /** The read failure, verbatim, when there was one. */
  readonly failure: string | null;
  readonly lines: ReadonlyArray<RawJsonlLine>;
}

/**
 * Read one JSONL shard WITHOUT coercing its rows into log entries.
 *
 * {@link readLogDay} exists to answer "what happened", so it drops every
 * row it cannot turn into a `BrainLogEntry` and never sees the chain
 * fields at all. The chain verifier's question is the opposite one -
 * "what is in this file, and does it link up" - and a row the reader
 * discarded is exactly the row it most needs to report. So the two share
 * the file and nothing else: this scanner parses JSON and stops there.
 *
 * Never throws. An unreadable shard reports `readable: false` with the
 * reason, because "not verified" is not "verified clean".
 */
export function scanJsonlRows(path: string): RawJsonlScan {
  let text: string;
  try {
    text = readFileSync(path, "utf8");
  } catch (err) {
    const message = (err as NodeJS.ErrnoException).message ?? String(err);
    return { readable: false, failure: message, lines: [] };
  }
  const lines: RawJsonlLine[] = [];
  const raw = text.split(/\r?\n/);
  for (let i = 0; i < raw.length; i++) {
    const line = raw[i]!;
    if (line.trim() === "") continue;
    let parsed: unknown;
    try {
      parsed = JSON.parse(line);
    } catch {
      lines.push({ lineNumber: i + 1, row: null });
      continue;
    }
    const isObject = parsed !== null && typeof parsed === "object" && !Array.isArray(parsed);
    lines.push({
      lineNumber: i + 1,
      row: isObject ? (parsed as Record<string, unknown>) : null,
    });
  }
  return { readable: true, failure: null, lines };
}

interface ReadJsonlResult {
  readonly entries: ReadonlyArray<BrainLogEntry>;
  readonly warnings: ReadonlyArray<BrainLogParseWarning>;
}

function readJsonl(path: string): ReadJsonlResult {
  let text: string;
  try {
    text = readFileSync(path, "utf8");
  } catch (err) {
    // §23 (v0.10.8): we got here because the directory listing showed
    // the file, so a subsequent read error is anomalous (race with
    // rotation, permission flip, fs-transient) and worth surfacing
    // rather than reporting an empty shard. Discipline-report and any
    // future doctor can flag it without falling back to a silently
    // incomplete count.
    const message = (err as NodeJS.ErrnoException).message ?? String(err);
    return {
      entries: [],
      warnings: [{ path, lineNumber: 0, message: `failed to read JSONL file: ${message}` }],
    };
  }

  const entries: BrainLogEntry[] = [];
  const warnings: BrainLogParseWarning[] = [];
  const lines = text.split(/\r?\n/);
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]!;
    if (line.trim() === "") continue;
    let parsed: unknown;
    try {
      parsed = JSON.parse(line);
    } catch {
      warnings.push({
        path,
        lineNumber: i + 1,
        // The least trusted value in the parser: any bytes that failed
        // JSON.parse, so the preview rides the same fence as ts/kind.
        message: `malformed JSONL line: ${fenceWarningValue(line.slice(0, 80))}`,
      });
      continue;
    }
    const entry = coerceEntry(parsed, path, i + 1, warnings);
    if (entry !== null) entries.push(entry);
  }
  return { entries, warnings };
}

function coerceEntry(
  raw: unknown,
  path: string,
  lineNumber: number,
  warnings: BrainLogParseWarning[],
): BrainLogEntry | null {
  if (raw === null || typeof raw !== "object") {
    warnings.push({ path, lineNumber, message: "JSONL row is not an object" });
    return null;
  }
  const obj = raw as Record<string, unknown>;
  const ts = obj["ts"];
  const kind = obj["kind"];
  const payload = obj["payload"];
  if (typeof ts !== "string" || typeof kind !== "string") {
    warnings.push({ path, lineNumber, message: "JSONL row missing ts/kind" });
    return null;
  }
  // `ts` must match the canonical ISO-8601 UTC shape produced by
  // `renderJsonlLine` (`YYYY-MM-DDTHH:MM:SSZ`, optional sub-second
  // precision). Anything looser would let arbitrary strings into
  // `BrainLogEntry.timestamp` and break downstream consumers that
  // assume the strict format (e.g. `parseIsoUtc` in `log.ts`).
  if (!ISO_UTC_TS_RE.test(ts)) {
    warnings.push({ path, lineNumber, message: `invalid ts format: ${fenceWarningValue(ts)}` });
    return null;
  }
  if (!BRAIN_LOG_EVENT_KIND_SET.has(kind)) {
    warnings.push({ path, lineNumber, message: `unknown event kind: ${fenceWarningValue(kind)}` });
    return null;
  }
  if (payload === null || typeof payload !== "object" || Array.isArray(payload)) {
    warnings.push({
      path,
      lineNumber,
      message: "JSONL row missing payload object",
    });
    return null;
  }
  const body: Record<string, string | ReadonlyArray<string>> = {};
  for (const [k, v] of Object.entries(payload as Record<string, unknown>)) {
    if (typeof v === "string") {
      body[k] = v;
    } else if (Array.isArray(v) && v.every((x) => typeof x === "string")) {
      body[k] = v as string[];
    } else {
      warnings.push({
        path,
        lineNumber,
        message: `JSONL payload key '${fenceWarningValue(k)}' has unsupported value type`,
      });
    }
  }
  // Mirror `parseLogDay`: surface the payload agent at the top level so
  // jsonl- and markdown-sourced entries are interchangeable downstream.
  const agent = typeof body["agent"] === "string" ? body["agent"] : undefined;
  return {
    timestamp: ts,
    eventType: kind as BrainLogEventKind,
    ...(agent !== undefined ? { agent } : {}),
    body: Object.freeze(body),
  };
}