Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.9 KB
Raw Blame
/**
 * Stage 0 of the dream pass: read the whole `Brain/` tree into memory.
 *
 * Pure with respect to the vault: it opens no writer, takes no clock and
 * touches nothing on disk, which is why it is the one part of the pass
 * that is provably runnable on its own (see `dream-step.ts`).
 *
 * A file whose frontmatter will not parse is collected into `corrupted`
 * rather than aborting the scan; the planning phase turns each entry into
 * a `skip-corrupted-frontmatter` log event.
 */

import { legacyVaultOf } from "./no-vault.ts";
import { existsSync, readdirSync } from "node:fs";
import { basename, join } from "node:path";

import { remoteDenyVisibilityTokens } from "../graph/privacy-markers.ts";
import type { FrontmatterMap } from "../types.ts";
import type {
  CorruptedEntry,
  PreferenceRecord,
  RetiredRecord,
  ScanResult,
  SignalRecord,
} from "./dream-plan.ts";
import {
  isTombstoned,
  SUPERSEDED_BY_KEY,
  buildChainLookup,
  resolveChainTip,
} from "./lifecycle/tombstone-read.ts";
import { chainVerdict } from "../search/correction-coupling.ts";
import { brainDirs, brainDirsOf } from "./paths.ts";
import {
  legacyMemoryPage,
  storeMemoryPages,
  type MemoryFolder,
  type MemoryPage,
} from "./memory-pages.ts";
import { parsePreferenceMeta } from "./preference.ts";
import { parseSignalPage } from "./signal.ts";
import {
  OPERATION,
  progressCounter,
  withProgress,
  type ProgressCounter,
  type ProgressSink,
} from "./progress.ts";
import type { Safeguard } from "./safeguard.ts";

const MARKDOWN_EXT = ".md";

/** The one stage this read has: it walks the four Brain directories. */
const SCAN_STAGE = "scan";

/**
 * What a caller may hand the scan.
 *
 * Both members are optional and absence means nobody asked, which is the
 * house observer idiom. They arrived together because the scan is one of
 * the two units `runDreamStep` runs on its own: reached that way it is a
 * whole call rather than a phase of one, and a call that can walk a large
 * tree needs a deadline and something to say while it does.
 */
export interface ScanBrainOptions {
  /**
   * Cooperative deadline; checked once per file read and once per
   * directory.
   *
   * `dream()` DOES pass this one, and the asymmetry with `onProgress`
   * below is deliberate rather than an oversight. A deadline is about the
   * wall-clock the work costs, and the work costs the same inside a pass
   * as it does on its own: the scan is where a large `Brain/` tree is
   * read, so a pass whose scan was unguarded would sit past its budget
   * with the next checkpoint hundreds of files away. A stream, by
   * contrast, is about who is speaking, and inside a pass that is the
   * pass.
   */
  readonly safeguard?: Safeguard;
  /**
   * Where this scan reports, when it is the whole run.
   *
   * `dream()` deliberately does NOT pass one: its own counter already
   * owns a `scan` stage and hands the same sink a second stream would
   * duplicate. A counter with no sink emits nothing, so the pass costs
   * the full run exactly what it did before.
   */
  readonly onProgress?: ProgressSink;
}

/** The per-file boundary every collector crosses: deadline, then tick. */
interface DirectoryWalk {
  step(): void;
}

/**
 * Absolute paths of the `.md` files directly inside `dir`, or nothing when it
 * does not exist. Sorted by name in code-unit order: `readdirSync` returns the
 * filesystem's storage order (hash order on ext4, name order on NTFS and
 * APFS), and every list the dream pass writes - `moved_to_processed`, the
 * evidence a promoted preference cites - inherits this order, so an unsorted
 * walk made the same corpus write different bytes on different machines.
 */
function markdownFilesIn(dir: string): string[] {
  if (!existsSync(dir)) return [];
  const out: string[] = [];
  for (const name of readdirSync(dir).toSorted()) {
    if (name.endsWith(MARKDOWN_EXT)) out.push(join(dir, name));
  }
  return out;
}

/**
 * The pages of one memory folder in the order {@link markdownFilesIn}
 * lists them: the files on an unswitched machine, the pages rebuilt from
 * the store's folds on a store-only one (task 2.10), already sorted by
 * name the same way.
 */
function memoryPagesIn(vault: string | null, folder: MemoryFolder): MemoryPage[] {
  return (
    storeMemoryPages(vault, folder) ??
    markdownFilesIn(brainDirs(legacyVaultOf(vault, "memoryPagesIn"))[folder]).map((full) =>
      legacyMemoryPage(legacyVaultOf(vault, "memoryPagesIn"), folder, basename(full)),
    )
  );
}

/**
 * A record's `visibility:` tokens plus any informal privacy marker's
 * reservation (`remoteDenyVisibilityTokens`), as a spreadable slice: `{}`
 * for default visibility, so a record from an untagged page keeps its
 * exact shape.
 *
 * The scan is the one place the pass holds a page's raw frontmatter, so
 * the marker union rides in here: the planner modules are pure folds over
 * the records' token lists (`strictestVisibilityOf` in the topic plan and
 * the cross-topic promotion), and a page saying `access: private` under
 * the informal key must reserve every page drafted from it exactly as a
 * formal `visibility: private` does.
 */
function visibilitySlice(meta: FrontmatterMap): { visibility?: ReadonlyArray<string> } {
  const tokens = remoteDenyVisibilityTokens(meta);
  return tokens.length > 0 ? { visibility: Object.freeze(tokens) } : {};
}

/**
 * Signals from one directory. `inbox/`, `processed/` and `archived/` differ
 * only in the flags they stamp on each record, so they share this walk.
 */
function collectSignals(
  files: ReadonlyArray<MemoryPage>,
  active: boolean,
  signals: SignalRecord[],
  corrupted: CorruptedEntry[],
  walk: DirectoryWalk,
  archived = false,
): void {
  for (const page of files) {
    walk.step();
    const full = page.path;
    try {
      // Belief lifecycle suite (t_7d5a3589): a tombstoned signal is
      // excluded from the dream pass so it is never re-clustered.
      const [meta, body] = page.read();
      if (isTombstoned(meta)) continue;
      signals.push({
        path: full,
        signal: parseSignalPage(meta, body, full),
        active,
        ...(archived ? { archived: true as const } : {}),
        ...visibilitySlice(meta),
      });
    } catch {
      corrupted.push({ path: full });
    }
  }
}

function collectPreferences(
  files: ReadonlyArray<MemoryPage>,
  preferences: PreferenceRecord[],
  corrupted: CorruptedEntry[],
  walk: DirectoryWalk,
  tombstoned: CorruptedEntry[],
): void {
  for (const page of files) {
    walk.step();
    const full = page.path;
    const rawMeta = page.read()[0];
    // The tombstoned file stays on disk for audit; the pass never plans
    // over it, but its path is still recorded (finding 64): the cross-
    // topic slug check must see a `pref-cross-*` filename it would
    // otherwise re-plan and collide with under `overwrite: false`.
    if (isTombstoned(rawMeta)) {
      tombstoned.push({ path: full });
      continue;
    }
    // Belief lifecycle suite (A4): capture the raw superseded_by pointer
    // (the typed parser drops it) so accelerated chain decay can see a
    // temporally-superseded ancestor.
    const supersededBy =
      typeof rawMeta["superseded_by"] === "string" && rawMeta["superseded_by"] !== ""
        ? (rawMeta["superseded_by"] as string)
        : null;
    try {
      preferences.push({
        path: full,
        pref: parsePreferenceMeta(rawMeta, full),
        supersededBy,
        ...visibilitySlice(rawMeta),
      });
    } catch {
      corrupted.push({ path: full });
    }
  }
}

function collectRetired(
  vault: string | null,
  files: ReadonlyArray<MemoryPage>,
  retired: RetiredRecord[],
  corrupted: CorruptedEntry[],
  walk: DirectoryWalk,
): void {
  // The serve-with-correction chain index, built on the first
  // pointer-bearing record and reused for the rest of the walk: a vault
  // whose retired records carry no pointer never pays for it. The scan
  // runs at the operator's own reach, so every on-disk page is indexed -
  // resolution here asks only whether the chain terminates, not who may
  // read it.
  let chainLookup: ReturnType<typeof buildChainLookup> | null = null;
  for (const page of files) {
    walk.step();
    const full = page.path;
    // Retired files we only need for topic + id (for supersede
    // bookkeeping) plus the optional `user_rejected_reason` that
    // drives signal-suppression (v0.10.1, _summary §6). We do a
    // lightweight frontmatter parse to avoid the strict folder
    // invariant check failing on permissive setups.
    try {
      const [meta] = page.read();
      const topic = typeof meta["topic"] === "string" ? meta["topic"] : "";
      const id = typeof meta["id"] === "string" ? meta["id"] : "";
      const principle = typeof meta["principle"] === "string" ? meta["principle"] : "";
      const scope = typeof meta["scope"] === "string" ? meta["scope"] : undefined;
      const userReason =
        typeof meta["user_rejected_reason"] === "string"
          ? (meta["user_rejected_reason"] as string).trim()
          : "";
      if (topic && id) {
        // Serve-with-correction coupling (truth-correctable-time-aware,
        // contract item 3): a retired record whose frontmatter declares
        // a successor - the serveable-retired regime - is scanned only
        // when that chain resolves to a tip, and is dropped fail-closed
        // when a hop dangles, so the pass never plans over a correction
        // that is not there. A record with no pointer is not in the
        // predicate's regime and is scanned as before.
        const pointer = meta[SUPERSEDED_BY_KEY];
        if (typeof pointer === "string" && pointer.trim() !== "") {
          chainLookup ??= buildChainLookup(vault);
          const verdict = chainVerdict(id, resolveChainTip(pointer, chainLookup));
          if (verdict.action === "drop") continue;
        }
        retired.push({
          path: full,
          topic,
          id,
          principle,
          ...(scope ? { scope } : {}),
          ...(userReason ? { user_rejected_reason: userReason } : {}),
          ...visibilitySlice(meta),
        });
      }
    } catch {
      corrupted.push({ path: full });
    }
  }
}

/**
 * Read the whole `Brain/` tree into memory. Pure: it opens no writer, takes
 * no clock and touches nothing on disk, which is why it is the one part of
 * the pass that is provably runnable on its own (see `dream-step.ts`).
 */
export function scanBrain(vault: string | null, opts: ScanBrainOptions = {}): ScanResult {
  const progress = progressCounter(OPERATION.dream, opts.onProgress);
  return withProgress(progress, () => scanBrainRun(vault, opts, progress));
}

function scanBrainRun(
  vault: string | null,
  opts: ScanBrainOptions,
  progress: ProgressCounter,
): ScanResult {
  const dirs = brainDirsOf(vault);
  // The four listings are `readdirSync` with no parse, so front-loading
  // them costs a directory read each and buys the stage its denominator:
  // the file count IS the unit this stage ticks in, and a reader watching
  // a bare counter cannot tell a scan a tenth of the way through from one
  // about to finish. The stage still opens before the first checkpoint,
  // so an already-elapsed guard leaves a stream that spoke once.
  const inbox = memoryPagesIn(vault, "inbox");
  const processed = memoryPagesIn(vault, "processed");
  // Archived signals are read as inactive history: the plan never counts
  // them, but a preference whose evidence link resolves to one still
  // derives its sign from it (see `deriveActiveSign`).
  const archived = memoryPagesIn(vault, "archived");
  const preferenceFiles = memoryPagesIn(vault, "preferences");
  const retiredFiles = memoryPagesIn(vault, "retired");
  progress.start(
    SCAN_STAGE,
    inbox.length +
      processed.length +
      archived.length +
      preferenceFiles.length +
      retiredFiles.length,
  );
  const signals: SignalRecord[] = [];
  const preferences: PreferenceRecord[] = [];
  const retired: RetiredRecord[] = [];
  const corrupted: CorruptedEntry[] = [];
  const tombstonedPreferences: CorruptedEntry[] = [];
  // One boundary for all four collectors: the per-file read is where the
  // work is, so it is where the deadline is honoured and where a reader
  // learns the scan is still moving. Collapsed into one object because
  // four collectors taking two more parameters each is four chances to
  // thread one of them and forget the other.
  const walk: DirectoryWalk = {
    step: () => {
      opts.safeguard?.checkpoint();
      progress.advance(SCAN_STAGE);
    },
  };

  // The directory boundary is a checkpoint too, and not only for
  // symmetry: a tree with no markdown in it crosses no per-file boundary
  // at all, and a deadline that only a populated vault can trip is a
  // deadline that is absent exactly where a caller cannot predict it.
  // The tick stays per file - a directory is not a unit of work a reader
  // can count.
  const collectors: ReadonlyArray<() => void> = [
    () => collectSignals(inbox, true, signals, corrupted, walk),
    () => collectSignals(processed, false, signals, corrupted, walk),
    () => collectSignals(archived, false, signals, corrupted, walk, true),
    () => collectPreferences(preferenceFiles, preferences, corrupted, walk, tombstonedPreferences),
    () => collectRetired(vault, retiredFiles, retired, corrupted, walk),
  ];
  for (const collect of collectors) {
    opts.safeguard?.checkpoint();
    collect();
  }

  return {
    signals,
    preferences,
    retired,
    corrupted,
    tombstonedPreferences,
    archivedNames: archivedNamesOf(vault, dirs.archived),
  };
}

/** The archived folder's entry names; the rebuilt pages' names on a store-only machine. */
function archivedNamesOf(vault: string | null, dir: string): ReadonlySet<string> {
  const stored = storeMemoryPages(vault, "archived");
  return stored === null ? entryNamesIn(dir) : new Set(stored.map((page) => page.name));
}

/** Every entry name directly inside `dir`, or none when it does not exist. */
function entryNamesIn(dir: string): ReadonlySet<string> {
  if (!existsSync(dir)) return new Set();
  return new Set(readdirSync(dir));
}

/** What the archive rule needs from the inbox, read the way the pass reads it. */
export interface InboxArchiveScan {
  /** Markdown files directly in `inbox/`, the files the pass walks. */
  readonly inboxFiles: number;
  /** Active inbox signal records: tombstoned and unparseable files left out. */
  readonly signals: ReadonlyArray<SignalRecord>;
  /** Every entry name in `inbox/archived/`. */
  readonly archivedNames: ReadonlySet<string>;
}

/**
 * The inbox half of {@link scanBrain}, for the `inbox-archivable` doctor
 * check: the same walk and the same per-file skips, so the check selects
 * from exactly the records the pass would.
 */
export function scanInboxForArchive(vault: string | null): InboxArchiveScan {
  const dirs = brainDirsOf(vault);
  const inbox = memoryPagesIn(vault, "inbox");
  const signals: SignalRecord[] = [];
  collectSignals(inbox, true, signals, [], { step: () => {} });
  return {
    inboxFiles: inbox.length,
    signals,
    archivedNames: archivedNamesOf(vault, dirs.archived),
  };
}