Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
18.3 KB
Raw Blame
/**
 * The render pass of the view writer (design D7 "Render protocol", tasks
 * 2.8 and 2.9). Only the view writer renders; every other machine's pass
 * writes no page and names the writer.
 *
 * A pass:
 *
 *  1. catches up on every held partition (and stops when a shard could
 *     not be read: a render over a partial read could hide records);
 *  2. hands the role over when the user moved it (`writer.ts`), and waits
 *     one catch-up after a fresh claim;
 *  3. under the device-local render lock, for every page:
 *     a. compares the file on disk with the writer's own renders of it;
 *        a file that is none of them is a human edit, diffed against the
 *        merge base of the last render and stored as human records
 *        (`corrections.ts`, `view-records.ts`);
 *     b. ingests each conflict copy a sync tool made of the page the same
 *        way, then removes it;
 *     c. renders the page from the store, three-way merges base, the
 *        user's file and the render (the user wins), and writes it only
 *        when its bytes change (`page-io.ts`), re-checking right before
 *        the replacement and capturing the bytes it replaces; a racing
 *        edit in those bytes is ingested and the page rendered again;
 *     d. writes the render record with the new merge base;
 *  4. before the pages, removes the pages of a retiring folder (topics,
 *     reports, sources) the store no longer renders (a renamed, merged or
 *     undone topic, a removed report), ingesting any edit they carry
 *     first, and moves a renamed topic's companion files to its new
 *     folder;
 *  5. writes a `view-pass` heartbeat when it wrote nothing for a day.
 *
 * A missing or empty page is never read as an edit (deleting a page does
 * not retire every rule on it); it is rendered again.
 */

import { PartitionUnavailableError } from "../store/appender.ts";
import { SETTING_VIEW_LANGUAGE } from "../store/constants.ts";
import { foldSettings, settingValue } from "../store/fold-settings.ts";
import { withLocalLock } from "../store/local-lock.ts";
import { textCarriesPrivateRegion, valueCarriesPrivateRegion } from "../store/private-region.ts";
import type { StoreHandle } from "../store/store-open.ts";
import {
  catchUp,
  catchUpSettings,
  partitionRecords,
  withOwnRecord,
  type ViewCatchUp,
} from "./catch-up.ts";
import { foldTopics } from "../topics/fold.ts";
import { VIEW_RENDER_ATTEMPTS } from "./constants.ts";
import { applyCorrections, foldCorrections, pageLines, reviewedEdits } from "./corrections.ts";
import { viewStrings } from "./i18n.ts";
import { matchLines, mergeLines } from "./line-diff.ts";
import { lanePages } from "./lane-view-pages.ts";
import {
  capturePage,
  conflictCopies,
  NotAPageFileError,
  readPage,
  replacePage,
  type ReplaceHooks,
} from "./page-io.ts";
import {
  aboutMePage,
  confirmedPreferences,
  decisionsPage,
  historyPages,
  sharedPreferences,
  viewEntries,
  type PageModel,
} from "./pages.ts";
import {
  bytesSum,
  clearIntent,
  heartbeat,
  ingestedSinceLastRender,
  ingestVersion,
  lastRender,
  loadBase,
  ownSums,
  pendingIntents,
  writeBase,
  writeIntent,
  writeRender,
  type RecordOptions,
  type RenderBase,
  type ReplacedReason,
} from "./view-records.ts";
import { claimSettled, releaseIfRequested, renderLockTarget } from "./writer.ts";
import { readPageStats, sameStat, statPage, writePageStats, type PageStats } from "./page-stats.ts";
import { pageFileInside } from "./page-path.ts";
import { retirePages, type RetireContext } from "./retire.ts";
import { topicPages } from "./topic-pages.ts";
import { foldViewWriter } from "./writer-fold.ts";

export interface ViewPassOptions extends RecordOptions {
  /** The view folder; `null` when none is configured (nothing is written). */
  readonly target: string | null;
  readonly hooks?: ReplaceHooks;
  /**
   * Asked before each page: `true` stops the pass there and reports the
   * remaining pages as deferred (a hook's time budget). The pass only
   * ever stops between pages.
   */
  readonly shouldStop?: () => boolean;
}

/** What happened to one page. */
export interface PageReport {
  readonly page: string;
  readonly status:
    | "written"
    | "unchanged"
    | "removed"
    | "refused"
    | "busy"
    | "base-pending"
    | "deferred";
  /** Human records written from edits of this page. */
  readonly ingested: number;
  readonly detail?: string;
}

export type ViewPassReport =
  | { readonly status: "incomplete-read"; readonly shards: ReadonlyArray<string> }
  | { readonly status: "no-writer" }
  | { readonly status: "not-writer"; readonly writer: string }
  | { readonly status: "released"; readonly to: string }
  | { readonly status: "waiting-for-claim" }
  | { readonly status: "no-target" }
  | { readonly status: "rendered"; readonly pages: ReadonlyArray<PageReport> };

/** Every page the store renders to now, with the user's corrections applied. */
export function buildPages(store: StoreHandle, read: ViewCatchUp): Map<string, PageModel> {
  const shared = partitionRecords(read, "shared");
  const lang = settingValue(foldSettings(shared), SETTING_VIEW_LANGUAGE);
  const strings = viewStrings(typeof lang === "string" ? lang : null);
  const all = sharedPreferences(shared);
  const prefs = confirmedPreferences(all);
  const entries = viewEntries(shared, all);
  const corrections = foldCorrections(shared);
  // Topic records that carry a `<private>` region stay off the pages, as
  // every other record family's do.
  const topics = foldTopics(shared.filter((r) => !valueCarriesPrivateRegion(r.body)));
  const models = [
    aboutMePage(store.storeId, prefs, strings),
    decisionsPage(store.storeId, entries, strings),
    ...historyPages(store.storeId, entries, strings),
    ...topicPages(store.storeId, topics, strings),
    ...lanePages(store.storeId, shared, strings),
  ];
  return new Map(models.map((m) => [m.page, applyCorrections(m, corrections)]));
}

function textOf(lines: ReadonlyArray<string>): string {
  return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
}

function hasContent(bytes: Buffer | null): bytes is Buffer {
  return bytes !== null && pageLines(bytes.toString("utf8")).some((l) => l.trim() !== "");
}

interface PageContext {
  readonly store: StoreHandle;
  readonly options: ViewPassOptions;
  read: ViewCatchUp;
  /** The pages as `read` renders them; dropped when an ingest changes the records. */
  models?: Map<string, PageModel>;
  /** Device-local stats of the pages as the writer last left them. */
  readonly stats: PageStats;
}

function modelOf(ctx: PageContext, page: string): PageModel | undefined {
  ctx.models ??= buildPages(ctx.store, ctx.read);
  return ctx.models.get(page);
}

/**
 * The no-change case without reading the page: its file is exactly as the
 * writer left it (same size, modification and change time, file id), that
 * text is the last render, no conflict copy exists, and the page would
 * render to the same bytes.
 */
function unchangedByStat(ctx: PageContext, page: string, path: string): boolean {
  const cached = ctx.stats.get(page);
  const last = lastRender(ctx.read, page);
  if (cached === undefined || last === null || last.record.body["sum"] !== cached.sum) return false;
  if (!sameStat(cached, statPage(path)) || conflictCopies(path).length > 0) return false;
  const model = modelOf(ctx, page);
  return model !== undefined && bytesSum(textOf(model.lines.map((l) => l.text))) === cached.sum;
}

/** Store a version that is not the writer's own render; returns the records written. */
function ingest(
  ctx: PageContext,
  page: string,
  base: RenderBase | null,
  text: string,
  reason: ReplacedReason,
): { records: number; isPrivate: boolean; refused: boolean } {
  // The same bytes seen twice (a conflict copy of an edit already read, a
  // race that delivered them again) are stored once.
  const seen = ingestedSinceLastRender(ctx.read, page).get(bytesSum(text));
  if (seen !== undefined) return { records: 0, isPrivate: seen === "private", refused: false };
  // An edit that cannot be trusted line by line (`reviewedEdits`) is kept
  // as bytes only (`view-replaced`, recoverable), never as human records.
  const reviewed =
    base === null || reason === "foreign"
      ? { records: [], refusal: null }
      : reviewedEdits(page, pageLines(base.text), base.keys, pageLines(text));
  const done = ingestVersion(ctx.store, page, text, reason, reviewed.records, ctx.options);
  ctx.read = catchUp(ctx.store, ctx.options.now);
  delete ctx.models;
  return {
    records: done.records,
    isPrivate: done.partition === "private",
    refused: reviewed.refusal !== null,
  };
}

function renderOnePage(ctx: PageContext, page: string, path: string): PageReport {
  if (unchangedByStat(ctx, page, path)) return { page, status: "unchanged", ingested: 0 };
  const report = renderPageNow(ctx, page, path);
  const stat = statPage(path);
  const last = lastRender(ctx.read, page);
  // Cached only when this pass left the writer's own render on disk: a
  // page left as found (its base not synced, refused, busy) must be read
  // again next pass, or an edit on it would never be ingested.
  const settled = report.status === "written" || report.status === "unchanged";
  if (stat !== null && last !== null && settled) {
    ctx.stats.set(page, { ...stat, sum: last.record.body["sum"] as string });
  } else {
    ctx.stats.delete(page);
  }
  return report;
}

function renderPageNow(ctx: PageContext, page: string, path: string): PageReport {
  let ingested = 0;
  for (let attempt = 0; attempt < VIEW_RENDER_ATTEMPTS; attempt++) {
    const last = lastRender(ctx.read, page);
    const base = last === null ? null : loadBase(ctx.store, last);
    if (last !== null && base === null) {
      return { page, status: "base-pending", ingested, detail: "the merge base has not synced" };
    }
    const own = ownSums(ctx.read, page);
    const disk = readPage(path);
    let edited = false;
    let privateEdit = false;
    if (hasContent(disk) && !own.has(bytesSum(disk))) {
      const done = ingest(ctx, page, base, disk.toString("utf8"), base ? "edit" : "foreign");
      ingested += done.records;
      // A refused edit is not merged: the page is rendered back whole.
      edited = base !== null && !done.refused;
      privateEdit = done.isPrivate;
    }
    for (const copy of conflictCopies(path)) {
      // A name that is not a regular file (a folder, a link) is refused
      // where it is. A copy is taken out of its name before it is read: one
      // a sync tool writes again meanwhile is left for the next pass.
      readPage(copy);
      const captured = capturePage(copy);
      if (captured === null) continue;
      try {
        if (hasContent(captured.bytes) && !own.has(bytesSum(captured.bytes))) {
          const done = ingest(ctx, page, base, captured.bytes.toString("utf8"), "conflict-copy");
          ingested += done.records;
          privateEdit ||= done.isPrivate;
        }
      } catch (err) {
        captured.restore();
        throw err;
      }
      captured.discard();
    }
    const model = modelOf(ctx, page);
    if (model === undefined) return { page, status: "unchanged", ingested };
    const rendered = model.lines.map((l) => l.text);
    let lines = rendered;
    if (edited && !privateEdit && base !== null) {
      lines = mergeLines(pageLines(base.text), pageLines(disk!.toString("utf8")), rendered);
      if (textCarriesPrivateRegion(textOf(lines))) lines = rendered;
    }
    const match = matchLines(lines, rendered);
    const keys = lines.map((_, i) => (match[i]! >= 0 ? model.lines[match[i]!]!.key : null));
    const text = textOf(lines);
    const bytes = Buffer.from(text, "utf8");
    const intent = {
      page,
      ...(model.topic !== undefined ? { topic: model.topic } : {}),
      sum: bytesSum(bytes),
      base: writeBase(ctx.store, page, { text, keys }),
      bytes: bytes.length,
    };
    if (disk !== null && disk.equals(bytes)) {
      // Already on disk: an edit the render now matches still needs its base recorded.
      if (!own.has(intent.sum)) {
        ctx.read = withOwnRecord(ctx.read, "shared", writeRender(ctx.store, intent, ctx.options));
      }
      return { page, status: edited ? "written" : "unchanged", ingested };
    }
    writeIntent(ctx.store, intent);
    const outcome = replacePage(path, bytes, disk, ctx.options.hooks);
    if (outcome.status === "changed") {
      clearIntent(ctx.store, page);
      continue;
    }
    if (outcome.status === "placed") {
      ctx.read = withOwnRecord(ctx.read, "shared", writeRender(ctx.store, intent, ctx.options));
    }
    clearIntent(ctx.store, page);
    const replaced = outcome.replaced;
    const raced =
      hasContent(replaced) &&
      (disk === null || !replaced.equals(disk)) &&
      !ownSums(ctx.read, page).has(bytesSum(replaced));
    if (raced) {
      const done = ingest(ctx, page, base, replaced.toString("utf8"), "race");
      ingested += done.records;
    }
    outcome.discard();
    if (outcome.status === "placed" && !raced) return { page, status: "written", ingested };
  }
  return { page, status: "busy", ingested, detail: "the page kept changing during the render" };
}

/** Write the render records a dead pass left only as intents. */
function recoverIntents(ctx: PageContext, target: string): void {
  for (const intent of pendingIntents(ctx.store)) {
    const path = pageFileInside(target, intent.page);
    let disk: Buffer | null = null;
    try {
      disk = path === null ? null : readPage(path);
    } catch (err) {
      if (!(err instanceof NotAPageFileError)) throw err;
    }
    const recorded = ownSums(ctx.read, intent.page).has(intent.sum);
    if (disk !== null && bytesSum(disk) === intent.sum && !recorded) {
      writeRender(ctx.store, intent, ctx.options);
    }
    clearIntent(ctx.store, intent.page);
  }
  ctx.read = catchUp(ctx.store, ctx.options.now);
}

/** Errors of a page another program holds; the page is rendered on a later pass. */
const PAGE_BUSY_CODES: ReadonlySet<string> = new Set(["EBUSY", "EPERM", "EACCES"]);

function renderPages(ctx: PageContext, target: string): PageReport[] {
  recoverIntents(ctx, target);
  const reports: PageReport[] = [];
  ctx.models ??= buildPages(ctx.store, ctx.read);
  const retire: RetireContext = {
    store: ctx.store,
    read: () => ctx.read,
    models: () => (ctx.models ??= buildPages(ctx.store, ctx.read)),
    stats: ctx.stats,
    ingest: (page, base, text, reason) => ingest(ctx, page, base, text, reason).records,
  };
  reports.push(...retirePages(retire, target, pageFailure, ctx.options.shouldStop));
  ctx.models ??= buildPages(ctx.store, ctx.read);
  // The page list of this pass: an ingest may rebuild the models, and a
  // page it renamed is retired by the next pass.
  for (const page of ctx.models.keys()) {
    if (ctx.options.shouldStop?.() === true) {
      reports.push({ page, status: "deferred", ingested: 0, detail: "the time budget ran out" });
      continue;
    }
    // A page whose folder is a link (or leads out of the view) is never written.
    const path = pageFileInside(target, page);
    if (path === null) {
      reports.push({
        page,
        status: "refused",
        ingested: 0,
        detail: "the page path leaves the view",
      });
      continue;
    }
    try {
      reports.push(renderOnePage(ctx, page, path));
    } catch (err) {
      reports.push(pageFailure(page, err));
    }
  }
  return reports;
}

/** The report of a page whose render or removal failed for a reason a later pass may not have. */
function pageFailure(page: string, err: unknown): PageReport {
  if (err instanceof PartitionUnavailableError || err instanceof NotAPageFileError) {
    return { page, status: "refused", ingested: 0, detail: err.message };
  }
  if (PAGE_BUSY_CODES.has((err as NodeJS.ErrnoException).code ?? "")) {
    // Windows refuses to rename a page another program holds open.
    return { page, status: "busy", ingested: 0, detail: (err as Error).message };
  }
  throw err;
}

/** One render pass on this machine. */
export function runViewPass(store: StoreHandle, options: ViewPassOptions): ViewPassReport {
  // The writer records first: a machine that is not the writer reads
  // nothing else.
  const settings = catchUpSettings(store, options.now);
  if (settings === null) return { status: "incomplete-read", shards: ["shared/settings"] };
  const state = foldViewWriter(settings);
  if (state.writer === null) return { status: "no-writer" };
  if (state.writer !== store.device) return { status: "not-writer", writer: state.writer };
  const to = releaseIfRequested(store, state, options);
  if (to !== null) return { status: "released", to };
  if (!claimSettled(store, state)) return { status: "waiting-for-claim" };
  return withLocalLock(renderLockTarget(store), (): ViewPassReport => {
    // Asked again under the lock: another pass may have handed the role
    // over while this one waited, and no render starts after a release.
    const settled = catchUpSettings(store, options.now);
    if (settled === null) return { status: "incomplete-read", shards: ["shared/settings"] };
    const current = foldViewWriter(settled);
    if (current.writer !== store.device) {
      return current.writer === null
        ? { status: "no-writer" }
        : { status: "not-writer", writer: current.writer };
    }
    const handed = releaseIfRequested(store, current, options);
    if (handed !== null) return { status: "released", to: handed };
    // The catch-up is taken under the lock too: a pass that rendered while
    // this one waited wrote render records a read from before the wait
    // would not know, and its pages would then read as a human edit.
    const read = catchUp(store, options.now);
    if (read.unreadable.length > 0) {
      return { status: "incomplete-read", shards: read.unreadable.map((s) => s.path) };
    }
    const ctx: PageContext = { store, options, read, stats: readPageStats(store) };
    const target = options.target;
    const pages = target === null ? [] : renderPages(ctx, target);
    writePageStats(store, ctx.stats);
    // The daily "alive" record only for a genuinely idle pass: a view
    // folder to render to, and every page already as rendered. A writer
    // that renders nothing (no folder, pages refused or busy) stays
    // silent, so the doctor's silent-writer check can say so.
    const idle = target !== null && pages.every((p) => p.status === "unchanged");
    if (idle) heartbeat(store, ctx.read, options);
    return target === null ? { status: "no-target" } : { status: "rendered", pages };
  });
}