Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
22.5 KB
Raw Blame
/**
 * Capture-note contract (Knowledge intake suite, seam 1, t_f8f5ef6a).
 *
 * One module owning the inbound-capture staging vocabulary: the frontmatter
 * kind, provenance (source channel, sender identity, capture timestamp), the
 * staging and archive path helpers, and the read/write/list/archive
 * functions. The inbound Telegram bot writes captures ONLY through this
 * contract; the inbox-drain pass reads captures ONLY through it. That single
 * ownership is what lets the two features share one on-disk shape without
 * either reaching into the other.
 *
 * The layout mirrors the existing inbox-versus-processed distinction in
 * `governance/forget-plan.ts`: a capture lands in `Brain/captures/`
 * (staging) and moves to `Brain/captures/processed/` (archive) once a drain
 * routes it. The captures subtree is deliberately separate from the signal
 * inbox so the dream pass and signal machinery stay byte-identical for vaults
 * that never capture.
 *
 * ## Guidance, and why it is not part of the body
 *
 * A capture may carry an optional `guidance` string: what the capturer
 * says should be DONE with the text, as opposed to the text itself. It
 * lands in a `## Guidance` body section under a frontmatter marker, and
 * it joins the id hash - two captures differing only in guidance are two
 * captures, and hashing the body alone would collapse them onto one stem
 * and hand the second to the `-2` allocator. Both additions are
 * conditional on the field being present, so a capture without guidance
 * is byte-identical to one written before the field existed.
 *
 * ## Write-time marker neutralization (t_c2ac4fc8)
 *
 * The body and the guidance are CALLER text - the one thing in the page
 * this contract did not author - so under the markdown-neutralization
 * gate's `enforce` mode they pass through the pure neutralizer
 * (`text/neutralize-markdown.ts`) before anything is composed: planted
 * wikilinks, `@osb` markers, ```osb fence lines and ATX headings are
 * backslash-escaped, and the conditional `neutralized_markers` key names
 * the kinds that changed. The writer's own `## Guidance` heading is
 * renderer-authored and never passes through the neutralizer, so a body
 * whose captured text quoted a guidance heading can never win the split
 * the frontmatter marker authorizes. The two halves are neutralized in
 * order and the guidance's detection runs in the fence context the
 * rendered prefix leaves (body, separator, heading): the page is one text
 * to every scanner, so a fence split across the two halves - opened by
 * the body, closed or left open by the guidance's own bytes - has
 * combined-page meaning neither half's standalone scan can see, and a
 * marker that page-scanner verdict makes live is caught in the guidance
 * half while a marker it masks stays untouched there. `captureBaseId`
 * hashes the PERSISTED post-neutralize body, so identity follows what is
 * on disk; a gate flip between captures of the same text therefore yields
 * a different id - an honest duplicate, documented rather than accidental.
 * Under `off` (the default) and `shadow` the bytes are untouched: the
 * seam never reaches the neutralizer under either mode.
 */

import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
import { dirname, join } from "node:path";
import { createHash } from "node:crypto";

import type { FrontmatterMap } from "../../types.ts";
import { atomicWriteText, renameWithRetry } from "../../fs-atomic.ts";
import {
  BRAIN_CAPTURES_PROCESSED_REL,
  BRAIN_CAPTURES_REL,
  allocateAndCreate,
  captureArchivePath,
  capturesDir,
  capturesProcessedDir,
  captureStagingPath,
  captureWatermarkPath,
  vaultRelative,
} from "../paths.ts";
import { declaredPageTypes } from "../schema-pack.ts";
import {
  MARKDOWN_NEUTRALIZE_MODE,
  neutralizeMarkdown,
  neutralizedMarkersFrontmatter,
  resolveMarkdownNeutralizeModeForProcess,
} from "../text/neutralize-markdown.ts";
import {
  assertLaneWrite,
  lanePageExists,
  lanePagesInStore,
  listLaneFiles,
  moveLanePage,
  parseLanePage,
  writeLanePage,
} from "../../store/lane-pages.ts";

/** Frontmatter `kind:` marker of a staged inbound capture. */
export const BRAIN_CAPTURE_KIND = "brain-capture";

/** Filename prefix of every capture page (`cap-<date>-<time>-<hash>`). */
export const CAPTURE_ID_PREFIX = "cap";

/** Length of the content-hash suffix that keeps same-second captures distinct. */
const CAPTURE_HASH_LEN = 8;

/** Provenance of one inbound capture. */
export interface CaptureProvenance {
  /** Capture channel, e.g. `telegram`. */
  readonly source: string;
  /** Sender identity within the channel, e.g. the Telegram chat id. */
  readonly sender: string;
  /** ISO-8601 whole-second capture timestamp. */
  readonly capturedAt: string;
}

/** One capture note as read back from disk. */
export interface CaptureNote {
  /** Filename stem (also the frontmatter id): `cap-<date>-<time>-<hash>`. */
  readonly id: string;
  /** Vault-relative path (POSIX separators). */
  readonly path: string;
  /** The captured text body, with any guidance section split back off. */
  readonly body: string;
  /**
   * What the capturer said should be DONE with this capture, verbatim, or
   * `null` when they said nothing. Never merged into {@link body}: the two
   * are written by the same person in one breath and mean different
   * things, and a drain that cannot tell them apart would route on an
   * instruction as though it were content.
   */
  readonly guidance: string | null;
  readonly provenance: CaptureProvenance;
  /** `true` in the staging area, `false` once archived. */
  readonly staged: boolean;
}

export interface WriteCaptureInput {
  readonly body: string;
  readonly provenance: CaptureProvenance;
  /**
   * Optional per-capture handling note, rendered as a `## Guidance` body
   * section and read back verbatim. Present-but-blank is refused rather
   * than stored as an empty section; omit the field to say nothing.
   */
  readonly guidance?: string;
}

/** Typed failure raised by every contract refusal - never a silent skip. */
export class CaptureContractError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "CaptureContractError";
  }
}

function requireNonEmpty(value: string, label: string): string {
  const trimmed = value.trim();
  if (trimmed.length === 0) {
    throw new CaptureContractError(`capture ${label} must not be empty`);
  }
  return trimmed;
}

/** Remedy carried by every undeclared-kind refusal. */
const DECLARE_KIND_REMEDY = "declare it in Brain/_brain.yaml under schema.page_types to accept it";

/**
 * Refuse a stamped `kind` the vault's pack does not declare
 * (trust-surface-hardening wave; kanban t_151a564c).
 *
 * Consulted only where the vault made a declaration: {@link declaredPageTypes}
 * returns `null` for an absent or declaration-free config, and that fails
 * open - today's behavior, byte for byte. A non-null declaration that omits
 * the kind refuses with this module's typed error naming the kind, the
 * declared set, and the remedy. Callers run this BEFORE any filesystem
 * effect, so a refused capture never creates a directory and never
 * consumes an allocator name.
 */
export function requireDeclaredCaptureKind(vault: string, kind: string): void {
  const declared = declaredPageTypes(vault);
  if (declared === null || declared.includes(kind)) return;
  throw new CaptureContractError(
    `capture kind '${kind}' is not declared in this vault's schema.page_types ` +
      `(declared: ${declared.join(", ")}); ${DECLARE_KIND_REMEDY}`,
  );
}

/**
 * Derive the filename stem from the capture timestamp plus a short content
 * hash. The date-then-time prefix makes lexical order match chronological
 * order; the hash keeps two same-second captures from the same sender
 * distinct before the allocator has to append a numeric suffix.
 */
function captureBaseId(
  body: string,
  provenance: CaptureProvenance,
  guidance: string | null,
): { prefix: string; slug: string } {
  const stamp = provenance.capturedAt.trim();
  const date = stamp.slice(0, 10);
  const time = stamp.slice(11, 19).replace(/:/g, "");
  // Guidance joins the hash because two captures differing only in it are
  // two captures - "read this" and "act on this" over the same text are
  // not the same message - and hashing only the body would collapse them
  // onto one stem and hand the second to the `-2` allocator, where the id
  // stops describing the content.
  //
  // The segment is CONDITIONAL, and that is the difference between an
  // additive field and a re-keying: with no guidance the hash input is
  // the historical three-part string byte for byte, so every capture the
  // Telegram bot writes today keeps the id it would have had.
  const hash = createHash("sha256")
    .update(
      `${provenance.source}\0${provenance.sender}\0${body}` +
        (guidance === null ? "" : `\0${guidance}`),
      "utf8",
    )
    .digest("hex")
    .slice(0, CAPTURE_HASH_LEN);
  return { prefix: `${CAPTURE_ID_PREFIX}-${date}`, slug: `${time}-${hash}` };
}

/** Heading of the body section carrying {@link WriteCaptureInput.guidance}. */
const GUIDANCE_HEADING = "## Guidance";

/**
 * Frontmatter marker saying the body carries a {@link GUIDANCE_HEADING}
 * section that this contract wrote.
 *
 * The marker is what makes the split unambiguous, and without it the
 * feature would be a misreading waiting to happen: a captured message may
 * itself end in a `## Guidance` heading - people quote documents - and a
 * parser that went looking for the heading would hand that quotation back
 * as an instruction. The writer sets the marker, so only a capture this
 * contract gave guidance to is ever split.
 */
const GUIDANCE_MARKER_FIELD = "capture_guidance";

/**
 * Read the marker. Both spellings, because the frontmatter reader hands
 * back the scalar as written and a YAML `true` arrives here as the string
 * `"true"` - the same two-form check `truth/contamination.ts:31` and
 * `trust/untrusted-provenance.ts:79` already make.
 */
function guidanceMarked(meta: FrontmatterMap): boolean {
  const value = meta[GUIDANCE_MARKER_FIELD];
  return value === true || value === "true";
}

/** Compose the on-disk body: the capture, then the guidance section. */
function renderCaptureBody(body: string, guidance: string | null): string {
  return guidance === null ? body : `${body}\n\n${GUIDANCE_HEADING}\n\n${guidance}`;
}

/**
 * The exact rendered bytes ahead of the guidance on the page: the body
 * half, the blank-line separator and the writer's own heading. This - not
 * the body alone - is the fence context the guidance's neutralization runs
 * in, because it is what every scanner of the rendered page sees before
 * the guidance's first byte (see the seam comment in
 * {@link writeCaptureNote}).
 */
function renderedGuidanceContext(body: string): string {
  return `${body}\n\n${GUIDANCE_HEADING}\n\n`;
}

/**
 * Split a marked body back into its two halves.
 *
 * The leading `[\s\S]*` is greedy, so it takes as much of the body as it
 * can and the separator matches at its LAST occurrence - which is the one
 * {@link renderCaptureBody} appended, whatever the captured text itself
 * quotes. A marked body with
 * no such section is a corrupted page rather than a guidance-less one, and
 * it is refused: reporting `guidance: null` there would silently return
 * the marker's own text as part of the capture.
 */
function splitGuidance(body: string, id: string): { body: string; guidance: string } {
  const match = new RegExp(String.raw`^([\s\S]*)\n\n${GUIDANCE_HEADING}\n\n([\s\S]*)$`).exec(body);
  if (match === null) {
    throw new CaptureContractError(
      `capture ${id} declares ${GUIDANCE_MARKER_FIELD} but its body carries no ` +
        `'${GUIDANCE_HEADING}' section`,
    );
  }
  return { body: match[1]!.trim(), guidance: match[2]!.trim() };
}

/** Write one capture note into the staging area through the contract. */
export function writeCaptureNote(vault: string, input: WriteCaptureInput): CaptureNote {
  // Vault-identity write guard (context-integrity-gates, Unit J).
  assertLaneWrite(vault);
  const body = requireNonEmpty(input.body, "body");
  const source = requireNonEmpty(input.provenance.source, "source");
  const sender = requireNonEmpty(input.provenance.sender, "sender");
  const capturedAt = requireNonEmpty(input.provenance.capturedAt, "captured_at");
  // Absent says nothing; present-but-blank is a caller that meant to say
  // something and sent whitespace, which is the shape a silent drop hides.
  const guidance =
    input.guidance === undefined ? null : requireNonEmpty(input.guidance, "guidance");

  // Pack vocabulary gate (t_151a564c), before ANY filesystem effect: a
  // refused capture creates no directory and consumes no allocator name.
  requireDeclaredCaptureKind(vault, BRAIN_CAPTURE_KIND);

  // Write-time marker neutralization (t_c2ac4fc8), resolved before the
  // id so the hash follows the bytes that will actually be persisted.
  // Only `enforce` writes neutralized bytes; `off` and `shadow` leave
  // the caller's text byte-identical - shadow's preview is the doctor's
  // planted-rows audit, which recomputes the would-be rows from the
  // vault's bytes on its own, so a shadow-mode seam has nothing to run.
  // The caller's body and guidance are the only caller-authored halves -
  // the guidance HEADING below is renderer-authored and never passes
  // through the neutralizer.
  //
  // The two halves are neutralized in order, and the guidance's detection
  // runs in the fence context the rendered prefix leaves: the page is one
  // text to every scanner, so a fence the body opens and the guidance's
  // own bytes close (or leave open) has combined-page meaning neither
  // half's standalone scan can see - the split-fence hole, where a
  // planted marker live on the combined page scanned clean in both
  // halves. The context is the exact rendered bytes ahead of the guidance
  // (the neutralized body, the separator, the writer heading), so the
  // half's verdict agrees with the combined page's in both directions.
  const gate = resolveMarkdownNeutralizeModeForProcess();
  let storedBody = body;
  let storedGuidance = guidance;
  let neutralized: FrontmatterMap = {};
  if (gate === MARKDOWN_NEUTRALIZE_MODE.enforce) {
    const neutralBody = neutralizeMarkdown(body);
    const neutralGuidance =
      guidance === null
        ? null
        : neutralizeMarkdown(guidance, {
            precedingContext: renderedGuidanceContext(neutralBody.text),
          });
    if (neutralBody.changed || (neutralGuidance !== null && neutralGuidance.changed)) {
      storedBody = neutralBody.text;
      if (neutralGuidance !== null) storedGuidance = neutralGuidance.text;
      neutralized = neutralizedMarkersFrontmatter(
        [...new Set([...neutralBody.kinds, ...(neutralGuidance?.kinds ?? [])])].toSorted(),
      );
    }
  }

  const dir = capturesDir(vault);
  const { prefix, slug } = captureBaseId(
    storedBody,
    { source, sender, capturedAt },
    storedGuidance,
  );
  const provenance: CaptureProvenance = { source, sender, capturedAt };

  // Allocation and creation are one step: two captures that hash to the
  // same stem race for the name, and the loser of the exclusive create
  // must fall through to `-2` rather than drop the message (GitHub
  // #161). The `id` is derived per attempt because it has to equal the
  // filename it ships in.
  const { allocation: allocated, value: id } = allocateAndCreate(
    { vault, targetDir: dir, prefix, slug },
    (allocation) => {
      const captureId = `${prefix}-${allocation.slug}`;
      // The kind and the vault-relative path its two siblings already
      // pass: without them a collision here reported a raw errno naming
      // an absolute path, which leaks the operator's home directory into
      // CLI output and MCP envelopes and never says what collided.
      writeLanePage(
        vault,
        allocation.path,
        captureFrontmatter(captureId, provenance, storedGuidance, neutralized),
        renderCaptureBody(storedBody, storedGuidance),
        {
          overwrite: false,
          existsErrorKind: "capture",
          vaultForRelativePath: vault,
        },
      );
      return captureId;
    },
  );
  return Object.freeze({
    id,
    path: vaultRelative(allocated.path, vault),
    // What was persisted - under enforce mode the neutralized bytes, so
    // the returned note answers for the file it shipped in.
    body: storedBody,
    guidance: storedGuidance,
    provenance,
    staged: true,
  });
}

function captureFrontmatter(
  id: string,
  provenance: CaptureProvenance,
  guidance: string | null,
  neutralized: FrontmatterMap,
): FrontmatterMap {
  return {
    kind: BRAIN_CAPTURE_KIND,
    id,
    capture_source: provenance.source,
    capture_sender: provenance.sender,
    captured_at: provenance.capturedAt,
    // Emitted only when there IS guidance, so a capture without it is
    // byte-identical to one written before this field existed.
    ...(guidance === null ? {} : { [GUIDANCE_MARKER_FIELD]: true }),
    // Emitted only when enforce-mode neutralization CHANGED bytes; a
    // clean capture adds nothing and stays byte-identical to the shape
    // this contract wrote before the gate existed.
    ...neutralized,
    tags: ["brain", "brain/capture"],
  };
}

function parseCaptureFile(
  vault: string,
  absPath: string,
  relPath: string,
  staged: boolean,
): CaptureNote | null {
  let meta: FrontmatterMap;
  let body: string;
  try {
    [meta, body] = parseLanePage(vault, absPath);
  } catch {
    return null;
  }
  if (meta["kind"] !== BRAIN_CAPTURE_KIND) return null;
  const id =
    typeof meta["id"] === "string" && meta["id"].length > 0 ? meta["id"] : basename(relPath);
  const source = typeof meta["capture_source"] === "string" ? meta["capture_source"] : "";
  const sender = typeof meta["capture_sender"] === "string" ? meta["capture_sender"] : "";
  const capturedAt = typeof meta["captured_at"] === "string" ? meta["captured_at"] : "";
  const split = guidanceMarked(meta)
    ? splitGuidance(body.trim(), id)
    : { body: body.trim(), guidance: null };
  return Object.freeze({
    id,
    path: relPath,
    body: split.body,
    guidance: split.guidance,
    provenance: Object.freeze({ source, sender, capturedAt }),
    staged,
  });
}

function basename(relPath: string): string {
  const parts = relPath.split("/");
  return (parts[parts.length - 1] ?? "").replace(/\.md$/u, "");
}

function listCapturesIn(
  vault: string,
  dir: string,
  relRoot: string,
  staged: boolean,
): CaptureNote[] {
  if (!lanePageExists(vault, dir)) return [];
  const out: CaptureNote[] = [];
  for (const name of listLaneFiles(vault, dir)) {
    const abs = join(dir, name);
    const note = parseCaptureFile(vault, abs, `${relRoot}/${name}`, staged);
    if (note !== null) out.push(note);
  }
  return out;
}

function sortById(notes: CaptureNote[]): CaptureNote[] {
  return notes.toSorted((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
}

/** Every staged (not-yet-drained) capture, sorted chronologically by id. */
export function listStagedCaptures(vault: string): CaptureNote[] {
  return sortById(listCapturesIn(vault, capturesDir(vault), BRAIN_CAPTURES_REL, true));
}

/** Every archived capture, sorted chronologically by id. */
export function listArchivedCaptures(vault: string): CaptureNote[] {
  return sortById(
    listCapturesIn(vault, capturesProcessedDir(vault), BRAIN_CAPTURES_PROCESSED_REL, false),
  );
}

/**
 * Every capture, staged or archived, sorted chronologically by id.
 * Archiving must never hide a capture from `/catchup`, so both areas are
 * unioned here.
 */
export function listAllCaptures(vault: string): CaptureNote[] {
  return sortById([...listStagedCaptures(vault), ...listArchivedCaptures(vault)]);
}

/** Read one capture by id from either the staging or archive area. */
export function readCaptureNote(vault: string, id: string): CaptureNote | null {
  const staging = captureStagingPath(vault, id);
  if (lanePageExists(vault, staging)) {
    return parseCaptureFile(vault, staging, `${BRAIN_CAPTURES_REL}/${id}.md`, true);
  }
  const archive = captureArchivePath(vault, id);
  if (lanePageExists(vault, archive)) {
    return parseCaptureFile(vault, archive, `${BRAIN_CAPTURES_PROCESSED_REL}/${id}.md`, false);
  }
  return null;
}

/** Move a staged capture into the processed archive through the contract. */
export function archiveCapture(vault: string, id: string): CaptureNote {
  // Vault-identity write guard (context-integrity-gates, Unit J).
  assertLaneWrite(vault);
  const staging = captureStagingPath(vault, id);
  if (!lanePageExists(vault, staging)) {
    throw new CaptureContractError(`no staged capture to archive: ${id}`);
  }
  const archive = captureArchivePath(vault, id);
  if (lanePagesInStore(vault)) {
    moveLanePage(vault, staging, archive);
  } else {
    mkdirSync(capturesProcessedDir(vault), { recursive: true });
    try {
      renameWithRetry(staging, archive);
    } catch {
      // Cross-device rename fallback: copy then remove the source.
      atomicWriteText(archive, readFileSync(staging, "utf8"));
      rmSync(staging, { force: true });
    }
  }
  const note = parseCaptureFile(vault, archive, `${BRAIN_CAPTURES_PROCESSED_REL}/${id}.md`, false);
  if (note === null) {
    throw new CaptureContractError(`archived capture is unreadable: ${id}`);
  }
  return note;
}

/** Captures newer than `watermark` (an id), staged or archived. */
export function capturesSince(vault: string, watermark: string | null): CaptureNote[] {
  const all = listAllCaptures(vault);
  if (watermark === null || watermark.length === 0) return all;
  return all.filter((c) => c.id > watermark);
}

interface WatermarkFile {
  readonly last_acknowledged: string;
}

/** Read the last capture id acknowledged by a `/catchup` reply, or null. */
export function readCatchupWatermark(vault: string): string | null {
  const path = captureWatermarkPath(vault);
  if (!existsSync(path)) return null;
  try {
    const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<WatermarkFile>;
    return typeof parsed.last_acknowledged === "string" && parsed.last_acknowledged.length > 0
      ? parsed.last_acknowledged
      : null;
  } catch {
    return null;
  }
}

/** Advance the catchup watermark to `id`. */
export function writeCatchupWatermark(vault: string, id: string): void {
  // Vault-identity write guard (context-integrity-gates, Unit J).
  assertLaneWrite(vault);
  // The watermark is machine state: in the vault on an unswitched machine,
  // in device-local state once switched (`machineStatePath`).
  const path = captureWatermarkPath(vault);
  mkdirSync(dirname(path), { recursive: true });
  const payload: WatermarkFile = { last_acknowledged: id };
  atomicWriteText(path, `${JSON.stringify(payload)}\n`);
}