Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
11.4 KB
Raw Blame
/**
 * Extraction dispatch and the derived section of a summary page.
 *
 * {@link extractSource} is the one place a source's bytes meet an extractor:
 * an exhaustive switch over the registry's extractor for the source's
 * format, so a format added to the registry with a new extractor does not
 * type-check until it has an arm here. {@link deriveSourceSection} is what
 * ingest writes from it onto a summary page: for an in-vault HTML, CSV or
 * TSV file the caller may read, the format, the content hash and one
 * section (`## Parts` or `## Table`); for every other source nothing.
 */

import { remoteDenyVisibilityTokens } from "../../graph/privacy-markers.ts";
import type { FrontmatterMap } from "../../types.ts";
import { FRONTMATTER_RE, parseFrontmatterText } from "../../vault.ts";
import { isSourceHidden, readSourceOrigin, SourceTrustError } from "../intake/source-trust.ts";
import {
  INTAKE_TRUST,
  type IntakeTrust,
  sourceContentHashFrontmatter,
} from "../trust/untrusted-provenance.ts";
import { extractHtml, type HtmlExtractResult, renderPartsSection } from "./extract-html.ts";
import {
  SOURCE_EXTRACT_SKIP_REASON,
  SOURCE_EXTRACTOR,
  SOURCE_FORMAT,
  type SourceExtractSkipReason,
  type SourceFormat,
  sourceFormatOf,
  sourceFormatSpec,
} from "./source-formats.ts";
import {
  type TableFormat,
  tableNote,
  tableNoteFrontmatter,
  type TableNoteRendered,
  type TableNoteResult,
  type TableNoteSkipped,
} from "./table-note.ts";

/** What the dispatch made of one source's bytes. */
export type SourceExtraction =
  | {
      readonly extractor: typeof SOURCE_EXTRACTOR.html;
      readonly format: SourceFormat;
      readonly html: HtmlExtractResult;
    }
  | {
      readonly extractor: typeof SOURCE_EXTRACTOR.table;
      readonly format: SourceFormat;
      readonly table: TableNoteResult;
    }
  | {
      readonly extractor: null | typeof SOURCE_EXTRACTOR.verbatim;
      readonly format: SourceFormat | null;
      readonly reason: SourceExtractSkipReason;
    };

/**
 * Dispatch `bytes` addressed by `path` to the extractor its format names.
 * An unregistered extension answers `format-unknown` with no format; a text
 * format is read verbatim and a named format without an extractor is
 * `format-not-extractable`. Pure: no I/O.
 */
export function extractSource(path: string, bytes: Uint8Array): SourceExtraction {
  const format = sourceFormatOf(path);
  if (format === null) {
    return { extractor: null, format: null, reason: SOURCE_EXTRACT_SKIP_REASON.formatUnknown };
  }
  const spec = sourceFormatSpec(format);
  switch (spec.extractor) {
    case null:
      return {
        extractor: null,
        format: spec.format,
        reason: SOURCE_EXTRACT_SKIP_REASON.formatNotExtractable,
      };
    case SOURCE_EXTRACTOR.verbatim:
      return {
        extractor: spec.extractor,
        format: spec.format,
        reason: SOURCE_EXTRACT_SKIP_REASON.formatReadVerbatim,
      };
    case SOURCE_EXTRACTOR.html:
      return {
        extractor: spec.extractor,
        format: spec.format,
        html: extractHtmlAfterFrontmatter(bytes),
      };
    case SOURCE_EXTRACTOR.table:
      return {
        extractor: spec.extractor,
        format: spec.format,
        table: tableNote(path, withoutLeadingFrontmatter(bytes).bytes),
      };
  }
}

/** The UTF-8 byte-order mark, which may precede a leading frontmatter block. */
const UTF8_BOM = Object.freeze([0xef, 0xbb, 0xbf]);
/** The first bytes of a frontmatter block. */
const FRONTMATTER_FENCE = "---";
const UTF8_STRICT = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
/** Invalid bytes become U+FFFD, as in a page read as text; the mark is kept for the parser. */
const UTF8_LENIENT = new TextDecoder("utf-8", { ignoreBOM: true });
const UTF8_ENCODER = new TextEncoder();

/**
 * `bytes` without one leading frontmatter block (after an optional
 * byte-order mark), so the block is never rendered as source data: a CSV
 * header of `---` or a `visibility: private` row. `skipped` is the number
 * of bytes left out (the mark and the block), 0 when nothing is. Bytes that
 * are not UTF-8 or open with no closed block are returned as they are, and
 * the extractor answers for them.
 */
function withoutLeadingFrontmatter(bytes: Uint8Array): {
  readonly bytes: Uint8Array;
  readonly skipped: number;
} {
  const whole = { bytes, skipped: 0 };
  const bom = UTF8_BOM.every((byte, i) => bytes[i] === byte) ? UTF8_BOM.length : 0;
  for (let i = 0; i < FRONTMATTER_FENCE.length; i++) {
    if (bytes[bom + i] !== FRONTMATTER_FENCE.charCodeAt(i)) return whole;
  }
  let text: string;
  try {
    text = UTF8_STRICT.decode(bytes.subarray(bom));
  } catch {
    return whole;
  }
  // The vault's own frontmatter pattern, so the block the reach predicate
  // reads a source's visibility from is exactly the block withheld here.
  const block = FRONTMATTER_RE.exec(text);
  if (block === null) return whole;
  const skipped = bom + UTF8_ENCODER.encode(block[0]).length;
  return { bytes: bytes.subarray(skipped), skipped };
}

/** The HTML extraction of `bytes` after any leading frontmatter block, offsets counted in the whole file. */
function extractHtmlAfterFrontmatter(bytes: Uint8Array): HtmlExtractResult {
  const rest = withoutLeadingFrontmatter(bytes);
  return extractHtml(rest.bytes, rest.skipped);
}

/** The HTML outcome on an ingest result: counts only, never the text. */
export type PartsOutcome =
  | { readonly extracted: true; readonly count: number; readonly omitted?: number }
  | { readonly extracted: false; readonly reason: SourceExtractSkipReason };

/** The table outcome on an ingest result: the rendering's counts, never its rows. */
export type TableOutcome = Omit<TableNoteRendered, "section"> | TableNoteSkipped;

/** What ingest adds to a summary page for a source with a derived section. */
export interface SourceDerivation {
  readonly format: SourceFormat;
  /** `source_format`, `source_content_hash`, then the table keys; empty when nothing was read. */
  readonly frontmatter: FrontmatterMap;
  /** The section written last on the page; `""` when there is none. */
  readonly section: string;
  readonly parts?: PartsOutcome;
  readonly table?: TableOutcome;
  /**
   * The source's own visibility tokens, present only when it declares some:
   * a page that copies source content is at most as visible as the source.
   */
  readonly visibility?: readonly string[];
}

/** Frontmatter key naming a derived page's source format. */
export const SOURCE_FORMAT_FRONTMATTER_KEY = "source_format";

/** Is this registered format one the table extractor reads? */
function isTableFormat(format: SourceFormat): format is TableFormat {
  return format === SOURCE_FORMAT.csv || format === SOURCE_FORMAT.tsv;
}

/** The derivation of a source whose bytes were not read: no keys, no section. */
function notLocal(format: SourceFormat): SourceDerivation {
  const reason = SOURCE_EXTRACT_SKIP_REASON.sourceNotLocal;
  if (isTableFormat(format)) {
    return { format, frontmatter: {}, section: "", table: { rendered: false, format, reason } };
  }
  return { format, frontmatter: {}, section: "", parts: { extracted: false, reason } };
}

/** The parts outcome and section of an HTML extraction. */
function fromHtml(html: HtmlExtractResult): { parts: PartsOutcome; section: string } {
  if (!html.extracted) return { parts: { extracted: false, reason: html.reason }, section: "" };
  const parts: PartsOutcome =
    html.partsOmitted > 0
      ? { extracted: true, count: html.parts.length, omitted: html.partsOmitted }
      : { extracted: true, count: html.parts.length };
  return { parts, section: renderPartsSection(html) };
}

/** The table outcome, extra keys and section of a table note. */
function fromTable(table: TableNoteResult): {
  table: TableOutcome;
  frontmatter: FrontmatterMap;
  section: string;
} {
  if (!table.rendered) return { table, frontmatter: {}, section: "" };
  const { section, ...counts } = table;
  return { table: counts, frontmatter: tableNoteFrontmatter(table), section };
}

/**
 * The derived section of the summary page for `source`, or `undefined` when
 * its format has no derived section (text, an unregistered extension, a
 * named format without an extractor): those pages are written exactly as
 * before.
 *
 * The order is the reach order: the format first (it reads nothing), then
 * the trusted lane, then the caller's reach, then ONE read whose digest
 * covers exactly the bytes extracted. A URL, an absent file and a file the
 * caller may not read all answer `source-not-local` with no digest and no
 * section, so the page cannot tell them apart.
 */
export function deriveSourceSection(
  vault: string,
  source: string,
  trust: IntakeTrust,
  readable?: (rel: string) => boolean,
): SourceDerivation | undefined {
  const format = sourceFormatOf(source);
  if (format === null) return undefined;
  const extractor = sourceFormatSpec(format).extractor;
  if (extractor !== SOURCE_EXTRACTOR.html && extractor !== SOURCE_EXTRACTOR.table) {
    return undefined;
  }
  if (trust !== INTAKE_TRUST.trusted || isSourceHidden(vault, source, readable)) {
    return notLocal(format);
  }
  let origin: ReturnType<typeof readSourceOrigin>;
  try {
    origin = readSourceOrigin(vault, source);
  } catch (cause) {
    // The intake already read this source; a refusal now (it grew past the
    // digest ceiling, or stopped being readable, between the two reads) is
    // the source not being there to derive from, not a reason to fail an
    // ingest whose entity pages are already written.
    if (cause instanceof SourceTrustError) return notLocal(format);
    throw cause;
  }
  if (origin.bytes === undefined || origin.contentHash === undefined) return notLocal(format);

  const base: FrontmatterMap = {
    [SOURCE_FORMAT_FRONTMATTER_KEY]: format,
    ...sourceContentHashFrontmatter(origin.contentHash),
  };
  const visibility = sourceVisibility(origin.bytes);
  const reach = visibility.length > 0 ? { visibility } : {};
  const extraction = extractSource(source, origin.bytes);
  if ("html" in extraction) {
    const { parts, section } = fromHtml(extraction.html);
    return { format, frontmatter: base, section, parts, ...reach };
  }
  if ("table" in extraction) {
    const derived = fromTable(extraction.table);
    return {
      format,
      frontmatter: { ...base, ...derived.frontmatter },
      section: derived.section,
      table: derived.table,
      ...reach,
    };
  }
  // The registry gave this format an html or table extractor above, so the
  // dispatch answered with that arm; anything else is a registry defect.
  throw new Error(`extractor ${extractor} answered without its arm for ${format}`);
}

/**
 * The visibility tokens the source's own frontmatter block declares, read
 * from the bytes the extractor strips that block from (so the two agree by
 * construction) and decoded the way the reach predicate
 * (`isPathReadableAtReach`) decodes a page: leniently, through the vault's
 * frontmatter parser.
 *
 * The tokens are {@link remoteDenyVisibilityTokens}, not the declared
 * `visibility:` alone: an informal privacy marker on the source
 * (`access: private` and its siblings) is a claim about who may read the
 * source, and a summary page that copies the source's rows or parts is at
 * most as visible as the source. Reading the declaration alone would write
 * those sections onto a default-visibility page - remotely served.
 */
function sourceVisibility(bytes: Uint8Array): readonly string[] {
  const [meta] = parseFrontmatterText(UTF8_LENIENT.decode(bytes));
  return [...remoteDenyVisibilityTokens(meta)];
}