Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
10.0 KB
Raw Blame
/**
 * `o2b brain extract <file> [--json] [--vault <vault>]`: preview what ingest
 * derives from one source file, for every registered format.
 *
 * An HTML file answers its title, its text and its heading-derived parts; a
 * CSV or TSV file answers its table note (the counts and the rendered
 * section). Every other format - text read as it is, a named format with no
 * extractor, an unknown extension - answers `extracted: false` with the
 * reason by name, without reading the file. So does an HTML or table file
 * the bounded reader refuses (not a regular file, larger than the read
 * limit). "Could not extract" is data: every such answer exits 0. Read-only
 * and deterministic; nothing is written.
 *
 * With a resolvable vault (`--vault`, else the configured one) and a source
 * the ingest manifest has already extracted, the preview also reports the
 * `appended` slice: the boundary-safe byte range the file grew by since its
 * recorded extraction watermark. It is additive data for an incremental
 * pass - a source with no watermark, a rewritten source, or no vault at all
 * answers without the section, never with a fake range.
 */

import { lstatSync } from "node:fs";
import { isAbsolute, relative, sep } from "node:path";

import { oneLine } from "../../../core/brain/architect/manifests.ts";
import {
  formatPartLine,
  HTML_EXTRACT_MAX_SOURCE_BYTES,
  type HtmlPart,
} from "../../../core/brain/ingest/extract-html.ts";
import { extractSource, type SourceExtraction } from "../../../core/brain/ingest/extract-source.ts";
import {
  appendedExtractionDelta,
  readSourceBounded,
  type SourceUnread,
} from "../../../core/brain/ingest/ingest.ts";
import { canonicalNotePath } from "../../../core/path-safety.ts";
import {
  SOURCE_EXTRACT_SKIP_REASON,
  SOURCE_EXTRACTOR,
  type SourceExtractor,
  sourceFormatOf,
  sourceFormatSpec,
  type SourceExtractSkipReason,
} from "../../../core/brain/ingest/source-formats.ts";
import { brainVerbContext, CliError, fail, ok, okJson, parse, usageError } from "../helpers.ts";

const USAGE = "usage: o2b brain extract <file> [--json] [--vault <vault>]";
/** The bytes handed to the dispatch for a format answered by name. */
const NO_BYTES = new Uint8Array(0);
/** How the text output names the format of an unregistered extension. */
const UNKNOWN_FORMAT_LABEL = "unknown";

/** The skip reason each bounded-read refusal answers. */
const UNREAD_REASON: Readonly<Record<SourceUnread, SourceExtractSkipReason>> = Object.freeze({
  "not a regular file": SOURCE_EXTRACT_SKIP_REASON.notARegularFile,
  "larger than the read limit": SOURCE_EXTRACT_SKIP_REASON.sourceTooLarge,
});

/** The extractors that read a source's bytes; every other format is answered by name. */
const BYTE_READING_EXTRACTORS: ReadonlySet<SourceExtractor> = new Set([
  SOURCE_EXTRACTOR.html,
  SOURCE_EXTRACTOR.table,
]);

/**
 * The dispatch answer for `file`. The path is lstat-ed first, before any
 * format is named: a missing path and a symbolic link are both errors. A
 * format with no byte-reading extractor (text read as it is, a named format
 * with no extractor, an unknown extension) is then answered by name without
 * reading the file. An HTML or table source goes through the bounded
 * reader, whose refusal answers in the same shape as a format skip.
 */
function extractFile(file: string): SourceExtraction {
  if (lstatSync(file).isSymbolicLink()) throw new Error(`symbolic link: ${file}`);
  const format = sourceFormatOf(file);
  const extractor = format === null ? null : sourceFormatSpec(format).extractor;
  if (extractor === null || !BYTE_READING_EXTRACTORS.has(extractor)) {
    return extractSource(file, NO_BYTES);
  }
  const read = readSourceBounded(file, HTML_EXTRACT_MAX_SOURCE_BYTES);
  if (read.unread !== undefined) {
    return { extractor: null, format, reason: UNREAD_REASON[read.unread] };
  }
  return extractSource(file, read.bytes);
}

/** The appended slice a previously-extracted source grew by, in bytes. */
interface AppendedSlice {
  readonly fromByte: number;
  readonly toByte: number;
}

/**
 * The vault the appended section answers against, when one resolves. The
 * preview itself is vault-independent - it answers any file - so an
 * unresolvable vault is not an error here; it only means there is no ingest
 * manifest to consult and the section stays absent.
 */
function resolveOptionalVault(
  flags: Record<string, string | boolean | string[] | undefined>,
): string | null {
  try {
    return brainVerbContext(flags).vault;
  } catch (err) {
    if (err instanceof CliError) return null;
    throw err;
  }
}

/**
 * The appended slice `file` grew by since its recorded extraction watermark,
 * when the file sits inside `vault` and the manifest records one. The
 * pipeline's own delta pass answers this (`appendedExtractionDelta`), so the
 * preview reports the same fact the pipeline would: a source whose leading
 * watermark bytes no longer match the recorded digest is a rewrite, not an
 * append, and answers undefined exactly as the ingest's delta report does -
 * as do a file outside the vault, no watermark, a tail with no complete
 * line yet, and a tail past the pipeline's delta cap. The section is
 * additive data for an incremental pass, never a second verdict on the
 * extraction.
 */
function appendedSlice(vault: string | null, file: string): AppendedSlice | undefined {
  if (vault === null) return undefined;
  const rel = relative(vault, file);
  if (rel.length === 0 || isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) {
    return undefined;
  }
  return appendedExtractionDelta(vault, file, canonicalNotePath(rel.split(sep).join("/")));
}

/** The wire shape of one HTML part. */
function partJson(part: HtmlPart): Record<string, unknown> {
  return {
    index: part.index,
    level: part.level,
    heading: part.heading,
    trail: part.trail,
    line_start: part.lineStart,
    line_end: part.lineEnd,
    source_offset: part.sourceOffset,
  };
}

/** The `--json` payload (without `ok`) for `path`. */
function extractionJson(path: string, extraction: SourceExtraction): Record<string, unknown> {
  if (extraction.extractor === SOURCE_EXTRACTOR.html) {
    const html = extraction.html;
    if (!html.extracted) {
      return { path, extracted: false, format: extraction.format, reason: html.reason };
    }
    return {
      path,
      extracted: true,
      format: extraction.format,
      title: html.title,
      text: html.text,
      parts: html.parts.map(partJson),
      ...(html.partsOmitted > 0 ? { parts_omitted: html.partsOmitted } : {}),
    };
  }
  if (extraction.extractor === SOURCE_EXTRACTOR.table) {
    const table = extraction.table;
    if (!table.rendered) {
      return {
        path,
        extracted: false,
        format: table.format,
        reason: table.reason,
        ...(table.detail !== undefined ? { detail: table.detail } : {}),
      };
    }
    return {
      path,
      extracted: true,
      format: table.format,
      delimiter: table.delimiter,
      columns: table.columns,
      rows: table.rows,
      rows_rendered: table.rowsRendered,
      ...(table.truncated.length > 0 ? { truncated: table.truncated } : {}),
      redacted_cells: table.redactedCells,
      section: table.section,
    };
  }
  return { path, extracted: false, format: extraction.format, reason: extraction.reason };
}

/** The "not extracted" line, with the detail when there is one. */
function notExtractedLine(reason: string, detail?: string): string {
  return `  not extracted: ${reason}${detail !== undefined ? ` (${oneLine(detail)})` : ""}`;
}

/** The text output for `path`, one entry per line. */
function extractionText(path: string, extraction: SourceExtraction): string[] {
  const lines = [`extract: ${path} (${extraction.format ?? UNKNOWN_FORMAT_LABEL})`];
  if (extraction.extractor === SOURCE_EXTRACTOR.html) {
    const html = extraction.html;
    if (!html.extracted) return [...lines, notExtractedLine(html.reason)];
    if (html.title !== null) lines.push(`  title: ${html.title}`);
    lines.push(`  ${html.parts.length} part(s)${html.parts.length > 0 ? ":" : ""}`);
    for (const part of html.parts) lines.push(`    ${formatPartLine(part)}`);
    if (html.partsOmitted > 0) lines.push(`  ${html.partsOmitted} more part(s) omitted`);
    return lines;
  }
  if (extraction.extractor === SOURCE_EXTRACTOR.table) {
    const table = extraction.table;
    if (!table.rendered) return [...lines, notExtractedLine(table.reason, table.detail)];
    const truncated =
      table.truncated.length > 0 ? `, truncated: ${table.truncated.join(", ")}` : "";
    lines.push(
      `  ${table.delimiter}-delimited, ${table.columns} column(s), ${table.rows} row(s), ` +
        `${table.rowsRendered} rendered, ${table.redactedCells} redacted cell(s)${truncated}`,
      table.section,
    );
    return lines;
  }
  return [...lines, notExtractedLine(extraction.reason)];
}

/** The text line naming the appended slice, for the non-JSON output. */
function appendedLine(appended: AppendedSlice): string {
  return `  appended: byte ${appended.fromByte}..${appended.toByte} since the recorded extraction watermark`;
}

export async function runBrainExtract(argv: string[]): Promise<number> {
  const { flags, positional } = parse(argv, {
    json: { type: "boolean" },
    vault: { type: "string" },
  });
  const file = positional[0];
  if (!file) return usageError(USAGE);

  try {
    const extraction = extractFile(file);
    // A corrupted manifest fails the command by name, like every ingest-lane
    // read of it; everything else about the appended section is additive.
    const appended = appendedSlice(resolveOptionalVault(flags), file);
    if (flags["json"]) {
      const payload = extractionJson(file, extraction);
      if (appended !== undefined) {
        payload["appended"] = { from_byte: appended.fromByte, to_byte: appended.toByte };
      }
      okJson(payload);
      return 0;
    }
    const lines = extractionText(file, extraction);
    if (appended !== undefined) lines.push(appendedLine(appended));
    for (const line of lines) ok(line);
    return 0;
  } catch (err) {
    return fail((err as Error).message ?? String(err));
  }
}