Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
9.7 KB
Raw Blame
/**
 * `o2b brain distill <source> --claims <json>` (t_2e2e959f): condense a source
 * into atomic claims with block-level provenance.
 *
 * Provider-agnostic: the agent supplies the atomic claims (and optional source
 * block ids) as JSON - a `[{ "text": "...", "block": "^abc" }]` array via
 * `--claims` or `--claims-file`. Open Second Brain validates them, checks every
 * quoted span against the cited block or the source, and writes one idempotent
 * distillation page per source. No model, no extraction here.
 *
 * `--strict-quotes` turns an unverified span into a refusal (exit 1, nothing
 * written); `--excerpt-file` stores the verbatim text read from a source the
 * vault does not hold, making the page `bounded-local`.
 *
 * Exit codes: 0 on success, 1 on an operational failure, 2 on usage errors.
 */

import { readFileSync } from "node:fs";

import {
  distillSource,
  DistillValidationError,
  parseDistillClaims,
  type DistillClaim,
  type DistillSourceResult,
} from "../../../core/brain/distill/distill-source.ts";
import {
  QuoteCheckError,
  type QuoteCheckReport,
} from "../../../core/brain/distill/quote-verdict.ts";
import {
  CAPTURE_SCOPE,
  CaptureExcerptError,
  type CaptureScope,
} from "../../../core/brain/provenance/capture-scope.ts";
import { ResponseShapeError } from "../../../core/brain/response-shape.ts";
import {
  parsePayloadJson,
  PayloadJsonError,
  payloadStripNote,
  type PayloadStrip,
} from "../../../core/brain/payload-json.ts";
import {
  INTAKE_TRUST,
  UNTRUSTED_SOURCE_FRONTMATTER_KEY,
  type IntakeTrust,
} from "../../../core/brain/trust/untrusted-provenance.ts";
import { brainVerbContext, fail, ok, okJson, parse, resolveBrainAgent } from "../helpers.ts";

const USAGE =
  "usage: o2b brain distill <source> (--claims <json> | --claims-file <path>) [--strict-quotes] [--excerpt-file <path>] [--agent <name>] [--vault <path>] [--json]";

/** Refusal for an excerpt file whose bytes are not UTF-8 text. */
const EXCERPT_NOT_UTF8_MESSAGE = "distill: excerpt file is not valid UTF-8";

/**
 * Strict UTF-8 decoder for the excerpt file: an invalid sequence throws
 * instead of becoming U+FFFD, and a leading byte-order mark is kept, so the
 * decoded text is exactly the file's bytes.
 */
const EXCERPT_DECODER = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });

/** The excerpt file's text, or `null` when its bytes are not valid UTF-8. */
function readExcerptFile(path: string): string | null {
  const bytes = readFileSync(path);
  try {
    return EXCERPT_DECODER.decode(bytes);
  } catch (err) {
    if (err instanceof TypeError) return null;
    throw err;
  }
}

/** Errors that are the operator's input, reported as `distill: <message>`. */
const OPERATOR_ERROR_CLASSES: ReadonlyArray<new (...args: never[]) => Error> = [
  DistillValidationError,
  ResponseShapeError,
  QuoteCheckError,
  CaptureExcerptError,
];

/** What `--claims` accepts, named in the operator's own terms. */
const CLAIMS_SHAPE_HINT = "claims must be a JSON array of { text, block? } objects";

/**
 * Unwrap the operator's JSON (a bare array, or an object with a `claims`
 * array) and hand it to the shared shape-checked ingress, so the CLI and the
 * MCP tool accept exactly the same payload under exactly the same rules.
 *
 * The parse goes through the shared strip-and-note parser (t_dac8bf7e): a
 * leading `<think>` block is stripped once and the fact returned, while a
 * refusal - plain or naming the strip attempt - propagates as the
 * {@link PayloadJsonError} the operator-input catch reports. The unwrap
 * failure is still reported HERE rather than by the ingress: at this point
 * the operator's mistake is the wrapper they typed, and a path inside a
 * payload the CLI never found does not tell them what to type instead. Once
 * a list is in hand, the ingress owns every complaint about its items.
 */
function parseClaims(raw: string): { claims: DistillClaim[]; strip?: PayloadStrip } {
  const { payload: parsed, strip } = parsePayloadJson(raw);
  const claims =
    !Array.isArray(parsed) && parsed !== null && typeof parsed === "object"
      ? (parsed as { claims?: unknown }).claims
      : parsed;
  if (!Array.isArray(claims)) throw new Error(CLAIMS_SHAPE_HINT);
  return { claims: parseDistillClaims(claims), ...(strip !== undefined ? { strip } : {}) };
}

/**
 * What the success line adds when the page was marked untrusted. The token is
 * the frontmatter key itself rather than a sentence, so the operator can grep
 * for the same string on disk - and a trusted run's line stays exactly as it
 * was. Silence here would report one success sentence for two different
 * outcomes.
 *
 * The marker is a MARKER, not a guarantee that the page is out of reach. The
 * exclusion it feeds is the retrieval trust gate, which
 * `search/pipeline/post-rank.ts` mounts only when `search_trust_gate_enabled`
 * is set - a flag that falls back to `false`. This comment used to claim "no
 * ordinary read will ever return" it, which is false on a default install; see
 * the argument in `src/mcp/brain/distill-tools.ts` for why the answer is to
 * state the condition rather than to flip the flag from here.
 */
function untrustedNote(trust: IntakeTrust): string {
  return trust === INTAKE_TRUST.untrusted ? ` [${UNTRUSTED_SOURCE_FRONTMATTER_KEY}]` : "";
}

/**
 * What the success line adds when an excerpt was stored. Only `bounded-local`
 * earns a token: `url-only` is already named by the untrusted marker, and a
 * `full-local` run keeps its line exactly as it was.
 */
function captureNote(scope: CaptureScope): string {
  return scope === CAPTURE_SCOPE.boundedLocal ? ` [${CAPTURE_SCOPE.boundedLocal}]` : "";
}

/**
 * What the success line adds when quoted spans were checked: the spans that
 * verified (in a block or in the whole source) and the spans unquoted.
 */
function quotesNote(quotes: QuoteCheckReport | undefined): string {
  if (quotes === undefined) return "";
  const verified = quotes.verified_in_block + quotes.verified_in_source;
  return ` [quotes verified:${verified} unquoted:${quotes.unquoted}]`;
}

/** The success line, suffixes in their pinned order. */
function successLine(res: DistillSourceResult): string {
  return (
    `distilled ${res.claimCount} claim(s) -> ${res.distillationPath}${res.created ? "" : " (updated)"}` +
    `${untrustedNote(res.trust)}${captureNote(res.captureScope)}${quotesNote(res.quotes)}`
  );
}

export async function cmdBrainDistill(argv: string[]): Promise<number> {
  const { flags, positional } = parse(argv, {
    vault: { type: "string" },
    agent: { type: "string" },
    claims: { type: "string" },
    "claims-file": { type: "string" },
    "strict-quotes": { type: "boolean" },
    "excerpt-file": { type: "string" },
    json: { type: "boolean" },
  });
  const source = positional[0];
  // Usage errors exit 2 (the command's documented contract), distinct from the
  // operational exit 1 the catch below returns.
  if (!source) {
    process.stderr.write(`${USAGE}\n`);
    return 2;
  }
  if (typeof flags["claims"] !== "string" && typeof flags["claims-file"] !== "string") {
    process.stderr.write(`${USAGE}\n`);
    return 2;
  }

  try {
    const { config, vault } = brainVerbContext(flags);
    // Read the claims source inside the try so a missing --claims-file is a
    // clean error, not an uncaught throw.
    const claimsRaw =
      typeof flags["claims"] === "string"
        ? (flags["claims"] as string)
        : readFileSync(flags["claims-file"] as string, "utf8");
    let claims: DistillClaim[];
    let strip: PayloadStrip | undefined;
    try {
      const parsed = parseClaims(claimsRaw);
      claims = parsed.claims;
      strip = parsed.strip;
    } catch (error) {
      // Reported here rather than by the ingress: at this point the mistake
      // is the JSON the operator typed, and a path inside a payload the CLI
      // never parsed would not tell them what to fix. The refusal rides the
      // verb's own `distill: <message>` stderr line; after a strip attempt
      // the message names the attempt (t_dac8bf7e).
      if (error instanceof PayloadJsonError) {
        process.stderr.write(`distill: ${error.message}\n`);
        return 1;
      }
      throw error;
    }
    // Read verbatim: the excerpt is stored byte for byte and its digest is
    // taken over exactly these bytes.
    const excerpt =
      typeof flags["excerpt-file"] === "string"
        ? readExcerptFile(flags["excerpt-file"] as string)
        : undefined;
    if (excerpt === null) {
      process.stderr.write(`${EXCERPT_NOT_UTF8_MESSAGE}\n`);
      return 2;
    }
    const res = distillSource(
      vault,
      { sourcePath: source, claims, ...(excerpt !== undefined ? { excerpt } : {}) },
      {
        agent: resolveBrainAgent(flags, config),
        now: new Date(),
        strictQuotes: flags["strict-quotes"] === true,
      },
    );
    if (flags["json"]) {
      okJson({
        distillation_path: res.distillationPath,
        created: res.created,
        claim_count: res.claimCount,
        // Absent when the source had no bytes to hash, so the operator reads
        // "not recorded" rather than a sentinel that looks like a digest.
        ...(res.sourceHash !== undefined ? { source_hash: res.sourceHash } : {}),
        trust: res.trust,
        capture_scope: res.captureScope,
        ...(res.quotes !== undefined ? { quotes: res.quotes } : {}),
        // Present only when a leading <think> block was stripped (t_dac8bf7e).
        ...(strip !== undefined ? { note: payloadStripNote(strip) } : {}),
      });
      return 0;
    }
    ok(successLine(res));
    if (strip !== undefined) ok(payloadStripNote(strip));
    return 0;
  } catch (err) {
    if (OPERATOR_ERROR_CLASSES.some((cls) => err instanceof cls)) {
      return fail(`distill: ${(err as Error).message}`);
    }
    return fail(`distill failed: ${(err as Error).message ?? err}`);
  }
}