Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
15.4 KB
Raw Blame
/**
 * Inbox-drain classify-and-route pass (Knowledge intake suite, I2,
 * t_b0bba8cb).
 *
 * Walks the staged captures via the seam-1 capture-note contract and, for
 * each, decides a route from STRUCTURAL signals only - never a
 * natural-language word list:
 *
 *   - source-reference: a URL-shaped body (the trimmed body is a single
 *     `http(s)://` token). Routed through {@link ingestSource}.
 *   - obligation: the body's first line begins with the explicit
 *     {@link CAPTURE_OBLIGATION_MARKER} token, optionally carrying a cadence
 *     from the obligation vocabulary. Routed through {@link addObligation}.
 *   - idea: everything else is an atomic idea. Routed to a create-or-merge
 *     note under `captured/`.
 *
 * Dry-run is the default and writes nothing. `apply` executes the route and
 * archives the capture through the contract, so a rerun after apply finds no
 * staged captures and is a no-op (the processed marker is the idempotency
 * key). Unroutable items are reported with a reason and left in place.
 *
 * The source-reference route sits behind the same storage-capacity
 * admission wall `brain_ingest_source` keeps: a vault measured at its
 * critical tier refuses the ingest route in BOTH modes (a dry run
 * previews the refusal faithfully), leaves the capture staged, and
 * still drains its non-ingest routes.
 *
 * ## The hub hook (t_23bd347d)
 *
 * An apply-mode idea route that succeeds stages its routed page's area-hub
 * candidate through `hub-candidates.ts` - a record in the repair lane's
 * staged store, never an edge: the edge lands exclusively through the lane's
 * apply + confirm + holdout gate. Obligation and source-reference routes
 * stage nothing. A store failure aborts the drain loudly before the archive:
 * a hub proposal nobody saw is the silence this hook exists to prevent, and
 * the still-staged capture makes a rerun recover it.
 */

import { existsSync, mkdirSync } from "node:fs";
import { dirname, join } from "node:path";

import type { FrontmatterMap } from "../../types.ts";
import { atomicWriteText } from "../../fs-atomic.ts";
import { ensureInsideVault } from "../../path-safety.ts";
import { formatFrontmatter, parseFrontmatter, slugify } from "../../vault.ts";
import { addObligation, obligationExists, parseCadence, ObligationError } from "../obligations.ts";
import { ingestSource } from "../ingest/ingest.ts";
import { CAPACITY_GATE_DECISION, evaluateIngestAdmission } from "../storage/gate.ts";
import type { IngestAdmission } from "../storage/gate.ts";
import { MAINTENANCE_CAPACITY_REMEDY } from "../maintenance/lane.ts";
import { isoSecond } from "../time.ts";
import {
  loadHubPagePool,
  selectHubCandidate,
  stageHubSelection,
  type HubPagePool,
} from "../link-graph/hub-candidates.ts";
import {
  archiveCapture,
  CaptureContractError,
  listStagedCaptures,
  requireDeclaredCaptureKind,
  type CaptureNote,
} from "./capture-note.ts";
import { assertVaultIdentityForWrite } from "../vault-identity.ts";

/** Explicit leading token that classifies a capture as an obligation. */
export const CAPTURE_OBLIGATION_MARKER = "@obligation";

/** Cadence used when an obligation marker carries no explicit cadence. */
export const DEFAULT_CAPTURE_OBLIGATION_CADENCE = "weekly";

/** Vault-relative directory that holds create-or-merge idea notes. */
export const CAPTURED_NOTES_DIR_REL = "captured";

/** Frontmatter `kind:` marker of a drained idea note. */
export const CAPTURED_IDEA_KIND = "captured-idea";

/** A single line-delimited URL with no interior whitespace. */
const URL_SHAPED_RE = /^https?:\/\/\S+$/u;

export type CaptureClass = "source-reference" | "obligation" | "idea";
export type DrainAction = "ingest-source" | "open-obligation" | "note" | "skip";

export interface DrainItem {
  readonly id: string;
  /** Vault-relative path of the staged capture. */
  readonly capturePath: string;
  readonly classification: CaptureClass | "unroutable" | "archive-failed";
  readonly action: DrainAction;
  readonly reason: string;
  /** Resolved route target (summary page, obligation slug, note path). */
  readonly target: string | null;
  /** `true` when the capture was routed and archived (apply mode only). */
  readonly routed: boolean;
}

export interface DrainReport {
  readonly mode: "dry-run" | "apply";
  readonly items: readonly DrainItem[];
  readonly routed: number;
  readonly unroutable: number;
  /**
   * Count of captures whose route executed but whose archive then failed
   * (route done, still staged). Reported apart from `unroutable` because the
   * route already ran, so a blind retry would re-execute it; the idea-note
   * write is idempotent so a re-run converges instead of duplicating.
   */
  readonly archiveFailed: number;
  /** Corrupt staged hub-store lines this drain dropped, counted rather than thrown. */
  readonly skippedCorrupt: number;
}

export interface DrainOptions {
  readonly apply: boolean;
  readonly agent: string;
  readonly now: Date;
  /** The explicit machine-config file the admission census resolves the store from. */
  readonly configPath?: string;
}

/** One item's route decided structurally, before any write. */
interface RoutePlan {
  readonly classification: CaptureClass;
  readonly action: DrainAction;
  readonly reason: string;
  /** Executes the route and returns the resolved target. */
  readonly execute: () => string;
}

/** A structural refusal that leaves the capture in place. */
class UnroutableCapture {
  constructor(readonly reason: string) {}
}

function classify(
  vault: string,
  note: CaptureNote,
  opts: DrainOptions,
  admission: IngestAdmission,
): RoutePlan | UnroutableCapture {
  const body = note.body.trim();
  const firstLine = body.split("\n", 1)[0]!.trim();

  if (firstLine.startsWith(CAPTURE_OBLIGATION_MARKER)) {
    return planObligation(vault, firstLine, opts);
  }
  if (URL_SHAPED_RE.test(body)) {
    return planSource(vault, body, opts, admission);
  }
  return planIdea(vault, body, opts);
}

function planObligation(
  vault: string,
  firstLine: string,
  opts: DrainOptions,
): RoutePlan | UnroutableCapture {
  const afterMarker = firstLine.slice(CAPTURE_OBLIGATION_MARKER.length);
  let cadenceRaw = DEFAULT_CAPTURE_OBLIGATION_CADENCE;
  let rest = afterMarker;
  if (afterMarker.startsWith(":")) {
    const [token, ...tail] = afterMarker.slice(1).trim().split(/\s+/u);
    cadenceRaw = token ?? "";
    rest = tail.join(" ");
  }
  let cadence: string;
  try {
    cadence = parseCadence(cadenceRaw);
  } catch {
    return new UnroutableCapture(`obligation marker has an unknown cadence: ${cadenceRaw}`);
  }
  const title = rest.trim();
  if (title.length === 0) {
    return new UnroutableCapture("obligation marker without a title");
  }
  const slug = slugify(title);
  if (obligationExists(vault, slug)) {
    return new UnroutableCapture(`obligation already exists: ${slug}`);
  }
  return {
    classification: "obligation",
    action: "open-obligation",
    reason: `obligation marker (cadence ${cadence})`,
    execute: () => addObligation(vault, { title, cadence, agent: opts.agent, now: opts.now }).slug,
  };
}

/**
 * The source-reference route, behind the same admission wall
 * `brain_ingest_source` keeps: only a MEASURED critical tier refuses, so
 * a capture an agent could not ingest by hand is not sneaked in by a
 * drain, and a capture the wall would admit is not refused by one. The
 * refusal is decided at PLAN time (one admission evaluation per drain,
 * never per capture - the census is a full vault walk) so a dry run is a
 * faithful preview of it, the same posture as the declared-vocabulary
 * gate below. The remedy is the lane's own sentence, quoted rather than
 * restated, so every surface that refuses on capacity says the same
 * thing.
 */
function planSource(
  vault: string,
  url: string,
  opts: DrainOptions,
  admission: IngestAdmission,
): RoutePlan | UnroutableCapture {
  // A storage block the wall cannot see through refuses, the same rule
  // brain_ingest_source applies: reading it as "no cap" would ingest past
  // the operator's bound. The capture stays staged for a rerun once the
  // block is fixed.
  if (admission.configRefusal !== undefined) {
    return new UnroutableCapture(`ingest refused (config_invalid): ${admission.configRefusal}`);
  }
  const decision = admission.decision;
  if (decision?.decision === CAPACITY_GATE_DECISION.skip) {
    return new UnroutableCapture(
      `ingest refused (capacity_exhausted): the vault measured at the ${decision.tier} ` +
        `storage tier. ${MAINTENANCE_CAPACITY_REMEDY}.`,
    );
  }
  return {
    classification: "source-reference",
    action: "ingest-source",
    reason: "url-shaped body",
    execute: () =>
      ingestSource(
        vault,
        {
          sourcePath: url,
          summary: `Captured source reference: ${url}`,
          extraction: { entities: [] },
        },
        { agent: opts.agent, now: opts.now },
      ).summaryPath,
  };
}

function planIdea(vault: string, body: string, opts: DrainOptions): RoutePlan | UnroutableCapture {
  // Pack vocabulary gate (t_151a564c), consulted at PLAN time so a dry run is
  // a faithful preview: the idea route stamps `captured-idea`, so a vault
  // that declares page_types without it is unroutable in BOTH modes with the
  // same named refusal - the capture stays staged and a rerun after the
  // operator declares the kind converges. Same gate, same refusal text, as
  // the staging contract's own writer.
  try {
    requireDeclaredCaptureKind(vault, CAPTURED_IDEA_KIND);
  } catch (err) {
    if (!(err instanceof CaptureContractError)) throw err;
    return new UnroutableCapture(err.message);
  }
  const slug = slugify(body);
  const relPath = `${CAPTURED_NOTES_DIR_REL}/${slug}.md`;
  const abs = ensureInsideVault(join(vault, relPath), vault);
  const merge = existsSync(abs);
  return {
    classification: "idea",
    action: "note",
    reason: merge ? "atomic idea (merge into existing note)" : "atomic idea (create note)",
    execute: () => {
      writeIdeaNote(vault, abs, body, opts, merge);
      return relPath;
    },
  };
}

function writeIdeaNote(
  vault: string,
  abs: string,
  body: string,
  opts: DrainOptions,
  merge: boolean,
): void {
  // The declared-vocabulary gate lives in `planIdea`, the classifier, so the
  // dry-run plan and the apply see the same refusal (t_151a564c); by execute
  // time the route is known declared, and nothing here writes before it.
  const stamp = isoSecond(opts.now);
  // `ensureInsideVault` hands back a native path, so the parent comes from
  // `dirname`: a hand-rolled split on "/" found no separator in a Windows
  // path and created the NOTE ITSELF as a directory, after which every
  // idea route failed on the rename into it.
  mkdirSync(dirname(abs), { recursive: true });
  if (!merge) {
    const meta: FrontmatterMap = {
      kind: CAPTURED_IDEA_KIND,
      created_at: stamp,
      updated_at: stamp,
      tags: ["brain/captured-idea"],
    };
    atomicWriteText(abs, formatFrontmatter(meta, body));
    return;
  }
  const [meta, existing] = parseFrontmatter(abs);
  const existingTrimmed = existing.trim();
  const incoming = body.trim();
  // Idempotent merge: if this exact block is already present, do not append
  // it again. This makes a re-run safe when a prior route succeeded but its
  // archive failed and left the capture staged (finding: split route/archive
  // handling), so the idea note is never duplicated.
  if (bodyContainsBlock(existingTrimmed, incoming)) return;
  const nextMeta: FrontmatterMap = { ...meta, updated_at: stamp };
  const mergedBody = `${existingTrimmed}\n\n${incoming}`;
  atomicWriteText(abs, formatFrontmatter(nextMeta, mergedBody));
}

/** True when `block` is already present as a paragraph block of `body`. */
function bodyContainsBlock(body: string, block: string): boolean {
  return body
    .split(/\n{2,}/u)
    .map((b) => b.trim())
    .includes(block);
}

/** Classify and (in apply mode) route every staged capture. */
export function drainInbox(vault: string, opts: DrainOptions): DrainReport {
  // Vault-identity write guard (context-integrity-gates, Unit J).
  assertVaultIdentityForWrite(vault);
  const items: DrainItem[] = [];
  let routed = 0;
  let unroutable = 0;
  let archiveFailed = 0;
  let skippedCorrupt = 0;
  // The hub page pool is a full-vault parse: loaded once, on the first idea.
  let hubPool: HubPagePool | undefined;
  // One admission evaluation per drain, not per capture: the census is a
  // full vault walk, and the answer cannot change between captures of
  // the same run. A null decision with no config refusal means no wall
  // stands this pass (no configured cap); a storage block the wall cannot
  // see through refuses the ingest route by name, like brain_ingest_source.
  const admission = evaluateIngestAdmission(vault, {
    now: opts.now,
    ...(opts.configPath !== undefined ? { configPath: opts.configPath } : {}),
  });

  for (const note of listStagedCaptures(vault)) {
    const plan = classify(vault, note, opts, admission);
    if (plan instanceof UnroutableCapture) {
      unroutable += 1;
      items.push({
        id: note.id,
        capturePath: note.path,
        classification: "unroutable",
        action: "skip",
        reason: plan.reason,
        target: null,
        routed: false,
      });
      continue;
    }

    if (!opts.apply) {
      items.push({
        id: note.id,
        capturePath: note.path,
        classification: plan.classification,
        action: plan.action,
        reason: plan.reason,
        target: null,
        routed: false,
      });
      continue;
    }

    // The route and the archive are handled apart: a route that succeeds but
    // whose archive then fails is NOT unroutable - the route already ran and
    // the capture is still staged. Folding it into `unroutable` would both
    // misreport it and invite a blind retry of an already-executed route.
    let target: string;
    try {
      target = plan.execute();
    } catch (err) {
      unroutable += 1;
      const reason =
        err instanceof ObligationError
          ? err.message
          : `routing failed: ${err instanceof Error ? err.message : String(err)}`;
      items.push({
        id: note.id,
        capturePath: note.path,
        classification: "unroutable",
        action: "skip",
        reason,
        target: null,
        routed: false,
      });
      continue;
    }

    // Hub hook (t_23bd347d): the moment the idea became a corpus page, stage
    // its hub outcome - a candidate or a named refusal - for the repair lane.
    // Only the idea route lands a page; the other routes stage nothing.
    if (plan.classification === "idea") {
      hubPool ??= loadHubPagePool(vault);
      skippedCorrupt += stageHubSelection(
        vault,
        selectHubCandidate(vault, target, hubPool),
      ).skippedCorrupt;
    }

    try {
      archiveCapture(vault, note.id);
    } catch (err) {
      archiveFailed += 1;
      items.push({
        id: note.id,
        capturePath: note.path,
        classification: "archive-failed",
        action: plan.action,
        reason: `route succeeded but archive failed: ${
          err instanceof Error ? err.message : String(err)
        }`,
        target,
        routed: false,
      });
      continue;
    }

    routed += 1;
    items.push({
      id: note.id,
      capturePath: note.path,
      classification: plan.classification,
      action: plan.action,
      reason: plan.reason,
      target,
      routed: true,
    });
  }

  return {
    mode: opts.apply ? "apply" : "dry-run",
    items,
    routed,
    unroutable,
    archiveFailed,
    skippedCorrupt,
  };
}