Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
9.0 KB
Raw Blame
/**
 * Pre-compact continuity extraction: the labelled lines a host flushes
 * before a context window is compacted, captured as `pre_compact_extract`
 * continuity records without an LLM call.
 *
 * ## The recognizer is STRUCTURAL, and that is the whole point
 *
 * This used to carry a five-entry English word list (`decision:`,
 * `commitment:`, `outcome:`, `rule:`, `open question:`) - the only prose
 * recognizer left on any write path, and a direct contradiction of the
 * rule its sibling {@link ../brain/fact-extract.ts} states in its own
 * header: we cannot enumerate the world's languages, so a per-language
 * phrase list is a defect, not a feature. A session captured in Russian,
 * Japanese or Arabic extracted NOTHING, and the zero was indistinguishable
 * from "this session recorded no decisions".
 *
 * What is recognised now is the SHAPE of a labelled line - an opening
 * label, a colon, and a body - exactly as {@link ../graph/agent-scope.ts}
 * treats an owner token: an opaque, language-neutral identifier, never a
 * closed enum. The label itself becomes the record's `extract_type`,
 * normalized to one token. English input is byte-identical to before
 * (`Decision:` -> `decision`, `Open question:` -> `open_question`), so
 * nothing downstream that reads those five values changes; every other
 * script now scores on the same structure instead of scoring zero.
 *
 * Precision beats recall, the same trade the sibling extractor makes: the
 * label may only carry letters, digits, spaces, `-` and `_`, must contain
 * at least one letter (so `10:30` is a clock time, not a label), is capped
 * at {@link LABEL_MAX_WORDS} words and {@link LABEL_MAX_CHARS} characters
 * (so a prose sentence ending in a colon is not a label), and a line whose
 * colon belongs to a URI scheme or a Windows path is refused outright.
 */

import { createHash } from "node:crypto";

import {
  appendContinuityRecord,
  buildContinuityRecord,
  listContinuityRecords,
} from "./continuity/store.ts";
import type {
  AppendContinuityRecordInput,
  ContinuityRecord,
  ContinuitySourceRef,
} from "./continuity/types.ts";

export interface PreCompactExtractInput {
  readonly sessionId: string;
  readonly turnStart: string;
  readonly turnEnd: string;
  readonly text: string;
  readonly host?: string;
  readonly createdAt?: string;
  readonly maxChars?: number;
  /**
   * True when this segment was flushed by an interrupted close
   * (SIGHUP/SIGTERM/force-quit/restart-drain). Recorded on the continuity
   * record so an interrupted capture is honestly distinguishable from a clean
   * one. Absent by default - omitted records stay byte-identical (t_c181f92b).
   */
  readonly interrupted?: boolean;
  /**
   * Preview mode (C2 / t_2c6cf3e2). When true, return the candidate
   * records extraction WOULD append WITHOUT touching the vault — no
   * `appendContinuityRecord`, no log event, no dream/retire trigger. Each
   * returned record is built through the SAME `buildContinuityRecord`
   * path the real write uses, so the preview predicts the real extraction
   * byte-for-byte. Absent/false → existing write-committing behavior,
   * byte-identical. Mirrors the `opts.dryRun` idiom in session `import.ts`.
   */
  readonly dryRun?: boolean;
}

export interface PreCompactExtractResult {
  readonly records: ReadonlyArray<ContinuityRecord>;
  readonly errors: ReadonlyArray<string>;
  readonly skipped: number;
}

interface ExtractedLine {
  /** Normalized label token - open vocabulary, never a closed enum. */
  readonly type: string;
  readonly text: string;
  readonly line: number;
}

/**
 * A labelled line: an optional list bullet, a label built only from
 * letters, digits, spaces, `-` and `_`, a colon, and a non-empty body.
 * The label class is what makes this structural rather than lexical -
 * it names no word in any language, only the characters a label may use.
 */
const LABELLED_LINE_RE = /^(?:[-*]\s*)?([\p{L}\p{N} _-]+?)\s*:\s*(\S.*)$/u;

/**
 * Lines whose colon is punctuation inside a locator rather than a label
 * separator. A URI scheme (`https://`, `file:///`) and a Windows drive
 * path (`C:\...`) both parse as `label: body` by shape alone, so they are
 * refused BEFORE the shape test rather than patched afterwards.
 */
const LOCATOR_LINE_RE = /^(?:[-*]\s*)?[\p{L}][\p{L}\p{N}+.-]*:(?:\/\/|\\)/u;

/** A label longer than this is prose that happens to end in a colon. */
const LABEL_MAX_CHARS = 32;

/** Likewise: real labels are one to three words ("open question"). */
const LABEL_MAX_WORDS = 3;

const DEFAULT_MAX_CHARS = 40_000;

/**
 * Reduce a raw label to the single token recorded as `extract_type`, or
 * `null` when the candidate is not a label at all.
 *
 * NFC + case-fold + collapse each internal whitespace run to `_`, which is
 * the same normalization {@link ../graph/agent-scope.ts} applies to an
 * owner token, plus the word joiner that keeps `Open question` and
 * `open_question` the same key. At least one letter is required so a clock
 * time (`10:30`) or a numbered list (`1: ...`) is never read as a label.
 */
export function normalizePreCompactLabel(raw: string): string | null {
  const trimmed = raw.normalize("NFC").trim();
  if (trimmed.length === 0 || trimmed.length > LABEL_MAX_CHARS) return null;
  if (!/\p{L}/u.test(trimmed)) return null;
  const words = trimmed.split(/\s+/u);
  if (words.length > LABEL_MAX_WORDS) return null;
  return words.join("_").toLowerCase();
}

export function extractPreCompactRecords(
  vault: string,
  input: PreCompactExtractInput,
): PreCompactExtractResult {
  const errors: string[] = [];
  const records: ContinuityRecord[] = [];
  const sourceRefs = extractSourceRefs(input);
  const boundedText = input.text.slice(0, input.maxChars ?? DEFAULT_MAX_CHARS);
  const extracted = extractLines(sanitizePreCompactText(boundedText));
  const createdAt = input.createdAt ?? new Date().toISOString();
  const host = input.host?.trim();
  // Preview mode reuses the exact record builder but never writes, so the
  // candidate output predicts the real extraction byte-for-byte (C2).
  const persist: (recordInput: AppendContinuityRecordInput) => ContinuityRecord =
    input.dryRun === true
      ? buildContinuityRecord
      : (recordInput) => appendContinuityRecord(vault, recordInput);

  for (const item of extracted) {
    try {
      const contentHash = hash(`${item.type}\n${item.text}`);
      const dedupeKey = [
        input.sessionId,
        input.turnStart,
        input.turnEnd,
        item.type,
        contentHash,
      ].join(":");
      const existing = findExistingExtract(vault, dedupeKey);
      if (existing !== null) {
        records.push(existing);
        continue;
      }
      records.push(
        persist({
          kind: "pre_compact_extract",
          createdAt,
          sourceRefs,
          payload: {
            extract_type: item.type,
            text: item.text,
            line: item.line,
            session_id: input.sessionId,
            turn_start: input.turnStart,
            turn_end: input.turnEnd,
            turn_range: `${input.turnStart}..${input.turnEnd}`,
            ...(host !== undefined && host.length > 0 ? { host } : {}),
            ...(input.interrupted === true ? { interrupted: true } : {}),
            content_hash: contentHash,
            dedupe_key: dedupeKey,
            truncated_input: boundedText.length < input.text.length,
          },
        }),
      );
    } catch (error) {
      errors.push(error instanceof Error ? error.message : String(error));
    }
  }

  return Object.freeze({
    records: Object.freeze(records),
    errors: Object.freeze(errors),
    skipped: extracted.length - records.length,
  });
}

export function sanitizePreCompactText(text: string): string {
  return text
    .replace(/data:[^\s;,]+;base64,[A-Za-z0-9+/=]+/g, "[base64]")
    .replace(/\b[A-Za-z0-9+/]{32,}={0,2}\b/g, "[base64]");
}

function extractLines(text: string): ExtractedLine[] {
  const items: ExtractedLine[] = [];
  const lines = text.split(/\r?\n/);
  for (let index = 0; index < lines.length; index += 1) {
    const line = lines[index]!.trim();
    if (line.length === 0) continue;
    if (LOCATOR_LINE_RE.test(line)) continue;
    const match = LABELLED_LINE_RE.exec(line);
    if (match === null) continue;
    const type = normalizePreCompactLabel(match[1]!);
    const body = match[2]!.trim();
    if (type === null || body.length === 0) continue;
    items.push({ type, text: body, line: index + 1 });
  }
  return items;
}

function extractSourceRefs(input: PreCompactExtractInput): ReadonlyArray<ContinuitySourceRef> {
  return Object.freeze([
    Object.freeze({ type: "session", id: input.sessionId }),
    Object.freeze({
      type: "turn_range",
      id: `${input.turnStart}..${input.turnEnd}`,
    }),
  ]);
}

function findExistingExtract(vault: string, dedupeKey: string): ContinuityRecord | null {
  return (
    listContinuityRecords(vault, { kind: "pre_compact_extract" }).find(
      (record) => record.payload["dedupe_key"] === dedupeKey,
    ) ?? null
  );
}

function hash(value: string): string {
  return createHash("sha256").update(value).digest("hex").slice(0, 16);
}