Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
22.4 KB
Raw Blame
import { existsSync, readFileSync } from "node:fs";

import { isStoreOnly } from "../store/store-only.ts";
import { escapeRegex } from "../strings.ts";
import { brainConfigPath } from "./paths.ts";
import {
  readStoreSchemaRecords,
  STORE_SCHEMA_PACK_SOURCE,
  type StoreSchemaRecords,
} from "./schema-pack-store.ts";
import {
  SCHEMA_VOCAB_CATEGORIES,
  resolveSchemaVocabulary,
  splitCompoundSchemaKey,
  validateSchemaDeclarations,
  validateSchemaToken,
  type BrainSchemaDeclarations,
  type BrainSchemaVocabulary,
  type SchemaVocabularyCategory,
} from "./schema-vocab.ts";

export interface SchemaPack {
  readonly declarations: BrainSchemaDeclarations;
  readonly vocabulary: BrainSchemaVocabulary;
  readonly aliases: Readonly<Record<string, ReadonlyArray<string>>>;
  readonly prefixes: Readonly<Record<string, string>>;
  readonly link_types: ReadonlyArray<string>;
  readonly extractable: ReadonlyArray<string>;
  readonly expert_routing: Readonly<Record<string, string>>;
  /** Controlled-vocabulary label dimensions: dimension -> allowed values. */
  readonly labels: Readonly<Record<string, ReadonlyArray<string>>>;
  /** Allowed endpoint pairs per link type, each stored as `source->target`. */
  readonly link_constraints: Readonly<Record<string, ReadonlyArray<string>>>;
  /** Per-type attribute descriptors: type -> field -> description. */
  readonly attributes: Readonly<Record<string, Readonly<Record<string, string>>>>;
  /** Frontmatter tier overrides: kind -> field -> tier. */
  readonly frontmatter_tiers: Readonly<Record<string, Readonly<Record<string, FrontmatterTier>>>>;
}

/**
 * Four-level frontmatter field tier model (EverOS-inspired):
 * `identity` - framework-owned join keys (hand-edit = corruption);
 * `system` - framework-written bookkeeping (hand-edit = drift);
 * `business` - agent-written domain fields; `user` - freely editable.
 */
export const FRONTMATTER_TIERS = Object.freeze(["identity", "system", "business", "user"] as const);

export type FrontmatterTier = (typeof FRONTMATTER_TIERS)[number];

/**
 * The config text parsed in place of an absent `Brain/_brain.yaml`.
 *
 * It declares a schema version and nothing else, so it yields a pack with
 * an empty vocabulary - which is a legitimate default for the many callers
 * that only want to know what the ontology permits. It is named rather
 * than inlined because the substitution used to be an anonymous literal in
 * two modules, and because a caller that needs to tell "no ontology" from
 * "an ontology that declares nothing" must be able to see that the
 * substitution happened at all. {@link readSchemaPackSource} is how it
 * sees that; {@link loadSchemaPack} deliberately keeps the defaulting
 * behaviour for everyone else.
 */
export const ABSENT_SCHEMA_CONFIG_TEXT = "schema_version: 1\n";

/**
 * The schema pack together with the provenance a caller needs to judge it:
 * where it came from, and whether that file was actually there.
 *
 * `present === false` means the config file does not exist and
 * {@link ABSENT_SCHEMA_CONFIG_TEXT} was parsed instead. A caller that only
 * wants the ontology can keep calling {@link loadSchemaPack} and ignore
 * this entirely.
 */
export interface SchemaPackSource {
  /**
   * Absolute path of `Brain/_brain.yaml`, whether or not it exists; on a
   * store-only machine the name of the settings records the pack lives in
   * ({@link STORE_SCHEMA_PACK_SOURCE}).
   */
  readonly path: string;
  /** False when the file is absent and the default text was substituted. */
  readonly present: boolean;
  /** The text that was parsed - the file's bytes, or the default. */
  readonly text: string;
  readonly pack: SchemaPack;
}

/** Read the schema pack and report whether its config file was there. */
export function readSchemaPackSource(vault: string | null): SchemaPackSource {
  // A store-only machine keeps the pack in the shared settings records
  // `brain.schema.<field>` (design D14 item 3), never in `_brain.yaml`.
  // No vault means store-only too.
  if (vault === null || isStoreOnly({ vault })) return readStoreSchemaPack().source;
  const path = brainConfigPath(vault);
  const present = existsSync(path);
  const text = present ? readFileSync(path, "utf8") : ABSENT_SCHEMA_CONFIG_TEXT;
  return Object.freeze({ path, present, text, pack: parseSchemaPack(text) });
}

/** The store's schema pack with the records it was read from, one read of them. */
export interface StoreSchemaPack extends StoreSchemaRecords {
  readonly source: SchemaPackSource;
}

/**
 * The pack of a store-only machine together with the field lists and the
 * deciding records it was parsed from, all from one read of the shared
 * settings stream (task 3.12, design D19 item 1). An apply and the
 * integrity read take the pack, its deciding records and the entries a
 * repair may remove from here, so no record can sync in between them.
 */
export function readStoreSchemaPack(configPath?: string): StoreSchemaPack {
  const records = readStoreSchemaRecords(configPath);
  const text = records.text ?? ABSENT_SCHEMA_CONFIG_TEXT;
  const source: SchemaPackSource = Object.freeze({
    path: STORE_SCHEMA_PACK_SOURCE,
    present: records.text !== null,
    text,
    // Fields merge per entry (task 3.12), so a removal from one device
    // can meet a reference another device added: repaired on read.
    pack: dropDanglingReferences(parseSchemaPack(text)),
  });
  return { ...records, source };
}

export function loadSchemaPack(vault: string | null): SchemaPack {
  return readSchemaPackSource(vault).pack;
}

/**
 * The page types this vault DECLARED, or `null` when it made no
 * declaration (trust-surface-hardening wave; kanban t_151a564c).
 *
 * `null` is the fail-open answer and it covers every undeclaring shape:
 * no config file at all, and a config that omits or empties
 * `page_types`. A non-null answer means the config is present and names
 * at least one page type - the one shape a writer must treat as a
 * closed vocabulary. The merged vocabulary (see
 * {@link resolveSchemaVocabulary}) always contains `note`, so the
 * declared list, not the merged one, is what tells a writer whether the
 * vault said anything at all.
 */
export function declaredPageTypes(vault: string): ReadonlyArray<string> | null {
  const source = readSchemaPackSource(vault);
  if (!source.present) return null;
  const declared = source.pack.declarations.page_types;
  if (declared === undefined || declared.length === 0) return null;
  return declared;
}

export function parseSchemaPack(configText: string): SchemaPack {
  const lines = extractSchemaLines(configText);
  const partial: Partial<Record<SchemaVocabularyCategory, ReadonlyArray<string>>> = {};
  for (const category of SCHEMA_VOCAB_CATEGORIES) {
    const values = parseListField(lines, category);
    if (values.length > 0) partial[category] = values;
  }
  const declarations = validateSchemaDeclarations(partial);
  const aliases = parseMapListField(lines, "aliases", "schema.aliases");
  const prefixes = parseMapScalarField(lines, "prefixes", "schema.prefixes");
  const linkTypes = parseListField(lines, "link_types").map((token, index) =>
    validateSchemaToken(token, `schema.link_types[${index}]`),
  );
  const extractable = parseListField(lines, "extractable").map((token, index) =>
    validateSchemaToken(token, `schema.extractable[${index}]`),
  );
  const expertRouting = parseMapScalarField(
    lines,
    "expert_routing",
    "schema.expert_routing",
    false,
  );
  const labels = parseMapListField(lines, "labels", "schema.labels");
  const linkConstraints = parseMapListField(
    lines,
    "link_constraints",
    "schema.link_constraints",
    validateEndpointPair,
  );
  const attributes = parseCompoundMapField(
    lines,
    "attributes",
    "schema.attributes",
    validateAttributeDescription,
  );
  const frontmatterTiers = parseCompoundMapField(
    lines,
    "frontmatter_tiers",
    "schema.frontmatter_tiers",
    validateFrontmatterTier,
  );

  return freezeSchemaPack({
    declarations,
    vocabulary: resolveSchemaVocabulary(declarations),
    aliases,
    prefixes,
    link_types: unique(linkTypes),
    extractable: unique(extractable),
    expert_routing: expertRouting,
    labels,
    link_constraints: linkConstraints,
    attributes,
    frontmatter_tiers: frontmatterTiers,
  });
}

/**
 * The pack without its references to undeclared tokens (design D19 item
 * 1): the aliases, prefixes, extractable entries, expert routes and
 * attributes of a token the vocabulary does not hold, the constraint pairs
 * whose source or target it does not hold, and the constraints of an
 * undeclared link type or of one left with no pair. That is the cascade
 * `remove_type` and `remove_link_type` apply. A store's fields merge per entry, so a type
 * removed on one device and a reference to it added on another meet in
 * the merged records; every machine drops the reference the same way when
 * it reads the pack, so the pack in force always passes the reference
 * check an apply runs, and the next apply writes the drop as removals.
 */
export function dropDanglingReferences(pack: SchemaPack): SchemaPack {
  const tokens = schemaPackTokens(pack);
  return freezeSchemaPack({
    declarations: pack.declarations,
    vocabulary: pack.vocabulary,
    aliases: keep(pack.aliases, (token) => tokens.has(token)),
    prefixes: keep(pack.prefixes, (_, token) => tokens.has(token)),
    link_types: pack.link_types,
    extractable: pack.extractable.filter((token) => tokens.has(token)),
    expert_routing: keep(pack.expert_routing, (token) => tokens.has(token)),
    labels: pack.labels,
    link_constraints: keep(
      Object.fromEntries(
        Object.entries(pack.link_constraints).map(([type, pairs]) => [
          type,
          Object.freeze(pairs.filter((pair) => pairEndpoints(pair).every((t) => tokens.has(t)))),
        ]),
      ),
      (type, pairs) => pack.link_types.includes(type) && pairs.length > 0,
    ),
    attributes: keep(pack.attributes, (type) => tokens.has(type)),
    frontmatter_tiers: pack.frontmatter_tiers,
  });
}

/** The entries of `record` that `ok` keeps, frozen. */
function keep<T>(
  record: Readonly<Record<string, T>>,
  ok: (key: string, value: T) => boolean,
): Readonly<Record<string, T>> {
  return Object.freeze(Object.fromEntries(Object.entries(record).filter(([k, v]) => ok(k, v))));
}

/** The source and target tokens of a normalized `source->target` pair. */
export function pairEndpoints(pair: string): [string, string] {
  const arrow = pair.indexOf("->");
  return [pair.slice(0, arrow), pair.slice(arrow + 2)];
}

/** Validate and normalize a `source->target` endpoint pair value. */
export function validateEndpointPair(value: string, field: string): string {
  const arrow = value.indexOf("->");
  if (arrow < 0) {
    throw new Error(`${field}: "${value}" must be a source->target pair of schema tokens`);
  }
  const source = validateSchemaToken(value.slice(0, arrow), field);
  const target = validateSchemaToken(value.slice(arrow + 2), field);
  return `${source}->${target}`;
}

function validateAttributeDescription(value: string, field: string): string {
  // Raw value checked for line breaks (same contract as the mutation
  // op in schema-mutate.ts); the trimmed form is what gets stored.
  if (/[\r\n]/.test(value)) throw new Error(`${field}: description must be a single line`);
  const trimmed = value.trim();
  if (trimmed.length === 0) throw new Error(`${field}: description must not be empty`);
  return trimmed;
}

function validateFrontmatterTier(value: string, field: string): FrontmatterTier {
  const normalized = value.trim().toLowerCase();
  if (!(FRONTMATTER_TIERS as ReadonlyArray<string>).includes(normalized)) {
    throw new Error(`${field}: tier must be one of ${FRONTMATTER_TIERS.join(", ")}`);
  }
  return normalized as FrontmatterTier;
}

export function renderSchemaBlock(pack: SchemaPack): string {
  const lines = ["schema:"];
  for (const category of SCHEMA_VOCAB_CATEGORIES) {
    const values = pack.declarations[category] ?? [];
    if (values.length === 0) continue;
    lines.push(`  ${category}:`);
    for (const token of values) lines.push(`    - ${token}`);
  }
  renderMapList(lines, "aliases", pack.aliases);
  renderMapScalar(lines, "prefixes", pack.prefixes);
  renderList(lines, "link_types", pack.link_types);
  renderList(lines, "extractable", pack.extractable);
  renderMapScalar(lines, "expert_routing", pack.expert_routing);
  renderMapList(lines, "labels", pack.labels);
  renderMapList(lines, "link_constraints", pack.link_constraints);
  renderCompoundMap(lines, "attributes", pack.attributes);
  renderCompoundMap(lines, "frontmatter_tiers", pack.frontmatter_tiers);
  if (lines.length === 1) lines.push("  {}");
  return lines.join("\n") + "\n";
}

export function replaceSchemaBlock(configText: string, schemaBlock: string): string {
  const normalized = configText.endsWith("\n") ? configText : configText + "\n";
  const lines = normalized.split("\n");
  const start = lines.findIndex((line) => /^schema:\s*$/.test(line));
  if (start < 0) return normalized + schemaBlock;

  let end = start + 1;
  while (end < lines.length) {
    const line = lines[end]!;
    if (line.length > 0 && !line.startsWith(" ") && !line.startsWith("\t")) break;
    end++;
  }
  const before = lines.slice(0, start).join("\n");
  const after = lines.slice(end).join("\n").replace(/^\n+/, "");
  return [before, schemaBlock.trimEnd(), after].filter((part) => part.length > 0).join("\n") + "\n";
}

export function schemaPackTokens(pack: SchemaPack): ReadonlySet<string> {
  const tokens = new Set<string>();
  for (const category of SCHEMA_VOCAB_CATEGORIES) {
    for (const token of pack.vocabulary[category]) tokens.add(token);
  }
  for (const token of pack.link_types) tokens.add(token);
  return tokens;
}

function extractSchemaLines(configText: string): string[] {
  const all = configText.split(/\r?\n/);
  const start = all.findIndex((line) => /^schema:\s*$/.test(line));
  if (start < 0) return [];
  const out: string[] = [];
  for (let index = start + 1; index < all.length; index++) {
    const line = all[index]!;
    if (line.length > 0 && !line.startsWith(" ") && !line.startsWith("\t")) break;
    out.push(line);
  }
  return out;
}

function parseListField(lines: ReadonlyArray<string>, key: string): string[] {
  const start = findField(lines, key);
  if (start < 0) return [];
  const rest = fieldRest(lines[start]!, key);
  if (rest.startsWith("[")) return parseInlineList(rest);
  if (rest === "{}") return [];

  const values: string[] = [];
  for (let index = start + 1; index < lines.length; index++) {
    const line = lines[index]!;
    if (/^  \S/.test(line)) break;
    const match = /^    -\s+(.+?)\s*$/.exec(line);
    if (match) values.push(stripQuotes(match[1]!));
  }
  return values;
}

function parseMapListField(
  lines: ReadonlyArray<string>,
  key: string,
  field: string,
  validateValue: (value: string, field: string) => string = validateSchemaToken,
): Record<string, ReadonlyArray<string>> {
  const start = findField(lines, key);
  if (start < 0) return freezeRecord({});
  const out: Record<string, string[]> = {};
  for (let index = start + 1; index < lines.length; index++) {
    const line = lines[index]!;
    if (/^  \S/.test(line)) break;
    const flat = /^    -\s+(.+?)\s*$/.exec(line);
    if (flat) {
      const [rawToken, rawAlias] = splitPair(flat[1]!);
      const token = validateSchemaToken(rawToken, `${field}.${rawToken}`);
      const alias = validateValue(rawAlias, `${field}.${token}`);
      out[token] = out[token] ?? [];
      if (!out[token]!.includes(alias)) out[token]!.push(alias);
      continue;
    }
    const entry = /^    ([^:]+):\s*(.*)$/.exec(line);
    if (!entry) continue;
    const token = validateSchemaToken(stripQuotes(entry[1]!), `${field}.${entry[1]}`);
    const rest = entry[2]!.trim();
    const values = rest.startsWith("[") ? parseInlineList(rest) : collectNestedList(lines, index);
    out[token] = unique(
      values.map((value, valueIndex) => validateValue(value, `${field}.${token}[${valueIndex}]`)),
    );
  }
  return freezeRecord(out);
}

/**
 * Parse a `- outer.inner=value` map-of-maps field. The compound key is
 * split on its first `.`; both halves must be schema tokens, the value
 * is checked by `validateValue` (free text stays untouched beyond it).
 */
function parseCompoundMapField<T extends string>(
  lines: ReadonlyArray<string>,
  key: string,
  field: string,
  validateValue: (value: string, field: string) => T,
): Readonly<Record<string, Readonly<Record<string, T>>>> {
  const start = findField(lines, key);
  if (start < 0) return Object.freeze({});
  const out: Record<string, Record<string, T>> = {};
  for (let index = start + 1; index < lines.length; index++) {
    const line = lines[index]!;
    if (/^  \S/.test(line)) break;
    const flat = /^    -\s+(.+?)\s*$/.exec(line);
    if (!flat) continue;
    const [rawKey, rawValue] = splitPair(flat[1]!);
    const halves = splitCompoundSchemaKey(rawKey);
    if (halves === null) {
      throw new Error(`${field}: "${rawKey}" must use a compound type.field key`);
    }
    const outer = validateSchemaToken(halves[0], `${field}.${rawKey}`);
    const inner = validateSchemaToken(halves[1], `${field}.${outer}`);
    const value = validateValue(rawValue, `${field}.${outer}.${inner}`);
    out[outer] = out[outer] ?? {};
    out[outer]![inner] = value;
  }
  return freezeNestedRecord(out);
}

function parseMapScalarField(
  lines: ReadonlyArray<string>,
  key: string,
  field: string,
  validateValue = true,
): Record<string, string> {
  const start = findField(lines, key);
  if (start < 0) return {};
  const out: Record<string, string> = {};
  for (let index = start + 1; index < lines.length; index++) {
    const line = lines[index]!;
    if (/^  \S/.test(line)) break;
    const flat = /^    -\s+(.+?)\s*$/.exec(line);
    if (flat) {
      const [rawName, rawValue] = splitPair(flat[1]!);
      const name = validateSchemaToken(rawName, `${field}.${rawName}`);
      out[name] = validateValue
        ? validateSchemaToken(rawValue, `${field}.${name}`)
        : rawValue.trim();
      continue;
    }
    const entry = /^    ([^:]+):\s*(.+?)\s*$/.exec(line);
    if (!entry) continue;
    const name = validateSchemaToken(stripQuotes(entry[1]!), `${field}.${entry[1]}`);
    const rawValue = stripQuotes(entry[2]!);
    out[name] = validateValue ? validateSchemaToken(rawValue, `${field}.${name}`) : rawValue.trim();
  }
  return freezeRecord(out);
}

function collectNestedList(lines: ReadonlyArray<string>, start: number): string[] {
  const values: string[] = [];
  for (let index = start + 1; index < lines.length; index++) {
    const line = lines[index]!;
    if (/^    \S/.test(line) || /^  \S/.test(line)) break;
    const match = /^      -\s+(.+?)\s*$/.exec(line);
    if (match) values.push(stripQuotes(match[1]!));
  }
  return values;
}

function findField(lines: ReadonlyArray<string>, key: string): number {
  return lines.findIndex((line) => new RegExp(`^  ${escapeRegex(key)}:\\s*`).test(line));
}

function fieldRest(line: string, key: string): string {
  return line.replace(new RegExp(`^  ${escapeRegex(key)}:\\s*`), "").trim();
}

function parseInlineList(raw: string): string[] {
  const trimmed = raw.trim();
  if (!trimmed.startsWith("[") || !trimmed.endsWith("]")) return [];
  const inner = trimmed.slice(1, -1).trim();
  if (inner.length === 0) return [];
  return inner.split(",").map((item) => stripQuotes(item));
}

function stripQuotes(raw: string): string {
  const trimmed = raw.trim();
  if (
    (trimmed.startsWith('"') && trimmed.endsWith('"')) ||
    (trimmed.startsWith("'") && trimmed.endsWith("'"))
  ) {
    return trimmed.slice(1, -1).trim();
  }
  return trimmed;
}

function renderList(lines: string[], key: string, values: ReadonlyArray<string>): void {
  if (values.length === 0) return;
  lines.push(`  ${key}:`);
  for (const value of values) lines.push(`    - ${value}`);
}

function renderMapList(
  lines: string[],
  key: string,
  values: Readonly<Record<string, ReadonlyArray<string>>>,
): void {
  const entries = Object.entries(values).toSorted(([left], [right]) => left.localeCompare(right));
  if (entries.length === 0) return;
  lines.push(`  ${key}:`);
  for (const [token, aliases] of entries) {
    for (const alias of aliases) lines.push(`    - ${token}=${alias}`);
  }
}

function renderMapScalar(
  lines: string[],
  key: string,
  values: Readonly<Record<string, string>>,
): void {
  const entries = Object.entries(values).toSorted(([left], [right]) => left.localeCompare(right));
  if (entries.length === 0) return;
  lines.push(`  ${key}:`);
  for (const [name, value] of entries) lines.push(`    - ${name}=${value}`);
}

function renderCompoundMap(
  lines: string[],
  key: string,
  values: Readonly<Record<string, Readonly<Record<string, string>>>>,
): void {
  const outers = Object.entries(values).toSorted(([left], [right]) => left.localeCompare(right));
  const rendered: string[] = [];
  for (const [outer, fields] of outers) {
    const inners = Object.entries(fields).toSorted(([left], [right]) => left.localeCompare(right));
    for (const [inner, value] of inners) rendered.push(`    - ${outer}.${inner}=${value}`);
  }
  if (rendered.length === 0) return;
  lines.push(`  ${key}:`, ...rendered);
}

function splitPair(raw: string): [string, string] {
  const index = raw.indexOf("=");
  if (index < 0) return [raw, ""];
  return [raw.slice(0, index).trim(), raw.slice(index + 1).trim()];
}

function freezeSchemaPack(
  input: Omit<SchemaPack, "vocabulary"> & { vocabulary: BrainSchemaVocabulary },
): SchemaPack {
  return Object.freeze({
    declarations: input.declarations,
    vocabulary: input.vocabulary,
    aliases: input.aliases,
    prefixes: input.prefixes,
    link_types: Object.freeze([...input.link_types]),
    extractable: Object.freeze([...input.extractable]),
    expert_routing: input.expert_routing,
    labels: input.labels,
    link_constraints: input.link_constraints,
    attributes: input.attributes,
    frontmatter_tiers: input.frontmatter_tiers,
  });
}

function freezeRecord<T>(record: Record<string, T>): Readonly<Record<string, T>> {
  return Object.freeze({ ...record });
}

function freezeNestedRecord<T extends string>(
  record: Record<string, Record<string, T>>,
): Readonly<Record<string, Readonly<Record<string, T>>>> {
  const out: Record<string, Readonly<Record<string, T>>> = {};
  for (const [key, inner] of Object.entries(record)) out[key] = Object.freeze({ ...inner });
  return Object.freeze(out);
}

function unique(values: ReadonlyArray<string>): string[] {
  const seen = new Set<string>();
  const out: string[] = [];
  for (const value of values) {
    if (seen.has(value)) continue;
    seen.add(value);
    out.push(value);
  }
  return out;
}