Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Recovery for frontmatter blocks js-yaml refuses (#5988).
*
* `parseMarkdown` calls `recoverFrontmatter` only after its unconditional
* pre-quote pass and a strict parse have failed, so a block that parses today
* is never re-read. Rules touch only the suspect run (the key line nearest the
* parser's error mark plus its non-key continuation lines); every other line
* stays byte-identical, and each line keeps its own CRLF ending.
*
* `quote` keeps the value text exactly and re-emits it as one double-quoted
* scalar, so it is the only kind an import accepts. `fold`, `dup` and
* `unclosed` guess at meaning: the block is classified `needs_interpretation`
* and the proposed block is kept for an approved repair, never imported.
* Protected keys (who may read a page, where it came from) are never
* rewritten, folded into or guessed; identity keys are never resolved from a
* duplicate.
*/
import { FRONTMATTER_SCHEMA, loadBoundedYaml as load } from './data-frontmatter.ts';
/** Bump when a rule widens; older holds are re-screened. */
export const RECOVERY_VERSION = 1;
/**
* Keys read by access-control and provenance code (`search/private-visibility.ts`,
* `repair/visibility.ts`, connector publication). `test/frontmatter-recovery.test.ts`
* greps those readers so a new key cannot be missed.
*/
export const PROTECTED_FRONTMATTER_KEYS: readonly string[] = ['visibility', 'derived_from', 'source_slug', 'synthesized_by', 'concepts', 'captured_via', 'event', 'pinned_question', 'access', 'confidentiality', 'readable_by', 'trust_tier'];
/** Keys that decide which page a file is. A duplicate is never resolved. */
export const IDENTITY_FRONTMATTER_KEYS: readonly string[] = ['slug', 'type', 'id', 'source_id'];
export type RecoveryKind = 'quote' | 'fold' | 'dup' | 'unclosed';
export type FrontmatterRecoveryStatus =
| 'clean' | 'recovered' | 'needs_interpretation' | 'ambiguous_identity_key' | 'ambiguous_protected_key' | 'unrecoverable';
export interface FrontmatterRecoveryStep {
kind: RecoveryKind;
key: string;
/** File line of the rewritten key line (for `dup`, the later occurrence, which wins). */
line: number;
/** For `dup`: the earlier occurrence, which the proposal drops. */
otherLine?: number;
original: string;
replacement: string | null;
}
export interface FrontmatterRecovery {
status: FrontmatterRecoveryStatus;
/** `recovered`: the block to import. `needs_interpretation`: the proposed block. Otherwise the input. */
block: string;
steps: FrontmatterRecoveryStep[];
key?: string;
line?: number;
column?: number;
/** js-yaml's reason phrase for the last failure; it never quotes the document. */
yamlReason?: string;
recovery_version: number;
}
const KEY_LINE = /^([A-Za-z_][\w-]*):(?=[ \t]|$)/;
interface Row { text: string; eol: string }
interface YamlFailure { reason: string; line: number | null; column: number | null }
function loadBlock(text: string): { ok: true; data: unknown } | { ok: false; failure: YamlFailure } {
try {
return { ok: true, data: load(text, { schema: FRONTMATTER_SCHEMA }) };
} catch (error) {
const e = error as { reason?: string; mark?: { line?: number; column?: number } };
return { ok: false, failure: { reason: e.reason ?? 'malformed YAML', line: e.mark?.line ?? null, column: e.mark?.column ?? null } };
}
}
function toRows(block: string): Row[] {
return block.split('\n').map(raw => raw.endsWith('\r') ? { text: raw.slice(0, -1), eol: '\r' } : { text: raw, eol: '' });
}
function fromRows(rows: Row[]): string {
return rows.map(row => row.text + row.eol).join('\n');
}
function keyOf(text: string): string | null {
return KEY_LINE.exec(text)?.[1] ?? null;
}
/** Location only: js-yaml's reason phrase plus line and column, never a document excerpt. */
export function yamlLocationMessage(reason: string, line: number | null | undefined, column?: number | null): string {
return `${reason}${typeof line === 'number' ? ` at line ${line}${typeof column === 'number' ? `, column ${column}` : ''}` : ''}`;
}
/** A value whose meaning is fixed, re-emitted as one double-quoted scalar. */
function quotedScalarText(value: string): string {
const q = value[0];
if ((q === '"' || q === "'") && value.length >= 2 && value.endsWith(q)) {
const inner = value.slice(1, -1);
if (new RegExp(q === '"' ? '(^|[^\\\\])"' : "'").test(inner)) return inner;
}
return value;
}
interface Rule { kind: RecoveryKind; replace: Row[]; removeThrough: number }
function ruleFor(rows: Row[], k: number, end: number): Rule | null {
const match = KEY_LINE.exec(rows[k]!.text)!;
const key = match[1]!;
const value = rows[k]!.text.slice(match[0].length).trim();
const continuation = rows.slice(k + 1, end);
const lastContent = continuation.reduce((last, row, i) => row.text.trim() === '' ? last : i, -1);
const content = continuation.slice(0, lastContent + 1);
const unindented = content.filter(row => row.text.trim() !== '' && !/^[ \t]/.test(row.text) && !/^-(?:[ \t]|$)/.test(row.text) && !row.text.startsWith('#'));
const standaloneOk = value === '' || loadBlock(`${key}: ${value}`).ok;
const emit = (text: string): Row[] => [{ text: `${key}: ${JSON.stringify(text)}`, eol: rows[k]!.eol }];
if (unindented.length > 0) {
if (unindented.length !== content.filter(row => row.text.trim() !== '').length) return null;
const head = standaloneOk && /^["']/.test(value) ? String((loadBlock(`v: ${value}`) as { data: { v: unknown } }).data.v) : value;
return { kind: 'fold', replace: emit([head, ...unindented.map(row => row.text)].filter(part => part !== '').join('\n')), removeThrough: k + 1 + lastContent };
}
if (standaloneOk || content.length > 0) return null;
if (/^["']/.test(value)) return { kind: 'quote', replace: emit(quotedScalarText(value)), removeThrough: k };
if (/^[[{]/.test(value)) return { kind: 'unclosed', replace: emit(value), removeThrough: k };
if (value.includes(': ') || value.endsWith(':') || /^[@`]/.test(value)) return { kind: 'quote', replace: emit(value), removeThrough: k };
return null;
}
/**
* Recover one frontmatter block (the text between the fences). `lineOffset`
* is the file line before the block's first row, so reported lines are file
* lines. Callers run it only after a strict parse failed.
*/
export function recoverFrontmatter(block: string, lineOffset = 0): FrontmatterRecovery {
let rows = toRows(block);
const steps: FrontmatterRecoveryStep[] = [];
const seen = new Set<string>();
let interpretive = false;
const done = (status: FrontmatterRecoveryStatus, extra: Partial<FrontmatterRecovery> = {}): FrontmatterRecovery =>
({ status, block: status === 'recovered' || status === 'needs_interpretation' ? fromRows(rows) : block, steps, recovery_version: RECOVERY_VERSION, ...extra });
const keyLines = rows.filter(row => keyOf(row.text) !== null).length;
for (let pass = 0; pass <= keyLines + 1; pass++) {
const loaded = loadBlock(fromRows(rows));
if (loaded.ok) return done(steps.length === 0 ? 'clean' : interpretive ? 'needs_interpretation' : 'recovered');
const failure = loaded.failure;
const mark = Math.min(Math.max(failure.line ?? rows.length - 1, 0), rows.length - 1);
const location = { yamlReason: failure.reason, column: failure.column === null ? undefined : failure.column + 1 };
let nearest = mark;
while (nearest > 0 && keyOf(rows[nearest]!.text) === null) nearest--;
const nearestKey = keyOf(rows[nearest]!.text) ?? undefined;
const at = { ...location, key: nearestKey, line: lineOffset + (nearestKey ? nearest : mark) + 1 };
if (/duplicated mapping key/.test(failure.reason) && nearestKey) {
const line = lineOffset + nearest + 1;
if (PROTECTED_FRONTMATTER_KEYS.includes(nearestKey)) return done('ambiguous_protected_key', { ...at, line });
if (IDENTITY_FRONTMATTER_KEYS.includes(nearestKey)) return done('ambiguous_identity_key', { ...at, line });
const earlier = rows.findIndex((row, i) => i < nearest && keyOf(row.text) === nearestKey);
if (earlier < 0 || seen.has(`dup:${nearestKey}`)) return done('unrecoverable', at);
seen.add(`dup:${nearestKey}`);
let earlierEnd = earlier + 1;
while (earlierEnd < rows.length && keyOf(rows[earlierEnd]!.text) === null && /^([ \t]|-(?:[ \t]|$))/.test(rows[earlierEnd]!.text)) earlierEnd++;
steps.push({ kind: 'dup', key: nearestKey, line: lineOffset + nearest + 1, otherLine: lineOffset + earlier + 1, original: rows[earlier]!.text, replacement: null });
interpretive = true;
rows = [...rows.slice(0, earlier), ...rows.slice(earlierEnd)];
continue;
}
let chosen: { k: number; rule: Rule } | null = null;
for (let k = mark; k >= 0 && !chosen; k--) {
const key = keyOf(rows[k]!.text);
if (key === null || seen.has(`line:${k}:${rows[k]!.text}`)) continue;
let end = k + 1;
while (end < rows.length && keyOf(rows[end]!.text) === null) end++;
const rule = ruleFor(rows, k, end);
if (rule) chosen = { k, rule };
}
if (!chosen) return done('unrecoverable', at);
const { k, rule } = chosen;
const key = keyOf(rows[k]!.text)!;
const line = lineOffset + k + 1;
if (PROTECTED_FRONTMATTER_KEYS.includes(key)) return done('ambiguous_protected_key', { ...location, key, line });
if (rule.kind !== 'quote' && IDENTITY_FRONTMATTER_KEYS.includes(key)) return done('ambiguous_identity_key', { ...location, key, line });
seen.add(`line:${k}:${rule.replace[0]!.text}`);
steps.push({ kind: rule.kind, key, line, original: rows[k]!.text, replacement: rule.replace[0]!.text });
if (rule.kind !== 'quote') interpretive = true;
rows = [...rows.slice(0, k), ...rule.replace, ...rows.slice(rule.removeThrough + 1)];
}
return done('unrecoverable', { yamlReason: 'too many recovery passes' });
}
/**
* A successfully parsed block can still hide who may read the page: a
* protected value the pre-quote pass rewrote (`visibility: private # note: x`
* imports as a different string), or a protected or identity key line that a
* multi-line quoted scalar swallowed. `quotedKeys` are the key names the
* pre-quote pass rewrote.
*/
export function frontmatterKeyHazard(
block: string,
data: Record<string, unknown>,
quotedKeys: ReadonlySet<string>,
lineOffset = 0,
): { status: 'ambiguous_protected_key' | 'ambiguous_identity_key'; key: string; line: number } | null {
const rows = toRows(block);
for (let i = 0; i < rows.length; i++) {
const key = keyOf(rows[i]!.text);
if (key === null) continue;
const line = lineOffset + i + 1;
const present = Object.prototype.hasOwnProperty.call(data, key);
if (PROTECTED_FRONTMATTER_KEYS.includes(key) && (quotedKeys.has(key) || !present)) return { status: 'ambiguous_protected_key', key, line };
if (IDENTITY_FRONTMATTER_KEYS.includes(key) && !present) return { status: 'ambiguous_identity_key', key, line };
}
return null;
}
/**
* An unclosed fence imports with no frontmatter at all, which would drop a
* protected key such as `visibility`. `lines` are the file lines after the
* opening fence; only the leading YAML-shaped run is inspected.
*/
export function unclosedFenceProtectedKey(lines: readonly string[], lineOffset = 0): { key: string; line: number } | null {
for (let i = 0; i < lines.length; i++) {
const text = lines[i]!.replace(/\r$/, '');
const key = keyOf(text);
if (key !== null) {
if (PROTECTED_FRONTMATTER_KEYS.includes(key)) return { key, line: lineOffset + i + 1 };
continue;
}
if (text.trim() === '' || /^[ \t]/.test(text) || /^-(?:[ \t]|$)/.test(text) || text.startsWith('#')) continue;
return null;
}
return null;
}
/**
* `title: #1 thing` is valid YAML whose value is a comment, so the key reads
* as null. Detection only: the parse is unchanged. `description: # TODO`
* (a space after `#`) is an ordinary empty value and is not reported.
*/
export function commentValueKeys(block: string, data: Record<string, unknown>, lineOffset = 0): Array<{ key: string; line: number }> {
const found: Array<{ key: string; line: number }> = [];
toRows(block).forEach((row, i) => {
const m = /^([A-Za-z_][\w-]*):[ \t]+#\S/.exec(row.text);
if (m && Object.prototype.hasOwnProperty.call(data, m[1]!) && data[m[1]!] === null) found.push({ key: m[1]!, line: lineOffset + i + 1 });
});
return found;
}