Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.6 KB
Raw Blame
/**
 * #5988 Lane A: the frontmatter reader recovers what it can read exactly
 * (quoting a value), holds what it would have to guess, and never reads a
 * protected key as a broader value.
 *
 * Protects: import of the issue's files, the hold classification every
 * ingestion path shares, and the upgrade invariant (a file that imports today
 * parses to the same values). Regressions it catches: the validator rejecting
 * files the parser reads (R2), a guessed value or a swallowed `visibility`
 * being imported, a rule rewriting a valid line, raw values leaking into
 * persisted messages.
 */
import { describe, expect, test } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import {
  classifyImportHold, parseMarkdown, recoverFrontmatter, HOLD_CODES, PROTECTED_FRONTMATTER_KEYS,
  IDENTITY_FRONTMATTER_KEYS, RECOVERY_VERSION, type ParsedMarkdown,
} from '../src/core/markdown.ts';
import { isPrivatePage } from '../src/core/search/private-visibility.ts';

const doc = (block: string, body = 'Body text.\n') => `---\n${block}\n---\n\n${body}`;
const parse = (content: string, path = 'notes/example.md') => parseMarkdown(content, path, { validate: true });
const hold = (parsed: ParsedMarkdown) => classifyImportHold(parsed);
const allFrontmatter = (parsed: ParsedMarkdown) => ({ ...parsed.frontmatter, title: parsed.title, type: parsed.type });

describe('recoverable shapes import with their exact text', () => {
  test("the issue's exact author line (unquoted ': ') imports and is not held", () => {
    const parsed = parse(doc('title: Payments roundup\nauthor: PYMNTS (citing Reuters / Bloomberg) (original: https://x.com/a/status/1)'));
    expect(parsed.frontmatter.author).toBe('PYMNTS (citing Reuters / Bloomberg) (original: https://x.com/a/status/1)');
    expect(hold(parsed)).toBeNull();
    const yaml = parsed.errors!.find(e => e.code === 'YAML_PARSE')!;
    expect(yaml.recoverable).toBe(true);
    expect(yaml.recovery).toEqual([{ kind: 'quote', key: 'author', line: 3, recovery_version: RECOVERY_VERSION }]);
    expect(parsed.warnings!.map(w => [w.code, w.key, w.line])).toEqual([['FRONTMATTER_RECOVERED', 'author', 3]]);
  });

  test('"quoted" trailing text keeps the whole value text', () => {
    const parsed = parse(doc('title: "Quoted" trailing words\ntype: note'));
    expect(parsed.title).toBe('"Quoted" trailing words');
    expect(parsed.recovery?.status).toBe('recovered');
    expect(hold(parsed)).toBeNull();
  });

  test('nested quotes keep the inner text, as the NESTED_QUOTES fix does', () => {
    const parsed = parse(doc('title: "Name "Nick" Last"'));
    expect(parsed.title).toBe('Name "Nick" Last');
    expect(parsed.errors!.map(e => e.code)).toContain('NESTED_QUOTES');
    expect(hold(parsed)).toBeNull();
  });

  test('a single-quoted scalar followed by text and an unclosed leading quote are quoted', () => {
    expect(parse(doc("title: 'it' s here")).title).toBe("'it' s here");
    expect(parse(doc('title: "Hello\ntype: note')).title).toBe('"Hello');
  });

  test('CRLF files recover and keep CRLF on every line, including the rewritten one', () => {
    const block = 'title: Re: hello\r\nstatus: draft # todo\r\n';
    const recovered = recoverFrontmatter(block);
    expect(recovered.status).toBe('recovered');
    expect(recovered.block).toBe('title: "Re: hello"\r\nstatus: draft # todo\r\n');
    const parsed = parse('---\r\ntitle: Re: hello\r\ntype: note\r\n---\r\nbody\r\n');
    expect(parsed.title).toBe('Re: hello');
    expect(hold(parsed)).toBeNull();
  });

  test('a BOM-prefixed file recovers', () => {
    const parsed = parse('\uFEFF---\ntitle: Re: hello\n---\nbody\n');
    expect(parsed.title).toBe('Re: hello');
    expect(hold(parsed)).toBeNull();
  });

  test('valid lines stay byte-identical; only the suspect line is rewritten', () => {
    const block = 'status: draft # todo\ntitle: "Hello" world\ntags: [a, b]\ndate: 2024-06-01\n';
    const recovered = recoverFrontmatter(block);
    expect(recovered.status).toBe('recovered');
    expect(recovered.block.split('\n')).toEqual(['status: draft # todo', 'title: "\\"Hello\\" world"', 'tags: [a, b]', 'date: 2024-06-01', '']);
    expect(recovered.steps.map(s => [s.kind, s.key, s.line])).toEqual([['quote', 'title', 2]]);
  });

  test('a recovered visibility-bearing file keeps its private visibility end to end', () => {
    const parsed = parse(doc('title: "Hello" world\nvisibility: private'));
    expect(hold(parsed)).toBeNull();
    expect(isPrivatePage({ type: parsed.type, frontmatter: parsed.frontmatter })).toBe(true);
  });
});

describe('interpretive shapes are held, never imported', () => {
  test('a value continuing on unquoted lines (fold) is needs_interpretation, with the proposal kept', () => {
    const parsed = parse(doc('title: First line of a post\nsecond line of the post\ntype: note'));
    expect(parsed.frontmatter).toEqual({});
    expect(hold(parsed)).toMatchObject({ code: 'invalid_frontmatter', reason: 'needs_interpretation', key: 'title', line: 2 });
    expect(parsed.recovery!.block).toContain('title: "First line of a post\\nsecond line of the post"');
  });

  test('a duplicated non-identity key is needs_interpretation naming both lines', () => {
    const parsed = parse(doc('title: a\ntype: note\ntitle: b'));
    expect(hold(parsed)).toMatchObject({ code: 'invalid_frontmatter', reason: 'needs_interpretation', key: 'title', line: 4 });
    expect(parsed.recovery!.steps[0]).toMatchObject({ kind: 'dup', key: 'title', line: 4, otherLine: 2 });
  });

  test('an unclosed [ is needs_interpretation', () => {
    expect(hold(parse(doc('tags: [a, b\ntype: note')))).toMatchObject({ reason: 'needs_interpretation', key: 'tags' });
  });

  test('shapes with no rule are unrecoverable yaml_parse holds', () => {
    expect(hold(parse(doc('tags:\n  - a\n - b')))).toMatchObject({ code: 'invalid_frontmatter', reason: 'yaml_parse', key: 'tags' });
    expect(recoverFrontmatter('title: ok\n  bad: indent\n').status).toBe('unrecoverable');
  });

  test('duplicated identity keys are held as ambiguous_identity_key', () => {
    for (const key of IDENTITY_FRONTMATTER_KEYS) {
      const h = hold(parse(doc(`${key}: one\ntitle: a\n${key}: two`)));
      expect(h).toMatchObject({ code: 'invalid_frontmatter', reason: 'ambiguous_identity_key', key, line: 4 });
      expect(h!.message.startsWith('Invalid YAML frontmatter: ambiguous identity key')).toBe(true);
    }
  });
});

describe('protected keys are never guessed', () => {
  test('a visibility swallowed by an unclosed quote is held although YAML parses it', () => {
    const parsed = parse(doc('title: "Hello\nvisibility: private\nsummary: x"'));
    expect(parsed.errors!.some(e => e.code === 'YAML_PARSE')).toBe(false);
    expect(isPrivatePage({ type: parsed.type, frontmatter: parsed.frontmatter })).toBe(false);
    expect(hold(parsed)).toMatchObject({ code: 'invalid_frontmatter', reason: 'ambiguous_protected_key', key: 'visibility', line: 3 });
  });

  test('a protected value the pre-quote pass rewrote is held', () => {
    const parsed = parse(doc('title: a\nvisibility: private # note: x'));
    expect(hold(parsed)).toMatchObject({ reason: 'ambiguous_protected_key', key: 'visibility', line: 3 });
  });

  test('an unclosed fence carrying visibility is held instead of importing with no frontmatter', () => {
    const parsed = parse('---\ntitle: a\nvisibility: private\n\n# Heading\nbody\n');
    expect(parsed.errors!.map(e => e.code)).toContain('MISSING_CLOSE');
    expect(hold(parsed)).toMatchObject({ reason: 'ambiguous_protected_key', key: 'visibility', line: 3 });
  });

  test('duplicated visibility, a quoted visibility with trailing text and a fold into derived_from are held', () => {
    expect(hold(parse(doc('visibility: private\ntitle: a\nvisibility: world')))).toMatchObject({ reason: 'ambiguous_protected_key', key: 'visibility' });
    expect(hold(parse(doc('visibility: "private" trailing')))).toMatchObject({ reason: 'ambiguous_protected_key', key: 'visibility' });
    expect(hold(parse(doc('derived_from: notes/a\nnotes/b\ntitle: x')))).toMatchObject({ reason: 'ambiguous_protected_key', key: 'derived_from' });
  });

  test('every frontmatter key the access-control and provenance readers use is protected', () => {
    const readers = ['src/core/search/private-visibility.ts', 'src/core/repair/visibility.ts', 'src/core/persistence/connector-sync.ts'];
    const keys = new Set<string>();
    for (const file of readers) {
      const text = readFileSync(join(import.meta.dir, '..', file), 'utf8');
      for (const m of text.matchAll(/frontmatter(?:->>?'|\??\.)([a-z_]+)/g)) keys.add(m[1]!);
    }
    expect(keys.size).toBeGreaterThan(3);
    for (const key of keys) expect(PROTECTED_FRONTMATTER_KEYS).toContain(key);
  });
});

describe('contract surfaces', () => {
  test('producer checks stay strict: recoverable YAML_PARSE stays in errors; warnings are separate', () => {
    const parsed = parse(doc('title: Re: hello'));
    expect(parsed.errors!.map(e => e.code)).toEqual(['YAML_PARSE']);
    expect(parsed.warnings!.map(w => w.code)).toEqual(['FRONTMATTER_RECOVERED']);
    expect(parse(doc('title: fine')).warnings).toEqual([]);
  });

  test('#-comment values are detected without changing the parse; a spaced comment is left alone', () => {
    const parsed = parse(doc('title: #1 thing\ndescription: # TODO'));
    expect(parsed.frontmatter.description).toBeNull();
    expect(parsed.title).toBe('Example');
    expect(parsed.warnings!.map(w => [w.code, w.key, w.line])).toEqual([['FRONTMATTER_COMMENT_VALUE', 'title', 2]]);
    expect(hold(parsed)).toBeNull();
  });

  test('hold messages and YAML_PARSE messages are location-only', () => {
    const secret = 'alice-example private note';
    for (const block of [`title: ${secret}\nmore of ${secret}`, `tags:\n  - ${secret}\n - b`, `title: "${secret}" x\ntitle: y`]) {
      const parsed = parse(doc(block));
      const h = hold(parsed)!;
      expect(h.message.startsWith('Invalid YAML frontmatter:')).toBe(true);
      expect(h.message).not.toContain('alice-example');
      for (const e of parsed.errors!) expect(e.message).not.toContain('alice-example');
    }
  });

  test('HOLD_CODES is the canonical hold set', () => {
    expect([...HOLD_CODES]).toEqual(['invalid_frontmatter', 'frontmatter_slug_conflict', 'file_too_large', 'rename_held', 'parser_regression', 'tag_too_long']);
  });

  test('slug conflicts hold unless the recorded-origin exemption applies', () => {
    const parsed = parse(doc('slug: other/page'), 'notes/page.md');
    expect(classifyImportHold(parsed, { expectedSlug: 'notes/page' })).toMatchObject({ code: 'frontmatter_slug_conflict', key: 'slug' });
    expect(classifyImportHold(parsed, { expectedSlug: 'notes/page', slugExempt: slug => slug === 'other/page' })).toBeNull();
  });
});

describe('upgrade invariant: files that import today parse to identical values', () => {
  const corpus: Array<[string, Record<string, unknown>]> = [
    ['title: foo # note: x\ntype: note', { title: 'foo # note: x', type: 'note' }],
    ['title: #1 thing\ntype: note', { title: 'Example', type: 'note' }],
    ['status: draft # todo\ntitle: Plain', { status: 'draft', title: 'Plain', type: 'note' }],
    ['title: Re: October booking\ntype: email', { title: 'Re: October booking', type: 'email' }],
    ['title: "Quoted: fine"\ntags: [a, "b c"]', { title: 'Quoted: fine', type: 'note' }],
    ["title: 'single \"inner\" quotes'", { title: 'single "inner" quotes', type: 'note' }],
    ['title: Ends with colon:\nvisibility: world', { title: 'Ends with colon:', visibility: 'world', type: 'note' }],
    ['title: >\n  folded block\n  text\ntype: concept', { title: 'folded block text', type: 'concept' }],
    ['description: |\n  line one\n  line two\ntitle: Block', { description: 'line one\nline two\n', title: 'Block', type: 'note' }],
    ['title: Multi\n  line plain\ntype: note', { title: 'Multi line plain', type: 'note' }],
    ['title: "Hello\n  continued"', { title: 'Hello continued', type: 'note' }],
    ['tags:\n- a\n- b\ntitle: Seq', { title: 'Seq', type: 'note' }],
    ['title: 2024-06-01', { title: '2024-06-01', type: 'note' }],
    ['date: 2024-02-30\ntitle: Bad date', { date: '2024-02-30', title: 'Bad date', type: 'note' }],
  ];
  for (const [block, expected] of corpus) {
    test(JSON.stringify(block), () => {
      const content = doc(block);
      const parsed = parse(content, 'notes/example.md');
      expect(parsed.recovery?.steps ?? []).toEqual([]);
      expect(parsed.recovery).toBeUndefined();
      expect(allFrontmatter(parsed)).toMatchObject(expected);
      expect(hold(parsed)).toBeNull();
      const plain = parseMarkdown(content, 'notes/example.md');
      expect(allFrontmatter(plain)).toEqual(allFrontmatter(parsed));
    });
  }
});