Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.3 KB
Raw Blame
/**
 * v0.20.0 Cathedral II Layer 8 D2 — markdown fence extraction tests.
 *
 * Validates that importing markdown with fenced code blocks produces
 * extra chunks with chunk_source='fenced_code', correct language
 * metadata, and respect for the fence-bomb DOS cap.
 */

import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { importFromContent } from '../src/core/import-file.ts';
import { scanFencedBlocks } from '../src/core/fence-scan.ts';
import { cpuMs } from './helpers/cpu-time.ts';

describe('Layer 8 D2 — markdown fence extraction', () => {
  let engine: PGLiteEngine;

  beforeAll(async () => {
    engine = new PGLiteEngine();
    await engine.connect({});
    await engine.initSchema();
  });

  afterAll(async () => {
    await engine.disconnect();
  }, 30_000);

  test('TypeScript fence becomes a fenced_code chunk with language=typescript', async () => {
    const md = `# Guide

Some intro prose about the chunker.

\`\`\`ts
export function hello(name: string): string {
  return \`Hello, \${name}\`;
}
\`\`\`

More prose.`;

    await importFromContent(engine, 'guides/fence-ts', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-ts');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks).toHaveLength(1);
    expect(fenceChunks[0]).toMatchObject({ language: 'typescript', symbol_name: 'hello', start_line: 1, end_line: 3, modality: 'text' });
    expect(fenceChunks[0].symbol_type).toMatch(/function|export/);
    expect(fenceChunks[0].chunk_text).toContain('fence.ts:1-3');
    expect(fenceChunks[0].chunk_text).toContain('return `Hello, ${name}`;');
    expect(chunks.map(c => c.chunk_index)).toEqual(chunks.map((_, i) => i));
    expect(chunks.at(-1)?.chunk_source).toBe('fenced_code');
  });

  test('Python fence → language=python, chunk_text contains the def', async () => {
    const md = `Docs.

\`\`\`python
def greet(name):
    return f"hi, {name}"
\`\`\`
`;
    await importFromContent(engine, 'guides/fence-py', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-py');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBeGreaterThan(0);
    expect(fenceChunks[0]!.language).toBe('python');
    expect(fenceChunks[0]!.chunk_text).toMatch(/def greet/);
  });

  test('Ruby fence → language=ruby', async () => {
    const md = `\`\`\`ruby
class Foo
  def bar; 42; end
end
\`\`\``;
    await importFromContent(engine, 'guides/fence-rb', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-rb');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBeGreaterThan(0);
    expect(fenceChunks[0]!.language).toBe('ruby');
  });

  test('unknown fence tag produces zero fenced_code chunks (graceful fallback)', async () => {
    const md = `Intro.

\`\`\`mermaid
graph TD
  A --> B
\`\`\`

\`\`\`unknown-lang-xyz
do stuff
\`\`\``;
    await importFromContent(engine, 'guides/fence-unknown', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-unknown');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    // No extraction — no chunks with fenced_code source. Prose still chunks normally.
    expect(fenceChunks.length).toBe(0);
  });

  test('missing fence language tag → no fenced_code chunks', async () => {
    const md = `Intro.

\`\`\`
some ambiguous code
\`\`\``;
    await importFromContent(engine, 'guides/fence-no-tag', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-no-tag');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBe(0);
  });

  test('multiple fences on one page all extract (under cap)', async () => {
    const md = `
\`\`\`ts
const a = 1;
\`\`\`

prose

\`\`\`python
x = 2
\`\`\`

\`\`\`bash
echo hi
\`\`\`
`;
    await importFromContent(engine, 'guides/fence-multi', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-multi');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    // Three fences, each produces at least one chunk. Languages vary.
    expect(fenceChunks.length).toBeGreaterThanOrEqual(3);
    const langs = new Set(fenceChunks.map(c => c.language));
    expect(langs.has('typescript')).toBe(true);
    expect(langs.has('python')).toBe(true);
    expect(langs.has('bash')).toBe(true);
  });

  test('empty fence body is skipped (no chunks)', async () => {
    const md = "Intro.\n\n```ts\n```\n";
    await importFromContent(engine, 'guides/fence-empty', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-empty');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBe(0);
  });

  // #2437 — extractFencedChunks skips marked.lexer entirely when the body has
  // no fence marker (the lexer transiently allocates ~60x the page size, which
  // OOMs the import worker under memory pressure). These two guard that the
  // fast-path neither breaks tilde fences nor lexes fence-less pages.
  test('tilde-fenced (~~~) code is still extracted after the no-fence fast-path (#2437)', async () => {
    const md = 'Docs.\n\n~~~ts\nexport const x = 1;\n~~~\n';
    await importFromContent(engine, 'guides/fence-tilde', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-tilde');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBeGreaterThan(0);
    expect(fenceChunks[0]!.language).toBe('typescript');
  });

  test('CR-only (\\r) line endings still extract a fenced chunk (#2437)', async () => {
    // marked normalizes \r → \n before lexing, so the fast-path's fence probe
    // must too; otherwise a classic-Mac line-ended page loses its fenced code.
    const md = 'intro\r```ts\rexport const x = 1;\r```\r';
    await importFromContent(engine, 'guides/fence-cr', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-cr');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBeGreaterThan(0);
    expect(fenceChunks[0]!.language).toBe('typescript');
  });

  // #2862 — a SINGLE fence marker used to route the whole body through
  // marked.lexer, which is quadratic on autolink-dense text under bun
  // (~50s for a 418KB body; OOM class on bigger ones). The linear line
  // scanner makes fence extraction O(lines) regardless of inline content.
  test('400KB autolink-dense body with one fence imports and still extracts it (#2862)', async () => {
    // ~420KB total — matches the issue's 418KB body and stays UNDER the
    // content-sanity bytes_block (500KB), which would otherwise embed-skip
    // the page and bypass fence extraction entirely.
    const line = '<https://example.com/very/long/path/segment?ticket=12345&session=abcdef>\n';
    const half = line.repeat(Math.ceil(210_000 / line.length));
    const md = '# Autolink flood\n\n' + half + '\n```ts\nexport const marker = 2862;\n```\n' + half;
    // The scanner itself must be linear-fast regardless of inline autolink
    // density: pre-fix, marked.lexer spent ~50s on this exact shape. 2s is
    // ~1000x headroom for a pure O(lines) pass even on a heavily loaded box
    // (an end-to-end import timing assertion proved too load-sensitive:
    // chunking + PGLite inserts dominate and vary wildly under parallel CI).
    // CPU time measures the scan's work rather than how long the scheduler made it wait.
    let fences!: ReturnType<typeof scanFencedBlocks>['fences'];
    const ms = cpuMs(() => { ({ fences } = scanFencedBlocks(md)); });
    expect(ms).toBeLessThan(2_000);
    expect(fences.length).toBe(1);
    // End-to-end: the fence still lands as a fenced_code chunk.
    await importFromContent(engine, 'guides/fence-autolink-flood', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-autolink-flood');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBeGreaterThan(0);
    expect(fenceChunks[0]!.language).toBe('typescript');
    expect(fenceChunks[0]!.chunk_text).toContain('marker = 2862');
  }, 120_000);

  test('import-file no longer routes fence extraction through marked (#2862)', async () => {
    // Structural pin: the quadratic-lexer class of bug can only come back if
    // someone re-imports marked into the import hot path.
    const { readFileSync } = await import('fs');
    for (const path of ['import-file.ts', 'markdown-chunks.ts']) {
      const src = readFileSync(new URL(`../src/core/${path}`, import.meta.url), 'utf8');
      expect(src).not.toMatch(/from ['"]marked['"]/);
    }
  });

  // #2862 — linear-scanner parity edges the marked walk used to handle.
  // Body fidelity is asserted on the scanner itself (the code chunker
  // downstream legitimately reshapes chunk text around parseable symbols).
  test('scanner: longer close fence closes a shorter opener; other-char stays literal (#2862)', () => {
    const md = 'intro\n\n```ts\nconst a = 1;\n~~~\nconst b = 2;\n`````\n\nprose\n';
    const { fences, capped } = scanFencedBlocks(md);
    expect(capped).toBe(false);
    expect(fences.length).toBe(1);
    expect(fences[0]!.lang).toBe('ts');
    // The ~~~ line is fence BODY (wrong char can't close a backtick fence);
    // the ````` line (5 >= 3 backticks) closes it.
    expect(fences[0]!.text).toBe('const a = 1;\n~~~\nconst b = 2;');
  });

  test('scanner: backtick opener with backtick in info string is inline code, not a fence (#2862)', () => {
    const md = '```inline `code` weirdness\nnot a fence body\n';
    const { fences } = scanFencedBlocks(md);
    expect(fences.length).toBe(0);
  });

  test('scanner: opener indentation is stripped from body lines (#2862)', () => {
    const md = '  ```ts\n  const x = 1;\n    deeper();\n  ```\n';
    const { fences } = scanFencedBlocks(md);
    expect(fences.length).toBe(1);
    expect(fences[0]!.text).toBe('const x = 1;\n  deeper();');
  });

  test('scanner: cap stops collection and reports capped (#2862)', () => {
    const md = Array.from({ length: 5 }, (_, i) => '```ts\nconst v' + i + ' = 1;\n```').join('\n\n');
    const { fences, capped } = scanFencedBlocks(md, 3);
    expect(capped).toBe(true);
    expect(fences.length).toBe(3);
  });

  test('unclosed fence runs to EOF (#2862)', async () => {
    const md = 'intro\n\n```python\ndef trailing():\n    return 1\n';
    await importFromContent(engine, 'guides/fence-unclosed', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-unclosed');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBeGreaterThan(0);
    expect(fenceChunks[0]!.language).toBe('python');
    expect(fenceChunks[0]!.chunk_text).toContain('def trailing');
  });

  test('lang is the first info-string word (```ts title=x) (#2862)', async () => {
    const md = '```ts title=example.ts\nexport const infoWord = true;\n```\n';
    await importFromContent(engine, 'guides/fence-info-word', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-info-word');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBeGreaterThan(0);
    expect(fenceChunks[0]!.language).toBe('typescript');
  });

  test('large fence-less table page imports with zero fenced chunks (no lexer pass) (#2437)', async () => {
    const cols = 16;
    const header = '| ' + Array.from({ length: cols }, (_, i) => 'col' + i).join(' | ') + ' |';
    const sep = '| ' + Array.from({ length: cols }, () => '---').join(' | ') + ' |';
    const body = Array.from({ length: 2000 }, (_, r) =>
      '| ' + Array.from({ length: cols }, (_, c) => 'v' + r + '_' + c).join(' | ') + ' |').join('\n');
    const md = '# Overview\n\n' + header + '\n' + sep + '\n' + body + '\n';
    // sanity: the page is genuinely fence-less, so the fast-path applies
    expect(/(^|[\r\n])[ \t]{0,3}(```|~~~)/.test(md)).toBe(false);
    await importFromContent(engine, 'guides/fence-less-table', md, { noEmbed: true });
    const chunks = await engine.getChunks('guides/fence-less-table');
    const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code');
    expect(fenceChunks.length).toBe(0);
  });
});