Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.2 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import {
  closesFence,
  decodeEntities,
  htmlToMarkdownInline,
  htmlToPlainText,
  imageTarget,
  inlineSpans,
  openingFence,
  quoteParagraphs,
  withoutClosingHashes,
} from '../inlineMarkup'

describe('inline HTML', () => {
  it('reads emphasis tags as their styles and <br> as a line break', () => {
    expect(inlineSpans('<b>a</b> <strong>b</strong> <i>c</i> <em>d</em> <del>e</del>')).toEqual([
      { text: 'a', bold: true },
      { text: ' ' },
      { text: 'b', bold: true },
      { text: ' ' },
      { text: 'c', italics: true },
      { text: ' ' },
      { text: 'd', italics: true },
      { text: ' ' },
      { text: 'e', strike: true },
    ])
    expect(htmlToMarkdownInline('one<br>two<BR/>three')).toBe('one\ntwo\nthree')
  })

  it('keeps links, turns img into an image reference and drops scripts and comments', () => {
    expect(htmlToMarkdownInline('<a href="https://x.test/a">site</a>')).toBe(
      '[site](https://x.test/a)'
    )
    expect(htmlToMarkdownInline('<img src="chart.png" alt="c">')).toBe('![](chart.png)')
    expect(htmlToMarkdownInline('a<script>alert(1)</script>b<!-- note -->')).toBe('ab')
  })

  it('leaves text that only looks like a tag, and HTML quoted in code', () => {
    expect(htmlToMarkdownInline('Vec<String> and a < b')).toBe('Vec<String> and a < b')
    expect(htmlToMarkdownInline('use `<br>` here')).toBe('use `<br>` here')
    expect(htmlToPlainText('if a<b and c>d or x<i then y>z')).toBe('if a<b and c>d or x<i then y>z')
  })

  it('reads a tag whose attributes have values, or are HTML boolean attributes', () => {
    expect(inlineSpans('<b class="x">a</b> <span data-n=1 title=\'t\'>b</span>')).toEqual([
      { text: 'a', bold: true },
      { text: ' b' },
    ])
    expect(htmlToMarkdownInline('<table><tr><td nowrap>1</td><td>2</td></tr></table>')).toBe('1 2')
    expect(inlineSpans('<b >a</b >')).toEqual([{ text: 'a', bold: true }])
  })

  it('keeps a bare element name that nothing closes as a placeholder', () => {
    expect(htmlToPlainText('kubectl logs --since=<time>')).toBe('kubectl logs --since=<time>')
    expect(htmlToPlainText('Replace <label> with your name')).toBe('Replace <label> with your name')
    expect(htmlToPlainText('Options <a> and <b> are shown')).toBe('Options <a> and <b> are shown')
    const spans = inlineSpans('Options <a> and <b> are shown')
    expect(spans.map(s => s.text).join('')).toBe('Options <a> and <b> are shown')
    expect(spans.some(s => s.bold)).toBe(false)
  })

  it('still reads a closed element, and one HTML lets stand unclosed, as markup', () => {
    expect(inlineSpans('a <b>bold</b> word')).toContainEqual({
      text: 'bold',
      bold: true,
    })
    expect(htmlToPlainText('<p>one<p>two')).toBe('one two')
    expect(htmlToPlainText('<ul><li>a<li>b</ul>')).toMatch(/a.*b/)
    expect(htmlToPlainText('line<hr>next')).not.toContain('<hr>')
  })

  it('keeps a placeholder such as <table name> that no closing tag makes a tag', () => {
    const sql = 'SELECT * FROM <table name> WHERE <column name> = 1 at <time of day>'
    expect(inlineSpans(sql)).toEqual([{ text: sql }])
    expect(htmlToPlainText(sql)).toBe(sql)
    expect(inlineSpans('A<table border><tr><td>x</td></tr></table>')).toEqual([{ text: 'A\nx' }])
  })

  it('drops a tag of a longer element name whatever its attributes', () => {
    expect(
      htmlToMarkdownInline(
        '<div data-role>inner</div><span aria-hidden>x</span><hr noshade><ol compact><li>one</li></ol>'
      )
    ).toBe('inner\nx\n1. one')
    expect(htmlToMarkdownInline('<span title="a>b">t</span> <a download>file</a>')).toBe('t file')
    expect(htmlToMarkdownInline('A<table border><tr><td>cell</td></tr></table>B')).toBe(
      'A\ncell\nB'
    )
  })

  it('breaks the line at block elements and drops their tags', () => {
    expect(htmlToMarkdownInline('<h2>Sales</h2><p>One</p><div>Two</div>')).toBe('Sales\nOne\nTwo')
    expect(
      htmlToMarkdownInline('<table><tr><th>A</th><th>B</th></tr><tr><td>1</td></tr></table>')
    ).toBe('A B\n1')
    expect(htmlToMarkdownInline('<details><summary>S</summary>D</details>')).toBe('S\nD')
  })

  it('keeps list items apart, numbered as their list is', () => {
    expect(htmlToMarkdownInline('<ul><li>a</li><li>b</li></ul><ol><li>c</li><li>d</li></ol>')).toBe(
      '\u2022 a\n\u2022 b\n1. c\n2. d'
    )
  })

  it('reduces a title to its text', () => {
    expect(htmlToPlainText('HTML <b>mixed</b> &amp; <i>title</i>')).toBe('HTML mixed & title')
    expect(htmlToPlainText('<h1>A</h1><p>B</p> <img src="x.png">')).toBe('A B')
  })
})

describe('entities', () => {
  it('decodes named and numeric references', () => {
    expect(decodeEntities('a &amp; b &lt;c&gt; &#65;&#x42; &nbsp;')).toBe('a & b <c> AB \u00A0')
  })

  it('leaves unknown and unrepresentable references as written', () => {
    expect(decodeEntities('&bogus; &#0; &#xD800;')).toBe('&bogus; &#0; &#xD800;')
  })

  it('prints an encoded marker instead of reading it as markup', () => {
    expect(inlineSpans('&#42;&#42;not bold&#42;&#42; &lt;b&gt;')).toEqual([
      { text: '**not bold** <b>' },
    ])
  })
})

describe('inline markdown', () => {
  it('reads ***text*** as bold italic, and nests emphasis', () => {
    expect(inlineSpans('A ***both*** B')).toEqual([
      { text: 'A ' },
      { text: 'both', bold: true, italics: true },
      { text: ' B' },
    ])
    expect(inlineSpans('~~old **bold**~~')).toEqual([
      { text: 'old ', strike: true },
      { text: 'bold', strike: true, bold: true },
    ])
  })

  it('nests emphasis written in HTML, in markdown or in both', () => {
    const noteImportant = [
      { text: 'Note: ', bold: true },
      { text: 'important', bold: true, italics: true },
    ]
    expect(inlineSpans('<b>Note: <i>important</i></b>')).toEqual(noteImportant)
    expect(inlineSpans('**Note: *important***')).toEqual(noteImportant)
    expect(inlineSpans('<b>Note: *important*</b>')).toEqual(noteImportant)
    const boldThenItalic = [
      { text: 'x', bold: true },
      { text: 'y', italics: true },
    ]
    expect(inlineSpans('<b>x</b><i>y</i>')).toEqual(boldThenItalic)
    expect(inlineSpans('**x***y*')).toEqual(boldThenItalic)
    const italicAroundBold = [
      { text: 'a ', italics: true },
      { text: 'b', italics: true, bold: true },
      { text: ' c', italics: true },
    ]
    expect(inlineSpans('<i>a <b>b</b> c</i>')).toEqual(italicAroundBold)
    expect(inlineSpans('*a **b** c*')).toEqual(italicAroundBold)
  })

  it('applies an HTML emphasis tag beside punctuation, where a markdown marker would not open', () => {
    expect(inlineSpans('Price<b>$5</b>, total<i>(est.)</i>')).toEqual([
      { text: 'Price' },
      { text: '$5', bold: true },
      { text: ', total' },
      { text: '(est.)', italics: true },
    ])
    expect(inlineSpans('**a</b> b**')).toEqual([{ text: 'a b', bold: true }])
  })

  it('prints underscores as written, so names such as __init__ keep them', () => {
    const text = 'the __init__ method, _private and snake_case_name'
    expect(inlineSpans(text)).toEqual([{ text }])
  })

  it('prints a noncharacter the input holds as nothing, not as a style', () => {
    expect(inlineSpans('a\uFDD0b\uFDD1c')).toEqual([{ text: 'abc' }])
  })

  it('prints an escaped marker as itself', () => {
    expect(
      inlineSpans('\\*not italic\\*')
        .map(s => s.text)
        .join('')
    ).toBe('*not italic*')
    expect(inlineSpans('\\*not italic\\*').some(s => s.italics)).toBe(false)
  })

  it('keeps multiplication signs that have spaces around them', () => {
    expect(inlineSpans('Math 5 * 3 * 2 = 30')).toEqual([{ text: 'Math 5 * 3 * 2 = 30' }])
  })

  it('keeps brackets in the label of an HTML link', () => {
    expect(inlineSpans('Source <a href="https://x.com/a">[1]</a> end')).toEqual([
      { text: 'Source ' },
      { text: '[1]', link: 'https://x.com/a' },
      { text: ' end' },
    ])
    expect(inlineSpans('[a \\] b](https://x.com/b)')).toEqual([
      { text: 'a ] b', link: 'https://x.com/b' },
    ])
  })

  it('prints backslashes in the label of an HTML link as the text around it prints them', () => {
    const labels = [
      'C:\\Users\\',
      'a\\',
      '\\',
      'a \\[1\\] b',
      '[\\]',
      'a\\\\]',
      '\\*x\\*',
      'path\\to',
      '&#92;[1]',
      '<span>C:\\Users\\</span>',
      'a\\<u>]</u> b',
      'x\\<b>\\</b>',
    ]
    for (const label of labels) {
      const outside = inlineSpans(label).map(s => ({ ...s, link: 'https://x.com/a' }))
      expect(inlineSpans(`<a href="https://x.com/a">${label}</a> end`), label).toEqual([
        ...outside,
        { text: ' end' },
      ])
    }
    expect(htmlToMarkdownInline('<a href="https://x.test/a">C:\\dir\\</a>')).toBe(
      '[C:\\dir\\\\](https://x.test/a)'
    )
    expect(htmlToMarkdownInline('<a href="https://x.test/a">a \\[1\\] b</a>')).toBe(
      '[a \\[1\\] b](https://x.test/a)'
    )
    expect(htmlToMarkdownInline('<a href="https://x.test/a">[1] a]b</a>')).toBe(
      '[\\[1\\] a\\]b](https://x.test/a)'
    )
  })

  it('ends a link label at its first unescaped bracket, so an escaped "](" names no target', () => {
    expect(inlineSpans('[a \\](b](https://x.com/a)')).toEqual([
      { text: 'a ](b', link: 'https://x.com/a' },
    ])
    expect(inlineSpans('[a \\](file:///etc/passwd](https://x.com/a)')).toEqual([
      { text: 'a ](file:///etc/passwd', link: 'https://x.com/a' },
    ])
    expect(inlineSpans('<a href="https://x.com/a">a](javascript:alert(1)</a>')).toEqual([
      { text: 'a](javascript:alert(1)', link: 'https://x.com/a' },
    ])
    expect(inlineSpans('<a href="https://x.com/a">![i](p.png) \\[1\\]</a>')).toEqual([
      { text: '![i](p.png) [1]', link: 'https://x.com/a' },
    ])
  })

  it('reads the label of a link as inline markdown, as CommonMark does', () => {
    const link = 'https://x.test'
    expect(inlineSpans(`[**Docs**](${link})`)).toEqual([{ text: 'Docs', bold: true, link }])
    expect(inlineSpans(`[a *b* \`c\`](${link})`)).toEqual([
      { text: 'a ', link },
      { text: 'b', italics: true, link },
      { text: ' ', link },
      { text: 'c', code: true, link },
    ])
    expect(inlineSpans(`<a href="${link}"><b>Docs</b> page</a>`)).toEqual([
      { text: 'Docs', bold: true, link },
      { text: ' page', link },
    ])
  })

  it('reads emphasis in a link label with the brackets beside it, as CommonMark does', () => {
    const link = 'https://x.test'
    // The closing * is followed by "]", so the rule of three keeps it apart from "**".
    expect(inlineSpans(`[**\`b*](${link})`)).toEqual([{ text: '**`b*', link }])
    // Emphasis does not cross the edge of a link.
    expect(inlineSpans(`[**a](${link})**`)).toEqual([{ text: '**a', link }, { text: '**' }])
    expect(inlineSpans(`**[a](${link})**`)).toEqual([{ text: 'a', bold: true, link }])
  })

  it('prints an escaped bracket in a link label as the bracket, as CommonMark does', () => {
    expect(inlineSpans('[a\\]](https://x.test)')).toEqual([{ text: 'a]', link: 'https://x.test' }])
  })

  it('keeps balanced parentheses in a link target', () => {
    expect(inlineSpans('[wiki](https://en.wikipedia.org/wiki/Foo_(bar)) end')).toEqual([
      { text: 'wiki', link: 'https://en.wikipedia.org/wiki/Foo_(bar)' },
      { text: ' end' },
    ])
  })

  it('names an image written in the text, with its alt text', () => {
    expect(inlineSpans('see ![Revenue](sales.png "t") now')).toEqual([
      { text: 'see ' },
      { text: 'Revenue', image: 'sales.png' },
      { text: ' now' },
    ])
    expect(inlineSpans('`![x](y.png)`')).toEqual([{ text: '![x](y.png)', code: true }])
  })
})

describe('code spans', () => {
  it('closes a code span with a run of as many backticks as opened it', () => {
    expect(inlineSpans('``a`b``')).toEqual([{ text: 'a`b', code: true }])
    expect(inlineSpans('`a``b`')).toEqual([{ text: 'a``b', code: true }])
    expect(inlineSpans('x ```a``b``` y')).toEqual([
      { text: 'x ' },
      { text: 'a``b', code: true },
      { text: ' y' },
    ])
  })

  it('drops one space on each side, so a span can show a backtick at its edge', () => {
    expect(inlineSpans('`` `x` ``')).toEqual([{ text: '`x`', code: true }])
    expect(inlineSpans('`  `')).toEqual([{ text: '  ', code: true }])
  })

  it('prints a run nothing closes, and an escaped backtick, as written', () => {
    expect(inlineSpans('``a`')).toEqual([{ text: '``a`' }])
    expect(inlineSpans('\\`a`')).toEqual([{ text: '`a`' }])
    expect(inlineSpans('```')).toEqual([{ text: '```' }])
  })

  it('reads no markup inside a code span', () => {
    expect(inlineSpans('``**a** `b` <b>c</b>``')).toEqual([
      { text: '**a** `b` <b>c</b>', code: true },
    ])
  })
})

describe('ATX headings', () => {
  it('drops a closing run of hashes only when a space precedes it', () => {
    expect(withoutClosingHashes('Section title ##')).toBe('Section title')
    expect(withoutClosingHashes('C#')).toBe('C#')
    expect(withoutClosingHashes('Issue #42')).toBe('Issue #42')
    expect(withoutClosingHashes('##')).toBe('')
  })
})

describe('image targets', () => {
  it('reads the file an image names, in each form Markdown allows', () => {
    expect(imageTarget('chart.png')).toBe('chart.png')
    expect(imageTarget(' <my chart.png> ')).toBe('my chart.png')
    expect(imageTarget('<my chart.png> "Sales"')).toBe('my chart.png')
    expect(imageTarget('chart.png "Quarterly sales"')).toBe('chart.png')
    expect(imageTarget("chart.png 'Sales'")).toBe('chart.png')
    expect(imageTarget('chart.png (Sales)')).toBe('chart.png')
    expect(imageTarget('my\\_chart.png')).toBe('my_chart.png')
    expect(imageTarget('a&amp;b.png')).toBe('a&b.png')
  })

  it('reads a name with spaces written without angle brackets whole', () => {
    expect(imageTarget('my chart.png')).toBe('my chart.png')
    expect(imageTarget('image (1).png')).toBe('image (1).png')
    expect(imageTarget('image (1).png "Costs"')).toBe('image (1).png')
    expect(imageTarget('my chart.png (Sales)')).toBe('my chart.png')
  })

  it('finds an image whose name holds a parenthesized part', () => {
    const [span] = inlineSpans('![Sales](image (1).png)')
    expect(span).toEqual({ text: 'Sales', image: 'image (1).png' })
  })

  it('reads a title as the one-pattern reader it replaced did, without its backtracking', () => {
    const titled = /^(.*?)\s+(?:"[^"\n]*"|'[^'\n]*'|\([^()\n]*\))$/
    const angled = /^<([^<>\n]*)>/
    // No backslash or ampersand, so unescaping and entity decoding leave the name as read.
    const alphabet = ['a', 'b', '.', ' ', '\t', '\n', '\r', '\u2028', '"', "'", '(', ')', '<', '>']
    let seed = 1
    const next = () => (seed = (seed * 1103515245 + 12345) % 2147483648)
    for (let i = 0; i < 20000; i++) {
      let input = ''
      for (let n = next() % 12; n > 0; n--) input += alphabet[next() % alphabet.length]
      const t = input.trim()
      const expected = angled.exec(t)?.[1] ?? titled.exec(t)?.[1] ?? t
      expect(imageTarget(input), JSON.stringify(input)).toBe(expected)
    }
  })

  it('reads a target with a long run of spaces in linear time', () => {
    const spaces = ' '.repeat(100000)
    for (const inside of [`a${spaces}b`, `a${spaces}"title"`, `a${spaces}(b`, `${spaces}x`]) {
      const started = performance.now()
      imageTarget(inside)
      // Quadratic work on these takes minutes; the bound leaves room for a loaded machine.
      expect(performance.now() - started).toBeLessThan(2000)
    }
  })
})

describe('block quotes', () => {
  it('ends a paragraph at an empty quoted line and runs the others on', () => {
    expect(quoteParagraphs(['One', 'runs on.', '', '  ', 'Two.', ''])).toEqual([
      'One runs on.',
      'Two.',
    ])
    expect(quoteParagraphs(['', ''])).toEqual([''])
  })
})

describe('code fences', () => {
  it('takes the whole run of backticks or tildes as the fence', () => {
    expect(openingFence('````markdown')).toEqual({ marker: '````', language: 'markdown' })
    expect(openingFence('  ~~~ js ')).toEqual({ marker: '~~~', language: 'js' })
    expect(openingFence('``x')).toBeUndefined()
  })

  it('closes a fence only on a run of its character at least as long', () => {
    expect(closesFence('```', '````')).toBe(false)
    expect(closesFence('~~~~', '```')).toBe(false)
    expect(closesFence('  `````', '````')).toBe(true)
  })

  it('keeps a fence open on a line that adds text after the run, as CommonMark does', () => {
    expect(closesFence('```js', '```')).toBe(false)
    expect(closesFence('``` x', '```')).toBe(false)
    expect(closesFence('```  \t', '```')).toBe(true)
    expect(closesFence('```\r', '```')).toBe(true)
  })

  it('reads a long run of backticks in linear time', () => {
    const line = `${'`'.repeat(200000)}\r`
    const started = performance.now()
    openingFence(line)
    closesFence(line, '```')
    expect(performance.now() - started).toBeLessThan(2000)
  })
})

describe('hostile input', () => {
  // Each input is about 200 KB on one line: an unclosed construct repeated, so
  // a pattern that rescans the rest of the line from every opening is quadratic.
  const inputs: Record<string, string> = {
    img: "<img src='x".repeat(20000),
    tag: '<b'.repeat(100000),
    tagSpaced: '<b '.repeat(66667),
    tagAttributes: '<b a=1'.repeat(30000),
    tagAttributesOpen: `<b ${'a=1 '.repeat(50000)}`,
    tagBoolean: `<td ${'nowrap '.repeat(28000)}`,
    tagLoose: '<div "'.repeat(33000),
    tagLooseQuote: '<div title="a>'.repeat(14000),
    tagLooseOpen: `<div ${'a="1" \'2\' '.repeat(20000)}`,
    placeholders: '<table name '.repeat(20000),
    placeholdersClosed: `${'<table name>'.repeat(15000)}</table>`,
    comment: '<!--'.repeat(50000),
    script: '<script>'.repeat(25000),
    anchor: '<a href="https://x.test">'.repeat(8000),
    anchorAttributes: `<a ${'href="http://x" '.repeat(12000)}>label`,
    list: '<ol><li>'.repeat(25000),
    image: '!['.repeat(100000),
    imageSpaced: '![ '.repeat(66667),
    imageOpen: '![a]('.repeat(40000),
    imageGroups: `![a](${'(x)'.repeat(60000)}`,
    imageGroupsRepeated: `![a](${'(x)'.repeat(5)}`.repeat(10000),
    imageTargetSpaces: `![a](a${' '.repeat(100000)}b)`,
    imageTitleSpaces: `![a](a${' '.repeat(100000)}"t")`,
    link: '[a](http://x'.repeat(16000),
    linkEscapes: `[${'\\]'.repeat(100000)}`,
    bracket: '['.repeat(200000),
    bold: '**a '.repeat(50000),
    boldItalic: '***a '.repeat(40000),
    italic: '*a '.repeat(66667),
    strike: '~~a '.repeat(50000),
    nested: '*a **a '.repeat(30000),
    nestedUnderscore: '_a __a '.repeat(30000),
    closers: 'a* '.repeat(66667),
    bothWays: 'a**b*'.repeat(40000),
    mixedRuns: '*_'.repeat(100000),
    atomsThenMarker: `${'`a` '.repeat(50000)}*`,
    tagMarks: '<b><i>x'.repeat(28000),
    code: '`a '.repeat(66667),
    codeDouble: '``a '.repeat(50000),
    codeMixed: '`a`` '.repeat(40000),
    backticks: '`'.repeat(200000),
    codeThenLink: `${'`a` '.repeat(50000)}[x](http://x)`,
    escape: '\\*'.repeat(100000),
    entity: '&#x1'.repeat(50000),
  }
  const parsers: Record<string, (text: string) => unknown> = {
    htmlToMarkdownInline,
    htmlToPlainText,
    inlineSpans,
  }

  for (const [name, input] of Object.entries(inputs)) {
    it(`reads ${name} in linear time`, () => {
      for (const [parser, parse] of Object.entries(parsers)) {
        const started = performance.now()
        parse(input)
        expect(performance.now() - started, `${parser} on ${name}`).toBeLessThan(2000)
      }
    })
  }
})