Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.0 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { createCanvas, loadImage } from '@napi-rs/canvas'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import { fitImageSize, loadEmbeddableImage, predecodeImages } from '../embeddedImages'
import { INTERNAL_TOOLS } from '../internalTools'

let outputDir: string

beforeEach(() => {
  outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'clerum-embed-'))
})

afterEach(() => {
  fs.rmSync(outputDir, { recursive: true, force: true })
})

async function writeImage(name: string, format: 'png' | 'jpeg' | 'webp', w = 40, h = 20) {
  const canvas = createCanvas(w, h)
  canvas.getContext('2d').fillRect(0, 0, w, h)
  const data = format === 'png' ? canvas.toBuffer('image/png') : await canvas.encode(format)
  fs.writeFileSync(path.join(outputDir, name), data)
}

/** A JPEG with an APP1 Exif segment carrying only the orientation tag. */
function withExifOrientation(jpeg: Buffer, orientation: number): Buffer {
  const tiff = Buffer.from([
    0x4d,
    0x4d,
    0x00,
    0x2a,
    0x00,
    0x00,
    0x00,
    0x08,
    0x00,
    0x01,
    0x01,
    0x12,
    0x00,
    0x03,
    0x00,
    0x00,
    0x00,
    0x01,
    0x00,
    orientation,
    0x00,
    0x00,
    0x00,
    0x00,
    0x00,
    0x00,
  ])
  const body = Buffer.concat([Buffer.from('Exif\0\0', 'latin1'), tiff])
  const header = Buffer.from([0xff, 0xe1, (body.length + 2) >> 8, (body.length + 2) & 0xff])
  return Buffer.concat([jpeg.subarray(0, 2), header, body, jpeg.subarray(2)])
}

async function opaquePixels(png: Buffer): Promise<number> {
  const image = await loadImage(png)
  const canvas = createCanvas(image.width, image.height)
  const ctx = canvas.getContext('2d')
  ctx.drawImage(image, 0, 0)
  const data = ctx.getImageData(0, 0, image.width, image.height).data
  let count = 0
  for (let i = 3; i < data.length; i += 4) if (data[i] > 0) count++
  return count
}

describe('fitImageSize', () => {
  const box = { width: 500, height: 300 }

  it('keeps proportions and never exceeds the box, even for an explicit size', () => {
    expect(fitImageSize({ width: 1000, height: 500 }, box)).toEqual({ width: 500, height: 250 })
    expect(fitImageSize({ width: 1000, height: 500 }, box, { width: 2000 })).toEqual({
      width: 500,
      height: 250,
    })
    expect(fitImageSize({ width: 100, height: 100 }, box, { height: 900 })).toEqual({
      width: 300,
      height: 300,
    })
  })

  it('reads a width and height given together as a box to fit in', () => {
    expect(fitImageSize({ width: 100, height: 100 }, box, { width: 200, height: 100 })).toEqual({
      width: 100,
      height: 100,
    })
    expect(fitImageSize({ width: 200, height: 100 }, box, { width: 400, height: 400 })).toEqual({
      width: 400,
      height: 200,
    })
    expect(fitImageSize({ width: 100, height: 100 }, box, { width: 1000, height: 1000 })).toEqual({
      width: 300,
      height: 300,
    })
  })
})

describe('loadEmbeddableImage', () => {
  it('accepts a bare filename or an object with path', async () => {
    await writeImage('a.png', 'png')
    const warnings: string[] = []
    expect(loadEmbeddableImage('a.png', outputDir, warnings)).toMatchObject({
      format: 'png',
      width: 40,
      height: 20,
    })
    expect(loadEmbeddableImage({ path: 'a.png' }, outputDir, warnings)?.format).toBe('png')
    expect(warnings).toEqual([])
  })

  it('reads the format from the bytes and converts decoded images to opaque PNG', async () => {
    await writeImage('photo.png', 'jpeg')
    await writeImage('anim.webp', 'webp')
    fs.writeFileSync(
      path.join(outputDir, 'logo.svg'),
      '<svg xmlns="http://www.w3.org/2000/svg" width="30" height="10"><rect width="30" height="10"/></svg>'
    )
    await predecodeImages({ images: ['anim.webp', { path: 'logo.svg' }] }, outputDir)
    const warnings: string[] = []
    expect(loadEmbeddableImage('photo.png', outputDir, warnings)?.format).toBe('jpeg')
    const webp = loadEmbeddableImage('anim.webp', outputDir, warnings)
    expect(webp?.format).toBe('png')
    expect(await opaquePixels(webp!.data)).toBe(40 * 20)
    const svg = loadEmbeddableImage('logo.svg', outputDir, warnings)
    expect(svg).toMatchObject({ format: 'png', width: 30, height: 10 })
    expect(await opaquePixels(svg!.data)).toBe(30 * 10)
    expect(warnings).toEqual([])
  })

  it('finds images named inside markdown bodies', async () => {
    await writeImage('inline.webp', 'webp')
    await predecodeImages({ body: 'Intro\n\n![sales](inline.webp)\n' }, outputDir)
    const image = loadEmbeddableImage('inline.webp', outputDir, [])
    expect(await opaquePixels(image!.data)).toBe(40 * 20)
  })

  it('finds a body image whose name holds spaces, in each form markdown writes it', async () => {
    await writeImage('my chart.webp', 'webp')
    await writeImage('image (1).webp', 'webp')
    await predecodeImages(
      { body: '![a](<my chart.webp>)\n\n![b](image (1).webp "Title")' },
      outputDir
    )
    expect(loadEmbeddableImage('my chart.webp', outputDir, [])).toBeDefined()
    expect(loadEmbeddableImage('image (1).webp', outputDir, [])).toBeDefined()
  })

  it('scans a body of unclosed image openings in linear time', async () => {
    const started = performance.now()
    await predecodeImages({ body: '!['.repeat(100000) }, outputDir)
    expect(performance.now() - started).toBeLessThan(5000)
  })

  it('reports an image it was not able to decode instead of drawing it blank', async () => {
    await writeImage('late.webp', 'webp')
    const warnings: string[] = []
    expect(loadEmbeddableImage('late.webp', outputDir, warnings)).toBeUndefined()
    expect(warnings).toEqual(["'late.webp' is not an image this tool can read and was left out."])
  })

  it('rasterizes a huge SVG at a bounded size instead of its declared one', async () => {
    // The canvas library aborts the process when it cannot allocate the bitmap.
    fs.writeFileSync(
      path.join(outputDir, 'huge.svg'),
      '<svg xmlns="http://www.w3.org/2000/svg" width="32000" height="16000"><rect width="32000" height="16000"/></svg>'
    )
    fs.writeFileSync(
      path.join(outputDir, 'inches.svg'),
      '<svg xmlns="http://www.w3.org/2000/svg" width="1000in" height="1000in" viewBox="0 0 10 10"><rect width="10" height="10"/></svg>'
    )
    await predecodeImages(['huge.svg', 'inches.svg'], outputDir)
    expect(loadEmbeddableImage('huge.svg', outputDir, [])).toMatchObject({
      width: 2048,
      height: 1024,
    })
    expect(loadEmbeddableImage('inches.svg', outputDir, [])).toMatchObject({
      width: 2048,
      height: 2048,
    })
  })

  it('bounds an SVG size written with an exponent and refuses one it cannot read', async () => {
    const svg = (attrs: string) =>
      `<svg xmlns="http://www.w3.org/2000/svg" ${attrs}><rect width="10" height="10"/></svg>`
    fs.writeFileSync(path.join(outputDir, 'exp.svg'), svg('width="1e9" height="5e8"'))
    fs.writeFileSync(path.join(outputDir, 'calc.svg'), svg('width="calc(1e9px)" height="10"'))
    fs.writeFileSync(
      path.join(outputDir, 'pct.svg'),
      svg('width="100%" height="100%" viewBox="0 0 50 40"')
    )
    await predecodeImages(['exp.svg', 'calc.svg', 'pct.svg'], outputDir)
    expect(loadEmbeddableImage('exp.svg', outputDir, [])).toMatchObject({
      width: 2048,
      height: 1024,
    })
    const warnings: string[] = []
    expect(loadEmbeddableImage('calc.svg', outputDir, warnings)).toBeUndefined()
    expect(warnings[0]).toMatch(/calc\.svg' is not an image this tool can read/)
    expect(loadEmbeddableImage('pct.svg', outputDir, [])).toMatchObject({ width: 50, height: 40 })
  })

  it('reads an SVG after the declaration, comments and doctype an XML file may open with', async () => {
    const root =
      '<svg xmlns="http://www.w3.org/2000/svg" width="12" height="8"><rect width="12" height="8"/></svg>'
    const files: Record<string, string> = {
      'decl.svg': `<?xml version="1.0" encoding="UTF-8"?>\n${root}`,
      'comments.svg': `\n  <!-- Generator: x -->\n<!---->\t<!-- <svg> named in a comment -->${root}`,
      'doctype.svg':
        '<?xml version="1.0" standalone="no"?>\n<!-- a -->\n' +
        '<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" ' +
        `"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">\n${root}`,
    }
    for (const [name, text] of Object.entries(files)) {
      fs.writeFileSync(path.join(outputDir, name), text)
    }
    await predecodeImages(Object.keys(files), outputDir)
    for (const name of Object.keys(files)) {
      expect(loadEmbeddableImage(name, outputDir, []), name).toMatchObject({
        format: 'png',
        width: 12,
        height: 8,
      })
    }
  })

  it('does not read as SVG a prolog that never reaches the <svg> root', async () => {
    const root =
      '<svg xmlns="http://www.w3.org/2000/svg" width="12" height="8"><rect width="12" height="8"/></svg>'
    const files: Record<string, string> = {
      'unclosed.svg': `<!-- never closed ${root}`,
      'html.svg': `<!-- a --><html>${root}</html>`,
      'stray.svg': `<!-- a --> text --> ${root}`,
    }
    for (const [name, text] of Object.entries(files)) {
      fs.writeFileSync(path.join(outputDir, name), text)
    }
    await predecodeImages(Object.keys(files), outputDir)
    for (const name of Object.keys(files)) {
      const warnings: string[] = []
      expect(loadEmbeddableImage(name, outputDir, warnings), name).toBeUndefined()
      expect(warnings).toEqual([`'${name}' is not an image this tool can read and was left out.`])
    }
  })

  it('reads a prolog of back-to-back comments with the last one open in linear time', async () => {
    // Every "--><!--" could end one comment or sit inside a longer one, so a
    // pattern that let a comment run past its first "-->" tried each way.
    fs.writeFileSync(path.join(outputDir, 'comments.svg'), `<!--${'--><!--'.repeat(30)}`)
    const started = performance.now()
    await predecodeImages(['comments.svg'], outputDir)
    expect(performance.now() - started).toBeLessThan(2000)
    expect(loadEmbeddableImage('comments.svg', outputDir, [])).toBeUndefined()
  })

  it('does not decode an image whose header declares an enormous size', async () => {
    const gif = Buffer.from(
      'R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==',
      'base64'
    )
    gif.writeUInt16LE(60000, 6)
    gif.writeUInt16LE(60000, 8)
    fs.writeFileSync(path.join(outputDir, 'huge.gif'), gif)
    await predecodeImages(['huge.gif'], outputDir)
    expect(loadEmbeddableImage('huge.gif', outputDir, [])).toBeUndefined()
  })

  it('leaves out, with a warning, what is missing or not an image', () => {
    fs.writeFileSync(path.join(outputDir, 'notes.png'), 'plain text')
    const warnings: string[] = []
    expect(loadEmbeddableImage('missing.png', outputDir, warnings)).toBeUndefined()
    expect(loadEmbeddableImage('notes.png', outputDir, warnings)).toBeUndefined()
    expect(loadEmbeddableImage({ width: 100 }, outputDir, warnings, 'images[2]')).toBeUndefined()
    expect(warnings).toEqual([
      "Image 'missing.png' was not found in the output folder and was left out.",
      "'notes.png' is not an image this tool can read and was left out.",
      'images[2] needs a file name, such as the one clerum__generate_chart returns; it was left out.',
    ])
  })

  it('uses the file of the same name when given an absolute path from elsewhere', async () => {
    await writeImage('chart.png', 'png')
    const warnings: string[] = []
    const image = loadEmbeddableImage('/output/chart.png', outputDir, warnings)
    expect(image?.path).toBe(path.join(path.resolve(outputDir), 'chart.png'))
    expect(warnings[0]).toContain("'/output/chart.png' is outside the output folder")
  })

  it('refuses a link inside the output folder that points outside it', async () => {
    const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'clerum-outside-'))
    try {
      const canvas = createCanvas(4, 4)
      canvas.getContext('2d').fillRect(0, 0, 4, 4)
      fs.writeFileSync(path.join(outside, 'secret.png'), canvas.toBuffer('image/png'))
      fs.symlinkSync(path.join(outside, 'secret.png'), path.join(outputDir, 'innocent.png'))
      expect(() => loadEmbeddableImage('innocent.png', outputDir, [])).toThrow(
        /path traversal blocked/
      )
    } finally {
      fs.rmSync(outside, { recursive: true, force: true })
    }
  })

  it('says a web address is not downloaded', () => {
    const warnings: string[] = []
    expect(
      loadEmbeddableImage('https://example.com/chart.png', outputDir, warnings, 'images[0]')
    ).toBeUndefined()
    expect(warnings[0]).toContain('is a web address or inline data, and images are not downloaded')
  })

  it('turns a photo tagged with an EXIF rotation upright for every format', async () => {
    // 40x20 with the left quarter blue, tagged "rotate 90° clockwise".
    const canvas = createCanvas(40, 20)
    const ctx = canvas.getContext('2d')
    ctx.fillStyle = '#ff0000'
    ctx.fillRect(0, 0, 40, 20)
    ctx.fillStyle = '#0000ff'
    ctx.fillRect(0, 0, 10, 20)
    fs.writeFileSync(
      path.join(outputDir, 'photo.jpg'),
      withExifOrientation(canvas.toBuffer('image/jpeg'), 6)
    )
    await predecodeImages(['photo.jpg'], outputDir)
    const image = loadEmbeddableImage('photo.jpg', outputDir, [])!
    expect(image).toMatchObject({ format: 'jpeg', width: 20, height: 40 })
    const decoded = await loadImage(image.data)
    const probe = createCanvas(decoded.width, decoded.height)
    probe.getContext('2d').drawImage(decoded, 0, 0)
    const [r, , b] = probe.getContext('2d').getImageData(10, 2, 1, 1).data
    expect(b).toBeGreaterThan(200)
    expect(r).toBeLessThan(60)
  })

  it('still refuses to leave the output folder', () => {
    expect(() => loadEmbeddableImage('../../etc/passwd', outputDir, [])).toThrow(
      /path traversal blocked/
    )
    expect(() => loadEmbeddableImage('/etc/passwd', outputDir, [])).toThrow(
      /path traversal blocked/
    )
  })
})

describe('images decoded for a call', () => {
  /** MAX_DECODED_BYTES: what the cache keeps once no call holds its images. */
  const BUDGET = 64 * 1024 * 1024

  /**
   * WebPs of noise named `names`, each about 0.5 MB on disk and about
   * 3.4 MB decoded, so 21 of them decode to more than the budget.
   */
  async function writeNoise(names: string[]) {
    const canvas = createCanvas(1000, 1000)
    const ctx = canvas.getContext('2d')
    const image = ctx.createImageData(1000, 1000)
    // xorshift32, so every run decodes to the same size.
    let s = 2463534242
    for (let i = 0; i < image.data.length; i++) {
      s ^= s << 13
      s ^= s >>> 17
      s ^= s << 5
      image.data[i] = i % 4 === 3 ? 255 : s & 255
    }
    ctx.putImageData(image, 0, 0)
    const webp = await canvas.encode('webp', 50)
    for (const name of names) fs.writeFileSync(path.join(outputDir, name), webp)
  }

  /** Decoded bytes the cache still holds for `names`. */
  function kept(names: string[]): number {
    return names.reduce(
      (sum, name) => sum + (loadEmbeddableImage(name, outputDir, [])?.data.length ?? 0),
      0
    )
  }

  const named = (prefix: string, count: number) =>
    Array.from({ length: count }, (_, i) => `${prefix}-${i}.webp`)

  it('stay readable while another call decodes more than the cache holds', async () => {
    await writeImage('held.webp', 'webp')
    const releaseHeld = await predecodeImages(['held.webp'], outputDir)
    const others = named('noise', 21)
    await writeNoise(others)
    const releaseOthers = await predecodeImages(others, outputDir)
    expect(kept(others)).toBeGreaterThan(BUDGET)
    releaseOthers()

    const warnings: string[] = []
    expect(loadEmbeddableImage('held.webp', outputDir, warnings)).toBeDefined()
    expect(warnings).toEqual([])
    releaseHeld()
  }, 60_000)

  it('fit the budget again once the tool call that decoded them ends', async () => {
    const images = named('photo', 21)
    await writeNoise(images)
    const markdown = INTERNAL_TOOLS.find(t => t.name === 'clerum__generate_markdown')!
    const result = await markdown.execute(
      { filename: 'photos.md', content: images.map(name => `![photo](${name})`).join('\n') },
      outputDir
    )
    expect(result.success, result.error).toBe(true)

    const left = kept(images)
    expect(left).toBeGreaterThan(0)
    expect(left).toBeLessThanOrEqual(BUDGET)
    // The oldest go first.
    expect(loadEmbeddableImage(images[0], outputDir, [])).toBeUndefined()
    expect(loadEmbeddableImage(images[20], outputDir, [])).toBeDefined()
  }, 60_000)

  it.each(['first', 'second'])(
    'keep what a running call holds when the %s of two calls ends',
    async ending => {
      const first = named('first', 11)
      const second = named('second', 11)
      await writeNoise([...first, ...second])
      const releaseFirst = await predecodeImages(first, outputDir)
      const releaseSecond = await predecodeImages(second, outputDir)
      const size = kept(first.slice(0, 1))
      expect(kept([...first, ...second])).toBe(22 * size)
      expect(22 * size).toBeGreaterThan(BUDGET)

      const [releaseEnding, releaseRunning] =
        ending === 'first' ? [releaseFirst, releaseSecond] : [releaseSecond, releaseFirst]
      const running = ending === 'first' ? second : first
      releaseEnding()
      expect(kept(running)).toBe(11 * size)
      expect(kept([...first, ...second])).toBeLessThanOrEqual(BUDGET)

      releaseRunning()
      releaseEnding()
      expect(kept([...first, ...second])).toBeLessThanOrEqual(BUDGET)
    },
    60_000
  )

  it('stay held by one call after another releases them twice', async () => {
    const shared = named('shared', 1)
    const others = named('other', 21)
    await writeNoise([...shared, ...others])
    const releaseOne = await predecodeImages(shared, outputDir)
    const releaseTwo = await predecodeImages(shared, outputDir)
    releaseOne()
    releaseOne()

    // The shared image is the oldest, so it would be the first to go.
    const releaseOthers = await predecodeImages(others, outputDir)
    releaseOthers()
    expect(kept(shared)).toBeGreaterThan(0)
    expect(kept([...shared, ...others])).toBeLessThanOrEqual(BUDGET)

    releaseTwo()
    expect(kept([...shared, ...others])).toBeLessThanOrEqual(BUDGET)
  }, 60_000)
})