Skip to content

An AI-powered search engine with a generative UI

4.4 KB
Raw Blame
/**
 * E2E test: verify that an LLM actually emits well-formed inline image
 * spec blocks when given the researcher system prompt and a mocked search
 * tool returning image results.
 *
 * This test hits a real LLM API and is therefore gated behind the
 * `RUN_LLM_E2E=1` environment variable. CI runs without the flag and the
 * test is skipped. To run locally:
 *
 *   RUN_LLM_E2E=1 OPENAI_API_KEY=sk-... bun run test llm-image-output.e2e
 */
import { generateText, isStepCount, tool } from 'ai'
import { describe, expect, test } from 'vitest'
import { z } from 'zod'

import { getAdaptiveModePrompt } from '@/lib/agents/prompts/search-mode-prompts'
import { createSearchTool } from '@/lib/tools/search'
import { assignCitationLabels } from '@/lib/utils/citation'
import { getModel } from '@/lib/utils/registry'

import { parseSpecBlock } from '../parse-spec-block'

const RUN = process.env.RUN_LLM_E2E === '1'
const MODEL = process.env.LLM_E2E_MODEL || 'openai:gpt-4o-mini'

const FIXTURE_IMAGES = [
  {
    url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/MtFuji_FujiCity.jpg/1280px-MtFuji_FujiCity.jpg',
    description: 'Mount Fuji seen from Fuji City'
  },
  {
    url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/da/070722_Mt.Fuji_Yoshidaguchi_Trail.jpg/1280px-070722_Mt.Fuji_Yoshidaguchi_Trail.jpg',
    description: 'Yoshida trail on Mount Fuji'
  },
  {
    url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/5b/MtFuji_FujiCity_winter.jpg/1280px-MtFuji_FujiCity_winter.jpg',
    description: 'Mount Fuji in winter'
  }
]

const FIXTURE_RESULTS = [
  {
    title: 'Mount Fuji - Wikipedia',
    url: 'https://en.wikipedia.org/wiki/Mount_Fuji',
    content:
      'Mount Fuji is the tallest mountain in Japan, standing 3,776 meters. It is an active stratovolcano located on Honshu island.'
  }
]

function createMockSearchTool() {
  return tool({
    description:
      'Search the web for information. Returns text results and related images.',
    inputSchema: z.object({
      query: z.string(),
      type: z.enum(['optimized', 'general']).optional(),
      max_results: z.number().optional()
    }),
    async execute({ query }) {
      return {
        state: 'complete' as const,
        query,
        // Labelled the way the production executor labels a real search, so
        // the model has the citation target its prompt requires.
        results: assignCitationLabels(FIXTURE_RESULTS, 1),
        images: FIXTURE_IMAGES,
        number_of_results: FIXTURE_RESULTS.length,
        toolCallId: 'mock-search-1'
      }
    },
    // Route the mock output through the real search tool's model-view filter
    // so the test exercises exactly what the model sees in production. A mock
    // without this hid the regression where toModelOutput stripped images.
    toModelOutput: createSearchTool(MODEL).toModelOutput as never
  })
}

function extractSpecBlocks(markdown: string): string[] {
  const blocks: string[] = []
  const regex = /```spec\n([\s\S]*?)```/g
  let match: RegExpExecArray | null
  while ((match = regex.exec(markdown)) !== null) {
    blocks.push(match[1])
  }
  return blocks
}

describe.skipIf(!RUN)('LLM inline image output (E2E)', () => {
  test('LLM emits inline image spec block using search tool images', async () => {
    const { text } = await generateText({
      model: getModel(MODEL),
      instructions: getAdaptiveModePrompt(),
      tools: {
        search: createMockSearchTool()
      },
      stopWhen: isStepCount(6),
      prompt:
        'Show me photos of Mount Fuji and a brief description. Please include relevant images inline.'
    })

    console.log('[E2E] Model output:\n', text)

    const blocks = extractSpecBlocks(text)
    expect(blocks.length).toBeGreaterThanOrEqual(1)

    // Parse every spec block and check at least one contains an Image.
    const fixtureSrcs = new Set(FIXTURE_IMAGES.map(i => i.url))
    let imageCount = 0

    for (const source of blocks) {
      const spec = parseSpecBlock(source)
      for (const el of Object.values(spec.elements) as Array<{
        type: string
        props: Record<string, unknown>
      }>) {
        if (el.type === 'Image') {
          imageCount++
          const src = el.props.src as string
          // src MUST be one of the fixture image URLs — no fabrication.
          expect(fixtureSrcs.has(src)).toBe(true)
        }
      }
    }

    expect(imageCount).toBeGreaterThanOrEqual(1)
  }, 120_000)
})