An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(search): keep images in model-facing tool output (#914)
Yoshiki Miurade98e8a29381
diff --git a/lib/render/__tests__/llm-image-output.e2e.test.ts b/lib/render/__tests__/llm-image-output.e2e.test.tsindex d0a2b54f..de039ea5 100644--- a/lib/render/__tests__/llm-image-output.e2e.test.ts+++ b/lib/render/__tests__/llm-image-output.e2e.test.ts@@ -14,6 +14,7 @@ import { describe, expect, test } from 'vitest'import { z } from 'zod'import { getAdaptiveModePrompt } from '@/lib/agents/prompts/search-mode-prompts'+import { createSearchTool } from '@/lib/tools/search'import { getModel } from '@/lib/utils/registry'import { parseSpecBlock } from '../parse-spec-block'@@ -63,7 +64,11 @@ function createMockSearchTool() {number_of_results: FIXTURE_RESULTS.length,toolCallId: 'mock-search-1'}- }+ },+ // Route the mock output through the real search tool's model-view filter+ // so the test exercises exactly what the model sees in production. A mock+ // without this hid the regression where toModelOutput stripped images.+ toModelOutput: createSearchTool(MODEL).toModelOutput as never})}diff --git a/lib/tools/__tests__/search-to-model-output.test.ts b/lib/tools/__tests__/search-to-model-output.test.tsindex 74e50e7a..d6f02d67 100644--- a/lib/tools/__tests__/search-to-model-output.test.ts+++ b/lib/tools/__tests__/search-to-model-output.test.ts@@ -2,10 +2,11 @@ import { describe, expect, it } from 'vitest'import { createSearchTool } from '@/lib/tools/search'-// The search tool's toModelOutput strips UI-only fields (citationMap duplicates-// results; images are display thumbnails) from what the model sees, halving the-// search payload's token weight. These fields must survive in the streamed/-// persisted output (for the UI) but never reach the model.+// The search tool's toModelOutput strips UI-only fields (citationMap+// duplicates results; state is a streaming marker) from what the model sees.+// images MUST reach the model — getImageSpecPrompt instructs it to embed URLs+// verbatim from that array. All fields must survive in the streamed/persisted+// output for the UI.describe('search tool toModelOutput', () => {const tool = createSearchTool('google:gemini-3-flash-preview')@@ -17,7 +18,7 @@ describe('search tool toModelOutput', () => {{ title: 'A', url: 'https://a.test', content: 'alpha' },{ title: 'B', url: 'https://b.test', content: 'beta' }],- images: [{ url: 'https://a.test/1.png' }],+ images: [{ url: 'https://a.test/1.png', description: 'one' }],citationMap: {1: { title: 'A', url: 'https://a.test', content: 'alpha' },2: { title: 'B', url: 'https://b.test', content: 'beta' }@@ -25,33 +26,26 @@ describe('search tool toModelOutput', () => {toolCallId: 'call_123'}- it('omits citationMap, images, and state from the model output', async () => {+ const getModelValue = async (output: unknown) => {const modelOutput = await tool.toModelOutput?.({toolCallId: 'call_123',input: {} as never,- output: fullOutput as never+ output: output as never})-- expect(modelOutput).toBeDefined()expect(modelOutput?.type).toBe('json')- const value = (- modelOutput as { type: 'json'; value: Record<string, unknown> }- ).value+ return (modelOutput as { type: 'json'; value: Record<string, unknown> })+ .value+ }++ it('omits citationMap and state from the model output', async () => {+ const value = await getModelValue(fullOutput)expect(value).not.toHaveProperty('citationMap')- expect(value).not.toHaveProperty('images')expect(value).not.toHaveProperty('state')})it('preserves the fields the model needs to answer and to cite', async () => {- const modelOutput = await tool.toModelOutput?.({- toolCallId: 'call_123',- input: {} as never,- output: fullOutput as never- })- const value = (- modelOutput as { type: 'json'; value: Record<string, unknown> }- ).value+ const value = await getModelValue(fullOutput)expect(value.results).toEqual(fullOutput.results)expect(value.query).toBe('test query')@@ -60,6 +54,12 @@ describe('search tool toModelOutput', () => {expect(value.toolCallId).toBe('call_123')})+ it('keeps images so the model can embed inline image specs', async () => {+ const value = await getModelValue(fullOutput)++ expect(value.images).toEqual(fullOutput.images)+ })+it('does not mutate the original output (UI/persistence keep all fields)', async () => {await tool.toModelOutput?.({toolCallId: 'call_123',@@ -68,7 +68,7 @@ describe('search tool toModelOutput', () => {})expect(fullOutput).toHaveProperty('citationMap')- expect(fullOutput).toHaveProperty('images')+ expect(fullOutput).toHaveProperty('state')})it('handles non-object output defensively', async () => {diff --git a/lib/tools/search.ts b/lib/tools/search.tsindex 6e19a468..5a088aeb 100644--- a/lib/tools/search.ts+++ b/lib/tools/search.ts@@ -165,11 +165,12 @@ export function createSearchTool(fullModel: string) {...searchResult}},- // Trim the model-facing tool result: images are UI-only thumbnails and- // state is a streaming marker. citationMap is no longer produced, but we- // still drop it defensively for any older persisted output replayed through- // here. toolCallId MUST stay: the prompt cites as [number](#toolCallId), so- // the model reads the id from here.+ // Trim the model-facing tool result: citationMap fully duplicates+ // `results` (dropped defensively for older persisted output) and state is+ // a streaming marker. images MUST stay — getImageSpecPrompt instructs the+ // model to embed URLs verbatim from this array. toolCallId MUST stay: the+ // prompt cites as [number](#toolCallId), so the model reads the id from+ // here.toModelOutput: ({ output }) => {if (!output || typeof output !== 'object') {return { type: 'json', value: (output ?? null) as JSONValue }@@ -178,7 +179,6 @@ export function createSearchTool(fullModel: string) {...(output as Record<string, unknown>)}delete modelView.citationMap- delete modelView.imagesdelete modelView.statereturn { type: 'json', value: modelView as JSONValue }}