Skip to content

An AI-powered search engine with a generative UI

fix(search): keep images in model-facing tool output (#914)

Yoshiki Miurade98e8a29381
diff --git a/lib/render/__tests__/llm-image-output.e2e.test.ts b/lib/render/__tests__/llm-image-output.e2e.test.ts
index d0a2b54f..de039ea5 100644
--- a/lib/render/__tests__/llm-image-output.e2e.test.ts
+++ b/lib/render/__tests__/llm-image-output.e2e.test.ts
@@ -14,6 +14,7 @@ import { describe, expect, test } from 'vitest'
import { z } from 'zod'
import { getAdaptiveModePrompt } from '@/lib/agents/prompts/search-mode-prompts'
+import { createSearchTool } from '@/lib/tools/search'
import { getModel } from '@/lib/utils/registry'
import { parseSpecBlock } from '../parse-spec-block'
@@ -63,7 +64,11 @@ function createMockSearchTool() {
number_of_results: FIXTURE_RESULTS.length,
toolCallId: 'mock-search-1'
}
- }
+ },
+ // Route the mock output through the real search tool's model-view filter
+ // so the test exercises exactly what the model sees in production. A mock
+ // without this hid the regression where toModelOutput stripped images.
+ toModelOutput: createSearchTool(MODEL).toModelOutput as never
})
}
diff --git a/lib/tools/__tests__/search-to-model-output.test.ts b/lib/tools/__tests__/search-to-model-output.test.ts
index 74e50e7a..d6f02d67 100644
--- a/lib/tools/__tests__/search-to-model-output.test.ts
+++ b/lib/tools/__tests__/search-to-model-output.test.ts
@@ -2,10 +2,11 @@ import { describe, expect, it } from 'vitest'
import { createSearchTool } from '@/lib/tools/search'
-// The search tool's toModelOutput strips UI-only fields (citationMap duplicates
-// results; images are display thumbnails) from what the model sees, halving the
-// search payload's token weight. These fields must survive in the streamed/
-// persisted output (for the UI) but never reach the model.
+// The search tool's toModelOutput strips UI-only fields (citationMap
+// duplicates results; state is a streaming marker) from what the model sees.
+// images MUST reach the model — getImageSpecPrompt instructs it to embed URLs
+// verbatim from that array. All fields must survive in the streamed/persisted
+// output for the UI.
describe('search tool toModelOutput', () => {
const tool = createSearchTool('google:gemini-3-flash-preview')
@@ -17,7 +18,7 @@ describe('search tool toModelOutput', () => {
{ title: 'A', url: 'https://a.test', content: 'alpha' },
{ title: 'B', url: 'https://b.test', content: 'beta' }
],
- images: [{ url: 'https://a.test/1.png' }],
+ images: [{ url: 'https://a.test/1.png', description: 'one' }],
citationMap: {
1: { title: 'A', url: 'https://a.test', content: 'alpha' },
2: { title: 'B', url: 'https://b.test', content: 'beta' }
@@ -25,33 +26,26 @@ describe('search tool toModelOutput', () => {
toolCallId: 'call_123'
}
- it('omits citationMap, images, and state from the model output', async () => {
+ const getModelValue = async (output: unknown) => {
const modelOutput = await tool.toModelOutput?.({
toolCallId: 'call_123',
input: {} as never,
- output: fullOutput as never
+ output: output as never
})
-
- expect(modelOutput).toBeDefined()
expect(modelOutput?.type).toBe('json')
- const value = (
- modelOutput as { type: 'json'; value: Record<string, unknown> }
- ).value
+ return (modelOutput as { type: 'json'; value: Record<string, unknown> })
+ .value
+ }
+
+ it('omits citationMap and state from the model output', async () => {
+ const value = await getModelValue(fullOutput)
expect(value).not.toHaveProperty('citationMap')
- expect(value).not.toHaveProperty('images')
expect(value).not.toHaveProperty('state')
})
it('preserves the fields the model needs to answer and to cite', async () => {
- const modelOutput = await tool.toModelOutput?.({
- toolCallId: 'call_123',
- input: {} as never,
- output: fullOutput as never
- })
- const value = (
- modelOutput as { type: 'json'; value: Record<string, unknown> }
- ).value
+ const value = await getModelValue(fullOutput)
expect(value.results).toEqual(fullOutput.results)
expect(value.query).toBe('test query')
@@ -60,6 +54,12 @@ describe('search tool toModelOutput', () => {
expect(value.toolCallId).toBe('call_123')
})
+ it('keeps images so the model can embed inline image specs', async () => {
+ const value = await getModelValue(fullOutput)
+
+ expect(value.images).toEqual(fullOutput.images)
+ })
+
it('does not mutate the original output (UI/persistence keep all fields)', async () => {
await tool.toModelOutput?.({
toolCallId: 'call_123',
@@ -68,7 +68,7 @@ describe('search tool toModelOutput', () => {
})
expect(fullOutput).toHaveProperty('citationMap')
- expect(fullOutput).toHaveProperty('images')
+ expect(fullOutput).toHaveProperty('state')
})
it('handles non-object output defensively', async () => {
diff --git a/lib/tools/search.ts b/lib/tools/search.ts
index 6e19a468..5a088aeb 100644
--- a/lib/tools/search.ts
+++ b/lib/tools/search.ts
@@ -165,11 +165,12 @@ export function createSearchTool(fullModel: string) {
...searchResult
}
},
- // Trim the model-facing tool result: images are UI-only thumbnails and
- // state is a streaming marker. citationMap is no longer produced, but we
- // still drop it defensively for any older persisted output replayed through
- // here. toolCallId MUST stay: the prompt cites as [number](#toolCallId), so
- // the model reads the id from here.
+ // Trim the model-facing tool result: citationMap fully duplicates
+ // `results` (dropped defensively for older persisted output) and state is
+ // a streaming marker. images MUST stay — getImageSpecPrompt instructs the
+ // model to embed URLs verbatim from this array. toolCallId MUST stay: the
+ // prompt cites as [number](#toolCallId), so the model reads the id from
+ // here.
toModelOutput: ({ output }) => {
if (!output || typeof output !== 'object') {
return { type: 'json', value: (output ?? null) as JSONValue }
@@ -178,7 +179,6 @@ export function createSearchTool(fullModel: string) {
...(output as Record<string, unknown>)
}
delete modelView.citationMap
- delete modelView.images
delete modelView.state
return { type: 'json', value: modelView as JSONValue }
}