Skip to content

An AI-powered search engine with a generative UI

perf(search): strip duplicated citationMap from model tool output (#882)

Yoshiki Miurab3450cc9b202
diff --git a/lib/agents/researcher.ts b/lib/agents/researcher.ts
index c404c65e..4115814e 100644
--- a/lib/agents/researcher.ts
+++ b/lib/agents/researcher.ts
@@ -23,6 +23,9 @@ function wrapSearchToolForQuickMode<
return tool({
description: originalTool.description,
inputSchema: originalTool.inputSchema,
+ // Preserve the original tool's model-output trimming (strips the duplicated
+ // citationMap / UI-only images) so quick mode gets the same payload savings.
+ toModelOutput: originalTool.toModelOutput,
async *execute(params, context) {
const executeFunc = originalTool.execute
if (!executeFunc) {
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index 38406d48..e290290d 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -23,6 +23,7 @@ import {
} from '../utils/context-window'
import { getTextFromParts } from '../utils/message-utils'
import { perfLog, perfTime } from '../utils/perf-logging'
+import { isUsageLogging, logUsage } from '../utils/usage-logging'
import { persistStreamResults } from './helpers/persist-stream-results'
import { prepareMessages } from './helpers/prepare-messages'
@@ -183,10 +184,29 @@ export async function createChatStreamResponse(
const result = await researchAgent.stream({
messages: modelMessages,
abortSignal,
- experimental_transform: smoothStream({ chunking: 'word' })
+ experimental_transform: smoothStream({ chunking: 'word' }),
+ ...(isUsageLogging() && {
+ onStepFinish: step => {
+ logUsage(
+ { scope: 'step', modelId: context.modelId },
+ step.usage,
+ step.providerMetadata
+ )
+ }
+ })
})
result.consumeStream()
+ // Log the session-total usage once the stream settles (does not block the
+ // response; consumeStream above already drives it to completion).
+ if (isUsageLogging()) {
+ Promise.resolve(result.totalUsage)
+ .then(usage =>
+ logUsage({ scope: 'total', modelId: context.modelId }, usage)
+ )
+ .catch(() => {})
+ }
+
return result.toUIMessageStreamResponse({
messageMetadata: ({ part }) => {
if (part.type === 'start') {
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index ec8de1e4..f2db0caf 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -20,6 +20,7 @@ import {
shouldTruncateMessages,
truncateMessages
} from '../utils/context-window'
+import { isUsageLogging, logUsage } from '../utils/usage-logging'
import { stripReasoningParts } from './helpers/strip-reasoning-parts'
import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'
@@ -93,13 +94,29 @@ export async function createEphemeralChatStreamResponse(
searchMode
})
+ const modelId = `${model.providerId}:${model.id}`
const result = await researchAgent.stream({
messages: modelMessages,
abortSignal,
- experimental_transform: smoothStream({ chunking: 'word' })
+ experimental_transform: smoothStream({ chunking: 'word' }),
+ ...(isUsageLogging() && {
+ onStepFinish: step => {
+ logUsage(
+ { scope: 'step', modelId },
+ step.usage,
+ step.providerMetadata
+ )
+ }
+ })
})
result.consumeStream()
+ if (isUsageLogging()) {
+ Promise.resolve(result.totalUsage)
+ .then(usage => logUsage({ scope: 'total', modelId }, usage))
+ .catch(() => {})
+ }
+
return result.toUIMessageStreamResponse({
messageMetadata: ({ part }) => {
if (part.type === 'start') {
diff --git a/lib/tools/__tests__/search-to-model-output.test.ts b/lib/tools/__tests__/search-to-model-output.test.ts
new file mode 100644
index 00000000..74e50e7a
--- /dev/null
+++ b/lib/tools/__tests__/search-to-model-output.test.ts
@@ -0,0 +1,83 @@
+import { describe, expect, it } from 'vitest'
+
+import { createSearchTool } from '@/lib/tools/search'
+
+// The search tool's toModelOutput strips UI-only fields (citationMap duplicates
+// results; images are display thumbnails) from what the model sees, halving the
+// search payload's token weight. These fields must survive in the streamed/
+// persisted output (for the UI) but never reach the model.
+describe('search tool toModelOutput', () => {
+ const tool = createSearchTool('google:gemini-3-flash-preview')
+
+ const fullOutput = {
+ state: 'complete',
+ query: 'test query',
+ number_of_results: 2,
+ results: [
+ { title: 'A', url: 'https://a.test', content: 'alpha' },
+ { title: 'B', url: 'https://b.test', content: 'beta' }
+ ],
+ images: [{ url: 'https://a.test/1.png' }],
+ citationMap: {
+ 1: { title: 'A', url: 'https://a.test', content: 'alpha' },
+ 2: { title: 'B', url: 'https://b.test', content: 'beta' }
+ },
+ toolCallId: 'call_123'
+ }
+
+ it('omits citationMap, images, and state from the model output', async () => {
+ const modelOutput = await tool.toModelOutput?.({
+ toolCallId: 'call_123',
+ input: {} as never,
+ output: fullOutput as never
+ })
+
+ expect(modelOutput).toBeDefined()
+ expect(modelOutput?.type).toBe('json')
+ const value = (
+ modelOutput as { type: 'json'; value: Record<string, unknown> }
+ ).value
+
+ expect(value).not.toHaveProperty('citationMap')
+ expect(value).not.toHaveProperty('images')
+ expect(value).not.toHaveProperty('state')
+ })
+
+ it('preserves the fields the model needs to answer and to cite', async () => {
+ const modelOutput = await tool.toModelOutput?.({
+ toolCallId: 'call_123',
+ input: {} as never,
+ output: fullOutput as never
+ })
+ const value = (
+ modelOutput as { type: 'json'; value: Record<string, unknown> }
+ ).value
+
+ expect(value.results).toEqual(fullOutput.results)
+ expect(value.query).toBe('test query')
+ expect(value.number_of_results).toBe(2)
+ // toolCallId is required: the prompt cites as [number](#toolCallId).
+ expect(value.toolCallId).toBe('call_123')
+ })
+
+ it('does not mutate the original output (UI/persistence keep all fields)', async () => {
+ await tool.toModelOutput?.({
+ toolCallId: 'call_123',
+ input: {} as never,
+ output: fullOutput as never
+ })
+
+ expect(fullOutput).toHaveProperty('citationMap')
+ expect(fullOutput).toHaveProperty('images')
+ })
+
+ it('handles non-object output defensively', async () => {
+ const modelOutput = await tool.toModelOutput?.({
+ toolCallId: 'call_123',
+ input: {} as never,
+ output: null as never
+ })
+
+ expect(modelOutput).toEqual({ type: 'json', value: null })
+ })
+})
diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.ts
index 7e781833..9bc16eee 100644
--- a/lib/tools/fetch.ts
+++ b/lib/tools/fetch.ts
@@ -2,6 +2,7 @@ import { tool, UIToolInvocation } from 'ai'
import { fetchSchema } from '@/lib/schema/fetch'
import { SearchResults as SearchResultsType } from '@/lib/types'
+import { logToolPayload } from '@/lib/utils/usage-logging'
const CONTENT_CHARACTER_LIMIT = 50000
const TITLE_CHARACTER_LIMIT = 100
@@ -183,6 +184,8 @@ export const fetchTool = tool({
}
}
+ logToolPayload('fetch', url, { results: results.results })
+
// Yield final results with complete state
yield {
state: 'complete' as const,
diff --git a/lib/tools/search.ts b/lib/tools/search.ts
index 24bf4249..c5f33020 100644
--- a/lib/tools/search.ts
+++ b/lib/tools/search.ts
@@ -1,4 +1,4 @@
-import { tool, UIToolInvocation } from 'ai'
+import { type JSONValue, tool, UIToolInvocation } from 'ai'
import { getSearchSchemaForModel } from '@/lib/schema/search'
import { SearchResultItem, SearchResults } from '@/lib/types'
@@ -7,6 +7,7 @@ import {
getSearchToolDescription
} from '@/lib/utils/search-config'
import { getBaseUrlString } from '@/lib/utils/url'
+import { logToolPayload } from '@/lib/utils/usage-logging'
import {
createSearchProvider,
@@ -141,7 +142,12 @@ export function createSearchTool(fullModel: string) {
throw error instanceof Error ? error : new Error('Unknown search error')
}
- // Add citation mapping and toolCallId to search results
+ // Add citation mapping and toolCallId to search results.
+ // citationMap fully duplicates results (citationMap[i+1] === results[i]);
+ // it is stripped from the model output (see toModelOutput below) but still
+ // lives in the persisted UI payload. Removing it at the source and having
+ // the UI index results[N-1] directly is a separate follow-up (UI +
+ // persisted-shape change, needs a fallback for existing messages).
if (searchResult.results && searchResult.results.length > 0) {
const citationMap: Record<number, SearchResultItem> = {}
searchResult.results.forEach((result, index) => {
@@ -157,11 +163,35 @@ export function createSearchTool(fullModel: string) {
console.log('completed search')
+ logToolPayload('search', query, {
+ results: searchResult.results,
+ citationMap: searchResult.citationMap,
+ images: searchResult.images
+ })
+
// Yield final results with complete state
yield {
state: 'complete' as const,
...searchResult
}
+ },
+ // citationMap duplicates `results` (citationMap[i+1] === results[i]) and is
+ // only consumed by the UI for citation rendering; images are display
+ // thumbnails. The model needs neither, so strip them from what it (and every
+ // replayed turn carrying this tool result) sees — this roughly halves the
+ // search payload's token weight. toolCallId MUST stay: the prompt requires
+ // the model to cite as [number](#toolCallId), so it reads the id from here.
+ toModelOutput: ({ output }) => {
+ if (!output || typeof output !== 'object') {
+ return { type: 'json', value: (output ?? null) as JSONValue }
+ }
+ const modelView: Record<string, unknown> = {
+ ...(output as Record<string, unknown>)
+ }
+ delete modelView.citationMap
+ delete modelView.images
+ delete modelView.state
+ return { type: 'json', value: modelView as JSONValue }
}
})
}
diff --git a/lib/utils/usage-logging.ts b/lib/utils/usage-logging.ts
new file mode 100644
index 00000000..7fd889cd
--- /dev/null
+++ b/lib/utils/usage-logging.ts
@@ -0,0 +1,93 @@
+// Token usage / prompt-cache logging utilities.
+//
+// The AI SDK normalizes every provider's cache accounting onto
+// LanguageModelUsage.inputTokenDetails.cacheReadTokens. For Gemini this comes
+// from usage.cachedContentTokenCount (implicit caching), so cache hits are
+// observable without any explicit cache setup. Enable with
+// ENABLE_USAGE_LOGGING=true to measure cache hit rate across a session.
+
+import type { LanguageModelUsage, ProviderMetadata } from 'ai'
+
+const isUsageLoggingEnabled = process.env.ENABLE_USAGE_LOGGING === 'true'
+
+export function isUsageLogging(): boolean {
+ return isUsageLoggingEnabled
+}
+
+interface UsageLogContext {
+ // Free-form scope label, e.g. "step" or "total".
+ scope: string
+ modelId?: string
+ // Step index for per-step logs (onStepFinish).
+ step?: number
+}
+
+function pct(part: number | undefined, whole: number | undefined): string {
+ if (!whole || whole <= 0 || part == null) return 'n/a'
+ return `${((part / whole) * 100).toFixed(1)}%`
+}
+
+// logUsage prints a single normalized usage line. cacheRead/cacheWrite come from
+// inputTokenDetails; the hit rate is cacheRead / inputTokens.
+export function logUsage(
+ ctx: UsageLogContext,
+ usage: LanguageModelUsage,
+ providerMetadata?: ProviderMetadata
+) {
+ if (!isUsageLoggingEnabled) return
+
+ const input = usage.inputTokens
+ const cacheRead = usage.inputTokenDetails?.cacheReadTokens
+ const cacheWrite = usage.inputTokenDetails?.cacheWriteTokens
+ const reasoning = usage.outputTokenDetails?.reasoningTokens
+
+ const parts = [
+ `scope=${ctx.scope}`,
+ ctx.step != null ? `step=${ctx.step}` : null,
+ ctx.modelId ? `model=${ctx.modelId}` : null,
+ `input=${input ?? '?'}`,
+ `cacheRead=${cacheRead ?? 0}`,
+ `cacheWrite=${cacheWrite ?? 0}`,
+ `hitRate=${pct(cacheRead, input)}`,
+ `output=${usage.outputTokens ?? '?'}`,
+ reasoning != null ? `reasoning=${reasoning}` : null,
+ `total=${usage.totalTokens ?? '?'}`
+ ].filter(Boolean)
+
+ console.log(`[USAGE] ${parts.join(' ')}`)
+
+ // Gemini reports nothing beyond cachedContentTokenCount today, but dump the
+ // raw provider metadata when present so we can spot fields the SDK does not
+ // yet normalize.
+ if (providerMetadata) {
+ console.log(`[USAGE] ${ctx.scope} providerMetadata=`, providerMetadata)
+ }
+}
+
+// Rough byte-based token estimate (~4 chars/token). Good enough for relative
+// payload breakdowns; avoids pulling a tokenizer into the tool path.
+function estTokens(value: unknown): number {
+ if (value == null) return 0
+ const s = typeof value === 'string' ? value : JSON.stringify(value)
+ return Math.ceil(s.length / 4)
+}
+
+// logToolPayload reports the token weight of what a tool injects into the next
+// request, broken down by sub-field. This is how we see which part of the
+// 100K+ input prompt is the real cost driver (search results vs the duplicated
+// citationMap vs fetched page bodies).
+export function logToolPayload(
+ toolName: string,
+ query: string | undefined,
+ breakdown: Record<string, unknown>
+) {
+ if (!isUsageLoggingEnabled) return
+
+ const total = estTokens(breakdown)
+ const parts = Object.entries(breakdown).map(
+ ([k, v]) => `${k}=${estTokens(v)}`
+ )
+ console.log(
+ `[PAYLOAD] tool=${toolName}${query ? ` q="${query.slice(0, 40)}"` : ''} total≈${total} ${parts.join(' ')}`
+ )
+}