An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
perf(search): strip duplicated citationMap from model tool output (#882)
Yoshiki Miurab3450cc9b202
diff --git a/lib/agents/researcher.ts b/lib/agents/researcher.tsindex c404c65e..4115814e 100644--- a/lib/agents/researcher.ts+++ b/lib/agents/researcher.ts@@ -23,6 +23,9 @@ function wrapSearchToolForQuickMode<return tool({description: originalTool.description,inputSchema: originalTool.inputSchema,+ // Preserve the original tool's model-output trimming (strips the duplicated+ // citationMap / UI-only images) so quick mode gets the same payload savings.+ toModelOutput: originalTool.toModelOutput,async *execute(params, context) {const executeFunc = originalTool.executeif (!executeFunc) {diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 38406d48..e290290d 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -23,6 +23,7 @@ import {} from '../utils/context-window'import { getTextFromParts } from '../utils/message-utils'import { perfLog, perfTime } from '../utils/perf-logging'+import { isUsageLogging, logUsage } from '../utils/usage-logging'import { persistStreamResults } from './helpers/persist-stream-results'import { prepareMessages } from './helpers/prepare-messages'@@ -183,10 +184,29 @@ export async function createChatStreamResponse(const result = await researchAgent.stream({messages: modelMessages,abortSignal,- experimental_transform: smoothStream({ chunking: 'word' })+ experimental_transform: smoothStream({ chunking: 'word' }),+ ...(isUsageLogging() && {+ onStepFinish: step => {+ logUsage(+ { scope: 'step', modelId: context.modelId },+ step.usage,+ step.providerMetadata+ )+ }+ })})result.consumeStream()+ // Log the session-total usage once the stream settles (does not block the+ // response; consumeStream above already drives it to completion).+ if (isUsageLogging()) {+ Promise.resolve(result.totalUsage)+ .then(usage =>+ logUsage({ scope: 'total', modelId: context.modelId }, usage)+ )+ .catch(() => {})+ }+return result.toUIMessageStreamResponse({messageMetadata: ({ part }) => {if (part.type === 'start') {diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.tsindex ec8de1e4..f2db0caf 100644--- a/lib/streaming/create-ephemeral-chat-stream-response.ts+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts@@ -20,6 +20,7 @@ import {shouldTruncateMessages,truncateMessages} from '../utils/context-window'+import { isUsageLogging, logUsage } from '../utils/usage-logging'import { stripReasoningParts } from './helpers/strip-reasoning-parts'import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'@@ -93,13 +94,29 @@ export async function createEphemeralChatStreamResponse(searchMode})+ const modelId = `${model.providerId}:${model.id}`const result = await researchAgent.stream({messages: modelMessages,abortSignal,- experimental_transform: smoothStream({ chunking: 'word' })+ experimental_transform: smoothStream({ chunking: 'word' }),+ ...(isUsageLogging() && {+ onStepFinish: step => {+ logUsage(+ { scope: 'step', modelId },+ step.usage,+ step.providerMetadata+ )+ }+ })})result.consumeStream()+ if (isUsageLogging()) {+ Promise.resolve(result.totalUsage)+ .then(usage => logUsage({ scope: 'total', modelId }, usage))+ .catch(() => {})+ }+return result.toUIMessageStreamResponse({messageMetadata: ({ part }) => {if (part.type === 'start') {diff --git a/lib/tools/__tests__/search-to-model-output.test.ts b/lib/tools/__tests__/search-to-model-output.test.tsnew file mode 100644index 00000000..74e50e7a--- /dev/null+++ b/lib/tools/__tests__/search-to-model-output.test.ts@@ -0,0 +1,83 @@+import { describe, expect, it } from 'vitest'++import { createSearchTool } from '@/lib/tools/search'++// The search tool's toModelOutput strips UI-only fields (citationMap duplicates+// results; images are display thumbnails) from what the model sees, halving the+// search payload's token weight. These fields must survive in the streamed/+// persisted output (for the UI) but never reach the model.+describe('search tool toModelOutput', () => {+ const tool = createSearchTool('google:gemini-3-flash-preview')++ const fullOutput = {+ state: 'complete',+ query: 'test query',+ number_of_results: 2,+ results: [+ { title: 'A', url: 'https://a.test', content: 'alpha' },+ { title: 'B', url: 'https://b.test', content: 'beta' }+ ],+ images: [{ url: 'https://a.test/1.png' }],+ citationMap: {+ 1: { title: 'A', url: 'https://a.test', content: 'alpha' },+ 2: { title: 'B', url: 'https://b.test', content: 'beta' }+ },+ toolCallId: 'call_123'+ }++ it('omits citationMap, images, and state from the model output', async () => {+ const modelOutput = await tool.toModelOutput?.({+ toolCallId: 'call_123',+ input: {} as never,+ output: fullOutput as never+ })++ expect(modelOutput).toBeDefined()+ expect(modelOutput?.type).toBe('json')+ const value = (+ modelOutput as { type: 'json'; value: Record<string, unknown> }+ ).value++ expect(value).not.toHaveProperty('citationMap')+ expect(value).not.toHaveProperty('images')+ expect(value).not.toHaveProperty('state')+ })++ it('preserves the fields the model needs to answer and to cite', async () => {+ const modelOutput = await tool.toModelOutput?.({+ toolCallId: 'call_123',+ input: {} as never,+ output: fullOutput as never+ })+ const value = (+ modelOutput as { type: 'json'; value: Record<string, unknown> }+ ).value++ expect(value.results).toEqual(fullOutput.results)+ expect(value.query).toBe('test query')+ expect(value.number_of_results).toBe(2)+ // toolCallId is required: the prompt cites as [number](#toolCallId).+ expect(value.toolCallId).toBe('call_123')+ })++ it('does not mutate the original output (UI/persistence keep all fields)', async () => {+ await tool.toModelOutput?.({+ toolCallId: 'call_123',+ input: {} as never,+ output: fullOutput as never+ })++ expect(fullOutput).toHaveProperty('citationMap')+ expect(fullOutput).toHaveProperty('images')+ })++ it('handles non-object output defensively', async () => {+ const modelOutput = await tool.toModelOutput?.({+ toolCallId: 'call_123',+ input: {} as never,+ output: null as never+ })++ expect(modelOutput).toEqual({ type: 'json', value: null })+ })+})diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.tsindex 7e781833..9bc16eee 100644--- a/lib/tools/fetch.ts+++ b/lib/tools/fetch.ts@@ -2,6 +2,7 @@ import { tool, UIToolInvocation } from 'ai'import { fetchSchema } from '@/lib/schema/fetch'import { SearchResults as SearchResultsType } from '@/lib/types'+import { logToolPayload } from '@/lib/utils/usage-logging'const CONTENT_CHARACTER_LIMIT = 50000const TITLE_CHARACTER_LIMIT = 100@@ -183,6 +184,8 @@ export const fetchTool = tool({}}+ logToolPayload('fetch', url, { results: results.results })+// Yield final results with complete stateyield {state: 'complete' as const,diff --git a/lib/tools/search.ts b/lib/tools/search.tsindex 24bf4249..c5f33020 100644--- a/lib/tools/search.ts+++ b/lib/tools/search.ts@@ -1,4 +1,4 @@-import { tool, UIToolInvocation } from 'ai'+import { type JSONValue, tool, UIToolInvocation } from 'ai'import { getSearchSchemaForModel } from '@/lib/schema/search'import { SearchResultItem, SearchResults } from '@/lib/types'@@ -7,6 +7,7 @@ import {getSearchToolDescription} from '@/lib/utils/search-config'import { getBaseUrlString } from '@/lib/utils/url'+import { logToolPayload } from '@/lib/utils/usage-logging'import {createSearchProvider,@@ -141,7 +142,12 @@ export function createSearchTool(fullModel: string) {throw error instanceof Error ? error : new Error('Unknown search error')}- // Add citation mapping and toolCallId to search results+ // Add citation mapping and toolCallId to search results.+ // citationMap fully duplicates results (citationMap[i+1] === results[i]);+ // it is stripped from the model output (see toModelOutput below) but still+ // lives in the persisted UI payload. Removing it at the source and having+ // the UI index results[N-1] directly is a separate follow-up (UI ++ // persisted-shape change, needs a fallback for existing messages).if (searchResult.results && searchResult.results.length > 0) {const citationMap: Record<number, SearchResultItem> = {}searchResult.results.forEach((result, index) => {@@ -157,11 +163,35 @@ export function createSearchTool(fullModel: string) {console.log('completed search')+ logToolPayload('search', query, {+ results: searchResult.results,+ citationMap: searchResult.citationMap,+ images: searchResult.images+ })+// Yield final results with complete stateyield {state: 'complete' as const,...searchResult}+ },+ // citationMap duplicates `results` (citationMap[i+1] === results[i]) and is+ // only consumed by the UI for citation rendering; images are display+ // thumbnails. The model needs neither, so strip them from what it (and every+ // replayed turn carrying this tool result) sees — this roughly halves the+ // search payload's token weight. toolCallId MUST stay: the prompt requires+ // the model to cite as [number](#toolCallId), so it reads the id from here.+ toModelOutput: ({ output }) => {+ if (!output || typeof output !== 'object') {+ return { type: 'json', value: (output ?? null) as JSONValue }+ }+ const modelView: Record<string, unknown> = {+ ...(output as Record<string, unknown>)+ }+ delete modelView.citationMap+ delete modelView.images+ delete modelView.state+ return { type: 'json', value: modelView as JSONValue }}})}diff --git a/lib/utils/usage-logging.ts b/lib/utils/usage-logging.tsnew file mode 100644index 00000000..7fd889cd--- /dev/null+++ b/lib/utils/usage-logging.ts@@ -0,0 +1,93 @@+// Token usage / prompt-cache logging utilities.+//+// The AI SDK normalizes every provider's cache accounting onto+// LanguageModelUsage.inputTokenDetails.cacheReadTokens. For Gemini this comes+// from usage.cachedContentTokenCount (implicit caching), so cache hits are+// observable without any explicit cache setup. Enable with+// ENABLE_USAGE_LOGGING=true to measure cache hit rate across a session.++import type { LanguageModelUsage, ProviderMetadata } from 'ai'++const isUsageLoggingEnabled = process.env.ENABLE_USAGE_LOGGING === 'true'++export function isUsageLogging(): boolean {+ return isUsageLoggingEnabled+}++interface UsageLogContext {+ // Free-form scope label, e.g. "step" or "total".+ scope: string+ modelId?: string+ // Step index for per-step logs (onStepFinish).+ step?: number+}++function pct(part: number | undefined, whole: number | undefined): string {+ if (!whole || whole <= 0 || part == null) return 'n/a'+ return `${((part / whole) * 100).toFixed(1)}%`+}++// logUsage prints a single normalized usage line. cacheRead/cacheWrite come from+// inputTokenDetails; the hit rate is cacheRead / inputTokens.+export function logUsage(+ ctx: UsageLogContext,+ usage: LanguageModelUsage,+ providerMetadata?: ProviderMetadata+) {+ if (!isUsageLoggingEnabled) return++ const input = usage.inputTokens+ const cacheRead = usage.inputTokenDetails?.cacheReadTokens+ const cacheWrite = usage.inputTokenDetails?.cacheWriteTokens+ const reasoning = usage.outputTokenDetails?.reasoningTokens++ const parts = [+ `scope=${ctx.scope}`,+ ctx.step != null ? `step=${ctx.step}` : null,+ ctx.modelId ? `model=${ctx.modelId}` : null,+ `input=${input ?? '?'}`,+ `cacheRead=${cacheRead ?? 0}`,+ `cacheWrite=${cacheWrite ?? 0}`,+ `hitRate=${pct(cacheRead, input)}`,+ `output=${usage.outputTokens ?? '?'}`,+ reasoning != null ? `reasoning=${reasoning}` : null,+ `total=${usage.totalTokens ?? '?'}`+ ].filter(Boolean)++ console.log(`[USAGE] ${parts.join(' ')}`)++ // Gemini reports nothing beyond cachedContentTokenCount today, but dump the+ // raw provider metadata when present so we can spot fields the SDK does not+ // yet normalize.+ if (providerMetadata) {+ console.log(`[USAGE] ${ctx.scope} providerMetadata=`, providerMetadata)+ }+}++// Rough byte-based token estimate (~4 chars/token). Good enough for relative+// payload breakdowns; avoids pulling a tokenizer into the tool path.+function estTokens(value: unknown): number {+ if (value == null) return 0+ const s = typeof value === 'string' ? value : JSON.stringify(value)+ return Math.ceil(s.length / 4)+}++// logToolPayload reports the token weight of what a tool injects into the next+// request, broken down by sub-field. This is how we see which part of the+// 100K+ input prompt is the real cost driver (search results vs the duplicated+// citationMap vs fetched page bodies).+export function logToolPayload(+ toolName: string,+ query: string | undefined,+ breakdown: Record<string, unknown>+) {+ if (!isUsageLoggingEnabled) return++ const total = estTokens(breakdown)+ const parts = Object.entries(breakdown).map(+ ([k, v]) => `${k}=${estTokens(v)}`+ )+ console.log(+ `[PAYLOAD] tool=${toolName}${query ? ` q="${query.slice(0, 40)}"` : ''} total≈${total} ${parts.join(' ')}`+ )+}