An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix: align PDF attachment token estimates (#991)
Sushant Tiwarif16c88ac7f2c
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex aefd10ba..c4f45ebd 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -26,6 +26,7 @@ import { perfLog, perfTime } from '../utils/perf-logging'import { isUsageLogging, logUsage } from '../utils/usage-logging'import { resolveAttachmentSizes } from './helpers/attachment-sizes'+import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'import { capHistoricalAttachments } from './helpers/cap-historical-attachments'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'@@ -212,6 +213,8 @@ export async function createChatStreamResponse())carriedContext = summarizeCarriedContext(messagesToConvert)+ const attachmentTokenEstimates =+ buildAttachmentTokenEstimates(messagesToConvert)// Convert to model messages and apply context window managementstreamErrorStage = 'convert-messages'@@ -220,10 +223,17 @@ export async function createChatStreamResponse(})streamErrorStage = 'truncate-messages'- if (shouldTruncateMessages(modelMessages, model)) {+ if (+ shouldTruncateMessages(modelMessages, model, attachmentTokenEstimates)+ ) {const maxTokens = getMaxAllowedTokens(model)const originalCount = modelMessages.length- modelMessages = truncateMessages(modelMessages, maxTokens, model.id)+ modelMessages = truncateMessages(+ modelMessages,+ maxTokens,+ model.id,+ attachmentTokenEstimates+ )contextWindowTruncated = trueif (process.env.NODE_ENV === 'development') {diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.tsindex 60eb3b92..7b2f770e 100644--- a/lib/streaming/create-ephemeral-chat-stream-response.ts+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts@@ -22,6 +22,7 @@ import {import { getTextFromParts } from '../utils/message-utils'import { isUsageLogging, logUsage } from '../utils/usage-logging'+import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'import { capHistoricalAttachments } from './helpers/cap-historical-attachments'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'@@ -136,6 +137,8 @@ export async function createEphemeralChatStreamResponse(capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec)))carriedContext = summarizeCarriedContext(messagesToConvert)+ const attachmentTokenEstimates =+ buildAttachmentTokenEstimates(messagesToConvert)streamErrorStage = 'convert-messages'let modelMessages = await convertToModelMessages(messagesToConvert, {@@ -143,9 +146,16 @@ export async function createEphemeralChatStreamResponse(})streamErrorStage = 'truncate-messages'- if (shouldTruncateMessages(modelMessages, model)) {+ if (+ shouldTruncateMessages(modelMessages, model, attachmentTokenEstimates)+ ) {const maxTokens = getMaxAllowedTokens(model)- modelMessages = truncateMessages(modelMessages, maxTokens, model.id)+ modelMessages = truncateMessages(+ modelMessages,+ maxTokens,+ model.id,+ attachmentTokenEstimates+ )contextWindowTruncated = true}diff --git a/lib/streaming/helpers/__tests__/attachment-token-estimates.test.ts b/lib/streaming/helpers/__tests__/attachment-token-estimates.test.tsnew file mode 100644index 00000000..26c7419a--- /dev/null+++ b/lib/streaming/helpers/__tests__/attachment-token-estimates.test.ts@@ -0,0 +1,80 @@+import { convertToModelMessages, type UIMessage } from 'ai'+import { describe, expect, it } from 'vitest'++import {+ estimateAttachmentTokens,+ UNKNOWN_ATTACHMENT_TOKENS+} from '@/lib/utils/attachment-tokens'+import { shouldTruncateMessages } from '@/lib/utils/context-window'++import { buildAttachmentTokenEstimates } from '../attachment-token-estimates'++function messages(): UIMessage[] {+ return [+ {+ id: 'u1',+ role: 'user',+ parts: [+ { type: 'text', text: 'Compare these files' },+ {+ type: 'file',+ mediaType: 'application/pdf',+ url: 'https://example.com/large.pdf',+ size: 20_000_000+ },+ {+ type: 'file',+ mediaType: 'application/pdf',+ url: 'https://example.com/unknown.pdf'+ }+ ]+ } as unknown as UIMessage+ ]+}++describe('buildAttachmentTokenEstimates', () => {+ it('indexes the exact pre-conversion estimate by attachment URL', () => {+ const estimates = buildAttachmentTokenEstimates(messages())++ expect(estimates.get('https://example.com/large.pdf')).toBe(+ estimateAttachmentTokens({+ mediaType: 'application/pdf',+ size: 20_000_000+ })+ )+ expect(estimates.get('https://example.com/unknown.pdf')).toBe(+ UNKNOWN_ATTACHMENT_TOKENS+ )+ })++ it('ignores non-file parts', () => {+ const estimates = buildAttachmentTokenEstimates([+ {+ id: 'u1',+ role: 'user',+ parts: [{ type: 'text', text: 'No files' }]+ }+ ])++ expect(estimates.size).toBe(0)+ })++ it('matches converted URL parts in the context-window guard', async () => {+ const input = messages()+ const estimates = buildAttachmentTokenEstimates(input)+ const modelMessages = await convertToModelMessages(input)++ expect(+ shouldTruncateMessages(+ modelMessages,+ {+ id: 'gpt-4o-mini',+ name: 'GPT-4o mini',+ provider: 'OpenAI',+ providerId: 'openai'+ },+ estimates+ )+ ).toBe(true)+ })+})diff --git a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.tsindex d1ff47d6..f3c85821 100644--- a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts+++ b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts@@ -235,7 +235,7 @@ describe('capHistoricalAttachments', () => {it('bounds heavy attachments below the count limit by weight blocks', () => {const messages = pdfThread(4).concat(userTurn('current', 0))- const capped = capHistoricalAttachments(messages, LIMIT, 200_000)+ const capped = capHistoricalAttachments(messages, LIMIT, 20_000)expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])expect(placeholders(capped)).toHaveLength(2)@@ -245,12 +245,12 @@ describe('capHistoricalAttachments', () => {const atFour = capHistoricalAttachments(pdfThread(4, 1_000_000).concat(userTurn('current', 0)),0,- 200_000+ 20_000)const atFive = capHistoricalAttachments(pdfThread(5, 1_000_000).concat(userTurn('current', 0)),0,- 200_000+ 20_000)const replayedPrefixLength = atFour.length - 1@@ -265,7 +265,7 @@ describe('capHistoricalAttachments', () => {const capped = capHistoricalAttachments(pdfThread(4).concat(userTurn('current', 0)),0,- 200_000+ 20_000)expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])@@ -280,11 +280,19 @@ describe('capHistoricalAttachments', () => {})it('leaves a thread within the weight budget untouched', () => {- const messages = thread(5, 1).concat(userTurn('current', 0))+ const messages = pdfThread(5, 100_000).concat(userTurn('current', 0))expect(capHistoricalAttachments(messages, LIMIT, 200_000)).toBe(messages)})+ it('bounds multiple large PDFs with the default weight budget', () => {+ const messages = pdfThread(4, 10_000_000).concat(userTurn('current', 0))+ const capped = capHistoricalAttachments(messages, LIMIT, 200_000)++ expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])+ expect(placeholders(capped)).toHaveLength(2)+ })+it('never applies the weight bound to the newest user message', () => {const messages = thread(1, 0).concat(userTurn('current', 0))messages.at(-1)?.parts.unshift({diff --git a/lib/streaming/helpers/__tests__/summarize-carried-context.test.ts b/lib/streaming/helpers/__tests__/summarize-carried-context.test.tsindex 44289e46..dc2f67d0 100644--- a/lib/streaming/helpers/__tests__/summarize-carried-context.test.ts+++ b/lib/streaming/helpers/__tests__/summarize-carried-context.test.ts@@ -2,7 +2,10 @@ import type { UIMessage } from 'ai'import { describe, expect, it } from 'vitest'import { summarizeCarriedContext } from '@/lib/streaming/helpers/summarize-carried-context'-import { IMAGE_ATTACHMENT_TOKENS } from '@/lib/utils/attachment-tokens'+import {+ IMAGE_ATTACHMENT_TOKENS,+ MIN_PDF_ATTACHMENT_TOKENS+} from '@/lib/utils/attachment-tokens'function createMessages(parts: unknown[]): UIMessage[] {return [{ id: 'message-1', role: 'user', parts: parts as UIMessage['parts'] }]@@ -34,7 +37,7 @@ describe('summarizeCarriedContext', () => {)).toEqual({attachments: 2,- attachmentTokens: IMAGE_ATTACHMENT_TOKENS + 10+ attachmentTokens: IMAGE_ATTACHMENT_TOKENS + MIN_PDF_ATTACHMENT_TOKENS})})@@ -65,6 +68,9 @@ describe('summarizeCarriedContext', () => {{ type: 'data-noteContext', data: { text: null } }]))- ).toEqual({ attachments: 1 })+ ).toEqual({+ attachments: 1,+ attachmentTokens: MIN_PDF_ATTACHMENT_TOKENS+ })})})diff --git a/lib/streaming/helpers/attachment-token-estimates.ts b/lib/streaming/helpers/attachment-token-estimates.tsnew file mode 100644index 00000000..ecd65cab--- /dev/null+++ b/lib/streaming/helpers/attachment-token-estimates.ts@@ -0,0 +1,30 @@+import type { UIMessage } from 'ai'++import { estimateAttachmentTokens } from '@/lib/utils/attachment-tokens'++import { isFilePart } from './attachment-parts'++/**+ * Builds the token estimates before model-message conversion removes custom+ * attachment metadata such as the resolved storage size. The URL is the stable+ * value shared by both representations and is not sent to providers as extra+ * metadata.+ */+export function buildAttachmentTokenEstimates(+ messages: UIMessage[]+): ReadonlyMap<string, number> {+ const estimates = new Map<string, number>()++ for (const message of messages) {+ for (const part of message.parts) {+ if (!isFilePart(part)) continue++ const file = part as { mediaType?: string; size?: number; url?: string }+ if (!file.url) continue++ estimates.set(file.url, estimateAttachmentTokens(file))+ }+ }++ return estimates+}diff --git a/lib/utils/__tests__/attachment-tokens.test.ts b/lib/utils/__tests__/attachment-tokens.test.tsindex a7a0e51a..2df6b6ca 100644--- a/lib/utils/__tests__/attachment-tokens.test.ts+++ b/lib/utils/__tests__/attachment-tokens.test.ts@@ -4,6 +4,8 @@ import {BYTES_PER_TOKEN,estimateAttachmentTokens,IMAGE_ATTACHMENT_TOKENS,+ MIN_PDF_ATTACHMENT_TOKENS,+ PDF_BYTES_PER_TOKEN,UNKNOWN_ATTACHMENT_TOKENS} from '../attachment-tokens'@@ -14,13 +16,19 @@ describe('estimateAttachmentTokens', () => {).toBe(IMAGE_ATTACHMENT_TOKENS)})- it('estimates known PDF sizes from their byte length', () => {+ it('uses a floor for small PDFs and scales large PDFs conservatively', () => {expect(estimateAttachmentTokens({ mediaType: 'application/pdf', size: 3_901 })- ).toBe(Math.ceil(3_901 / BYTES_PER_TOKEN))+ ).toBe(MIN_PDF_ATTACHMENT_TOKENS)+ expect(+ estimateAttachmentTokens({+ mediaType: 'application/pdf',+ size: 20_000_000+ })+ ).toBe(Math.ceil(20_000_000 / PDF_BYTES_PER_TOKEN))})- it('uses the unknown estimate when size is unavailable', () => {+ it('uses a conservative PDF estimate when size is unavailable', () => {expect(estimateAttachmentTokens({ mediaType: 'application/pdf' })).toBe(UNKNOWN_ATTACHMENT_TOKENS)diff --git a/lib/utils/__tests__/context-window.test.ts b/lib/utils/__tests__/context-window.test.tsindex c577a612..8c64195a 100644--- a/lib/utils/__tests__/context-window.test.ts+++ b/lib/utils/__tests__/context-window.test.ts@@ -100,6 +100,11 @@ describe('context-window', () => {type: 'file',mediaType: 'application/pdf',data: 'https://example.com/report.pdf'+ },+ {+ type: 'file',+ mediaType: 'application/pdf',+ data: 'https://example.com/appendix.pdf'}]}@@ -108,6 +113,47 @@ describe('context-window', () => {expect(shouldTruncateMessages(messages, unknownModel)).toBe(true)})+ test('does not treat a URL-backed PDF as an unknown-size attachment', () => {+ const unknownModel: Model = { ...mockModel, id: 'unknown-model' }+ const url = 'https://example.com/report.pdf'+ const messages: ModelMessage[] = [+ {+ role: 'user',+ content: [+ {+ type: 'file',+ mediaType: 'application/pdf',+ data: { type: 'url', url: new URL(url) }+ }+ ]+ }+ ]++ expect(+ shouldTruncateMessages(messages, unknownModel, new Map([[url, 10_000]]))+ ).toBe(false)+ })++ test('uses a resolved large-PDF estimate in the context guard', () => {+ const url = 'https://example.com/large-report.pdf'+ const messages: ModelMessage[] = [+ {+ role: 'user',+ content: [+ {+ type: 'file',+ mediaType: 'application/pdf',+ data: { type: 'url', url: new URL(url) }+ }+ ]+ }+ ]++ expect(+ shouldTruncateMessages(messages, mockModel, new Map([[url, 400_000]]))+ ).toBe(true)+ })+test('handles null/undefined messages gracefully', () => {expect(shouldTruncateMessages(null as any, mockModel)).toBe(false)expect(shouldTruncateMessages(undefined as any, mockModel)).toBe(false)@@ -209,6 +255,31 @@ describe('context-window', () => {expect(result.length).toBeGreaterThan(0)})+ test('uses resolved attachment estimates while truncating', () => {+ const url = 'https://example.com/large-report.pdf'+ const messages: ModelMessage[] = [+ {+ role: 'user',+ content: [+ {+ type: 'file',+ mediaType: 'application/pdf',+ data: { type: 'url', url: new URL(url) }+ }+ ]+ }+ ]++ expect(+ truncateMessages(+ messages,+ 100_000,+ mockModel.id,+ new Map([[url, 400_000]])+ )+ ).toEqual([])+ })+test('handles undefined content gracefully', () => {const messages: ModelMessage[] = [{ role: 'user', content: '' },diff --git a/lib/utils/attachment-tokens.ts b/lib/utils/attachment-tokens.tsindex 17735e6c..5387b771 100644--- a/lib/utils/attachment-tokens.ts+++ b/lib/utils/attachment-tokens.ts@@ -1,4 +1,8 @@export const IMAGE_ATTACHMENT_TOKENS = 10_000+export const MIN_PDF_ATTACHMENT_TOKENS = 10_000+// A 500 KB PDF reaches the small-document floor; larger containers scale so+// multi-page files cannot all hide behind a flat estimate.+export const PDF_BYTES_PER_TOKEN = 50export const BYTES_PER_TOKEN = 3.9export const UNKNOWN_ATTACHMENT_TOKENS = 50_000@@ -9,10 +13,25 @@ export function estimateAttachmentTokens({mediaType?: stringsize?: number | null}): number {- // An image is tiled by the provider, so its cost does not follow its byte- // length. Every other document is charged for what its bytes expand into.+ // Provider processing for images is based on rendered content, so compressed+ // file size is not a useful proxy for its token cost.if (mediaType?.startsWith('image/')) return IMAGE_ATTACHMENT_TOKENS+ // PDF bytes are compressed and therefore expand into substantially fewer+ // tokens than text bytes. Keep a floor for small documents while retaining a+ // conservative size signal for long, multi-page files.+ if (mediaType === 'application/pdf') {+ if (typeof size === 'number' && Number.isFinite(size) && size >= 0) {+ return Math.max(+ MIN_PDF_ATTACHMENT_TOKENS,+ Math.ceil(size / PDF_BYTES_PER_TOKEN)+ )+ }++ return UNKNOWN_ATTACHMENT_TOKENS+ }++ // Retain byte-based estimation for text-like formats added in the future.if (typeof size === 'number' && Number.isFinite(size) && size >= 0) {return Math.ceil(size / BYTES_PER_TOKEN)}diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.tsindex 1df12b21..b09571f5 100644--- a/lib/utils/context-window.ts+++ b/lib/utils/context-window.ts@@ -5,6 +5,8 @@ import { Model } from '../types/models'import { estimateAttachmentTokens } from './attachment-tokens'+type AttachmentTokenEstimates = ReadonlyMap<string, number>+interface ModelContextInfo {contextWindow: numberoutputTokens: number@@ -158,20 +160,26 @@ function getEncoder(modelId: string) {*/function estimateTokenCount(content: ModelMessage['content'],- modelId?: string+ modelId?: string,+ attachmentTokenEstimates?: AttachmentTokenEstimates): number {const text = extractTextContent(content)const attachmentTokens = Array.isArray(content)? content.reduce((total, part) => {if (part.type !== 'file') return total+ const url = getFileDataUrl(part.data)+ const resolvedEstimate = url+ ? attachmentTokenEstimates?.get(url)+ : undefined+return (total +- estimateAttachmentTokens({- mediaType: part.mediaType,- size:- part.data instanceof Uint8Array ? part.data.byteLength : undefined- })+ (resolvedEstimate ??+ estimateAttachmentTokens({+ mediaType: part.mediaType,+ size: getInlineFileByteLength(part.data)+ })))}, 0): 0@@ -206,13 +214,45 @@ function estimateTokenCount(return baseCount + overhead + attachmentTokens}+function getFileDataUrl(data: unknown): string | undefined {+ if (typeof data === 'string') return data+ if (data instanceof URL) return data.href+ if (!data || typeof data !== 'object') return undefined++ const fileData = data as { type?: unknown; url?: unknown }+ if (fileData.type !== 'url') return undefined++ if (fileData.url instanceof URL) return fileData.url.href+ return typeof fileData.url === 'string' ? fileData.url : undefined+}++function getInlineFileByteLength(data: unknown): number | undefined {+ if (data instanceof Uint8Array || data instanceof ArrayBuffer) {+ return data.byteLength+ }+ if (!data || typeof data !== 'object') return undefined++ const fileData = data as { type?: unknown; data?: unknown }+ if (fileData.type !== 'data') return undefined++ if (+ fileData.data instanceof Uint8Array ||+ fileData.data instanceof ArrayBuffer+ ) {+ return fileData.data.byteLength+ }++ return undefined+}+/*** Smart message truncation with priority for context preservation*/export function truncateMessages(messages: ModelMessage[],maxTokens: number,- modelId?: string+ modelId?: string,+ attachmentTokenEstimates?: AttachmentTokenEstimates): ModelMessage[] {// Input validationif (!messages || messages.length === 0) return []@@ -228,7 +268,7 @@ export function truncateMessages(// Calculate token counts for all messagesconst messageTokenCounts = messages.map(msg => ({message: msg,- tokens: estimateTokenCount(msg.content, modelId)+ tokens: estimateTokenCount(msg.content, modelId, attachmentTokenEstimates)}))// Calculate total tokens@@ -250,7 +290,8 @@ export function truncateMessages(if (firstUserMessage) {const firstUserTokens = estimateTokenCount(firstUserMessage.content,- modelId+ modelId,+ attachmentTokenEstimates)if (firstUserTokens < maxTokens * 0.3) {// Don't let first message take more than 30%@@ -278,7 +319,11 @@ export function truncateMessages(while (recentMessages.length > 0 && usedTokens + tokens > maxTokens) {const removed = recentMessages.shift()if (removed) {- usedTokens -= estimateTokenCount(removed.content, modelId)+ usedTokens -= estimateTokenCount(+ removed.content,+ modelId,+ attachmentTokenEstimates+ )}}if (usedTokens + tokens <= maxTokens) {@@ -312,13 +357,15 @@ export function truncateMessages(*/export function shouldTruncateMessages(messages: ModelMessage[],- model: Model+ model: Model,+ attachmentTokenEstimates?: AttachmentTokenEstimates): boolean {if (!messages || messages.length === 0) return falseconst maxTokens = getMaxAllowedTokens(model)const totalTokens = messages.reduce(- (sum, msg) => sum + estimateTokenCount(msg.content, model.id),+ (sum, msg) =>+ sum + estimateTokenCount(msg.content, model.id, attachmentTokenEstimates),0)return totalTokens > maxTokens