Skip to content

An AI-powered search engine with a generative UI

fix: align PDF attachment token estimates (#991)

Sushant Tiwarif16c88ac7f2c
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index aefd10ba..c4f45ebd 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -26,6 +26,7 @@ import { perfLog, perfTime } from '../utils/perf-logging'
import { isUsageLogging, logUsage } from '../utils/usage-logging'
import { resolveAttachmentSizes } from './helpers/attachment-sizes'
+import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'
import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
@@ -212,6 +213,8 @@ export async function createChatStreamResponse(
)
)
carriedContext = summarizeCarriedContext(messagesToConvert)
+ const attachmentTokenEstimates =
+ buildAttachmentTokenEstimates(messagesToConvert)
// Convert to model messages and apply context window management
streamErrorStage = 'convert-messages'
@@ -220,10 +223,17 @@ export async function createChatStreamResponse(
})
streamErrorStage = 'truncate-messages'
- if (shouldTruncateMessages(modelMessages, model)) {
+ if (
+ shouldTruncateMessages(modelMessages, model, attachmentTokenEstimates)
+ ) {
const maxTokens = getMaxAllowedTokens(model)
const originalCount = modelMessages.length
- modelMessages = truncateMessages(modelMessages, maxTokens, model.id)
+ modelMessages = truncateMessages(
+ modelMessages,
+ maxTokens,
+ model.id,
+ attachmentTokenEstimates
+ )
contextWindowTruncated = true
if (process.env.NODE_ENV === 'development') {
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index 60eb3b92..7b2f770e 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -22,6 +22,7 @@ import {
import { getTextFromParts } from '../utils/message-utils'
import { isUsageLogging, logUsage } from '../utils/usage-logging'
+import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'
import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
@@ -136,6 +137,8 @@ export async function createEphemeralChatStreamResponse(
capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec))
)
carriedContext = summarizeCarriedContext(messagesToConvert)
+ const attachmentTokenEstimates =
+ buildAttachmentTokenEstimates(messagesToConvert)
streamErrorStage = 'convert-messages'
let modelMessages = await convertToModelMessages(messagesToConvert, {
@@ -143,9 +146,16 @@ export async function createEphemeralChatStreamResponse(
})
streamErrorStage = 'truncate-messages'
- if (shouldTruncateMessages(modelMessages, model)) {
+ if (
+ shouldTruncateMessages(modelMessages, model, attachmentTokenEstimates)
+ ) {
const maxTokens = getMaxAllowedTokens(model)
- modelMessages = truncateMessages(modelMessages, maxTokens, model.id)
+ modelMessages = truncateMessages(
+ modelMessages,
+ maxTokens,
+ model.id,
+ attachmentTokenEstimates
+ )
contextWindowTruncated = true
}
diff --git a/lib/streaming/helpers/__tests__/attachment-token-estimates.test.ts b/lib/streaming/helpers/__tests__/attachment-token-estimates.test.ts
new file mode 100644
index 00000000..26c7419a
--- /dev/null
+++ b/lib/streaming/helpers/__tests__/attachment-token-estimates.test.ts
@@ -0,0 +1,80 @@
+import { convertToModelMessages, type UIMessage } from 'ai'
+import { describe, expect, it } from 'vitest'
+
+import {
+ estimateAttachmentTokens,
+ UNKNOWN_ATTACHMENT_TOKENS
+} from '@/lib/utils/attachment-tokens'
+import { shouldTruncateMessages } from '@/lib/utils/context-window'
+
+import { buildAttachmentTokenEstimates } from '../attachment-token-estimates'
+
+function messages(): UIMessage[] {
+ return [
+ {
+ id: 'u1',
+ role: 'user',
+ parts: [
+ { type: 'text', text: 'Compare these files' },
+ {
+ type: 'file',
+ mediaType: 'application/pdf',
+ url: 'https://example.com/large.pdf',
+ size: 20_000_000
+ },
+ {
+ type: 'file',
+ mediaType: 'application/pdf',
+ url: 'https://example.com/unknown.pdf'
+ }
+ ]
+ } as unknown as UIMessage
+ ]
+}
+
+describe('buildAttachmentTokenEstimates', () => {
+ it('indexes the exact pre-conversion estimate by attachment URL', () => {
+ const estimates = buildAttachmentTokenEstimates(messages())
+
+ expect(estimates.get('https://example.com/large.pdf')).toBe(
+ estimateAttachmentTokens({
+ mediaType: 'application/pdf',
+ size: 20_000_000
+ })
+ )
+ expect(estimates.get('https://example.com/unknown.pdf')).toBe(
+ UNKNOWN_ATTACHMENT_TOKENS
+ )
+ })
+
+ it('ignores non-file parts', () => {
+ const estimates = buildAttachmentTokenEstimates([
+ {
+ id: 'u1',
+ role: 'user',
+ parts: [{ type: 'text', text: 'No files' }]
+ }
+ ])
+
+ expect(estimates.size).toBe(0)
+ })
+
+ it('matches converted URL parts in the context-window guard', async () => {
+ const input = messages()
+ const estimates = buildAttachmentTokenEstimates(input)
+ const modelMessages = await convertToModelMessages(input)
+
+ expect(
+ shouldTruncateMessages(
+ modelMessages,
+ {
+ id: 'gpt-4o-mini',
+ name: 'GPT-4o mini',
+ provider: 'OpenAI',
+ providerId: 'openai'
+ },
+ estimates
+ )
+ ).toBe(true)
+ })
+})
diff --git a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
index d1ff47d6..f3c85821 100644
--- a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
+++ b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
@@ -235,7 +235,7 @@ describe('capHistoricalAttachments', () => {
it('bounds heavy attachments below the count limit by weight blocks', () => {
const messages = pdfThread(4).concat(userTurn('current', 0))
- const capped = capHistoricalAttachments(messages, LIMIT, 200_000)
+ const capped = capHistoricalAttachments(messages, LIMIT, 20_000)
expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])
expect(placeholders(capped)).toHaveLength(2)
@@ -245,12 +245,12 @@ describe('capHistoricalAttachments', () => {
const atFour = capHistoricalAttachments(
pdfThread(4, 1_000_000).concat(userTurn('current', 0)),
0,
- 200_000
+ 20_000
)
const atFive = capHistoricalAttachments(
pdfThread(5, 1_000_000).concat(userTurn('current', 0)),
0,
- 200_000
+ 20_000
)
const replayedPrefixLength = atFour.length - 1
@@ -265,7 +265,7 @@ describe('capHistoricalAttachments', () => {
const capped = capHistoricalAttachments(
pdfThread(4).concat(userTurn('current', 0)),
0,
- 200_000
+ 20_000
)
expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])
@@ -280,11 +280,19 @@ describe('capHistoricalAttachments', () => {
})
it('leaves a thread within the weight budget untouched', () => {
- const messages = thread(5, 1).concat(userTurn('current', 0))
+ const messages = pdfThread(5, 100_000).concat(userTurn('current', 0))
expect(capHistoricalAttachments(messages, LIMIT, 200_000)).toBe(messages)
})
+ it('bounds multiple large PDFs with the default weight budget', () => {
+ const messages = pdfThread(4, 10_000_000).concat(userTurn('current', 0))
+ const capped = capHistoricalAttachments(messages, LIMIT, 200_000)
+
+ expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])
+ expect(placeholders(capped)).toHaveLength(2)
+ })
+
it('never applies the weight bound to the newest user message', () => {
const messages = thread(1, 0).concat(userTurn('current', 0))
messages.at(-1)?.parts.unshift({
diff --git a/lib/streaming/helpers/__tests__/summarize-carried-context.test.ts b/lib/streaming/helpers/__tests__/summarize-carried-context.test.ts
index 44289e46..dc2f67d0 100644
--- a/lib/streaming/helpers/__tests__/summarize-carried-context.test.ts
+++ b/lib/streaming/helpers/__tests__/summarize-carried-context.test.ts
@@ -2,7 +2,10 @@ import type { UIMessage } from 'ai'
import { describe, expect, it } from 'vitest'
import { summarizeCarriedContext } from '@/lib/streaming/helpers/summarize-carried-context'
-import { IMAGE_ATTACHMENT_TOKENS } from '@/lib/utils/attachment-tokens'
+import {
+ IMAGE_ATTACHMENT_TOKENS,
+ MIN_PDF_ATTACHMENT_TOKENS
+} from '@/lib/utils/attachment-tokens'
function createMessages(parts: unknown[]): UIMessage[] {
return [{ id: 'message-1', role: 'user', parts: parts as UIMessage['parts'] }]
@@ -34,7 +37,7 @@ describe('summarizeCarriedContext', () => {
)
).toEqual({
attachments: 2,
- attachmentTokens: IMAGE_ATTACHMENT_TOKENS + 10
+ attachmentTokens: IMAGE_ATTACHMENT_TOKENS + MIN_PDF_ATTACHMENT_TOKENS
})
})
@@ -65,6 +68,9 @@ describe('summarizeCarriedContext', () => {
{ type: 'data-noteContext', data: { text: null } }
])
)
- ).toEqual({ attachments: 1 })
+ ).toEqual({
+ attachments: 1,
+ attachmentTokens: MIN_PDF_ATTACHMENT_TOKENS
+ })
})
})
diff --git a/lib/streaming/helpers/attachment-token-estimates.ts b/lib/streaming/helpers/attachment-token-estimates.ts
new file mode 100644
index 00000000..ecd65cab
--- /dev/null
+++ b/lib/streaming/helpers/attachment-token-estimates.ts
@@ -0,0 +1,30 @@
+import type { UIMessage } from 'ai'
+
+import { estimateAttachmentTokens } from '@/lib/utils/attachment-tokens'
+
+import { isFilePart } from './attachment-parts'
+
+/**
+ * Builds the token estimates before model-message conversion removes custom
+ * attachment metadata such as the resolved storage size. The URL is the stable
+ * value shared by both representations and is not sent to providers as extra
+ * metadata.
+ */
+export function buildAttachmentTokenEstimates(
+ messages: UIMessage[]
+): ReadonlyMap<string, number> {
+ const estimates = new Map<string, number>()
+
+ for (const message of messages) {
+ for (const part of message.parts) {
+ if (!isFilePart(part)) continue
+
+ const file = part as { mediaType?: string; size?: number; url?: string }
+ if (!file.url) continue
+
+ estimates.set(file.url, estimateAttachmentTokens(file))
+ }
+ }
+
+ return estimates
+}
diff --git a/lib/utils/__tests__/attachment-tokens.test.ts b/lib/utils/__tests__/attachment-tokens.test.ts
index a7a0e51a..2df6b6ca 100644
--- a/lib/utils/__tests__/attachment-tokens.test.ts
+++ b/lib/utils/__tests__/attachment-tokens.test.ts
@@ -4,6 +4,8 @@ import {
BYTES_PER_TOKEN,
estimateAttachmentTokens,
IMAGE_ATTACHMENT_TOKENS,
+ MIN_PDF_ATTACHMENT_TOKENS,
+ PDF_BYTES_PER_TOKEN,
UNKNOWN_ATTACHMENT_TOKENS
} from '../attachment-tokens'
@@ -14,13 +16,19 @@ describe('estimateAttachmentTokens', () => {
).toBe(IMAGE_ATTACHMENT_TOKENS)
})
- it('estimates known PDF sizes from their byte length', () => {
+ it('uses a floor for small PDFs and scales large PDFs conservatively', () => {
expect(
estimateAttachmentTokens({ mediaType: 'application/pdf', size: 3_901 })
- ).toBe(Math.ceil(3_901 / BYTES_PER_TOKEN))
+ ).toBe(MIN_PDF_ATTACHMENT_TOKENS)
+ expect(
+ estimateAttachmentTokens({
+ mediaType: 'application/pdf',
+ size: 20_000_000
+ })
+ ).toBe(Math.ceil(20_000_000 / PDF_BYTES_PER_TOKEN))
})
- it('uses the unknown estimate when size is unavailable', () => {
+ it('uses a conservative PDF estimate when size is unavailable', () => {
expect(estimateAttachmentTokens({ mediaType: 'application/pdf' })).toBe(
UNKNOWN_ATTACHMENT_TOKENS
)
diff --git a/lib/utils/__tests__/context-window.test.ts b/lib/utils/__tests__/context-window.test.ts
index c577a612..8c64195a 100644
--- a/lib/utils/__tests__/context-window.test.ts
+++ b/lib/utils/__tests__/context-window.test.ts
@@ -100,6 +100,11 @@ describe('context-window', () => {
type: 'file',
mediaType: 'application/pdf',
data: 'https://example.com/report.pdf'
+ },
+ {
+ type: 'file',
+ mediaType: 'application/pdf',
+ data: 'https://example.com/appendix.pdf'
}
]
}
@@ -108,6 +113,47 @@ describe('context-window', () => {
expect(shouldTruncateMessages(messages, unknownModel)).toBe(true)
})
+ test('does not treat a URL-backed PDF as an unknown-size attachment', () => {
+ const unknownModel: Model = { ...mockModel, id: 'unknown-model' }
+ const url = 'https://example.com/report.pdf'
+ const messages: ModelMessage[] = [
+ {
+ role: 'user',
+ content: [
+ {
+ type: 'file',
+ mediaType: 'application/pdf',
+ data: { type: 'url', url: new URL(url) }
+ }
+ ]
+ }
+ ]
+
+ expect(
+ shouldTruncateMessages(messages, unknownModel, new Map([[url, 10_000]]))
+ ).toBe(false)
+ })
+
+ test('uses a resolved large-PDF estimate in the context guard', () => {
+ const url = 'https://example.com/large-report.pdf'
+ const messages: ModelMessage[] = [
+ {
+ role: 'user',
+ content: [
+ {
+ type: 'file',
+ mediaType: 'application/pdf',
+ data: { type: 'url', url: new URL(url) }
+ }
+ ]
+ }
+ ]
+
+ expect(
+ shouldTruncateMessages(messages, mockModel, new Map([[url, 400_000]]))
+ ).toBe(true)
+ })
+
test('handles null/undefined messages gracefully', () => {
expect(shouldTruncateMessages(null as any, mockModel)).toBe(false)
expect(shouldTruncateMessages(undefined as any, mockModel)).toBe(false)
@@ -209,6 +255,31 @@ describe('context-window', () => {
expect(result.length).toBeGreaterThan(0)
})
+ test('uses resolved attachment estimates while truncating', () => {
+ const url = 'https://example.com/large-report.pdf'
+ const messages: ModelMessage[] = [
+ {
+ role: 'user',
+ content: [
+ {
+ type: 'file',
+ mediaType: 'application/pdf',
+ data: { type: 'url', url: new URL(url) }
+ }
+ ]
+ }
+ ]
+
+ expect(
+ truncateMessages(
+ messages,
+ 100_000,
+ mockModel.id,
+ new Map([[url, 400_000]])
+ )
+ ).toEqual([])
+ })
+
test('handles undefined content gracefully', () => {
const messages: ModelMessage[] = [
{ role: 'user', content: '' },
diff --git a/lib/utils/attachment-tokens.ts b/lib/utils/attachment-tokens.ts
index 17735e6c..5387b771 100644
--- a/lib/utils/attachment-tokens.ts
+++ b/lib/utils/attachment-tokens.ts
@@ -1,4 +1,8 @@
export const IMAGE_ATTACHMENT_TOKENS = 10_000
+export const MIN_PDF_ATTACHMENT_TOKENS = 10_000
+// A 500 KB PDF reaches the small-document floor; larger containers scale so
+// multi-page files cannot all hide behind a flat estimate.
+export const PDF_BYTES_PER_TOKEN = 50
export const BYTES_PER_TOKEN = 3.9
export const UNKNOWN_ATTACHMENT_TOKENS = 50_000
@@ -9,10 +13,25 @@ export function estimateAttachmentTokens({
mediaType?: string
size?: number | null
}): number {
- // An image is tiled by the provider, so its cost does not follow its byte
- // length. Every other document is charged for what its bytes expand into.
+ // Provider processing for images is based on rendered content, so compressed
+ // file size is not a useful proxy for its token cost.
if (mediaType?.startsWith('image/')) return IMAGE_ATTACHMENT_TOKENS
+ // PDF bytes are compressed and therefore expand into substantially fewer
+ // tokens than text bytes. Keep a floor for small documents while retaining a
+ // conservative size signal for long, multi-page files.
+ if (mediaType === 'application/pdf') {
+ if (typeof size === 'number' && Number.isFinite(size) && size >= 0) {
+ return Math.max(
+ MIN_PDF_ATTACHMENT_TOKENS,
+ Math.ceil(size / PDF_BYTES_PER_TOKEN)
+ )
+ }
+
+ return UNKNOWN_ATTACHMENT_TOKENS
+ }
+
+ // Retain byte-based estimation for text-like formats added in the future.
if (typeof size === 'number' && Number.isFinite(size) && size >= 0) {
return Math.ceil(size / BYTES_PER_TOKEN)
}
diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.ts
index 1df12b21..b09571f5 100644
--- a/lib/utils/context-window.ts
+++ b/lib/utils/context-window.ts
@@ -5,6 +5,8 @@ import { Model } from '../types/models'
import { estimateAttachmentTokens } from './attachment-tokens'
+type AttachmentTokenEstimates = ReadonlyMap<string, number>
+
interface ModelContextInfo {
contextWindow: number
outputTokens: number
@@ -158,20 +160,26 @@ function getEncoder(modelId: string) {
*/
function estimateTokenCount(
content: ModelMessage['content'],
- modelId?: string
+ modelId?: string,
+ attachmentTokenEstimates?: AttachmentTokenEstimates
): number {
const text = extractTextContent(content)
const attachmentTokens = Array.isArray(content)
? content.reduce((total, part) => {
if (part.type !== 'file') return total
+ const url = getFileDataUrl(part.data)
+ const resolvedEstimate = url
+ ? attachmentTokenEstimates?.get(url)
+ : undefined
+
return (
total +
- estimateAttachmentTokens({
- mediaType: part.mediaType,
- size:
- part.data instanceof Uint8Array ? part.data.byteLength : undefined
- })
+ (resolvedEstimate ??
+ estimateAttachmentTokens({
+ mediaType: part.mediaType,
+ size: getInlineFileByteLength(part.data)
+ }))
)
}, 0)
: 0
@@ -206,13 +214,45 @@ function estimateTokenCount(
return baseCount + overhead + attachmentTokens
}
+function getFileDataUrl(data: unknown): string | undefined {
+ if (typeof data === 'string') return data
+ if (data instanceof URL) return data.href
+ if (!data || typeof data !== 'object') return undefined
+
+ const fileData = data as { type?: unknown; url?: unknown }
+ if (fileData.type !== 'url') return undefined
+
+ if (fileData.url instanceof URL) return fileData.url.href
+ return typeof fileData.url === 'string' ? fileData.url : undefined
+}
+
+function getInlineFileByteLength(data: unknown): number | undefined {
+ if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
+ return data.byteLength
+ }
+ if (!data || typeof data !== 'object') return undefined
+
+ const fileData = data as { type?: unknown; data?: unknown }
+ if (fileData.type !== 'data') return undefined
+
+ if (
+ fileData.data instanceof Uint8Array ||
+ fileData.data instanceof ArrayBuffer
+ ) {
+ return fileData.data.byteLength
+ }
+
+ return undefined
+}
+
/**
* Smart message truncation with priority for context preservation
*/
export function truncateMessages(
messages: ModelMessage[],
maxTokens: number,
- modelId?: string
+ modelId?: string,
+ attachmentTokenEstimates?: AttachmentTokenEstimates
): ModelMessage[] {
// Input validation
if (!messages || messages.length === 0) return []
@@ -228,7 +268,7 @@ export function truncateMessages(
// Calculate token counts for all messages
const messageTokenCounts = messages.map(msg => ({
message: msg,
- tokens: estimateTokenCount(msg.content, modelId)
+ tokens: estimateTokenCount(msg.content, modelId, attachmentTokenEstimates)
}))
// Calculate total tokens
@@ -250,7 +290,8 @@ export function truncateMessages(
if (firstUserMessage) {
const firstUserTokens = estimateTokenCount(
firstUserMessage.content,
- modelId
+ modelId,
+ attachmentTokenEstimates
)
if (firstUserTokens < maxTokens * 0.3) {
// Don't let first message take more than 30%
@@ -278,7 +319,11 @@ export function truncateMessages(
while (recentMessages.length > 0 && usedTokens + tokens > maxTokens) {
const removed = recentMessages.shift()
if (removed) {
- usedTokens -= estimateTokenCount(removed.content, modelId)
+ usedTokens -= estimateTokenCount(
+ removed.content,
+ modelId,
+ attachmentTokenEstimates
+ )
}
}
if (usedTokens + tokens <= maxTokens) {
@@ -312,13 +357,15 @@ export function truncateMessages(
*/
export function shouldTruncateMessages(
messages: ModelMessage[],
- model: Model
+ model: Model,
+ attachmentTokenEstimates?: AttachmentTokenEstimates
): boolean {
if (!messages || messages.length === 0) return false
const maxTokens = getMaxAllowedTokens(model)
const totalTokens = messages.reduce(
- (sum, msg) => sum + estimateTokenCount(msg.content, model.id),
+ (sum, msg) =>
+ sum + estimateTokenCount(msg.content, model.id, attachmentTokenEstimates),
0
)
return totalTokens > maxTokens