An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
perf(chat): cap pasted content replayed from earlier turns (#1064)
Yoshiki Miurab874413033fa
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.mdindex 685918ec..7a8d29db 100644--- a/docs/CONFIGURATION.md+++ b/docs/CONFIGURATION.md@@ -368,6 +368,20 @@ told to ask for a re-attach if it needs one again.HISTORY_ATTACHMENT_REPLAY_LIMIT=10```+Pasted content is replayed the same way, and a thread that pastes a whole file on every+turn carries every earlier copy. Older pasted blocks are replaced with a placeholder+giving their size once the earlier turns hold more than the budget below. The paste stays+visible in the conversation, and the model is told to ask for it again if it needs it.++```bash+# Optional: characters of pasted content replayed from earlier turns, excluding the+# newest message. Dropped in whole blocks so the prompt prefix stays cacheable, so the+# amount replayed can reach about twice the budget. The most recent earlier paste is+# always kept, even when it alone exceeds the budget. Set to 0 to replay every pasted+# block (default: 200000)+HISTORY_PASTED_CONTENT_CHAR_BUDGET=200000+```+### Content ExtractionUse Jina for enhanced content extraction:diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 0b215010..28fc07f2 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -29,6 +29,7 @@ import { isUsageLogging, logUsage } from '../utils/usage-logging'import { resolveAttachmentSizes } from './helpers/attachment-sizes'import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'import { capHistoricalAttachments } from './helpers/cap-historical-attachments'+import { capHistoricalPastedContent } from './helpers/cap-historical-pasted-content'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'@@ -233,7 +234,7 @@ export async function createChatStreamResponse(userId)const coldStartHistory = trimColdStartHistory(- messagesWithAttachmentSizes,+ capHistoricalPastedContent(messagesWithAttachmentSizes),{limit:COLD_START_HISTORY_TOKEN_LIMIT > 0diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.tsindex 01fcdecf..b66843e7 100644--- a/lib/streaming/create-ephemeral-chat-stream-response.ts+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts@@ -25,6 +25,7 @@ import { isUsageLogging, logUsage } from '../utils/usage-logging'import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'import { capHistoricalAttachments } from './helpers/cap-historical-attachments'+import { capHistoricalPastedContent } from './helpers/cap-historical-pasted-content'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'@@ -135,7 +136,11 @@ export async function createEphemeralChatStreamResponse(const messagesWithNonces = assignDataPartNonces(messages)const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)const messagesToConvert = dedupeAttachments(- capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec))+ capHistoricalPastedContent(+ capHistoricalAttachments(+ compactHistoricalMessages(messagesWithoutSpec)+ )+ ))carriedContext = summarizeCarriedContext(messagesToConvert)const attachmentTokenEstimates =diff --git a/lib/streaming/helpers/__tests__/cap-historical-pasted-content.test.ts b/lib/streaming/helpers/__tests__/cap-historical-pasted-content.test.tsnew file mode 100644index 00000000..dead0974--- /dev/null+++ b/lib/streaming/helpers/__tests__/cap-historical-pasted-content.test.ts@@ -0,0 +1,178 @@+import type { UIMessage } from 'ai'+import { describe, expect, it } from 'vitest'++import {+ capHistoricalPastedContent,+ parsePastedContentCharBudget+} from '../cap-historical-pasted-content'+import { trimColdStartHistory } from '../trim-cold-start-history'++const BUDGET = 100++function pasteTurn(id: string, length: number, text = 'Is this right?') {+ return {+ id,+ role: 'user',+ parts: [+ {+ type: 'data-pastedContent',+ data: { text: 'x'.repeat(length), nonce: 'abcd1234' }+ },+ { type: 'text', text }+ ]+ } as unknown as UIMessage+}++function textTurn(id: string, role: 'user' | 'assistant' = 'assistant') {+ return {+ id,+ role,+ parts: [{ type: 'text', text: 'Reply' }]+ } as unknown as UIMessage+}++function thread(pasteLengths: number[]): UIMessage[] {+ return pasteLengths.flatMap((length, i) => [+ pasteTurn(`u${i}`, length),+ textTurn(`a${i}`)+ ])+}++function omittedIds(messages: UIMessage[]): string[] {+ return messages+ .filter(message =>+ message.parts.some(+ part =>+ part.type === 'text' &&+ part.text.startsWith('[Pasted content omitted from history')+ )+ )+ .map(message => message.id)+}++describe('capHistoricalPastedContent', () => {+ it('leaves a thread within the budget untouched', () => {+ const messages = [...thread([40, 40]), textTurn('u2', 'user')]+ expect(capHistoricalPastedContent(messages, BUDGET)).toBe(messages)+ })++ it('replaces the oldest block with a placeholder naming its size', () => {+ const messages = [...thread([60, 60, 60, 60]), textTurn('u4', 'user')]+ const capped = capHistoricalPastedContent(messages, BUDGET)++ expect(omittedIds(capped)).toEqual(['u0', 'u1'])+ expect(capped[0].parts).toEqual([+ {+ type: 'text',+ text: '[Pasted content omitted from history: 60 characters. Ask the user to paste it again if you need to look at it again.]'+ },+ { type: 'text', text: 'Is this right?' }+ ])+ expect(capped[4]).toBe(messages[4])+ })++ it('keeps the replayed prefix stable between block crossings', () => {+ const lengths = [60, 60, 60, 60]+ const before = capHistoricalPastedContent(+ [...thread(lengths), textTurn('u4', 'user')],+ BUDGET+ )+ const after = capHistoricalPastedContent(+ [...thread([...lengths, 60]), textTurn('u5', 'user')],+ BUDGET+ )++ expect(omittedIds(after)).toEqual(omittedIds(before))+ expect(after.slice(0, before.length - 1)).toEqual(before.slice(0, -1))+ })++ it('keeps only the newest historical paste when each exceeds the budget', () => {+ const dropped = [1, 2, 3, 4].map(count =>+ omittedIds(+ capHistoricalPastedContent(+ [...thread(Array(count).fill(500)), textTurn('next', 'user')],+ BUDGET+ )+ )+ )++ expect(dropped).toEqual([[], ['u0'], ['u0', 'u1'], ['u0', 'u1', 'u2']])+ })++ it('keeps a small paste that follows an oversized one', () => {+ const messages = [...thread([500, 500, 10]), textTurn('u3', 'user')]++ expect(omittedIds(capHistoricalPastedContent(messages, BUDGET))).toEqual([+ 'u0'+ ])+ })++ it('never caps the newest user message', () => {+ const messages = [...thread([60, 60, 60, 60]), pasteTurn('u4', 5000)]+ const capped = capHistoricalPastedContent(messages, BUDGET)++ expect(capped[capped.length - 1]).toBe(messages[messages.length - 1])+ })++ it('ignores pasted parts without text', () => {+ const empty = {+ id: 'empty',+ role: 'user',+ parts: [{ type: 'data-pastedContent', data: {} }]+ } as unknown as UIMessage+ const messages = [empty, ...thread([60, 60]), textTurn('u2', 'user')]++ expect(capHistoricalPastedContent(messages, BUDGET)).toBe(messages)+ })++ it('is disabled by a zero budget', () => {+ const messages = [...thread([500, 500, 500, 500]), textTurn('u4', 'user')]+ expect(capHistoricalPastedContent(messages, 0)).toBe(messages)+ })+})++describe('capHistoricalPastedContent before the cold-start trim', () => {+ const start = Date.parse('2026-01-01T00:00:00Z')+ const minute = 60 * 1000++ function at(message: UIMessage, minutes: number): UIMessage {+ return {+ ...message,+ metadata: { createdAt: new Date(start + minutes * minute).toISOString() }+ } as UIMessage+ }++ const history = [0, 1, 2, 3, 4].flatMap(i => [+ at(pasteTurn(`u${i}`, 4000), i),+ textTurn(`a${i}`)+ ])+ const messages = [...history, at(textTurn('u5', 'user'), 120)]+ const options = { now: new Date(start + 120 * minute), limit: 2500 }++ it('keeps the turns a cold return would otherwise drop', () => {+ const uncapped = trimColdStartHistory(messages, options).messages+ const capped = trimColdStartHistory(+ capHistoricalPastedContent(messages, 4000),+ options+ ).messages++ expect(uncapped.length).toBeLessThan(messages.length)+ expect(capped.map(message => message.id)).toEqual(+ messages.map(message => message.id)+ )+ })+})++describe('parsePastedContentCharBudget', () => {+ it('falls back to the default on unusable values', () => {+ for (const raw of [undefined, '', ' ', 'abc', '-1']) {+ expect(parsePastedContentCharBudget(raw)).toBe(200_000)+ }+ })++ it('disables the budget only on an explicit zero', () => {+ expect(parsePastedContentCharBudget('0')).toBe(0)+ expect(parsePastedContentCharBudget('0.5')).toBe(1)+ expect(parsePastedContentCharBudget('1200.9')).toBe(1200)+ })+})diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.tsindex c2fa0e83..8b00ccd2 100644--- a/lib/streaming/helpers/cap-historical-attachments.ts+++ b/lib/streaming/helpers/cap-historical-attachments.ts@@ -81,7 +81,11 @@ function getDroppedCount(total: number, limit: number): number {* exceeds the budget. That overshoot is the cost of prefix stability while* guaranteeing that at least one historical attachment always survives.*/-function getWeightDroppedCount(weights: number[], tokenBudget: number): number {+export function getWeightDroppedCount(+ weights: number[],+ tokenBudget: number,+ minBlockSize: number = MIN_WEIGHT_BLOCK_ATTACHMENTS+): number {let blockTokens = 0let blockAttachmentCount = 0let latestBlockEnd = 0@@ -90,10 +94,7 @@ function getWeightDroppedCount(weights: number[], tokenBudget: number): number {for (let i = 0; i < weights.length; i++) {blockTokens += weights[i]blockAttachmentCount += 1- if (- blockTokens < tokenBudget ||- blockAttachmentCount < MIN_WEIGHT_BLOCK_ATTACHMENTS- ) {+ if (blockTokens < tokenBudget || blockAttachmentCount < minBlockSize) {continue}diff --git a/lib/streaming/helpers/cap-historical-pasted-content.ts b/lib/streaming/helpers/cap-historical-pasted-content.tsnew file mode 100644index 00000000..c76e9a2b--- /dev/null+++ b/lib/streaming/helpers/cap-historical-pasted-content.ts@@ -0,0 +1,84 @@+import type { UIMessage } from 'ai'++import { getWeightDroppedCount } from './cap-historical-attachments'+import { isSourceContextMessage } from './compact-historical-messages'++const DEFAULT_PASTED_CONTENT_CHAR_BUDGET = 200_000++export function parsePastedContentCharBudget(raw: string | undefined): number {+ const value = raw?.trim()+ if (!value) return DEFAULT_PASTED_CONTENT_CHAR_BUDGET++ const parsed = Number(value)+ if (!Number.isFinite(parsed) || parsed < 0) {+ return DEFAULT_PASTED_CONTENT_CHAR_BUDGET+ }++ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))+}++export const HISTORY_PASTED_CONTENT_CHAR_BUDGET = parsePastedContentCharBudget(+ process.env.HISTORY_PASTED_CONTENT_CHAR_BUDGET+)++function getPastedLength(part: UIMessage['parts'][number]): number {+ if (part.type !== 'data-pastedContent') return 0++ const text = (part as { data?: { text?: unknown } }).data?.text+ return typeof text === 'string' ? text.length : 0+}++/**+ * Caps how much pasted content from earlier turns is replayed to the model.+ *+ * Older blocks become a text placeholder giving their size. They are dropped+ * by the same weight quantization as attachments, so the prompt prefix stays+ * cacheable between block crossings. A block may be a single paste: when one+ * paste exceeds the budget, moving the boundary on each new paste rewrites far+ * less than carrying a second copy would. The newest user message and the+ * newest historical paste are never capped.+ */+export function capHistoricalPastedContent(+ messages: UIMessage[],+ charBudget: number = HISTORY_PASTED_CONTENT_CHAR_BUDGET+): UIMessage[] {+ if (charBudget <= 0) return messages++ const currentTurnIndex = messages.findLastIndex(+ message => message.role === 'user' && !isSourceContextMessage(message)+ )+ const historyEnd =+ currentTurnIndex === -1 ? messages.length : currentTurnIndex++ const weights: number[] = []+ for (let i = 0; i < historyEnd; i++) {+ for (const part of messages[i].parts) {+ const length = getPastedLength(part)+ if (length > 0) weights.push(length)+ }+ }++ const dropCount = getWeightDroppedCount(weights, charBudget, 1)+ if (dropCount === 0) return messages++ let seen = 0+ return messages.map((message, index) => {+ if (index >= historyEnd || seen >= dropCount) return message+ if (!message.parts.some(part => getPastedLength(part) > 0)) return message++ const parts = message.parts.map(part => {+ const length = getPastedLength(part)+ if (length === 0) return part++ seen += 1+ if (seen > dropCount) return part++ return {+ type: 'text' as const,+ text: `[Pasted content omitted from history: ${length} characters. Ask the user to paste it again if you need to look at it again.]`+ }+ })++ return { ...message, parts }+ })+}