Skip to content

An AI-powered search engine with a generative UI

perf(chat): cap pasted content replayed from earlier turns (#1064)

Yoshiki Miurab874413033fa
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index 685918ec..7a8d29db 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -368,6 +368,20 @@ told to ask for a re-attach if it needs one again.
HISTORY_ATTACHMENT_REPLAY_LIMIT=10
```
+Pasted content is replayed the same way, and a thread that pastes a whole file on every
+turn carries every earlier copy. Older pasted blocks are replaced with a placeholder
+giving their size once the earlier turns hold more than the budget below. The paste stays
+visible in the conversation, and the model is told to ask for it again if it needs it.
+
+```bash
+# Optional: characters of pasted content replayed from earlier turns, excluding the
+# newest message. Dropped in whole blocks so the prompt prefix stays cacheable, so the
+# amount replayed can reach about twice the budget. The most recent earlier paste is
+# always kept, even when it alone exceeds the budget. Set to 0 to replay every pasted
+# block (default: 200000)
+HISTORY_PASTED_CONTENT_CHAR_BUDGET=200000
+```
+
### Content Extraction
Use Jina for enhanced content extraction:
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index 0b215010..28fc07f2 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -29,6 +29,7 @@ import { isUsageLogging, logUsage } from '../utils/usage-logging'
import { resolveAttachmentSizes } from './helpers/attachment-sizes'
import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'
import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
+import { capHistoricalPastedContent } from './helpers/cap-historical-pasted-content'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
@@ -233,7 +234,7 @@ export async function createChatStreamResponse(
userId
)
const coldStartHistory = trimColdStartHistory(
- messagesWithAttachmentSizes,
+ capHistoricalPastedContent(messagesWithAttachmentSizes),
{
limit:
COLD_START_HISTORY_TOKEN_LIMIT > 0
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index 01fcdecf..b66843e7 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -25,6 +25,7 @@ import { isUsageLogging, logUsage } from '../utils/usage-logging'
import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'
import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
+import { capHistoricalPastedContent } from './helpers/cap-historical-pasted-content'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
@@ -135,7 +136,11 @@ export async function createEphemeralChatStreamResponse(
const messagesWithNonces = assignDataPartNonces(messages)
const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)
const messagesToConvert = dedupeAttachments(
- capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec))
+ capHistoricalPastedContent(
+ capHistoricalAttachments(
+ compactHistoricalMessages(messagesWithoutSpec)
+ )
+ )
)
carriedContext = summarizeCarriedContext(messagesToConvert)
const attachmentTokenEstimates =
diff --git a/lib/streaming/helpers/__tests__/cap-historical-pasted-content.test.ts b/lib/streaming/helpers/__tests__/cap-historical-pasted-content.test.ts
new file mode 100644
index 00000000..dead0974
--- /dev/null
+++ b/lib/streaming/helpers/__tests__/cap-historical-pasted-content.test.ts
@@ -0,0 +1,178 @@
+import type { UIMessage } from 'ai'
+import { describe, expect, it } from 'vitest'
+
+import {
+ capHistoricalPastedContent,
+ parsePastedContentCharBudget
+} from '../cap-historical-pasted-content'
+import { trimColdStartHistory } from '../trim-cold-start-history'
+
+const BUDGET = 100
+
+function pasteTurn(id: string, length: number, text = 'Is this right?') {
+ return {
+ id,
+ role: 'user',
+ parts: [
+ {
+ type: 'data-pastedContent',
+ data: { text: 'x'.repeat(length), nonce: 'abcd1234' }
+ },
+ { type: 'text', text }
+ ]
+ } as unknown as UIMessage
+}
+
+function textTurn(id: string, role: 'user' | 'assistant' = 'assistant') {
+ return {
+ id,
+ role,
+ parts: [{ type: 'text', text: 'Reply' }]
+ } as unknown as UIMessage
+}
+
+function thread(pasteLengths: number[]): UIMessage[] {
+ return pasteLengths.flatMap((length, i) => [
+ pasteTurn(`u${i}`, length),
+ textTurn(`a${i}`)
+ ])
+}
+
+function omittedIds(messages: UIMessage[]): string[] {
+ return messages
+ .filter(message =>
+ message.parts.some(
+ part =>
+ part.type === 'text' &&
+ part.text.startsWith('[Pasted content omitted from history')
+ )
+ )
+ .map(message => message.id)
+}
+
+describe('capHistoricalPastedContent', () => {
+ it('leaves a thread within the budget untouched', () => {
+ const messages = [...thread([40, 40]), textTurn('u2', 'user')]
+ expect(capHistoricalPastedContent(messages, BUDGET)).toBe(messages)
+ })
+
+ it('replaces the oldest block with a placeholder naming its size', () => {
+ const messages = [...thread([60, 60, 60, 60]), textTurn('u4', 'user')]
+ const capped = capHistoricalPastedContent(messages, BUDGET)
+
+ expect(omittedIds(capped)).toEqual(['u0', 'u1'])
+ expect(capped[0].parts).toEqual([
+ {
+ type: 'text',
+ text: '[Pasted content omitted from history: 60 characters. Ask the user to paste it again if you need to look at it again.]'
+ },
+ { type: 'text', text: 'Is this right?' }
+ ])
+ expect(capped[4]).toBe(messages[4])
+ })
+
+ it('keeps the replayed prefix stable between block crossings', () => {
+ const lengths = [60, 60, 60, 60]
+ const before = capHistoricalPastedContent(
+ [...thread(lengths), textTurn('u4', 'user')],
+ BUDGET
+ )
+ const after = capHistoricalPastedContent(
+ [...thread([...lengths, 60]), textTurn('u5', 'user')],
+ BUDGET
+ )
+
+ expect(omittedIds(after)).toEqual(omittedIds(before))
+ expect(after.slice(0, before.length - 1)).toEqual(before.slice(0, -1))
+ })
+
+ it('keeps only the newest historical paste when each exceeds the budget', () => {
+ const dropped = [1, 2, 3, 4].map(count =>
+ omittedIds(
+ capHistoricalPastedContent(
+ [...thread(Array(count).fill(500)), textTurn('next', 'user')],
+ BUDGET
+ )
+ )
+ )
+
+ expect(dropped).toEqual([[], ['u0'], ['u0', 'u1'], ['u0', 'u1', 'u2']])
+ })
+
+ it('keeps a small paste that follows an oversized one', () => {
+ const messages = [...thread([500, 500, 10]), textTurn('u3', 'user')]
+
+ expect(omittedIds(capHistoricalPastedContent(messages, BUDGET))).toEqual([
+ 'u0'
+ ])
+ })
+
+ it('never caps the newest user message', () => {
+ const messages = [...thread([60, 60, 60, 60]), pasteTurn('u4', 5000)]
+ const capped = capHistoricalPastedContent(messages, BUDGET)
+
+ expect(capped[capped.length - 1]).toBe(messages[messages.length - 1])
+ })
+
+ it('ignores pasted parts without text', () => {
+ const empty = {
+ id: 'empty',
+ role: 'user',
+ parts: [{ type: 'data-pastedContent', data: {} }]
+ } as unknown as UIMessage
+ const messages = [empty, ...thread([60, 60]), textTurn('u2', 'user')]
+
+ expect(capHistoricalPastedContent(messages, BUDGET)).toBe(messages)
+ })
+
+ it('is disabled by a zero budget', () => {
+ const messages = [...thread([500, 500, 500, 500]), textTurn('u4', 'user')]
+ expect(capHistoricalPastedContent(messages, 0)).toBe(messages)
+ })
+})
+
+describe('capHistoricalPastedContent before the cold-start trim', () => {
+ const start = Date.parse('2026-01-01T00:00:00Z')
+ const minute = 60 * 1000
+
+ function at(message: UIMessage, minutes: number): UIMessage {
+ return {
+ ...message,
+ metadata: { createdAt: new Date(start + minutes * minute).toISOString() }
+ } as UIMessage
+ }
+
+ const history = [0, 1, 2, 3, 4].flatMap(i => [
+ at(pasteTurn(`u${i}`, 4000), i),
+ textTurn(`a${i}`)
+ ])
+ const messages = [...history, at(textTurn('u5', 'user'), 120)]
+ const options = { now: new Date(start + 120 * minute), limit: 2500 }
+
+ it('keeps the turns a cold return would otherwise drop', () => {
+ const uncapped = trimColdStartHistory(messages, options).messages
+ const capped = trimColdStartHistory(
+ capHistoricalPastedContent(messages, 4000),
+ options
+ ).messages
+
+ expect(uncapped.length).toBeLessThan(messages.length)
+ expect(capped.map(message => message.id)).toEqual(
+ messages.map(message => message.id)
+ )
+ })
+})
+
+describe('parsePastedContentCharBudget', () => {
+ it('falls back to the default on unusable values', () => {
+ for (const raw of [undefined, '', ' ', 'abc', '-1']) {
+ expect(parsePastedContentCharBudget(raw)).toBe(200_000)
+ }
+ })
+
+ it('disables the budget only on an explicit zero', () => {
+ expect(parsePastedContentCharBudget('0')).toBe(0)
+ expect(parsePastedContentCharBudget('0.5')).toBe(1)
+ expect(parsePastedContentCharBudget('1200.9')).toBe(1200)
+ })
+})
diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.ts
index c2fa0e83..8b00ccd2 100644
--- a/lib/streaming/helpers/cap-historical-attachments.ts
+++ b/lib/streaming/helpers/cap-historical-attachments.ts
@@ -81,7 +81,11 @@ function getDroppedCount(total: number, limit: number): number {
* exceeds the budget. That overshoot is the cost of prefix stability while
* guaranteeing that at least one historical attachment always survives.
*/
-function getWeightDroppedCount(weights: number[], tokenBudget: number): number {
+export function getWeightDroppedCount(
+ weights: number[],
+ tokenBudget: number,
+ minBlockSize: number = MIN_WEIGHT_BLOCK_ATTACHMENTS
+): number {
let blockTokens = 0
let blockAttachmentCount = 0
let latestBlockEnd = 0
@@ -90,10 +94,7 @@ function getWeightDroppedCount(weights: number[], tokenBudget: number): number {
for (let i = 0; i < weights.length; i++) {
blockTokens += weights[i]
blockAttachmentCount += 1
- if (
- blockTokens < tokenBudget ||
- blockAttachmentCount < MIN_WEIGHT_BLOCK_ATTACHMENTS
- ) {
+ if (blockTokens < tokenBudget || blockAttachmentCount < minBlockSize) {
continue
}
diff --git a/lib/streaming/helpers/cap-historical-pasted-content.ts b/lib/streaming/helpers/cap-historical-pasted-content.ts
new file mode 100644
index 00000000..c76e9a2b
--- /dev/null
+++ b/lib/streaming/helpers/cap-historical-pasted-content.ts
@@ -0,0 +1,84 @@
+import type { UIMessage } from 'ai'
+
+import { getWeightDroppedCount } from './cap-historical-attachments'
+import { isSourceContextMessage } from './compact-historical-messages'
+
+const DEFAULT_PASTED_CONTENT_CHAR_BUDGET = 200_000
+
+export function parsePastedContentCharBudget(raw: string | undefined): number {
+ const value = raw?.trim()
+ if (!value) return DEFAULT_PASTED_CONTENT_CHAR_BUDGET
+
+ const parsed = Number(value)
+ if (!Number.isFinite(parsed) || parsed < 0) {
+ return DEFAULT_PASTED_CONTENT_CHAR_BUDGET
+ }
+
+ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))
+}
+
+export const HISTORY_PASTED_CONTENT_CHAR_BUDGET = parsePastedContentCharBudget(
+ process.env.HISTORY_PASTED_CONTENT_CHAR_BUDGET
+)
+
+function getPastedLength(part: UIMessage['parts'][number]): number {
+ if (part.type !== 'data-pastedContent') return 0
+
+ const text = (part as { data?: { text?: unknown } }).data?.text
+ return typeof text === 'string' ? text.length : 0
+}
+
+/**
+ * Caps how much pasted content from earlier turns is replayed to the model.
+ *
+ * Older blocks become a text placeholder giving their size. They are dropped
+ * by the same weight quantization as attachments, so the prompt prefix stays
+ * cacheable between block crossings. A block may be a single paste: when one
+ * paste exceeds the budget, moving the boundary on each new paste rewrites far
+ * less than carrying a second copy would. The newest user message and the
+ * newest historical paste are never capped.
+ */
+export function capHistoricalPastedContent(
+ messages: UIMessage[],
+ charBudget: number = HISTORY_PASTED_CONTENT_CHAR_BUDGET
+): UIMessage[] {
+ if (charBudget <= 0) return messages
+
+ const currentTurnIndex = messages.findLastIndex(
+ message => message.role === 'user' && !isSourceContextMessage(message)
+ )
+ const historyEnd =
+ currentTurnIndex === -1 ? messages.length : currentTurnIndex
+
+ const weights: number[] = []
+ for (let i = 0; i < historyEnd; i++) {
+ for (const part of messages[i].parts) {
+ const length = getPastedLength(part)
+ if (length > 0) weights.push(length)
+ }
+ }
+
+ const dropCount = getWeightDroppedCount(weights, charBudget, 1)
+ if (dropCount === 0) return messages
+
+ let seen = 0
+ return messages.map((message, index) => {
+ if (index >= historyEnd || seen >= dropCount) return message
+ if (!message.parts.some(part => getPastedLength(part) > 0)) return message
+
+ const parts = message.parts.map(part => {
+ const length = getPastedLength(part)
+ if (length === 0) return part
+
+ seen += 1
+ if (seen > dropCount) return part
+
+ return {
+ type: 'text' as const,
+ text: `[Pasted content omitted from history: ${length} characters. Ask the user to paste it again if you need to look at it again.]`
+ }
+ })
+
+ return { ...message, parts }
+ })
+}