Skip to content

An AI-powered search engine with a generative UI

feat(streaming): cap how many attachments are replayed to the model (#942)

Yoshiki Miura3dae330a53bd
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index 8fa1b769..b394dadf 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -286,6 +286,21 @@ R2_SIGNED_URL_STABILITY_WINDOW_SECONDS=900
If storage variables are not configured, `/api/upload` returns `400` and uploads are disabled.
+Attachments are re-sent to the model on every turn, and an image costs thousands of
+input tokens, so long conversations spend most of their context re-reading pictures
+they already answered about. Older attachments are replaced with a placeholder naming
+the file once a conversation collects more than the limit below. Uploads themselves are
+not restricted: files stay stored and stay visible in the conversation, and the model is
+told to ask for a re-attach if it needs one again.
+
+```bash
+# Optional: attachments replayed from earlier turns, excluding the newest message.
+# Dropped in whole blocks so the prompt prefix stays cacheable, so the effective
+# number replayed is between the limit and twice it. Set to 0 to replay every
+# attachment (default: 10)
+HISTORY_ATTACHMENT_REPLAY_LIMIT=10
+```
+
### Content Extraction
Use Jina for enhanced content extraction:
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index 7b763988..466fb391 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -21,6 +21,7 @@ import { getTextFromParts } from '../utils/message-utils'
import { perfLog, perfTime } from '../utils/perf-logging'
import { isUsageLogging, logUsage } from '../utils/usage-logging'
+import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
@@ -143,7 +144,9 @@ export async function createChatStreamResponse(
const messagesWithNonces = assignDataPartNonces(messagesToModel)
const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)
- const messagesToConvert = compactHistoricalMessages(messagesWithoutSpec)
+ const messagesToConvert = capHistoricalAttachments(
+ compactHistoricalMessages(messagesWithoutSpec)
+ )
// Convert to model messages and apply context window management
let modelMessages = await convertToModelMessages(messagesToConvert, {
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index 73bcf44e..3b98c956 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -17,6 +17,7 @@ import {
} from '../utils/context-window'
import { isUsageLogging, logUsage } from '../utils/usage-logging'
+import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
@@ -80,7 +81,9 @@ export async function createEphemeralChatStreamResponse(
try {
const messagesWithNonces = assignDataPartNonces(messages)
const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)
- const messagesToConvert = compactHistoricalMessages(messagesWithoutSpec)
+ const messagesToConvert = capHistoricalAttachments(
+ compactHistoricalMessages(messagesWithoutSpec)
+ )
let modelMessages = await convertToModelMessages(messagesToConvert, {
convertDataPart
diff --git a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
new file mode 100644
index 00000000..8e2751f0
--- /dev/null
+++ b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
@@ -0,0 +1,236 @@
+import type { UIMessage } from 'ai'
+import { describe, expect, it } from 'vitest'
+
+import {
+ capHistoricalAttachments,
+ parseReplayLimit
+} from '../cap-historical-attachments'
+
+const LIMIT = 10
+
+function userTurn(
+ id: string,
+ fileCount: number,
+ text = 'Look at this'
+): UIMessage {
+ const files = Array.from({ length: fileCount }, (_, i) => ({
+ type: 'file' as const,
+ mediaType: 'image/png',
+ filename: `${id}-${i}.png`,
+ url: `https://uploads.example.com/${id}-${i}.png?sig=abc`
+ }))
+
+ return {
+ id,
+ role: 'user',
+ parts: [...files, { type: 'text', text }]
+ } as unknown as UIMessage
+}
+
+function assistantTurn(id: string): UIMessage {
+ return {
+ id,
+ role: 'assistant',
+ parts: [{ type: 'text', text: 'Here is what I see.' }]
+ } as unknown as UIMessage
+}
+
+/** One user turn carrying `perTurn` files, followed by an assistant reply. */
+function thread(turns: number, perTurn: number): UIMessage[] {
+ return Array.from({ length: turns }, (_, i) => [
+ userTurn(`u${i}`, perTurn),
+ assistantTurn(`a${i}`)
+ ]).flat()
+}
+
+function filenamesReaching(messages: UIMessage[]): string[] {
+ return messages.flatMap(message =>
+ message.parts
+ .filter(part => part.type === 'file')
+ .map(part => (part as unknown as { filename: string }).filename)
+ )
+}
+
+function placeholders(messages: UIMessage[]): string[] {
+ return messages.flatMap(message =>
+ message.parts
+ .filter(
+ part =>
+ part.type === 'text' &&
+ (part as unknown as { text: string }).text.startsWith(
+ '[Attachment omitted'
+ )
+ )
+ .map(part => (part as unknown as { text: string }).text)
+ )
+}
+
+describe('capHistoricalAttachments', () => {
+ it('leaves a conversation under the limit untouched', () => {
+ const messages = thread(5, 1).concat(userTurn('current', 1))
+
+ expect(capHistoricalAttachments(messages, LIMIT)).toEqual(messages)
+ })
+
+ it('leaves attachments alone until a whole block can be dropped', () => {
+ // 19 in history is over the limit, but not yet a full block, so dropping
+ // any of them would move the boundary again next turn.
+ const messages = thread(19, 1).concat(userTurn('current', 0))
+
+ expect(capHistoricalAttachments(messages, LIMIT)).toEqual(messages)
+ })
+
+ it('drops the oldest block once history crosses it', () => {
+ const messages = thread(20, 1).concat(userTurn('current', 0))
+ const capped = capHistoricalAttachments(messages, LIMIT)
+
+ const kept = filenamesReaching(capped)
+ expect(kept).toHaveLength(LIMIT)
+ expect(kept[0]).toBe('u10-0.png')
+ expect(kept.at(-1)).toBe('u19-0.png')
+ expect(placeholders(capped)).toHaveLength(LIMIT)
+ })
+
+ it('keeps the dropped set identical while inside the same block', () => {
+ // The cache property: appending turns must not change what was already
+ // sent, or the prompt prefix is rewritten on every turn.
+ const at20 = capHistoricalAttachments(
+ thread(20, 1).concat(userTurn('current', 0)),
+ LIMIT
+ )
+ const at25 = capHistoricalAttachments(
+ thread(25, 1).concat(userTurn('current', 0)),
+ LIMIT
+ )
+
+ const prefixLength = at20.length - 1
+ expect(at25.slice(0, prefixLength)).toEqual(at20.slice(0, prefixLength))
+ })
+
+ it('advances the boundary by exactly one block on the next crossing', () => {
+ const capped = capHistoricalAttachments(
+ thread(30, 1).concat(userTurn('current', 0)),
+ LIMIT
+ )
+
+ expect(placeholders(capped)).toHaveLength(2 * LIMIT)
+ expect(filenamesReaching(capped)[0]).toBe('u20-0.png')
+ })
+
+ it('never keeps more than two blocks regardless of history size', () => {
+ for (const turns of [21, 47, 70, 200]) {
+ const kept = filenamesReaching(
+ capHistoricalAttachments(
+ thread(turns, 1).concat(userTurn('current', 0)),
+ LIMIT
+ )
+ )
+ expect(kept.length).toBeGreaterThanOrEqual(LIMIT)
+ expect(kept.length).toBeLessThan(2 * LIMIT)
+ }
+ })
+
+ it('never caps the newest user message', () => {
+ const messages = thread(30, 1).concat(userTurn('current', 4))
+ const capped = capHistoricalAttachments(messages, LIMIT)
+
+ expect(capped.at(-1)).toEqual(messages.at(-1))
+ expect(
+ filenamesReaching(capped).filter(n => n.startsWith('current'))
+ ).toHaveLength(4)
+ })
+
+ it('keeps a message that held only attachments non-empty', () => {
+ const messages: UIMessage[] = [
+ {
+ id: 'files-only',
+ role: 'user',
+ parts: [
+ {
+ type: 'file',
+ mediaType: 'image/png',
+ filename: 'only.png',
+ url: 'https://uploads.example.com/only.png'
+ }
+ ]
+ } as unknown as UIMessage,
+ assistantTurn('a0'),
+ ...thread(20, 1),
+ userTurn('current', 0)
+ ]
+
+ const capped = capHistoricalAttachments(messages, LIMIT)
+
+ expect(capped[0].parts).toHaveLength(1)
+ expect(capped[0].parts[0]).toMatchObject({ type: 'text' })
+ })
+
+ it('puts no url or timestamp in the placeholder and escapes the filename', () => {
+ const messages: UIMessage[] = [
+ {
+ id: 'hostile',
+ role: 'user',
+ parts: [
+ {
+ type: 'file',
+ mediaType: 'image/png',
+ filename: '<script>alert(1)</script>\nsecond line.png',
+ url: 'https://uploads.example.com/x.png?X-Amz-Signature=deadbeef'
+ }
+ ]
+ } as unknown as UIMessage,
+ assistantTurn('a0'),
+ ...thread(20, 1),
+ userTurn('current', 0)
+ ]
+
+ const [placeholder] = placeholders(
+ capHistoricalAttachments(messages, LIMIT)
+ )
+
+ expect(placeholder).toContain('&lt;script&gt;')
+ expect(placeholder).not.toContain('<script>')
+ expect(placeholder).not.toContain('\n')
+ expect(placeholder).not.toContain('X-Amz-Signature')
+ expect(placeholder).not.toContain('https://')
+ })
+
+ it('replays everything when the limit is disabled', () => {
+ const messages = thread(70, 1).concat(userTurn('current', 0))
+
+ expect(capHistoricalAttachments(messages, 0)).toEqual(messages)
+ })
+
+ it('counts attachments across turns, not per message', () => {
+ // 7 turns x 3 files = 21 in history -> one block of 10 is droppable.
+ const capped = capHistoricalAttachments(
+ thread(7, 3).concat(userTurn('current', 0)),
+ LIMIT
+ )
+
+ expect(placeholders(capped)).toHaveLength(LIMIT)
+ expect(filenamesReaching(capped)).toHaveLength(11)
+ })
+})
+
+describe('parseReplayLimit', () => {
+ it('falls back to the default for anything unusable', () => {
+ // An empty string is the one that matters: templated environments define
+ // variables as '' all the time, and Number('') is 0, which would look like
+ // a deliberate "replay everything".
+ for (const raw of [undefined, '', ' ', 'ten', '-1', 'NaN', 'Infinity']) {
+ expect(parseReplayLimit(raw)).toBe(10)
+ }
+ })
+
+ it('disables the cap only on an explicit zero', () => {
+ expect(parseReplayLimit('0')).toBe(0)
+ expect(parseReplayLimit(' 0 ')).toBe(0)
+ })
+
+ it('keeps at least one attachment for any positive value', () => {
+ expect(parseReplayLimit('0.5')).toBe(1)
+ expect(parseReplayLimit('4')).toBe(4)
+ expect(parseReplayLimit('12.9')).toBe(12)
+ })
+})
diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.ts
new file mode 100644
index 00000000..cf75404d
--- /dev/null
+++ b/lib/streaming/helpers/cap-historical-attachments.ts
@@ -0,0 +1,123 @@
+import type { UIMessage } from 'ai'
+
+const DEFAULT_REPLAY_LIMIT = 10
+
+/**
+ * Only an explicit `0` disables the cap.
+ *
+ * Templated environments routinely define a variable as an empty string, and
+ * `Number('')` is `0` — which would silently replay every attachment while
+ * looking configured. Anything unusable falls back to the default instead, and
+ * a positive value always keeps at least one attachment.
+ */
+export function parseReplayLimit(raw: string | undefined): number {
+ const value = raw?.trim()
+ if (!value) return DEFAULT_REPLAY_LIMIT
+
+ const parsed = Number(value)
+ if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_REPLAY_LIMIT
+
+ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))
+}
+
+export const HISTORY_ATTACHMENT_REPLAY_LIMIT = parseReplayLimit(
+ process.env.HISTORY_ATTACHMENT_REPLAY_LIMIT
+)
+
+const MAX_FILENAME_CHARS = 80
+
+function describeAttachment(part: { mediaType?: string; filename?: string }) {
+ // The filename is user-supplied, so neutralize it the way source excerpts are
+ // neutralized: no markup, no newlines, bounded length.
+ const filename = (part.filename ?? '')
+ .replace(/</g, '&lt;')
+ .replace(/>/g, '&gt;')
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, MAX_FILENAME_CHARS)
+ const mediaType = (part.mediaType ?? 'file').replace(/[^\w.+/-]/g, '')
+
+ return filename
+ ? `"${filename}" (${mediaType})`
+ : `an earlier ${mediaType} attachment`
+}
+
+/**
+ * Number of leading attachments to drop, quantized to whole blocks of `limit`.
+ *
+ * A plain "keep the newest N" boundary moves by one on every turn that adds an
+ * attachment, and because the boundary sits near the START of the conversation
+ * it would rewrite the prompt prefix — and so drop the provider's cache for the
+ * whole request — on exactly the long threads this is meant to make cheaper.
+ *
+ * Quantizing means the dropped set is identical for every turn between two
+ * block crossings, so the prefix survives; the cache is only reset once per
+ * `limit` new attachments. The cost of that is a looser bound: the number kept
+ * lands anywhere in [limit, 2 * limit).
+ */
+function getDroppedCount(total: number, limit: number): number {
+ if (total <= limit) return 0
+ return Math.floor((total - limit) / limit) * limit
+}
+
+function isFilePart(part: UIMessage['parts'][number]) {
+ return part.type === 'file'
+}
+
+/**
+ * Caps how many of a conversation's attachments are replayed to the model.
+ *
+ * Attachments are re-sent on every turn, and an image costs thousands of input
+ * tokens, so a thread that collects a few screenshots per turn ends up spending
+ * most of its context re-reading pictures it already answered about. Older ones
+ * become a text placeholder naming the file, which keeps the message valid when
+ * it held nothing but attachments, and lets the model ask for a re-attach
+ * instead of silently losing the reference. The file itself is untouched: it
+ * stays stored and stays visible in the conversation.
+ *
+ * The newest user message is never capped — that is the request being answered.
+ */
+export function capHistoricalAttachments(
+ messages: UIMessage[],
+ limit: number = HISTORY_ATTACHMENT_REPLAY_LIMIT
+): UIMessage[] {
+ if (limit <= 0) return messages
+
+ const currentTurnIndex = messages.findLastIndex(
+ message => message.role === 'user'
+ )
+ const historyEnd =
+ currentTurnIndex === -1 ? messages.length : currentTurnIndex
+
+ let total = 0
+ for (let i = 0; i < historyEnd; i++) {
+ total += messages[i].parts.filter(isFilePart).length
+ }
+
+ const dropCount = getDroppedCount(total, limit)
+ if (dropCount === 0) return messages
+
+ let seen = 0
+ return messages.map((message, index) => {
+ if (index >= historyEnd || !message.parts.some(isFilePart)) {
+ return message
+ }
+
+ const parts = message.parts.map(part => {
+ if (!isFilePart(part)) return part
+
+ seen += 1
+ if (seen > dropCount) return part
+
+ const file = part as { mediaType?: string; filename?: string }
+ return {
+ type: 'text' as const,
+ text: `[Attachment omitted from history: ${describeAttachment(
+ file
+ )}. Ask the user to re-attach it if you need to look at it again.]`
+ }
+ })
+
+ return { ...message, parts }
+ })
+}