An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
feat(streaming): cap how many attachments are replayed to the model (#942)
Yoshiki Miura3dae330a53bd
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.mdindex 8fa1b769..b394dadf 100644--- a/docs/CONFIGURATION.md+++ b/docs/CONFIGURATION.md@@ -286,6 +286,21 @@ R2_SIGNED_URL_STABILITY_WINDOW_SECONDS=900If storage variables are not configured, `/api/upload` returns `400` and uploads are disabled.+Attachments are re-sent to the model on every turn, and an image costs thousands of+input tokens, so long conversations spend most of their context re-reading pictures+they already answered about. Older attachments are replaced with a placeholder naming+the file once a conversation collects more than the limit below. Uploads themselves are+not restricted: files stay stored and stay visible in the conversation, and the model is+told to ask for a re-attach if it needs one again.++```bash+# Optional: attachments replayed from earlier turns, excluding the newest message.+# Dropped in whole blocks so the prompt prefix stays cacheable, so the effective+# number replayed is between the limit and twice it. Set to 0 to replay every+# attachment (default: 10)+HISTORY_ATTACHMENT_REPLAY_LIMIT=10+```+### Content ExtractionUse Jina for enhanced content extraction:diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 7b763988..466fb391 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -21,6 +21,7 @@ import { getTextFromParts } from '../utils/message-utils'import { perfLog, perfTime } from '../utils/perf-logging'import { isUsageLogging, logUsage } from '../utils/usage-logging'+import { capHistoricalAttachments } from './helpers/cap-historical-attachments'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'@@ -143,7 +144,9 @@ export async function createChatStreamResponse(const messagesWithNonces = assignDataPartNonces(messagesToModel)const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)- const messagesToConvert = compactHistoricalMessages(messagesWithoutSpec)+ const messagesToConvert = capHistoricalAttachments(+ compactHistoricalMessages(messagesWithoutSpec)+ )// Convert to model messages and apply context window managementlet modelMessages = await convertToModelMessages(messagesToConvert, {diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.tsindex 73bcf44e..3b98c956 100644--- a/lib/streaming/create-ephemeral-chat-stream-response.ts+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts@@ -17,6 +17,7 @@ import {} from '../utils/context-window'import { isUsageLogging, logUsage } from '../utils/usage-logging'+import { capHistoricalAttachments } from './helpers/cap-historical-attachments'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'@@ -80,7 +81,9 @@ export async function createEphemeralChatStreamResponse(try {const messagesWithNonces = assignDataPartNonces(messages)const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)- const messagesToConvert = compactHistoricalMessages(messagesWithoutSpec)+ const messagesToConvert = capHistoricalAttachments(+ compactHistoricalMessages(messagesWithoutSpec)+ )let modelMessages = await convertToModelMessages(messagesToConvert, {convertDataPartdiff --git a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.tsnew file mode 100644index 00000000..8e2751f0--- /dev/null+++ b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts@@ -0,0 +1,236 @@+import type { UIMessage } from 'ai'+import { describe, expect, it } from 'vitest'++import {+ capHistoricalAttachments,+ parseReplayLimit+} from '../cap-historical-attachments'++const LIMIT = 10++function userTurn(+ id: string,+ fileCount: number,+ text = 'Look at this'+): UIMessage {+ const files = Array.from({ length: fileCount }, (_, i) => ({+ type: 'file' as const,+ mediaType: 'image/png',+ filename: `${id}-${i}.png`,+ url: `https://uploads.example.com/${id}-${i}.png?sig=abc`+ }))++ return {+ id,+ role: 'user',+ parts: [...files, { type: 'text', text }]+ } as unknown as UIMessage+}++function assistantTurn(id: string): UIMessage {+ return {+ id,+ role: 'assistant',+ parts: [{ type: 'text', text: 'Here is what I see.' }]+ } as unknown as UIMessage+}++/** One user turn carrying `perTurn` files, followed by an assistant reply. */+function thread(turns: number, perTurn: number): UIMessage[] {+ return Array.from({ length: turns }, (_, i) => [+ userTurn(`u${i}`, perTurn),+ assistantTurn(`a${i}`)+ ]).flat()+}++function filenamesReaching(messages: UIMessage[]): string[] {+ return messages.flatMap(message =>+ message.parts+ .filter(part => part.type === 'file')+ .map(part => (part as unknown as { filename: string }).filename)+ )+}++function placeholders(messages: UIMessage[]): string[] {+ return messages.flatMap(message =>+ message.parts+ .filter(+ part =>+ part.type === 'text' &&+ (part as unknown as { text: string }).text.startsWith(+ '[Attachment omitted'+ )+ )+ .map(part => (part as unknown as { text: string }).text)+ )+}++describe('capHistoricalAttachments', () => {+ it('leaves a conversation under the limit untouched', () => {+ const messages = thread(5, 1).concat(userTurn('current', 1))++ expect(capHistoricalAttachments(messages, LIMIT)).toEqual(messages)+ })++ it('leaves attachments alone until a whole block can be dropped', () => {+ // 19 in history is over the limit, but not yet a full block, so dropping+ // any of them would move the boundary again next turn.+ const messages = thread(19, 1).concat(userTurn('current', 0))++ expect(capHistoricalAttachments(messages, LIMIT)).toEqual(messages)+ })++ it('drops the oldest block once history crosses it', () => {+ const messages = thread(20, 1).concat(userTurn('current', 0))+ const capped = capHistoricalAttachments(messages, LIMIT)++ const kept = filenamesReaching(capped)+ expect(kept).toHaveLength(LIMIT)+ expect(kept[0]).toBe('u10-0.png')+ expect(kept.at(-1)).toBe('u19-0.png')+ expect(placeholders(capped)).toHaveLength(LIMIT)+ })++ it('keeps the dropped set identical while inside the same block', () => {+ // The cache property: appending turns must not change what was already+ // sent, or the prompt prefix is rewritten on every turn.+ const at20 = capHistoricalAttachments(+ thread(20, 1).concat(userTurn('current', 0)),+ LIMIT+ )+ const at25 = capHistoricalAttachments(+ thread(25, 1).concat(userTurn('current', 0)),+ LIMIT+ )++ const prefixLength = at20.length - 1+ expect(at25.slice(0, prefixLength)).toEqual(at20.slice(0, prefixLength))+ })++ it('advances the boundary by exactly one block on the next crossing', () => {+ const capped = capHistoricalAttachments(+ thread(30, 1).concat(userTurn('current', 0)),+ LIMIT+ )++ expect(placeholders(capped)).toHaveLength(2 * LIMIT)+ expect(filenamesReaching(capped)[0]).toBe('u20-0.png')+ })++ it('never keeps more than two blocks regardless of history size', () => {+ for (const turns of [21, 47, 70, 200]) {+ const kept = filenamesReaching(+ capHistoricalAttachments(+ thread(turns, 1).concat(userTurn('current', 0)),+ LIMIT+ )+ )+ expect(kept.length).toBeGreaterThanOrEqual(LIMIT)+ expect(kept.length).toBeLessThan(2 * LIMIT)+ }+ })++ it('never caps the newest user message', () => {+ const messages = thread(30, 1).concat(userTurn('current', 4))+ const capped = capHistoricalAttachments(messages, LIMIT)++ expect(capped.at(-1)).toEqual(messages.at(-1))+ expect(+ filenamesReaching(capped).filter(n => n.startsWith('current'))+ ).toHaveLength(4)+ })++ it('keeps a message that held only attachments non-empty', () => {+ const messages: UIMessage[] = [+ {+ id: 'files-only',+ role: 'user',+ parts: [+ {+ type: 'file',+ mediaType: 'image/png',+ filename: 'only.png',+ url: 'https://uploads.example.com/only.png'+ }+ ]+ } as unknown as UIMessage,+ assistantTurn('a0'),+ ...thread(20, 1),+ userTurn('current', 0)+ ]++ const capped = capHistoricalAttachments(messages, LIMIT)++ expect(capped[0].parts).toHaveLength(1)+ expect(capped[0].parts[0]).toMatchObject({ type: 'text' })+ })++ it('puts no url or timestamp in the placeholder and escapes the filename', () => {+ const messages: UIMessage[] = [+ {+ id: 'hostile',+ role: 'user',+ parts: [+ {+ type: 'file',+ mediaType: 'image/png',+ filename: '<script>alert(1)</script>\nsecond line.png',+ url: 'https://uploads.example.com/x.png?X-Amz-Signature=deadbeef'+ }+ ]+ } as unknown as UIMessage,+ assistantTurn('a0'),+ ...thread(20, 1),+ userTurn('current', 0)+ ]++ const [placeholder] = placeholders(+ capHistoricalAttachments(messages, LIMIT)+ )++ expect(placeholder).toContain('<script>')+ expect(placeholder).not.toContain('<script>')+ expect(placeholder).not.toContain('\n')+ expect(placeholder).not.toContain('X-Amz-Signature')+ expect(placeholder).not.toContain('https://')+ })++ it('replays everything when the limit is disabled', () => {+ const messages = thread(70, 1).concat(userTurn('current', 0))++ expect(capHistoricalAttachments(messages, 0)).toEqual(messages)+ })++ it('counts attachments across turns, not per message', () => {+ // 7 turns x 3 files = 21 in history -> one block of 10 is droppable.+ const capped = capHistoricalAttachments(+ thread(7, 3).concat(userTurn('current', 0)),+ LIMIT+ )++ expect(placeholders(capped)).toHaveLength(LIMIT)+ expect(filenamesReaching(capped)).toHaveLength(11)+ })+})++describe('parseReplayLimit', () => {+ it('falls back to the default for anything unusable', () => {+ // An empty string is the one that matters: templated environments define+ // variables as '' all the time, and Number('') is 0, which would look like+ // a deliberate "replay everything".+ for (const raw of [undefined, '', ' ', 'ten', '-1', 'NaN', 'Infinity']) {+ expect(parseReplayLimit(raw)).toBe(10)+ }+ })++ it('disables the cap only on an explicit zero', () => {+ expect(parseReplayLimit('0')).toBe(0)+ expect(parseReplayLimit(' 0 ')).toBe(0)+ })++ it('keeps at least one attachment for any positive value', () => {+ expect(parseReplayLimit('0.5')).toBe(1)+ expect(parseReplayLimit('4')).toBe(4)+ expect(parseReplayLimit('12.9')).toBe(12)+ })+})diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.tsnew file mode 100644index 00000000..cf75404d--- /dev/null+++ b/lib/streaming/helpers/cap-historical-attachments.ts@@ -0,0 +1,123 @@+import type { UIMessage } from 'ai'++const DEFAULT_REPLAY_LIMIT = 10++/**+ * Only an explicit `0` disables the cap.+ *+ * Templated environments routinely define a variable as an empty string, and+ * `Number('')` is `0` — which would silently replay every attachment while+ * looking configured. Anything unusable falls back to the default instead, and+ * a positive value always keeps at least one attachment.+ */+export function parseReplayLimit(raw: string | undefined): number {+ const value = raw?.trim()+ if (!value) return DEFAULT_REPLAY_LIMIT++ const parsed = Number(value)+ if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_REPLAY_LIMIT++ return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))+}++export const HISTORY_ATTACHMENT_REPLAY_LIMIT = parseReplayLimit(+ process.env.HISTORY_ATTACHMENT_REPLAY_LIMIT+)++const MAX_FILENAME_CHARS = 80++function describeAttachment(part: { mediaType?: string; filename?: string }) {+ // The filename is user-supplied, so neutralize it the way source excerpts are+ // neutralized: no markup, no newlines, bounded length.+ const filename = (part.filename ?? '')+ .replace(/</g, '<')+ .replace(/>/g, '>')+ .replace(/\s+/g, ' ')+ .trim()+ .slice(0, MAX_FILENAME_CHARS)+ const mediaType = (part.mediaType ?? 'file').replace(/[^\w.+/-]/g, '')++ return filename+ ? `"${filename}" (${mediaType})`+ : `an earlier ${mediaType} attachment`+}++/**+ * Number of leading attachments to drop, quantized to whole blocks of `limit`.+ *+ * A plain "keep the newest N" boundary moves by one on every turn that adds an+ * attachment, and because the boundary sits near the START of the conversation+ * it would rewrite the prompt prefix — and so drop the provider's cache for the+ * whole request — on exactly the long threads this is meant to make cheaper.+ *+ * Quantizing means the dropped set is identical for every turn between two+ * block crossings, so the prefix survives; the cache is only reset once per+ * `limit` new attachments. The cost of that is a looser bound: the number kept+ * lands anywhere in [limit, 2 * limit).+ */+function getDroppedCount(total: number, limit: number): number {+ if (total <= limit) return 0+ return Math.floor((total - limit) / limit) * limit+}++function isFilePart(part: UIMessage['parts'][number]) {+ return part.type === 'file'+}++/**+ * Caps how many of a conversation's attachments are replayed to the model.+ *+ * Attachments are re-sent on every turn, and an image costs thousands of input+ * tokens, so a thread that collects a few screenshots per turn ends up spending+ * most of its context re-reading pictures it already answered about. Older ones+ * become a text placeholder naming the file, which keeps the message valid when+ * it held nothing but attachments, and lets the model ask for a re-attach+ * instead of silently losing the reference. The file itself is untouched: it+ * stays stored and stays visible in the conversation.+ *+ * The newest user message is never capped — that is the request being answered.+ */+export function capHistoricalAttachments(+ messages: UIMessage[],+ limit: number = HISTORY_ATTACHMENT_REPLAY_LIMIT+): UIMessage[] {+ if (limit <= 0) return messages++ const currentTurnIndex = messages.findLastIndex(+ message => message.role === 'user'+ )+ const historyEnd =+ currentTurnIndex === -1 ? messages.length : currentTurnIndex++ let total = 0+ for (let i = 0; i < historyEnd; i++) {+ total += messages[i].parts.filter(isFilePart).length+ }++ const dropCount = getDroppedCount(total, limit)+ if (dropCount === 0) return messages++ let seen = 0+ return messages.map((message, index) => {+ if (index >= historyEnd || !message.parts.some(isFilePart)) {+ return message+ }++ const parts = message.parts.map(part => {+ if (!isFilePart(part)) return part++ seen += 1+ if (seen > dropCount) return part++ const file = part as { mediaType?: string; filename?: string }+ return {+ type: 'text' as const,+ text: `[Attachment omitted from history: ${describeAttachment(+ file+ )}. Ask the user to re-attach it if you need to look at it again.]`+ }+ })++ return { ...message, parts }+ })+}