An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(streaming): stabilize attachment weight boundary (#974)
Sushant Tiwarif4c62c318ce8
diff --git a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.tsindex db939190..d1ff47d6 100644--- a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts+++ b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts@@ -44,6 +44,26 @@ function thread(turns: number, perTurn: number): UIMessage[] {]).flat()}+function pdfThread(turns: number, size = 500_000): UIMessage[] {+ return thread(turns, 0).map(message => {+ if (message.role !== 'user') return message++ return {+ ...message,+ parts: [+ {+ type: 'file',+ mediaType: 'application/pdf',+ filename: `${message.id}.pdf`,+ url: `https://example.com/${message.id}.pdf`,+ size+ },+ ...message.parts+ ]+ } as unknown as UIMessage+ })+}+function filenamesReaching(messages: UIMessage[]): string[] {return messages.flatMap(message =>message.parts@@ -213,33 +233,50 @@ describe('capHistoricalAttachments', () => {expect(filenamesReaching(capped)).toHaveLength(11)})- it('bounds heavy attachments below the count limit by weight', () => {- const heavy = [- userTurn('old', 0),- assistantTurn('a0'),- userTurn('recent', 0),- assistantTurn('a1'),- userTurn('current', 0)- ]- heavy[0].parts.unshift({- type: 'file',- mediaType: 'application/pdf',- filename: 'old.pdf',- url: 'https://example.com/old.pdf',- size: 500_000- } as never)- heavy[2].parts.unshift({- type: 'file',- mediaType: 'application/pdf',- filename: 'recent.pdf',- url: 'https://example.com/recent.pdf',- size: 500_000- } as never)+ it('bounds heavy attachments below the count limit by weight blocks', () => {+ const messages = pdfThread(4).concat(userTurn('current', 0))+ const capped = capHistoricalAttachments(messages, LIMIT, 200_000)++ expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])+ expect(placeholders(capped)).toHaveLength(2)+ })- const capped = capHistoricalAttachments(heavy, LIMIT, 200_000)+ it('keeps the weighted replay prefix stable between block crossings', () => {+ const atFour = capHistoricalAttachments(+ pdfThread(4, 1_000_000).concat(userTurn('current', 0)),+ 0,+ 200_000+ )+ const atFive = capHistoricalAttachments(+ pdfThread(5, 1_000_000).concat(userTurn('current', 0)),+ 0,+ 200_000+ )- expect(filenamesReaching(capped)).toEqual(['recent.pdf'])- expect(placeholders(capped)).toHaveLength(1)+ const replayedPrefixLength = atFour.length - 1+ expect(atFive.slice(0, replayedPrefixLength)).toEqual(+ atFour.slice(0, replayedPrefixLength)+ )+ expect(placeholders(atFour)).toHaveLength(2)+ expect(placeholders(atFive)).toHaveLength(2)+ })++ it('still applies the weight bound when the count cap is disabled', () => {+ const capped = capHistoricalAttachments(+ pdfThread(4).concat(userTurn('current', 0)),+ 0,+ 200_000+ )++ expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])+ })++ it('keeps a historical attachment when one file exceeds the budget', () => {+ const messages = pdfThread(1, 1_000_000).concat(userTurn('current', 0))++ expect(filenamesReaching(capHistoricalAttachments(messages, 0, 1))).toEqual(+ ['u0.pdf']+ )})it('leaves a thread within the weight budget untouched', () => {diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.tsindex 311e6d08..6a24369f 100644--- a/lib/streaming/helpers/cap-historical-attachments.ts+++ b/lib/streaming/helpers/cap-historical-attachments.ts@@ -6,6 +6,7 @@ import { describeAttachment, isFilePart } from './attachment-parts'const DEFAULT_REPLAY_LIMIT = 10const DEFAULT_ATTACHMENT_TOKEN_BUDGET = 200_000+const MIN_WEIGHT_BLOCK_ATTACHMENTS = 2/*** Only an explicit `0` disables the count cap. The weight budget below is a@@ -64,6 +65,46 @@ function getDroppedCount(total: number, limit: number): number {return Math.floor((total - limit) / limit) * limit}+/**+ * Number of leading attachments to drop, quantized by cumulative token weight.+ *+ * Blocks are anchored to the start of the surviving history and complete once+ * they reach `tokenBudget`, with at least two attachments per block. The+ * minimum is the stability bound: even an attachment heavier than the entire+ * budget cannot make the boundary advance on consecutive turns. We retain the+ * newest complete block plus any incomplete tail, so appending an attachment+ * cannot move the boundary until it completes another block.+ *+ * This deliberately permits nearly two blocks of replayed weight. A block can+ * exceed the budget by its final attachment, or more when an attachment alone+ * exceeds the budget. That overshoot is the cost of prefix stability while+ * guaranteeing that at least one historical attachment always survives.+ */+function getWeightDroppedCount(weights: number[], tokenBudget: number): number {+ let blockTokens = 0+ let blockAttachmentCount = 0+ let latestBlockEnd = 0+ let droppedCount = 0++ for (let i = 0; i < weights.length; i++) {+ blockTokens += weights[i]+ blockAttachmentCount += 1+ if (+ blockTokens < tokenBudget ||+ blockAttachmentCount < MIN_WEIGHT_BLOCK_ATTACHMENTS+ ) {+ continue+ }++ droppedCount = latestBlockEnd+ latestBlockEnd = i + 1+ blockTokens = 0+ blockAttachmentCount = 0+ }++ return droppedCount+}+/*** Caps how many of a conversation's attachments are replayed to the model.*@@ -96,8 +137,7 @@ export function capHistoricalAttachments(}const countDropCount = limit > 0 ? getDroppedCount(total, limit) : 0- let survivingTokens = 0- let survivingCount = 0+ const survivingWeights: number[] = []let seenForBudget = 0for (let i = 0; i < historyEnd; i++) {@@ -111,38 +151,12 @@ export function capHistoricalAttachments(mediaType?: stringsize?: number}- survivingTokens += estimateAttachmentTokens(file)- survivingCount += 1+ survivingWeights.push(estimateAttachmentTokens(file))}}- // The weight pass drops just enough of the oldest survivors to fit, without- // the block quantization the count cap uses. It does not need it: history- // only grows, so this boundary only ever moves forward, and each crossing- // rewrites the prefix once instead of on every turn.- let weightDropCount = 0- if (tokenBudget > 0 && survivingTokens > tokenBudget) {- seenForBudget = 0- for (let i = 0; i < historyEnd; i++) {- for (const part of messages[i].parts) {- if (!isFilePart(part)) continue-- seenForBudget += 1- if (seenForBudget <= countDropCount) continue- if (survivingTokens <= tokenBudget || survivingCount <= 1) break-- const file = part as {- mediaType?: string- size?: number- }- survivingTokens -= estimateAttachmentTokens(file)- survivingCount -= 1- weightDropCount += 1- }-- if (survivingTokens <= tokenBudget || survivingCount <= 1) break- }- }+ const weightDropCount =+ tokenBudget > 0 ? getWeightDroppedCount(survivingWeights, tokenBudget) : 0const dropCount = countDropCount + weightDropCountif (dropCount === 0) return messages