Skip to content

An AI-powered search engine with a generative UI

fix(streaming): stabilize attachment weight boundary (#974)

Sushant Tiwarif4c62c318ce8
diff --git a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
index db939190..d1ff47d6 100644
--- a/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
+++ b/lib/streaming/helpers/__tests__/cap-historical-attachments.test.ts
@@ -44,6 +44,26 @@ function thread(turns: number, perTurn: number): UIMessage[] {
]).flat()
}
+function pdfThread(turns: number, size = 500_000): UIMessage[] {
+ return thread(turns, 0).map(message => {
+ if (message.role !== 'user') return message
+
+ return {
+ ...message,
+ parts: [
+ {
+ type: 'file',
+ mediaType: 'application/pdf',
+ filename: `${message.id}.pdf`,
+ url: `https://example.com/${message.id}.pdf`,
+ size
+ },
+ ...message.parts
+ ]
+ } as unknown as UIMessage
+ })
+}
+
function filenamesReaching(messages: UIMessage[]): string[] {
return messages.flatMap(message =>
message.parts
@@ -213,33 +233,50 @@ describe('capHistoricalAttachments', () => {
expect(filenamesReaching(capped)).toHaveLength(11)
})
- it('bounds heavy attachments below the count limit by weight', () => {
- const heavy = [
- userTurn('old', 0),
- assistantTurn('a0'),
- userTurn('recent', 0),
- assistantTurn('a1'),
- userTurn('current', 0)
- ]
- heavy[0].parts.unshift({
- type: 'file',
- mediaType: 'application/pdf',
- filename: 'old.pdf',
- url: 'https://example.com/old.pdf',
- size: 500_000
- } as never)
- heavy[2].parts.unshift({
- type: 'file',
- mediaType: 'application/pdf',
- filename: 'recent.pdf',
- url: 'https://example.com/recent.pdf',
- size: 500_000
- } as never)
+ it('bounds heavy attachments below the count limit by weight blocks', () => {
+ const messages = pdfThread(4).concat(userTurn('current', 0))
+ const capped = capHistoricalAttachments(messages, LIMIT, 200_000)
+
+ expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])
+ expect(placeholders(capped)).toHaveLength(2)
+ })
- const capped = capHistoricalAttachments(heavy, LIMIT, 200_000)
+ it('keeps the weighted replay prefix stable between block crossings', () => {
+ const atFour = capHistoricalAttachments(
+ pdfThread(4, 1_000_000).concat(userTurn('current', 0)),
+ 0,
+ 200_000
+ )
+ const atFive = capHistoricalAttachments(
+ pdfThread(5, 1_000_000).concat(userTurn('current', 0)),
+ 0,
+ 200_000
+ )
- expect(filenamesReaching(capped)).toEqual(['recent.pdf'])
- expect(placeholders(capped)).toHaveLength(1)
+ const replayedPrefixLength = atFour.length - 1
+ expect(atFive.slice(0, replayedPrefixLength)).toEqual(
+ atFour.slice(0, replayedPrefixLength)
+ )
+ expect(placeholders(atFour)).toHaveLength(2)
+ expect(placeholders(atFive)).toHaveLength(2)
+ })
+
+ it('still applies the weight bound when the count cap is disabled', () => {
+ const capped = capHistoricalAttachments(
+ pdfThread(4).concat(userTurn('current', 0)),
+ 0,
+ 200_000
+ )
+
+ expect(filenamesReaching(capped)).toEqual(['u2.pdf', 'u3.pdf'])
+ })
+
+ it('keeps a historical attachment when one file exceeds the budget', () => {
+ const messages = pdfThread(1, 1_000_000).concat(userTurn('current', 0))
+
+ expect(filenamesReaching(capHistoricalAttachments(messages, 0, 1))).toEqual(
+ ['u0.pdf']
+ )
})
it('leaves a thread within the weight budget untouched', () => {
diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.ts
index 311e6d08..6a24369f 100644
--- a/lib/streaming/helpers/cap-historical-attachments.ts
+++ b/lib/streaming/helpers/cap-historical-attachments.ts
@@ -6,6 +6,7 @@ import { describeAttachment, isFilePart } from './attachment-parts'
const DEFAULT_REPLAY_LIMIT = 10
const DEFAULT_ATTACHMENT_TOKEN_BUDGET = 200_000
+const MIN_WEIGHT_BLOCK_ATTACHMENTS = 2
/**
* Only an explicit `0` disables the count cap. The weight budget below is a
@@ -64,6 +65,46 @@ function getDroppedCount(total: number, limit: number): number {
return Math.floor((total - limit) / limit) * limit
}
+/**
+ * Number of leading attachments to drop, quantized by cumulative token weight.
+ *
+ * Blocks are anchored to the start of the surviving history and complete once
+ * they reach `tokenBudget`, with at least two attachments per block. The
+ * minimum is the stability bound: even an attachment heavier than the entire
+ * budget cannot make the boundary advance on consecutive turns. We retain the
+ * newest complete block plus any incomplete tail, so appending an attachment
+ * cannot move the boundary until it completes another block.
+ *
+ * This deliberately permits nearly two blocks of replayed weight. A block can
+ * exceed the budget by its final attachment, or more when an attachment alone
+ * exceeds the budget. That overshoot is the cost of prefix stability while
+ * guaranteeing that at least one historical attachment always survives.
+ */
+function getWeightDroppedCount(weights: number[], tokenBudget: number): number {
+ let blockTokens = 0
+ let blockAttachmentCount = 0
+ let latestBlockEnd = 0
+ let droppedCount = 0
+
+ for (let i = 0; i < weights.length; i++) {
+ blockTokens += weights[i]
+ blockAttachmentCount += 1
+ if (
+ blockTokens < tokenBudget ||
+ blockAttachmentCount < MIN_WEIGHT_BLOCK_ATTACHMENTS
+ ) {
+ continue
+ }
+
+ droppedCount = latestBlockEnd
+ latestBlockEnd = i + 1
+ blockTokens = 0
+ blockAttachmentCount = 0
+ }
+
+ return droppedCount
+}
+
/**
* Caps how many of a conversation's attachments are replayed to the model.
*
@@ -96,8 +137,7 @@ export function capHistoricalAttachments(
}
const countDropCount = limit > 0 ? getDroppedCount(total, limit) : 0
- let survivingTokens = 0
- let survivingCount = 0
+ const survivingWeights: number[] = []
let seenForBudget = 0
for (let i = 0; i < historyEnd; i++) {
@@ -111,38 +151,12 @@ export function capHistoricalAttachments(
mediaType?: string
size?: number
}
- survivingTokens += estimateAttachmentTokens(file)
- survivingCount += 1
+ survivingWeights.push(estimateAttachmentTokens(file))
}
}
- // The weight pass drops just enough of the oldest survivors to fit, without
- // the block quantization the count cap uses. It does not need it: history
- // only grows, so this boundary only ever moves forward, and each crossing
- // rewrites the prefix once instead of on every turn.
- let weightDropCount = 0
- if (tokenBudget > 0 && survivingTokens > tokenBudget) {
- seenForBudget = 0
- for (let i = 0; i < historyEnd; i++) {
- for (const part of messages[i].parts) {
- if (!isFilePart(part)) continue
-
- seenForBudget += 1
- if (seenForBudget <= countDropCount) continue
- if (survivingTokens <= tokenBudget || survivingCount <= 1) break
-
- const file = part as {
- mediaType?: string
- size?: number
- }
- survivingTokens -= estimateAttachmentTokens(file)
- survivingCount -= 1
- weightDropCount += 1
- }
-
- if (survivingTokens <= tokenBudget || survivingCount <= 1) break
- }
- }
+ const weightDropCount =
+ tokenBudget > 0 ? getWeightDroppedCount(survivingWeights, tokenBudget) : 0
const dropCount = countDropCount + weightDropCount
if (dropCount === 0) return messages